From 91bd8a78fa439eb0fc1789fd505ec52556111080 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 29 Jul 2026 16:53:15 -0500 Subject: [PATCH 1/2] feat(agentic): remove Normalized E2E / Session Time / Prefill TPS x-axis modes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Retire the three experimental agentic x-axis modes and everything that existed solely to serve them, leaving Interactivity / E2E Latency / TTFT. - chart: drop the three mode buttons, DERIVED_X_MODE_SPECS and the derived-metric fetch/remap plumbing in ChartDisplay; the remaining modes apply to both scenario kinds, so the agentic-only button filter (and its `mounted` SSR guard) goes too - API/hook/db: delete /api/v1/derived-agentic-metrics, the useDerivedAgenticMetrics hook, and queries/derived-agentic-metrics.ts - aggregate_stats v6: drop normalizedSessionTimeS, p90PrefillTpsPerUser and normalizedE2e400; mergeProfileStatsUpgrade no longer carries them forward, and the backfill's profile-only fast path now covers every v3+ bundle instead of only v3 - drop the NORMALIZED_E2E_OUTPUT_TOKENS constant and the overlay suppression helper that existed only for Normalized E2E 中文：移除三个实验性智能体 X 轴模式（Normalized E2E、会话时长、 Prefill TPS / user）及其专属实现，仅保留交互性 / 端到端延迟 / TTFT。 同时删除对应的 API 路由、React Query hook 与数据库查询模块； aggregate_stats 升级至 v6，去掉三个已废弃字段，backfill 的 profile-only 快速路径扩展至所有 v3 及以上版本。 Co-Authored-By: Claude Fable 5 --- .../app/cypress/e2e/ttft-x-axis-toggle.cy.ts | 71 --- .../src/app/api/v1/agentic-cache-keys.test.ts | 6 - .../api/v1/derived-agentic-metrics/route.ts | 51 -- .../components/inference/InferenceContext.tsx | 46 +- .../inference/hooks/useChartData.ts | 39 +- .../app/src/components/inference/types.ts | 17 +- .../components/inference/ui/ChartDisplay.tsx | 577 +++++++----------- .../src/components/inference/utils.test.ts | 21 +- .../app/src/components/inference/utils.ts | 14 - .../components/inference/utils/e2eFrontier.ts | 2 +- .../api/use-derived-agentic-metrics.test.ts | 13 - .../hooks/api/use-derived-agentic-metrics.ts | 55 -- packages/app/src/lib/benchmark-id.ts | 2 +- packages/constants/src/agentic.ts | 2 - packages/constants/src/index.ts | 1 - packages/db/src/backfill-aggregate-stats.ts | 8 +- .../src/etl/compute-aggregate-stats.test.ts | 49 +- .../db/src/etl/compute-aggregate-stats.ts | 41 +- .../db/src/queries/agentic-aggregates.test.ts | 11 +- packages/db/src/queries/agentic-aggregates.ts | 17 +- packages/db/src/queries/agentic-shared.ts | 7 +- .../queries/derived-agentic-metrics.test.ts | 256 -------- .../db/src/queries/derived-agentic-metrics.ts | 318 ---------- 23 files changed, 297 insertions(+), 1327 deletions(-) delete mode 100644 packages/app/src/app/api/v1/derived-agentic-metrics/route.ts delete mode 100644 packages/app/src/hooks/api/use-derived-agentic-metrics.test.ts delete mode 100644 packages/app/src/hooks/api/use-derived-agentic-metrics.ts delete mode 100644 packages/constants/src/agentic.ts delete mode 100644 packages/db/src/queries/derived-agentic-metrics.test.ts delete mode 100644 packages/db/src/queries/derived-agentic-metrics.ts diff --git a/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts b/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts index 5023877ea..f3a897b1b 100644 --- a/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts +++ b/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts @@ -1,27 +1,5 @@ import { unlockAgenticGate } from '../support/e2e'; -const interceptDerivedMetrics = () => { - cy.intercept('GET', '/api/v1/derived-agentic-metrics*', (request) => { - const ids = new URL(request.url).searchParams.get('ids')?.split(',').filter(Boolean) ?? []; - request.reply({ - body: Object.fromEntries( - ids.map((id, index) => [ - id, - { - id: Number(id), - normalized_session_time_s: 60 + index, - p90_prefill_tps_per_user: 100 + index, - p75_normalized_e2e_400_s: 8 + index, - p90_normalized_e2e_400_s: 12 + index, - p75_osl_per_e2el: 40 + index, - p90_osl_per_e2el: 25 + index, - }, - ]), - ), - }); - }).as('derivedAgenticMetrics'); -}; - // This spec exercises the agentic x-axis modes, which only exist when the // selected model resolves to the Agentic Traces scenario. The default e2e // fixtures (cypress/fixtures/api/*.json) have NO agentic rows for any model, and @@ -148,10 +126,6 @@ const interceptFixedSequenceData = () => { describe('X-Axis Mode Toggle (inference chart)', () => { before(() => { interceptAgenticData(); - // Stub derived metrics before the visit: if the agentic DEFAULT x-axis mode - // is (or becomes) a derived mode that fetches /derived-agentic-metrics on - // mount, the chart must not sit on its loading skeleton. - interceptDerivedMetrics(); cy.visit('/inference?i_seq=agentic-traces', { onBeforeLoad(win) { win.localStorage.setItem('inferencex-star-modal-dismissed', String(Date.now())); @@ -166,7 +140,6 @@ describe('X-Axis Mode Toggle (inference chart)', () => { cy.get('[data-testid="scenario-selector"]').should('contain.text', 'Agentic Traces'); cy.get('[data-testid="x-axis-mode-ttft"]').should('be.visible'); cy.get('[data-testid="x-axis-mode-e2e"]').should('be.visible'); - cy.get('[data-testid="x-axis-mode-normalized-e2e"]').should('be.visible'); cy.get('[data-testid="x-axis-mode-interactivity"]') .should('be.visible') .and('have.attr', 'aria-selected', 'true'); @@ -255,32 +228,6 @@ describe('X-Axis Mode Toggle (inference chart)', () => { cy.get('[data-testid="chart-figure"] svg').should('contain.text', 'P90 End-to-end Latency (s)'); }); - it('switches to request-level normalized E2E at 400 output tokens', () => { - interceptDerivedMetrics(); - cy.get('[data-testid="x-axis-mode-normalized-e2e"]').click(); - cy.wait('@derivedAgenticMetrics'); - cy.get('[data-testid="x-axis-mode-normalized-e2e"]').should( - 'have.attr', - 'aria-selected', - 'true', - ); - cy.get('[data-testid="chart-figure"] h2').should( - 'contain.text', - 'P90 Normalized E2E @ 400 output tokens', - ); - cy.get('[data-testid="chart-figure"] svg').should( - 'contain.text', - 'P90 Normalized E2E @ 400 output tokens (s)', - ); - - cy.get('[data-testid="percentile-selector"]').click(); - cy.contains('[role="option"]', 'p75').click(); - cy.get('[data-testid="chart-figure"] h2').should( - 'contain.text', - 'P75 Normalized E2E @ 400 output tokens', - ); - }); - it('switches back to Interactivity', () => { cy.get('[data-testid="x-axis-mode-interactivity"]').click(); cy.get('[data-testid="x-axis-mode-interactivity"]').should( @@ -411,9 +358,6 @@ const interceptAgenticDataWithOverlay = () => { describe('X-Axis Mode Toggle — overlay path (finding #8 regression guard)', () => { before(() => { interceptAgenticDataWithOverlay(); - // Same as the main suite: keep the visit independent of the agentic default - // x-axis mode (a derived default fetches /derived-agentic-metrics on mount). - interceptDerivedMetrics(); cy.visit(`/inference?unofficialrun=${OVERLAY_RUN_ID}&i_seq=agentic-traces`, { onBeforeLoad(win) { win.localStorage.setItem('inferencex-star-modal-dismissed', String(Date.now())); @@ -462,19 +406,4 @@ describe('X-Axis Mode Toggle — overlay path (finding #8 regression guard)', () expect(total).to.be.greaterThan(0); }); }); - - it('normalized-e2e mode shows suppression banner for unofficial-run overlays', () => { - interceptDerivedMetrics(); - cy.get('[data-testid="x-axis-mode-normalized-e2e"]').click(); - cy.get('[data-testid="x-axis-mode-normalized-e2e"]').should( - 'have.attr', - 'aria-selected', - 'true', - ); - // The suppression message appears because isUnofficialRun is true and the - // mode is 'normalized-e2e' (documented in ChartDisplay.tsx ~line 640). - cy.contains( - 'Normalized E2E requires persisted per-request traces, so unofficial-run overlays are unavailable for this experimental view.', - ).should('be.visible'); - }); }); diff --git a/packages/app/src/app/api/v1/agentic-cache-keys.test.ts b/packages/app/src/app/api/v1/agentic-cache-keys.test.ts index 3a6287cf6..7177f9960 100644 --- a/packages/app/src/app/api/v1/agentic-cache-keys.test.ts +++ b/packages/app/src/app/api/v1/agentic-cache-keys.test.ts @@ -28,17 +28,12 @@ import { STATS_VERSION } from '@semianalysisai/inferencex-db/queries/agentic-agg import { REQUEST_TIMELINE_VERSION } from '@semianalysisai/inferencex-db/etl/compute-request-timeline'; import { TRACE_SERVER_METRICS_VERSION } from '@semianalysisai/inferencex-db/queries/trace-server-metrics'; -import { CACHE_KEY_PREFIX as derivedAgenticMetricsKey } from './derived-agentic-metrics/route'; import { CACHE_KEY_PREFIX as agenticAggregatesKey } from './agentic-aggregates/route'; import { CACHE_KEY_PREFIX as requestTimelineKey } from './request-timeline/route'; import { CACHE_KEY_PREFIX as traceServerMetricsKey } from './trace-server-metrics/route'; import { CACHE_KEY_PREFIX as traceHistogramsKey } from './trace-histograms/route'; describe('agentic blob-cache keys are version-derived', () => { - it('derived-agentic-metrics key embeds STATS_VERSION', () => { - expect(derivedAgenticMetricsKey).toBe(`derived-agentic-metrics-v${STATS_VERSION}`); - }); - it('agentic-aggregates key embeds STATS_VERSION', () => { expect(agenticAggregatesKey).toBe(`agentic-aggregates-v${STATS_VERSION}`); }); @@ -57,7 +52,6 @@ describe('agentic blob-cache keys are version-derived', () => { it('every key actually contains a version segment (no unversioned literals)', () => { for (const key of [ - derivedAgenticMetricsKey, agenticAggregatesKey, requestTimelineKey, traceServerMetricsKey, diff --git a/packages/app/src/app/api/v1/derived-agentic-metrics/route.ts b/packages/app/src/app/api/v1/derived-agentic-metrics/route.ts deleted file mode 100644 index 373eb370d..000000000 --- a/packages/app/src/app/api/v1/derived-agentic-metrics/route.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { STATS_VERSION } from '@semianalysisai/inferencex-db/queries/agentic-aggregates'; -import { getDb } from '@semianalysisai/inferencex-db/connection'; - -import { - getDerivedAgenticMetrics, - type DerivedAgenticMetricMap, -} from '@semianalysisai/inferencex-db/queries/derived-agentic-metrics'; - -import { cachedQuery } from '@/lib/api-cache'; - -import { idsQueryRoute } from '../id-routes'; - -export const dynamic = 'force-dynamic'; - -// blobOnly: the response is one entry per id with two numbers, but the -// derivation work parses thousands of JSONL records per blob — cache the -// computed result so a chart-refresh hits the warm path. -// -// The cache key is derived from STATS_VERSION (the payload governs the derived -// metrics read out of `aggregate_stats`). blobSet is write-once and nothing -// purges post-backfill, so a hand-written version string would serve stale -// data forever after a bump — deriving the key from the constant means a -// STATS_VERSION bump automatically rolls the cache namespace. -/** Version-derived blob-cache key namespace (exported for the key-derivation test). */ -export const CACHE_KEY_PREFIX = `derived-agentic-metrics-v${STATS_VERSION}`; - -const getCachedDerivedAgenticMetrics = cachedQuery( - (ids: number[]): Promise => getDerivedAgenticMetrics(getDb(), ids), - CACHE_KEY_PREFIX, - { blobOnly: true }, -); - -/** - * GET /api/v1/derived-agentic-metrics?ids=1,2,3 - * - * Returns per-id derived metrics computed live from the stored aiperf - * profile_export.jsonl blobs: - * - normalized_session_time_s: mean across sessions of session e2e time - * (Σ per-turn request_latency) rescaled by mean_load / session_load. - * - p90_prefill_tps_per_user: P90 of per-turn prefill TPS/user (ISL / TTFT) - * across every turn in every session. - * - p75/p90_normalized_e2e_400_s: percentile of per-request - * TTFT + 399 × observed ITL. - * - * Ids without a trace_replay blob or with unparseable records are omitted. - */ -export const GET = idsQueryRoute({ - maxIds: 200, - logLabel: 'derived agentic metrics', - fetch: getCachedDerivedAgenticMetrics, -}); diff --git a/packages/app/src/components/inference/InferenceContext.tsx b/packages/app/src/components/inference/InferenceContext.tsx index 6d5a2b935..8b837ca5a 100644 --- a/packages/app/src/components/inference/InferenceContext.tsx +++ b/packages/app/src/components/inference/InferenceContext.tsx @@ -60,12 +60,7 @@ import { } from '@/lib/exclusion'; import { filterRunsByModel, getDisplayLabel } from '@/lib/utils'; -import { - isAgenticOnlyXAxisMode, - useChartData, - X_AXIS_MODES, - type XAxisMode, -} from './hooks/useChartData'; +import { useChartData, X_AXIS_MODES, type XAxisMode } from './hooks/useChartData'; import { resolveComparisonEntries } from './utils/comparisonEntry'; import { comparisonDefaultGroup, @@ -610,40 +605,23 @@ export function InferenceProvider({ }, [labelScenarioKind, sequenceResolved, getUrlParam]); // Reconcile the x-axis mode with the scenario kind: - // - On mount with no `i_xmode` URL param: snap to the kind's natural default - // (interactivity for both agentic and fixed-sequence scenarios). The state was initialized - // to a SSR-stable constant so server and client render the same DOM; this - // effect fixes it up after hydration. - // - When the user later switches sequence kinds: snap to the new kind's - // natural default (the prior selection was for a different kind, so it - // doesn't carry over). + // - On mount with no `i_xmode` URL param: snap to the natural default + // (interactivity for both agentic and fixed-sequence scenarios). The state + // was initialized to a SSR-stable constant so server and client render the + // same DOM; this effect fixes it up after hydration. + // - When the user later switches sequence kinds: snap back to that default + // (the prior selection was for a different kind, so it doesn't carry over). const lastSeqKindRef = useRef | null>(null); useEffect(() => { const kind = sequenceKind(effectiveSequence); const isInitialMount = lastSeqKindRef.current === null; - const isAgenticOnlyMode = isAgenticOnlyXAxisMode(selectedXAxisMode); - // On a stale render where kind hasn't changed, bail unless the current - // mode is agentic-only and we just landed on a fixed-seq scenario — in - // that case force the snap so the chart doesn't try to plot trace-derived - // metrics against rows that have no trace_replay. - if (!isInitialMount && lastSeqKindRef.current === kind) { - if (kind === 'fixed-seq' && isAgenticOnlyMode) { - handleSetXAxisMode('interactivity'); - } - return; - } + // Stale render where the kind hasn't changed — nothing to reconcile. + if (!isInitialMount && lastSeqKindRef.current === kind) return; lastSeqKindRef.current = kind; - if ( - isInitialMount && - xAxisModeFromUrlRef.current && - !(kind === 'fixed-seq' && isAgenticOnlyMode) - ) { - // URL-restored agentic-only mode on a fixed-seq sequence makes no sense - // — fall through to the default snap below. - return; - } + // A URL-restored mode wins on first mount; later kind switches reset it. + if (isInitialMount && xAxisModeFromUrlRef.current) return; handleSetXAxisMode('interactivity'); - }, [effectiveSequence, selectedXAxisMode, handleSetXAxisMode]); + }, [effectiveSequence, handleSetXAxisMode]); // Reconcile selectedE2eXAxisMetric whenever the mode, sequence kind, or // agentic percentile changes. For fixed-seq the JSONB only carries diff --git a/packages/app/src/components/inference/hooks/useChartData.ts b/packages/app/src/components/inference/hooks/useChartData.ts index c125e9e9e..c5593ee98 100644 --- a/packages/app/src/components/inference/hooks/useChartData.ts +++ b/packages/app/src/components/inference/hooks/useChartData.ts @@ -39,41 +39,18 @@ import { /** * Chart x-axis variant selected by the mode buttons above the plot. This is * the single definition — InferenceContext (URL/state) and ChartDisplay - * (buttons, derived-metric remapping) import it from here. + * (buttons) import it from here. */ -export type XAxisMode = - | 'ttft' - | 'e2e' - | 'normalized-e2e' - | 'interactivity' - | 'session-time' - | 'prefill-tps'; - -export const X_AXIS_MODES: readonly XAxisMode[] = [ - 'ttft', - 'e2e', - 'normalized-e2e', - 'interactivity', - 'session-time', - 'prefill-tps', -]; +export type XAxisMode = 'ttft' | 'e2e' | 'interactivity'; -/** - * Modes whose x metric is derived from persisted per-request traces — - * these only exist for agentic scenarios (fixed-seq rows have no - * trace_replay blob to derive them from). - */ -export function isAgenticOnlyXAxisMode(mode: XAxisMode): boolean { - return mode === 'normalized-e2e' || mode === 'session-time' || mode === 'prefill-tps'; -} +export const X_AXIS_MODES: readonly XAxisMode[] = ['ttft', 'e2e', 'interactivity']; /** * Compute the set of benchmark_results.id values that sit on the * (e2e_latency, y) Pareto frontier within each (hwKey, precision, date) - * group. Used to restrict the non-e2e xmode charts (ttft, interactivity, - * session-time, prefill-tps) so they show *only* the points that win on - * end-to-end latency — preventing benchmark-hacking where a config tops - * one axis while tanking the other. + * group. Used to restrict the non-e2e xmode charts (ttft, interactivity) so + * they show *only* the points that win on end-to-end latency — preventing + * benchmark-hacking where a config tops one axis while tanking the other. * * Returns null when the y-metric has no roofline direction declared on * the e2e chart (caller falls back to no filtering in that case). @@ -220,7 +197,7 @@ export function useChartData( /** * Current x-axis mode. When set to anything other than 'e2e', the displayed * data is filtered to the (e2e-latency, y) Pareto frontier so the ttft / - * interactivity / session-time / prefill-tps charts show only points that + * interactivity charts show only points that * also win on end-to-end latency — preventing benchmark-hacking where a * config tops one metric while tanking the other. The 'e2e' mode is the * source of truth and keeps the full point set. @@ -529,7 +506,7 @@ export function useChartData( // non-optimal configs from view. // // Fixed-seq workloads keep the existing per-axis Pareto since - // there's no separate "session-time" notion of total latency — + // there's no separate session-level notion of total latency — // their e2e IS the request latency, so a TTFT hack there reads // honestly on e2e too. The anti-hack constraint is specifically // about multi-turn agentic where TTFT measures a tiny fraction diff --git a/packages/app/src/components/inference/types.ts b/packages/app/src/components/inference/types.ts index 8bdc04333..ae73710bd 100644 --- a/packages/app/src/components/inference/types.ts +++ b/packages/app/src/components/inference/types.ts @@ -258,7 +258,7 @@ export interface InferenceData extends Partial void; + selectedXAxisMode: 'ttft' | 'e2e' | 'interactivity'; + setSelectedXAxisMode: (mode: 'ttft' | 'e2e' | 'interactivity') => void; scaleType: 'auto' | 'linear' | 'log'; setScaleType: (type: 'auto' | 'linear' | 'log') => void; /** Coarse vendor / framework / agg-disagg / mtp-stp filters applied to the chart point set. */ diff --git a/packages/app/src/components/inference/ui/ChartDisplay.tsx b/packages/app/src/components/inference/ui/ChartDisplay.tsx index 76d3e6f41..6329df644 100644 --- a/packages/app/src/components/inference/ui/ChartDisplay.tsx +++ b/packages/app/src/components/inference/ui/ChartDisplay.tsx @@ -1,8 +1,5 @@ 'use client'; -import { - DISPLAY_MODEL_TO_DB, - NORMALIZED_E2E_OUTPUT_TOKENS, -} from '@semianalysisai/inferencex-constants'; +import { DISPLAY_MODEL_TO_DB } from '@semianalysisai/inferencex-constants'; import { track } from '@/lib/analytics'; import dynamic from 'next/dynamic'; import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; @@ -17,10 +14,7 @@ import type { OverlayData, TrendDataPoint, } from '@/components/inference/types'; -import { - processOverlayChartData, - selectUnofficialOverlayForMode, -} from '@/components/inference/utils'; +import { processOverlayChartData } from '@/components/inference/utils'; import { isRunComparisonEntry, makeRunComparisonEntry, @@ -59,12 +53,7 @@ import { sequenceKind, } from '@/lib/data-mappings'; import { useComparisonChangelogs } from '@/hooks/api/use-comparison-changelogs'; -import { - useDerivedAgenticMetrics, - type DerivedAgenticMetric, -} from '@/hooks/api/use-derived-agentic-metrics'; -import { isAgenticOnlyXAxisMode, type XAxisMode } from '@/components/inference/hooks/useChartData'; -import { isPersistedBenchmarkId } from '@/lib/benchmark-id'; +import type { XAxisMode } from '@/components/inference/hooks/useChartData'; import { useTrendData } from '@/components/inference/hooks/useTrendData'; import { getHardwareConfig, hardwareKeyMatchesAnyBase } from '@/lib/constants'; import { useLocale } from '@/lib/use-locale'; @@ -95,8 +84,6 @@ const STRINGS = { sourceUnofficial: 'Source: UNOFFICIAL', sourceOfficial: 'Source: SemiAnalysis InferenceX™', updated: 'Updated:', - normalizedE2eDisclaimer: - 'Normalized E2E requires persisted per-request traces, so unofficial-run overlays are unavailable for this experimental view.', selectDateRange: 'Select a date range or add a run to view GPU comparison', performanceOverTime: 'Performance Over Time', performanceOverTimeDesc: @@ -114,8 +101,6 @@ const STRINGS = { sourceUnofficial: '来源：非官方', sourceOfficial: '来源：SemiAnalysis InferenceX™', updated: '更新时间：', - normalizedE2eDisclaimer: - 'Normalized E2E 需要持久化的逐请求 trace 数据，因此该实验性视图不支持非官方运行覆盖。', selectDateRange: '请选择日期范围或添加运行以查看 GPU 对比', performanceOverTime: '性能趋势', performanceOverTimeDesc: '双击散点图上的数据点以追踪配置随时间的变化。', @@ -149,61 +134,8 @@ const X_AXIS_MODE_BUTTONS: { value: XAxisMode; label: string; labelZh: string }[ { value: 'interactivity', label: 'Interactivity', labelZh: '交互性' }, { value: 'e2e', label: 'E2E Latency', labelZh: '端到端延迟' }, { value: 'ttft', label: 'TTFT', labelZh: 'TTFT' }, - { value: 'normalized-e2e', label: 'Normalized E2E', labelZh: 'Normalized E2E' }, - { value: 'session-time', label: 'Session Time', labelZh: '会话时长' }, - { value: 'prefill-tps', label: 'Prefill TPS / user', labelZh: 'Prefill TPS / user' }, ]; -/** - * Presentation + data plumbing for the trace-derived x-axis modes (the - * agentic-only modes). One spec per mode keeps the x-label, chart heading, - * roofline corner, and derived-metric accessor in sync instead of scattering - * `selectedXAxisMode === …` conditionals through the render. - */ -interface DerivedXModeSpec { - xLabel: (percentileLabel: string) => string; - /** Chinese x-label; omit to reuse the English one (technical terms). */ - xLabelZh?: (percentileLabel: string) => string; - /** Chart heading suffix ("vs. …") shown above the plot. */ - heading: (percentileLabel: string) => string; - /** Chinese heading suffix; omit to reuse the English one. */ - headingZh?: (percentileLabel: string) => string; - rooflineCorner: 'upper_right' | 'upper_left'; - /** Pull the raw metric for this mode off the derived-metrics payload. */ - value: (m: DerivedAgenticMetric | undefined, percentile: string) => number | null | undefined; - /** Convert the raw metric to the plotted x value. */ - toX: (raw: number) => number; -} - -const DERIVED_X_MODE_SPECS: Partial> = { - 'session-time': { - xLabel: () => 'Mean Normalized Session Time (min)', - xLabelZh: () => '平均归一化会话时长（min）', - heading: () => 'vs. Mean Normalized Session Time', - headingZh: () => 'vs. 平均归一化会话时长', - rooflineCorner: 'upper_right', - value: (m) => m?.normalized_session_time_s, - toX: (raw) => raw / 60, - }, - 'normalized-e2e': { - xLabel: (pctl) => `${pctl} Normalized E2E @ ${NORMALIZED_E2E_OUTPUT_TOKENS} output tokens (s)`, - xLabelZh: (pctl) => `${pctl} Normalized E2E @ ${NORMALIZED_E2E_OUTPUT_TOKENS} 输出 token（s）`, - heading: (pctl) => `vs. ${pctl} Normalized E2E @ ${NORMALIZED_E2E_OUTPUT_TOKENS} output tokens`, - headingZh: (pctl) => `vs. ${pctl} Normalized E2E @ ${NORMALIZED_E2E_OUTPUT_TOKENS} 输出 token`, - rooflineCorner: 'upper_right', - value: (m, percentile) => - percentile === 'p75' ? m?.p75_normalized_e2e_400_s : m?.p90_normalized_e2e_400_s, - toX: (raw) => raw, - }, - 'prefill-tps': { - xLabel: () => 'P90 Prefill TPS per user (tok/s)', - heading: () => 'vs. P90 Prefill TPS / user', - rooflineCorner: 'upper_left', - value: (m) => m?.p90_prefill_tps_per_user, - toX: (raw) => raw, - }, -}; - const VIEW_MODE_OPTIONS: SegmentedToggleOption[] = [ { value: 'chart', @@ -263,9 +195,6 @@ export default function ChartDisplay() { totalDatesQueried, } = useComparisonChangelogs(selectedGPUs, selectedDateRange, dateRangeAvailableDates); - const [mounted, setMounted] = useState(false); - useEffect(() => setMounted(true), []); - const modelDbKeys = useMemo( () => DISPLAY_MODEL_TO_DB[selectedModel] ?? [selectedModel], [selectedModel], @@ -628,258 +557,191 @@ export default function ChartDisplay() { }, [effectiveGraphs, selectedXAxisMode]); const isAgenticSequence = sequenceKind(selectedSequence) === 'agentic'; - const useDerived = isAgenticSequence && isAgenticOnlyXAxisMode(selectedXAxisMode); - const derivedTargetIds = useMemo(() => { - if (!useDerived) return [] as number[]; - const ids = new Set(); - for (const graph of visibleGraphs) { - for (const point of graph.data) { - // Overlay-only agentic points carry no persisted id — skip them so we - // never request `?ids=0`/`?ids=NaN` (which 400s and errors the chart). - if (point.benchmark_type === 'agentic_traces' && isPersistedBenchmarkId(point.id)) { - ids.add(point.id); - } - } - } - return [...ids]; - }, [useDerived, visibleGraphs]); - const derivedQuery = useDerivedAgenticMetrics(derivedTargetIds, useDerived); - const derivedMetrics = derivedQuery.data; - const isDerivedLoading = - useDerived && - derivedTargetIds.length > 0 && - (derivedQuery.isPending || derivedQuery.isFetching) && - !derivedMetrics; - - // Set only when the user is on a derived (agentic-only) x-axis mode; the - // specs are module constants so this is referentially stable per mode. - const derivedSpec = useDerived ? DERIVED_X_MODE_SPECS[selectedXAxisMode] : undefined; - - const renderableGraphs = useMemo(() => { - if (!derivedSpec) return visibleGraphs; - if (!derivedMetrics) return visibleGraphs.map((graph) => ({ ...graph, data: [] })); - const xLabelFn = - locale === 'zh' && derivedSpec.xLabelZh ? derivedSpec.xLabelZh : derivedSpec.xLabel; - const xLabel = xLabelFn(selectedPercentile.toUpperCase()); - return visibleGraphs.map((graph) => { - const chartDefinition = { - ...graph.chartDefinition, - x_label: xLabel, - y_latency_limit: undefined, - [`${selectedYAxisMetric}_roofline` as keyof typeof graph.chartDefinition]: - derivedSpec.rooflineCorner, - }; - const data = graph.data - .map((point) => { - if (!isPersistedBenchmarkId(point.id)) return null; - const raw = derivedSpec.value(derivedMetrics[point.id], selectedPercentile); - if (raw === null || raw === undefined || !Number.isFinite(raw)) return null; - return { ...point, x: derivedSpec.toX(raw) }; - }) - .filter((point): point is NonNullable => point !== null); - return { ...graph, chartDefinition, data }; - }); - }, [derivedSpec, visibleGraphs, derivedMetrics, selectedYAxisMetric, selectedPercentile, locale]); - - const displayGraphs = - isFirstLoad || isDerivedLoading - ? [ - - - - - , - ] - : renderableGraphs.length === 0 - ? [] - : renderableGraphs.map((graph, graphIndex) => { - const isTimelineMode = Boolean( - selectedDateRange.startDate && selectedDateRange.endDate && selectedGPUs.length > 0, - ); - const replayAvailable = getViewMode(graphIndex) === 'chart' && !isTimelineMode; - return ( -
-
- + + + + , + ] + : visibleGraphs.length === 0 + ? [] + : visibleGraphs.map((graph, graphIndex) => { + const isTimelineMode = Boolean( + selectedDateRange.startDate && selectedDateRange.endDate && selectedGPUs.length > 0, + ); + const replayAvailable = getViewMode(graphIndex) === 'chart' && !isTimelineMode; + return ( +
+
+ handleViewModeChange(graphIndex, v)} + ariaLabel={t.viewMode} + testId={`inference-view-toggle-${graphIndex}`} + /> + } + hideImageExport={getViewMode(graphIndex) === 'table'} + setIsLegendExpanded={setIsLegendExpanded} + exportFileName={`InferenceX_${selectedModel}_${graph.chartDefinition.chartType}`} + onExportMp4={ + replayAvailable ? () => replayHandlesRef.current[graphIndex]?.open() : undefined + } + onExportCsv={() => { + const candidateVisibleData = isTimelineMode + ? graph.data.filter((d) => activeDates.has(`${d.date}_${d.hwKey}`)) + : graph.data; + const overlay = overlayDataByChartType[graph.chartDefinition.chartType]; + const { officialRows: visibleData, overlayRows: visibleOverlayRowsForExport } = isTimelineMode - ? 'gpu_timeseries' - : graph.chartDefinition.chartType === 'e2e' - ? 'latency' - : 'interactivity' - } - leadingControls={ - handleViewModeChange(graphIndex, v)} - ariaLabel={t.viewMode} - testId={`inference-view-toggle-${graphIndex}`} - /> - } - hideImageExport={getViewMode(graphIndex) === 'table'} - setIsLegendExpanded={setIsLegendExpanded} - exportFileName={`InferenceX_${selectedModel}_${graph.chartDefinition.chartType}`} - onExportMp4={ - replayAvailable - ? () => replayHandlesRef.current[graphIndex]?.open() - : undefined - } - onExportCsv={() => { - const candidateVisibleData = isTimelineMode - ? graph.data.filter((d) => activeDates.has(`${d.date}_${d.hwKey}`)) - : graph.data; - const overlay = selectUnofficialOverlayForMode( - selectedXAxisMode, - graph.chartDefinition.chartType, - overlayDataByChartType, - ); - const { - officialRows: visibleData, - overlayRows: visibleOverlayRowsForExport, - } = isTimelineMode ? { officialRows: candidateVisibleData, overlayRows: [] } : visibleComparisonRows(candidateVisibleData, overlay); - const { headers, rows } = inferenceChartToCsv( - visibleData, - graph.model, - graph.sequence, - ); - // Match warnings against the same series the chart annotates, - // including visible unofficial-run overlay series. - const issueNotes = matchKnownConfigIssues(graph.model, [ - ...visibleData, - ...visibleOverlayRowsForExport, - ]).map((issue) => - knownIssueCsvNote(issue, getDisplayLabel(getHardwareConfig(issue.hwKey))), - ); - exportToCsv( - `InferenceX_${selectedModel}_${graph.chartDefinition.chartType}`, - headers, - rows, - issueNotes, - ); - }} - /> - - {(() => { - const chartCaption = ( - <> -

- {metricTitle(graph.chartDefinition, selectedYAxisMetric, locale)}{' '} - {(() => { - // For Input metrics with dynamic x-axis, use dynamic heading. - // Classify off the ENGLISH title — the localized one has no - // 'input' substring to match on zh pages. - const isInputMetric = metricTitle( - graph.chartDefinition, - selectedYAxisMetric, - 'en', - ) - .toLowerCase() - .includes('input'); - if ( - graph.chartDefinition.chartType === 'interactivity' && - isInputMetric && - selectedXAxisMetric - ) { - if (selectedXAxisMetric === 'p99_ttft') { - return t.vsTtft('P99'); - } else if (selectedXAxisMetric === 'median_ttft') { - return t.vsTtft('Median'); - } + const { headers, rows } = inferenceChartToCsv( + visibleData, + graph.model, + graph.sequence, + ); + // Match warnings against the same series the chart annotates, + // including visible unofficial-run overlay series. + const issueNotes = matchKnownConfigIssues(graph.model, [ + ...visibleData, + ...visibleOverlayRowsForExport, + ]).map((issue) => + knownIssueCsvNote(issue, getDisplayLabel(getHardwareConfig(issue.hwKey))), + ); + exportToCsv( + `InferenceX_${selectedModel}_${graph.chartDefinition.chartType}`, + headers, + rows, + issueNotes, + ); + }} + /> + + {(() => { + const chartCaption = ( + <> +

+ {metricTitle(graph.chartDefinition, selectedYAxisMetric, locale)}{' '} + {(() => { + // For Input metrics with dynamic x-axis, use dynamic heading. + // Classify off the ENGLISH title — the localized one has no + // 'input' substring to match on zh pages. + const isInputMetric = metricTitle( + graph.chartDefinition, + selectedYAxisMetric, + 'en', + ) + .toLowerCase() + .includes('input'); + if ( + graph.chartDefinition.chartType === 'interactivity' && + isInputMetric && + selectedXAxisMetric + ) { + if (selectedXAxisMetric === 'p99_ttft') { + return t.vsTtft('P99'); + } else if (selectedXAxisMetric === 'median_ttft') { + return t.vsTtft('Median'); } + } - // The e2e chart heading follows the branch-level x-axis mode - // selector, including agentic-only derived metrics. - if (graph.chartDefinition.chartType === 'e2e') { - const modeSpec = DERIVED_X_MODE_SPECS[selectedXAxisMode]; - if (modeSpec) { - const heading = - locale === 'zh' && modeSpec.headingZh - ? modeSpec.headingZh - : modeSpec.heading; - return heading(selectedPercentile.toUpperCase()); - } - if (selectedE2eXAxisMetric?.endsWith('_ttft')) { - const percentile = selectedE2eXAxisMetric.replace(/_ttft$/u, ''); - const word = - percentile === 'median' ? 'Median' : percentile.toUpperCase(); - return t.vsTtft(word); - } - return isAgenticSequence - ? t.vsE2eLatency(selectedPercentile.toUpperCase()) - : t.vsE2eLatency(); + // The e2e chart heading follows the branch-level x-axis + // mode selector. + if (graph.chartDefinition.chartType === 'e2e') { + if (selectedE2eXAxisMetric?.endsWith('_ttft')) { + const percentile = selectedE2eXAxisMetric.replace(/_ttft$/u, ''); + const word = + percentile === 'median' ? 'Median' : percentile.toUpperCase(); + return t.vsTtft(word); } + return isAgenticSequence + ? t.vsE2eLatency(selectedPercentile.toUpperCase()) + : t.vsE2eLatency(); + } - // Fall back to configured heading - const configured = - graph.chartDefinition[ - `${selectedYAxisMetric}_heading` as keyof typeof graph.chartDefinition - ] || graph.chartDefinition.heading; - return locale === 'zh' ? zhHeading(String(configured)) : configured; - })()} -

-

- {getModelLabel(graph.model as Model)} •{' '} - {selectedPrecisions - .map((prec) => getPrecisionLabel(prec as Precision)) - .join(', ')}{' '} - • {getSequenceLabel(graph.sequence as Sequence)} •{' '} - {isUnofficialRun ? t.sourceUnofficial : t.sourceOfficial} - {selectedRunDate && ( - <> - {' '} - • {t.updated}{' '} - {new Date(`${selectedRunDate}T00:00:00Z`).toLocaleDateString( - locale === 'zh' ? 'zh-CN' : 'en-US', - { - year: 'numeric', - month: '2-digit', - day: '2-digit', - timeZone: 'UTC', - }, - )} - - )} -

- - {isUnofficialRun && selectedXAxisMode === 'normalized-e2e' && ( -

- {t.normalizedE2eDisclaimer} -

+ // Fall back to configured heading + const configured = + graph.chartDefinition[ + `${selectedYAxisMetric}_heading` as keyof typeof graph.chartDefinition + ] || graph.chartDefinition.heading; + return locale === 'zh' ? zhHeading(String(configured)) : configured; + })()} +

+

+ {getModelLabel(graph.model as Model)} •{' '} + {selectedPrecisions + .map((prec) => getPrecisionLabel(prec as Precision)) + .join(', ')}{' '} + • {getSequenceLabel(graph.sequence as Sequence)} •{' '} + {isUnofficialRun ? t.sourceUnofficial : t.sourceOfficial} + {selectedRunDate && ( + <> + {' '} + • {t.updated}{' '} + {new Date(`${selectedRunDate}T00:00:00Z`).toLocaleDateString( + locale === 'zh' ? 'zh-CN' : 'en-US', + { + year: 'numeric', + month: '2-digit', + day: '2-digit', + timeZone: 'UTC', + }, + )} + )} - +

+ + + + ); + + if (getViewMode(graphIndex) === 'table') { + const overlay = overlayDataByChartType[graph.chartDefinition.chartType]; + const { officialRows, overlayRows } = visibleComparisonRows( + graph.data, + overlay, + ); + return ( + <> + {chartCaption} + ); + } - if (getViewMode(graphIndex) === 'table') { - const overlay = selectUnofficialOverlayForMode( - selectedXAxisMode, - graph.chartDefinition.chartType, - overlayDataByChartType, - ); - const { officialRows, overlayRows } = visibleComparisonRows( - graph.data, - overlay, - ); - return ( - <> - {chartCaption} - - - ); - } - - return selectedGPUs.length > 0 && - ((selectedDateRange.startDate && selectedDateRange.endDate) || - selectedDates.length > 0) ? ( - 0 && + ((selectedDateRange.startDate && selectedDateRange.endDate) || + selectedDates.length > 0) ? ( + + ) : ( +
+ - ) : ( -
- - {selectedGPUs.length > 0 && - (!selectedDateRange.startDate || !selectedDateRange.endDate) && - selectedDates.length === 0 && ( -
-

- {t.selectDateRange} -

-
- )} -
- ); - })()} - {replayAvailable && ( - { - replayHandlesRef.current[graphIndex] = handle; - }} - parentChartId={`chart-${graphIndex}`} - chartDefinition={graph.chartDefinition} - yLabel={metricLabel(graph.chartDefinition, selectedYAxisMetric, locale)} - xLabel={graph.chartDefinition.x_label} - /> - )} - -
-
- ); - }); + {selectedGPUs.length > 0 && + (!selectedDateRange.startDate || !selectedDateRange.endDate) && + selectedDates.length === 0 && ( +
+

+ {t.selectDateRange} +

+
+ )} + + ); + })()} + {replayAvailable && ( + { + replayHandlesRef.current[graphIndex] = handle; + }} + parentChartId={`chart-${graphIndex}`} + chartDefinition={graph.chartDefinition} + yLabel={metricLabel(graph.chartDefinition, selectedYAxisMetric, locale)} + xLabel={graph.chartDefinition.x_label} + /> + )} + +
+
+ ); + }); return (
@@ -1001,12 +847,7 @@ export default function ChartDisplay() { data-testid="x-axis-mode-buttons" className="flex-wrap justify-center gap-x-1 gap-y-1.5 sm:gap-x-1.5" > - {X_AXIS_MODE_BUTTONS.filter(({ value }) => { - if (!isAgenticOnlyXAxisMode(value)) return true; - // Before mount, render all buttons so SSR and first client render match. - if (!mounted) return true; - return isAgenticSequence; - }).map(({ value, label, labelZh }) => ( + {X_AXIS_MODE_BUTTONS.map(({ value, label, labelZh }) => ( { - const overlays = { e2e: { id: 'e2e' }, interactivity: { id: 'interactivity' } }; - - it('suppresses raw unofficial E2E data for normalized E2E mode', () => { - expect(selectUnofficialOverlayForMode('normalized-e2e', 'e2e', overlays)).toBeNull(); - }); - - it('preserves matching unofficial overlays for supported modes', () => { - expect(selectUnofficialOverlayForMode('e2e', 'e2e', overlays)).toBe(overlays.e2e); - expect(selectUnofficialOverlayForMode('interactivity', 'interactivity', overlays)).toBe( - overlays.interactivity, - ); - }); -}); +import { filterDataByCostLimit, processOverlayChartData } from '@/components/inference/utils'; // --------------------------------------------------------------------------- // fixture factories diff --git a/packages/app/src/components/inference/utils.ts b/packages/app/src/components/inference/utils.ts index 2b8074d9c..e88e5b0da 100644 --- a/packages/app/src/components/inference/utils.ts +++ b/packages/app/src/components/inference/utils.ts @@ -10,20 +10,6 @@ import { resolveXAxisField } from '@/components/inference/utils/resolveXAxisFiel import type { ChartDefinition, InferenceData, YAxisMetricKey } from './types'; -/** - * Select the matching unofficial-run overlay for a chart mode. Normalized E2E - * is intentionally excluded: unofficial benchmark rows do not include the - * persisted per-request trace needed to normalize before taking percentiles. - */ -export function selectUnofficialOverlayForMode( - xAxisMode: string, - chartType: 'e2e' | 'interactivity', - overlays: { e2e: T | null; interactivity: T | null }, -): T | null { - if (xAxisMode === 'normalized-e2e') return null; - return overlays[chartType]; -} - /** * Filters data points based on cost limits defined in the chart definition. * Only applies filtering for cost-related metrics, and only filters based on diff --git a/packages/app/src/components/inference/utils/e2eFrontier.ts b/packages/app/src/components/inference/utils/e2eFrontier.ts index 588afdd3e..65c3434a8 100644 --- a/packages/app/src/components/inference/utils/e2eFrontier.ts +++ b/packages/app/src/components/inference/utils/e2eFrontier.ts @@ -2,7 +2,7 @@ * @file e2eFrontier.ts * @description Shared seed for the anti-benchmark-hacking roofline restriction. * - * On the non-e2e xmode charts (interactivity, ttft, session-time, prefill-tps) + * On the non-e2e xmode charts (interactivity, ttft) * the roofline is restricted to the configs that ALSO win on end-to-end latency, * so a config can't top interactivity while tanking decode (or vice versa). Both * the official path (`useChartData` → benchmark ids → `isOnE2eFrontier`) and the diff --git a/packages/app/src/hooks/api/use-derived-agentic-metrics.test.ts b/packages/app/src/hooks/api/use-derived-agentic-metrics.test.ts deleted file mode 100644 index 2e54f4187..000000000 --- a/packages/app/src/hooks/api/use-derived-agentic-metrics.test.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { describe, expect, it } from 'vitest'; - -import { chunkDerivedAgenticMetricIds } from './use-derived-agentic-metrics'; - -describe('chunkDerivedAgenticMetricIds', () => { - it('keeps every id while respecting the API limit', () => { - const ids = Array.from({ length: 401 }, (_, index) => index + 1); - const chunks = chunkDerivedAgenticMetricIds(ids); - - expect(chunks.map((chunk) => chunk.length)).toEqual([200, 200, 1]); - expect(chunks.flat()).toEqual(ids); - }); -}); diff --git a/packages/app/src/hooks/api/use-derived-agentic-metrics.ts b/packages/app/src/hooks/api/use-derived-agentic-metrics.ts deleted file mode 100644 index 388563d96..000000000 --- a/packages/app/src/hooks/api/use-derived-agentic-metrics.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { bulkIdsFetcher, useBulkIdsQuery } from './benchmark-id-query'; - -export interface DerivedAgenticMetric { - id: number; - /** Mean across sessions of e2e time (Σ per-turn request_latency) rescaled - * by mean_load / session_load. Null when the JSONL had no usable records. */ - normalized_session_time_s: number | null; - /** P90 of per-turn ISL/TTFT across every turn in every session. - * Null when no prefill rates could be computed. */ - p90_prefill_tps_per_user: number | null; - /** P75 normalized per-request E2E at a fixed 400-token output length. */ - p75_normalized_e2e_400_s: number | null; - /** P90 normalized per-request E2E at a fixed 400-token output length. */ - p90_normalized_e2e_400_s: number | null; -} - -export type DerivedAgenticMetricMap = Record; - -const MAX_IDS_PER_REQUEST = 200; - -export function chunkDerivedAgenticMetricIds(ids: number[]): number[][] { - const chunks: number[][] = []; - for (let i = 0; i < ids.length; i += MAX_IDS_PER_REQUEST) { - chunks.push(ids.slice(i, i + MAX_IDS_PER_REQUEST)); - } - return chunks; -} - -const fetchChunk = bulkIdsFetcher('derived-agentic-metrics'); - -// Unlike the other bulk endpoints, dashboards can put >200 agentic points on -// screen at once, so this fetcher splits the id set across parallel requests -// to stay under the route's MAX_IDS_PER_REQUEST. -async function fetchDerivedAgenticMetrics( - ids: number[], - signal?: AbortSignal, -): Promise { - if (ids.length === 0) return {}; - const maps = await Promise.all( - chunkDerivedAgenticMetricIds(ids).map((chunk) => fetchChunk(chunk, signal)), - ); - return Object.assign({}, ...maps) as DerivedAgenticMetricMap; -} - -/** - * Fetch per-id derived agentic metrics (session time + p90 prefill TPS/user) - * computed live from the stored aiperf profile_export.jsonl. Used to drive - * the "Session Time" and "Prefill TPS/user" chart variants. - * - * Ids without a trace_replay blob (older or non-aiperf agentic runs) are - * silently omitted from the response. - */ -export function useDerivedAgenticMetrics(ids: number[], enabled = true) { - return useBulkIdsQuery('derived-agentic-metrics', ids, enabled, fetchDerivedAgenticMetrics); -} diff --git a/packages/app/src/lib/benchmark-id.ts b/packages/app/src/lib/benchmark-id.ts index b1ccb8bcc..6b1030df3 100644 --- a/packages/app/src/lib/benchmark-id.ts +++ b/packages/app/src/lib/benchmark-id.ts @@ -8,7 +8,7 @@ * * A bare `typeof id === 'number'` check is NOT enough: `NaN` and `0` are both * `number` yet neither is a real row. Passing them to the id-keyed endpoints - * (`/api/v1/derived-agentic-metrics?ids=…`, `…?id=…`) yields a 400 (the routes + * (`/api/v1/agentic-aggregates?ids=…`, `…?id=…`) yields a 400 (the routes * filter to `Number.isFinite(n) && n > 0`), and building an * `/inference/agentic/` link out of one points at a non-existent row. * diff --git a/packages/constants/src/agentic.ts b/packages/constants/src/agentic.ts deleted file mode 100644 index 42eab306e..000000000 --- a/packages/constants/src/agentic.ts +++ /dev/null @@ -1,2 +0,0 @@ -/** Fixed output length used by the experimental normalized-E2E chart metric. */ -export const NORMALIZED_E2E_OUTPUT_TOKENS = 400; diff --git a/packages/constants/src/index.ts b/packages/constants/src/index.ts index 7d3d6783a..e767e500c 100644 --- a/packages/constants/src/index.ts +++ b/packages/constants/src/index.ts @@ -1,4 +1,3 @@ -export * from './agentic'; export * from './framework-aliases'; export * from './github'; export * from './gpu-keys'; diff --git a/packages/db/src/backfill-aggregate-stats.ts b/packages/db/src/backfill-aggregate-stats.ts index 8aaa1214c..a50f6ea02 100644 --- a/packages/db/src/backfill-aggregate-stats.ts +++ b/packages/db/src/backfill-aggregate-stats.ts @@ -94,12 +94,16 @@ async function main(): Promise { } let stats: AggregateStats; - if (row.aggregate_stats?.version === 3) { + // v3 onwards → current is a profile-only change (the server-derived + // fields haven't changed since v3), so skip re-reading the huge server + // blob and carry its KV/prefix distributions forward. + const storedVersion = row.aggregate_stats?.version; + if (storedVersion !== undefined && storedVersion >= 3 && storedVersion < STATS_VERSION) { const profileStats = await computeAggregateStats({ profileBlob: row.profile_export_jsonl_gz, serverBlob: null, }); - stats = mergeProfileStatsUpgrade(row.aggregate_stats, profileStats); + stats = mergeProfileStatsUpgrade(row.aggregate_stats!, profileStats); } else { const [serverRow] = await sql<{ server_metrics_json_gz: Buffer | null }[]>` select server_metrics_json_gz diff --git a/packages/db/src/etl/compute-aggregate-stats.test.ts b/packages/db/src/etl/compute-aggregate-stats.test.ts index 7b745c098..1a237664d 100644 --- a/packages/db/src/etl/compute-aggregate-stats.test.ts +++ b/packages/db/src/etl/compute-aggregate-stats.test.ts @@ -66,12 +66,9 @@ describe('computeAggregateStats', () => { expect(stats.osl).toBeNull(); expect(stats.kvCacheUtil).toBeNull(); expect(stats.prefixCacheHitRate).toBeNull(); - expect(stats.normalizedSessionTimeS).toBeNull(); - expect(stats.p90PrefillTpsPerUser).toBeNull(); - expect(stats.normalizedE2e400).toBeNull(); }); - it('computes ISL/OSL percentiles + derived metrics from the profile blob', async () => { + it('computes ISL/OSL percentiles from the profile blob', async () => { const profileBlob = makeProfileBlob([ { isl: 100, osl: 50, rl: 1000, ttft: 100 }, { isl: 200, osl: 75, rl: 2000, ttft: 200 }, @@ -87,16 +84,6 @@ describe('computeAggregateStats', () => { // Server-side metrics still null when there's no server blob. expect(stats.kvCacheUtil).toBeNull(); expect(stats.prefixCacheHitRate).toBeNull(); - - // Derived: prefill TPS per turn = isl / (ttft/1000) = 1000 for each, so p90 = 1000. - expect(stats.p90PrefillTpsPerUser).toBeCloseTo(1000, 6); - // Normalized session time: T̃_i = T_i × (mean_load / load_i), then mean. - // loads = [150, 275, 400], mean_load = 275 - // scaled times (s) = [1×275/150, 2×275/275, 3×275/400] = [1.8333, 2, 2.0625] - // mean ≈ 1.9653 - expect(stats.normalizedSessionTimeS).toBeCloseTo(1.9653, 3); - expect(stats.normalizedE2e400?.n).toBe(3); - expect(stats.normalizedE2e400?.p90).toBeGreaterThan(0); }); it('computes KV util + prefix hit rate from the server blob alone', async () => { @@ -112,9 +99,6 @@ describe('computeAggregateStats', () => { // Profile-derived metrics absent. expect(stats.isl).toBeNull(); expect(stats.osl).toBeNull(); - expect(stats.normalizedSessionTimeS).toBeNull(); - expect(stats.p90PrefillTpsPerUser).toBeNull(); - expect(stats.normalizedE2e400).toBeNull(); }); it('tolerates a malformed profile blob by leaving its metrics null', async () => { @@ -123,9 +107,6 @@ describe('computeAggregateStats', () => { const stats = await computeAggregateStats({ profileBlob: garbage, serverBlob: null }); expect(stats.isl).toBeNull(); expect(stats.osl).toBeNull(); - expect(stats.normalizedSessionTimeS).toBeNull(); - expect(stats.p90PrefillTpsPerUser).toBeNull(); - expect(stats.normalizedE2e400).toBeNull(); // Version still set so the row is considered "computed". expect(stats.version).toBe(STATS_VERSION); }); @@ -145,8 +126,34 @@ describe('mergeProfileStatsUpgrade', () => { const merged = mergeProfileStatsUpgrade(existing, profile); expect(merged.version).toBe(STATS_VERSION); expect(merged.isl?.mean).toBe(100); - expect(merged.normalizedE2e400?.p90).toBeGreaterThan(0); expect(merged.kvCacheUtil).toEqual(existing.kvCacheUtil); expect(merged.prefixCacheHitRate).toEqual(existing.prefixCacheHitRate); }); + + it('drops the retired derived fields when upgrading a pre-v6 bundle', async () => { + // Bundles written before v6 carry normalizedSessionTimeS / + // p90PrefillTpsPerUser / normalizedE2e400 — the upgrade must not + // resurrect them into the new bundle. + const legacy = { + version: STATS_VERSION - 1, + isl: null, + osl: null, + kvCacheUtil: { mean: 0.4, p50: 0.4, p75: 0.5, p90: 0.6, p99: 0.7, n: 3 }, + prefixCacheHitRate: null, + normalizedSessionTimeS: 999, + p90PrefillTpsPerUser: 999, + normalizedE2e400: { mean: 1, p50: 1, p75: 1, p90: 2, p99: 3, n: 5 }, + }; + const profile = await computeAggregateStats({ + profileBlob: makeProfileBlob([{ isl: 100, osl: 100, rl: 2080, ttft: 100 }]), + serverBlob: null, + }); + + const merged = mergeProfileStatsUpgrade(legacy, profile); + expect(merged.version).toBe(STATS_VERSION); + expect(merged.kvCacheUtil).toEqual(legacy.kvCacheUtil); + expect(merged).not.toHaveProperty('normalizedSessionTimeS'); + expect(merged).not.toHaveProperty('p90PrefillTpsPerUser'); + expect(merged).not.toHaveProperty('normalizedE2e400'); + }); }); diff --git a/packages/db/src/etl/compute-aggregate-stats.ts b/packages/db/src/etl/compute-aggregate-stats.ts index f0385f6ec..d4a76e27b 100644 --- a/packages/db/src/etl/compute-aggregate-stats.ts +++ b/packages/db/src/etl/compute-aggregate-stats.ts @@ -1,9 +1,8 @@ /** * Pre-compute the per-row aggregate stats for an `agentic_trace_replay` * blob pair. The output lands in the `aggregate_stats` JSONB column so the - * detail page can serve the "Aggregates across configs" view and the - * derived chart x-axis modes from a single SQL row read, instead of - * parsing the raw blobs on demand. + * detail page can serve the "Aggregates across configs" view from a single + * SQL row read, instead of parsing the raw blobs on demand. * * Shape is intentionally versioned — bump `STATS_VERSION` whenever the * computation changes so the backfill script knows which rows to recompute. @@ -12,7 +11,6 @@ import { gunzipSync } from 'node:zlib'; import { gunzipJsonWithinLimit, streamCollectKeys } from './gzip-json-stream'; -import { computeDerivedFromBlob } from '../queries/derived-agentic-metrics'; import { STATS_VERSION, extractIslOsl, @@ -29,12 +27,19 @@ export interface AggregateStats { osl: MetricPercentiles | null; kvCacheUtil: MetricPercentiles | null; prefixCacheHitRate: MetricPercentiles | null; - /** Mean of (per-session e2e time × mean_load / session_load) across sessions. */ - normalizedSessionTimeS: number | null; - /** P90 of per-turn ISL/TTFT pooled across every session's turns. */ - p90PrefillTpsPerUser: number | null; - /** Per-request normalized E2E distribution at a fixed 400-token OSL. */ - normalizedE2e400: MetricPercentiles | null; +} + +/** + * The subset of an older-version bundle a profile-only upgrade carries + * forward. Pre-v6 bundles also carry the since-retired derived metrics + * (normalizedSessionTimeS, p90PrefillTpsPerUser, normalizedE2e400) — spreading + * `profile` first drops them from the merged result. + */ +interface ProfileUpgradeCarryover { + isl: MetricPercentiles | null; + osl: MetricPercentiles | null; + kvCacheUtil: MetricPercentiles | null; + prefixCacheHitRate: MetricPercentiles | null; } /** @@ -43,17 +48,13 @@ export interface AggregateStats { * while preserving its already-computed KV/cache distributions. */ export function mergeProfileStatsUpgrade( - existing: Omit & { - normalizedE2e400?: MetricPercentiles | null; - }, + existing: ProfileUpgradeCarryover, profile: AggregateStats, ): AggregateStats { return { ...profile, isl: profile.isl ?? existing.isl, osl: profile.osl ?? existing.osl, - normalizedSessionTimeS: profile.normalizedSessionTimeS ?? existing.normalizedSessionTimeS, - p90PrefillTpsPerUser: profile.p90PrefillTpsPerUser ?? existing.p90PrefillTpsPerUser, kvCacheUtil: existing.kvCacheUtil, prefixCacheHitRate: existing.prefixCacheHitRate, }; @@ -92,9 +93,6 @@ export async function computeAggregateStats(args: { }): Promise { let islPct: MetricPercentiles | null = null; let oslPct: MetricPercentiles | null = null; - let normalized: number | null = null; - let prefillP90: number | null = null; - let normalizedE2e400: MetricPercentiles | null = null; if (args.profileBlob) { try { @@ -102,10 +100,6 @@ export async function computeAggregateStats(args: { const { isl, osl } = extractIslOsl(jsonl); islPct = percentilesOf(isl); oslPct = percentilesOf(osl); - const derived = computeDerivedFromBlob(jsonl); - normalized = derived.normalized_session_time_s; - prefillP90 = derived.p90_prefill_tps_per_user; - normalizedE2e400 = derived.normalized_e2e_400; } catch { // ignore malformed blob — leave nulls } @@ -136,8 +130,5 @@ export async function computeAggregateStats(args: { osl: oslPct, kvCacheUtil: kvPct, prefixCacheHitRate: prefixPct, - normalizedSessionTimeS: normalized, - p90PrefillTpsPerUser: prefillP90, - normalizedE2e400, }; } diff --git a/packages/db/src/queries/agentic-aggregates.test.ts b/packages/db/src/queries/agentic-aggregates.test.ts index 0c4dbc890..d0daa2a39 100644 --- a/packages/db/src/queries/agentic-aggregates.test.ts +++ b/packages/db/src/queries/agentic-aggregates.test.ts @@ -129,9 +129,6 @@ interface WrittenStats { osl: unknown; kvCacheUtil: { mean: number } | null; prefixCacheHitRate: unknown; - normalizedSessionTimeS: number | null; - p90PrefillTpsPerUser: number | null; - normalizedE2e400: unknown; } /** Capture SQL template text + bound values for the write-back assertions. */ @@ -228,11 +225,11 @@ describe('getAgenticAggregates write-back', () => { expect(written.version).toBe(STATS_VERSION); // Server field FRESHLY recomputed (0.25), not the stale 0.9 carried forward. expect(written.kvCacheUtil?.mean).toBeCloseTo(0.25, 6); - // Derived fields FRESHLY recomputed (not the stale 999s). - expect(written.normalizedSessionTimeS).toBeCloseTo(3, 6); - expect(written.p90PrefillTpsPerUser).toBeCloseTo(200, 6); - expect(written.normalizedE2e400).not.toBeNull(); expect(written.isl).not.toBeNull(); + // The retired derived fields must not survive the recompute. + expect(written).not.toHaveProperty('normalizedSessionTimeS'); + expect(written).not.toHaveProperty('p90PrefillTpsPerUser'); + expect(written).not.toHaveProperty('normalizedE2e400'); }); it('does not write back for an id whose profile blob is missing/malformed', async () => { diff --git a/packages/db/src/queries/agentic-aggregates.ts b/packages/db/src/queries/agentic-aggregates.ts index 4917dc4c6..901defb9f 100644 --- a/packages/db/src/queries/agentic-aggregates.ts +++ b/packages/db/src/queries/agentic-aggregates.ts @@ -24,7 +24,6 @@ import { streamObject } from 'stream-json/streamers/stream-object.js'; import { gunzipJsonWithinLimit } from '../etl/gzip-json-stream'; import type { DbClient } from '../connection.js'; -import { computeDerivedFromBlob } from './derived-agentic-metrics'; import { extractIslOsl, fetchAggregateStatsRows, @@ -316,12 +315,8 @@ export async function getAgenticAggregates( const oslPct = percentilesOf(osl); result[id].isl = islPct; result[id].osl = oslPct; - // Recompute the profile-derived fields too (same jsonl, no extra - // read) so the self-healed bundle is a faithful full recompute — not - // a carry-forward of stale derived numbers stamped with a new - // version. Server-derived fields are filled in Pass 2 (or stay null - // when the server blob is absent, which is the correct complete value). - const derived = computeDerivedFromBlob(jsonl); + // Server-derived fields are filled in Pass 2 (or stay null when the + // server blob is absent, which is the correct complete value). pendingById.set(id, { traceReplayId: Number(row.trace_replay_id), stats: { @@ -330,9 +325,6 @@ export async function getAgenticAggregates( osl: oslPct, kvCacheUtil: null, prefixCacheHitRate: null, - normalizedSessionTimeS: derived.normalized_session_time_s, - p90PrefillTpsPerUser: derived.p90_prefill_tps_per_user, - normalizedE2e400: derived.normalized_e2e_400, }, }); } catch { @@ -404,8 +396,6 @@ interface AggregateStatsRow { osl: MetricPercentiles | null; kvCacheUtil: MetricPercentiles | null; prefixCacheHitRate: MetricPercentiles | null; - normalizedSessionTimeS: number | null; - p90PrefillTpsPerUser: number | null; } /** @@ -419,9 +409,6 @@ interface FullAggregateStats { osl: MetricPercentiles | null; kvCacheUtil: MetricPercentiles | null; prefixCacheHitRate: MetricPercentiles | null; - normalizedSessionTimeS: number | null; - p90PrefillTpsPerUser: number | null; - normalizedE2e400: MetricPercentiles | null; } function blankAggregate(id: number): AgenticAggregate { diff --git a/packages/db/src/queries/agentic-shared.ts b/packages/db/src/queries/agentic-shared.ts index d8673a075..d704abe02 100644 --- a/packages/db/src/queries/agentic-shared.ts +++ b/packages/db/src/queries/agentic-shared.ts @@ -31,8 +31,13 @@ import type { DbClient } from '../connection.js'; * * v5: reject osl <= 0 in extractTurn to exclude cancelled/empty-output turns * whose decode-interval math would explode normalized E2E to thousands of seconds. + * + * v6: drop the retired per-point derived metrics (normalizedSessionTimeS, + * p90PrefillTpsPerUser, normalizedE2e400) along with the experimental chart + * modes they fed. The bundle is now isl/osl + the server-derived kvCacheUtil + * and prefixCacheHitRate. */ -export const STATS_VERSION = 5; +export const STATS_VERSION = 6; interface ProfileRecord { metadata?: { benchmark_phase?: string }; diff --git a/packages/db/src/queries/derived-agentic-metrics.test.ts b/packages/db/src/queries/derived-agentic-metrics.test.ts deleted file mode 100644 index a39de6700..000000000 --- a/packages/db/src/queries/derived-agentic-metrics.test.ts +++ /dev/null @@ -1,256 +0,0 @@ -import { gzipSync } from 'node:zlib'; - -import { describe, expect, it } from 'vitest'; - -import { STATS_VERSION } from './agentic-shared'; -import type { DbClient } from '../connection.js'; - -import { computeDerivedFromBlob, getDerivedAgenticMetrics } from './derived-agentic-metrics.js'; - -/** Build one aiperf JSONL record for the synthetic fixture. */ -function rec( - conversation_id: string, - turn_index: number, - fields: { isl: number; osl: number; ttft_ms: number; latency_ms: number }, -): string { - return JSON.stringify({ - metadata: { conversation_id, turn_index, benchmark_phase: 'profiling' }, - metrics: { - request_latency: { value: fields.latency_ms, unit: 'ms' }, - time_to_first_token: { value: fields.ttft_ms, unit: 'ms' }, - input_sequence_length: { value: fields.isl, unit: 'tokens' }, - output_sequence_length: { value: fields.osl, unit: 'tokens' }, - }, - }); -} - -describe('computeDerivedFromBlob', () => { - it('returns nulls when no usable records', () => { - const out = computeDerivedFromBlob(''); - expect(out.normalized_session_time_s).toBeNull(); - expect(out.p90_prefill_tps_per_user).toBeNull(); - expect(out.normalized_e2e_400).toBeNull(); - }); - - it('normalizes each request to 400 output tokens before taking percentiles', () => { - const jsonl = [ - // Both requests have TTFT=2s and ITL=20ms, despite very different OSL/E2E. - rec('s1', 0, { isl: 100, osl: 100, ttft_ms: 2000, latency_ms: 3980 }), - rec('s2', 0, { isl: 100, osl: 1000, ttft_ms: 2000, latency_ms: 21_980 }), - ].join('\n'); - - const out = computeDerivedFromBlob(jsonl); - // 2s TTFT + 399 × 20ms ITL = 9.98s for both requests. - expect(out.normalized_e2e_400?.n).toBe(2); - expect(out.normalized_e2e_400?.p75).toBeCloseTo(9.98, 8); - expect(out.normalized_e2e_400?.p90).toBeCloseTo(9.98, 8); - }); - - it('rescales single-session time and computes P90 prefill', () => { - // One session, two turns. load = (100+50) + (200+50) = 400. - // Single session ⇒ mean_load = load_i ⇒ T̃ = T = (1000+2000) ms = 3.0 s. - const jsonl = [ - rec('s1', 0, { isl: 100, osl: 50, ttft_ms: 500, latency_ms: 1000 }), - rec('s1', 1, { isl: 200, osl: 50, ttft_ms: 1000, latency_ms: 2000 }), - ].join('\n'); - const out = computeDerivedFromBlob(jsonl); - expect(out.normalized_session_time_s).toBeCloseTo(3, 6); - // Prefill TPS per turn: 100/0.5=200, 200/1.0=200 → global P90 = 200. - expect(out.p90_prefill_tps_per_user).toBeCloseTo(200, 6); - }); - - it('rescales times across sessions with unequal load', () => { - // s1: 1 turn, load = 100, T = 1s - // s2: 1 turn, load = 300, T = 3s - // mean_load = 200; T̃_1 = 1 * 200/100 = 2; T̃_2 = 3 * 200/300 = 2 - // Mean T̃ = 2.0 - const jsonl = [ - rec('s1', 0, { isl: 90, osl: 10, ttft_ms: 500, latency_ms: 1000 }), - rec('s2', 0, { isl: 270, osl: 30, ttft_ms: 500, latency_ms: 3000 }), - ].join('\n'); - const out = computeDerivedFromBlob(jsonl); - expect(out.normalized_session_time_s).toBeCloseTo(2, 6); - }); - - it('drops records missing required fields and skips non-profiling phase', () => { - const lines = [ - rec('s1', 0, { isl: 100, osl: 50, ttft_ms: 500, latency_ms: 1000 }), - // missing TTFT — should be skipped - JSON.stringify({ - metadata: { conversation_id: 's1', turn_index: 1, benchmark_phase: 'profiling' }, - metrics: { - request_latency: { value: 1000, unit: 'ms' }, - input_sequence_length: { value: 100, unit: 'tokens' }, - output_sequence_length: { value: 50, unit: 'tokens' }, - }, - }), - // warmup phase — should be skipped - JSON.stringify({ - metadata: { conversation_id: 's2', turn_index: 0, benchmark_phase: 'warmup' }, - metrics: { - request_latency: { value: 9999, unit: 'ms' }, - time_to_first_token: { value: 9999, unit: 'ms' }, - input_sequence_length: { value: 100, unit: 'tokens' }, - output_sequence_length: { value: 50, unit: 'tokens' }, - }, - }), - ]; - const out = computeDerivedFromBlob(lines.join('\n')); - expect(out.normalized_session_time_s).toBeCloseTo(1, 6); - expect(out.p90_prefill_tps_per_user).toBeCloseTo(200, 6); - }); - - it('p90 across turns: 10-turn session picks the right rank', () => { - // Prefill rates 100..1000 (per turn isl/ttft); p90 of 10 values (linear) = 910. - const turns = Array.from({ length: 10 }, (_, i) => - rec('s1', i, { - isl: (i + 1) * 100, // 100, 200, ..., 1000 tokens - osl: 10, - ttft_ms: 1000, // 1 second → rates: 100..1000 tps - latency_ms: 1500, - }), - ); - const out = computeDerivedFromBlob(turns.join('\n')); - expect(out.p90_prefill_tps_per_user).toBeCloseTo(910, 6); - }); - - it('excludes osl=0 (cancelled/empty-output) turns from normalized E2E', () => { - // Two normal turns + one cancelled turn (osl=0, latency=30s, ttft=1s). - // - // The cancelled turn must be excluded because observedDecodeIntervals collapses - // to max(0-1,1)=1, making itlMs=(30000-1000)/1=29000ms and normalizedMs explode - // to ~11 572 s — roughly 386× the real scale. (Pre-fix behavior for reference; - // this number is intentionally not asserted below to avoid enshrining the bug.) - // - // Normal turn A: isl=100, osl=50, ttft=500ms, latency=1000ms - // observedDecodeIntervals = max(49,1) = 49 - // itlMs = (1000-500)/49 - // normalizedMs = 500 + 399*(500/49) - // - // Normal turn B: isl=200, osl=100, ttft=1000ms, latency=3000ms - // observedDecodeIntervals = max(99,1) = 99 - // itlMs = (3000-1000)/99 - // normalizedMs = 1000 + 399*(2000/99) - const normA = (500 + (399 * 500) / 49) / 1000; // seconds - const normB = (1000 + (399 * 2000) / 99) / 1000; // seconds - - const jsonl = [ - rec('s1', 0, { isl: 100, osl: 50, ttft_ms: 500, latency_ms: 1000 }), - rec('s1', 1, { isl: 200, osl: 100, ttft_ms: 1000, latency_ms: 3000 }), - // Cancelled / empty-output turn — osl=0 must be rejected by extractTurn. - rec('s2', 0, { isl: 150, osl: 0, ttft_ms: 1000, latency_ms: 30000 }), - ].join('\n'); - - const out = computeDerivedFromBlob(jsonl); - - // Only the 2 normal turns contribute; osl=0 record is silently excluded. - expect(out.normalized_e2e_400?.n).toBe(2); - - // p90 of [normA, normB] sorted ascending (normA < normB): - // pos = 1*0.9 = 0.9; result = normA + (normB - normA)*0.9 - const expectedP90 = normA + (normB - normA) * 0.9; - expect(out.normalized_e2e_400?.p90).toBeCloseTo(expectedP90, 6); - - // Sanity: p90 should be single-digit seconds, not thousands. - expect(out.normalized_e2e_400!.p90).toBeLessThan(20); - }); -}); - -/** Capture SQL template text + bound values for the write-back assertions. */ -function mockSql(queue: unknown[][]): { - sql: DbClient; - calls: { text: string; values: unknown[] }[]; -} { - const responses = [...queue]; - const calls: { text: string; values: unknown[] }[] = []; - const sql = ((strings: TemplateStringsArray, ...values: unknown[]) => { - calls.push({ text: strings.join('?'), values }); - return Promise.resolve(responses.shift() ?? []); - }) as unknown as DbClient; - return { sql, calls }; -} - -describe('getDerivedAgenticMetrics write-back', () => { - it('self-heals aggregate_stats from the profile blob, carrying server fields forward', async () => { - const jsonl = [ - rec('s1', 0, { isl: 100, osl: 50, ttft_ms: 500, latency_ms: 1000 }), - rec('s1', 1, { isl: 200, osl: 50, ttft_ms: 1000, latency_ms: 2000 }), - ].join('\n'); - const blob = gzipSync(Buffer.from(jsonl)); - - // Stale v(N-1) row that DOES carry server-derived fields — they must be - // preserved in the healed bundle (derived route can't recompute them). - const staleServerKv = { mean: 0.4, p50: 0.4, p75: 0.5, p90: 0.6, p99: 0.7, n: 3 }; - const staleStats = { - version: STATS_VERSION - 1, - isl: null, - osl: null, - kvCacheUtil: staleServerKv, - prefixCacheHitRate: null, - normalizedSessionTimeS: 999, - p90PrefillTpsPerUser: 999, - normalizedE2e400: null, - }; - - const { sql, calls } = mockSql([ - // fetchAggregateStatsRows - [{ benchmark_result_id: 7, stats: staleStats }], - // fallback profile-blob query - [{ benchmark_result_id: 7, trace_replay_id: 870, blob }], - ]); - - const result = await getDerivedAgenticMetrics(sql, [7]); - - // Response is the freshly recomputed value, not the stale 999s. - expect(result[7]?.normalized_session_time_s).toBeCloseTo(3, 6); - expect(result[7]?.p90_prefill_tps_per_user).toBeCloseTo(200, 6); - - // 3 calls: stats read, blob read, write-back UPDATE. - expect(calls).toHaveLength(3); - expect(calls[2]!.text).toContain('update agentic_trace_replay set aggregate_stats'); - expect(calls[2]!.text).toContain('::jsonb where id'); - - // The write-back binds a COMPLETE, version-stamped bundle at the new version, - // recomputing profile fields and carrying server fields forward untouched. - // The payload OBJECT is bound directly (not stringified — that would - // double-encode into a JSONB string). - interface WrittenStats { - version: number; - isl: unknown; - osl: unknown; - kvCacheUtil: unknown; - normalizedSessionTimeS: number | null; - p90PrefillTpsPerUser: number | null; - } - const [written, traceReplayId] = calls[2]!.values as [WrittenStats, number]; - expect(traceReplayId).toBe(870); - expect(written.version).toBe(STATS_VERSION); - expect(written.normalizedSessionTimeS).toBeCloseTo(3, 6); - expect(written.p90PrefillTpsPerUser).toBeCloseTo(200, 6); - expect(written.isl).not.toBeNull(); - expect(written.osl).not.toBeNull(); - // Server-derived field carried forward from the stale row (not re-read). - expect(written.kvCacheUtil).toEqual(staleServerKv); - }); - - it('takes the fast path (no blob read, no write-back) when stats are current', async () => { - const currentStats = { - version: STATS_VERSION, - isl: null, - osl: null, - kvCacheUtil: null, - prefixCacheHitRate: null, - normalizedSessionTimeS: 1.5, - p90PrefillTpsPerUser: 42, - normalizedE2e400: { mean: 1, p50: 1, p75: 1, p90: 2, p99: 3, n: 5 }, - }; - const { sql, calls } = mockSql([[{ benchmark_result_id: 7, stats: currentStats }]]); - - const result = await getDerivedAgenticMetrics(sql, [7]); - - expect(result[7]?.normalized_session_time_s).toBe(1.5); - expect(result[7]?.p90_normalized_e2e_400_s).toBe(2); - // Only the stats read — no fallback blob query, no write-back. - expect(calls).toHaveLength(1); - }); -}); diff --git a/packages/db/src/queries/derived-agentic-metrics.ts b/packages/db/src/queries/derived-agentic-metrics.ts deleted file mode 100644 index 626ab9c77..000000000 --- a/packages/db/src/queries/derived-agentic-metrics.ts +++ /dev/null @@ -1,318 +0,0 @@ -/** - * Live-computed per-point metrics derived from the stored aiperf - * `profile_export.jsonl` blob. These aren't precomputed in the metrics JSONB - * because they require grouping by `conversation_id` and aggregating per - * session — work that's cheap once per agentic point but adds up to be - * meaningful only when actually plotted. - * - * - normalized_session_time_s: per the "Mean Normalized Session Time" proposal - * (https://gist.github.com/xinli-sw/115d370c17f6d1b977878b68530981fa). Sum of - * per-turn `request_latency` per session (inter-turn tool/thinking gaps are - * inherently excluded since we only sum the active GPU time, not wallclock). - * Each session's time is rescaled by `mean_load / session_load`, where load - * is Σ(ISL+OSL) across turns. The plotted value is the mean across sessions. - * - * - p90_prefill_tps_per_user: per the same gist's "Prefill" Pareto chart. - * Per turn: prefill_tps = ISL / TTFT_seconds. Single P90 across every turn - * in every session — the per-session percentile + cross-session mean - * sandwich was discarded because it just dampens tail behavior. - */ - -import { gunzipSync } from 'node:zlib'; - -import { NORMALIZED_E2E_OUTPUT_TOKENS } from '@semianalysisai/inferencex-constants'; - -import type { DbClient } from '../connection.js'; -import { - extractIslOsl, - fetchAggregateStatsRows, - meanOf, - percentilesOf, - quantile, - readNum, - STATS_VERSION, - writeBackTraceReplayJsonb, - type MetricPercentiles, -} from './agentic-shared'; - -export interface DerivedAgenticMetric { - /** benchmark_results.id this entry belongs to. */ - id: number; - /** Mean normalized session time in seconds. */ - normalized_session_time_s: number | null; - /** P90 of per-turn prefill tps/user (ISL / TTFT) across every turn in every session. */ - p90_prefill_tps_per_user: number | null; - /** P75 normalized per-request E2E at a fixed 400-token output length. */ - p75_normalized_e2e_400_s: number | null; - /** P90 normalized per-request E2E at a fixed 400-token output length. */ - p90_normalized_e2e_400_s: number | null; -} - -export type DerivedAgenticMetricMap = Record; - -/** - * The full `aggregate_stats` JSONB shape (mirrors `AggregateStats` in - * etl/compute-aggregate-stats.ts). Duplicated here rather than imported to keep - * this module off the etl import graph. When we self-heal from the profile blob - * alone, the server-derived fields (kvCacheUtil, prefixCacheHitRate) are carried - * forward untouched from the stale row — never re-reading the huge server blob. - * This mirrors the profile-only upgrade `backfill-aggregate-stats.ts` performs; - * the agentic-aggregates route (which does read the server blob) heals those - * server fields. - */ -interface StoredAggregateStats { - version: number; - isl: MetricPercentiles | null; - osl: MetricPercentiles | null; - kvCacheUtil: MetricPercentiles | null; - prefixCacheHitRate: MetricPercentiles | null; - normalizedSessionTimeS: number | null; - p90PrefillTpsPerUser: number | null; - normalizedE2e400: MetricPercentiles | null; -} - -/** - * JSONL blobs can be ~1-2 MB compressed (~5-10 MB raw) and Neon's serverless - * HTTP driver caps responses at 64 MB — chunk to stay well under. - */ -const QUERY_CHUNK_SIZE = 6; - -interface RecordMetrics { - request_latency?: { value?: number; unit?: string } | number; - time_to_first_token?: { value?: number; unit?: string } | number; - input_sequence_length?: { value?: number } | number; - output_sequence_length?: { value?: number } | number; -} - -interface RecordMetadata { - conversation_id?: string; - turn_index?: number; - benchmark_phase?: string; -} - -interface ProfileRecord { - metadata?: RecordMetadata; - metrics?: RecordMetrics; -} - -interface TurnFields { - request_latency_ms: number; - ttft_ms: number; - isl: number; - osl: number; -} - -function extractTurn(rec: ProfileRecord): TurnFields | null { - const m = rec.metrics ?? {}; - const rl = readNum(m.request_latency); - const tt = readNum(m.time_to_first_token); - const isl = readNum(m.input_sequence_length); - const osl = readNum(m.output_sequence_length); - if (rl === undefined || tt === undefined || isl === undefined || osl === undefined) return null; - if (rl <= 0 || tt <= 0 || isl <= 0 || osl <= 0) return null; - return { request_latency_ms: rl, ttft_ms: tt, isl, osl }; -} - -/** - * Parse one point's JSONL and return the two derived metrics. Returns - * `{ session_time: null, prefill: null }` if the blob has no usable records. - */ -export function computeDerivedFromBlob(jsonl: string): { - normalized_session_time_s: number | null; - p90_prefill_tps_per_user: number | null; - normalized_e2e_400: MetricPercentiles | null; -} { - // Group records by conversation_id, filter to the profiling phase. - const bySession = new Map(); - for (const line of jsonl.split('\n')) { - if (!line) continue; - let rec: ProfileRecord; - try { - rec = JSON.parse(line) as ProfileRecord; - } catch { - continue; - } - if (rec.metadata?.benchmark_phase && rec.metadata.benchmark_phase !== 'profiling') continue; - const sid = rec.metadata?.conversation_id; - if (!sid) continue; - const turn = extractTurn(rec); - if (!turn) continue; - let list = bySession.get(sid); - if (!list) { - list = []; - bySession.set(sid, list); - } - list.push(turn); - } - if (bySession.size === 0) { - return { - normalized_session_time_s: null, - p90_prefill_tps_per_user: null, - normalized_e2e_400: null, - }; - } - - // Per-session aggregates for session time; per-turn prefill rates pool into - // a single global array so the percentile sees the full distribution. - const sessionTimesS: number[] = []; - const sessionLoads: number[] = []; - const allPrefillRates: number[] = []; - const allNormalizedE2eS: number[] = []; - for (const turns of bySession.values()) { - let timeMs = 0; - let load = 0; - for (const t of turns) { - timeMs += t.request_latency_ms; - load += t.isl + t.osl; - const ttftSec = t.ttft_ms / 1000; - if (ttftSec > 0) allPrefillRates.push(t.isl / ttftSec); - - // Keep the observed TTFT, then project the request's mean decode - // interval to a fixed output length. Do this per request before taking - // percentiles so long original outputs do not dominate the tail. - const observedDecodeIntervals = Math.max(t.osl - 1, 1); - const itlMs = (t.request_latency_ms - t.ttft_ms) / observedDecodeIntervals; - const normalizedMs = t.ttft_ms + (NORMALIZED_E2E_OUTPUT_TOKENS - 1) * itlMs; - if ( - Number.isFinite(itlMs) && - itlMs >= 0 && - Number.isFinite(normalizedMs) && - normalizedMs > 0 - ) { - allNormalizedE2eS.push(normalizedMs / 1000); - } - } - if (load > 0) { - sessionTimesS.push(timeMs / 1000); - sessionLoads.push(load); - } - } - - // Normalized session time: T̃_i = T_i × (mean_load / load_i), then mean. - let normalized: number | null = null; - if (sessionTimesS.length > 0) { - const meanLoad = meanOf(sessionLoads); - if (meanLoad > 0) { - const scaled: number[] = []; - for (let i = 0; i < sessionTimesS.length; i++) { - const ti = sessionTimesS[i]!; - const li = sessionLoads[i]!; - if (li > 0) scaled.push(ti * (meanLoad / li)); - } - normalized = scaled.length > 0 ? meanOf(scaled) : null; - } - } - - let prefill: number | null = null; - if (allPrefillRates.length > 0) { - allPrefillRates.sort((a, b) => a - b); - prefill = quantile(allPrefillRates, 0.9); - } - - return { - normalized_session_time_s: normalized, - p90_prefill_tps_per_user: prefill, - normalized_e2e_400: percentilesOf(allNormalizedE2eS), - }; -} - -export async function getDerivedAgenticMetrics( - sql: DbClient, - benchmarkResultIds: number[], -): Promise { - if (benchmarkResultIds.length === 0) return {}; - - const result: DerivedAgenticMetricMap = {}; - - // Fast path: read the pre-computed values out of `aggregate_stats`. The - // ingest pipeline computes both metrics in the same pass that produces the - // percentile bundles, so a single SQL round-trip covers most ids without - // touching the gzipped profile blob. - const statsRows = await fetchAggregateStatsRows(sql, benchmarkResultIds); - - const idsNeedingBlob: number[] = []; - // Carry each stale/missing row's existing stats into the fallback so a - // self-heal preserves the server-derived fields (kvCacheUtil, - // prefixCacheHitRate) it can't recompute from the profile blob alone. - const staleStatsById = new Map(); - for (const row of statsRows) { - const id = Number(row.benchmark_result_id); - if (row.stats && Number(row.stats.version) === STATS_VERSION) { - result[id] = { - id, - normalized_session_time_s: row.stats.normalizedSessionTimeS ?? null, - p90_prefill_tps_per_user: row.stats.p90PrefillTpsPerUser ?? null, - p75_normalized_e2e_400_s: row.stats.normalizedE2e400?.p75 ?? null, - p90_normalized_e2e_400_s: row.stats.normalizedE2e400?.p90 ?? null, - }; - } else { - idsNeedingBlob.push(id); - staleStatsById.set(id, row.stats ?? null); - } - } - - if (idsNeedingBlob.length === 0) return result; - - // Fallback: parse the profile blob directly. Used for rows whose - // `aggregate_stats` is null or computed by an older STATS_VERSION; the - // backfill script drains the population so this path should be rare. - // `trace_replay_id` + the (small) stale `aggregate_stats` come along on the - // same join — no extra round-trip — so we can self-heal after recompute. - const rows: { - benchmark_result_id: number; - trace_replay_id: number; - blob: Buffer; - }[] = []; - for (let i = 0; i < idsNeedingBlob.length; i += QUERY_CHUNK_SIZE) { - const chunk = idsNeedingBlob.slice(i, i + QUERY_CHUNK_SIZE); - const chunkRows = (await sql` - select - br.id as benchmark_result_id, - atr.id as trace_replay_id, - atr.profile_export_jsonl_gz as blob - from benchmark_results br - join agentic_trace_replay atr on atr.id = br.trace_replay_id - where br.id = any(${chunk}::bigint[]) - and atr.profile_export_jsonl_gz is not null - `) as { benchmark_result_id: number; trace_replay_id: number; blob: Buffer }[]; - rows.push(...chunkRows); - } - - for (const row of rows) { - const id = Number(row.benchmark_result_id); - try { - const jsonl = gunzipSync(row.blob).toString('utf8'); - const { normalized_session_time_s, p90_prefill_tps_per_user, normalized_e2e_400 } = - computeDerivedFromBlob(jsonl); - result[id] = { - id, - normalized_session_time_s, - p90_prefill_tps_per_user, - p75_normalized_e2e_400_s: normalized_e2e_400?.p75 ?? null, - p90_normalized_e2e_400_s: normalized_e2e_400?.p90 ?? null, - }; - - // Self-heal the shared `aggregate_stats` bundle. We only have the profile - // blob here, so recompute the profile-derived fields (isl/osl + the three - // derived metrics) and carry the stale row's server-derived fields - // forward untouched — the profile-only upgrade the backfill CLI also - // performs. Fire-and-forget, best-effort (no-ops on a read-only replica). - const { isl, osl } = extractIslOsl(jsonl); - const prior = staleStatsById.get(id) ?? null; - const merged: StoredAggregateStats = { - version: STATS_VERSION, - isl: percentilesOf(isl), - osl: percentilesOf(osl), - kvCacheUtil: prior?.kvCacheUtil ?? null, - prefixCacheHitRate: prior?.prefixCacheHitRate ?? null, - normalizedSessionTimeS: normalized_session_time_s, - p90PrefillTpsPerUser: p90_prefill_tps_per_user, - normalizedE2e400: normalized_e2e_400, - }; - writeBackTraceReplayJsonb(sql, 'aggregate_stats', Number(row.trace_replay_id), merged); - } catch { - // Skip malformed blobs silently — frontend treats missing ids as "no data". - } - } - return result; -} From 52fb7f65aae3a8f7c18a7d2d8a52c471dbf36a39 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 29 Jul 2026 17:07:11 -0500 Subject: [PATCH 2/2] test(e2e): make the Interactivity percentile assertion self-contained MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "switches back to Interactivity" test asserted a P75 axis label, but nothing in it selected p75 — it inherited the selector state from the Normalized E2E test that ran before it. Removing that test left the selector on p90 and the assertion failed. Assert P90 there (the default) and cover the p75 case in its own test that selects the percentile itself. 中文：「switches back to Interactivity」用例断言 P75 轴标签，但其自身 并未切换分位数，而是依赖此前 Normalized E2E 用例遗留的选择器状态。 该用例被移除后选择器停留在 p90，断言随之失败。现改为断言默认的 P90， 并将 p75 场景拆分为独立用例，由其自行切换分位数。 Co-Authored-By: Claude Fable 5 --- packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts b/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts index f3a897b1b..bec88b5b3 100644 --- a/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts +++ b/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts @@ -236,7 +236,18 @@ describe('X-Axis Mode Toggle (inference chart)', () => { 'true', ); cy.get('[data-testid="chart-figure"] h2').should('contain.text', 'Interactivity'); - // Percentile was switched to p75 in the previous test — the axis label follows. + cy.get('[data-testid="chart-figure"] svg').should( + 'contain.text', + 'P90 Interactivity (tok/s/user)', + ); + }); + + it('follows the percentile selector in the Interactivity axis label', () => { + // Select p75 here rather than inheriting it from another test — the axis + // label must track the selector on its own. + cy.get('[data-testid="x-axis-mode-interactivity"]').click(); + cy.get('[data-testid="percentile-selector"]').click(); + cy.contains('[role="option"]', 'p75').click(); cy.get('[data-testid="chart-figure"] svg').should( 'contain.text', 'P75 Interactivity (tok/s/user)',