diff --git a/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts b/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts index 5023877e..bec88b5b 100644 --- a/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts +++ b/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts @@ -1,27 +1,5 @@ import { unlockAgenticGate } from '../support/e2e'; -const interceptDerivedMetrics = () => { - cy.intercept('GET', '/api/v1/derived-agentic-metrics*', (request) => { - const ids = new URL(request.url).searchParams.get('ids')?.split(',').filter(Boolean) ?? []; - request.reply({ - body: Object.fromEntries( - ids.map((id, index) => [ - id, - { - id: Number(id), - normalized_session_time_s: 60 + index, - p90_prefill_tps_per_user: 100 + index, - p75_normalized_e2e_400_s: 8 + index, - p90_normalized_e2e_400_s: 12 + index, - p75_osl_per_e2el: 40 + index, - p90_osl_per_e2el: 25 + index, - }, - ]), - ), - }); - }).as('derivedAgenticMetrics'); -}; - // This spec exercises the agentic x-axis modes, which only exist when the // selected model resolves to the Agentic Traces scenario. The default e2e // fixtures (cypress/fixtures/api/*.json) have NO agentic rows for any model, and @@ -148,10 +126,6 @@ const interceptFixedSequenceData = () => { describe('X-Axis Mode Toggle (inference chart)', () => { before(() => { interceptAgenticData(); - // Stub derived metrics before the visit: if the agentic DEFAULT x-axis mode - // is (or becomes) a derived mode that fetches /derived-agentic-metrics on - // mount, the chart must not sit on its loading skeleton. - interceptDerivedMetrics(); cy.visit('/inference?i_seq=agentic-traces', { onBeforeLoad(win) { win.localStorage.setItem('inferencex-star-modal-dismissed', String(Date.now())); @@ -166,7 +140,6 @@ describe('X-Axis Mode Toggle (inference chart)', () => { cy.get('[data-testid="scenario-selector"]').should('contain.text', 'Agentic Traces'); cy.get('[data-testid="x-axis-mode-ttft"]').should('be.visible'); cy.get('[data-testid="x-axis-mode-e2e"]').should('be.visible'); - cy.get('[data-testid="x-axis-mode-normalized-e2e"]').should('be.visible'); cy.get('[data-testid="x-axis-mode-interactivity"]') .should('be.visible') .and('have.attr', 'aria-selected', 'true'); @@ -255,41 +228,26 @@ describe('X-Axis Mode Toggle (inference chart)', () => { cy.get('[data-testid="chart-figure"] svg').should('contain.text', 'P90 End-to-end Latency (s)'); }); - it('switches to request-level normalized E2E at 400 output tokens', () => { - interceptDerivedMetrics(); - cy.get('[data-testid="x-axis-mode-normalized-e2e"]').click(); - cy.wait('@derivedAgenticMetrics'); - cy.get('[data-testid="x-axis-mode-normalized-e2e"]').should( + it('switches back to Interactivity', () => { + cy.get('[data-testid="x-axis-mode-interactivity"]').click(); + cy.get('[data-testid="x-axis-mode-interactivity"]').should( 'have.attr', 'aria-selected', 'true', ); - cy.get('[data-testid="chart-figure"] h2').should( - 'contain.text', - 'P90 Normalized E2E @ 400 output tokens', - ); + cy.get('[data-testid="chart-figure"] h2').should('contain.text', 'Interactivity'); cy.get('[data-testid="chart-figure"] svg').should( 'contain.text', - 'P90 Normalized E2E @ 400 output tokens (s)', - ); - - cy.get('[data-testid="percentile-selector"]').click(); - cy.contains('[role="option"]', 'p75').click(); - cy.get('[data-testid="chart-figure"] h2').should( - 'contain.text', - 'P75 Normalized E2E @ 400 output tokens', + 'P90 Interactivity (tok/s/user)', ); }); - it('switches back to Interactivity', () => { + it('follows the percentile selector in the Interactivity axis label', () => { + // Select p75 here rather than inheriting it from another test — the axis + // label must track the selector on its own. cy.get('[data-testid="x-axis-mode-interactivity"]').click(); - cy.get('[data-testid="x-axis-mode-interactivity"]').should( - 'have.attr', - 'aria-selected', - 'true', - ); - cy.get('[data-testid="chart-figure"] h2').should('contain.text', 'Interactivity'); - // Percentile was switched to p75 in the previous test — the axis label follows. + cy.get('[data-testid="percentile-selector"]').click(); + cy.contains('[role="option"]', 'p75').click(); cy.get('[data-testid="chart-figure"] svg').should( 'contain.text', 'P75 Interactivity (tok/s/user)', @@ -411,9 +369,6 @@ const interceptAgenticDataWithOverlay = () => { describe('X-Axis Mode Toggle — overlay path (finding #8 regression guard)', () => { before(() => { interceptAgenticDataWithOverlay(); - // Same as the main suite: keep the visit independent of the agentic default - // x-axis mode (a derived default fetches /derived-agentic-metrics on mount). - interceptDerivedMetrics(); cy.visit(`/inference?unofficialrun=${OVERLAY_RUN_ID}&i_seq=agentic-traces`, { onBeforeLoad(win) { win.localStorage.setItem('inferencex-star-modal-dismissed', String(Date.now())); @@ -462,19 +417,4 @@ describe('X-Axis Mode Toggle — overlay path (finding #8 regression guard)', () expect(total).to.be.greaterThan(0); }); }); - - it('normalized-e2e mode shows suppression banner for unofficial-run overlays', () => { - interceptDerivedMetrics(); - cy.get('[data-testid="x-axis-mode-normalized-e2e"]').click(); - cy.get('[data-testid="x-axis-mode-normalized-e2e"]').should( - 'have.attr', - 'aria-selected', - 'true', - ); - // The suppression message appears because isUnofficialRun is true and the - // mode is 'normalized-e2e' (documented in ChartDisplay.tsx ~line 640). - cy.contains( - 'Normalized E2E requires persisted per-request traces, so unofficial-run overlays are unavailable for this experimental view.', - ).should('be.visible'); - }); }); diff --git a/packages/app/src/app/api/v1/agentic-cache-keys.test.ts b/packages/app/src/app/api/v1/agentic-cache-keys.test.ts index 3a6287cf..7177f996 100644 --- a/packages/app/src/app/api/v1/agentic-cache-keys.test.ts +++ b/packages/app/src/app/api/v1/agentic-cache-keys.test.ts @@ -28,17 +28,12 @@ import { STATS_VERSION } from '@semianalysisai/inferencex-db/queries/agentic-agg import { REQUEST_TIMELINE_VERSION } from '@semianalysisai/inferencex-db/etl/compute-request-timeline'; import { TRACE_SERVER_METRICS_VERSION } from '@semianalysisai/inferencex-db/queries/trace-server-metrics'; -import { CACHE_KEY_PREFIX as derivedAgenticMetricsKey } from './derived-agentic-metrics/route'; import { CACHE_KEY_PREFIX as agenticAggregatesKey } from './agentic-aggregates/route'; import { CACHE_KEY_PREFIX as requestTimelineKey } from './request-timeline/route'; import { CACHE_KEY_PREFIX as traceServerMetricsKey } from './trace-server-metrics/route'; import { CACHE_KEY_PREFIX as traceHistogramsKey } from './trace-histograms/route'; describe('agentic blob-cache keys are version-derived', () => { - it('derived-agentic-metrics key embeds STATS_VERSION', () => { - expect(derivedAgenticMetricsKey).toBe(`derived-agentic-metrics-v${STATS_VERSION}`); - }); - it('agentic-aggregates key embeds STATS_VERSION', () => { expect(agenticAggregatesKey).toBe(`agentic-aggregates-v${STATS_VERSION}`); }); @@ -57,7 +52,6 @@ describe('agentic blob-cache keys are version-derived', () => { it('every key actually contains a version segment (no unversioned literals)', () => { for (const key of [ - derivedAgenticMetricsKey, agenticAggregatesKey, requestTimelineKey, traceServerMetricsKey, diff --git a/packages/app/src/app/api/v1/derived-agentic-metrics/route.ts b/packages/app/src/app/api/v1/derived-agentic-metrics/route.ts deleted file mode 100644 index 373eb370..00000000 --- a/packages/app/src/app/api/v1/derived-agentic-metrics/route.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { STATS_VERSION } from '@semianalysisai/inferencex-db/queries/agentic-aggregates'; -import { getDb } from '@semianalysisai/inferencex-db/connection'; - -import { - getDerivedAgenticMetrics, - type DerivedAgenticMetricMap, -} from '@semianalysisai/inferencex-db/queries/derived-agentic-metrics'; - -import { cachedQuery } from '@/lib/api-cache'; - -import { idsQueryRoute } from '../id-routes'; - -export const dynamic = 'force-dynamic'; - -// blobOnly: the response is one entry per id with two numbers, but the -// derivation work parses thousands of JSONL records per blob — cache the -// computed result so a chart-refresh hits the warm path. -// -// The cache key is derived from STATS_VERSION (the payload governs the derived -// metrics read out of `aggregate_stats`). blobSet is write-once and nothing -// purges post-backfill, so a hand-written version string would serve stale -// data forever after a bump — deriving the key from the constant means a -// STATS_VERSION bump automatically rolls the cache namespace. -/** Version-derived blob-cache key namespace (exported for the key-derivation test). */ -export const CACHE_KEY_PREFIX = `derived-agentic-metrics-v${STATS_VERSION}`; - -const getCachedDerivedAgenticMetrics = cachedQuery( - (ids: number[]): Promise => getDerivedAgenticMetrics(getDb(), ids), - CACHE_KEY_PREFIX, - { blobOnly: true }, -); - -/** - * GET /api/v1/derived-agentic-metrics?ids=1,2,3 - * - * Returns per-id derived metrics computed live from the stored aiperf - * profile_export.jsonl blobs: - * - normalized_session_time_s: mean across sessions of session e2e time - * (Σ per-turn request_latency) rescaled by mean_load / session_load. - * - p90_prefill_tps_per_user: P90 of per-turn prefill TPS/user (ISL / TTFT) - * across every turn in every session. - * - p75/p90_normalized_e2e_400_s: percentile of per-request - * TTFT + 399 × observed ITL. - * - * Ids without a trace_replay blob or with unparseable records are omitted. - */ -export const GET = idsQueryRoute({ - maxIds: 200, - logLabel: 'derived agentic metrics', - fetch: getCachedDerivedAgenticMetrics, -}); diff --git a/packages/app/src/components/inference/InferenceContext.tsx b/packages/app/src/components/inference/InferenceContext.tsx index 6d5a2b93..8b837ca5 100644 --- a/packages/app/src/components/inference/InferenceContext.tsx +++ b/packages/app/src/components/inference/InferenceContext.tsx @@ -60,12 +60,7 @@ import { } from '@/lib/exclusion'; import { filterRunsByModel, getDisplayLabel } from '@/lib/utils'; -import { - isAgenticOnlyXAxisMode, - useChartData, - X_AXIS_MODES, - type XAxisMode, -} from './hooks/useChartData'; +import { useChartData, X_AXIS_MODES, type XAxisMode } from './hooks/useChartData'; import { resolveComparisonEntries } from './utils/comparisonEntry'; import { comparisonDefaultGroup, @@ -610,40 +605,23 @@ export function InferenceProvider({ }, [labelScenarioKind, sequenceResolved, getUrlParam]); // Reconcile the x-axis mode with the scenario kind: - // - On mount with no `i_xmode` URL param: snap to the kind's natural default - // (interactivity for both agentic and fixed-sequence scenarios). The state was initialized - // to a SSR-stable constant so server and client render the same DOM; this - // effect fixes it up after hydration. - // - When the user later switches sequence kinds: snap to the new kind's - // natural default (the prior selection was for a different kind, so it - // doesn't carry over). + // - On mount with no `i_xmode` URL param: snap to the natural default + // (interactivity for both agentic and fixed-sequence scenarios). The state + // was initialized to a SSR-stable constant so server and client render the + // same DOM; this effect fixes it up after hydration. + // - When the user later switches sequence kinds: snap back to that default + // (the prior selection was for a different kind, so it doesn't carry over). const lastSeqKindRef = useRef | null>(null); useEffect(() => { const kind = sequenceKind(effectiveSequence); const isInitialMount = lastSeqKindRef.current === null; - const isAgenticOnlyMode = isAgenticOnlyXAxisMode(selectedXAxisMode); - // On a stale render where kind hasn't changed, bail unless the current - // mode is agentic-only and we just landed on a fixed-seq scenario — in - // that case force the snap so the chart doesn't try to plot trace-derived - // metrics against rows that have no trace_replay. - if (!isInitialMount && lastSeqKindRef.current === kind) { - if (kind === 'fixed-seq' && isAgenticOnlyMode) { - handleSetXAxisMode('interactivity'); - } - return; - } + // Stale render where the kind hasn't changed — nothing to reconcile. + if (!isInitialMount && lastSeqKindRef.current === kind) return; lastSeqKindRef.current = kind; - if ( - isInitialMount && - xAxisModeFromUrlRef.current && - !(kind === 'fixed-seq' && isAgenticOnlyMode) - ) { - // URL-restored agentic-only mode on a fixed-seq sequence makes no sense - // — fall through to the default snap below. - return; - } + // A URL-restored mode wins on first mount; later kind switches reset it. + if (isInitialMount && xAxisModeFromUrlRef.current) return; handleSetXAxisMode('interactivity'); - }, [effectiveSequence, selectedXAxisMode, handleSetXAxisMode]); + }, [effectiveSequence, handleSetXAxisMode]); // Reconcile selectedE2eXAxisMetric whenever the mode, sequence kind, or // agentic percentile changes. For fixed-seq the JSONB only carries diff --git a/packages/app/src/components/inference/hooks/useChartData.ts b/packages/app/src/components/inference/hooks/useChartData.ts index c125e9e9..c5593ee9 100644 --- a/packages/app/src/components/inference/hooks/useChartData.ts +++ b/packages/app/src/components/inference/hooks/useChartData.ts @@ -39,41 +39,18 @@ import { /** * Chart x-axis variant selected by the mode buttons above the plot. This is * the single definition — InferenceContext (URL/state) and ChartDisplay - * (buttons, derived-metric remapping) import it from here. + * (buttons) import it from here. */ -export type XAxisMode = - | 'ttft' - | 'e2e' - | 'normalized-e2e' - | 'interactivity' - | 'session-time' - | 'prefill-tps'; - -export const X_AXIS_MODES: readonly XAxisMode[] = [ - 'ttft', - 'e2e', - 'normalized-e2e', - 'interactivity', - 'session-time', - 'prefill-tps', -]; +export type XAxisMode = 'ttft' | 'e2e' | 'interactivity'; -/** - * Modes whose x metric is derived from persisted per-request traces — - * these only exist for agentic scenarios (fixed-seq rows have no - * trace_replay blob to derive them from). - */ -export function isAgenticOnlyXAxisMode(mode: XAxisMode): boolean { - return mode === 'normalized-e2e' || mode === 'session-time' || mode === 'prefill-tps'; -} +export const X_AXIS_MODES: readonly XAxisMode[] = ['ttft', 'e2e', 'interactivity']; /** * Compute the set of benchmark_results.id values that sit on the * (e2e_latency, y) Pareto frontier within each (hwKey, precision, date) - * group. Used to restrict the non-e2e xmode charts (ttft, interactivity, - * session-time, prefill-tps) so they show *only* the points that win on - * end-to-end latency — preventing benchmark-hacking where a config tops - * one axis while tanking the other. + * group. Used to restrict the non-e2e xmode charts (ttft, interactivity) so + * they show *only* the points that win on end-to-end latency — preventing + * benchmark-hacking where a config tops one axis while tanking the other. * * Returns null when the y-metric has no roofline direction declared on * the e2e chart (caller falls back to no filtering in that case). @@ -220,7 +197,7 @@ export function useChartData( /** * Current x-axis mode. When set to anything other than 'e2e', the displayed * data is filtered to the (e2e-latency, y) Pareto frontier so the ttft / - * interactivity / session-time / prefill-tps charts show only points that + * interactivity charts show only points that * also win on end-to-end latency — preventing benchmark-hacking where a * config tops one metric while tanking the other. The 'e2e' mode is the * source of truth and keeps the full point set. @@ -529,7 +506,7 @@ export function useChartData( // non-optimal configs from view. // // Fixed-seq workloads keep the existing per-axis Pareto since - // there's no separate "session-time" notion of total latency — + // there's no separate session-level notion of total latency — // their e2e IS the request latency, so a TTFT hack there reads // honestly on e2e too. The anti-hack constraint is specifically // about multi-turn agentic where TTFT measures a tiny fraction diff --git a/packages/app/src/components/inference/types.ts b/packages/app/src/components/inference/types.ts index 8bdc0433..ae73710b 100644 --- a/packages/app/src/components/inference/types.ts +++ b/packages/app/src/components/inference/types.ts @@ -258,7 +258,7 @@ export interface InferenceData extends Partial void; + selectedXAxisMode: 'ttft' | 'e2e' | 'interactivity'; + setSelectedXAxisMode: (mode: 'ttft' | 'e2e' | 'interactivity') => void; scaleType: 'auto' | 'linear' | 'log'; setScaleType: (type: 'auto' | 'linear' | 'log') => void; /** Coarse vendor / framework / agg-disagg / mtp-stp filters applied to the chart point set. */ diff --git a/packages/app/src/components/inference/ui/ChartDisplay.tsx b/packages/app/src/components/inference/ui/ChartDisplay.tsx index 76d3e6f4..6329df64 100644 --- a/packages/app/src/components/inference/ui/ChartDisplay.tsx +++ b/packages/app/src/components/inference/ui/ChartDisplay.tsx @@ -1,8 +1,5 @@ 'use client'; -import { - DISPLAY_MODEL_TO_DB, - NORMALIZED_E2E_OUTPUT_TOKENS, -} from '@semianalysisai/inferencex-constants'; +import { DISPLAY_MODEL_TO_DB } from '@semianalysisai/inferencex-constants'; import { track } from '@/lib/analytics'; import dynamic from 'next/dynamic'; import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; @@ -17,10 +14,7 @@ import type { OverlayData, TrendDataPoint, } from '@/components/inference/types'; -import { - processOverlayChartData, - selectUnofficialOverlayForMode, -} from '@/components/inference/utils'; +import { processOverlayChartData } from '@/components/inference/utils'; import { isRunComparisonEntry, makeRunComparisonEntry, @@ -59,12 +53,7 @@ import { sequenceKind, } from '@/lib/data-mappings'; import { useComparisonChangelogs } from '@/hooks/api/use-comparison-changelogs'; -import { - useDerivedAgenticMetrics, - type DerivedAgenticMetric, -} from '@/hooks/api/use-derived-agentic-metrics'; -import { isAgenticOnlyXAxisMode, type XAxisMode } from '@/components/inference/hooks/useChartData'; -import { isPersistedBenchmarkId } from '@/lib/benchmark-id'; +import type { XAxisMode } from '@/components/inference/hooks/useChartData'; import { useTrendData } from '@/components/inference/hooks/useTrendData'; import { getHardwareConfig, hardwareKeyMatchesAnyBase } from '@/lib/constants'; import { useLocale } from '@/lib/use-locale'; @@ -95,8 +84,6 @@ const STRINGS = { sourceUnofficial: 'Source: UNOFFICIAL', sourceOfficial: 'Source: SemiAnalysis InferenceX™', updated: 'Updated:', - normalizedE2eDisclaimer: - 'Normalized E2E requires persisted per-request traces, so unofficial-run overlays are unavailable for this experimental view.', selectDateRange: 'Select a date range or add a run to view GPU comparison', performanceOverTime: 'Performance Over Time', performanceOverTimeDesc: @@ -114,8 +101,6 @@ const STRINGS = { sourceUnofficial: '来源:非官方', sourceOfficial: '来源:SemiAnalysis InferenceX™', updated: '更新时间:', - normalizedE2eDisclaimer: - 'Normalized E2E 需要持久化的逐请求 trace 数据,因此该实验性视图不支持非官方运行覆盖。', selectDateRange: '请选择日期范围或添加运行以查看 GPU 对比', performanceOverTime: '性能趋势', performanceOverTimeDesc: '双击散点图上的数据点以追踪配置随时间的变化。', @@ -149,61 +134,8 @@ const X_AXIS_MODE_BUTTONS: { value: XAxisMode; label: string; labelZh: string }[ { value: 'interactivity', label: 'Interactivity', labelZh: '交互性' }, { value: 'e2e', label: 'E2E Latency', labelZh: '端到端延迟' }, { value: 'ttft', label: 'TTFT', labelZh: 'TTFT' }, - { value: 'normalized-e2e', label: 'Normalized E2E', labelZh: 'Normalized E2E' }, - { value: 'session-time', label: 'Session Time', labelZh: '会话时长' }, - { value: 'prefill-tps', label: 'Prefill TPS / user', labelZh: 'Prefill TPS / user' }, ]; -/** - * Presentation + data plumbing for the trace-derived x-axis modes (the - * agentic-only modes). One spec per mode keeps the x-label, chart heading, - * roofline corner, and derived-metric accessor in sync instead of scattering - * `selectedXAxisMode === …` conditionals through the render. - */ -interface DerivedXModeSpec { - xLabel: (percentileLabel: string) => string; - /** Chinese x-label; omit to reuse the English one (technical terms). */ - xLabelZh?: (percentileLabel: string) => string; - /** Chart heading suffix ("vs. …") shown above the plot. */ - heading: (percentileLabel: string) => string; - /** Chinese heading suffix; omit to reuse the English one. */ - headingZh?: (percentileLabel: string) => string; - rooflineCorner: 'upper_right' | 'upper_left'; - /** Pull the raw metric for this mode off the derived-metrics payload. */ - value: (m: DerivedAgenticMetric | undefined, percentile: string) => number | null | undefined; - /** Convert the raw metric to the plotted x value. */ - toX: (raw: number) => number; -} - -const DERIVED_X_MODE_SPECS: Partial> = { - 'session-time': { - xLabel: () => 'Mean Normalized Session Time (min)', - xLabelZh: () => '平均归一化会话时长(min)', - heading: () => 'vs. Mean Normalized Session Time', - headingZh: () => 'vs. 平均归一化会话时长', - rooflineCorner: 'upper_right', - value: (m) => m?.normalized_session_time_s, - toX: (raw) => raw / 60, - }, - 'normalized-e2e': { - xLabel: (pctl) => `${pctl} Normalized E2E @ ${NORMALIZED_E2E_OUTPUT_TOKENS} output tokens (s)`, - xLabelZh: (pctl) => `${pctl} Normalized E2E @ ${NORMALIZED_E2E_OUTPUT_TOKENS} 输出 token(s)`, - heading: (pctl) => `vs. ${pctl} Normalized E2E @ ${NORMALIZED_E2E_OUTPUT_TOKENS} output tokens`, - headingZh: (pctl) => `vs. ${pctl} Normalized E2E @ ${NORMALIZED_E2E_OUTPUT_TOKENS} 输出 token`, - rooflineCorner: 'upper_right', - value: (m, percentile) => - percentile === 'p75' ? m?.p75_normalized_e2e_400_s : m?.p90_normalized_e2e_400_s, - toX: (raw) => raw, - }, - 'prefill-tps': { - xLabel: () => 'P90 Prefill TPS per user (tok/s)', - heading: () => 'vs. P90 Prefill TPS / user', - rooflineCorner: 'upper_left', - value: (m) => m?.p90_prefill_tps_per_user, - toX: (raw) => raw, - }, -}; - const VIEW_MODE_OPTIONS: SegmentedToggleOption[] = [ { value: 'chart', @@ -263,9 +195,6 @@ export default function ChartDisplay() { totalDatesQueried, } = useComparisonChangelogs(selectedGPUs, selectedDateRange, dateRangeAvailableDates); - const [mounted, setMounted] = useState(false); - useEffect(() => setMounted(true), []); - const modelDbKeys = useMemo( () => DISPLAY_MODEL_TO_DB[selectedModel] ?? [selectedModel], [selectedModel], @@ -628,258 +557,191 @@ export default function ChartDisplay() { }, [effectiveGraphs, selectedXAxisMode]); const isAgenticSequence = sequenceKind(selectedSequence) === 'agentic'; - const useDerived = isAgenticSequence && isAgenticOnlyXAxisMode(selectedXAxisMode); - const derivedTargetIds = useMemo(() => { - if (!useDerived) return [] as number[]; - const ids = new Set(); - for (const graph of visibleGraphs) { - for (const point of graph.data) { - // Overlay-only agentic points carry no persisted id — skip them so we - // never request `?ids=0`/`?ids=NaN` (which 400s and errors the chart). - if (point.benchmark_type === 'agentic_traces' && isPersistedBenchmarkId(point.id)) { - ids.add(point.id); - } - } - } - return [...ids]; - }, [useDerived, visibleGraphs]); - const derivedQuery = useDerivedAgenticMetrics(derivedTargetIds, useDerived); - const derivedMetrics = derivedQuery.data; - const isDerivedLoading = - useDerived && - derivedTargetIds.length > 0 && - (derivedQuery.isPending || derivedQuery.isFetching) && - !derivedMetrics; - - // Set only when the user is on a derived (agentic-only) x-axis mode; the - // specs are module constants so this is referentially stable per mode. - const derivedSpec = useDerived ? DERIVED_X_MODE_SPECS[selectedXAxisMode] : undefined; - - const renderableGraphs = useMemo(() => { - if (!derivedSpec) return visibleGraphs; - if (!derivedMetrics) return visibleGraphs.map((graph) => ({ ...graph, data: [] })); - const xLabelFn = - locale === 'zh' && derivedSpec.xLabelZh ? derivedSpec.xLabelZh : derivedSpec.xLabel; - const xLabel = xLabelFn(selectedPercentile.toUpperCase()); - return visibleGraphs.map((graph) => { - const chartDefinition = { - ...graph.chartDefinition, - x_label: xLabel, - y_latency_limit: undefined, - [`${selectedYAxisMetric}_roofline` as keyof typeof graph.chartDefinition]: - derivedSpec.rooflineCorner, - }; - const data = graph.data - .map((point) => { - if (!isPersistedBenchmarkId(point.id)) return null; - const raw = derivedSpec.value(derivedMetrics[point.id], selectedPercentile); - if (raw === null || raw === undefined || !Number.isFinite(raw)) return null; - return { ...point, x: derivedSpec.toX(raw) }; - }) - .filter((point): point is NonNullable => point !== null); - return { ...graph, chartDefinition, data }; - }); - }, [derivedSpec, visibleGraphs, derivedMetrics, selectedYAxisMetric, selectedPercentile, locale]); - - const displayGraphs = - isFirstLoad || isDerivedLoading - ? [ - - - - - , - ] - : renderableGraphs.length === 0 - ? [] - : renderableGraphs.map((graph, graphIndex) => { - const isTimelineMode = Boolean( - selectedDateRange.startDate && selectedDateRange.endDate && selectedGPUs.length > 0, - ); - const replayAvailable = getViewMode(graphIndex) === 'chart' && !isTimelineMode; - return ( -
-
- + + + + , + ] + : visibleGraphs.length === 0 + ? [] + : visibleGraphs.map((graph, graphIndex) => { + const isTimelineMode = Boolean( + selectedDateRange.startDate && selectedDateRange.endDate && selectedGPUs.length > 0, + ); + const replayAvailable = getViewMode(graphIndex) === 'chart' && !isTimelineMode; + return ( +
+
+ handleViewModeChange(graphIndex, v)} + ariaLabel={t.viewMode} + testId={`inference-view-toggle-${graphIndex}`} + /> + } + hideImageExport={getViewMode(graphIndex) === 'table'} + setIsLegendExpanded={setIsLegendExpanded} + exportFileName={`InferenceX_${selectedModel}_${graph.chartDefinition.chartType}`} + onExportMp4={ + replayAvailable ? () => replayHandlesRef.current[graphIndex]?.open() : undefined + } + onExportCsv={() => { + const candidateVisibleData = isTimelineMode + ? graph.data.filter((d) => activeDates.has(`${d.date}_${d.hwKey}`)) + : graph.data; + const overlay = overlayDataByChartType[graph.chartDefinition.chartType]; + const { officialRows: visibleData, overlayRows: visibleOverlayRowsForExport } = isTimelineMode - ? 'gpu_timeseries' - : graph.chartDefinition.chartType === 'e2e' - ? 'latency' - : 'interactivity' - } - leadingControls={ - handleViewModeChange(graphIndex, v)} - ariaLabel={t.viewMode} - testId={`inference-view-toggle-${graphIndex}`} - /> - } - hideImageExport={getViewMode(graphIndex) === 'table'} - setIsLegendExpanded={setIsLegendExpanded} - exportFileName={`InferenceX_${selectedModel}_${graph.chartDefinition.chartType}`} - onExportMp4={ - replayAvailable - ? () => replayHandlesRef.current[graphIndex]?.open() - : undefined - } - onExportCsv={() => { - const candidateVisibleData = isTimelineMode - ? graph.data.filter((d) => activeDates.has(`${d.date}_${d.hwKey}`)) - : graph.data; - const overlay = selectUnofficialOverlayForMode( - selectedXAxisMode, - graph.chartDefinition.chartType, - overlayDataByChartType, - ); - const { - officialRows: visibleData, - overlayRows: visibleOverlayRowsForExport, - } = isTimelineMode ? { officialRows: candidateVisibleData, overlayRows: [] } : visibleComparisonRows(candidateVisibleData, overlay); - const { headers, rows } = inferenceChartToCsv( - visibleData, - graph.model, - graph.sequence, - ); - // Match warnings against the same series the chart annotates, - // including visible unofficial-run overlay series. - const issueNotes = matchKnownConfigIssues(graph.model, [ - ...visibleData, - ...visibleOverlayRowsForExport, - ]).map((issue) => - knownIssueCsvNote(issue, getDisplayLabel(getHardwareConfig(issue.hwKey))), - ); - exportToCsv( - `InferenceX_${selectedModel}_${graph.chartDefinition.chartType}`, - headers, - rows, - issueNotes, - ); - }} - /> - - {(() => { - const chartCaption = ( - <> -

- {metricTitle(graph.chartDefinition, selectedYAxisMetric, locale)}{' '} - {(() => { - // For Input metrics with dynamic x-axis, use dynamic heading. - // Classify off the ENGLISH title — the localized one has no - // 'input' substring to match on zh pages. - const isInputMetric = metricTitle( - graph.chartDefinition, - selectedYAxisMetric, - 'en', - ) - .toLowerCase() - .includes('input'); - if ( - graph.chartDefinition.chartType === 'interactivity' && - isInputMetric && - selectedXAxisMetric - ) { - if (selectedXAxisMetric === 'p99_ttft') { - return t.vsTtft('P99'); - } else if (selectedXAxisMetric === 'median_ttft') { - return t.vsTtft('Median'); - } + const { headers, rows } = inferenceChartToCsv( + visibleData, + graph.model, + graph.sequence, + ); + // Match warnings against the same series the chart annotates, + // including visible unofficial-run overlay series. + const issueNotes = matchKnownConfigIssues(graph.model, [ + ...visibleData, + ...visibleOverlayRowsForExport, + ]).map((issue) => + knownIssueCsvNote(issue, getDisplayLabel(getHardwareConfig(issue.hwKey))), + ); + exportToCsv( + `InferenceX_${selectedModel}_${graph.chartDefinition.chartType}`, + headers, + rows, + issueNotes, + ); + }} + /> + + {(() => { + const chartCaption = ( + <> +

+ {metricTitle(graph.chartDefinition, selectedYAxisMetric, locale)}{' '} + {(() => { + // For Input metrics with dynamic x-axis, use dynamic heading. + // Classify off the ENGLISH title — the localized one has no + // 'input' substring to match on zh pages. + const isInputMetric = metricTitle( + graph.chartDefinition, + selectedYAxisMetric, + 'en', + ) + .toLowerCase() + .includes('input'); + if ( + graph.chartDefinition.chartType === 'interactivity' && + isInputMetric && + selectedXAxisMetric + ) { + if (selectedXAxisMetric === 'p99_ttft') { + return t.vsTtft('P99'); + } else if (selectedXAxisMetric === 'median_ttft') { + return t.vsTtft('Median'); } + } - // The e2e chart heading follows the branch-level x-axis mode - // selector, including agentic-only derived metrics. - if (graph.chartDefinition.chartType === 'e2e') { - const modeSpec = DERIVED_X_MODE_SPECS[selectedXAxisMode]; - if (modeSpec) { - const heading = - locale === 'zh' && modeSpec.headingZh - ? modeSpec.headingZh - : modeSpec.heading; - return heading(selectedPercentile.toUpperCase()); - } - if (selectedE2eXAxisMetric?.endsWith('_ttft')) { - const percentile = selectedE2eXAxisMetric.replace(/_ttft$/u, ''); - const word = - percentile === 'median' ? 'Median' : percentile.toUpperCase(); - return t.vsTtft(word); - } - return isAgenticSequence - ? t.vsE2eLatency(selectedPercentile.toUpperCase()) - : t.vsE2eLatency(); + // The e2e chart heading follows the branch-level x-axis + // mode selector. + if (graph.chartDefinition.chartType === 'e2e') { + if (selectedE2eXAxisMetric?.endsWith('_ttft')) { + const percentile = selectedE2eXAxisMetric.replace(/_ttft$/u, ''); + const word = + percentile === 'median' ? 'Median' : percentile.toUpperCase(); + return t.vsTtft(word); } + return isAgenticSequence + ? t.vsE2eLatency(selectedPercentile.toUpperCase()) + : t.vsE2eLatency(); + } - // Fall back to configured heading - const configured = - graph.chartDefinition[ - `${selectedYAxisMetric}_heading` as keyof typeof graph.chartDefinition - ] || graph.chartDefinition.heading; - return locale === 'zh' ? zhHeading(String(configured)) : configured; - })()} -

-

- {getModelLabel(graph.model as Model)} •{' '} - {selectedPrecisions - .map((prec) => getPrecisionLabel(prec as Precision)) - .join(', ')}{' '} - • {getSequenceLabel(graph.sequence as Sequence)} •{' '} - {isUnofficialRun ? t.sourceUnofficial : t.sourceOfficial} - {selectedRunDate && ( - <> - {' '} - • {t.updated}{' '} - {new Date(`${selectedRunDate}T00:00:00Z`).toLocaleDateString( - locale === 'zh' ? 'zh-CN' : 'en-US', - { - year: 'numeric', - month: '2-digit', - day: '2-digit', - timeZone: 'UTC', - }, - )} - - )} -

- - {isUnofficialRun && selectedXAxisMode === 'normalized-e2e' && ( -

- {t.normalizedE2eDisclaimer} -

+ // Fall back to configured heading + const configured = + graph.chartDefinition[ + `${selectedYAxisMetric}_heading` as keyof typeof graph.chartDefinition + ] || graph.chartDefinition.heading; + return locale === 'zh' ? zhHeading(String(configured)) : configured; + })()} +

+

+ {getModelLabel(graph.model as Model)} •{' '} + {selectedPrecisions + .map((prec) => getPrecisionLabel(prec as Precision)) + .join(', ')}{' '} + • {getSequenceLabel(graph.sequence as Sequence)} •{' '} + {isUnofficialRun ? t.sourceUnofficial : t.sourceOfficial} + {selectedRunDate && ( + <> + {' '} + • {t.updated}{' '} + {new Date(`${selectedRunDate}T00:00:00Z`).toLocaleDateString( + locale === 'zh' ? 'zh-CN' : 'en-US', + { + year: 'numeric', + month: '2-digit', + day: '2-digit', + timeZone: 'UTC', + }, + )} + )} - +

+ + + + ); + + if (getViewMode(graphIndex) === 'table') { + const overlay = overlayDataByChartType[graph.chartDefinition.chartType]; + const { officialRows, overlayRows } = visibleComparisonRows( + graph.data, + overlay, + ); + return ( + <> + {chartCaption} + ); + } - if (getViewMode(graphIndex) === 'table') { - const overlay = selectUnofficialOverlayForMode( - selectedXAxisMode, - graph.chartDefinition.chartType, - overlayDataByChartType, - ); - const { officialRows, overlayRows } = visibleComparisonRows( - graph.data, - overlay, - ); - return ( - <> - {chartCaption} - - - ); - } - - return selectedGPUs.length > 0 && - ((selectedDateRange.startDate && selectedDateRange.endDate) || - selectedDates.length > 0) ? ( - 0 && + ((selectedDateRange.startDate && selectedDateRange.endDate) || + selectedDates.length > 0) ? ( + + ) : ( +
+ - ) : ( -
- - {selectedGPUs.length > 0 && - (!selectedDateRange.startDate || !selectedDateRange.endDate) && - selectedDates.length === 0 && ( -
-

- {t.selectDateRange} -

-
- )} -
- ); - })()} - {replayAvailable && ( - { - replayHandlesRef.current[graphIndex] = handle; - }} - parentChartId={`chart-${graphIndex}`} - chartDefinition={graph.chartDefinition} - yLabel={metricLabel(graph.chartDefinition, selectedYAxisMetric, locale)} - xLabel={graph.chartDefinition.x_label} - /> - )} - -
-
- ); - }); + {selectedGPUs.length > 0 && + (!selectedDateRange.startDate || !selectedDateRange.endDate) && + selectedDates.length === 0 && ( +
+

+ {t.selectDateRange} +

+
+ )} + + ); + })()} + {replayAvailable && ( + { + replayHandlesRef.current[graphIndex] = handle; + }} + parentChartId={`chart-${graphIndex}`} + chartDefinition={graph.chartDefinition} + yLabel={metricLabel(graph.chartDefinition, selectedYAxisMetric, locale)} + xLabel={graph.chartDefinition.x_label} + /> + )} + +
+
+ ); + }); return (
@@ -1001,12 +847,7 @@ export default function ChartDisplay() { data-testid="x-axis-mode-buttons" className="flex-wrap justify-center gap-x-1 gap-y-1.5 sm:gap-x-1.5" > - {X_AXIS_MODE_BUTTONS.filter(({ value }) => { - if (!isAgenticOnlyXAxisMode(value)) return true; - // Before mount, render all buttons so SSR and first client render match. - if (!mounted) return true; - return isAgenticSequence; - }).map(({ value, label, labelZh }) => ( + {X_AXIS_MODE_BUTTONS.map(({ value, label, labelZh }) => ( { - const overlays = { e2e: { id: 'e2e' }, interactivity: { id: 'interactivity' } }; - - it('suppresses raw unofficial E2E data for normalized E2E mode', () => { - expect(selectUnofficialOverlayForMode('normalized-e2e', 'e2e', overlays)).toBeNull(); - }); - - it('preserves matching unofficial overlays for supported modes', () => { - expect(selectUnofficialOverlayForMode('e2e', 'e2e', overlays)).toBe(overlays.e2e); - expect(selectUnofficialOverlayForMode('interactivity', 'interactivity', overlays)).toBe( - overlays.interactivity, - ); - }); -}); +import { filterDataByCostLimit, processOverlayChartData } from '@/components/inference/utils'; // --------------------------------------------------------------------------- // fixture factories diff --git a/packages/app/src/components/inference/utils.ts b/packages/app/src/components/inference/utils.ts index 2b8074d9..e88e5b0d 100644 --- a/packages/app/src/components/inference/utils.ts +++ b/packages/app/src/components/inference/utils.ts @@ -10,20 +10,6 @@ import { resolveXAxisField } from '@/components/inference/utils/resolveXAxisFiel import type { ChartDefinition, InferenceData, YAxisMetricKey } from './types'; -/** - * Select the matching unofficial-run overlay for a chart mode. Normalized E2E - * is intentionally excluded: unofficial benchmark rows do not include the - * persisted per-request trace needed to normalize before taking percentiles. - */ -export function selectUnofficialOverlayForMode( - xAxisMode: string, - chartType: 'e2e' | 'interactivity', - overlays: { e2e: T | null; interactivity: T | null }, -): T | null { - if (xAxisMode === 'normalized-e2e') return null; - return overlays[chartType]; -} - /** * Filters data points based on cost limits defined in the chart definition. * Only applies filtering for cost-related metrics, and only filters based on diff --git a/packages/app/src/components/inference/utils/e2eFrontier.ts b/packages/app/src/components/inference/utils/e2eFrontier.ts index 588afdd3..65c3434a 100644 --- a/packages/app/src/components/inference/utils/e2eFrontier.ts +++ b/packages/app/src/components/inference/utils/e2eFrontier.ts @@ -2,7 +2,7 @@ * @file e2eFrontier.ts * @description Shared seed for the anti-benchmark-hacking roofline restriction. * - * On the non-e2e xmode charts (interactivity, ttft, session-time, prefill-tps) + * On the non-e2e xmode charts (interactivity, ttft) * the roofline is restricted to the configs that ALSO win on end-to-end latency, * so a config can't top interactivity while tanking decode (or vice versa). Both * the official path (`useChartData` → benchmark ids → `isOnE2eFrontier`) and the diff --git a/packages/app/src/hooks/api/use-derived-agentic-metrics.test.ts b/packages/app/src/hooks/api/use-derived-agentic-metrics.test.ts deleted file mode 100644 index 2e54f418..00000000 --- a/packages/app/src/hooks/api/use-derived-agentic-metrics.test.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { describe, expect, it } from 'vitest'; - -import { chunkDerivedAgenticMetricIds } from './use-derived-agentic-metrics'; - -describe('chunkDerivedAgenticMetricIds', () => { - it('keeps every id while respecting the API limit', () => { - const ids = Array.from({ length: 401 }, (_, index) => index + 1); - const chunks = chunkDerivedAgenticMetricIds(ids); - - expect(chunks.map((chunk) => chunk.length)).toEqual([200, 200, 1]); - expect(chunks.flat()).toEqual(ids); - }); -}); diff --git a/packages/app/src/hooks/api/use-derived-agentic-metrics.ts b/packages/app/src/hooks/api/use-derived-agentic-metrics.ts deleted file mode 100644 index 388563d9..00000000 --- a/packages/app/src/hooks/api/use-derived-agentic-metrics.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { bulkIdsFetcher, useBulkIdsQuery } from './benchmark-id-query'; - -export interface DerivedAgenticMetric { - id: number; - /** Mean across sessions of e2e time (Σ per-turn request_latency) rescaled - * by mean_load / session_load. Null when the JSONL had no usable records. */ - normalized_session_time_s: number | null; - /** P90 of per-turn ISL/TTFT across every turn in every session. - * Null when no prefill rates could be computed. */ - p90_prefill_tps_per_user: number | null; - /** P75 normalized per-request E2E at a fixed 400-token output length. */ - p75_normalized_e2e_400_s: number | null; - /** P90 normalized per-request E2E at a fixed 400-token output length. */ - p90_normalized_e2e_400_s: number | null; -} - -export type DerivedAgenticMetricMap = Record; - -const MAX_IDS_PER_REQUEST = 200; - -export function chunkDerivedAgenticMetricIds(ids: number[]): number[][] { - const chunks: number[][] = []; - for (let i = 0; i < ids.length; i += MAX_IDS_PER_REQUEST) { - chunks.push(ids.slice(i, i + MAX_IDS_PER_REQUEST)); - } - return chunks; -} - -const fetchChunk = bulkIdsFetcher('derived-agentic-metrics'); - -// Unlike the other bulk endpoints, dashboards can put >200 agentic points on -// screen at once, so this fetcher splits the id set across parallel requests -// to stay under the route's MAX_IDS_PER_REQUEST. -async function fetchDerivedAgenticMetrics( - ids: number[], - signal?: AbortSignal, -): Promise { - if (ids.length === 0) return {}; - const maps = await Promise.all( - chunkDerivedAgenticMetricIds(ids).map((chunk) => fetchChunk(chunk, signal)), - ); - return Object.assign({}, ...maps) as DerivedAgenticMetricMap; -} - -/** - * Fetch per-id derived agentic metrics (session time + p90 prefill TPS/user) - * computed live from the stored aiperf profile_export.jsonl. Used to drive - * the "Session Time" and "Prefill TPS/user" chart variants. - * - * Ids without a trace_replay blob (older or non-aiperf agentic runs) are - * silently omitted from the response. - */ -export function useDerivedAgenticMetrics(ids: number[], enabled = true) { - return useBulkIdsQuery('derived-agentic-metrics', ids, enabled, fetchDerivedAgenticMetrics); -} diff --git a/packages/app/src/lib/benchmark-id.ts b/packages/app/src/lib/benchmark-id.ts index b1ccb8bc..6b1030df 100644 --- a/packages/app/src/lib/benchmark-id.ts +++ b/packages/app/src/lib/benchmark-id.ts @@ -8,7 +8,7 @@ * * A bare `typeof id === 'number'` check is NOT enough: `NaN` and `0` are both * `number` yet neither is a real row. Passing them to the id-keyed endpoints - * (`/api/v1/derived-agentic-metrics?ids=…`, `…?id=…`) yields a 400 (the routes + * (`/api/v1/agentic-aggregates?ids=…`, `…?id=…`) yields a 400 (the routes * filter to `Number.isFinite(n) && n > 0`), and building an * `/inference/agentic/` link out of one points at a non-existent row. * diff --git a/packages/constants/src/agentic.ts b/packages/constants/src/agentic.ts deleted file mode 100644 index 42eab306..00000000 --- a/packages/constants/src/agentic.ts +++ /dev/null @@ -1,2 +0,0 @@ -/** Fixed output length used by the experimental normalized-E2E chart metric. */ -export const NORMALIZED_E2E_OUTPUT_TOKENS = 400; diff --git a/packages/constants/src/index.ts b/packages/constants/src/index.ts index 7d3d6783..e767e500 100644 --- a/packages/constants/src/index.ts +++ b/packages/constants/src/index.ts @@ -1,4 +1,3 @@ -export * from './agentic'; export * from './framework-aliases'; export * from './github'; export * from './gpu-keys'; diff --git a/packages/db/src/backfill-aggregate-stats.ts b/packages/db/src/backfill-aggregate-stats.ts index 8aaa1214..a50f6ea0 100644 --- a/packages/db/src/backfill-aggregate-stats.ts +++ b/packages/db/src/backfill-aggregate-stats.ts @@ -94,12 +94,16 @@ async function main(): Promise { } let stats: AggregateStats; - if (row.aggregate_stats?.version === 3) { + // v3 onwards → current is a profile-only change (the server-derived + // fields haven't changed since v3), so skip re-reading the huge server + // blob and carry its KV/prefix distributions forward. + const storedVersion = row.aggregate_stats?.version; + if (storedVersion !== undefined && storedVersion >= 3 && storedVersion < STATS_VERSION) { const profileStats = await computeAggregateStats({ profileBlob: row.profile_export_jsonl_gz, serverBlob: null, }); - stats = mergeProfileStatsUpgrade(row.aggregate_stats, profileStats); + stats = mergeProfileStatsUpgrade(row.aggregate_stats!, profileStats); } else { const [serverRow] = await sql<{ server_metrics_json_gz: Buffer | null }[]>` select server_metrics_json_gz diff --git a/packages/db/src/etl/compute-aggregate-stats.test.ts b/packages/db/src/etl/compute-aggregate-stats.test.ts index 7b745c09..1a237664 100644 --- a/packages/db/src/etl/compute-aggregate-stats.test.ts +++ b/packages/db/src/etl/compute-aggregate-stats.test.ts @@ -66,12 +66,9 @@ describe('computeAggregateStats', () => { expect(stats.osl).toBeNull(); expect(stats.kvCacheUtil).toBeNull(); expect(stats.prefixCacheHitRate).toBeNull(); - expect(stats.normalizedSessionTimeS).toBeNull(); - expect(stats.p90PrefillTpsPerUser).toBeNull(); - expect(stats.normalizedE2e400).toBeNull(); }); - it('computes ISL/OSL percentiles + derived metrics from the profile blob', async () => { + it('computes ISL/OSL percentiles from the profile blob', async () => { const profileBlob = makeProfileBlob([ { isl: 100, osl: 50, rl: 1000, ttft: 100 }, { isl: 200, osl: 75, rl: 2000, ttft: 200 }, @@ -87,16 +84,6 @@ describe('computeAggregateStats', () => { // Server-side metrics still null when there's no server blob. expect(stats.kvCacheUtil).toBeNull(); expect(stats.prefixCacheHitRate).toBeNull(); - - // Derived: prefill TPS per turn = isl / (ttft/1000) = 1000 for each, so p90 = 1000. - expect(stats.p90PrefillTpsPerUser).toBeCloseTo(1000, 6); - // Normalized session time: T̃_i = T_i × (mean_load / load_i), then mean. - // loads = [150, 275, 400], mean_load = 275 - // scaled times (s) = [1×275/150, 2×275/275, 3×275/400] = [1.8333, 2, 2.0625] - // mean ≈ 1.9653 - expect(stats.normalizedSessionTimeS).toBeCloseTo(1.9653, 3); - expect(stats.normalizedE2e400?.n).toBe(3); - expect(stats.normalizedE2e400?.p90).toBeGreaterThan(0); }); it('computes KV util + prefix hit rate from the server blob alone', async () => { @@ -112,9 +99,6 @@ describe('computeAggregateStats', () => { // Profile-derived metrics absent. expect(stats.isl).toBeNull(); expect(stats.osl).toBeNull(); - expect(stats.normalizedSessionTimeS).toBeNull(); - expect(stats.p90PrefillTpsPerUser).toBeNull(); - expect(stats.normalizedE2e400).toBeNull(); }); it('tolerates a malformed profile blob by leaving its metrics null', async () => { @@ -123,9 +107,6 @@ describe('computeAggregateStats', () => { const stats = await computeAggregateStats({ profileBlob: garbage, serverBlob: null }); expect(stats.isl).toBeNull(); expect(stats.osl).toBeNull(); - expect(stats.normalizedSessionTimeS).toBeNull(); - expect(stats.p90PrefillTpsPerUser).toBeNull(); - expect(stats.normalizedE2e400).toBeNull(); // Version still set so the row is considered "computed". expect(stats.version).toBe(STATS_VERSION); }); @@ -145,8 +126,34 @@ describe('mergeProfileStatsUpgrade', () => { const merged = mergeProfileStatsUpgrade(existing, profile); expect(merged.version).toBe(STATS_VERSION); expect(merged.isl?.mean).toBe(100); - expect(merged.normalizedE2e400?.p90).toBeGreaterThan(0); expect(merged.kvCacheUtil).toEqual(existing.kvCacheUtil); expect(merged.prefixCacheHitRate).toEqual(existing.prefixCacheHitRate); }); + + it('drops the retired derived fields when upgrading a pre-v6 bundle', async () => { + // Bundles written before v6 carry normalizedSessionTimeS / + // p90PrefillTpsPerUser / normalizedE2e400 — the upgrade must not + // resurrect them into the new bundle. + const legacy = { + version: STATS_VERSION - 1, + isl: null, + osl: null, + kvCacheUtil: { mean: 0.4, p50: 0.4, p75: 0.5, p90: 0.6, p99: 0.7, n: 3 }, + prefixCacheHitRate: null, + normalizedSessionTimeS: 999, + p90PrefillTpsPerUser: 999, + normalizedE2e400: { mean: 1, p50: 1, p75: 1, p90: 2, p99: 3, n: 5 }, + }; + const profile = await computeAggregateStats({ + profileBlob: makeProfileBlob([{ isl: 100, osl: 100, rl: 2080, ttft: 100 }]), + serverBlob: null, + }); + + const merged = mergeProfileStatsUpgrade(legacy, profile); + expect(merged.version).toBe(STATS_VERSION); + expect(merged.kvCacheUtil).toEqual(legacy.kvCacheUtil); + expect(merged).not.toHaveProperty('normalizedSessionTimeS'); + expect(merged).not.toHaveProperty('p90PrefillTpsPerUser'); + expect(merged).not.toHaveProperty('normalizedE2e400'); + }); }); diff --git a/packages/db/src/etl/compute-aggregate-stats.ts b/packages/db/src/etl/compute-aggregate-stats.ts index f0385f6e..d4a76e27 100644 --- a/packages/db/src/etl/compute-aggregate-stats.ts +++ b/packages/db/src/etl/compute-aggregate-stats.ts @@ -1,9 +1,8 @@ /** * Pre-compute the per-row aggregate stats for an `agentic_trace_replay` * blob pair. The output lands in the `aggregate_stats` JSONB column so the - * detail page can serve the "Aggregates across configs" view and the - * derived chart x-axis modes from a single SQL row read, instead of - * parsing the raw blobs on demand. + * detail page can serve the "Aggregates across configs" view from a single + * SQL row read, instead of parsing the raw blobs on demand. * * Shape is intentionally versioned — bump `STATS_VERSION` whenever the * computation changes so the backfill script knows which rows to recompute. @@ -12,7 +11,6 @@ import { gunzipSync } from 'node:zlib'; import { gunzipJsonWithinLimit, streamCollectKeys } from './gzip-json-stream'; -import { computeDerivedFromBlob } from '../queries/derived-agentic-metrics'; import { STATS_VERSION, extractIslOsl, @@ -29,12 +27,19 @@ export interface AggregateStats { osl: MetricPercentiles | null; kvCacheUtil: MetricPercentiles | null; prefixCacheHitRate: MetricPercentiles | null; - /** Mean of (per-session e2e time × mean_load / session_load) across sessions. */ - normalizedSessionTimeS: number | null; - /** P90 of per-turn ISL/TTFT pooled across every session's turns. */ - p90PrefillTpsPerUser: number | null; - /** Per-request normalized E2E distribution at a fixed 400-token OSL. */ - normalizedE2e400: MetricPercentiles | null; +} + +/** + * The subset of an older-version bundle a profile-only upgrade carries + * forward. Pre-v6 bundles also carry the since-retired derived metrics + * (normalizedSessionTimeS, p90PrefillTpsPerUser, normalizedE2e400) — spreading + * `profile` first drops them from the merged result. + */ +interface ProfileUpgradeCarryover { + isl: MetricPercentiles | null; + osl: MetricPercentiles | null; + kvCacheUtil: MetricPercentiles | null; + prefixCacheHitRate: MetricPercentiles | null; } /** @@ -43,17 +48,13 @@ export interface AggregateStats { * while preserving its already-computed KV/cache distributions. */ export function mergeProfileStatsUpgrade( - existing: Omit & { - normalizedE2e400?: MetricPercentiles | null; - }, + existing: ProfileUpgradeCarryover, profile: AggregateStats, ): AggregateStats { return { ...profile, isl: profile.isl ?? existing.isl, osl: profile.osl ?? existing.osl, - normalizedSessionTimeS: profile.normalizedSessionTimeS ?? existing.normalizedSessionTimeS, - p90PrefillTpsPerUser: profile.p90PrefillTpsPerUser ?? existing.p90PrefillTpsPerUser, kvCacheUtil: existing.kvCacheUtil, prefixCacheHitRate: existing.prefixCacheHitRate, }; @@ -92,9 +93,6 @@ export async function computeAggregateStats(args: { }): Promise { let islPct: MetricPercentiles | null = null; let oslPct: MetricPercentiles | null = null; - let normalized: number | null = null; - let prefillP90: number | null = null; - let normalizedE2e400: MetricPercentiles | null = null; if (args.profileBlob) { try { @@ -102,10 +100,6 @@ export async function computeAggregateStats(args: { const { isl, osl } = extractIslOsl(jsonl); islPct = percentilesOf(isl); oslPct = percentilesOf(osl); - const derived = computeDerivedFromBlob(jsonl); - normalized = derived.normalized_session_time_s; - prefillP90 = derived.p90_prefill_tps_per_user; - normalizedE2e400 = derived.normalized_e2e_400; } catch { // ignore malformed blob — leave nulls } @@ -136,8 +130,5 @@ export async function computeAggregateStats(args: { osl: oslPct, kvCacheUtil: kvPct, prefixCacheHitRate: prefixPct, - normalizedSessionTimeS: normalized, - p90PrefillTpsPerUser: prefillP90, - normalizedE2e400, }; } diff --git a/packages/db/src/queries/agentic-aggregates.test.ts b/packages/db/src/queries/agentic-aggregates.test.ts index 0c4dbc89..d0daa2a3 100644 --- a/packages/db/src/queries/agentic-aggregates.test.ts +++ b/packages/db/src/queries/agentic-aggregates.test.ts @@ -129,9 +129,6 @@ interface WrittenStats { osl: unknown; kvCacheUtil: { mean: number } | null; prefixCacheHitRate: unknown; - normalizedSessionTimeS: number | null; - p90PrefillTpsPerUser: number | null; - normalizedE2e400: unknown; } /** Capture SQL template text + bound values for the write-back assertions. */ @@ -228,11 +225,11 @@ describe('getAgenticAggregates write-back', () => { expect(written.version).toBe(STATS_VERSION); // Server field FRESHLY recomputed (0.25), not the stale 0.9 carried forward. expect(written.kvCacheUtil?.mean).toBeCloseTo(0.25, 6); - // Derived fields FRESHLY recomputed (not the stale 999s). - expect(written.normalizedSessionTimeS).toBeCloseTo(3, 6); - expect(written.p90PrefillTpsPerUser).toBeCloseTo(200, 6); - expect(written.normalizedE2e400).not.toBeNull(); expect(written.isl).not.toBeNull(); + // The retired derived fields must not survive the recompute. + expect(written).not.toHaveProperty('normalizedSessionTimeS'); + expect(written).not.toHaveProperty('p90PrefillTpsPerUser'); + expect(written).not.toHaveProperty('normalizedE2e400'); }); it('does not write back for an id whose profile blob is missing/malformed', async () => { diff --git a/packages/db/src/queries/agentic-aggregates.ts b/packages/db/src/queries/agentic-aggregates.ts index 4917dc4c..901defb9 100644 --- a/packages/db/src/queries/agentic-aggregates.ts +++ b/packages/db/src/queries/agentic-aggregates.ts @@ -24,7 +24,6 @@ import { streamObject } from 'stream-json/streamers/stream-object.js'; import { gunzipJsonWithinLimit } from '../etl/gzip-json-stream'; import type { DbClient } from '../connection.js'; -import { computeDerivedFromBlob } from './derived-agentic-metrics'; import { extractIslOsl, fetchAggregateStatsRows, @@ -316,12 +315,8 @@ export async function getAgenticAggregates( const oslPct = percentilesOf(osl); result[id].isl = islPct; result[id].osl = oslPct; - // Recompute the profile-derived fields too (same jsonl, no extra - // read) so the self-healed bundle is a faithful full recompute — not - // a carry-forward of stale derived numbers stamped with a new - // version. Server-derived fields are filled in Pass 2 (or stay null - // when the server blob is absent, which is the correct complete value). - const derived = computeDerivedFromBlob(jsonl); + // Server-derived fields are filled in Pass 2 (or stay null when the + // server blob is absent, which is the correct complete value). pendingById.set(id, { traceReplayId: Number(row.trace_replay_id), stats: { @@ -330,9 +325,6 @@ export async function getAgenticAggregates( osl: oslPct, kvCacheUtil: null, prefixCacheHitRate: null, - normalizedSessionTimeS: derived.normalized_session_time_s, - p90PrefillTpsPerUser: derived.p90_prefill_tps_per_user, - normalizedE2e400: derived.normalized_e2e_400, }, }); } catch { @@ -404,8 +396,6 @@ interface AggregateStatsRow { osl: MetricPercentiles | null; kvCacheUtil: MetricPercentiles | null; prefixCacheHitRate: MetricPercentiles | null; - normalizedSessionTimeS: number | null; - p90PrefillTpsPerUser: number | null; } /** @@ -419,9 +409,6 @@ interface FullAggregateStats { osl: MetricPercentiles | null; kvCacheUtil: MetricPercentiles | null; prefixCacheHitRate: MetricPercentiles | null; - normalizedSessionTimeS: number | null; - p90PrefillTpsPerUser: number | null; - normalizedE2e400: MetricPercentiles | null; } function blankAggregate(id: number): AgenticAggregate { diff --git a/packages/db/src/queries/agentic-shared.ts b/packages/db/src/queries/agentic-shared.ts index d8673a07..d704abe0 100644 --- a/packages/db/src/queries/agentic-shared.ts +++ b/packages/db/src/queries/agentic-shared.ts @@ -31,8 +31,13 @@ import type { DbClient } from '../connection.js'; * * v5: reject osl <= 0 in extractTurn to exclude cancelled/empty-output turns * whose decode-interval math would explode normalized E2E to thousands of seconds. + * + * v6: drop the retired per-point derived metrics (normalizedSessionTimeS, + * p90PrefillTpsPerUser, normalizedE2e400) along with the experimental chart + * modes they fed. The bundle is now isl/osl + the server-derived kvCacheUtil + * and prefixCacheHitRate. */ -export const STATS_VERSION = 5; +export const STATS_VERSION = 6; interface ProfileRecord { metadata?: { benchmark_phase?: string }; diff --git a/packages/db/src/queries/derived-agentic-metrics.test.ts b/packages/db/src/queries/derived-agentic-metrics.test.ts deleted file mode 100644 index a39de670..00000000 --- a/packages/db/src/queries/derived-agentic-metrics.test.ts +++ /dev/null @@ -1,256 +0,0 @@ -import { gzipSync } from 'node:zlib'; - -import { describe, expect, it } from 'vitest'; - -import { STATS_VERSION } from './agentic-shared'; -import type { DbClient } from '../connection.js'; - -import { computeDerivedFromBlob, getDerivedAgenticMetrics } from './derived-agentic-metrics.js'; - -/** Build one aiperf JSONL record for the synthetic fixture. */ -function rec( - conversation_id: string, - turn_index: number, - fields: { isl: number; osl: number; ttft_ms: number; latency_ms: number }, -): string { - return JSON.stringify({ - metadata: { conversation_id, turn_index, benchmark_phase: 'profiling' }, - metrics: { - request_latency: { value: fields.latency_ms, unit: 'ms' }, - time_to_first_token: { value: fields.ttft_ms, unit: 'ms' }, - input_sequence_length: { value: fields.isl, unit: 'tokens' }, - output_sequence_length: { value: fields.osl, unit: 'tokens' }, - }, - }); -} - -describe('computeDerivedFromBlob', () => { - it('returns nulls when no usable records', () => { - const out = computeDerivedFromBlob(''); - expect(out.normalized_session_time_s).toBeNull(); - expect(out.p90_prefill_tps_per_user).toBeNull(); - expect(out.normalized_e2e_400).toBeNull(); - }); - - it('normalizes each request to 400 output tokens before taking percentiles', () => { - const jsonl = [ - // Both requests have TTFT=2s and ITL=20ms, despite very different OSL/E2E. - rec('s1', 0, { isl: 100, osl: 100, ttft_ms: 2000, latency_ms: 3980 }), - rec('s2', 0, { isl: 100, osl: 1000, ttft_ms: 2000, latency_ms: 21_980 }), - ].join('\n'); - - const out = computeDerivedFromBlob(jsonl); - // 2s TTFT + 399 × 20ms ITL = 9.98s for both requests. - expect(out.normalized_e2e_400?.n).toBe(2); - expect(out.normalized_e2e_400?.p75).toBeCloseTo(9.98, 8); - expect(out.normalized_e2e_400?.p90).toBeCloseTo(9.98, 8); - }); - - it('rescales single-session time and computes P90 prefill', () => { - // One session, two turns. load = (100+50) + (200+50) = 400. - // Single session ⇒ mean_load = load_i ⇒ T̃ = T = (1000+2000) ms = 3.0 s. - const jsonl = [ - rec('s1', 0, { isl: 100, osl: 50, ttft_ms: 500, latency_ms: 1000 }), - rec('s1', 1, { isl: 200, osl: 50, ttft_ms: 1000, latency_ms: 2000 }), - ].join('\n'); - const out = computeDerivedFromBlob(jsonl); - expect(out.normalized_session_time_s).toBeCloseTo(3, 6); - // Prefill TPS per turn: 100/0.5=200, 200/1.0=200 → global P90 = 200. - expect(out.p90_prefill_tps_per_user).toBeCloseTo(200, 6); - }); - - it('rescales times across sessions with unequal load', () => { - // s1: 1 turn, load = 100, T = 1s - // s2: 1 turn, load = 300, T = 3s - // mean_load = 200; T̃_1 = 1 * 200/100 = 2; T̃_2 = 3 * 200/300 = 2 - // Mean T̃ = 2.0 - const jsonl = [ - rec('s1', 0, { isl: 90, osl: 10, ttft_ms: 500, latency_ms: 1000 }), - rec('s2', 0, { isl: 270, osl: 30, ttft_ms: 500, latency_ms: 3000 }), - ].join('\n'); - const out = computeDerivedFromBlob(jsonl); - expect(out.normalized_session_time_s).toBeCloseTo(2, 6); - }); - - it('drops records missing required fields and skips non-profiling phase', () => { - const lines = [ - rec('s1', 0, { isl: 100, osl: 50, ttft_ms: 500, latency_ms: 1000 }), - // missing TTFT — should be skipped - JSON.stringify({ - metadata: { conversation_id: 's1', turn_index: 1, benchmark_phase: 'profiling' }, - metrics: { - request_latency: { value: 1000, unit: 'ms' }, - input_sequence_length: { value: 100, unit: 'tokens' }, - output_sequence_length: { value: 50, unit: 'tokens' }, - }, - }), - // warmup phase — should be skipped - JSON.stringify({ - metadata: { conversation_id: 's2', turn_index: 0, benchmark_phase: 'warmup' }, - metrics: { - request_latency: { value: 9999, unit: 'ms' }, - time_to_first_token: { value: 9999, unit: 'ms' }, - input_sequence_length: { value: 100, unit: 'tokens' }, - output_sequence_length: { value: 50, unit: 'tokens' }, - }, - }), - ]; - const out = computeDerivedFromBlob(lines.join('\n')); - expect(out.normalized_session_time_s).toBeCloseTo(1, 6); - expect(out.p90_prefill_tps_per_user).toBeCloseTo(200, 6); - }); - - it('p90 across turns: 10-turn session picks the right rank', () => { - // Prefill rates 100..1000 (per turn isl/ttft); p90 of 10 values (linear) = 910. - const turns = Array.from({ length: 10 }, (_, i) => - rec('s1', i, { - isl: (i + 1) * 100, // 100, 200, ..., 1000 tokens - osl: 10, - ttft_ms: 1000, // 1 second → rates: 100..1000 tps - latency_ms: 1500, - }), - ); - const out = computeDerivedFromBlob(turns.join('\n')); - expect(out.p90_prefill_tps_per_user).toBeCloseTo(910, 6); - }); - - it('excludes osl=0 (cancelled/empty-output) turns from normalized E2E', () => { - // Two normal turns + one cancelled turn (osl=0, latency=30s, ttft=1s). - // - // The cancelled turn must be excluded because observedDecodeIntervals collapses - // to max(0-1,1)=1, making itlMs=(30000-1000)/1=29000ms and normalizedMs explode - // to ~11 572 s — roughly 386× the real scale. (Pre-fix behavior for reference; - // this number is intentionally not asserted below to avoid enshrining the bug.) - // - // Normal turn A: isl=100, osl=50, ttft=500ms, latency=1000ms - // observedDecodeIntervals = max(49,1) = 49 - // itlMs = (1000-500)/49 - // normalizedMs = 500 + 399*(500/49) - // - // Normal turn B: isl=200, osl=100, ttft=1000ms, latency=3000ms - // observedDecodeIntervals = max(99,1) = 99 - // itlMs = (3000-1000)/99 - // normalizedMs = 1000 + 399*(2000/99) - const normA = (500 + (399 * 500) / 49) / 1000; // seconds - const normB = (1000 + (399 * 2000) / 99) / 1000; // seconds - - const jsonl = [ - rec('s1', 0, { isl: 100, osl: 50, ttft_ms: 500, latency_ms: 1000 }), - rec('s1', 1, { isl: 200, osl: 100, ttft_ms: 1000, latency_ms: 3000 }), - // Cancelled / empty-output turn — osl=0 must be rejected by extractTurn. - rec('s2', 0, { isl: 150, osl: 0, ttft_ms: 1000, latency_ms: 30000 }), - ].join('\n'); - - const out = computeDerivedFromBlob(jsonl); - - // Only the 2 normal turns contribute; osl=0 record is silently excluded. - expect(out.normalized_e2e_400?.n).toBe(2); - - // p90 of [normA, normB] sorted ascending (normA < normB): - // pos = 1*0.9 = 0.9; result = normA + (normB - normA)*0.9 - const expectedP90 = normA + (normB - normA) * 0.9; - expect(out.normalized_e2e_400?.p90).toBeCloseTo(expectedP90, 6); - - // Sanity: p90 should be single-digit seconds, not thousands. - expect(out.normalized_e2e_400!.p90).toBeLessThan(20); - }); -}); - -/** Capture SQL template text + bound values for the write-back assertions. */ -function mockSql(queue: unknown[][]): { - sql: DbClient; - calls: { text: string; values: unknown[] }[]; -} { - const responses = [...queue]; - const calls: { text: string; values: unknown[] }[] = []; - const sql = ((strings: TemplateStringsArray, ...values: unknown[]) => { - calls.push({ text: strings.join('?'), values }); - return Promise.resolve(responses.shift() ?? []); - }) as unknown as DbClient; - return { sql, calls }; -} - -describe('getDerivedAgenticMetrics write-back', () => { - it('self-heals aggregate_stats from the profile blob, carrying server fields forward', async () => { - const jsonl = [ - rec('s1', 0, { isl: 100, osl: 50, ttft_ms: 500, latency_ms: 1000 }), - rec('s1', 1, { isl: 200, osl: 50, ttft_ms: 1000, latency_ms: 2000 }), - ].join('\n'); - const blob = gzipSync(Buffer.from(jsonl)); - - // Stale v(N-1) row that DOES carry server-derived fields — they must be - // preserved in the healed bundle (derived route can't recompute them). - const staleServerKv = { mean: 0.4, p50: 0.4, p75: 0.5, p90: 0.6, p99: 0.7, n: 3 }; - const staleStats = { - version: STATS_VERSION - 1, - isl: null, - osl: null, - kvCacheUtil: staleServerKv, - prefixCacheHitRate: null, - normalizedSessionTimeS: 999, - p90PrefillTpsPerUser: 999, - normalizedE2e400: null, - }; - - const { sql, calls } = mockSql([ - // fetchAggregateStatsRows - [{ benchmark_result_id: 7, stats: staleStats }], - // fallback profile-blob query - [{ benchmark_result_id: 7, trace_replay_id: 870, blob }], - ]); - - const result = await getDerivedAgenticMetrics(sql, [7]); - - // Response is the freshly recomputed value, not the stale 999s. - expect(result[7]?.normalized_session_time_s).toBeCloseTo(3, 6); - expect(result[7]?.p90_prefill_tps_per_user).toBeCloseTo(200, 6); - - // 3 calls: stats read, blob read, write-back UPDATE. - expect(calls).toHaveLength(3); - expect(calls[2]!.text).toContain('update agentic_trace_replay set aggregate_stats'); - expect(calls[2]!.text).toContain('::jsonb where id'); - - // The write-back binds a COMPLETE, version-stamped bundle at the new version, - // recomputing profile fields and carrying server fields forward untouched. - // The payload OBJECT is bound directly (not stringified — that would - // double-encode into a JSONB string). - interface WrittenStats { - version: number; - isl: unknown; - osl: unknown; - kvCacheUtil: unknown; - normalizedSessionTimeS: number | null; - p90PrefillTpsPerUser: number | null; - } - const [written, traceReplayId] = calls[2]!.values as [WrittenStats, number]; - expect(traceReplayId).toBe(870); - expect(written.version).toBe(STATS_VERSION); - expect(written.normalizedSessionTimeS).toBeCloseTo(3, 6); - expect(written.p90PrefillTpsPerUser).toBeCloseTo(200, 6); - expect(written.isl).not.toBeNull(); - expect(written.osl).not.toBeNull(); - // Server-derived field carried forward from the stale row (not re-read). - expect(written.kvCacheUtil).toEqual(staleServerKv); - }); - - it('takes the fast path (no blob read, no write-back) when stats are current', async () => { - const currentStats = { - version: STATS_VERSION, - isl: null, - osl: null, - kvCacheUtil: null, - prefixCacheHitRate: null, - normalizedSessionTimeS: 1.5, - p90PrefillTpsPerUser: 42, - normalizedE2e400: { mean: 1, p50: 1, p75: 1, p90: 2, p99: 3, n: 5 }, - }; - const { sql, calls } = mockSql([[{ benchmark_result_id: 7, stats: currentStats }]]); - - const result = await getDerivedAgenticMetrics(sql, [7]); - - expect(result[7]?.normalized_session_time_s).toBe(1.5); - expect(result[7]?.p90_normalized_e2e_400_s).toBe(2); - // Only the stats read — no fallback blob query, no write-back. - expect(calls).toHaveLength(1); - }); -}); diff --git a/packages/db/src/queries/derived-agentic-metrics.ts b/packages/db/src/queries/derived-agentic-metrics.ts deleted file mode 100644 index 626ab9c7..00000000 --- a/packages/db/src/queries/derived-agentic-metrics.ts +++ /dev/null @@ -1,318 +0,0 @@ -/** - * Live-computed per-point metrics derived from the stored aiperf - * `profile_export.jsonl` blob. These aren't precomputed in the metrics JSONB - * because they require grouping by `conversation_id` and aggregating per - * session — work that's cheap once per agentic point but adds up to be - * meaningful only when actually plotted. - * - * - normalized_session_time_s: per the "Mean Normalized Session Time" proposal - * (https://gist.github.com/xinli-sw/115d370c17f6d1b977878b68530981fa). Sum of - * per-turn `request_latency` per session (inter-turn tool/thinking gaps are - * inherently excluded since we only sum the active GPU time, not wallclock). - * Each session's time is rescaled by `mean_load / session_load`, where load - * is Σ(ISL+OSL) across turns. The plotted value is the mean across sessions. - * - * - p90_prefill_tps_per_user: per the same gist's "Prefill" Pareto chart. - * Per turn: prefill_tps = ISL / TTFT_seconds. Single P90 across every turn - * in every session — the per-session percentile + cross-session mean - * sandwich was discarded because it just dampens tail behavior. - */ - -import { gunzipSync } from 'node:zlib'; - -import { NORMALIZED_E2E_OUTPUT_TOKENS } from '@semianalysisai/inferencex-constants'; - -import type { DbClient } from '../connection.js'; -import { - extractIslOsl, - fetchAggregateStatsRows, - meanOf, - percentilesOf, - quantile, - readNum, - STATS_VERSION, - writeBackTraceReplayJsonb, - type MetricPercentiles, -} from './agentic-shared'; - -export interface DerivedAgenticMetric { - /** benchmark_results.id this entry belongs to. */ - id: number; - /** Mean normalized session time in seconds. */ - normalized_session_time_s: number | null; - /** P90 of per-turn prefill tps/user (ISL / TTFT) across every turn in every session. */ - p90_prefill_tps_per_user: number | null; - /** P75 normalized per-request E2E at a fixed 400-token output length. */ - p75_normalized_e2e_400_s: number | null; - /** P90 normalized per-request E2E at a fixed 400-token output length. */ - p90_normalized_e2e_400_s: number | null; -} - -export type DerivedAgenticMetricMap = Record; - -/** - * The full `aggregate_stats` JSONB shape (mirrors `AggregateStats` in - * etl/compute-aggregate-stats.ts). Duplicated here rather than imported to keep - * this module off the etl import graph. When we self-heal from the profile blob - * alone, the server-derived fields (kvCacheUtil, prefixCacheHitRate) are carried - * forward untouched from the stale row — never re-reading the huge server blob. - * This mirrors the profile-only upgrade `backfill-aggregate-stats.ts` performs; - * the agentic-aggregates route (which does read the server blob) heals those - * server fields. - */ -interface StoredAggregateStats { - version: number; - isl: MetricPercentiles | null; - osl: MetricPercentiles | null; - kvCacheUtil: MetricPercentiles | null; - prefixCacheHitRate: MetricPercentiles | null; - normalizedSessionTimeS: number | null; - p90PrefillTpsPerUser: number | null; - normalizedE2e400: MetricPercentiles | null; -} - -/** - * JSONL blobs can be ~1-2 MB compressed (~5-10 MB raw) and Neon's serverless - * HTTP driver caps responses at 64 MB — chunk to stay well under. - */ -const QUERY_CHUNK_SIZE = 6; - -interface RecordMetrics { - request_latency?: { value?: number; unit?: string } | number; - time_to_first_token?: { value?: number; unit?: string } | number; - input_sequence_length?: { value?: number } | number; - output_sequence_length?: { value?: number } | number; -} - -interface RecordMetadata { - conversation_id?: string; - turn_index?: number; - benchmark_phase?: string; -} - -interface ProfileRecord { - metadata?: RecordMetadata; - metrics?: RecordMetrics; -} - -interface TurnFields { - request_latency_ms: number; - ttft_ms: number; - isl: number; - osl: number; -} - -function extractTurn(rec: ProfileRecord): TurnFields | null { - const m = rec.metrics ?? {}; - const rl = readNum(m.request_latency); - const tt = readNum(m.time_to_first_token); - const isl = readNum(m.input_sequence_length); - const osl = readNum(m.output_sequence_length); - if (rl === undefined || tt === undefined || isl === undefined || osl === undefined) return null; - if (rl <= 0 || tt <= 0 || isl <= 0 || osl <= 0) return null; - return { request_latency_ms: rl, ttft_ms: tt, isl, osl }; -} - -/** - * Parse one point's JSONL and return the two derived metrics. Returns - * `{ session_time: null, prefill: null }` if the blob has no usable records. - */ -export function computeDerivedFromBlob(jsonl: string): { - normalized_session_time_s: number | null; - p90_prefill_tps_per_user: number | null; - normalized_e2e_400: MetricPercentiles | null; -} { - // Group records by conversation_id, filter to the profiling phase. - const bySession = new Map(); - for (const line of jsonl.split('\n')) { - if (!line) continue; - let rec: ProfileRecord; - try { - rec = JSON.parse(line) as ProfileRecord; - } catch { - continue; - } - if (rec.metadata?.benchmark_phase && rec.metadata.benchmark_phase !== 'profiling') continue; - const sid = rec.metadata?.conversation_id; - if (!sid) continue; - const turn = extractTurn(rec); - if (!turn) continue; - let list = bySession.get(sid); - if (!list) { - list = []; - bySession.set(sid, list); - } - list.push(turn); - } - if (bySession.size === 0) { - return { - normalized_session_time_s: null, - p90_prefill_tps_per_user: null, - normalized_e2e_400: null, - }; - } - - // Per-session aggregates for session time; per-turn prefill rates pool into - // a single global array so the percentile sees the full distribution. - const sessionTimesS: number[] = []; - const sessionLoads: number[] = []; - const allPrefillRates: number[] = []; - const allNormalizedE2eS: number[] = []; - for (const turns of bySession.values()) { - let timeMs = 0; - let load = 0; - for (const t of turns) { - timeMs += t.request_latency_ms; - load += t.isl + t.osl; - const ttftSec = t.ttft_ms / 1000; - if (ttftSec > 0) allPrefillRates.push(t.isl / ttftSec); - - // Keep the observed TTFT, then project the request's mean decode - // interval to a fixed output length. Do this per request before taking - // percentiles so long original outputs do not dominate the tail. - const observedDecodeIntervals = Math.max(t.osl - 1, 1); - const itlMs = (t.request_latency_ms - t.ttft_ms) / observedDecodeIntervals; - const normalizedMs = t.ttft_ms + (NORMALIZED_E2E_OUTPUT_TOKENS - 1) * itlMs; - if ( - Number.isFinite(itlMs) && - itlMs >= 0 && - Number.isFinite(normalizedMs) && - normalizedMs > 0 - ) { - allNormalizedE2eS.push(normalizedMs / 1000); - } - } - if (load > 0) { - sessionTimesS.push(timeMs / 1000); - sessionLoads.push(load); - } - } - - // Normalized session time: T̃_i = T_i × (mean_load / load_i), then mean. - let normalized: number | null = null; - if (sessionTimesS.length > 0) { - const meanLoad = meanOf(sessionLoads); - if (meanLoad > 0) { - const scaled: number[] = []; - for (let i = 0; i < sessionTimesS.length; i++) { - const ti = sessionTimesS[i]!; - const li = sessionLoads[i]!; - if (li > 0) scaled.push(ti * (meanLoad / li)); - } - normalized = scaled.length > 0 ? meanOf(scaled) : null; - } - } - - let prefill: number | null = null; - if (allPrefillRates.length > 0) { - allPrefillRates.sort((a, b) => a - b); - prefill = quantile(allPrefillRates, 0.9); - } - - return { - normalized_session_time_s: normalized, - p90_prefill_tps_per_user: prefill, - normalized_e2e_400: percentilesOf(allNormalizedE2eS), - }; -} - -export async function getDerivedAgenticMetrics( - sql: DbClient, - benchmarkResultIds: number[], -): Promise { - if (benchmarkResultIds.length === 0) return {}; - - const result: DerivedAgenticMetricMap = {}; - - // Fast path: read the pre-computed values out of `aggregate_stats`. The - // ingest pipeline computes both metrics in the same pass that produces the - // percentile bundles, so a single SQL round-trip covers most ids without - // touching the gzipped profile blob. - const statsRows = await fetchAggregateStatsRows(sql, benchmarkResultIds); - - const idsNeedingBlob: number[] = []; - // Carry each stale/missing row's existing stats into the fallback so a - // self-heal preserves the server-derived fields (kvCacheUtil, - // prefixCacheHitRate) it can't recompute from the profile blob alone. - const staleStatsById = new Map(); - for (const row of statsRows) { - const id = Number(row.benchmark_result_id); - if (row.stats && Number(row.stats.version) === STATS_VERSION) { - result[id] = { - id, - normalized_session_time_s: row.stats.normalizedSessionTimeS ?? null, - p90_prefill_tps_per_user: row.stats.p90PrefillTpsPerUser ?? null, - p75_normalized_e2e_400_s: row.stats.normalizedE2e400?.p75 ?? null, - p90_normalized_e2e_400_s: row.stats.normalizedE2e400?.p90 ?? null, - }; - } else { - idsNeedingBlob.push(id); - staleStatsById.set(id, row.stats ?? null); - } - } - - if (idsNeedingBlob.length === 0) return result; - - // Fallback: parse the profile blob directly. Used for rows whose - // `aggregate_stats` is null or computed by an older STATS_VERSION; the - // backfill script drains the population so this path should be rare. - // `trace_replay_id` + the (small) stale `aggregate_stats` come along on the - // same join — no extra round-trip — so we can self-heal after recompute. - const rows: { - benchmark_result_id: number; - trace_replay_id: number; - blob: Buffer; - }[] = []; - for (let i = 0; i < idsNeedingBlob.length; i += QUERY_CHUNK_SIZE) { - const chunk = idsNeedingBlob.slice(i, i + QUERY_CHUNK_SIZE); - const chunkRows = (await sql` - select - br.id as benchmark_result_id, - atr.id as trace_replay_id, - atr.profile_export_jsonl_gz as blob - from benchmark_results br - join agentic_trace_replay atr on atr.id = br.trace_replay_id - where br.id = any(${chunk}::bigint[]) - and atr.profile_export_jsonl_gz is not null - `) as { benchmark_result_id: number; trace_replay_id: number; blob: Buffer }[]; - rows.push(...chunkRows); - } - - for (const row of rows) { - const id = Number(row.benchmark_result_id); - try { - const jsonl = gunzipSync(row.blob).toString('utf8'); - const { normalized_session_time_s, p90_prefill_tps_per_user, normalized_e2e_400 } = - computeDerivedFromBlob(jsonl); - result[id] = { - id, - normalized_session_time_s, - p90_prefill_tps_per_user, - p75_normalized_e2e_400_s: normalized_e2e_400?.p75 ?? null, - p90_normalized_e2e_400_s: normalized_e2e_400?.p90 ?? null, - }; - - // Self-heal the shared `aggregate_stats` bundle. We only have the profile - // blob here, so recompute the profile-derived fields (isl/osl + the three - // derived metrics) and carry the stale row's server-derived fields - // forward untouched — the profile-only upgrade the backfill CLI also - // performs. Fire-and-forget, best-effort (no-ops on a read-only replica). - const { isl, osl } = extractIslOsl(jsonl); - const prior = staleStatsById.get(id) ?? null; - const merged: StoredAggregateStats = { - version: STATS_VERSION, - isl: percentilesOf(isl), - osl: percentilesOf(osl), - kvCacheUtil: prior?.kvCacheUtil ?? null, - prefixCacheHitRate: prior?.prefixCacheHitRate ?? null, - normalizedSessionTimeS: normalized_session_time_s, - p90PrefillTpsPerUser: p90_prefill_tps_per_user, - normalizedE2e400: normalized_e2e_400, - }; - writeBackTraceReplayJsonb(sql, 'aggregate_stats', Number(row.trace_replay_id), merged); - } catch { - // Skip malformed blobs silently — frontend treats missing ids as "no data". - } - } - return result; -}