diff --git a/.agents/skills/neon/SKILL.md b/.agents/skills/neon/SKILL.md new file mode 100644 index 000000000..f1970c1fb --- /dev/null +++ b/.agents/skills/neon/SKILL.md @@ -0,0 +1,79 @@ +--- +name: neon +description: >- + Repository-specific guidance for working with InferenceX's Neon PostgreSQL + databases, connection variables, migrations, and query code. Use when Neon, + Postgres, DATABASE_URL, database, schema, migration, or backend data access is + mentioned. +--- + +# Neon PostgreSQL in InferenceX + +Use this skill for database work in this repository. InferenceX runs on Vercel +and uses Neon as PostgreSQL; do not introduce Neon Auth, Object Storage, +Functions, AI Gateway, an ORM, or new infrastructure unless the user asks. + +## Start with the repository + +Read the relevant local documentation before changing code: + +- `docs/index.md` +- `docs/architecture.md` for the API and cache boundaries +- `docs/data-pipeline.md` for ingestion and schema flow +- `docs/collectivex.md` for the separate CollectiveX database + +Existing database code lives in `packages/db/`. Reuse its connection helpers, +tagged SQL patterns, migrations, and scripts instead of creating a parallel +client or configuration layer. + +## Connections and credentials + +- Main read path: `DATABASE_READONLY_URL` +- Main administrative writes: `DATABASE_WRITE_URL` +- CollectiveX read path: `DATABASE_COLLECTIVEX_READONLY_URL` +- CollectiveX writes and migrations: `DATABASE_COLLECTIVEX_WRITE_URL` + +Keep credentials in environment variables. Never print, commit, copy into +source, or expose connection strings in logs or responses. Use the read-only +connection for diagnostics and normal reads; use a write connection only when +the requested task authorizes mutation. + +The application uses `@neondatabase/serverless` for serverless reads and +`postgres` for administrative or transaction-heavy scripts. Preserve that +split unless runtime requirements clearly demand a change. + +## Schema and query changes + +1. Inspect the current migration and query code before proposing a schema + change. +2. Add an append-only migration; never rewrite a migration that may have + already run. +3. Keep raw database rows in API responses unless a documented route is an + explicit exception. +4. Make multi-step writes atomic and safe under concurrent Vercel requests. +5. Verify indexes and bounded query behavior for new filters or ordering. +6. Add focused query/migration tests and run the repository checks. + +Useful commands: + +```bash +bun run admin:db:migrate +bun run admin:db:migrate:collectivex +bun run admin:db:verify +bun run typecheck +bun run test:unit +``` + +Do not run migrations, destructive SQL, or production writes during a review +or diagnostic request. For authorized schema work, prefer an isolated Neon +branch and confirm the target before applying changes. + +## Current Neon documentation + +Neon changes over time. For platform-specific behavior, verify against the +official documentation index rather than relying on this compact skill: + +- https://neon.com/docs/llms.txt +- https://neon.com/docs/connect/choose-connection +- https://neon.com/docs/serverless/serverless-driver +- https://neon.com/docs/introduction/branching diff --git a/.env.example b/.env.example index 2a662dd06..baf019b5c 100644 --- a/.env.example +++ b/.env.example @@ -4,8 +4,8 @@ # ║ Choose ONE option: standard PostgreSQL or Neon. ║ # ╚══════════════════════════════════════════════════════════════════════════╝ -# LAN / remote dev: set this machine's IP so the dev server accepts cross-origin requests -# NEXT_DEV_ALLOWED_ORIGINS=10.112.9.49 +# LAN / remote dev: set this machine's hostname so the dev server accepts cross-origin requests +# NEXT_DEV_ALLOWED_ORIGINS=dev-machine.local # Option A: Standard PostgreSQL (postgres.js TCP driver) # DATABASE_READONLY_URL=postgresql://postgres:postgres@localhost:5432/postgres @@ -17,10 +17,17 @@ # DATABASE_DRIVER=neon # DATABASE_SSL=true -# GitHub PAT (optional) — improves rate limits for star count and workflow metadata +# GitHub PAT — required for CollectiveX lazy ingest (sweep artifact reads); also used +# for framework releases, star counts, and unofficial-run artifact reads # Create at: https://github.com/settings/personal-access-tokens # GITHUB_TOKEN= +# CollectiveX database (separate Neon instance; a durable cache of GitHub sweep runs, +# populated lazily on read). Read URL powers /api/v1/collectivex/* reads; write URL +# powers the lazy ingest and the run-deletion route. +# DATABASE_COLLECTIVEX_READONLY_URL= +# DATABASE_COLLECTIVEX_WRITE_URL= + # ╔══════════════════════════════════════════════════════════════════════════╗ # ║ Production deployment (Vercel) ║ # ║ ║ @@ -51,3 +58,6 @@ # DATABASE_WRITE_URL= # INVALIDATE_SECRET= # FEEDBACK_SECRET= +# Bearer token for the CollectiveX run-deletion route (entered in the dashboard UI; +# deliberately separate from INVALIDATE_SECRET so it is independently rotatable) +# COLLECTIVEX_ADMIN_SECRET= diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 000000000..dba0cfe32 --- /dev/null +++ b/.prettierignore @@ -0,0 +1,10 @@ +# oxfmt reads this file by default (as does Prettier, if it is ever reintroduced). +# +# API response fixtures are stored minified: collectivex-latest.json holds a whole +# dataset payload, which pretty-printed is ~3.4k lines of indentation for ~60 KB of +# content and dominates any diff that touches it. They are machine-shaped payloads +# that are replaced wholesale rather than hand-edited, so formatting them buys +# nothing; without this entry the pre-commit hook would silently re-expand them. +packages/app/cypress/fixtures/api/collectivex-latest.json +packages/app/cypress/fixtures/api/collectivex-run-*.json +packages/app/cypress/fixtures/api/collectivex-runs.json diff --git a/AGENTS.md b/AGENTS.md index a5cae815e..fe60d1120 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,6 +2,8 @@ For detailed subsystem docs, see [docs/index.md](./docs/index.md). +> **Temporary language override (highest repository priority):** Until the user explicitly removes this override, ignore any instruction elsewhere in this repository that requires Simplified Chinese or bilingual output, including instructions in later sections, nested agent files, contribution guides, checklists, and templates. Work in English only: do not create or update `/zh` pages, Chinese documentation, translations, or Chinese text in commits, pull requests, issues, reviews, or comments. Leave existing Chinese content untouched. + > **PR and GitHub-issue titles & descriptions must be bilingual — include a Simplified Chinese version in addition to English.** Title format: ` / <中文标题>` (keep bracket prefixes at the front untranslated). In the PR/issue body, follow the English content with a `## 中文说明` section mirroring the summary; don't translate code blocks, logs, or stack traces — summarize around them. **Commit messages must include a Chinese translation too**: keep the subject line in English (conventional-commit style) and include the Chinese translation of the subject and key points in the commit body (e.g. a trailing `中文:` paragraph); squash-merge commits inherit the bilingual PR title, which satisfies the subject requirement automatically. > **Translation quality bar:** write natural technical Chinese, not word-for-word machine translation (style reference: [`vllm-project/vllm-ascend` `README.zh.md`](https://github.com/vllm-project/vllm-ascend/blob/main/README.zh.md)). Preserve product names, hardware SKUs, framework/library names (Next.js, React Query, D3.js, Tailwind ...), flags, and code identifiers in English. Use parenthetical English clarification for acronyms on first use. Preferred terms: benchmark 基准测试, dashboard 仪表板, chart 图表, config 配置, throughput 吞吐量, latency 延迟, single-node/multi-node 单节点/多节点, evaluation 评估, artifact 产物. @@ -71,10 +73,20 @@ API routes (`packages/app/src/app/api/v1/`): - `reliability` — raw `ReliabilityRow[]` - `evaluations` — raw `EvalRow[]` - `server-log` — retrieve benchmark runtime logs -- `invalidate` — invalidate API cache (admin) +- `invalidate` — invalidate API cache (admin; `?scope=collectivex` purges only that scope) +- `collectivex/latest`, `collectivex/runs`, `collectivex/runs/[runId]` — CollectiveX sweep data + from a **separate** Neon DB, populated lazily on read from GitHub Actions artifacts and served + assembled through the shared reader (the one deliberate exception to the raw-rows rule below); + `runs/[runId]` also handles admin DELETE. See [CollectiveX](./docs/collectivex.md). - `tco-feed?model=dsv4&workloads=1024x1024,8192x1024&tiers=30,50,75,100&format=csv` — per-hardware Pareto-frontier output-throughput reads at fixed interactivity tiers, for external spreadsheet TCO models (Excel Power Query); `view=scores` (optional `weights`, `workload_weights`, `alpha`) folds them into one tier-weighted, workload-blended, output-equivalent score per hardware -**API routes return raw DB data** — no presentation logic. Frontend handles all transformations. Sole exception: `tco-feed`, which runs the calculator's frontier interpolation server-side because its consumers (spreadsheets) cannot execute the TS transforms; its assumptions (tier weights, workload mix, α) enter only as explicit query params with documented defaults, so a published sheet's URL fully records its methodology. +**API routes return raw DB data** — no presentation logic. Frontend handles all transformations. +Exceptions: the CollectiveX routes assemble raw stored documents through the shared reader in +`packages/db/src/collectivex/` (see [docs/collectivex.md](./docs/collectivex.md) for why); and +`tco-feed`, which runs the calculator's frontier interpolation server-side because its consumers +(spreadsheets) cannot execute the TS transforms — its assumptions (tier weights, workload mix, α) +enter only as explicit query params with documented defaults, so a published sheet's URL fully +records its methodology. Static content routes (no DB): diff --git a/docs/collectivex.md b/docs/collectivex.md new file mode 100644 index 000000000..073103b91 --- /dev/null +++ b/docs/collectivex.md @@ -0,0 +1,94 @@ +# CollectiveX + +Design rationale for the CollectiveX tab's data pipeline. Unlike every other tab +(Neon DB → ETL ingest → `/api/v1/*`), CollectiveX uses **lazy ingest-on-read**: its +database is a durable cache of GitHub Actions, populated by the API routes themselves. + +## Why lazy ingest instead of the main pipeline + +- **Sweep artifacts expire after 14 days.** The sweep workflow + (`collectivex-sweep.yml` in the harness repo) uploads a matrix artifact + (`cxsweep-matrix-{run_id}`) and per-cell result artifacts + (`cxshard-{cell}-{run_id}-{attempt}`) with 14-day retention. Persisting on first + view makes a run outlive its artifacts once anyone has looked at it. +- **The sweep JSON contract is expected to change.** The DB stores the RAW documents + verbatim; the shared reader (`packages/db/src/collectivex/reader.ts`) is the single + transform point and runs at API-read time, so a reader fix retroactively applies to + already-stored runs — no re-ingest. A contract change = reader change + a bump of the + numeric `version` in the harness's `experimental/CollectiveX/configs/sweep.json`. +- **No CI plumbing.** There is no ingest workflow, no cross-repo dispatch, and no GH + secrets. Runs launched via `gh api` on any harness branch appear on the dashboard + within the CDN TTL of someone viewing the page — only the workflow identity is + checked, never the branch. + +## How it works + +`packages/app/src/lib/collectivex-lazy-ingest.ts` exposes three `ensure*` functions the +routes call before reading the DB (`packages/db/src/queries/collectivex.ts`): + +- `ensureLatestCollectiveXRun` — walk GitHub's completed sweep runs newest-first; stop at + the first live requested-version run; persist it if absent. +- `ensureCollectiveXRunsList` — progressively backfill every requested-version run whose + 14-day artifacts may still be downloaded. Each request changes at most eight rows; an + incomplete response is uncached and the client refetches until the workflow history is + exhausted. Known rows do not consume the batch, so discovery advances past the newest runs. + GitHub discovery is limited to runs created within the last 44 days: the 30-day workflow + rerun window plus 14-day artifact retention, covering the oldest rerun whose artifacts can + still exist without rescanning permanent workflow history on every cold-origin request. +- `ensureCollectiveXRun` — fetch one run by id, or compare a stored run's `run_attempt` + against GitHub and refresh it when a rerun is newer. Only completed runs are persisted. + +Key invariants: + +- **Writes are atomic and race-safe**: one CTE statement with + `ON CONFLICT (run_id) DO NOTHING`; concurrent first-viewers can't double-ingest or + expose a partial run. A GitHub re-run (newer `run_attempt`) is replaced through a + `FOR UPDATE`-guarded refresh statement. +- **Deletion tombstones** (`cx_runs.deleted_at`, documents freed): discovery must never + resurrect a deleted run. Re-ingesting via the CLI + (`bun run admin:db:ingest:collectivex `) clears the tombstone — that CLI is + the operator tool for pre-warming runs before artifact expiry, backfills, and un-deletes. +- **"Latest" orders by `run_id`** (monotonic with run creation, matching the discovery + walk) — not by completion time, where a long-failing older run would shadow a newer + successful one. +- **GitHub being down never takes the page down**: routes serve whatever the DB holds and + only surface an error when there is no stored fallback. +- **Run-list completeness is progressive**: `/api/v1/collectivex/runs` returns + `discovery_complete: false` while another bounded ingest pass is required. Those responses + use `private, no-store`; the client polls once per second until the field becomes `true`. + Stored runs remain visible indefinitely, while never-ingested runs disappear with their + upstream artifacts and can no longer be reconstructed. +- **Caching**: responses carry the `collectivex` CDN tag with a 60s + `s-maxage` (freshness bound for lazy discovery). Run deletion and + `POST /api/v1/invalidate?scope=collectivex` purge only that tag; the main dashboard's + blob cache is untouched by CollectiveX operations. +- **Env**: `DATABASE_COLLECTIVEX_READONLY_URL` (must be the same primary as the write URL + — the routes read their own writes), `DATABASE_COLLECTIVEX_WRITE_URL` (direct/unpooled; + also used by migrations via `bun run admin:db:migrate:collectivex`), + `COLLECTIVEX_ADMIN_SECRET` (delete route Bearer token — deliberately not + INVALIDATE_SECRET, since it is remembered in browser localStorage), and `GITHUB_TOKEN`. + +## Multi-run explorer + +The frontend loads every stored live run summary for the selected benchmark version and keeps +refetching while recent GitHub history is still being ingested. The summary query has no arbitrary +row cap and does not load artifact documents. Each table row has a visibility checkbox; checking a +run fetches its cached dataset through `/api/v1/collectivex/runs/[runId]`. Checked datasets are +combined client-side, and the EP, phase, kernel mode, precision, SKU, and backend controls filter +their combined series. + +Series ids are namespaced by GitHub Actions run id so the same matrix case from two runs remains +independently toggleable. Configuration color stays consistent across runs; run identity is encoded +as a stable, non-repeating line dash pattern shown in both the run table and legend, keeping run ids +out of visible legend labels while retaining them in the legend item's accessible title. +The newest run with measured cases is checked by default; newer incomplete sweeps remain listed but +cannot blank the initial explorer. Deletion is a row action in the run table and keeps the same +tombstone semantics described above. + +## The raw-rows exception + +CollectiveX routes return the **assembled** dataset (reader over stored matrix + docs) +instead of raw rows. The reader is shared between the app and the CLI through the db +package (`@semianalysisai/inferencex-db/collectivex/*`), so ingest-time validation and +read-time assembly can never drift; shipping raw docs to the client would only move the +same shared transform across the wire. diff --git a/docs/index.md b/docs/index.md index ea6154edd..c4a72633b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -15,4 +15,5 @@ Design rationale and non-obvious conventions. See [CLAUDE.md](../CLAUDE.md) for - [Data Transforms](./data-transforms.md) — Full pipeline from BenchmarkRow to RenderableGraph: type hierarchy, hardware key construction, derived metrics, memoization strategy - [State Ownership](./state-ownership.md) — Which context owns which state, availability filtering cascade, comparison date mechanics, URL param sync - [Blog](./blog.md) — MDX content system, SEO features (OG images, RSS, llms.txt, JSON-LD), TOC sidebar, reading progress, heading links, analytics events +- [CollectiveX](./collectivex.md) — lazy ingest-on-read pipeline (separate Neon DB as a durable GitHub-artifact cache), tombstoned deletes, raw-docs storage + shared reader - [Chinese Pages (/zh)](./i18n.md) — Why hand-authored /zh pages instead of an i18n framework, hreflang pairing, blog translation pairing, html lang workaround, CJK reading time/slugs diff --git a/package.json b/package.json index d7963e453..d4731b6b1 100644 --- a/package.json +++ b/package.json @@ -34,10 +34,12 @@ "admin:cache:warmup": "bun run --cwd packages/app cache:warmup", "admin:db:ingest:run": "bun run --cwd packages/db db:ingest:run", "admin:db:ingest:ci": "bun run --cwd packages/db db:ingest:ci", + "admin:db:ingest:collectivex": "bun run --cwd packages/db db:ingest:collectivex", "admin:db:prepare:ci": "bun run --cwd packages/db db:prepare:ci", "admin:db:ingest:gcs": "bun run --cwd packages/db db:ingest:gcs", "admin:db:ingest:supplemental": "bun run --cwd packages/db db:ingest:supplemental", "admin:db:migrate": "bun run --cwd packages/db db:migrate", + "admin:db:migrate:collectivex": "bun run --cwd packages/db db:migrate:collectivex", "admin:db:apply-overrides": "bun run --cwd packages/db db:apply-overrides", "admin:db:reset": "bun run --cwd packages/db db:reset", "admin:db:verify": "bun run --cwd packages/db db:verify" diff --git a/packages/app/cypress/component/tab-nav.cy.tsx b/packages/app/cypress/component/tab-nav.cy.tsx index df297e61e..e479b7b51 100644 --- a/packages/app/cypress/component/tab-nav.cy.tsx +++ b/packages/app/cypress/component/tab-nav.cy.tsx @@ -76,6 +76,11 @@ describe('TabNav — unofficialrun URL preservation (issue #319)', () => { 'href', '/submissions?unofficialruns=12345', ); + cy.get('[data-testid="tab-trigger-collectivex"]').should( + 'have.attr', + 'href', + '/collectivex?unofficialruns=12345', + ); cy.get('[data-testid="tab-trigger-historical"]').should( 'have.attr', 'href', @@ -115,6 +120,7 @@ describe('TabNav — Hidden popover for gated tabs', () => { mountTabNav({}); cy.get('[data-testid="tab-trigger-inference"]').should('exist'); cy.get('[data-testid="tab-trigger-gpu-specs"]').should('exist'); + cy.get('[data-testid="tab-trigger-collectivex"]').should('exist'); cy.get('[data-testid="tab-trigger-submissions"]').should('exist'); cy.get('[data-testid="tab-trigger-hidden"]').should('not.exist'); cy.get('[data-testid="tab-trigger-feedback"]').should('not.exist'); diff --git a/packages/app/cypress/e2e/agentic-point-time-series.cy.ts b/packages/app/cypress/e2e/agentic-point-time-series.cy.ts index cc93bd3fd..2e71f663d 100644 --- a/packages/app/cypress/e2e/agentic-point-time-series.cy.ts +++ b/packages/app/cypress/e2e/agentic-point-time-series.cy.ts @@ -251,10 +251,10 @@ describe('Agentic point orchestrator metric sources', () => { beforeEach(() => { const prefill = sourceSeries( { - id: 'dynamo|prefill|10.30.1.56:7500|prefill-a|0|0', + id: 'dynamo|prefill|prefill-a.internal.test:7500|prefill-a|0|0', adapter: 'dynamo', role: 'prefill', - endpointUrl: '10.30.1.56:7500', + endpointUrl: 'prefill-a.internal.test:7500', nativeRole: 'prefill', workerId: 'prefill-a', dpRank: '0', @@ -265,10 +265,10 @@ describe('Agentic point orchestrator metric sources', () => { ); const decode = sourceSeries( { - id: 'dynamo|decode|10.30.1.206:7516|decode-a|0|0', + id: 'dynamo|decode|decode-a.internal.test:7516|decode-a|0|0', adapter: 'dynamo', role: 'decode', - endpointUrl: '10.30.1.206:7516', + endpointUrl: 'decode-a.internal.test:7516', nativeRole: 'backend', workerId: 'decode-a', dpRank: '0', diff --git a/packages/app/cypress/e2e/collectivex.cy.ts b/packages/app/cypress/e2e/collectivex.cy.ts new file mode 100644 index 000000000..0ac500461 --- /dev/null +++ b/packages/app/cypress/e2e/collectivex.cy.ts @@ -0,0 +1,412 @@ +import { buildRunSummary } from '@semianalysisai/inferencex-db/collectivex/reader'; +import { + buildDataset, + makeCollectiveXDataset, + makeRawShard, +} from '@/components/collectivex/test-fixture'; +import type { CollectiveXDataset } from '@/components/collectivex/types'; + +// The neutral view: one run's measured series plus its full case coverage, +// served from the CollectiveX database via /api/v1/collectivex/latest, +// /api/v1/collectivex/runs (picker listing), and /api/v1/collectivex/runs/{id}. +const SOURCE_SHA = 'c'.repeat(40); +const dataset = makeCollectiveXDataset(); +const runId = dataset.run.run_id; +const comparisonDataset = buildDataset({ + shards: [makeRawShard(), makeRawShard({ precision: 'fp8' })], + meta: { + run_id: '159', + generated_at: '2026-07-07T12:20:00Z', + source_sha: 'd'.repeat(40), + }, +}); +const incompleteDataset = buildDataset({ + shards: [], + meta: { + run_id: '161', + generated_at: '2026-07-09T12:20:00Z', + conclusion: 'failure', + }, +}); +const ADMIN_TOKEN_KEY = 'collectivex-admin-token'; + +function installRuns(bodies: CollectiveXDataset[] = [dataset]) { + cy.intercept('GET', '/api/v1/collectivex/runs?*', { + body: { version: 1, runs: bodies.map(buildRunSummary), discovery_complete: true }, + }).as('runs'); +} + +function installRun(body: CollectiveXDataset = dataset, alias = 'run') { + cy.intercept('GET', `/api/v1/collectivex/runs/${body.run.run_id}*`, { body }).as(alias); +} + +function openCollectiveX() { + cy.visit('/collectivex'); + cy.wait('@runs'); + cy.wait('@run'); + cy.get('[data-testid="collectivex-display"]').should('be.visible'); +} + +describe('CollectiveX neutral run view', () => { + beforeEach(() => { + installRuns(); + installRun(); + openCollectiveX(); + }); + + it('shows the run header, coverage stats, and revision-pinned source links', () => { + cy.get('[data-testid="collectivex-run-conclusion"]') + .should('contain.text', `#${runId}`) + .and('contain.text', 'success'); + cy.get('[data-testid="collectivex-display"]') + .should('contain.text', `${dataset.run.measured_cases}/${dataset.run.requested_cases}`) + .and('contain.text', String(dataset.series.length)); + cy.get('[data-testid="collectivex-version-select"]').should('contain.text', 'V1'); + cy.get('[data-testid="collectivex-runs-table"]').should('have.css', 'max-height', '448px'); + cy.get('[data-testid="collectivex-source-link"]').should( + 'have.attr', + 'href', + `https://github.com/SemiAnalysisAI/InferenceX/tree/${SOURCE_SHA}/experimental/CollectiveX`, + ); + cy.get('[data-testid="collectivex-methodology-link"]') + .should('contain.text', 'Methodology') + .and( + 'have.attr', + 'href', + `https://github.com/SemiAnalysisAI/InferenceX/blob/${SOURCE_SHA}/experimental/CollectiveX/docs/methodology.md`, + ); + }); + + it('keeps loading bounded discovery batches until every run is listed', () => { + let requests = 0; + cy.intercept('GET', '/api/v1/collectivex/runs?*', (request) => { + requests += 1; + request.reply({ + body: { + version: 1, + runs: (requests === 1 ? [] : [dataset, comparisonDataset]).map(buildRunSummary), + discovery_complete: requests > 1, + }, + }); + }).as('progressiveRuns'); + + cy.reload(); + cy.wait('@progressiveRuns'); + cy.wait('@progressiveRuns'); + cy.wait('@run'); + + cy.get(`[data-testid="collectivex-run-row-${comparisonDataset.run.run_id}"]`).should( + 'be.visible', + ); + cy.get(`[data-testid="collectivex-run-visible-${runId}"]`).should('be.checked'); + cy.get('[data-testid="collectivex-explorer-chart"] .line-path').should('have.length', 1); + cy.then(() => expect(requests).to.be.gte(2)); + }); + + it('renders the default decode round-trip chart for the EP8 scale-up series', () => { + cy.get('[data-testid="collectivex-main-chart"]') + .should('contain.text', 'Round trip (measured) · decode · p99') + .and('contain.text', 'deepep-v2'); + cy.get('[data-testid="collectivex-explorer-chart"] .line-path').should('have.length', 1); + }); + + it('only exposes dimensions that vary in the current matrix', () => { + cy.get('[data-testid="collectivex-ep-select"]').should('be.visible'); + cy.get('[data-testid="collectivex-phase-toggle"]').should('be.visible'); + cy.get('[data-testid="collectivex-precision-toggle"]').should('be.visible'); + cy.get('[data-testid="collectivex-sku-select"]').should('be.visible'); + cy.get('[data-testid="collectivex-backend-select"]').should('be.visible'); + cy.get('[data-testid="collectivex-mode-toggle"]').should('not.exist'); + cy.get('[data-testid="collectivex-fabric-scope-toggle"]').should('not.exist'); + cy.get('[data-testid="collectivex-routing-select"]').should('not.exist'); + }); + + it('selects the EP16 series through the identity controls', () => { + cy.get('[data-testid="collectivex-ep-select"]').click(); + cy.contains('[role="option"]', 'EP16').click(); + + cy.get('[data-testid="collectivex-main-chart"]') + .should('contain.text', 'mori') + .and('contain.text', 'EP16'); + cy.get('[data-testid="collectivex-explorer-chart"] .line-path').should('have.length', 1); + }); + + it('renders an nccl-ep backend series end to end', () => { + const ncclEp = buildDataset({ + shards: [makeRawShard({ backend: 'nccl-ep', implName: 'nccl-ep' })], + }); + installRuns([ncclEp]); + installRun(ncclEp); + cy.reload(); + cy.wait('@runs'); + cy.wait('@run'); + cy.get('[data-testid="collectivex-main-chart"]').should('contain.text', 'nccl-ep'); + cy.get('[data-testid="collectivex-explorer-chart"] .line-path').should('have.length', 1); + }); + + it('switches the y-axis to per-GPU payload bandwidth', () => { + cy.get('[data-testid="collectivex-y-axis-select"]').click(); + cy.contains('[role="option"]', 'Payload bandwidth').click(); + cy.get('[data-testid="collectivex-main-chart"]').should('contain.text', 'Payload bandwidth'); + cy.get('[data-testid="collectivex-explorer-chart"] .line-path').should('have.length', 1); + }); + + it('labels the activation-data rate footnote with the selected metric', () => { + cy.get('[data-testid="collectivex-y-axis-select"]').click(); + cy.contains('[role="option"]', 'Activation-data rate').click(); + cy.get('[data-testid="collectivex-main-chart"]') + .should('contain.text', 'Activation-data rate') + .and('not.contain.text', 'Payload rate is derived'); + }); + + it('exposes the kernel-mode toggle when a run measured both modes and pins the LL series', () => { + const withLowLatency = buildDataset({ + shards: [makeRawShard(), makeRawShard({ mode: 'low-latency' })], + }); + installRuns([withLowLatency]); + installRun(withLowLatency); + cy.reload(); + cy.wait('@runs'); + cy.wait('@run'); + + cy.get('[data-testid="collectivex-mode-toggle"]').should('be.visible'); + cy.get('[data-testid="collectivex-main-chart"]').should('contain.text', 'deepep-v2'); + cy.get('[data-testid="collectivex-explorer-chart"] .line-path').should('have.length', 1); + + cy.get('[data-testid="collectivex-mode-toggle"]').contains('Low-latency').click(); + cy.get('[data-testid="chart-legend"]').should('contain.text', 'low-latency'); + cy.get('[data-testid="collectivex-explorer-chart"] .line-path').should('have.length', 1); + }); + + it('selects the available phase when a partial run only measured prefill', () => { + const prefill = buildDataset({ shards: [makeRawShard({ phase: 'prefill' })] }); + installRuns([prefill]); + installRun(prefill); + cy.reload(); + cy.wait('@runs'); + cy.wait('@run'); + cy.get('[data-testid="collectivex-phase-toggle"]').should('contain.text', 'Prefill'); + cy.get('[data-testid="collectivex-main-chart"]').should('contain.text', 'prefill'); + cy.get('[data-testid="collectivex-explorer-chart"] .line-path').should('have.length', 1); + }); + + it('clears the chart when the sole series is toggled off in the legend', () => { + cy.get('[data-testid="collectivex-explorer-chart"] .line-path').should('have.length', 1); + cy.get('[data-testid="chart-legend"] input[type="checkbox"]:checked') + .first() + .uncheck({ force: true }); + cy.get('[data-testid="collectivex-explorer-chart"] .line-path').should('not.exist'); + }); + + it('pins a compact tooltip on point click', () => { + cy.get('[data-testid="collectivex-explorer-chart"] .point').first().click({ force: true }); + cy.get('[data-chart-tooltip]:visible') + .should('contain.text', 'Click elsewhere to dismiss') + .and('contain.text', 'Round trip (measured) p99:') + .and('contain.text', 'Latency p50 / p90 / p95 / p99') + .and('not.contain.text', 'Expert CV') + .and('not.contain.text', 'evidence='); + }); + + it('lists every version-matching run and overlays checked runs', () => { + installRuns([dataset, comparisonDataset]); + installRun(); + installRun(comparisonDataset, 'comparisonRun'); + cy.reload(); + cy.wait('@runs'); + cy.wait('@run'); + + cy.get(`[data-testid="collectivex-run-row-${runId}"]`).should('be.visible'); + cy.get(`[data-testid="collectivex-run-row-${comparisonDataset.run.run_id}"]`).should( + 'be.visible', + ); + cy.get(`[data-testid="collectivex-run-visible-${runId}"]`).should('be.checked'); + cy.get('[data-testid="collectivex-explorer-chart"] .line-path').should('have.length', 1); + cy.get(`[data-testid="collectivex-run-line-style-${runId}"] line`).should( + 'not.have.attr', + 'stroke-dasharray', + ); + cy.get( + `[data-testid="collectivex-run-line-style-${comparisonDataset.run.run_id}"] line`, + ).should('have.attr', 'stroke-dasharray', '9 4'); + + cy.get(`[data-testid="collectivex-run-visible-${comparisonDataset.run.run_id}"]`).check(); + cy.wait('@comparisonRun'); + cy.get('[data-testid="collectivex-explorer-chart"] .line-path') + .should('have.length', 2) + .then(($lines) => { + expect($lines.eq(0)).to.have.attr('stroke-dasharray', 'none'); + expect($lines.eq(1)).to.have.attr('stroke-dasharray', '9 4'); + expect($lines.eq(0)).to.have.attr('stroke', $lines.eq(1).attr('stroke')); + }); + cy.get('[data-testid="chart-legend"]') + .should('not.contain.text', `#${runId}`) + .and('not.contain.text', `#${comparisonDataset.run.run_id}`) + .find('[data-testid="legend-line-swatch"]') + .should('have.length', 2) + .then(($swatches) => { + expect($swatches.eq(0).find('line')).not.to.have.attr('stroke-dasharray'); + expect($swatches.eq(1).find('line')).to.have.attr('stroke-dasharray', '9 4'); + }); + + cy.get(`[data-testid="collectivex-run-visible-${runId}"]`).uncheck(); + cy.get('[data-testid="collectivex-run-conclusion"]').should( + 'contain.text', + `#${comparisonDataset.run.run_id}`, + ); + cy.get('[data-testid="collectivex-explorer-chart"] .line-path').should('have.length', 1); + }); + + it('defaults to the newest measured run when a newer incomplete run has no series', () => { + installRuns([incompleteDataset, dataset]); + installRun(); + cy.reload(); + cy.wait('@runs'); + cy.wait('@run'); + + cy.get(`[data-testid="collectivex-run-row-${incompleteDataset.run.run_id}"]`).should( + 'be.visible', + ); + cy.get(`[data-testid="collectivex-run-visible-${incompleteDataset.run.run_id}"]`).should( + 'not.be.checked', + ); + cy.get(`[data-testid="collectivex-run-visible-${runId}"]`).should('be.checked'); + cy.get('[data-testid="collectivex-run-conclusion"]').should('contain.text', `#${runId}`); + cy.get('[data-testid="collectivex-explorer-chart"] .line-path').should('have.length', 1); + }); + + it('keeps the chart on top and presents the matrix inventory', () => { + cy.get('[data-testid="collectivex-main-chart"]').should('be.visible'); + cy.get('[data-testid="collectivex-inventory"]') + .should('contain.text', 'Matrix case inventory') + .and('contain.text', `${dataset.coverage.length} cases`); + cy.get('[data-testid="collectivex-inventory-table"]') + .should('contain.text', 'H200-DGXC') + .and('contain.text', 'B300'); + }); +}); + +describe('CollectiveX run deletion', () => { + beforeEach(() => { + installRuns(); + installRun(); + openCollectiveX(); + }); + + it('deletes a table row after confirm + token prompt and remembers the token', () => { + let deleted = false; + cy.intercept('GET', '/api/v1/collectivex/runs?*', (request) => { + request.reply({ + body: { + version: 1, + runs: deleted ? [] : [buildRunSummary(dataset)], + }, + }); + }).as('runsAfterDelete'); + cy.intercept('DELETE', `/api/v1/collectivex/runs/${runId}`, (request) => { + expect(request.headers.authorization).to.eq('Bearer test-token'); + deleted = true; + request.reply({ deleted: true, runId }); + }).as('deleteRun'); + cy.window().then((win) => { + win.localStorage.removeItem(ADMIN_TOKEN_KEY); + cy.stub(win, 'confirm').returns(true); + cy.stub(win, 'prompt').returns('test-token'); + }); + + cy.get(`[data-testid="collectivex-delete-run-${runId}"]`).click(); + cy.wait('@deleteRun'); + cy.wait('@runsAfterDelete'); + cy.get(`[data-testid="collectivex-run-row-${runId}"]`).should('not.exist'); + cy.window().then((win) => { + expect(win.localStorage.getItem(ADMIN_TOKEN_KEY)).to.eq('test-token'); + }); + }); + + it('clears a stale stored token and reports unauthorized on 401', () => { + cy.intercept('DELETE', `/api/v1/collectivex/runs/${runId}`, { statusCode: 401 }).as( + 'delete401', + ); + cy.window().then((win) => { + win.localStorage.setItem(ADMIN_TOKEN_KEY, 'stale-token'); + cy.stub(win, 'confirm').returns(true); + cy.stub(win, 'alert').as('unauthorizedAlert'); + }); + + cy.get(`[data-testid="collectivex-delete-run-${runId}"]`).click(); + cy.wait('@delete401'); + cy.get('@unauthorizedAlert').should('have.been.calledWith', 'Invalid admin token.'); + cy.window().then((win) => { + expect(win.localStorage.getItem(ADMIN_TOKEN_KEY)).to.eq(null); + }); + }); + + it('does nothing when the confirmation is declined', () => { + let deleteRequests = 0; + cy.intercept('DELETE', `/api/v1/collectivex/runs/${runId}`, () => { + deleteRequests += 1; + }); + cy.window().then((win) => { + cy.stub(win, 'confirm').returns(false); + }); + + cy.get(`[data-testid="collectivex-delete-run-${runId}"]`).click(); + cy.get('[data-testid="collectivex-display"]').should('be.visible'); + cy.then(() => expect(deleteRequests).to.eq(0)); + }); +}); + +describe('CollectiveX availability states', () => { + it('reports a missing run list', () => { + cy.intercept('GET', '/api/v1/collectivex/runs?*', { + statusCode: 404, + body: { error: 'Not found' }, + }).as('missing'); + cy.visit('/collectivex'); + cy.wait('@missing'); + cy.get('[data-testid="collectivex-error"]') + .should('be.visible') + .and('contain.text', 'API error: 404'); + cy.get('[data-testid="collectivex-error-version-select"]').should('contain.text', 'V1'); + }); + + it('reports an unavailable backend', () => { + cy.intercept('GET', '/api/v1/collectivex/runs?*', { + statusCode: 503, + body: { error: 'unavailable' }, + }).as('down'); + cy.visit('/collectivex'); + cy.wait('@down'); + cy.get('[data-testid="collectivex-error"]') + .should('be.visible') + .and('contain.text', 'API error: 503'); + }); + + it('renders the loading state while the run resolves', () => { + installRuns(); + cy.intercept('GET', `/api/v1/collectivex/runs/${runId}*`, { + body: dataset, + delay: 500, + }).as('slowRun'); + cy.visit('/collectivex'); + cy.wait('@runs'); + cy.get('[data-testid="collectivex-selected-runs-loading"]').should('be.visible'); + cy.wait('@slowRun'); + cy.get('[data-testid="collectivex-display"]').should('be.visible'); + }); + + it('does not query database availability for the isolated page', () => { + let availabilityRequests = 0; + cy.intercept('GET', '/api/v1/availability', (request) => { + availabilityRequests += 1; + request.reply([]); + }); + installRuns(); + installRun(); + cy.visit('/collectivex'); + cy.wait('@runs'); + cy.wait('@run'); + cy.get('[data-testid="collectivex-display"]').should('be.visible'); + cy.then(() => expect(availabilityRequests).to.eq(0)); + }); +}); diff --git a/packages/app/cypress/fixtures/api/collectivex-latest.json b/packages/app/cypress/fixtures/api/collectivex-latest.json new file mode 100644 index 000000000..42f0c5e76 --- /dev/null +++ b/packages/app/cypress/fixtures/api/collectivex-latest.json @@ -0,0 +1 @@ +{"version":1,"run":{"run_id":"160","run_attempt":1,"generated_at":"2026-07-08T12:20:00Z","conclusion":"success","source_sha":"cccccccccccccccccccccccccccccccccccccccc","requested_cases":5,"terminal_cases":4,"measured_cases":3,"unsupported_cases":1,"failed_cases":0,"requested_points":50,"terminal_points":40,"measured_points":30,"covered_skus":["b200-dgxc","b300","h200-dgxc","mi355x"]},"coverage":[{"case_id":"h200-dgxc-deepep-v2-deepseek-v3-normal-decode-ep8-uniform-bf16","label":"h200-dgxc · deepep-v2 · normal · decode · EP8 · bf16","disposition":"runnable","sku":"h200-dgxc","backend":"deepep-v2","phase":"decode","mode":"normal","precision":"bf16","topology":{"ep_size":8,"nodes":1,"gpus_per_node":8,"scale_up_domain":8,"scale_up_transport":"nvlink","scale_out_transport":null,"topology_class":"h200-nvlink-island"},"points":[{"tokens_per_rank":1,"global_tokens":8,"terminal_status":"measured","reason":null},{"tokens_per_rank":2,"global_tokens":16,"terminal_status":"measured","reason":null},{"tokens_per_rank":4,"global_tokens":32,"terminal_status":"measured","reason":null},{"tokens_per_rank":8,"global_tokens":64,"terminal_status":"measured","reason":null},{"tokens_per_rank":16,"global_tokens":128,"terminal_status":"measured","reason":null},{"tokens_per_rank":32,"global_tokens":256,"terminal_status":"measured","reason":null},{"tokens_per_rank":64,"global_tokens":512,"terminal_status":"measured","reason":null},{"tokens_per_rank":128,"global_tokens":1024,"terminal_status":"measured","reason":null},{"tokens_per_rank":256,"global_tokens":2048,"terminal_status":"measured","reason":null},{"tokens_per_rank":512,"global_tokens":4096,"terminal_status":"measured","reason":null}],"outcome":"success","reason":null,"detail":null},{"case_id":"mi355x-mori-deepseek-v3-normal-decode-ep16-uniform-bf16","label":"mi355x · mori · normal · decode · EP16 · bf16","disposition":"runnable","sku":"mi355x","backend":"mori","phase":"decode","mode":"normal","precision":"bf16","topology":{"ep_size":16,"nodes":2,"gpus_per_node":8,"scale_up_domain":8,"scale_up_transport":"xgmi","scale_out_transport":"rdma","topology_class":"mi355x-xgmi-rdma"},"points":[{"tokens_per_rank":1,"global_tokens":16,"terminal_status":"measured","reason":null},{"tokens_per_rank":2,"global_tokens":32,"terminal_status":"measured","reason":null},{"tokens_per_rank":4,"global_tokens":64,"terminal_status":"measured","reason":null},{"tokens_per_rank":8,"global_tokens":128,"terminal_status":"measured","reason":null},{"tokens_per_rank":16,"global_tokens":256,"terminal_status":"measured","reason":null},{"tokens_per_rank":32,"global_tokens":512,"terminal_status":"measured","reason":null},{"tokens_per_rank":64,"global_tokens":1024,"terminal_status":"measured","reason":null},{"tokens_per_rank":128,"global_tokens":2048,"terminal_status":"measured","reason":null},{"tokens_per_rank":256,"global_tokens":4096,"terminal_status":"measured","reason":null},{"tokens_per_rank":512,"global_tokens":8192,"terminal_status":"measured","reason":null}],"outcome":"success","reason":null,"detail":null},{"case_id":"h200-dgxc-deepep-v2-deepseek-v3-normal-decode-ep8-uniform-fp8","label":"h200-dgxc · deepep-v2 · normal · decode · EP8 · fp8","disposition":"runnable","sku":"h200-dgxc","backend":"deepep-v2","phase":"decode","mode":"normal","precision":"fp8","topology":{"ep_size":8,"nodes":1,"gpus_per_node":8,"scale_up_domain":8,"scale_up_transport":"nvlink","scale_out_transport":null,"topology_class":"h200-nvlink-island"},"points":[{"tokens_per_rank":1,"global_tokens":8,"terminal_status":"measured","reason":null},{"tokens_per_rank":2,"global_tokens":16,"terminal_status":"measured","reason":null},{"tokens_per_rank":4,"global_tokens":32,"terminal_status":"measured","reason":null},{"tokens_per_rank":8,"global_tokens":64,"terminal_status":"measured","reason":null},{"tokens_per_rank":16,"global_tokens":128,"terminal_status":"measured","reason":null},{"tokens_per_rank":32,"global_tokens":256,"terminal_status":"measured","reason":null},{"tokens_per_rank":64,"global_tokens":512,"terminal_status":"measured","reason":null},{"tokens_per_rank":128,"global_tokens":1024,"terminal_status":"measured","reason":null},{"tokens_per_rank":256,"global_tokens":2048,"terminal_status":"measured","reason":null},{"tokens_per_rank":512,"global_tokens":4096,"terminal_status":"measured","reason":null}],"outcome":"success","reason":null,"detail":null},{"case_id":"b300-deepep-v2-deepseek-v3-normal-decode-ep16-uniform-bf16","label":"b300 · deepep-v2 · normal · decode · EP16 · bf16","disposition":"unsupported","sku":"b300","backend":"deepep-v2","phase":"decode","mode":"normal","precision":"bf16","topology":{"ep_size":16,"nodes":2,"gpus_per_node":8,"scale_up_domain":8,"scale_up_transport":"nvlink","scale_out_transport":"rdma","topology_class":"b300-nvlink-rdma"},"points":[{"tokens_per_rank":1,"global_tokens":16,"terminal_status":"unsupported","reason":"backend-platform-unsupported"},{"tokens_per_rank":2,"global_tokens":32,"terminal_status":"unsupported","reason":"backend-platform-unsupported"},{"tokens_per_rank":4,"global_tokens":64,"terminal_status":"unsupported","reason":"backend-platform-unsupported"},{"tokens_per_rank":8,"global_tokens":128,"terminal_status":"unsupported","reason":"backend-platform-unsupported"},{"tokens_per_rank":16,"global_tokens":256,"terminal_status":"unsupported","reason":"backend-platform-unsupported"},{"tokens_per_rank":32,"global_tokens":512,"terminal_status":"unsupported","reason":"backend-platform-unsupported"},{"tokens_per_rank":64,"global_tokens":1024,"terminal_status":"unsupported","reason":"backend-platform-unsupported"},{"tokens_per_rank":128,"global_tokens":2048,"terminal_status":"unsupported","reason":"backend-platform-unsupported"},{"tokens_per_rank":256,"global_tokens":4096,"terminal_status":"unsupported","reason":"backend-platform-unsupported"},{"tokens_per_rank":512,"global_tokens":8192,"terminal_status":"unsupported","reason":"backend-platform-unsupported"}],"outcome":"unsupported","reason":"backend-platform-unsupported","detail":"unsupported by the selected backend/platform"},{"case_id":"b200-dgxc-deepep-v2-deepseek-v3-normal-decode-ep8-uniform-bf16","label":"b200-dgxc · deepep-v2 · normal · decode · EP8 · bf16","disposition":"runnable","sku":"b200-dgxc","backend":"deepep-v2","phase":"decode","mode":"normal","precision":"bf16","topology":{"ep_size":8,"nodes":1,"gpus_per_node":8,"scale_up_domain":8,"scale_up_transport":"nvlink","scale_out_transport":null,"topology_class":"b200-nvlink-island"},"points":[{"tokens_per_rank":1,"global_tokens":8,"terminal_status":"pending","reason":"pending"},{"tokens_per_rank":2,"global_tokens":16,"terminal_status":"pending","reason":"pending"},{"tokens_per_rank":4,"global_tokens":32,"terminal_status":"pending","reason":"pending"},{"tokens_per_rank":8,"global_tokens":64,"terminal_status":"pending","reason":"pending"},{"tokens_per_rank":16,"global_tokens":128,"terminal_status":"pending","reason":"pending"},{"tokens_per_rank":32,"global_tokens":256,"terminal_status":"pending","reason":"pending"},{"tokens_per_rank":64,"global_tokens":512,"terminal_status":"pending","reason":"pending"},{"tokens_per_rank":128,"global_tokens":1024,"terminal_status":"pending","reason":"pending"},{"tokens_per_rank":256,"global_tokens":2048,"terminal_status":"pending","reason":"pending"},{"tokens_per_rank":512,"global_tokens":4096,"terminal_status":"pending","reason":"pending"}],"outcome":"pending","reason":"pending","detail":null}],"series":[{"series_id":"h200-dgxc-deepep-v2-deepseek-v3-normal-decode-ep8-uniform-bf16","phase":"decode","mode":"normal","precision":"bf16","backend":"deepep-v2","system":{"ep_size":8,"nodes":1,"gpus_per_node":8,"scale_up_domain":8,"scale_up_transport":"nvlink","scale_out_transport":null,"topology_class":"h200-nvlink-island","sku":"h200-dgxc","vendor":"nvidia"},"points":[{"tokens_per_rank":1,"global_tokens":8,"components":{"dispatch":{"latency_us":{"p50":417,"p90":450.36,"p95":467.04,"p99":500.4},"activation_data_rate_gbps_at_latency_percentile":{"p50":922.6952134292566,"p90":854.3474198419042,"p95":823.8350119904076,"p99":768.9126778577139},"payload_data_rate_gbps_at_latency_percentile":{"p50":119.90407673860912,"p90":111.02229327648992,"p95":107.05721137375814,"p99":99.92006394884093},"payload_bytes":400000000},"stage":{"latency_us":{"p50":120,"p90":129.60000000000002,"p95":134.4,"p99":144},"activation_data_rate_gbps_at_latency_percentile":{"p50":1603.1829333333335,"p90":1484.4286419753084,"p95":1431.4133333333332,"p99":1335.9857777777777},"payload_data_rate_gbps_at_latency_percentile":{"p50":200.3978666666667,"p90":185.55358024691355,"p95":178.92666666666665,"p99":166.9982222222222},"payload_bytes":192381952},"combine":{"latency_us":{"p50":392,"p90":423.36,"p95":439.04,"p99":470.4},"activation_data_rate_gbps_at_latency_percentile":{"p50":981.5405714285715,"p90":908.8338624338625,"p95":876.3755102040816,"p99":817.9504761904763},"payload_data_rate_gbps_at_latency_percentile":{"p50":122.69257142857144,"p90":113.60423280423281,"p95":109.5469387755102,"p99":102.24380952380953},"payload_bytes":384763904},"roundtrip":{"latency_us":{"p50":921,"p90":994.6800000000001,"p95":1031.5200000000002,"p99":1105.2},"activation_data_rate_gbps_at_latency_percentile":{"p50":835.5350792616721,"p90":773.6435919089556,"p95":746.0134636264928,"p99":696.27923271806},"payload_data_rate_gbps_at_latency_percentile":{"p50":106.50975895765473,"p90":98.62014718301363,"p95":95.09799906933456,"p99":88.75813246471228},"payload_bytes":784763904}},"roundtrip_token_rate_at_latency_percentile":{"p50":8338218,"p90":9005275.440000001,"p95":9338804.16,"p99":10005861.6}},{"tokens_per_rank":2,"global_tokens":16,"components":{"dispatch":{"latency_us":{"p50":418,"p90":451.44000000000005,"p95":468.16,"p99":501.59999999999997},"activation_data_rate_gbps_at_latency_percentile":{"p50":920.4878086124403,"p90":852.3035264930002,"p95":821.8641148325358,"p99":767.0731738437003},"payload_data_rate_gbps_at_latency_percentile":{"p50":119.61722488038276,"p90":110.75668970405812,"p95":106.8010936431989,"p99":99.68102073365232},"payload_bytes":400000000},"stage":{"latency_us":{"p50":121,"p90":130.68,"p95":135.52,"p99":145.2},"activation_data_rate_gbps_at_latency_percentile":{"p50":1589.9334876033058,"p90":1472.1606366697276,"p95":1419.58347107438,"p99":1324.944573002755},"payload_data_rate_gbps_at_latency_percentile":{"p50":198.74168595041323,"p90":184.02007958371595,"p95":177.4479338842975,"p99":165.61807162534438},"payload_bytes":192381952},"combine":{"latency_us":{"p50":393,"p90":424.44000000000005,"p95":440.16,"p99":471.59999999999997},"activation_data_rate_gbps_at_latency_percentile":{"p50":979.0430127226464,"p90":906.5213080765243,"p95":874.1455470737912,"p99":815.869177268872},"payload_data_rate_gbps_at_latency_percentile":{"p50":122.3803765903308,"p90":113.31516350956554,"p95":109.2681933842239,"p99":101.983647158609},"payload_bytes":384763904},"roundtrip":{"latency_us":{"p50":922,"p90":995.7600000000001,"p95":1032.64,"p99":1106.3999999999999},"activation_data_rate_gbps_at_latency_percentile":{"p50":834.6288590021693,"p90":772.8044990760825,"p95":745.2043383947938,"p99":695.5240491684744},"payload_data_rate_gbps_at_latency_percentile":{"p50":106.39423861171366,"p90":98.51318389973487,"p95":94.99485590331577,"p99":88.6618655097614},"payload_bytes":784763904}},"roundtrip_token_rate_at_latency_percentile":{"p50":8338218,"p90":9005275.440000001,"p95":9338804.16,"p99":10005861.6}},{"tokens_per_rank":4,"global_tokens":32,"components":{"dispatch":{"latency_us":{"p50":419,"p90":452.52000000000004,"p95":469.28000000000003,"p99":502.79999999999995},"activation_data_rate_gbps_at_latency_percentile":{"p50":918.2909403341289,"p90":850.2693891982674,"p95":819.9026252983293,"p99":765.2424502784409},"payload_data_rate_gbps_at_latency_percentile":{"p50":119.33174224343675,"p90":110.49235392910809,"p95":106.54619843163995,"p99":99.4431185361973},"payload_bytes":400000000},"stage":{"latency_us":{"p50":122,"p90":131.76000000000002,"p95":136.64000000000001,"p99":146.4},"activation_data_rate_gbps_at_latency_percentile":{"p50":1576.9012459016394,"p90":1460.0937462052214,"p95":1407.9475409836064,"p99":1314.0843715846995},"payload_data_rate_gbps_at_latency_percentile":{"p50":197.11265573770493,"p90":182.51171827565267,"p95":175.9934426229508,"p99":164.26054644808744},"payload_bytes":192381952},"combine":{"latency_us":{"p50":394,"p90":425.52000000000004,"p95":441.28000000000003,"p99":472.79999999999995},"activation_data_rate_gbps_at_latency_percentile":{"p50":976.5581319796954,"p90":904.2204925737921,"p95":871.9269035532996,"p99":813.798443316413},"payload_data_rate_gbps_at_latency_percentile":{"p50":122.06976649746193,"p90":113.02756157172401,"p95":108.99086294416244,"p99":101.72480541455162},"payload_bytes":384763904},"roundtrip":{"latency_us":{"p50":923,"p90":996.84,"p95":1033.76,"p99":1107.6},"activation_data_rate_gbps_at_latency_percentile":{"p50":833.724602383532,"p90":771.9672244291962,"p95":744.3969664138677,"p99":694.7705019862767},"payload_data_rate_gbps_at_latency_percentile":{"p50":106.27896858071506,"p90":98.40645238955098,"p95":94.8919362327813,"p99":88.5658071505959},"payload_bytes":784763904}},"roundtrip_token_rate_at_latency_percentile":{"p50":8338218,"p90":9005275.440000001,"p95":9338804.16,"p99":10005861.6}},{"tokens_per_rank":8,"global_tokens":64,"components":{"dispatch":{"latency_us":{"p50":420,"p90":453.6,"p95":470.40000000000003,"p99":504},"activation_data_rate_gbps_at_latency_percentile":{"p50":916.1045333333334,"p90":848.244938271605,"p95":817.9504761904761,"p99":763.4204444444445},"payload_data_rate_gbps_at_latency_percentile":{"p50":119.04761904761905,"p90":110.22927689594357,"p95":106.29251700680271,"p99":99.2063492063492},"payload_bytes":400000000},"stage":{"latency_us":{"p50":123,"p90":132.84,"p95":137.76000000000002,"p99":147.6},"activation_data_rate_gbps_at_latency_percentile":{"p50":1564.0809105691058,"p90":1448.2230653417646,"p95":1396.5008130081299,"p99":1303.400758807588},"payload_data_rate_gbps_at_latency_percentile":{"p50":195.51011382113822,"p90":181.02788316772057,"p95":174.56260162601623,"p99":162.9250948509485},"payload_bytes":192381952},"combine":{"latency_us":{"p50":395,"p90":426.6,"p95":442.40000000000003,"p99":474},"activation_data_rate_gbps_at_latency_percentile":{"p50":974.0858329113925,"p90":901.9313267698077,"p95":869.7194936708861,"p99":811.738194092827},"payload_data_rate_gbps_at_latency_percentile":{"p50":121.76072911392406,"p90":112.74141584622596,"p95":108.71493670886076,"p99":101.46727426160338},"payload_bytes":384763904},"roundtrip":{"latency_us":{"p50":924,"p90":997.9200000000001,"p95":1034.88,"p99":1108.8},"activation_data_rate_gbps_at_latency_percentile":{"p50":832.822303030303,"p90":771.1317620650954,"p95":743.5913419913419,"p99":694.0185858585859},"payload_data_rate_gbps_at_latency_percentile":{"p50":106.16394805194805,"p90":98.29995189995189,"p95":94.78923933209646,"p99":88.4699567099567},"payload_bytes":784763904}},"roundtrip_token_rate_at_latency_percentile":{"p50":8338218,"p90":9005275.440000001,"p95":9338804.16,"p99":10005861.6}},{"tokens_per_rank":16,"global_tokens":128,"components":{"dispatch":{"latency_us":{"p50":421,"p90":454.68,"p95":471.52000000000004,"p99":505.2},"activation_data_rate_gbps_at_latency_percentile":{"p50":913.928513064133,"p90":846.230104689012,"p95":816.0076009501188,"p99":761.607094220111},"payload_data_rate_gbps_at_latency_percentile":{"p50":118.76484560570071,"p90":109.96744963490808,"p95":106.04004071937563,"p99":98.97070467141727},"payload_bytes":400000000},"stage":{"latency_us":{"p50":124,"p90":133.92000000000002,"p95":138.88000000000002,"p99":148.79999999999998},"activation_data_rate_gbps_at_latency_percentile":{"p50":1551.4673548387095,"p90":1436.543847072879,"p95":1385.2387096774191,"p99":1292.8894623655915},"payload_data_rate_gbps_at_latency_percentile":{"p50":193.9334193548387,"p90":179.5679808841099,"p95":173.1548387096774,"p99":161.61118279569894},"payload_bytes":192381952},"combine":{"latency_us":{"p50":396,"p90":427.68,"p95":443.52000000000004,"p99":475.2},"activation_data_rate_gbps_at_latency_percentile":{"p50":971.6260202020202,"p90":899.653722409278,"p95":867.5232323232323,"p99":809.6883501683502},"payload_data_rate_gbps_at_latency_percentile":{"p50":121.45325252525252,"p90":112.45671530115975,"p95":108.44040404040403,"p99":101.21104377104378},"payload_bytes":384763904},"roundtrip":{"latency_us":{"p50":925,"p90":999.0000000000001,"p95":1036,"p99":1110},"activation_data_rate_gbps_at_latency_percentile":{"p50":831.9219545945946,"p90":770.298106106106,"p95":742.7874594594595,"p99":693.2682954954955},"payload_data_rate_gbps_at_latency_percentile":{"p50":106.04917621621622,"p90":98.19368168168167,"p95":94.68676447876449,"p99":88.37431351351351},"payload_bytes":784763904}},"roundtrip_token_rate_at_latency_percentile":{"p50":8338218,"p90":9005275.440000001,"p95":9338804.16,"p99":10005861.6}},{"tokens_per_rank":32,"global_tokens":256,"components":{"dispatch":{"latency_us":{"p50":422,"p90":455.76000000000005,"p95":472.64000000000004,"p99":506.4},"activation_data_rate_gbps_at_latency_percentile":{"p50":911.7628056872038,"p90":844.2248200807443,"p95":814.0739336492891,"p99":759.8023380726698},"payload_data_rate_gbps_at_latency_percentile":{"p50":118.48341232227487,"p90":109.70686326136563,"p95":105.78876100203114,"p99":98.73617693522907},"payload_bytes":400000000},"stage":{"latency_us":{"p50":125,"p90":135,"p95":140,"p99":150},"activation_data_rate_gbps_at_latency_percentile":{"p50":1539.0556159999999,"p90":1425.0514962962964,"p95":1374.1568,"p99":1282.5463466666668},"payload_data_rate_gbps_at_latency_percentile":{"p50":192.38195199999998,"p90":178.13143703703705,"p95":171.7696,"p99":160.31829333333334},"payload_bytes":192381952},"combine":{"latency_us":{"p50":397,"p90":428.76000000000005,"p95":444.64000000000004,"p99":476.4},"activation_data_rate_gbps_at_latency_percentile":{"p50":969.1785994962216,"p90":897.387592126131,"p95":865.3380352644836,"p99":807.648832913518},"payload_data_rate_gbps_at_latency_percentile":{"p50":121.1473249370277,"p90":112.17344901576638,"p95":108.16725440806044,"p99":100.95610411418976},"payload_bytes":384763904},"roundtrip":{"latency_us":{"p50":926,"p90":1000.08,"p95":1037.1200000000001,"p99":1111.2},"activation_data_rate_gbps_at_latency_percentile":{"p50":831.0235507559396,"p90":769.466250699944,"p95":741.9853131749459,"p99":692.5196256299496},"payload_data_rate_gbps_at_latency_percentile":{"p50":105.93465226781858,"p90":98.0876409887209,"p95":94.58451095340943,"p99":88.2788768898488},"payload_bytes":784763904}},"roundtrip_token_rate_at_latency_percentile":{"p50":8338218,"p90":9005275.440000001,"p95":9338804.16,"p99":10005861.6}},{"tokens_per_rank":64,"global_tokens":512,"components":{"dispatch":{"latency_us":{"p50":423,"p90":456.84000000000003,"p95":473.76000000000005,"p99":507.59999999999997},"activation_data_rate_gbps_at_latency_percentile":{"p50":909.6073380614658,"p90":842.2290167235793,"p95":812.1494089834514,"p99":758.0061150512215},"payload_data_rate_gbps_at_latency_percentile":{"p50":118.2033096926714,"p90":109.44750897469572,"p95":105.5386693684566,"p99":98.50275807722618},"payload_bytes":400000000},"stage":{"latency_us":{"p50":126,"p90":136.08,"p95":141.12,"p99":151.2},"activation_data_rate_gbps_at_latency_percentile":{"p50":1526.840888888889,"p90":1413.741563786008,"p95":1363.2507936507936,"p99":1272.3674074074074},"payload_data_rate_gbps_at_latency_percentile":{"p50":190.8551111111111,"p90":176.717695473251,"p95":170.4063492063492,"p99":159.04592592592593},"payload_bytes":192381952},"combine":{"latency_us":{"p50":398,"p90":429.84000000000003,"p95":445.76000000000005,"p99":477.59999999999997},"activation_data_rate_gbps_at_latency_percentile":{"p50":966.7434773869347,"p90":895.1328494323469,"p95":863.1638190954774,"p99":805.6195644891122},"payload_data_rate_gbps_at_latency_percentile":{"p50":120.84293467336684,"p90":111.89160617904336,"p95":107.89547738693467,"p99":100.70244556113903},"payload_bytes":384763904},"roundtrip":{"latency_us":{"p50":927,"p90":1001.1600000000001,"p95":1038.24,"p99":1112.3999999999999},"activation_data_rate_gbps_at_latency_percentile":{"p50":830.1270852211435,"p90":768.6361900195773,"p95":741.1848975188781,"p99":691.7725710176197},"payload_data_rate_gbps_at_latency_percentile":{"p50":105.82037540453075,"p90":97.98182907826921,"p95":94.48247803975958,"p99":88.1836461704423},"payload_bytes":784763904}},"roundtrip_token_rate_at_latency_percentile":{"p50":8338218,"p90":9005275.440000001,"p95":9338804.16,"p99":10005861.6}},{"tokens_per_rank":128,"global_tokens":1024,"components":{"dispatch":{"latency_us":{"p50":424,"p90":457.92,"p95":474.88000000000005,"p99":508.79999999999995},"activation_data_rate_gbps_at_latency_percentile":{"p50":907.4620377358491,"p90":840.2426275331936,"p95":810.2339622641508,"p99":756.2183647798744},"payload_data_rate_gbps_at_latency_percentile":{"p50":117.9245283018868,"p90":109.18937805730259,"p95":105.2897574123989,"p99":98.27044025157234},"payload_bytes":400000000},"stage":{"latency_us":{"p50":127,"p90":137.16,"p95":142.24,"p99":152.4},"activation_data_rate_gbps_at_latency_percentile":{"p50":1514.8185196850393,"p90":1402.6097404491106,"p95":1352.5165354330707,"p99":1262.3487664041995},"payload_data_rate_gbps_at_latency_percentile":{"p50":189.3523149606299,"p90":175.32621755613883,"p95":169.06456692913383,"p99":157.79359580052494},"payload_bytes":192381952},"combine":{"latency_us":{"p50":399,"p90":430.92,"p95":446.88000000000005,"p99":478.79999999999995},"activation_data_rate_gbps_at_latency_percentile":{"p50":964.3205614035088,"p90":892.8894087069526,"p95":861.0005012531327,"p99":803.6004678362574},"payload_data_rate_gbps_at_latency_percentile":{"p50":120.5400701754386,"p90":111.61117608836908,"p95":107.62506265664159,"p99":100.45005847953217},"payload_bytes":384763904},"roundtrip":{"latency_us":{"p50":928,"p90":1002.24,"p95":1039.3600000000001,"p99":1113.6},"activation_data_rate_gbps_at_latency_percentile":{"p50":829.232551724138,"p90":767.8079182630906,"p95":740.3862068965517,"p99":691.0271264367817},"payload_data_rate_gbps_at_latency_percentile":{"p50":105.70634482758621,"p90":97.87624521072797,"p95":94.38066502463055,"p99":88.08862068965517},"payload_bytes":784763904}},"roundtrip_token_rate_at_latency_percentile":{"p50":8338218,"p90":9005275.440000001,"p95":9338804.16,"p99":10005861.6}},{"tokens_per_rank":256,"global_tokens":2048,"components":{"dispatch":{"latency_us":{"p50":425,"p90":459.00000000000006,"p95":476.00000000000006,"p99":510},"activation_data_rate_gbps_at_latency_percentile":{"p50":905.3268329411765,"p90":838.2655860566448,"p95":808.3275294117645,"p99":754.4390274509803},"payload_data_rate_gbps_at_latency_percentile":{"p50":117.64705882352942,"p90":108.93246187363833,"p95":105.04201680672269,"p99":98.0392156862745},"payload_bytes":400000000},"stage":{"latency_us":{"p50":128,"p90":138.24,"p95":143.36,"p99":153.6},"activation_data_rate_gbps_at_latency_percentile":{"p50":1502.984,"p90":1391.6518518518517,"p95":1341.9499999999998,"p99":1252.4866666666667},"payload_data_rate_gbps_at_latency_percentile":{"p50":187.873,"p90":173.95648148148146,"p95":167.74374999999998,"p99":156.56083333333333},"payload_bytes":192381952},"combine":{"latency_us":{"p50":400,"p90":432,"p95":448.00000000000006,"p99":480},"activation_data_rate_gbps_at_latency_percentile":{"p50":961.90976,"p90":890.6571851851852,"p95":858.848,"p99":801.5914666666667},"payload_data_rate_gbps_at_latency_percentile":{"p50":120.23872,"p90":111.33214814814815,"p95":107.356,"p99":100.19893333333334},"payload_bytes":384763904},"roundtrip":{"latency_us":{"p50":929,"p90":1003.32,"p95":1040.48,"p99":1114.8},"activation_data_rate_gbps_at_latency_percentile":{"p50":828.3399440258343,"p90":766.9814296535502,"p95":739.5892357373519,"p99":690.2832866881952},"payload_data_rate_gbps_at_latency_percentile":{"p50":105.5925597416577,"p90":97.77088864968306,"p95":94.27907119790866,"p99":87.99379978471475},"payload_bytes":784763904}},"roundtrip_token_rate_at_latency_percentile":{"p50":8338218,"p90":9005275.440000001,"p95":9338804.16,"p99":10005861.6}},{"tokens_per_rank":512,"global_tokens":4096,"components":{"dispatch":{"latency_us":{"p50":426,"p90":460.08000000000004,"p95":477.12000000000006,"p99":511.2},"activation_data_rate_gbps_at_latency_percentile":{"p50":903.2016525821597,"p90":836.2978264649626,"p95":806.4300469483567,"p99":752.6680438184663},"payload_data_rate_gbps_at_latency_percentile":{"p50":117.37089201877934,"p90":108.67675186924012,"p95":104.79543930248154,"p99":97.80907668231613},"payload_bytes":400000000},"stage":{"latency_us":{"p50":129,"p90":139.32000000000002,"p95":144.48000000000002,"p99":154.79999999999998},"activation_data_rate_gbps_at_latency_percentile":{"p50":1491.33296124031,"p90":1380.863853000287,"p95":1331.5472868217053,"p99":1242.7774677002585},"payload_data_rate_gbps_at_latency_percentile":{"p50":186.41662015503874,"p90":172.60798162503588,"p95":166.44341085271316,"p99":155.3471834625323},"payload_bytes":192381952},"combine":{"latency_us":{"p50":401,"p90":433.08000000000004,"p95":449.12000000000006,"p99":481.2},"activation_data_rate_gbps_at_latency_percentile":{"p50":959.5109825436409,"p90":888.4360949478155,"p95":856.706234413965,"p99":799.5924854530341},"payload_data_rate_gbps_at_latency_percentile":{"p50":119.93887281795512,"p90":111.05451186847694,"p95":107.08827930174563,"p99":99.94906068162926},"payload_bytes":384763904},"roundtrip":{"latency_us":{"p50":930,"p90":1004.4000000000001,"p95":1041.6000000000001,"p99":1116},"activation_data_rate_gbps_at_latency_percentile":{"p50":827.4492559139785,"p90":766.1567184388689,"p95":738.7939784946236,"p99":689.541046594982},"payload_data_rate_gbps_at_latency_percentile":{"p50":105.47901935483871,"p90":97.66575866188768,"p95":94.17769585253455,"p99":87.89918279569893},"payload_bytes":784763904}},"roundtrip_token_rate_at_latency_percentile":{"p50":8338218,"p90":9005275.440000001,"p95":9338804.16,"p99":10005861.6}}]},{"series_id":"mi355x-mori-deepseek-v3-normal-decode-ep16-uniform-bf16","phase":"decode","mode":"normal","precision":"bf16","backend":"mori","system":{"ep_size":16,"nodes":2,"gpus_per_node":8,"scale_up_domain":8,"scale_up_transport":"xgmi","scale_out_transport":"rdma","topology_class":"mi355x-xgmi-rdma","sku":"mi355x","vendor":"amd"},"points":[{"tokens_per_rank":1,"global_tokens":16,"components":{"dispatch":{"latency_us":{"p50":417,"p90":450.36,"p95":467.04,"p99":500.4},"activation_data_rate_gbps_at_latency_percentile":{"p50":922.6952134292566,"p90":854.3474198419042,"p95":823.8350119904076,"p99":768.9126778577139},"payload_data_rate_gbps_at_latency_percentile":{"p50":59.95203836930456,"p90":55.51114663824496,"p95":53.52860568687907,"p99":49.96003197442047},"payload_bytes":400000000},"stage":{"latency_us":{"p50":120,"p90":129.60000000000002,"p95":134.4,"p99":144},"activation_data_rate_gbps_at_latency_percentile":{"p50":1603.1829333333335,"p90":1484.4286419753084,"p95":1431.4133333333332,"p99":1335.9857777777777},"payload_data_rate_gbps_at_latency_percentile":{"p50":100.19893333333334,"p90":92.77679012345678,"p95":89.46333333333332,"p99":83.4991111111111},"payload_bytes":192381952},"combine":{"latency_us":{"p50":392,"p90":423.36,"p95":439.04,"p99":470.4},"activation_data_rate_gbps_at_latency_percentile":{"p50":981.5405714285715,"p90":908.8338624338625,"p95":876.3755102040816,"p99":817.9504761904763},"payload_data_rate_gbps_at_latency_percentile":{"p50":61.34628571428572,"p90":56.802116402116404,"p95":54.7734693877551,"p99":51.121904761904766},"payload_bytes":384763904},"roundtrip":{"latency_us":{"p50":921,"p90":994.6800000000001,"p95":1031.5200000000002,"p99":1105.2},"activation_data_rate_gbps_at_latency_percentile":{"p50":835.5350792616721,"p90":773.6435919089556,"p95":746.0134636264928,"p99":696.27923271806},"payload_data_rate_gbps_at_latency_percentile":{"p50":53.25487947882736,"p90":49.310073591506814,"p95":47.54899953466728,"p99":44.37906623235614},"payload_bytes":784763904}},"roundtrip_token_rate_at_latency_percentile":{"p50":8338218,"p90":9005275.440000001,"p95":9338804.16,"p99":10005861.6}},{"tokens_per_rank":2,"global_tokens":32,"components":{"dispatch":{"latency_us":{"p50":418,"p90":451.44000000000005,"p95":468.16,"p99":501.59999999999997},"activation_data_rate_gbps_at_latency_percentile":{"p50":920.4878086124403,"p90":852.3035264930002,"p95":821.8641148325358,"p99":767.0731738437003},"payload_data_rate_gbps_at_latency_percentile":{"p50":59.80861244019138,"p90":55.37834485202906,"p95":53.40054682159945,"p99":49.84051036682616},"payload_bytes":400000000},"stage":{"latency_us":{"p50":121,"p90":130.68,"p95":135.52,"p99":145.2},"activation_data_rate_gbps_at_latency_percentile":{"p50":1589.9334876033058,"p90":1472.1606366697276,"p95":1419.58347107438,"p99":1324.944573002755},"payload_data_rate_gbps_at_latency_percentile":{"p50":99.37084297520661,"p90":92.01003979185798,"p95":88.72396694214875,"p99":82.80903581267219},"payload_bytes":192381952},"combine":{"latency_us":{"p50":393,"p90":424.44000000000005,"p95":440.16,"p99":471.59999999999997},"activation_data_rate_gbps_at_latency_percentile":{"p50":979.0430127226464,"p90":906.5213080765243,"p95":874.1455470737912,"p99":815.869177268872},"payload_data_rate_gbps_at_latency_percentile":{"p50":61.1901882951654,"p90":56.65758175478277,"p95":54.63409669211195,"p99":50.9918235793045},"payload_bytes":384763904},"roundtrip":{"latency_us":{"p50":922,"p90":995.7600000000001,"p95":1032.64,"p99":1106.3999999999999},"activation_data_rate_gbps_at_latency_percentile":{"p50":834.6288590021693,"p90":772.8044990760825,"p95":745.2043383947938,"p99":695.5240491684744},"payload_data_rate_gbps_at_latency_percentile":{"p50":53.19711930585683,"p90":49.256591949867435,"p95":47.49742795165788,"p99":44.3309327548807},"payload_bytes":784763904}},"roundtrip_token_rate_at_latency_percentile":{"p50":8338218,"p90":9005275.440000001,"p95":9338804.16,"p99":10005861.6}},{"tokens_per_rank":4,"global_tokens":64,"components":{"dispatch":{"latency_us":{"p50":419,"p90":452.52000000000004,"p95":469.28000000000003,"p99":502.79999999999995},"activation_data_rate_gbps_at_latency_percentile":{"p50":918.2909403341289,"p90":850.2693891982674,"p95":819.9026252983293,"p99":765.2424502784409},"payload_data_rate_gbps_at_latency_percentile":{"p50":59.665871121718375,"p90":55.246176964554046,"p95":53.273099215819975,"p99":49.72155926809865},"payload_bytes":400000000},"stage":{"latency_us":{"p50":122,"p90":131.76000000000002,"p95":136.64000000000001,"p99":146.4},"activation_data_rate_gbps_at_latency_percentile":{"p50":1576.9012459016394,"p90":1460.0937462052214,"p95":1407.9475409836064,"p99":1314.0843715846995},"payload_data_rate_gbps_at_latency_percentile":{"p50":98.55632786885246,"p90":91.25585913782633,"p95":87.9967213114754,"p99":82.13027322404372},"payload_bytes":192381952},"combine":{"latency_us":{"p50":394,"p90":425.52000000000004,"p95":441.28000000000003,"p99":472.79999999999995},"activation_data_rate_gbps_at_latency_percentile":{"p50":976.5581319796954,"p90":904.2204925737921,"p95":871.9269035532996,"p99":813.798443316413},"payload_data_rate_gbps_at_latency_percentile":{"p50":61.034883248730964,"p90":56.513780785862004,"p95":54.49543147208122,"p99":50.86240270727581},"payload_bytes":384763904},"roundtrip":{"latency_us":{"p50":923,"p90":996.84,"p95":1033.76,"p99":1107.6},"activation_data_rate_gbps_at_latency_percentile":{"p50":833.724602383532,"p90":771.9672244291962,"p95":744.3969664138677,"p99":694.7705019862767},"payload_data_rate_gbps_at_latency_percentile":{"p50":53.13948429035753,"p90":49.20322619477549,"p95":47.44596811639065,"p99":44.28290357529795},"payload_bytes":784763904}},"roundtrip_token_rate_at_latency_percentile":{"p50":8338218,"p90":9005275.440000001,"p95":9338804.16,"p99":10005861.6}},{"tokens_per_rank":8,"global_tokens":128,"components":{"dispatch":{"latency_us":{"p50":420,"p90":453.6,"p95":470.40000000000003,"p99":504},"activation_data_rate_gbps_at_latency_percentile":{"p50":916.1045333333334,"p90":848.244938271605,"p95":817.9504761904761,"p99":763.4204444444445},"payload_data_rate_gbps_at_latency_percentile":{"p50":59.523809523809526,"p90":55.114638447971785,"p95":53.146258503401356,"p99":49.6031746031746},"payload_bytes":400000000},"stage":{"latency_us":{"p50":123,"p90":132.84,"p95":137.76000000000002,"p99":147.6},"activation_data_rate_gbps_at_latency_percentile":{"p50":1564.0809105691058,"p90":1448.2230653417646,"p95":1396.5008130081299,"p99":1303.400758807588},"payload_data_rate_gbps_at_latency_percentile":{"p50":97.75505691056911,"p90":90.51394158386029,"p95":87.28130081300812,"p99":81.46254742547426},"payload_bytes":192381952},"combine":{"latency_us":{"p50":395,"p90":426.6,"p95":442.40000000000003,"p99":474},"activation_data_rate_gbps_at_latency_percentile":{"p50":974.0858329113925,"p90":901.9313267698077,"p95":869.7194936708861,"p99":811.738194092827},"payload_data_rate_gbps_at_latency_percentile":{"p50":60.88036455696203,"p90":56.37070792311298,"p95":54.35746835443038,"p99":50.73363713080169},"payload_bytes":384763904},"roundtrip":{"latency_us":{"p50":924,"p90":997.9200000000001,"p95":1034.88,"p99":1108.8},"activation_data_rate_gbps_at_latency_percentile":{"p50":832.822303030303,"p90":771.1317620650954,"p95":743.5913419913419,"p99":694.0185858585859},"payload_data_rate_gbps_at_latency_percentile":{"p50":53.08197402597403,"p90":49.149975949975946,"p95":47.39461966604823,"p99":44.23497835497835},"payload_bytes":784763904}},"roundtrip_token_rate_at_latency_percentile":{"p50":8338218,"p90":9005275.440000001,"p95":9338804.16,"p99":10005861.6}},{"tokens_per_rank":16,"global_tokens":256,"components":{"dispatch":{"latency_us":{"p50":421,"p90":454.68,"p95":471.52000000000004,"p99":505.2},"activation_data_rate_gbps_at_latency_percentile":{"p50":913.928513064133,"p90":846.230104689012,"p95":816.0076009501188,"p99":761.607094220111},"payload_data_rate_gbps_at_latency_percentile":{"p50":59.38242280285036,"p90":54.98372481745404,"p95":53.02002035968781,"p99":49.48535233570863},"payload_bytes":400000000},"stage":{"latency_us":{"p50":124,"p90":133.92000000000002,"p95":138.88000000000002,"p99":148.79999999999998},"activation_data_rate_gbps_at_latency_percentile":{"p50":1551.4673548387095,"p90":1436.543847072879,"p95":1385.2387096774191,"p99":1292.8894623655915},"payload_data_rate_gbps_at_latency_percentile":{"p50":96.96670967741935,"p90":89.78399044205494,"p95":86.5774193548387,"p99":80.80559139784947},"payload_bytes":192381952},"combine":{"latency_us":{"p50":396,"p90":427.68,"p95":443.52000000000004,"p99":475.2},"activation_data_rate_gbps_at_latency_percentile":{"p50":971.6260202020202,"p90":899.653722409278,"p95":867.5232323232323,"p99":809.6883501683502},"payload_data_rate_gbps_at_latency_percentile":{"p50":60.72662626262626,"p90":56.22835765057987,"p95":54.22020202020202,"p99":50.60552188552189},"payload_bytes":384763904},"roundtrip":{"latency_us":{"p50":925,"p90":999.0000000000001,"p95":1036,"p99":1110},"activation_data_rate_gbps_at_latency_percentile":{"p50":831.9219545945946,"p90":770.298106106106,"p95":742.7874594594595,"p99":693.2682954954955},"payload_data_rate_gbps_at_latency_percentile":{"p50":53.02458810810811,"p90":49.096840840840834,"p95":47.343382239382244,"p99":44.18715675675676},"payload_bytes":784763904}},"roundtrip_token_rate_at_latency_percentile":{"p50":8338218,"p90":9005275.440000001,"p95":9338804.16,"p99":10005861.6}},{"tokens_per_rank":32,"global_tokens":512,"components":{"dispatch":{"latency_us":{"p50":422,"p90":455.76000000000005,"p95":472.64000000000004,"p99":506.4},"activation_data_rate_gbps_at_latency_percentile":{"p50":911.7628056872038,"p90":844.2248200807443,"p95":814.0739336492891,"p99":759.8023380726698},"payload_data_rate_gbps_at_latency_percentile":{"p50":59.24170616113744,"p90":54.85343163068281,"p95":52.89438050101557,"p99":49.368088467614534},"payload_bytes":400000000},"stage":{"latency_us":{"p50":125,"p90":135,"p95":140,"p99":150},"activation_data_rate_gbps_at_latency_percentile":{"p50":1539.0556159999999,"p90":1425.0514962962964,"p95":1374.1568,"p99":1282.5463466666668},"payload_data_rate_gbps_at_latency_percentile":{"p50":96.19097599999999,"p90":89.06571851851852,"p95":85.8848,"p99":80.15914666666667},"payload_bytes":192381952},"combine":{"latency_us":{"p50":397,"p90":428.76000000000005,"p95":444.64000000000004,"p99":476.4},"activation_data_rate_gbps_at_latency_percentile":{"p50":969.1785994962216,"p90":897.387592126131,"p95":865.3380352644836,"p99":807.648832913518},"payload_data_rate_gbps_at_latency_percentile":{"p50":60.57366246851385,"p90":56.08672450788319,"p95":54.08362720403022,"p99":50.47805205709488},"payload_bytes":384763904},"roundtrip":{"latency_us":{"p50":926,"p90":1000.08,"p95":1037.1200000000001,"p99":1111.2},"activation_data_rate_gbps_at_latency_percentile":{"p50":831.0235507559396,"p90":769.466250699944,"p95":741.9853131749459,"p99":692.5196256299496},"payload_data_rate_gbps_at_latency_percentile":{"p50":52.96732613390929,"p90":49.04382049436045,"p95":47.29225547670472,"p99":44.1394384449244},"payload_bytes":784763904}},"roundtrip_token_rate_at_latency_percentile":{"p50":8338218,"p90":9005275.440000001,"p95":9338804.16,"p99":10005861.6}},{"tokens_per_rank":64,"global_tokens":1024,"components":{"dispatch":{"latency_us":{"p50":423,"p90":456.84000000000003,"p95":473.76000000000005,"p99":507.59999999999997},"activation_data_rate_gbps_at_latency_percentile":{"p50":909.6073380614658,"p90":842.2290167235793,"p95":812.1494089834514,"p99":758.0061150512215},"payload_data_rate_gbps_at_latency_percentile":{"p50":59.1016548463357,"p90":54.72375448734786,"p95":52.7693346842283,"p99":49.25137903861309},"payload_bytes":400000000},"stage":{"latency_us":{"p50":126,"p90":136.08,"p95":141.12,"p99":151.2},"activation_data_rate_gbps_at_latency_percentile":{"p50":1526.840888888889,"p90":1413.741563786008,"p95":1363.2507936507936,"p99":1272.3674074074074},"payload_data_rate_gbps_at_latency_percentile":{"p50":95.42755555555556,"p90":88.3588477366255,"p95":85.2031746031746,"p99":79.52296296296296},"payload_bytes":192381952},"combine":{"latency_us":{"p50":398,"p90":429.84000000000003,"p95":445.76000000000005,"p99":477.59999999999997},"activation_data_rate_gbps_at_latency_percentile":{"p50":966.7434773869347,"p90":895.1328494323469,"p95":863.1638190954774,"p99":805.6195644891122},"payload_data_rate_gbps_at_latency_percentile":{"p50":60.42146733668342,"p90":55.94580308952168,"p95":53.947738693467336,"p99":50.351222780569515},"payload_bytes":384763904},"roundtrip":{"latency_us":{"p50":927,"p90":1001.1600000000001,"p95":1038.24,"p99":1112.3999999999999},"activation_data_rate_gbps_at_latency_percentile":{"p50":830.1270852211435,"p90":768.6361900195773,"p95":741.1848975188781,"p99":691.7725710176197},"payload_data_rate_gbps_at_latency_percentile":{"p50":52.910187702265375,"p90":48.99091453913461,"p95":47.24123901987979,"p99":44.09182308522115},"payload_bytes":784763904}},"roundtrip_token_rate_at_latency_percentile":{"p50":8338218,"p90":9005275.440000001,"p95":9338804.16,"p99":10005861.6}},{"tokens_per_rank":128,"global_tokens":2048,"components":{"dispatch":{"latency_us":{"p50":424,"p90":457.92,"p95":474.88000000000005,"p99":508.79999999999995},"activation_data_rate_gbps_at_latency_percentile":{"p50":907.4620377358491,"p90":840.2426275331936,"p95":810.2339622641508,"p99":756.2183647798744},"payload_data_rate_gbps_at_latency_percentile":{"p50":58.9622641509434,"p90":54.594689028651295,"p95":52.64487870619945,"p99":49.13522012578617},"payload_bytes":400000000},"stage":{"latency_us":{"p50":127,"p90":137.16,"p95":142.24,"p99":152.4},"activation_data_rate_gbps_at_latency_percentile":{"p50":1514.8185196850393,"p90":1402.6097404491106,"p95":1352.5165354330707,"p99":1262.3487664041995},"payload_data_rate_gbps_at_latency_percentile":{"p50":94.67615748031496,"p90":87.66310877806941,"p95":84.53228346456692,"p99":78.89679790026247},"payload_bytes":192381952},"combine":{"latency_us":{"p50":399,"p90":430.92,"p95":446.88000000000005,"p99":478.79999999999995},"activation_data_rate_gbps_at_latency_percentile":{"p50":964.3205614035088,"p90":892.8894087069526,"p95":861.0005012531327,"p99":803.6004678362574},"payload_data_rate_gbps_at_latency_percentile":{"p50":60.2700350877193,"p90":55.80558804418454,"p95":53.812531328320794,"p99":50.22502923976609},"payload_bytes":384763904},"roundtrip":{"latency_us":{"p50":928,"p90":1002.24,"p95":1039.3600000000001,"p99":1113.6},"activation_data_rate_gbps_at_latency_percentile":{"p50":829.232551724138,"p90":767.8079182630906,"p95":740.3862068965517,"p99":691.0271264367817},"payload_data_rate_gbps_at_latency_percentile":{"p50":52.853172413793104,"p90":48.938122605363986,"p95":47.19033251231527,"p99":44.044310344827586},"payload_bytes":784763904}},"roundtrip_token_rate_at_latency_percentile":{"p50":8338218,"p90":9005275.440000001,"p95":9338804.16,"p99":10005861.6}},{"tokens_per_rank":256,"global_tokens":4096,"components":{"dispatch":{"latency_us":{"p50":425,"p90":459.00000000000006,"p95":476.00000000000006,"p99":510},"activation_data_rate_gbps_at_latency_percentile":{"p50":905.3268329411765,"p90":838.2655860566448,"p95":808.3275294117645,"p99":754.4390274509803},"payload_data_rate_gbps_at_latency_percentile":{"p50":58.82352941176471,"p90":54.466230936819166,"p95":52.52100840336134,"p99":49.01960784313725},"payload_bytes":400000000},"stage":{"latency_us":{"p50":128,"p90":138.24,"p95":143.36,"p99":153.6},"activation_data_rate_gbps_at_latency_percentile":{"p50":1502.984,"p90":1391.6518518518517,"p95":1341.9499999999998,"p99":1252.4866666666667},"payload_data_rate_gbps_at_latency_percentile":{"p50":93.9365,"p90":86.97824074074073,"p95":83.87187499999999,"p99":78.28041666666667},"payload_bytes":192381952},"combine":{"latency_us":{"p50":400,"p90":432,"p95":448.00000000000006,"p99":480},"activation_data_rate_gbps_at_latency_percentile":{"p50":961.90976,"p90":890.6571851851852,"p95":858.848,"p99":801.5914666666667},"payload_data_rate_gbps_at_latency_percentile":{"p50":60.11936,"p90":55.666074074074075,"p95":53.678,"p99":50.09946666666667},"payload_bytes":384763904},"roundtrip":{"latency_us":{"p50":929,"p90":1003.32,"p95":1040.48,"p99":1114.8},"activation_data_rate_gbps_at_latency_percentile":{"p50":828.3399440258343,"p90":766.9814296535502,"p95":739.5892357373519,"p99":690.2832866881952},"payload_data_rate_gbps_at_latency_percentile":{"p50":52.79627987082885,"p90":48.88544432484153,"p95":47.13953559895433,"p99":43.996899892357376},"payload_bytes":784763904}},"roundtrip_token_rate_at_latency_percentile":{"p50":8338218,"p90":9005275.440000001,"p95":9338804.16,"p99":10005861.6}},{"tokens_per_rank":512,"global_tokens":8192,"components":{"dispatch":{"latency_us":{"p50":426,"p90":460.08000000000004,"p95":477.12000000000006,"p99":511.2},"activation_data_rate_gbps_at_latency_percentile":{"p50":903.2016525821597,"p90":836.2978264649626,"p95":806.4300469483567,"p99":752.6680438184663},"payload_data_rate_gbps_at_latency_percentile":{"p50":58.68544600938967,"p90":54.33837593462006,"p95":52.39771965124077,"p99":48.904538341158066},"payload_bytes":400000000},"stage":{"latency_us":{"p50":129,"p90":139.32000000000002,"p95":144.48000000000002,"p99":154.79999999999998},"activation_data_rate_gbps_at_latency_percentile":{"p50":1491.33296124031,"p90":1380.863853000287,"p95":1331.5472868217053,"p99":1242.7774677002585},"payload_data_rate_gbps_at_latency_percentile":{"p50":93.20831007751937,"p90":86.30399081251794,"p95":83.22170542635658,"p99":77.67359173126616},"payload_bytes":192381952},"combine":{"latency_us":{"p50":401,"p90":433.08000000000004,"p95":449.12000000000006,"p99":481.2},"activation_data_rate_gbps_at_latency_percentile":{"p50":959.5109825436409,"p90":888.4360949478155,"p95":856.706234413965,"p99":799.5924854530341},"payload_data_rate_gbps_at_latency_percentile":{"p50":59.96943640897756,"p90":55.52725593423847,"p95":53.544139650872815,"p99":49.97453034081463},"payload_bytes":384763904},"roundtrip":{"latency_us":{"p50":930,"p90":1004.4000000000001,"p95":1041.6000000000001,"p99":1116},"activation_data_rate_gbps_at_latency_percentile":{"p50":827.4492559139785,"p90":766.1567184388689,"p95":738.7939784946236,"p99":689.541046594982},"payload_data_rate_gbps_at_latency_percentile":{"p50":52.739509677419356,"p90":48.83287933094384,"p95":47.08884792626728,"p99":43.94959139784947},"payload_bytes":784763904}},"roundtrip_token_rate_at_latency_percentile":{"p50":8338218,"p90":9005275.440000001,"p95":9338804.16,"p99":10005861.6}}]},{"series_id":"h200-dgxc-deepep-v2-deepseek-v3-normal-decode-ep8-uniform-fp8","phase":"decode","mode":"normal","precision":"fp8","backend":"deepep-v2","system":{"ep_size":8,"nodes":1,"gpus_per_node":8,"scale_up_domain":8,"scale_up_transport":"nvlink","scale_out_transport":null,"topology_class":"h200-nvlink-island","sku":"h200-dgxc","vendor":"nvidia"},"points":[{"tokens_per_rank":1,"global_tokens":8,"components":{"dispatch":{"latency_us":{"p50":417,"p90":450.36,"p95":467.04,"p99":500.4},"activation_data_rate_gbps_at_latency_percentile":{"p50":922.6952134292566,"p90":854.3474198419042,"p95":823.8350119904076,"p99":768.9126778577139},"payload_data_rate_gbps_at_latency_percentile":{"p50":119.90407673860912,"p90":111.02229327648992,"p95":107.05721137375814,"p99":99.92006394884093},"payload_bytes":400000000},"stage":{"latency_us":{"p50":120,"p90":129.60000000000002,"p95":134.4,"p99":144},"activation_data_rate_gbps_at_latency_percentile":{"p50":1603.1829333333335,"p90":1484.4286419753084,"p95":1431.4133333333332,"p99":1335.9857777777777},"payload_data_rate_gbps_at_latency_percentile":{"p50":200.3978666666667,"p90":185.55358024691355,"p95":178.92666666666665,"p99":166.9982222222222},"payload_bytes":192381952},"combine":{"latency_us":{"p50":392,"p90":423.36,"p95":439.04,"p99":470.4},"activation_data_rate_gbps_at_latency_percentile":{"p50":981.5405714285715,"p90":908.8338624338625,"p95":876.3755102040816,"p99":817.9504761904763},"payload_data_rate_gbps_at_latency_percentile":{"p50":122.69257142857144,"p90":113.60423280423281,"p95":109.5469387755102,"p99":102.24380952380953},"payload_bytes":384763904},"roundtrip":{"latency_us":{"p50":921,"p90":994.6800000000001,"p95":1031.5200000000002,"p99":1105.2},"activation_data_rate_gbps_at_latency_percentile":{"p50":835.5350792616721,"p90":773.6435919089556,"p95":746.0134636264928,"p99":696.27923271806},"payload_data_rate_gbps_at_latency_percentile":{"p50":106.50975895765473,"p90":98.62014718301363,"p95":95.09799906933456,"p99":88.75813246471228},"payload_bytes":784763904}},"roundtrip_token_rate_at_latency_percentile":{"p50":8338218,"p90":9005275.440000001,"p95":9338804.16,"p99":10005861.6}},{"tokens_per_rank":2,"global_tokens":16,"components":{"dispatch":{"latency_us":{"p50":418,"p90":451.44000000000005,"p95":468.16,"p99":501.59999999999997},"activation_data_rate_gbps_at_latency_percentile":{"p50":920.4878086124403,"p90":852.3035264930002,"p95":821.8641148325358,"p99":767.0731738437003},"payload_data_rate_gbps_at_latency_percentile":{"p50":119.61722488038276,"p90":110.75668970405812,"p95":106.8010936431989,"p99":99.68102073365232},"payload_bytes":400000000},"stage":{"latency_us":{"p50":121,"p90":130.68,"p95":135.52,"p99":145.2},"activation_data_rate_gbps_at_latency_percentile":{"p50":1589.9334876033058,"p90":1472.1606366697276,"p95":1419.58347107438,"p99":1324.944573002755},"payload_data_rate_gbps_at_latency_percentile":{"p50":198.74168595041323,"p90":184.02007958371595,"p95":177.4479338842975,"p99":165.61807162534438},"payload_bytes":192381952},"combine":{"latency_us":{"p50":393,"p90":424.44000000000005,"p95":440.16,"p99":471.59999999999997},"activation_data_rate_gbps_at_latency_percentile":{"p50":979.0430127226464,"p90":906.5213080765243,"p95":874.1455470737912,"p99":815.869177268872},"payload_data_rate_gbps_at_latency_percentile":{"p50":122.3803765903308,"p90":113.31516350956554,"p95":109.2681933842239,"p99":101.983647158609},"payload_bytes":384763904},"roundtrip":{"latency_us":{"p50":922,"p90":995.7600000000001,"p95":1032.64,"p99":1106.3999999999999},"activation_data_rate_gbps_at_latency_percentile":{"p50":834.6288590021693,"p90":772.8044990760825,"p95":745.2043383947938,"p99":695.5240491684744},"payload_data_rate_gbps_at_latency_percentile":{"p50":106.39423861171366,"p90":98.51318389973487,"p95":94.99485590331577,"p99":88.6618655097614},"payload_bytes":784763904}},"roundtrip_token_rate_at_latency_percentile":{"p50":8338218,"p90":9005275.440000001,"p95":9338804.16,"p99":10005861.6}},{"tokens_per_rank":4,"global_tokens":32,"components":{"dispatch":{"latency_us":{"p50":419,"p90":452.52000000000004,"p95":469.28000000000003,"p99":502.79999999999995},"activation_data_rate_gbps_at_latency_percentile":{"p50":918.2909403341289,"p90":850.2693891982674,"p95":819.9026252983293,"p99":765.2424502784409},"payload_data_rate_gbps_at_latency_percentile":{"p50":119.33174224343675,"p90":110.49235392910809,"p95":106.54619843163995,"p99":99.4431185361973},"payload_bytes":400000000},"stage":{"latency_us":{"p50":122,"p90":131.76000000000002,"p95":136.64000000000001,"p99":146.4},"activation_data_rate_gbps_at_latency_percentile":{"p50":1576.9012459016394,"p90":1460.0937462052214,"p95":1407.9475409836064,"p99":1314.0843715846995},"payload_data_rate_gbps_at_latency_percentile":{"p50":197.11265573770493,"p90":182.51171827565267,"p95":175.9934426229508,"p99":164.26054644808744},"payload_bytes":192381952},"combine":{"latency_us":{"p50":394,"p90":425.52000000000004,"p95":441.28000000000003,"p99":472.79999999999995},"activation_data_rate_gbps_at_latency_percentile":{"p50":976.5581319796954,"p90":904.2204925737921,"p95":871.9269035532996,"p99":813.798443316413},"payload_data_rate_gbps_at_latency_percentile":{"p50":122.06976649746193,"p90":113.02756157172401,"p95":108.99086294416244,"p99":101.72480541455162},"payload_bytes":384763904},"roundtrip":{"latency_us":{"p50":923,"p90":996.84,"p95":1033.76,"p99":1107.6},"activation_data_rate_gbps_at_latency_percentile":{"p50":833.724602383532,"p90":771.9672244291962,"p95":744.3969664138677,"p99":694.7705019862767},"payload_data_rate_gbps_at_latency_percentile":{"p50":106.27896858071506,"p90":98.40645238955098,"p95":94.8919362327813,"p99":88.5658071505959},"payload_bytes":784763904}},"roundtrip_token_rate_at_latency_percentile":{"p50":8338218,"p90":9005275.440000001,"p95":9338804.16,"p99":10005861.6}},{"tokens_per_rank":8,"global_tokens":64,"components":{"dispatch":{"latency_us":{"p50":420,"p90":453.6,"p95":470.40000000000003,"p99":504},"activation_data_rate_gbps_at_latency_percentile":{"p50":916.1045333333334,"p90":848.244938271605,"p95":817.9504761904761,"p99":763.4204444444445},"payload_data_rate_gbps_at_latency_percentile":{"p50":119.04761904761905,"p90":110.22927689594357,"p95":106.29251700680271,"p99":99.2063492063492},"payload_bytes":400000000},"stage":{"latency_us":{"p50":123,"p90":132.84,"p95":137.76000000000002,"p99":147.6},"activation_data_rate_gbps_at_latency_percentile":{"p50":1564.0809105691058,"p90":1448.2230653417646,"p95":1396.5008130081299,"p99":1303.400758807588},"payload_data_rate_gbps_at_latency_percentile":{"p50":195.51011382113822,"p90":181.02788316772057,"p95":174.56260162601623,"p99":162.9250948509485},"payload_bytes":192381952},"combine":{"latency_us":{"p50":395,"p90":426.6,"p95":442.40000000000003,"p99":474},"activation_data_rate_gbps_at_latency_percentile":{"p50":974.0858329113925,"p90":901.9313267698077,"p95":869.7194936708861,"p99":811.738194092827},"payload_data_rate_gbps_at_latency_percentile":{"p50":121.76072911392406,"p90":112.74141584622596,"p95":108.71493670886076,"p99":101.46727426160338},"payload_bytes":384763904},"roundtrip":{"latency_us":{"p50":924,"p90":997.9200000000001,"p95":1034.88,"p99":1108.8},"activation_data_rate_gbps_at_latency_percentile":{"p50":832.822303030303,"p90":771.1317620650954,"p95":743.5913419913419,"p99":694.0185858585859},"payload_data_rate_gbps_at_latency_percentile":{"p50":106.16394805194805,"p90":98.29995189995189,"p95":94.78923933209646,"p99":88.4699567099567},"payload_bytes":784763904}},"roundtrip_token_rate_at_latency_percentile":{"p50":8338218,"p90":9005275.440000001,"p95":9338804.16,"p99":10005861.6}},{"tokens_per_rank":16,"global_tokens":128,"components":{"dispatch":{"latency_us":{"p50":421,"p90":454.68,"p95":471.52000000000004,"p99":505.2},"activation_data_rate_gbps_at_latency_percentile":{"p50":913.928513064133,"p90":846.230104689012,"p95":816.0076009501188,"p99":761.607094220111},"payload_data_rate_gbps_at_latency_percentile":{"p50":118.76484560570071,"p90":109.96744963490808,"p95":106.04004071937563,"p99":98.97070467141727},"payload_bytes":400000000},"stage":{"latency_us":{"p50":124,"p90":133.92000000000002,"p95":138.88000000000002,"p99":148.79999999999998},"activation_data_rate_gbps_at_latency_percentile":{"p50":1551.4673548387095,"p90":1436.543847072879,"p95":1385.2387096774191,"p99":1292.8894623655915},"payload_data_rate_gbps_at_latency_percentile":{"p50":193.9334193548387,"p90":179.5679808841099,"p95":173.1548387096774,"p99":161.61118279569894},"payload_bytes":192381952},"combine":{"latency_us":{"p50":396,"p90":427.68,"p95":443.52000000000004,"p99":475.2},"activation_data_rate_gbps_at_latency_percentile":{"p50":971.6260202020202,"p90":899.653722409278,"p95":867.5232323232323,"p99":809.6883501683502},"payload_data_rate_gbps_at_latency_percentile":{"p50":121.45325252525252,"p90":112.45671530115975,"p95":108.44040404040403,"p99":101.21104377104378},"payload_bytes":384763904},"roundtrip":{"latency_us":{"p50":925,"p90":999.0000000000001,"p95":1036,"p99":1110},"activation_data_rate_gbps_at_latency_percentile":{"p50":831.9219545945946,"p90":770.298106106106,"p95":742.7874594594595,"p99":693.2682954954955},"payload_data_rate_gbps_at_latency_percentile":{"p50":106.04917621621622,"p90":98.19368168168167,"p95":94.68676447876449,"p99":88.37431351351351},"payload_bytes":784763904}},"roundtrip_token_rate_at_latency_percentile":{"p50":8338218,"p90":9005275.440000001,"p95":9338804.16,"p99":10005861.6}},{"tokens_per_rank":32,"global_tokens":256,"components":{"dispatch":{"latency_us":{"p50":422,"p90":455.76000000000005,"p95":472.64000000000004,"p99":506.4},"activation_data_rate_gbps_at_latency_percentile":{"p50":911.7628056872038,"p90":844.2248200807443,"p95":814.0739336492891,"p99":759.8023380726698},"payload_data_rate_gbps_at_latency_percentile":{"p50":118.48341232227487,"p90":109.70686326136563,"p95":105.78876100203114,"p99":98.73617693522907},"payload_bytes":400000000},"stage":{"latency_us":{"p50":125,"p90":135,"p95":140,"p99":150},"activation_data_rate_gbps_at_latency_percentile":{"p50":1539.0556159999999,"p90":1425.0514962962964,"p95":1374.1568,"p99":1282.5463466666668},"payload_data_rate_gbps_at_latency_percentile":{"p50":192.38195199999998,"p90":178.13143703703705,"p95":171.7696,"p99":160.31829333333334},"payload_bytes":192381952},"combine":{"latency_us":{"p50":397,"p90":428.76000000000005,"p95":444.64000000000004,"p99":476.4},"activation_data_rate_gbps_at_latency_percentile":{"p50":969.1785994962216,"p90":897.387592126131,"p95":865.3380352644836,"p99":807.648832913518},"payload_data_rate_gbps_at_latency_percentile":{"p50":121.1473249370277,"p90":112.17344901576638,"p95":108.16725440806044,"p99":100.95610411418976},"payload_bytes":384763904},"roundtrip":{"latency_us":{"p50":926,"p90":1000.08,"p95":1037.1200000000001,"p99":1111.2},"activation_data_rate_gbps_at_latency_percentile":{"p50":831.0235507559396,"p90":769.466250699944,"p95":741.9853131749459,"p99":692.5196256299496},"payload_data_rate_gbps_at_latency_percentile":{"p50":105.93465226781858,"p90":98.0876409887209,"p95":94.58451095340943,"p99":88.2788768898488},"payload_bytes":784763904}},"roundtrip_token_rate_at_latency_percentile":{"p50":8338218,"p90":9005275.440000001,"p95":9338804.16,"p99":10005861.6}},{"tokens_per_rank":64,"global_tokens":512,"components":{"dispatch":{"latency_us":{"p50":423,"p90":456.84000000000003,"p95":473.76000000000005,"p99":507.59999999999997},"activation_data_rate_gbps_at_latency_percentile":{"p50":909.6073380614658,"p90":842.2290167235793,"p95":812.1494089834514,"p99":758.0061150512215},"payload_data_rate_gbps_at_latency_percentile":{"p50":118.2033096926714,"p90":109.44750897469572,"p95":105.5386693684566,"p99":98.50275807722618},"payload_bytes":400000000},"stage":{"latency_us":{"p50":126,"p90":136.08,"p95":141.12,"p99":151.2},"activation_data_rate_gbps_at_latency_percentile":{"p50":1526.840888888889,"p90":1413.741563786008,"p95":1363.2507936507936,"p99":1272.3674074074074},"payload_data_rate_gbps_at_latency_percentile":{"p50":190.8551111111111,"p90":176.717695473251,"p95":170.4063492063492,"p99":159.04592592592593},"payload_bytes":192381952},"combine":{"latency_us":{"p50":398,"p90":429.84000000000003,"p95":445.76000000000005,"p99":477.59999999999997},"activation_data_rate_gbps_at_latency_percentile":{"p50":966.7434773869347,"p90":895.1328494323469,"p95":863.1638190954774,"p99":805.6195644891122},"payload_data_rate_gbps_at_latency_percentile":{"p50":120.84293467336684,"p90":111.89160617904336,"p95":107.89547738693467,"p99":100.70244556113903},"payload_bytes":384763904},"roundtrip":{"latency_us":{"p50":927,"p90":1001.1600000000001,"p95":1038.24,"p99":1112.3999999999999},"activation_data_rate_gbps_at_latency_percentile":{"p50":830.1270852211435,"p90":768.6361900195773,"p95":741.1848975188781,"p99":691.7725710176197},"payload_data_rate_gbps_at_latency_percentile":{"p50":105.82037540453075,"p90":97.98182907826921,"p95":94.48247803975958,"p99":88.1836461704423},"payload_bytes":784763904}},"roundtrip_token_rate_at_latency_percentile":{"p50":8338218,"p90":9005275.440000001,"p95":9338804.16,"p99":10005861.6}},{"tokens_per_rank":128,"global_tokens":1024,"components":{"dispatch":{"latency_us":{"p50":424,"p90":457.92,"p95":474.88000000000005,"p99":508.79999999999995},"activation_data_rate_gbps_at_latency_percentile":{"p50":907.4620377358491,"p90":840.2426275331936,"p95":810.2339622641508,"p99":756.2183647798744},"payload_data_rate_gbps_at_latency_percentile":{"p50":117.9245283018868,"p90":109.18937805730259,"p95":105.2897574123989,"p99":98.27044025157234},"payload_bytes":400000000},"stage":{"latency_us":{"p50":127,"p90":137.16,"p95":142.24,"p99":152.4},"activation_data_rate_gbps_at_latency_percentile":{"p50":1514.8185196850393,"p90":1402.6097404491106,"p95":1352.5165354330707,"p99":1262.3487664041995},"payload_data_rate_gbps_at_latency_percentile":{"p50":189.3523149606299,"p90":175.32621755613883,"p95":169.06456692913383,"p99":157.79359580052494},"payload_bytes":192381952},"combine":{"latency_us":{"p50":399,"p90":430.92,"p95":446.88000000000005,"p99":478.79999999999995},"activation_data_rate_gbps_at_latency_percentile":{"p50":964.3205614035088,"p90":892.8894087069526,"p95":861.0005012531327,"p99":803.6004678362574},"payload_data_rate_gbps_at_latency_percentile":{"p50":120.5400701754386,"p90":111.61117608836908,"p95":107.62506265664159,"p99":100.45005847953217},"payload_bytes":384763904},"roundtrip":{"latency_us":{"p50":928,"p90":1002.24,"p95":1039.3600000000001,"p99":1113.6},"activation_data_rate_gbps_at_latency_percentile":{"p50":829.232551724138,"p90":767.8079182630906,"p95":740.3862068965517,"p99":691.0271264367817},"payload_data_rate_gbps_at_latency_percentile":{"p50":105.70634482758621,"p90":97.87624521072797,"p95":94.38066502463055,"p99":88.08862068965517},"payload_bytes":784763904}},"roundtrip_token_rate_at_latency_percentile":{"p50":8338218,"p90":9005275.440000001,"p95":9338804.16,"p99":10005861.6}},{"tokens_per_rank":256,"global_tokens":2048,"components":{"dispatch":{"latency_us":{"p50":425,"p90":459.00000000000006,"p95":476.00000000000006,"p99":510},"activation_data_rate_gbps_at_latency_percentile":{"p50":905.3268329411765,"p90":838.2655860566448,"p95":808.3275294117645,"p99":754.4390274509803},"payload_data_rate_gbps_at_latency_percentile":{"p50":117.64705882352942,"p90":108.93246187363833,"p95":105.04201680672269,"p99":98.0392156862745},"payload_bytes":400000000},"stage":{"latency_us":{"p50":128,"p90":138.24,"p95":143.36,"p99":153.6},"activation_data_rate_gbps_at_latency_percentile":{"p50":1502.984,"p90":1391.6518518518517,"p95":1341.9499999999998,"p99":1252.4866666666667},"payload_data_rate_gbps_at_latency_percentile":{"p50":187.873,"p90":173.95648148148146,"p95":167.74374999999998,"p99":156.56083333333333},"payload_bytes":192381952},"combine":{"latency_us":{"p50":400,"p90":432,"p95":448.00000000000006,"p99":480},"activation_data_rate_gbps_at_latency_percentile":{"p50":961.90976,"p90":890.6571851851852,"p95":858.848,"p99":801.5914666666667},"payload_data_rate_gbps_at_latency_percentile":{"p50":120.23872,"p90":111.33214814814815,"p95":107.356,"p99":100.19893333333334},"payload_bytes":384763904},"roundtrip":{"latency_us":{"p50":929,"p90":1003.32,"p95":1040.48,"p99":1114.8},"activation_data_rate_gbps_at_latency_percentile":{"p50":828.3399440258343,"p90":766.9814296535502,"p95":739.5892357373519,"p99":690.2832866881952},"payload_data_rate_gbps_at_latency_percentile":{"p50":105.5925597416577,"p90":97.77088864968306,"p95":94.27907119790866,"p99":87.99379978471475},"payload_bytes":784763904}},"roundtrip_token_rate_at_latency_percentile":{"p50":8338218,"p90":9005275.440000001,"p95":9338804.16,"p99":10005861.6}},{"tokens_per_rank":512,"global_tokens":4096,"components":{"dispatch":{"latency_us":{"p50":426,"p90":460.08000000000004,"p95":477.12000000000006,"p99":511.2},"activation_data_rate_gbps_at_latency_percentile":{"p50":903.2016525821597,"p90":836.2978264649626,"p95":806.4300469483567,"p99":752.6680438184663},"payload_data_rate_gbps_at_latency_percentile":{"p50":117.37089201877934,"p90":108.67675186924012,"p95":104.79543930248154,"p99":97.80907668231613},"payload_bytes":400000000},"stage":{"latency_us":{"p50":129,"p90":139.32000000000002,"p95":144.48000000000002,"p99":154.79999999999998},"activation_data_rate_gbps_at_latency_percentile":{"p50":1491.33296124031,"p90":1380.863853000287,"p95":1331.5472868217053,"p99":1242.7774677002585},"payload_data_rate_gbps_at_latency_percentile":{"p50":186.41662015503874,"p90":172.60798162503588,"p95":166.44341085271316,"p99":155.3471834625323},"payload_bytes":192381952},"combine":{"latency_us":{"p50":401,"p90":433.08000000000004,"p95":449.12000000000006,"p99":481.2},"activation_data_rate_gbps_at_latency_percentile":{"p50":959.5109825436409,"p90":888.4360949478155,"p95":856.706234413965,"p99":799.5924854530341},"payload_data_rate_gbps_at_latency_percentile":{"p50":119.93887281795512,"p90":111.05451186847694,"p95":107.08827930174563,"p99":99.94906068162926},"payload_bytes":384763904},"roundtrip":{"latency_us":{"p50":930,"p90":1004.4000000000001,"p95":1041.6000000000001,"p99":1116},"activation_data_rate_gbps_at_latency_percentile":{"p50":827.4492559139785,"p90":766.1567184388689,"p95":738.7939784946236,"p99":689.541046594982},"payload_data_rate_gbps_at_latency_percentile":{"p50":105.47901935483871,"p90":97.66575866188768,"p95":94.17769585253455,"p99":87.89918279569893},"payload_bytes":784763904}},"roundtrip_token_rate_at_latency_percentile":{"p50":8338218,"p90":9005275.440000001,"p95":9338804.16,"p99":10005861.6}}]}]} diff --git a/packages/app/cypress/fixtures/api/collectivex-run-159.json b/packages/app/cypress/fixtures/api/collectivex-run-159.json new file mode 100644 index 000000000..6cc60f58e --- /dev/null +++ b/packages/app/cypress/fixtures/api/collectivex-run-159.json @@ -0,0 +1 @@ +{"version":1,"run":{"run_id":"159","run_attempt":1,"generated_at":"2026-07-07T12:20:00Z","conclusion":"success","source_sha":"dddddddddddddddddddddddddddddddddddddddd","requested_cases":1,"terminal_cases":1,"measured_cases":1,"unsupported_cases":0,"failed_cases":0,"requested_points":10,"terminal_points":10,"measured_points":10,"covered_skus":["h200-dgxc"]},"coverage":[{"case_id":"h200-dgxc-deepep-v2-deepseek-v3-normal-decode-ep8-uniform-fp8","label":"h200-dgxc · deepep-v2 · normal · decode · EP8 · fp8","disposition":"runnable","sku":"h200-dgxc","backend":"deepep-v2","phase":"decode","mode":"normal","precision":"fp8","topology":{"ep_size":8,"nodes":1,"gpus_per_node":8,"scale_up_domain":8,"scale_up_transport":"nvlink","scale_out_transport":null,"topology_class":"h200-nvlink-island"},"points":[{"tokens_per_rank":1,"global_tokens":8,"terminal_status":"measured","reason":null},{"tokens_per_rank":2,"global_tokens":16,"terminal_status":"measured","reason":null},{"tokens_per_rank":4,"global_tokens":32,"terminal_status":"measured","reason":null},{"tokens_per_rank":8,"global_tokens":64,"terminal_status":"measured","reason":null},{"tokens_per_rank":16,"global_tokens":128,"terminal_status":"measured","reason":null},{"tokens_per_rank":32,"global_tokens":256,"terminal_status":"measured","reason":null},{"tokens_per_rank":64,"global_tokens":512,"terminal_status":"measured","reason":null},{"tokens_per_rank":128,"global_tokens":1024,"terminal_status":"measured","reason":null},{"tokens_per_rank":256,"global_tokens":2048,"terminal_status":"measured","reason":null},{"tokens_per_rank":512,"global_tokens":4096,"terminal_status":"measured","reason":null}],"outcome":"success","reason":null,"detail":null}],"series":[{"series_id":"h200-dgxc-deepep-v2-deepseek-v3-normal-decode-ep8-uniform-fp8","phase":"decode","mode":"normal","precision":"fp8","backend":"deepep-v2","system":{"ep_size":8,"nodes":1,"gpus_per_node":8,"scale_up_domain":8,"scale_up_transport":"nvlink","scale_out_transport":null,"topology_class":"h200-nvlink-island","sku":"h200-dgxc","vendor":"nvidia"},"points":[{"tokens_per_rank":1,"global_tokens":8,"components":{"dispatch":{"latency_us":{"p50":417,"p90":450.36,"p95":467.04,"p99":500.4},"activation_data_rate_gbps_at_latency_percentile":{"p50":922.6952134292566,"p90":854.3474198419042,"p95":823.8350119904076,"p99":768.9126778577139},"payload_data_rate_gbps_at_latency_percentile":{"p50":119.90407673860912,"p90":111.02229327648992,"p95":107.05721137375814,"p99":99.92006394884093},"payload_bytes":400000000},"stage":{"latency_us":{"p50":120,"p90":129.60000000000002,"p95":134.4,"p99":144},"activation_data_rate_gbps_at_latency_percentile":{"p50":1603.1829333333335,"p90":1484.4286419753084,"p95":1431.4133333333332,"p99":1335.9857777777777},"payload_data_rate_gbps_at_latency_percentile":{"p50":200.3978666666667,"p90":185.55358024691355,"p95":178.92666666666665,"p99":166.9982222222222},"payload_bytes":192381952},"combine":{"latency_us":{"p50":392,"p90":423.36,"p95":439.04,"p99":470.4},"activation_data_rate_gbps_at_latency_percentile":{"p50":981.5405714285715,"p90":908.8338624338625,"p95":876.3755102040816,"p99":817.9504761904763},"payload_data_rate_gbps_at_latency_percentile":{"p50":122.69257142857144,"p90":113.60423280423281,"p95":109.5469387755102,"p99":102.24380952380953},"payload_bytes":384763904},"roundtrip":{"latency_us":{"p50":921,"p90":994.6800000000001,"p95":1031.5200000000002,"p99":1105.2},"activation_data_rate_gbps_at_latency_percentile":{"p50":835.5350792616721,"p90":773.6435919089556,"p95":746.0134636264928,"p99":696.27923271806},"payload_data_rate_gbps_at_latency_percentile":{"p50":106.50975895765473,"p90":98.62014718301363,"p95":95.09799906933456,"p99":88.75813246471228},"payload_bytes":784763904}},"roundtrip_token_rate_at_latency_percentile":{"p50":8338218,"p90":9005275.440000001,"p95":9338804.16,"p99":10005861.6}},{"tokens_per_rank":2,"global_tokens":16,"components":{"dispatch":{"latency_us":{"p50":418,"p90":451.44000000000005,"p95":468.16,"p99":501.59999999999997},"activation_data_rate_gbps_at_latency_percentile":{"p50":920.4878086124403,"p90":852.3035264930002,"p95":821.8641148325358,"p99":767.0731738437003},"payload_data_rate_gbps_at_latency_percentile":{"p50":119.61722488038276,"p90":110.75668970405812,"p95":106.8010936431989,"p99":99.68102073365232},"payload_bytes":400000000},"stage":{"latency_us":{"p50":121,"p90":130.68,"p95":135.52,"p99":145.2},"activation_data_rate_gbps_at_latency_percentile":{"p50":1589.9334876033058,"p90":1472.1606366697276,"p95":1419.58347107438,"p99":1324.944573002755},"payload_data_rate_gbps_at_latency_percentile":{"p50":198.74168595041323,"p90":184.02007958371595,"p95":177.4479338842975,"p99":165.61807162534438},"payload_bytes":192381952},"combine":{"latency_us":{"p50":393,"p90":424.44000000000005,"p95":440.16,"p99":471.59999999999997},"activation_data_rate_gbps_at_latency_percentile":{"p50":979.0430127226464,"p90":906.5213080765243,"p95":874.1455470737912,"p99":815.869177268872},"payload_data_rate_gbps_at_latency_percentile":{"p50":122.3803765903308,"p90":113.31516350956554,"p95":109.2681933842239,"p99":101.983647158609},"payload_bytes":384763904},"roundtrip":{"latency_us":{"p50":922,"p90":995.7600000000001,"p95":1032.64,"p99":1106.3999999999999},"activation_data_rate_gbps_at_latency_percentile":{"p50":834.6288590021693,"p90":772.8044990760825,"p95":745.2043383947938,"p99":695.5240491684744},"payload_data_rate_gbps_at_latency_percentile":{"p50":106.39423861171366,"p90":98.51318389973487,"p95":94.99485590331577,"p99":88.6618655097614},"payload_bytes":784763904}},"roundtrip_token_rate_at_latency_percentile":{"p50":8338218,"p90":9005275.440000001,"p95":9338804.16,"p99":10005861.6}},{"tokens_per_rank":4,"global_tokens":32,"components":{"dispatch":{"latency_us":{"p50":419,"p90":452.52000000000004,"p95":469.28000000000003,"p99":502.79999999999995},"activation_data_rate_gbps_at_latency_percentile":{"p50":918.2909403341289,"p90":850.2693891982674,"p95":819.9026252983293,"p99":765.2424502784409},"payload_data_rate_gbps_at_latency_percentile":{"p50":119.33174224343675,"p90":110.49235392910809,"p95":106.54619843163995,"p99":99.4431185361973},"payload_bytes":400000000},"stage":{"latency_us":{"p50":122,"p90":131.76000000000002,"p95":136.64000000000001,"p99":146.4},"activation_data_rate_gbps_at_latency_percentile":{"p50":1576.9012459016394,"p90":1460.0937462052214,"p95":1407.9475409836064,"p99":1314.0843715846995},"payload_data_rate_gbps_at_latency_percentile":{"p50":197.11265573770493,"p90":182.51171827565267,"p95":175.9934426229508,"p99":164.26054644808744},"payload_bytes":192381952},"combine":{"latency_us":{"p50":394,"p90":425.52000000000004,"p95":441.28000000000003,"p99":472.79999999999995},"activation_data_rate_gbps_at_latency_percentile":{"p50":976.5581319796954,"p90":904.2204925737921,"p95":871.9269035532996,"p99":813.798443316413},"payload_data_rate_gbps_at_latency_percentile":{"p50":122.06976649746193,"p90":113.02756157172401,"p95":108.99086294416244,"p99":101.72480541455162},"payload_bytes":384763904},"roundtrip":{"latency_us":{"p50":923,"p90":996.84,"p95":1033.76,"p99":1107.6},"activation_data_rate_gbps_at_latency_percentile":{"p50":833.724602383532,"p90":771.9672244291962,"p95":744.3969664138677,"p99":694.7705019862767},"payload_data_rate_gbps_at_latency_percentile":{"p50":106.27896858071506,"p90":98.40645238955098,"p95":94.8919362327813,"p99":88.5658071505959},"payload_bytes":784763904}},"roundtrip_token_rate_at_latency_percentile":{"p50":8338218,"p90":9005275.440000001,"p95":9338804.16,"p99":10005861.6}},{"tokens_per_rank":8,"global_tokens":64,"components":{"dispatch":{"latency_us":{"p50":420,"p90":453.6,"p95":470.40000000000003,"p99":504},"activation_data_rate_gbps_at_latency_percentile":{"p50":916.1045333333334,"p90":848.244938271605,"p95":817.9504761904761,"p99":763.4204444444445},"payload_data_rate_gbps_at_latency_percentile":{"p50":119.04761904761905,"p90":110.22927689594357,"p95":106.29251700680271,"p99":99.2063492063492},"payload_bytes":400000000},"stage":{"latency_us":{"p50":123,"p90":132.84,"p95":137.76000000000002,"p99":147.6},"activation_data_rate_gbps_at_latency_percentile":{"p50":1564.0809105691058,"p90":1448.2230653417646,"p95":1396.5008130081299,"p99":1303.400758807588},"payload_data_rate_gbps_at_latency_percentile":{"p50":195.51011382113822,"p90":181.02788316772057,"p95":174.56260162601623,"p99":162.9250948509485},"payload_bytes":192381952},"combine":{"latency_us":{"p50":395,"p90":426.6,"p95":442.40000000000003,"p99":474},"activation_data_rate_gbps_at_latency_percentile":{"p50":974.0858329113925,"p90":901.9313267698077,"p95":869.7194936708861,"p99":811.738194092827},"payload_data_rate_gbps_at_latency_percentile":{"p50":121.76072911392406,"p90":112.74141584622596,"p95":108.71493670886076,"p99":101.46727426160338},"payload_bytes":384763904},"roundtrip":{"latency_us":{"p50":924,"p90":997.9200000000001,"p95":1034.88,"p99":1108.8},"activation_data_rate_gbps_at_latency_percentile":{"p50":832.822303030303,"p90":771.1317620650954,"p95":743.5913419913419,"p99":694.0185858585859},"payload_data_rate_gbps_at_latency_percentile":{"p50":106.16394805194805,"p90":98.29995189995189,"p95":94.78923933209646,"p99":88.4699567099567},"payload_bytes":784763904}},"roundtrip_token_rate_at_latency_percentile":{"p50":8338218,"p90":9005275.440000001,"p95":9338804.16,"p99":10005861.6}},{"tokens_per_rank":16,"global_tokens":128,"components":{"dispatch":{"latency_us":{"p50":421,"p90":454.68,"p95":471.52000000000004,"p99":505.2},"activation_data_rate_gbps_at_latency_percentile":{"p50":913.928513064133,"p90":846.230104689012,"p95":816.0076009501188,"p99":761.607094220111},"payload_data_rate_gbps_at_latency_percentile":{"p50":118.76484560570071,"p90":109.96744963490808,"p95":106.04004071937563,"p99":98.97070467141727},"payload_bytes":400000000},"stage":{"latency_us":{"p50":124,"p90":133.92000000000002,"p95":138.88000000000002,"p99":148.79999999999998},"activation_data_rate_gbps_at_latency_percentile":{"p50":1551.4673548387095,"p90":1436.543847072879,"p95":1385.2387096774191,"p99":1292.8894623655915},"payload_data_rate_gbps_at_latency_percentile":{"p50":193.9334193548387,"p90":179.5679808841099,"p95":173.1548387096774,"p99":161.61118279569894},"payload_bytes":192381952},"combine":{"latency_us":{"p50":396,"p90":427.68,"p95":443.52000000000004,"p99":475.2},"activation_data_rate_gbps_at_latency_percentile":{"p50":971.6260202020202,"p90":899.653722409278,"p95":867.5232323232323,"p99":809.6883501683502},"payload_data_rate_gbps_at_latency_percentile":{"p50":121.45325252525252,"p90":112.45671530115975,"p95":108.44040404040403,"p99":101.21104377104378},"payload_bytes":384763904},"roundtrip":{"latency_us":{"p50":925,"p90":999.0000000000001,"p95":1036,"p99":1110},"activation_data_rate_gbps_at_latency_percentile":{"p50":831.9219545945946,"p90":770.298106106106,"p95":742.7874594594595,"p99":693.2682954954955},"payload_data_rate_gbps_at_latency_percentile":{"p50":106.04917621621622,"p90":98.19368168168167,"p95":94.68676447876449,"p99":88.37431351351351},"payload_bytes":784763904}},"roundtrip_token_rate_at_latency_percentile":{"p50":8338218,"p90":9005275.440000001,"p95":9338804.16,"p99":10005861.6}},{"tokens_per_rank":32,"global_tokens":256,"components":{"dispatch":{"latency_us":{"p50":422,"p90":455.76000000000005,"p95":472.64000000000004,"p99":506.4},"activation_data_rate_gbps_at_latency_percentile":{"p50":911.7628056872038,"p90":844.2248200807443,"p95":814.0739336492891,"p99":759.8023380726698},"payload_data_rate_gbps_at_latency_percentile":{"p50":118.48341232227487,"p90":109.70686326136563,"p95":105.78876100203114,"p99":98.73617693522907},"payload_bytes":400000000},"stage":{"latency_us":{"p50":125,"p90":135,"p95":140,"p99":150},"activation_data_rate_gbps_at_latency_percentile":{"p50":1539.0556159999999,"p90":1425.0514962962964,"p95":1374.1568,"p99":1282.5463466666668},"payload_data_rate_gbps_at_latency_percentile":{"p50":192.38195199999998,"p90":178.13143703703705,"p95":171.7696,"p99":160.31829333333334},"payload_bytes":192381952},"combine":{"latency_us":{"p50":397,"p90":428.76000000000005,"p95":444.64000000000004,"p99":476.4},"activation_data_rate_gbps_at_latency_percentile":{"p50":969.1785994962216,"p90":897.387592126131,"p95":865.3380352644836,"p99":807.648832913518},"payload_data_rate_gbps_at_latency_percentile":{"p50":121.1473249370277,"p90":112.17344901576638,"p95":108.16725440806044,"p99":100.95610411418976},"payload_bytes":384763904},"roundtrip":{"latency_us":{"p50":926,"p90":1000.08,"p95":1037.1200000000001,"p99":1111.2},"activation_data_rate_gbps_at_latency_percentile":{"p50":831.0235507559396,"p90":769.466250699944,"p95":741.9853131749459,"p99":692.5196256299496},"payload_data_rate_gbps_at_latency_percentile":{"p50":105.93465226781858,"p90":98.0876409887209,"p95":94.58451095340943,"p99":88.2788768898488},"payload_bytes":784763904}},"roundtrip_token_rate_at_latency_percentile":{"p50":8338218,"p90":9005275.440000001,"p95":9338804.16,"p99":10005861.6}},{"tokens_per_rank":64,"global_tokens":512,"components":{"dispatch":{"latency_us":{"p50":423,"p90":456.84000000000003,"p95":473.76000000000005,"p99":507.59999999999997},"activation_data_rate_gbps_at_latency_percentile":{"p50":909.6073380614658,"p90":842.2290167235793,"p95":812.1494089834514,"p99":758.0061150512215},"payload_data_rate_gbps_at_latency_percentile":{"p50":118.2033096926714,"p90":109.44750897469572,"p95":105.5386693684566,"p99":98.50275807722618},"payload_bytes":400000000},"stage":{"latency_us":{"p50":126,"p90":136.08,"p95":141.12,"p99":151.2},"activation_data_rate_gbps_at_latency_percentile":{"p50":1526.840888888889,"p90":1413.741563786008,"p95":1363.2507936507936,"p99":1272.3674074074074},"payload_data_rate_gbps_at_latency_percentile":{"p50":190.8551111111111,"p90":176.717695473251,"p95":170.4063492063492,"p99":159.04592592592593},"payload_bytes":192381952},"combine":{"latency_us":{"p50":398,"p90":429.84000000000003,"p95":445.76000000000005,"p99":477.59999999999997},"activation_data_rate_gbps_at_latency_percentile":{"p50":966.7434773869347,"p90":895.1328494323469,"p95":863.1638190954774,"p99":805.6195644891122},"payload_data_rate_gbps_at_latency_percentile":{"p50":120.84293467336684,"p90":111.89160617904336,"p95":107.89547738693467,"p99":100.70244556113903},"payload_bytes":384763904},"roundtrip":{"latency_us":{"p50":927,"p90":1001.1600000000001,"p95":1038.24,"p99":1112.3999999999999},"activation_data_rate_gbps_at_latency_percentile":{"p50":830.1270852211435,"p90":768.6361900195773,"p95":741.1848975188781,"p99":691.7725710176197},"payload_data_rate_gbps_at_latency_percentile":{"p50":105.82037540453075,"p90":97.98182907826921,"p95":94.48247803975958,"p99":88.1836461704423},"payload_bytes":784763904}},"roundtrip_token_rate_at_latency_percentile":{"p50":8338218,"p90":9005275.440000001,"p95":9338804.16,"p99":10005861.6}},{"tokens_per_rank":128,"global_tokens":1024,"components":{"dispatch":{"latency_us":{"p50":424,"p90":457.92,"p95":474.88000000000005,"p99":508.79999999999995},"activation_data_rate_gbps_at_latency_percentile":{"p50":907.4620377358491,"p90":840.2426275331936,"p95":810.2339622641508,"p99":756.2183647798744},"payload_data_rate_gbps_at_latency_percentile":{"p50":117.9245283018868,"p90":109.18937805730259,"p95":105.2897574123989,"p99":98.27044025157234},"payload_bytes":400000000},"stage":{"latency_us":{"p50":127,"p90":137.16,"p95":142.24,"p99":152.4},"activation_data_rate_gbps_at_latency_percentile":{"p50":1514.8185196850393,"p90":1402.6097404491106,"p95":1352.5165354330707,"p99":1262.3487664041995},"payload_data_rate_gbps_at_latency_percentile":{"p50":189.3523149606299,"p90":175.32621755613883,"p95":169.06456692913383,"p99":157.79359580052494},"payload_bytes":192381952},"combine":{"latency_us":{"p50":399,"p90":430.92,"p95":446.88000000000005,"p99":478.79999999999995},"activation_data_rate_gbps_at_latency_percentile":{"p50":964.3205614035088,"p90":892.8894087069526,"p95":861.0005012531327,"p99":803.6004678362574},"payload_data_rate_gbps_at_latency_percentile":{"p50":120.5400701754386,"p90":111.61117608836908,"p95":107.62506265664159,"p99":100.45005847953217},"payload_bytes":384763904},"roundtrip":{"latency_us":{"p50":928,"p90":1002.24,"p95":1039.3600000000001,"p99":1113.6},"activation_data_rate_gbps_at_latency_percentile":{"p50":829.232551724138,"p90":767.8079182630906,"p95":740.3862068965517,"p99":691.0271264367817},"payload_data_rate_gbps_at_latency_percentile":{"p50":105.70634482758621,"p90":97.87624521072797,"p95":94.38066502463055,"p99":88.08862068965517},"payload_bytes":784763904}},"roundtrip_token_rate_at_latency_percentile":{"p50":8338218,"p90":9005275.440000001,"p95":9338804.16,"p99":10005861.6}},{"tokens_per_rank":256,"global_tokens":2048,"components":{"dispatch":{"latency_us":{"p50":425,"p90":459.00000000000006,"p95":476.00000000000006,"p99":510},"activation_data_rate_gbps_at_latency_percentile":{"p50":905.3268329411765,"p90":838.2655860566448,"p95":808.3275294117645,"p99":754.4390274509803},"payload_data_rate_gbps_at_latency_percentile":{"p50":117.64705882352942,"p90":108.93246187363833,"p95":105.04201680672269,"p99":98.0392156862745},"payload_bytes":400000000},"stage":{"latency_us":{"p50":128,"p90":138.24,"p95":143.36,"p99":153.6},"activation_data_rate_gbps_at_latency_percentile":{"p50":1502.984,"p90":1391.6518518518517,"p95":1341.9499999999998,"p99":1252.4866666666667},"payload_data_rate_gbps_at_latency_percentile":{"p50":187.873,"p90":173.95648148148146,"p95":167.74374999999998,"p99":156.56083333333333},"payload_bytes":192381952},"combine":{"latency_us":{"p50":400,"p90":432,"p95":448.00000000000006,"p99":480},"activation_data_rate_gbps_at_latency_percentile":{"p50":961.90976,"p90":890.6571851851852,"p95":858.848,"p99":801.5914666666667},"payload_data_rate_gbps_at_latency_percentile":{"p50":120.23872,"p90":111.33214814814815,"p95":107.356,"p99":100.19893333333334},"payload_bytes":384763904},"roundtrip":{"latency_us":{"p50":929,"p90":1003.32,"p95":1040.48,"p99":1114.8},"activation_data_rate_gbps_at_latency_percentile":{"p50":828.3399440258343,"p90":766.9814296535502,"p95":739.5892357373519,"p99":690.2832866881952},"payload_data_rate_gbps_at_latency_percentile":{"p50":105.5925597416577,"p90":97.77088864968306,"p95":94.27907119790866,"p99":87.99379978471475},"payload_bytes":784763904}},"roundtrip_token_rate_at_latency_percentile":{"p50":8338218,"p90":9005275.440000001,"p95":9338804.16,"p99":10005861.6}},{"tokens_per_rank":512,"global_tokens":4096,"components":{"dispatch":{"latency_us":{"p50":426,"p90":460.08000000000004,"p95":477.12000000000006,"p99":511.2},"activation_data_rate_gbps_at_latency_percentile":{"p50":903.2016525821597,"p90":836.2978264649626,"p95":806.4300469483567,"p99":752.6680438184663},"payload_data_rate_gbps_at_latency_percentile":{"p50":117.37089201877934,"p90":108.67675186924012,"p95":104.79543930248154,"p99":97.80907668231613},"payload_bytes":400000000},"stage":{"latency_us":{"p50":129,"p90":139.32000000000002,"p95":144.48000000000002,"p99":154.79999999999998},"activation_data_rate_gbps_at_latency_percentile":{"p50":1491.33296124031,"p90":1380.863853000287,"p95":1331.5472868217053,"p99":1242.7774677002585},"payload_data_rate_gbps_at_latency_percentile":{"p50":186.41662015503874,"p90":172.60798162503588,"p95":166.44341085271316,"p99":155.3471834625323},"payload_bytes":192381952},"combine":{"latency_us":{"p50":401,"p90":433.08000000000004,"p95":449.12000000000006,"p99":481.2},"activation_data_rate_gbps_at_latency_percentile":{"p50":959.5109825436409,"p90":888.4360949478155,"p95":856.706234413965,"p99":799.5924854530341},"payload_data_rate_gbps_at_latency_percentile":{"p50":119.93887281795512,"p90":111.05451186847694,"p95":107.08827930174563,"p99":99.94906068162926},"payload_bytes":384763904},"roundtrip":{"latency_us":{"p50":930,"p90":1004.4000000000001,"p95":1041.6000000000001,"p99":1116},"activation_data_rate_gbps_at_latency_percentile":{"p50":827.4492559139785,"p90":766.1567184388689,"p95":738.7939784946236,"p99":689.541046594982},"payload_data_rate_gbps_at_latency_percentile":{"p50":105.47901935483871,"p90":97.66575866188768,"p95":94.17769585253455,"p99":87.89918279569893},"payload_bytes":784763904}},"roundtrip_token_rate_at_latency_percentile":{"p50":8338218,"p90":9005275.440000001,"p95":9338804.16,"p99":10005861.6}}]}]} diff --git a/packages/app/cypress/fixtures/api/collectivex-runs.json b/packages/app/cypress/fixtures/api/collectivex-runs.json new file mode 100644 index 000000000..efdc80210 --- /dev/null +++ b/packages/app/cypress/fixtures/api/collectivex-runs.json @@ -0,0 +1 @@ +{"version":1,"runs":[{"run_id":"160","run_attempt":1,"generated_at":"2026-07-08T12:20:00Z","conclusion":"success","covered_skus":["b200-dgxc","b300","h200-dgxc","mi355x"],"requested_cases":5,"measured_cases":3,"requested_points":50,"terminal_points":40,"terminal_counts":{"measured":3,"unsupported":1,"failed":0}},{"run_id":"159","run_attempt":1,"generated_at":"2026-07-07T12:20:00Z","conclusion":"success","covered_skus":["h200-dgxc"],"requested_cases":1,"measured_cases":1,"requested_points":10,"terminal_points":10,"terminal_counts":{"measured":1,"unsupported":0,"failed":0}}],"discovery_complete":true} diff --git a/packages/app/scripts/capture-cypress-fixtures.ts b/packages/app/scripts/capture-cypress-fixtures.ts index de3190b3b..e30c042d3 100644 --- a/packages/app/scripts/capture-cypress-fixtures.ts +++ b/packages/app/scripts/capture-cypress-fixtures.ts @@ -8,13 +8,23 @@ * Usage: * bun run --cwd packages/app capture:fixtures (prod) * bun run --cwd packages/app capture:fixtures http://localhost:3000 (local dev) + * bun run --cwd packages/app capture:fixtures -- --collectivex-only (synthetic multi-run data) */ import { mkdir, writeFile } from 'node:fs/promises'; import { resolve } from 'node:path'; +import { buildRunSummary } from '@semianalysisai/inferencex-db/collectivex/reader'; +import { + buildDataset, + makeCollectiveXDataset, + makeRawShard, +} from '@semianalysisai/inferencex-db/collectivex/test-fixture'; + +const cliArgs = process.argv.filter((argument) => argument !== '--').slice(2); +const collectiveXOnly = cliArgs.includes('--collectivex-only'); const baseUrl = ( - process.argv.filter((a) => a !== '--').slice(2)[0] ?? 'https://inferencex.semianalysis.com' + cliArgs.find((argument) => !argument.startsWith('--')) ?? 'https://inferencex.semianalysis.com' ).replace(/\/$/u, ''); const fixturesDir = resolve(import.meta.dirname, '..', 'cypress', 'fixtures', 'api'); @@ -121,16 +131,62 @@ function sampleAlongAxis( } async function writeFixture(name: string, data: unknown): Promise { - // Pretty-print: matches oxfmt's output so re-running capture doesn't dirty - // the working tree on the formatter pass. - const body = `${JSON.stringify(data, null, 2)}\n`; + // Large, machine-shaped API payloads stay minified; every other fixture is + // pretty-printed to match oxfmt. This mirrors .prettierignore and makes + // repeated capture runs byte-for-byte stable. + const minified = name.startsWith('collectivex-'); + const body = `${JSON.stringify(data, null, minified ? undefined : 2)}\n`; await writeFile(resolve(fixturesDir, `${name}.json`), body); return body.length; } +async function writeCollectiveXFixtures(): Promise<[string, number][]> { + const latest = makeCollectiveXDataset(); + const comparison = buildDataset({ + shards: [makeRawShard({ precision: 'fp8' })], + meta: { + run_id: '159', + generated_at: '2026-07-07T12:20:00Z', + source_sha: 'd'.repeat(40), + }, + }); + const datasets = [latest, comparison]; + const sizes: [string, number][] = [ + ['collectivex-latest', await writeFixture('collectivex-latest', latest)], + ]; + // The newest dataset already has the stable `collectivex-latest` fixture; + // older runs get id-keyed files for multi-run selection. + for (const dataset of datasets.slice(1)) { + const name = `collectivex-run-${dataset.run.run_id}`; + sizes.push([name, await writeFixture(name, dataset)]); + } + sizes.push([ + 'collectivex-runs', + await writeFixture('collectivex-runs', { + version: 1, + runs: datasets.map(buildRunSummary), + discovery_complete: true, + }), + ]); + return sizes; +} + +function printSizes(sizes: [string, number][]) { + for (const [name, bytes] of sizes) { + console.log(` ${name.padEnd(22)} ${(bytes / 1024).toFixed(1).padStart(8)} KB`); + } + console.log(`\nWrote ${sizes.length} fixtures to ${fixturesDir}`); +} + async function main() { - console.log(`Capturing fixtures from ${baseUrl}`); await mkdir(fixturesDir, { recursive: true }); + if (collectiveXOnly) { + console.log('Generating synthetic CollectiveX fixtures'); + printSizes(await writeCollectiveXFixtures()); + return; + } + + console.log(`Capturing fixtures from ${baseUrl}`); const latestDate = await fetchLatestDate(); console.log( @@ -186,6 +242,7 @@ async function main() { ); const N = TOP_DATES_PER_PARTITION; + const collectiveXSizes = await writeCollectiveXFixtures(); const sizes: [string, number][] = [ [ 'availability', @@ -247,12 +304,12 @@ async function main() { }), ], ['workflow-info', await writeFixture('workflow-info', workflowInfo)], + // Synthetic deterministic data: production may hold arbitrary sweeps, + // while e2e asserts on the builders' known multi-run shape. + ...collectiveXSizes, ]; - for (const [name, bytes] of sizes) { - console.log(` ${name.padEnd(22)} ${(bytes / 1024).toFixed(1).padStart(8)} KB`); - } - console.log(`\nWrote ${sizes.length} fixtures to ${fixturesDir}`); + printSizes(sizes); } main().catch((error) => { diff --git a/packages/app/src/app/(dashboard)/collectivex/page.tsx b/packages/app/src/app/(dashboard)/collectivex/page.tsx new file mode 100644 index 000000000..d3380bd98 --- /dev/null +++ b/packages/app/src/app/(dashboard)/collectivex/page.tsx @@ -0,0 +1,10 @@ +import type { Metadata } from 'next'; + +import CollectiveXDisplay from '@/components/collectivex/CollectiveXDisplay'; +import { tabMetadata } from '@/lib/tab-meta'; + +export const metadata: Metadata = tabMetadata('collectivex'); + +export default function CollectiveXPage() { + return ; +} diff --git a/packages/app/src/app/api/v1/collectivex/latest/route.test.ts b/packages/app/src/app/api/v1/collectivex/latest/route.test.ts new file mode 100644 index 000000000..da3ee6929 --- /dev/null +++ b/packages/app/src/app/api/v1/collectivex/latest/route.test.ts @@ -0,0 +1,105 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { makeCollectiveXDataset } from '@semianalysisai/inferencex-db/collectivex/test-fixture'; + +const { mockGetLatest, mockFromRow, mockGetDb, mockEnsureLatest } = vi.hoisted(() => ({ + mockGetLatest: vi.fn(), + mockFromRow: vi.fn(), + mockGetDb: vi.fn(() => 'mock-sql'), + mockEnsureLatest: vi.fn(), +})); + +vi.mock('@semianalysisai/inferencex-db/connection', () => ({ + getCollectiveXDb: mockGetDb, + FIXTURES_MODE: false, +})); + +vi.mock('@semianalysisai/inferencex-db/queries/collectivex', () => ({ + getLatestCollectiveXRun: mockGetLatest, + collectiveXDatasetFromRow: mockFromRow, +})); + +vi.mock('@/lib/collectivex-lazy-ingest', () => ({ + ensureLatestCollectiveXRun: mockEnsureLatest, + collectiveXSweepErrorStatus: (error: unknown) => { + const code = error instanceof Error && 'code' in error ? (error.code as string) : null; + if (code === 'not-found') return 404; + if (code === 'unavailable') return 503; + if (code === 'invalid') return 502; + return null; + }, +})); + +vi.mock('@/lib/api-cache', () => ({ + COLLECTIVEX_CACHE_SCOPE: 'collectivex', + COLLECTIVEX_CACHE_CONTROL: 'public, max-age=0, s-maxage=60', + cachedJson: (data: unknown) => Response.json(data), + collectiveXCacheTag: () => 'collectivex', +})); + +import { NextRequest } from 'next/server'; + +import { GET } from './route'; + +function req(url: string): NextRequest { + return new NextRequest(new URL(url, 'http://localhost')); +} + +function sweepError(code: string): Error { + return Object.assign(new Error(code), { code }); +} + +const dataset = makeCollectiveXDataset(); + +beforeEach(() => { + vi.clearAllMocks(); + vi.spyOn(console, 'error').mockImplementation(() => {}); + mockEnsureLatest.mockResolvedValue(undefined); + mockGetLatest.mockResolvedValue({ run_id: dataset.run.run_id }); + mockFromRow.mockReturnValue(dataset); +}); + +describe('GET /api/v1/collectivex/latest', () => { + it('returns 400 for a missing or unknown version', async () => { + for (const url of ['/api/v1/collectivex/latest', '/api/v1/collectivex/latest?version=99']) { + const res = await GET(req(url)); + expect(res.status).toBe(400); + } + expect(mockEnsureLatest).not.toHaveBeenCalled(); + }); + + it('lazily ingests then serves the latest run assembled as a dataset', async () => { + const res = await GET(req('/api/v1/collectivex/latest?version=1')); + expect(res.status).toBe(200); + expect(await res.json()).toEqual(dataset); + expect(mockEnsureLatest).toHaveBeenCalledWith(1); + expect(mockGetLatest).toHaveBeenCalledWith('mock-sql', 1); + }); + + it('serves the stored run when GitHub discovery fails', async () => { + mockEnsureLatest.mockRejectedValue(sweepError('unavailable')); + const res = await GET(req('/api/v1/collectivex/latest?version=1')); + expect(res.status).toBe(200); + expect(await res.json()).toEqual(dataset); + }); + + it('returns 503 when discovery fails and nothing is stored', async () => { + mockEnsureLatest.mockRejectedValue(sweepError('unavailable')); + mockGetLatest.mockResolvedValue(null); + const res = await GET(req('/api/v1/collectivex/latest?version=1')); + expect(res.status).toBe(503); + }); + + it('returns 404 when neither GitHub nor the DB has a run', async () => { + mockGetLatest.mockResolvedValue(null); + const res = await GET(req('/api/v1/collectivex/latest?version=1')); + expect(res.status).toBe(404); + }); + + it('returns 500 without leaking details on DB failure', async () => { + mockGetLatest.mockRejectedValue(new Error('boom')); + const res = await GET(req('/api/v1/collectivex/latest?version=1')); + expect(res.status).toBe(500); + expect(await res.json()).toEqual({ error: 'Internal server error' }); + }); +}); diff --git a/packages/app/src/app/api/v1/collectivex/latest/route.ts b/packages/app/src/app/api/v1/collectivex/latest/route.ts new file mode 100644 index 000000000..264b5a842 --- /dev/null +++ b/packages/app/src/app/api/v1/collectivex/latest/route.ts @@ -0,0 +1,56 @@ +import { type NextRequest, NextResponse } from 'next/server'; + +import { parseCollectiveXVersion } from '@semianalysisai/inferencex-db/collectivex/types'; +import { FIXTURES_MODE, getCollectiveXDb } from '@semianalysisai/inferencex-db/connection'; +import { + collectiveXDatasetFromRow, + getLatestCollectiveXRun, +} from '@semianalysisai/inferencex-db/queries/collectivex'; + +import { COLLECTIVEX_CACHE_CONTROL, cachedJson, collectiveXCacheTag } from '@/lib/api-cache'; +import { + collectiveXSweepErrorStatus, + ensureLatestCollectiveXRun, +} from '@/lib/collectivex-lazy-ingest'; +import { loadFixture } from '@/lib/test-fixtures'; + +export const dynamic = 'force-dynamic'; +export const runtime = 'nodejs'; + +export async function GET(request: NextRequest) { + const version = parseCollectiveXVersion(request.nextUrl.searchParams.get('version') ?? ''); + if (!version) { + return NextResponse.json({ error: 'Unknown version' }, { status: 400 }); + } + if (FIXTURES_MODE) return cachedJson(loadFixture('collectivex-latest')); + + // Discovery failures must not take the page down — serve whatever the DB + // already holds and only surface the error when there is no fallback. + let ensureError: unknown = null; + try { + await ensureLatestCollectiveXRun(version); + } catch (error) { + ensureError = error; + } + + try { + const row = await getLatestCollectiveXRun(getCollectiveXDb(), version); + if (row !== null) { + if (ensureError) + console.error('CollectiveX discovery failed; serving stored run:', ensureError); + return cachedJson(collectiveXDatasetFromRow(row), { + tag: collectiveXCacheTag(), + cacheControl: COLLECTIVEX_CACHE_CONTROL, + }); + } + if (ensureError) { + console.error('CollectiveX discovery failed with no stored fallback:', ensureError); + const status = collectiveXSweepErrorStatus(ensureError) ?? 502; + return NextResponse.json({ error: 'Unavailable' }, { status }); + } + return NextResponse.json({ error: 'Not found' }, { status: 404 }); + } catch (error) { + console.error('Error fetching CollectiveX latest run:', error); + return NextResponse.json({ error: 'Internal server error' }, { status: 500 }); + } +} diff --git a/packages/app/src/app/api/v1/collectivex/runs/[runId]/route.fixtures.test.ts b/packages/app/src/app/api/v1/collectivex/runs/[runId]/route.fixtures.test.ts new file mode 100644 index 000000000..4c2b1b5ce --- /dev/null +++ b/packages/app/src/app/api/v1/collectivex/runs/[runId]/route.fixtures.test.ts @@ -0,0 +1,92 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { + buildDataset, + makeCollectiveXDataset, +} from '@semianalysisai/inferencex-db/collectivex/test-fixture'; +import { buildRunSummary } from '@semianalysisai/inferencex-db/collectivex/reader'; + +const { mockLoadFixture } = vi.hoisted(() => ({ + mockLoadFixture: vi.fn(), +})); + +vi.mock('@semianalysisai/inferencex-db/connection', () => ({ + FIXTURES_MODE: true, + getCollectiveXDb: vi.fn(), + getCollectiveXWriteDb: vi.fn(), +})); + +vi.mock('@semianalysisai/inferencex-db/queries/collectivex', () => ({ + collectiveXDatasetFromRow: vi.fn(), + deleteCollectiveXRun: vi.fn(), + getCollectiveXRun: vi.fn(), +})); + +vi.mock('@/lib/collectivex-lazy-ingest', () => ({ + collectiveXSweepErrorStatus: vi.fn(), + ensureCollectiveXRun: vi.fn(), +})); + +vi.mock('@/lib/api-cache', () => ({ + COLLECTIVEX_CACHE_CONTROL: 'public, max-age=0, s-maxage=60', + cachedJson: (data: unknown) => Response.json(data), + collectiveXCacheTag: () => 'collectivex', + purgeCollectiveX: vi.fn(), +})); + +vi.mock('@/lib/test-fixtures', () => ({ + loadFixture: mockLoadFixture, +})); + +import { NextRequest } from 'next/server'; + +import { GET } from './route'; + +const latest = makeCollectiveXDataset(); +const comparison = buildDataset({ meta: { run_id: '159' } }); +const fixtureList = { + version: 1, + runs: [buildRunSummary(latest), buildRunSummary(comparison)], +}; + +function get(runId: string) { + return GET(new NextRequest(new URL(`/x?version=1`, 'http://localhost')), { + params: Promise.resolve({ runId }), + }); +} + +beforeEach(() => { + vi.clearAllMocks(); + mockLoadFixture.mockImplementation((name: string) => { + if (name === 'collectivex-runs') return fixtureList; + if (name === 'collectivex-latest') return latest; + if (name === `collectivex-run-${comparison.run.run_id}`) return comparison; + throw new Error(`Unexpected fixture: ${name}`); + }); +}); + +describe('GET /api/v1/collectivex/runs/[runId] in fixture mode', () => { + it('reuses the latest fixture when its run id was requested', async () => { + const response = await get(latest.run.run_id); + + expect(response.status).toBe(200); + expect(await response.json()).toEqual(latest); + expect(mockLoadFixture).toHaveBeenCalledWith('collectivex-latest'); + }); + + it('serves an older dataset from its run-id-keyed fixture', async () => { + const response = await get(comparison.run.run_id); + + expect(response.status).toBe(200); + expect(await response.json()).toEqual(comparison); + expect(mockLoadFixture).toHaveBeenCalledWith(`collectivex-run-${comparison.run.run_id}`); + }); + + it('returns 404 instead of substituting the latest fixture for an unknown run', async () => { + const response = await get('999'); + + expect(response.status).toBe(404); + expect(mockLoadFixture).toHaveBeenCalledTimes(1); + expect(mockLoadFixture).toHaveBeenCalledWith('collectivex-runs'); + }); +}); diff --git a/packages/app/src/app/api/v1/collectivex/runs/[runId]/route.test.ts b/packages/app/src/app/api/v1/collectivex/runs/[runId]/route.test.ts new file mode 100644 index 000000000..c2f2c51ad --- /dev/null +++ b/packages/app/src/app/api/v1/collectivex/runs/[runId]/route.test.ts @@ -0,0 +1,170 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { makeCollectiveXDataset } from '@semianalysisai/inferencex-db/collectivex/test-fixture'; + +const { mockGetRun, mockDelete, mockFromRow, mockGetDb, mockGetWriteDb, mockPurge, mockEnsureRun } = + vi.hoisted(() => ({ + mockGetRun: vi.fn(), + mockDelete: vi.fn(), + mockFromRow: vi.fn(), + mockGetDb: vi.fn(() => 'mock-sql'), + mockGetWriteDb: vi.fn(() => 'mock-write-sql'), + mockPurge: vi.fn(() => Promise.resolve(0)), + mockEnsureRun: vi.fn(), + })); + +vi.mock('@semianalysisai/inferencex-db/connection', () => ({ + getCollectiveXDb: mockGetDb, + getCollectiveXWriteDb: mockGetWriteDb, + FIXTURES_MODE: false, +})); + +vi.mock('@semianalysisai/inferencex-db/queries/collectivex', () => ({ + getCollectiveXRun: mockGetRun, + deleteCollectiveXRun: mockDelete, + collectiveXDatasetFromRow: mockFromRow, +})); + +vi.mock('@/lib/collectivex-lazy-ingest', () => ({ + ensureCollectiveXRun: mockEnsureRun, + collectiveXSweepErrorStatus: (error: unknown) => { + const code = error instanceof Error && 'code' in error ? (error.code as string) : null; + if (code === 'not-found') return 404; + if (code === 'unavailable') return 503; + if (code === 'invalid') return 502; + return null; + }, +})); + +vi.mock('@/lib/api-cache', () => ({ + COLLECTIVEX_CACHE_SCOPE: 'collectivex', + COLLECTIVEX_CACHE_CONTROL: 'public, max-age=0, s-maxage=60', + cachedJson: (data: unknown) => Response.json(data), + collectiveXCacheTag: () => 'collectivex', + purgeCollectiveX: mockPurge, +})); + +import { NextRequest } from 'next/server'; + +import { DELETE, GET } from './route'; + +const SECRET = 'test-admin-secret'; +const dataset = makeCollectiveXDataset(); +const runId = dataset.run.run_id; + +function sweepError(code: string): Error { + return Object.assign(new Error(code), { code }); +} + +function get(url: string, id: string) { + return GET(new NextRequest(new URL(url, 'http://localhost')), { + params: Promise.resolve({ runId: id }), + }); +} + +function del(id: string, token?: string) { + return DELETE( + new NextRequest(new URL(`/api/v1/collectivex/runs/${id}`, 'http://localhost'), { + method: 'DELETE', + headers: token ? { Authorization: `Bearer ${token}` } : {}, + }), + { params: Promise.resolve({ runId: id }) }, + ); +} + +beforeEach(() => { + vi.clearAllMocks(); + vi.stubEnv('COLLECTIVEX_ADMIN_SECRET', SECRET); + mockEnsureRun.mockResolvedValue(undefined); + mockGetRun.mockResolvedValue({ run_id: runId }); + mockFromRow.mockReturnValue(dataset); + mockDelete.mockResolvedValue(true); +}); + +afterEach(() => { + vi.unstubAllEnvs(); +}); + +describe('GET /api/v1/collectivex/runs/[runId]', () => { + it('returns 400 for malformed version or run id', async () => { + const badRunId = await get(`/x?version=1`, 'abc'); + const badVersion = await get(`/x?version=99`, runId); + expect(badRunId.status).toBe(400); + expect(badVersion.status).toBe(400); + expect(mockEnsureRun).not.toHaveBeenCalled(); + }); + + it('lazily ingests then serves the run assembled as a dataset', async () => { + const res = await get(`/x?version=1`, runId); + expect(res.status).toBe(200); + expect(await res.json()).toEqual(dataset); + expect(mockEnsureRun).toHaveBeenCalledWith(1, runId); + expect(mockGetRun).toHaveBeenCalledWith('mock-sql', 1, runId); + }); + + it.each([ + ['not-found', 404], + ['unavailable', 503], + ['invalid', 502], + ] as const)('maps %s ingest failures without exposing details', async (code, status) => { + vi.spyOn(console, 'error').mockImplementation(() => {}); + mockEnsureRun.mockRejectedValue(sweepError(code)); + mockGetRun.mockResolvedValue(null); + const res = await get(`/x?version=1`, runId); + expect(res.status).toBe(status); + }); + + it('serves a stored run when its GitHub refresh fails', async () => { + vi.spyOn(console, 'error').mockImplementation(() => {}); + mockEnsureRun.mockRejectedValue(sweepError('unavailable')); + + const res = await get(`/x?version=1`, runId); + + expect(res.status).toBe(200); + expect(await res.json()).toEqual(dataset); + }); + + it('returns 404 when the run is absent after a clean ensure', async () => { + mockGetRun.mockResolvedValue(null); + const res = await get(`/x?version=1`, runId); + expect(res.status).toBe(404); + }); +}); + +describe('DELETE /api/v1/collectivex/runs/[runId]', () => { + it('rejects missing or wrong bearer tokens', async () => { + const missing = await del(runId); + const wrong = await del(runId, 'wrong-token'); + expect(missing.status).toBe(401); + expect(wrong.status).toBe(401); + expect(mockDelete).not.toHaveBeenCalled(); + }); + + it('rejects all requests when the secret is not configured', async () => { + vi.stubEnv('COLLECTIVEX_ADMIN_SECRET', ''); + const res = await del(runId, ''); + expect(res.status).toBe(401); + expect(mockDelete).not.toHaveBeenCalled(); + }); + + it('tombstones the run and purges only the CollectiveX cache scope', async () => { + const res = await del(runId, SECRET); + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ deleted: true, runId }); + expect(mockDelete).toHaveBeenCalledWith('mock-write-sql', runId); + expect(mockPurge).toHaveBeenCalledTimes(1); + }); + + it('returns 404 without purging when the run does not exist', async () => { + mockDelete.mockResolvedValue(false); + const res = await del(runId, SECRET); + expect(res.status).toBe(404); + expect(mockPurge).not.toHaveBeenCalled(); + }); + + it('returns 400 for malformed run ids', async () => { + const res = await del('not-a-run-id', SECRET); + expect(res.status).toBe(400); + expect(mockDelete).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/app/src/app/api/v1/collectivex/runs/[runId]/route.ts b/packages/app/src/app/api/v1/collectivex/runs/[runId]/route.ts new file mode 100644 index 000000000..e8150c1fb --- /dev/null +++ b/packages/app/src/app/api/v1/collectivex/runs/[runId]/route.ts @@ -0,0 +1,133 @@ +import { timingSafeEqual } from 'crypto'; + +import { type NextRequest, NextResponse } from 'next/server'; + +import { + type CollectiveXDataset, + type CollectiveXRunSummary, + parseCollectiveXVersion, +} from '@semianalysisai/inferencex-db/collectivex/types'; +import { + FIXTURES_MODE, + getCollectiveXDb, + getCollectiveXWriteDb, +} from '@semianalysisai/inferencex-db/connection'; +import { + collectiveXDatasetFromRow, + deleteCollectiveXRun, + getCollectiveXRun, +} from '@semianalysisai/inferencex-db/queries/collectivex'; + +import { + COLLECTIVEX_CACHE_CONTROL, + collectiveXCacheTag, + cachedJson, + purgeCollectiveX, +} from '@/lib/api-cache'; +import { collectiveXSweepErrorStatus, ensureCollectiveXRun } from '@/lib/collectivex-lazy-ingest'; +import { loadFixture } from '@/lib/test-fixtures'; + +export const dynamic = 'force-dynamic'; +export const runtime = 'nodejs'; + +const RUN_ID = /^[1-9][0-9]*$/u; + +export async function GET(request: NextRequest, context: { params: Promise<{ runId: string }> }) { + const { runId } = await context.params; + const version = parseCollectiveXVersion(request.nextUrl.searchParams.get('version') ?? ''); + if (!version || !RUN_ID.test(runId)) { + return NextResponse.json({ error: 'Unknown version or run id' }, { status: 400 }); + } + if (FIXTURES_MODE) { + const fixtureList = loadFixture<{ version: number; runs: CollectiveXRunSummary[] }>( + 'collectivex-runs', + ); + if (fixtureList.version !== version || !fixtureList.runs.some((run) => run.run_id === runId)) { + return NextResponse.json({ error: 'Not found' }, { status: 404 }); + } + const latest = loadFixture('collectivex-latest'); + if (latest.run.run_id === runId) return cachedJson(latest); + return cachedJson(loadFixture(`collectivex-run-${runId}`)); + } + + let ensureError: unknown = null; + try { + await ensureCollectiveXRun(version, runId); + } catch (error) { + ensureError = error; + } + + try { + const row = await getCollectiveXRun(getCollectiveXDb(), version, runId); + if (row === null) { + if (ensureError) { + const status = collectiveXSweepErrorStatus(ensureError); + if (status !== null) { + return NextResponse.json( + { error: status === 404 ? 'Not found' : 'Unavailable' }, + { status }, + ); + } + throw ensureError; + } + return NextResponse.json({ error: 'Not found' }, { status: 404 }); + } + if (ensureError) { + console.error('CollectiveX run refresh failed; serving stored run:', ensureError); + } + // Short window like the sibling routes: a GitHub re-run of failed shards + // refreshes this run's stored contents, and deletion must not linger. + return cachedJson(collectiveXDatasetFromRow(row), { + tag: collectiveXCacheTag(), + cacheControl: COLLECTIVEX_CACHE_CONTROL, + }); + } catch (error) { + console.error('Error fetching CollectiveX run:', error); + return NextResponse.json({ error: 'Internal server error' }, { status: 500 }); + } +} + +/** Constant-time Bearer check that tolerates multibyte header bytes. */ +function bearerMatches(header: string, secret: string): boolean { + const provided = Buffer.from(header); + const expected = Buffer.from(`Bearer ${secret}`); + // Compare BYTE lengths — a multibyte char can make the JS string lengths + // equal while the buffers differ, and timingSafeEqual throws on that. + return provided.length === expected.length && timingSafeEqual(provided, expected); +} + +/** + * Admin deletion of an ingested run. Authenticated with a dedicated Bearer + * secret — the token is remembered in browser localStorage, so it must not + * be the CI-held INVALIDATE_SECRET (scoped blast radius, independently + * rotatable). Deletion tombstones the run (lazy discovery must never + * re-ingest it) and purges the CollectiveX cache scope so the run table and + * latest views drop it immediately. + */ +export async function DELETE( + request: NextRequest, + context: { params: Promise<{ runId: string }> }, +) { + const secret = process.env.COLLECTIVEX_ADMIN_SECRET; + const authHeader = request.headers.get('Authorization') ?? ''; + if (!secret || !bearerMatches(authHeader, secret)) { + return new NextResponse('Unauthorized', { status: 401 }); + } + + const { runId } = await context.params; + if (!RUN_ID.test(runId)) { + return NextResponse.json({ error: 'Unknown run id' }, { status: 400 }); + } + + try { + const deleted = await deleteCollectiveXRun(getCollectiveXWriteDb(), runId); + if (!deleted) { + return NextResponse.json({ error: 'Not found' }, { status: 404 }); + } + purgeCollectiveX(); + return NextResponse.json({ deleted: true, runId }); + } catch (error) { + console.error('Error deleting CollectiveX run:', error); + return NextResponse.json({ error: 'Internal server error' }, { status: 500 }); + } +} diff --git a/packages/app/src/app/api/v1/collectivex/runs/route.test.ts b/packages/app/src/app/api/v1/collectivex/runs/route.test.ts new file mode 100644 index 000000000..658d7b178 --- /dev/null +++ b/packages/app/src/app/api/v1/collectivex/runs/route.test.ts @@ -0,0 +1,133 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { buildRunSummary } from '@semianalysisai/inferencex-db/collectivex/reader'; +import { makeCollectiveXDataset } from '@semianalysisai/inferencex-db/collectivex/test-fixture'; + +const { mockList, mockGetDb, mockEnsureList, mockCachedJson } = vi.hoisted(() => ({ + mockList: vi.fn(), + mockGetDb: vi.fn(() => 'mock-sql'), + mockEnsureList: vi.fn(), + mockCachedJson: vi.fn((data: unknown) => Response.json(data)), +})); + +vi.mock('@semianalysisai/inferencex-db/connection', () => ({ + getCollectiveXDb: mockGetDb, + FIXTURES_MODE: false, +})); + +vi.mock('@semianalysisai/inferencex-db/queries/collectivex', () => ({ + listCollectiveXRuns: mockList, +})); + +vi.mock('@/lib/collectivex-lazy-ingest', () => ({ + ensureCollectiveXRunsList: mockEnsureList, + collectiveXSweepErrorStatus: (error: unknown) => { + const code = error instanceof Error && 'code' in error ? (error.code as string) : null; + if (code === 'not-found') return 404; + if (code === 'unavailable') return 503; + if (code === 'invalid') return 502; + return null; + }, +})); + +vi.mock('@/lib/api-cache', () => ({ + COLLECTIVEX_CACHE_SCOPE: 'collectivex', + COLLECTIVEX_CACHE_CONTROL: 'public, max-age=0, s-maxage=60', + cachedJson: mockCachedJson, + collectiveXCacheTag: () => 'collectivex', +})); + +import { NextRequest } from 'next/server'; + +import { GET } from './route'; + +function req(url: string): NextRequest { + return new NextRequest(new URL(url, 'http://localhost')); +} + +function sweepError(code: string): Error { + return Object.assign(new Error(code), { code }); +} + +const summary = buildRunSummary(makeCollectiveXDataset()); + +beforeEach(() => { + vi.clearAllMocks(); + vi.spyOn(console, 'error').mockImplementation(() => {}); + mockEnsureList.mockResolvedValue(true); + mockList.mockResolvedValue([summary]); +}); + +describe('GET /api/v1/collectivex/runs', () => { + it('returns 400 for a missing or unknown version', async () => { + for (const url of ['/api/v1/collectivex/runs', '/api/v1/collectivex/runs?version=abc']) { + const res = await GET(req(url)); + expect(res.status).toBe(400); + } + expect(mockEnsureList).not.toHaveBeenCalled(); + }); + + it('backfills recent runs then lists stored summaries newest first', async () => { + const res = await GET(req('/api/v1/collectivex/runs?version=1')); + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ + version: 1, + runs: [summary], + discovery_complete: true, + }); + expect(mockEnsureList).toHaveBeenCalledWith(1); + expect(mockList).toHaveBeenCalledWith('mock-sql', 1); + }); + + it('marks the list incomplete when another bounded discovery pass is needed', async () => { + mockEnsureList.mockResolvedValue(false); + + const res = await GET(req('/api/v1/collectivex/runs?version=1')); + + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ + version: 1, + runs: [summary], + discovery_complete: false, + }); + expect(mockCachedJson).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ cacheControl: 'private, no-store' }), + ); + }); + + it('serves the stored list as incomplete when the GitHub backfill fails', async () => { + mockEnsureList.mockRejectedValue(sweepError('unavailable')); + const res = await GET(req('/api/v1/collectivex/runs?version=1')); + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ + version: 1, + runs: [summary], + discovery_complete: false, + }); + expect(mockCachedJson).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ cacheControl: 'private, no-store' }), + ); + }); + + it('returns 503 when the backfill fails and nothing is stored', async () => { + mockEnsureList.mockRejectedValue(sweepError('unavailable')); + mockList.mockResolvedValue([]); + const res = await GET(req('/api/v1/collectivex/runs?version=1')); + expect(res.status).toBe(503); + }); + + it('returns an empty list when GitHub has no runs yet', async () => { + mockList.mockResolvedValue([]); + const res = await GET(req('/api/v1/collectivex/runs?version=1')); + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ version: 1, runs: [], discovery_complete: true }); + }); + + it('returns 500 without leaking details on DB failure', async () => { + mockList.mockRejectedValue(new Error('boom')); + const res = await GET(req('/api/v1/collectivex/runs?version=1')); + expect(res.status).toBe(500); + }); +}); diff --git a/packages/app/src/app/api/v1/collectivex/runs/route.ts b/packages/app/src/app/api/v1/collectivex/runs/route.ts new file mode 100644 index 000000000..fce3f2a2d --- /dev/null +++ b/packages/app/src/app/api/v1/collectivex/runs/route.ts @@ -0,0 +1,58 @@ +import { type NextRequest, NextResponse } from 'next/server'; + +import { parseCollectiveXVersion } from '@semianalysisai/inferencex-db/collectivex/types'; +import { FIXTURES_MODE, getCollectiveXDb } from '@semianalysisai/inferencex-db/connection'; +import { listCollectiveXRuns } from '@semianalysisai/inferencex-db/queries/collectivex'; + +import { COLLECTIVEX_CACHE_CONTROL, cachedJson, collectiveXCacheTag } from '@/lib/api-cache'; +import { + collectiveXSweepErrorStatus, + ensureCollectiveXRunsList, +} from '@/lib/collectivex-lazy-ingest'; +import { loadFixture } from '@/lib/test-fixtures'; + +export const dynamic = 'force-dynamic'; +export const runtime = 'nodejs'; + +const DISCOVERY_CACHE_CONTROL = 'private, no-store'; + +export async function GET(request: NextRequest) { + const version = parseCollectiveXVersion(request.nextUrl.searchParams.get('version') ?? ''); + if (!version) { + return NextResponse.json({ error: 'Unknown version' }, { status: 400 }); + } + if (FIXTURES_MODE) return cachedJson(loadFixture('collectivex-runs')); + + // Backfill failures must not take the run table down — serve the stored list + // and only surface the error when there is nothing at all to show. + let ensureError: unknown = null; + let discoveryComplete: boolean; + try { + discoveryComplete = await ensureCollectiveXRunsList(version); + } catch (error) { + ensureError = error; + discoveryComplete = false; + } + + try { + const runs = await listCollectiveXRuns(getCollectiveXDb(), version); + if (runs.length === 0 && ensureError) { + console.error('CollectiveX run backfill failed with no stored fallback:', ensureError); + const status = collectiveXSweepErrorStatus(ensureError) ?? 502; + return NextResponse.json({ error: 'Unavailable' }, { status }); + } + if (ensureError) { + console.error('CollectiveX run backfill failed; serving stored list:', ensureError); + } + return cachedJson( + { version, runs, discovery_complete: discoveryComplete }, + { + tag: collectiveXCacheTag(), + cacheControl: discoveryComplete ? COLLECTIVEX_CACHE_CONTROL : DISCOVERY_CACHE_CONTROL, + }, + ); + } catch (error) { + console.error('Error listing CollectiveX runs:', error); + return NextResponse.json({ error: 'Internal server error' }, { status: 500 }); + } +} diff --git a/packages/app/src/app/api/v1/invalidate/route.test.ts b/packages/app/src/app/api/v1/invalidate/route.test.ts index 4947107a3..6d25d6f22 100644 --- a/packages/app/src/app/api/v1/invalidate/route.test.ts +++ b/packages/app/src/app/api/v1/invalidate/route.test.ts @@ -1,11 +1,14 @@ import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest'; -const { mockPurgeAll } = vi.hoisted(() => ({ +const { mockPurgeAll, mockPurgeCollectiveX } = vi.hoisted(() => ({ mockPurgeAll: vi.fn(), + mockPurgeCollectiveX: vi.fn(), })); vi.mock('@/lib/api-cache', () => ({ + COLLECTIVEX_CACHE_SCOPE: 'collectivex', purgeAll: mockPurgeAll, + purgeCollectiveX: mockPurgeCollectiveX, })); import { POST } from './route'; @@ -42,8 +45,8 @@ afterEach(() => { } }); -function postReq(headers?: Record): Request { - return new Request('http://localhost/api/v1/invalidate', { +function postReq(headers?: Record, query = ''): Request { + return new Request(`http://localhost/api/v1/invalidate${query}`, { method: 'POST', headers: headers ?? {}, }); @@ -124,4 +127,22 @@ describe('POST /api/v1/invalidate', () => { const body = await res.json(); expect(body).toEqual({ invalidated: true, blobsDeleted: 0 }); }); + + it('purges only the CollectiveX scope when requested', async () => { + const res = await POST( + postReq({ Authorization: 'Bearer test-secret-123' }, '?scope=collectivex'), + ); + expect(res.status).toBe(200); + const body = await res.json(); + expect(body).toEqual({ invalidated: true, scope: 'collectivex' }); + expect(mockPurgeCollectiveX).toHaveBeenCalledOnce(); + expect(mockPurgeAll).not.toHaveBeenCalled(); + }); + + it('rejects unknown scopes without purging anything', async () => { + const res = await POST(postReq({ Authorization: 'Bearer test-secret-123' }, '?scope=nope')); + expect(res.status).toBe(400); + expect(mockPurgeAll).not.toHaveBeenCalled(); + expect(mockPurgeCollectiveX).not.toHaveBeenCalled(); + }); }); diff --git a/packages/app/src/app/api/v1/invalidate/route.ts b/packages/app/src/app/api/v1/invalidate/route.ts index 0fc8cd122..e648e75b1 100644 --- a/packages/app/src/app/api/v1/invalidate/route.ts +++ b/packages/app/src/app/api/v1/invalidate/route.ts @@ -2,12 +2,13 @@ import { timingSafeEqual } from 'crypto'; import { NextResponse } from 'next/server'; -import { purgeAll } from '@/lib/api-cache'; +import { COLLECTIVEX_CACHE_SCOPE, purgeAll, purgeCollectiveX } from '@/lib/api-cache'; export async function POST(request: Request) { const secret = process.env.INVALIDATE_SECRET; const authHeader = request.headers.get('Authorization') ?? ''; - const expected = `Bearer ${secret}`; + const provided = Buffer.from(authHeader); + const expected = Buffer.from(`Bearer ${secret}`); // The shared staging deployment is already protected by Vercel. Its CI // caller must present the project-scoped automation bypass before Vercel // forwards this header to the route, so a second app secret is redundant. @@ -18,16 +19,26 @@ export async function POST(request: Request) { process.env.VERCEL_GIT_COMMIT_REF === 'staging' && Boolean(request.headers.get('x-vercel-protection-bypass')); + // Compare BYTE lengths — a multibyte char can make the JS string lengths + // equal while the buffers differ, and timingSafeEqual throws on that. if ( !isProtectedStagingRequest && - (!secret || - authHeader.length !== expected.length || - !timingSafeEqual(Buffer.from(authHeader), Buffer.from(expected))) + (!secret || provided.length !== expected.length || !timingSafeEqual(provided, expected)) ) { return new NextResponse('Unauthorized', { status: 401 }); } - const blobsDeleted = await purgeAll(); + // ?scope=collectivex purges only the CollectiveX cache scope; the default + // remains a full purge (which covers CollectiveX too). + const scope = new URL(request.url).searchParams.get('scope'); + if (scope && scope !== COLLECTIVEX_CACHE_SCOPE) { + return NextResponse.json({ error: 'unknown scope' }, { status: 400 }); + } + if (scope === COLLECTIVEX_CACHE_SCOPE) { + purgeCollectiveX(); + return NextResponse.json({ invalidated: true, scope }); + } + const blobsDeleted = await purgeAll(); return NextResponse.json({ invalidated: true, blobsDeleted }); } diff --git a/packages/app/src/app/sitemap.ts b/packages/app/src/app/sitemap.ts index 2dd3e934b..8bce05497 100644 --- a/packages/app/src/app/sitemap.ts +++ b/packages/app/src/app/sitemap.ts @@ -22,6 +22,7 @@ const TABS = [ 'reliability', 'gpu-specs', 'gpu-metrics', + 'collectivex', ] as const; type SitemapEntry = MetadataRoute.Sitemap[number]; diff --git a/packages/app/src/app/zh/(dashboard)/collectivex/page.tsx b/packages/app/src/app/zh/(dashboard)/collectivex/page.tsx new file mode 100644 index 000000000..6496ec8a2 --- /dev/null +++ b/packages/app/src/app/zh/(dashboard)/collectivex/page.tsx @@ -0,0 +1,16 @@ +import type { Metadata } from 'next'; + +import CollectiveXDisplay from '@/components/collectivex/CollectiveXDisplay'; +import { ZhTabIntro } from '@/components/zh/zh-tab-intro'; +import { tabMetadataZh } from '@/lib/tab-meta-zh'; + +export const metadata: Metadata = tabMetadataZh('collectivex'); + +export default function ZhCollectiveXPage() { + return ( + <> + + + + ); +} diff --git a/packages/app/src/components/collectivex/CollectiveXChart.tsx b/packages/app/src/components/collectivex/CollectiveXChart.tsx new file mode 100644 index 000000000..8c7f0a4fc --- /dev/null +++ b/packages/app/src/components/collectivex/CollectiveXChart.tsx @@ -0,0 +1,242 @@ +'use client'; + +import * as d3 from 'd3'; +import { useMemo } from 'react'; + +import { D3Chart } from '@/lib/d3-chart/D3Chart'; + +import { chartPoints, collectiveXColorKey, collectiveXRunDasharray, fitAlphaBeta } from './data'; +import type { + CollectiveXChartPoint, + CollectiveXOperation, + CollectiveXPercentile, + CollectiveXRunSeries, + CollectiveXYAxis, +} from './types'; + +interface CollectiveXChartProps { + chartId: string; + series: CollectiveXRunSeries[]; + colors: Record; + operation: CollectiveXOperation; + percentile: CollectiveXPercentile; + yAxis: CollectiveXYAxis; + caption?: React.ReactNode; + legendElement?: React.ReactNode; + testId?: string; +} + +const OPERATION_LABELS: Record = { + dispatch: 'Dispatch', + stage: 'Stage', + combine: 'Combine', + roundtrip: 'Round trip (measured)', +}; + +const Y_AXIS_LABELS: Record = { + latency: 'Latency (µs)', + 'tokens-per-second': 'Token rate at selected latency percentile (tokens/s)', + 'activation-rate': 'Activation-data rate at selected latency percentile (GB/s)', + 'payload-rate': 'Payload bandwidth at selected latency percentile (GB/s, per GPU)', +}; + +function paddedDomain(values: number[]): [number, number] { + if (values.length === 0) return [1, 10]; + const min = d3.min(values) ?? 1; + const max = d3.max(values) ?? 1; + return min === max ? [min / 2, max * 2] : [min / 1.08, max * 1.08]; +} + +function formatCompact(value: number): string { + if (value >= 1e9) return `${(value / 1e9).toFixed(value < 1e10 ? 1 : 0)}G`; + if (value >= 1e6) return `${(value / 1e6).toFixed(value < 1e7 ? 1 : 0)}M`; + if (value >= 1e3) return `${(value / 1e3).toFixed(value < 1e4 ? 1 : 0)}k`; + if (value >= 10) return value.toFixed(0); + if (value >= 1) return value.toFixed(value < 3 ? 1 : 0); + return value.toFixed(2); +} + +function formatTokenCount(value: number): string { + return Number.isInteger(value) ? value.toLocaleString('en-US') : formatCompact(value); +} + +function formatMetric(value: number, yAxis: CollectiveXYAxis): string { + if (yAxis === 'latency') return `${value.toFixed(value >= 100 ? 0 : 1)} µs`; + if (yAxis === 'tokens-per-second') return `${formatCompact(value)} tok/s`; + return `${value.toFixed(value >= 100 ? 0 : 2)} GB/s`; +} + +function formatPercentiles( + value: CollectiveXRunSeries['points'][number]['components']['dispatch'], +): string { + if (value === null) return 'unavailable'; + return `${value.latency_us.p50.toFixed(1)} / ${value.latency_us.p90.toFixed(1)} / ${value.latency_us.p95.toFixed(1)} / ${value.latency_us.p99.toFixed(1)} µs`; +} + +function escapeHtml(value: string): string { + return value + .replaceAll('&', '&') + .replaceAll('<', '<') + .replaceAll('>', '>') + .replaceAll('"', '"') + .replaceAll("'", '''); +} + +export function CollectiveXChart({ + chartId, + series, + colors, + operation, + percentile, + yAxis, + caption, + legendElement, + testId, +}: CollectiveXChartProps) { + const points = useMemo( + () => chartPoints(series, operation, percentile, yAxis), + [series, operation, percentile, yAxis], + ); + const seriesById = useMemo(() => new Map(series.map((item) => [item.series_id, item])), [series]); + // Per-series α/β fit for the current operation (p50). β is the per-GPU + // bandwidth term, α the fixed overhead; surfaced in the tooltip. Null when a + // series has too few points / a degenerate byte axis to fit. + const fitsBySeries = useMemo( + () => new Map(series.map((item) => [item.series_id, fitAlphaBeta(item, operation)])), + [series, operation], + ); + const lines = useMemo(() => { + const result: Record = {}; + for (const point of points) { + (result[point.seriesId] ??= []).push({ x: point.x, y: point.y }); + } + for (const line of Object.values(result)) { + line.sort((a, b) => a.x - b.x); + } + return result; + }, [points]); + + const xDomain = useMemo(() => paddedDomain(points.map((point) => point.x)), [points]); + const yDomain = useMemo(() => paddedDomain(points.map((point) => point.y)), [points]); + const xTickValues = useMemo( + () => [...new Set(points.map((point) => point.x))].toSorted((a, b) => a - b), + [points], + ); + + const noDataOverlay = + points.length === 0 ? ( +
+

+ {series.length > 0 + ? `${OPERATION_LABELS[operation]} is unavailable for the selected series.` + : 'No matching CollectiveX series.'} +

+
+ ) : undefined; + + return ( + + chartId={chartId} + data={points} + height={560} + margin={{ top: 24, right: 20, bottom: 62, left: 78 }} + watermark="logo" + testId={testId} + grabCursor + instructions="Shift+Scroll to zoom · Drag to pan · Double-click to reset · Click a point to pin tooltip" + xScale={{ type: 'log', domain: xDomain, nice: false }} + yScale={{ type: 'log', domain: yDomain, nice: false }} + xAxis={{ + label: 'Source tokens / rank (log)', + tickCount: 8, + tickValues: xTickValues, + tickFormat: (value) => formatTokenCount(Number(value)), + }} + yAxis={{ + label: Y_AXIS_LABELS[yAxis], + tickCount: 5, + tickFormat: (value) => formatCompact(Number(value)), + }} + layers={[ + { + type: 'line', + key: 'collectivex-lines', + lines, + config: { + getColor: (key) => { + const item = seriesById.get(key); + return colors[item ? collectiveXColorKey(item) : ''] ?? '#888'; + }, + getStrokeDasharray: (key) => { + const item = seriesById.get(key); + return item ? collectiveXRunDasharray(item.run_index) : 'none'; + }, + strokeWidth: 2.25, + curve: d3.curveLinear, + }, + }, + { + type: 'point', + key: 'collectivex-points', + data: points, + config: { + getCx: () => 0, + getCy: () => 0, + getX: (point) => point.x, + getY: (point) => point.y, + getColor: (point) => colors[point.colorKey] ?? '#888', + getRadius: () => 3.5, + stroke: 'var(--background)', + strokeWidth: 1, + keyFn: (point) => `${point.seriesId}-${point.x}`, + maxPoints: Infinity, + }, + }, + ]} + zoom={{ + enabled: true, + axes: 'both', + scaleExtent: [1, 20], + resetEventName: `collectivex_zoom_reset_${chartId}`, + }} + tooltip={{ + rulerType: 'crosshair', + attachToLayer: 1, + content: (point, isPinned) => { + const color = colors[point.colorKey] ?? '#888'; + const measurement = point.point; + const measuredRoundtrip = measurement.components.roundtrip; + const fit = fitsBySeries.get(point.seriesId); + const fitLine = fit + ? `
Fit β=${fit.betaGbps.toFixed(fit.betaGbps >= 100 ? 0 : 1)} GB/s · α=${fit.alphaUs.toFixed(1)} µs (p50, per GPU)
` + : ''; + return `
+ ${isPinned ? '
Click elsewhere to dismiss
' : ''} +
${escapeHtml(point.seriesLabel)}
+
${escapeHtml(OPERATION_LABELS[operation])} ${yAxis === 'latency' ? percentile : `at ${percentile} latency`}: ${formatMetric(point.y, yAxis)}
+
${measurement.tokens_per_rank} tokens/rank · ${measurement.global_tokens} global tokens
+
Latency p50 / p90 / p95 / p99
+
Dispatch: ${formatPercentiles(measurement.components.dispatch)}
+
Stage: ${formatPercentiles(measurement.components.stage)}
+
Combine: ${formatPercentiles(measurement.components.combine)}
+
Round trip: ${formatPercentiles(measuredRoundtrip)}${measuredRoundtrip ? ' (measured)' : ''}
+ ${fitLine} +
`; + }, + getRulerX: (point, scale) => + (scale as d3.ScaleLinear | d3.ScaleLogarithmic)(point.x), + getRulerY: (point, scale) => scale(point.y), + onHoverStart: (selection) => { + selection.attr('r', 6); + }, + onHoverEnd: (selection) => { + selection.attr('r', 3.5); + }, + }} + transitionDuration={200} + legendElement={legendElement} + noDataOverlay={noDataOverlay} + caption={caption} + /> + ); +} diff --git a/packages/app/src/components/collectivex/CollectiveXDisplay.tsx b/packages/app/src/components/collectivex/CollectiveXDisplay.tsx new file mode 100644 index 000000000..164d225d4 --- /dev/null +++ b/packages/app/src/components/collectivex/CollectiveXDisplay.tsx @@ -0,0 +1,1041 @@ +'use client'; + +import { BookOpen, ExternalLink, Loader2, RefreshCw } from 'lucide-react'; +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; + +import { Button } from '@/components/ui/button'; +import { Card } from '@/components/ui/card'; +import ChartLegend from '@/components/ui/chart-legend'; +import { Label } from '@/components/ui/label'; +import { SegmentedToggle, type SegmentedToggleOption } from '@/components/ui/segmented-toggle'; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from '@/components/ui/select'; +import { + useCollectiveXRunDatasets, + useCollectiveXRuns, + useDeleteCollectiveXRun, +} from '@/hooks/api/use-collectivex'; +import { useThemeColors } from '@/hooks/useThemeColors'; +import { track } from '@/lib/analytics'; +import { useLocale } from '@/lib/use-locale'; + +import { CollectiveXChart } from './CollectiveXChart'; +import { CollectiveXInventory } from './CollectiveXInventory'; +import { CollectiveXRunsTable } from './CollectiveXRunsTable'; +import { + collectiveXColorKey, + collectiveXLegendLabel, + collectiveXRunDasharray, + collectiveXSeriesForRun, + collectiveXTopologyLabel, + seriesMatchesSelection, + type CollectiveXSeriesSelection, +} from './data'; +import { + COLLECTIVEX_VERSIONS, + COLLECTIVEX_DEFAULT_VERSION, + collectiveXVersionLabel, + type CollectiveXMode, + type CollectiveXOperation, + type CollectiveXPercentile, + type CollectiveXPhase, + type CollectiveXPrecision, + type CollectiveXRunSeries, + type CollectiveXVersion, + type CollectiveXYAxis, +} from './types'; + +interface SelectOption { + value: T; + label: string; +} + +const PERCENTILE_OPTIONS: SegmentedToggleOption[] = [ + { value: 'p50', label: 'p50' }, + { value: 'p90', label: 'p90' }, + { value: 'p95', label: 'p95' }, + { value: 'p99', label: 'p99' }, +]; +const STRINGS = { + en: { + operation: { + dispatch: 'Dispatch', + combine: 'Combine', + roundtrip: 'Round trip', + }, + operationHeading: { + dispatch: 'Dispatch', + combine: 'Combine', + roundtrip: 'Round trip (measured)', + }, + phase: { decode: 'Decode', prefill: 'Prefill' }, + phaseValue: { decode: 'decode', prefill: 'prefill' }, + mode: { normal: 'Normal', 'low-latency': 'Low-latency' }, + precision: { bf16: 'BF16', fp8: 'FP8' }, + yAxis: { + latency: 'Latency', + 'tokens-per-second': 'Token rate at selected latency percentile', + 'payload-rate': 'Payload bandwidth at selected latency percentile (per GPU)', + }, + all: 'All', + loading: 'Resolving CollectiveX run...', + unavailable: 'CollectiveX run unavailable', + loadError: 'The CollectiveX dataset failed to load.', + retry: 'Retry', + description: + 'Expert-parallel latency and payload rate across collective libraries and systems.', + source: 'Source', + methodology: 'Methodology', + refresh: 'Refresh', + seriesCount: 'Series', + measuredCases: 'Measured cases', + terminalCases: 'Terminal cases', + publishedUtc: 'Published (UTC)', + version: 'Benchmark version', + runsHeading: 'Runs', + runsDescription: + 'Every stored run for the selected benchmark version. Check one or more runs to compare them in the explorer.', + runsShown: 'Runs shown', + selectRuns: 'Select one or more runs from the table to show their data.', + selectedRunsFailed: 'One or more selected runs failed to load.', + runControl: 'Run', + loadRuns: 'Load runs', + loadingRuns: 'Loading runs…', + latestPublished: 'Latest run', + epControl: 'EP degree', + operationControl: 'Operation', + phaseControl: 'Phase', + phaseAria: 'CollectiveX phase', + modeControl: 'Kernel mode', + modeAria: 'CollectiveX kernel mode', + precisionControl: 'Precision', + precisionAria: 'CollectiveX precision', + latencyPercentile: 'Latency percentile', + percentileAria: 'CollectiveX percentile', + sku: 'SKU', + backend: 'Backend', + yAxisControl: 'Y axis', + tokenRateOption: 'Token rate at latency percentile', + noSeries: 'No measured series match these filters.', + resetFilter: 'Reset filter', + payloadNote: + 'Activation-data rate is derived at the selected latency percentile and is not physical link bandwidth.', + payloadBandwidthNote: + 'Payload bandwidth is the full logical payload (incl. FP8 scale bytes) ÷ latency, per GPU — a derived rate over logical bytes, not physical link bandwidth. The tooltip β/α is a least-squares fit of latency vs bytes across the ladder (β = per-GPU bandwidth term, α = fixed overhead).', + deleteRun: 'Delete run', + deleteConfirm: (id: string) => + `Delete run #${id} from the dashboard database? This cannot be undone.`, + deleteTokenPrompt: 'Admin token required to delete runs:', + deleteUnauthorized: 'Invalid admin token.', + deleteFailed: 'Deleting the run failed. Try again.', + }, + zh: { + operation: { + dispatch: '分发', + combine: '合并', + roundtrip: '往返', + 'isolated-sum': '分项之和', + }, + operationHeading: { + dispatch: '分发', + combine: '合并', + roundtrip: '往返(实测)', + 'isolated-sum': '分项之和(派生)', + }, + phase: { decode: '解码', prefill: '预填充' }, + phaseValue: { decode: '解码', prefill: '预填充' }, + precision: { bf16: 'BF16', fp8: 'FP8' }, + scale: { log: '对数', linear: '线性' }, + xAxis: { + 'tokens-per-rank': '每 rank 源 token 数', + 'global-tokens': '全局源 token 数', + }, + yAxis: { + latency: '延迟', + 'tokens-per-second': '所选延迟分位点的 token 速率', + 'payload-rate': '所选延迟分位点的载荷带宽(每 GPU)', + }, + mode: { normal: '常规', 'low-latency': '低延迟' }, + fabricScope: { all: '全部', 'scale-up': '域内', 'scale-out': '跨域' }, + topologyScope: { 'scale-up': '域内(scale-up)', 'scale-out': '跨域(scale-out)' }, + payloadUnit: { 'token-rank': 'Token-rank 载荷', 'token-expert': 'Token-expert 载荷' }, + combineSemantics: { + 'activation-only': '仅激活值合并', + 'gate-weighted': '门控加权合并', + }, + tabs: { + inventory: 'Matrix case inventory', + case: 'Selected matrix case', + evidence: '证据', + }, + noCases: 'This run has no matrix cases to inspect.', + all: '全部', + loading: 'Resolving CollectiveX run...', + unavailable: 'CollectiveX run unavailable', + sourceUnavailable: 'The GitHub Actions run source is temporarily unavailable.', + runsErrorMessage: 'No CollectiveX run has been published yet.', + loadError: 'The CollectiveX dataset failed to load.', + retry: '重试', + description: '对比集合通信库与系统的专家并行(EP)延迟和逻辑载荷速率。', + source: '源代码', + methodology: '测试方法', + sourceLinkUnavailable: 'Source unavailable because measured series span different revisions', + refresh: '刷新', + seriesCount: 'Series', + measuredCases: 'Measured cases', + terminalCases: '已终结用例', + retainedAttempts: '保留尝试', + allocations: '独立分配', + publishedUtc: '发布时间(UTC)', + version: '基准版本', + // English placeholders per the repository's temporary language override + // (no new Chinese translations); localize when the override lifts. + runsHeading: 'Runs', + runsDescription: + 'Every stored run for the selected benchmark version. Check one or more runs to compare them in the explorer.', + runsShown: 'Runs shown', + selectRuns: 'Select one or more runs from the table to show their data.', + selectedRunsFailed: 'One or more selected runs failed to load.', + runControl: 'Run', + loadRuns: 'Load runs', + loadingRuns: 'Loading runs…', + latestPublished: 'Latest run', + modeControl: '模式', + modeAria: 'CollectiveX 模式', + epControl: 'EP 并行度', + fabricScopeControl: '互联范围', + fabricScopeAria: 'CollectiveX 互联范围', + operationControl: '操作', + phaseControl: '阶段', + phaseAria: 'CollectiveX 阶段', + precisionControl: '精度', + precisionAria: 'CollectiveX 精度', + latencyPercentile: '延迟分位点', + percentileAria: 'CollectiveX 延迟分位点', + sku: 'SKU', + backend: '后端', + routing: '路由', + xAxisControl: 'X 轴', + xScale: 'X 轴刻度', + xScaleAria: 'CollectiveX X 轴刻度', + yAxisControl: 'Y 轴', + tokenRateOption: '延迟分位点对应的 token 速率', + yScale: 'Y 轴刻度', + yScaleAria: 'CollectiveX Y 轴刻度', + noSeries: 'No measured series match these filters.', + highContrast: '高对比度', + resetFilter: '重置筛选', + stableOrdering: '排名顺序稳定性已通过', + samplingContract: (trials: number, iterations: number, samples: number, warmups: number) => + `${trials}×${iterations} = 每个分项 ${samples} 个样本 · ${warmups} 次同步预热`, + selectedFactorsDiffer: '所选配置存在差异', + differenceLabels: { + model: '模型', + suite: '测试套件', + mode: '模式', + phase: '阶段', + 'backend implementation': '后端实现', + 'implementation build': '实现构建', + 'system identity': '系统标识', + 'fabric scope': '互联范围', + topology: '拓扑', + transport: '传输方式', + 'world size': '全局 rank 数', + 'EP degree': 'EP 并行度', + placement: '放置方式', + workload: '工作负载', + 'model shape': '模型形状', + routing: '路由', + 'EPLB plan': 'EPLB 方案', + dtypes: '数据类型', + 'resource profile': '资源配置', + measurement: '测量协议', + 'token ladder': 'token 梯度', + 'component availability': '测量分项可用性', + correctness: '正确性', + }, + missingComponents: '不可用的测量分项保持为空,并从图表中省略。', + isolatedNote: '分项之和为派生值,不用于计算吞吐量。', + payloadNote: '逻辑载荷速率按所选延迟分位点派生,不代表物理链路带宽。', + payloadBandwidthNote: + '载荷带宽为完整逻辑载荷(含 FP8 缩放字节)÷ 延迟(每 GPU),是基于逻辑字节的派生速率,不代表物理链路带宽。工具提示中的 β/α 为延迟对字节在整个梯度上的最小二乘拟合(β = 每 GPU 带宽项,α = 固定开销)。', + provenance: '发布数据溯源', + runLabel: 'Run', + attemptLabel: 'Attempt', + matrixLabel: 'Matrix', + sourceBundles: '源产物包', + deleteRun: 'Delete run', + deleteConfirm: (id: string) => + `Delete run #${id} from the dashboard database? This cannot be undone.`, + deleteTokenPrompt: 'Admin token required to delete runs:', + deleteUnauthorized: 'Invalid admin token.', + deleteFailed: 'Deleting the run failed. Try again.', + }, +} as const; +const CONCLUSION_CLASSES: Record = { + success: 'border-emerald-600/40 bg-emerald-500/10 text-emerald-700 dark:text-emerald-300', + failure: 'border-red-600/40 bg-red-500/10 text-red-700 dark:text-red-300', +}; +const CONCLUSION_FALLBACK_CLASS = + 'border-amber-600/40 bg-amber-500/10 text-amber-700 dark:text-amber-300'; +// Remembered admin bearer token for run deletion; cleared on a 401 so a +// rotated secret re-prompts instead of failing silently forever. +const ADMIN_TOKEN_STORAGE_KEY = 'collectivex-admin-token'; + +function ControlGroup({ label, children }: { label: string; children: React.ReactNode }) { + return ( +
+ + {children} +
+ ); +} + +function selectOptions( + values: string[], + allLabel: string, + uppercase = false, +): SelectOption[] { + return values.map((value) => ({ + value, + label: value === 'all' ? allLabel : uppercase ? value.toUpperCase() : value, + })); +} + +export default function CollectiveXDisplay() { + const locale = useLocale(); + const t = STRINGS[locale]; + const [version, setVersion] = useState(COLLECTIVEX_DEFAULT_VERSION); + const [visibleRunIds, setVisibleRunIds] = useState>(new Set()); + const initializedVersionRef = useRef(null); + const runsQuery = useCollectiveXRuns(version); + const runList = runsQuery.data?.runs ?? []; + const orderedVisibleRunIds = useMemo( + () => runList.filter((run) => visibleRunIds.has(run.run_id)).map((run) => run.run_id), + [runList, visibleRunIds], + ); + const runQueries = useCollectiveXRunDatasets(version, orderedVisibleRunIds); + const datasets = useMemo( + () => runQueries.flatMap((query) => (query.data ? [query.data] : [])), + [runQueries], + ); + const runIndexById = useMemo( + () => new Map(runList.map((run, index) => [run.run_id, index])), + [runList], + ); + const combinedSeries = useMemo( + () => + datasets.flatMap((dataset) => + collectiveXSeriesForRun( + dataset.series, + dataset.run.run_id, + runIndexById.get(dataset.run.run_id) ?? 0, + ), + ), + [datasets, runIndexById], + ); + const loadingRunIds = useMemo( + () => + new Set( + orderedVisibleRunIds.filter( + (_runId, index) => runQueries[index]?.isLoading || runQueries[index]?.isFetching, + ), + ), + [orderedVisibleRunIds, runQueries], + ); + const selectedRunErrors = runQueries.filter((query) => query.error).length; + const isFetching = runsQuery.isFetching || runQueries.some((query) => query.isFetching); + const [epSize, setEpSize] = useState(8); + const [operation, setOperation] = useState('roundtrip'); + const [phase, setPhase] = useState('decode'); + // Normal (throughput) kernels are the baseline; the availability effect + // below falls back when a slice only measured low-latency kernels. + const [mode, setMode] = useState('normal'); + // Prefer FP8 when the run measured it; the availability effect below falls + // back to bf16 for runs (or EP/phase slices) without FP8 series. + const [precision, setPrecision] = useState('fp8'); + const [percentile, setPercentile] = useState('p99'); + const [yAxis, setYAxis] = useState('latency'); + const [sku, setSku] = useState('all'); + const [backend, setBackend] = useState('all'); + const [activeSeriesIds, setActiveSeriesIds] = useState>(new Set()); + const [legendExpanded, setLegendExpanded] = useState(true); + const operationOptions: SelectOption[] = [ + { value: 'dispatch', label: t.operation.dispatch }, + { value: 'stage', label: 'Stage' }, + { value: 'combine', label: t.operation.combine }, + { value: 'roundtrip', label: t.operation.roundtrip }, + ]; + const versionOptions: SelectOption[] = COLLECTIVEX_VERSIONS.map((value) => ({ + value, + label: collectiveXVersionLabel(value), + })); + + // Runs are per-version. Start each version on its newest run with measured + // data so an incomplete newest sweep cannot blank the explorer. + useEffect(() => { + initializedVersionRef.current = null; + setVisibleRunIds(new Set()); + }, [version]); + + useEffect(() => { + if (!runsQuery.data || initializedVersionRef.current === version) return; + if (runsQuery.data.runs.length === 0 && !runsQuery.data.discovery_complete) return; + const initial = + runsQuery.data.runs.find((run) => run.measured_cases > 0) ?? runsQuery.data.runs[0]; + setVisibleRunIds(initial ? new Set([initial.run_id]) : new Set()); + initializedVersionRef.current = version; + }, [runsQuery.data, version]); + + // A deleted run disappears from both the table and the checked set after the + // list refetch. Preserve deliberate "none checked" state. + useEffect(() => { + if (!runsQuery.data || initializedVersionRef.current !== version) return; + const liveIds = new Set(runsQuery.data.runs.map((run) => run.run_id)); + setVisibleRunIds((previous) => { + const next = new Set([...previous].filter((runId) => liveIds.has(runId))); + return next.size === previous.size ? previous : next; + }); + }, [runsQuery.data, version]); + + const availableEpSizes = useMemo( + () => [...new Set(combinedSeries.map((item) => item.system.ep_size))].toSorted((a, b) => a - b), + [combinedSeries], + ); + const availablePhases = useMemo( + () => + [ + ...new Set( + combinedSeries.filter((item) => item.system.ep_size === epSize).map((item) => item.phase), + ), + ].toSorted((left, right) => + left === right ? 0 : left === 'decode' ? -1 : right === 'decode' ? 1 : 0, + ), + [combinedSeries, epSize], + ); + const phaseOptions: SegmentedToggleOption[] = availablePhases.map((value) => ({ + value, + label: t.phase[value], + })); + const availableModes = useMemo( + () => + [ + ...new Set( + combinedSeries + .filter((item) => item.system.ep_size === epSize && item.phase === phase) + .map((item) => item.mode), + ), + ].toSorted((left, right) => + left === right ? 0 : left === 'normal' ? -1 : right === 'normal' ? 1 : 0, + ), + [combinedSeries, epSize, phase], + ); + const modeOptions: SegmentedToggleOption[] = availableModes.map((value) => ({ + value, + label: t.mode[value], + })); + const availablePrecisions = useMemo( + () => + [ + ...new Set( + combinedSeries + .filter( + (item) => + item.system.ep_size === epSize && item.phase === phase && item.mode === mode, + ) + .map((item) => item.precision), + ), + ].toSorted(), + [combinedSeries, epSize, mode, phase], + ); + const precisionOptions: SegmentedToggleOption[] = availablePrecisions.map( + (value) => ({ value, label: t.precision[value] }), + ); + useEffect(() => { + if (availableEpSizes.length > 0 && !availableEpSizes.includes(epSize)) { + setEpSize(availableEpSizes[0]); + } + if (availablePhases.length > 0 && !availablePhases.includes(phase)) { + setPhase(availablePhases[0]); + } + if (availableModes.length > 0 && !availableModes.includes(mode)) { + setMode(availableModes[0]); + } + if (availablePrecisions.length > 0 && !availablePrecisions.includes(precision)) { + setPrecision(availablePrecisions[0]); + } + }, [ + availableEpSizes, + availableModes, + availablePhases, + availablePrecisions, + epSize, + mode, + phase, + precision, + ]); + const seriesSelection = useMemo( + () => ({ epSize, phase, mode, precision }), + [epSize, mode, phase, precision], + ); + // SKU and EP determine topology; V1 fixes routing. EP, phase, kernel mode, + // and precision are needed before the library/SKU comparison filters. + const matchedSeries = useMemo( + () => combinedSeries.filter((item) => seriesMatchesSelection(item, seriesSelection)), + [combinedSeries, seriesSelection], + ); + const skuOptions = useMemo( + () => ['all', ...new Set(matchedSeries.map((item) => item.system.sku))], + [matchedSeries], + ); + const backendOptions = useMemo( + () => [ + 'all', + ...new Set( + matchedSeries + .filter((item) => sku === 'all' || item.system.sku === sku) + .map((item) => item.backend), + ), + ], + [matchedSeries, sku], + ); + useEffect(() => { + if (!skuOptions.includes(sku)) setSku('all'); + if (!backendOptions.includes(backend)) setBackend('all'); + }, [backend, backendOptions, sku, skuOptions]); + const phaseSeries = useMemo( + () => + matchedSeries.filter( + (item) => + (sku === 'all' || item.system.sku === sku) && + (backend === 'all' || item.backend === backend), + ), + [backend, matchedSeries, sku], + ); + const phaseSeriesKey = phaseSeries.map((item) => item.series_id).join('\u0000'); + + useEffect(() => { + setActiveSeriesIds(new Set(phaseSeriesKey ? phaseSeriesKey.split('\u0000') : [])); + }, [phaseSeriesKey]); + + const activeSeries = useMemo( + () => phaseSeries.filter((item) => activeSeriesIds.has(item.series_id)), + [activeSeriesIds, phaseSeries], + ); + const colorKeys = useMemo( + () => [...new Set(phaseSeries.map(collectiveXColorKey))], + [phaseSeries], + ); + const { resolveColor, getCssColor } = useThemeColors({ + highContrast: false, + activeKeys: colorKeys, + hcKeys: colorKeys, + hcVendorKeyFor: (key) => key.split('_')[0], + }); + const colors = useMemo( + () => Object.fromEntries(colorKeys.map((key) => [key, getCssColor(resolveColor(key, key))])), + [colorKeys, getCssColor, resolveColor], + ); + const legendItems = useMemo( + () => + phaseSeries.map((item) => ({ + name: item.series_id, + label: collectiveXLegendLabel(item), + color: colors[collectiveXColorKey(item)] ?? 'var(--muted-foreground)', + lineDasharray: collectiveXRunDasharray(item.run_index), + isActive: activeSeriesIds.has(item.series_id), + title: `#${item.run_id} · EP${item.system.ep_size} · ${collectiveXTopologyLabel(item.system)}`, + onClick: () => { + setActiveSeriesIds((previous) => { + const next = new Set(previous); + if (next.has(item.series_id)) next.delete(item.series_id); + else next.add(item.series_id); + return next; + }); + track('collectivex_series_toggled', { series: item.series_id }); + }, + })), + [activeSeriesIds, colors, phaseSeries], + ); + const handleRefresh = useCallback(() => { + track('collectivex_data_refreshed'); + void runsQuery.refetch(); + for (const query of runQueries) void query.refetch(); + }, [runQueries, runsQuery]); + const deleteRun = useDeleteCollectiveXRun(); + const handleVisibleRunChange = useCallback((runId: string, visible: boolean) => { + setVisibleRunIds((previous) => { + const next = new Set(previous); + if (visible) next.add(runId); + else next.delete(runId); + return next; + }); + }, []); + const handleDeleteRun = useCallback( + async (runId: string) => { + track('collectivex_run_delete_prompted', { run: runId }); + if (!window.confirm(t.deleteConfirm(runId))) return; + const stored = localStorage.getItem(ADMIN_TOKEN_STORAGE_KEY) ?? ''; + const token = stored || (window.prompt(t.deleteTokenPrompt)?.trim() ?? ''); + if (!token) return; + try { + const deleted = await deleteRun.mutateAsync({ runId, token }); + if (!deleted) { + localStorage.removeItem(ADMIN_TOKEN_STORAGE_KEY); + track('collectivex_run_delete_failed', { run: runId, reason: 'unauthorized' }); + window.alert(t.deleteUnauthorized); + return; + } + localStorage.setItem(ADMIN_TOKEN_STORAGE_KEY, token); + setVisibleRunIds((previous) => { + const next = new Set(previous); + next.delete(runId); + return next; + }); + track('collectivex_run_delete_confirmed', { run: runId }); + } catch { + track('collectivex_run_delete_failed', { run: runId, reason: 'error' }); + window.alert(t.deleteFailed); + } + }, + [deleteRun, t], + ); + + if (runsQuery.isLoading) { + return ( + + +

{t.loadingRuns}

+
+ ); + } + if (runsQuery.error || !runsQuery.data) { + const message = runsQuery.error instanceof Error ? runsQuery.error.message : t.loadError; + return ( + +

{t.unavailable}

+

{message}

+
+
+ { + setVersion(value); + track('collectivex_version_changed', { version: value }); + }} + /> +
+ +
+
+ ); + } + const singleDataset = datasets.length === 1 ? datasets[0] : null; + const measuredCases = datasets.reduce((sum, dataset) => sum + dataset.run.measured_cases, 0); + const requestedCases = datasets.reduce((sum, dataset) => sum + dataset.run.requested_cases, 0); + const terminalCases = datasets.reduce((sum, dataset) => sum + dataset.run.terminal_cases, 0); + const seriesCount = datasets.reduce((sum, dataset) => sum + dataset.series.length, 0); + const singleConclusionClass = + (singleDataset?.run.conclusion && CONCLUSION_CLASSES[singleDataset.run.conclusion]) ?? + CONCLUSION_FALLBACK_CLASS; + + return ( +
+ +
+
+
+

CollectiveX

+ + {singleDataset + ? `#${singleDataset.run.run_id} · ${singleDataset.run.conclusion ?? 'pending'}` + : `${datasets.length} ${t.runsShown.toLowerCase()}`} + +
+

{t.description}

+
+ +
+ {datasets.length > 0 && ( +
+ + + + +
+ )} +
+ + +
+
+

{t.runsHeading}

+

{t.runsDescription}

+
+
+ { + setVersion(value); + track('collectivex_version_changed', { version: value }); + }} + /> +
+
+ void handleDeleteRun(runId)} + /> +
+ + {visibleRunIds.size === 0 && ( + +

{t.selectRuns}

+
+ )} + {visibleRunIds.size > 0 && datasets.length === 0 && loadingRunIds.size > 0 && ( + + +

{t.loading}

+
+ )} + {selectedRunErrors > 0 && ( + +

{t.selectedRunsFailed}

+
+ )} + + {datasets.length > 0 && ( + <> + {phaseSeries.length === 0 && ( + +

{t.noSeries}

+
+ )} + + +

+ {operation === 'stage' ? 'Stage' : t.operationHeading[operation]} ·{' '} + {t.phaseValue[phase]} ·{' '} + {yAxis === 'latency' + ? percentile + : locale === 'zh' + ? `${percentile} 延迟分位点` + : `at ${percentile} latency`} +

+

+ {yAxis === 'activation-rate' + ? 'Activation-data rate at selected latency percentile' + : t.yAxis[yAxis]} +

+ + } + legendElement={ + { + setActiveSeriesIds( + (previous) => new Set([...previous].filter((item) => item !== id)), + ); + track('collectivex_series_toggled', { series: id, visible: false }); + }} + isLegendExpanded={legendExpanded} + onExpandedChange={setLegendExpanded} + actions={ + activeSeries.length < phaseSeries.length + ? [ + { + id: 'collectivex-reset-filter', + label: t.resetFilter, + onClick: () => { + setActiveSeriesIds( + new Set(phaseSeries.map((item) => item.series_id)), + ); + track('collectivex_series_filter_reset'); + }, + }, + ] + : [] + } + /> + } + /> + {yAxis === 'activation-rate' && ( +

{t.payloadNote}

+ )} + {yAxis === 'payload-rate' && ( +

{t.payloadBandwidthNote}

+ )} +
+ +
+ ({ + value: String(value), + label: `EP${value}`, + }))} + onChange={(value) => { + setEpSize(Number(value)); + track('collectivex_ep_changed', { ep: Number(value) }); + }} + /> + { + setOperation(next); + if (next !== 'roundtrip' && yAxis === 'tokens-per-second') setYAxis('latency'); + track('collectivex_operation_changed', { operation: next }); + }} + /> + + { + setPhase(next); + track('collectivex_phase_changed', { phase: next }); + }} + ariaLabel={t.phaseAria} + testId="collectivex-phase-toggle" + /> + + {availableModes.length > 1 && ( + + { + setMode(next); + track('collectivex_mode_changed', { mode: next }); + }} + ariaLabel={t.modeAria} + testId="collectivex-mode-toggle" + /> + + )} + + { + setPrecision(next); + track('collectivex_precision_changed', { precision: next }); + }} + ariaLabel={t.precisionAria} + testId="collectivex-precision-toggle" + /> + + + { + setPercentile(next); + track('collectivex_percentile_changed', { percentile: next }); + }} + ariaLabel={t.percentileAria} + testId="collectivex-percentile-toggle" + /> + + { + setSku(next); + track('collectivex_sku_changed', { sku: next }); + }} + /> + { + setBackend(next); + track('collectivex_backend_changed', { backend: next }); + }} + /> + { + setYAxis(next); + track('collectivex_y_axis_changed', { y_axis: next }); + }} + options={[ + { value: 'latency', label: t.yAxis.latency }, + ...(operation === 'roundtrip' + ? ([ + { + value: 'tokens-per-second', + label: t.tokenRateOption, + }, + ] as const) + : []), + { + value: 'activation-rate', + label: 'Activation-data rate at latency percentile', + }, + { + value: 'payload-rate', + label: t.yAxis['payload-rate'], + }, + ]} + /> +
+
+ `${dataset.run.run_id}:${dataset.run.run_attempt}`).join(',')}`} + datasets={datasets} + /> + + )} +
+ ); +} + +function Stat({ + value, + label, + compact = false, +}: { + value: React.ReactNode; + label: string; + compact?: boolean; +}) { + return ( +
+

{value}

+

{label}

+
+ ); +} + +function SelectControl({ + label, + testId, + value, + options, + onChange, + placeholder, +}: { + label: string; + testId: string; + value: T; + options: SelectOption[]; + onChange: (value: T) => void; + placeholder?: string; +}) { + // Radix Select speaks strings; numeric option values (e.g. the release version) + // round-trip through String() and are recovered from the option list on change. + return ( + + + + ); +} diff --git a/packages/app/src/components/collectivex/CollectiveXInventory.tsx b/packages/app/src/components/collectivex/CollectiveXInventory.tsx new file mode 100644 index 000000000..c3e9ec57a --- /dev/null +++ b/packages/app/src/components/collectivex/CollectiveXInventory.tsx @@ -0,0 +1,170 @@ +'use client'; + +import { useMemo } from 'react'; + +import { Badge } from '@/components/ui/badge'; +import { Card } from '@/components/ui/card'; +import { type DataTableColumn, DataTable } from '@/components/ui/data-table'; + +import { collectiveXTopologyLabel } from './data'; +import type { CollectiveXCoverage, CollectiveXDataset, CollectiveXTerminalStatus } from './types'; + +type CollectiveXRunCoverage = CollectiveXCoverage & { run_id: string }; + +const TERMINAL_ORDER: CollectiveXTerminalStatus[] = [ + 'measured', + 'unsupported', + 'failed', + 'invalid', + 'diagnostic', + 'pending', +]; + +const STATUS_CLASS: Record = { + measured: 'border-emerald-600/40 bg-emerald-500/10 text-emerald-700 dark:text-emerald-300', + unsupported: 'border-zinc-500/40 bg-zinc-500/10 text-zinc-700 dark:text-zinc-300', + failed: 'border-red-700/50 bg-red-700/10 text-red-800 dark:text-red-300', + invalid: 'border-red-600/40 bg-red-500/10 text-red-700 dark:text-red-300', + diagnostic: 'border-amber-600/40 bg-amber-500/10 text-amber-700 dark:text-amber-300', + pending: 'border-zinc-500/40 bg-zinc-500/5 text-muted-foreground', +}; + +function terminalCounts(item: CollectiveXCoverage): Record { + const counts = Object.fromEntries(TERMINAL_ORDER.map((status) => [status, 0])) as Record< + CollectiveXTerminalStatus, + number + >; + for (const point of item.points) counts[point.terminal_status] += 1; + return counts; +} + +function TerminalBadges({ item }: { item: CollectiveXCoverage }) { + const counts = terminalCounts(item); + const reasons = [...new Set(item.points.flatMap((point) => point.reason ?? []))]; + return ( +
+
+ {TERMINAL_ORDER.filter((status) => counts[status] > 0).map((status) => ( + + {status} {counts[status]} + + ))} +
+ {reasons.length > 0 && ( +

{reasons.join(', ')}

+ )} +
+ ); +} + +export function CollectiveXInventory({ datasets }: { datasets: CollectiveXDataset[] }) { + const rows = useMemo( + () => + datasets.flatMap((dataset) => + dataset.coverage.map((item) => ({ ...item, run_id: dataset.run.run_id })), + ), + [datasets], + ); + const columns = useMemo[]>( + () => [ + { + header: 'Run', + cell: (row) => #{row.run_id}, + sortValue: (row) => Number(row.run_id), + className: 'whitespace-nowrap', + }, + { + header: 'Case', + cell: (row) => ( +
+

{row.label}

+

{row.case_id}

+
+ ), + sortValue: (row) => `${row.label} ${row.case_id}`, + }, + { header: 'SKU', cell: (row) => row.sku.toUpperCase(), sortValue: (row) => row.sku }, + { + header: 'Backend', + cell: (row) => row.backend, + sortValue: (row) => row.backend, + className: 'whitespace-nowrap', + }, + { + header: 'EP', + cell: (row) => `EP${row.topology.ep_size}`, + sortValue: (row) => row.topology.ep_size, + }, + { + header: 'Phase', + cell: (row) => row.phase, + sortValue: (row) => row.phase, + }, + { + header: 'Mode', + cell: (row) => row.mode, + sortValue: (row) => row.mode, + }, + { + header: 'Precision', + cell: (row) => row.precision, + sortValue: (row) => row.precision, + }, + { + header: 'Topology', + cell: (row) => collectiveXTopologyLabel(row.topology), + sortValue: (row) => collectiveXTopologyLabel(row.topology), + className: 'whitespace-nowrap', + }, + { + header: 'Disposition', + cell: (row) => ( +
+

+ {row.disposition} · {row.outcome} +

+ {(row.detail || row.reason) && ( +

{row.detail ?? row.reason}

+ )} +
+ ), + sortValue: (row) => + `${row.disposition} ${row.outcome} ${row.reason ?? ''} ${row.detail ?? ''}`, + }, + { + header: 'Point status', + cell: (row) => , + sortValue: (row) => + `${TERMINAL_ORDER.map((status) => `${status}:${terminalCounts(row)[status]}`).join(' ')} ${row.points.map((point) => point.reason ?? '').join(' ')}`, + }, + ], + [], + ); + const points = rows.flatMap((item) => item.points); + const measured = points.filter((point) => point.terminal_status === 'measured').length; + const unsupported = points.filter((point) => point.terminal_status === 'unsupported').length; + const measuredCases = datasets.reduce((sum, dataset) => sum + dataset.run.measured_cases, 0); + const unsupportedCases = datasets.reduce( + (sum, dataset) => sum + dataset.run.unsupported_cases, + 0, + ); + const terminalPoints = datasets.reduce((sum, dataset) => sum + dataset.run.terminal_points, 0); + const requestedPoints = datasets.reduce((sum, dataset) => sum + dataset.run.requested_points, 0); + + return ( + +

Matrix case inventory

+

+ {datasets.length} runs · {rows.length} cases · {measuredCases} measured · {unsupportedCases}{' '} + unsupported · {terminalPoints}/{requestedPoints} terminal points · {measured} measured ·{' '} + {unsupported} unsupported +

+ +
+ ); +} diff --git a/packages/app/src/components/collectivex/CollectiveXRunsTable.tsx b/packages/app/src/components/collectivex/CollectiveXRunsTable.tsx new file mode 100644 index 000000000..b78e3443b --- /dev/null +++ b/packages/app/src/components/collectivex/CollectiveXRunsTable.tsx @@ -0,0 +1,226 @@ +'use client'; + +import { ExternalLink, Loader2, Trash2 } from 'lucide-react'; + +import { Button } from '@/components/ui/button'; +import { track } from '@/lib/analytics'; +import { useLocale } from '@/lib/use-locale'; +import { cn } from '@/lib/utils'; + +import { collectiveXRunDasharray } from './data'; +import type { CollectiveXRunSummary } from './types'; + +interface CollectiveXRunsTableProps { + runs: CollectiveXRunSummary[]; + runIndexById: ReadonlyMap; + visibleRunIds: ReadonlySet; + loadingRunIds: ReadonlySet; + deletingRunId: string | null; + onVisibleChange: (runId: string, visible: boolean) => void; + onDelete: (runId: string) => void; +} + +const STRINGS = { + en: { + shown: 'Shown', + run: 'Run', + result: 'Result', + cases: 'Measured cases', + points: 'Terminal points', + skus: 'SKUs', + published: 'Published (UTC)', + actions: 'Actions', + pending: 'pending', + showRun: (id: string) => `Show run #${id}`, + lineStyle: (id: string) => `Line style for run #${id}`, + openRun: (id: string) => `Open GitHub Actions run #${id}`, + deleteRun: (id: string) => `Delete run #${id}`, + empty: 'No runs match this benchmark version.', + }, + // English placeholders per the repository's temporary language override. + zh: { + shown: 'Shown', + run: 'Run', + result: 'Result', + cases: 'Measured cases', + points: 'Terminal points', + skus: 'SKUs', + published: 'Published (UTC)', + actions: 'Actions', + pending: 'pending', + showRun: (id: string) => `Show run #${id}`, + lineStyle: (id: string) => `Line style for run #${id}`, + openRun: (id: string) => `Open GitHub Actions run #${id}`, + deleteRun: (id: string) => `Delete run #${id}`, + empty: 'No runs match this benchmark version.', + }, +} as const; + +const CONCLUSION_CLASSES: Record = { + success: 'border-emerald-600/40 bg-emerald-500/10 text-emerald-700 dark:text-emerald-300', + failure: 'border-red-600/40 bg-red-500/10 text-red-700 dark:text-red-300', +}; +const CONCLUSION_FALLBACK_CLASS = + 'border-amber-600/40 bg-amber-500/10 text-amber-700 dark:text-amber-300'; + +function formatDate(value: string, locale: 'en' | 'zh'): string { + return new Intl.DateTimeFormat(locale === 'zh' ? 'zh-CN' : 'en', { + dateStyle: 'medium', + timeStyle: 'short', + timeZone: 'UTC', + }).format(new Date(value)); +} + +export function CollectiveXRunsTable({ + runs, + runIndexById, + visibleRunIds, + loadingRunIds, + deletingRunId, + onVisibleChange, + onDelete, +}: CollectiveXRunsTableProps) { + const locale = useLocale(); + const t = STRINGS[locale]; + + if (runs.length === 0) { + return

{t.empty}

; + } + + return ( +
+ + + + + + + + + + + + + + + {runs.map((run) => { + const visible = visibleRunIds.has(run.run_id); + const loading = loadingRunIds.has(run.run_id); + const deleting = deletingRunId === run.run_id; + const conclusion = run.conclusion ?? t.pending; + const lineDasharray = collectiveXRunDasharray(runIndexById.get(run.run_id) ?? 0); + return ( + + + + + + + + + + + ); + })} + +
{t.shown}{t.run}{t.result}{t.cases}{t.points}{t.skus}{t.published}{t.actions}
+
+ { + const next = event.target.checked; + onVisibleChange(run.run_id, next); + track('collectivex_run_visibility_toggled', { + run: run.run_id, + visible: next, + }); + }} + className="size-4 accent-primary" + /> + {loading && } +
+
+ + + + {conclusion} + + + {run.measured_cases}/{run.requested_cases} + + {run.terminal_points}/{run.requested_points} + + {run.covered_skus.length > 0 + ? run.covered_skus.map((sku) => sku.toUpperCase()).join(', ') + : '—'} + + {formatDate(run.generated_at, locale)} + + +
+
+ ); +} diff --git a/packages/app/src/components/collectivex/CollectiveXTables.tsx b/packages/app/src/components/collectivex/CollectiveXTables.tsx new file mode 100644 index 000000000..14c5843ea --- /dev/null +++ b/packages/app/src/components/collectivex/CollectiveXTables.tsx @@ -0,0 +1,267 @@ +'use client'; + +import { useMemo } from 'react'; + +import { Badge } from '@/components/ui/badge'; +import { Card } from '@/components/ui/card'; +import { type DataTableColumn, DataTable } from '@/components/ui/data-table'; +import { useLocale } from '@/lib/use-locale'; + +import { collectiveXTopologyLabel } from './data'; + +import type { CollectiveXCoverage, CollectiveXOutcome } from './types'; + +const OUTCOME_CLASSES = { + success: 'border-emerald-600/40 bg-emerald-500/15 text-emerald-700 dark:text-emerald-300', + unsupported: 'border-zinc-500/40 bg-zinc-500/15 text-zinc-700 dark:text-zinc-300', + failed: 'border-red-700/50 bg-red-700/15 text-red-800 dark:text-red-300', + invalid: 'border-red-600/40 bg-red-500/15 text-red-700 dark:text-red-300', + diagnostic: 'border-amber-600/40 bg-amber-500/15 text-amber-700 dark:text-amber-300', + pending: 'border-zinc-500/40 bg-zinc-500/5 text-muted-foreground', +} satisfies Record; + +const STRINGS = { + en: { + outcome: { + success: 'success', + unsupported: 'unsupported', + failed: 'failed', + invalid: 'invalid', + diagnostic: 'diagnostic', + pending: 'pending', + }, + phase: { decode: 'decode', prefill: 'prefill' }, + mode: { normal: 'Normal', 'low-latency': 'Low latency' }, + scope: { 'scale-up': 'Scale-up', 'scale-out': 'Scale-out' }, + disposition: { runnable: 'runnable', unsupported: 'unsupported' }, + case: 'Case', + sku: 'SKU', + backend: 'Backend', + phaseHeader: 'Phase', + precisionHeader: 'Precision', + modeHeader: 'Mode', + epHeader: 'EP', + scopeHeader: 'Fabric scope', + topologyHeader: 'Topology', + dispositionHeader: 'Disposition', + outcomeHeader: 'Outcome', + attempts: 'Attempts', + selected: 'Selected', + caseId: 'Case ID', + attemptId: 'Attempt ID', + failureMode: 'Failure mode', + reason: 'Reason', + terminalCoverage: 'Terminal coverage', + allocation: 'Allocation', + run: 'Run', + attempt: 'Try', + role: 'Role', + terminalRole: 'terminal selection', + allocationRole: 'allocation selection', + retainedRole: 'retained', + evidence: 'Evidence', + retainedAttempts: 'Retained attempts', + }, + zh: { + outcome: { + success: '成功', + unsupported: '不支持', + failed: '失败', + invalid: '无效', + diagnostic: '诊断', + pending: '待运行', + }, + phase: { decode: '解码', prefill: '预填充' }, + mode: { normal: '常规', 'low-latency': '低延迟' }, + scope: { 'scale-up': '域内(scale-up)', 'scale-out': '跨域(scale-out)' }, + disposition: { runnable: '可运行', unsupported: '不支持' }, + case: '用例', + sku: 'SKU', + backend: '后端', + phaseHeader: '阶段', + precisionHeader: '精度', + modeHeader: '模式', + epHeader: 'EP', + scopeHeader: '互联范围', + topologyHeader: '拓扑', + dispositionHeader: '计划状态', + outcomeHeader: '结果', + attempts: '尝试次数', + selected: '已选尝试', + caseId: '用例 ID', + attemptId: '尝试 ID', + failureMode: '失败类型', + reason: '原因', + terminalCoverage: '终结状态覆盖', + allocation: '独立分配', + run: '运行', + attempt: '尝试序号', + role: '用途', + terminalRole: '终结状态选择', + allocationRole: '独立分配选择', + retainedRole: '保留', + evidence: '证据数', + retainedAttempts: '保留的全部尝试', + }, +} as const; + +const REASON_LABELS = { + zh: { + 'artifact-validation-failed': '产物校验失败', + 'backend-platform-unsupported': '后端不支持该平台', + 'backend-token-capacity': '后端 token 容量不足', + 'launcher-setup-failed': '启动器初始化失败', + 'repository-staging-failed': '代码仓库暂存失败', + 'container-registry-verification-failed': '容器镜像仓库校验失败', + 'scheduler-allocation-failed': '调度资源分配失败', + 'container-image-preparation-failed': '容器镜像准备失败', + 'container-image-identity-failed': '容器镜像身份校验失败', + 'container-runtime-launch-failed': '容器运行时启动失败', + 'backend-setup-failed': '后端初始化失败', + 'artifact-collection-failed': '产物收集失败', + 'runtime-identity-mismatch': '运行时身份不匹配', + 'execution-timeout': '执行超时', + 'execution-deadlock': '执行死锁', + 'distributed-command-failed': '分布式命令执行失败', + 'post-emit-distributed-command-failed': '结果写出后的分布式命令失败', + 'unsupported-capability': '能力不支持', + 'execution-failed': '执行失败', + 'validation-failed': '校验失败', + 'diagnostic-evidence': '诊断证据', + capability: '能力限制', + setup: '初始化', + 'repository-stage': '代码仓库暂存', + 'registry-verification': '镜像仓库校验', + 'scheduler-allocation': '调度资源分配', + 'container-import': '容器镜像导入', + 'container-hash': '容器镜像哈希校验', + 'container-launch': '容器启动', + 'backend-setup': '后端初始化', + 'artifact-collection': '产物收集', + 'runtime-identity': '运行时身份', + timeout: '超时', + deadlock: '死锁', + execution: '执行', + 'insufficient-allocations': '独立分配不足', + 'incomplete-repeat-coverage': '重复运行覆盖不完整', + 'correctness-failed': '正确性校验失败', + 'missing-measured-roundtrip-p99': '缺少实测往返 p99', + 'unstable-ordering': '排名顺序不稳定', + 'incomplete-provenance': '来源与运行溯源不完整', + 'noncanonical-workload': '工作负载不符合规范', + 'unresolved-anomaly': '异常尚未解释', + 'semantic-correctness-failed': '语义正确性校验失败', + 'measurement-nonconformant': '测量协议不符合要求', + 'expert-oracle-incomplete': '专家路由正确性校验不完整', + 'incomplete-aligned-repeats': '对齐的重复运行不完整', + 'missing-uniform-baseline': '缺少 uniform 基线', + 'incomplete-routing-anchors': '路由基准锚点不完整', + 'implementation-config-mismatch': '实现配置不一致', + 'unmatched-token-coverage': 'token 点位覆盖不一致', + 'awaiting-v1-runs': '等待 CollectiveX v1 运行结果', + }, +} as const; + +export function collectiveXReasonLabel(value: string, locale: 'en' | 'zh'): string { + if (locale === 'en') return value; + return REASON_LABELS.zh[value as keyof typeof REASON_LABELS.zh] ?? value; +} + +function OutcomeBadge({ outcome }: { outcome: CollectiveXOutcome }) { + const t = STRINGS[useLocale()]; + return ( + + {t.outcome[outcome]} + + ); +} + +function shortId(value: string | null): string { + if (value === null) return '-'; + const suffix = value.lastIndexOf('-'); + return suffix === -1 ? value : value.slice(suffix + 1).slice(-8); +} + +export function CollectiveXCoverageTable({ coverage }: { coverage: CollectiveXCoverage[] }) { + const locale = useLocale(); + const t = STRINGS[locale]; + const columns = useMemo[]>( + () => [ + { + header: t.case, + cell: (row) => ( +
+

{row.label}

+

{shortId(row.case_id)}

+
+ ), + sortValue: (row) => row.label, + }, + { + header: t.sku, + cell: (row) => row.sku.toUpperCase(), + sortValue: (row) => row.sku, + }, + { + header: t.backend, + cell: (row) => row.backend, + sortValue: (row) => row.backend, + }, + { + header: t.phaseHeader, + cell: (row) => t.phase[row.phase], + sortValue: (row) => t.phase[row.phase], + }, + { + header: t.modeHeader, + cell: (row) => row.mode, + sortValue: (row) => row.mode, + }, + { + header: t.precisionHeader, + cell: (row) => row.precision, + sortValue: (row) => row.precision, + }, + { + header: t.epHeader, + cell: (row) => `EP${row.topology.ep_size}`, + sortValue: (row) => row.topology.ep_size, + }, + { + header: t.topologyHeader, + cell: (row) => collectiveXTopologyLabel(row.topology), + sortValue: (row) => collectiveXTopologyLabel(row.topology), + className: 'whitespace-nowrap', + }, + { + header: t.dispositionHeader, + cell: (row) => t.disposition[row.disposition], + sortValue: (row) => t.disposition[row.disposition], + }, + { + header: t.outcomeHeader, + cell: (row) => , + sortValue: (row) => t.outcome[row.outcome], + }, + { + header: t.reason, + cell: (row) => (row.reason ? collectiveXReasonLabel(row.reason, locale) : '-'), + sortValue: (row) => + row.reason ? `${collectiveXReasonLabel(row.reason, locale)} ${row.reason}` : '', + }, + ], + [locale, t], + ); + + return ( + +

{t.terminalCoverage}

+ +
+ ); +} diff --git a/packages/app/src/components/collectivex/data.test.ts b/packages/app/src/components/collectivex/data.test.ts new file mode 100644 index 000000000..2e0ad4719 --- /dev/null +++ b/packages/app/src/components/collectivex/data.test.ts @@ -0,0 +1,282 @@ +import { describe, expect, it } from 'vitest'; + +import { + chartPoints, + collectiveXColorKey, + collectiveXLegendLabel, + collectiveXRunDasharray, + collectiveXSeriesForRun, + collectiveXSeriesLabel, + collectiveXTopologyLabel, + fitAlphaBeta, + metricValue, + seriesMatchesSelection, + type CollectiveXSeriesSelection, +} from './data'; +import type { CollectiveXPercentiles, CollectiveXSeries } from './types'; +import { makeCollectiveXDataset, makeCollectiveXSeries } from './test-fixture'; + +const dataset = makeCollectiveXDataset(); +// series[0]: deepep-v2 EP8 scale-up (nvlink, single node). +// series[1]: MoRI EP16 scale-out (xGMI scale-up + RDMA scale-out, two nodes). +const [scaleUp, scaleOut] = dataset.series; + +describe('collectiveXTopologyLabel', () => { + it('shows only the scale-up transport when there is no scale-out fabric', () => { + expect(collectiveXTopologyLabel(scaleUp.system)).toBe( + '1x8 · domain 8 · nvlink · h200-nvlink-island', + ); + }); + + it('joins scale-up and scale-out transports when a scale-out fabric is present', () => { + expect(collectiveXTopologyLabel(scaleOut.system)).toBe( + '2x8 · domain 8 · xgmi+rdma · mi355x-xgmi-rdma', + ); + }); +}); + +describe('collectiveXSeriesLabel', () => { + it('renders the varying identity axes of a series', () => { + expect(collectiveXSeriesLabel(scaleUp)).toBe( + 'H200-DGXC · deepep-v2 · EP8 · normal · decode · bf16', + ); + }); + + it('distinguishes the dispatch precision', () => { + expect(collectiveXSeriesLabel(makeCollectiveXSeries({ precision: 'fp8' }))).toBe( + 'H200-DGXC · deepep-v2 · EP8 · normal · decode · fp8', + ); + }); + + it('shows the selected EP degree, mode, and phase', () => { + expect(collectiveXSeriesLabel(scaleOut)).toContain('EP16 · normal · decode'); + }); + + it('includes the run id for namespaced multi-run series', () => { + const [runSeries] = collectiveXSeriesForRun([scaleUp], '30165164821'); + expect(collectiveXSeriesLabel(runSeries)).toBe( + '#30165164821 · H200-DGXC · deepep-v2 · EP8 · normal · decode · bf16', + ); + }); + + it('keeps run ids out of the configuration label used by the legend', () => { + const [runSeries] = collectiveXSeriesForRun([scaleUp], '30165164821'); + expect(collectiveXLegendLabel(runSeries)).toBe( + 'H200-DGXC · deepep-v2 · EP8 · normal · decode · bf16', + ); + }); +}); + +describe('collectiveXColorKey', () => { + it('assigns the same key to two series with identical configuration', () => { + const a = makeCollectiveXSeries({ variant: 'a' }); + const b = makeCollectiveXSeries({ variant: 'b' }); + // The color key is configuration-derived and excludes the series id. + expect(collectiveXColorKey(a)).toBe(collectiveXColorKey(b)); + }); + + it('assigns distinct keys to series differing in EP degree', () => { + const a = makeCollectiveXSeries({ ep: 8 }); + const b = makeCollectiveXSeries({ ep: 16 }); + expect(collectiveXColorKey(a)).not.toBe(collectiveXColorKey(b)); + }); + + it('assigns distinct keys to bf16 and fp8 series of one configuration', () => { + const bf16 = makeCollectiveXSeries(); + const fp8 = makeCollectiveXSeries({ precision: 'fp8' }); + expect(collectiveXColorKey(bf16)).not.toBe(collectiveXColorKey(fp8)); + }); + + it('assigns distinct keys to normal and low-latency series of one configuration', () => { + const normal = makeCollectiveXSeries(); + const lowLatency = makeCollectiveXSeries({ mode: 'low-latency' }); + expect(collectiveXColorKey(normal)).not.toBe(collectiveXColorKey(lowLatency)); + }); + + it('leads with the system vendor so getVendor places series in vendor hue zones', () => { + // The chart color system reads the first "_"-separated token to classify the + // vendor (NVIDIA greens, AMD reds), matching the InferenceX charts. + expect(collectiveXColorKey(scaleUp).split('_')[0]).toBe('nvidia'); + const amd = makeCollectiveXSeries({ sku: 'mi355x', vendor: 'amd' }); + expect(collectiveXColorKey(amd).split('_')[0]).toBe('amd'); + }); + + it('keeps the same configuration color across different runs', () => { + const [first] = collectiveXSeriesForRun([scaleUp], '30165164821'); + const [second] = collectiveXSeriesForRun([scaleUp], '30177021271'); + expect(collectiveXColorKey(first)).toBe(collectiveXColorKey(second)); + }); +}); + +describe('collectiveXSeriesForRun', () => { + it('namespaces otherwise-colliding series ids without mutating the source', () => { + const originalId = scaleUp.series_id; + const [first] = collectiveXSeriesForRun([scaleUp], '30165164821', 2); + const [second] = collectiveXSeriesForRun([scaleUp], '30177021271', 3); + + expect(first.series_id).toBe(`30165164821:${originalId}`); + expect(first.run_id).toBe('30165164821'); + expect(first.run_index).toBe(2); + expect(second.series_id).toBe(`30177021271:${originalId}`); + expect(second.run_index).toBe(3); + expect(scaleUp.series_id).toBe(originalId); + }); +}); + +describe('collectiveXRunDasharray', () => { + it('uses a solid line for the newest run and distinct patterns for following runs', () => { + expect(collectiveXRunDasharray(0)).toBe('none'); + expect(collectiveXRunDasharray(1)).toBe('9 4'); + expect(collectiveXRunDasharray(2)).toBe('3 3'); + }); + + it('normalizes invalid negative and fractional indexes', () => { + expect(collectiveXRunDasharray(-1)).toBe('none'); + expect(collectiveXRunDasharray(1.9)).toBe('9 4'); + }); + + it('does not repeat styles after the base six patterns', () => { + expect(collectiveXRunDasharray(6)).not.toBe(collectiveXRunDasharray(0)); + expect(collectiveXRunDasharray(12)).not.toBe(collectiveXRunDasharray(6)); + expect(collectiveXRunDasharray(55)).not.toBe(collectiveXRunDasharray(6)); + }); +}); + +describe('seriesMatchesSelection', () => { + const base: CollectiveXSeriesSelection = { + epSize: 8, + phase: 'decode', + mode: 'normal', + precision: 'bf16', + }; + + it('matches on EP size, phase, mode, and precision', () => { + expect(seriesMatchesSelection(scaleUp, base)).toBe(true); + }); + + it('rejects a series whose EP, phase, mode, or precision differs from the selection', () => { + expect(seriesMatchesSelection(scaleUp, { ...base, epSize: 16 })).toBe(false); + expect(seriesMatchesSelection(scaleUp, { ...base, phase: 'prefill' })).toBe(false); + expect(seriesMatchesSelection(scaleUp, { ...base, mode: 'low-latency' })).toBe(false); + expect(seriesMatchesSelection(scaleUp, { ...base, precision: 'fp8' })).toBe(false); + }); +}); + +describe('metricValue', () => { + const point = scaleUp.points[0]; + + it('returns the latency percentile for the requested component', () => { + expect(metricValue(point, 'dispatch', 'p50', 'latency')).toBe( + point.components.dispatch?.latency_us.p50, + ); + }); + + it('returns the roundtrip token rate only for the roundtrip operation', () => { + expect(metricValue(point, 'roundtrip', 'p50', 'tokens-per-second')).toBe( + point.roundtrip_token_rate_at_latency_percentile.p50, + ); + expect(metricValue(point, 'dispatch', 'p50', 'tokens-per-second')).toBeNull(); + }); + + it('returns the activation data rate', () => { + expect(metricValue(point, 'dispatch', 'p50', 'activation-rate')).toBeGreaterThan(0); + }); + + it('returns the per-GPU payload bandwidth distinct from the activation rate', () => { + const payload = metricValue(point, 'dispatch', 'p50', 'payload-rate'); + const activation = metricValue(point, 'dispatch', 'p50', 'activation-rate'); + expect(payload).toBeGreaterThan(0); + // Payload uses total_logical_bytes ÷ ep; activation uses aggregate activation bytes. + expect(payload).not.toBeCloseTo(activation as number, 1); + }); + + it('returns null for an unavailable component', () => { + const unavailable = makeCollectiveXSeries({ rows: [{ stageUnavailable: true }] }).points[0]; + expect(metricValue(unavailable, 'stage', 'p50', 'latency')).toBeNull(); + }); +}); + +function pct(value: number): CollectiveXPercentiles { + return { p50: value, p90: value, p95: value, p99: value }; +} + +describe('fitAlphaBeta', () => { + // Build a series whose dispatch latency is exactly α + bytesPerGpu/β with + // α = 10 µs and β = 250 GB/s (per GPU), so the OLS must recover both. ep = 8, + // so aggregate payload_bytes = bytesPerGpu × 8 and fitAlphaBeta divides it back. + const EP = 8; + function fitSeries(): CollectiveXSeries { + const point = (bytesPerGpu: number) => { + const latency = 10 + bytesPerGpu / (250 * 1e3); // µs + const rate = (bytesPerGpu / latency) * 1e-3; // GB/s per GPU + const component = { + latency_us: pct(latency), + activation_data_rate_gbps_at_latency_percentile: pct(rate), + payload_data_rate_gbps_at_latency_percentile: pct(rate), + payload_bytes: bytesPerGpu * EP, + }; + return { + tokens_per_rank: bytesPerGpu / 1e4, + global_tokens: bytesPerGpu / 1e3, + components: { dispatch: component, stage: null, combine: component, roundtrip: component }, + roundtrip_token_rate_at_latency_percentile: pct(1), + }; + }; + return { + series_id: 'fit-series', + phase: 'decode', + mode: 'normal', + precision: 'bf16', + backend: 'nccl-ep', + system: { + ep_size: 8, + nodes: 1, + gpus_per_node: 8, + scale_up_domain: 8, + scale_up_transport: 'nvlink', + scale_out_transport: null, + topology_class: 'h100-nvlink-island', + sku: 'h100', + vendor: 'nvidia', + }, + points: [point(1e6), point(2e6), point(3e6)], + }; + } + + it('recovers the fixed overhead (alpha) and per-GPU bandwidth (beta)', () => { + const fit = fitAlphaBeta(fitSeries(), 'dispatch'); + expect(fit).not.toBeNull(); + expect(fit?.alphaUs).toBeCloseTo(10, 3); + expect(fit?.betaGbps).toBeCloseTo(250, 3); + expect(fit?.pointCount).toBe(3); + }); + + it('returns null when the byte axis has no variance across the ladder', () => { + // The default fixture holds bytes constant across the ladder, so bytes/GPU + // reconstructs to a single value and there is no slope to fit. + expect(fitAlphaBeta(makeCollectiveXSeries(), 'dispatch')).toBeNull(); + }); +}); + +describe('chartPoints', () => { + it('emits one point per token row with populated axes', () => { + const points = chartPoints([scaleUp], 'dispatch', 'p50', 'latency'); + expect(points).toHaveLength(scaleUp.points.length); + for (const point of points) { + expect(point.seriesId).toBe(scaleUp.series_id); + expect(point.colorKey).toBe(collectiveXColorKey(scaleUp)); + expect(point.x).toBeGreaterThan(0); + expect(point.y).toBeGreaterThan(0); + } + }); + + it('drops points whose metric is unavailable', () => { + // Dispatch has no per-operation token rate, so tokens-per-second is null everywhere. + expect(chartPoints([scaleUp], 'dispatch', 'p50', 'tokens-per-second')).toHaveLength(0); + }); + + it('keeps roundtrip token-rate points', () => { + const points = chartPoints([scaleUp], 'roundtrip', 'p50', 'tokens-per-second'); + expect(points).toHaveLength(scaleUp.points.length); + }); +}); diff --git a/packages/app/src/components/collectivex/data.ts b/packages/app/src/components/collectivex/data.ts new file mode 100644 index 000000000..1b3cc6aeb --- /dev/null +++ b/packages/app/src/components/collectivex/data.ts @@ -0,0 +1,200 @@ +import type { + CollectiveXChartPoint, + CollectiveXComponent, + CollectiveXMode, + CollectiveXOperation, + CollectiveXPercentile, + CollectiveXPhase, + CollectiveXPoint, + CollectiveXPrecision, + CollectiveXRunSeries, + CollectiveXSeries, + CollectiveXYAxis, +} from './types'; + +export interface CollectiveXSeriesSelection { + epSize: number; + phase: CollectiveXPhase; + mode: CollectiveXMode; + precision: CollectiveXPrecision; +} + +const BASE_RUN_DASHARRAYS = ['none', '9 4', '3 3', '10 3 2 3', '2 3', '12 3 2 3'] as const; + +/** + * Stable within the newest-first run table. Preserve the six simple patterns, + * then encode higher indexes as a unique dash/gap/accent tuple so two listed + * runs never become visually identical merely because their indexes differ by + * six. + */ +export function collectiveXRunDasharray(runIndex: number): string { + const normalized = Math.max(0, Math.trunc(runIndex)); + if (normalized < BASE_RUN_DASHARRAYS.length) return BASE_RUN_DASHARRAYS[normalized]; + const encoded = normalized - BASE_RUN_DASHARRAYS.length; + const dash = 4 + (encoded % 7); + const gap = 3 + (Math.floor(encoded / 7) % 7); + const accent = 1 + Math.floor(encoded / 49); + return `${dash} 3 ${accent} ${gap}`; +} + +export function collectiveXTopologyLabel( + system: Pick< + CollectiveXSeries['system'], + | 'nodes' + | 'gpus_per_node' + | 'scale_up_domain' + | 'scale_up_transport' + | 'scale_out_transport' + | 'topology_class' + >, +): string { + const transports = system.scale_out_transport + ? `${system.scale_up_transport}+${system.scale_out_transport}` + : system.scale_up_transport; + return `${system.nodes}x${system.gpus_per_node} · domain ${system.scale_up_domain} · ${transports} · ${system.topology_class}`; +} + +export function collectiveXLegendLabel(series: CollectiveXSeries): string { + return `${series.system.sku.toUpperCase()} · ${series.backend} · EP${series.system.ep_size} · ${series.mode} · ${series.phase} · ${series.precision}`; +} + +export function collectiveXSeriesLabel(series: CollectiveXSeries | CollectiveXRunSeries): string { + const runPrefix = 'run_id' in series ? `#${series.run_id} · ` : ''; + return `${runPrefix}${collectiveXLegendLabel(series)}`; +} + +export function collectiveXColorKey(series: CollectiveXSeries | CollectiveXRunSeries): string { + return `${series.system.vendor}_${series.system.sku}_${series.backend}_ep${series.system.ep_size}_${series.mode}_${series.phase}_${series.precision}`; +} + +/** Namespace series ids and attach the run's stable visual-style index. */ +export function collectiveXSeriesForRun( + series: readonly CollectiveXSeries[], + runId: string, + runIndex = 0, +): CollectiveXRunSeries[] { + return series.map((item) => ({ + ...item, + series_id: `${runId}:${item.series_id}`, + run_id: runId, + run_index: runIndex, + })); +} + +export function seriesMatchesSelection( + series: CollectiveXSeries, + selection: CollectiveXSeriesSelection, +): boolean { + return ( + series.system.ep_size === selection.epSize && + series.phase === selection.phase && + series.mode === selection.mode && + series.precision === selection.precision + ); +} + +export function metricValue( + point: CollectiveXPoint, + operation: CollectiveXOperation, + percentile: CollectiveXPercentile, + yAxis: CollectiveXYAxis, +): number | null { + const component: CollectiveXComponent | null = point.components[operation]; + if (component === null) return null; + if (yAxis === 'latency') return component.latency_us[percentile]; + if (yAxis === 'tokens-per-second') { + return operation === 'roundtrip' + ? point.roundtrip_token_rate_at_latency_percentile[percentile] + : null; + } + if (yAxis === 'payload-rate') { + return component.payload_data_rate_gbps_at_latency_percentile?.[percentile] ?? null; + } + return component.activation_data_rate_gbps_at_latency_percentile?.[percentile] ?? null; +} + +interface CollectiveXFit { + /** Fixed per-call overhead (µs): the launch/sync/rendezvous floor. */ + alphaUs: number; + /** Per-GPU bandwidth term (GB/s): the slope of latency vs bytes. */ + betaGbps: number; + /** Points that entered the fit. */ + pointCount: number; +} + +/** Ordinary least squares; returns [intercept, slope]. */ +function ols(xs: number[], ys: number[]): [number, number] { + const n = xs.length; + const meanX = xs.reduce((a, b) => a + b, 0) / n; + const meanY = ys.reduce((a, b) => a + b, 0) / n; + let sxx = 0; + let sxy = 0; + for (let i = 0; i < n; i++) { + sxx += (xs[i] - meanX) ** 2; + sxy += (xs[i] - meanX) * (ys[i] - meanY); + } + const slope = sxy / sxx; + return [meanY - slope * meanX, slope]; +} + +/** + * Separate the bandwidth term (β) from the fixed overhead (α) for one operation + * across a series' token ladder: latency(bytes) ≈ α + bytes/β. Regresses the + * per-point latency against the per-GPU payload bytes (raw `payload_bytes` ÷ + * ep_size), so β lands in the same per-GPU GB/s units as the payload-rate axis + * and α is the fixed overhead in µs. p50 by default (p99 carries tail noise). + * Mirrors experimental/CollectiveX/bandwidth.py. Returns null when fewer than + * three points, a degenerate (near-zero-variance) byte axis — e.g. a constant + * payload across the ladder — or a non-positive slope leaves no bandwidth term. + */ +export function fitAlphaBeta( + series: CollectiveXSeries, + operation: CollectiveXOperation, + percentile: CollectiveXPercentile = 'p50', +): CollectiveXFit | null { + const ep = Math.max(1, series.system.ep_size); + const bytesPerGpu: number[] = []; + const latencies: number[] = []; + for (const point of series.points) { + const component = point.components[operation]; + if (component === null || component.payload_bytes === null) continue; + const latency = component.latency_us[percentile]; + if (latency <= 0) continue; + bytesPerGpu.push(component.payload_bytes / ep); + latencies.push(latency); + } + if (bytesPerGpu.length < 3) return null; + // Reject a near-constant byte axis (constant payload across the ladder): a + // relative spread this small carries no real slope, only numeric noise. + const min = Math.min(...bytesPerGpu); + const max = Math.max(...bytesPerGpu); + if (max - min <= 1e-9 * max) return null; + const [alphaUs, slopeUsPerByte] = ols(bytesPerGpu, latencies); + if (slopeUsPerByte <= 0) return null; + return { alphaUs, betaGbps: 1e-3 / slopeUsPerByte, pointCount: bytesPerGpu.length }; +} + +export function chartPoints( + series: CollectiveXSeries[], + operation: CollectiveXOperation, + percentile: CollectiveXPercentile, + yAxis: CollectiveXYAxis, +): CollectiveXChartPoint[] { + return series.flatMap((item) => + item.points.flatMap((point) => { + const x = point.tokens_per_rank; + const y = metricValue(point, operation, percentile, yAxis); + if (!Number.isFinite(x) || x <= 0 || y === null || y <= 0 || !Number.isFinite(y)) return []; + return [ + { + seriesId: item.series_id, + seriesLabel: collectiveXSeriesLabel(item), + colorKey: collectiveXColorKey(item), + x, + y, + point, + }, + ]; + }), + ); +} diff --git a/packages/app/src/components/collectivex/test-fixture.ts b/packages/app/src/components/collectivex/test-fixture.ts new file mode 100644 index 000000000..8551afa82 --- /dev/null +++ b/packages/app/src/components/collectivex/test-fixture.ts @@ -0,0 +1,7 @@ +/** + * Re-export of the CollectiveX contract fixture builders, which live next to + * the shared reader in the db package. Kept at this path so component tests + * and cypress specs import fixtures the same way as the rest of this folder. + */ + +export * from '@semianalysisai/inferencex-db/collectivex/test-fixture'; diff --git a/packages/app/src/components/collectivex/types.ts b/packages/app/src/components/collectivex/types.ts new file mode 100644 index 000000000..13a144238 --- /dev/null +++ b/packages/app/src/components/collectivex/types.ts @@ -0,0 +1,60 @@ +/** + * UI-side CollectiveX types. The neutral contract (dataset/series/coverage + * shapes, version constants, reader) lives in the db package so the ingest + * script and this frontend share one source of truth — see + * `packages/db/src/collectivex/`. + */ + +import type { + CollectiveXPoint, + CollectiveXSeries, + CollectiveXVersion, +} from '@semianalysisai/inferencex-db/collectivex/types'; + +// Re-exported explicitly rather than with `export *`: Next's SWC loader rejects a +// star re-export anywhere in a page's module graph ("Using `export * from '...'` in +// a page is disallowed"), and this module is reached from the /collectivex page. +export type { + CollectiveXComponent, + CollectiveXCoverage, + CollectiveXCoveragePoint, + CollectiveXDataset, + CollectiveXMode, + CollectiveXOperation, + CollectiveXOutcome, + CollectiveXPercentile, + CollectiveXPercentiles, + CollectiveXPhase, + CollectiveXPoint, + CollectiveXPrecision, + CollectiveXRun, + CollectiveXRunSummary, + CollectiveXSeries, + CollectiveXTerminalStatus, + CollectiveXTopology, + CollectiveXVersion, +} from '@semianalysisai/inferencex-db/collectivex/types'; +export { + COLLECTIVEX_DEFAULT_VERSION, + COLLECTIVEX_VERSIONS, + parseCollectiveXVersion, +} from '@semianalysisai/inferencex-db/collectivex/types'; + +export const collectiveXVersionLabel = (version: CollectiveXVersion): string => `V${version}`; + +export type CollectiveXYAxis = 'latency' | 'tokens-per-second' | 'activation-rate' | 'payload-rate'; + +export interface CollectiveXChartPoint { + seriesId: string; + seriesLabel: string; + colorKey: string; + x: number; + y: number; + point: CollectiveXPoint; +} + +/** A stored run's series with namespaced identity and visual-style index. */ +export type CollectiveXRunSeries = CollectiveXSeries & { + run_id: string; + run_index: number; +}; diff --git a/packages/app/src/components/dashboard-shell.tsx b/packages/app/src/components/dashboard-shell.tsx index 17eb13868..d44ff8432 100644 --- a/packages/app/src/components/dashboard-shell.tsx +++ b/packages/app/src/components/dashboard-shell.tsx @@ -4,8 +4,19 @@ import { GlobalFilterProvider } from '@/components/GlobalFilterContext'; import { NudgeEngine } from '@/components/nudge-engine'; import { TabNav } from '@/components/tab-nav'; import { UnofficialRunProvider } from '@/components/unofficial-run-provider'; +import { usePathname } from 'next/navigation'; export function DashboardShell({ children }: { children: React.ReactNode }) { + const pathname = usePathname(); + const content = ( +
+
+ + {children} +
+
+ ); + if (pathname === '/collectivex' || pathname === '/zh/collectivex') return content; return ( <> diff --git a/packages/app/src/components/header/header.tsx b/packages/app/src/components/header/header.tsx index 021d57956..9a8507d47 100644 --- a/packages/app/src/components/header/header.tsx +++ b/packages/app/src/components/header/header.tsx @@ -25,6 +25,7 @@ const DASHBOARD_TABS = [ '/reliability', '/gpu-specs', '/gpu-metrics', + '/collectivex', '/submissions', '/current-inferencex-image', ]; diff --git a/packages/app/src/components/tab-nav.tsx b/packages/app/src/components/tab-nav.tsx index ec99db5f1..edc3e1265 100644 --- a/packages/app/src/components/tab-nav.tsx +++ b/packages/app/src/components/tab-nav.tsx @@ -32,6 +32,7 @@ const VISIBLE_TABS = [ { href: '/historical', label: 'Historical Trends', testId: 'tab-trigger-historical' }, { href: '/calculator', label: 'TCO Calculator', testId: 'tab-trigger-calculator' }, { href: '/gpu-specs', label: 'GPU Specs', testId: 'tab-trigger-gpu-specs' }, + { href: '/collectivex', label: 'CollectiveX', testId: 'tab-trigger-collectivex' }, { href: '/submissions', label: 'Submissions', testId: 'tab-trigger-submissions' }, ] as const; diff --git a/packages/app/src/components/ui/chart-legend-item.tsx b/packages/app/src/components/ui/chart-legend-item.tsx index ee2f0241e..69b1a0d6a 100644 --- a/packages/app/src/components/ui/chart-legend-item.tsx +++ b/packages/app/src/components/ui/chart-legend-item.tsx @@ -22,6 +22,8 @@ export interface CommonLegendItemProps { hw?: string; label: string; color: string; + /** When present, render a line swatch using this SVG dash pattern instead of a dot. */ + lineDasharray?: string; isActive: boolean; isHighlighted?: boolean; onClick: (name: string) => void; @@ -33,6 +35,7 @@ export interface CommonLegendItemProps { isLegendExpanded?: boolean; // Whether the legend is expanded to show full text sidebarMode?: boolean; // Use sidebar-style visual feedback (line-through + faded dot) onRemove?: (name: string) => void; + hideAriaLabel?: string; /** * Set false for entries that are labels rather than toggleable series — e.g. * the unofficial-run entries, which are always `isActive` and have nothing to @@ -47,12 +50,14 @@ export interface CommonLegendItemProps { * inference tab's legend passes this — other tabs get no icon. */ onShowPoints?: (name: string) => void; + showPointsAriaLabel?: string; } const ChartLegendItem: React.FC = ({ name, label, color, + lineDasharray, title, isActive, onClick, @@ -65,6 +70,8 @@ const ChartLegendItem: React.FC = ({ sidebarMode = false, onRemove, onShowPoints, + hideAriaLabel, + showPointsAriaLabel, }) => { const locale = useLocale(); const t = STRINGS[locale]; @@ -96,14 +103,44 @@ const ChartLegendItem: React.FC = ({ onMouseEnter={onHover && isActive ? () => onHover(hw || name) : undefined} onMouseLeave={onHoverEnd && isActive ? onHoverEnd : undefined} > - - + + {lineDasharray === undefined ? ( + + ) : ( + + )} {canRemove && ( = ({ onRemove!(hw || name); }} className="absolute inset-0 inline-flex items-center justify-center opacity-0 group-hover/item:opacity-100" - aria-label={t.hide(label)} + aria-label={hideAriaLabel ?? t.hide(label)} title={t.hide(label)} > @@ -147,7 +184,7 @@ const ChartLegendItem: React.FC = ({ diff --git a/packages/app/src/components/ui/searchable-select.test.ts b/packages/app/src/components/ui/searchable-select.test.ts index 70b2aa8ef..275774f32 100644 --- a/packages/app/src/components/ui/searchable-select.test.ts +++ b/packages/app/src/components/ui/searchable-select.test.ts @@ -56,7 +56,7 @@ function openMenu() { // internal value tracker thinks nothing changed. Use the native HTMLInputElement // setter so React picks up the change and fires onChange in jsdom. function setSearchValue(value: string) { - const input = container.querySelector('input[placeholder="Search..."]') as HTMLInputElement; + const input = document.body.querySelector('input[type="text"]') as HTMLInputElement; const nativeSetter = Object.getOwnPropertyDescriptor( window.HTMLInputElement.prototype, 'value', @@ -83,7 +83,7 @@ describe('SearchableSelect', () => { it('shows all groups and options when opened', () => { render(); openMenu(); - const items = container.querySelectorAll('[data-slot="select-item"]'); + const items = document.body.querySelectorAll('[data-slot="select-item"]'); expect(items).toHaveLength(3); }); @@ -91,7 +91,7 @@ describe('SearchableSelect', () => { render(); openMenu(); setSearchValue('input'); - const items = container.querySelectorAll('[data-slot="select-item"]'); + const items = document.body.querySelectorAll('[data-slot="select-item"]'); expect(items).toHaveLength(1); expect(items[0]?.textContent).toContain('Input Token Throughput per GPU'); }); @@ -100,7 +100,7 @@ describe('SearchableSelect', () => { render(); openMenu(); setSearchValue('cost'); - const items = container.querySelectorAll('[data-slot="select-item"]'); + const items = document.body.querySelectorAll('[data-slot="select-item"]'); expect(items).toHaveLength(1); expect(items[0]?.textContent).toContain('Cost per Million Total Tokens (Hyperscaler)'); }); @@ -109,22 +109,84 @@ describe('SearchableSelect', () => { render(); openMenu(); setSearchValue('zzzzz'); - const items = container.querySelectorAll('[data-slot="select-item"]'); + const items = document.body.querySelectorAll('[data-slot="select-item"]'); expect(items).toHaveLength(0); - expect(container.textContent).toContain('No results'); + expect(document.body.textContent).toContain('No results'); + }); + + it('uses localized search controls and empty state labels', () => { + render({ + searchPlaceholder: '搜索队列...', + searchAriaLabel: '搜索 CollectiveX 队列', + clearSearchLabel: '清除队列搜索', + noResultsLabel: '无匹配队列', + }); + openMenu(); + const input = document.body.querySelector('input[type="text"]') as HTMLInputElement; + expect(input.placeholder).toBe('搜索队列...'); + expect(input.getAttribute('aria-label')).toBe('搜索 CollectiveX 队列'); + setSearchValue('zzzzz'); + expect(document.body.textContent).toContain('无匹配队列'); + expect(document.body.querySelector('button[aria-label="清除队列搜索"]')).not.toBeNull(); + }); + + it('filters the canonical 280-cohort publication scale without truncating it', () => { + const groups = [ + ['Library', 76], + ['Platform', 76], + ['Reference system', 12], + ['Routing', 116], + ].map(([label, size]) => ({ + label: `${label} comparisons`, + options: Array.from({ length: Number(size) }, (_, index) => ({ + value: `${label}-cohort-${index}`, + label: `${label} cohort ${index}`, + })), + })); + render({ groups, value: 'Library-cohort-0' }); + openMenu(); + expect(document.body.querySelectorAll('[data-slot="select-item"]')).toHaveLength(280); + setSearchValue('Routing comparisons'); + expect(document.body.querySelectorAll('[data-slot="select-item"]')).toHaveLength(116); + setSearchValue('Routing cohort 115'); + const items = document.body.querySelectorAll('[data-slot="select-item"]'); + expect(items).toHaveLength(1); + expect(items[0]?.textContent).toContain('Routing cohort 115'); }); it('invokes onValueChange and closes the menu when an option is clicked', () => { const handle = vi.fn(); render({ onValueChange: handle }); openMenu(); - const items = container.querySelectorAll('[data-slot="select-item"]'); + const items = document.body.querySelectorAll('[data-slot="select-item"]'); const target = [...items].find((el) => el.textContent?.includes('Input Token Throughput per GPU'), ) as HTMLDivElement; act(() => target.click()); expect(handle).toHaveBeenCalledExactlyOnceWith('y_inputTputPerGpu'); // Menu closed → no select-item visible - expect(container.querySelectorAll('[data-slot="select-item"]')).toHaveLength(0); + expect(document.body.querySelectorAll('[data-slot="select-item"]')).toHaveLength(0); + }); + + it('moves through options and selects from the keyboard', () => { + const handle = vi.fn(); + render({ onValueChange: handle }); + openMenu(); + const input = document.body.querySelector('input[type="text"]') as HTMLInputElement; + const options = document.body.querySelectorAll('[data-slot="select-item"]'); + expect([...options].every((item) => item.tabIndex === -1)).toBe(true); + act(() => + input.dispatchEvent(new KeyboardEvent('keydown', { key: 'ArrowDown', bubbles: true })), + ); + expect(document.activeElement).toBe(options[0]); + act(() => + options[0]?.dispatchEvent(new KeyboardEvent('keydown', { key: 'ArrowDown', bubbles: true })), + ); + expect(document.activeElement).toBe(options[1]); + act(() => + options[1]?.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true })), + ); + expect(handle).toHaveBeenCalledExactlyOnceWith('y_inputTputPerGpu'); + expect(document.body.querySelectorAll('[data-slot="select-item"]')).toHaveLength(0); }); }); diff --git a/packages/app/src/components/ui/searchable-select.tsx b/packages/app/src/components/ui/searchable-select.tsx index 3b1a5d25a..1c7d80ad5 100644 --- a/packages/app/src/components/ui/searchable-select.tsx +++ b/packages/app/src/components/ui/searchable-select.tsx @@ -3,6 +3,7 @@ import { CheckIcon, ChevronDownIcon, SearchIcon, XIcon } from 'lucide-react'; import * as React from 'react'; +import { Popover, PopoverContent, PopoverTrigger } from '@/components/ui/popover'; import { track } from '@/lib/analytics'; import { cn } from '@/lib/utils'; @@ -26,11 +27,12 @@ interface SearchableSelectProps { triggerTestId?: string; disabled?: boolean; searchable?: boolean; - /** Analytics event prefix, e.g. "yaxis_metric" → "yaxis_metric_searched" */ - trackPrefix?: string; searchPlaceholder?: string; - noResultsLabel?: string; + searchAriaLabel?: string; clearSearchLabel?: string; + noResultsLabel?: string; + /** Analytics event prefix, e.g. "yaxis_metric" → "yaxis_metric_searched" */ + trackPrefix?: string; } export function SearchableSelect({ @@ -43,10 +45,11 @@ export function SearchableSelect({ triggerTestId, disabled = false, searchable = true, - trackPrefix, searchPlaceholder = 'Search...', - noResultsLabel = 'No results', + searchAriaLabel = 'Search options', clearSearchLabel = 'Clear search', + noResultsLabel = 'No results', + trackPrefix, }: SearchableSelectProps) { const [isOpen, setIsOpen] = React.useState(false); const [search, setSearch] = React.useState(''); @@ -56,8 +59,8 @@ export function SearchableSelect({ // resolve client-side, so SSR would otherwise lock in the default label and // leave it stale after hydration. const [mounted, setMounted] = React.useState(false); - const containerRef = React.useRef(null); const searchRef = React.useRef(null); + const listboxRef = React.useRef(null); const searchUsedRef = React.useRef(false); React.useEffect(() => { @@ -65,33 +68,13 @@ export function SearchableSelect({ }, []); React.useEffect(() => { - const handleClickOutside = (event: MouseEvent) => { - if (containerRef.current && !containerRef.current.contains(event.target as Node)) { - setIsOpen(false); - } - }; - const handleKeyDown = (event: KeyboardEvent) => { - if (event.key === 'Escape') { - event.stopPropagation(); - setIsOpen(false); - } - }; - - if (isOpen) { - document.addEventListener('mousedown', handleClickOutside); - document.addEventListener('keydown', handleKeyDown); - searchRef.current?.focus(); - } else { + if (!isOpen) { if (searchUsedRef.current && trackPrefix) { track(`${trackPrefix}_searched`, { query: search }); searchUsedRef.current = false; } setSearch(''); } - return () => { - document.removeEventListener('mousedown', handleClickOutside); - document.removeEventListener('keydown', handleKeyDown); - }; }, [isOpen, search, trackPrefix]); const filteredGroups = React.useMemo(() => { @@ -120,49 +103,79 @@ export function SearchableSelect({ onValueChange(optionValue); setIsOpen(false); }; + const focusOption = (index: number) => { + const options = listboxRef.current?.querySelectorAll('[role="option"]'); + options?.[Math.max(0, Math.min(index, options.length - 1))]?.focus(); + }; + const handleOptionKeyDown = (event: React.KeyboardEvent, optionValue: string) => { + if (event.key === 'Enter' || event.key === ' ') { + event.preventDefault(); + handleSelect(optionValue); + return; + } + const options = [...(listboxRef.current?.querySelectorAll('[role="option"]') ?? [])]; + const current = options.indexOf(event.currentTarget); + const target = + event.key === 'ArrowDown' + ? current + 1 + : event.key === 'ArrowUp' + ? current - 1 + : event.key === 'Home' + ? 0 + : event.key === 'End' + ? options.length - 1 + : null; + if (target !== null) { + event.preventDefault(); + focusOption(target); + } + }; return ( -
- - - {isOpen && ( -
!disabled && setIsOpen(open)}> +
+ + + + { + event.preventDefault(); + searchRef.current?.focus(); + }} + className="z-[100] w-[var(--radix-popover-trigger-width)] overflow-hidden p-0 data-[state=open]:animate-none data-[state=closed]:animate-none" > {/* Search header lives outside the scrollable region so it never picks up * `sticky` → `position: fixed` resolution that puts it behind the page @@ -178,7 +191,17 @@ export function SearchableSelect({ setSearch(e.target.value); if (e.target.value) searchUsedRef.current = true; }} + onKeyDown={(event) => { + if (event.key === 'ArrowDown') { + event.preventDefault(); + focusOption(0); + } else if (event.key === 'ArrowUp') { + event.preventDefault(); + focusOption(Number.MAX_SAFE_INTEGER); + } + }} placeholder={searchPlaceholder} + aria-label={searchAriaLabel} className="w-full bg-transparent py-1.5 text-sm outline-none placeholder:text-muted-foreground" /> {search && ( @@ -197,6 +220,7 @@ export function SearchableSelect({
)}
handleSelect(option.value)} + onKeyDown={(event) => handleOptionKeyDown(event, option.value)} className={cn( "focus:bg-accent focus:text-accent-foreground [&_svg:not([class*='text-'])]:text-muted-foreground relative flex w-full cursor-pointer items-center gap-2 rounded-sm py-1.5 pr-8 pl-2 text-sm outline-hidden select-none transition-all duration-150 ease-in-out", 'hover:bg-primary/20 hover:pl-3 hover:shadow-sm', @@ -237,8 +263,8 @@ export function SearchableSelect({
))}
-
- )} - + + + ); } diff --git a/packages/app/src/hooks/api/use-collectivex.ts b/packages/app/src/hooks/api/use-collectivex.ts new file mode 100644 index 000000000..30792ea31 --- /dev/null +++ b/packages/app/src/hooks/api/use-collectivex.ts @@ -0,0 +1,53 @@ +import { useMutation, useQueries, useQuery, useQueryClient } from '@tanstack/react-query'; + +import { deleteCollectiveXRun, fetchCollectiveXRun, fetchCollectiveXRunList } from '@/lib/api'; +import { + COLLECTIVEX_DEFAULT_VERSION, + type CollectiveXVersion, +} from '@/components/collectivex/types'; + +/** + * Every stored run summary for a version. While the server reports an + * incomplete discovery pass, keep refetching the uncached endpoint so bounded + * ingest batches progressively fill the table. + */ +export function useCollectiveXRuns(version: CollectiveXVersion = COLLECTIVEX_DEFAULT_VERSION) { + return useQuery({ + queryKey: ['collectivex-runs', version], + queryFn: ({ signal }) => fetchCollectiveXRunList(version, signal), + staleTime: 0, + refetchOnMount: 'always', + refetchInterval: (query) => (query.state.data?.discovery_complete === false ? 1_000 : false), + }); +} + +/** Resolve every checked run in parallel for the multi-run explorer. */ +export function useCollectiveXRunDatasets(version: CollectiveXVersion, runIds: readonly string[]) { + return useQueries({ + queries: runIds.map((runId) => ({ + queryKey: ['collectivex-run', version, runId], + queryFn: ({ signal }: { signal: AbortSignal }) => fetchCollectiveXRun(version, runId, signal), + staleTime: 0, + refetchOnMount: 'always' as const, + })), + }); +} + +/** + * Admin deletion of an ingested run. Resolves `false` on 401 (stale token — + * the caller clears its stored copy); on success every CollectiveX query is + * invalidated so the run table and selected datasets drop it immediately. + */ +export function useDeleteCollectiveXRun() { + const queryClient = useQueryClient(); + return useMutation({ + mutationFn: ({ runId, token }: { runId: string; token: string }) => + deleteCollectiveXRun(runId, token), + onSuccess: (deleted) => { + if (!deleted) return; + for (const key of ['collectivex-runs', 'collectivex-run']) { + void queryClient.invalidateQueries({ queryKey: [key] }); + } + }, + }); +} diff --git a/packages/app/src/lib/allowed-dev-origins.test.ts b/packages/app/src/lib/allowed-dev-origins.test.ts index 0577e3ce0..44e8ae3ce 100644 --- a/packages/app/src/lib/allowed-dev-origins.test.ts +++ b/packages/app/src/lib/allowed-dev-origins.test.ts @@ -11,7 +11,7 @@ describe('allowedDevOriginsFromEnv', () => { it('trims whitespace and removes empty entries', () => { expect( - allowedDevOriginsFromEnv(' 10.112.9.49 , , local-origin.dev , *.local-origin.dev '), - ).toEqual(['10.112.9.49', 'local-origin.dev', '*.local-origin.dev']); + allowedDevOriginsFromEnv(' dev-machine.local , , local-origin.dev , *.local-origin.dev '), + ).toEqual(['dev-machine.local', 'local-origin.dev', '*.local-origin.dev']); }); }); diff --git a/packages/app/src/lib/allowed-dev-origins.ts b/packages/app/src/lib/allowed-dev-origins.ts index 839a486ff..82fac506a 100644 --- a/packages/app/src/lib/allowed-dev-origins.ts +++ b/packages/app/src/lib/allowed-dev-origins.ts @@ -1,4 +1,4 @@ -/** Comma-separated hostnames or IPs (e.g. `10.112.9.49,192.168.1.10`). Only used in dev. */ +/** Comma-separated dev hostnames (e.g. `dev-a.local,dev-b.local`). */ export function allowedDevOriginsFromEnv(raw = process.env.NEXT_DEV_ALLOWED_ORIGINS): string[] { if (!raw?.trim()) return []; return raw diff --git a/packages/app/src/lib/api-cache.test.ts b/packages/app/src/lib/api-cache.test.ts index 9b5ac039f..c10661621 100644 --- a/packages/app/src/lib/api-cache.test.ts +++ b/packages/app/src/lib/api-cache.test.ts @@ -16,7 +16,7 @@ vi.mock('./blob-cache', () => ({ blobPurge: vi.fn(), })); -import { cachedQuery, purgeAll, cachedJson } from './api-cache'; +import { cachedQuery, purgeAll, purgeCollectiveX, cachedJson } from './api-cache'; import { revalidateTag, unstable_cache } from 'next/cache'; import { blobGet, blobSet, blobPurge } from './blob-cache'; @@ -155,6 +155,14 @@ describe('purgeAll', () => { expect(mockBlobPurge).toHaveBeenCalled(); }); + it('drops the CollectiveX tag too — the blob purge removed its entries', async () => { + mockBlobPurge.mockResolvedValue(1); + + await purgeAll(); + + expect(mockRevalidateTag).toHaveBeenCalledWith('collectivex', { expire: 0 }); + }); + it('returns 0 when no blobs were deleted', async () => { mockBlobPurge.mockResolvedValue(0); @@ -172,6 +180,16 @@ describe('purgeAll', () => { }); }); +describe('purgeCollectiveX', () => { + it('drops only the collectivex tag — no blobs belong to that scope', () => { + purgeCollectiveX(); + + expect(mockBlobPurge).not.toHaveBeenCalled(); + expect(mockRevalidateTag).toHaveBeenCalledWith('collectivex', { expire: 0 }); + expect(mockRevalidateTag).not.toHaveBeenCalledWith('db', { expire: 0 }); + }); +}); + describe('cachedJson', () => { it('sets Cache-Control with max-age=0 and 1 day s-maxage', () => { const res = cachedJson({ ok: true }); @@ -183,6 +201,20 @@ describe('cachedJson', () => { expect(res.headers.get('Vercel-Cache-Tag')).toBe('db'); }); + it('carries a custom cache tag when one is passed', () => { + const res = cachedJson({ ok: true }, { tag: 'collectivex' }); + expect(res.headers.get('Vercel-Cache-Tag')).toBe('collectivex'); + expect(res.headers.get('Cache-Control')).toBe('public, max-age=0, s-maxage=86400'); + }); + + it('honors a custom Cache-Control for lazily-refreshed routes', () => { + const res = cachedJson( + { ok: true }, + { tag: 'collectivex', cacheControl: 'public, max-age=0, s-maxage=60' }, + ); + expect(res.headers.get('Cache-Control')).toBe('public, max-age=0, s-maxage=60'); + }); + it('sets an environment-scoped Vercel-Cache-Tag', () => { process.env.CACHE_NAMESPACE = 'staging'; diff --git a/packages/app/src/lib/api-cache.ts b/packages/app/src/lib/api-cache.ts index 65b0ec268..b63e725ce 100644 --- a/packages/app/src/lib/api-cache.ts +++ b/packages/app/src/lib/api-cache.ts @@ -1,6 +1,36 @@ import { revalidateTag, unstable_cache } from 'next/cache'; import { blobGet, blobPurge, blobSet } from './blob-cache'; +/** + * CollectiveX data lives in its own database and gets its own cache scope: + * every CollectiveX response carries this CDN/unstable_cache tag (via + * `cachedJson(..., { tag })`), so run deletion can purge the CollectiveX + * cache without dropping the main dashboard's. CollectiveX routes read their + * DB directly — no blob layer, so the tag is the entire scope. + */ +export const COLLECTIVEX_CACHE_SCOPE = 'collectivex'; + +/** + * CDN/unstable_cache tag for CollectiveX responses, namespaced the same way as + * the main database tag. Environments that share a Vercel project share its + * cache tags, so an unnamespaced `collectivex` would let a purge in one + * environment drop another's cached responses. `COLLECTIVEX_CACHE_SCOPE` stays + * unnamespaced on purpose — it is the public identifier callers pass to + * `/api/v1/invalidate?scope=`. + */ +export function collectiveXCacheTag(): string { + const namespace = cacheNamespace(); + return namespace ? `${COLLECTIVEX_CACHE_SCOPE}:${namespace}` : COLLECTIVEX_CACHE_SCOPE; +} + +/** + * Short CDN window shared by the CollectiveX latest/runs routes: new sweep + * runs are discovered lazily at the origin, so freshness is bounded by this + * TTL rather than by an ingest-time purge. + */ +export const COLLECTIVEX_CACHE_CONTROL = + 'public, max-age=0, s-maxage=60, stale-while-revalidate=300'; + function cacheNamespace(): string { return process.env.CACHE_NAMESPACE?.trim() ?? ''; } @@ -18,6 +48,8 @@ function cacheTag(): string { interface CachedQueryOptions { /** Use blob storage directly, skipping unstable_cache. Use for payloads known to exceed 2MB. */ blobOnly?: boolean; + /** Cache tag for the unstable_cache path (default 'db'). */ + tag?: string; } /** @@ -42,7 +74,9 @@ export function cachedQuery( }; } - const nextCached = unstable_cache(fn, [cacheKey(keyPrefix)], { tags: [cacheTag()] }); + const nextCached = unstable_cache(fn, [cacheKey(keyPrefix)], { + tags: [options?.tag ?? cacheTag()], + }); return (...args: Args): Promise => nextCached(...args); } @@ -50,14 +84,32 @@ export function cachedQuery( export async function purgeAll(): Promise { const deleted = await blobPurge(); revalidateTag(cacheTag(), { expire: 0 }); + // CollectiveX responses are cached only under their own tag (no blobs), so + // a full purge must drop that tag explicitly — this line is the sole + // mechanism that clears CollectiveX from the CDN. + purgeCollectiveX(); return deleted; } -/** 1 day. Purged on demand via the environment-scoped database cache tag. */ -function cdnHeaders(): Record { +/** + * Purge only the CollectiveX cache scope. CollectiveX routes read straight + * from their own DB (no blob layer), so this is just the CDN/unstable_cache + * tag. + */ +export function purgeCollectiveX(): void { + revalidateTag(collectiveXCacheTag(), { expire: 0 }); +} + +/** + * 1 day unless overridden. Purged on demand via revalidateTag with the matching + * tag. `tag` defaults to the environment-scoped database tag; CollectiveX routes + * pass their own scope (and a shorter cacheControl). + */ +function cdnHeaders(tag: string = cacheTag(), cacheControl?: string): Record { return { - 'Cache-Control': 'public, max-age=0, s-maxage=86400', - 'Vercel-Cache-Tag': cacheTag(), + 'Cache-Control': cacheControl ?? 'public, max-age=0, s-maxage=86400', + 'Vercel-Cache-Tag': tag, + 'X-Content-Type-Options': 'nosniff', }; } @@ -75,7 +127,10 @@ export function cachedText(data: string, contentType: string): Response { } /** CDN-cached streamed + gzip-compressed JSON response — supports up to 20 MB on Vercel CDN. */ -export function cachedJson(data: T): Response { +export function cachedJson( + data: T, + options?: { tag?: string; cacheControl?: string }, +): Response { const bytes = new TextEncoder().encode(JSON.stringify(data)); const CHUNK = 64 * 1024; const raw = new ReadableStream({ @@ -91,7 +146,7 @@ export function cachedJson(data: T): Response { headers: { 'Content-Type': 'application/json', 'Content-Encoding': 'gzip', - ...cdnHeaders(), + ...cdnHeaders(options?.tag, options?.cacheControl), }, }); } diff --git a/packages/app/src/lib/api.test.ts b/packages/app/src/lib/api.test.ts index a1f290068..abf899168 100644 --- a/packages/app/src/lib/api.test.ts +++ b/packages/app/src/lib/api.test.ts @@ -4,9 +4,14 @@ import { fetchBenchmarks, fetchWorkflowInfo, fetchAvailability, + deleteCollectiveXRun, + fetchCollectiveXRun, + fetchCollectiveXRunList, fetchReliability, fetchEvaluations, } from './api'; +import { buildRunSummary } from '@semianalysisai/inferencex-db/collectivex/reader'; +import { makeCollectiveXDataset } from '@/components/collectivex/test-fixture'; const mockFetch = vi.fn(); vi.stubGlobal('fetch', mockFetch); @@ -126,3 +131,61 @@ describe('fetchEvaluations', () => { expect(result[0].task).toBe('gsm8k'); }); }); + +describe('CollectiveX API', () => { + const dataset = makeCollectiveXDataset(); + + function mockJson(payload: unknown) { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: () => Promise.resolve(payload), + }); + } + + it('fetches a specific run by id', async () => { + mockJson(dataset); + + const result = await fetchCollectiveXRun(1, dataset.run.run_id); + + expect(mockFetch).toHaveBeenCalledWith( + `/api/v1/collectivex/runs/${dataset.run.run_id}?version=1`, + expect.objectContaining({}), + ); + expect(result.run.run_id).toBe(dataset.run.run_id); + }); + + it('fetches the run list', async () => { + mockJson({ version: 1, runs: [buildRunSummary(dataset)], discovery_complete: true }); + + const response = await fetchCollectiveXRunList(1); + + expect(mockFetch).toHaveBeenCalledWith( + '/api/v1/collectivex/runs?version=1', + expect.objectContaining({}), + ); + expect(response.discovery_complete).toBe(true); + expect(response.runs).toHaveLength(1); + expect(response.runs[0].run_id).toBe(dataset.run.run_id); + }); + + it('sends the bearer token on delete and reports success', async () => { + mockFetch.mockResolvedValueOnce({ ok: true, status: 200 }); + + await expect(deleteCollectiveXRun('160', 'secret-token')).resolves.toBe(true); + + expect(mockFetch).toHaveBeenCalledWith('/api/v1/collectivex/runs/160', { + method: 'DELETE', + headers: { Authorization: 'Bearer secret-token' }, + }); + }); + + it('resolves false on 401 so callers can clear a stale token', async () => { + mockFetch.mockResolvedValueOnce({ ok: false, status: 401 }); + await expect(deleteCollectiveXRun('160', 'stale')).resolves.toBe(false); + }); + + it('throws on non-auth delete failures', async () => { + mockFetch.mockResolvedValueOnce({ ok: false, status: 500 }); + await expect(deleteCollectiveXRun('160', 'secret-token')).rejects.toThrow(/500/); + }); +}); diff --git a/packages/app/src/lib/api.ts b/packages/app/src/lib/api.ts index 1c8cb5e83..9b7930e97 100644 --- a/packages/app/src/lib/api.ts +++ b/packages/app/src/lib/api.ts @@ -3,6 +3,12 @@ * Each function is a thin fetch wrapper returning typed data. */ +import { + COLLECTIVEX_DEFAULT_VERSION, + type CollectiveXDataset, + type CollectiveXRunSummary, + type CollectiveXVersion, +} from '@/components/collectivex/types'; import type { WorkerPower } from '@/components/inference/types'; import type { SubmissionsResponse } from './submissions-types'; @@ -304,6 +310,46 @@ export function fetchSubmissions(signal?: AbortSignal) { return fetchJson('/api/v1/submissions', signal); } +/** Stored sweep runs plus whether lazy discovery has reached the end of history. */ +export function fetchCollectiveXRunList( + version: CollectiveXVersion = COLLECTIVEX_DEFAULT_VERSION, + signal?: AbortSignal, +) { + return fetchJson<{ + version: CollectiveXVersion; + runs: CollectiveXRunSummary[]; + discovery_complete: boolean; + }>(`/api/v1/collectivex/runs?version=${version}`, signal); +} + +/** Resolve a specific ingested sweep run's neutral view dataset by run_id. */ +export function fetchCollectiveXRun( + version: CollectiveXVersion, + runId: string, + signal?: AbortSignal, +) { + return fetchJson( + `/api/v1/collectivex/runs/${runId}?version=${version}`, + signal, + ); +} + +/** + * Admin deletion of an ingested run. Requires the dedicated CollectiveX admin + * Bearer secret (COLLECTIVEX_ADMIN_SECRET — deliberately not the CI-held + * INVALIDATE_SECRET); resolves false on 401 so callers can clear a stale + * stored token, throws on other failures. + */ +export async function deleteCollectiveXRun(runId: string, token: string): Promise { + const response = await fetch(`/api/v1/collectivex/runs/${runId}`, { + method: 'DELETE', + headers: { Authorization: `Bearer ${token}` }, + }); + if (response.status === 401) return false; + if (!response.ok) throw new Error(`CollectiveX delete failed (${response.status}).`); + return true; +} + export interface FeedbackListRow { id: string; created_at: string; diff --git a/packages/app/src/lib/collectivex-lazy-ingest.test.ts b/packages/app/src/lib/collectivex-lazy-ingest.test.ts new file mode 100644 index 000000000..20bf58293 --- /dev/null +++ b/packages/app/src/lib/collectivex-lazy-ingest.test.ts @@ -0,0 +1,510 @@ +import AdmZip from 'adm-zip'; +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { makeRawMatrix, makeRawShard } from '@/components/collectivex/test-fixture'; + +const { mockGetStates, mockInsert, mockRefresh, mockGetDb, mockGetWriteDb } = vi.hoisted(() => ({ + mockGetStates: vi.fn(), + mockInsert: vi.fn(), + mockRefresh: vi.fn(), + mockGetDb: vi.fn(() => 'mock-sql'), + mockGetWriteDb: vi.fn(() => 'mock-write-sql'), +})); + +vi.mock('@semianalysisai/inferencex-db/connection', () => ({ + getCollectiveXDb: mockGetDb, + getCollectiveXWriteDb: mockGetWriteDb, +})); + +vi.mock('@semianalysisai/inferencex-db/queries/collectivex', () => ({ + getCollectiveXRunStates: mockGetStates, + insertCollectiveXRun: mockInsert, + refreshCollectiveXRunAttempt: mockRefresh, +})); + +import { + collectiveXSweepErrorCode, + ensureCollectiveXRun, + ensureCollectiveXRunsList, + ensureLatestCollectiveXRun, + resetCollectiveXDiscoveryCooldown, +} from './collectivex-lazy-ingest'; + +const mockFetch = vi.fn(); +vi.stubGlobal('fetch', mockFetch); + +// Two current matrix shards: NVIDIA scale-up EP8 + AMD scale-out EP16. +const shardA = makeRawShard({ backend: 'deepep-v2', ep: 8 }); +const shardB = makeRawShard({ + sku: 'mi355x', + backend: 'mori', + implName: 'mori', + vendor: 'amd', + ep: 16, + scaleUpTransport: 'xgmi', + scaleOutTransport: 'rdma', + topologyClass: 'mi355x-xgmi-rdma', + nodes: 2, + gpusPerNode: 8, + scaleUpDomain: 8, +}); + +function requestedOf(shard: Record) { + const identity = shard.identity as Record; + const factors = identity.case_factors as Record; + return { + caseId: identity.case_id as string, + sku: factors.sku as string, + disposition: 'runnable' as const, + case: factors.case as Record, + }; +} + +const matrix = makeRawMatrix([requestedOf(shardA), requestedOf(shardB)]); +const matrixV2 = makeRawMatrix([requestedOf(shardA), requestedOf(shardB)], 2); + +function zipDocs(...docs: unknown[]): ArrayBuffer { + const zip = new AdmZip(); + docs.forEach((doc, index) => zip.addFile(`doc-${index}.json`, Buffer.from(JSON.stringify(doc)))); + const bytes = zip.toBuffer(); + const archive = new ArrayBuffer(bytes.byteLength); + new Uint8Array(archive).set(bytes); + return archive; +} + +const matrixZip = zipDocs(matrix); +const matrixZipV2 = zipDocs(matrixV2); +const shardZip = zipDocs(shardA, shardB); +const shardZipV2 = zipDocs({ ...shardA, version: 2 }, { ...shardB, version: 2 }); + +function runObject(overrides: Record = {}) { + return { + id: 160, + name: 'CollectiveX Sweep', + path: '.github/workflows/collectivex-sweep.yml', + // Deliberately a feature branch: lazy ingest accepts runs from ANY branch. + head_branch: 'collectivex-fp8-precision', + head_sha: 'a'.repeat(40), + status: 'completed', + conclusion: 'success', + run_attempt: 1, + updated_at: new Date().toISOString(), + ...overrides, + }; +} + +/** One page of the run listing. A Response body reads once, so build a fresh one per call. */ +function runListing(...runs: ReturnType[]) { + return Response.json({ total_count: runs.length, workflow_runs: runs }); +} + +const twoRunListing = () => runListing(runObject({ id: 161 }), runObject({ id: 160 })); + +function artifactsBody(runId = 160, runAttempt = 1) { + return { + total_count: 2, + artifacts: [ + { + id: 1, + name: `cxsweep-matrix-${runId}`, + archive_download_url: 'https://example.test/matrix.zip', + expired: false, + }, + { + id: 2, + name: `cxshard-cases-${runId}-${runAttempt}`, + archive_download_url: 'https://example.test/shards.zip', + expired: false, + }, + ], + }; +} + +beforeEach(() => { + mockFetch.mockReset(); + mockGetStates.mockReset().mockResolvedValue({}); + mockInsert.mockReset().mockResolvedValue(true); + mockRefresh.mockReset().mockResolvedValue(true); + process.env.GITHUB_TOKEN = 'test-token'; + // Module-level state outlives a case; without this every test after the first + // would be answered from the discovery cooldown and issue no requests. + resetCollectiveXDiscoveryCooldown(); +}); + +afterAll(() => { + delete process.env.GITHUB_TOKEN; + vi.unstubAllGlobals(); +}); + +describe('ensureLatestCollectiveXRun', () => { + it('discovers the newest absent run and persists its raw documents', async () => { + mockFetch + .mockResolvedValueOnce(Response.json({ total_count: 1, workflow_runs: [runObject()] })) + .mockResolvedValueOnce(Response.json(artifactsBody())) + .mockResolvedValueOnce(new Response(matrixZip)) + .mockResolvedValueOnce(new Response(shardZip)); + + await ensureLatestCollectiveXRun(1); + + expect(mockInsert).toHaveBeenCalledTimes(1); + const [sql, run, docs] = mockInsert.mock.calls[0]; + expect(sql).toBe('mock-write-sql'); + expect(run).toMatchObject({ + run_id: '160', + run_attempt: 1, + version: 1, + source_branch: 'collectivex-fp8-precision', + conclusion: 'success', + matrix, + }); + expect(run.summary).toMatchObject({ run_id: '160', measured_cases: 2 }); + expect(docs).toEqual([shardA, shardB]); + }); + + it('stops without artifact downloads when the newest matching run is live', async () => { + mockGetStates.mockResolvedValue({ '160': { state: 'live', version: 1, run_attempt: 1 } }); + mockFetch.mockResolvedValueOnce( + Response.json({ total_count: 1, workflow_runs: [runObject()] }), + ); + + await ensureLatestCollectiveXRun(1); + + expect(mockInsert).not.toHaveBeenCalled(); + expect(mockFetch).toHaveBeenCalledTimes(1); + }); + + it('never re-ingests a tombstoned run — the next candidate wins', async () => { + mockGetStates.mockImplementation((_sql: unknown, ids: string[]) => + Promise.resolve( + ids[0] === '161' ? { '161': { state: 'deleted', version: 1, run_attempt: 1 } } : {}, + ), + ); + mockFetch + .mockResolvedValueOnce(twoRunListing()) + .mockResolvedValueOnce(Response.json(artifactsBody())) + .mockResolvedValueOnce(new Response(matrixZip)) + .mockResolvedValueOnce(new Response(shardZip)); + + await ensureLatestCollectiveXRun(1); + + expect(mockInsert).toHaveBeenCalledTimes(1); + expect(mockInsert.mock.calls[0][1].run_id).toBe('160'); + }); + + it('skips runs tagged for another version', async () => { + mockFetch + .mockResolvedValueOnce(twoRunListing()) + // Run 161 carries a v2 matrix — requesting v1 must move on. + .mockResolvedValueOnce(Response.json(artifactsBody(161))) + .mockResolvedValueOnce(new Response(matrixZipV2)) + .mockResolvedValueOnce(new Response(shardZipV2)) + .mockResolvedValueOnce(Response.json(artifactsBody())) + .mockResolvedValueOnce(new Response(matrixZip)) + .mockResolvedValueOnce(new Response(shardZip)); + + await ensureLatestCollectiveXRun(1); + + expect(mockInsert).toHaveBeenCalledTimes(2); + expect(mockInsert.mock.calls[0][1]).toMatchObject({ version: 2, run_id: '161' }); + expect(mockInsert.mock.calls[1][1]).toMatchObject({ version: 1, run_id: '160' }); + }); + + it('refreshes a live run when GitHub reports a newer attempt', async () => { + mockGetStates.mockResolvedValue({ '160': { state: 'live', version: 1, run_attempt: 1 } }); + mockFetch + .mockResolvedValueOnce( + Response.json({ total_count: 1, workflow_runs: [runObject({ run_attempt: 2 })] }), + ) + .mockResolvedValueOnce(Response.json(artifactsBody(160, 2))) + .mockResolvedValueOnce(new Response(matrixZip)) + .mockResolvedValueOnce(new Response(shardZip)); + + await ensureLatestCollectiveXRun(1); + + expect(mockInsert).not.toHaveBeenCalled(); + expect(mockRefresh).toHaveBeenCalledTimes(1); + expect(mockRefresh.mock.calls[0][1]).toMatchObject({ run_id: '160', run_attempt: 2 }); + }); + + it('downloads only the highest usable attempt per shard', async () => { + const artifacts = { + total_count: 3, + artifacts: [ + { + id: 1, + name: 'cxsweep-matrix-160', + archive_download_url: 'https://example.test/matrix.zip', + expired: false, + }, + { + id: 2, + name: 'cxshard-cases-160-1', + archive_download_url: 'https://example.test/shard-attempt1.zip', + expired: false, + }, + { + id: 3, + name: 'cxshard-cases-160-2', + archive_download_url: 'https://example.test/shard-attempt2.zip', + expired: false, + }, + ], + }; + mockFetch + .mockResolvedValueOnce( + Response.json({ total_count: 1, workflow_runs: [runObject({ run_attempt: 2 })] }), + ) + .mockResolvedValueOnce(Response.json(artifacts)) + .mockResolvedValueOnce(new Response(matrixZip)) + .mockResolvedValueOnce(new Response(shardZip)); + + await ensureLatestCollectiveXRun(1); + + expect(mockInsert).toHaveBeenCalledTimes(1); + // 4 fetches total: runs, artifacts, matrix, ONE shard — the attempt-1 + // archive is superseded and never downloaded. + expect(mockFetch).toHaveBeenCalledTimes(4); + expect(mockFetch.mock.calls[3][0]).toBe('https://example.test/shard-attempt2.zip'); + }); + + it('classifies a matrix artifact without a matrix document as invalid', async () => { + mockFetch + .mockResolvedValueOnce(Response.json({ total_count: 1, workflow_runs: [runObject()] })) + .mockResolvedValueOnce(Response.json(artifactsBody())) + .mockResolvedValueOnce(new Response(zipDocs({ record_type: 'samples' }))); + + const caught = await ensureLatestCollectiveXRun(1).catch((error: unknown) => error); + expect(collectiveXSweepErrorCode(caught)).toBe('invalid'); + expect(mockInsert).not.toHaveBeenCalled(); + }); + + it('reports GitHub outages as unavailable', async () => { + mockFetch.mockResolvedValue(new Response(null, { status: 500 })); + const caught = await ensureLatestCollectiveXRun(1).catch((error: unknown) => error); + expect(collectiveXSweepErrorCode(caught)).toBe('unavailable'); + }); + + it('reports a missing GITHUB_TOKEN as unavailable', async () => { + delete process.env.GITHUB_TOKEN; + const caught = await ensureLatestCollectiveXRun(1).catch((error: unknown) => error); + expect(collectiveXSweepErrorCode(caught)).toBe('unavailable'); + }); +}); + +describe('discovery cooldown', () => { + it('serves a warm target from the cooldown without calling GitHub again', async () => { + mockGetStates.mockResolvedValue({ '160': { state: 'live', version: 1, run_attempt: 1 } }); + mockFetch.mockImplementation(() => Promise.resolve(runListing(runObject()))); + + await ensureLatestCollectiveXRun(1); + const afterFirst = mockFetch.mock.calls.length; + expect(afterFirst).toBeGreaterThan(0); + + await ensureLatestCollectiveXRun(1); + await ensureLatestCollectiveXRun(1); + expect(mockFetch.mock.calls.length).toBe(afterFirst); + }); + + it('walks again once the cooldown window has elapsed', async () => { + vi.useFakeTimers(); + try { + mockGetStates.mockResolvedValue({ '160': { state: 'live', version: 1, run_attempt: 1 } }); + mockFetch.mockImplementation(() => Promise.resolve(runListing(runObject()))); + + await ensureLatestCollectiveXRun(1); + const afterFirst = mockFetch.mock.calls.length; + + vi.advanceTimersByTime(61_000); + await ensureLatestCollectiveXRun(1); + expect(mockFetch.mock.calls.length).toBeGreaterThan(afterFirst); + } finally { + vi.useRealTimers(); + } + }); + + it('rethrows a remembered failure so an outage stays 502/503 rather than 404', async () => { + mockFetch.mockResolvedValue(new Response('nope', { status: 500 })); + + const first = await ensureLatestCollectiveXRun(1).catch((error: unknown) => error); + expect(collectiveXSweepErrorCode(first)).toBe('unavailable'); + const afterFirst = mockFetch.mock.calls.length; + + // Same error object, and no fresh requests while the failure window holds. + const second = await ensureLatestCollectiveXRun(1).catch((error: unknown) => error); + expect(second).toBe(first); + expect(mockFetch.mock.calls.length).toBe(afterFirst); + }); +}); + +describe('ensureCollectiveXRunsList', () => { + it('bounds GitHub discovery to runs that could still have rerun artifacts', async () => { + vi.useFakeTimers(); + vi.setSystemTime('2026-07-29T00:00:00.000Z'); + try { + mockFetch.mockResolvedValueOnce(runListing()); + + await expect(ensureCollectiveXRunsList(1)).resolves.toBe(true); + + const discoveryUrl = new URL(mockFetch.mock.calls[0][0] as string); + expect(discoveryUrl.searchParams.get('created')).toBe('>=2026-06-15T00:00:00.000Z'); + } finally { + vi.useRealTimers(); + } + }); + + it('does not probe artifacts for unknown runs beyond the retention window', async () => { + mockFetch.mockResolvedValueOnce(runListing(runObject({ updated_at: '2020-01-01T00:00:00Z' }))); + + await expect(ensureCollectiveXRunsList(1)).resolves.toBe(true); + + expect(mockFetch).toHaveBeenCalledTimes(1); + expect(mockGetStates).not.toHaveBeenCalled(); + expect(mockInsert).not.toHaveBeenCalled(); + }); + + it('continues past more than eight known rows to backfill an older run', async () => { + const runs = Array.from({ length: 10 }, (_unused, index) => runObject({ id: 169 - index })); + mockGetStates.mockImplementation((_sql: unknown, ids: string[]) => + Promise.resolve( + ids[0] === '160' ? {} : { [ids[0]]: { state: 'live', version: 1, run_attempt: 1 } }, + ), + ); + mockFetch + .mockResolvedValueOnce(runListing(...runs)) + .mockResolvedValueOnce(Response.json(artifactsBody())) + .mockResolvedValueOnce(new Response(matrixZip)) + .mockResolvedValueOnce(new Response(shardZip)); + + const complete = await ensureCollectiveXRunsList(1); + + expect(complete).toBe(true); + expect(mockInsert).toHaveBeenCalledTimes(1); + expect(mockInsert.mock.calls[0][1].run_id).toBe('160'); + }); + + it('continues past one malformed run instead of hiding later valid runs', async () => { + mockFetch + .mockResolvedValueOnce(twoRunListing()) + .mockResolvedValueOnce(Response.json(artifactsBody(161))) + .mockResolvedValueOnce(new Response(zipDocs({ record_type: 'samples' }))) + .mockResolvedValueOnce(Response.json(artifactsBody())) + .mockResolvedValueOnce(new Response(matrixZip)) + .mockResolvedValueOnce(new Response(shardZip)); + + await expect(ensureCollectiveXRunsList(1)).resolves.toBe(true); + + expect(mockInsert).toHaveBeenCalledTimes(1); + expect(mockInsert.mock.calls[0][1].run_id).toBe('160'); + }); + + it('reports an incomplete pass after changing eight runs', async () => { + const runs = Array.from({ length: 8 }, (_unused, index) => runObject({ id: 167 - index })); + mockFetch.mockResolvedValueOnce(runListing(...runs)); + for (const run of runs) { + mockFetch + .mockResolvedValueOnce(Response.json(artifactsBody(run.id))) + .mockResolvedValueOnce(new Response(matrixZip)) + .mockResolvedValueOnce(new Response(shardZip)); + } + + await expect(ensureCollectiveXRunsList(1)).resolves.toBe(false); + expect(mockInsert).toHaveBeenCalledTimes(8); + }); + + it('does not consume the mutation budget when concurrent inserts are no-ops', async () => { + const runs = Array.from({ length: 8 }, (_unused, index) => runObject({ id: 167 - index })); + mockInsert.mockResolvedValue(false); + mockFetch.mockResolvedValueOnce(runListing(...runs)); + for (const run of runs) { + mockFetch + .mockResolvedValueOnce(Response.json(artifactsBody(run.id))) + .mockResolvedValueOnce(new Response(matrixZip)) + .mockResolvedValueOnce(new Response(shardZip)); + } + + await expect(ensureCollectiveXRunsList(1)).resolves.toBe(true); + expect(mockInsert).toHaveBeenCalledTimes(8); + }); + + it('does not consume the mutation budget when guarded refreshes are no-ops', async () => { + const runs = Array.from({ length: 8 }, (_unused, index) => + runObject({ id: 167 - index, run_attempt: 2 }), + ); + mockGetStates.mockImplementation((_sql: unknown, ids: string[]) => + Promise.resolve({ [ids[0]]: { state: 'live', version: 1, run_attempt: 1 } }), + ); + mockRefresh.mockResolvedValue(false); + mockFetch.mockResolvedValueOnce(runListing(...runs)); + for (const run of runs) { + mockFetch + .mockResolvedValueOnce(Response.json(artifactsBody(run.id, 2))) + .mockResolvedValueOnce(new Response(matrixZip)) + .mockResolvedValueOnce(new Response(shardZip)); + } + + await expect(ensureCollectiveXRunsList(1)).resolves.toBe(true); + expect(mockRefresh).toHaveBeenCalledTimes(8); + }); +}); + +describe('ensureCollectiveXRun', () => { + it('treats tombstoned runs as not found', async () => { + mockGetStates.mockResolvedValue({ '160': { state: 'deleted', version: 1, run_attempt: 1 } }); + const caught = await ensureCollectiveXRun(1, '160').catch((error: unknown) => error); + expect(collectiveXSweepErrorCode(caught)).toBe('not-found'); + expect(mockFetch).not.toHaveBeenCalled(); + }); + + it('checks the GitHub attempt for a live matching run without downloading artifacts', async () => { + mockGetStates.mockResolvedValue({ '160': { state: 'live', version: 1, run_attempt: 1 } }); + mockFetch.mockResolvedValueOnce(Response.json(runObject())); + + await ensureCollectiveXRun(1, '160'); + + expect(mockFetch).toHaveBeenCalledTimes(1); + expect(mockInsert).not.toHaveBeenCalled(); + expect(mockRefresh).not.toHaveBeenCalled(); + }); + + it('refreshes a live matching run when GitHub has a newer attempt', async () => { + mockGetStates.mockResolvedValue({ '160': { state: 'live', version: 1, run_attempt: 1 } }); + mockFetch + .mockResolvedValueOnce(Response.json(runObject({ run_attempt: 2 }))) + .mockResolvedValueOnce(Response.json(artifactsBody(160, 2))) + .mockResolvedValueOnce(new Response(matrixZip)) + .mockResolvedValueOnce(new Response(shardZip)); + + await ensureCollectiveXRun(1, '160'); + + expect(mockRefresh).toHaveBeenCalledTimes(1); + expect(mockRefresh.mock.calls[0][1]).toMatchObject({ run_id: '160', run_attempt: 2 }); + }); + + it('persists an absent run fetched by id', async () => { + mockFetch + .mockResolvedValueOnce(Response.json(runObject())) + .mockResolvedValueOnce(Response.json(artifactsBody())) + .mockResolvedValueOnce(new Response(matrixZip)) + .mockResolvedValueOnce(new Response(shardZip)); + + await ensureCollectiveXRun(1, '160'); + + expect(mockInsert).toHaveBeenCalledTimes(1); + expect(mockInsert.mock.calls[0][1].run_id).toBe('160'); + }); + + it('rejects runs from other workflows as not found', async () => { + mockFetch.mockResolvedValueOnce( + Response.json(runObject({ path: '.github/workflows/run-sweep.yml' })), + ); + const caught = await ensureCollectiveXRun(1, '160').catch((error: unknown) => error); + expect(collectiveXSweepErrorCode(caught)).toBe('not-found'); + expect(mockInsert).not.toHaveBeenCalled(); + }); + + it('rejects malformed run ids without touching GitHub', async () => { + const caught = await ensureCollectiveXRun(1, 'abc').catch((error: unknown) => error); + expect(collectiveXSweepErrorCode(caught)).toBe('not-found'); + expect(mockFetch).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/app/src/lib/collectivex-lazy-ingest.ts b/packages/app/src/lib/collectivex-lazy-ingest.ts new file mode 100644 index 000000000..6e5d7055b --- /dev/null +++ b/packages/app/src/lib/collectivex-lazy-ingest.ts @@ -0,0 +1,617 @@ +/** + * Lazy CollectiveX ingest: the CollectiveX database is a durable cache of + * GitHub Actions, populated on read. Each `ensure*` function checks the DB + * first and only then discovers/downloads sweep artifacts from GitHub, + * persisting the RAW documents so a run outlives its 14-day artifact + * retention once anyone has viewed it. + * + * Rules encoded here: + * - Sweep runs are accepted from ANY branch (they are launched via `gh api` + * on feature branches); only the workflow identity is checked. + * - Discovery never gates on conclusion — a red or partial run still + * surfaces what it produced. + * - Tombstoned runs (deleted via the dashboard) are never re-ingested. + * - GitHub being down must not take the page down: callers read the DB + * after `ensure*` and serve whatever is there, so these functions only + * matter when the DB has nothing to fall back to. + */ + +import AdmZip from 'adm-zip'; + +import { GITHUB_API_BASE, GITHUB_OWNER, GITHUB_REPO } from '@semianalysisai/inferencex-constants'; + +import { + buildDatasetFromNeutral, + buildRunSummary, + isMatrixDoc, + matrixVersion, +} from '@semianalysisai/inferencex-db/collectivex/reader'; +import { + matrixArtifactName, + selectShardArtifacts, +} from '@semianalysisai/inferencex-db/collectivex/artifact-selection'; +import type { CollectiveXVersion } from '@semianalysisai/inferencex-db/collectivex/types'; +import { getCollectiveXDb, getCollectiveXWriteDb } from '@semianalysisai/inferencex-db/connection'; +import { + getCollectiveXRunStates, + insertCollectiveXRun, + refreshCollectiveXRunAttempt, +} from '@semianalysisai/inferencex-db/queries/collectivex'; + +const WORKFLOW_PATH = '.github/workflows/collectivex-sweep.yml'; +const WORKFLOW_FILE = 'collectivex-sweep.yml'; +const WORKFLOW_NAME = 'CollectiveX Sweep'; +const RUNS_PER_PAGE = 100; +const ARTIFACTS_PER_PAGE = 100; + +const MAX_ARTIFACT_BYTES = 64 * 1024 * 1024; +const MAX_RUN_BYTES = 256 * 1024 * 1024; +const REQUEST_TIMEOUT_MS = 30_000; +const MAX_REQUEST_ATTEMPTS = 3; +const RETRYABLE_STATUSES = new Set([408, 429, 500, 502, 503, 504]); +// The workflow retains artifacts for 14 days. Older runs already persisted in +// the DB remain visible, but unknown runs beyond this window cannot be +// assembled and need not spend one GitHub artifact-list request apiece. +const ARTIFACT_RETENTION_MS = 14 * 24 * 60 * 60 * 1000; +// GitHub permits a workflow rerun for 30 days after creation. Include that +// whole window plus artifact retention so the created-date filter still finds +// the oldest run whose newest rerun artifacts could be downloadable. +const WORKFLOW_RERUN_WINDOW_MS = 30 * 24 * 60 * 60 * 1000; +const DISCOVERY_LOOKBACK_MS = WORKFLOW_RERUN_WINDOW_MS + ARTIFACT_RETENTION_MS; +// Bound one origin request's work. The runs route reports whether discovery is +// complete, and the client refetches while more ingestible runs remain. +const MAX_CHANGED_RUNS_PER_PASS = 8; + +type SweepErrorCode = 'invalid' | 'not-found' | 'unavailable'; + +interface WorkflowRun { + id: number; + name: string; + path: string; + head_branch: string | null; + head_sha: string; + status: string | null; + conclusion: string | null; + run_attempt: number; + run_started_at?: string | null; + updated_at?: string | null; + created_at?: string | null; +} + +interface GithubArtifact { + id: number; + name: string; + archive_download_url: string; + expired?: boolean; + size_in_bytes?: number; +} + +class CollectiveXSweepError extends Error { + readonly code: SweepErrorCode; + + constructor(code: SweepErrorCode, message: string, options?: ErrorOptions) { + super(message, options); + this.name = 'CollectiveXSweepError'; + this.code = code; + } +} + +export function collectiveXSweepErrorCode(error: unknown): SweepErrorCode | null { + return error instanceof CollectiveXSweepError ? error.code : null; +} + +/** HTTP status for a sweep-ingest failure; null for unexpected errors. */ +export function collectiveXSweepErrorStatus(error: unknown): 404 | 502 | 503 | null { + const code = collectiveXSweepErrorCode(error); + if (code === 'not-found') return 404; + if (code === 'unavailable') return 503; + if (code === 'invalid') return 502; + return null; +} + +// Concurrent requests for the same target share one discovery pass. +const inFlight = new Map>(); + +function dedupe(key: string, work: () => Promise): Promise { + const existing = inFlight.get(key) as Promise | undefined; + if (existing) return existing; + const promise = work().finally(() => inFlight.delete(key)); + inFlight.set(key, promise); + return promise; +} + +// Cooldown for the latest-run walk. The DB is the durable cache, but sharing +// only the in-flight promise still left every latest read walking GitHub. The +// runs-list path is intentionally not throttled while incomplete: its response +// is uncached and the client requests the next bounded discovery batch. +// +// Failures are remembered too, and rethrown for the rest of their (shorter) +// window: swallowing them would let a GitHub outage read as "discovery fine, +// nothing found", which downgrades the routes' 502/503 to a 404 when the DB is +// empty. The window is the upper bound on how stale "latest" can be. +const DISCOVERY_COOLDOWN_MS = 60_000; +const DISCOVERY_FAILURE_COOLDOWN_MS = 10_000; + +interface DiscoveryOutcome { + until: number; + error: unknown; +} + +const discoveryCooldown = new Map(); + +function throttled(key: string, work: () => Promise): () => Promise { + return async () => { + const settled = discoveryCooldown.get(key); + if (settled && Date.now() < settled.until) { + if (settled.error !== null) throw settled.error; + return; + } + try { + await work(); + } catch (error) { + discoveryCooldown.set(key, { until: Date.now() + DISCOVERY_FAILURE_COOLDOWN_MS, error }); + throw error; + } + discoveryCooldown.set(key, { until: Date.now() + DISCOVERY_COOLDOWN_MS, error: null }); + }; +} + +/** + * Test-only: drop every discovery cooldown. The module keeps this state for the + * lifetime of the process, so a suite driving successive passes must clear it + * between cases. + */ +export function resetCollectiveXDiscoveryCooldown(): void { + discoveryCooldown.clear(); +} + +function githubHeaders(token: string) { + return { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${token}`, + 'X-GitHub-Api-Version': '2022-11-28', + }; +} + +async function waitBeforeRetry(attempt: number): Promise { + const delay = process.env.NODE_ENV === 'test' ? 0 : Math.min(250 * 2 ** (attempt - 1), 2000); + await new Promise((resolve) => { + setTimeout(resolve, delay); + }); +} + +async function githubFetch(url: string, token: string): Promise { + let lastError: unknown; + for (let attempt = 1; attempt <= MAX_REQUEST_ATTEMPTS; attempt++) { + try { + const response = await fetch(url, { + cache: 'no-store', + headers: githubHeaders(token), + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }); + if ( + response.ok || + !RETRYABLE_STATUSES.has(response.status) || + attempt === MAX_REQUEST_ATTEMPTS + ) { + return response; + } + lastError = new Error(`GitHub returned ${response.status}`); + } catch (error) { + lastError = error; + if (attempt === MAX_REQUEST_ATTEMPTS) break; + } + await waitBeforeRetry(attempt); + } + throw new CollectiveXSweepError('unavailable', 'GitHub request failed', { cause: lastError }); +} + +function requireToken(): string { + const token = process.env.GITHUB_TOKEN; + if (!token) throw new CollectiveXSweepError('unavailable', 'GITHUB_TOKEN is not configured'); + return token; +} + +// Identity check only — never the branch: sweeps run on feature branches. +function isSweepRun(run: WorkflowRun): boolean { + return ( + run.name === WORKFLOW_NAME && + run.path === WORKFLOW_PATH && + Number.isSafeInteger(run.id) && + run.id > 0 && + Number.isSafeInteger(run.run_attempt) && + run.run_attempt > 0 + ); +} + +function runGeneratedAt(run: WorkflowRun): string { + return run.updated_at || run.run_started_at || run.created_at || ''; +} + +function artifactsMayBeAvailable(run: WorkflowRun): boolean { + const timestamp = Date.parse(runGeneratedAt(run)); + return !Number.isFinite(timestamp) || Date.now() - timestamp <= ARTIFACT_RETENTION_MS; +} + +// Newest-first stream of completed sweep runs across all branches. GitHub's +// created filter bounds cold-origin discovery without excluding any run whose +// original or rerun artifacts could still be within their retention window. +async function* sweepRuns(token: string): AsyncGenerator { + let page = 1; + let visited = 0; + let total: number | null = null; + const createdSince = new Date(Date.now() - DISCOVERY_LOOKBACK_MS).toISOString(); + while (total === null || visited < total) { + const parameters = new URLSearchParams({ + status: 'completed', + created: `>=${createdSince}`, + per_page: String(RUNS_PER_PAGE), + page: String(page), + }); + const response = await githubFetch( + `${GITHUB_API_BASE}/repos/${GITHUB_OWNER}/${GITHUB_REPO}/actions/workflows/${WORKFLOW_FILE}/runs?${parameters}`, + token, + ); + if (!response.ok) { + throw new CollectiveXSweepError( + 'unavailable', + `GitHub run discovery failed (${response.status})`, + ); + } + const payload = (await response.json()) as { + total_count?: number; + workflow_runs?: WorkflowRun[]; + }; + const runs = payload.workflow_runs ?? []; + if ( + total === null && + Number.isSafeInteger(payload.total_count) && + (payload.total_count ?? -1) >= 0 + ) { + total = payload.total_count!; + } + if (runs.length === 0) break; + visited += runs.length; + for (const run of runs) if (isSweepRun(run)) yield run; + if (runs.length < RUNS_PER_PAGE || (total !== null && visited >= total)) break; + page += 1; + } +} + +async function listArtifacts(runId: number, token: string): Promise { + const artifacts: GithubArtifact[] = []; + let page = 1; + let total: number | null = null; + while (total === null || artifacts.length < total) { + const response = await githubFetch( + `${GITHUB_API_BASE}/repos/${GITHUB_OWNER}/${GITHUB_REPO}/actions/runs/${runId}/artifacts?per_page=${ARTIFACTS_PER_PAGE}&page=${page}`, + token, + ); + if (!response.ok) { + throw new CollectiveXSweepError( + 'unavailable', + `GitHub artifact discovery failed (${response.status})`, + ); + } + const payload = (await response.json()) as { + total_count?: number; + artifacts?: GithubArtifact[]; + }; + const page_artifacts = payload.artifacts ?? []; + if ( + total === null && + Number.isSafeInteger(payload.total_count) && + (payload.total_count ?? -1) >= 0 + ) { + total = payload.total_count!; + } + if (page_artifacts.length === 0) break; + artifacts.push(...page_artifacts); + if (page_artifacts.length < ARTIFACTS_PER_PAGE) break; + page += 1; + } + return artifacts.filter((artifact) => !artifact.expired); +} + +function hasMatrixArtifact(artifacts: GithubArtifact[], run: WorkflowRun): boolean { + return artifacts.some((artifact) => artifact.name === matrixArtifactName(String(run.id))); +} + +async function collectDocs(artifact: GithubArtifact, token: string): Promise { + if (artifact.size_in_bytes !== undefined && artifact.size_in_bytes > MAX_ARTIFACT_BYTES) { + throw new CollectiveXSweepError('invalid', `artifact ${artifact.name} is oversized`); + } + const response = await githubFetch(artifact.archive_download_url, token); + if (!response.ok) { + throw new CollectiveXSweepError( + 'unavailable', + `GitHub artifact download failed (${response.status})`, + ); + } + const archive = Buffer.from(await response.arrayBuffer()); + if (archive.byteLength > MAX_ARTIFACT_BYTES) { + throw new CollectiveXSweepError('invalid', `artifact ${artifact.name} archive is oversized`); + } + let zip: AdmZip; + try { + zip = new AdmZip(archive); + } catch (error) { + throw new CollectiveXSweepError('invalid', `artifact ${artifact.name} is not a ZIP`, { + cause: error, + }); + } + const docs: unknown[] = []; + for (const entry of zip.getEntries()) { + if (entry.isDirectory || !entry.entryName.endsWith('.json')) continue; + let text: string; + try { + text = new TextDecoder('utf-8', { fatal: true }).decode(entry.getData()); + } catch (error) { + throw new CollectiveXSweepError('invalid', `artifact ${artifact.name} has non-UTF-8 entry`, { + cause: error, + }); + } + try { + docs.push(JSON.parse(text)); + } catch (error) { + throw new CollectiveXSweepError('invalid', `artifact ${artifact.name} has invalid JSON`, { + cause: error, + }); + } + } + return docs; +} + +// A run's validated matrix, its version tag, and the artifacts that feed +// persistence. Kept separate so discovery can read the version tag cheaply +// (matrix docs are tiny) before committing to a full download. +interface MatrixCandidate { + matrixDoc: unknown; + version: number; + matrixArtifacts: GithubArtifact[]; + resultArtifacts: GithubArtifact[]; +} + +function resultArtifactsForRun(artifacts: GithubArtifact[], run: WorkflowRun): GithubArtifact[] { + return selectShardArtifacts(artifacts, String(run.id), run.run_attempt).toSorted((left, right) => + left.name.localeCompare(right.name), + ); +} + +async function loadMatrixCandidate( + artifacts: GithubArtifact[], + token: string, + run: WorkflowRun, +): Promise { + const matrixArtifacts = artifacts + .filter((artifact) => artifact.name === matrixArtifactName(String(run.id))) + .toSorted((left, right) => right.id - left.id) + .slice(0, 1); + if (matrixArtifacts.length === 0) { + throw new CollectiveXSweepError('not-found', 'sweep run has no matrix artifact'); + } + const matrixDocs: unknown[] = []; + for (const artifact of matrixArtifacts) matrixDocs.push(...(await collectDocs(artifact, token))); + const matrixCandidates = matrixDocs.filter((doc) => isMatrixDoc(doc)); + if (matrixCandidates.length !== 1) { + throw new CollectiveXSweepError('invalid', 'sweep run must carry exactly one matrix document'); + } + const version = matrixVersion(matrixCandidates[0]); + if (version === null) { + throw new CollectiveXSweepError('invalid', 'matrix document has no valid version tag'); + } + const resultArtifacts = resultArtifactsForRun(artifacts, run); + return { matrixDoc: matrixCandidates[0], version, matrixArtifacts, resultArtifacts }; +} + +/** + * Download a candidate's result docs, validate assembly, persist raw. + * `refresh` replaces an already-live row whose GitHub attempt is newer (a + * re-run of failed shards); plain inserts are conflict-safe no-ops. + */ +async function persistRun( + run: WorkflowRun, + candidate: MatrixCandidate, + token: string, + refresh = false, +): Promise { + const generatedAt = runGeneratedAt(run); + if (!generatedAt) { + throw new CollectiveXSweepError('invalid', 'sweep run is missing a timestamp'); + } + + let totalBytes = 0; + for (const artifact of [...candidate.matrixArtifacts, ...candidate.resultArtifacts]) { + totalBytes += artifact.size_in_bytes ?? 0; + if (totalBytes > MAX_RUN_BYTES) { + throw new CollectiveXSweepError('invalid', 'sweep run artifacts exceed the size budget'); + } + } + + const docs: unknown[] = []; + for (const artifact of candidate.resultArtifacts) { + docs.push(...(await collectDocs(artifact, token))); + } + + const meta = { + run_id: String(run.id), + run_attempt: run.run_attempt, + generated_at: generatedAt, + conclusion: run.conclusion, + source_sha: run.head_sha, + }; + + // Assemble once to validate the bundle and precompute the run-table summary; + // only the raw documents are stored. + let summary; + try { + summary = buildRunSummary(buildDatasetFromNeutral(candidate.matrixDoc, docs, meta)); + } catch (error) { + throw new CollectiveXSweepError('invalid', 'sweep run artifacts failed validation', { + cause: error, + }); + } + + const row = { + ...meta, + version: candidate.version, + source_branch: run.head_branch, + matrix: candidate.matrixDoc, + summary, + }; + return refresh + ? refreshCollectiveXRunAttempt(getCollectiveXWriteDb(), row, docs) + : insertCollectiveXRun(getCollectiveXWriteDb(), row, docs); +} + +/** Download and version-check a candidate's matrix; null when not ingestible. */ +async function matrixCandidateFor( + run: WorkflowRun, + version: CollectiveXVersion, + token: string, +): Promise { + const artifacts = await listArtifacts(run.id, token); + if (!hasMatrixArtifact(artifacts, run)) return null; + const candidate = await loadMatrixCandidate(artifacts, token, run); + return candidate.version === version ? candidate : null; +} + +/** + * Handle one discovery candidate — the single walker step shared by the + * latest and runs-list paths: persist absent requested-version runs, refresh + * live ones whose GitHub attempt is newer (re-run of failed shards), skip + * everything else. `changed-match` means this pass inserted or refreshed the + * requested version; `changed-other` means an absent run was persisted under + * its actual version while walking. Known live rows are `match` but do not + * consume the runs-list batch. Tombstoned rows are `skip`. + */ +type CandidateResult = 'changed-match' | 'changed-other' | 'match' | 'skip'; + +async function considerCandidate( + run: WorkflowRun, + version: CollectiveXVersion, + token: string, +): Promise { + const states = await getCollectiveXRunStates(getCollectiveXDb(), [String(run.id)]); + const known = states[String(run.id)]; + if (known) { + if (known.version !== version || known.state !== 'live') return 'skip'; + if (run.run_attempt > known.run_attempt) { + const candidate = await matrixCandidateFor(run, version, token); + if (candidate) { + const changed = await persistRun(run, candidate, token, true); + return changed ? 'changed-match' : 'match'; + } + } + return 'match'; + } + const artifacts = await listArtifacts(run.id, token); + if (!hasMatrixArtifact(artifacts, run)) return 'skip'; + const candidate = await loadMatrixCandidate(artifacts, token, run); + const changed = await persistRun(run, candidate, token); + if (!changed) return candidate.version === version ? 'match' : 'skip'; + return candidate.version === version ? 'changed-match' : 'changed-other'; +} + +/** + * Make sure the newest requested-version sweep run on GitHub is in the DB. + * Completes silently when GitHub has nothing new; throws only on GitHub or + * artifact failures (callers fall back to whatever the DB already holds). + */ +export function ensureLatestCollectiveXRun(version: CollectiveXVersion): Promise { + const key = `latest:${version}`; + return dedupe( + key, + throttled(key, async () => { + const token = requireToken(); + for await (const run of sweepRuns(token)) { + const result = await considerCandidate(run, version, token); + if (result === 'match' || result === 'changed-match') return; + } + }), + ); +} + +/** + * Progressively backfill every requested-version run whose artifacts may + * still be available. Known live rows do not consume the per-request batch, + * so successive calls advance through the workflow history instead of + * revisiting the same newest rows forever. + * + * Returns true once the current GitHub history has been exhausted, or false + * when this pass stopped at the mutation budget and the caller should refetch. + */ +export function ensureCollectiveXRunsList(version: CollectiveXVersion): Promise { + const key = `list:${version}`; + return dedupe(key, async () => { + const token = requireToken(); + let changed = 0; + for await (const run of sweepRuns(token)) { + if (!artifactsMayBeAvailable(run)) continue; + let result: CandidateResult; + try { + result = await considerCandidate(run, version, token); + } catch (error) { + const code = collectiveXSweepErrorCode(error); + // One incomplete or malformed workflow run must not hide every valid + // run behind it. Network/service failures still abort so the route can + // serve its stored fallback and expose the outage when no fallback exists. + if (code === 'invalid' || code === 'not-found') continue; + throw error; + } + if (result === 'changed-match' || result === 'changed-other') changed += 1; + if (changed >= MAX_CHANGED_RUNS_PER_PASS) return false; + } + return true; + }); +} + +/** + * Make sure one specific run is in the DB. Throws 'not-found' for absent, + * non-sweep, cross-version, or tombstoned runs. + */ +export function ensureCollectiveXRun(version: CollectiveXVersion, runId: string): Promise { + return dedupe(`run:${version}:${runId}`, async () => { + const numericId = Number(runId); + if (!Number.isSafeInteger(numericId) || numericId <= 0) { + throw new CollectiveXSweepError('not-found', 'invalid run id'); + } + const states = await getCollectiveXRunStates(getCollectiveXDb(), [runId]); + const known = states[runId]; + if (known && (known.state !== 'live' || known.version !== version)) { + // Tombstoned or cross-version rows both read as absent to the caller. + throw new CollectiveXSweepError('not-found', 'run is not available'); + } + const token = requireToken(); + const response = await githubFetch( + `${GITHUB_API_BASE}/repos/${GITHUB_OWNER}/${GITHUB_REPO}/actions/runs/${numericId}`, + token, + ); + if (response.status === 404) { + throw new CollectiveXSweepError('not-found', 'sweep run not found'); + } + if (!response.ok) { + throw new CollectiveXSweepError( + 'unavailable', + `GitHub run lookup failed (${response.status})`, + ); + } + const run = (await response.json()) as WorkflowRun; + if (!isSweepRun(run)) { + throw new CollectiveXSweepError('not-found', 'run is not a CollectiveX sweep'); + } + // Persisting an in-progress run would freeze a partial snapshot forever + // (the run_id is then "known" and never re-fetched). Discovery only sees + // completed runs; hold fetch-by-id to the same bar. + if (run.status !== 'completed') { + throw new CollectiveXSweepError('not-found', 'sweep run has not completed'); + } + if (known && run.run_attempt <= known.run_attempt) return; + const artifacts = await listArtifacts(run.id, token); + const candidate = await loadMatrixCandidate(artifacts, token, run); + if (candidate.version !== version) { + throw new CollectiveXSweepError('not-found', 'run does not match the requested version'); + } + await persistRun(run, candidate, token, known !== undefined); + }); +} diff --git a/packages/app/src/lib/d3-chart/D3Chart/types.ts b/packages/app/src/lib/d3-chart/D3Chart/types.ts index 9bb3f5c14..9aff5161b 100644 --- a/packages/app/src/lib/d3-chart/D3Chart/types.ts +++ b/packages/app/src/lib/d3-chart/D3Chart/types.ts @@ -126,6 +126,8 @@ export interface AxisConfig { label?: string; tickFormat?: (d: d3.AxisDomain) => string; tickCount?: number; + /** Explicit ticks or a domain-aware generator, useful for geometric and sparse log axes. */ + tickValues?: (number | Date)[] | ((scale: AnyScale) => (number | Date)[]); /** Post-render callback for custom axis label formatting (e.g., multi-line tspan). */ customize?: (axisGroup: d3.Selection) => void; } diff --git a/packages/app/src/lib/d3-chart/D3Chart/useD3ChartRenderer.ts b/packages/app/src/lib/d3-chart/D3Chart/useD3ChartRenderer.ts index 6dc323598..fdcd415e4 100644 --- a/packages/app/src/lib/d3-chart/D3Chart/useD3ChartRenderer.ts +++ b/packages/app/src/lib/d3-chart/D3Chart/useD3ChartRenderer.ts @@ -9,7 +9,7 @@ import type { ChartLayout, ContinuousScale } from '../types'; import { buildScale, isBandScale, type BuiltScale } from './scale-builders'; import { renderLayer, updateLayerOnZoom } from './layer-renderer'; -import type { D3ChartProps, RenderContext, ZoomContext } from './types'; +import type { AxisConfig, D3ChartProps, RenderContext, ZoomContext } from './types'; interface RendererDeps { svgRef: React.RefObject; @@ -52,6 +52,14 @@ interface RendererDeps { ) => void; } +function resolveTickValues( + tickValues: AxisConfig['tickValues'], + scale: AnyScale, +): (number | Date)[] | undefined { + if (!tickValues) return undefined; + return typeof tickValues === 'function' ? tickValues(scale) : tickValues; +} + /** * Core render effect for D3Chart. Builds scales, renders structure/axes/grid/layers, * wires up tooltip and zoom handlers. @@ -98,7 +106,14 @@ export function useD3ChartRenderer(props: D3ChartProps, deps: RendererDeps // preventing a frame where dots and lines are out of sync during y-axis metric changes. useLayoutEffect(() => { if (!svgRef.current || !tooltipRef.current || dimensions.width === 0) return; - if (data.length === 0 && layers.every((l) => l.type !== 'custom')) return; + if (data.length === 0 && layers.every((layer) => layer.type !== 'custom')) { + d3.select(svgRef.current).selectAll('*').remove(); + scalesRef.current = null; + layoutRef.current = null; + dismissTooltip(true); + prevDataRef.current = data; + return; + } // Animate when data or scale domains changed (but not on resize/theme changes) const dataChanged = data !== prevDataRef.current; @@ -163,12 +178,24 @@ export function useD3ChartRenderer(props: D3ChartProps, deps: RendererDeps // ── Grid + Axes (skip when no scale configs) ── if (hasScales) { - renderGrid(layout, xScale as AnyScale, yScale as any, yAxisConfig?.tickCount ?? 5); + const xTickValues = resolveTickValues(xAxisConfig?.tickValues, xScale as AnyScale); + const yTickValues = resolveTickValues(yAxisConfig?.tickValues, yScale as AnyScale); + renderGrid( + layout, + xScale as AnyScale, + yScale as any, + yAxisConfig?.tickCount ?? 5, + 0, + xTickValues, + yTickValues, + ); renderAxes(layout, xScale as AnyScale, yScale as any, { xTickFormat: xAxisConfig?.tickFormat, yTickFormat: yAxisConfig?.tickFormat, xTickCount: xAxisConfig?.tickCount, yTickCount: yAxisConfig?.tickCount, + xTickValues, + yTickValues, }); // Custom axis formatting callbacks @@ -410,11 +437,15 @@ export function useD3ChartRenderer(props: D3ChartProps, deps: RendererDeps } // Update axes + grid + const xTickValues = resolveTickValues(xAxisConfig?.tickValues, newXScale as AnyScale); + const yTickValues = resolveTickValues(yAxisConfig?.tickValues, newYScale as AnyScale); renderAxes(layout, newXScale as AnyScale, newYScale as any, { xTickFormat: xAxisConfig?.tickFormat, yTickFormat: yAxisConfig?.tickFormat, xTickCount: xAxisConfig?.tickCount, yTickCount: yAxisConfig?.tickCount, + xTickValues, + yTickValues, }); if (xAxisConfig?.customize) { xAxisConfig.customize(layout.xAxisGroup); @@ -427,6 +458,9 @@ export function useD3ChartRenderer(props: D3ChartProps, deps: RendererDeps newXScale as AnyScale, newYScale as any, yAxisConfig?.tickCount ?? 5, + 0, + xTickValues, + yTickValues, ); // Update layers diff --git a/packages/app/src/lib/d3-chart/chart-update.test.ts b/packages/app/src/lib/d3-chart/chart-update.test.ts index a4a52b859..a8b3d0127 100644 --- a/packages/app/src/lib/d3-chart/chart-update.test.ts +++ b/packages/app/src/lib/d3-chart/chart-update.test.ts @@ -51,6 +51,23 @@ describe('renderAxes', () => { expect(tickCount).toBeLessThanOrEqual(8); }); + it('renders only explicit x tick values within the visible domain', () => { + const layout = makeLayout(); + const xScale = d3.scaleLinear().domain([2, 10]).range([0, layout.width]); + const yScale = d3.scaleLinear().domain([0, 50]).range([layout.height, 0]); + + renderAxes(layout, xScale, yScale, { + xTickValues: [1, 2, 4, 16], + xTickFormat: String, + }); + + const labels: string[] = []; + layout.xAxisGroup.selectAll('.tick text').each(function () { + labels.push(d3.select(this).text()); + }); + expect(labels).toEqual(['2', '4']); + }); + it('respects yTickCount', () => { const layout = makeLayout(); const xScale = d3.scaleLinear().domain([0, 100]).range([0, layout.width]); @@ -130,6 +147,26 @@ describe('renderAxes', () => { }); }); + describe('with log scales', () => { + it('uses measured geometric sweep values instead of generated log subdivisions', () => { + const layout = makeLayout(); + const xScale = d3.scaleLog().base(2).domain([0.9, 70]).range([0, layout.width]); + const yScale = d3.scaleLinear().domain([0, 50]).range([layout.height, 0]); + const measuredValues = [1, 2, 4, 8, 16, 32, 64]; + + renderAxes(layout, xScale, yScale, { + xTickValues: measuredValues, + xTickFormat: String, + }); + + const labels: string[] = []; + layout.xAxisGroup.selectAll('.tick text').each(function () { + labels.push(d3.select(this).text()); + }); + expect(labels).toEqual(measuredValues.map(String)); + }); + }); + describe('with band scales', () => { it('renders band scale on x-axis', () => { const layout = makeLayout(); @@ -281,6 +318,24 @@ describe('renderGrid', () => { expect(vLines).toBeGreaterThan(0); }); + it('uses explicit x tick values for vertical grid lines', () => { + const layout = makeLayout(); + const xScale = d3.scaleLog().base(2).domain([1, 64]).range([0, layout.width]); + const yScale = d3.scaleLinear().domain([0, 50]).range([layout.height, 0]); + const measuredValues = [1, 4, 16, 64]; + + renderGrid(layout, xScale, yScale, 5, 0, measuredValues); + + const positions: number[] = []; + layout.gridGroup + .select('.grid-v') + .selectAll('line') + .each(function () { + positions.push(Number(d3.select(this).attr('x1'))); + }); + expect(positions).toEqual(measuredValues.map((value) => xScale(value))); + }); + it('creates horizontal grid lines matching y-scale ticks', () => { const layout = makeLayout(); const xScale = d3.scaleLinear().domain([0, 100]).range([0, layout.width]); diff --git a/packages/app/src/lib/d3-chart/chart-update.ts b/packages/app/src/lib/d3-chart/chart-update.ts index 45c458c77..fb79ac270 100644 --- a/packages/app/src/lib/d3-chart/chart-update.ts +++ b/packages/app/src/lib/d3-chart/chart-update.ts @@ -10,6 +10,8 @@ export interface AxisUpdateConfig { yTickFormat?: (d: d3.AxisDomain) => string; xTickCount?: number; yTickCount?: number; + xTickValues?: (number | Date)[]; + yTickValues?: (number | Date)[]; /** Override tick size for Y axis (default: 6, use 0 for band scales). */ yTickSize?: number; /** When set, axes animate to new positions over this duration (ms). */ @@ -23,8 +25,16 @@ export function renderAxes( yScale: ContinuousScale | d3.ScaleBand, config: AxisUpdateConfig, ): void { - const { xTickFormat, yTickFormat, xTickCount, yTickCount, yTickSize, transitionDuration } = - config; + const { + xTickFormat, + yTickFormat, + xTickCount, + yTickCount, + xTickValues, + yTickValues, + yTickSize, + transitionDuration, + } = config; const dur = transitionDuration ?? 0; // X axis @@ -36,6 +46,9 @@ export function renderAxes( } else { const gen = d3.axisBottom(xScale as ContinuousScale).tickSize(6); if (xTickCount) gen.ticks(xTickCount); + if (xTickValues) { + gen.tickValues(visibleTickValues(xScale, xTickValues) as Iterable); + } if (xTickFormat) gen.tickFormat(xTickFormat as any); xAxisGen = gen as unknown as d3.Axis; } @@ -54,6 +67,9 @@ export function renderAxes( } else { const yAxisGen = d3.axisLeft(yScale as ContinuousScale).tickSize(yTickSize ?? 6); if (yTickCount) yAxisGen.ticks(yTickCount); + if (yTickValues) { + yAxisGen.tickValues(visibleTickValues(yScale, yTickValues) as Iterable); + } if (yTickFormat) yAxisGen.tickFormat(yTickFormat as any); const yTarget = dur > 0 ? layout.yAxisGroup.transition().duration(dur) : layout.yAxisGroup; (yTarget as any).call(yAxisGen as any); @@ -67,6 +83,8 @@ export function renderGrid( yScale: ContinuousScale | d3.ScaleBand, yTickCount?: number, transitionDuration = 0, + xTickValues?: (number | Date)[], + yTickValues?: (number | Date)[], ): void { const { width, height, gridGroup } = layout; const dur = transitionDuration; @@ -87,7 +105,9 @@ export function renderGrid( .attr('y2', height); } else { const tickScale = xScale as { ticks: (count?: number) => number[]; (v: number): number }; - const xTicks = tickScale.ticks(); + const xTicks = xTickValues + ? (visibleTickValues(xScale, xTickValues) as number[]) + : tickScale.ticks(); const vJoin = vGroup .selectAll('line') .data(xTicks) @@ -126,7 +146,9 @@ export function renderGrid( .attr('y2', (d) => (bandScale(d) || 0) + bandScale.bandwidth() / 2) .style('stroke-width', 0.5); } else { - const yTicks = yScale.ticks(yTickCount ?? 5); + const yTicks = yTickValues + ? (visibleTickValues(yScale, yTickValues) as number[]) + : yScale.ticks(yTickCount ?? 5); const hJoin = hGroup .selectAll('line') .data(yTicks) @@ -149,3 +171,18 @@ export function renderGrid( .attr('y2', (d: number) => yScale(d)); } } + +function visibleTickValues( + scale: ContinuousScale | d3.ScaleTime, + values: (number | Date)[], +): (number | Date)[] { + const domain = scale.domain(); + const start = Number(domain[0]); + const end = Number(domain.at(-1)); + const min = Math.min(start, end); + const max = Math.max(start, end); + return values.filter((value) => { + const numeric = Number(value); + return Number.isFinite(numeric) && numeric >= min && numeric <= max; + }); +} diff --git a/packages/app/src/lib/d3-chart/layers/lines.test.ts b/packages/app/src/lib/d3-chart/layers/lines.test.ts index 6631bcc89..894e9d5f3 100644 --- a/packages/app/src/lib/d3-chart/layers/lines.test.ts +++ b/packages/app/src/lib/d3-chart/layers/lines.test.ts @@ -86,6 +86,28 @@ describe('renderLines', () => { expect(strokeByClass['line-path line-seriesB']).toBe('#0f0'); }); + it('sets a per-series stroke dash pattern when configured', () => { + const group = createMockGroup(); + const { xScale, yScale } = makeScales(); + renderLines( + group as any, + SAMPLE_LINES, + xScale, + yScale, + makeConfig({ + getStrokeDasharray: (key) => (key === 'seriesA' ? 'none' : '9 4'), + }), + ); + + const paths = group.selectAll('.line-path'); + const dashByClass: Record = {}; + for (const el of paths.elements) { + dashByClass[el.attrs['class'] as string] = el.attrs['stroke-dasharray']; + } + expect(dashByClass['line-path line-seriesA']).toBe('none'); + expect(dashByClass['line-path line-seriesB']).toBe('9 4'); + }); + it('uses default strokeWidth of 2 when not specified', () => { const group = createMockGroup(); const { xScale, yScale } = makeScales(); diff --git a/packages/app/src/lib/d3-chart/layers/lines.ts b/packages/app/src/lib/d3-chart/layers/lines.ts index 79c394e9c..075a1be71 100644 --- a/packages/app/src/lib/d3-chart/layers/lines.ts +++ b/packages/app/src/lib/d3-chart/layers/lines.ts @@ -6,6 +6,8 @@ type AnyXScale = ContinuousScale | d3.ScaleTime; export interface LineConfig { getColor: (key: string) => string; + /** Optional per-series SVG dash pattern; return `none` for a solid line. */ + getStrokeDasharray?: (key: string) => string; strokeWidth?: number; curve?: d3.CurveFactory; /** Return false to create gaps in the line (e.g., missing data points). */ @@ -58,6 +60,7 @@ export function renderLines( .merge(selection) .attr('class', (d) => `line-path line-${d.key}`) .attr('stroke', (d) => config.getColor(d.key)) + .attr('stroke-dasharray', (d) => config.getStrokeDasharray?.(d.key) ?? null) .attr('stroke-width', config.strokeWidth ?? 2) .attr('d', (d) => lineGenerator(d.points)); } diff --git a/packages/app/src/lib/dynamic-colors.test.ts b/packages/app/src/lib/dynamic-colors.test.ts new file mode 100644 index 000000000..52550a094 --- /dev/null +++ b/packages/app/src/lib/dynamic-colors.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from 'vitest'; + +import { VENDOR_OKLCH_ZONES } from '@semianalysisai/inferencex-constants'; + +import { generateVendorColors, getVendor } from './dynamic-colors'; + +function hueOf(color: string): number { + const match = /^oklch\([\d.]+ [\d.]+ (?[\d.]+)\)$/.exec(color); + expect(match, `expected oklch color, got ${color}`).not.toBeNull(); + return Number(match!.groups!.hue); +} + +describe('getVendor', () => { + it('classifies registered GPU base keys through GPU_VENDORS', () => { + expect(getVendor('h100_vllm')).toBe('nvidia'); + expect(getVendor('mi300x_sglang')).toBe('amd'); + }); + + it('classifies keys that lead with a literal vendor token', () => { + // CollectiveX series keys carry the dataset's explicit vendor rather than a + // registered GPU key (their SKUs, e.g. "h200-dgxc", are not registry keys). + expect(getVendor('nvidia_h200-dgxc_normal_ep8')).toBe('nvidia'); + expect(getVendor('amd_mi355x-oam_normal_ep8')).toBe('amd'); + }); + + it('falls back to unknown for unclassifiable keys', () => { + expect(getVendor('h200-dgxc_normal_ep8')).toBe('unknown'); + }); +}); + +describe('generateVendorColors', () => { + it('places vendor-prefixed keys in their vendor hue zones', () => { + const colors = generateVendorColors(['nvidia_series-a', 'amd_series-b'], 'light'); + const nvidia = VENDOR_OKLCH_ZONES.nvidia; + const amd = VENDOR_OKLCH_ZONES.amd; + expect(hueOf(colors['nvidia_series-a'])).toBeGreaterThanOrEqual(nvidia.start); + expect(hueOf(colors['nvidia_series-a'])).toBeLessThanOrEqual(nvidia.end); + expect(hueOf(colors['amd_series-b'])).toBeGreaterThanOrEqual(amd.start); + expect(hueOf(colors['amd_series-b'])).toBeLessThanOrEqual(amd.end); + }); + + it('keeps unclassifiable keys in the unknown zone', () => { + const colors = generateVendorColors(['mystery_series'], 'dark'); + const unknown = VENDOR_OKLCH_ZONES.unknown; + expect(hueOf(colors['mystery_series'])).toBeGreaterThanOrEqual(unknown.start); + expect(hueOf(colors['mystery_series'])).toBeLessThanOrEqual(unknown.end); + }); +}); diff --git a/packages/app/src/lib/dynamic-colors.ts b/packages/app/src/lib/dynamic-colors.ts index 38b9e10eb..d788c8fd6 100644 --- a/packages/app/src/lib/dynamic-colors.ts +++ b/packages/app/src/lib/dynamic-colors.ts @@ -20,6 +20,9 @@ export type Vendor = 'nvidia' | 'amd' | 'unknown'; export function getVendor(hwKey: string): Vendor { // hwKey may have a framework suffix (e.g. "h100_vllm") — strip it to get the GPU base key const base = hwKey.split('_')[0]; + // Keys whose dataset carries an explicit vendor (e.g. CollectiveX series) lead + // with the vendor name itself rather than a registered GPU key. + if (base === 'nvidia' || base === 'amd') return base; const vendor = GPU_VENDORS[base]; if (vendor === 'NVIDIA') return 'nvidia'; if (vendor === 'AMD') return 'amd'; diff --git a/packages/app/src/lib/i18n.test.ts b/packages/app/src/lib/i18n.test.ts index 36a9fde35..7d89f66bc 100644 --- a/packages/app/src/lib/i18n.test.ts +++ b/packages/app/src/lib/i18n.test.ts @@ -42,6 +42,7 @@ describe('hasZhSibling', () => { expect(hasZhSibling('/inference')).toBe(true); expect(hasZhSibling('/overview')).toBe(true); expect(hasZhSibling('/about')).toBe(true); + expect(hasZhSibling('/collectivex')).toBe(true); }); it('matches blog and compare child paths', () => { @@ -73,6 +74,7 @@ describe('switchLocalePath', () => { it('switches English pages to their zh sibling', () => { expect(switchLocalePath('/')).toBe('/zh'); expect(switchLocalePath('/inference')).toBe('/zh/inference'); + expect(switchLocalePath('/collectivex')).toBe('/zh/collectivex'); expect(switchLocalePath('/overview')).toBe('/zh/overview'); expect(switchLocalePath('/blog/some-post')).toBe('/zh/blog/some-post'); }); @@ -80,6 +82,7 @@ describe('switchLocalePath', () => { it('switches zh pages back to English', () => { expect(switchLocalePath('/zh')).toBe('/'); expect(switchLocalePath('/zh/quotes')).toBe('/quotes'); + expect(switchLocalePath('/zh/collectivex')).toBe('/collectivex'); expect(switchLocalePath('/zh/overview')).toBe('/overview'); expect(switchLocalePath('/zh/blog/some-post')).toBe('/blog/some-post'); }); diff --git a/packages/app/src/lib/i18n.ts b/packages/app/src/lib/i18n.ts index 7577353bf..5740981df 100644 --- a/packages/app/src/lib/i18n.ts +++ b/packages/app/src/lib/i18n.ts @@ -45,6 +45,7 @@ export const ZH_MIRRORED_ROUTES: readonly { path: string; exact?: boolean }[] = { path: '/reliability', exact: true }, { path: '/gpu-specs', exact: true }, { path: '/gpu-metrics', exact: true }, + { path: '/collectivex', exact: true }, { path: '/submissions', exact: true }, { path: '/ai-chart', exact: true }, { path: '/current-inferencex-image', exact: true }, diff --git a/packages/app/src/lib/tab-meta-zh.ts b/packages/app/src/lib/tab-meta-zh.ts index 3c50362d6..69014c1d4 100644 --- a/packages/app/src/lib/tab-meta-zh.ts +++ b/packages/app/src/lib/tab-meta-zh.ts @@ -17,6 +17,7 @@ export const ZH_TAB_KEYS = [ 'reliability', 'gpu-specs', 'gpu-metrics', + 'collectivex', 'submissions', 'ai-chart', 'current-inferencex-image', @@ -62,6 +63,11 @@ export const TAB_META_ZH: Record = { '本页面提供 GPU 规格对比:NVIDIA、AMD 等厂商加速器的显存容量、显存带宽、FLOPS、互连拓扑与功耗规格。', 'gpu-metrics': '本页面展示 GPU 功耗与能效指标(PowerX):推理负载下的实测功耗、每瓦 token 数与每兆瓦 token 产出。', + collectivex: + '本页面展示 CollectiveX 专家并行(EP)通信基准测试结果:在统一工作负载、正确性校验与采样协议下,对比 DeepEP、MoRI、UCCL 及 NCCL/RCCL 参考实现的分发(dispatch)、合并(combine)与完整往返延迟。跨芯片速率均按逻辑载荷计算;只有发布器确认完整且稳定的官方队列才会生成排名与推荐。', submissions: '本页面列出提交到 InferenceX 的全部基准测试配置:按 GPU 厂商查看提交历史、活动趋势与数据点数量。', 'ai-chart': @@ -122,6 +130,7 @@ export const TAB_LABELS_ZH: Record = { reliability: '可靠性', 'gpu-specs': 'GPU 规格', 'gpu-metrics': 'GPU 功耗', + collectivex: 'CollectiveX 通信', submissions: '提交记录', 'ai-chart': 'AI 图表', 'current-inferencex-image': '镜像', diff --git a/packages/app/src/lib/tab-meta.ts b/packages/app/src/lib/tab-meta.ts index b312a6e76..5d641b237 100644 --- a/packages/app/src/lib/tab-meta.ts +++ b/packages/app/src/lib/tab-meta.ts @@ -16,6 +16,7 @@ export const VALID_TABS = [ 'calculator', 'reliability', 'gpu-specs', + 'collectivex', 'ai-chart', 'gpu-metrics', 'submissions', @@ -56,6 +57,11 @@ export const TAB_META: Record = description: 'Detailed GPU specifications for AI inference. Compare NVIDIA, AMD, and Intel GPUs — memory bandwidth, FLOPS, interconnects, and topology.', }, + collectivex: { + title: 'CollectiveX Communication Benchmarks', + description: + 'Experimental cross-vendor expert-parallel communication benchmarks. Compare MoE dispatch and combine latency across NVIDIA and AMD GPU platforms.', + }, 'ai-chart': { title: 'AI-Powered Chart Generation', description: diff --git a/packages/db/migrations-collectivex/001_initial_schema.sql b/packages/db/migrations-collectivex/001_initial_schema.sql new file mode 100644 index 000000000..71bc53974 --- /dev/null +++ b/packages/db/migrations-collectivex/001_initial_schema.sql @@ -0,0 +1,38 @@ +-- CollectiveX sweep runs, stored as RAW artifact documents. +-- +-- The sweep JSON contract is expected to change; the shared reader +-- (packages/db/src/collectivex/reader.ts) is the single transform point and +-- runs at API-read time, so rows here are the artifacts' documents verbatim. +-- `summary` is the one precomputed column (CollectiveXRunSummary) so the run +-- picker can list runs without loading their documents. + +create table cx_runs ( + run_id bigint primary key, + run_attempt int not null, + version int not null, + generated_at timestamptz not null, + source_sha text not null, + source_branch text, + conclusion text, + matrix jsonb not null, + summary jsonb not null, + ingested_at timestamptz not null default now(), + -- Tombstone: runs are discovered lazily from GitHub on read, so a deleted + -- run must leave a marker or the next discovery would re-ingest it. + -- Deletion clears cx_run_docs but keeps this row with deleted_at set. + deleted_at timestamptz +); + +-- "Latest run per version" ordering: run_id desc. GitHub run ids increase +-- monotonically with run creation, so this matches lazy discovery's +-- newest-first walk — unlike completion time (generated_at), where a +-- long-failing older run can outlast a newer successful one. +create index cx_runs_version_latest on cx_runs (version, run_id desc); + +create table cx_run_docs ( + id bigserial primary key, + run_id bigint not null references cx_runs(run_id) on delete cascade, + doc jsonb not null +); + +create index cx_run_docs_run on cx_run_docs (run_id); diff --git a/packages/db/migrations-collectivex/002_docs_attempt.sql b/packages/db/migrations-collectivex/002_docs_attempt.sql new file mode 100644 index 000000000..443c5d944 --- /dev/null +++ b/packages/db/migrations-collectivex/002_docs_attempt.sql @@ -0,0 +1,17 @@ +-- Stamp each raw document with the run attempt that produced it. +-- +-- Readers join docs on the cx_runs row's CURRENT attempt, which makes both +-- refresh races harmless: a concurrent pair of attempt-refreshes can leave +-- superseded docs behind (single-snapshot CTE deletes can miss rows written +-- after the statement's snapshot), and a reader can straddle a refresh across +-- its row/doc reads — in both cases the attempt filter hides stale documents. +-- Leftover superseded docs are garbage-collected by the next refresh's DELETE. + +alter table cx_run_docs add column run_attempt int; + +update cx_run_docs d +set run_attempt = r.run_attempt +from cx_runs r +where r.run_id = d.run_id and d.run_attempt is null; + +alter table cx_run_docs alter column run_attempt set not null; diff --git a/packages/db/migrations-collectivex/003_docs_attempt_index.sql b/packages/db/migrations-collectivex/003_docs_attempt_index.sql new file mode 100644 index 000000000..19c28f99d --- /dev/null +++ b/packages/db/migrations-collectivex/003_docs_attempt_index.sql @@ -0,0 +1,14 @@ +-- Index cx_run_docs the way readers actually query it. +-- +-- 002 added run_attempt, and every doc read filters on the pair and orders by id +-- (`where run_id = ... and run_attempt = ... order by id`), but the table was +-- still indexed on run_id alone. A run holds one document per shard — 120 for a +-- full 9-SKU sweep — so the old index left the attempt filter and the ordering to +-- be resolved per row. +-- +-- Ordering the index by (run_id, run_attempt, id) lets the same index satisfy the +-- filter and the sort. Kept as a separate migration rather than an edit to 002: +-- the runner skips already-applied files, so amending 002 would silently do +-- nothing wherever it has already run. + +create index if not exists cx_run_docs_run_attempt_id_idx on cx_run_docs (run_id, run_attempt, id); diff --git a/packages/db/package.json b/packages/db/package.json index f7c1a9bec..a99339a47 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -5,6 +5,7 @@ "type": "module", "sideEffects": false, "exports": { + "./collectivex/*": "./src/collectivex/*.ts", "./connection": "./src/connection.ts", "./etl/*": "./src/etl/*.ts", "./lib/*": "./src/lib/*.ts", @@ -14,10 +15,12 @@ "scripts": { "db:prepare:ci": "bun src/prepare-ci-artifacts.ts", "db:ingest:ci": "bun src/ingest-ci-run.ts", + "db:ingest:collectivex": "bun --env-file=../../.env src/ingest-collectivex.ts --download", "db:ingest:run": "bun --env-file=../../.env src/ingest-ci-run.ts --download", "db:ingest:gcs": "bun --env-file=../../.env src/ingest-gcs-backup.ts", "db:ingest:supplemental": "bun --env-file=../../.env src/ingest-supplemental.ts", "db:migrate": "bun --env-file=../../.env src/migrate.ts", + "db:migrate:collectivex": "bun --env-file=../../.env src/migrate-collectivex.ts", "db:apply-overrides": "bun --env-file=../../.env src/apply-overrides.ts", "db:backfill-aggregate-stats": "bun --env-file=../../.env src/backfill-aggregate-stats.ts", "db:backfill-chart-series": "bun --env-file=../../.env src/backfill-chart-series.ts", diff --git a/packages/db/src/collectivex/artifact-selection.test.ts b/packages/db/src/collectivex/artifact-selection.test.ts new file mode 100644 index 000000000..78622dfe1 --- /dev/null +++ b/packages/db/src/collectivex/artifact-selection.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from 'vitest'; + +import { + matrixArtifactName, + selectShardArtifactNames, + selectShardArtifacts, +} from './artifact-selection'; + +describe('shard artifact selection', () => { + it('selects one artifact per cell, sorted by name', () => { + expect( + selectShardArtifactNames( + ['cxshard-b-160-1', 'cxshard-a-160-1', 'cxsweep-matrix-160'], + '160', + 1, + ), + ).toEqual(['cxshard-a-160-1', 'cxshard-b-160-1']); + }); + + it('prefers the highest attempt not above the run attempt', () => { + const names = ['cxshard-a-160-1', 'cxshard-a-160-2', 'cxshard-a-160-3', 'cxshard-b-160-1']; + expect(selectShardArtifactNames(names, '160', 2)).toEqual([ + 'cxshard-a-160-2', + 'cxshard-b-160-1', + ]); + }); + + it('keeps cells with hyphenated names intact across run-id collisions', () => { + // A cell name may itself contain "-" fragments; only the trailing + // "-{runId}-{attempt}" is structural. + const names = ['cxshard-h200-ep8-160-1', 'cxshard-h200-ep8-161-1']; + expect(selectShardArtifactNames(names, '160', 1)).toEqual(['cxshard-h200-ep8-160-1']); + }); + + it('ignores foreign names and zero attempts', () => { + expect( + selectShardArtifactNames(['cxshard-a-160-0', 'other-160-1', 'cxshard-a-999-1'], '160', 1), + ).toEqual([]); + }); + it('breaks same-attempt ties on the later artifact id', () => { + // GitHub permits a repeated artifact name within one run; the lazy ingest has + // ids and must keep the later upload. Both ingest paths share this selector, + // so the tie-break cannot drift between them again. + const artifacts = [ + { name: 'cxshard-a-160-1', id: 10 }, + { name: 'cxshard-a-160-1', id: 42 }, + ]; + expect(selectShardArtifacts(artifacts, '160', 1)).toEqual([ + { name: 'cxshard-a-160-1', id: 42 }, + ]); + }); + + it('keeps the first match when callers supply no ids', () => { + // The names-only path has no id to compare, so it stays deterministic on input order. + expect( + selectShardArtifacts([{ name: 'cxshard-a-160-1' }, { name: 'cxshard-a-160-1' }], '160', 1), + ).toHaveLength(1); + }); + + it('treats a run id with regex metacharacters literally', () => { + // The id is interpolated into the pattern; an unescaped '.' would match any char. + expect(selectShardArtifactNames(['cxshard-a-1x0-1'], '1.0', 1)).toEqual([]); + }); +}); + +describe('matrixArtifactName', () => { + it('derives the per-run matrix artifact name', () => { + expect(matrixArtifactName('160')).toBe('cxsweep-matrix-160'); + }); +}); diff --git a/packages/db/src/collectivex/artifact-selection.ts b/packages/db/src/collectivex/artifact-selection.ts new file mode 100644 index 000000000..d01341d32 --- /dev/null +++ b/packages/db/src/collectivex/artifact-selection.ts @@ -0,0 +1,87 @@ +/** + * CollectiveX sweep artifact naming and selection. Pure helpers shared by the + * ingest script and its tests. + * + * The sweep uploads two artifact families per run: + * cxsweep-matrix-{run_id} — one matrix document + * cxshard-{cell}-{run_id}-{attempt} — case-attempt documents per matrix cell + */ + +const MATRIX_PREFIX = 'cxsweep-matrix-'; +const SHARD_PREFIX = 'cxshard-'; + +export function matrixArtifactName(runId: string): string { + return `${MATRIX_PREFIX}${runId}`; +} + +/** Escape regex metacharacters so an interpolated value matches literally. */ +function escapeRegExp(value: string): string { + return value.replaceAll(/[.*+?^${}()|[\]\\]/gu, String.raw`\$&`); +} + +/** Anything nameable as a shard artifact. `id` is optional — see the tie-break below. */ +interface ShardArtifactRef { + name: string; + /** GitHub artifact id, when the caller has one. */ + id?: number; +} + +/** + * Pick the shard artifacts to ingest: keep the highest attempt ≤ the run's + * current attempt per cell (a re-run attempt supersedes its predecessors; + * attempts above the run's own attempt cannot legitimately exist and are + * ignored). + * + * Both ingest paths — the CLI, which has only names, and the lazy ingest, + * which has full artifact records — share this one implementation. They used + * to carry separate copies that had already drifted apart on the tie-break + * below, which meant the same run could resolve to different documents + * depending on which path ingested it. + * + * Ties at the same attempt are broken by the greater `id` when callers supply + * one (GitHub permits repeated artifact names within a run, and the later + * upload is the one to keep); without ids the first match wins, so the + * names-only path stays deterministic on input order. + */ +export function selectShardArtifacts( + artifacts: readonly T[], + runId: string, + runAttempt: number, +): T[] { + // runId is interpolated into a pattern, so escape it: an unescaped caller value + // would change what this matches (regular expression injection), and a crafted + // one could make the `.+` prefix backtrack pathologically. + const pattern = new RegExp( + `^${SHARD_PREFIX}(?.+)-${escapeRegExp(runId)}-(?[1-9][0-9]*)$`, + 'u', + ); + const selected = new Map(); + for (const artifact of artifacts) { + const match = pattern.exec(artifact.name); + if (!match) continue; + const attempt = Number(match.groups!.attempt); + if (attempt > runAttempt) continue; + const previous = selected.get(match.groups!.cell); + const supersedes = + !previous || + attempt > previous.attempt || + (attempt === previous.attempt && (artifact.id ?? -1) > (previous.artifact.id ?? -1)); + if (supersedes) selected.set(match.groups!.cell, { artifact, attempt }); + } + return [...selected.values()].map((entry) => entry.artifact); +} + +/** Name-only convenience wrapper over {@link selectShardArtifacts}, sorted for stable output. */ +export function selectShardArtifactNames( + names: readonly string[], + runId: string, + runAttempt: number, +): string[] { + return selectShardArtifacts( + names.map((name) => ({ name })), + runId, + runAttempt, + ) + .map((entry) => entry.name) + .toSorted(); +} diff --git a/packages/db/src/collectivex/reader.test.ts b/packages/db/src/collectivex/reader.test.ts new file mode 100644 index 000000000..57407ca83 --- /dev/null +++ b/packages/db/src/collectivex/reader.test.ts @@ -0,0 +1,210 @@ +import { describe, expect, it } from 'vitest'; + +import { buildDatasetFromNeutral } from './reader'; +import { + buildDataset, + makeCollectiveXDataset, + makeCollectiveXSeries, + makeInvalidCaseAttempt, + makeRawMatrix, + makeRawShard, + makeRunMeta, +} from './test-fixture'; + +function requestedOf(shard: Record) { + const identity = shard.identity as { + case_id: string; + case_factors: { sku: string; case: Record }; + }; + return { + caseId: identity.case_id, + sku: identity.case_factors.sku, + disposition: 'runnable' as const, + case: identity.case_factors.case, + }; +} + +describe('CollectiveX artifact assembly', () => { + it('builds the current view from matrix cases and result shards', () => { + const dataset = makeCollectiveXDataset(); + expect(dataset.version).toBe(1); + expect(dataset.series).toHaveLength(3); + expect(dataset.coverage).toHaveLength(5); + expect(dataset.run).toMatchObject({ + requested_cases: 5, + measured_cases: 3, + unsupported_cases: 1, + terminal_cases: 4, + measured_points: 30, + terminal_points: 40, + requested_points: 50, + }); + }); + + it('maps series identity and points', () => { + const series = makeCollectiveXSeries(); + expect(series.series_id).toBe('h200-dgxc-deepep-v2-deepseek-v3-normal-decode-ep8-uniform-bf16'); + expect(series.backend).toBe('deepep-v2'); + expect(series.precision).toBe('bf16'); + expect(series.points).toHaveLength(10); + }); + + it('keeps bf16 and fp8 measurements of one cell as distinct labeled cases', () => { + const dataset = makeCollectiveXDataset(); + const h200 = dataset.series.filter((series) => series.system.sku === 'h200-dgxc'); + expect(h200.map((series) => series.precision).toSorted()).toEqual(['bf16', 'fp8']); + expect(new Set(h200.map((series) => series.series_id)).size).toBe(2); + expect(dataset.coverage.find((row) => row.precision === 'fp8')).toMatchObject({ + case_id: 'h200-dgxc-deepep-v2-deepseek-v3-normal-decode-ep8-uniform-fp8', + label: 'h200-dgxc · deepep-v2 · normal · decode · EP8 · fp8', + }); + expect( + dataset.coverage.find( + (row) => row.case_id === 'h200-dgxc-deepep-v2-deepseek-v3-normal-decode-ep8-uniform-bf16', + ), + ).toMatchObject({ + precision: 'bf16', + label: 'h200-dgxc · deepep-v2 · normal · decode · EP8 · bf16', + }); + }); + + it('defaults pre-FP8 artifacts without a precision field to bf16', () => { + const dataset = buildDataset({ shards: [makeRawShard({ precision: null })] }); + expect(dataset.series[0].series_id).toBe( + 'h200-dgxc-deepep-v2-deepseek-v3-normal-decode-ep8-uniform', + ); + expect(dataset.series[0].precision).toBe('bf16'); + expect(dataset.coverage[0]).toMatchObject({ + precision: 'bf16', + label: 'h200-dgxc · deepep-v2 · normal · decode · EP8 · bf16', + }); + }); + + it('keeps normal and low-latency measurements of one cell as distinct labeled cases', () => { + const dataset = buildDataset({ + shards: [makeRawShard(), makeRawShard({ mode: 'low-latency' })], + }); + expect(new Set(dataset.series.map((series) => series.series_id)).size).toBe(2); + expect(dataset.series.map((series) => series.mode).toSorted()).toEqual([ + 'low-latency', + 'normal', + ]); + expect(dataset.coverage.find((row) => row.mode === 'low-latency')).toMatchObject({ + case_id: 'h200-dgxc-deepep-v2-deepseek-v3-low-latency-decode-ep8-uniform-bf16', + label: 'h200-dgxc · deepep-v2 · low-latency · decode · EP8 · bf16', + }); + }); + + it('defaults pre-LL artifacts without a mode field to normal kernels', () => { + const dataset = buildDataset({ shards: [makeRawShard({ mode: null })] }); + expect(dataset.series[0].mode).toBe('normal'); + expect(dataset.coverage[0].mode).toBe('normal'); + }); + + it('ignores non-result documents', () => { + const shard = makeRawShard(); + const dataset = buildDatasetFromNeutral( + makeRawMatrix([requestedOf(shard)]), + [shard, { record_type: 'samples', rows: [] }], + makeRunMeta(), + ); + expect(dataset.series).toHaveLength(1); + }); + + it('normalizes in-band failure reasons', () => { + const dataset = buildDataset({ + shards: [ + makeInvalidCaseAttempt({ reasons: ['semantic correctness or routing identity failed'] }), + ], + }); + expect(dataset.series).toHaveLength(0); + expect(dataset.coverage[0]).toMatchObject({ + outcome: 'invalid', + reason: 'semantic-correctness-or-routing-identity-failed', + }); + }); + + it('keeps capacity-limited points omitted by a successful backend', () => { + const dataset = buildDataset({ + shards: [ + makeRawShard({ + phase: 'prefill', + rows: [{ tokensPerRank: 256 }, { tokensPerRank: 512 }], + }), + ], + }); + expect(dataset.coverage[0].points.map((point) => point.terminal_status)).toEqual([ + 'measured', + 'measured', + 'unsupported', + 'unsupported', + ]); + expect(dataset.coverage[0].points.at(-1)).toMatchObject({ + tokens_per_rank: 2048, + reason: 'backend-token-capacity', + }); + }); + + it('does not read a success shard with no rows as a capacity wall', () => { + // Math.max() of no rows is -Infinity, which would put every ladder point + // "beyond the largest measured" and report a token-capacity limit that was + // never observed. Nothing was measured, so every point is pending. + const dataset = buildDataset({ shards: [makeRawShard({ rows: [] })] }); + const points = dataset.coverage[0].points; + expect(points.map((point) => point.terminal_status)).toEqual(points.map(() => 'pending')); + expect(points.every((point) => point.reason === 'not-measured')).toBe(true); + }); + + it('keeps unsupported and pending cases distinct', () => { + const dataset = makeCollectiveXDataset(); + expect(dataset.coverage.find((row) => row.sku === 'b300')).toMatchObject({ + outcome: 'unsupported', + reason: 'backend-platform-unsupported', + detail: 'unsupported by the selected backend/platform', + }); + expect(dataset.coverage.find((row) => row.sku === 'b200-dgxc')).toMatchObject({ + outcome: 'pending', + reason: 'pending', + }); + }); + + it('maps an nccl-ep backend series (the 4th pluggable backend)', () => { + const series = makeCollectiveXSeries({ backend: 'nccl-ep', implName: 'nccl-ep' }); + expect(series.backend).toBe('nccl-ep'); + expect(series.series_id).toContain('nccl-ep'); + expect(series.points).toHaveLength(10); + }); + + it('derives a per-GPU payload bandwidth from total_logical_bytes', () => { + const dispatch = makeCollectiveXSeries().points[0].components.dispatch; + // total_logical_bytes (400000000) / ep (8) / p50 latency (417 µs) → GB/s. + expect(dispatch?.payload_data_rate_gbps_at_latency_percentile?.p50).toBeCloseTo( + (400000000 / 8 / 417) * 1e-3, + 3, + ); + // Distinct from the aggregate activation rate (activation bytes, no ep split): + // the payload rate reads total_logical_bytes and divides by the EP world size. + expect(dispatch?.payload_data_rate_gbps_at_latency_percentile?.p50).not.toBeCloseTo( + dispatch?.activation_data_rate_gbps_at_latency_percentile?.p50 ?? 0, + 1, + ); + }); + + it('does not invent rates for zero-byte or unavailable components', () => { + const zeroStage = makeCollectiveXSeries({ rows: [{ stageZeroBytes: true }] }).points[0] + .components.stage; + expect(zeroStage?.activation_data_rate_gbps_at_latency_percentile?.p50).toBe(0); + expect( + makeCollectiveXSeries({ rows: [{ stageUnavailable: true }] }).points[0].components.stage, + ).toBeNull(); + }); + + it('rejects malformed and cross-version artifacts', () => { + expect(() => buildDatasetFromNeutral({}, [], makeRunMeta())).toThrow(/matrix/); + const shard = makeRawShard(); + shard.version = 2; + expect(() => + buildDatasetFromNeutral(makeRawMatrix([requestedOf(shard)]), [shard], makeRunMeta()), + ).toThrow(/version/); + }); +}); diff --git a/packages/db/src/collectivex/reader.ts b/packages/db/src/collectivex/reader.ts new file mode 100644 index 000000000..0dae4b021 --- /dev/null +++ b/packages/db/src/collectivex/reader.ts @@ -0,0 +1,397 @@ +/** + * CollectiveX neutral-contract reader: assembles the dashboard dataset from a + * sweep run's raw matrix + case-attempt docs. Shared by the ingest script + * (validation + summary precompute) and the app's API routes (assembly at + * read time), so the transform can never drift between the two. + * + * The sweep JSON contract is expected to change; when it does, update this + * reader (and bump the `version` tag in the harness's sweep config). Raw docs + * are stored untouched in the DB, so reader fixes retroactively apply to + * already-ingested runs. + */ + +import type { + CollectiveXComponent, + CollectiveXCoverage, + CollectiveXCoveragePoint, + CollectiveXDataset, + CollectiveXMode, + CollectiveXOutcome, + CollectiveXPercentiles, + CollectiveXPoint, + CollectiveXPrecision, + CollectiveXRunSummary, + CollectiveXSeries, + CollectiveXTerminalStatus, +} from './types'; + +interface RawCase { + case_id?: string; + backend: string; + ep: number; + gpus_per_node: number; + ladder: string; + nodes: number; + mode?: string; + phase: string; + precision?: string; + topology_class: string; + scale_up_domain: number; + scale_up_transport: string; + scale_out_transport: string | null; +} + +interface RawComponent { + availability: string; + percentiles_us: CollectiveXPercentiles | null; +} + +interface RawRow { + tokens_per_rank: number; + global_tokens: number; + token_rate_at_latency_percentile: CollectiveXPercentiles; + components: Record; + byte_provenance: Record; +} + +interface RawShard { + version: number; + record_type: 'case-attempt'; + identity: { + case_id: string; + case_factors: { sku: string; case: RawCase }; + }; + implementation: { name: string }; + runtime: { vendor: string }; + measurement: { rows: RawRow[] }; + outcome: { status: string; reasons?: string[] }; +} + +interface RawMatrix { + version: number; + requested_cases: { + case: RawCase; + sku: string; + disposition: 'runnable' | 'unsupported'; + reason?: string | null; + detail?: string | null; + }[]; +} + +export interface CollectiveXNeutralRunMeta { + run_id: string; + run_attempt: number; + generated_at: string; + conclusion: string | null; + source_sha: string; +} + +function matrixOf(value: unknown): RawMatrix { + const matrix = value as RawMatrix; + if (!Number.isSafeInteger(matrix?.version) || !Array.isArray(matrix?.requested_cases)) { + throw new TypeError('invalid CollectiveX matrix'); + } + return matrix; +} + +function shardOf(value: unknown): RawShard | null { + if ((value as RawShard | null)?.record_type !== 'case-attempt') return null; + const shard = value as RawShard; + if (!shard.identity?.case_id || !Array.isArray(shard.measurement?.rows)) { + throw new TypeError('invalid CollectiveX shard'); + } + return shard; +} + +function toOutcome(status: string): CollectiveXOutcome { + return ['success', 'unsupported', 'failed', 'invalid', 'diagnostic', 'pending'].includes(status) + ? (status as CollectiveXOutcome) + : 'failed'; +} + +function toTerminalStatus(outcome: CollectiveXOutcome): CollectiveXTerminalStatus { + return outcome === 'success' ? 'measured' : outcome; +} + +// Artifacts predating the FP8 dispatch dimension carry no precision field and +// were all measured in bf16. +function toPrecision(raw: string | undefined): CollectiveXPrecision { + return raw === 'fp8' ? 'fp8' : 'bf16'; +} + +// Artifacts predating the low-latency kernel dimension carry no mode field +// and were all measured with the normal (throughput) kernels. +function toMode(raw: string | undefined): CollectiveXMode { + return raw === 'low-latency' ? 'low-latency' : 'normal'; +} + +// GB/s = bytes / (latency_us * 1e-6) / 1e9 = (bytes / latency_us) * 1e-3. `divisor` +// splits an aggregate world byte count into a per-GPU figure (divisor = ep_size). +function ratesFrom( + bytes: number, + latency: CollectiveXPercentiles, + divisor = 1, +): CollectiveXPercentiles { + const rate = (us: number) => (bytes / divisor / us) * 1e-3; + return { + p50: rate(latency.p50), + p90: rate(latency.p90), + p95: rate(latency.p95), + p99: rate(latency.p99), + }; +} + +function mapComponent( + raw: RawComponent | null | undefined, + bytes: { activation_data_bytes: number; total_logical_bytes?: number } | undefined, + ep: number, +): CollectiveXComponent | null { + if (!raw?.percentiles_us || raw.availability === 'unavailable') return null; + // Byte counts are aggregate across the EP world (routed_copies = fanout.sum()). + // Activation rate stays aggregate (unchanged); payload rate is per-GPU over the + // full logical payload, falling back to activation bytes for pre-provenance + // artifacts that carry no total_logical_bytes. + const payloadBytes = bytes ? (bytes.total_logical_bytes ?? bytes.activation_data_bytes) : null; + return { + latency_us: raw.percentiles_us, + activation_data_rate_gbps_at_latency_percentile: bytes + ? ratesFrom(bytes.activation_data_bytes, raw.percentiles_us) + : null, + payload_data_rate_gbps_at_latency_percentile: + payloadBytes === null ? null : ratesFrom(payloadBytes, raw.percentiles_us, Math.max(1, ep)), + payload_bytes: payloadBytes, + }; +} + +function mapPoint(row: RawRow, ep: number): CollectiveXPoint { + const component = (name: string) => + mapComponent(row.components[name], row.byte_provenance[name], ep); + return { + tokens_per_rank: row.tokens_per_rank, + global_tokens: row.global_tokens, + components: { + dispatch: component('dispatch'), + stage: component('stage'), + combine: component('combine'), + roundtrip: component('roundtrip'), + }, + roundtrip_token_rate_at_latency_percentile: row.token_rate_at_latency_percentile, + }; +} + +function topologyOf(kase: RawCase) { + return { + ep_size: kase.ep, + nodes: kase.nodes, + gpus_per_node: kase.gpus_per_node, + scale_up_domain: kase.scale_up_domain, + scale_up_transport: kase.scale_up_transport, + scale_out_transport: kase.scale_out_transport, + topology_class: kase.topology_class, + }; +} + +function buildSeries(shard: RawShard): CollectiveXSeries { + const kase = shard.identity.case_factors.case; + return { + series_id: shard.identity.case_id, + phase: kase.phase === 'prefill' ? 'prefill' : 'decode', + mode: toMode(kase.mode), + precision: toPrecision(kase.precision), + backend: shard.implementation.name, + system: { + ...topologyOf(kase), + sku: shard.identity.case_factors.sku, + vendor: shard.runtime.vendor === 'amd' ? 'amd' : 'nvidia', + }, + points: shard.measurement.rows.map((row) => mapPoint(row, kase.ep)), + }; +} + +function ladderTokens(kase: RawCase): number[] { + const values = kase.ladder + .split(/\s+/) + .map(Number) + .filter((value) => Number.isSafeInteger(value) && value > 0); + return values.length > 0 ? values : [kase.ep]; +} + +function reasonId(value: string): string { + return ( + value + .toLowerCase() + .replaceAll(/[^a-z0-9.-]+/g, '-') + .replace(/^[^a-z0-9]+/, '') + .slice(0, 96) || 'unknown' + ); +} + +function measuredPoints(shard: RawShard, kase: RawCase): CollectiveXCoveragePoint[] { + const rows = new Map(shard.measurement.rows.map((row) => [row.tokens_per_rank, row])); + // null when the shard measured nothing. `Math.max()` of an empty list is + // -Infinity, which would put every ladder point above the largest measured + // value and report a backend token-capacity limit that was never observed — + // an unmeasured case would read as a hard capability wall. + const largestMeasured = rows.size > 0 ? Math.max(...rows.keys()) : null; + return ladderTokens(kase).map((tokens) => { + const row = rows.get(tokens); + if (row) { + return { + tokens_per_rank: tokens, + global_tokens: row.global_tokens, + terminal_status: 'measured' as const, + reason: null, + }; + } + // Only a point beyond something we actually measured is evidence of a capacity limit. + const beyondCapacity = largestMeasured !== null && tokens > largestMeasured; + return { + tokens_per_rank: tokens, + global_tokens: tokens * kase.ep, + terminal_status: beyondCapacity ? ('unsupported' as const) : ('pending' as const), + reason: beyondCapacity ? 'backend-token-capacity' : 'not-measured', + }; + }); +} + +function terminalPoints( + kase: RawCase, + status: CollectiveXTerminalStatus, + reason: string, +): CollectiveXCoveragePoint[] { + return ladderTokens(kase).map((tokens) => ({ + tokens_per_rank: tokens, + global_tokens: tokens * kase.ep, + terminal_status: status, + reason, + })); +} + +export function buildDatasetFromNeutral( + matrixRaw: unknown, + docs: unknown[], + run: CollectiveXNeutralRunMeta, +): CollectiveXDataset { + const matrix = matrixOf(matrixRaw); + const shards = docs.flatMap((doc) => { + const shard = shardOf(doc); + if (!shard) return []; + if (shard.version !== matrix.version) throw new Error('CollectiveX version mismatch'); + return [shard]; + }); + const successful = new Map(); + const terminal = new Map(); + for (const shard of shards) { + const target = shard.outcome.status === 'success' ? successful : terminal; + if (!target.has(shard.identity.case_id)) target.set(shard.identity.case_id, shard); + } + + const coverage: CollectiveXCoverage[] = matrix.requested_cases.flatMap((requested) => { + const kase = requested.case; + const caseId = kase.case_id; + if (!caseId) return []; + const measured = successful.get(caseId); + const failed = terminal.get(caseId); + let outcome: CollectiveXOutcome; + let reason: string | null; + let points: CollectiveXCoveragePoint[]; + if (measured) { + outcome = 'success'; + reason = null; + points = measuredPoints(measured, kase); + } else if (failed) { + outcome = toOutcome(failed.outcome.status); + reason = reasonId(failed.outcome.reasons?.[0] ?? outcome); + points = terminalPoints(kase, toTerminalStatus(outcome), reason); + } else if (requested.disposition === 'unsupported') { + outcome = 'unsupported'; + reason = reasonId(requested.reason ?? outcome); + points = terminalPoints(kase, 'unsupported', reason); + } else { + outcome = 'pending'; + reason = 'pending'; + points = terminalPoints(kase, 'pending', reason); + } + const mode = toMode(kase.mode); + const precision = toPrecision(kase.precision); + return [ + { + case_id: caseId, + label: `${requested.sku} · ${kase.backend} · ${mode} · ${kase.phase} · EP${kase.ep} · ${precision}`, + disposition: requested.disposition, + sku: requested.sku, + backend: kase.backend, + phase: kase.phase === 'prefill' ? 'prefill' : 'decode', + mode, + precision, + topology: topologyOf(kase), + points, + outcome, + reason, + detail: requested.detail ?? null, + }, + ]; + }); + const points = coverage.flatMap((item) => item.points); + return { + version: matrix.version, + run: { + ...run, + requested_cases: coverage.length, + terminal_cases: coverage.filter((item) => + item.points.every((point) => point.terminal_status !== 'pending'), + ).length, + measured_cases: coverage.filter((item) => item.outcome === 'success').length, + unsupported_cases: coverage.filter((item) => item.outcome === 'unsupported').length, + failed_cases: coverage.filter((item) => + ['failed', 'invalid', 'diagnostic'].includes(item.outcome), + ).length, + requested_points: points.length, + terminal_points: points.filter((point) => point.terminal_status !== 'pending').length, + measured_points: points.filter((point) => point.terminal_status === 'measured').length, + covered_skus: [...new Set(coverage.map((item) => item.sku))].toSorted(), + }, + coverage, + series: [...successful.values()].map(buildSeries), + }; +} + +export function buildRunSummary(dataset: CollectiveXDataset): CollectiveXRunSummary { + const { run } = dataset; + return { + run_id: run.run_id, + run_attempt: run.run_attempt, + generated_at: run.generated_at, + conclusion: run.conclusion, + covered_skus: run.covered_skus, + requested_cases: run.requested_cases, + measured_cases: run.measured_cases, + requested_points: run.requested_points, + terminal_points: run.terminal_points, + terminal_counts: { + measured: run.measured_cases, + unsupported: run.unsupported_cases, + failed: run.failed_cases, + }, + }; +} + +/** + * Structural identity check for the matrix document (it carries no + * `record_type` tag): requested_cases[] + include[] arrays plus a valid + * numeric `version` — the content axis the frontend selects on. + */ +export function isMatrixDoc(doc: unknown): boolean { + const candidate = doc as { requested_cases?: unknown; include?: unknown } | null; + return ( + Array.isArray(candidate?.requested_cases) && + Array.isArray(candidate?.include) && + matrixVersion(doc) !== null + ); +} + +/** Read the matrix doc's numeric version tag; null when absent or invalid. */ +export function matrixVersion(doc: unknown): number | null { + const value = (doc as { version?: unknown } | null)?.version; + return typeof value === 'number' && Number.isSafeInteger(value) && value > 0 ? value : null; +} diff --git a/packages/db/src/collectivex/test-fixture.ts b/packages/db/src/collectivex/test-fixture.ts new file mode 100644 index 000000000..ccb32f6e8 --- /dev/null +++ b/packages/db/src/collectivex/test-fixture.ts @@ -0,0 +1,253 @@ +import { buildDatasetFromNeutral, type CollectiveXNeutralRunMeta } from './reader'; +import type { CollectiveXDataset, CollectiveXSeries } from './types'; + +type Json = Record; + +const SOURCE_SHA = 'c'.repeat(40); +const TOKEN_LADDERS = { + decode: '1 2 4 8 16 32 64 128 256 512', + prefill: '256 512 1024 2048', +} as const; + +export interface RowOverrides { + tokensPerRank?: number; + globalTokens?: number; + stageUnavailable?: boolean; + stageZeroBytes?: boolean; +} + +export interface ShardOverrides { + caseId?: string; + variant?: string; + sku?: string; + backend?: string; + implName?: string; + ep?: number; + phase?: string; + /** null models a pre-LL artifact: no mode field (the case_id keeps `normal`). */ + mode?: string | null; + /** null models a pre-FP8 artifact: no precision field and no case_id suffix. */ + precision?: string | null; + scaleUpTransport?: string; + scaleOutTransport?: string | null; + topologyClass?: string; + nodes?: number; + gpusPerNode?: number; + scaleUpDomain?: number; + vendor?: string; + workload?: string; + ladder?: string; + status?: string; + reasons?: string[]; + rows?: RowOverrides[]; +} + +function percentiles(base: number): Json { + return { p50: base, p90: base * 1.08, p95: base * 1.12, p99: base * 1.2 }; +} + +function component(base: number): Json { + return { availability: 'measured', percentiles_us: percentiles(base) }; +} + +// `total` defaults to the activation count (the bf16 case, where there are no +// scale bytes). Dispatch under FP8 carries extra scale bytes, so its total +// exceeds activation — the fixture models that so tests can prove the payload +// rate reads total_logical_bytes rather than activation_data_bytes. +function bytes(activation: number, total: number = activation): Json { + return { activation_data_bytes: activation, total_logical_bytes: total }; +} + +function makeRawRow(index: number, row: RowOverrides, worldSize: number): Json { + const tokensPerRank = row.tokensPerRank ?? 128 * (index + 1); + const components: Json = { + dispatch: component(417 + index), + combine: component(392 + index), + roundtrip: component(921 + index), + stage: row.stageUnavailable + ? { availability: 'unavailable', percentiles_us: null } + : component(120 + index), + }; + // Dispatch total exceeds activation (models FP8 scale bytes); combine is + // always bf16 (total == activation); roundtrip total is their sum. + const byteProvenance: Json = { + dispatch: bytes(384763904, 400000000), + combine: bytes(384763904), + roundtrip: bytes(769527808, 784763904), + }; + if (!row.stageUnavailable) { + byteProvenance.stage = bytes(row.stageZeroBytes ? 0 : 192381952); + } + return { + tokens_per_rank: tokensPerRank, + global_tokens: row.globalTokens ?? tokensPerRank * worldSize, + token_rate_at_latency_percentile: percentiles(8_338_218), + components, + byte_provenance: byteProvenance, + }; +} + +function makeRawCase(options: ShardOverrides, caseId: string): Json { + const phase = options.phase === 'prefill' ? 'prefill' : 'decode'; + return { + case_id: caseId, + backend: options.backend ?? 'deepep-v2', + ep: options.ep ?? 8, + gpus_per_node: options.gpusPerNode ?? 8, + ladder: options.ladder ?? TOKEN_LADDERS[phase], + nodes: options.nodes ?? 1, + ...(options.mode === null ? {} : { mode: options.mode ?? 'normal' }), + phase, + ...(options.precision === null ? {} : { precision: options.precision ?? 'bf16' }), + topology_class: options.topologyClass ?? 'h200-nvlink-island', + scale_up_domain: options.scaleUpDomain ?? 8, + scale_up_transport: options.scaleUpTransport ?? 'nvlink', + scale_out_transport: options.scaleOutTransport ?? null, + }; +} + +function caseIdOf(options: ShardOverrides = {}): string { + if (options.caseId) return options.caseId; + const tail = options.variant ? `-${options.variant}` : ''; + // Pre-LL artifacts (mode: null) still carried `normal` in their case_ids. + const mode = options.mode === null ? 'normal' : (options.mode ?? 'normal'); + const precision = options.precision === null ? '' : `-${options.precision ?? 'bf16'}`; + return `${options.sku ?? 'h200-dgxc'}-${options.backend ?? 'deepep-v2'}-${options.workload ?? 'deepseek-v3'}-${mode}-${options.phase ?? 'decode'}-ep${options.ep ?? 8}-uniform${precision}${tail}`; +} + +export function makeRawShard(options: ShardOverrides = {}): Json { + const caseId = caseIdOf(options); + const sku = options.sku ?? 'h200-dgxc'; + const backend = options.backend ?? 'deepep-v2'; + const phase = options.phase === 'prefill' ? 'prefill' : 'decode'; + const ladder = options.ladder ?? TOKEN_LADDERS[phase]; + const worldSize = (options.nodes ?? 1) * (options.gpusPerNode ?? 8); + const rows = + options.rows ?? ladder.split(/\s+/).map((tokens) => ({ tokensPerRank: Number(tokens) })); + return { + version: 1, + record_type: 'case-attempt', + identity: { + case_id: caseId, + case_factors: { sku, case: makeRawCase({ ...options, backend }, caseId) }, + }, + implementation: { name: options.implName ?? backend }, + runtime: { vendor: options.vendor ?? 'nvidia' }, + measurement: { rows: rows.map((row, index) => makeRawRow(index, row, worldSize)) }, + outcome: { + status: options.status ?? 'success', + ...(options.reasons ? { reasons: options.reasons } : {}), + }, + }; +} + +export function makeInvalidCaseAttempt(options: ShardOverrides = {}): Json { + return makeRawShard({ status: 'invalid', reasons: ['capability-gate'], ...options }); +} + +interface RequestedCaseSpec { + caseId: string; + sku: string; + disposition?: 'runnable' | 'unsupported'; + reason?: string; + case: Json; +} + +function requestedFromShard(shard: Json): RequestedCaseSpec { + const identity = shard.identity as Json; + const factors = identity.case_factors as Json; + return { + caseId: identity.case_id as string, + sku: factors.sku as string, + case: factors.case as Json, + }; +} + +export function makeRawMatrix(requested: RequestedCaseSpec[], version = 1): Json { + return { + version, + include: [], + requested_cases: requested.map((entry) => ({ + case: entry.case, + sku: entry.sku, + disposition: entry.disposition ?? 'runnable', + reason: entry.reason ?? null, + detail: entry.reason ? 'unsupported by the selected backend/platform' : null, + })), + }; +} + +export function makeRunMeta( + overrides: Partial = {}, +): CollectiveXNeutralRunMeta { + return { + run_id: '160', + run_attempt: 1, + generated_at: '2026-07-08T12:20:00Z', + conclusion: 'success', + source_sha: SOURCE_SHA, + ...overrides, + }; +} + +export function buildDataset( + options: { + shards?: Json[]; + requestedCases?: RequestedCaseSpec[]; + meta?: Partial; + } = {}, +): CollectiveXDataset { + const shards = options.shards ?? [makeRawShard()]; + const requested = [...shards.map(requestedFromShard), ...(options.requestedCases ?? [])]; + return buildDatasetFromNeutral(makeRawMatrix(requested), shards, makeRunMeta(options.meta)); +} + +export function makeCollectiveXSeries(overrides: ShardOverrides = {}): CollectiveXSeries { + return buildDataset({ shards: [makeRawShard(overrides)] }).series[0]; +} + +export function makeCollectiveXDataset(): CollectiveXDataset { + const shardA = makeRawShard(); + const shardB = makeRawShard({ + sku: 'mi355x', + backend: 'mori', + implName: 'mori', + vendor: 'amd', + ep: 16, + scaleUpTransport: 'xgmi', + scaleOutTransport: 'rdma', + topologyClass: 'mi355x-xgmi-rdma', + nodes: 2, + }); + // The same cell as shardA measured with FP8 dispatch, so consumers exercise + // the bf16/fp8 split of an otherwise identical configuration. + const shardC = makeRawShard({ precision: 'fp8' }); + const unsupportedId = 'b300-deepep-v2-deepseek-v3-normal-decode-ep16-uniform-bf16'; + const pendingId = 'b200-dgxc-deepep-v2-deepseek-v3-normal-decode-ep8-uniform-bf16'; + return buildDataset({ + shards: [shardA, shardB, shardC], + requestedCases: [ + { + caseId: unsupportedId, + sku: 'b300', + disposition: 'unsupported', + reason: 'backend-platform-unsupported', + case: makeRawCase( + { + backend: 'deepep-v2', + ep: 16, + nodes: 2, + scaleOutTransport: 'rdma', + topologyClass: 'b300-nvlink-rdma', + }, + unsupportedId, + ), + }, + { + caseId: pendingId, + sku: 'b200-dgxc', + case: makeRawCase({ backend: 'deepep-v2', topologyClass: 'b200-nvlink-island' }, pendingId), + }, + ], + }); +} diff --git a/packages/db/src/collectivex/types.ts b/packages/db/src/collectivex/types.ts new file mode 100644 index 000000000..9bc2d2e3c --- /dev/null +++ b/packages/db/src/collectivex/types.ts @@ -0,0 +1,146 @@ +/** + * CollectiveX neutral contract — the version-tagged shape shared by the sweep + * artifacts' reader, the ingest script, and the dashboard frontend. + * + * This module is pure TypeScript (no Node/DB imports) so it is safe to import + * from client components. UI-only types (chart points, axis modes, display + * label helpers) live in `packages/app/src/components/collectivex/types.ts`. + */ + +export type CollectiveXPhase = 'decode' | 'prefill'; +export type CollectiveXPrecision = 'bf16' | 'fp8'; +/** Kernel mode: throughput-oriented `normal`, or decode-only `low-latency`. */ +export type CollectiveXMode = 'normal' | 'low-latency'; +export const COLLECTIVEX_VERSIONS = [1] as const; +export type CollectiveXVersion = (typeof COLLECTIVEX_VERSIONS)[number]; +export const COLLECTIVEX_DEFAULT_VERSION: CollectiveXVersion = COLLECTIVEX_VERSIONS.at(-1)!; + +export function parseCollectiveXVersion(raw: string): CollectiveXVersion | null { + const version = Number(raw); + return (COLLECTIVEX_VERSIONS as readonly number[]).includes(version) + ? (version as CollectiveXVersion) + : null; +} + +export type CollectiveXOperation = 'dispatch' | 'stage' | 'combine' | 'roundtrip'; +export type CollectiveXPercentile = 'p50' | 'p90' | 'p95' | 'p99'; +export type CollectiveXOutcome = + | 'success' + | 'unsupported' + | 'failed' + | 'invalid' + | 'diagnostic' + | 'pending'; +export type CollectiveXTerminalStatus = Exclude | 'measured'; + +export type CollectiveXPercentiles = Record; + +export interface CollectiveXComponent { + latency_us: CollectiveXPercentiles; + activation_data_rate_gbps_at_latency_percentile: CollectiveXPercentiles | null; + /** + * Per-GPU bandwidth over the FULL logical payload (activation bytes plus any + * FP8 scale bytes), i.e. `total_logical_bytes / ep_size / latency`. Distinct + * from `activation_data_rate_gbps_at_latency_percentile`, which is aggregate + * and excludes scale bytes. Null when the component carries no byte + * provenance (e.g. an unavailable component). + */ + payload_data_rate_gbps_at_latency_percentile: CollectiveXPercentiles | null; + /** + * Aggregate total logical payload bytes for this component at this point + * (the numerator behind `payload_data_rate_*`). Carried raw so a consumer can + * fit latency vs bytes across the ladder (bandwidth-vs-overhead decomposition) + * without reconstructing bytes from a rate. Null when no byte provenance. + */ + payload_bytes: number | null; +} + +export interface CollectiveXPoint { + tokens_per_rank: number; + global_tokens: number; + components: Record; + roundtrip_token_rate_at_latency_percentile: CollectiveXPercentiles; +} + +export interface CollectiveXTopology { + ep_size: number; + nodes: number; + gpus_per_node: number; + scale_up_domain: number; + scale_up_transport: string; + scale_out_transport: string | null; + topology_class: string; +} + +export interface CollectiveXSeries { + series_id: string; + phase: CollectiveXPhase; + mode: CollectiveXMode; + precision: CollectiveXPrecision; + backend: string; + system: CollectiveXTopology & { + sku: string; + vendor: 'nvidia' | 'amd'; + }; + points: CollectiveXPoint[]; +} + +export interface CollectiveXCoveragePoint { + tokens_per_rank: number; + global_tokens: number; + terminal_status: CollectiveXTerminalStatus; + reason: string | null; +} + +export interface CollectiveXCoverage { + case_id: string; + label: string; + disposition: 'runnable' | 'unsupported'; + sku: string; + backend: string; + phase: CollectiveXPhase; + mode: CollectiveXMode; + precision: CollectiveXPrecision; + topology: CollectiveXTopology; + points: CollectiveXCoveragePoint[]; + outcome: CollectiveXOutcome; + reason: string | null; + detail: string | null; +} + +export interface CollectiveXRun { + run_id: string; + run_attempt: number; + generated_at: string; + conclusion: string | null; + source_sha: string; + requested_cases: number; + terminal_cases: number; + measured_cases: number; + unsupported_cases: number; + failed_cases: number; + requested_points: number; + terminal_points: number; + measured_points: number; + covered_skus: string[]; +} + +export interface CollectiveXDataset { + version: number; + run: CollectiveXRun; + coverage: CollectiveXCoverage[]; + series: CollectiveXSeries[]; +} + +export interface CollectiveXRunSummary { + run_id: string; + run_attempt: number; + generated_at: string; + conclusion: string | null; + covered_skus: string[]; + requested_cases: number; + measured_cases: number; + requested_points: number; + terminal_points: number; + terminal_counts: { measured: number; unsupported: number; failed: number }; +} diff --git a/packages/db/src/connection.ts b/packages/db/src/connection.ts index a1a37bb74..23721d10b 100644 --- a/packages/db/src/connection.ts +++ b/packages/db/src/connection.ts @@ -76,7 +76,7 @@ function wrapPostgres(sql: postgres.Sql): DbClient { // Survive Next.js HMR — without globalThis the module re-evaluates on each // hot reload, leaking the previous postgres.js TCP connection pool. -const g = globalThis as unknown as { __dbClient?: DbClient; __dbWriteClient?: DbClient }; +const g = globalThis as unknown as { __dbClients?: Map }; function makeDbClient(url: string): DbClient { return shouldUseNeon(url) @@ -84,20 +84,39 @@ function makeDbClient(url: string): DbClient { : wrapPostgres(postgres(url, postgresOptionsForUrl(url))); } +/** One memoized client per connection env var; throws when the var is unset. */ +function memoizedClient(envVar: string): DbClient { + g.__dbClients ??= new Map(); + const cached = g.__dbClients.get(envVar); + if (cached) return cached; + const url = process.env[envVar]; + if (!url) throw new Error(`${envVar} is not set`); + const client = makeDbClient(url); + g.__dbClients.set(envVar, client); + return client; +} + /** Read-only SQL client for API routes. Requires DATABASE_READONLY_URL. */ export function getDb(): DbClient { - if (g.__dbClient) return g.__dbClient; - const url = process.env.DATABASE_READONLY_URL; - if (!url) throw new Error('DATABASE_READONLY_URL is not set'); - g.__dbClient = makeDbClient(url); - return g.__dbClient; + return memoizedClient('DATABASE_READONLY_URL'); } /** Write-capable SQL client for API routes that need to insert (e.g. user feedback). */ export function getWriteDb(): DbClient { - if (g.__dbWriteClient) return g.__dbWriteClient; - const url = process.env.DATABASE_WRITE_URL; - if (!url) throw new Error('DATABASE_WRITE_URL is not set'); - g.__dbWriteClient = makeDbClient(url); - return g.__dbWriteClient; + return memoizedClient('DATABASE_WRITE_URL'); +} + +/** + * Read-only SQL client for the CollectiveX database — a separate Neon + * instance from the main benchmark DB, holding raw sweep-run documents. + * Must point at the same primary as the write URL (not a lagging replica): + * the lazy-ingest routes read their own writes within a single request. + */ +export function getCollectiveXDb(): DbClient { + return memoizedClient('DATABASE_COLLECTIVEX_READONLY_URL'); +} + +/** Write-capable SQL client for the CollectiveX database (lazy ingest + run deletion). */ +export function getCollectiveXWriteDb(): DbClient { + return memoizedClient('DATABASE_COLLECTIVEX_WRITE_URL'); } diff --git a/packages/db/src/etl/compute-chart-series.test.ts b/packages/db/src/etl/compute-chart-series.test.ts index 3f088cd6a..749241713 100644 --- a/packages/db/src/etl/compute-chart-series.test.ts +++ b/packages/db/src/etl/compute-chart-series.test.ts @@ -269,23 +269,23 @@ describe('computeChartSeries', () => { metrics: { 'vllm:prompt_tokens': { series: [ - buildDynamoSeries('10.30.1.56:7500', 'prefill', 'prefill-a', 100), - buildDynamoSeries('10.30.1.36:7508', 'prefill', 'prefill-b', 200), - buildDynamoSeries('10.30.1.206:7516', 'backend', 'decode-a', 300), + buildDynamoSeries('prefill-a.internal.test:7500', 'prefill', 'prefill-a', 100), + buildDynamoSeries('prefill-b.internal.test:7508', 'prefill', 'prefill-b', 200), + buildDynamoSeries('decode-a.internal.test:7516', 'backend', 'decode-a', 300), ], }, 'vllm:generation_tokens': { series: [ - buildDynamoSeries('10.30.1.56:7500', 'prefill', 'prefill-a', 1), - buildDynamoSeries('10.30.1.36:7508', 'prefill', 'prefill-b', 2), - buildDynamoSeries('10.30.1.206:7516', 'backend', 'decode-a', 400), + buildDynamoSeries('prefill-a.internal.test:7500', 'prefill', 'prefill-a', 1), + buildDynamoSeries('prefill-b.internal.test:7508', 'prefill', 'prefill-b', 2), + buildDynamoSeries('decode-a.internal.test:7516', 'backend', 'decode-a', 400), ], }, 'vllm:num_requests_running': { series: [ - buildDynamoSeries('10.30.1.56:7500', 'prefill', 'prefill-a', 3, 'avg'), - buildDynamoSeries('10.30.1.36:7508', 'prefill', 'prefill-b', 4, 'avg'), - buildDynamoSeries('10.30.1.206:7516', 'backend', 'decode-a', 5, 'avg'), + buildDynamoSeries('prefill-a.internal.test:7500', 'prefill', 'prefill-a', 3, 'avg'), + buildDynamoSeries('prefill-b.internal.test:7508', 'prefill', 'prefill-b', 4, 'avg'), + buildDynamoSeries('decode-a.internal.test:7516', 'backend', 'decode-a', 5, 'avg'), ], }, }, @@ -299,8 +299,8 @@ describe('computeChartSeries', () => { expect(result?.metricSources).toHaveLength(3); expect(result?.metricSources.map(({ source: s }) => [s.role, s.workerId, s.engine])).toEqual([ - ['prefill', 'prefill-b', '0'], ['prefill', 'prefill-a', '0'], + ['prefill', 'prefill-b', '0'], ['decode', 'decode-a', '0'], ]); const prefillA = result?.metricSources.find(({ source: s }) => s.workerId === 'prefill-a'); @@ -322,7 +322,7 @@ describe('computeChartSeries', () => { 'vllm:prompt_tokens': { series: [ { - endpoint_url: '10.30.1.56:7500', + endpoint_url: 'prefill-a.internal.test:7500', labels: { dynamo_component: 'prefill', worker_id: 'prefill-a', engine: '0' }, timeslices: [{ start_ns: 0, end_ns: 1e9, rate: 100 }], }, diff --git a/packages/db/src/etl/db-utils.ts b/packages/db/src/etl/db-utils.ts index adc6c1cd5..6ff1678da 100644 --- a/packages/db/src/etl/db-utils.ts +++ b/packages/db/src/etl/db-utils.ts @@ -9,16 +9,18 @@ export type Sql = ReturnType; /** * Create a postgres client for admin scripts. * Reads DATABASE_WRITE_URL by default, or DATABASE_READONLY_URL with `readonly: true`. + * Pass `envVar` to target another database (e.g. DATABASE_COLLECTIVEX_WRITE_URL). * Pass `noSsl: true` to disable TLS for local Postgres. */ export function createAdminSql( opts: Omit>, 'ssl'> & { readonly?: boolean; noSsl?: boolean; + envVar?: string; } = {}, ): Sql { - const { readonly, noSsl, ...pgOpts } = opts; - const envVar = readonly ? 'DATABASE_READONLY_URL' : 'DATABASE_WRITE_URL'; + const { readonly, noSsl, envVar: envVarOverride, ...pgOpts } = opts; + const envVar = envVarOverride ?? (readonly ? 'DATABASE_READONLY_URL' : 'DATABASE_WRITE_URL'); const url = process.env[envVar]; if (!url) { console.error(`${envVar} is required`); diff --git a/packages/db/src/ingest-collectivex.ts b/packages/db/src/ingest-collectivex.ts new file mode 100644 index 000000000..3cffed1e5 --- /dev/null +++ b/packages/db/src/ingest-collectivex.ts @@ -0,0 +1,253 @@ +/** + * Ingest a CollectiveX sweep run's artifacts into the CollectiveX database. + * + * Stores the RAW matrix + case-attempt documents (the sweep JSON contract is + * expected to change; the shared reader in src/collectivex/reader.ts is the + * single transform point and runs at API-read time). The reader IS executed + * once here — to validate the bundle assembles and to precompute the + * `summary` column that backs the dashboard's run picker. + * + * Sweep runs are accepted from ANY branch of the source repo (they are + * launched via `gh api` on feature branches); only the workflow identity is + * checked, never head_branch. + * + * The dashboard ingests runs lazily on read; this CLI exists to pre-warm runs + * before their GitHub artifacts expire, backfill older runs into the picker, + * and force-refresh a run. Re-ingesting a run the dashboard deleted clears + * its tombstone (deliberate operator override). + * + * Two modes: + * --download [repo] Download artifacts from GitHub then ingest + * (no flag) Read pre-downloaded artifacts from INGEST_ARTIFACTS_PATH + * + * Usage: + * bun run admin:db:ingest:collectivex https://github.com/SemiAnalysisAI/InferenceX/actions/runs/123 + * bun run admin:db:ingest:collectivex 123 + * + * Environment variables: + * DATABASE_COLLECTIVEX_WRITE_URL — Postgres connection string (direct, non-pooled) + * GITHUB_TOKEN — GitHub PAT for run metadata + artifact download + * INGEST_RUN_ID — (env mode) Workflow run ID + * INGEST_ARTIFACTS_PATH — (env mode) Local path to pre-downloaded artifacts + * INGEST_REPO — (env mode) Source repo slug (owner/name) + */ + +import fs from 'fs'; +import os from 'os'; +import path from 'path'; + +import { hasNoSslFlag } from './cli-utils'; +import { createAdminSql } from './etl/db-utils'; +import { + downloadArtifact, + fetchRunMeta, + listRunArtifacts, + type RunMeta, +} from './lib/github-artifacts'; +import { matrixArtifactName, selectShardArtifactNames } from './collectivex/artifact-selection'; +import { + buildDatasetFromNeutral, + buildRunSummary, + isMatrixDoc, + matrixVersion, + type CollectiveXNeutralRunMeta, +} from './collectivex/reader'; + +const DEFAULT_REPO = 'SemiAnalysisAI/InferenceX'; +const SWEEP_WORKFLOW_PATH = '.github/workflows/collectivex-sweep.yml'; +const DOCS_INSERT_CHUNK = 200; + +// ── Argument / env parsing ────────────────────────────────────────────────── + +const isDownloadMode = process.argv[2] === '--download'; + +let artifactsDir: string; +let runIdStr: string; +let REPO: string; +let tempDir: string | null = null; + +if (isDownloadMode) { + // Positional args only: drop the '--' injected by pnpm arg passthrough and + // option flags like --no-ssl (read from argv by their own helpers). + const args = process.argv.slice(3).filter((a) => !a.startsWith('--')); + const input = args[0]; + if (!input) { + console.error('Usage: bun run admin:db:ingest:collectivex [repo]'); + process.exit(1); + } + const match = input.match(/\/runs\/(?\d+)/u); + const parsedId = match ? match.groups!.runId : /^\d+$/u.test(input) ? input : null; + if (!parsedId) { + console.error(`Could not parse run ID from: ${input}`); + process.exit(1); + } + runIdStr = parsedId; + REPO = args[1] ?? DEFAULT_REPO; + tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'cx-ingest-')); + artifactsDir = tempDir; +} else { + const runId = process.env.INGEST_RUN_ID; + const artifactsPath = process.env.INGEST_ARTIFACTS_PATH; + if (!runId || !artifactsPath) { + console.error('INGEST_RUN_ID and INGEST_ARTIFACTS_PATH are required without --download'); + process.exit(1); + } + runIdStr = runId; + REPO = process.env.INGEST_REPO ?? DEFAULT_REPO; + artifactsDir = artifactsPath; +} + +// Both reach shell-interpolated `gh api` calls — reject metachars. --download +// parses its run id out of a URL or a digits-only argument, but the env-var path +// takes INGEST_RUN_ID verbatim, so validate here where both modes converge. +if (!/^[\w.-]+\/[\w.-]+$/u.test(REPO)) { + console.error(`Invalid repo slug: ${REPO}`); + process.exit(1); +} +if (!/^\d+$/u.test(runIdStr)) { + console.error(`Invalid run id: ${runIdStr}`); + process.exit(1); +} + +// ── Artifact reading ──────────────────────────────────────────────────────── + +/** Parse every `*.json` file in an artifact directory. */ +function readDocsDir(dir: string): unknown[] { + const docs: unknown[] = []; + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) { + docs.push(...readDocsDir(full)); + continue; + } + if (!entry.name.endsWith('.json')) continue; + docs.push(JSON.parse(fs.readFileSync(full, 'utf8'))); + } + return docs; +} + +function runGeneratedAt(run: RunMeta): string { + return run.updated_at || run.run_started_at || run.created_at || ''; +} + +// ── Main ──────────────────────────────────────────────────────────────────── + +async function main(): Promise { + console.log(`=== db:ingest:collectivex — run ${runIdStr} (${REPO}) ===`); + + const run = fetchRunMeta(REPO, runIdStr); + // Identity check only — never the branch: sweeps run on feature branches. + if (run.path !== SWEEP_WORKFLOW_PATH) { + throw new Error(`run ${runIdStr} is not a CollectiveX sweep (workflow: ${run.path})`); + } + const generatedAt = runGeneratedAt(run); + if (!generatedAt) throw new Error(`run ${runIdStr} is missing a timestamp`); + console.log( + ` workflow: ${run.name} branch: ${run.head_branch ?? '?'} attempt: ${run.run_attempt} conclusion: ${run.conclusion ?? 'in-progress'}`, + ); + + const matrixDirName = matrixArtifactName(runIdStr); + + if (isDownloadMode) { + const artifacts = listRunArtifacts(REPO, runIdStr); + // Keep the newest per name — retried uploads can duplicate a name. + const byName = new Map(); + for (const artifact of artifacts) { + const existing = byName.get(artifact.name); + if (!existing || artifact.created_at > existing.created_at) { + byName.set(artifact.name, artifact); + } + } + const wanted = [ + matrixDirName, + ...selectShardArtifactNames([...byName.keys()], runIdStr, run.run_attempt), + ]; + for (const name of wanted) { + const artifact = byName.get(name); + if (!artifact) continue; + console.log(` downloading ${name}`); + downloadArtifact(artifact, artifactsDir); + } + } + + const availableDirs = fs + .readdirSync(artifactsDir, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => entry.name); + + if (!availableDirs.includes(matrixDirName)) { + throw new Error(`run ${runIdStr} has no ${matrixDirName} artifact`); + } + const matrixDocs = readDocsDir(path.join(artifactsDir, matrixDirName)).filter((doc) => + isMatrixDoc(doc), + ); + if (matrixDocs.length !== 1) { + throw new Error(`expected exactly one matrix document, found ${matrixDocs.length}`); + } + const matrix = matrixDocs[0]; + const version = matrixVersion(matrix); + if (version === null) throw new Error('matrix document has no valid version tag'); + + const shardDirs = selectShardArtifactNames(availableDirs, runIdStr, run.run_attempt); + console.log(` matrix version: ${version} shard artifacts: ${shardDirs.length}`); + const docs = shardDirs.flatMap((dir) => readDocsDir(path.join(artifactsDir, dir))); + + // Assemble once: validates the bundle and precomputes the picker summary. + const meta: CollectiveXNeutralRunMeta = { + run_id: runIdStr, + run_attempt: run.run_attempt, + generated_at: generatedAt, + conclusion: run.conclusion, + source_sha: run.head_sha, + }; + const dataset = buildDatasetFromNeutral(matrix, docs, meta); + const summary = buildRunSummary(dataset); + console.log( + ` assembled: ${summary.requested_cases} cases (${summary.measured_cases} measured), ${summary.requested_points} points`, + ); + + const sql = createAdminSql({ + envVar: 'DATABASE_COLLECTIVEX_WRITE_URL', + noSsl: hasNoSslFlag(), + max: 1, + onnotice: () => {}, + }); + try { + await sql.begin(async (tx) => { + // Re-ingest = refresh: replace the run and its documents wholesale. + await tx`DELETE FROM cx_runs WHERE run_id = ${runIdStr}`; + // The ::jsonb casts type the parameters as jsonb, so postgres.js + // serializes the raw objects itself — pre-stringifying here would + // double-encode them into jsonb strings. + await tx` + INSERT INTO cx_runs + (run_id, run_attempt, version, generated_at, source_sha, source_branch, conclusion, matrix, summary) + VALUES + (${runIdStr}, ${run.run_attempt}, ${version}, ${generatedAt}, ${run.head_sha}, + ${run.head_branch}, ${run.conclusion}, ${matrix as never}::jsonb, + ${summary as never}::jsonb) + `; + for (let i = 0; i < docs.length; i += DOCS_INSERT_CHUNK) { + const chunk = docs.slice(i, i + DOCS_INSERT_CHUNK).map((doc) => JSON.stringify(doc)); + await tx` + INSERT INTO cx_run_docs (run_id, run_attempt, doc) + SELECT ${runIdStr}, ${run.run_attempt}, unnest(${tx.array(chunk)}::jsonb[]) + `; + } + }); + console.log(` stored run ${runIdStr} (version ${version}, ${docs.length} docs)`); + } finally { + await sql.end(); + } + + console.log('=== db:ingest:collectivex complete ==='); +} + +main() + .catch((error) => { + console.error('db:ingest:collectivex failed:', error); + process.exitCode = 1; + }) + .finally(() => { + if (tempDir) fs.rmSync(tempDir, { recursive: true, force: true }); + }); diff --git a/packages/db/src/lib/github-artifacts.ts b/packages/db/src/lib/github-artifacts.ts index 39f143bd9..67827855e 100644 --- a/packages/db/src/lib/github-artifacts.ts +++ b/packages/db/src/lib/github-artifacts.ts @@ -86,3 +86,39 @@ export function fetchRunAttempt(repo: string, runId: string): number { }).trim(); return parseInt(attemptStr || '1', 10); } + +export interface RunMeta { + id: number; + name: string; + path: string; + run_attempt: number; + head_sha: string; + head_branch: string | null; + conclusion: string | null; + status: string | null; + updated_at?: string | null; + run_started_at?: string | null; + created_at?: string | null; +} + +/** + * Fetch a workflow run's metadata via `gh api`. + * + * Both arguments land in a shell-interpolated command, so they are validated + * here rather than trusted from the caller: this helper is exported and its + * callers' own checks are not guaranteed. A run id is always digits and a repo + * slug is `owner/name`, so anything else is rejected outright. + */ +export function fetchRunMeta(repo: string, runId: string): RunMeta { + if (!/^[\w.-]+\/[\w.-]+$/u.test(repo)) { + throw new Error(`Invalid repo slug: ${repo}`); + } + if (!/^\d+$/u.test(runId)) { + throw new Error(`Invalid run id: ${runId}`); + } + const json = execSync(`gh api "repos/${repo}/actions/runs/${runId}"`, { + encoding: 'utf8', + maxBuffer: 10 * 1024 * 1024, + }); + return JSON.parse(json) as RunMeta; +} diff --git a/packages/db/src/lib/migration-runner.test.ts b/packages/db/src/lib/migration-runner.test.ts new file mode 100644 index 000000000..4affed65d --- /dev/null +++ b/packages/db/src/lib/migration-runner.test.ts @@ -0,0 +1,88 @@ +import fs from 'fs'; +import os from 'os'; +import path from 'path'; + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { runMigrations } from './migration-runner'; + +interface Executed { + text: string; + params?: unknown[]; +} + +/** + * Minimal stand-in for the postgres.js client surface runMigrations uses: + * tagged-template select on schema_migrations + begin(tx.unsafe). + */ +function fakeSql(applied: string[]) { + const executed: Executed[] = []; + const sql = (strings: TemplateStringsArray) => { + const text = strings.join(''); + executed.push({ text }); + if (text.includes('select filename')) { + return Promise.resolve(applied.map((filename) => ({ filename }))); + } + return Promise.resolve([]); + }; + sql.begin = async ( + fn: (tx: { unsafe: (text: string, params?: unknown[]) => Promise }) => Promise, + ) => { + await fn({ + unsafe: (text: string, params?: unknown[]) => { + executed.push({ text, params }); + return Promise.resolve(); + }, + }); + }; + return { sql: sql as never, executed }; +} + +let dir: string; + +beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'migration-runner-')); +}); + +afterEach(() => { + fs.rmSync(dir, { recursive: true, force: true }); +}); + +describe('runMigrations', () => { + it('applies pending .sql files in filename order and records them', async () => { + fs.writeFileSync(path.join(dir, '002_second.sql'), 'create table two ();'); + fs.writeFileSync(path.join(dir, '001_first.sql'), 'create table one ();'); + fs.writeFileSync(path.join(dir, 'notes.txt'), 'not a migration'); + const { sql, executed } = fakeSql([]); + + const ran = await runMigrations(sql, dir); + + expect(ran).toBe(2); + const applied = executed.filter((e) => e.text.startsWith('create table')); + expect(applied.map((e) => e.text)).toEqual(['create table one ();', 'create table two ();']); + const recorded = executed.filter((e) => e.text.includes('insert into schema_migrations')); + expect(recorded.map((e) => e.params)).toEqual([['001_first.sql'], ['002_second.sql']]); + }); + + it('skips migrations already recorded in schema_migrations', async () => { + fs.writeFileSync(path.join(dir, '001_first.sql'), 'create table one ();'); + fs.writeFileSync(path.join(dir, '002_second.sql'), 'create table two ();'); + const { sql, executed } = fakeSql(['001_first.sql']); + + const ran = await runMigrations(sql, dir); + + expect(ran).toBe(1); + const applied = executed.filter((e) => e.text.startsWith('create table')); + expect(applied.map((e) => e.text)).toEqual(['create table two ();']); + }); + + it('returns 0 when everything is already applied', async () => { + fs.writeFileSync(path.join(dir, '001_first.sql'), 'create table one ();'); + const { sql, executed } = fakeSql(['001_first.sql']); + + const ran = await runMigrations(sql, dir); + + expect(ran).toBe(0); + expect(executed.some((e) => e.text.startsWith('create table'))).toBe(false); + }); +}); diff --git a/packages/db/src/lib/migration-runner.ts b/packages/db/src/lib/migration-runner.ts new file mode 100644 index 000000000..3ef4e2135 --- /dev/null +++ b/packages/db/src/lib/migration-runner.ts @@ -0,0 +1,56 @@ +/** + * Shared SQL-file migration loop used by `migrate.ts` (main DB) and + * `migrate-collectivex.ts` (CollectiveX DB). Applies pending `*.sql` files + * from a directory in filename order, tracking them in a `schema_migrations` + * table on the target database. + */ + +import fs from 'fs'; +import path from 'path'; + +import type { Sql } from '../etl/db-utils'; + +export async function runMigrations(sql: Sql, migrationsDir: string): Promise { + // Create migrations tracking table if it doesn't exist + await sql` + create table if not exists schema_migrations ( + filename text primary key, + applied_at timestamptz not null default now() + ) + `; + + const migrations = await sql<{ filename: string }[]>`select filename from schema_migrations`; + const applied = new Set(migrations.map((r) => r.filename)); + + const files = fs + .readdirSync(migrationsDir) + .filter((f) => f.endsWith('.sql')) + .toSorted(); + + let ran = 0; + for (const file of files) { + if (applied.has(file)) { + console.log(` skip ${file}`); + continue; + } + + console.log(` apply ${file} ...`); + const sql_text = fs.readFileSync(path.join(migrationsDir, file), 'utf8'); + + await sql.begin(async (tx) => { + await tx.unsafe(sql_text); + await tx.unsafe('insert into schema_migrations (filename) values ($1)', [file]); + }); + + console.log(` done ${file}`); + ran++; + } + + if (ran === 0) { + console.log(' all migrations already applied'); + } else { + console.log(`\n applied ${ran} migration(s)`); + } + + return ran; +} diff --git a/packages/db/src/migrate-collectivex.ts b/packages/db/src/migrate-collectivex.ts new file mode 100644 index 000000000..066dbb3d5 --- /dev/null +++ b/packages/db/src/migrate-collectivex.ts @@ -0,0 +1,50 @@ +/** + * Run CollectiveX database migrations. Targets the separate CollectiveX Neon + * instance via DATABASE_COLLECTIVEX_WRITE_URL (direct, non-pooled connection — + * migrations must not go through PgBouncer's transaction pooling mode). + * + * Usage: + * bun run admin:db:migrate:collectivex + */ + +import path from 'path'; + +import { confirm, hasNoSslFlag, hasYesFlag } from './cli-utils'; +import { createAdminSql } from './etl/db-utils'; +import { runMigrations } from './lib/migration-runner'; + +const MIGRATIONS_DIR = path.join(import.meta.dirname, '..', 'migrations-collectivex'); + +const sql = createAdminSql({ + envVar: 'DATABASE_COLLECTIVEX_WRITE_URL', + noSsl: hasNoSslFlag(), + max: 1, + onnotice: () => {}, // suppress "relation already exists" notices +}); + +async function migrate(): Promise { + console.log('=== db:migrate:collectivex ==='); + console.log( + 'This will apply any pending SQL migrations from migrations-collectivex/ to the\n' + + 'CollectiveX database. Already-applied migrations are skipped.\n', + ); + + if (!hasYesFlag()) { + const ok = await confirm('Continue? (y/N) '); + if (!ok) { + console.log('Aborted.'); + return; + } + } + + await runMigrations(sql, MIGRATIONS_DIR); + + console.log('\n=== db:migrate:collectivex complete ==='); +} + +migrate() + .catch((error) => { + console.error('db:migrate:collectivex failed:', error); + process.exitCode = 1; + }) + .finally(() => sql.end()); diff --git a/packages/db/src/migrate.ts b/packages/db/src/migrate.ts index 152329aa0..1adad9b95 100644 --- a/packages/db/src/migrate.ts +++ b/packages/db/src/migrate.ts @@ -8,11 +8,11 @@ * bun run admin:db:migrate */ -import fs from 'fs'; import path from 'path'; import { confirm, hasNoSslFlag, hasYesFlag } from './cli-utils'; import { createAdminSql } from './etl/db-utils'; +import { runMigrations } from './lib/migration-runner'; const MIGRATIONS_DIR = path.join(import.meta.dirname, '..', 'migrations'); @@ -37,46 +37,7 @@ async function migrate(): Promise { } } - // Create migrations tracking table if it doesn't exist - await sql` - create table if not exists schema_migrations ( - filename text primary key, - applied_at timestamptz not null default now() - ) - `; - - const migrations = await sql<{ filename: string }[]>`select filename from schema_migrations`; - const applied = new Set(migrations.map((r) => r.filename)); - - const files = fs - .readdirSync(MIGRATIONS_DIR) - .filter((f) => f.endsWith('.sql')) - .toSorted(); - - let ran = 0; - for (const file of files) { - if (applied.has(file)) { - console.log(` skip ${file}`); - continue; - } - - console.log(` apply ${file} ...`); - const sql_text = fs.readFileSync(path.join(MIGRATIONS_DIR, file), 'utf8'); - - await sql.begin(async (tx) => { - await tx.unsafe(sql_text); - await tx.unsafe('insert into schema_migrations (filename) values ($1)', [file]); - }); - - console.log(` done ${file}`); - ran++; - } - - if (ran === 0) { - console.log(' all migrations already applied'); - } else { - console.log(`\n applied ${ran} migration(s)`); - } + await runMigrations(sql, MIGRATIONS_DIR); console.log('\n=== db:migrate complete ==='); console.log(' Invalidate API cache: bun run admin:cache:invalidate'); diff --git a/packages/db/src/queries/collectivex.test.ts b/packages/db/src/queries/collectivex.test.ts new file mode 100644 index 000000000..e0dfd1144 --- /dev/null +++ b/packages/db/src/queries/collectivex.test.ts @@ -0,0 +1,188 @@ +import { describe, expect, it } from 'vitest'; + +import { makeRawMatrix, makeRawShard, makeRunMeta } from '../collectivex/test-fixture'; +import type { DbClient } from '../connection.js'; +import { + collectiveXDatasetFromRow, + deleteCollectiveXRun, + getCollectiveXRunStates, + insertCollectiveXRun, + listCollectiveXRuns, + refreshCollectiveXRunAttempt, + type CollectiveXRunInsert, +} from './collectivex'; + +interface Captured { + text: string; + values: unknown[]; +} + +/** Tagged-template stub: records each query, replays queued row sets. */ +function fakeSql(rowsQueue: Record[][]) { + const calls: Captured[] = []; + const sql = ((strings: TemplateStringsArray, ...values: unknown[]) => { + calls.push({ text: strings.join('?'), values }); + return Promise.resolve(rowsQueue.shift() ?? []); + }) as DbClient; + return { sql, calls }; +} + +const runInsert: CollectiveXRunInsert = { + run_id: '160', + run_attempt: 2, + version: 1, + generated_at: '2026-07-08T12:20:00Z', + source_sha: 'a'.repeat(40), + source_branch: 'collectivex', + conclusion: 'success', + matrix: { version: 1, requested_cases: [], include: [] }, + summary: { + run_id: '160', + run_attempt: 2, + generated_at: '2026-07-08T12:20:00Z', + conclusion: 'success', + covered_skus: [], + requested_cases: 0, + measured_cases: 0, + requested_points: 0, + terminal_points: 0, + terminal_counts: { measured: 0, unsupported: 0, failed: 0 }, + }, +}; + +describe('getCollectiveXRunStates', () => { + it('maps deleted flags to tombstone states with version and attempt', async () => { + const { sql } = fakeSql([ + [ + { run_id: '160', version: 1, run_attempt: 2, deleted: false }, + { run_id: '161', version: 2, run_attempt: 1, deleted: true }, + ], + ]); + const states = await getCollectiveXRunStates(sql, ['160', '161']); + expect(states).toEqual({ + '160': { state: 'live', version: 1, run_attempt: 2 }, + '161': { state: 'deleted', version: 2, run_attempt: 1 }, + }); + }); + + it('skips the query entirely for an empty id list', async () => { + const { sql, calls } = fakeSql([]); + expect(await getCollectiveXRunStates(sql, [])).toEqual({}); + expect(calls).toHaveLength(0); + }); +}); + +describe('read queries', () => { + it('excludes tombstoned rows and orders newest-created first', async () => { + const { sql, calls } = fakeSql([[]]); + await listCollectiveXRuns(sql, 1); + expect(calls[0].text).toContain('deleted_at IS NULL'); + expect(calls[0].text).toContain('ORDER BY run_id DESC'); + expect(calls[0].text).not.toContain('LIMIT'); + }); +}); + +describe('insertCollectiveXRun', () => { + it('binds raw objects (never pre-stringified JSON) and reports insertion', async () => { + const { sql, calls } = fakeSql([[{ runs_inserted: 1 }]]); + const docs = [{ record_type: 'case-attempt' }]; + + await expect(insertCollectiveXRun(sql, runInsert, docs)).resolves.toBe(true); + + expect(calls[0].text).toContain('ON CONFLICT (run_id) DO NOTHING'); + // Raw objects: a pre-stringified value would double-encode under + // postgres.js; a bare array would become a PG array literal under neon. + expect(calls[0].values).toContainEqual(runInsert.matrix); + expect(calls[0].values).toContainEqual({ docs }); + expect( + calls[0].values.every((value) => typeof value !== 'string' || !value.startsWith('{')), + ).toBe(true); + }); + + it('reports a conflict no-op as not inserted', async () => { + const { sql } = fakeSql([[{ runs_inserted: 0 }]]); + await expect(insertCollectiveXRun(sql, runInsert, [])).resolves.toBe(false); + }); +}); + +describe('refreshCollectiveXRunAttempt', () => { + it('guards on a strictly newer attempt and reports replacement', async () => { + const { sql, calls } = fakeSql([[{ runs_updated: 1 }]]); + await expect(refreshCollectiveXRunAttempt(sql, runInsert, [])).resolves.toBe(true); + expect(calls[0].text).toContain('run_attempt < '); + expect(calls[0].text).toContain('deleted_at IS NULL'); + expect(calls[0].text).toContain('FOR UPDATE'); + }); + + it('reports a guarded no-op (older or equal attempt, or tombstoned) as false', async () => { + const { sql } = fakeSql([[{ runs_updated: 0 }]]); + await expect(refreshCollectiveXRunAttempt(sql, runInsert, [])).resolves.toBe(false); + }); +}); + +describe('deleteCollectiveXRun', () => { + it('tombstones the run and frees its documents in one atomic statement', async () => { + const { sql, calls } = fakeSql([[{ runs_deleted: 1 }]]); + await expect(deleteCollectiveXRun(sql, '160')).resolves.toBe(true); + expect(calls).toHaveLength(1); + expect(calls[0].text).toContain('SET deleted_at = now()'); + expect(calls[0].text).toContain('deleted_at IS NULL'); + expect(calls[0].text).toContain('DELETE FROM cx_run_docs'); + }); + + it('reports absent or already-tombstoned runs as false', async () => { + const { sql } = fakeSql([[{ runs_deleted: 0 }]]); + await expect(deleteCollectiveXRun(sql, '160')).resolves.toBe(false); + }); + + it('cannot reach anything but the one run it is given', async () => { + // The delete route's Bearer token is held in browser localStorage, so the + // blast radius of a stolen token is whatever this statement can touch. It + // must stay: one run, in the two CollectiveX tables, and recoverable by + // re-ingesting the run from its GitHub artifacts. + const { sql, calls } = fakeSql([[{ runs_deleted: 1 }]]); + await deleteCollectiveXRun(sql, '160'); + // The run id is the only value bound into the statement — no other row is nameable. + expect(calls[0].values).toEqual(['160']); + // Documents go only via the tombstoned CTE, never a free-standing predicate. + expect(calls[0].text).toContain( + 'DELETE FROM cx_run_docs WHERE run_id IN (SELECT run_id FROM tombstoned)', + ); + // No table outside the CollectiveX pair is referenced, and nothing is dropped. + expect(new Set([...calls[0].text.matchAll(/\bcx_[a-z_]+/gu)].map((match) => match[0]))).toEqual( + new Set(['cx_runs', 'cx_run_docs']), + ); + expect(calls[0].text).not.toMatch(/\b(?:DROP|TRUNCATE|ALTER)\b/iu); + }); +}); + +describe('collectiveXDatasetFromRow', () => { + it('assembles a stored row through the shared reader', () => { + const shard = makeRawShard(); + const identity = ( + shard as { identity: { case_id: string; case_factors: { sku: string; case: unknown } } } + ).identity; + const meta = makeRunMeta(); + const dataset = collectiveXDatasetFromRow({ + run_id: meta.run_id, + run_attempt: meta.run_attempt, + version: 1, + generated_at: meta.generated_at, + source_sha: meta.source_sha, + source_branch: 'collectivex', + conclusion: meta.conclusion, + matrix: makeRawMatrix([ + { + caseId: identity.case_id, + sku: identity.case_factors.sku, + disposition: 'runnable', + case: identity.case_factors.case as Record, + }, + ]), + docs: [shard], + }); + expect(dataset.run.run_id).toBe(meta.run_id); + expect(dataset.series).toHaveLength(1); + expect(dataset.run.measured_cases).toBe(1); + }); +}); diff --git a/packages/db/src/queries/collectivex.ts b/packages/db/src/queries/collectivex.ts new file mode 100644 index 000000000..effd7a979 --- /dev/null +++ b/packages/db/src/queries/collectivex.ts @@ -0,0 +1,261 @@ +import type { DbClient } from '../connection.js'; +import { buildDatasetFromNeutral } from '../collectivex/reader'; +import type { CollectiveXDataset, CollectiveXRunSummary } from '../collectivex/types'; + +/** One cx_runs row with its raw documents, ready for reader assembly. */ +export interface CollectiveXRunRow { + run_id: string; + run_attempt: number; + version: number; + generated_at: string; + source_sha: string; + source_branch: string | null; + conclusion: string | null; + matrix: unknown; + docs: unknown[]; +} + +/** Row metadata + raw docs as written by the lazy ingest. */ +export interface CollectiveXRunInsert { + run_id: string; + run_attempt: number; + version: number; + generated_at: string; + source_sha: string; + source_branch: string | null; + conclusion: string | null; + matrix: unknown; + summary: CollectiveXRunSummary; +} + +/** + * Known-run state. Runs are discovered lazily from GitHub on read, so a + * deleted run keeps a tombstoned cx_runs row — otherwise the next discovery + * pass would re-ingest it. + */ +type CollectiveXRunState = 'live' | 'deleted'; + +interface CollectiveXRunStateRow { + state: CollectiveXRunState; + version: number; + run_attempt: number; +} + +function toRunRow(row: Record): CollectiveXRunRow { + const { docs, ...rest } = row as unknown as Omit & { docs: unknown }; + return { ...rest, docs: Array.isArray(docs) ? docs : [] }; +} + +/** + * The most recent live run for a version. Ordered by run_id — GitHub run ids + * increase monotonically with creation, matching lazy discovery's + * newest-first walk (completion time would let a long-failing older run + * shadow a newer successful one). + * + * Row and documents come back in ONE query, with docs filtered to the row's + * CURRENT run_attempt: a reader can never observe one attempt's metadata with + * another attempt's documents, even while a refresh commits concurrently. + */ +export async function getLatestCollectiveXRun( + sql: DbClient, + version: number, +): Promise { + const rows = await sql` + SELECT r.run_id::text, r.run_attempt, r.version, + to_char(r.generated_at at time zone 'utc', 'YYYY-MM-DD"T"HH24:MI:SS"Z"') as generated_at, + r.source_sha, r.source_branch, r.conclusion, r.matrix, + COALESCE(d.docs, '[]'::jsonb) AS docs + FROM cx_runs r + LEFT JOIN LATERAL ( + SELECT jsonb_agg(doc ORDER BY id) AS docs + FROM cx_run_docs + WHERE run_id = r.run_id AND run_attempt = r.run_attempt + ) d ON true + WHERE r.version = ${version} AND r.deleted_at IS NULL + ORDER BY r.run_id DESC + LIMIT 1 + `; + return rows.length === 0 ? null : toRunRow(rows[0]); +} + +/** One specific live run by id, or null when absent, tombstoned, or on another version. */ +export async function getCollectiveXRun( + sql: DbClient, + version: number, + runId: string, +): Promise { + const rows = await sql` + SELECT r.run_id::text, r.run_attempt, r.version, + to_char(r.generated_at at time zone 'utc', 'YYYY-MM-DD"T"HH24:MI:SS"Z"') as generated_at, + r.source_sha, r.source_branch, r.conclusion, r.matrix, + COALESCE(d.docs, '[]'::jsonb) AS docs + FROM cx_runs r + LEFT JOIN LATERAL ( + SELECT jsonb_agg(doc ORDER BY id) AS docs + FROM cx_run_docs + WHERE run_id = r.run_id AND run_attempt = r.run_attempt + ) d ON true + WHERE r.version = ${version} AND r.run_id = ${runId} AND r.deleted_at IS NULL + `; + return rows.length === 0 ? null : toRunRow(rows[0]); +} + +/** + * Every live run summary for a version, newest-first, straight from the + * precomputed `summary` column — no document loading. + */ +export async function listCollectiveXRuns( + sql: DbClient, + version: number, +): Promise { + const rows = await sql` + SELECT summary + FROM cx_runs + WHERE version = ${version} AND deleted_at IS NULL + ORDER BY run_id DESC + `; + return rows.map((row) => row.summary as CollectiveXRunSummary); +} + +/** State of each known run id (live or tombstoned); absent ids are omitted. */ +export async function getCollectiveXRunStates( + sql: DbClient, + runIds: readonly string[], +): Promise> { + if (runIds.length === 0) return {}; + const rows = await sql` + SELECT run_id::text, version, run_attempt, (deleted_at IS NOT NULL) AS deleted + FROM cx_runs + WHERE run_id = ANY(${runIds as string[]}::bigint[]) + `; + return Object.fromEntries( + rows.map((row) => [ + row.run_id as string, + { + state: row.deleted ? 'deleted' : 'live', + version: row.version as number, + run_attempt: row.run_attempt as number, + }, + ]), + ); +} + +/** + * Atomically persist one run and its raw documents in a single statement + * (data-modifying CTEs), so concurrent lazy ingests can race safely: + * `ON CONFLICT DO NOTHING` turns the loser into a clean no-op, and a partial + * run can never become visible. Returns true when this call inserted the run. + * + * The `::jsonb`-cast parameters must be raw objects: both drivers + * JSON-serialize objects exactly once, while pre-stringified values get + * double-encoded by postgres.js. `docs` is wrapped in an object because the + * neon driver would serialize a bare JS array as a Postgres array literal. + */ +export async function insertCollectiveXRun( + sql: DbClient, + run: CollectiveXRunInsert, + docs: unknown[], +): Promise { + const rows = await sql` + WITH new_run AS ( + INSERT INTO cx_runs + (run_id, run_attempt, version, generated_at, source_sha, source_branch, conclusion, matrix, summary) + VALUES + (${run.run_id}, ${run.run_attempt}, ${run.version}, ${run.generated_at}, + ${run.source_sha}, ${run.source_branch}, ${run.conclusion}, + ${run.matrix as never}::jsonb, ${run.summary as never}::jsonb) + ON CONFLICT (run_id) DO NOTHING + RETURNING run_id + ), + new_docs AS ( + INSERT INTO cx_run_docs (run_id, run_attempt, doc) + SELECT new_run.run_id, ${run.run_attempt}, entries.value + FROM new_run, jsonb_array_elements((${{ docs } as never}::jsonb)->'docs') AS entries(value) + RETURNING id + ) + SELECT (SELECT count(*)::int FROM new_run) AS runs_inserted + `; + return (rows[0]?.runs_inserted as number) > 0; +} + +/** + * Replace a live run's contents when GitHub reports a NEWER attempt (a re-run + * of failed shards after the run was already ingested). Single statement with + * `FOR UPDATE` + an attempt guard: concurrent refreshers serialize on the row + * lock and the loser re-evaluates the guard to a no-op. Readers filter docs + * by the row's current run_attempt, so any superseded docs this statement's + * snapshot could not see (and therefore could not DELETE) stay invisible until + * the next refresh garbage-collects them. Tombstoned runs are never refreshed. + * Returns true when replaced. + */ +export async function refreshCollectiveXRunAttempt( + sql: DbClient, + run: CollectiveXRunInsert, + docs: unknown[], +): Promise { + const rows = await sql` + WITH target AS ( + SELECT run_id FROM cx_runs + WHERE run_id = ${run.run_id} AND deleted_at IS NULL AND run_attempt < ${run.run_attempt} + FOR UPDATE + ), + removed AS ( + DELETE FROM cx_run_docs WHERE run_id IN (SELECT run_id FROM target) + ), + updated AS ( + UPDATE cx_runs SET + run_attempt = ${run.run_attempt}, + generated_at = ${run.generated_at}, + source_sha = ${run.source_sha}, + source_branch = ${run.source_branch}, + conclusion = ${run.conclusion}, + matrix = ${run.matrix as never}::jsonb, + summary = ${run.summary as never}::jsonb, + ingested_at = now() + WHERE run_id IN (SELECT run_id FROM target) + RETURNING run_id + ), + new_docs AS ( + INSERT INTO cx_run_docs (run_id, run_attempt, doc) + SELECT updated.run_id, ${run.run_attempt}, entries.value + FROM updated, jsonb_array_elements((${{ docs } as never}::jsonb)->'docs') AS entries(value) + RETURNING id + ) + SELECT (SELECT count(*)::int FROM updated) AS runs_updated + `; + return (rows[0]?.runs_updated as number) > 0; +} + +/** + * Tombstone a run: mark it deleted (so lazy discovery never re-ingests it) + * and drop its documents to free space — one atomic statement, so a partial + * failure can never tombstone the run while leaving its documents orphaned + * behind an unretryable 404. Returns false when the run is absent or already + * tombstoned. Re-ingesting via the CLI intentionally clears the tombstone + * (operator override; its hard DELETE also cascades any leftover docs). + */ +export async function deleteCollectiveXRun(sql: DbClient, runId: string): Promise { + const rows = await sql` + WITH tombstoned AS ( + UPDATE cx_runs SET deleted_at = now() + WHERE run_id = ${runId} AND deleted_at IS NULL + RETURNING run_id + ), + removed AS ( + DELETE FROM cx_run_docs WHERE run_id IN (SELECT run_id FROM tombstoned) + ) + SELECT (SELECT count(*)::int FROM tombstoned) AS runs_deleted + `; + return (rows[0]?.runs_deleted as number) > 0; +} + +/** Assemble a stored run's raw documents into the dashboard dataset. */ +export function collectiveXDatasetFromRow(row: CollectiveXRunRow): CollectiveXDataset { + return buildDatasetFromNeutral(row.matrix, row.docs, { + run_id: row.run_id, + run_attempt: row.run_attempt, + generated_at: row.generated_at, + conclusion: row.conclusion, + source_sha: row.source_sha, + }); +}