From 38dfc85c10e1e50dcc53e9bd54947c371752be05 Mon Sep 17 00:00:00 2001 From: Ib Green Date: Fri, 7 Aug 2026 06:46:03 -0400 Subject: [PATCH 1/2] docs(experimental): demonstrate table-first luVS benchmarks (#2950) --- docs/api-reference/experimental/luvs.md | 15 + docs/whats-new.md | 3 + .../luvs-live-benchmark-docs.node.spec.ts | 262 ++++++ test/examples/luvs-live-benchmark.spec.ts | 88 ++ .../components/docs/luvs-benchmark-runtime.ts | 783 ++++++++++++++++++ .../src/components/docs/luvs-benchmark.tsx | 235 ++++++ 6 files changed, 1386 insertions(+) create mode 100644 test/examples/luvs-live-benchmark-docs.node.spec.ts create mode 100644 test/examples/luvs-live-benchmark.spec.ts create mode 100644 website/src/components/docs/luvs-benchmark-runtime.ts create mode 100644 website/src/components/docs/luvs-benchmark.tsx diff --git a/docs/api-reference/experimental/luvs.md b/docs/api-reference/experimental/luvs.md index 1f74232468..0f883f4f3d 100644 --- a/docs/api-reference/experimental/luvs.md +++ b/docs/api-reference/experimental/luvs.md @@ -1,4 +1,5 @@ import {ExperimentalDocsTabs} from '@site/src/components/docs/experimental-docs-tabs'; +import {LuvsBenchmark} from '@site/src/components/docs/luvs-benchmark'; # luVS: GPU Vector Similarity and Clustering @@ -214,6 +215,20 @@ kernels, or FAISS implementations are copied into this module. It is not affilia by NVIDIA or the RAPIDS project, and it neither implements a compatible cuVS API nor claims feature parity. +## Live CPU versus WebGPU benchmark + +Run the benchmark explicitly to compare the same deterministic vectors on your browser's CPU and +WebGPU adapter. Dataset size, dimensions, query count, K, selection density, IVF list count, and +probe count are configurable. The exact GPU paths are checked against an independent CPU oracle; +the approximate IVF-flat path reports recall@K against exact search. + + + +GPU query measurements include command encoding, submission, and an explicit completion fence. +Initial upload, IVF training/index construction, and correctness readback are reported separately. +Warmups precede repeated measured runs, and displayed query times are medians. Results depend on +the current browser, WebGPU adapter, data dimensions, filtering, thermal conditions, and workload. + ## Fixed-size GPU table embedding columns High-dimensional values such as 384-, 768-, or 1,536-component embeddings are not GPU vertex diff --git a/docs/whats-new.md b/docs/whats-new.md index 3016ff573b..c78b17b308 100644 --- a/docs/whats-new.md +++ b/docs/whats-new.md @@ -69,6 +69,7 @@ Target Release Date: Q3, 2026 - **Reusable command-graph contributors** - `GPUCommandGraphContributor` gives small algorithm libraries a structural `addToGraph()` contract, while public aligned-view binding and typed transient-view helpers let those libraries extend command graphs without a runtime registry or hidden submission. - **Optional GPU geospatial kernels** - The side-effect-free `@luma.gl/experimental/geospatial` subpath contributes cuSpatial-compatible sinusoidal projection, haversine distance, pairwise planar distances, four-state point-in-polygon classification, nearest-linestring results, grid indexing, and point spatial queries to caller-owned command graphs, including raw binary64 coordinate inputs. - **GPU projection patches** - The optional `@luma.gl/experimental/luproj` subpath compiles arbitrary CPU projection providers into adaptive local polynomial patches and projects chunk-preserving coordinate vectors through WebGPU command graphs without discarding raw binary64 source precision. +- **GPU vector similarity and clustering** - The optional [`@luma.gl/experimental/luvs`](/docs/api-reference/experimental/luvs) backend searches ordinary fixed-size-list GPU table columns with exact squared-Euclidean, cosine, and inner-product rankings; GPU-resident linked-selection masks; deterministic bounded top-K output; GPU k-means; and explicitly approximate IVF-flat search. Existing `@luma.gl/arrow` table adapters upload Arrow embedding columns, while source IDs, validity, batch boundaries, ownership, and rendering remain caller-controlled. - **GPU scan, compaction, and indirect drawing** - Typed graph views compose hierarchical `uint32` scan, stable ID compaction, and GPU-written `DrawCommandBuffer` instance counts. Scan and compaction accept fixed-width `GPUVector` imports as one logical sequence while preserving chunk topology. The [GPU Trace Viewer](/examples/experimental/gpu-trace-viewer) demonstrates the path over up to four million spans, while [GPU Frustum Culling](/examples/experimental/gpu-frustum-culling) applies it to indexed indirect rendering of a 3D instance field. - **GPU virtual-geometry selection** - [`GPUVirtualGeometrySelection`](/docs/api-reference/experimental/gpu-primitives/gpu-virtual-geometry-selection) traverses breadth-level cluster forests with conservative sphere-frustum tests and pixel-scale geometric error, then reuses stable visibility compaction to publish a deterministic cluster frontier and capacity-safe indirect instance count without CPU readback. - **Virtual Geometry Canyon** - The [WebGPU showcase](/examples/experimental/virtual-geometry-canyon) drives a 4×4, six-refinement terrain forest through GPU-only LOD selection and one indexed indirect draw. A shared grid, exact parent-triangle geomorphing, and skirts visualize more than 41 million potential leaf triangles without a per-frame traversal or readback on the CPU. @@ -96,6 +97,7 @@ Target Release Date: Q3, 2026 - **Composite GPU inputs** - `GPUInputSchema.attributeNames` maps one logical table column to several shader attributes, allowing a shared matrix buffer to feed portable vertex attributes or a WebGPU storage binding without repacking. Ordinary inputs retain the singular `attributeName`. - **Generic GPU tables** - Canonical `GPUData`, `GPUVector`, `GPURecordBatch`, and `GPUTable` runtime classes for reusable non-Arrow-specific GPU table ownership and batching. +- **Fixed-size-list GPU columns** - First-class `fixed-size-list` formats describe arbitrary fixed-width storage rows without inventing unsupported vertex formats; vectors retain logical table-row counts, flattened element counts, preserved batches, and caller-owned storage. - **Table-backed rendering** - `GPUTableModel` draws preserved table batches, and `GPUTableGeometry` exposes packed static GPU tables as renderable geometry. - **Vertex storage planning** - `GPUTableBufferPlanner` now checks vertex-stage storage buffer limits before choosing storage-backed table attributes, allowing core WebGPU devices to fall back to vertex attributes when needed. - **Execution helpers** - `TableTransform`, `GPUTableComputation`, generated-buffer batch planning, and `GPUTableBufferPlanner` now live beside the generic table runtime instead of the Arrow adapter module. @@ -110,6 +112,7 @@ Target Release Date: Q3, 2026 - **Arrow shader layouts** - `getArrowBufferLayout()` maps Arrow scalar and `FixedSizeList` columns to shader attribute formats from a shader-first layout, including direct `arrow.Vector` sources and Arrow table path mappings. - **Arrow GPU adapters** - Arrow factories, append helpers, and readback helpers bridge Apache Arrow inputs into `@luma.gl/tables` objects and preserve chunked UTF-8 GPU vector input for text workflows. +- **High-dimensional Arrow storage columns** - Existing Arrow table/vector adapters map wide `FixedSizeList` values directly into row-aligned fixed-size-list GPU columns, with optional named validity siblings and preserved parent/child nulls, record batches, and source identity. - **Variable-length Arrow attribute lists** - `GPUVector` can retain chunked nested list columns whose elements contain one to four numeric components, covering scalar streams plus tuple-style data such as XY, XYZ, and XYZM coordinates for future path-rendering workflows. - **Closed Arrow path normalization** - `closeArrowPaths()` appends explicit closing vertices only for closed Float32 absolute or origin-relative delta path rows whose endpoints differ beyond an epsilon, using WebGPU compute when available with equivalent CPU fallback semantics. - **`ArrowPathModel`** - New attribute-backed path renderer consumes prepared Float32 XY, XYZ, and XYZM path props, expands path rows into packed per-segment render records, and supports Float64 source paths through CPU-prepared Float32 deltas plus CPU-updated view origins. diff --git a/test/examples/luvs-live-benchmark-docs.node.spec.ts b/test/examples/luvs-live-benchmark-docs.node.spec.ts new file mode 100644 index 0000000000..df0889990f --- /dev/null +++ b/test/examples/luvs-live-benchmark-docs.node.spec.ts @@ -0,0 +1,262 @@ +// luma.gl +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: Copyright (c) vis.gl contributors + +import {readFileSync} from 'node:fs'; +import {createRequire} from 'node:module'; + +import React from 'react'; +import {renderToString} from 'react-dom/server'; +import typescript from 'typescript'; +import {beforeEach, describe, expect, test, vi} from 'vitest'; + +type MockLuvsBenchmarkPanelProps = { + title: string; + description: string; + runLabel: string; + onRun: () => Promise; +}; + +const benchmarkSource = readFileSync( + new URL('../../website/src/components/docs/luvs-benchmark.tsx', import.meta.url), + 'utf8' +); +const benchmarkRuntimeSource = readFileSync( + new URL('../../website/src/components/docs/luvs-benchmark-runtime.ts', import.meta.url), + 'utf8' +); +const documentationSource = readFileSync( + new URL('../../docs/api-reference/experimental/luvs.md', import.meta.url), + 'utf8' +); +const navigationSource = readFileSync( + new URL('../../website/src/components/docs/experimental-docs-tabs.tsx', import.meta.url), + 'utf8' +); +const sidebarSource = readFileSync( + new URL('../../docs/table-of-contents.json', import.meta.url), + 'utf8' +); +const experimentalPackage = JSON.parse( + readFileSync(new URL('../../modules/experimental/package.json', import.meta.url), 'utf8') +) as {exports: Record}; +const experimentalMainIndex = readFileSync( + new URL('../../modules/experimental/src/index.ts', import.meta.url), + 'utf8' +); + +const transpiledBenchmark = typescript.transpileModule(benchmarkSource, { + compilerOptions: { + esModuleInterop: true, + jsx: typescript.JsxEmit.ReactJSX, + module: typescript.ModuleKind.CommonJS, + target: typescript.ScriptTarget.ES2022 + } +}); + +const createDevice = vi.fn(); +const runLuvsBenchmark = vi.fn(); +let selectedDevice: Record | undefined; +let benchmarkPanelProps: MockLuvsBenchmarkPanelProps | undefined; + +const nativeRequire = createRequire(import.meta.url); +function requireLuvsBenchmarkDependency(moduleName: string): unknown { + if (moduleName === './luvs-benchmark-runtime') { + return { + LUVS_BENCHMARK_MEASURED_ITERATIONS: 5, + LUVS_BENCHMARK_WARMUP_ITERATIONS: 1, + runLuvsBenchmark + }; + } + if (moduleName === '../../react-luma/store/device-store') { + return { + createDevice, + useStore: (selector: (state: {presentationDevice?: unknown; device?: unknown}) => unknown) => + selector({presentationDevice: selectedDevice}) + }; + } + if (moduleName === './live-benchmark-panel') { + return { + LiveBenchmarkPanel: (props: MockLuvsBenchmarkPanelProps) => { + benchmarkPanelProps = props; + return React.createElement( + 'section', + null, + React.createElement('h3', null, props.title), + React.createElement('p', null, props.description), + React.createElement('button', null, props.runLabel) + ); + } + }; + } + + return nativeRequire(moduleName); +} + +const benchmarkModule: {exports: Record} = {exports: {}}; +const loadLuvsBenchmark = new Function( + 'require', + 'module', + 'exports', + transpiledBenchmark.outputText +); +loadLuvsBenchmark(requireLuvsBenchmarkDependency, benchmarkModule, benchmarkModule.exports); +const LuvsBenchmark = benchmarkModule.exports.LuvsBenchmark; + +beforeEach(() => { + createDevice.mockReset(); + runLuvsBenchmark.mockReset(); + selectedDevice = undefined; + benchmarkPanelProps = undefined; +}); + +describe('luVS live vector-similarity benchmark documentation', () => { + test('publishes one optional package entry and embeds its reference in both sidebars', () => { + expect(experimentalPackage.exports['./luvs']).toMatchObject({ + import: './dist/luvs/index.js', + types: './dist/luvs/index.d.ts' + }); + expect(experimentalMainIndex).not.toContain("from './luvs"); + expect(documentationSource).toContain( + "import {LuvsBenchmark} from '@site/src/components/docs/luvs-benchmark';" + ); + expect(documentationSource).toContain(''); + expect(sidebarSource.match(/"api-reference\/experimental\/luvs"/g)).toHaveLength(2); + expect(navigationSource).toContain("href: '/docs/api-reference/experimental/luvs'"); + }); + + test('documents fixed-size GPU table columns, Arrow ingestion, filters, and approximate IVF', () => { + expect(documentationSource).toContain("GPUVector<'fixed-size-list'>"); + expect(documentationSource).toContain("from '@luma.gl/arrow'"); + expect(documentationSource).toContain('makeGPUTableFromArrowTable'); + expect(documentationSource).toContain('validityColumns'); + expect(documentationSource).toContain('importGPUEmbeddingTable'); + expect(documentationSource).toContain('Null source identifiers are'); + expect(documentationSource).toContain('Nullable embedding data without a selected'); + expect(documentationSource).not.toContain('makeGPUEmbeddingMatrixFromArrow'); + expect(documentationSource).not.toContain('ownsValues'); + expect(documentationSource).toContain('filterMask: selection.mask'); + expect(documentationSource).toContain('GPUKMeans'); + expect(documentationSource).toContain('GPUIVFFlatIndex'); + expect(documentationSource).toContain('listRowIndices'); + expect(documentationSource).toContain('traverse the selected inverted lists directly'); + expect(documentationSource).toContain('bounded GPU hash'); + expect(documentationSource).toContain('Float32 distance or inner product overflows'); + expect(documentationSource).toContain('**approximate**'); + expect(documentationSource).toContain('zero-copy'); + }); + + test('server-renders every workload control without creating a GPU device or starting work', () => { + const markup = renderToString(React.createElement(LuvsBenchmark)); + + for (const label of [ + 'Dataset rows', + 'Dimensions', + 'Queries', + 'Nearest neighbors (K)', + 'Selected rows (%)', + 'IVF lists', + 'IVF probes', + 'Run live CPU and WebGPU vector benchmark' + ]) { + expect(markup).toContain(label); + } + expect(markup).toContain('384'); + expect(markup).toContain('768'); + expect(markup).toContain('1,536'); + expect(createDevice).not.toHaveBeenCalled(); + expect(runLuvsBenchmark).not.toHaveBeenCalled(); + }); + + test('requests a WebGPU device only when the reader starts the benchmark', async () => { + createDevice.mockRejectedValue(new Error('Deferred WebGPU device request')); + renderToString(React.createElement(LuvsBenchmark)); + + expect(createDevice).not.toHaveBeenCalled(); + await expect(benchmarkPanelProps!.onRun()).rejects.toThrow('Deferred WebGPU device request'); + expect(createDevice).toHaveBeenCalledOnce(); + expect(createDevice).toHaveBeenCalledWith('webgpu-core'); + }); + + test('renders verified CPU, exact, filtered, and IVF results after an explicit run', async () => { + selectedDevice = {type: 'webgpu'}; + runLuvsBenchmark.mockResolvedValue({ + results: [ + {label: 'CPU exact', medianMilliseconds: 2, resultCount: 10, candidateCount: 2048}, + {label: 'WebGPU exact', medianMilliseconds: 1, resultCount: 10, candidateCount: 2048}, + { + label: 'WebGPU exact + selection', + medianMilliseconds: 0.8, + resultCount: 10, + candidateCount: 512 + }, + { + label: 'WebGPU IVF-flat + selection', + medianMilliseconds: 0.5, + resultCount: 10, + candidateCount: 128, + recall: 0.75 + } + ], + uploadMilliseconds: 3, + indexBuildMilliseconds: 4, + indexByteLength: 4096, + options: { + datasetRowCount: 2048, + dimensions: 128, + queryCount: 4, + resultCount: 10, + filterPercentage: 25, + listCount: 8, + probeCount: 2 + }, + timestampQueries: false, + deviceLabel: 'Reader GPU' + }); + renderToString(React.createElement(LuvsBenchmark)); + + const output = await benchmarkPanelProps!.onRun(); + const markup = renderToString(output as React.ReactElement); + + expect(createDevice).not.toHaveBeenCalled(); + expect(runLuvsBenchmark).toHaveBeenCalledOnce(); + expect(markup).toContain('CPU exact'); + expect(markup).toContain('WebGPU exact + selection'); + expect(markup).toContain('WebGPU IVF-flat + selection'); + expect(markup).toContain('75.0%'); + expect(markup).toContain('Recall@K'); + expect(markup).toContain('Reader GPU'); + expect(markup).toContain('completion fence'); + }); + + test('runs independent CPU, exact, filtered, and IVF paths with fenced and separated timings', () => { + expect(benchmarkRuntimeSource).toContain('runCPUEmbeddingSearch(fixture, options)'); + expect(benchmarkRuntimeSource).toContain( + 'score = Math.fround(score + Math.fround(difference * difference))' + ); + expect(benchmarkRuntimeSource).toContain('FLOAT32_RANKING_TOLERANCE'); + expect(benchmarkRuntimeSource).toContain('new GPUTable({batches})'); + expect(benchmarkRuntimeSource).toContain('new GPURecordBatch({'); + expect(benchmarkRuntimeSource).toContain('fixed-size-list'); + expect(benchmarkRuntimeSource).toContain('importGPUEmbeddingTable(graph, dataset'); + expect(benchmarkRuntimeSource).not.toContain('GPUEmbeddingMatrix'); + expect(benchmarkRuntimeSource).toContain('new GPUSimilaritySearch({'); + expect(benchmarkRuntimeSource).toContain('new GPUIVFFlatIndex({'); + expect(benchmarkRuntimeSource).toContain('buffers.listRowIndices'); + expect(benchmarkRuntimeSource).toContain("'list-row-indices'"); + expect(benchmarkRuntimeSource).toContain('filterMask: importLuvsView('); + expect(benchmarkRuntimeSource).toContain("fallback: 'none'"); + expect(benchmarkRuntimeSource).toContain( + 'validateLuvsOutput(actual, oracle, label, approximate)' + ); + expect(benchmarkRuntimeSource).toContain('device.submit(commandEncoder.finish())'); + expect(benchmarkRuntimeSource).toContain('const fence = device.createFence()'); + expect(benchmarkRuntimeSource).toContain('await fence.signaled'); + expect(benchmarkRuntimeSource).toContain('execution.encoding.readTimings()'); + expect(benchmarkRuntimeSource).toContain('uploadMilliseconds'); + expect(benchmarkRuntimeSource).toContain('indexBuildMilliseconds'); + expect(benchmarkRuntimeSource).toContain('readbackMilliseconds'); + expect(benchmarkRuntimeSource).toContain('rerankMilliseconds'); + expect(benchmarkSource).toContain('Recall@K'); + }); +}); diff --git a/test/examples/luvs-live-benchmark.spec.ts b/test/examples/luvs-live-benchmark.spec.ts new file mode 100644 index 0000000000..e82ac70330 --- /dev/null +++ b/test/examples/luvs-live-benchmark.spec.ts @@ -0,0 +1,88 @@ +// luma.gl +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: Copyright (c) vis.gl contributors + +import {getWebGPUTestDevice} from '@luma.gl/test-utils'; +import {expect, test} from 'vitest'; +import { + runLuvsBenchmark, + validateLuvsOutput +} from '../../website/src/components/docs/luvs-benchmark-runtime'; + +test('luVS benchmark accepts only Float32-equivalent exact-neighbor rank swaps', () => { + const makeOutput = (ids: number[], scores: number[]) => ({ + ids: Uint32Array.from(ids), + scores: Float32Array.from(scores), + resultCounts: Uint32Array.from([ids.length]), + candidateCounts: Uint32Array.from([4]) + }); + const firstScore = Math.fround(1); + const nearlyTiedScore = Math.fround(1 + 2 ** -22); + const oracle = makeOutput([10, 20], [firstScore, nearlyTiedScore]); + const equivalentSwap = makeOutput([20, 10], [firstScore, nearlyTiedScore]); + + expect(validateLuvsOutput(equivalentSwap, oracle, 'WebGPU exact', false)).toBe(1); + expect(() => + validateLuvsOutput( + makeOutput([99, 10], [firstScore, nearlyTiedScore]), + oracle, + 'WebGPU exact', + false + ) + ).toThrow('different nearest-neighbor set'); + expect(() => + validateLuvsOutput( + makeOutput([20, 10], [1, 2]), + makeOutput([10, 20], [1, 2]), + 'WebGPU exact', + false + ) + ).toThrow('different nearest-neighbor order'); +}); + +test('luVS documentation benchmark executes real exact, filtered, and IVF-flat WebGPU work', async () => { + const device = await getWebGPUTestDevice(); + if (!device) return; + + const report = await runLuvsBenchmark(device, { + datasetRowCount: 32, + dimensions: 4, + queryCount: 1, + resultCount: 2, + filterPercentage: 50, + listCount: 2, + probeCount: 1 + }); + + expect(report.results.map(result => result.label)).toEqual([ + 'CPU exact', + 'WebGPU exact', + 'WebGPU exact + selection', + 'WebGPU IVF-flat + selection' + ]); + const [cpuExact, gpuExact, gpuFiltered, gpuApproximate] = report.results; + expect(cpuExact.resultCount).toBe(2); + expect(gpuExact.resultCount).toBe(cpuExact.resultCount); + expect(gpuExact.candidateCount).toBe(32); + expect(gpuFiltered.candidateCount).toBeGreaterThan(0); + expect(gpuFiltered.candidateCount).toBeLessThan(32); + expect(gpuApproximate.candidateCount).toBeLessThanOrEqual(gpuFiltered.candidateCount); + expect(gpuApproximate.recall).toBeGreaterThanOrEqual(0); + expect(gpuApproximate.recall).toBeLessThanOrEqual(1); + expect(report.uploadMilliseconds).toBeGreaterThanOrEqual(0); + expect(report.indexBuildMilliseconds).toBeGreaterThanOrEqual(0); + expect(report.indexByteLength).toBe( + (report.options.listCount * report.options.dimensions + + report.options.datasetRowCount * 3 + + report.options.listCount * 2 + + 4) * + Uint32Array.BYTES_PER_ELEMENT + ); + for (const result of report.results) { + expect(result.medianMilliseconds).toBeGreaterThanOrEqual(0); + } + for (const result of report.results.slice(1)) { + expect(result.encodeMilliseconds).toBeGreaterThanOrEqual(0); + expect(result.readbackMilliseconds).toBeGreaterThanOrEqual(0); + } +}); diff --git a/website/src/components/docs/luvs-benchmark-runtime.ts b/website/src/components/docs/luvs-benchmark-runtime.ts new file mode 100644 index 0000000000..2abf4eefb5 --- /dev/null +++ b/website/src/components/docs/luvs-benchmark-runtime.ts @@ -0,0 +1,783 @@ +// luma.gl +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: Copyright (c) vis.gl contributors + +import {Buffer, type Device, type QuerySet} from '@luma.gl/core'; +import { + type CompiledGPUCommandGraph, + GPUCommandGraph, + type GPUCommandGraphEncoding, + type GraphDataView +} from '@luma.gl/experimental'; +import { + GPUIVFFlatIndex, + GPUSimilaritySearch, + importGPUEmbeddingTable +} from '@luma.gl/experimental/luvs'; +import {GPUData, GPURecordBatch, GPUTable, type FixedSizeList} from '@luma.gl/tables'; + +export const LUVS_BENCHMARK_WARMUP_ITERATIONS = 1; +export const LUVS_BENCHMARK_MEASURED_ITERATIONS = 5; +const INDEX_TRAINING_ITERATIONS = 4; +const INVALID_SOURCE_ROW = 0xffff_ffff; +const FLOAT32_RANKING_TOLERANCE = 8 * 2 ** -23; + +export type LuvsBenchmarkOptions = { + datasetRowCount: number; + dimensions: number; + queryCount: number; + resultCount: number; + filterPercentage: number; + listCount: number; + probeCount: number; +}; + +type LuvsBenchmarkFixture = { + dataset: Float32Array; + queries: Float32Array; + filter: Uint32Array; +}; + +type LuvsSearchOutput = { + ids: Buffer; + scores: Buffer; + resultCounts: Buffer; + candidateCounts: Buffer; +}; + +type LuvsCPUOutput = { + ids: Uint32Array; + scores: Float32Array; + resultCounts: Uint32Array; + candidateCounts: Uint32Array; +}; + +type LuvsIndexBuffers = { + centroids: Buffer; + labels: Buffer; + listCounts: Buffer; + listOffsets: Buffer; + listSourceIds: Buffer; + listRowIndices: Buffer; + status: Buffer; +}; + +export type LuvsBenchmarkResult = { + label: string; + medianMilliseconds: number; + encodeMilliseconds?: number; + readbackMilliseconds?: number; + rerankMilliseconds?: number; + resultCount: number; + candidateCount: number; + recall?: number; +}; + +type LuvsGraphExecution = { + milliseconds: number; + encoding: GPUCommandGraphEncoding; +}; + +export type LuvsBenchmarkReport = { + results: LuvsBenchmarkResult[]; + uploadMilliseconds: number; + indexBuildMilliseconds: number; + indexByteLength: number; + options: LuvsBenchmarkOptions; + timestampQueries: boolean; + deviceLabel: string; +}; + +export async function runLuvsBenchmark( + device: Device, + options: LuvsBenchmarkOptions +): Promise { + const fixture = makeLuvsBenchmarkFixture(options); + const exactCPU = measureCPUEmbeddingSearch(fixture, options); + const filteredOracle = runCPUEmbeddingSearch(fixture, options, fixture.filter); + const ownedBuffers: Buffer[] = []; + const ownedTables: GPUTable[] = []; + const compiledGraphs: CompiledGPUCommandGraph[] = []; + + try { + const uploadStartTime = performance.now(); + const dataset = createBenchmarkEmbeddingTable( + device, + ownedTables, + fixture.dataset, + options.datasetRowCount, + options.dimensions, + 'dataset', + 2 + ); + const queries = createBenchmarkEmbeddingTable( + device, + ownedTables, + fixture.queries, + options.queryCount, + options.dimensions, + 'queries', + 1 + ); + const filterBuffer = createLuvsInputBuffer(device, ownedBuffers, fixture.filter); + await waitForLuvsCompletion(device); + const uploadMilliseconds = performance.now() - uploadStartTime; + + const exactOutput = createLuvsSearchOutput(device, ownedBuffers, options); + const filteredOutput = createLuvsSearchOutput(device, ownedBuffers, options); + const approximateOutput = createLuvsSearchOutput(device, ownedBuffers, options); + const indexBuffers = createLuvsIndexBuffers(device, ownedBuffers, options); + + const exactGraph = compileLuvsExactGraph(device, dataset, queries, exactOutput, options); + compiledGraphs.push(exactGraph); + const filteredGraph = compileLuvsExactGraph( + device, + dataset, + queries, + filteredOutput, + options, + filterBuffer + ); + compiledGraphs.push(filteredGraph); + const indexBuildGraph = compileLuvsIndexBuild(device, dataset, indexBuffers, options); + compiledGraphs.push(indexBuildGraph); + const approximateGraph = compileLuvsIndexSearch( + device, + dataset, + queries, + indexBuffers, + approximateOutput, + filterBuffer, + options + ); + compiledGraphs.push(approximateGraph); + + const exactGPU = await measureLuvsGraph( + device, + exactGraph, + exactOutput, + exactCPU.output, + options, + 'WebGPU exact' + ); + const filteredGPU = await measureLuvsGraph( + device, + filteredGraph, + filteredOutput, + filteredOracle, + options, + 'WebGPU exact + selection' + ); + + await executeLuvsGraph(device, indexBuildGraph, 'luvs-index-warmup'); + const indexBuild = await executeLuvsGraph(device, indexBuildGraph, 'luvs-index-build'); + const approximateGPU = await measureLuvsGraph( + device, + approximateGraph, + approximateOutput, + filteredOracle, + options, + 'WebGPU IVF-flat + selection', + true + ); + + const indexByteLength = Object.values(indexBuffers).reduce( + (byteLength, buffer) => byteLength + buffer.byteLength, + 0 + ); + return { + results: [exactCPU.result, exactGPU, filteredGPU, approximateGPU], + uploadMilliseconds, + indexBuildMilliseconds: indexBuild.milliseconds, + indexByteLength, + options, + timestampQueries: device.features.has('timestamp-query'), + deviceLabel: device.info.renderer || device.info.vendor || device.info.gpu + }; + } finally { + for (const graph of compiledGraphs) graph.destroy(); + for (const table of ownedTables) table.destroy(); + for (const buffer of ownedBuffers) buffer.destroy(); + } +} + +function makeLuvsBenchmarkFixture(options: LuvsBenchmarkOptions): LuvsBenchmarkFixture { + const dataset = new Float32Array(options.datasetRowCount * options.dimensions); + const queries = new Float32Array(options.queryCount * options.dimensions); + const filter = new Uint32Array(options.datasetRowCount); + let randomState = 0x4c55_5653; + + for (let elementIndex = 0; elementIndex < dataset.length; elementIndex++) { + randomState = (Math.imul(randomState, 1_664_525) + 1_013_904_223) >>> 0; + dataset[elementIndex] = randomState / 0x8000_0000 - 1; + } + for (let queryIndex = 0; queryIndex < options.queryCount; queryIndex++) { + const sourceRow = Math.floor( + ((queryIndex + 1) * options.datasetRowCount) / (options.queryCount + 1) + ); + const sourceOffset = sourceRow * options.dimensions; + const queryOffset = queryIndex * options.dimensions; + for (let dimension = 0; dimension < options.dimensions; dimension++) { + queries[queryOffset + dimension] = Math.fround( + dataset[sourceOffset + dimension] + ((dimension % 7) - 3) * 0.0001 + ); + } + } + for (let rowIndex = 0; rowIndex < options.datasetRowCount; rowIndex++) { + const selection = (Math.imul(rowIndex + 1, 2_654_435_761) >>> 0) % 100; + filter[rowIndex] = Number(selection < options.filterPercentage); + } + + return {dataset, queries, filter}; +} + +function measureCPUEmbeddingSearch( + fixture: LuvsBenchmarkFixture, + options: LuvsBenchmarkOptions +): {result: LuvsBenchmarkResult; output: LuvsCPUOutput} { + for (let iteration = 0; iteration < LUVS_BENCHMARK_WARMUP_ITERATIONS; iteration++) { + runCPUEmbeddingSearch(fixture, options); + } + + const durations: number[] = []; + let output = runCPUEmbeddingSearch(fixture, options); + for (let iteration = 0; iteration < LUVS_BENCHMARK_MEASURED_ITERATIONS; iteration++) { + const startTime = performance.now(); + output = runCPUEmbeddingSearch(fixture, options); + durations.push(performance.now() - startTime); + } + + return { + result: { + label: 'CPU exact', + medianMilliseconds: getLuvsMedian(durations), + resultCount: sumLuvsCounts(output.resultCounts), + candidateCount: sumLuvsCounts(output.candidateCounts) + }, + output + }; +} + +function runCPUEmbeddingSearch( + fixture: LuvsBenchmarkFixture, + options: LuvsBenchmarkOptions, + filter?: Uint32Array +): LuvsCPUOutput { + const output: LuvsCPUOutput = { + ids: new Uint32Array(options.queryCount * options.resultCount).fill(INVALID_SOURCE_ROW), + scores: new Float32Array(options.queryCount * options.resultCount).fill(Infinity), + resultCounts: new Uint32Array(options.queryCount), + candidateCounts: new Uint32Array(options.queryCount) + }; + + for (let queryIndex = 0; queryIndex < options.queryCount; queryIndex++) { + const resultOffset = queryIndex * options.resultCount; + for (let rowIndex = 0; rowIndex < options.datasetRowCount; rowIndex++) { + if (filter && filter[rowIndex] === 0) continue; + output.candidateCounts[queryIndex]++; + let score = 0; + const queryOffset = queryIndex * options.dimensions; + const rowOffset = rowIndex * options.dimensions; + for (let dimension = 0; dimension < options.dimensions; dimension++) { + const difference = Math.fround( + fixture.queries[queryOffset + dimension] - fixture.dataset[rowOffset + dimension] + ); + score = Math.fround(score + Math.fround(difference * difference)); + } + + let insertionIndex = Math.min(output.resultCounts[queryIndex], options.resultCount); + while (insertionIndex > 0) { + const precedingIndex = resultOffset + insertionIndex - 1; + if ( + output.scores[precedingIndex] < score || + (output.scores[precedingIndex] === score && output.ids[precedingIndex] < rowIndex) + ) { + break; + } + if (insertionIndex < options.resultCount) { + output.scores[resultOffset + insertionIndex] = output.scores[precedingIndex]; + output.ids[resultOffset + insertionIndex] = output.ids[precedingIndex]; + } + insertionIndex--; + } + if (insertionIndex < options.resultCount) { + output.ids[resultOffset + insertionIndex] = rowIndex; + output.scores[resultOffset + insertionIndex] = score; + output.resultCounts[queryIndex] = Math.min( + output.resultCounts[queryIndex] + 1, + options.resultCount + ); + } + } + } + + return output; +} + +function createBenchmarkEmbeddingTable( + device: Device, + ownedTables: GPUTable[], + values: Float32Array, + rowCount: number, + dimensions: number, + identifier: string, + chunkCount: number +): GPUTable { + const batches: GPURecordBatch[] = []; + const format: FixedSizeList<'float32'> = `fixed-size-list`; + + for (let chunkIndex = 0; chunkIndex < chunkCount; chunkIndex++) { + const sourceRowOffset = Math.floor((chunkIndex * rowCount) / chunkCount); + const endRowOffset = Math.floor(((chunkIndex + 1) * rowCount) / chunkCount); + const chunkValues = values.subarray( + sourceRowOffset * dimensions, + endRowOffset * dimensions + ); + const buffer = device.createBuffer({ + id: `${identifier}-chunk-${chunkIndex}`, + data: chunkValues, + usage: Buffer.STORAGE | Buffer.COPY_DST + }); + const embedding = new GPUData({ + buffer, + format, + length: endRowOffset - sourceRowOffset, + ownsBuffer: true + }); + batches.push( + new GPURecordBatch({ + gpuData: {embedding}, + bufferLayout: [], + sourceInfo: { + sourceBatchIndex: chunkIndex, + sourceRowIndexOffset: sourceRowOffset, + sourceRowCount: endRowOffset - sourceRowOffset + } + }) + ); + } + + const table = new GPUTable({batches}); + ownedTables.push(table); + return table; +} + +function createLuvsInputBuffer( + device: Device, + ownedBuffers: Buffer[], + data: Float32Array | Uint32Array +): Buffer { + const buffer = device.createBuffer({data, usage: Buffer.STORAGE | Buffer.COPY_DST}); + ownedBuffers.push(buffer); + return buffer; +} + +function createLuvsStorageBuffer( + device: Device, + ownedBuffers: Buffer[], + valueCount: number +): Buffer { + const buffer = device.createBuffer({ + byteLength: Math.max(valueCount, 1) * Uint32Array.BYTES_PER_ELEMENT, + usage: Buffer.STORAGE | Buffer.COPY_SRC + }); + ownedBuffers.push(buffer); + return buffer; +} + +function createLuvsSearchOutput( + device: Device, + ownedBuffers: Buffer[], + options: LuvsBenchmarkOptions +): LuvsSearchOutput { + const valueCount = options.queryCount * options.resultCount; + return { + ids: createLuvsStorageBuffer(device, ownedBuffers, valueCount), + scores: createLuvsStorageBuffer(device, ownedBuffers, valueCount), + resultCounts: createLuvsStorageBuffer(device, ownedBuffers, options.queryCount), + candidateCounts: createLuvsStorageBuffer(device, ownedBuffers, options.queryCount) + }; +} + +function createLuvsIndexBuffers( + device: Device, + ownedBuffers: Buffer[], + options: LuvsBenchmarkOptions +): LuvsIndexBuffers { + return { + centroids: createLuvsStorageBuffer( + device, + ownedBuffers, + options.listCount * options.dimensions + ), + labels: createLuvsStorageBuffer(device, ownedBuffers, options.datasetRowCount), + listCounts: createLuvsStorageBuffer(device, ownedBuffers, options.listCount), + listOffsets: createLuvsStorageBuffer(device, ownedBuffers, options.listCount + 1), + listSourceIds: createLuvsStorageBuffer(device, ownedBuffers, options.datasetRowCount), + listRowIndices: createLuvsStorageBuffer(device, ownedBuffers, options.datasetRowCount), + status: createLuvsStorageBuffer(device, ownedBuffers, 3) + }; +} + +function compileLuvsExactGraph( + device: Device, + dataset: GPUTable, + queries: GPUTable, + output: LuvsSearchOutput, + options: LuvsBenchmarkOptions, + filterBuffer?: Buffer +): CompiledGPUCommandGraph { + const graph = new GPUCommandGraph(device, { + id: filterBuffer ? 'docs-luvs-filtered' : 'docs-luvs-exact' + }); + new GPUSimilaritySearch({ + id: filterBuffer ? 'filtered-search' : 'exact-search', + dataset: importGPUEmbeddingTable(graph, dataset, {column: 'embedding', id: 'dataset'}), + queries: importGPUEmbeddingTable(graph, queries, {column: 'embedding', id: 'queries'}), + ...importLuvsSearchOutput(graph, output, options), + k: options.resultCount, + metric: 'squared-euclidean', + ...(filterBuffer + ? { + filterMask: importLuvsView( + graph, + 'selection-mask', + filterBuffer, + 'uint32', + options.datasetRowCount + ) + } + : {}) + }).addToGraph(graph); + return graph.compile(); +} + +function compileLuvsIndexBuild( + device: Device, + dataset: GPUTable, + buffers: LuvsIndexBuffers, + options: LuvsBenchmarkOptions +): CompiledGPUCommandGraph { + const graph = new GPUCommandGraph(device, {id: 'docs-luvs-index-build'}); + const index = createLuvsIndex(graph, dataset, buffers, options); + index.addToGraph(graph); + return graph.compile(); +} + +function compileLuvsIndexSearch( + device: Device, + dataset: GPUTable, + queries: GPUTable, + buffers: LuvsIndexBuffers, + output: LuvsSearchOutput, + filterBuffer: Buffer, + options: LuvsBenchmarkOptions +): CompiledGPUCommandGraph { + const graph = new GPUCommandGraph(device, {id: 'docs-luvs-index-search'}); + const index = createLuvsIndex(graph, dataset, buffers, options); + index.addSearchToGraph(graph, { + id: 'approximate-search', + queries: importGPUEmbeddingTable(graph, queries, {column: 'embedding', id: 'queries'}), + ...importLuvsSearchOutput(graph, output, options), + k: options.resultCount, + metric: 'squared-euclidean', + probeCount: options.probeCount, + filterMask: importLuvsView( + graph, + 'selection-mask', + filterBuffer, + 'uint32', + options.datasetRowCount + ), + fallback: 'none' + }); + return graph.compile(); +} + +function createLuvsIndex( + graph: GPUCommandGraph, + dataset: GPUTable, + buffers: LuvsIndexBuffers, + options: LuvsBenchmarkOptions +): GPUIVFFlatIndex { + return new GPUIVFFlatIndex({ + id: 'docs-luvs-index', + dataset: importGPUEmbeddingTable(graph, dataset, {column: 'embedding', id: 'dataset'}), + listCount: options.listCount, + centroids: importLuvsView( + graph, + 'centroids', + buffers.centroids, + 'float32', + options.listCount * options.dimensions + ), + labels: importLuvsView(graph, 'labels', buffers.labels, 'uint32', options.datasetRowCount), + listCounts: importLuvsView( + graph, + 'list-counts', + buffers.listCounts, + 'uint32', + options.listCount + ), + listOffsets: importLuvsView( + graph, + 'list-offsets', + buffers.listOffsets, + 'uint32', + options.listCount + 1 + ), + listSourceIds: importLuvsView( + graph, + 'list-source-ids', + buffers.listSourceIds, + 'uint32', + options.datasetRowCount + ), + listRowIndices: importLuvsView( + graph, + 'list-row-indices', + buffers.listRowIndices, + 'uint32', + options.datasetRowCount + ), + status: importLuvsView(graph, 'index-status', buffers.status, 'uint32', 3), + maxIterations: INDEX_TRAINING_ITERATIONS + }); +} + +function importLuvsSearchOutput( + graph: GPUCommandGraph, + output: LuvsSearchOutput, + options: LuvsBenchmarkOptions +): { + outputIds: GraphDataView<'uint32'>; + outputScores: GraphDataView<'float32'>; + resultCounts: GraphDataView<'uint32'>; + candidateCounts: GraphDataView<'uint32'>; +} { + const valueCount = options.queryCount * options.resultCount; + return { + outputIds: importLuvsView(graph, 'output-ids', output.ids, 'uint32', valueCount), + outputScores: importLuvsView(graph, 'output-scores', output.scores, 'float32', valueCount), + resultCounts: importLuvsView( + graph, + 'result-counts', + output.resultCounts, + 'uint32', + options.queryCount + ), + candidateCounts: importLuvsView( + graph, + 'candidate-counts', + output.candidateCounts, + 'uint32', + options.queryCount + ) + }; +} + +function importLuvsView( + graph: GPUCommandGraph, + identifier: string, + buffer: Buffer, + format: Format, + length: number +): GraphDataView { + const handle = graph.importBuffer( + {id: identifier, byteLength: buffer.byteLength, usage: buffer.usage}, + buffer + ); + return graph.createDataView(handle, {format, length}); +} + +async function measureLuvsGraph( + device: Device, + graph: CompiledGPUCommandGraph, + output: LuvsSearchOutput, + oracle: LuvsCPUOutput, + options: LuvsBenchmarkOptions, + label: string, + approximate = false +): Promise { + await executeLuvsGraph(device, graph, `${label}-validation`); + const validationStartTime = performance.now(); + const actual = await readLuvsSearchOutput(output, options); + const readbackMilliseconds = performance.now() - validationStartTime; + const recall = validateLuvsOutput(actual, oracle, label, approximate); + + for (let iteration = 0; iteration < LUVS_BENCHMARK_WARMUP_ITERATIONS; iteration++) { + await executeLuvsGraph(device, graph, `${label}-warmup-${iteration}`); + } + + const measurements: LuvsGraphExecution[] = []; + for (let iteration = 0; iteration < LUVS_BENCHMARK_MEASURED_ITERATIONS; iteration++) { + measurements.push(await executeLuvsGraph(device, graph, `${label}-measurement-${iteration}`)); + } + + const rerankMilliseconds = await profileLuvsCandidatePasses(device, graph, label); + return { + label, + medianMilliseconds: getLuvsMedian(measurements.map(result => result.milliseconds)), + encodeMilliseconds: getLuvsMedian( + measurements.map(result => result.encoding.stats.cpuEncodeTimeMilliseconds) + ), + readbackMilliseconds, + resultCount: sumLuvsCounts(actual.resultCounts), + candidateCount: sumLuvsCounts(actual.candidateCounts), + ...(rerankMilliseconds === undefined ? {} : {rerankMilliseconds}), + ...(approximate ? {recall} : {}) + }; +} + +async function executeLuvsGraph( + device: Device, + graph: CompiledGPUCommandGraph, + identifier: string, + querySet?: QuerySet +): Promise { + const commandEncoder = device.createCommandEncoder({ + id: identifier, + ...(querySet ? {timeProfilingQuerySet: querySet} : {}) + }); + let submitted = false; + const startTime = performance.now(); + + try { + const encoding = graph.encode(commandEncoder, {parameters: undefined}); + device.submit(commandEncoder.finish()); + submitted = true; + await waitForLuvsCompletion(device); + return {milliseconds: performance.now() - startTime, encoding}; + } catch (error) { + if (!submitted) commandEncoder.destroy(); + throw error; + } +} + +async function waitForLuvsCompletion(device: Device): Promise { + const fence = device.createFence(); + try { + await fence.signaled; + } finally { + fence.destroy(); + } +} + +async function profileLuvsCandidatePasses( + device: Device, + graph: CompiledGPUCommandGraph, + identifier: string +): Promise { + if (!device.features.has('timestamp-query')) return undefined; + const querySet = device.createQuerySet({ + id: `${identifier}-timestamps`, + type: 'timestamp', + count: Math.max(graph.stats.nodeOrder.length * 2, 2) + }); + + try { + const execution = await executeLuvsGraph(device, graph, `${identifier}-profile`, querySet); + if (!execution.encoding.canReadGPUTimings) return undefined; + const report = await execution.encoding.readTimings(); + const candidatePasses = report.nodes.filter(node => + /candidate|distance|score|rerank|search|select/i.test(node.id) + ); + const measuredPasses = candidatePasses.filter(node => node.gpuTimeMilliseconds !== undefined); + if (measuredPasses.length === 0) return undefined; + return measuredPasses.reduce((duration, node) => duration + (node.gpuTimeMilliseconds ?? 0), 0); + } finally { + querySet.destroy(); + } +} + +async function readLuvsSearchOutput( + output: LuvsSearchOutput, + options: LuvsBenchmarkOptions +): Promise { + const [ids, scores, resultCounts, candidateCounts] = await Promise.all([ + output.ids.readAsync(), + output.scores.readAsync(), + output.resultCounts.readAsync(), + output.candidateCounts.readAsync() + ]); + const outputLength = options.queryCount * options.resultCount; + return { + ids: new Uint32Array(ids.buffer, ids.byteOffset, outputLength), + scores: new Float32Array(scores.buffer, scores.byteOffset, outputLength), + resultCounts: new Uint32Array( + resultCounts.buffer, + resultCounts.byteOffset, + options.queryCount + ), + candidateCounts: new Uint32Array( + candidateCounts.buffer, + candidateCounts.byteOffset, + options.queryCount + ) + }; +} + +/** Validates independent top-K membership while accepting only Float32-equivalent rank swaps. */ +export function validateLuvsOutput( + actual: LuvsCPUOutput, + oracle: LuvsCPUOutput, + label: string, + approximate: boolean +): number { + let matchingResults = 0; + let expectedResults = 0; + const resultCapacity = actual.ids.length / actual.resultCounts.length; + + for (let queryIndex = 0; queryIndex < actual.resultCounts.length; queryIndex++) { + const actualCount = actual.resultCounts[queryIndex]; + const expectedCount = oracle.resultCounts[queryIndex]; + if (actualCount > resultCapacity || (!approximate && actualCount !== expectedCount)) { + throw new Error(`${label} returned an invalid result count for query ${queryIndex}.`); + } + if (!approximate && actual.candidateCounts[queryIndex] !== oracle.candidateCounts[queryIndex]) { + throw new Error(`${label} returned an incorrect eligible-candidate count.`); + } + + const resultOffset = queryIndex * resultCapacity; + const expectedIds = new Map(); + for (let resultIndex = 0; resultIndex < expectedCount; resultIndex++) { + expectedIds.set(oracle.ids[resultOffset + resultIndex], resultIndex); + } + const actualIds = new Set(); + expectedResults += expectedCount; + for (let resultIndex = 0; resultIndex < actualCount; resultIndex++) { + const actualIdentifier = actual.ids[resultOffset + resultIndex]; + if (expectedIds.has(actualIdentifier)) matchingResults++; + if (!approximate) { + const expectedPosition = expectedIds.get(actualIdentifier); + if (expectedPosition === undefined || actualIds.has(actualIdentifier)) { + throw new Error(`${label} returned a different nearest-neighbor set than the CPU oracle.`); + } + actualIds.add(actualIdentifier); + const expectedRankingScore = oracle.scores[resultOffset + resultIndex]; + const expectedScore = oracle.scores[resultOffset + expectedPosition]; + const rankingTolerance = + Math.max(1, Math.abs(expectedRankingScore), Math.abs(expectedScore)) * + FLOAT32_RANKING_TOLERANCE; + if (Math.abs(expectedRankingScore - expectedScore) > rankingTolerance) { + throw new Error(`${label} returned a different nearest-neighbor order than the CPU oracle.`); + } + const actualScore = actual.scores[resultOffset + resultIndex]; + if (Math.abs(actualScore - expectedScore) > Math.max(0.0001, expectedScore * 0.0001)) { + throw new Error(`${label} returned a score outside the Float32 accuracy tolerance.`); + } + } + } + } + + return expectedResults === 0 ? 1 : matchingResults / expectedResults; +} + +function getLuvsMedian(values: number[]): number { + const orderedValues = [...values].sort((first, second) => first - second); + return orderedValues[Math.floor(orderedValues.length / 2)]; +} + +function sumLuvsCounts(values: Uint32Array): number { + return values.reduce((total, value) => total + value, 0); +} diff --git a/website/src/components/docs/luvs-benchmark.tsx b/website/src/components/docs/luvs-benchmark.tsx new file mode 100644 index 0000000000..da39dd50ff --- /dev/null +++ b/website/src/components/docs/luvs-benchmark.tsx @@ -0,0 +1,235 @@ +import React, {type ReactNode, useEffect, useId, useState} from 'react'; +import {createDevice, useStore} from '../../react-luma/store/device-store'; +import {LiveBenchmarkPanel} from './live-benchmark-panel'; +import { + LUVS_BENCHMARK_MEASURED_ITERATIONS, + LUVS_BENCHMARK_WARMUP_ITERATIONS, + runLuvsBenchmark, + type LuvsBenchmarkReport +} from './luvs-benchmark-runtime'; + +const DATASET_ROW_COUNTS = [512, 2_048, 8_192] as const; +const EMBEDDING_DIMENSIONS = [32, 128, 384, 768, 1_536] as const; +const QUERY_COUNTS = [1, 4, 8] as const; +const RESULT_COUNTS = [1, 5, 10, 20] as const; +const FILTER_PERCENTAGES = [5, 25, 50, 100] as const; +const IVF_LIST_COUNTS = [4, 8, 16] as const; +const IVF_PROBE_COUNTS = [1, 2, 4, 8, 16] as const; + +/** Compares real CPU, exact WebGPU, filtered WebGPU, and IVF-flat embedding searches. */ +export function LuvsBenchmark(): ReactNode { + const selectedDevice = useStore(store => store.presentationDevice || store.device); + const [datasetRowCount, setDatasetRowCount] = useState(2_048); + const [dimensions, setDimensions] = useState(128); + const [queryCount, setQueryCount] = useState(4); + const [resultCount, setResultCount] = useState(10); + const [filterPercentage, setFilterPercentage] = useState(25); + const [listCount, setListCount] = useState(8); + const [probeCount, setProbeCount] = useState(2); + const [unsupportedReason, setUnsupportedReason] = useState(); + const identifier = useId(); + + useEffect(() => { + if (typeof navigator === 'undefined' || !('gpu' in navigator)) { + setUnsupportedReason('WebGPU is unavailable in this browser or secure context.'); + } + }, []); + + return ( +
+
+ + + + + + { + setListCount(nextListCount); + setProbeCount(Math.min(probeCount, nextListCount)); + }} + /> + count <= listCount)} + value={probeCount} + onChange={setProbeCount} + /> +
+ + { + const device = + selectedDevice?.type === 'webgpu' ? selectedDevice : await createDevice('webgpu-core'); + const report = await runLuvsBenchmark(device, { + datasetRowCount, + dimensions, + queryCount, + resultCount, + filterPercentage, + listCount, + probeCount + }); + return ; + }} + /> +
+ ); +} + +function LuvsBenchmarkControl({ + identifier, + label, + options, + value, + onChange +}: { + identifier: string; + label: string; + options: readonly number[]; + value: number; + onChange: (value: number) => void; +}): ReactNode { + return ( + + ); +} + +function LuvsBenchmarkResults({report}: {report: LuvsBenchmarkReport}): ReactNode { + const { + results, + uploadMilliseconds, + indexBuildMilliseconds, + indexByteLength, + options, + timestampQueries, + deviceLabel + } = report; + const cpuMilliseconds = results[0].medianMilliseconds; + + return ( + <> +

+ {options.datasetRowCount.toLocaleString()} rows ×{' '} + {options.dimensions.toLocaleString()} dimensions ·{' '} + {options.queryCount} queries · K = {options.resultCount} ·{' '} + {deviceLabel} +

+ + + + + + + + + + + + + + + {results.map(result => ( + + + + + + + + + + + ))} + +
ExecutionMedian queryCPU encodeCandidate GPUReadbackCPU comparisonEligible rowsRecall@K
{result.label}{formatLuvsMilliseconds(result.medianMilliseconds)}{formatOptionalLuvsMilliseconds(result.encodeMilliseconds)}{formatOptionalLuvsMilliseconds(result.rerankMilliseconds)}{formatOptionalLuvsMilliseconds(result.readbackMilliseconds)} + {(cpuMilliseconds / Math.max(result.medianMilliseconds, Number.EPSILON)).toFixed(2)} + × + {result.candidateCount.toLocaleString()} + {result.recall === undefined ? 'Exact' : `${(result.recall * 100).toFixed(1)}%`} +
+

+ One-time source upload: {formatLuvsMilliseconds(uploadMilliseconds)}; + {` ${options.listCount}-list IVF training and index build: `} + {formatLuvsMilliseconds(indexBuildMilliseconds)}; reusable index storage:{' '} + {(indexByteLength / 1024).toFixed(1)} KiB. The approximate query probes{' '} + {options.probeCount} lists without fallback expansion. +

+

+ Query medians include encoding, submission, and an explicit GPU completion fence, but + exclude upload, graph compilation, index training, and the separately reported correctness + readback. Medians use {LUVS_BENCHMARK_WARMUP_ITERATIONS} warmup and {LUVS_BENCHMARK_MEASURED_ITERATIONS} measured runs. + {timestampQueries + ? ' Candidate GPU time uses available per-pass timestamp queries.' + : ' Candidate GPU timing is unavailable because this adapter has no timestamp queries.'} +

+ + ); +} + +function formatLuvsMilliseconds(milliseconds: number): string { + return `${milliseconds.toFixed(3)} ms`; +} + +function formatOptionalLuvsMilliseconds(milliseconds: number | undefined): string { + return milliseconds === undefined ? '—' : formatLuvsMilliseconds(milliseconds); +} From cc0a6b32ef3228bd7e2155976bd59b82d9b4c694 Mon Sep 17 00:00:00 2001 From: Ib Green Date: Fri, 7 Aug 2026 13:26:47 -0400 Subject: [PATCH 2/2] fix(experimental): report accurate luVS benchmark candidate metrics --- .../luvs-live-benchmark-docs.node.spec.ts | 9 ++++-- test/examples/luvs-live-benchmark.spec.ts | 11 +++++++ .../components/docs/luvs-benchmark-runtime.ts | 30 ++++++++++--------- .../src/components/docs/luvs-benchmark.tsx | 10 ++++--- 4 files changed, 40 insertions(+), 20 deletions(-) diff --git a/test/examples/luvs-live-benchmark-docs.node.spec.ts b/test/examples/luvs-live-benchmark-docs.node.spec.ts index df0889990f..40beb6fee6 100644 --- a/test/examples/luvs-live-benchmark-docs.node.spec.ts +++ b/test/examples/luvs-live-benchmark-docs.node.spec.ts @@ -182,8 +182,8 @@ describe('luVS live vector-similarity benchmark documentation', () => { selectedDevice = {type: 'webgpu'}; runLuvsBenchmark.mockResolvedValue({ results: [ - {label: 'CPU exact', medianMilliseconds: 2, resultCount: 10, candidateCount: 2048}, - {label: 'WebGPU exact', medianMilliseconds: 1, resultCount: 10, candidateCount: 2048}, + {label: 'CPU exact', medianMilliseconds: 2, resultCount: 10, candidateCount: 8192}, + {label: 'WebGPU exact', medianMilliseconds: 1, resultCount: 10, candidateCount: 8192}, { label: 'WebGPU exact + selection', medianMilliseconds: 0.8, @@ -225,6 +225,9 @@ describe('luVS live vector-similarity benchmark documentation', () => { expect(markup).toContain('WebGPU IVF-flat + selection'); expect(markup).toContain('75.0%'); expect(markup).toContain('Recall@K'); + expect(markup).toContain('Candidate evaluations'); + expect(markup).toContain('8,192'); + expect(markup).not.toContain('Eligible rows'); expect(markup).toContain('Reader GPU'); expect(markup).toContain('completion fence'); }); @@ -257,6 +260,8 @@ describe('luVS live vector-similarity benchmark documentation', () => { expect(benchmarkRuntimeSource).toContain('indexBuildMilliseconds'); expect(benchmarkRuntimeSource).toContain('readbackMilliseconds'); expect(benchmarkRuntimeSource).toContain('rerankMilliseconds'); + expect(benchmarkRuntimeSource).toContain('isLuvsCandidatePass(node.id)'); + expect(benchmarkRuntimeSource).not.toContain('rerank|search|select'); expect(benchmarkSource).toContain('Recall@K'); }); }); diff --git a/test/examples/luvs-live-benchmark.spec.ts b/test/examples/luvs-live-benchmark.spec.ts index e82ac70330..4267a1370e 100644 --- a/test/examples/luvs-live-benchmark.spec.ts +++ b/test/examples/luvs-live-benchmark.spec.ts @@ -5,10 +5,21 @@ import {getWebGPUTestDevice} from '@luma.gl/test-utils'; import {expect, test} from 'vitest'; import { + isLuvsCandidatePass, runLuvsBenchmark, validateLuvsOutput } from '../../website/src/components/docs/luvs-benchmark-runtime'; +test('luVS candidate timings exclude common graph prefixes, initialization, and IVF probes', () => { + expect(isLuvsCandidatePass('exact-search-initialize')).toBe(false); + expect(isLuvsCandidatePass('filtered-search-clear-output')).toBe(false); + expect(isLuvsCandidatePass('approximate-search-probe-query-0')).toBe(false); + expect(isLuvsCandidatePass('exact-search-count-candidates-0-0')).toBe(true); + expect(isLuvsCandidatePass('filtered-search-score-tile-1')).toBe(true); + expect(isLuvsCandidatePass('approximate-search-rerank-query-0-tile-1')).toBe(true); + expect(isLuvsCandidatePass('exact-search-select-top-k')).toBe(true); +}); + test('luVS benchmark accepts only Float32-equivalent exact-neighbor rank swaps', () => { const makeOutput = (ids: number[], scores: number[]) => ({ ids: Uint32Array.from(ids), diff --git a/website/src/components/docs/luvs-benchmark-runtime.ts b/website/src/components/docs/luvs-benchmark-runtime.ts index 2abf4eefb5..b705c4836d 100644 --- a/website/src/components/docs/luvs-benchmark-runtime.ts +++ b/website/src/components/docs/luvs-benchmark-runtime.ts @@ -329,10 +329,7 @@ function createBenchmarkEmbeddingTable( for (let chunkIndex = 0; chunkIndex < chunkCount; chunkIndex++) { const sourceRowOffset = Math.floor((chunkIndex * rowCount) / chunkCount); const endRowOffset = Math.floor(((chunkIndex + 1) * rowCount) / chunkCount); - const chunkValues = values.subarray( - sourceRowOffset * dimensions, - endRowOffset * dimensions - ); + const chunkValues = values.subarray(sourceRowOffset * dimensions, endRowOffset * dimensions); const buffer = device.createBuffer({ id: `${identifier}-chunk-${chunkIndex}`, data: chunkValues, @@ -679,9 +676,7 @@ async function profileLuvsCandidatePasses( const execution = await executeLuvsGraph(device, graph, `${identifier}-profile`, querySet); if (!execution.encoding.canReadGPUTimings) return undefined; const report = await execution.encoding.readTimings(); - const candidatePasses = report.nodes.filter(node => - /candidate|distance|score|rerank|search|select/i.test(node.id) - ); + const candidatePasses = report.nodes.filter(node => isLuvsCandidatePass(node.id)); const measuredPasses = candidatePasses.filter(node => node.gpuTimeMilliseconds !== undefined); if (measuredPasses.length === 0) return undefined; return measuredPasses.reduce((duration, node) => duration + (node.gpuTimeMilliseconds ?? 0), 0); @@ -690,6 +685,13 @@ async function profileLuvsCandidatePasses( } } +/** Excludes shared graph prefixes and unrelated initialization or centroid-probe passes. */ +export function isLuvsCandidatePass(nodeIdentifier: string): boolean { + return /(?:^|[-/:])(?:candidates?|distances?|scores?|rerank|select)(?:[-/:]|$)/i.test( + nodeIdentifier + ); +} + async function readLuvsSearchOutput( output: LuvsSearchOutput, options: LuvsBenchmarkOptions @@ -704,11 +706,7 @@ async function readLuvsSearchOutput( return { ids: new Uint32Array(ids.buffer, ids.byteOffset, outputLength), scores: new Float32Array(scores.buffer, scores.byteOffset, outputLength), - resultCounts: new Uint32Array( - resultCounts.buffer, - resultCounts.byteOffset, - options.queryCount - ), + resultCounts: new Uint32Array(resultCounts.buffer, resultCounts.byteOffset, options.queryCount), candidateCounts: new Uint32Array( candidateCounts.buffer, candidateCounts.byteOffset, @@ -751,7 +749,9 @@ export function validateLuvsOutput( if (!approximate) { const expectedPosition = expectedIds.get(actualIdentifier); if (expectedPosition === undefined || actualIds.has(actualIdentifier)) { - throw new Error(`${label} returned a different nearest-neighbor set than the CPU oracle.`); + throw new Error( + `${label} returned a different nearest-neighbor set than the CPU oracle.` + ); } actualIds.add(actualIdentifier); const expectedRankingScore = oracle.scores[resultOffset + resultIndex]; @@ -760,7 +760,9 @@ export function validateLuvsOutput( Math.max(1, Math.abs(expectedRankingScore), Math.abs(expectedScore)) * FLOAT32_RANKING_TOLERANCE; if (Math.abs(expectedRankingScore - expectedScore) > rankingTolerance) { - throw new Error(`${label} returned a different nearest-neighbor order than the CPU oracle.`); + throw new Error( + `${label} returned a different nearest-neighbor order than the CPU oracle.` + ); } const actualScore = actual.scores[resultOffset + resultIndex]; if (Math.abs(actualScore - expectedScore) > Math.max(0.0001, expectedScore * 0.0001)) { diff --git a/website/src/components/docs/luvs-benchmark.tsx b/website/src/components/docs/luvs-benchmark.tsx index da39dd50ff..484f084ddd 100644 --- a/website/src/components/docs/luvs-benchmark.tsx +++ b/website/src/components/docs/luvs-benchmark.tsx @@ -183,7 +183,7 @@ function LuvsBenchmarkResults({report}: {report: LuvsBenchmarkReport}): ReactNod Candidate GPU Readback CPU comparison - Eligible rows + Candidate evaluations Recall@K @@ -215,9 +215,11 @@ function LuvsBenchmarkResults({report}: {report: LuvsBenchmarkReport}): ReactNod {options.probeCount} lists without fallback expansion.

- Query medians include encoding, submission, and an explicit GPU completion fence, but - exclude upload, graph compilation, index training, and the separately reported correctness - readback. Medians use {LUVS_BENCHMARK_WARMUP_ITERATIONS} warmup and {LUVS_BENCHMARK_MEASURED_ITERATIONS} measured runs. + Candidate evaluations sum eligible candidates across all selected queries. Query medians + include encoding, submission, and an explicit GPU completion fence, but exclude upload, + graph compilation, index training, and the separately reported correctness readback. Medians + use {LUVS_BENCHMARK_WARMUP_ITERATIONS} warmup and {LUVS_BENCHMARK_MEASURED_ITERATIONS}{' '} + measured runs. {timestampQueries ? ' Candidate GPU time uses available per-pass timestamp queries.' : ' Candidate GPU timing is unavailable because this adapter has no timestamp queries.'}