diff --git a/docs/api-reference/experimental/README.md b/docs/api-reference/experimental/README.md index ee4fb01d26..87ce471a72 100644 --- a/docs/api-reference/experimental/README.md +++ b/docs/api-reference/experimental/README.md @@ -147,6 +147,20 @@ exact force-layout coordinates, neighborhood highlighting, stable GPU picking, d An opt-in live benchmark compares six actual CPU and WebGPU graph workloads across five graph families while reporting command encoding, completion fences, setup costs, and layout accuracy. +## GPU-Resident Dataframes + +

+ WebGPU required +

+ +[`luDF`](/docs/api-reference/experimental/ludf) adds immutable, GPU-resident dataframe queries on +top of existing `GPUTable` batches. Its optional `@luma.gl/experimental/ludf` entry point provides +nullable expressions, derived columns, categorical and global aggregation, histograms, stable +per-batch sorting, top-K selection, and bounded unique-right joins without hidden GPU submission, +source repacking, or CPU readback. The existing +[GPU Data Analysis example](/examples/experimental/gpu-data-analysis) demonstrates Arrow ingestion +and opt-in, explicitly fenced GPU-versus-CPU benchmarks. + ## GPU-resident Linked Crossfiltering

diff --git a/docs/api-reference/experimental/ludf.md b/docs/api-reference/experimental/ludf.md new file mode 100644 index 0000000000..ee27203aa1 --- /dev/null +++ b/docs/api-reference/experimental/ludf.md @@ -0,0 +1,422 @@ +import {ExperimentalDocsTabs} from '@site/src/components/docs/experimental-docs-tabs'; + +# luDF: GPU-Resident Dataframes + + + +`@luma.gl/experimental/ludf` provides immutable, visualization-oriented dataframe operations on +existing WebGPU-resident tables. Filters, derived columns, reductions, histograms, categorical +grouping, stable per-batch sorting, and bounded hash joins compile into reusable +`GPUCommandGraph` work. Source record batches, null masks, stable row identifiers, and results stay +on the GPU until an application explicitly chooses to read them. + +luDF is inspired by the GPU-resident dataframe ideas pioneered by +[NVIDIA RAPIDS cuDF](https://github.com/rapidsai/cudf). It is an independent browser-native WebGPU +implementation, not a CUDA port, a compatible cuDF API, a SQL engine, or a claim of feature parity. + +## Attribution and licensing + +We gratefully acknowledge NVIDIA and the RAPIDS contributors for pioneering GPU-resident dataframe +analytics. [NVIDIA RAPIDS cuDF](https://github.com/rapidsai/cudf) is distributed under the +[Apache License 2.0](https://github.com/rapidsai/cudf/blob/main/LICENSE). + +luDF is an independently written, [MIT-licensed](https://github.com/visgl/luma.gl/blob/master/LICENSE) +vis.gl implementation for browser-native WebGPU; it does not copy or translate cuDF source code, +including CUDA or Python implementations. It does not claim cuDF API compatibility or feature +parity, and is neither affiliated with nor endorsed by NVIDIA. + +## Try the interactive example + +The [GPU Data Analysis example](/examples/experimental/gpu-data-analysis) uploads real Apache Arrow +tables and compares GPU filtering, dense grouping, stable sorting, and unique-right joins against +CPU references. Its luDF benchmark is opt-in and separately reports upload, graph compilation, +index construction, fenced GPU execution, explicit validation readback, and CPU execution. + +## Supported data and package boundaries + +| Capability | Supported behavior | +| --- | --- | +| GPU scalar storage | Packed `float32`, `sint32`, and `uint32` columns; Arrow `Int32` maps to `sint32`. | +| Categories | Explicit adapter-owned UTF-8 dictionary labels with GPU-resident 32-bit indices. Dense grouping and joins require `uint32` indices. | +| Nullable values | Separate source-row-aligned `GPUVector<'uint32'>` validity masks. Nullable columns with unknown validity cannot be evaluated. | +| Source topology | Every original `GPURecordBatch`, including empty batches, remains independently identifiable. | +| Row identity | Stable original source-row identifiers, including caller-provided batch offsets. | +| Execution | One browser WebGPU device and caller-owned command encoding, submission, and optional readback. | + +Import the dataframe facade only from its optional subpath. Arrow-specific upload helpers belong to +`@luma.gl/arrow`; generic GPU storage remains in `@luma.gl/tables`. Neither `@luma.gl/tables` nor +luDF requires Apache Arrow as a runtime dependency. + +```ts +import {makeGPUAnalyticsTableFromArrowTable} from '@luma.gl/arrow'; +import {GPUCommandGraph} from '@luma.gl/experimental'; +import { + LuDataFrame, + and, + column, + literal, + parameter, + type LuDataFrameQueryParameters +} from '@luma.gl/experimental/ludf'; +``` + +The root `@luma.gl/experimental` entry point does not export `LuDataFrame`; applications that do +not import `/ludf` do not take on the dataframe facade. + +## Upload Arrow data or borrow an existing table + +`makeGPUAnalyticsTableFromArrowTable` uploads numeric values and dictionary indices into the +existing `GPUData`, `GPUVector`, `GPURecordBatch`, and `GPUTable` storage model. It does not require +a renderer `ShaderLayout` and preserves sliced Arrow offsets, record-batch boundaries, null counts, +and ordered dictionary metadata. + +```ts +import * as arrow from 'apache-arrow'; + +const arrowTable = arrow.tableFromArrays({ + fare: new Float32Array([12, 24, 36]), + customerId: new Uint32Array([3, 7, 9]) +}); + +const uploaded = makeGPUAnalyticsTableFromArrowTable(device, arrowTable, { + columns: ['fare', 'customerId'] +}); + +const dataframe = new LuDataFrame({...uploaded, ownership: 'owned'}); + +dataframe.schema; +dataframe.columnNames; +dataframe.numRows; +dataframe.batches; +dataframe.sourceInfo; +dataframe.column('fare'); +dataframe.validity.fare; +dataframe.dictionaries; +uploaded.nullCounts; +``` + +Each selected nullable Arrow field receives its own batch-aligned `uint32` validity vector, where +`0` means null and `1` means valid. The helper accounts for sliced bitmap offsets; dictionary labels +remain explicit CPU metadata rather than pretending arbitrary strings are GPU-native scalars. + +Applications with an existing generic GPU table can provide their own masks and dictionaries: + +```ts +const borrowed = new LuDataFrame({ + table: sourceTable, + validity: {fare: fareValidity}, + dictionaries: {category: {values: ['Local', 'Express'], ordered: false}}, + ownership: 'borrowed' +}); + +const fares = borrowed.select(['fare']); +``` + +Projection creates independently borrowed views without calling the destructive +`GPUTable.select()` operation. Source batches, backing buffers, and sibling projections remain +intact. + +## Plan expressions and filters without GPU work + +Constructing a dataframe, selecting columns, and creating query plans never allocates GPU outputs, +encodes commands, or submits work. Expressions are immutable typed trees; column names and scalar +parameters never become unchecked WGSL identifiers or source strings. + +```ts +const query = dataframe + .filter( + and( + column('fare').greaterThan(parameter('minimumFare', 10)), + column('customerId').isValid() + ) + ) + .select(['fare', 'customerId']); +``` + +Scalar expressions provide arithmetic, comparisons, `isValid()`, and `isNull()`. Compose predicates +with `and`, `or`, and `not`; use `literal(value)` for fixed numeric or boolean values and +`parameter(name, initialValue)` for values updated when encoding an already-compiled graph. + +Nullable expressions follow SQL-style three-valued logic: + +| Expression | Result | +| --- | --- | +| `false AND null` | `false` | +| `true AND null` | `null` | +| `true OR null` | `true` | +| `false OR null` | `null` | +| `NOT null` | `null` | +| `isValid(null)` / `isNull(null)` | `false` / `true` | + +A filter accepts only a valid `true` predicate. A nonempty nullable source field without an +explicit validity sidecar is rejected instead of silently treating its rows as valid. + +## Add nullable derived columns + +`withColumn` appends a new logical column, preserves existing query immutability, and propagates +the expression's null validity into a separate GPU-backed sidecar when needed: + +```ts +const adjusted = dataframe + .withColumn('adjustedFare', column('fare').multiply(literal(1.2)), { + format: 'float32' + }) + .withColumn('serviceCharge', column('adjustedFare').subtract(column('fare'))) + .filter(column('serviceCharge').greaterThan(literal(1))) + .select(['customerId', 'adjustedFare', 'serviceCharge']); +``` + +Later derived expressions may reference earlier derived columns; hidden dependencies remain +available even when the final projection excludes them. Formats are inferred from compatible +source operands, and an explicit format must match the inferred scalar format. Replacing an +existing column, implicit casts, arbitrary string values, and mixed scalar arithmetic are not +supported. + +## Compile, encode, and retain GPU-resident results + +Each query compiles into one caller-provided command graph. Encoding updates named parameters +without recompiling and records work into an application-owned command encoder: + +```ts +const graph = new GPUCommandGraph(device); +const compiled = query.compile(graph); + +const commandEncoder = device.createCommandEncoder({id: 'ludf-interaction'}); +compiled.encode(commandEncoder, {minimumFare: 25}); +device.submit(commandEncoder.finish()); + +compiled.table; +compiled.validity; +compiled.dictionaries; +compiled.selectionMask; +compiled.rowIndices; +compiled.selectedCounts; +``` + +`selectionMask` is source-aligned, `rowIndices` contains stable selected source identifiers, and +`selectedCounts` contains one GPU count per original batch. Derived values, reductions, category +groups, histograms, and joined row identifiers are also exposed as GPU-backed tables or vectors. +No luDF method submits the command encoder or performs implicit CPU readback. + +Compile each independent plan into a new `GPUCommandGraph`; a graph becomes immutable once +compiled. Re-encode the same compiled query with new parameters for repeated interactions. + +## Group dense categorical values + +Group keys must use `uint32` GPU storage. Dictionary-backed keys infer their dense group count from +the adapter-owned labels; raw `uint32` keys require an explicit `groupCount`. The following example +assumes the dataframe also contains a dictionary-backed `category` column. + +```ts +const grouped = dataframe + .filter(column('fare').greaterThan(parameter('minimumFare', 10))) + .groupBy('category') + .aggregate({ + rides: 'count', + totalFare: {sum: 'fare'}, + minimumFare: {min: 'fare'}, + maximumFare: {max: 'fare'}, + averageFare: {mean: 'fare'} + }); + +const explicitGroups = dataframe.groupBy('category', {groupCount: 4}); +``` + +Grouping preserves the category dictionary and publishes one row for every dense group, including +empty groups. Nullable keys are excluded. Count results are `uint32`; summed, minimum, maximum, +and mean values currently require `float32` input. Null, NaN, and infinite metric values do not +contribute. Empty numeric groups have an explicit invalid output mask; their sum payload is zero +and minimum, maximum, and mean payloads are NaN. + +Cross-batch grouping accumulates contributions from every original source batch without repacking +the source table. `CompiledLuDataFrameGroupedAggregation.groupCount` exposes the dense domain. + +## Compute global reductions and explicit histograms + +Global reductions support packed `float32`, `sint32`, and `uint32` metric columns: + +```ts +const totals = dataframe.aggregate({ + rows: 'count', + totalFare: {sum: 'fare'}, + minimumFare: {min: 'fare'}, + maximumFare: {max: 'fare'}, + averageFare: {mean: 'fare'} +}); + +const equalWidth = dataframe.histogram('fare', { + bins: 8, + domain: [0, 80] +}); + +const customEdges = dataframe.histogram('fare', { + edges: [0, 10, 25, 50, 100] +}); +``` + +`count` counts selected source rows and produces `uint32`. A metric's sum, minimum, and maximum +retain its input format; its mean is `float32`. Metric nulls and nonfinite floating-point values +are excluded independently, and each potentially empty metric has an explicit one-row validity +mask. Native integer sums wrap to their 32-bit representation, floating-point reductions retain +`float32` precision, and oversized row counts are rejected instead of silently overflowing. + +Histograms publish a dense GPU table of `uint32` `bin` identifiers and `count` values. Supply either +an explicit equal-width domain or 2–257 strictly ascending literal edges; automatic domains are not +supported because masked or nullable source values must not influence an inferred extent. Existing +filters, null masks, derived columns, and repeated query parameters apply before binning. + +## Sort and select top-K rows per source batch + +Numeric ordering is stable for `uint32`, `sint32`, and `float32` keys. Sorting returns GPU-resident +stable source-row identifiers rather than rewriting the source table: + +```ts +const sorted = dataframe.sortBy('fare', { + direction: 'ascending', + nulls: 'last', + nans: 'last', + algorithm: 'auto' +}); + +const highestPerBatch = dataframe.topK('fare', 10, { + direction: 'descending', + nulls: 'last' +}); + +const lowestPerBatch = dataframe.sortBy('fare').topK(10); +``` + +`sortBy` defaults to ascending order; direct `topK` defaults to descending order; calling `topK` on +an existing sorted plan preserves its established ordering. `nulls` places nulls outside all +nonnull values, while `nans` orders NaNs among the remaining nonnull floating-point values. Positive +and negative zero compare equally and retain stable source order; infinities are ordinary numeric +values. Deselected rows never enter the published selected prefix. + +Sorting and top-K are performed independently within every original source batch. There is no +implicit global cross-batch materialization or global top-K. Compiled results expose the original +table, sorted `rowIndices`, updated `selectionMask`, and one selected count per preserved batch. + +## Join or look up unique right-side keys + +luDF supports bounded, unique-right-key `uint32` inner joins and source-aligned left lookups. Left +and right tables may have different batch topologies, empty chunks, nullable keys, and explicit +original source-row offsets. The right-side hash index is built directly from its original batches; +neither side is concatenated or repacked. + +```ts +const joined = customers + .filter(column('customerId').isValid()) + .innerJoin(accounts, { + leftOn: 'customerId', + rightOn: 'accountId', + capacity: 1024, + indexCapacity: 4096, + maxProbeCount: 64 + }) + .compile(new GPUCommandGraph(device)); + +joined.rowIndices; +joined.rightRowIndices; +joined.requiredCounts; +joined.selectedCounts; +joined.overflows; +joined.indexStatistics; +joined.lookupStatistics; +joined.contractViolation; +joined.rightTable; + +const lookups = customers + .lookup(accounts, {leftOn: 'customerId', rightOn: 'accountId'}) + .compile(new GPUCommandGraph(device)); + +lookups.rowIndices; +lookups.rightRowIndices; +lookups.matchMask; +lookups.probeCounts; +lookups.indexStatistics; +lookups.contractViolation; +``` + +For an inner join, `rowIndices` and `rightRowIndices` contain paired stable source identifiers; +`selectedCounts` gives the published prefix while `requiredCounts` reports all matches before +capacity truncation. `overflows` flags insufficient output capacity per original left batch. +Lookups instead preserve source-aligned right identifiers and expose a match flag and probe count +for every left row. + +The six GPU-resident index statistic words are, in order, unique entries, duplicate keys, index +overflow, invalid keys, total probe count, and maximum probe count. A valid key equal to +`0xffffffff` is reserved and therefore invalid; nullable right rows are ignored. Duplicate right +keys, reserved valid keys, or incomplete hash-index construction set `contractViolation` and +suppress all published matches instead of returning ambiguous results. Dictionary-encoded keys +must have identical labels and ordering on both sides. + +Many-to-many joins, outer joins, multi-key joins, string-key hashing, and CPU-side result +materialization are intentionally unsupported. + +## Share GPU outputs with rendering and LuxFilter + +Visualization shaders can consume `compiled.selectionMask`, `compiled.rowIndices`, aggregated GPU +columns, and joined source-row identifiers directly as storage or vertex buffers. Import the same +existing table vectors into a separate [`LuxFilter`](/docs/api-reference/experimental/luxfilter) +graph when an application needs linked ranges, brushes, histograms, or visibility views: + +```ts +const interactionGraph = new GPUCommandGraph(device); +const fare = interactionGraph.importGPUVector('fare', dataframe.table.gpuVectors.fare); +const category = interactionGraph.importGPUVector('category', dataframe.table.gpuVectors.category); + +const fareValidity = dataframe.validity.fare + ? interactionGraph.importGPUVector('fare-validity', dataframe.validity.fare) + : undefined; +``` + +LuxFilter and other lower-level consumers do not automatically interpret luDF's nullable sidecars; +combine the explicit validity mask into their selection before treating nullable values as valid. +Sharing vectors does not transfer ownership, merge source batches, or require CPU row readback. + +## Measure GPU work without hiding synchronization + +The opt-in [GPU Data Analysis benchmark](/examples/experimental/gpu-data-analysis) reports separate +durations for: + +1. Uploading Arrow columns, explicit validity masks, and dictionaries. +2. Compiling caller-owned luDF command graphs. +3. Building a standalone right-side hash index equivalent to the join's index. +4. Encoding and executing GPU filtering, grouping, sorting, and joining. +5. Explicitly reading only the outputs required for validation. +6. Computing the corresponding CPU reference results. + +GPU durations wait for `device.createFence().signaled` rather than measuring command submission +alone. The separately reported index-build phase is an equivalent standalone measurement; the +complete join execution still includes construction of its own index. Timings must not be added or +subtracted as if those duplicated builds were one disjoint operation. + +Validation compares GPU results with CPU references for filtering, grouped aggregation, stable +sorting, and unique-right joins. This benchmark's bounded result readback is explicit and optional; +ordinary luDF query execution never reads source rows or results back implicitly. + +## Ownership, fallback, and intentional limits + +`ownership: 'borrowed'` is the default: destroying a dataframe or its projections does not destroy +the caller's original table or validity vectors. With `ownership: 'owned'`, the original table and +provided validity sidecars are released only after every borrowed projection and compiled query +has released its shared source lease: + +```ts +const owner = new LuDataFrame({...uploaded, ownership: 'owned'}); +const retained = owner.filter(column('fare').isValid()).compile( + new GPUCommandGraph(device) +); + +owner.destroy(); +retained.destroy(); +``` + +Always call `destroy()` on compiled queries and owned frames when they are no longer needed; calls +are idempotent. Applications without an available WebGPU adapter must offer their own CPU path or +display an unsupported-device state. luDF does not transparently switch execution backends. + +Native GPU `float64` and `int64`, arbitrary GPU strings, distributed or multi-GPU execution, global +cross-batch sorting, full SQL semantics, and complete cuDF compatibility are outside the supported +scope. See [GPU Primitives and Command Graphs](/docs/api-reference/experimental/gpu-primitives) for +the underlying WebGPU execution infrastructure. diff --git a/docs/table-of-contents.json b/docs/table-of-contents.json index ddc2e5019d..bd91a4fae9 100644 --- a/docs/table-of-contents.json +++ b/docs/table-of-contents.json @@ -213,6 +213,7 @@ "api-reference/experimental/luraster/README", "api-reference/experimental/luproj", "api-reference/experimental/lugraph", + "api-reference/experimental/ludf", "api-reference/experimental/luxfilter", "api-reference/experimental/lutrace", "api-reference/experimental/g-buffer", @@ -347,6 +348,7 @@ "api-reference/experimental/luraster/README", "api-reference/experimental/luproj", "api-reference/experimental/lugraph", + "api-reference/experimental/ludf", "api-reference/experimental/luxfilter", "api-reference/experimental/lutrace", "api-reference/experimental/g-buffer", diff --git a/docs/whats-new.md b/docs/whats-new.md index e12f4bc4f4..f21f767d2e 100644 --- a/docs/whats-new.md +++ b/docs/whats-new.md @@ -54,6 +54,11 @@ Target Release Date: Q3, 2026 - **Shared interactive GPU-accelerated ray tracing** - `RayTracingSceneRenderer` composes world-space instance bounds, dirty-only Morton-sorted object/instance TLAS construction, retained-permutation transform refits, topology-only Morton-sorted per-mesh triangle BLAS construction, nearest-hit traversal, bounded direct-light shadows, adaptive internal resolution, interleaved frame-budget coverage, stable-identity temporal reprojection, progressive accumulation, and upsampled HDR presentation through WebGPU compute/command graphs. Frame pacing uses ordinary animation intervals, the tracing pass uses exactly eight storage buffers, every TLAS or BLAS construction pass stays within the eight-storage-buffer WebGPU CORE limit, and command submission stays application-owned. - **[Generated physical lighting environments](/docs/api-reference/experimental/pbr-environment)** - `PBREnvironmentGenerator` and `preparePBREnvironment()` integrate equirectangular source textures into GGX-prefiltered specular cubemap mip chains, diffuse irradiance cubemaps, and split-sum BRDF lookup textures on both WebGL and WebGPU. - **Scene-color transmission and volume attenuation** - The shared forward renderer captures opaque scene color automatically for transmissive surfaces, then applies screen-space refraction, roughness, Fresnel response, index of refraction, thickness, and Beer-Lambert attenuation while preserving physically opaque output. +- **[`luDF` GPU-resident dataframes](/docs/api-reference/experimental/ludf)** - The optional + `@luma.gl/experimental/ludf` entry point adds immutable nullable expressions, derived columns, + dense categorical and global aggregations, explicit-domain histograms, stable per-batch sorting + and top-K, and bounded unique-right joins over existing `GPUTable` batches. Applications retain + ownership of GPU command submission, source lifetimes, and optional result readback. - **`HTMLTexture`** - Experimental copied texture binding source copies HTML-in-Canvas DOM subtrees into GPU textures while the browser API is still experimental. - **OIT resolve pipelines** - A-buffer and weighted-blended order-independent transparency now resolve captured fragments through exported `ShaderPassPipeline` factories, allowing WBOIT to @@ -109,6 +114,9 @@ Target Release Date: Q3, 2026 **@luma.gl/arrow** NEW MODULE +- **Renderer-independent Arrow analytics upload** - `makeGPUAnalyticsTableFromArrowTable()` + preserves Arrow record batches, sliced validity bitmaps, nullable column masks, and explicit + dictionary labels while uploading portable scalar columns without requiring a `ShaderLayout`. - **Arrow shader layouts** - `getArrowBufferLayout()` maps Arrow scalar and `FixedSizeList` columns to shader attribute formats from a shader-first layout, including direct `arrow.Vector` sources and Arrow table path mappings. - **Arrow GPU adapters** - Arrow factories, append helpers, and readback helpers bridge Apache Arrow inputs into `@luma.gl/tables` objects and preserve chunked UTF-8 GPU vector input for text workflows. - **Variable-length Arrow attribute lists** - `GPUVector` can retain chunked nested list columns whose elements contain one to four numeric components, covering scalar streams plus tuple-style data such as XY, XYZ, and XYZM coordinates for future path-rendering workflows. diff --git a/examples/experimental/gpu-data-analysis/src/app-shell.ts b/examples/experimental/gpu-data-analysis/src/app-shell.ts index 17f21c99e3..78d1c71df9 100644 --- a/examples/experimental/gpu-data-analysis/src/app-shell.ts +++ b/examples/experimental/gpu-data-analysis/src/app-shell.ts @@ -124,6 +124,33 @@ export const GPU_DATA_ANALYSIS_TEMPLATE = `

+
+
+
+

ARROW / CPU + GPU BENCHMARK

+

Measure the complete dataframe pipeline.

+

+ Compare filtering, grouped aggregation, stable top-K, and hash joins over genuine + nullable, dictionary-encoded Arrow batches. Nothing runs until you ask. +

+
+ +
+

+ Filter, group, stable top-K, and unique-key joins remain GPU-resident. +

+
+
+

DISTRIBUTION / HISTOGRAM + CDF

@@ -413,6 +440,33 @@ export const GPU_DATA_ANALYSIS_STYLES = ` .query-status[data-state="verified"] { color: var(--green); } + .benchmark-lab { + margin-top: 23px; + padding: 23px; + border: 1px solid rgba(115, 170, 250, 0.26); + border-radius: 12px; + background: linear-gradient(135deg, rgba(16, 24, 40, 0.95), rgba(10, 18, 29, 0.98)); + } + + .benchmark-heading { + display: flex; + align-items: center; + justify-content: space-between; + gap: 20px; + } + + .benchmark-heading .eyebrow { color: var(--blue); } + .benchmark-heading h2 { margin: 0 0 7px; font-size: 21px; letter-spacing: -0.4px; } + .benchmark-heading > div > p:last-child { max-width: 680px; margin: 0; color: var(--muted); } + .analysis-example .benchmark-button { width: 230px; border-color: rgba(115, 170, 250, 0.5); color: var(--blue); } + .benchmark-status { margin: 16px 0 0; color: var(--muted); font-size: 11px; } + .benchmark-results table { width: 100%; margin-top: 13px; border-collapse: collapse; } + .benchmark-results th, .benchmark-results td { padding: 10px; border-bottom: 1px solid var(--border); text-align: left; } + .benchmark-results thead th { color: var(--muted); font-size: 10px; letter-spacing: 0.5px; } + .benchmark-results tbody th { color: var(--text); font-size: 12px; font-weight: 500; } + .benchmark-results td { color: var(--green); text-align: right; font-variant-numeric: tabular-nums; } + .benchmark-results[data-state="error"] { color: #ff8585; } + .visualizations { display: grid; grid-template-columns: 1fr 1fr; @@ -485,6 +539,9 @@ export const GPU_DATA_ANALYSIS_STYLES = ` .pipeline > span:last-child { border-bottom: 0; } .pipeline strong { display: inline; margin-left: 7px; } .dataframe-lab { padding: 15px; } + .benchmark-lab { padding: 15px; } + .benchmark-heading { align-items: flex-start; flex-direction: column; } + .analysis-example .benchmark-button { width: 100%; } .visualizations { grid-template-columns: 1fr; } .heatmap-card { grid-column: auto; } .expression-code p { font-size: 10px; } diff --git a/examples/experimental/gpu-data-analysis/src/app.ts b/examples/experimental/gpu-data-analysis/src/app.ts index 76c19caac3..24d885d290 100644 --- a/examples/experimental/gpu-data-analysis/src/app.ts +++ b/examples/experimental/gpu-data-analysis/src/app.ts @@ -25,6 +25,7 @@ import {GPURecordBatch, GPUTable, type GPUVector} from '@luma.gl/tables'; import {webgpuAdapter} from '@luma.gl/webgpu'; import * as arrow from 'apache-arrow'; import {GPU_DATA_ANALYSIS_STYLES, GPU_DATA_ANALYSIS_TEMPLATE} from './app-shell'; +import {runLuDataFrameBenchmark, type LuDataFrameBenchmarkResult} from './ludf-benchmark'; const APP_ID = 'gpu-data-analysis-app'; const STYLE_ID = 'gpu-data-analysis-style'; @@ -61,6 +62,9 @@ type ExampleElements = { luDataFrameRun: HTMLButtonElement; luDataFrameSelected: HTMLElement; luDataFrameThreshold: HTMLInputElement; + ludfBenchmark: HTMLButtonElement; + ludfBenchmarkResults: HTMLElement; + ludfBenchmarkStatus: HTMLElement; nodes: HTMLElement; reuse: HTMLElement; run: HTMLButtonElement; @@ -86,6 +90,7 @@ class GPUDataAnalysisExample { private readonly elements: ExampleElements; private device: Device | null = null; private resources: ExampleResources | null = null; + private benchmarkController: AbortController | null = null; private destroyed = false; private hasRunLuDataFrameDemo = false; private runVersion = 0; @@ -96,6 +101,7 @@ class GPUDataAnalysisExample { private readonly handleLuDataFrameChange = (): void => { if (this.hasRunLuDataFrameDemo) void this.runLuDataFrameDemo(); }; + private readonly handleLuDataFrameBenchmark = (): void => void this.runBenchmark(); constructor(root: HTMLElement) { this.elements = getElements(root); @@ -115,6 +121,7 @@ class GPUDataAnalysisExample { control.addEventListener('change', this.handleLuDataFrameChange); } this.updateLuDataFrameExpression(); + this.elements.ludfBenchmark.addEventListener('click', this.handleLuDataFrameBenchmark); } async initialize(): Promise { @@ -130,20 +137,26 @@ class GPUDataAnalysisExample { } this.device = device; await this.run(); + if (!this.destroyed) { + this.elements.ludfBenchmark.disabled = false; + } } catch (error) { this.setStatus(getErrorMessage(error), true); + this.elements.ludfBenchmarkStatus.textContent = 'WebGPU is unavailable on this device.'; } } destroy(): void { if (this.destroyed) return; this.destroyed = true; + this.benchmarkController?.abort(); this.elements.run.removeEventListener('click', this.handleRun); this.elements.luDataFrameRun.removeEventListener('click', this.handleLuDataFrameRun); for (const control of this.getLuDataFrameControls()) { control.removeEventListener('input', this.handleLuDataFrameInput); control.removeEventListener('change', this.handleLuDataFrameChange); } + this.elements.ludfBenchmark.removeEventListener('click', this.handleLuDataFrameBenchmark); for (const element of [ this.elements.dataset, this.elements.bins, @@ -718,6 +731,38 @@ class GPUDataAnalysisExample { this.elements.status.textContent = message; this.elements.status.dataset.state = error ? 'error' : 'ok'; } + + /** Executes optional, bounded CPU/GPU comparisons only after an explicit user request. */ + private async runBenchmark(): Promise { + if (!this.device || this.destroyed || this.benchmarkController) return; + const controller = new AbortController(); + this.benchmarkController = controller; + this.elements.ludfBenchmark.disabled = true; + this.elements.ludfBenchmarkResults.dataset.state = 'running'; + this.elements.ludfBenchmarkResults.dataset.validated = 'false'; + this.elements.ludfBenchmarkStatus.textContent = + 'Uploading a nullable Arrow dictionary dataset and validating GPU dataframe queries...'; + + try { + const result = await runLuDataFrameBenchmark(this.device, { + rowCount: 384, + signal: controller.signal + }); + if (this.destroyed || controller.signal.aborted) return; + renderLuDataFrameBenchmark(this.elements, result); + } catch (error) { + if (this.destroyed || controller.signal.aborted) return; + this.elements.ludfBenchmarkResults.dataset.state = 'error'; + this.elements.ludfBenchmarkStatus.textContent = getErrorMessage(error); + } finally { + if (this.benchmarkController === controller) { + this.benchmarkController = null; + } + if (!this.destroyed) { + this.elements.ludfBenchmark.disabled = false; + } + } + } } function makeDataset(length: number): { @@ -1001,6 +1046,9 @@ function getElements(root: HTMLElement): ExampleElements { luDataFrameRun: get('[data-ludf-run]'), luDataFrameSelected: get('[data-ludf-selected]'), luDataFrameThreshold: get('[data-ludf-threshold]'), + ludfBenchmark: get('[data-ludf-benchmark]'), + ludfBenchmarkResults: get('[data-ludf-benchmark-phases]'), + ludfBenchmarkStatus: get('[data-ludf-benchmark-status]'), nodes: get('[data-nodes]'), reuse: get('[data-reuse]'), run: get('[data-run]'), @@ -1017,6 +1065,33 @@ function ensureStyles(): void { document.head.appendChild(style); } +/** Renders only bounded, independently fenced phase timings and explicit CPU-oracle validation. */ +function renderLuDataFrameBenchmark( + elements: ExampleElements, + result: LuDataFrameBenchmarkResult +): void { + const timingRows = [ + ['upload', 'Arrow upload', result.timings.uploadMilliseconds], + ['compile', 'Graph compilation', result.timings.compileMilliseconds], + ['index', 'Standalone hash-index build', result.timings.indexMilliseconds], + ['execution', 'Fenced WebGPU execution', result.timings.executionMilliseconds], + ['readback', 'Bounded result readback', result.timings.readbackMilliseconds], + ['cpu', 'Equivalent CPU reference', result.timings.cpuMilliseconds] + ] as const; + elements.ludfBenchmarkResults.innerHTML = `${timingRows + .map( + ([phase, label, milliseconds]) => + `` + ) + .join('')}
PhaseMilliseconds
${label}${milliseconds.toFixed(2)}
`; + const validated = Object.values(result.validation).every(Boolean); + elements.ludfBenchmarkResults.dataset.state = validated ? 'ok' : 'error'; + elements.ludfBenchmarkResults.dataset.validated = String(validated); + elements.ludfBenchmarkStatus.textContent = validated + ? `${result.rowCount.toLocaleString()} Arrow rows · batches ${result.batchRowCounts.join(' / ')} · filter, grouping, sorting, and joins match the CPU reference · ${result.readbackBytes.toLocaleString()} summary bytes read` + : 'GPU dataframe results did not match their equivalent CPU reference.'; +} + function getErrorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error); } diff --git a/examples/experimental/gpu-data-analysis/src/ludf-benchmark.ts b/examples/experimental/gpu-data-analysis/src/ludf-benchmark.ts new file mode 100644 index 0000000000..916823fa08 --- /dev/null +++ b/examples/experimental/gpu-data-analysis/src/ludf-benchmark.ts @@ -0,0 +1,749 @@ +// luma.gl +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: Copyright (c) vis.gl contributors + +import {makeGPUAnalyticsTableFromArrowTable} from '@luma.gl/arrow'; +import {Buffer, type Device} from '@luma.gl/core'; +import { + GPUBatchHashIndex, + GPUCommandGraph, + GPU_HASH_INDEX_STATISTICS_LENGTH, + type CompiledGPUCommandGraph +} from '@luma.gl/experimental'; +import { + LuDataFrame, + and, + column, + literal, + type CompiledLuDataFrameGroupedAggregation, + type CompiledLuDataFrameJoin, + type CompiledLuDataFrameQuery, + type CompiledLuDataFrameSort, + type LuDataFrameQueryParameters +} from '@luma.gl/experimental/ludf'; +import {GPUVector, type GPUData} from '@luma.gl/tables'; +import * as arrow from 'apache-arrow'; + +const DEFAULT_ROW_COUNT = 512; +const MAXIMUM_ROW_COUNT = 4096; +const SLICE_OFFSET = 9; +const CATEGORY_LABELS = ['North', 'East', 'South', 'West'] as const; +const MINIMUM_FARE = 20; +const DRIVER_TIP = 2.5; +const TOP_K_LIMIT = 5; +const JOIN_CAPACITY = 8; +const INDEX_CAPACITY = 8; +const UINT32_BYTE_LENGTH = Uint32Array.BYTES_PER_ELEMENT; + +type BenchmarkDictionary = arrow.Dictionary; + +type BenchmarkArrowColumns = { + fare: arrow.Float32; + category: BenchmarkDictionary; + tripId: arrow.Uint32; +}; + +type BenchmarkRightArrowColumns = { + category: BenchmarkDictionary; + weight: arrow.Float32; +}; + +type BenchmarkColumns = { + fare: 'float32'; + category: 'uint32'; + tripId: 'uint32'; +}; + +type BenchmarkDerivedColumns = BenchmarkColumns & {adjustedFare: 'float32'}; + +type BenchmarkRightColumns = { + category: 'uint32'; + weight: 'float32'; +}; + +type BenchmarkDataset = { + left: arrow.Table; + right: arrow.Table; + rows: readonly BenchmarkSourceRow[]; + batchRowCounts: readonly number[]; +}; + +type BenchmarkSourceRow = { + rowId: number; + batchIndex: number; + fare: number | null; + category: number | null; +}; + +type BenchmarkReference = { + filterCounts: number[]; + groupCounts: number[]; + groupSums: number[]; + topKRowIds: number[][]; + joinRequiredCounts: number[]; + joinLeftRowIds: number[][]; + joinRightRowIds: number[][]; +}; + +type BenchmarkGraphs = { + filter: CompiledLuDataFrameQuery; + groups: CompiledLuDataFrameGroupedAggregation<{ + category: 'uint32'; + count: 'uint32'; + totalAdjustedFare: 'float32'; + }>; + sorting: CompiledLuDataFrameSort; + join: CompiledLuDataFrameJoin; + index: CompiledGPUCommandGraph; +}; + +/** Explicit timing phases recorded from real local Arrow, CPU, and fence-synchronized GPU work. */ +export type LuDataFrameBenchmarkTimings = { + /** Construction and completed upload of both original Arrow analytics tables. */ + uploadMilliseconds: number; + /** Compilation of four dataframe graphs and one separate equivalent right-index graph. */ + compileMilliseconds: number; + /** Independently submitted and fence-synchronized equivalent right-index construction. */ + indexMilliseconds: number; + /** Four dataframe encodings, submissions, and completed GPU execution fences. */ + executionMilliseconds: number; + /** Explicit bounded output readback required only for correctness verification. */ + readbackMilliseconds: number; + /** Equivalent JavaScript filter, grouping, batch-local top-K, and bounded join work. */ + cpuMilliseconds: number; +}; + +/** Small independently verified outputs; full Arrow columns are never read back from the GPU. */ +export type LuDataFrameBenchmarkSummaries = { + filterCount: number; + groupCounts: number[]; + topKRowIds: number[][]; + joinCounts: number[]; + joinLeftRowIds: number[][]; + joinRightRowIds: number[][]; +}; + +/** Completed opt-in Arrow-to-luDF benchmark, measured on the caller's actual WebGPU device. */ +export type LuDataFrameBenchmarkResult = { + rowCount: number; + batchRowCounts: number[]; + timings: LuDataFrameBenchmarkTimings; + validation: { + filter: boolean; + groups: boolean; + sorting: boolean; + join: boolean; + }; + summaries: LuDataFrameBenchmarkSummaries; + readbackBytes: number; +}; + +/** Runs real Arrow uploads and correctness-gated dataframe workloads only when explicitly invoked. */ +export async function runLuDataFrameBenchmark( + device: Device, + {rowCount = DEFAULT_ROW_COUNT, signal}: {rowCount?: number; signal?: AbortSignal} = {} +): Promise { + if (device.type !== 'webgpu') { + throw new Error('The luDF benchmark requires a WebGPU device'); + } + if (!Number.isSafeInteger(rowCount) || rowCount < 2 || rowCount > MAXIMUM_ROW_COUNT) { + throw new Error(`The luDF benchmark requires between 2 and ${MAXIMUM_ROW_COUNT} rows`); + } + signal?.throwIfAborted(); + + const dataset = createBenchmarkDataset(rowCount); + const cpuStarted = performance.now(); + const reference = createBenchmarkReference(dataset.rows, dataset.batchRowCounts); + const cpuMilliseconds = performance.now() - cpuStarted; + + const ownedBuffers: Buffer[] = []; + let left: LuDataFrame | undefined; + let right: LuDataFrame | undefined; + let graphs: BenchmarkGraphs | undefined; + try { + const uploadStarted = performance.now(); + const leftUpload = makeGPUAnalyticsTableFromArrowTable(device, dataset.left); + try { + left = new LuDataFrame({...leftUpload, ownership: 'owned'}); + } catch (error) { + leftUpload.table.destroy(); + for (const validity of Object.values(leftUpload.validity)) validity?.destroy(); + throw error; + } + const rightUpload = makeGPUAnalyticsTableFromArrowTable(device, dataset.right); + try { + right = new LuDataFrame({...rightUpload, ownership: 'owned'}); + } catch (error) { + rightUpload.table.destroy(); + for (const validity of Object.values(rightUpload.validity)) validity?.destroy(); + throw error; + } + await waitForBenchmarkGPU(device, signal); + const uploadMilliseconds = performance.now() - uploadStarted; + signal?.throwIfAborted(); + + const compileStarted = performance.now(); + graphs = compileBenchmarkGraphs(device, left, right, ownedBuffers); + const compileMilliseconds = performance.now() - compileStarted; + signal?.throwIfAborted(); + + const indexMilliseconds = await executeBenchmarkGraph( + device, + graphs.index, + 'ludf-benchmark-standalone-index', + signal + ); + let executionMilliseconds = 0; + for (const [name, graph] of [ + ['filter', graphs.filter], + ['groups', graphs.groups], + ['sorting', graphs.sorting], + ['join', graphs.join] + ] as const) { + executionMilliseconds += await executeBenchmarkGraph( + device, + graph, + `ludf-benchmark-${name}`, + signal + ); + } + + const readbackStarted = performance.now(); + const bytes = {value: 0}; + const filterCounts = await readBenchmarkScalars(graphs.filter.selectedCounts, bytes, signal); + const groupCounts = await readBenchmarkUint32( + graphs.groups.table.gpuVectors.count.data[0], + CATEGORY_LABELS.length, + bytes, + signal + ); + const groupSums = await readBenchmarkFloat32( + graphs.groups.table.gpuVectors.totalAdjustedFare.data[0], + CATEGORY_LABELS.length, + bytes, + signal + ); + const topKCounts = await readBenchmarkScalars(graphs.sorting.selectedCounts, bytes, signal); + const topKRowIds = await readBenchmarkPrefixes( + graphs.sorting.rowIndices, + topKCounts, + bytes, + signal + ); + const joinCounts = await readBenchmarkScalars(graphs.join.selectedCounts, bytes, signal); + const requiredCounts = await readBenchmarkScalars(graphs.join.requiredCounts, bytes, signal); + const overflows = await readBenchmarkScalars(graphs.join.overflows, bytes, signal); + const joinLeftRowIds = await readBenchmarkPrefixes( + graphs.join.rowIndices, + joinCounts, + bytes, + signal + ); + const joinRightRowIds = await readBenchmarkPrefixes( + graphs.join.rightRowIndices, + joinCounts, + bytes, + signal + ); + const contractViolation = await readBenchmarkUint32( + graphs.join.contractViolation.data[0], + 1, + bytes, + signal + ); + const readbackMilliseconds = performance.now() - readbackStarted; + + const filter = compareNumberArrays(filterCounts, reference.filterCounts); + const groups = + compareNumberArrays(groupCounts, reference.groupCounts) && + groupSums.every((sum, index) => approximatelyEqualBenchmark(sum, reference.groupSums[index])); + const sorting = compareNestedNumberArrays(topKRowIds, reference.topKRowIds); + const join = + contractViolation[0] === 0 && + compareNumberArrays(requiredCounts, reference.joinRequiredCounts) && + overflows.every( + (overflow, index) => + overflow === Number(reference.joinRequiredCounts[index] > JOIN_CAPACITY) + ) && + compareNestedNumberArrays(joinLeftRowIds, reference.joinLeftRowIds) && + compareNestedNumberArrays(joinRightRowIds, reference.joinRightRowIds); + + if (!filter || !groups || !sorting || !join) { + throw new Error('The luDF benchmark GPU outputs do not match the shared CPU reference'); + } + signal?.throwIfAborted(); + return { + rowCount, + batchRowCounts: [...dataset.batchRowCounts], + timings: { + uploadMilliseconds, + compileMilliseconds, + indexMilliseconds, + executionMilliseconds, + readbackMilliseconds, + cpuMilliseconds + }, + validation: {filter, groups, sorting, join}, + summaries: { + filterCount: filterCounts.reduce((total, count) => total + count, 0), + groupCounts, + topKRowIds, + joinCounts, + joinLeftRowIds, + joinRightRowIds + }, + readbackBytes: bytes.value + }; + } finally { + if (graphs) { + graphs.filter.destroy(); + graphs.groups.destroy(); + graphs.sorting.destroy(); + graphs.join.destroy(); + graphs.index.destroy(); + } + left?.destroy(); + right?.destroy(); + for (const buffer of ownedBuffers) buffer.destroy(); + } +} + +/** Constructs sliced nullable Arrow columns and dictionary-compatible independent right batches. */ +function createBenchmarkDataset(rowCount: number): BenchmarkDataset { + const totalRows = rowCount + SLICE_OFFSET; + const fares: (number | null)[] = []; + const categories = new Uint32Array(totalRows); + const tripIds = new Uint32Array(totalRows); + const categoryBitmap = new Uint8Array(Math.ceil(totalRows / 8)); + let categoryNullCount = 0; + + for (let index = 0; index < totalRows; index++) { + fares.push(index % 13 === 0 ? null : Math.fround(((index * 37) % 121) - 30 + (index % 7) / 10)); + categories[index] = index % CATEGORY_LABELS.length; + tripIds[index] = index - SLICE_OFFSET >= 0 ? index - SLICE_OFFSET : 0; + if (index % 11 === 0) { + categoryNullCount++; + } else { + categoryBitmap[index >> 3] |= 1 << (index & 7); + } + } + + const dictionaryType = new arrow.Dictionary(new arrow.Utf8(), new arrow.Uint32(), 41, true); + const dictionary = arrow.vectorFromArray([...CATEGORY_LABELS], new arrow.Utf8()); + const categoryData = arrow.makeData({ + type: dictionaryType, + length: totalRows, + data: categories, + nullBitmap: categoryBitmap, + nullCount: categoryNullCount, + dictionary + }); + const categoryVector = new arrow.Vector([categoryData]); + const fareVector = arrow.vectorFromArray(fares, new arrow.Float32()); + const tripIdVector = arrow.makeVector(tripIds); + const fields = [ + new arrow.Field('fare', new arrow.Float32(), true, new Map([['unit', 'USD']])), + new arrow.Field('category', dictionaryType, true), + new arrow.Field('tripId', new arrow.Uint32(), false) + ]; + const schema = new arrow.Schema( + fields, + new Map([['dataset', 'arrow-ludf-taxi']]) + ); + const midpoint = Math.floor(rowCount / 2); + const rowRanges: readonly [number, number][] = [ + [SLICE_OFFSET, SLICE_OFFSET + midpoint], + [SLICE_OFFSET + midpoint, SLICE_OFFSET + midpoint], + [SLICE_OFFSET + midpoint, SLICE_OFFSET + rowCount] + ]; + const batches = rowRanges.map(([start, end], batchIndex) => { + const batchSchema = new arrow.Schema( + fields, + new Map([['sourceBatch', String(batchIndex)]]) + ); + return new arrow.RecordBatch( + batchSchema, + arrow.makeData({ + type: new arrow.Struct(batchSchema.fields), + length: end - start, + children: [ + fareVector.slice(start, end).data[0], + categoryVector.slice(start, end).data[0], + tripIdVector.slice(start, end).data[0] + ] + }) + ); + }); + + const rightFields = [ + new arrow.Field('category', dictionaryType, false), + new arrow.Field('weight', new arrow.Float32(), false) + ]; + const rightSchema = new arrow.Schema( + rightFields, + new Map([['dataset', 'arrow-ludf-categories']]) + ); + const rightCategories = new arrow.Vector([ + arrow.makeData({ + type: dictionaryType, + length: CATEGORY_LABELS.length, + data: Uint32Array.from(CATEGORY_LABELS, (_, index) => index), + dictionary + }) + ]); + const rightWeights = arrow.makeVector(Float32Array.from([1, 1.5, 2, 2.5])); + const rightRanges: readonly [number, number][] = [ + [0, 2], + [2, 2], + [2, 4] + ]; + const rightBatches = rightRanges.map(([start, end], batchIndex) => { + const batchSchema = new arrow.Schema( + rightFields, + new Map([['sourceBatch', String(batchIndex)]]) + ); + return new arrow.RecordBatch( + batchSchema, + arrow.makeData({ + type: new arrow.Struct(batchSchema.fields), + length: end - start, + children: [ + rightCategories.slice(start, end).data[0], + rightWeights.slice(start, end).data[0] + ] + }) + ); + }); + + const rows: BenchmarkSourceRow[] = []; + for (let rowId = 0; rowId < rowCount; rowId++) { + const sourceIndex = rowId + SLICE_OFFSET; + rows.push({ + rowId, + batchIndex: rowId < midpoint ? 0 : 2, + fare: fares[sourceIndex], + category: sourceIndex % 11 === 0 ? null : categories[sourceIndex] + }); + } + return { + left: new arrow.Table(schema, batches), + right: new arrow.Table(rightSchema, rightBatches), + rows, + batchRowCounts: [midpoint, 0, rowCount - midpoint] + }; +} + +/** Computes exact source-batch-aware CPU oracles for every independent GPU workload. */ +function createBenchmarkReference( + rows: readonly BenchmarkSourceRow[], + batchRowCounts: readonly number[] +): BenchmarkReference { + const filterCounts = batchRowCounts.map(() => 0); + const groupCounts = CATEGORY_LABELS.map(() => 0); + const groupSums = CATEGORY_LABELS.map(() => 0); + const selectedByBatch = batchRowCounts.map( + () => [] as Array<{rowId: number; adjusted: number; category: number}> + ); + + for (const row of rows) { + if (row.fare === null || row.fare <= MINIMUM_FARE || row.category === null) { + continue; + } + const adjusted = Math.fround(row.fare + DRIVER_TIP); + filterCounts[row.batchIndex]++; + groupCounts[row.category]++; + groupSums[row.category] += adjusted; + selectedByBatch[row.batchIndex].push({rowId: row.rowId, adjusted, category: row.category}); + } + + const topKRowIds = selectedByBatch.map(values => + [...values] + .sort((left, right) => right.adjusted - left.adjusted || left.rowId - right.rowId) + .slice(0, TOP_K_LIMIT) + .map(row => row.rowId) + ); + const joinRequiredCounts = selectedByBatch.map(values => values.length); + const joinLeftRowIds = selectedByBatch.map(values => + values.slice(0, JOIN_CAPACITY).map(row => row.rowId) + ); + const joinRightRowIds = selectedByBatch.map(values => + values.slice(0, JOIN_CAPACITY).map(row => row.category) + ); + + return { + filterCounts, + groupCounts, + groupSums, + topKRowIds, + joinRequiredCounts, + joinLeftRowIds, + joinRightRowIds + }; +} + +/** Compiles four reusable dataframe workloads plus one truthful, standalone equivalent index. */ +function compileBenchmarkGraphs( + device: Device, + left: LuDataFrame, + right: LuDataFrame, + ownedBuffers: Buffer[] +): BenchmarkGraphs { + const query = left + .withColumn('adjustedFare', column('fare').add(literal(DRIVER_TIP)), {format: 'float32'}) + .filter(and(column('fare').greaterThan(literal(MINIMUM_FARE)), column('category').isValid())); + + const filter = query.compile( + new GPUCommandGraph(device, {id: 'ludf-benchmark-filter'}) + ); + let groups: BenchmarkGraphs['groups'] | undefined; + let sorting: BenchmarkGraphs['sorting'] | undefined; + let join: BenchmarkGraphs['join'] | undefined; + let index: BenchmarkGraphs['index'] | undefined; + try { + groups = query + .groupBy('category') + .aggregate({count: 'count', totalAdjustedFare: {sum: 'adjustedFare'}}) + .compile(new GPUCommandGraph(device, {id: 'ludf-benchmark-groups'})); + sorting = query + .topK('adjustedFare', TOP_K_LIMIT) + .compile(new GPUCommandGraph(device, {id: 'ludf-benchmark-sorting'})); + join = query + .innerJoin(right, {leftOn: 'category', rightOn: 'category', capacity: JOIN_CAPACITY}) + .compile(new GPUCommandGraph(device, {id: 'ludf-benchmark-join'})); + index = compileStandaloneBenchmarkIndex(device, right, ownedBuffers); + return {filter, groups, sorting, join, index}; + } catch (error) { + filter.destroy(); + groups?.destroy(); + sorting?.destroy(); + join?.destroy(); + index?.destroy(); + throw error; + } +} + +/** Builds exactly the same chunk-preserving unique-right hash index as the joined dataframe. */ +function compileStandaloneBenchmarkIndex( + device: Device, + right: LuDataFrame, + ownedBuffers: Buffer[] +): CompiledGPUCommandGraph { + const graph = new GPUCommandGraph(device, { + id: 'ludf-benchmark-equivalent-index' + }); + const category = right.column('category'); + if (!(category instanceof GPUVector)) { + throw new Error('The luDF benchmark right category must be a GPU vector'); + } + const keys = graph.importGPUVector('ludf-index-source', category); + const keyBuffer = createBenchmarkIndexBuffer( + device, + 'ludf-index-keys', + INDEX_CAPACITY, + ownedBuffers + ); + const valueBuffer = createBenchmarkIndexBuffer( + device, + 'ludf-index-values', + INDEX_CAPACITY, + ownedBuffers + ); + const statisticsBuffer = createBenchmarkIndexBuffer( + device, + 'ludf-index-statistics', + GPU_HASH_INDEX_STATISTICS_LENGTH, + ownedBuffers + ); + const tableKeys = importBenchmarkUint32( + graph, + 'ludf-index-table-keys', + keyBuffer, + INDEX_CAPACITY + ); + const tableValues = importBenchmarkUint32( + graph, + 'ludf-index-table-values', + valueBuffer, + INDEX_CAPACITY + ); + const statistics = importBenchmarkUint32( + graph, + 'ludf-index-build-statistics', + statisticsBuffer, + GPU_HASH_INDEX_STATISTICS_LENGTH + ); + const firstValues = right.batches.map(batch => batch.sourceInfo?.sourceRowIndexOffset ?? 0); + new GPUBatchHashIndex({ + id: 'ludf-equivalent-right-index', + keys, + firstValues, + tableKeys, + tableValues, + statistics, + maxProbeCount: INDEX_CAPACITY + }).addToGraph(graph); + return graph.compile(); +} + +/** Creates one explicitly owned small hash-index buffer for the isolated build measurement. */ +function createBenchmarkIndexBuffer( + device: Device, + id: string, + length: number, + ownedBuffers: Buffer[] +): Buffer { + const buffer = device.createBuffer({ + id, + byteLength: length * UINT32_BYTE_LENGTH, + usage: Buffer.STORAGE | Buffer.COPY_SRC | Buffer.COPY_DST + }); + ownedBuffers.push(buffer); + return buffer; +} + +/** Imports a packed standalone-index buffer through the normal application-owned graph boundary. */ +function importBenchmarkUint32( + graph: GPUCommandGraph, + id: string, + buffer: Buffer, + length: number +) { + const handle = graph.importBuffer( + {id, byteLength: buffer.byteLength, usage: buffer.usage}, + buffer + ); + return graph.createDataView(handle, {format: 'uint32', length}); +} + +/** Measures actual encode, submission, and completed execution without timestamp assumptions. */ +async function executeBenchmarkGraph( + device: Device, + graph: + | CompiledGPUCommandGraph + | CompiledLuDataFrameQuery + | CompiledLuDataFrameGroupedAggregation<{ + category: 'uint32'; + count: 'uint32'; + totalAdjustedFare: 'float32'; + }>, + id: string, + signal: AbortSignal | undefined +): Promise { + signal?.throwIfAborted(); + const commandEncoder = device.createCommandEncoder({id}); + const started = performance.now(); + let submitted = false; + try { + if ('table' in graph) { + graph.encode(commandEncoder); + } else { + graph.encode(commandEncoder, {parameters: {}}); + } + device.submit(commandEncoder.finish()); + submitted = true; + await waitForBenchmarkGPU(device, signal); + return performance.now() - started; + } catch (error) { + if (!submitted) commandEncoder.destroy(); + throw error; + } +} + +/** Waits for submitted GPU work using the portable luma.gl fence instead of private device state. */ +async function waitForBenchmarkGPU(device: Device, signal: AbortSignal | undefined): Promise { + signal?.throwIfAborted(); + const fence = device.createFence(); + try { + await fence.signaled; + signal?.throwIfAborted(); + } finally { + fence.destroy(); + } +} + +/** Reads one uint32 scalar per preserved GPU output batch without touching source rows. */ +async function readBenchmarkScalars( + vector: GPUVector<'uint32'>, + bytes: {value: number}, + signal: AbortSignal | undefined +): Promise { + const values: number[] = []; + for (const data of vector.data) { + values.push((await readBenchmarkUint32(data, 1, bytes, signal))[0]); + } + return values; +} + +/** Reads only the already-bounded published output prefix from each preserved source batch. */ +async function readBenchmarkPrefixes( + vector: GPUVector<'uint32'>, + counts: readonly number[], + bytes: {value: number}, + signal: AbortSignal | undefined +): Promise { + const values: number[][] = []; + for (const [batchIndex, data] of vector.data.entries()) { + values.push(await readBenchmarkUint32(data, counts[batchIndex], bytes, signal)); + } + return values; +} + +/** Counts and reads a caller-specified bounded unsigned output prefix. */ +async function readBenchmarkUint32( + data: GPUData, + length: number, + bytes: {value: number}, + signal: AbortSignal | undefined +): Promise { + signal?.throwIfAborted(); + if (data.format !== 'uint32') { + throw new Error('The luDF benchmark expected uint32 GPU output'); + } + if (length === 0) return []; + const byteLength = length * UINT32_BYTE_LENGTH; + const result = await data.buffer.readAsync(data.byteOffset, byteLength); + bytes.value += byteLength; + signal?.throwIfAborted(); + return Array.from(new Uint32Array(result.buffer, result.byteOffset, length)); +} + +/** Reads only fixed-cardinality grouped floating summaries for CPU-oracle verification. */ +async function readBenchmarkFloat32( + data: GPUData, + length: number, + bytes: {value: number}, + signal: AbortSignal | undefined +): Promise { + signal?.throwIfAborted(); + if (data.format !== 'float32') { + throw new Error('The luDF benchmark expected float32 GPU output'); + } + const byteLength = length * Float32Array.BYTES_PER_ELEMENT; + const result = await data.buffer.readAsync(data.byteOffset, byteLength); + bytes.value += byteLength; + signal?.throwIfAborted(); + return Array.from(new Float32Array(result.buffer, result.byteOffset, length)); +} + +/** Compares integer result vectors without sorting or obscuring stable row identity. */ +function compareNumberArrays(actual: readonly number[], expected: readonly number[]): boolean { + return ( + actual.length === expected.length && actual.every((value, index) => value === expected[index]) + ); +} + +/** Compares independent source-batch result prefixes without flattening their original topology. */ +function compareNestedNumberArrays( + actual: readonly (readonly number[])[], + expected: readonly (readonly number[])[] +): boolean { + return ( + actual.length === expected.length && + actual.every((values, index) => compareNumberArrays(values, expected[index])) + ); +} + +/** Accounts for the documented nondeterministic floating-point order of GPU grouped atomics. */ +function approximatelyEqualBenchmark(actual: number, expected: number): boolean { + return Math.abs(actual - expected) <= Math.max(0.001, Math.abs(expected) * 0.00001); +} diff --git a/modules/experimental/src/ludf/index.ts b/modules/experimental/src/ludf/index.ts index 94863ca7fb..e6323de748 100644 --- a/modules/experimental/src/ludf/index.ts +++ b/modules/experimental/src/ludf/index.ts @@ -1,6 +1,7 @@ // luma.gl // SPDX-License-Identifier: MIT // SPDX-FileCopyrightText: Copyright (c) vis.gl contributors +// SPDX-FileComment: Independently implemented for WebGPU; inspired by NVIDIA RAPIDS cuDF. export {LuDataFrame} from './lu-data-frame'; export type { diff --git a/modules/experimental/src/ludf/lu-analytics-compiler-utils.ts b/modules/experimental/src/ludf/lu-analytics-compiler-utils.ts index 4abe053c49..f47e75edc6 100644 --- a/modules/experimental/src/ludf/lu-analytics-compiler-utils.ts +++ b/modules/experimental/src/ludf/lu-analytics-compiler-utils.ts @@ -1,6 +1,7 @@ // luma.gl // SPDX-License-Identifier: MIT // SPDX-FileCopyrightText: Copyright (c) vis.gl contributors +// SPDX-FileComment: Independently implemented for WebGPU; inspired by NVIDIA RAPIDS cuDF. import {Buffer, type Binding, type Device} from '@luma.gl/core'; import {Computation} from '@luma.gl/engine'; diff --git a/modules/experimental/src/ludf/lu-data-frame-query.ts b/modules/experimental/src/ludf/lu-data-frame-query.ts index 3a7c938b25..146de869f3 100644 --- a/modules/experimental/src/ludf/lu-data-frame-query.ts +++ b/modules/experimental/src/ludf/lu-data-frame-query.ts @@ -1,6 +1,7 @@ // luma.gl // SPDX-License-Identifier: MIT // SPDX-FileCopyrightText: Copyright (c) vis.gl contributors +// SPDX-FileComment: Independently implemented for WebGPU; inspired by NVIDIA RAPIDS cuDF. import type {GPUTypeMap} from '@luma.gl/tables'; import type {GPUCommandGraph} from '../gpu-primitives/gpu-command-graph'; diff --git a/modules/experimental/src/ludf/lu-data-frame.ts b/modules/experimental/src/ludf/lu-data-frame.ts index a1d27ae5fa..9fea8ce770 100644 --- a/modules/experimental/src/ludf/lu-data-frame.ts +++ b/modules/experimental/src/ludf/lu-data-frame.ts @@ -1,6 +1,7 @@ // luma.gl // SPDX-License-Identifier: MIT // SPDX-FileCopyrightText: Copyright (c) vis.gl contributors +// SPDX-FileComment: Independently implemented for WebGPU; inspired by NVIDIA RAPIDS cuDF. import type {BufferLayout} from '@luma.gl/core'; import { diff --git a/modules/experimental/src/ludf/lu-expression-shader.ts b/modules/experimental/src/ludf/lu-expression-shader.ts index d43a22d200..d9ff00cce4 100644 --- a/modules/experimental/src/ludf/lu-expression-shader.ts +++ b/modules/experimental/src/ludf/lu-expression-shader.ts @@ -1,6 +1,7 @@ // luma.gl // SPDX-License-Identifier: MIT // SPDX-FileCopyrightText: Copyright (c) vis.gl contributors +// SPDX-FileComment: Independently implemented for WebGPU; inspired by NVIDIA RAPIDS cuDF. import type {GPUTypeMap} from '@luma.gl/tables'; import type {LuDataFrame} from './lu-data-frame'; diff --git a/modules/experimental/src/ludf/lu-expression.ts b/modules/experimental/src/ludf/lu-expression.ts index f3ca6c4fc1..f437be8ee0 100644 --- a/modules/experimental/src/ludf/lu-expression.ts +++ b/modules/experimental/src/ludf/lu-expression.ts @@ -1,6 +1,7 @@ // luma.gl // SPDX-License-Identifier: MIT // SPDX-FileCopyrightText: Copyright (c) vis.gl contributors +// SPDX-FileComment: Independently implemented for WebGPU; inspired by NVIDIA RAPIDS cuDF. /** Portable scalar values accepted by GPU-resident dataframe expressions. */ export type LuExpressionValue = number | boolean | null; diff --git a/modules/experimental/src/ludf/lu-global-aggregation-compiler.ts b/modules/experimental/src/ludf/lu-global-aggregation-compiler.ts index 7ab1c82c96..9c4f543778 100644 --- a/modules/experimental/src/ludf/lu-global-aggregation-compiler.ts +++ b/modules/experimental/src/ludf/lu-global-aggregation-compiler.ts @@ -1,6 +1,7 @@ // luma.gl // SPDX-License-Identifier: MIT // SPDX-FileCopyrightText: Copyright (c) vis.gl contributors +// SPDX-FileComment: Independently implemented for WebGPU; inspired by NVIDIA RAPIDS cuDF. import {GPUVector, type GPUField, type GPUTypeMap} from '@luma.gl/tables'; import { diff --git a/modules/experimental/src/ludf/lu-global-aggregation-query.ts b/modules/experimental/src/ludf/lu-global-aggregation-query.ts index 9c2760b112..80fd51a128 100644 --- a/modules/experimental/src/ludf/lu-global-aggregation-query.ts +++ b/modules/experimental/src/ludf/lu-global-aggregation-query.ts @@ -1,6 +1,7 @@ // luma.gl // SPDX-License-Identifier: MIT // SPDX-FileCopyrightText: Copyright (c) vis.gl contributors +// SPDX-FileComment: Independently implemented for WebGPU; inspired by NVIDIA RAPIDS cuDF. import type {GPUTypeMap} from '@luma.gl/tables'; import type {GPUCommandGraph} from '../gpu-primitives/gpu-command-graph'; diff --git a/modules/experimental/src/ludf/lu-group-aggregation-compiler.ts b/modules/experimental/src/ludf/lu-group-aggregation-compiler.ts index 6226771b75..427553eb3a 100644 --- a/modules/experimental/src/ludf/lu-group-aggregation-compiler.ts +++ b/modules/experimental/src/ludf/lu-group-aggregation-compiler.ts @@ -1,6 +1,7 @@ // luma.gl // SPDX-License-Identifier: MIT // SPDX-FileCopyrightText: Copyright (c) vis.gl contributors +// SPDX-FileComment: Independently implemented for WebGPU; inspired by NVIDIA RAPIDS cuDF. import {Buffer, type Binding, type Device} from '@luma.gl/core'; import {Computation} from '@luma.gl/engine'; diff --git a/modules/experimental/src/ludf/lu-group-by-query.ts b/modules/experimental/src/ludf/lu-group-by-query.ts index 427471d457..14f6b1775d 100644 --- a/modules/experimental/src/ludf/lu-group-by-query.ts +++ b/modules/experimental/src/ludf/lu-group-by-query.ts @@ -1,6 +1,7 @@ // luma.gl // SPDX-License-Identifier: MIT // SPDX-FileCopyrightText: Copyright (c) vis.gl contributors +// SPDX-FileComment: Independently implemented for WebGPU; inspired by NVIDIA RAPIDS cuDF. import type {GPUTypeMap} from '@luma.gl/tables'; import type {GPUCommandGraph} from '../gpu-primitives/gpu-command-graph'; diff --git a/modules/experimental/src/ludf/lu-histogram-compiler.ts b/modules/experimental/src/ludf/lu-histogram-compiler.ts index a69e9d5360..beda0cf92c 100644 --- a/modules/experimental/src/ludf/lu-histogram-compiler.ts +++ b/modules/experimental/src/ludf/lu-histogram-compiler.ts @@ -1,6 +1,7 @@ // luma.gl // SPDX-License-Identifier: MIT // SPDX-FileCopyrightText: Copyright (c) vis.gl contributors +// SPDX-FileComment: Independently implemented for WebGPU; inspired by NVIDIA RAPIDS cuDF. import {GPUVector, type GPUField, type GPUTypeMap} from '@luma.gl/tables'; import {type GPUCommandGraph, type GraphDataView} from '../gpu-primitives/gpu-command-graph'; diff --git a/modules/experimental/src/ludf/lu-histogram-query.ts b/modules/experimental/src/ludf/lu-histogram-query.ts index 7608cf0f5f..218daa420c 100644 --- a/modules/experimental/src/ludf/lu-histogram-query.ts +++ b/modules/experimental/src/ludf/lu-histogram-query.ts @@ -1,6 +1,7 @@ // luma.gl // SPDX-License-Identifier: MIT // SPDX-FileCopyrightText: Copyright (c) vis.gl contributors +// SPDX-FileComment: Independently implemented for WebGPU; inspired by NVIDIA RAPIDS cuDF. import type {GPUTypeMap} from '@luma.gl/tables'; import type {GPUCommandGraph} from '../gpu-primitives/gpu-command-graph'; diff --git a/modules/experimental/src/ludf/lu-join-compiler.ts b/modules/experimental/src/ludf/lu-join-compiler.ts index a34946da59..56f24b9d9b 100644 --- a/modules/experimental/src/ludf/lu-join-compiler.ts +++ b/modules/experimental/src/ludf/lu-join-compiler.ts @@ -1,6 +1,7 @@ // luma.gl // SPDX-License-Identifier: MIT // SPDX-FileCopyrightText: Copyright (c) vis.gl contributors +// SPDX-FileComment: Independently implemented for WebGPU; inspired by NVIDIA RAPIDS cuDF. import {Buffer, type Device} from '@luma.gl/core'; import {GPUData, GPUVector, type GPUTable, type GPUTypeMap} from '@luma.gl/tables'; diff --git a/modules/experimental/src/ludf/lu-join-query.ts b/modules/experimental/src/ludf/lu-join-query.ts index 37b5941b7f..c46ba71c8d 100644 --- a/modules/experimental/src/ludf/lu-join-query.ts +++ b/modules/experimental/src/ludf/lu-join-query.ts @@ -1,6 +1,7 @@ // luma.gl // SPDX-License-Identifier: MIT // SPDX-FileCopyrightText: Copyright (c) vis.gl contributors +// SPDX-FileComment: Independently implemented for WebGPU; inspired by NVIDIA RAPIDS cuDF. import type {GPUTypeMap} from '@luma.gl/tables'; import type {GPUCommandGraph} from '../gpu-primitives/gpu-command-graph'; diff --git a/modules/experimental/src/ludf/lu-query-compiler.ts b/modules/experimental/src/ludf/lu-query-compiler.ts index a1ab03d819..61f760010d 100644 --- a/modules/experimental/src/ludf/lu-query-compiler.ts +++ b/modules/experimental/src/ludf/lu-query-compiler.ts @@ -1,6 +1,7 @@ // luma.gl // SPDX-License-Identifier: MIT // SPDX-FileCopyrightText: Copyright (c) vis.gl contributors +// SPDX-FileComment: Independently implemented for WebGPU; inspired by NVIDIA RAPIDS cuDF. import { Buffer, diff --git a/modules/experimental/src/ludf/lu-sort-compiler.ts b/modules/experimental/src/ludf/lu-sort-compiler.ts index a40fbb3b86..4e21e06488 100644 --- a/modules/experimental/src/ludf/lu-sort-compiler.ts +++ b/modules/experimental/src/ludf/lu-sort-compiler.ts @@ -1,6 +1,7 @@ // luma.gl // SPDX-License-Identifier: MIT // SPDX-FileCopyrightText: Copyright (c) vis.gl contributors +// SPDX-FileComment: Independently implemented for WebGPU; inspired by NVIDIA RAPIDS cuDF. import type {GPUTypeMap} from '@luma.gl/tables'; import {GPUBatchSort} from '../gpu-primitives/gpu-batch-sort'; diff --git a/modules/experimental/src/ludf/lu-sort-query.ts b/modules/experimental/src/ludf/lu-sort-query.ts index 57fd20ab53..c4e409ef10 100644 --- a/modules/experimental/src/ludf/lu-sort-query.ts +++ b/modules/experimental/src/ludf/lu-sort-query.ts @@ -1,6 +1,7 @@ // luma.gl // SPDX-License-Identifier: MIT // SPDX-FileCopyrightText: Copyright (c) vis.gl contributors +// SPDX-FileComment: Independently implemented for WebGPU; inspired by NVIDIA RAPIDS cuDF. import type {GPUTypeMap} from '@luma.gl/tables'; import type {GPUCommandGraph} from '../gpu-primitives/gpu-command-graph'; diff --git a/test/examples/ludf-analysis.node.spec.ts b/test/examples/ludf-analysis.node.spec.ts new file mode 100644 index 0000000000..b517a74ade --- /dev/null +++ b/test/examples/ludf-analysis.node.spec.ts @@ -0,0 +1,134 @@ +// luma.gl +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: Copyright (c) vis.gl contributors + +import {readFileSync} from 'node:fs'; +import path from 'node:path'; +import {describe, expect, test} from 'vitest'; + +type DocumentationEntry = + | string + | {type: 'doc'; id: string; label?: string} + | {type: 'category'; label: string; items: DocumentationEntry[]}; + +type DocumentationCategory = Extract; + +const REPOSITORY_DIRECTORY = process.cwd(); +const LUDF_DOCUMENT_IDENTIFIER = 'api-reference/experimental/ludf'; +const LUDF_DOCUMENT_PATH = '/docs/api-reference/experimental/ludf'; +const LUDF_EXAMPLE_PATH = '/examples/experimental/gpu-data-analysis'; + +describe('luDF dataframe documentation and opt-in Arrow benchmark integration', () => { + test('registers the dedicated luDF reference throughout both experimental navigation trees', () => { + const tableOfContents = JSON.parse( + readFileSync(path.join(REPOSITORY_DIRECTORY, 'docs/table-of-contents.json'), 'utf8') + ) as DocumentationEntry[]; + const experimentalCategories: DocumentationCategory[] = []; + + const visit = (entries: readonly DocumentationEntry[]): void => { + for (const entry of entries) { + if (typeof entry === 'string' || entry.type !== 'category') { + continue; + } + if (entry.label === '@luma.gl/experimental') { + experimentalCategories.push(entry); + } + visit(entry.items); + } + }; + visit(tableOfContents); + + expect(experimentalCategories).toHaveLength(2); + for (const category of experimentalCategories) { + expect(category.items).toContain(LUDF_DOCUMENT_IDENTIFIER); + } + + const documentation = readRepositoryFile('docs/api-reference/experimental/ludf.md'); + const overview = readRepositoryFile('docs/api-reference/experimental/README.md'); + const releaseNotes = readRepositoryFile('docs/whats-new.md'); + const experimentalTabs = readRepositoryFile( + 'website/src/components/docs/experimental-docs-tabs.tsx' + ); + + expect(documentation).toContain(''); + expect(overview).toContain(LUDF_DOCUMENT_PATH); + expect(releaseNotes).toContain(LUDF_DOCUMENT_PATH); + expect(experimentalTabs).toMatch( + /id:\s*['"]ludf['"][^}]*href:\s*['"]\/docs\/api-reference\/experimental\/ludf['"]/ + ); + }); + + test('documents real Arrow ingestion, owned GPU work, supported joins, and accurate limitations', () => { + const documentation = readRepositoryFile('docs/api-reference/experimental/ludf.md'); + + expect(documentation).toContain("from '@luma.gl/arrow'"); + expect(documentation).toContain("from '@luma.gl/experimental/ludf'"); + expect(documentation).toContain('makeGPUAnalyticsTableFromArrowTable'); + expect(documentation).toContain('GPUCommandGraph'); + expect(documentation).toContain('selectionMask'); + expect(documentation).toContain('rowIndices'); + expect(documentation).toContain('selectedCounts'); + expect(documentation).toContain('innerJoin'); + expect(documentation).toContain('lookup'); + expect(documentation).toContain('float32'); + expect(documentation).toContain('sint32'); + expect(documentation).toContain('uint32'); + expect(documentation).toMatch(/validity/i); + expect(documentation).toMatch(/readback/i); + expect(documentation).toContain(LUDF_EXAMPLE_PATH); + }); + + test('keeps the existing WebGPU example route and benchmark explicitly opt-in', () => { + const example = readRepositoryFile('examples/experimental/gpu-data-analysis/src/app.ts'); + const exampleShell = readRepositoryFile( + 'examples/experimental/gpu-data-analysis/src/app-shell.ts' + ); + const websiteExample = readRepositoryFile( + 'website/content/examples/experimental/gpu-data-analysis.mdx' + ); + + expect(websiteExample).toContain(''); + expect(example).toContain("from './ludf-benchmark'"); + expect(exampleShell).toContain('analysis-ludf-benchmark-run'); + expect(exampleShell).toContain('analysis-ludf-benchmark-status'); + expect(exampleShell).toContain('analysis-ludf-benchmark-results'); + expect(exampleShell).toContain('data-ludf-benchmark'); + expect(exampleShell).toContain('data-ludf-benchmark-phases'); + expect(exampleShell).toContain('data-state="idle"'); + expect(exampleShell).toContain('data-validated="false"'); + expect(example).toContain("addEventListener('click', this.handleLuDataFrameBenchmark)"); + expect(example).toContain('this.benchmarkController?.abort()'); + expect(example).toContain('rowCount: 384'); + + for (const phase of ['upload', 'compile', 'index', 'execution', 'readback', 'cpu']) { + expect(example).toContain(`['${phase}',`); + } + }); + + test('keeps Arrow conversion in its adapter and explicitly fences bounded CPU/GPU comparisons', () => { + const benchmark = readRepositoryFile( + 'examples/experimental/gpu-data-analysis/src/ludf-benchmark.ts' + ); + const tablesPackage = JSON.parse(readRepositoryFile('modules/tables/package.json')) as { + dependencies?: Record; + }; + const gpuPackage = JSON.parse(readRepositoryFile('modules/gpgpu/package.json')) as { + dependencies?: Record; + }; + + expect(benchmark).toContain("from '@luma.gl/arrow'"); + expect(benchmark).toContain("from '@luma.gl/experimental/ludf'"); + expect(benchmark).toContain("from 'apache-arrow'"); + expect(benchmark).toContain('makeGPUAnalyticsTableFromArrowTable'); + expect(benchmark).toContain('runLuDataFrameBenchmark'); + expect(benchmark).toContain('createFence'); + expect(benchmark).toContain('readbackBytes'); + expect(benchmark).toContain('signal'); + expect(tablesPackage.dependencies?.['apache-arrow']).toBeUndefined(); + expect(gpuPackage.dependencies?.['apache-arrow']).toBeUndefined(); + }); +}); + +function readRepositoryFile(relativePath: string): string { + return readFileSync(path.join(REPOSITORY_DIRECTORY, relativePath), 'utf8'); +} diff --git a/test/examples/ludf-analysis.spec.ts b/test/examples/ludf-analysis.spec.ts new file mode 100644 index 0000000000..bac60a1337 --- /dev/null +++ b/test/examples/ludf-analysis.spec.ts @@ -0,0 +1,172 @@ +// luma.gl +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: Copyright (c) vis.gl contributors + +import {luma} from '@luma.gl/core'; +import {getWebGPUTestDevice} from '@luma.gl/test-utils'; +import {describe, expect, test, vi} from 'vitest'; +import { + initializeGPUDataAnalysisExample, + type GPUDataAnalysisExampleHandle +} from '../../examples/experimental/gpu-data-analysis/src/app'; +import {runLuDataFrameBenchmark} from '../../examples/experimental/gpu-data-analysis/src/ludf-benchmark'; + +const BENCHMARK_PHASES = ['upload', 'compile', 'index', 'execution', 'readback', 'cpu'] as const; + +describe('Arrow-driven luDF dataframe example', () => { + test('validates real filtering, grouping, stable sorting, and joins against equivalent CPU work', async () => { + const device = await getWebGPUTestDevice(); + if (!device) { + return; + } + + const result = await runLuDataFrameBenchmark(device, {rowCount: 128}); + + expect(result.rowCount).toBe(128); + expect(result.batchRowCounts).toHaveLength(3); + expect(result.batchRowCounts[1]).toBe(0); + expect(result.batchRowCounts.reduce((count, batchRows) => count + batchRows, 0)).toBe(128); + expect(result.validation).toEqual({ + filter: true, + groups: true, + sorting: true, + join: true + }); + expect(result.summaries.filterCount).toBeGreaterThan(0); + expect(result.summaries.groupCounts).toHaveLength(4); + expect(result.summaries.topKRowIds).toHaveLength(3); + expect(result.summaries.joinCounts).toHaveLength(3); + expect(result.summaries.joinLeftRowIds).toHaveLength(3); + expect(result.summaries.joinRightRowIds).toHaveLength(3); + expect(result.summaries.joinLeftRowIds[1]).toEqual([]); + expect(result.summaries.joinRightRowIds[1]).toEqual([]); + expect(result.readbackBytes).toBeGreaterThan(0); + expect(result.readbackBytes).toBeLessThanOrEqual(1024); + + for (const phase of [ + result.timings.uploadMilliseconds, + result.timings.compileMilliseconds, + result.timings.indexMilliseconds, + result.timings.executionMilliseconds, + result.timings.readbackMilliseconds, + result.timings.cpuMilliseconds + ]) { + expect(Number.isFinite(phase)).toBe(true); + expect(phase).toBeGreaterThanOrEqual(0); + } + }, 30_000); + + test('runs the real WebGPU benchmark only after an explicit interactive request', async () => { + const availableDevice = await getWebGPUTestDevice(); + if (!availableDevice) { + return; + } + + const root = document.createElement('main'); + root.id = 'gpu-data-analysis-app'; + document.body.append(root); + let example: GPUDataAnalysisExampleHandle | undefined; + + try { + example = initializeGPUDataAnalysisExample(); + + const dataset = getRequiredElement(root, '[data-dataset]'); + dataset.value = 'small'; + + const button = getRequiredElement(root, '#analysis-ludf-benchmark-run'); + const status = getRequiredElement(root, '#analysis-ludf-benchmark-status'); + const results = getRequiredElement(root, '#analysis-ludf-benchmark-results'); + + expect(button.matches('[data-ludf-benchmark]')).toBe(true); + expect(status.matches('[data-ludf-benchmark-status]')).toBe(true); + expect(results.matches('[data-ludf-benchmark-phases]')).toBe(true); + expect(results.dataset.state).toBe('idle'); + expect(results.dataset.validated).toBe('false'); + expect(results.querySelectorAll('[data-ludf-phase]')).toHaveLength(0); + + await vi.waitFor( + () => { + expect(getRequiredElement(root, '[data-validation]').dataset.state).toBe( + 'ok' + ); + expect(button.disabled).toBe(false); + }, + {timeout: 20_000, interval: 25} + ); + + expect(results.dataset.state).toBe('idle'); + expect(results.querySelectorAll('[data-ludf-phase]')).toHaveLength(0); + + button.click(); + expect(results.dataset.state).toBe('running'); + expect(button.disabled).toBe(true); + + await vi.waitFor( + () => { + expect(results.dataset.state).toBe('ok'); + expect(results.dataset.validated).toBe('true'); + expect(button.disabled).toBe(false); + }, + {timeout: 25_000, interval: 25} + ); + + for (const phase of BENCHMARK_PHASES) { + const row = getRequiredElement(results, `[data-ludf-phase="${phase}"]`); + const milliseconds = Number(row.querySelector('td')?.textContent); + expect(Number.isFinite(milliseconds)).toBe(true); + expect(milliseconds).toBeGreaterThanOrEqual(0); + } + expect(status.textContent).toContain('384'); + expect(status.textContent).toMatch(/filter, grouping, sorting, and joins match/i); + expect(status.textContent).toMatch(/summary bytes read/i); + } finally { + example?.destroy(); + root.remove(); + } + }, 45_000); + + test('keeps the opt-in benchmark disabled when WebGPU initialization fails', async () => { + const root = document.createElement('main'); + root.id = 'gpu-data-analysis-app'; + document.body.append(root); + const createDevice = vi + .spyOn(luma, 'createDevice') + .mockRejectedValue(new Error('WebGPU test adapter unavailable')); + let example: GPUDataAnalysisExampleHandle | undefined; + + try { + example = initializeGPUDataAnalysisExample(); + + await vi.waitFor( + () => { + const status = getRequiredElement(root, '[data-status]'); + expect(status.dataset.state).toBe('error'); + expect(status.textContent).toContain('WebGPU test adapter unavailable'); + }, + {timeout: 2_000, interval: 10} + ); + + expect( + getRequiredElement(root, '#analysis-ludf-benchmark-run').disabled + ).toBe(true); + expect( + getRequiredElement(root, '#analysis-ludf-benchmark-status').textContent + ).toMatch(/WebGPU is unavailable/i); + expect( + getRequiredElement(root, '#analysis-ludf-benchmark-results').dataset.state + ).toBe('idle'); + } finally { + example?.destroy(); + createDevice.mockRestore(); + root.remove(); + } + }); +}); + +function getRequiredElement(root: ParentNode, selector: string): T { + const element = root.querySelector(selector); + if (!element) { + throw new Error(`Missing luDF benchmark element ${selector}`); + } + return element; +} diff --git a/test/examples/ludf-attribution.node.spec.ts b/test/examples/ludf-attribution.node.spec.ts new file mode 100644 index 0000000000..d607f0c375 --- /dev/null +++ b/test/examples/ludf-attribution.node.spec.ts @@ -0,0 +1,52 @@ +// luma.gl +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: Copyright (c) vis.gl contributors + +import {readdirSync, readFileSync} from 'node:fs'; +import {describe, expect, test} from 'vitest'; + +const LUDF_SOURCE_DIRECTORY = new URL('../../modules/experimental/src/ludf/', import.meta.url); +const LUDF_DOCUMENTATION = new URL( + '../../docs/api-reference/experimental/ludf.md', + import.meta.url +); +const LUDF_SOURCE_FILES = readdirSync(LUDF_SOURCE_DIRECTORY) + .filter(fileName => fileName.endsWith('.ts')) + .sort(); +const LUDF_SPDX_HEADER = [ + '// luma.gl', + '// SPDX-License-Identifier: MIT', + '// SPDX-FileCopyrightText: Copyright (c) vis.gl contributors', + '// SPDX-FileComment: Independently implemented for WebGPU; inspired by NVIDIA RAPIDS cuDF.' +].join('\n'); + +describe('luDF RAPIDS attribution and independent MIT licensing', () => { + test('discovers every production TypeScript implementation', () => { + expect(LUDF_SOURCE_FILES.length).toBeGreaterThan(0); + expect(LUDF_SOURCE_FILES).toContain('index.ts'); + expect(LUDF_SOURCE_FILES).toContain('lu-data-frame.ts'); + }); + + test.each(LUDF_SOURCE_FILES)('preserves accurate SPDX attribution in %s', fileName => { + const source = readFileSync(new URL(fileName, LUDF_SOURCE_DIRECTORY), 'utf8'); + + expect(source.startsWith(`${LUDF_SPDX_HEADER}\n`)).toBe(true); + expect(source).not.toMatch(/^\/\/ SPDX-FileCopyrightText:.*NVIDIA/m); + expect(source).not.toMatch(/^\/\/ SPDX-License-Identifier:.*Apache-2\.0/m); + }); + + test('documents the distinct upstream Apache and original vis.gl MIT licenses', () => { + const documentation = readFileSync(LUDF_DOCUMENTATION, 'utf8'); + + expect(documentation).toContain('## Attribution and licensing'); + expect(documentation).toContain('https://github.com/rapidsai/cudf'); + expect(documentation).toContain('https://github.com/rapidsai/cudf/blob/main/LICENSE'); + expect(documentation).toContain('https://github.com/visgl/luma.gl/blob/master/LICENSE'); + expect(documentation).toContain('Apache License 2.0'); + expect(documentation).toContain('MIT-licensed'); + expect(documentation).toContain('does not copy or translate cuDF source code'); + expect(documentation).toContain('including CUDA or Python implementations'); + expect(documentation).toContain('neither affiliated with nor endorsed by NVIDIA'); + expect(documentation).toContain('feature parity'); + }); +}); diff --git a/website/content/examples/experimental/gpu-data-analysis.mdx b/website/content/examples/experimental/gpu-data-analysis.mdx index 6757e24bb1..22bbe340c1 100644 --- a/website/content/examples/experimental/gpu-data-analysis.mdx +++ b/website/content/examples/experimental/gpu-data-analysis.mdx @@ -1,12 +1,12 @@ --- -title: Graph-native GPU data analysis +title: Graph-native GPU data analysis and luDF sidebar_label: GPU data analysis -description: Compose reductions, histograms, scans, grouped statistics, and spatial bins in one GPU command graph. +description: Compose graph-native GPU analytics and run an opt-in Arrow-driven luDF dataframe benchmark against equivalent CPU results. sidebar_custom_props: backends: [webgpu] difficulty: advanced maturity: experimental - topics: [compute, data, gpgpu, analytics, command-graphs] + topics: [compute, analytics, command-graphs, arrow, dataframes] --- import {GPUDataAnalysisExample} from '@site/src/examples'; diff --git a/website/src/components/docs/experimental-docs-tabs.tsx b/website/src/components/docs/experimental-docs-tabs.tsx index 074e857826..18788aaa73 100644 --- a/website/src/components/docs/experimental-docs-tabs.tsx +++ b/website/src/components/docs/experimental-docs-tabs.tsx @@ -11,6 +11,7 @@ export type ExperimentalDocsTabId = | 'pbr-environment' | 'luproj' | 'lugraph' + | 'ludf' | 'luxfilter' | 'lutrace' | 'g-buffer' @@ -44,6 +45,7 @@ const EXPERIMENTAL_DOCS_TABS: ExperimentalDocsTab[] = [ }, {id: 'luproj', label: 'GPU Projection', href: '/docs/api-reference/experimental/luproj'}, {id: 'lugraph', label: 'GPU Graphs', href: '/docs/api-reference/experimental/lugraph'}, + {id: 'ludf', label: 'luDF', href: '/docs/api-reference/experimental/ludf'}, {id: 'luxfilter', label: 'LuxFilter', href: '/docs/api-reference/experimental/luxfilter'}, {id: 'lutrace', label: 'GPU Traces', href: '/docs/api-reference/experimental/lutrace'}, {id: 'g-buffer', label: 'GBuffer', href: '/docs/api-reference/experimental/g-buffer'},