From 266faaf055d248378087ac3b507ed9fec0c348de Mon Sep 17 00:00:00 2001 From: Ib Green Date: Tue, 4 Aug 2026 22:06:51 -0400 Subject: [PATCH 1/3] feat(experimental): add luDF reductions and histograms --- modules/experimental/src/ludf/index.ts | 12 + .../src/ludf/lu-analytics-compiler-utils.ts | 272 +++++++ .../src/ludf/lu-data-frame-query.ts | 21 + .../experimental/src/ludf/lu-data-frame.ts | 28 + .../ludf/lu-global-aggregation-compiler.ts | 423 +++++++++++ .../src/ludf/lu-global-aggregation-query.ts | 178 +++++ .../src/ludf/lu-histogram-compiler.ts | 189 +++++ .../src/ludf/lu-histogram-query.ts | 150 ++++ modules/experimental/test/index.ts | 1 + .../test/ludf/lu-global-aggregation.spec.ts | 710 ++++++++++++++++++ .../lu-reductions-histograms.node.spec.ts | 366 +++++++++ 11 files changed, 2350 insertions(+) create mode 100644 modules/experimental/src/ludf/lu-analytics-compiler-utils.ts create mode 100644 modules/experimental/src/ludf/lu-global-aggregation-compiler.ts create mode 100644 modules/experimental/src/ludf/lu-global-aggregation-query.ts create mode 100644 modules/experimental/src/ludf/lu-histogram-compiler.ts create mode 100644 modules/experimental/src/ludf/lu-histogram-query.ts create mode 100644 modules/experimental/test/ludf/lu-global-aggregation.spec.ts create mode 100644 modules/experimental/test/ludf/lu-reductions-histograms.node.spec.ts diff --git a/modules/experimental/src/ludf/index.ts b/modules/experimental/src/ludf/index.ts index 9b5b7bde68..3378902852 100644 --- a/modules/experimental/src/ludf/index.ts +++ b/modules/experimental/src/ludf/index.ts @@ -29,6 +29,16 @@ export type { LuDataFrameGroupByOptions, LuDataFrameGroupedAggregationResult } from './lu-group-by-query'; +export {LuDataFrameAggregationQuery} from './lu-global-aggregation-query'; +export type { + LuDataFrameAnalyticScalarFormat, + LuDataFrameGlobalAggregationDefinitions, + LuDataFrameGlobalAggregationResult, + LuDataFrameGlobalAggregationValue, + LuDataFrameScalarColumnNames +} from './lu-global-aggregation-query'; +export {LuDataFrameHistogramQuery} from './lu-histogram-query'; +export type {LuDataFrameHistogramOptions} from './lu-histogram-query'; export {and, column, literal, LuExpression, not, or, parameter} from './lu-expression'; export type { LuExpressionBinaryOperator, @@ -39,3 +49,5 @@ export type { export {CompiledLuDataFrameQuery} from './lu-query-compiler'; export type {LuDataFrameQueryParameters} from './lu-query-compiler'; export {CompiledLuDataFrameGroupedAggregation} from './lu-group-aggregation-compiler'; +export {CompiledLuDataFrameAggregation} from './lu-global-aggregation-compiler'; +export {CompiledLuDataFrameHistogram} from './lu-histogram-compiler'; diff --git a/modules/experimental/src/ludf/lu-analytics-compiler-utils.ts b/modules/experimental/src/ludf/lu-analytics-compiler-utils.ts new file mode 100644 index 0000000000..e5bebeed4e --- /dev/null +++ b/modules/experimental/src/ludf/lu-analytics-compiler-utils.ts @@ -0,0 +1,272 @@ +// luma.gl +// SPDX-License-Identifier: MIT +// Copyright (c) vis.gl contributors + +import {Buffer, type Binding, type Device} from '@luma.gl/core'; +import {Computation} from '@luma.gl/engine'; +import { + GPUData, + GPURecordBatch, + GPUTable, + GPUVector, + type GPUField, + type GPUTypeMap +} from '@luma.gl/tables'; +import { + GraphVectorView, + type GPUCommandGraph, + type GraphBufferUse, + type GraphDataView +} from '../gpu-primitives/gpu-command-graph'; +import {GPUMask} from '../gpu-primitives/gpu-mask'; +import { + createTransientVectorView, + createTransientView, + getViewBinding +} from '../gpu-primitives/graph-data-view-utils'; +import type {LuDataFrame} from './lu-data-frame'; +import type { + LuDataFrameQueryExtensionContext, + LuDataFrameQueryParameters +} from './lu-query-compiler'; + +/** Portable scalar GPU formats accepted by global analytics and numeric histograms. @internal */ +export type LuAnalyticsScalarFormat = 'float32' | 'sint32' | 'uint32'; + +/** Portable compute-group size shared by source-aligned analytics helper passes. @internal */ +export const LU_ANALYTICS_WORKGROUP_SIZE = 256; + +const UINT32_BYTE_LENGTH = Uint32Array.BYTES_PER_ELEMENT; +const MAXIMUM_UINT32 = 0xffffffff; + +/** Rejects unsupported constants, unknown source nullability, and possible uint32-count overflow. */ +export function validateLuAnalyticsSource( + source: LuDataFrame, + columnNames: readonly string[] +): void { + if (source.numRows > MAXIMUM_UINT32) { + throw new Error('LuDataFrame analytics counts cannot represent more than uint32 source rows'); + } + for (const name of new Set(columnNames)) { + if (source.table.gpuConstants[name]) { + throw new Error(`LuDataFrame analytics column "${name}" must contain GPU vector data`); + } + const field = source.schema.fields.find(candidate => candidate.name === name); + if (field?.nullable && source.numRows > 0 && !source.validity[name as keyof Source & string]) { + throw new Error(`LuDataFrame nullable analytics column "${name}" requires GPU validity`); + } + } +} + +/** Validates dense output storage and existing one-dimensional histogram dispatch constraints. */ +export function validateLuAnalyticsOutputLength( + graph: GPUCommandGraph, + length: number +): void { + if (!Number.isSafeInteger(length) || length <= 0 || length > MAXIMUM_UINT32) { + throw new Error('LuDataFrame analytics output requires a positive uint32 length'); + } + if (length > graph.device.limits.maxComputeWorkgroupsPerDimension * LU_ANALYTICS_WORKGROUP_SIZE) { + throw new Error('LuDataFrame analytics output exceeds the supported dispatch capacity'); + } + const byteLength = length * UINT32_BYTE_LENGTH; + if ( + byteLength > graph.device.limits.maxBufferSize || + byteLength > graph.device.limits.maxStorageBufferBindingSize + ) { + throw new Error('LuDataFrame analytics output exceeds available GPU buffer capacity'); + } +} + +/** Resolves one packed selected source/derived vector or a schema-only empty vector. */ +export function getLuAnalyticsVector( + context: LuDataFrameQueryExtensionContext, + name: string +): GPUVector { + if (context.table.gpuConstants[name]) { + throw new Error(`LuDataFrame analytics column "${name}" must contain GPU vector data`); + } + const field = context.table.schema.fields.find(candidate => candidate.name === name); + const vector = context.table.gpuVectors[name]; + const format = vector?.format ?? field?.format; + if ( + !field || + (format !== 'float32' && format !== 'sint32' && format !== 'uint32') || + (!vector && context.table.batches.length > 0) + ) { + throw new Error(`LuDataFrame analytics column "${name}" requires a 32-bit scalar GPU vector`); + } + if (!vector) { + return new GPUVector({type: 'data', name, format, data: [], ownsData: false}); + } + return vector as GPUVector; +} + +/** Intersects query selection with an explicit nullable source/derived validity sidecar. */ +export function getLuAnalyticsSelectionMask( + context: LuDataFrameQueryExtensionContext, + name: string, + id: string +): GraphVectorView<'uint32'> { + const field = context.table.schema.fields.find(candidate => candidate.name === name); + if (!field?.nullable) { + return context.selectionMask; + } + const validity = context.validity[name as keyof Selection & string]; + if (!validity) { + if (context.selectionMask.length === 0) { + return context.selectionMask; + } + throw new Error(`LuDataFrame nullable analytics column "${name}" requires GPU validity`); + } + const validityView = context.graph.importGPUVector(`${id}-validity`, validity); + const output = createTransientVectorView( + context.graph, + `${id}-combined-mask`, + context.selectionMask + ); + new GPUMask({ + id: `${id}-combine-validity`, + inputs: [context.selectionMask, validityView], + output + }).addToGraph(context.graph); + return output; +} + +/** Creates graph-owned scalar scratch while retaining exact source row and batch topology. */ +export function createLuAnalyticsTransientVector( + graph: GPUCommandGraph, + id: string, + template: GraphVectorView, + format: Format +): GraphVectorView { + let emptyChunk: GraphDataView | undefined; + const data = template.data.map((chunk, chunkIndex) => { + if (chunk.length === 0) { + emptyChunk ??= createTransientView(graph, `${id}-empty`, format, 0); + return emptyChunk; + } + return createTransientView(graph, `${id}-chunk-${chunkIndex}`, format, chunk.length); + }); + return new GraphVectorView({ + id, + name: id, + format, + length: template.length, + valueLength: template.length, + stride: 1, + byteStride: UINT32_BYTE_LENGTH, + rowByteLength: UINT32_BYTE_LENGTH, + data + }); +} + +/** Allocates one explicitly owned, renderer-compatible scalar result vector. */ +export function createLuAnalyticsOutputVector( + device: Device, + name: string, + format: Format, + length: number +): GPUVector { + const buffer = device.createBuffer({ + id: name, + byteLength: Math.max(length, 1) * UINT32_BYTE_LENGTH, + usage: Buffer.STORAGE | Buffer.VERTEX | Buffer.COPY_SRC | Buffer.COPY_DST + }); + try { + const data = new GPUData({buffer, format, length, ownsBuffer: true}); + return new GPUVector({type: 'data', name, format, data: [data], ownsData: true}); + } catch (error) { + buffer.destroy(); + throw error; + } +} + +/** Builds one dense output batch from borrowed wrappers while preserving source schema metadata. */ +export function createLuAnalyticsResultTable( + source: GPUTable, + fields: readonly GPUField[], + vectors: ReadonlyMap> +): GPUTable { + const gpuData: Record = {}; + for (const field of fields) { + const data = vectors.get(field.name)?.data[0]; + if (!data) { + throw new Error(`LuDataFrame analytics result is missing column "${field.name}"`); + } + gpuData[field.name] = new GPUData({ + buffer: data.buffer, + format: data.format, + length: data.length, + ownsBuffer: false + }); + } + const batch = new GPURecordBatch({ + gpuData, + fields: [...fields], + metadata: new Map(source.schema.metadata) + }); + try { + return new GPUTable({batches: [batch]}); + } catch (error) { + batch.destroy(); + throw error; + } +} + +/** Adds one safely bound scalar computation without changing graph ownership or submission. */ +export function addLuAnalyticsComputePass( + graph: GPUCommandGraph, + props: { + id: string; + source: string; + resources: readonly GraphBufferUse[]; + bindings: Readonly>; + length: number; + } +): void { + graph.addComputePass({ + id: props.id, + resources: [...props.resources], + compile: ({device}) => { + const computation = new Computation(device, { + id: props.id, + source: props.source, + shaderLayout: { + bindings: Object.keys(props.bindings).map((name, location) => ({ + name, + type: 'storage' as const, + group: 0, + location + })) + } + }); + return { + encode: ({computePass, getBuffer}) => { + const bindings: Record = {}; + for (const [name, view] of Object.entries(props.bindings)) { + bindings[name] = getViewBinding(view, getBuffer); + } + computation.setBindings(bindings); + computation.dispatch( + computePass, + Math.max(1, Math.ceil(props.length / LU_ANALYTICS_WORKGROUP_SIZE)) + ); + }, + destroy: () => computation.destroy() + }; + } + }); +} + +/** Maps only closed scalar storage formats to native WGSL scalar names. */ +export function getLuAnalyticsShaderType(format: LuAnalyticsScalarFormat): 'f32' | 'i32' | 'u32' { + switch (format) { + case 'float32': + return 'f32'; + case 'sint32': + return 'i32'; + case 'uint32': + return 'u32'; + } +} diff --git a/modules/experimental/src/ludf/lu-data-frame-query.ts b/modules/experimental/src/ludf/lu-data-frame-query.ts index 7c7162cb50..cb0038a750 100644 --- a/modules/experimental/src/ludf/lu-data-frame-query.ts +++ b/modules/experimental/src/ludf/lu-data-frame-query.ts @@ -6,11 +6,17 @@ import type {GPUTypeMap} from '@luma.gl/tables'; import type {GPUCommandGraph} from '../gpu-primitives/gpu-command-graph'; import type {LuDataFrame} from './lu-data-frame'; import {getLuExpressionColumnNames, LuExpression, type LuExpressionNode} from './lu-expression'; +import { + LuDataFrameAggregationQuery, + type LuDataFrameGlobalAggregationDefinitions, + type LuDataFrameScalarColumnNames +} from './lu-global-aggregation-query'; import { LuDataFrameGroupByQuery, type LuDataFrameColumnNamesOfFormat, type LuDataFrameGroupByOptions } from './lu-group-by-query'; +import {LuDataFrameHistogramQuery, type LuDataFrameHistogramOptions} from './lu-histogram-query'; import { compileLuDataFrameQuery, type CompiledLuDataFrameQuery, @@ -157,6 +163,21 @@ export class LuDataFrameQuery< return new LuDataFrameGroupByQuery(this, key, options); } + /** Plans globally reduced scalar statistics without allocating or submitting GPU work. */ + aggregate>( + definitions: Definitions + ): LuDataFrameAggregationQuery { + return new LuDataFrameAggregationQuery(this, definitions); + } + + /** Plans explicit-domain histogram binning without reading or materializing source data. */ + histogram>( + column: Column, + options: LuDataFrameHistogramOptions + ): LuDataFrameHistogramQuery { + return new LuDataFrameHistogramQuery(this, column, options); + } + /** Materializes reusable GPU graph passes and compiler-owned selection/index/count outputs. */ compile( graph: GPUCommandGraph diff --git a/modules/experimental/src/ludf/lu-data-frame.ts b/modules/experimental/src/ludf/lu-data-frame.ts index 1aba87bac4..c95874d56f 100644 --- a/modules/experimental/src/ludf/lu-data-frame.ts +++ b/modules/experimental/src/ludf/lu-data-frame.ts @@ -22,11 +22,17 @@ import { type LuDataFrameDerivedColumnOptions } from './lu-data-frame-query'; import type {LuExpression} from './lu-expression'; +import type { + LuDataFrameAggregationQuery, + LuDataFrameGlobalAggregationDefinitions, + LuDataFrameScalarColumnNames +} from './lu-global-aggregation-query'; import type { LuDataFrameColumnNamesOfFormat, LuDataFrameGroupByOptions, LuDataFrameGroupByQuery } from './lu-group-by-query'; +import type {LuDataFrameHistogramOptions, LuDataFrameHistogramQuery} from './lu-histogram-query'; /** Whether a dataframe borrows its source resources or releases them after its final view. */ export type LuDataFrameOwnership = 'borrowed' | 'owned'; @@ -208,6 +214,28 @@ export class LuDataFrame { ); } + /** Plans global numeric reductions without allocating or retaining any GPU resources. */ + aggregate>( + definitions: Definitions + ): LuDataFrameAggregationQuery { + this.assertAvailable(); + return new LuDataFrameQuery(this, [], this.columnNames).aggregate( + definitions + ); + } + + /** Plans fixed-domain or irregular-edge histogram binning entirely on the CPU. */ + histogram>( + column: Column, + options: LuDataFrameHistogramOptions + ): LuDataFrameHistogramQuery { + this.assertAvailable(); + return new LuDataFrameQuery(this, [], this.columnNames).histogram( + column, + options + ); + } + /** * Returns an independent borrowed projection without mutating or destroying source columns. * diff --git a/modules/experimental/src/ludf/lu-global-aggregation-compiler.ts b/modules/experimental/src/ludf/lu-global-aggregation-compiler.ts new file mode 100644 index 0000000000..adc5027e63 --- /dev/null +++ b/modules/experimental/src/ludf/lu-global-aggregation-compiler.ts @@ -0,0 +1,423 @@ +// luma.gl +// SPDX-License-Identifier: MIT +// Copyright (c) vis.gl contributors + +import {GPUVector, type GPUField, type GPUTypeMap} from '@luma.gl/tables'; +import { + type GPUCommandGraph, + type GraphBufferUse, + type GraphDataView, + type GraphVectorView +} from '../gpu-primitives/gpu-command-graph'; +import {GPUReduction} from '../gpu-primitives/gpu-reduction'; +import {createTransientView, getViewElementOffset} from '../gpu-primitives/graph-data-view-utils'; +import { + LU_ANALYTICS_WORKGROUP_SIZE, + addLuAnalyticsComputePass, + createLuAnalyticsOutputVector, + createLuAnalyticsResultTable, + createLuAnalyticsTransientVector, + getLuAnalyticsSelectionMask, + getLuAnalyticsShaderType, + getLuAnalyticsVector, + validateLuAnalyticsSource, + type LuAnalyticsScalarFormat +} from './lu-analytics-compiler-utils'; +import type {LuDataFrame, LuDataFrameDictionaries, LuDataFrameValidity} from './lu-data-frame'; +import type {LuDataFrameDerivedColumn} from './lu-data-frame-query'; +import type {LuExpression} from './lu-expression'; +import type {LuDataFrameAggregationDefinition} from './lu-group-by-query'; +import { + CompiledLuDataFrameQuery, + compileLuDataFrameQuery, + type LuDataFrameQueryExtensionContext, + type LuDataFrameQueryExtensionResult, + type LuDataFrameQueryParameters +} from './lu-query-compiler'; + +type LuGlobalMetricState = { + values: GraphVectorView; + acceptedRows: GraphVectorView<'uint32'>; + acceptedCount: GraphDataView<'uint32'>; + validity: GPUVector<'uint32'>; + sanitized: Map>; +}; + +/** One-row GPU-resident global aggregation with source-aligned selection and explicit validity. */ +export class CompiledLuDataFrameAggregation< + T extends GPUTypeMap = GPUTypeMap +> extends CompiledLuDataFrameQuery {} + +/** Lowers global statistics into masked, chunk-preserving GPU reductions before graph compilation. */ +export function compileLuDataFrameAggregation< + Source extends GPUTypeMap, + Selection extends GPUTypeMap, + Result extends GPUTypeMap +>( + source: LuDataFrame, + predicates: readonly LuExpression[], + selectedColumns: readonly (keyof Selection & string)[], + derivedColumns: readonly LuDataFrameDerivedColumn[], + definitions: readonly LuDataFrameAggregationDefinition[], + graph: GPUCommandGraph +): CompiledLuDataFrameAggregation { + validateLuAnalyticsSource( + source, + definitions.flatMap(definition => (definition.column ? [definition.column] : [])) + ); + return compileLuDataFrameQuery>( + source, + predicates, + selectedColumns, + graph, + derivedColumns, + { + allowEmptyPredicates: true, + prepare: context => addLuGlobalAggregationsToGraph(context, definitions) + } + ); +} + +/** Materializes independent one-row statistics while sharing per-column acceptance and validity. */ +function addLuGlobalAggregationsToGraph( + context: LuDataFrameQueryExtensionContext, + definitions: readonly LuDataFrameAggregationDefinition[] +): LuDataFrameQueryExtensionResult> { + const prefix = `${context.queryId}-global`; + const ownedVectors: GPUVector[] = []; + let table: ReturnType> | undefined; + + try { + const vectors = new Map>(); + const fields: GPUField[] = []; + const validity: Record> = {}; + const metricStates = new Map(); + + for (const [definitionIndex, definition] of definitions.entries()) { + const id = `${prefix}-metric-${definitionIndex}`; + if (definition.operation === 'count') { + const output = createLuAnalyticsOutputVector(context.graph.device, id, 'uint32', 1); + ownedVectors.push(output); + vectors.set(definition.name, output); + fields.push({ + name: definition.name, + format: 'uint32', + nullable: false, + metadata: new Map() + }); + new GPUReduction({ + id, + input: context.selectionMask, + output: context.graph.importGPUVector(`${id}-output`, output).data[0], + operation: 'sum' + }).addToGraph(context.graph); + continue; + } + + const name = definition.column; + if (!name) { + throw new Error('LuDataFrame global statistics require a numeric source column'); + } + let state = metricStates.get(name); + if (!state) { + state = createLuGlobalMetricState( + context, + name, + `${prefix}-values-${metricStates.size}`, + ownedVectors + ); + metricStates.set(name, state); + } + + const format = definition.operation === 'mean' ? 'float32' : state.values.format; + const output = createLuAnalyticsOutputVector(context.graph.device, id, format, 1); + ownedVectors.push(output); + vectors.set(definition.name, output); + validity[definition.name] = state.validity; + fields.push({name: definition.name, format, nullable: true, metadata: new Map()}); + + const sanitized = getLuSanitizedMetricValues(context, state, definition.operation, id); + const outputView = context.graph.importGPUVector(`${id}-output`, output).data[0]; + new GPUReduction({ + id: `${id}-reduce`, + input: sanitized, + output: outputView, + operation: definition.operation === 'mean' ? 'sum' : definition.operation + }).addToGraph(context.graph); + if (definition.operation !== 'sum') { + addLuFinalizeGlobalMetricPass( + context.graph, + `${id}-finalize`, + outputView, + state.acceptedCount, + definition.operation + ); + } + } + + table = createLuAnalyticsResultTable(context.table, fields, vectors); + return { + table, + validity: Object.freeze(validity) as Readonly>, + dictionaries: Object.freeze({}) as Readonly>, + ownedTables: [table], + ownedVectors, + createCompiled: props => new CompiledLuDataFrameAggregation(props) + }; + } catch (error) { + table?.destroy(); + for (const vector of ownedVectors) { + vector.destroy(); + } + throw error; + } +} + +/** Builds one source-aligned null/finite mask, one accepted-row reduction, and one validity flag. */ +function createLuGlobalMetricState( + context: LuDataFrameQueryExtensionContext, + name: string, + id: string, + ownedVectors: GPUVector[] +): LuGlobalMetricState { + const vector = getLuAnalyticsVector(context, name); + const values = context.graph.importGPUVector(`${id}-input`, vector); + const selectedRows = getLuAnalyticsSelectionMask(context, name, id); + const acceptedRows = + values.format === 'float32' + ? createLuGlobalFiniteMask( + context.graph, + `${id}-finite`, + values as GraphVectorView<'float32'>, + selectedRows + ) + : selectedRows; + const acceptedCount = createTransientView(context.graph, `${id}-accepted-count`, 'uint32', 1); + new GPUReduction({ + id: `${id}-count-valid`, + input: acceptedRows, + output: acceptedCount, + operation: 'sum' + }).addToGraph(context.graph); + + const validity = createLuAnalyticsOutputVector( + context.graph.device, + `${id}-output-validity`, + 'uint32', + 1 + ); + ownedVectors.push(validity); + addLuGlobalValidityPass( + context.graph, + `${id}-normalize-validity`, + acceptedCount, + context.graph.importGPUVector(`${id}-validity-vector`, validity).data[0] + ); + return {values, acceptedRows, acceptedCount, validity, sanitized: new Map()}; +} + +/** Removes NaN and infinity after query selection and nullable source validity are applied. */ +function createLuGlobalFiniteMask( + graph: GPUCommandGraph, + id: string, + values: GraphVectorView<'float32'>, + selectedRows: GraphVectorView<'uint32'> +): GraphVectorView<'uint32'> { + const output = createLuAnalyticsTransientVector(graph, id, selectedRows, 'uint32'); + for (const [chunkIndex, mask] of output.data.entries()) { + if (mask.length === 0) { + continue; + } + const input = values.data[chunkIndex]; + const selection = selectedRows.data[chunkIndex]; + const source = /* wgsl */ ` +const ELEMENT_COUNT: u32 = ${mask.length}u; +const INPUT_OFFSET: u32 = ${getViewElementOffset(input)}u; +const SELECTION_OFFSET: u32 = ${getViewElementOffset(selection)}u; +const OUTPUT_OFFSET: u32 = ${getViewElementOffset(mask)}u; +@group(0) @binding(0) var inputValues: array; +@group(0) @binding(1) var selectionMask: array; +@group(0) @binding(2) var outputMask: array; + +@compute @workgroup_size(${LU_ANALYTICS_WORKGROUP_SIZE}) +fn main(@builtin(global_invocation_id) globalId: vec3) { + let index = globalId.x; + if (index < ELEMENT_COUNT) { + let value = inputValues[INPUT_OFFSET + index]; + let finite = value == value && abs(value) <= 3.402823466e+38; + outputMask[OUTPUT_OFFSET + index] = select( + 0u, + 1u, + selectionMask[SELECTION_OFFSET + index] != 0u && finite + ); + } +}`; + addLuAnalyticsComputePass(graph, { + id: `${id}-chunk-${chunkIndex}`, + source, + resources: [ + {buffer: input, usage: 'storage-read'}, + {buffer: selection, usage: 'storage-read'}, + {buffer: mask, usage: 'storage-write'} + ], + bindings: {inputValues: input, selectionMask: selection, outputMask: mask}, + length: mask.length + }); + } + return output; +} + +/** Creates cached source-aligned values with operation-specific identity rows for invalid inputs. */ +function getLuSanitizedMetricValues( + context: LuDataFrameQueryExtensionContext, + state: LuGlobalMetricState, + operation: Exclude, + id: string +): GraphVectorView { + const outputFormat = operation === 'mean' ? 'float32' : state.values.format; + const cacheKey = `${outputFormat}-${operation === 'mean' ? 'sum' : operation}`; + const cached = state.sanitized.get(cacheKey); + if (cached) { + return cached; + } + const output = createLuAnalyticsTransientVector( + context.graph, + `${id}-sanitized`, + state.values, + outputFormat + ); + const inputType = getLuAnalyticsShaderType(state.values.format); + const outputType = getLuAnalyticsShaderType(outputFormat); + const identity = getLuGlobalIdentity(outputFormat, operation); + + for (const [chunkIndex, destination] of output.data.entries()) { + if (destination.length === 0) { + continue; + } + const input = state.values.data[chunkIndex]; + const mask = state.acceptedRows.data[chunkIndex]; + const source = /* wgsl */ ` +const ELEMENT_COUNT: u32 = ${destination.length}u; +const INPUT_OFFSET: u32 = ${getViewElementOffset(input)}u; +const MASK_OFFSET: u32 = ${getViewElementOffset(mask)}u; +const OUTPUT_OFFSET: u32 = ${getViewElementOffset(destination)}u; +@group(0) @binding(0) var inputValues: array<${inputType}>; +@group(0) @binding(1) var inputMask: array; +@group(0) @binding(2) var outputValues: array<${outputType}>; + +@compute @workgroup_size(${LU_ANALYTICS_WORKGROUP_SIZE}) +fn main(@builtin(global_invocation_id) globalId: vec3) { + let index = globalId.x; + if (index < ELEMENT_COUNT) { + let value = ${outputType}(inputValues[INPUT_OFFSET + index]); + outputValues[OUTPUT_OFFSET + index] = select( + ${identity}, + value, + inputMask[MASK_OFFSET + index] != 0u + ); + } +}`; + addLuAnalyticsComputePass(context.graph, { + id: `${id}-sanitize-chunk-${chunkIndex}`, + source, + resources: [ + {buffer: input, usage: 'storage-read'}, + {buffer: mask, usage: 'storage-read'}, + {buffer: destination, usage: 'storage-write'} + ], + bindings: {inputValues: input, inputMask: mask, outputValues: destination}, + length: destination.length + }); + } + + state.sanitized.set(cacheKey, output); + return output; +} + +/** Returns the operation identity used by masked scalar reductions. */ +function getLuGlobalIdentity( + format: LuAnalyticsScalarFormat, + operation: Exclude +): string { + if (operation === 'sum' || operation === 'mean') { + return format === 'float32' ? '0.0' : format === 'uint32' ? '0u' : '0i'; + } + if (format === 'float32') { + return operation === 'min' ? '3.402823466e+38' : '-3.402823466e+38'; + } + if (operation === 'min') { + return format === 'uint32' ? '0xffffffffu' : '2147483647i'; + } + return format === 'uint32' ? '0u' : 'bitcast(0x80000000u)'; +} + +/** Publishes one canonical 0/1 validity value without destroying the accepted-row count. */ +function addLuGlobalValidityPass( + graph: GPUCommandGraph, + id: string, + acceptedCount: GraphDataView<'uint32'>, + validity: GraphDataView<'uint32'> +): void { + const source = /* wgsl */ ` +const COUNT_OFFSET: u32 = ${getViewElementOffset(acceptedCount)}u; +const VALIDITY_OFFSET: u32 = ${getViewElementOffset(validity)}u; +@group(0) @binding(0) var acceptedCounts: array; +@group(0) @binding(1) var outputValidity: array; + +@compute @workgroup_size(1) +fn main() { + outputValidity[VALIDITY_OFFSET] = select(0u, 1u, acceptedCounts[COUNT_OFFSET] != 0u); +}`; + addLuAnalyticsComputePass(graph, { + id, + source, + resources: [ + {buffer: acceptedCount, usage: 'storage-read'}, + {buffer: validity, usage: 'storage-write'} + ], + bindings: {acceptedCounts: acceptedCount, outputValidity: validity}, + length: 1 + }); +} + +/** Applies empty-input semantics and computes floating-point means from valid contribution counts. */ +function addLuFinalizeGlobalMetricPass( + graph: GPUCommandGraph, + id: string, + output: GraphDataView, + acceptedCount: GraphDataView<'uint32'>, + operation: 'min' | 'max' | 'mean' +): void { + const outputType = getLuAnalyticsShaderType(output.format); + const empty = output.format === 'float32' ? 'bitcast(count | 0x7fc00000u)' : '0'; + const populated = + operation === 'mean' + ? 'outputValues[OUTPUT_OFFSET] / f32(count)' + : 'outputValues[OUTPUT_OFFSET]'; + const source = /* wgsl */ ` +const COUNT_OFFSET: u32 = ${getViewElementOffset(acceptedCount)}u; +const OUTPUT_OFFSET: u32 = ${getViewElementOffset(output)}u; +@group(0) @binding(0) var acceptedCounts: array; +@group(0) @binding(1) var outputValues: array<${outputType}>; + +@compute @workgroup_size(1) +fn main() { + let count = acceptedCounts[COUNT_OFFSET]; + if (count == 0u) { + outputValues[OUTPUT_OFFSET] = ${empty}; + } else { + outputValues[OUTPUT_OFFSET] = ${populated}; + } +}`; + const resources: GraphBufferUse[] = [ + {buffer: acceptedCount, usage: 'storage-read'}, + {buffer: output, usage: 'storage-read-write'} + ]; + addLuAnalyticsComputePass(graph, { + id, + source, + resources, + bindings: {acceptedCounts: acceptedCount, outputValues: output}, + length: 1 + }); +} diff --git a/modules/experimental/src/ludf/lu-global-aggregation-query.ts b/modules/experimental/src/ludf/lu-global-aggregation-query.ts new file mode 100644 index 0000000000..db7c49a0ec --- /dev/null +++ b/modules/experimental/src/ludf/lu-global-aggregation-query.ts @@ -0,0 +1,178 @@ +// luma.gl +// SPDX-License-Identifier: MIT +// Copyright (c) vis.gl contributors + +import type {GPUTypeMap} from '@luma.gl/tables'; +import type {GPUCommandGraph} from '../gpu-primitives/gpu-command-graph'; +import type {LuDataFrameQuery} from './lu-data-frame-query'; +import {getLuExpressionColumnNames} from './lu-expression'; +import { + compileLuDataFrameAggregation, + type CompiledLuDataFrameAggregation +} from './lu-global-aggregation-compiler'; +import type {LuDataFrameAggregationDefinition} from './lu-group-by-query'; +import type {LuDataFrameQueryParameters} from './lu-query-compiler'; + +/** Fixed-width numeric formats supported by GPU reductions and histogram binning. */ +export type LuDataFrameAnalyticScalarFormat = 'float32' | 'sint32' | 'uint32'; + +/** Selected logical column names carrying one portable GPU-native numeric scalar value. */ +export type LuDataFrameScalarColumnNames< + T extends GPUTypeMap, + SelectedColumns extends keyof T & string +> = { + [Name in SelectedColumns]: T[Name] extends LuDataFrameAnalyticScalarFormat ? Name : never; +}[SelectedColumns]; + +/** One source-row count or a global numeric statistic referencing a selected scalar column. */ +export type LuDataFrameGlobalAggregationValue< + T extends GPUTypeMap, + SelectedColumns extends keyof T & string +> = + | 'count' + | Readonly<{sum: LuDataFrameScalarColumnNames}> + | Readonly<{min: LuDataFrameScalarColumnNames}> + | Readonly<{max: LuDataFrameScalarColumnNames}> + | Readonly<{mean: LuDataFrameScalarColumnNames}>; + +/** Caller-defined global reduction output names and closed statistic descriptions. */ +export type LuDataFrameGlobalAggregationDefinitions< + T extends GPUTypeMap, + SelectedColumns extends keyof T & string = keyof T & string +> = Readonly>>; + +/** Exact result scalar format retained for one requested global statistic. */ +type LuDataFrameGlobalAggregationValueFormat = Value extends 'count' + ? 'uint32' + : Value extends {mean: string} + ? 'float32' + : Value extends {sum: infer Column extends keyof T & string} + ? Extract + : Value extends {min: infer Column extends keyof T & string} + ? Extract + : Value extends {max: infer Column extends keyof T & string} + ? Extract + : never; + +/** Exact one-row GPU table formats produced by caller-defined global numeric reductions. */ +export type LuDataFrameGlobalAggregationResult< + T extends GPUTypeMap, + Definitions extends Readonly> +> = { + [Name in keyof Definitions & string]: LuDataFrameGlobalAggregationValueFormat< + T, + Definitions[Name] + >; +}; + +/** + * Immutable one-row global count, sum, minimum, maximum, and mean reduction query. + * + * Sum/min/max retain the source column format; mean always produces float32, and count produces + * uint32. Explicit GPU validity sidecars distinguish empty or fully rejected statistic inputs. + */ +export class LuDataFrameAggregationQuery< + Logical extends GPUTypeMap, + SelectedColumns extends keyof Logical & string, + Definitions extends LuDataFrameGlobalAggregationDefinitions, + Source extends GPUTypeMap = Logical +> { + /** Original immutable filtered, projected, or derived source-row query. */ + readonly query: LuDataFrameQuery; + /** Closed GPU reduction operations in stable caller-provided output-column order. */ + readonly definitions: readonly LuDataFrameAggregationDefinition[]; + + /** Validates selected 32-bit scalar inputs without acquiring a GPU source lease. @internal */ + constructor(query: LuDataFrameQuery, definitions: Definitions) { + this.query = query; + this.definitions = Object.freeze(normalizeLuDataFrameGlobalAggregations(query, definitions)); + Object.freeze(this); + } + + /** Adds filtered, null-aware reductions to one reusable application-owned command graph. */ + compile( + graph: GPUCommandGraph + ): CompiledLuDataFrameAggregation> { + return compileLuDataFrameAggregation< + Source, + Pick, + LuDataFrameGlobalAggregationResult + >( + this.query.source, + this.query.predicates, + this.query.selectedColumns, + this.query.derivedColumns, + this.definitions, + graph + ); + } +} + +/** Resolves one logical source/derived scalar format from canonical GPU column metadata. @internal */ +export function getLuDataFrameAnalyticColumnFormat< + Logical extends GPUTypeMap, + SelectedColumns extends keyof Logical & string, + Source extends GPUTypeMap +>( + query: LuDataFrameQuery, + name: string +): LuDataFrameAnalyticScalarFormat | undefined { + const formats = new Map( + query.source.schema.fields.map( + field => + [field.name, query.source.table.gpuColumns[field.name]?.format ?? field.format] as const + ) + ); + for (const definition of query.derivedColumns) { + const firstReference = getLuExpressionColumnNames(definition.expression)[0]; + formats.set(definition.name, definition.format ?? formats.get(firstReference) ?? 'float32'); + } + const format = formats.get(name); + return format === 'float32' || format === 'sint32' || format === 'uint32' ? format : undefined; +} + +/** Rejects unselected columns and non-closed aggregation operators before touching GPU state. */ +function normalizeLuDataFrameGlobalAggregations< + Logical extends GPUTypeMap, + SelectedColumns extends keyof Logical & string, + Source extends GPUTypeMap +>( + query: LuDataFrameQuery, + definitions: LuDataFrameGlobalAggregationDefinitions +): LuDataFrameAggregationDefinition[] { + const entries = Object.entries(definitions); + if (entries.length === 0) { + throw new Error('LuDataFrame global reductions require at least one aggregation'); + } + + const normalized: LuDataFrameAggregationDefinition[] = []; + for (const [name, value] of entries) { + if (name.length === 0) { + throw new Error('LuDataFrame global aggregations require nonempty output names'); + } + if (value === 'count') { + normalized.push(Object.freeze({name, operation: 'count'})); + continue; + } + if (!value || typeof value !== 'object') { + throw new Error(`LuDataFrame global aggregation "${name}" has an unsupported operation`); + } + const entries = Object.entries(value); + if (entries.length !== 1) { + throw new Error(`LuDataFrame global aggregation "${name}" requires exactly one operation`); + } + const [operation, column] = entries[0]; + if (operation !== 'sum' && operation !== 'min' && operation !== 'max' && operation !== 'mean') { + throw new Error(`LuDataFrame global aggregation "${name}" has an unsupported operation`); + } + if (typeof column !== 'string' || !query.selectedColumns.includes(column as SelectedColumns)) { + throw new Error(`LuDataFrame global aggregation "${name}" requires a selected input column`); + } + if (!getLuDataFrameAnalyticColumnFormat(query, column)) { + throw new Error(`LuDataFrame global aggregation "${name}" requires a scalar GPU column`); + } + normalized.push(Object.freeze({name, operation, column})); + } + + return normalized; +} diff --git a/modules/experimental/src/ludf/lu-histogram-compiler.ts b/modules/experimental/src/ludf/lu-histogram-compiler.ts new file mode 100644 index 0000000000..e20d6908e2 --- /dev/null +++ b/modules/experimental/src/ludf/lu-histogram-compiler.ts @@ -0,0 +1,189 @@ +// luma.gl +// SPDX-License-Identifier: MIT +// Copyright (c) vis.gl contributors + +import {GPUVector, type GPUField, type GPUTypeMap} from '@luma.gl/tables'; +import {type GPUCommandGraph, type GraphDataView} from '../gpu-primitives/gpu-command-graph'; +import {GPUHistogram} from '../gpu-primitives/gpu-histogram'; +import {getViewElementOffset} from '../gpu-primitives/graph-data-view-utils'; +import { + LU_ANALYTICS_WORKGROUP_SIZE, + addLuAnalyticsComputePass, + createLuAnalyticsOutputVector, + createLuAnalyticsResultTable, + getLuAnalyticsSelectionMask, + getLuAnalyticsVector, + validateLuAnalyticsOutputLength, + validateLuAnalyticsSource, + type LuAnalyticsScalarFormat +} from './lu-analytics-compiler-utils'; +import type {LuDataFrame, LuDataFrameDictionaries, LuDataFrameValidity} from './lu-data-frame'; +import type {LuDataFrameDerivedColumn} from './lu-data-frame-query'; +import type {LuExpression} from './lu-expression'; +import type {LuDataFrameHistogramOptions} from './lu-histogram-query'; +import { + CompiledLuDataFrameQuery, + compileLuDataFrameQuery, + type CompiledLuDataFrameQueryProps, + type LuDataFrameQueryExtensionContext, + type LuDataFrameQueryExtensionResult, + type LuDataFrameQueryParameters +} from './lu-query-compiler'; + +type LuHistogramResult = {bin: 'uint32'; count: 'uint32'}; + +/** Dense source-aligned histogram with explicit, immutable numeric bin metadata. */ +export class CompiledLuDataFrameHistogram extends CompiledLuDataFrameQuery { + /** Number of dense GPU-resident histogram bins. */ + readonly binCount: number; + /** Inclusive equal-width source domain, when explicit boundaries are not supplied. */ + readonly domain?: readonly [number, number]; + /** Explicit strictly increasing bin boundaries, when using irregular intervals. */ + readonly edges?: readonly number[]; + + /** @internal */ + constructor( + props: CompiledLuDataFrameQueryProps, + options: LuDataFrameHistogramOptions + ) { + super(props); + if ('edges' in options) { + this.edges = Object.freeze([...options.edges]); + this.binCount = options.edges.length - 1; + } else { + this.domain = Object.freeze([...options.domain]) as readonly [number, number]; + this.binCount = options.bins; + } + } +} + +/** Adds filtered numeric histogram work to the same graph as the source dataframe query. */ +export function compileLuDataFrameHistogram< + Source extends GPUTypeMap, + Selection extends GPUTypeMap +>( + source: LuDataFrame, + predicates: readonly LuExpression[], + selectedColumns: readonly (keyof Selection & string)[], + derivedColumns: readonly LuDataFrameDerivedColumn[], + column: keyof Selection & string, + options: LuDataFrameHistogramOptions, + graph: GPUCommandGraph +): CompiledLuDataFrameHistogram { + validateLuAnalyticsSource(source, [column]); + validateLuAnalyticsOutputLength( + graph, + 'edges' in options ? options.edges.length - 1 : options.bins + ); + return compileLuDataFrameQuery< + Source, + Selection, + LuHistogramResult, + CompiledLuDataFrameHistogram + >(source, predicates, selectedColumns, graph, derivedColumns, { + allowEmptyPredicates: true, + prepare: context => addLuHistogramToGraph(context, column, options) + }); +} + +/** Initializes dense bin IDs and applies source null/selection masks to native histogram passes. */ +function addLuHistogramToGraph( + context: LuDataFrameQueryExtensionContext, + column: keyof Selection & string, + options: LuDataFrameHistogramOptions +): LuDataFrameQueryExtensionResult { + const prefix = `${context.queryId}-histogram`; + const binCount = 'edges' in options ? options.edges.length - 1 : options.bins; + const ownedVectors: GPUVector[] = []; + let table: + | ReturnType> + | undefined; + + try { + const vector = getLuAnalyticsVector(context, column); + const input = context.graph.importGPUVector(`${prefix}-input`, vector); + const mask = getLuAnalyticsSelectionMask(context, column, prefix); + const bins = createLuAnalyticsOutputVector( + context.graph.device, + `${prefix}-bins`, + 'uint32', + binCount + ); + ownedVectors.push(bins); + const counts = createLuAnalyticsOutputVector( + context.graph.device, + `${prefix}-counts`, + 'uint32', + binCount + ); + ownedVectors.push(counts); + const binView = context.graph.importGPUVector(`${prefix}-bin-vector`, bins).data[0]; + const output = context.graph.importGPUVector(`${prefix}-count-vector`, counts).data[0]; + addLuHistogramBinIdentityPass(context.graph, `${prefix}-initialize-bins`, binView); + + if ('edges' in options) { + new GPUHistogram({id: prefix, input, output, mask, edges: options.edges}).addToGraph( + context.graph + ); + } else { + new GPUHistogram({id: prefix, input, output, mask, domain: options.domain}).addToGraph( + context.graph + ); + } + + const fields: GPUField[] = [ + {name: 'bin', format: 'uint32', nullable: false, metadata: new Map()}, + {name: 'count', format: 'uint32', nullable: false, metadata: new Map()} + ]; + const vectors = new Map>([ + ['bin', bins], + ['count', counts] + ]); + table = createLuAnalyticsResultTable( + context.table, + fields, + vectors + ); + + return { + table, + validity: Object.freeze({}) as Readonly>, + dictionaries: Object.freeze({}) as Readonly>, + ownedTables: [table], + ownedVectors, + createCompiled: props => new CompiledLuDataFrameHistogram(props, options) + }; + } catch (error) { + table?.destroy(); + for (const vector of ownedVectors) { + vector.destroy(); + } + throw error; + } +} + +/** Publishes deterministic dense histogram-bin identities directly from a GPU compute pass. */ +function addLuHistogramBinIdentityPass( + graph: GPUCommandGraph, + id: string, + output: GraphDataView<'uint32'> +): void { + const source = /* wgsl */ ` +const ELEMENT_COUNT: u32 = ${output.length}u; +const OUTPUT_OFFSET: u32 = ${getViewElementOffset(output)}u; +@group(0) @binding(0) var outputBins: array; + +@compute @workgroup_size(${LU_ANALYTICS_WORKGROUP_SIZE}) +fn main(@builtin(global_invocation_id) globalId: vec3) { + if (globalId.x < ELEMENT_COUNT) { + outputBins[OUTPUT_OFFSET + globalId.x] = globalId.x; + } +}`; + addLuAnalyticsComputePass(graph, { + id, + source, + resources: [{buffer: output, usage: 'storage-write'}], + bindings: {outputBins: output}, + length: output.length + }); +} diff --git a/modules/experimental/src/ludf/lu-histogram-query.ts b/modules/experimental/src/ludf/lu-histogram-query.ts new file mode 100644 index 0000000000..2c65273480 --- /dev/null +++ b/modules/experimental/src/ludf/lu-histogram-query.ts @@ -0,0 +1,150 @@ +// luma.gl +// SPDX-License-Identifier: MIT +// Copyright (c) vis.gl contributors + +import type {GPUTypeMap} from '@luma.gl/tables'; +import type {GPUCommandGraph} from '../gpu-primitives/gpu-command-graph'; +import type {LuDataFrameQuery} from './lu-data-frame-query'; +import { + getLuDataFrameAnalyticColumnFormat, + type LuDataFrameScalarColumnNames +} from './lu-global-aggregation-query'; +import { + compileLuDataFrameHistogram, + type CompiledLuDataFrameHistogram +} from './lu-histogram-compiler'; +import type {LuDataFrameQueryParameters} from './lu-query-compiler'; + +const MAXIMUM_UINT32 = 0xffffffff; +const MAXIMUM_LITERAL_HISTOGRAM_EDGES = 257; + +/** Explicit equal-width domain or strictly increasing irregular numeric histogram boundaries. */ +export type LuDataFrameHistogramOptions = + | Readonly<{bins: number; domain: readonly [number, number]}> + | Readonly<{edges: readonly number[]}>; + +/** + * Immutable source-aligned numeric histogram query. + * + * Domains are always explicit because automatic whole-vector extents would include rows excluded + * by filters or explicit null masks. Histogram planning never allocates GPU storage or reads data. + */ +export class LuDataFrameHistogramQuery< + Logical extends GPUTypeMap, + SelectedColumns extends keyof Logical & string, + Column extends LuDataFrameScalarColumnNames, + Source extends GPUTypeMap = Logical +> { + /** Original immutable filtered/derived source plan. */ + readonly query: LuDataFrameQuery; + /** Selected float32, sint32, or uint32 source column. */ + readonly column: Column; + /** Deep-frozen explicit histogram domain or irregular edges. */ + readonly options: LuDataFrameHistogramOptions; + /** Number of dense GPU-resident output bins. */ + readonly binCount: number; + + /** Validates representable source-domain metadata without touching GPU resources. @internal */ + constructor( + query: LuDataFrameQuery, + column: Column, + options: LuDataFrameHistogramOptions + ) { + if (!query.selectedColumns.includes(column)) { + throw new Error(`LuDataFrame histogram column "${column}" is not selected`); + } + const format = getLuDataFrameAnalyticColumnFormat(query, column); + if (!format) { + throw new Error(`LuDataFrame histogram column "${column}" requires scalar GPU data`); + } + + const normalized = normalizeLuDataFrameHistogramOptions(options, format); + this.query = query; + this.column = column; + this.options = normalized; + this.binCount = 'edges' in normalized ? normalized.edges.length - 1 : normalized.bins; + Object.freeze(this); + } + + /** Adds histogram work to the same reusable graph as source filtering and derived expressions. */ + compile(graph: GPUCommandGraph): CompiledLuDataFrameHistogram { + return compileLuDataFrameHistogram>( + this.query.source, + this.query.predicates, + this.query.selectedColumns, + this.query.derivedColumns, + this.column, + this.options, + graph + ); + } +} + +/** Validates and deeply freezes explicit histogram metadata before GPU compilation. */ +function normalizeLuDataFrameHistogramOptions( + options: LuDataFrameHistogramOptions, + format: 'float32' | 'sint32' | 'uint32' +): LuDataFrameHistogramOptions { + if (!options || typeof options !== 'object') { + throw new Error('LuDataFrame histograms require an explicit domain or literal edges'); + } + if ('edges' in options) { + if ('bins' in options || 'domain' in options) { + throw new Error('LuDataFrame histogram edges cannot be combined with an equal-width domain'); + } + if ( + !Array.isArray(options.edges) || + options.edges.length < 2 || + options.edges.length > MAXIMUM_LITERAL_HISTOGRAM_EDGES + ) { + throw new Error('LuDataFrame histogram edges require between 2 and 257 values'); + } + const edges = options.edges.map(value => normalizeLuDataFrameHistogramBoundary(value, format)); + if (edges.some((value, index) => index > 0 && value <= edges[index - 1])) { + throw new Error('LuDataFrame histogram edges must be strictly increasing'); + } + return Object.freeze({edges: Object.freeze(edges)}); + } + + if (!('bins' in options) || !('domain' in options)) { + throw new Error('LuDataFrame histograms require an explicit bin count and domain'); + } + if (!Number.isSafeInteger(options.bins) || options.bins < 1 || options.bins > MAXIMUM_UINT32) { + throw new Error('LuDataFrame histograms require a positive uint32 bin count'); + } + if (!Array.isArray(options.domain) || options.domain.length !== 2) { + throw new Error('LuDataFrame histograms require a finite [min, max] domain'); + } + const minimum = normalizeLuDataFrameHistogramBoundary(options.domain[0], format); + const maximum = normalizeLuDataFrameHistogramBoundary(options.domain[1], format); + if (minimum > maximum) { + throw new Error('LuDataFrame histogram domain minimum cannot exceed its maximum'); + } + return Object.freeze({ + bins: options.bins, + domain: Object.freeze([minimum, maximum] as [number, number]) + }); +} + +/** Ensures literal histogram boundaries fit their exact GPU scalar storage representation. */ +function normalizeLuDataFrameHistogramBoundary( + value: number, + format: 'float32' | 'sint32' | 'uint32' +): number { + if (typeof value !== 'number' || !Number.isFinite(value)) { + throw new Error('LuDataFrame histogram boundaries must contain finite numbers'); + } + if (format === 'float32') { + const rounded = Math.fround(value); + if (!Number.isFinite(rounded)) { + throw new Error('LuDataFrame histogram boundaries must fit their float32 column'); + } + return rounded; + } + const minimum = format === 'uint32' ? 0 : -0x80000000; + const maximum = format === 'uint32' ? MAXIMUM_UINT32 : 0x7fffffff; + if (!Number.isInteger(value) || value < minimum || value > maximum) { + throw new Error(`LuDataFrame histogram boundaries must fit their ${format} column`); + } + return value; +} diff --git a/modules/experimental/test/index.ts b/modules/experimental/test/index.ts index a21b3e2ddd..fed4dc005a 100644 --- a/modules/experimental/test/index.ts +++ b/modules/experimental/test/index.ts @@ -43,6 +43,7 @@ import './ludf/lu-data-frame.spec'; import './ludf/lu-data-frame-query.spec'; import './ludf/lu-derived-columns.spec'; import './ludf/lu-group-aggregation.spec'; +import './ludf/lu-global-aggregation.spec'; import './luraster'; import './luxfilter'; import './luproj/luproj.spec'; diff --git a/modules/experimental/test/ludf/lu-global-aggregation.spec.ts b/modules/experimental/test/ludf/lu-global-aggregation.spec.ts new file mode 100644 index 0000000000..12e0f6f9c4 --- /dev/null +++ b/modules/experimental/test/ludf/lu-global-aggregation.spec.ts @@ -0,0 +1,710 @@ +// luma.gl +// SPDX-License-Identifier: MIT +// Copyright (c) vis.gl contributors + +import {Buffer, type Device} from '@luma.gl/core'; +import {GPUCommandGraph} from '@luma.gl/experimental'; +import { + LuDataFrame, + column, + literal, + parameter, + type LuDataFrameQueryParameters +} from '@luma.gl/experimental/ludf'; +import {GPUData, GPURecordBatch, GPUTable, GPUVector} from '@luma.gl/tables'; +import {getWebGPUTestDevice} from '@luma.gl/test-utils'; +import test from 'test/utils/vitest-tape'; +import {vi} from 'vitest'; + +type LuAnalyticsSourceSchema = { + category: 'uint32'; + fare: 'float32'; + distance: 'sint32'; +}; + +type LuAnalyticsFixture = { + frame: LuDataFrame; + sourceBuffers: Buffer[]; +}; + +test('LuDataFrame reduces nullable scalar GPU vectors without flattening source batches', async testContext => { + const device = await getWebGPUTestDevice(); + if (!device) { + testContext.comment('WebGPU is not available'); + testContext.end(); + return; + } + + const fixture = createLuAnalyticsFixture(device); + const createBufferSpy = vi.spyOn(device, 'createBuffer'); + const submitSpy = vi.spyOn(device, 'submit'); + const query = fixture.frame.aggregate({ + rowCount: 'count', + totalFare: {sum: 'fare'}, + minimumFare: {min: 'fare'}, + maximumFare: {max: 'fare'}, + averageFare: {mean: 'fare'}, + totalDistance: {sum: 'distance'}, + minimumDistance: {min: 'distance'}, + averageDistance: {mean: 'distance'}, + maximumCategory: {max: 'category'}, + totalCategory: {sum: 'category'}, + averageCategory: {mean: 'category'} + }); + + testContext.equal( + createBufferSpy.mock.calls.length, + 0, + 'immutable global-aggregation planning allocates no GPU storage' + ); + testContext.equal( + submitSpy.mock.calls.length, + 0, + 'immutable global-aggregation planning submits no GPU commands' + ); + + const graph = new GPUCommandGraph(device, { + id: 'ludf-global-nullable-aggregation' + }); + const compiled = query.compile(graph); + + try { + testContext.deepEqual( + compiled.table.batches.map(batch => batch.numRows), + [1], + 'global GPU reductions publish exactly one owned result row' + ); + testContext.deepEqual( + compiled.table.schema.fields.map(field => ({name: field.name, format: field.format})), + [ + {name: 'rowCount', format: 'uint32'}, + {name: 'totalFare', format: 'float32'}, + {name: 'minimumFare', format: 'float32'}, + {name: 'maximumFare', format: 'float32'}, + {name: 'averageFare', format: 'float32'}, + {name: 'totalDistance', format: 'sint32'}, + {name: 'minimumDistance', format: 'sint32'}, + {name: 'averageDistance', format: 'float32'}, + {name: 'maximumCategory', format: 'uint32'}, + {name: 'totalCategory', format: 'uint32'}, + {name: 'averageCategory', format: 'float32'} + ], + 'sum/min/max preserve scalar formats while means and counts use float32 and uint32' + ); + testContext.deepEqual( + compiled.selectionMask.data.map(chunk => chunk.length), + [2, 0, 3], + 'global reductions retain independent source selection batches' + ); + + fixture.frame.destroy(); + testContext.ok( + fixture.sourceBuffers.every(buffer => !buffer.destroyed), + 'global reductions retain their original owned source lease' + ); + + const commandEncoder = device.createCommandEncoder({id: 'ludf-global-reduction-encode'}); + compiled.encode(commandEncoder); + testContext.equal( + submitSpy.mock.calls.length, + 0, + 'global aggregation only records work into a caller-owned command encoder' + ); + device.submit(commandEncoder.finish()); + + testContext.equal( + await readUint32Scalar(compiled.table.gpuVectors.rowCount), + 5, + 'global count includes every selected source row, independent of column nulls' + ); + testContext.equal( + await readFloat32Scalar(compiled.table.gpuVectors.totalFare), + 159, + 'floating-point sums exclude explicit nullable rows across preserved batches' + ); + testContext.equal( + await readFloat32Scalar(compiled.table.gpuVectors.minimumFare), + 10, + 'floating-point minimums exclude explicit null rows' + ); + testContext.equal( + await readFloat32Scalar(compiled.table.gpuVectors.maximumFare), + 99, + 'global maximums include valid rows regardless of other columns nullability' + ); + testContext.equal( + await readFloat32Scalar(compiled.table.gpuVectors.averageFare), + 39.75, + 'floating-point means divide only accepted non-null contributions' + ); + testContext.equal( + await readSignedScalar(compiled.table.gpuVectors.totalDistance), + 12, + 'signed integer sums preserve native 32-bit output formats' + ); + testContext.equal( + await readSignedScalar(compiled.table.gpuVectors.minimumDistance), + -2, + 'signed minimums preserve negative input values' + ); + testContext.ok( + Math.abs((await readFloat32Scalar(compiled.table.gpuVectors.averageDistance)) - 2.4) < + 0.000001, + 'signed scalar means convert contributions to float32 before averaging' + ); + testContext.equal( + await readUint32Scalar(compiled.table.gpuVectors.maximumCategory), + 1, + 'unsigned maximums reject null category values' + ); + testContext.equal( + await readUint32Scalar(compiled.table.gpuVectors.totalCategory), + 2, + 'unsigned sums exclude the nullable category sidecar' + ); + testContext.equal( + await readFloat32Scalar(compiled.table.gpuVectors.averageCategory), + 0.5, + 'unsigned nullable scalar means divide only accepted categorical values' + ); + for (const metric of [ + 'totalFare', + 'minimumFare', + 'maximumFare', + 'averageFare', + 'totalDistance', + 'minimumDistance', + 'averageDistance', + 'maximumCategory', + 'totalCategory', + 'averageCategory' + ] as const) { + const validity = compiled.validity[metric]; + if (!validity) { + throw new Error(`Expected explicit global validity for ${metric}`); + } + testContext.equal(await readUint32Scalar(validity), 1, `${metric} has valid contributions`); + } + testContext.equal(compiled.validity.rowCount, undefined, 'global row counts are nonnullable'); + + const outputBuffers = Object.values(compiled.table.gpuVectors).flatMap(vector => + vector.data.map(getLuAnalyticsBuffer) + ); + compiled.destroy(); + testContext.ok( + outputBuffers.every(buffer => buffer.destroyed), + 'compiled global reductions release their owned scalar result buffers' + ); + testContext.ok( + fixture.sourceBuffers.every(buffer => buffer.destroyed), + 'owned source resources release only after their final aggregation lease' + ); + } finally { + compiled.destroy(); + fixture.frame.destroy(); + createBufferSpy.mockRestore(); + submitSpy.mockRestore(); + } + + testContext.end(); +}); + +test('LuDataFrame updates filtered and derived global statistics within one command encoder', async testContext => { + const device = await getWebGPUTestDevice(); + if (!device) { + testContext.comment('WebGPU is not available'); + testContext.end(); + return; + } + + const fixture = createLuAnalyticsFixture(device); + const graph = new GPUCommandGraph(device, { + id: 'ludf-filtered-derived-reduction' + }); + const compiled = fixture.frame + .withColumn('adjustedFare', column('fare').add(literal(5)), {format: 'float32'}) + .filter(column('adjustedFare').greaterThan(parameter('minimumFare', 0))) + .aggregate({ + rowCount: 'count', + totalFare: {sum: 'adjustedFare'}, + averageFare: {mean: 'adjustedFare'} + }) + .compile(graph); + + const firstCount = device.createBuffer({ + id: 'ludf-global-first-count', + byteLength: Uint32Array.BYTES_PER_ELEMENT, + usage: Buffer.COPY_SRC | Buffer.COPY_DST + }); + const firstSum = device.createBuffer({ + id: 'ludf-global-first-sum', + byteLength: Float32Array.BYTES_PER_ELEMENT, + usage: Buffer.COPY_SRC | Buffer.COPY_DST + }); + + try { + const commandEncoder = device.createCommandEncoder({id: 'ludf-global-two-encodes'}); + compiled.encode(commandEncoder, {minimumFare: 20}); + commandEncoder.copyBufferToBuffer({ + sourceBuffer: getLuAnalyticsBuffer(compiled.table.gpuVectors.rowCount.data[0]), + destinationBuffer: firstCount, + size: Uint32Array.BYTES_PER_ELEMENT + }); + commandEncoder.copyBufferToBuffer({ + sourceBuffer: getLuAnalyticsBuffer(compiled.table.gpuVectors.totalFare.data[0]), + destinationBuffer: firstSum, + size: Float32Array.BYTES_PER_ELEMENT + }); + compiled.encode(commandEncoder, {minimumFare: 40}); + device.submit(commandEncoder.finish()); + + testContext.equal( + (await readLuUint32Buffer(firstCount, 1))[0], + 3, + 'first filter accepts three rows' + ); + testContext.equal( + (await readLuFloat32Buffer(firstSum, 1))[0], + 164, + 'first derived sum is 25 + 35 + 104' + ); + testContext.equal( + await readUint32Scalar(compiled.table.gpuVectors.rowCount), + 1, + 'the second encoder state accepts only the valid fare above forty' + ); + testContext.equal( + await readFloat32Scalar(compiled.table.gpuVectors.totalFare), + 104, + 'global reductions observe encoder-ordered derived parameter updates' + ); + testContext.equal( + await readFloat32Scalar(compiled.table.gpuVectors.averageFare), + 104, + 'global mean follows the second filtered contribution count' + ); + } finally { + firstCount.destroy(); + firstSum.destroy(); + compiled.destroy(); + fixture.frame.destroy(); + } + + testContext.end(); +}); + +test('LuDataFrame drops NaN and infinity from scalar statistics without dropping row counts', async testContext => { + const device = await getWebGPUTestDevice(); + if (!device) { + testContext.comment('WebGPU is not available'); + testContext.end(); + return; + } + + const fixture = createLuAnalyticsFixture(device); + const fare = fixture.frame.table.gpuVectors.fare; + getLuAnalyticsBuffer(fare.data[0]).write(Float32Array.from([10, Number.NaN])); + getLuAnalyticsBuffer(fare.data[2]).write(Float32Array.from([Number.POSITIVE_INFINITY, 99, 50])); + + const graph = new GPUCommandGraph(device, { + id: 'ludf-nonfinite-reduction' + }); + const compiled = fixture.frame + .aggregate({rowCount: 'count', totalFare: {sum: 'fare'}, averageFare: {mean: 'fare'}}) + .compile(graph); + + try { + const commandEncoder = device.createCommandEncoder({id: 'ludf-nonfinite-reduction-encode'}); + compiled.encode(commandEncoder); + device.submit(commandEncoder.finish()); + + testContext.equal( + await readUint32Scalar(compiled.table.gpuVectors.rowCount), + 5, + 'NaN rows remain counted' + ); + testContext.equal( + await readFloat32Scalar(compiled.table.gpuVectors.totalFare), + 109, + 'NaN and infinity do not poison floating sums' + ); + testContext.equal( + await readFloat32Scalar(compiled.table.gpuVectors.averageFare), + 54.5, + 'means count finite 10 and 99 only' + ); + } finally { + compiled.destroy(); + fixture.frame.destroy(); + } + + testContext.end(); +}); + +test('LuDataFrame builds nullable, filtered numeric histograms with literal domains and edges', async testContext => { + const device = await getWebGPUTestDevice(); + if (!device) { + testContext.comment('WebGPU is not available'); + testContext.end(); + return; + } + + const fixture = createLuAnalyticsFixture(device); + const createBufferSpy = vi.spyOn(device, 'createBuffer'); + const domainQuery = fixture.frame.histogram('fare', {bins: 4, domain: [0, 80]}); + testContext.equal( + createBufferSpy.mock.calls.length, + 0, + 'histogram planning does not allocate buffers' + ); + createBufferSpy.mockRestore(); + + const domainGraph = new GPUCommandGraph(device, { + id: 'ludf-domain-histogram' + }); + const domainHistogram = domainQuery.compile(domainGraph); + const edgeGraph = new GPUCommandGraph(device, { + id: 'ludf-irregular-histogram' + }); + const edgeHistogram = fixture.frame + .histogram('fare', {edges: [0, 15, 25, 40, 100]}) + .compile(edgeGraph); + const filteredGraph = new GPUCommandGraph(device, { + id: 'ludf-filtered-histogram' + }); + const filteredHistogram = fixture.frame + .filter(column('category').isValid()) + .histogram('fare', {bins: 4, domain: [0, 80]}) + .compile(filteredGraph); + + try { + const commandEncoder = device.createCommandEncoder({id: 'ludf-histograms-encode'}); + domainHistogram.encode(commandEncoder); + edgeHistogram.encode(commandEncoder); + filteredHistogram.encode(commandEncoder); + device.submit(commandEncoder.finish()); + + testContext.equal(domainHistogram.binCount, 4, 'equal-width histogram exposes its bin count'); + testContext.deepEqual(domainHistogram.domain, [0, 80], 'equal-width domain remains explicit'); + testContext.deepEqual( + edgeHistogram.edges, + [0, 15, 25, 40, 100], + 'irregular boundaries remain explicit' + ); + testContext.deepEqual( + await readUint32Vector(domainHistogram.table.gpuVectors.bin), + [0, 1, 2, 3], + 'histogram output publishes dense GPU-written bin identities' + ); + testContext.deepEqual( + await readUint32Vector(domainHistogram.table.gpuVectors.count), + [1, 2, 0, 0], + 'equal-width histogram excludes null and out-of-domain values' + ); + testContext.deepEqual( + await readUint32Vector(edgeHistogram.table.gpuVectors.count), + [1, 1, 1, 1], + 'irregular histogram bins retain explicit nullable semantics' + ); + testContext.deepEqual( + await readUint32Vector(filteredHistogram.table.gpuVectors.count), + [1, 2, 0, 0], + 'histogram selection masks combine filters with independent value validity' + ); + testContext.deepEqual( + filteredHistogram.selectionMask.data.map(chunk => chunk.length), + [2, 0, 3], + 'histogram source masks retain every original batch boundary' + ); + } finally { + domainHistogram.destroy(); + edgeHistogram.destroy(); + filteredHistogram.destroy(); + fixture.frame.destroy(); + } + + testContext.end(); +}); + +test('LuDataFrame retains explicit invalid scalar outputs and zero histograms for empty inputs', async testContext => { + const device = await getWebGPUTestDevice(); + if (!device) { + testContext.comment('WebGPU is not available'); + testContext.end(); + return; + } + + const frame = new LuDataFrame({ + table: new GPUTable({ + schema: { + fields: [ + {name: 'category', format: 'uint32', nullable: false}, + {name: 'fare', format: 'float32', nullable: true}, + {name: 'distance', format: 'sint32', nullable: false} + ], + metadata: new Map([['dataset', 'empty-analytics']]) + }, + bufferLayout: [ + {name: 'category', format: 'uint32', byteStride: 4}, + {name: 'fare', format: 'float32', byteStride: 4}, + {name: 'distance', format: 'sint32', byteStride: 4} + ] + }), + ownership: 'owned' + }); + + const reductionGraph = new GPUCommandGraph(device, { + id: 'ludf-empty-reductions' + }); + const reductions = frame + .aggregate({ + rowCount: 'count', + totalFare: {sum: 'fare'}, + minimumFare: {min: 'fare'}, + averageFare: {mean: 'fare'}, + minimumDistance: {min: 'distance'} + }) + .compile(reductionGraph); + const histogramGraph = new GPUCommandGraph(device, { + id: 'ludf-empty-histogram' + }); + const histogram = frame.histogram('fare', {bins: 3, domain: [0, 60]}).compile(histogramGraph); + + try { + const commandEncoder = device.createCommandEncoder({id: 'ludf-empty-analytics-encode'}); + reductions.encode(commandEncoder); + histogram.encode(commandEncoder); + device.submit(commandEncoder.finish()); + + testContext.equal( + await readUint32Scalar(reductions.table.gpuVectors.rowCount), + 0, + 'empty count remains nonnullable zero' + ); + testContext.equal( + await readFloat32Scalar(reductions.table.gpuVectors.totalFare), + 0, + 'empty floating sums retain zero payloads' + ); + testContext.ok( + Number.isNaN(await readFloat32Scalar(reductions.table.gpuVectors.minimumFare)), + 'empty floating minimum uses a NaN payload' + ); + testContext.ok( + Number.isNaN(await readFloat32Scalar(reductions.table.gpuVectors.averageFare)), + 'empty floating mean uses a NaN payload' + ); + testContext.equal( + await readSignedScalar(reductions.table.gpuVectors.minimumDistance), + 0, + 'empty signed minimum uses zero payload' + ); + for (const name of ['totalFare', 'minimumFare', 'averageFare', 'minimumDistance'] as const) { + const validity = reductions.validity[name]; + if (!validity) { + throw new Error(`Expected empty reduction validity for ${name}`); + } + testContext.equal(await readUint32Scalar(validity), 0, `${name} is explicitly invalid`); + } + testContext.deepEqual( + await readUint32Vector(histogram.table.gpuVectors.count), + [0, 0, 0], + 'empty source tables still publish deterministic zero-valued histogram bins' + ); + testContext.deepEqual( + histogram.selectedCounts.data, + [], + 'empty sources retain no synthetic source batches' + ); + testContext.equal( + reductions.table.schema.metadata.get('dataset'), + 'empty-analytics', + 'scalar output schemas preserve original source metadata' + ); + } finally { + reductions.destroy(); + histogram.destroy(); + frame.destroy(); + } + + testContext.end(); +}); + +test('LuDataFrame bins signed, unsigned, and derived scalar values using typed GPU domains', async testContext => { + const device = await getWebGPUTestDevice(); + if (!device) { + testContext.comment('WebGPU is not available'); + testContext.end(); + return; + } + + const fixture = createLuAnalyticsFixture(device); + const signedGraph = new GPUCommandGraph(device, { + id: 'ludf-signed-histogram' + }); + const signed = fixture.frame + .histogram('distance', {bins: 5, domain: [-2, 8]}) + .compile(signedGraph); + const unsignedGraph = new GPUCommandGraph(device, { + id: 'ludf-unsigned-histogram' + }); + const unsigned = fixture.frame.histogram('category', {edges: [0, 1, 2]}).compile(unsignedGraph); + const derivedGraph = new GPUCommandGraph(device, { + id: 'ludf-derived-histogram' + }); + const derived = fixture.frame + .withColumn('adjustedFare', column('fare').add(literal(5)), {format: 'float32'}) + .histogram('adjustedFare', {bins: 4, domain: [0, 120]}) + .compile(derivedGraph); + + try { + const commandEncoder = device.createCommandEncoder({id: 'ludf-typed-histograms-encode'}); + signed.encode(commandEncoder); + unsigned.encode(commandEncoder); + derived.encode(commandEncoder); + device.submit(commandEncoder.finish()); + + testContext.deepEqual( + await readUint32Vector(signed.table.gpuVectors.count), + [2, 0, 1, 1, 1], + 'signed histograms retain negative values and include the upper domain endpoint' + ); + testContext.deepEqual( + await readUint32Vector(unsigned.table.gpuVectors.count), + [2, 2], + 'unsigned irregular histograms exclude explicit categorical nulls' + ); + testContext.deepEqual( + await readUint32Vector(derived.table.gpuVectors.count), + [2, 1, 0, 1], + 'histograms consume nullable derived floating-point vectors directly from the graph' + ); + } finally { + signed.destroy(); + unsigned.destroy(); + derived.destroy(); + fixture.frame.destroy(); + } + + testContext.end(); +}); + +function createLuAnalyticsFixture(device: Device): LuAnalyticsFixture { + const sourceBuffers: Buffer[] = []; + const categories = [Uint32Array.from([0, 1]), new Uint32Array(0), Uint32Array.from([0, 2, 1])]; + const categoryValidity = [ + Uint32Array.from([1, 1]), + new Uint32Array(0), + Uint32Array.from([1, 0, 1]) + ]; + const fares = [Float32Array.from([10, 20]), new Float32Array(0), Float32Array.from([30, 99, 50])]; + const fareValidity = [Uint32Array.from([1, 1]), new Uint32Array(0), Uint32Array.from([1, 1, 0])]; + const distances = [Int32Array.from([-2, 3]), new Int32Array(0), Int32Array.from([-1, 4, 8])]; + const categoryValidityChunks: GPUData<'uint32'>[] = []; + const fareValidityChunks: GPUData<'uint32'>[] = []; + let sourceRowIndexOffset = 40; + + const batches = categories.map((values, batchIndex) => { + const batch = new GPURecordBatch({ + gpuData: { + category: createLuAnalyticsData(device, sourceBuffers, values, 'uint32'), + fare: createLuAnalyticsData(device, sourceBuffers, fares[batchIndex], 'float32'), + distance: createLuAnalyticsData(device, sourceBuffers, distances[batchIndex], 'sint32') + }, + fields: [ + {name: 'category', format: 'uint32', nullable: true}, + {name: 'fare', format: 'float32', nullable: true}, + {name: 'distance', format: 'sint32', nullable: false} + ], + sourceInfo: { + sourceBatchIndex: batchIndex + 4, + sourceRowIndexOffset, + sourceRowCount: values.length + } + }); + sourceRowIndexOffset += values.length; + categoryValidityChunks.push( + createLuAnalyticsData(device, sourceBuffers, categoryValidity[batchIndex], 'uint32') + ); + fareValidityChunks.push( + createLuAnalyticsData(device, sourceBuffers, fareValidity[batchIndex], 'uint32') + ); + return batch; + }); + + return { + frame: new LuDataFrame({ + table: new GPUTable({batches}), + validity: { + category: new GPUVector<'uint32'>({ + type: 'data', + name: 'ludf-analytics-category-validity', + format: 'uint32', + data: categoryValidityChunks, + ownsData: true + }), + fare: new GPUVector<'uint32'>({ + type: 'data', + name: 'ludf-analytics-fare-validity', + format: 'uint32', + data: fareValidityChunks, + ownsData: true + }) + }, + dictionaries: { + category: {values: ['economy', 'standard', 'premium'], ordered: false} + }, + ownership: 'owned' + }), + sourceBuffers + }; +} + +function createLuAnalyticsData( + device: Device, + sourceBuffers: Buffer[], + values: Float32Array | Int32Array | Uint32Array, + format: Format +): GPUData { + const buffer = device.createBuffer({ + byteLength: Math.max(values.byteLength, Uint32Array.BYTES_PER_ELEMENT), + usage: Buffer.STORAGE | Buffer.VERTEX | Buffer.COPY_SRC | Buffer.COPY_DST, + ...(values.byteLength > 0 ? {data: values} : {}) + }); + sourceBuffers.push(buffer); + return new GPUData({buffer, format, length: values.length, ownsBuffer: true}); +} + +function getLuAnalyticsBuffer(data: GPUData): Buffer { + return data.buffer instanceof Buffer ? data.buffer : data.buffer.buffer; +} + +async function readLuUint32Buffer(buffer: Buffer, length: number): Promise { + const values = await buffer.readAsync(0, length * Uint32Array.BYTES_PER_ELEMENT); + return Array.from(new Uint32Array(values.buffer, values.byteOffset, length)); +} + +async function readLuFloat32Buffer(buffer: Buffer, length: number): Promise { + const values = await buffer.readAsync(0, length * Float32Array.BYTES_PER_ELEMENT); + return Array.from(new Float32Array(values.buffer, values.byteOffset, length)); +} + +async function readUint32Vector(vector: GPUVector): Promise { + return readLuUint32Buffer(getLuAnalyticsBuffer(vector.data[0]), vector.length); +} + +async function readUint32Scalar(vector: GPUVector): Promise { + return (await readUint32Vector(vector))[0]; +} + +async function readFloat32Scalar(vector: GPUVector): Promise { + return (await readLuFloat32Buffer(getLuAnalyticsBuffer(vector.data[0]), 1))[0]; +} + +async function readSignedScalar(vector: GPUVector): Promise { + const values = await getLuAnalyticsBuffer(vector.data[0]).readAsync( + 0, + Int32Array.BYTES_PER_ELEMENT + ); + return new Int32Array(values.buffer, values.byteOffset, 1)[0]; +} diff --git a/modules/experimental/test/ludf/lu-reductions-histograms.node.spec.ts b/modules/experimental/test/ludf/lu-reductions-histograms.node.spec.ts new file mode 100644 index 0000000000..2f9bb4222c --- /dev/null +++ b/modules/experimental/test/ludf/lu-reductions-histograms.node.spec.ts @@ -0,0 +1,366 @@ +// luma.gl +// SPDX-License-Identifier: MIT +// Copyright (c) vis.gl contributors + +import {Buffer} from '@luma.gl/core'; +import { + column, + CompiledLuDataFrameAggregation, + CompiledLuDataFrameHistogram, + literal, + LuDataFrame, + LuDataFrameAggregationQuery, + LuDataFrameHistogramQuery, + parameter, + type LuDataFrameGlobalAggregationDefinitions, + type LuDataFrameGlobalAggregationResult, + type LuDataFrameHistogramOptions +} from '@luma.gl/experimental/ludf'; +import { + GPUData, + GPURecordBatch, + GPUTable, + type GPUField, + type GPURecordBatchSourceInfo +} from '@luma.gl/tables'; +import {NullDevice} from '@luma.gl/test-utils'; +import {describe, expect, expectTypeOf, test, vi} from 'vitest'; + +type AnalyticSourceColumns = { + fare: 'float32'; + distance: 'sint32'; + category: 'uint32'; + coordinates: 'float32x2'; +}; + +type AnalyticSourceFixture = { + device: NullDevice; + table: GPUTable; + buffers: Buffer[]; +}; + +describe('LuDataFrame immutable global-reduction planning', () => { + test('plans every scalar statistic without GPU allocation, submission, or source retention', () => { + const fixture = createAnalyticSourceFixture([2, 0, 3]); + const source = new LuDataFrame({table: fixture.table, ownership: 'owned'}); + const createBuffer = vi.spyOn(fixture.device, 'createBuffer'); + const createCommandEncoder = vi.spyOn(fixture.device, 'createCommandEncoder'); + const submit = vi.spyOn(fixture.device, 'submit'); + const tableSelect = vi.spyOn(fixture.table, 'select'); + + const aggregated = source.aggregate({ + rowCount: 'count', + totalFare: {sum: 'fare'}, + minimumDistance: {min: 'distance'}, + maximumCategory: {max: 'category'}, + averageFare: {mean: 'fare'}, + totalDistance: {sum: 'distance'} + }); + + expect(aggregated).toBeInstanceOf(LuDataFrameAggregationQuery); + expect(aggregated.query.source).toBe(source); + expect(aggregated.definitions).toEqual([ + {name: 'rowCount', operation: 'count'}, + {name: 'totalFare', operation: 'sum', column: 'fare'}, + {name: 'minimumDistance', operation: 'min', column: 'distance'}, + {name: 'maximumCategory', operation: 'max', column: 'category'}, + {name: 'averageFare', operation: 'mean', column: 'fare'}, + {name: 'totalDistance', operation: 'sum', column: 'distance'} + ]); + expect(Object.isFrozen(aggregated)).toBe(true); + expect(Object.isFrozen(aggregated.definitions)).toBe(true); + expect(Object.isFrozen(aggregated.definitions[0])).toBe(true); + expect(source.batches.map(batch => batch.numRows)).toEqual([2, 0, 3]); + expect(createBuffer).not.toHaveBeenCalled(); + expect(createCommandEncoder).not.toHaveBeenCalled(); + expect(submit).not.toHaveBeenCalled(); + expect(tableSelect).not.toHaveBeenCalled(); + + source.destroy(); + expect(fixture.buffers.every(buffer => buffer.destroyed)).toBe(true); + + createBuffer.mockRestore(); + createCommandEncoder.mockRestore(); + submit.mockRestore(); + tableSelect.mockRestore(); + }); + + test('retains native sum/min/max formats, float32 means, and exact uint32 row counts', () => { + const fixture = createAnalyticSourceFixture([2]); + const source = new LuDataFrame({table: fixture.table}); + const aggregated = source.aggregate({ + rowCount: 'count', + totalFare: {sum: 'fare'}, + minimumDistance: {min: 'distance'}, + maximumCategory: {max: 'category'}, + averageDistance: {mean: 'distance'} + }); + + expectTypeOf(aggregated.compile).returns.toEqualTypeOf< + CompiledLuDataFrameAggregation<{ + rowCount: 'uint32'; + totalFare: 'float32'; + minimumDistance: 'sint32'; + maximumCategory: 'uint32'; + averageDistance: 'float32'; + }> + >(); + expectTypeOf< + LuDataFrameGlobalAggregationResult< + AnalyticSourceColumns, + {accepted: 'count'; totalDistance: {sum: 'distance'}; averageCategory: {mean: 'category'}} + > + >().toEqualTypeOf<{ + accepted: 'uint32'; + totalDistance: 'sint32'; + averageCategory: 'float32'; + }>(); + + source.destroy(); + fixture.table.destroy(); + }); + + test('retains immutable source filters, interaction parameters, derived columns, and empty batches', () => { + for (const batchLengths of [[], [0], [2, 0, 3]] as const) { + const fixture = createAnalyticSourceFixture(batchLengths, {nullableFare: true}); + const source = new LuDataFrame({table: fixture.table}); + const filtered = source + .filter(column('fare').greaterThan(parameter('minimumFare', 10))) + .select(['fare', 'distance']); + const adjusted = filtered.withColumn('adjustedFare', column('fare').add(literal(2))); + const reduced = adjusted.aggregate({ + accepted: 'count', + averageAdjustedFare: {mean: 'adjustedFare'} + }); + + expect(reduced.query.predicates[0]).toBe(filtered.predicates[0]); + expect(reduced.query.derivedColumns.map(({name}) => name)).toEqual(['adjustedFare']); + expect(reduced.query.source.batches.map(batch => batch.numRows)).toEqual(batchLengths); + expect(reduced.query.source.validity.fare).toBeUndefined(); + expect(filtered.columnNames).toEqual(['fare', 'distance']); + + source.destroy(); + fixture.table.destroy(); + } + }); + + test('rejects empty operations, hidden metrics, unsupported formats, and invalid operation shapes', () => { + const fixture = createAnalyticSourceFixture([2]); + const source = new LuDataFrame({table: fixture.table}); + const createBuffer = vi.spyOn(fixture.device, 'createBuffer'); + + expect(() => source.aggregate({})).toThrow(/aggregation|reduction/i); + expect(() => source.aggregate({'': 'count'})).toThrow(/name/i); + expect(() => + // @ts-expect-error Global statistics require scalar, not vector-valued, columns. + source.aggregate({invalid: {sum: 'coordinates'}}) + ).toThrow(/scalar|column/i); + expect(() => + // @ts-expect-error Global statistics require an existing source metric column. + source.aggregate({invalid: {sum: 'missing'}}) + ).toThrow(/selected|column/i); + const selected = source.filter(column('fare').greaterThan(literal(0))).select(['fare']); + expect(() => + // @ts-expect-error Global reductions cannot consume hidden projected columns. + selected.aggregate({invalid: {sum: 'distance'}}) + ).toThrow(/selected|column/i); + + const unsupported = { + broken: {median: 'fare'} + } as unknown as LuDataFrameGlobalAggregationDefinitions; + expect(() => source.aggregate(unsupported)).toThrow(/operation/i); + const multiple = { + broken: {sum: 'fare', max: 'fare'} + } as unknown as LuDataFrameGlobalAggregationDefinitions; + expect(() => source.aggregate(multiple)).toThrow(/one|operation/i); + expect(createBuffer).not.toHaveBeenCalled(); + + createBuffer.mockRestore(); + source.destroy(); + fixture.table.destroy(); + }); +}); + +describe('LuDataFrame immutable explicit-domain histogram planning', () => { + test('deep-clones explicit domains and edges without allocating or retaining GPU resources', () => { + const fixture = createAnalyticSourceFixture([2, 0, 3]); + const source = new LuDataFrame({table: fixture.table, ownership: 'owned'}); + const createBuffer = vi.spyOn(fixture.device, 'createBuffer'); + const createCommandEncoder = vi.spyOn(fixture.device, 'createCommandEncoder'); + const submit = vi.spyOn(fixture.device, 'submit'); + const mutableDomain: [number, number] = [0, 80]; + const mutableEdges = [0, 15, 25, 40, 100]; + + const uniform = source.histogram('fare', {bins: 4, domain: mutableDomain}); + const irregular = source.histogram('fare', {edges: mutableEdges}); + mutableDomain[0] = 10; + mutableEdges[1] = 20; + + expect(uniform).toBeInstanceOf(LuDataFrameHistogramQuery); + expect(irregular).toBeInstanceOf(LuDataFrameHistogramQuery); + expect(uniform.column).toBe('fare'); + expect(uniform.binCount).toBe(4); + expect(uniform.options).toEqual({bins: 4, domain: [0, 80]}); + expect(irregular.binCount).toBe(4); + expect(irregular.options).toEqual({edges: [0, 15, 25, 40, 100]}); + expect(Object.isFrozen(uniform)).toBe(true); + expect(Object.isFrozen(uniform.options)).toBe(true); + expect(Object.isFrozen('domain' in uniform.options ? uniform.options.domain : undefined)).toBe( + true + ); + expect(Object.isFrozen(irregular.options)).toBe(true); + expect( + Object.isFrozen('edges' in irregular.options ? irregular.options.edges : undefined) + ).toBe(true); + expect(createBuffer).not.toHaveBeenCalled(); + expect(createCommandEncoder).not.toHaveBeenCalled(); + expect(submit).not.toHaveBeenCalled(); + + source.destroy(); + expect(fixture.buffers.every(buffer => buffer.destroyed)).toBe(true); + + createBuffer.mockRestore(); + createCommandEncoder.mockRestore(); + submit.mockRestore(); + }); + + test('supports exact typed floating, signed, unsigned, filtered, and derived histogram columns', () => { + const fixture = createAnalyticSourceFixture([2]); + const source = new LuDataFrame({table: fixture.table}); + const floating = source.histogram('fare', {bins: 4, domain: [0, 80]}); + const signed = source.histogram('distance', {edges: [-10, 0, 10]}); + const unsigned = source.histogram('category', {bins: 3, domain: [0, 3]}); + const derived = source + .filter(column('fare').greaterThan(parameter('minimumFare', 5))) + .withColumn('doubleFare', column('fare').multiply(literal(2))) + .histogram('doubleFare', {edges: [0, 20, 40, 80]}); + + expectTypeOf(floating.compile).returns.toEqualTypeOf(); + expect(floating.column).toBe('fare'); + expect(signed.column).toBe('distance'); + expect(unsigned.column).toBe('category'); + expect(derived.column).toBe('doubleFare'); + expect(derived.query.predicates).toHaveLength(1); + expect(derived.query.derivedColumns.map(({name}) => name)).toEqual(['doubleFare']); + + source.destroy(); + fixture.table.destroy(); + }); + + test('rejects absent domains, invalid counts, nonrepresentable boundaries, and ambiguous options', () => { + const fixture = createAnalyticSourceFixture([2]); + const source = new LuDataFrame({table: fixture.table}); + const createBuffer = vi.spyOn(fixture.device, 'createBuffer'); + + expect(() => + // @ts-expect-error Histograms require a selected portable numeric scalar column. + source.histogram('coordinates', {bins: 2, domain: [0, 1]}) + ).toThrow(/scalar|column/i); + expect(() => + // @ts-expect-error Histogram input columns must exist in the selected schema. + source.histogram('missing', {bins: 2, domain: [0, 1]}) + ).toThrow(/selected|column/i); + + for (const bins of [0, -1, 1.5, Number.NaN, 0x1_0000_0000]) { + expect(() => source.histogram('fare', {bins, domain: [0, 1]})).toThrow(/bin|count/i); + } + expect(() => source.histogram('fare', {bins: 2, domain: [5, 1]})).toThrow(/domain|minimum/i); + expect(() => + source.histogram('fare', {bins: 2, domain: [0, Number.POSITIVE_INFINITY]}) + ).toThrow(/finite|boundary/i); + expect(() => source.histogram('category', {bins: 2, domain: [-1, 2]})).toThrow(/uint32/i); + expect(() => source.histogram('distance', {bins: 2, domain: [-2, 2.5]})).toThrow(/sint32/i); + expect(() => source.histogram('fare', {edges: [1]})).toThrow(/edge|257/i); + expect(() => source.histogram('fare', {edges: [1, 1]})).toThrow(/increasing/i); + expect(() => source.histogram('fare', {edges: [3, 2]})).toThrow(/increasing/i); + expect(() => + source.histogram('fare', {edges: Array.from({length: 258}, (_, index) => index)}) + ).toThrow(/257|edge/i); + expect(() => source.histogram('fare', {edges: [1, 1 + Number.EPSILON]})).toThrow(/increasing/i); + expect(() => source.histogram('fare', {edges: [0, Number.POSITIVE_INFINITY]})).toThrow( + /finite|boundary/i + ); + + const mixed = {bins: 2, domain: [0, 2], edges: [0, 1, 2]} as LuDataFrameHistogramOptions; + expect(() => source.histogram('fare', mixed)).toThrow(/domain|combined/i); + expect(createBuffer).not.toHaveBeenCalled(); + + createBuffer.mockRestore(); + source.destroy(); + fixture.table.destroy(); + }); + + test('rejects new analytic plans after their source dataframe was explicitly destroyed', () => { + const fixture = createAnalyticSourceFixture([2]); + const source = new LuDataFrame({table: fixture.table}); + source.destroy(); + + expect(() => source.aggregate({count: 'count'})).toThrow(/destroyed/i); + expect(() => source.histogram('fare', {bins: 2, domain: [0, 1]})).toThrow(/destroyed/i); + fixture.table.destroy(); + }); +}); + +function createAnalyticSourceFixture( + batchLengths: readonly number[], + options: {nullableFare?: boolean} = {} +): AnalyticSourceFixture { + const device = new NullDevice({id: 'ludf-analytic-reductions-node-device'}); + const buffers: Buffer[] = []; + const fields: GPUField[] = [ + {name: 'fare', format: 'float32', nullable: options.nullableFare ?? false}, + {name: 'distance', format: 'sint32', nullable: false}, + {name: 'category', format: 'uint32', nullable: false}, + {name: 'coordinates', format: 'float32x2', nullable: false} + ]; + let sourceRowIndexOffset = 40; + const batches = batchLengths.map((length, sourceBatchIndex) => { + const sourceInfo: GPURecordBatchSourceInfo = { + sourceBatchIndex, + sourceRowIndexOffset, + sourceRowCount: length + }; + sourceRowIndexOffset += length; + + return new GPURecordBatch({ + gpuData: { + fare: makeAnalyticSourceData(device, buffers, length, 'float32'), + distance: makeAnalyticSourceData(device, buffers, length, 'sint32'), + category: makeAnalyticSourceData(device, buffers, length, 'uint32'), + coordinates: makeAnalyticSourceData(device, buffers, length, 'float32x2') + }, + fields, + numRows: length, + sourceInfo + }); + }); + + const table = + batches.length > 0 + ? new GPUTable({batches}) + : new GPUTable({ + schema: {fields, metadata: new Map()}, + bufferLayout: [ + {name: 'fare', format: 'float32'}, + {name: 'distance', format: 'sint32'}, + {name: 'category', format: 'uint32'}, + {name: 'coordinates', format: 'float32x2'} + ] + }); + return {device, table, buffers}; +} + +function makeAnalyticSourceData( + device: NullDevice, + buffers: Buffer[], + length: number, + format: Format +): GPUData { + const byteStride = format === 'float32x2' ? 8 : Uint32Array.BYTES_PER_ELEMENT; + const buffer = device.createBuffer({ + byteLength: Math.max(length, 1) * byteStride, + usage: Buffer.STORAGE | Buffer.COPY_SRC | Buffer.COPY_DST + }); + buffers.push(buffer); + return new GPUData({buffer, format, length, ownsBuffer: true}); +} From 29b96de01ebf845a43f93207bcedbae740f2450f Mon Sep 17 00:00:00 2001 From: Ib Green Date: Tue, 4 Aug 2026 22:25:34 -0400 Subject: [PATCH 2/3] fix(experimental): reject strided luDF analytics columns --- .../src/ludf/lu-analytics-compiler-utils.ts | 26 +++++++ .../lu-reductions-histograms.node.spec.ts | 67 +++++++++++++++++-- 2 files changed, 87 insertions(+), 6 deletions(-) diff --git a/modules/experimental/src/ludf/lu-analytics-compiler-utils.ts b/modules/experimental/src/ludf/lu-analytics-compiler-utils.ts index e5bebeed4e..42c796313e 100644 --- a/modules/experimental/src/ludf/lu-analytics-compiler-utils.ts +++ b/modules/experimental/src/ludf/lu-analytics-compiler-utils.ts @@ -51,6 +51,10 @@ export function validateLuAnalyticsSource( if (source.table.gpuConstants[name]) { throw new Error(`LuDataFrame analytics column "${name}" must contain GPU vector data`); } + const vector = source.table.gpuVectors[name]; + if (vector) { + validateLuAnalyticsVectorLayout(vector, name); + } const field = source.schema.fields.find(candidate => candidate.name === name); if (field?.nullable && source.numRows > 0 && !source.validity[name as keyof Source & string]) { throw new Error(`LuDataFrame nullable analytics column "${name}" requires GPU validity`); @@ -99,9 +103,31 @@ export function getLuAnalyticsVector( if (!vector) { return new GPUVector({type: 'data', name, format, data: [], ownsData: false}); } + validateLuAnalyticsVectorLayout(vector, name); return vector as GPUVector; } +/** Rejects interleaved, padded, or misaligned rows before packed scalar shaders can consume them. */ +function validateLuAnalyticsVectorLayout(vector: GPUVector, name: string): void { + if ( + vector.bufferLayout || + vector.stride !== 1 || + vector.byteStride !== UINT32_BYTE_LENGTH || + vector.rowByteLength !== UINT32_BYTE_LENGTH || + vector.data.some( + chunk => + chunk.stride !== 1 || + chunk.byteStride !== UINT32_BYTE_LENGTH || + chunk.rowByteLength !== UINT32_BYTE_LENGTH || + chunk.byteOffset % UINT32_BYTE_LENGTH !== 0 + ) + ) { + throw new Error( + `LuDataFrame analytics column "${name}" requires packed, uint32-aligned scalar GPU data` + ); + } +} + /** Intersects query selection with an explicit nullable source/derived validity sidecar. */ export function getLuAnalyticsSelectionMask( context: LuDataFrameQueryExtensionContext, diff --git a/modules/experimental/test/ludf/lu-reductions-histograms.node.spec.ts b/modules/experimental/test/ludf/lu-reductions-histograms.node.spec.ts index 2f9bb4222c..04a7b6bf2c 100644 --- a/modules/experimental/test/ludf/lu-reductions-histograms.node.spec.ts +++ b/modules/experimental/test/ludf/lu-reductions-histograms.node.spec.ts @@ -3,6 +3,7 @@ // Copyright (c) vis.gl contributors import {Buffer} from '@luma.gl/core'; +import {GPUCommandGraph} from '@luma.gl/experimental'; import { column, CompiledLuDataFrameAggregation, @@ -14,7 +15,8 @@ import { parameter, type LuDataFrameGlobalAggregationDefinitions, type LuDataFrameGlobalAggregationResult, - type LuDataFrameHistogramOptions + type LuDataFrameHistogramOptions, + type LuDataFrameQueryParameters } from '@luma.gl/experimental/ludf'; import { GPUData, @@ -179,6 +181,51 @@ describe('LuDataFrame immutable global-reduction planning', () => { source.destroy(); fixture.table.destroy(); }); + + test('rejects strided scalar metrics before allocating global or histogram GPU outputs', () => { + const fixture = createAnalyticSourceFixture([2, 0, 3], {stridedFare: true}); + Object.defineProperty(fixture.device, 'type', {value: 'webgpu'}); + const source = new LuDataFrame({table: fixture.table}); + const globalGraph = new GPUCommandGraph(fixture.device, { + id: 'ludf-strided-global-metric' + }); + const histogramGraph = new GPUCommandGraph(fixture.device, { + id: 'ludf-strided-histogram-metric' + }); + const createBuffer = vi.spyOn(fixture.device, 'createBuffer'); + const addGlobalPass = vi.spyOn(globalGraph, 'addComputePass'); + const addHistogramPass = vi.spyOn(histogramGraph, 'addComputePass'); + const sourceVector = fixture.table.gpuVectors['fare']; + + expect(sourceVector.byteStride).toBe(8); + expect(sourceVector.rowByteLength).toBe(4); + expect(sourceVector.stride).toBe(1); + expect(sourceVector.data.map(chunk => chunk.byteStride)).toEqual([8, 8, 8]); + expect(source.batches.map(batch => batch.numRows)).toEqual([2, 0, 3]); + + expect(() => + source + .aggregate({ + total: {sum: 'fare'}, + minimum: {min: 'fare'}, + maximum: {max: 'fare'}, + average: {mean: 'fare'} + }) + .compile(globalGraph) + ).toThrow(/packed|stride|aligned/i); + expect(() => + source.histogram('fare', {bins: 4, domain: [0, 80]}).compile(histogramGraph) + ).toThrow(/packed|stride|aligned/i); + expect(createBuffer).not.toHaveBeenCalled(); + expect(addGlobalPass).not.toHaveBeenCalled(); + expect(addHistogramPass).not.toHaveBeenCalled(); + + createBuffer.mockRestore(); + addGlobalPass.mockRestore(); + addHistogramPass.mockRestore(); + source.destroy(); + fixture.table.destroy(); + }); }); describe('LuDataFrame immutable explicit-domain histogram planning', () => { @@ -303,7 +350,7 @@ describe('LuDataFrame immutable explicit-domain histogram planning', () => { function createAnalyticSourceFixture( batchLengths: readonly number[], - options: {nullableFare?: boolean} = {} + options: {nullableFare?: boolean; stridedFare?: boolean} = {} ): AnalyticSourceFixture { const device = new NullDevice({id: 'ludf-analytic-reductions-node-device'}); const buffers: Buffer[] = []; @@ -324,7 +371,13 @@ function createAnalyticSourceFixture( return new GPURecordBatch({ gpuData: { - fare: makeAnalyticSourceData(device, buffers, length, 'float32'), + fare: makeAnalyticSourceData( + device, + buffers, + length, + 'float32', + options.stridedFare ? {byteStride: 8, rowByteLength: 4, stride: 1} : undefined + ), distance: makeAnalyticSourceData(device, buffers, length, 'sint32'), category: makeAnalyticSourceData(device, buffers, length, 'uint32'), coordinates: makeAnalyticSourceData(device, buffers, length, 'float32x2') @@ -354,13 +407,15 @@ function makeAnalyticSourceData { - const byteStride = format === 'float32x2' ? 8 : Uint32Array.BYTES_PER_ELEMENT; + const byteStride = + layout?.byteStride ?? (format === 'float32x2' ? 8 : Uint32Array.BYTES_PER_ELEMENT); const buffer = device.createBuffer({ byteLength: Math.max(length, 1) * byteStride, usage: Buffer.STORAGE | Buffer.COPY_SRC | Buffer.COPY_DST }); buffers.push(buffer); - return new GPUData({buffer, format, length, ownsBuffer: true}); + return new GPUData({buffer, format, length, ownsBuffer: true, ...layout}); } From 08b0f791e526e6fee5c9b665211ed1603430c470 Mon Sep 17 00:00:00 2001 From: Ib Green Date: Fri, 7 Aug 2026 09:22:28 -0400 Subject: [PATCH 3/3] fix(experimental): add SPDX headers for luDF analytics --- modules/experimental/src/ludf/lu-analytics-compiler-utils.ts | 2 +- modules/experimental/src/ludf/lu-global-aggregation-compiler.ts | 2 +- modules/experimental/src/ludf/lu-global-aggregation-query.ts | 2 +- modules/experimental/src/ludf/lu-histogram-compiler.ts | 2 +- modules/experimental/src/ludf/lu-histogram-query.ts | 2 +- modules/experimental/test/ludf/lu-global-aggregation.spec.ts | 2 +- .../test/ludf/lu-reductions-histograms.node.spec.ts | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/modules/experimental/src/ludf/lu-analytics-compiler-utils.ts b/modules/experimental/src/ludf/lu-analytics-compiler-utils.ts index 42c796313e..4abe053c49 100644 --- a/modules/experimental/src/ludf/lu-analytics-compiler-utils.ts +++ b/modules/experimental/src/ludf/lu-analytics-compiler-utils.ts @@ -1,6 +1,6 @@ // luma.gl // SPDX-License-Identifier: MIT -// Copyright (c) vis.gl contributors +// SPDX-FileCopyrightText: Copyright (c) vis.gl contributors import {Buffer, type Binding, type Device} from '@luma.gl/core'; import {Computation} from '@luma.gl/engine'; diff --git a/modules/experimental/src/ludf/lu-global-aggregation-compiler.ts b/modules/experimental/src/ludf/lu-global-aggregation-compiler.ts index adc5027e63..7ab1c82c96 100644 --- a/modules/experimental/src/ludf/lu-global-aggregation-compiler.ts +++ b/modules/experimental/src/ludf/lu-global-aggregation-compiler.ts @@ -1,6 +1,6 @@ // luma.gl // SPDX-License-Identifier: MIT -// Copyright (c) vis.gl contributors +// SPDX-FileCopyrightText: Copyright (c) vis.gl contributors import {GPUVector, type GPUField, type GPUTypeMap} from '@luma.gl/tables'; import { diff --git a/modules/experimental/src/ludf/lu-global-aggregation-query.ts b/modules/experimental/src/ludf/lu-global-aggregation-query.ts index db7c49a0ec..9c2760b112 100644 --- a/modules/experimental/src/ludf/lu-global-aggregation-query.ts +++ b/modules/experimental/src/ludf/lu-global-aggregation-query.ts @@ -1,6 +1,6 @@ // luma.gl // SPDX-License-Identifier: MIT -// Copyright (c) vis.gl contributors +// SPDX-FileCopyrightText: Copyright (c) vis.gl contributors import type {GPUTypeMap} from '@luma.gl/tables'; import type {GPUCommandGraph} from '../gpu-primitives/gpu-command-graph'; diff --git a/modules/experimental/src/ludf/lu-histogram-compiler.ts b/modules/experimental/src/ludf/lu-histogram-compiler.ts index e20d6908e2..a69e9d5360 100644 --- a/modules/experimental/src/ludf/lu-histogram-compiler.ts +++ b/modules/experimental/src/ludf/lu-histogram-compiler.ts @@ -1,6 +1,6 @@ // luma.gl // SPDX-License-Identifier: MIT -// Copyright (c) vis.gl contributors +// SPDX-FileCopyrightText: Copyright (c) vis.gl contributors import {GPUVector, type GPUField, type GPUTypeMap} from '@luma.gl/tables'; import {type GPUCommandGraph, type GraphDataView} from '../gpu-primitives/gpu-command-graph'; diff --git a/modules/experimental/src/ludf/lu-histogram-query.ts b/modules/experimental/src/ludf/lu-histogram-query.ts index 2c65273480..7608cf0f5f 100644 --- a/modules/experimental/src/ludf/lu-histogram-query.ts +++ b/modules/experimental/src/ludf/lu-histogram-query.ts @@ -1,6 +1,6 @@ // luma.gl // SPDX-License-Identifier: MIT -// Copyright (c) vis.gl contributors +// SPDX-FileCopyrightText: Copyright (c) vis.gl contributors import type {GPUTypeMap} from '@luma.gl/tables'; import type {GPUCommandGraph} from '../gpu-primitives/gpu-command-graph'; diff --git a/modules/experimental/test/ludf/lu-global-aggregation.spec.ts b/modules/experimental/test/ludf/lu-global-aggregation.spec.ts index 12e0f6f9c4..c6491649ef 100644 --- a/modules/experimental/test/ludf/lu-global-aggregation.spec.ts +++ b/modules/experimental/test/ludf/lu-global-aggregation.spec.ts @@ -1,6 +1,6 @@ // luma.gl // SPDX-License-Identifier: MIT -// Copyright (c) vis.gl contributors +// SPDX-FileCopyrightText: Copyright (c) vis.gl contributors import {Buffer, type Device} from '@luma.gl/core'; import {GPUCommandGraph} from '@luma.gl/experimental'; diff --git a/modules/experimental/test/ludf/lu-reductions-histograms.node.spec.ts b/modules/experimental/test/ludf/lu-reductions-histograms.node.spec.ts index 04a7b6bf2c..ee06b24984 100644 --- a/modules/experimental/test/ludf/lu-reductions-histograms.node.spec.ts +++ b/modules/experimental/test/ludf/lu-reductions-histograms.node.spec.ts @@ -1,6 +1,6 @@ // luma.gl // SPDX-License-Identifier: MIT -// Copyright (c) vis.gl contributors +// SPDX-FileCopyrightText: Copyright (c) vis.gl contributors import {Buffer} from '@luma.gl/core'; import {GPUCommandGraph} from '@luma.gl/experimental';