diff --git a/docs/api-reference/gltf/README.md b/docs/api-reference/gltf/README.md index 123efdede8..d4e61f62d4 100644 --- a/docs/api-reference/gltf/README.md +++ b/docs/api-reference/gltf/README.md @@ -77,8 +77,11 @@ bundle contains: | --- | --- | | `scenes` | One `@luma.gl/engine` `GroupNode` root per source scene. | | `materials` | Shared engine materials in source glTF material order. | +| `variants` | Source-aware runtime controller for authored material variants. | +| `cameras` | Runtime camera projections updated by supported animation pointers. | | `animator` | A `GLTFAnimator` backed by the shared engine animation mixer. | | `animations` | Decoded source clips, including supported animation-pointer channels. | +| `skins` | Automatically updated source skin bindings and reusable joint palettes. | | `lights` | World-space directional, point, and spot lights from `KHR_lights_punctual`. | | `extensionSupport` | A map describing support for extensions reported by the asset. | | `sceneBounds` | World-space bounds and camera-framing recommendations for each scene. | @@ -149,6 +152,7 @@ and `CUBICSPLINE` tracks through the format-independent engine mixer. Existing s and morph-target helpers preserve authored joint attributes, target deltas, and per-node weights. See [glTF animation and deformation](/docs/api-reference/gltf/gltf-animation), the +[GPU-animated crowd reference](/docs/api-reference/gltf/gltf-animated-crowd), the [engine animation guide](/docs/api-guide/engine/animation), and [glTF extension support](/docs/api-reference/gltf/gltf-extensions) for details and limitations. diff --git a/docs/api-reference/gltf/gltf-animated-crowd.md b/docs/api-reference/gltf/gltf-animated-crowd.md new file mode 100644 index 0000000000..55bce613e5 --- /dev/null +++ b/docs/api-reference/gltf/gltf-animated-crowd.md @@ -0,0 +1,217 @@ +import {GltfDocsTabs} from '@site/src/components/docs/gltf-docs-tabs'; + +# GPU-Animated glTF Crowds + + + +`GLTFAnimatedCrowd` renders independently animated characters using one shared GPU model and one +instanced draw per compatible source primitive. Actors retain independent animation clocks, +hierarchies, joint palettes, and placement transforms without duplicating geometry, materials, or +draw calls for every character. + +## Create and render a crowd + +```ts +import {load} from '@loaders.gl/core'; +import {GLTFLoader, postProcessGLTF} from '@loaders.gl/gltf'; +import {createGLTFAnimatedCrowd} from '@luma.gl/gltf'; +import {Matrix4} from '@math.gl/core'; + +const asset = await load('/models/character.glb', GLTFLoader); +const gltf = postProcessGLTF(asset); +const crowd = createGLTFAnimatedCrowd(device, gltf, {capacity: 256}); + +const [walker, runner] = crowd.addActors([ + { + id: 'walker', + clip: 'Walking', + phase: 0, + transform: new Matrix4().translate([-2, 0, 0]) + }, + { + id: 'runner', + clip: 'Running', + phase: 0.35, + speed: 1.5, + transform: new Matrix4().translate([2, 0, 0]) + } +]); + +function renderFrame( + deltaSeconds: number, + viewProjectionMatrix: Matrix4, + cameraPosition: [number, number, number] +): void { + crowd.update(deltaSeconds); + + const modelMatrix = new Matrix4(); + for (const model of crowd.models) { + model.shaderInputs.setProps({ + pbrProjection: { + camera: cameraPosition, + modelViewProjectionMatrix: viewProjectionMatrix, + modelMatrix, + normalMatrix: modelMatrix + } + }); + } + + const renderPass = device.beginRenderPass({clearColor: [0, 0, 0, 1], clearDepth: 1}); + const drawCount = crowd.draw(renderPass); + renderPass.end(); + device.submit(); + + console.log({actors: crowd.actorCount, draws: drawCount}); +} +``` + +`update()` takes a frame delta in **seconds**, not the absolute millisecond value supplied by +`requestAnimationFrame()`. `addActors()` prepares every actor first, then uploads the complete +group in one refresh; prefer it over repeatedly calling `addActor()` when building large crowds. +Provide initial placement matrices in the batched actor options to avoid a separate upload per +transform. `removeActors()` similarly compacts many actor slots with one upload. +Placement, clip-selection, and seek operations refresh their packed data immediately; use +`update()` to advance every actor's independent playback clock. + +Only active actor transforms and joint-palette slots are uploaded; unused fixed-capacity storage +is not rewritten every frame. + +The default capacity is 16 actors. Capacity is fixed so shared GPU allocations and binding layouts +remain stable; creating more actors than the configured capacity is rejected. + +The glTF Asset Studio exposes this path through its **GPU Crowd Actors** control, supporting +1–100 actors and reporting the actual number of shared GPU draws. +Its default CC0 Robot Expressive model provides 14 named actions, including walking, running, +dancing, waving, and idling. Neighboring actors can play different actions without splitting a +shared primitive into separate draw calls. + +## Independent playback + +Every actor has its own lightweight node hierarchy, existing glTF animator, engine animation +mixer, and joint-palette state: + +```ts +walker.selectClip('Running', {crossFadeDuration: 0.4}); +runner.selectClip('Idle', {phase: 0.5}); + +walker.pause(); +runner.setSpeed(2); +crowd.update(0.25); + +walker.play(); +walker.seek(1.25); +runner.setPhase(0.75); +runner.setLoop('ping-pong', 3); +runner.setTransform(new Matrix4().translate([4, 0, 0])); + +console.log(walker.activeClip, runner.activeClip); +console.log(walker.time, runner.speed, runner.playing); +``` + +Clip times, crossfade durations, and update deltas are measured in seconds. Normalized `phase` +values select a position within the active clip. Loop modes are `once`, `repeat`, and +`ping-pong`; negative playback speeds run the selected clip backward. + +`actor.root` and `actor.getNode(indexOrId)` expose private CPU-side scenegraph nodes. Those nodes +do not own duplicate `Model` objects. GPU models, source geometry, and runtime materials belong +to the single shared `crowd.scenegraphs` bundle. + +## Batching model + +Each compatible source mesh primitive owns one shared luma.gl `Model`. Its draw uses the number +of live crowd actors as its instance count. Different source primitives, materials, primitive +topologies, or render-state requirements remain separate draw groups. + +For example, a character containing 19 source primitives requires approximately 19 instanced +draws whether the crowd contains two actors or 100. Rendering 100 independent scenegraphs +would instead require approximately 1,900 draws. This API does **not** claim that arbitrary +multi-primitive or multi-material models collapse into one universal draw call. + +Animated rigid node transforms are uploaded as per-actor instance attributes. Authored source +skins additionally read a palette selected by the GPU instance index, so actors playing different +clips or phases deform differently while sharing the same vertex buffers. + +The existing CPU `AnimationMixer` evaluates glTF keyframes and builds each actor's joint +matrices. Vertex shaders then apply those actor-specific matrices on the GPU. This is GPU +instanced skinning, **not** GPU sampling of baked animation clips or GPU-side interpolation. +Batching reduces GPU draw calls, but large crowds can still be limited by CPU animation work. + +## Graphics backends + +| Backend | Actor joint-palette storage | GPU access | +| --- | --- | --- | +| WebGPU | One packed read-only storage buffer per skinned primitive draw group. | Vertex shaders index the buffer with the instance and joint indices. | +| WebGL 2 | One nearest-sampled `rgba32float` palette texture per skinned primitive draw group. | Vertex shaders retrieve four matrix columns using `texelFetch()` and `gl_InstanceID`. | + +A joint matrix occupies 64 bytes. One 43-joint palette for 100 actors therefore requires +approximately 275 KB of packed GPU data per pose update. Each skinned primitive draw group owns +its own packed palette; assets with multiple skinned primitives allocate one palette per group. + +WebGPU capacity is constrained by storage-buffer and binding-size limits. WebGL 2 capacity is +constrained by vertex-stage texture support and maximum texture dimensions: a palette texture is +`4 × jointCount` texels wide and one row per actor. Float linear filtering is unnecessary because +palette data is read at exact texel coordinates. + +If a backend cannot support the required storage or float-texture path, crowd rendering is not +silently replaced with one ordinary draw per actor. + +## Current boundaries + +- Source geometry and materials are shared; per-actor material factors, material variants, + texture-transform pointers, camera/light pointers, and renderer state are not isolated. +- Actor morph weights can advance independently on their CPU-side nodes, but independently + deformed morph-target vertex data is not yet evaluated or drawn per actor. +- Per-actor visibility, culling, transparency sorting, and source-authored + `EXT_mesh_gpu_instancing` composition are not promised by this crowd API. +- Each source primitive still has its own draw, and every actor's clip evaluation and palette + preparation currently occur on the CPU. +- Crowd buffers have a fixed capacity; recreate the crowd to increase it. + +Use the regular [glTF animation reference](/docs/api-reference/gltf/gltf-animation) when an asset +requires independently updated materials, morph geometry, cameras, or lights without batching. + +## Ownership and cleanup + +```ts +crowd.getActor('runner'); +crowd.removeActors(['walker']); +crowd.update(1 / 60); + +crowd.destroy(); +crowd.destroy(); +``` + +Removing an actor releases only its private CPU-side animation state; remaining actors continue +using the shared GPU models. Destroying the crowd releases its packed palette resources and calls +the shared `scenegraphs.destroy()` lifecycle exactly once. Destruction is idempotent and does not +destroy the application-owned device or borrowed image-based-lighting textures. + +## Public API + +```ts +import { + createGLTFAnimatedCrowd, + GLTFAnimatedCrowd, + type GLTFAnimatedCrowdOptions, + GLTFCrowdActor, + type GLTFCrowdActorOptions, + type GLTFCrowdClipSelectionOptions, + type GLTFCrowdPrimitiveGroup +} from '@luma.gl/gltf'; +``` + +| API | Purpose | +| --- | --- | +| `createGLTFAnimatedCrowd(device, gltf, options?)` | Parse one postprocessed asset and allocate shared crowd resources. | +| `crowd.addActors(options[])`, `crowd.addActor(options?)` | Add lightweight actors; the batched form uploads once. | +| `crowd.getActor(id)`, `crowd.removeActor(id)` | Inspect or remove one independent actor. | +| `crowd.removeActors(ids)` | Remove and compact many actors with one upload. | +| `crowd.actorCount`, `crowd.capacity`, `crowd.actors` | Inspect live actors and fixed storage capacity. | +| `crowd.scenegraphs`, `crowd.models`, `crowd.primitiveGroups` | Inspect the shared parsed asset and primitive draw groups. | +| `crowd.update(deltaSeconds)` | Evaluate actor clips and upload current transforms and palettes. | +| `crowd.draw(renderPass)` | Issue one instanced draw per compatible source primitive. | +| `crowd.destroy()` | Release owned actor, palette, and shared scenegraph resources. | +| `actor.selectClip()`, `actor.seek()`, `actor.setPhase()` | Select, crossfade, or reposition an independent clip. | +| `actor.play()`, `actor.pause()`, `actor.setSpeed()`, `actor.setLoop()` | Configure independent playback. | +| `actor.setTransform()`, `actor.root`, `actor.getNode()` | Set placement and inspect private source-node state. | +| `actor.update(deltaSeconds)`, `actor.destroy()` | Advance or remove one independent actor. | diff --git a/docs/api-reference/gltf/gltf-animation.md b/docs/api-reference/gltf/gltf-animation.md index 4dcb88a20e..19d4bce67f 100644 --- a/docs/api-reference/gltf/gltf-animation.md +++ b/docs/api-reference/gltf/gltf-animation.md @@ -95,6 +95,9 @@ See the [engine animation guide](/docs/api-guide/engine/animation) and [AnimationMixer API reference](/docs/api-reference/engine/animation/animation-mixer) for pause, seek, reverse playback, once/repeat/ping-pong loops, weighted blending, and crossfading. +To share GPU models across independently posed actors, see +[GPU-animated glTF crowds](/docs/api-reference/gltf/gltf-animated-crowd). + ## Supported channels and interpolation | Source channel | Runtime target | diff --git a/docs/table-of-contents.json b/docs/table-of-contents.json index ddc2e5019d..f38994a937 100644 --- a/docs/table-of-contents.json +++ b/docs/table-of-contents.json @@ -331,6 +331,7 @@ "api-reference/gltf/gltf-materials", "api-reference/gltf/gltf-native-extensions", "api-reference/gltf/gltf-animation", + "api-reference/gltf/gltf-animated-crowd", "api-reference/gltf/gltf-interchange", "api-reference/gltf/gltf-extensions" ] diff --git a/examples/showcase/gltf/app.ts b/examples/showcase/gltf/app.ts index 4ac1fc1ffc..a2ebc5293e 100644 --- a/examples/showcase/gltf/app.ts +++ b/examples/showcase/gltf/app.ts @@ -20,6 +20,7 @@ import { makeHtmlCustomPanel } from '../../example-panels'; import GLTFCatalogApp, { + GLTF_CROWD_INFO_ID, GLTF_MODEL_INFO_ID, saveOptions, type GLTFCatalogModel, @@ -89,12 +90,13 @@ void main(void) { `; const GLTF_DESCRIPTION_HTML = `\ -

Browse production-quality glTF sample assets with interactive camera and animation controls.

+

Explore animated glTF characters, skeletal rigs, and expressive motion.

Drag to orbit. Use the mouse wheel or trackpad to zoom.

+
@@ -138,11 +140,11 @@ export default class AppAnimationLoopTemplate extends GLTFCatalogApp { } getDefaultModelName(): string { - return 'DamagedHelmet'; + return 'RobotExpressive'; } getModelStorageKey(): string { - return 'showcase-last-gltf-model-v2'; + return 'showcase-last-gltf-model-v3'; } getClearColor(): [number, number, number, number] { @@ -266,6 +268,7 @@ export default class AppAnimationLoopTemplate extends GLTFCatalogApp { return { extensionName: this.extensionName, modelValue: this.selectedModelValue || LOADING_MODEL_VALUE, + instanceCount: this.getAnimationInstanceCount(), useModelLights: this.options['useModelLights'], cameraAnimation: this.options['cameraAnimation'], gltfAnimation: this.options['gltfAnimation'] @@ -294,6 +297,11 @@ export default class AppAnimationLoopTemplate extends GLTFCatalogApp { this.selectModel(modelValue); return; } + const instanceCount = getChangedSetting(changedSettings, 'instanceCount')?.nextValue; + if (typeof instanceCount === 'number') { + this.setAnimationInstanceCount(instanceCount); + return; + } for (const optionName of ['useModelLights', 'cameraAnimation', 'gltfAnimation'] as const) { const nextValue = getChangedSetting(changedSettings, optionName)?.nextValue; if (typeof nextValue === 'boolean') { @@ -417,6 +425,7 @@ function encodeModelOption(modelOption: GLTFModelReference): string { type GltfSettingsState = { extensionName: string; modelValue: string; + instanceCount: number; useModelLights: boolean; cameraAnimation: boolean; gltfAnimation: boolean; @@ -469,6 +478,16 @@ export function makeGltfSettingsSchema( name: 'Animation', initiallyCollapsed: false, settings: [ + { + name: 'instanceCount', + label: 'GPU Crowd Actors', + type: 'number', + persist: 'none', + min: 1, + max: 100, + step: 1, + sliderDebounceMs: 120 + }, { name: 'useModelLights', label: 'Use Model Lights', diff --git a/examples/showcase/gltf/gltf-catalog-app.ts b/examples/showcase/gltf/gltf-catalog-app.ts index 31291d6b13..dacefd79a7 100644 --- a/examples/showcase/gltf/gltf-catalog-app.ts +++ b/examples/showcase/gltf/gltf-catalog-app.ts @@ -6,8 +6,14 @@ import {AnimationLoopTemplate, AnimationProps, ModelNode} from '@luma.gl/engine' import {Color, Device, RenderPass, log} from '@luma.gl/core'; import {load} from '@loaders.gl/core'; import {Light, LightingProps, type PBRMaterialUniforms} from '@luma.gl/shadertools'; -import {createScenegraphsFromGLTF, type PBREnvironment} from '@luma.gl/gltf'; -import {GLTFLoader, postProcessGLTF} from '@loaders.gl/gltf'; +import { + createGLTFAnimatedCrowd, + createScenegraphsFromGLTF, + type GLTFAnimatedCrowd, + type GLTFCrowdActorOptions, + type PBREnvironment +} from '@luma.gl/gltf'; +import {GLTFLoader, postProcessGLTF, type GLTFPostprocessed} from '@loaders.gl/gltf'; import {Matrix4} from '@math.gl/core'; /* eslint-disable camelcase */ @@ -15,16 +21,28 @@ import {Matrix4} from '@math.gl/core'; const MODEL_DIRECTORY_URL = 'https://raw.githubusercontent.com/KhronosGroup/glTF-Sample-Assets/main/Models'; const MODEL_LIST_URL = `${MODEL_DIRECTORY_URL}/model-index.json`; +const ROBOT_EXPRESSIVE_MODEL_URL = + 'https://raw.githubusercontent.com/mrdoob/three.js/dev/examples/models/gltf/RobotExpressive/RobotExpressive.glb'; const LAST_GLTF_MODEL_STORAGE_KEY = 'last-gltf-model'; const GLTF_OPTIONS_STORAGE_KEY = 'showcase-gltf-options'; const GLTF_LOADING_STYLE_ID = 'gltf-loading-indicator-style'; export const GLTF_MODEL_INFO_ID = 'model-info'; +export const GLTF_CROWD_INFO_ID = 'gltf-crowd-info'; export const GLTF_CONTROL_PANEL_STYLE = 'display: grid; gap: 8px;'; export const GLTF_CONTROL_ROW_STYLE = 'display: grid; grid-template-columns: 7rem minmax(0, 1fr); align-items: center; column-gap: 0.75rem;'; export const GLTF_SELECT_STYLE = 'width: 100%; min-width: 0;'; const MAX_CAMERA_TILT = 0.7; const CAMERA_TILT_HEIGHT_FACTOR = 0.35; +const AUTOMATIC_CAMERA_ORBIT_SPEED = 0.00012; +const MANUAL_CAMERA_ORBIT_SPEED = 0.001; +const MAXIMUM_GLTF_CROWD_ACTORS = 100; +const ADDITIONAL_ANIMATED_GLTF_MODELS = new Set([ + 'Fox', + 'MorphStressTest', + 'RobotExpressive', + 'SimpleMorph' +]); const lightSources = { ambientLight: { @@ -69,6 +87,7 @@ const INFO_HTML = `\
+
`; @@ -87,14 +106,36 @@ export type GLTFCatalogModel = { tags?: string[]; variants?: Record; }; + +/** Identifies animated Khronos samples without fetching every model document. */ +export function isAnimatedGLTFCatalogModel( + model: Pick +): boolean { + return ( + ADDITIONAL_ANIMATED_GLTF_MODELS.has(model.name) || + /animat/i.test(model.name) || + /(?:\.gif(?:[?#]|$)|(?:^|[/._-])animated(?:[._-]|$))/i.test(model.screenshot || '') + ); +} + type GLTFModelMetadata = Pick; const GLTF_MODEL_METADATA_OVERRIDES: Record = { PotOfCoalsAnimationPointer: { description: 'A non-reflective bumpy glass-like surface distorts the hot coals underneath, using KHR_animation_pointer to animate the heat refraction effect.' + }, + RobotExpressive: { + summary: + 'An expressive skinned robot with 14 named actions, facial expressions, and independently animated crowd playback.' } }; +const ROBOT_EXPRESSIVE_CATALOG_MODEL: GLTFCatalogModel = { + label: 'Robot Expressive', + name: 'RobotExpressive', + ...GLTF_MODEL_METADATA_OVERRIDES['RobotExpressive'], + variants: {'glTF-Binary': 'RobotExpressive.glb'} +}; export type GLTFModelReference = { name: string; variant?: string; @@ -107,6 +148,11 @@ export default class AppAnimationLoopTemplate extends AnimationLoopTemplate { device: Device; availableModels: GLTFCatalogModel[] = []; scenegraphsFromGLTF?: ReturnType; + animatedCrowd?: GLTFAnimatedCrowd; + activeScenegraphOptions: Parameters[2] = {}; + loadedGLTF?: GLTFPostprocessed; + previousCrowdFrameTime?: number; + crowdActionNames: string[] = []; modelLights: Light[] = []; center = [0, 0, 0]; cameraHeight = 0; @@ -144,9 +190,15 @@ export default class AppAnimationLoopTemplate extends AnimationLoopTemplate { return; } this.availableModels = models; - const cleanupModelMenu = this.initializeModelMenus(models, initialModelName); + const currentModelName = models.some(model => model.name === initialModelName) + ? initialModelName + : models.find(model => model.name === this.getDefaultModelName())?.name || + models[0]?.name || + initialModelName; + window.localStorage[modelStorageKey] = currentModelName; + const cleanupModelMenu = this.initializeModelMenus(models, currentModelName); this.cleanupCallbacks.push(cleanupModelMenu); - this.loadGLTF(initialModelName); + this.loadGLTF(currentModelName); }) .catch(error => { log.error( @@ -197,11 +249,16 @@ export default class AppAnimationLoopTemplate extends AnimationLoopTemplate { cleanupCallback(); } this.cleanupCallbacks = []; + this.animatedCrowd?.destroy(); + this.animatedCrowd = undefined; + this.crowdActionNames = []; destroyScenegraphs(this.scenegraphsFromGLTF); this.scenegraphsFromGLTF = undefined; + this.loadedGLTF = undefined; this.modelLights = []; this.setViewerLoadingState(false); updateModelInfoBox(); + updateCrowdInfo(); updateExtensionSupportTable(); } @@ -248,6 +305,82 @@ export default class AppAnimationLoopTemplate extends AnimationLoopTemplate { drawBackground(_renderPass: RenderPass): void {} + /** Returns the number of independently animated actors sharing the current asset. */ + getAnimationInstanceCount(): number { + return this.animatedCrowd?.actorCount || 1; + } + + /** Places independently phased actors in one GPU-instanced draw per source primitive. */ + setAnimationInstanceCount(instanceCount: number): void { + const actorCount = Math.max(1, Math.min(MAXIMUM_GLTF_CROWD_ACTORS, Math.floor(instanceCount))); + if (!this.loadedGLTF || (!this.animatedCrowd && actorCount === 1)) { + return; + } + + if (!this.animatedCrowd) { + if (this.scenegraphsFromGLTF?.extensionSupport.has('EXT_mesh_gpu_instancing')) { + showError(new Error('GPU animated crowds cannot nest EXT_mesh_gpu_instancing.')); + return; + } + + const previousScenegraphs = this.scenegraphsFromGLTF; + try { + this.animatedCrowd = createGLTFAnimatedCrowd(this.device, this.loadedGLTF, { + ...this.activeScenegraphOptions, + capacity: MAXIMUM_GLTF_CROWD_ACTORS + }); + } catch (error) { + showError(error); + return; + } + destroyScenegraphs(previousScenegraphs); + this.scenegraphsFromGLTF = this.animatedCrowd.scenegraphs; + this.modelLights = this.animatedCrowd.scenegraphs.lights; + this.previousCrowdFrameTime = undefined; + showError(); + } + + const clipNames = getAnimationClipNames(this.animatedCrowd.scenegraphs); + const preferredClips = ['Walking', 'Running', 'Dance', 'Wave', 'Idle']; + const availableClips = preferredClips.filter(name => clipNames.includes(name)); + const playableClips = availableClips.length ? availableClips : clipNames; + + const actorOptions: GLTFCrowdActorOptions[] = []; + const spacing = Math.max(this.sceneRadius * 2.5, 0.75); + for (let actorIndex = this.animatedCrowd.actorCount; actorIndex < actorCount; actorIndex++) { + const clip = playableClips.length + ? playableClips[actorIndex % playableClips.length] + : undefined; + const angle = actorIndex * 2.39996322973; + const radius = Math.sqrt(actorIndex) * spacing; + actorOptions.push({ + id: `gltf-crowd-actor-${actorIndex}`, + ...(clip ? {clip} : {}), + phase: (actorIndex * 0.61803398875) % 1, + speed: 0.8 + (actorIndex % 5) * 0.1, + transform: new Matrix4().translate([Math.cos(angle) * radius, 0, Math.sin(angle) * radius]) + }); + } + if (actorOptions.length) { + this.animatedCrowd.addActors(actorOptions); + } + + if (this.animatedCrowd.actorCount > actorCount) { + this.animatedCrowd.removeActors( + this.animatedCrowd.actors.slice(actorCount).map(actor => actor.id) + ); + } + + this.crowdActionNames = Array.from( + new Set( + this.animatedCrowd.actors + .map(actor => actor.activeClip) + .filter((name): name is string => Boolean(name)) + ) + ); + updateCrowdInfo(actorCount, this.animatedCrowd.models.length, this.crowdActionNames); + } + onRender({aspect, device, time}: AnimationProps): void { const renderPass = device.beginRenderPass({clearColor: this.getClearColor(), clearDepth: 1}); this.drawBackground(renderPass); @@ -259,12 +392,20 @@ export default class AppAnimationLoopTemplate extends AnimationLoopTemplate { updateModelLightIndicator(this.modelLights, this.options['useModelLights']); - const orbitDistance = this.cameraOrbitDistance; - const far = Math.max(orbitDistance + this.sceneRadius * 8, 10); + const actorCount = this.getAnimationInstanceCount(); + const actorSpacing = Math.max(this.sceneRadius * 2.5, 0.75); + const crowdRadius = + actorCount > 1 + ? Math.sqrt(actorCount - 1) * actorSpacing + this.sceneRadius + : this.sceneRadius; + const orbitDistance = + this.cameraOrbitDistance * Math.max(1, crowdRadius / Math.max(this.sceneRadius, 0.001)); + const far = Math.max(orbitDistance + crowdRadius * 2, 10); const near = Math.max(this.sceneRadius / 1000, 0.01); const projectionMatrix = new Matrix4().perspective({fovy: Math.PI / 3, aspect, near, far}); - const cameraTime = this.options['cameraAnimation'] ? time : this.mouseCameraTime; - const orbitAngle = 0.001 * cameraTime; + const orbitAngle = this.options['cameraAnimation'] + ? time * AUTOMATIC_CAMERA_ORBIT_SPEED + : this.mouseCameraTime * MANUAL_CAMERA_ORBIT_SPEED; const horizontalOrbitScale = Math.cos(this.mouseCameraTilt); const verticalOrbitOffset = orbitDistance * CAMERA_TILT_HEIGHT_FACTOR * Math.sin(this.mouseCameraTilt); @@ -274,7 +415,7 @@ export default class AppAnimationLoopTemplate extends AnimationLoopTemplate { orbitDistance * horizontalOrbitScale * Math.cos(orbitAngle) ]; - if (this.options['gltfAnimation']) { + if (this.options['gltfAnimation'] && !this.animatedCrowd) { this.scenegraphsFromGLTF.animator?.setTime(time); } @@ -283,6 +424,44 @@ export default class AppAnimationLoopTemplate extends AnimationLoopTemplate { const pbrMaterialProps = this.getPBRMaterialProps(); const hasPBRMaterialProps = Object.keys(pbrMaterialProps).length > 0; + if (this.animatedCrowd) { + const deltaSeconds = + this.previousCrowdFrameTime === undefined + ? 0 + : Math.min(Math.max(time - this.previousCrowdFrameTime, 0) / 1000, 0.1); + this.previousCrowdFrameTime = time; + if (this.options['gltfAnimation']) { + this.animatedCrowd.update(deltaSeconds); + } + + const modelMatrix = new Matrix4(); + const modelViewProjectionMatrix = new Matrix4(projectionMatrix).multiplyRight(viewMatrix); + for (const model of this.animatedCrowd.models) { + const sceneShaderInputProps: Record = { + lighting: this.getLightingProps(), + pbrProjection: { + camera: cameraPos, + modelViewProjectionMatrix, + modelMatrix, + normalMatrix: modelMatrix + } + }; + if (hasPBRMaterialProps) { + if (model.material?.ownsModule('pbrMaterial')) { + model.material.setProps({pbrMaterial: pbrMaterialProps}); + } else { + sceneShaderInputProps.pbrMaterial = pbrMaterialProps; + } + } + model.shaderInputs.setProps(sceneShaderInputProps); + } + + const drawCount = this.animatedCrowd.draw(renderPass); + updateCrowdInfo(this.animatedCrowd.actorCount, drawCount, this.crowdActionNames); + renderPass.end(); + return; + } + this.scenegraphsFromGLTF.scenes[0].traverse((node, {worldMatrix: modelMatrix}) => { const {model} = node as ModelNode; @@ -320,10 +499,12 @@ export default class AppAnimationLoopTemplate extends AnimationLoopTemplate { async fetchModelList(): Promise { const response = await fetch(MODEL_LIST_URL); const models = (await response.json()) as GLTFCatalogModel[]; - return models.map(model => ({ - ...model, - hasGLBVariant: Boolean(model.variants?.['glTF-Binary']) - })); + return [ROBOT_EXPRESSIVE_CATALOG_MODEL, ...models.filter(isAnimatedGLTFCatalogModel)].map( + model => ({ + ...model, + hasGLBVariant: Boolean(model.variants?.['glTF-Binary']) + }) + ); } async loadGLTF(modelReference: string | GLTFModelReference) { @@ -354,12 +535,17 @@ export default class AppAnimationLoopTemplate extends AnimationLoopTemplate { )(); const processedGLTF = postProcessGLTF(gltf); - const scenegraphsFromGLTF = createScenegraphsFromGLTF(this.device, processedGLTF, { + const scenegraphOptions = { lights: true, imageBasedLightingEnvironment, pbrDebug: false, useTangents: true - }); + }; + const scenegraphsFromGLTF = createScenegraphsFromGLTF( + this.device, + processedGLTF, + scenegraphOptions + ); log.log(0, `Created glTF scenegraphs: ${modelDescription}`)(); if (this.isLoadStale(loadGeneration)) { @@ -367,10 +553,21 @@ export default class AppAnimationLoopTemplate extends AnimationLoopTemplate { return; } + this.animatedCrowd?.destroy(); + this.animatedCrowd = undefined; + this.crowdActionNames = []; destroyScenegraphs(this.scenegraphsFromGLTF); this.scenegraphsFromGLTF = scenegraphsFromGLTF; + this.loadedGLTF = processedGLTF; + this.activeScenegraphOptions = scenegraphOptions; + this.previousCrowdFrameTime = undefined; this.modelLights = scenegraphsFromGLTF.lights; - this.updateModelInfo(resolvedModelReference, loadGeneration); + updateCrowdInfo(); + this.updateModelInfo( + resolvedModelReference, + loadGeneration, + getAnimationClipNames(scenegraphsFromGLTF) + ); updateExtensionSupportTable(scenegraphsFromGLTF.extensionSupport); const activeSceneBounds = @@ -416,10 +613,11 @@ export default class AppAnimationLoopTemplate extends AnimationLoopTemplate { private updateModelInfo( modelReference: Required, - loadGeneration: number + loadGeneration: number, + actionNames: readonly string[] = [] ): void { const catalogModel = this.availableModels.find(model => model.name === modelReference.name); - updateModelInfoBox(catalogModel, modelReference); + updateModelInfoBox(catalogModel, modelReference, actionNames); void this.fetchModelMetadata(modelReference.name).then(modelMetadata => { if (this.isLoadStale(loadGeneration)) { @@ -431,7 +629,7 @@ export default class AppAnimationLoopTemplate extends AnimationLoopTemplate { ); if (mutableCatalogModel) { Object.assign(mutableCatalogModel, modelMetadata); - updateModelInfoBox(mutableCatalogModel, modelReference); + updateModelInfoBox(mutableCatalogModel, modelReference, actionNames); return; } @@ -441,7 +639,8 @@ export default class AppAnimationLoopTemplate extends AnimationLoopTemplate { name: modelReference.name, ...modelMetadata }, - modelReference + modelReference, + actionNames ); }); } @@ -577,6 +776,10 @@ async function loadPreferredGLTF(candidateModelReferences: Required): string { + if (modelReference.name === 'RobotExpressive') { + return ROBOT_EXPRESSIVE_MODEL_URL; + } + return `${MODEL_DIRECTORY_URL}/${modelReference.name}/${modelReference.variant}/${modelReference.fileName}`; } @@ -671,6 +874,10 @@ function getModelLoadingLabel(modelDescription: string): string { } async function loadModelMetadata(modelName: string): Promise { + if (modelName === 'RobotExpressive') { + return GLTF_MODEL_METADATA_OVERRIDES[modelName] || {}; + } + const readmeUrl = `${MODEL_DIRECTORY_URL}/${modelName}/README.md`; try { @@ -749,7 +956,8 @@ function setLoadingState(isLoading: boolean, message?: string): void { function updateModelInfoBox( model?: Pick, - modelReference?: Required + modelReference?: Required, + actionNames: readonly string[] = [] ): void { const container = document.getElementById(GLTF_MODEL_INFO_ID) as HTMLDivElement | null; if (!container) { @@ -782,6 +990,14 @@ function updateModelInfoBox( container.append(variant); } + if (actionNames.length) { + const actions = document.createElement('div'); + actions.style.fontSize = '12px'; + actions.style.marginBottom = '6px'; + actions.textContent = `Actions: ${actionNames.join(' · ')}`; + container.append(actions); + } + if (summary) { const summaryParagraph = document.createElement('p'); summaryParagraph.style.margin = '0 0 6px 0'; @@ -827,6 +1043,41 @@ function updateModelLightIndicator(modelLights: Light[], useModelLights: boolean indicator.textContent = `Model lights: ${summary}; ${activeSource}.`; } +function getAnimationClipNames( + scenegraphs: Pick, 'animations'> +): string[] { + return Array.from( + new Set( + scenegraphs.animations + .map(animation => animation.name) + .filter((name): name is string => Boolean(name)) + ) + ); +} + +function updateCrowdInfo( + actorCount: number = 1, + drawCount: number = 0, + actionNames: readonly string[] = [] +): void { + const container = document.getElementById(GLTF_CROWD_INFO_ID) as HTMLDivElement | null; + if (!container) { + return; + } + + container.hidden = actorCount <= 1; + const actionSummary = actionNames.length + ? ` · ${actionNames.length > 1 ? 'Mixed actions' : 'Action'}: ${actionNames.join(', ')}` + : ''; + const summary = + actorCount > 1 + ? `${actorCount.toLocaleString()} independently animated actors · ${drawCount} shared GPU draws${actionSummary}` + : ''; + if (container.textContent !== summary) { + container.textContent = summary; + } +} + function updateExtensionSupportTable(extensionSupport?: GLTFExtensionSupportMap) { const container = document.getElementById('extension-support') as HTMLDivElement; if (!container) { diff --git a/examples/showcase/gltf/index.html b/examples/showcase/gltf/index.html index 555ac05e6b..e3fd8d4eb3 100644 --- a/examples/showcase/gltf/index.html +++ b/examples/showcase/gltf/index.html @@ -1,5 +1,8 @@ + + + glTF Animation Studio - + + + diff --git a/modules/arrow/src/arrow/gpu/arrow-gpu-analytics-adapters.ts b/modules/arrow/src/arrow/gpu/arrow-gpu-analytics-adapters.ts index 45befb7811..9524644d75 100644 --- a/modules/arrow/src/arrow/gpu/arrow-gpu-analytics-adapters.ts +++ b/modules/arrow/src/arrow/gpu/arrow-gpu-analytics-adapters.ts @@ -1,6 +1,6 @@ // luma.gl // SPDX-License-Identifier: MIT -// Copyright (c) vis.gl contributors +// SPDX-FileCopyrightText: Copyright (c) vis.gl contributors import {Buffer, type Device} from '@luma.gl/core'; import { diff --git a/modules/arrow/test/arrow/arrow-gpu-analytics-adapters.node.spec.ts b/modules/arrow/test/arrow/arrow-gpu-analytics-adapters.node.spec.ts index 574947a0bd..97e4f77e24 100644 --- a/modules/arrow/test/arrow/arrow-gpu-analytics-adapters.node.spec.ts +++ b/modules/arrow/test/arrow/arrow-gpu-analytics-adapters.node.spec.ts @@ -1,6 +1,6 @@ // luma.gl // SPDX-License-Identifier: MIT -// Copyright (c) vis.gl contributors +// SPDX-FileCopyrightText: Copyright (c) vis.gl contributors import {readFileSync} from 'node:fs'; diff --git a/modules/arrow/test/arrow/arrow-gpu-analytics-adapters.spec.ts b/modules/arrow/test/arrow/arrow-gpu-analytics-adapters.spec.ts index 45547cb078..16d55d735d 100644 --- a/modules/arrow/test/arrow/arrow-gpu-analytics-adapters.spec.ts +++ b/modules/arrow/test/arrow/arrow-gpu-analytics-adapters.spec.ts @@ -1,6 +1,6 @@ // luma.gl // SPDX-License-Identifier: MIT -// Copyright (c) vis.gl contributors +// SPDX-FileCopyrightText: Copyright (c) vis.gl contributors import {makeGPUAnalyticsTableFromArrowTable} from '@luma.gl/arrow'; import {Buffer} from '@luma.gl/core'; diff --git a/modules/gltf/src/gltf/create-gltf-model.ts b/modules/gltf/src/gltf/create-gltf-model.ts index ab40046048..7d9ef8ff18 100644 --- a/modules/gltf/src/gltf/create-gltf-model.ts +++ b/modules/gltf/src/gltf/create-gltf-model.ts @@ -52,6 +52,9 @@ struct VertexInputs { @location(10) instanceModelMatrixCol2: vec4f, @location(11) instanceModelMatrixCol3: vec4f, #endif +#ifdef HAS_INSTANCED_SKIN + @builtin(instance_index) instanceIndex: u32, +#endif }; struct FragmentInputs { @@ -99,7 +102,16 @@ fn vertexMain(inputs: VertexInputs) -> FragmentInputs { tangent = inputs.TANGENT; #endif #ifdef HAS_SKIN +#ifdef HAS_INSTANCED_SKIN + let skinMatrix = getInstancedSkinMatrix( + inputs.WEIGHTS_0, + inputs.JOINTS_0, + inputs.instanceIndex, + u32(CROWD_JOINTS_PER_INSTANCE) + ); +#else let skinMatrix = getSkinMatrix(inputs.WEIGHTS_0, inputs.JOINTS_0); +#endif position = skinMatrix * position; normal = normalize((skinMatrix * vec4f(normal, 0.0)).xyz); #ifdef HAS_TANGENTS @@ -221,7 +233,16 @@ const vs = /* glsl */ `\ vec4 pos = positions; #ifdef HAS_SKIN + #ifdef HAS_INSTANCED_SKIN + mat4 skinMat = getInstancedSkinMatrix( + WEIGHTS_0, + JOINTS_0, + uint(gl_InstanceID), + uint(CROWD_JOINTS_PER_INSTANCE) + ); + #else mat4 skinMat = getSkinMatrix(WEIGHTS_0, JOINTS_0); + #endif pos = skinMat * pos; _NORMAL = skinMat * _NORMAL; _TANGENT = vec4((skinMat * vec4(_TANGENT.xyz, 0.)).xyz, _TANGENT.w); @@ -280,6 +301,21 @@ export type CreateGLTFMaterialOptions = { materialFactory?: MaterialFactory; }; +/** Internal shared primitive configuration supplied by the glTF crowd adapter. */ +export type GLTFCrowdModelConfiguration = { + capacity: number; + jointsPerInstance: number; +}; + +/** Internal instance resources owned by exactly one canonical glTF primitive model. */ +export type GLTFCrowdModelResources = { + transformBuffers: readonly Buffer[]; + transformColumns: readonly Float32Array[]; + skinJointMatrices?: Buffer | Texture; + jointMatrices?: Float32Array; + jointsPerInstance: number; +}; + export function createGLTFMaterial(device: Device, options: CreateGLTFMaterialOptions): Material { const materialFactory = options.materialFactory || new MaterialFactory(device, {modules: [pbrMaterial]}); @@ -314,6 +350,12 @@ export function createGLTFModel(device: Device, options: CreateGLTFModelOptions) modelOptions = {}, instanceMatrices } = options; + const crowd = modelOptions.userData?.['gltfAnimatedCrowd'] as + | GLTFCrowdModelConfiguration + | undefined; + if (crowd && instanceMatrices) { + throw new Error('Nested glTF crowd instancing is unsupported'); + } log.info(4, 'createGLTFModel defines: ', parsedPPBRMaterial.defines)(); @@ -333,10 +375,12 @@ export function createGLTFModel(device: Device, options: CreateGLTFModelOptions) const instanceAttributes: Record = {}; const instanceBufferLayout: BufferLayout[] = []; - if (instanceMatrices) { + const transformBuffers: Buffer[] = []; + const transformColumns: Float32Array[] = []; + if (instanceMatrices || crowd) { for (let columnIndex = 0; columnIndex < 4; columnIndex++) { - const values = new Float32Array(instanceMatrices.length * 4); - instanceMatrices.forEach((matrix, instanceIndex) => { + const values = new Float32Array((crowd?.capacity || instanceMatrices?.length || 0) * 4); + instanceMatrices?.forEach((matrix, instanceIndex) => { for (let rowIndex = 0; rowIndex < 4; rowIndex++) { values[instanceIndex * 4 + rowIndex] = matrix[columnIndex * 4 + rowIndex]; } @@ -350,12 +394,39 @@ export function createGLTFModel(device: Device, options: CreateGLTFModelOptions) instanceAttributes[attributeName] = buffer; instanceBufferLayout.push({name: attributeName, format: 'float32x4', stepMode: 'instance'}); managedResources.push(buffer); + transformBuffers.push(buffer); + transformColumns.push(values); } } + const hasInstancedSkin = Boolean(crowd && parsedPPBRMaterial.defines['HAS_SKIN']); + let skinJointMatrices: Buffer | Texture | undefined; + let jointMatrices: Float32Array | undefined; + if (crowd && hasInstancedSkin) { + jointMatrices = new Float32Array(crowd.capacity * crowd.jointsPerInstance * 16); + skinJointMatrices = + device.type === 'webgpu' + ? device.createBuffer({ + id: `${id || 'gltf'}-crowd-joint-matrices`, + byteLength: jointMatrices.byteLength, + usage: Buffer.STORAGE | Buffer.COPY_DST + }) + : device.createTexture({ + id: `${id || 'gltf'}-crowd-joint-matrices`, + format: 'rgba32float', + width: crowd.jointsPerInstance * 4, + height: crowd.capacity, + usage: Texture.SAMPLE | Texture.COPY_DST, + sampler: {minFilter: 'nearest', magFilter: 'nearest', mipmapFilter: 'nearest'} + }); + managedResources.push(skinJointMatrices); + } + const modelProps: ModelProps = { id, - source: SHADER, + source: hasInstancedSkin + ? SHADER.replace('u32(CROWD_JOINTS_PER_INSTANCE)', `u32(${crowd!.jointsPerInstance})`) + : SHADER, vs, fs, geometry, @@ -364,18 +435,21 @@ export function createGLTFModel(device: Device, options: CreateGLTFModelOptions) modules: [pbrMaterial, skin], ...modelOptions, - ...(instanceMatrices + ...(instanceMatrices || crowd ? { attributes: {...modelOptions.attributes, ...instanceAttributes}, bufferLayout: [...(modelOptions.bufferLayout || []), ...instanceBufferLayout], - instanceCount: instanceMatrices.length, + instanceCount: instanceMatrices?.length || 0, isInstanced: true } : {}), defines: { ...parsedPPBRMaterial.defines, - ...(instanceMatrices ? {HAS_GLTF_INSTANCING: true} : {}), - ...modelOptions.defines + ...modelOptions.defines, + ...(instanceMatrices || crowd ? {HAS_GLTF_INSTANCING: true} : {}), + ...(hasInstancedSkin + ? {HAS_INSTANCED_SKIN: true, CROWD_JOINTS_PER_INSTANCE: crowd!.jointsPerInstance} + : {}) }, parameters: {...parameters, ...parsedPPBRMaterial.parameters, ...modelOptions.parameters} }; @@ -402,12 +476,25 @@ export function createGLTFModel(device: Device, options: CreateGLTFModelOptions) sceneShaderInputValues ); model.shaderInputs.setProps(sceneShaderInputProps); - return new ModelNode({ + if (skinJointMatrices) { + model.shaderInputs.setProps({skin: {jointMatrices: [], skinJointMatrices}}); + } + const modelNode = new ModelNode({ managedResources, model, bounds: options.bounds, instanceMatrices }); + if (crowd) { + modelNode.userData['gltfAnimatedCrowd'] = { + transformBuffers, + transformColumns, + skinJointMatrices, + jointMatrices, + jointsPerInstance: crowd.jointsPerInstance + } satisfies GLTFCrowdModelResources; + } + return modelNode; } function isMaterialBindingResource(value: unknown): boolean { diff --git a/modules/gltf/src/gltf/gltf-animated-crowd.ts b/modules/gltf/src/gltf/gltf-animated-crowd.ts new file mode 100644 index 0000000000..cb740464d5 --- /dev/null +++ b/modules/gltf/src/gltf/gltf-animated-crowd.ts @@ -0,0 +1,643 @@ +// luma.gl +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: Copyright (c) vis.gl contributors + +import type {GLTFPostprocessed} from '@loaders.gl/gltf'; +import {assert, Buffer, type Device, type RenderPass, Texture} from '@luma.gl/core'; +import { + type AnimationLoopMode, + type AnimationMixer, + GroupNode, + ModelNode, + updateSkinJointMatrices +} from '@luma.gl/engine'; +import type {Model} from '@luma.gl/engine'; +import {Matrix4, type NumericArray} from '@math.gl/core'; +import type {ParseGLTFOptions} from '../parsers/parse-gltf'; +import {createScenegraphsFromGLTF, type GLTFScenegraphs} from './create-scenegraph-from-gltf'; +import type {GLTFCrowdModelConfiguration, GLTFCrowdModelResources} from './create-gltf-model'; +import {type GLTFAnimationSelectionOptions, GLTFAnimator} from './gltf-animator'; +import {GLTFSkinController} from './gltf-skin'; + +/** Fixed shared-model and GPU-buffer configuration for an independently animated glTF crowd. */ +export type GLTFAnimatedCrowdOptions = ParseGLTFOptions & { + /** Maximum simultaneous actors; fixes GPU buffer and palette-atlas allocations. Defaults to 16. */ + capacity?: number; +}; + +/** Initial placement and independent playback controls for one lightweight crowd actor. */ +export type GLTFCrowdActorOptions = { + id?: string; + clip?: string; + /** Initial clip-local time in seconds; takes precedence over normalized phase. */ + time?: number; + /** Initial normalized clip phase. */ + phase?: number; + speed?: number; + loop?: AnimationLoopMode; + repetitions?: number; + playing?: boolean; + /** Actor placement matrix, multiplied by independently animated authored node transforms. */ + transform?: Readonly; +}; + +/** Crossfade and initial playback position for a lightweight crowd actor. */ +export type GLTFCrowdClipSelectionOptions = GLTFAnimationSelectionOptions & { + time?: number; + phase?: number; +}; + +/** One canonical glTF primitive submitted exactly once for every compatible actor pose. */ +export type GLTFCrowdPrimitiveGroup = { + /** Source node whose animated world transform places this primitive. */ + nodeIndex: number; + /** One shared immutable-geometry/material model used for every actor. */ + model: Model; + /** Four shared per-instance matrix-column vertex buffers. */ + transformBuffers: readonly Buffer[]; + /** Number of joints in the authored skin driving this primitive. */ + jointCount: number; + /** Actor-major CPU staging palette uploaded to the shared GPU skin resource. */ + jointMatrices?: Float32Array; + /** WebGPU read-only storage buffer or WebGL float-texture palette atlas. */ + skinJointMatrices?: Buffer | Texture; +}; + +type GLTFCrowdActorNodes = { + root: GroupNode; + scenes: GroupNode[]; + nodesByIndex: Map; + nodesById: Map; +}; + +/** Lightweight independent animation state targeting shared, GPU-instanced glTF primitives. */ +export class GLTFCrowdActor { + readonly id: string; + /** Actor-local CPU node hierarchy; contains no render Models or mutable vertex buffers. */ + readonly root: GroupNode; + /** Existing reusable glTF animator backed by one actor-local engine animation mixer. */ + readonly animator: GLTFAnimator; + /** Existing glTF skin controller producing actor-local mesh-space joint palettes. */ + readonly skins: GLTFSkinController; + + private readonly crowd: GLTFAnimatedCrowd; + private readonly nodesByIndex: Map; + private readonly nodesById: Map; + private isPlaying: boolean; + private isDestroyed = false; + private skinMatricesNeedUpdate = true; + + /** @internal Actors are created and owned by {@link GLTFAnimatedCrowd}. */ + constructor(crowd: GLTFAnimatedCrowd, id: string, options: GLTFCrowdActorOptions = {}) { + this.crowd = crowd; + this.id = id; + const hierarchy = createActorNodes(crowd.scenegraphs, id); + this.root = hierarchy.root; + this.nodesByIndex = hierarchy.nodesByIndex; + this.nodesById = hierarchy.nodesById; + this.isPlaying = options.playing ?? true; + + this.skins = new GLTFSkinController({ + gltf: crowd.gltf, + scenes: hierarchy.scenes, + gltfNodeIndexToNodeMap: this.nodesByIndex + }); + this.animator = new GLTFAnimator({ + animations: crowd.scenegraphs.animations.map(animation => ({ + name: animation.name, + channels: animation.channels.filter(channel => channel.type === 'node') + })), + gltfNodeIdToNodeMap: this.nodesById, + autoplay: 'first', + onUpdate: () => { + this.skinMatricesNeedUpdate = true; + } + }); + + if (options.transform) { + this.root.setMatrix(options.transform); + } + + const clip = options.clip || this.animator.activeClip; + if (clip) { + this.animator.selectClip(clip); + const action = this.mixer.getAction(clip); + if (options.loop) { + action?.setLoop(options.loop, options.repetitions); + } + if (options.time !== undefined || options.phase !== undefined) { + this.seek(options.time ?? (options.phase || 0) * (action?.clip.duration || 0)); + } else { + this.animator.update(0); + } + } + + this.setSpeed(options.speed ?? 1); + if (!this.isPlaying) { + this.pause(); + } + } + + get mixer(): AnimationMixer { + return this.animator.mixer; + } + + get activeClip(): string | undefined { + return this.animator.activeClip; + } + + /** Selected clip-local time in seconds. */ + get time(): number { + return this.activeClip ? this.mixer.getAction(this.activeClip)?.time || 0 : 0; + } + + get speed(): number { + return this.mixer.timeScale; + } + + get playing(): boolean { + return this.isPlaying; + } + + get destroyed(): boolean { + return this.isDestroyed; + } + + /** Returns this actor's private authored node by glTF source index or identifier. */ + getNode(node: number | string): GroupNode | undefined { + return typeof node === 'number' ? this.nodesByIndex.get(node) : this.nodesById.get(node); + } + + /** Replaces actor placement without mutating another actor or shared source nodes. */ + setTransform(transform: Readonly): this { + this.root.setMatrix(transform); + this.crowd.refresh(); + return this; + } + + /** Selects or crossfades an authored clip independently from every neighboring actor. */ + selectClip(name: string, options: GLTFCrowdClipSelectionOptions = {}): this { + const clip = this.animator.selectClip(name, options); + if (options.time !== undefined || options.phase !== undefined) { + this.seek(options.time ?? (options.phase || 0) * clip.clip.duration); + } + if (!this.isPlaying) { + this.pause(); + } + this.crowd.refresh(); + return this; + } + + /** Seeks all active actor-local actions to seconds and updates its joint palettes once. */ + seek(timeSeconds: number): this { + for (const clip of this.animator.getAnimations()) { + if (clip.action.playing) { + clip.action.setTime(timeSeconds); + } + } + this.animator.update(0); + this.crowd.refresh(); + return this; + } + + setPhase(phase: number): this { + const duration = this.activeClip + ? this.mixer.getAction(this.activeClip)?.clip.duration || 0 + : 0; + return this.seek(duration * phase); + } + + setSpeed(speed: number): this { + this.mixer.timeScale = speed; + return this; + } + + setLoop(loop: AnimationLoopMode, repetitions?: number): this { + if (this.activeClip) { + this.mixer.getAction(this.activeClip)?.setLoop(loop, repetitions); + } + return this; + } + + play(): this { + this.isPlaying = true; + for (const clip of this.animator.getAnimations()) { + if (clip.action.playing) { + clip.action.resume(); + } + } + return this; + } + + pause(): this { + this.isPlaying = false; + for (const clip of this.animator.getAnimations()) { + if (clip.action.playing) { + clip.action.pause(); + } + } + return this; + } + + /** Advances only this actor and immediately refreshes the existing shared instance buffers. */ + update(deltaSeconds: number): this { + this.advance(deltaSeconds); + this.crowd.refresh(); + return this; + } + + /** @internal Allows the crowd to upload all actors together after one shared frame update. */ + advance(deltaSeconds: number): void { + if (this.isPlaying && !this.isDestroyed) { + this.animator.update(deltaSeconds); + } + } + + /** @internal Reuses the crowd's one scene traversal for both placement and joint palettes. */ + updateSkinMatrices(worldMatrices: ReadonlyMap): void { + if (!this.skinMatricesNeedUpdate) { + return; + } + + for (const binding of this.skins.bindings) { + updateSkinJointMatrices({ + joints: binding.joints, + meshNode: binding.node, + worldMatrices, + inverseBindMatrices: binding.inverseBindMatrices, + target: binding.jointMatrices + }); + } + this.skinMatricesNeedUpdate = false; + } + + /** Releases lightweight actor-local state without touching shared GPU models or source data. */ + destroy(): void { + if (this.isDestroyed) { + return; + } + this.isDestroyed = true; + this.isPlaying = false; + this.crowd.removeActor(this.id); + this.root.destroy(); + } +} + +/** + * Draws independently animated glTF actors through one shared instanced Model per primitive. + * + * The source is parsed once. Every actor owns only CPU transforms, animation actions, and joint + * staging palettes; immutable geometry, materials, pipelines, instance buffers, and draw calls + * are shared across the entire crowd. + */ +export class GLTFAnimatedCrowd { + readonly device: Device; + readonly gltf: GLTFPostprocessed; + readonly scenegraphs: GLTFScenegraphs; + readonly capacity: number; + readonly primitiveGroups: readonly GLTFCrowdPrimitiveGroup[]; + readonly models: readonly Model[]; + + private readonly actorsById = new Map(); + private nextActorIndex = 0; + private isDestroyed = false; + private suspendedRefreshCount = 0; + + constructor(device: Device, gltf: GLTFPostprocessed, options: GLTFAnimatedCrowdOptions = {}) { + const {capacity = 16, ...parseOptions} = options; + // Fixed capacity keeps GPU instance and joint-palette buffers stable for the crowd lifetime. + assert(Number.isSafeInteger(capacity) && capacity > 0); + this.device = device; + this.gltf = gltf; + this.capacity = capacity; + + const jointsPerInstance = Math.max(0, ...(gltf.skins || []).map(skin => skin.joints.length)); + const configuration: GLTFCrowdModelConfiguration = {capacity, jointsPerInstance}; + this.scenegraphs = createScenegraphsFromGLTF(device, gltf, { + ...parseOptions, + modelOptions: { + ...parseOptions.modelOptions, + userData: {...parseOptions.modelOptions?.userData, gltfAnimatedCrowd: configuration} + } + }); + this.primitiveGroups = createPrimitiveGroups(this.scenegraphs); + this.models = this.primitiveGroups.map(group => group.model); + } + + get actors(): readonly GLTFCrowdActor[] { + return [...this.actorsById.values()]; + } + + get actorCount(): number { + return this.actorsById.size; + } + + get destroyed(): boolean { + return this.isDestroyed; + } + + /** Adds independent CPU clip/node state without parsing the source or allocating GPU models. */ + addActor(options: GLTFCrowdActorOptions = {}): GLTFCrowdActor { + // Fixed-capacity crowd buffers cannot represent additional actors after destruction or overflow. + assert(!this.isDestroyed && this.actorsById.size < this.capacity); + const id = options.id || `gltf-crowd-actor-${this.nextActorIndex++}`; + // Actor identifiers are stable keys for removal and application scene integrations. + assert(!this.actorsById.has(id)); + + let actor: GLTFCrowdActor; + this.suspendedRefreshCount++; + try { + actor = new GLTFCrowdActor(this, id, options); + this.actorsById.set(id, actor); + } finally { + this.suspendedRefreshCount--; + } + this.refresh(); + return actor; + } + + /** Adds many independently initialized actors while uploading shared GPU buffers only once. */ + addActors(options: readonly GLTFCrowdActorOptions[]): GLTFCrowdActor[] { + // Validate fixed capacity before allocating any actor-local animation or hierarchy state. + assert(!this.isDestroyed && this.actorsById.size + options.length <= this.capacity); + const actors: GLTFCrowdActor[] = []; + this.suspendedRefreshCount++; + try { + for (const actorOptions of options) { + actors.push(this.addActor(actorOptions)); + } + } finally { + this.suspendedRefreshCount--; + this.refresh(); + } + return actors; + } + + getActor(id: string): GLTFCrowdActor | undefined { + return this.actorsById.get(id); + } + + /** Compacts actor slots and updates every shared primitive without recreating its pipeline. */ + removeActor(id: string): boolean { + const actor = this.actorsById.get(id); + if (!actor) { + return false; + } + this.actorsById.delete(id); + if (!actor.destroyed) { + actor.destroy(); + } + this.refresh(); + return true; + } + + /** Removes and compacts many actors while uploading surviving instance slots only once. */ + removeActors(ids: readonly string[]): number { + let removedActorCount = 0; + this.suspendedRefreshCount++; + try { + for (const id of ids) { + if (this.removeActor(id)) { + removedActorCount++; + } + } + } finally { + this.suspendedRefreshCount--; + this.refresh(); + } + return removedActorCount; + } + + /** Evaluates independent clips in seconds and uploads all actor transforms/palettes once. */ + update(deltaSeconds: number): this { + for (const actor of this.actorsById.values()) { + actor.advance(deltaSeconds); + } + this.refresh(); + return this; + } + + /** Issues exactly one instanced draw for each compatible source primitive. */ + draw(renderPass: RenderPass): number { + if (this.isDestroyed || this.actorCount === 0) { + return 0; + } + let drawCount = 0; + for (const group of this.primitiveGroups) { + if (group.model.draw(renderPass)) { + drawCount++; + } + } + return drawCount; + } + + /** @internal Reuses fixed GPU buffers while packing current actor node and joint transforms. */ + refresh(): void { + if (this.isDestroyed || this.suspendedRefreshCount > 0 || !this.primitiveGroups) { + return; + } + + const actors = [...this.actorsById.values()]; + const actorWorldMatrices = actors.map(actor => { + const worldMatrices = collectNodeWorldMatrices(actor.root); + actor.updateSkinMatrices(worldMatrices); + return worldMatrices; + }); + + for (const group of this.primitiveGroups) { + const modelNode = findCrowdModelNode(this.scenegraphs, group.nodeIndex, group.model); + if (!modelNode) { + continue; + } + const resources = modelNode.userData['gltfAnimatedCrowd'] as GLTFCrowdModelResources; + + for (let actorIndex = 0; actorIndex < actors.length; actorIndex++) { + const actor = actors[actorIndex]; + const actorNode = actor.getNode(group.nodeIndex); + const matrix = actorNode && actorWorldMatrices[actorIndex].get(actorNode); + for (let columnIndex = 0; columnIndex < 4; columnIndex++) { + for (let rowIndex = 0; rowIndex < 4; rowIndex++) { + resources.transformColumns[columnIndex][actorIndex * 4 + rowIndex] = + matrix?.[columnIndex * 4 + rowIndex] || 0; + } + } + + if (resources.jointMatrices) { + const jointPalette = actor.skins.getBinding(group.nodeIndex)?.jointMatrices; + const offset = actorIndex * resources.jointsPerInstance * 16; + resources.jointMatrices.fill(0, offset, offset + resources.jointsPerInstance * 16); + if (jointPalette) { + resources.jointMatrices.set(jointPalette, offset); + } + } + } + + if (actors.length > 0) { + for (let columnIndex = 0; columnIndex < resources.transformBuffers.length; columnIndex++) { + resources.transformBuffers[columnIndex].write( + resources.transformColumns[columnIndex].subarray(0, actors.length * 4) + ); + } + if (resources.jointMatrices && resources.skinJointMatrices) { + const jointMatrices = resources.jointMatrices.subarray( + 0, + actors.length * resources.jointsPerInstance * 16 + ); + if (resources.skinJointMatrices instanceof Buffer) { + resources.skinJointMatrices.write(jointMatrices); + } else { + resources.skinJointMatrices.writeData(jointMatrices, { + width: resources.jointsPerInstance * 4, + height: actors.length + }); + } + } + } + group.model.setInstanceCount(actors.length); + } + } + + /** Destroys actor CPU state and the one canonical source scenegraph exactly once. */ + destroy(): void { + if (this.isDestroyed) { + return; + } + this.isDestroyed = true; + for (const actor of [...this.actorsById.values()]) { + actor.destroy(); + } + this.actorsById.clear(); + this.scenegraphs.destroy(); + } +} + +/** Parses one source asset once and creates a shared-model GPU-instanced animation crowd. */ +export function createGLTFAnimatedCrowd( + device: Device, + gltf: GLTFPostprocessed, + options: GLTFAnimatedCrowdOptions = {} +): GLTFAnimatedCrowd { + return new GLTFAnimatedCrowd(device, gltf, options); +} + +function createActorNodes(scenegraphs: GLTFScenegraphs, id: string): GLTFCrowdActorNodes { + const {gltf, gltfNodeIndexToNodeMap} = scenegraphs; + const nodesByIndex = new Map(); + const nodesById = new Map(); + + for (let nodeIndex = 0; nodeIndex < gltf.nodes.length; nodeIndex++) { + const sourceNode = gltf.nodes[nodeIndex]; + const sourceRuntimeNode = gltfNodeIndexToNodeMap.get(nodeIndex); + if (!sourceRuntimeNode) { + continue; + } + const node = new GroupNode({ + id: sourceRuntimeNode.id, + position: Array.from(sourceRuntimeNode.position), + rotation: Array.from(sourceRuntimeNode.rotation), + scale: Array.from(sourceRuntimeNode.scale), + matrix: Array.from(sourceRuntimeNode.matrix), + display: sourceRuntimeNode.display + }); + const morphWeights = sourceRuntimeNode.userData['morphWeights']; + if (Array.isArray(morphWeights)) { + node.userData['morphWeights'] = [...morphWeights]; + } + nodesByIndex.set(nodeIndex, node); + nodesById.set(sourceNode.id, node); + } + + for (let nodeIndex = 0; nodeIndex < gltf.nodes.length; nodeIndex++) { + const sourceNode = gltf.nodes[nodeIndex]; + const node = nodesByIndex.get(nodeIndex); + if (!node) { + continue; + } + for (const child of sourceNode.children || []) { + const childNode = nodesById.get(child.id); + if (childNode) { + node.add(childNode); + } + } + if (sourceNode.mesh) { + const mesh = new GroupNode({id: sourceNode.mesh.name || sourceNode.mesh.id}); + node.userData['gltfMesh'] = mesh; + node.add(mesh); + } + } + + const scenes = gltf.scenes.map( + (scene, sceneIndex) => + new GroupNode({ + id: `${id}-scene-${sceneIndex}`, + children: (scene.nodes || []).flatMap(sourceNode => { + const node = nodesById.get(sourceNode.id); + return node ? [node] : []; + }) + }) + ); + + return { + root: new GroupNode({id: `${id}-root`, children: [...scenes]}), + scenes, + nodesByIndex, + nodesById + }; +} + +function createPrimitiveGroups(scenegraphs: GLTFScenegraphs): GLTFCrowdPrimitiveGroup[] { + const groups: GLTFCrowdPrimitiveGroup[] = []; + for (const [nodeIndex, sourceNode] of scenegraphs.gltf.nodes.entries()) { + if (!sourceNode.mesh) { + continue; + } + const node = scenegraphs.gltfNodeIndexToNodeMap.get(nodeIndex); + const mesh = node?.userData['gltfMesh']; + if (!(mesh instanceof GroupNode)) { + continue; + } + const skinBinding = scenegraphs.skins.getBinding(nodeIndex); + for (const child of mesh.children) { + if (!(child instanceof ModelNode)) { + continue; + } + const resources = child.userData['gltfAnimatedCrowd'] as GLTFCrowdModelResources | undefined; + if (!resources) { + continue; + } + groups.push({ + nodeIndex, + model: child.model, + transformBuffers: resources.transformBuffers, + jointCount: skinBinding?.joints.length || 0, + ...(resources.jointMatrices ? {jointMatrices: resources.jointMatrices} : {}), + ...(resources.skinJointMatrices ? {skinJointMatrices: resources.skinJointMatrices} : {}) + }); + } + } + return groups; +} + +function findCrowdModelNode( + scenegraphs: GLTFScenegraphs, + nodeIndex: number, + model: Model +): ModelNode | undefined { + const mesh = scenegraphs.gltfNodeIndexToNodeMap.get(nodeIndex)?.userData['gltfMesh']; + if (!(mesh instanceof GroupNode)) { + return undefined; + } + return mesh.children.find( + (node): node is ModelNode => node instanceof ModelNode && node.model === model + ); +} + +function collectNodeWorldMatrices(root: GroupNode): Map { + const result = new Map(); + root.preorderTraversal((node, {worldMatrix}) => { + if (node instanceof GroupNode) { + result.set(node, worldMatrix); + } + }); + return result; +} diff --git a/modules/gltf/src/index.ts b/modules/gltf/src/index.ts index 4468cc0c9d..1f3064c3e4 100644 --- a/modules/gltf/src/index.ts +++ b/modules/gltf/src/index.ts @@ -25,6 +25,15 @@ export { GLTFAnimator, type GLTFAnimatorProps } from './gltf/gltf-animator'; +export { + createGLTFAnimatedCrowd, + GLTFAnimatedCrowd, + type GLTFAnimatedCrowdOptions, + GLTFCrowdActor, + type GLTFCrowdActorOptions, + type GLTFCrowdClipSelectionOptions, + type GLTFCrowdPrimitiveGroup +} from './gltf/gltf-animated-crowd'; export { type GLTFExtensionSupport, type GLTFExtensionSupportLevel, diff --git a/modules/gltf/test/gltf/gltf-animated-crowd.node.spec.ts b/modules/gltf/test/gltf/gltf-animated-crowd.node.spec.ts new file mode 100644 index 0000000000..a5b59b469f --- /dev/null +++ b/modules/gltf/test/gltf/gltf-animated-crowd.node.spec.ts @@ -0,0 +1,230 @@ +// luma.gl +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: Copyright (c) vis.gl contributors + +import {readFile} from 'node:fs/promises'; +import {parse} from '@loaders.gl/core'; +import {GLTFLoader, type GLTFPostprocessed, postProcessGLTF} from '@loaders.gl/gltf'; +import {Texture, type RenderPass} from '@luma.gl/core'; +import {ModelNode} from '@luma.gl/engine'; +import {createGLTFAnimatedCrowd} from '@luma.gl/gltf'; +import {NullDevice} from '@luma.gl/test-utils'; +import {Matrix4} from '@math.gl/core'; +import {describe, expect, test, vi} from 'vitest'; + +async function loadCrowdFixture( + fixture: 'SimpleSkin.gltf' | 'AnimatedMorphCube.glb' +): Promise { + const data = await readFile( + new URL(`../../../../examples/showcase/anari/public/gltf/${fixture}`, import.meta.url) + ); + return postProcessGLTF(await parse(data, GLTFLoader, {gltf: {loadImages: false}})); +} + +function getActiveResourceCount(device: NullDevice, resource: 'Buffers' | 'Textures'): number { + return device.statsManager.getStats('Resource Counts').get(`${resource} Active`).count; +} + +describe('GPU-instanced glTF animation crowds', () => { + test('shares one parsed skinned primitive while keeping actor poses and joint palettes independent', async () => { + const source = await loadCrowdFixture('SimpleSkin.gltf'); + const sourceNodeTransforms = source.nodes.map(node => ({ + translation: node.translation ? [...node.translation] : undefined, + rotation: node.rotation ? [...node.rotation] : undefined + })); + const device = new NullDevice({}); + const initialBuffers = getActiveResourceCount(device, 'Buffers'); + const initialTextures = getActiveResourceCount(device, 'Textures'); + const crowd = createGLTFAnimatedCrowd(device, source, {capacity: 8}); + + expect(crowd.primitiveGroups).toHaveLength(1); + expect(crowd.models).toHaveLength(1); + expect(crowd.models[0].isInstanced).toBe(true); + expect(crowd.primitiveGroups[0].jointCount).toBe(2); + expect(crowd.primitiveGroups[0].skinJointMatrices).toBeInstanceOf(Texture); + const allocatedBuffers = getActiveResourceCount(device, 'Buffers'); + const allocatedTextures = getActiveResourceCount(device, 'Textures'); + + const buffer = crowd.primitiveGroups[0].transformBuffers[0]; + const write = vi.spyOn(Object.getPrototypeOf(buffer), 'write'); + let first; + let second; + try { + [first, second] = crowd.addActors([ + { + id: 'left', + phase: 0, + speed: 1, + transform: new Matrix4().translate([-3, 0, 0]) + }, + { + id: 'right', + phase: 0.25, + speed: 2, + transform: new Matrix4().translate([6, 0, 0]) + } + ]); + expect(write).toHaveBeenCalledTimes(4); + expect(write.mock.calls.every(([data]) => data.byteLength === 2 * 4 * 4)).toBe(true); + } finally { + write.mockRestore(); + } + + expect(first).toBeDefined(); + expect(second).toBeDefined(); + if (!first || !second) { + throw new Error('Missing crowd actors'); + } + + expect(crowd.actorCount).toBe(2); + expect(crowd.getActor('left')).toBe(first); + expect(crowd.getActor('right')).toBe(second); + expect(first.root).not.toBe(second.root); + expect(first.animator).not.toBe(second.animator); + expect(first.skins).not.toBe(second.skins); + expect(first.skins.bindings).toHaveLength(1); + expect(second.skins.bindings).toHaveLength(1); + expect(first.skins.bindings[0].models).toHaveLength(0); + expect(second.skins.bindings[0].models).toHaveLength(0); + expect(Array.from(first.skins.bindings[0].jointMatrices)).not.toEqual( + Array.from(second.skins.bindings[0].jointMatrices) + ); + expect(getActiveResourceCount(device, 'Buffers')).toBe(allocatedBuffers); + expect(getActiveResourceCount(device, 'Textures')).toBe(allocatedTextures); + expect(crowd.models[0].instanceCount).toBe(2); + + for (const actor of crowd.actors) { + actor.root.preorderTraversal(node => expect(node).not.toBeInstanceOf(ModelNode)); + } + + const positionBytes = await crowd.primitiveGroups[0].transformBuffers[3].readAsync(); + const positions = new Float32Array( + positionBytes.buffer, + positionBytes.byteOffset, + positionBytes.byteLength / Float32Array.BYTES_PER_ELEMENT + ); + expect(positions[0]).toBeCloseTo(-3); + expect(positions[4]).toBeCloseTo(6); + + const packedJointMatrices = crowd.primitiveGroups[0].jointMatrices; + expect(packedJointMatrices).toBeDefined(); + if (packedJointMatrices) { + expect(Array.from(packedJointMatrices.subarray(0, 32))).not.toEqual( + Array.from(packedJointMatrices.subarray(32, 64)) + ); + } + + const firstTime = first.time; + const secondTime = second.time; + crowd.update(0.1); + expect(first.time - firstTime).toBeCloseTo(0.1); + expect(second.time - secondTime).toBeCloseTo(0.2); + expect( + source.nodes.map(node => ({ + translation: node.translation ? [...node.translation] : undefined, + rotation: node.rotation ? [...node.rotation] : undefined + })) + ).toEqual(sourceNodeTransforms); + + first.pause(); + const pausedTime = first.time; + crowd.update(0.1); + expect(first.time).toBe(pausedTime); + expect(second.time).toBeGreaterThan(secondTime); + + expect(crowd.removeActors(['left', 'missing'])).toBe(1); + expect(first.destroyed).toBe(true); + expect(second.destroyed).toBe(false); + expect(crowd.models[0].instanceCount).toBe(1); + + crowd.destroy(); + crowd.destroy(); + expect(second.destroyed).toBe(true); + expect(getActiveResourceCount(device, 'Buffers')).toBe(initialBuffers); + expect(getActiveResourceCount(device, 'Textures')).toBe(initialTextures); + device.destroy(); + }); + + test('evaluates private morph weights without rewriting shared immutable primitive geometry', async () => { + const source = await loadCrowdFixture('AnimatedMorphCube.glb'); + const device = new NullDevice({}); + const crowd = createGLTFAnimatedCrowd(device, source, {capacity: 3}); + const nodeIndex = source.nodes.findIndex(node => + node.mesh?.primitives.some(primitive => Boolean(primitive.targets?.length)) + ); + const group = crowd.primitiveGroups.find(candidate => candidate.nodeIndex === nodeIndex); + expect(group).toBeDefined(); + if (!group) { + throw new Error('Missing morph primitive'); + } + + const geometryBuffer = group.model._gpuGeometry.attributes['geometry']; + const initialGeometry = Array.from(await geometryBuffer.readAsync()); + const [first, second] = crowd.addActors([ + {id: 'first', phase: 0.1}, + {id: 'second', phase: 0.6} + ]); + + expect(first.getNode(nodeIndex)?.userData['morphWeights']).not.toEqual( + second.getNode(nodeIndex)?.userData['morphWeights'] + ); + expect(first.getNode(nodeIndex)?.userData['morphMeshes']).toBeUndefined(); + expect(second.getNode(nodeIndex)?.userData['morphMeshes']).toBeUndefined(); + crowd.update(0.25); + expect(Array.from(await geometryBuffer.readAsync())).toEqual(initialGeometry); + expect(group.model.instanceCount).toBe(2); + + crowd.destroy(); + device.destroy(); + }); + + test('plays different named actions through the same shared instanced draw', async () => { + const source = await loadCrowdFixture('SimpleSkin.gltf'); + const sourceAnimation = source.animations?.[0]; + expect(sourceAnimation).toBeDefined(); + if (!sourceAnimation) { + throw new Error('Missing source animation'); + } + source.animations = [ + {...sourceAnimation, name: 'Walking'}, + {...sourceAnimation, name: 'Wave'} + ]; + + const device = new NullDevice({}); + const crowd = createGLTFAnimatedCrowd(device, source, {capacity: 2}); + const [walker, waver] = crowd.addActors([ + {id: 'walker', clip: 'Walking', phase: 0}, + {id: 'waver', clip: 'Wave', phase: 0.25} + ]); + const draw = vi.spyOn(crowd.models[0], 'draw').mockReturnValue(true); + + expect(walker.activeClip).toBe('Walking'); + expect(waver.activeClip).toBe('Wave'); + expect(crowd.models).toHaveLength(1); + expect(crowd.models[0].instanceCount).toBe(2); + expect(crowd.draw({} as RenderPass)).toBe(1); + expect(draw).toHaveBeenCalledOnce(); + + draw.mockRestore(); + crowd.destroy(); + device.destroy(); + }); + + test('keeps fixed capacity and rejects duplicate actor identifiers', async () => { + const source = await loadCrowdFixture('SimpleSkin.gltf'); + const device = new NullDevice({}); + const crowd = createGLTFAnimatedCrowd(device, source, {capacity: 2}); + + crowd.addActor({id: 'first'}); + expect(() => crowd.addActor({id: 'first'})).toThrow(); + expect(() => crowd.addActors([{id: 'second'}, {id: 'third'}])).toThrow(); + expect(crowd.actorCount).toBe(1); + crowd.addActor({id: 'second'}); + expect(crowd.actorCount).toBe(2); + expect(() => crowd.addActor({id: 'third'})).toThrow(); + + crowd.destroy(); + expect(() => crowd.addActor()).toThrow(); + device.destroy(); + }); +}); diff --git a/modules/gltf/test/gltf/gltf-animated-crowd.spec.ts b/modules/gltf/test/gltf/gltf-animated-crowd.spec.ts new file mode 100644 index 0000000000..767a148e3d --- /dev/null +++ b/modules/gltf/test/gltf/gltf-animated-crowd.spec.ts @@ -0,0 +1,136 @@ +// luma.gl +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: Copyright (c) vis.gl contributors + +import {load} from '@loaders.gl/core'; +import {GLTFLoader, postProcessGLTF} from '@loaders.gl/gltf'; +import {Buffer, Texture} from '@luma.gl/core'; +import {createGLTFAnimatedCrowd} from '@luma.gl/gltf'; +import {getTestDevices, getWebGLTestDevice} from '@luma.gl/test-utils'; +import {Matrix4} from '@math.gl/core'; +import test from 'test/utils/vitest-tape'; + +test('glTF crowds draw independent SimpleSkin actors in one real WebGL and WebGPU call', async testContext => { + const source = postProcessGLTF( + await load('/examples/showcase/anari/public/gltf/SimpleSkin.gltf', GLTFLoader, { + gltf: {loadImages: false} + }) + ); + const [webglDevice, webgpuDevices] = await Promise.all([ + getWebGLTestDevice(), + getTestDevices(['webgpu']) + ]); + const devices = webglDevice ? [webglDevice, ...webgpuDevices] : webgpuDevices; + + for (const device of devices) { + const crowd = createGLTFAnimatedCrowd(device, source, {capacity: 8}); + const colorTexture = device.createTexture({ + width: 32, + height: 32, + format: device.preferredColorFormat, + usage: Texture.RENDER | Texture.COPY_SRC + }); + const depthTexture = device.createTexture({ + width: 32, + height: 32, + format: 'depth24plus', + usage: Texture.RENDER + }); + const framebuffer = device.createFramebuffer({ + width: 32, + height: 32, + colorAttachments: [colorTexture], + depthStencilAttachment: depthTexture + }); + + try { + const [first, second, third] = crowd.addActors([ + {id: 'left', phase: 0, transform: new Matrix4().translate([-0.5, 0, 0])}, + {id: 'center', phase: 0.35, transform: new Matrix4()}, + {id: 'right', phase: 0.7, speed: 2, transform: new Matrix4().translate([0.5, 0, 0])} + ]); + const model = crowd.models[0]; + const identity = Array.from(new Matrix4()); + model.shaderInputs.setProps({ + pbrProjection: { + modelViewProjectionMatrix: identity, + modelMatrix: identity, + normalMatrix: identity, + camera: [0, 0, 4] + } + }); + + const renderPass = device.beginRenderPass({ + framebuffer, + clearColor: [0, 0, 0, 0], + clearDepth: 1 + }); + const drawCount = crowd.draw(renderPass); + renderPass.end(); + device.submit(); + + if ( + device.info.gpu !== 'software' && + device.info.gpuType !== 'cpu' && + !device.info.fallback + ) { + const memoryLayout = colorTexture.computeMemoryLayout({width: 32, height: 32}); + const readbackBuffer = device.createBuffer({ + byteLength: memoryLayout.byteLength, + usage: Buffer.COPY_DST | Buffer.MAP_READ + }); + try { + colorTexture.readBuffer({width: 32, height: 32}, readbackBuffer); + const pixels = await readbackBuffer.readAsync(0, memoryLayout.byteLength); + testContext.ok( + pixels.some((value, index) => index % 4 === 3 && value > 0), + `${device.type} writes visible crowd geometry into the real framebuffer` + ); + } finally { + readbackBuffer.destroy(); + } + } + + testContext.equal(drawCount, 1, `${device.type} submits one shared primitive draw`); + testContext.equal(model.instanceCount, 3, `${device.type} draws all three actor instances`); + testContext.ok(model.isInstanced, `${device.type} enables instanced primitive rendering`); + testContext.notDeepEqual( + Array.from(first.skins.bindings[0].jointMatrices), + Array.from(second.skins.bindings[0].jointMatrices), + `${device.type} preserves independent actor joint poses` + ); + const resource = crowd.primitiveGroups[0].skinJointMatrices; + testContext.ok( + device.type === 'webgpu' ? resource instanceof Buffer : resource instanceof Texture, + `${device.type} selects its native crowd palette binding` + ); + + const initialModel = model; + const initialThirdTime = third.time; + crowd.update(0.1); + testContext.equal(crowd.models[0], initialModel, `${device.type} reuses its compiled model`); + testContext.ok(third.time > initialThirdTime, `${device.type} advances actor-local playback`); + + const updatedRenderPass = device.beginRenderPass({ + framebuffer, + clearColor: [0, 0, 0, 0], + clearDepth: 1 + }); + testContext.equal( + crowd.draw(updatedRenderPass), + 1, + `${device.type} keeps draw count constant after animation` + ); + updatedRenderPass.end(); + device.submit(); + } finally { + crowd.destroy(); + framebuffer.destroy(); + colorTexture.destroy(); + depthTexture.destroy(); + } + } + + testContext.ok(devices.length > 0, 'at least one live graphics backend is exercised'); + testContext.end(); +}); diff --git a/modules/gltf/test/index.ts b/modules/gltf/test/index.ts index 1f618e9ac9..a65fbc4c34 100644 --- a/modules/gltf/test/index.ts +++ b/modules/gltf/test/index.ts @@ -5,6 +5,7 @@ import './webgl-to-webgpu/convert-webgl-sampler.spec'; import './gltf/gltf-animator.spec'; +import './gltf/gltf-animated-crowd.spec'; import './gltf/gltf.spec'; import './gltf/gltf-extension-support.spec'; import './gltf/lights.spec'; diff --git a/modules/shadertools/src/modules/engine/skin/skin.ts b/modules/shadertools/src/modules/engine/skin/skin.ts index 35fa43310a..9fd43c6dfd 100644 --- a/modules/shadertools/src/modules/engine/skin/skin.ts +++ b/modules/shadertools/src/modules/engine/skin/skin.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: MIT // SPDX-FileCopyrightText: Copyright (c) vis.gl contributors +import type {Binding} from '@luma.gl/core'; import {Matrix4} from '@math.gl/core'; import {ShaderModule} from '../../../lib/shader-module/shader-module'; @@ -16,6 +17,23 @@ struct skinUniforms { @group(0) @binding(auto) var skin: skinUniforms; +#ifdef HAS_INSTANCED_SKIN +@group(0) @binding(auto) var skinJointMatrices: array>; + +fn getInstancedSkinMatrix( + weights: vec4f, + joints: vec4u, + instanceIndex: u32, + jointsPerInstance: u32 +) -> mat4x4 { + let firstJoint = instanceIndex * jointsPerInstance; + return (weights.x * skinJointMatrices[firstJoint + joints.x]) + + (weights.y * skinJointMatrices[firstJoint + joints.y]) + + (weights.z * skinJointMatrices[firstJoint + joints.z]) + + (weights.w * skinJointMatrices[firstJoint + joints.w]); +} +#endif + fn getSkinMatrix(weights: vec4f, joints: vec4u) -> mat4x4 { return (weights.x * skin.jointMatrix[joints.x]) + (weights.y * skin.jointMatrix[joints.y]) @@ -30,6 +48,33 @@ layout(std140) uniform skinUniforms { mat4 jointMatrix[SKIN_MAX_JOINTS]; } skin; +#ifdef HAS_INSTANCED_SKIN +uniform highp sampler2D skinJointMatrices; + +mat4 getInstancedJointMatrix(uint jointIndex, uint instanceIndex) { + int firstColumn = int(jointIndex * 4u); + int row = int(instanceIndex); + return mat4( + texelFetch(skinJointMatrices, ivec2(firstColumn, row), 0), + texelFetch(skinJointMatrices, ivec2(firstColumn + 1, row), 0), + texelFetch(skinJointMatrices, ivec2(firstColumn + 2, row), 0), + texelFetch(skinJointMatrices, ivec2(firstColumn + 3, row), 0) + ); +} + +mat4 getInstancedSkinMatrix( + vec4 weights, + uvec4 joints, + uint instanceIndex, + uint jointsPerInstance +) { + return (weights.x * getInstancedJointMatrix(joints.x, instanceIndex)) + + (weights.y * getInstancedJointMatrix(joints.y, instanceIndex)) + + (weights.z * getInstancedJointMatrix(joints.z, instanceIndex)) + + (weights.w * getInstancedJointMatrix(joints.w, instanceIndex)); +} +#endif + mat4 getSkinMatrix(vec4 weights, uvec4 joints) { return (weights.x * skin.jointMatrix[joints.x]) + (weights.y * skin.jointMatrix[joints.y]) @@ -49,6 +94,8 @@ export type SkinProps = { skinIndex?: number; /** Adapter-owned joint palette, already expressed in the skinned mesh's local space. */ jointMatrices?: Float32Array | readonly number[]; + /** Instance-packed joint palettes: WebGPU storage buffer or WebGL float texture. */ + skinJointMatrices?: Binding; /** Optional mesh transform used to convert world-space joints into mesh-local space. */ meshWorldMatrix?: readonly number[]; }; @@ -57,12 +104,21 @@ export type SkinUniforms = { jointMatrix?: any; }; +type SkinBindings = { + /** WebGPU read-only storage or WebGL vertex-sampled float texture. */ + skinJointMatrices?: Binding; +}; + export const skin = { props: {} as SkinProps, uniforms: {} as SkinUniforms, + bindings: {} as SkinBindings, name: 'skin', - bindingLayout: [{name: 'skin', group: 0}], + bindingLayout: [ + {name: 'skin', group: 0}, + {name: 'skinJointMatrices', group: 0, visibility: 1} + ], dependencies: [], source, vs, @@ -72,15 +128,25 @@ export const skin = { SKIN_MAX_JOINTS }, - getUniforms: (props: SkinProps = {}, _previousUniforms?: SkinUniforms): SkinUniforms => { - const {jointMatrices, scenegraphsFromGLTF, skinIndex = 0, meshWorldMatrix} = props; + getUniforms: ( + props: SkinProps = {}, + _previousUniforms?: SkinUniforms + ): SkinUniforms & SkinBindings => { + const { + jointMatrices, + skinJointMatrices, + scenegraphsFromGLTF, + skinIndex = 0, + meshWorldMatrix + } = props; + const bindings = skinJointMatrices ? {skinJointMatrices} : {}; if (jointMatrices) { - return {jointMatrix: makeJointPalette(jointMatrices)}; + return {jointMatrix: makeJointPalette(jointMatrices), ...bindings}; } const sourceSkin = scenegraphsFromGLTF?.gltf?.skins?.[skinIndex]; if (!sourceSkin) { - return {jointMatrix: []}; + return {jointMatrix: [], ...bindings}; } const {inverseBindMatrices, joints, skeleton} = sourceSkin; @@ -116,13 +182,13 @@ export const skin = { jointPalette.set(jointMatrix, jointIndex * 16); } - return {jointMatrix: jointPalette}; + return {jointMatrix: jointPalette, ...bindings}; }, uniformTypes: { jointMatrix: ['mat4x4', SKIN_MAX_JOINTS] } -} as const satisfies ShaderModule; +} as const satisfies ShaderModule; function makeJointPalette(jointMatrices: Float32Array | readonly number[]): Float32Array { const jointPalette = new Float32Array(SKIN_MAX_JOINTS * 16); diff --git a/modules/shadertools/test/modules/engine/skin.spec.ts b/modules/shadertools/test/modules/engine/skin.spec.ts index 04b2ea0558..f999188072 100644 --- a/modules/shadertools/test/modules/engine/skin.spec.ts +++ b/modules/shadertools/test/modules/engine/skin.spec.ts @@ -2,11 +2,35 @@ // SPDX-License-Identifier: MIT // SPDX-FileCopyrightText: Copyright (c) vis.gl contributors -import {GroupNode} from '@luma.gl/engine'; -import {SKIN_MAX_JOINTS, skin} from '@luma.gl/shadertools'; +import {Buffer} from '@luma.gl/core'; +import {GroupNode, ShaderInputs} from '@luma.gl/engine'; +import { + assembleGLSLShaderPair, + SKIN_MAX_JOINTS, + skin, + WGSLShaderAssembler, + type PlatformInfo +} from '@luma.gl/shadertools'; +import {getWebGLTestDevice, getWebGPUTestDevice, NullDevice} from '@luma.gl/test-utils'; import {Matrix4} from '@math.gl/core'; import test from 'test/utils/vitest-tape'; +const GLSL_PLATFORM_INFO: PlatformInfo = { + type: 'webgl', + gpu: 'test-gpu', + shaderLanguage: 'glsl', + shaderLanguageVersion: 300, + features: new Set() +}; + +const WGSL_PLATFORM_INFO: PlatformInfo = { + type: 'webgpu', + gpu: 'test-gpu', + shaderLanguage: 'wgsl', + shaderLanguageVersion: 300, + features: new Set() +}; + test('shadertools#skin returns empty uniforms without a glTF skin', t => { t.deepEqual( skin.getUniforms({ @@ -121,3 +145,181 @@ test('shadertools#skin accepts a format-independent precomputed joint palette', t.equal(uniforms.jointMatrix?.length, SKIN_MAX_JOINTS * 16, 'pads the uniform palette'); t.end(); }); + +test('shadertools#skin keeps instance palettes feature-specialized in WGSL', async t => { + const shaderAssembler = new WGSLShaderAssembler(); + const source = /* wgsl */ ` +@vertex +fn vertexMain(@builtin(instance_index) instanceIndex: u32) -> @builtin(position) vec4f { +#ifdef HAS_INSTANCED_SKIN + return getInstancedSkinMatrix(vec4f(1.0, 0.0, 0.0, 0.0), vec4u(0u), instanceIndex, 2u) + * vec4f(0.0, 0.0, 0.0, 1.0); +#else + return getSkinMatrix(vec4f(1.0, 0.0, 0.0, 0.0), vec4u(0u)) + * vec4f(0.0, 0.0, 0.0, 1.0); +#endif +}`; + const uninstancedShader = shaderAssembler.assembleWGSLShader({ + platformInfo: WGSL_PLATFORM_INFO, + source, + modules: [skin] + }); + const instancedShader = shaderAssembler.assembleWGSLShader({ + platformInfo: WGSL_PLATFORM_INFO, + source, + modules: [skin], + defines: {HAS_INSTANCED_SKIN: true} + }); + + t.notOk( + uninstancedShader.bindingTable.some(binding => binding.name === 'skinJointMatrices'), + 'ordinary skinning requires no crowd storage binding' + ); + t.notOk( + uninstancedShader.source.includes('getInstancedSkinMatrix'), + 'ordinary shaders exclude crowd skinning helpers' + ); + t.ok( + instancedShader.bindingTable.some( + binding => binding.name === 'skinJointMatrices' && binding.kind === 'read-only-storage' + ), + 'instanced WebGPU skinning uses read-only packed matrix storage' + ); + t.ok( + instancedShader.source.includes('instanceIndex * jointsPerInstance'), + 'each drawn instance indexes its own contiguous joint palette' + ); + t.ok( + instancedShader.bindingTable.some(binding => binding.name === 'skin'), + 'the existing compatible skin uniform remains available' + ); + t.equal( + skin.bindingLayout.find(binding => binding.name === 'skinJointMatrices')?.visibility, + 1, + 'packed palettes bind to the vertex stage only' + ); + + if (typeof document !== 'undefined') { + const device = await getWebGPUTestDevice(); + if (device) { + const shader = device.createShader({ + id: 'instanced-skin-storage-vertex', + source: instancedShader.source + }); + try { + const errors = (await shader.getCompilationInfo()).filter( + message => message.type === 'error' + ); + t.equal( + errors.length, + 0, + `the actual WebGPU backend compiles indexed storage skinning${ + errors.length ? `: ${errors.map(error => error.message).join('; ')}` : '' + }` + ); + } finally { + shader.destroy(); + } + } + } + + t.end(); +}); + +test('shadertools#skin assembles portable float-texture instance palettes for WebGL', async t => { + const assembledShader = assembleGLSLShaderPair({ + platformInfo: GLSL_PLATFORM_INFO, + vs: /* glsl */ `#version 300 es +in vec4 positions; +void main(void) { + gl_Position = getInstancedSkinMatrix( + vec4(1.0, 0.0, 0.0, 0.0), uvec4(0u), uint(gl_InstanceID), 2u + ) * positions; +}`, + fs: /* glsl */ `#version 300 es +precision highp float; +out vec4 fragmentColor; +void main(void) { + fragmentColor = vec4(1.0); +}`, + modules: [skin], + defines: {HAS_INSTANCED_SKIN: true} + }); + + t.ok( + assembledShader.vs.includes('uniform highp sampler2D skinJointMatrices'), + 'WebGL binds instance palettes as a vertex-sampled float texture' + ); + t.ok( + assembledShader.vs.includes('texelFetch(skinJointMatrices'), + 'joint matrices use exact unfiltered float texels' + ); + t.ok(assembledShader.vs.includes('uint(gl_InstanceID)'), 'WebGL indexes the drawn instance'); + t.notOk(assembledShader.vs.includes('var message.type === 'error' + ); + t.equal( + errors.length, + 0, + `the actual WebGL backend compiles indexed float-texture skinning${ + errors.length ? `: ${errors.map(error => error.message).join('; ')}` : '' + }` + ); + } finally { + shader.destroy(); + } + } + + t.end(); +}); + +test('shadertools#skin preserves uniforms while forwarding backend-native palette resources', t => { + const device = new NullDevice({}); + const jointMatrices = new Float32Array(new Matrix4().translate([9, 0, 0])); + const paletteBuffer = device.createBuffer({ + data: jointMatrices, + usage: Buffer.STORAGE | Buffer.COPY_DST + }); + const paletteTexture = device.createTexture({ + width: 4, + height: 1, + format: 'rgba32float' + }); + + try { + const storageUniforms = skin.getUniforms({jointMatrices, skinJointMatrices: paletteBuffer}); + const textureUniforms = skin.getUniforms({jointMatrices, skinJointMatrices: paletteTexture}); + const shaderInputs = new ShaderInputs({skin}); + shaderInputs.setProps({skin: {jointMatrices, skinJointMatrices: paletteBuffer}}); + + t.equal(storageUniforms.jointMatrix?.[12], 9, 'retains the existing uniform palette'); + t.equal(storageUniforms.skinJointMatrices, paletteBuffer, 'forwards WebGPU matrix storage'); + t.equal(textureUniforms.skinJointMatrices, paletteTexture, 'forwards WebGL float textures'); + t.equal( + shaderInputs.getBindingValues().skinJointMatrices, + paletteBuffer, + 'ShaderInputs recognizes the optional palette as a binding' + ); + t.equal( + shaderInputs.getUniformValues().skin?.jointMatrix?.[12], + 9, + 'ShaderInputs retains the compatible padded uniform values' + ); + } finally { + paletteBuffer.destroy(); + paletteTexture.destroy(); + device.destroy(); + } + + t.end(); +}); diff --git a/test/examples/example-panels.spec.ts b/test/examples/example-panels.spec.ts index 5331a078d3..fd876d95e1 100644 --- a/test/examples/example-panels.spec.ts +++ b/test/examples/example-panels.spec.ts @@ -21,6 +21,7 @@ import { setTextSpaceCrawlColorKind } from '../../examples/text-space-crawl-color'; import {makeGltfSettingsSchema} from '../../examples/showcase/gltf/app'; +import {isAnimatedGLTFCatalogModel} from '../../examples/showcase/gltf/gltf-catalog-app'; import { flattenEffectSettings, getEffectResolutionScale, @@ -758,6 +759,39 @@ describe('postprocessing effect settings', () => { }); describe('glTF controls', () => { + test('only offers source models with authored animation', () => { + expect( + isAnimatedGLTFCatalogModel({ + name: 'RobotExpressive', + screenshot: 'screenshot/screenshot.png' + }) + ).toBe(true); + expect( + isAnimatedGLTFCatalogModel({ + name: 'CesiumMan', + screenshot: 'screenshot/screenshot.gif' + }) + ).toBe(true); + expect( + isAnimatedGLTFCatalogModel({ + name: 'Fox', + screenshot: 'screenshot/screenshot.jpg' + }) + ).toBe(true); + expect( + isAnimatedGLTFCatalogModel({ + name: 'DamagedHelmet', + screenshot: 'screenshot/screenshot.png' + }) + ).toBe(false); + expect( + isAnimatedGLTFCatalogModel({ + name: 'MorphPrimitivesTest', + screenshot: 'screenshot/screenshot.png' + }) + ).toBe(false); + }); + test('keeps the model selector in the settings schema', () => { expect(getSettingDefinitions(makeGltfSettingsSchema()).get('modelValue')).toEqual( expect.objectContaining({ @@ -766,6 +800,18 @@ describe('glTF controls', () => { }) ); }); + + test('exposes a genuinely instanced animated crowd of up to one hundred actors', () => { + expect(getSettingDefinitions(makeGltfSettingsSchema()).get('instanceCount')).toEqual( + expect.objectContaining({ + label: 'GPU Crowd Actors', + type: 'number', + min: 1, + max: 100, + step: 1 + }) + ); + }); }); function makeMemoryStorage(initialValues: Record = {}): Storage { diff --git a/website/src/components/docs/gltf-docs-tabs.tsx b/website/src/components/docs/gltf-docs-tabs.tsx index 9f31ed6dad..356d2a0755 100644 --- a/website/src/components/docs/gltf-docs-tabs.tsx +++ b/website/src/components/docs/gltf-docs-tabs.tsx @@ -6,7 +6,7 @@ type GltfDocsTab = {id: NativeGltfDocsTabId; label: string; href: string}; /** glTF documentation tab identifiers. */ export type GltfDocsTabId = 'overview' | 'materials' | 'animation' | 'interchange' | 'extensions'; -type NativeGltfDocsTabId = GltfDocsTabId | 'native-extensions'; +type NativeGltfDocsTabId = GltfDocsTabId | 'native-extensions' | 'animated-crowd'; const GLTF_DOCS_TABS: GltfDocsTab[] = [ {id: 'overview', label: 'Overview', href: '/docs/api-reference/gltf'}, @@ -17,6 +17,11 @@ const GLTF_DOCS_TABS: GltfDocsTab[] = [ href: '/docs/api-reference/gltf/gltf-native-extensions' }, {id: 'animation', label: 'Animation', href: '/docs/api-reference/gltf/gltf-animation'}, + { + id: 'animated-crowd', + label: 'Animated Crowd', + href: '/docs/api-reference/gltf/gltf-animated-crowd' + }, {id: 'interchange', label: 'Interchange', href: '/docs/api-reference/gltf/gltf-interchange'}, {id: 'extensions', label: 'Extensions', href: '/docs/api-reference/gltf/gltf-extensions'} ];