diff --git a/docs/Features/3D-Layers.md b/docs/Features/3D-Layers.md index bd0fe4a70..6456d4de2 100644 --- a/docs/Features/3D-Layers.md +++ b/docs/Features/3D-Layers.md @@ -86,7 +86,9 @@ Camera clips expose their own Properties tab with: - Near plane - Far plane -The Transform tab becomes scene-navigation controls for the active camera clip. In FPS mode, the preview accepts WASD/QE navigation plus mouse look. Free scene navigation now belongs to camera clips rather than gaussian-splat clips. +The Transform tab becomes scene-navigation controls for the active camera clip. In FPS mode, the preview accepts WASD/QE navigation plus uncapped mouse look. Free scene navigation now belongs to camera clips rather than gaussian-splat clips. + +Camera rotation keyframes interpolate through the shortest angular path so timeline flights do not spin the long way around when yaw, pitch, or roll crosses a 360-degree wrap. FPS-look camera segments with keyed position/forward travel render through world-pose interpolation, keeping vertical-look roll moves from drifting away between keyframes. ## Gaussian Splats @@ -98,6 +100,7 @@ Gaussian splat clips are imported through the SuperSplat-compatible `@playcanvas - Realtime splat rendering uses a worker-backed back-to-front order buffer based on the SuperSplat/PlayCanvas sorter approach. Precise export can still fall back to the existing GPU sort path. - Sequence splats follow the same shared runtime contract and are no longer treated as a permanent legacy-only scene path. - The Transform tab now exposes normal object transforms for gaussian splats. Scene navigation lives on camera clips. +- Large gaussian splats show viewport loading progress during project restore, URL fetch, parser work, normalization, and GPU upload. Some gaussian-splat settings exist in the data model and export pipeline but are not yet surfaced as a full dedicated UI: diff --git a/src/App.css b/src/App.css index 40243c905..f81a2f425 100644 --- a/src/App.css +++ b/src/App.css @@ -10041,6 +10041,63 @@ input[type="checkbox"] { pointer-events: none; } +.preview-splat-progress-overlay { + position: absolute; + left: 50%; + bottom: 42px; + width: min(460px, calc(100% - 32px)); + transform: translateX(-50%); + padding: 10px 12px; + color: var(--text-primary); + background: rgba(10, 12, 16, 0.86); + border: 1px solid rgba(255, 255, 255, 0.14); + border-radius: 6px; + box-shadow: 0 10px 32px rgba(0, 0, 0, 0.32); + backdrop-filter: blur(8px); + z-index: 24; + pointer-events: none; +} + +.preview-splat-progress-header { + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; + font-size: 12px; + font-weight: 600; + line-height: 1.3; +} + +.preview-splat-progress-name { + margin-top: 3px; + color: var(--text-secondary); + font-size: 11px; + line-height: 1.35; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.preview-splat-progress-track { + height: 5px; + margin-top: 8px; + overflow: hidden; + background: rgba(255, 255, 255, 0.14); + border-radius: 999px; +} + +.preview-splat-progress-fill { + height: 100%; + width: 0; + background: var(--accent); + border-radius: inherit; + transition: width 0.16s ease-out; +} + +.preview-splat-progress-overlay.error .preview-splat-progress-fill { + background: var(--danger); +} + .preview-canvas-wrapper { transform-origin: center center; transition: transform 0.1s ease-out; diff --git a/src/changelog-data.json b/src/changelog-data.json index 51cdb023e..a687de356 100644 --- a/src/changelog-data.json +++ b/src/changelog-data.json @@ -1,4 +1,25 @@ [ + { + "date": "2026-04-24", + "type": "fix", + "title": "3D Video Planes Now Render Through the Active Scene Camera", + "description": "Video clips converted into 3D layers now share the same renderable scene camera path as splats and nested compositions, so the 3D viewport no longer disappears or breaks when video planes are added.", + "section": "3D / Native Scene" + }, + { + "date": "2026-04-24", + "type": "improve", + "title": "Gaussian Splat Loading Shows Viewport Progress", + "description": "Large splats now report fetch, read, parse, and upload progress directly in the preview viewport during import, refresh, and timeline restore.", + "section": "3D / Gaussian Splats" + }, + { + "date": "2026-04-24", + "type": "fix", + "title": "FPS Camera Rotation Tweens as a Real World Pose", + "description": "FPS camera look is no longer pitch-capped, and camera keyframes now interpolate the recorded world pose so vertical look and roll-style moves avoid unwanted fly-out arcs between keyframes.", + "section": "3D / Camera Controls" + }, { "date": "2026-04-24", "type": "new", diff --git a/src/components/preview/Preview.tsx b/src/components/preview/Preview.tsx index c126a90fb..6a1ab4148 100644 --- a/src/components/preview/Preview.tsx +++ b/src/components/preview/Preview.tsx @@ -8,6 +8,7 @@ const log = Logger.create('Preview'); import { useEngine } from '../../hooks/useEngine'; import { useShortcut } from '../../hooks/useShortcut'; import { + selectActiveGaussianSplatLoadProgress, selectSceneNavClipId, selectSceneNavFpsMode, selectSceneNavFpsMoveSpeed, @@ -56,7 +57,34 @@ function getSharedSceneDefaultCameraDistance(fovDegrees: number): number { } const CAMERA_NAV_FPS_LOOK_SPEED = 0.18; -const CAMERA_NAV_FPS_PITCH_LIMIT = 89.5; + +function formatSplatLoadPercent(percent: number): number { + if (!Number.isFinite(percent)) { + return 0; + } + return Math.round(Math.max(0, Math.min(1, percent)) * 100); +} + +function getSplatLoadPhaseLabel(phase: string): string { + switch (phase) { + case 'fetching': + return 'Fetching splat'; + case 'reading': + return 'Reading splat'; + case 'parsing': + return 'Parsing splat'; + case 'normalizing': + return 'Preparing splat'; + case 'uploading': + return 'Uploading splat'; + case 'complete': + return 'Splat loaded'; + case 'error': + return 'Splat load failed'; + default: + return 'Loading splat'; + } +} interface PreviewProps { panelId: string; @@ -75,6 +103,7 @@ export function Preview({ panelId, source, showTransparencyGrid }: PreviewProps) const sceneNavClipId = useEngineStore(selectSceneNavClipId); const sceneNavFpsMode = useEngineStore(selectSceneNavFpsMode); const sceneNavFpsMoveSpeed = useEngineStore(selectSceneNavFpsMoveSpeed); + const activeSplatLoadProgress = useEngineStore(selectActiveGaussianSplatLoadProgress); const setSceneNavFpsMoveSpeed = useEngineStore((s) => s.setSceneNavFpsMoveSpeed); const { clips, selectedClipIds, primarySelectedClipId, selectClip, updateClipTransform, maskEditMode, layers, selectedLayerId, selectLayer, updateLayer, tracks } = useTimelineStore(useShallow(s => ({ clips: s.clips, @@ -738,10 +767,7 @@ export function Preview({ panelId, source, showTransparencyGrid }: PreviewProps) if (deltaX === 0 && deltaY === 0) return; - const nextPitch = Math.max( - -CAMERA_NAV_FPS_PITCH_LIMIT, - Math.min(CAMERA_NAV_FPS_PITCH_LIMIT, freshTransform.rotation.x + deltaY * CAMERA_NAV_FPS_LOOK_SPEED), - ); + const nextPitch = freshTransform.rotation.x + deltaY * CAMERA_NAV_FPS_LOOK_SPEED; const nextYaw = freshTransform.rotation.y - deltaX * CAMERA_NAV_FPS_LOOK_SPEED; const nextTranslation = resolveOrbitCameraTranslationForFixedEye( freshTransform, @@ -1131,6 +1157,12 @@ export function Preview({ panelId, source, showTransparencyGrid }: PreviewProps) const viewTransform = editMode ? { transform: `scale(${viewZoom}) translate(${viewPan.x / viewZoom}px, ${viewPan.y / viewZoom}px)`, } : {}; + const splatLoadPercent = activeSplatLoadProgress + ? formatSplatLoadPercent(activeSplatLoadProgress.percent) + : 0; + const splatLoadPhaseLabel = activeSplatLoadProgress + ? getSplatLoadPhaseLabel(activeSplatLoadProgress.phase) + : ''; return (
+ {activeSplatLoadProgress && ( +
+
+ {splatLoadPhaseLabel} + {splatLoadPercent}% +
+
+ {activeSplatLoadProgress.fileName} +
+
+
+
+
+ )} + {/* Edit mode overlay - covers full container for pasteboard support */} {editMode && isEngineReady && ( 0 - ? getInterpolatedClipTransform(keyframes, nestedLocalTime, baseTransform) + ? getInterpolatedClipTransform(keyframes, nestedLocalTime, baseTransform, { + rotationMode: nestedClip.source?.type === 'camera' ? 'shortest' : 'linear', + }) : baseTransform; // Interpolate effect parameters if there are effect keyframes diff --git a/src/engine/export/ExportLayerBuilder.ts b/src/engine/export/ExportLayerBuilder.ts index b722feabc..876b35e7f 100644 --- a/src/engine/export/ExportLayerBuilder.ts +++ b/src/engine/export/ExportLayerBuilder.ts @@ -636,7 +636,9 @@ function buildNestedBaseLayer(nestedClip: TimelineClip, nestedClipLocalTime: num // Interpolate transform using keyframes (supports opacity fades, position animations, etc.) const transform = keyframes.length > 0 - ? getInterpolatedClipTransform(keyframes, nestedClipLocalTime, baseTransform) + ? getInterpolatedClipTransform(keyframes, nestedClipLocalTime, baseTransform, { + rotationMode: nestedClip.source?.type === 'camera' ? 'shortest' : 'linear', + }) : baseTransform; // Interpolate effect parameters if there are effect keyframes diff --git a/src/engine/gaussian/index.ts b/src/engine/gaussian/index.ts index 7bc520e08..3272f7a2a 100644 --- a/src/engine/gaussian/index.ts +++ b/src/engine/gaussian/index.ts @@ -18,6 +18,9 @@ export type { GaussianSplatBuffer, GaussianSplatFrame, GaussianSplatAsset, + GaussianSplatLoadOptions, + GaussianSplatLoadProgress, + GaussianSplatLoadProgressCallback, SplatCache, } from './loaders'; diff --git a/src/engine/gaussian/loaders/SplatTransformLoader.ts b/src/engine/gaussian/loaders/SplatTransformLoader.ts index 2ea7fcbbc..4279b2e11 100644 --- a/src/engine/gaussian/loaders/SplatTransformLoader.ts +++ b/src/engine/gaussian/loaders/SplatTransformLoader.ts @@ -18,11 +18,14 @@ import type { GaussianSplatAsset, GaussianSplatBuffer, GaussianSplatFormat, + GaussianSplatLoadOptions, + GaussianSplatLoadProgressCallback, } from './types.ts'; import { FLOATS_PER_SPLAT, SH_C0 } from './types.ts'; import { buildMetadata, computeBoundingBox, normalizeQuaternion, sigmoid } from './normalize.ts'; const BLOB_CHUNK_SIZE = 4 * 1024 * 1024; +type BlobReadProgressReporter = (loadedBytes: number, totalBytes: number) => void; const DEFAULT_OPTIONS: SplatTransformOptions = { iterations: 10, @@ -44,16 +47,19 @@ const SUPPORTED_INPUT_FORMATS = new Set([ class BlobReadStream extends ReadStream { private readonly blob: Blob; private readonly end: number; + private readonly onReadProgress?: BlobReadProgressReporter; private offset: number; constructor( blob: Blob, start: number, end: number, + onReadProgress?: BlobReadProgressReporter, ) { super(end - start); this.blob = blob; this.end = end; + this.onReadProgress = onReadProgress; this.offset = start; } @@ -87,6 +93,7 @@ class BlobReadStream extends ReadStream { target.set(bytes); this.offset += bytesToRead; this.bytesRead += bytesToRead; + this.onReadProgress?.(this.offset, this.blob.size); return bytesToRead; } } @@ -96,10 +103,29 @@ class BlobReadSource implements ReadSource { readonly seekable = true; private closed = false; private readonly blob: Blob; + private readonly onProgress?: GaussianSplatLoadProgressCallback; + private maxReadOffset = 0; - constructor(blob: Blob) { + constructor(blob: Blob, onProgress?: GaussianSplatLoadProgressCallback) { this.blob = blob; this.size = blob.size; + this.onProgress = onProgress; + } + + private reportReadProgress(loadedBytes: number, totalBytes: number): void { + const nextLoadedBytes = Math.max(this.maxReadOffset, loadedBytes); + if (nextLoadedBytes === this.maxReadOffset && this.maxReadOffset > 0) { + return; + } + + this.maxReadOffset = nextLoadedBytes; + this.onProgress?.({ + phase: 'reading', + loadedBytes: nextLoadedBytes, + totalBytes, + percent: totalBytes > 0 ? nextLoadedBytes / totalBytes : 0, + message: 'Reading splat file', + }); } read(start = 0, end = this.size): ReadStream { @@ -110,7 +136,12 @@ class BlobReadSource implements ReadSource { const clampedStart = Math.max(0, Math.min(start, this.size)); const clampedEnd = Math.max(clampedStart, Math.min(end, this.size)); return new BufferedReadStream( - new BlobReadStream(this.blob, clampedStart, clampedEnd), + new BlobReadStream( + this.blob, + clampedStart, + clampedEnd, + (loadedBytes, totalBytes) => this.reportReadProgress(loadedBytes, totalBytes), + ), BLOB_CHUNK_SIZE, ); } @@ -122,6 +153,11 @@ class BlobReadSource implements ReadSource { class BlobReadFileSystem implements ReadFileSystem { private readonly files = new Map(); + private readonly onProgress?: GaussianSplatLoadProgressCallback; + + constructor(onProgress?: GaussianSplatLoadProgressCallback) { + this.onProgress = onProgress; + } set(name: string, blob: Blob): void { this.files.set(name.toLowerCase(), blob); @@ -132,7 +168,7 @@ class BlobReadFileSystem implements ReadFileSystem { if (!blob) { throw new Error(`File not found: ${filename}`); } - return new BlobReadSource(blob); + return new BlobReadSource(blob, this.onProgress); } } @@ -313,13 +349,22 @@ export function canLoadWithSplatTransform(file: File, format?: GaussianSplatForm export async function loadWithSplatTransform( file: File, format?: GaussianSplatFormat, + options?: GaussianSplatLoadOptions, ): Promise { const inputFormat = detectSplatTransformInputFormat(file, format); if (!inputFormat) { throw new Error(`Unsupported splat-transform input format for ${file.name}`); } - const fileSystem = new BlobReadFileSystem(); + options?.onProgress?.({ + phase: 'reading', + loadedBytes: 0, + totalBytes: file.size, + percent: 0, + message: 'Reading splat file', + }); + + const fileSystem = new BlobReadFileSystem(options?.onProgress); fileSystem.set(file.name, file); let tables: DataTable[]; @@ -350,6 +395,14 @@ export async function loadWithSplatTransform( }); } + options?.onProgress?.({ + phase: 'parsing', + loadedBytes: file.size, + totalBytes: file.size, + percent: 0.76, + message: 'Converting splat tables', + }); + const table = tables[0]; if (!table) { throw new Error(`No splat data tables found in ${file.name}`); @@ -357,8 +410,23 @@ export async function loadWithSplatTransform( const alreadyMortonOrdered = inputFormat === 'sog' || lowerName.endsWith('.compressed.ply'); if (!alreadyMortonOrdered) { + options?.onProgress?.({ + phase: 'parsing', + loadedBytes: file.size, + totalBytes: file.size, + percent: 0.84, + message: 'Sorting splats', + }); sortTableByMortonOrder(table); } + options?.onProgress?.({ + phase: 'parsing', + loadedBytes: file.size, + totalBytes: file.size, + percent: 0.92, + message: 'Building splat buffers', + }); + return convertTableToAsset(table, file, inputFormat); } diff --git a/src/engine/gaussian/loaders/index.ts b/src/engine/gaussian/loaders/index.ts index a387f03d4..3a16feefa 100644 --- a/src/engine/gaussian/loaders/index.ts +++ b/src/engine/gaussian/loaders/index.ts @@ -9,6 +9,7 @@ import type { GaussianSplatFormat, GaussianSplatAsset, GaussianSplatMetadata, + GaussianSplatLoadOptions, } from './types.ts'; import { detectFormat } from './parseHeader.ts'; import { parseGaussianSplatHeader as parseHeader } from './parseHeader.ts'; @@ -69,6 +70,7 @@ function applyAssetBasisCorrection(asset: GaussianSplatAsset): GaussianSplatAsse export async function loadGaussianSplatAsset( file: File, format?: GaussianSplatFormat, + options?: GaussianSplatLoadOptions, ): Promise { const resolvedFormat = format ?? detectFormat(file); @@ -91,7 +93,7 @@ export async function loadGaussianSplatAsset( if (!canLoadWithSplatTransform(file, resolvedFormat)) { throw new Error(`Format "${resolvedFormat}" is not supported by the splat-transform loader.`); } - asset = await loadWithSplatTransform(file, resolvedFormat); + asset = await loadWithSplatTransform(file, resolvedFormat, options); } catch (err) { log.error('Failed to load gaussian splat asset', { name: file.name, @@ -101,6 +103,13 @@ export async function loadGaussianSplatAsset( throw err; } + options?.onProgress?.({ + phase: 'normalizing', + loadedBytes: file.size, + totalBytes: file.size, + percent: 0.96, + message: 'Normalizing scene basis', + }); asset = applyAssetBasisCorrection(asset); log.info('Gaussian splat asset loaded', { @@ -125,6 +134,7 @@ export async function loadGaussianSplatAssetCached( mediaFileId: string, file: File, format?: GaussianSplatFormat, + options?: GaussianSplatLoadOptions, ): Promise { const cache = getSplatCache(); @@ -132,11 +142,18 @@ export async function loadGaussianSplatAssetCached( const cached = cache.get(mediaFileId); if (cached && cached.frames[0]?.buffer.data.length > 0) { log.debug('Cache hit', { mediaFileId, splatCount: cached.metadata.splatCount }); + options?.onProgress?.({ + phase: 'normalizing', + loadedBytes: file.size, + totalBytes: file.size, + percent: 1, + message: 'Using cached splat data', + }); return cached; } // Cache miss — load from file - const asset = await loadGaussianSplatAsset(file, format); + const asset = await loadGaussianSplatAsset(file, format, options); // Store in cache cache.put(mediaFileId, asset); @@ -162,6 +179,9 @@ export type { GaussianSplatBuffer, GaussianSplatFrame, GaussianSplatAsset, + GaussianSplatLoadOptions, + GaussianSplatLoadProgress, + GaussianSplatLoadProgressCallback, } from './types.ts'; // Re-export cache diff --git a/src/engine/gaussian/loaders/types.ts b/src/engine/gaussian/loaders/types.ts index 13b4f8fde..1e9319afc 100644 --- a/src/engine/gaussian/loaders/types.ts +++ b/src/engine/gaussian/loaders/types.ts @@ -2,6 +2,22 @@ export type GaussianSplatFormat = 'ply' | 'splat' | 'ksplat' | 'gsplat-zip' | 'spz' | 'sog' | 'lcc'; +export type GaussianSplatLoadProgressPhase = 'reading' | 'parsing' | 'normalizing'; + +export interface GaussianSplatLoadProgress { + phase: GaussianSplatLoadProgressPhase; + percent?: number; + loadedBytes?: number; + totalBytes?: number; + message?: string; +} + +export type GaussianSplatLoadProgressCallback = (progress: GaussianSplatLoadProgress) => void; + +export interface GaussianSplatLoadOptions { + onProgress?: GaussianSplatLoadProgressCallback; +} + export interface GaussianSplatMetadata { format: GaussianSplatFormat; splatCount: number; diff --git a/src/engine/render/NestedCompRenderer.ts b/src/engine/render/NestedCompRenderer.ts index ffd4ba056..830ab222f 100644 --- a/src/engine/render/NestedCompRenderer.ts +++ b/src/engine/render/NestedCompRenderer.ts @@ -21,7 +21,7 @@ import { getCopiedHtmlVideoPreviewFrame } from './htmlVideoPreviewFallback'; import { splitLayerEffects } from './layerEffectStack'; import { collectActiveSceneSplatEffectors } from '../scene/SceneEffectorUtils'; import { collectScene3DLayers } from '../scene/SceneLayerCollector'; -import { resolveSharedSceneCamera } from '../scene/SceneCameraUtils'; +import { resolveRenderableSharedSceneCamera } from '../scene/SceneCameraUtils'; import { getNativeSceneRenderer } from '../native3d/NativeSceneRenderer'; const log = Logger.create('NestedCompRenderer'); @@ -621,7 +621,7 @@ export class NestedCompRenderer { const textureView = renderer.renderScene( this.device, layers3D, - resolveSharedSceneCamera({ width, height }, currentTime ?? 0, sceneContext), + resolveRenderableSharedSceneCamera({ width, height }, currentTime ?? 0, sceneContext), activeSplatEffectors, isRealtimePlayback, ); diff --git a/src/engine/render/RenderDispatcher.ts b/src/engine/render/RenderDispatcher.ts index 00b72a83d..f75a10c40 100644 --- a/src/engine/render/RenderDispatcher.ts +++ b/src/engine/render/RenderDispatcher.ts @@ -27,11 +27,12 @@ import { flags } from '../featureFlags'; import type { NativeSceneRenderer } from '../native3d/NativeSceneRenderer'; import { collectActiveSceneSplatEffectors } from '../scene/SceneEffectorUtils'; import { collectScene3DLayers } from '../scene/SceneLayerCollector'; -import { resolveRenderableSharedSceneCamera, resolveSharedSceneCamera } from '../scene/SceneCameraUtils'; +import { resolveRenderableSharedSceneCamera } from '../scene/SceneCameraUtils'; import type { SceneSplatEffectorRuntimeData, SceneSplatLayer } from '../scene/types'; import { useMediaStore } from '../../stores/mediaStore'; +import { useEngineStore, type GaussianSplatLoadPhase } from '../../stores/engineStore'; import { getGaussianSplatGpuRenderer } from '../gaussian/core/GaussianSplatGpuRenderer'; -import { loadGaussianSplatAssetCached } from '../gaussian/loaders'; +import { loadGaussianSplatAssetCached, type GaussianSplatLoadProgress } from '../gaussian/loaders'; import { DEFAULT_GAUSSIAN_SPLAT_SETTINGS } from '../gaussian/types'; import { buildSharedSplatRuntimeRequest, @@ -42,6 +43,21 @@ const log = Logger.create('RenderDispatcher'); const GAUSSIAN_PLAYBACK_SORT_FREQUENCY = 6; const MAX_EXPORT_LAYER_NESTING_DEPTH = 8; +type GaussianSplatSceneLoadRequest = { + sceneKey: string; + clipId?: string; + url?: string; + fileName: string; + file?: File; +}; + +function clampUnitInterval(value: number): number { + if (!Number.isFinite(value)) { + return 0; + } + return Math.max(0, Math.min(1, value)); +} + /** * Mutable deps bag — the engine updates these references as they change * (e.g. after device loss/restore, canvas attach/detach). @@ -140,13 +156,7 @@ export class RenderDispatcher { ); } - async ensureGaussianSplatSceneLoaded(options: { - sceneKey: string; - clipId?: string; - url?: string; - fileName: string; - file?: File; - }): Promise { + async ensureGaussianSplatSceneLoaded(options: GaussianSplatSceneLoadRequest): Promise { if (!options.url && !options.file) return false; const device = this.deps.getDevice(); @@ -158,6 +168,7 @@ export class RenderDispatcher { } if (renderer.hasScene(options.sceneKey)) { + useEngineStore.getState().clearGaussianSplatLoadProgress(options.sceneKey); return true; } @@ -787,7 +798,7 @@ export class RenderDispatcher { includeLayer: (data) => includedLayers.has(data), }); - const camera = resolveSharedSceneCamera( + const camera = resolveRenderableSharedSceneCamera( { width, height }, this.getEffectiveTimelineTime(), ); @@ -1169,15 +1180,119 @@ export class RenderDispatcher { return summary; } + private setGaussianSplatLoadProgress( + request: GaussianSplatSceneLoadRequest, + progress: { + phase: GaussianSplatLoadPhase; + percent?: number; + loadedBytes?: number; + totalBytes?: number; + message?: string; + }, + ): void { + useEngineStore.getState().setGaussianSplatLoadProgress({ + sceneKey: request.sceneKey, + clipId: request.clipId, + fileName: request.file?.name || request.fileName || 'splat.ply', + ...progress, + }); + } + + private clearGaussianSplatLoadProgressSoon(sceneKey: string, delayMs: number): void { + globalThis.setTimeout(() => { + useEngineStore.getState().clearGaussianSplatLoadProgress(sceneKey); + }, delayMs); + } + + private mapGaussianAssetLoadProgress( + progress: GaussianSplatLoadProgress, + startPercent: number, + endPercent: number, + ): number { + const bytePercent = progress.totalBytes && progress.totalBytes > 0 + ? (progress.loadedBytes ?? 0) / progress.totalBytes + : 0; + const rawPercent = clampUnitInterval(progress.percent ?? bytePercent); + return startPercent + (endPercent - startPercent) * rawPercent; + } + + private async fetchGaussianSplatFile(request: GaussianSplatSceneLoadRequest): Promise { + if (!request.url) { + throw new Error('Cannot fetch gaussian splat without a URL.'); + } + + this.setGaussianSplatLoadProgress(request, { + phase: 'fetching', + percent: 0.02, + loadedBytes: 0, + message: 'Fetching splat file', + }); + + const response = await fetch(request.url); + if (!response.ok) { + throw new Error(`Failed to fetch gaussian splat: ${response.status} ${response.statusText}`); + } + + const contentLengthHeader = response.headers.get('content-length'); + const parsedTotalBytes = contentLengthHeader ? Number.parseInt(contentLengthHeader, 10) : Number.NaN; + const totalBytes = Number.isFinite(parsedTotalBytes) && parsedTotalBytes > 0 + ? parsedTotalBytes + : undefined; + + if (!response.body) { + const arrayBuffer = await response.arrayBuffer(); + this.setGaussianSplatLoadProgress(request, { + phase: 'fetching', + percent: 0.35, + loadedBytes: arrayBuffer.byteLength, + totalBytes: totalBytes ?? arrayBuffer.byteLength, + message: 'Fetched splat file', + }); + return new File([arrayBuffer], request.fileName || 'splat.ply'); + } + + const reader = response.body.getReader(); + const chunks: BlobPart[] = []; + let loadedBytes = 0; + + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + if (!value) { + continue; + } + + chunks.push(value.slice()); + loadedBytes += value.byteLength; + const rawPercent = totalBytes + ? loadedBytes / totalBytes + : Math.min(0.95, loadedBytes / (64 * 1024 * 1024)); + + this.setGaussianSplatLoadProgress(request, { + phase: 'fetching', + percent: 0.02 + clampUnitInterval(rawPercent) * 0.33, + loadedBytes, + totalBytes, + message: 'Fetching splat file', + }); + } + + this.setGaussianSplatLoadProgress(request, { + phase: 'fetching', + percent: 0.35, + loadedBytes, + totalBytes: totalBytes ?? loadedBytes, + message: 'Fetched splat file', + }); + + return new File(chunks, request.fileName || 'splat.ply'); + } + /** Async helper: fetch splat file, parse, and upload to GPU renderer */ private async loadAndUploadSplatScene( - request: { - sceneKey: string; - clipId?: string; - url?: string; - fileName: string; - file?: File; - }, + request: GaussianSplatSceneLoadRequest, renderer: ReturnType, ): Promise { if (this.splatLoadingClips.has(request.sceneKey)) return; @@ -1185,15 +1300,35 @@ export class RenderDispatcher { try { let file = request.file; + let loadStartPercent = 0.02; if (!file) { if (!request.url) { return; } - const response = await fetch(request.url); - const arrayBuffer = await response.arrayBuffer(); - file = new File([arrayBuffer], request.fileName || 'splat.ply'); + file = await this.fetchGaussianSplatFile(request); + loadStartPercent = 0.35; + } else { + this.setGaussianSplatLoadProgress(request, { + phase: 'reading', + percent: loadStartPercent, + loadedBytes: 0, + totalBytes: file.size, + message: 'Reading splat file', + }); } - const asset = await loadGaussianSplatAssetCached(request.sceneKey, file); + + const loadFile = file; + const asset = await loadGaussianSplatAssetCached(request.sceneKey, loadFile, undefined, { + onProgress: (progress) => { + this.setGaussianSplatLoadProgress(request, { + phase: progress.phase, + percent: this.mapGaussianAssetLoadProgress(progress, loadStartPercent, 0.9), + loadedBytes: progress.loadedBytes, + totalBytes: progress.totalBytes ?? loadFile.size, + message: progress.message, + }); + }, + }); if (asset?.frames[0]?.buffer) { if (asset.metadata?.boundingBox) { @@ -1202,10 +1337,25 @@ export class RenderDispatcher { this.splatSceneBounds.set(request.clipId, asset.metadata.boundingBox); } } + this.setGaussianSplatLoadProgress(request, { + phase: 'uploading', + percent: 0.94, + loadedBytes: loadFile.size, + totalBytes: loadFile.size, + message: 'Uploading splat scene', + }); renderer.uploadScene(request.sceneKey, { splatCount: asset.frames[0].buffer.splatCount, data: asset.frames[0].buffer.data, }); + this.setGaussianSplatLoadProgress(request, { + phase: 'complete', + percent: 1, + loadedBytes: loadFile.size, + totalBytes: loadFile.size, + message: 'Splat scene loaded', + }); + this.clearGaussianSplatLoadProgressSoon(request.sceneKey, 350); log.info('Gaussian splat scene uploaded', { clipId: request.clipId ?? request.sceneKey, sceneKey: request.sceneKey, @@ -1219,6 +1369,12 @@ export class RenderDispatcher { sceneKey: request.sceneKey, err, }); + this.setGaussianSplatLoadProgress(request, { + phase: 'error', + percent: 1, + message: err instanceof Error ? err.message : 'Failed to load gaussian splat scene', + }); + this.clearGaussianSplatLoadProgressSoon(request.sceneKey, 1500); } finally { this.splatLoadingClips.delete(request.sceneKey); } diff --git a/src/engine/scene/SceneCameraUtils.ts b/src/engine/scene/SceneCameraUtils.ts index 2a1e65746..fe807d879 100644 --- a/src/engine/scene/SceneCameraUtils.ts +++ b/src/engine/scene/SceneCameraUtils.ts @@ -1,8 +1,10 @@ import { useMediaStore, DEFAULT_SCENE_CAMERA_SETTINGS } from '../../stores/mediaStore'; import { selectSceneNavClipId, useEngineStore } from '../../stores/engineStore'; import { useTimelineStore } from '../../stores/timeline'; -import type { TimelineClip } from '../../stores/timeline/types'; -import { resolveOrbitCameraPose } from '../gaussian/core/SplatCameraUtils'; +import type { Keyframe, TimelineClip } from '../../stores/timeline/types'; +import { resolveOrbitCameraFrame, resolveOrbitCameraPose } from '../gaussian/core/SplatCameraUtils'; +import { normalizeEasingType } from '../../utils/easing'; +import { easingFunctions } from '../../utils/keyframeInterpolation'; import type { SceneCamera, SceneCameraConfig, SceneViewport } from './types'; import { resolveSceneClipTransform, type SceneTimelineContext } from './SceneTimelineUtils'; @@ -24,6 +26,218 @@ export function getSharedSceneDefaultCameraDistance(fovDegrees: number): number return worldHeight / (2 * Math.tan(fovRadians * 0.5)); } +type CameraVector3 = { x: number; y: number; z: number }; +type CameraQuaternion = { x: number; y: number; z: number; w: number }; + +const CAMERA_ROTATION_PROPERTIES = new Set(['rotation.x', 'rotation.y', 'rotation.z']); +const CAMERA_POSE_TRIGGER_PROPERTIES = new Set([ + 'position.x', + 'position.y', + 'position.z', + 'scale.z', +]); +const CAMERA_POSE_PROPERTIES = new Set([ + ...CAMERA_ROTATION_PROPERTIES, + ...CAMERA_POSE_TRIGGER_PROPERTIES, + 'scale.x', + 'scale.y', +]); + +function lerpNumber(a: number, b: number, t: number): number { + return a + (b - a) * t; +} + +function lerpVector(a: CameraVector3, b: CameraVector3, t: number): CameraVector3 { + return { + x: lerpNumber(a.x, b.x, t), + y: lerpNumber(a.y, b.y, t), + z: lerpNumber(a.z, b.z, t), + }; +} + +function addVector(a: CameraVector3, b: CameraVector3): CameraVector3 { + return { x: a.x + b.x, y: a.y + b.y, z: a.z + b.z }; +} + +function scaleVector(v: CameraVector3, scale: number): CameraVector3 { + return { x: v.x * scale, y: v.y * scale, z: v.z * scale }; +} + +function distanceBetween(a: CameraVector3, b: CameraVector3): number { + return Math.hypot(a.x - b.x, a.y - b.y, a.z - b.z); +} + +function normalizeQuaternion(q: CameraQuaternion): CameraQuaternion { + const length = Math.hypot(q.x, q.y, q.z, q.w); + if (length <= 1e-8) { + return { x: 0, y: 0, z: 0, w: 1 }; + } + return { + x: q.x / length, + y: q.y / length, + z: q.z / length, + w: q.w / length, + }; +} + +function quaternionFromCameraBasis( + right: CameraVector3, + up: CameraVector3, + forward: CameraVector3, +): CameraQuaternion { + const back = scaleVector(forward, -1); + const m00 = right.x; + const m01 = up.x; + const m02 = back.x; + const m10 = right.y; + const m11 = up.y; + const m12 = back.y; + const m20 = right.z; + const m21 = up.z; + const m22 = back.z; + const trace = m00 + m11 + m22; + + if (trace > 0) { + const scale = Math.sqrt(trace + 1) * 2; + return normalizeQuaternion({ + w: 0.25 * scale, + x: (m21 - m12) / scale, + y: (m02 - m20) / scale, + z: (m10 - m01) / scale, + }); + } + + if (m00 > m11 && m00 > m22) { + const scale = Math.sqrt(1 + m00 - m11 - m22) * 2; + return normalizeQuaternion({ + w: (m21 - m12) / scale, + x: 0.25 * scale, + y: (m01 + m10) / scale, + z: (m02 + m20) / scale, + }); + } + + if (m11 > m22) { + const scale = Math.sqrt(1 + m11 - m00 - m22) * 2; + return normalizeQuaternion({ + w: (m02 - m20) / scale, + x: (m01 + m10) / scale, + y: 0.25 * scale, + z: (m12 + m21) / scale, + }); + } + + const scale = Math.sqrt(1 + m22 - m00 - m11) * 2; + return normalizeQuaternion({ + w: (m10 - m01) / scale, + x: (m02 + m20) / scale, + y: (m12 + m21) / scale, + z: 0.25 * scale, + }); +} + +function slerpQuaternion(a: CameraQuaternion, b: CameraQuaternion, t: number): CameraQuaternion { + let next = b; + let dot = a.x * b.x + a.y * b.y + a.z * b.z + a.w * b.w; + + if (dot < 0) { + dot = -dot; + next = { x: -b.x, y: -b.y, z: -b.z, w: -b.w }; + } + + if (dot > 0.9995) { + return normalizeQuaternion({ + x: lerpNumber(a.x, next.x, t), + y: lerpNumber(a.y, next.y, t), + z: lerpNumber(a.z, next.z, t), + w: lerpNumber(a.w, next.w, t), + }); + } + + const theta0 = Math.acos(Math.max(-1, Math.min(1, dot))); + const theta = theta0 * t; + const sinTheta = Math.sin(theta); + const sinTheta0 = Math.sin(theta0); + const s0 = Math.cos(theta) - dot * sinTheta / sinTheta0; + const s1 = sinTheta / sinTheta0; + + return normalizeQuaternion({ + x: a.x * s0 + next.x * s1, + y: a.y * s0 + next.y * s1, + z: a.z * s0 + next.z * s1, + w: a.w * s0 + next.w * s1, + }); +} + +function rotateVectorByQuaternion(v: CameraVector3, q: CameraQuaternion): CameraVector3 { + const tx = 2 * (q.y * v.z - q.z * v.y); + const ty = 2 * (q.z * v.x - q.x * v.z); + const tz = 2 * (q.x * v.y - q.y * v.x); + + return { + x: v.x + q.w * tx + (q.y * tz - q.z * ty), + y: v.y + q.w * ty + (q.z * tx - q.x * tz), + z: v.z + q.w * tz + (q.x * ty - q.y * tx), + }; +} + +function hasCameraPoseInterpolationKeyframes(keyframes: Keyframe[]): boolean { + const hasRotation = keyframes.some((keyframe) => CAMERA_ROTATION_PROPERTIES.has(keyframe.property)); + const hasPoseTrigger = keyframes.some((keyframe) => CAMERA_POSE_TRIGGER_PROPERTIES.has(keyframe.property)); + return hasRotation && hasPoseTrigger; +} + +function getCameraPoseKeyframeTimes(keyframes: Keyframe[]): number[] { + return [...new Set( + keyframes + .filter((keyframe) => CAMERA_POSE_PROPERTIES.has(keyframe.property)) + .map((keyframe) => keyframe.time), + )].toSorted((a, b) => a - b); +} + +function getCameraPoseSegment( + keyframes: Keyframe[], + clipLocalTime: number, +): { startTime: number; endTime: number } | null { + const times = getCameraPoseKeyframeTimes(keyframes); + if (times.length < 2 || clipLocalTime <= times[0] || clipLocalTime >= times[times.length - 1]) { + return null; + } + + for (let i = 1; i < times.length; i += 1) { + const endTime = times[i]; + if (clipLocalTime <= endTime) { + return { startTime: times[i - 1], endTime }; + } + } + + return null; +} + +function getCameraPoseInterpolationT( + keyframes: Keyframe[], + startTime: number, + endTime: number, + clipLocalTime: number, +): number { + const range = endTime - startTime; + if (range <= 0) { + return 0; + } + + const rawT = Math.max(0, Math.min(1, (clipLocalTime - startTime) / range)); + const segmentKeyframe = keyframes.find((keyframe) => + keyframe.time === startTime && + CAMERA_POSE_PROPERTIES.has(keyframe.property) && + CAMERA_ROTATION_PROPERTIES.has(keyframe.property), + ) ?? keyframes.find((keyframe) => + keyframe.time === startTime && + CAMERA_POSE_PROPERTIES.has(keyframe.property), + ); + const easing = normalizeEasingType(segmentKeyframe?.easing, 'linear'); + return easing === 'bezier' ? rawT : easingFunctions[easing](rawT); +} + function lookAt( eyeX: number, eyeY: number, @@ -91,6 +305,98 @@ function perspective(fovYRadians: number, aspect: number, near: number, far: num return matrix; } +function buildPoseInterpolatedCameraConfigFromClip( + cameraClip: TimelineClip, + clipLocalTime: number, + viewport: SceneViewport, + context: Pick, +): SceneCameraConfig | null { + if (cameraClip.source?.type !== 'camera') { + return null; + } + + const keyframes = context.clipKeyframes?.get(cameraClip.id) ?? []; + if (!hasCameraPoseInterpolationKeyframes(keyframes)) { + return null; + } + + const segment = getCameraPoseSegment(keyframes, clipLocalTime); + if (!segment) { + return null; + } + + const cameraSettings = cameraClip.source.cameraSettings ?? DEFAULT_SCENE_CAMERA_SETTINGS; + const defaultDistance = getSharedSceneDefaultCameraDistance(cameraSettings.fov); + const settings = { + nearPlane: cameraSettings.near, + farPlane: cameraSettings.far, + fov: cameraSettings.fov, + minimumDistance: defaultDistance, + }; + const startTimelineTime = cameraClip.startTime + segment.startTime; + const endTimelineTime = cameraClip.startTime + segment.endTime; + const startTransform = resolveSceneClipTransform( + cameraClip, + segment.startTime, + startTimelineTime, + context, + ); + const endTransform = resolveSceneClipTransform( + cameraClip, + segment.endTime, + endTimelineTime, + context, + ); + const startFrame = resolveOrbitCameraFrame( + { + position: startTransform.position, + scale: startTransform.scale, + rotation: startTransform.rotation, + }, + settings, + viewport, + ); + const endFrame = resolveOrbitCameraFrame( + { + position: endTransform.position, + scale: endTransform.scale, + rotation: endTransform.rotation, + }, + settings, + viewport, + ); + const t = getCameraPoseInterpolationT( + keyframes, + segment.startTime, + segment.endTime, + clipLocalTime, + ); + const startOrientation = quaternionFromCameraBasis(startFrame.right, startFrame.cameraUp, startFrame.forward); + const endOrientation = quaternionFromCameraBasis(endFrame.right, endFrame.cameraUp, endFrame.forward); + const orientation = slerpQuaternion(startOrientation, endOrientation, t); + const eye = lerpVector(startFrame.eye, endFrame.eye, t); + const forward = rotateVectorByQuaternion({ x: 0, y: 0, z: -1 }, orientation); + const up = rotateVectorByQuaternion({ x: 0, y: 1, z: 0 }, orientation); + const focusDistance = Math.max( + 0.001, + lerpNumber( + distanceBetween(startFrame.eye, startFrame.target), + distanceBetween(endFrame.eye, endFrame.target), + t, + ), + ); + + return { + position: eye, + target: addVector(eye, scaleVector(forward, focusDistance)), + up, + fov: cameraSettings.fov, + near: cameraSettings.near, + far: cameraSettings.far, + applyDefaultDistance: false, + }; +} + function buildCameraConfigFromClip( cameraClip: TimelineClip, timelineTime: number, @@ -102,6 +408,16 @@ function buildCameraConfigFromClip( } const clipLocalTime = timelineTime - cameraClip.startTime; + const poseInterpolatedConfig = buildPoseInterpolatedCameraConfigFromClip( + cameraClip, + clipLocalTime, + viewport, + context, + ); + if (poseInterpolatedConfig) { + return poseInterpolatedConfig; + } + const transform = resolveSceneClipTransform(cameraClip, clipLocalTime, timelineTime, context); const cameraSettings = cameraClip.source.cameraSettings ?? DEFAULT_SCENE_CAMERA_SETTINGS; const defaultDistance = getSharedSceneDefaultCameraDistance(cameraSettings.fov); diff --git a/src/engine/scene/SceneTimelineUtils.ts b/src/engine/scene/SceneTimelineUtils.ts index f08332d29..c4c5ed11e 100644 --- a/src/engine/scene/SceneTimelineUtils.ts +++ b/src/engine/scene/SceneTimelineUtils.ts @@ -43,7 +43,9 @@ export function resolveSceneClipTransform( const baseTransform = buildBaseTransform(clip); const ownTransform = keyframes.length === 0 ? baseTransform - : getInterpolatedClipTransform(keyframes, clipLocalTime, baseTransform); + : getInterpolatedClipTransform(keyframes, clipLocalTime, baseTransform, { + rotationMode: clip.source?.type === 'camera' ? 'shortest' : 'linear', + }); if (!clip.parentClipId) { return ownTransform; diff --git a/src/services/layerBuilder/LayerBuilderService.ts b/src/services/layerBuilder/LayerBuilderService.ts index dd24b2e08..785b1e91e 100644 --- a/src/services/layerBuilder/LayerBuilderService.ts +++ b/src/services/layerBuilder/LayerBuilderService.ts @@ -1192,7 +1192,9 @@ export class LayerBuilderService { // Interpolate transform using keyframes (supports opacity fades, position animations, etc.) const transform = keyframes.length > 0 - ? getInterpolatedClipTransform(keyframes, nestedClipLocalTime, baseTransform) + ? getInterpolatedClipTransform(keyframes, nestedClipLocalTime, baseTransform, { + rotationMode: nestedClip.source?.type === 'camera' ? 'shortest' : 'linear', + }) : baseTransform; // Interpolate effect parameters if there are effect keyframes diff --git a/src/stores/engineStore.ts b/src/stores/engineStore.ts index 8b363be06..578ac9428 100644 --- a/src/stores/engineStore.ts +++ b/src/stores/engineStore.ts @@ -5,6 +5,34 @@ import { create } from 'zustand'; import { subscribeWithSelector } from 'zustand/middleware'; import type { EngineStats } from '../types'; +export type GaussianSplatLoadPhase = + | 'fetching' + | 'reading' + | 'parsing' + | 'normalizing' + | 'uploading' + | 'complete' + | 'error'; + +export interface GaussianSplatLoadProgressEntry { + sceneKey: string; + clipId?: string; + fileName: string; + phase: GaussianSplatLoadPhase; + percent: number; + loadedBytes?: number; + totalBytes?: number; + message?: string; + updatedAt: number; +} + +export type GaussianSplatLoadProgressUpdate = Omit< + GaussianSplatLoadProgressEntry, + 'updatedAt' | 'percent' +> & { + percent?: number; +}; + interface EngineState { // Engine status isEngineReady: boolean; @@ -16,6 +44,7 @@ interface EngineState { sceneNavClipId: string | null; sceneNavFpsMode: boolean; sceneNavFpsMoveSpeed: number; + gaussianSplatLoadProgress: Record; // Actions setEngineReady: (ready: boolean) => void; @@ -27,6 +56,8 @@ interface EngineState { setSceneNavClipId: (clipId: string | null) => void; setSceneNavFpsMode: (enabled: boolean) => void; setSceneNavFpsMoveSpeed: (speed: number) => void; + setGaussianSplatLoadProgress: (progress: GaussianSplatLoadProgressUpdate) => void; + clearGaussianSplatLoadProgress: (sceneKey: string) => void; } export const SCENE_NAV_FPS_MOVE_SPEED_STEPS = [ @@ -64,6 +95,13 @@ export function stepSceneNavFpsMoveSpeed(speed: number, direction: -1 | 1): numb return SCENE_NAV_FPS_MOVE_SPEED_STEPS[nextIndex] ?? 1; } +function clampProgressPercent(percent: number | undefined): number { + if (typeof percent !== 'number' || !Number.isFinite(percent)) { + return 0; + } + return Math.max(0, Math.min(1, percent)); +} + export function selectSceneNavClipId( state: Pick, ): string | null { @@ -82,6 +120,17 @@ export function selectSceneNavFpsMoveSpeed( return state.sceneNavFpsMoveSpeed ?? 1; } +export function selectActiveGaussianSplatLoadProgress( + state: Pick, +): GaussianSplatLoadProgressEntry | null { + const entries = Object.values(state.gaussianSplatLoadProgress ?? {}); + if (entries.length === 0) { + return null; + } + + return entries.toSorted((a, b) => b.updatedAt - a.updatedAt)[0] ?? null; +} + // Check if Linux Vulkan warning was already dismissed const LINUX_VULKAN_DISMISSED_KEY = 'linux-vulkan-warning-dismissed'; @@ -96,6 +145,7 @@ export const useEngineStore = create()( sceneNavClipId: null, sceneNavFpsMode: false, sceneNavFpsMoveSpeed: 1, + gaussianSplatLoadProgress: {}, engineStats: { fps: 0, frameTime: 0, @@ -150,5 +200,37 @@ export const useEngineStore = create()( setSceneNavFpsMoveSpeed: (speed: number) => { set({ sceneNavFpsMoveSpeed: snapSceneNavFpsMoveSpeed(speed) }); }, + + setGaussianSplatLoadProgress: (progress: GaussianSplatLoadProgressUpdate) => { + set((state) => { + const previous = state.gaussianSplatLoadProgress[progress.sceneKey]; + const nextPercent = Math.max( + previous?.percent ?? 0, + clampProgressPercent(progress.percent), + ); + return { + gaussianSplatLoadProgress: { + ...state.gaussianSplatLoadProgress, + [progress.sceneKey]: { + ...previous, + ...progress, + percent: nextPercent, + updatedAt: Date.now(), + }, + }, + }; + }); + }, + + clearGaussianSplatLoadProgress: (sceneKey: string) => { + set((state) => { + if (!state.gaussianSplatLoadProgress[sceneKey]) { + return {}; + } + const remaining = { ...state.gaussianSplatLoadProgress }; + delete remaining[sceneKey]; + return { gaussianSplatLoadProgress: remaining }; + }); + }, })) ); diff --git a/src/stores/timeline/keyframeSlice.ts b/src/stores/timeline/keyframeSlice.ts index 157365cf1..d004853be 100644 --- a/src/stores/timeline/keyframeSlice.ts +++ b/src/stores/timeline/keyframeSlice.ts @@ -151,7 +151,9 @@ export const createKeyframeSlice: SliceCreator = (set, get) => const keyframes = clipKeyframes.get(clipId) || []; const ownTransform = keyframes.length === 0 ? baseTransform - : getInterpolatedClipTransform(keyframes, clipLocalTime, baseTransform); + : getInterpolatedClipTransform(keyframes, clipLocalTime, baseTransform, { + rotationMode: clip.source?.type === 'camera' ? 'shortest' : 'linear', + }); // If clip has a parent, compose with parent's transform if (clip.parentClipId) { diff --git a/src/utils/keyframeInterpolation.ts b/src/utils/keyframeInterpolation.ts index 909134c7a..4dc8c7657 100644 --- a/src/utils/keyframeInterpolation.ts +++ b/src/utils/keyframeInterpolation.ts @@ -17,6 +17,27 @@ export const PRESET_BEZIER: Record, { p1: [number, 'ease-in-out': { p1: [0.42, 0], p2: [0.58, 1] }, }; +export interface KeyframeInterpolationOptions { + angleMode?: 'linear' | 'shortest'; +} + +export interface ClipTransformInterpolationOptions { + rotationMode?: 'linear' | 'shortest'; +} + +export function getShortestAngleDeltaDegrees(from: number, to: number): number { + const rawDelta = to - from; + let delta = (((rawDelta % 360) + 540) % 360) - 180; + if (delta === -180 && rawDelta > 0) { + delta = 180; + } + return delta; +} + +export function interpolateAngleDegrees(from: number, to: number, t: number): number { + return from + getShortestAngleDeltaDegrees(from, to) * t; +} + /** * Solve cubic bezier for Y given X (time) using Newton-Raphson iteration. * Uses standard CSS cubic-bezier format where X controls timing and Y controls output. @@ -136,7 +157,8 @@ export function interpolateKeyframes( keyframes: Keyframe[], property: AnimatableProperty, time: number, - defaultValue: number + defaultValue: number, + options?: KeyframeInterpolationOptions ): number { // Filter keyframes for this property and sort by time const propKeyframes = keyframes @@ -175,24 +197,33 @@ export function interpolateKeyframes( // Use bezier interpolation if easing is 'bezier' or if keyframe has custom handles const easing = normalizeEasingType(prevKey.easing, 'linear'); + const nextValue = options?.angleMode === 'shortest' + ? prevKey.value + getShortestAngleDeltaDegrees(prevKey.value, nextKey.value) + : nextKey.value; + const valueDelta = nextValue - prevKey.value; if (easing === 'bezier' || prevKey.handleOut || nextKey.handleIn) { - return interpolateBezier(prevKey, nextKey, t); + return interpolateBezier(prevKey, { ...nextKey, value: nextValue }, t); } // Apply preset easing from the previous keyframe const easedT = easingFunctions[easing](t); // Linear interpolation between values - return prevKey.value + (nextKey.value - prevKey.value) * easedT; + return prevKey.value + valueDelta * easedT; } // Get full interpolated transform at a given time export function getInterpolatedClipTransform( keyframes: Keyframe[], time: number, - baseTransform: ClipTransform + baseTransform: ClipTransform, + options?: ClipTransformInterpolationOptions ): ClipTransform { + const rotationOptions: KeyframeInterpolationOptions | undefined = options?.rotationMode === 'shortest' + ? { angleMode: 'shortest' } + : undefined; + return { opacity: interpolateKeyframes(keyframes, 'opacity', time, baseTransform.opacity), blendMode: baseTransform.blendMode, // Not animatable @@ -209,9 +240,9 @@ export function getInterpolatedClipTransform( : {}), }, rotation: { - x: interpolateKeyframes(keyframes, 'rotation.x', time, baseTransform.rotation.x), - y: interpolateKeyframes(keyframes, 'rotation.y', time, baseTransform.rotation.y), - z: interpolateKeyframes(keyframes, 'rotation.z', time, baseTransform.rotation.z), + x: interpolateKeyframes(keyframes, 'rotation.x', time, baseTransform.rotation.x, rotationOptions), + y: interpolateKeyframes(keyframes, 'rotation.y', time, baseTransform.rotation.y, rotationOptions), + z: interpolateKeyframes(keyframes, 'rotation.z', time, baseTransform.rotation.z, rotationOptions), }, }; } diff --git a/src/version.ts b/src/version.ts index 74062f67c..5991729a6 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1,6 +1,6 @@ // App version // Format: MAJOR.MINOR.PATCH -export const APP_VERSION = '1.5.9'; +export const APP_VERSION = '1.5.10'; export interface ChangelogNotice { type: 'info' | 'warning' | 'success' | 'danger'; diff --git a/tests/stores/timeline/keyframeSlice.test.ts b/tests/stores/timeline/keyframeSlice.test.ts index 924555f78..c6d34510f 100644 --- a/tests/stores/timeline/keyframeSlice.test.ts +++ b/tests/stores/timeline/keyframeSlice.test.ts @@ -598,6 +598,27 @@ describe('keyframeSlice', () => { expect(t.opacity).toBeCloseTo(0.5, 1); }); + it('getInterpolatedTransform: uses shortest-path rotation for camera clips', () => { + const cameraClip = createMockClip({ + id: 'camera-1', + trackId: 'video-1', + startTime: 0, + duration: 10, + source: { + type: 'camera', + naturalDuration: Number.MAX_SAFE_INTEGER, + cameraSettings: { fov: 60, near: 0.1, far: 1000 }, + }, + }); + store = createTestTimelineStore({ clips: [cameraClip] } as any); + + store.getState().addKeyframe('camera-1', 'rotation.y', 350, 0); + store.getState().addKeyframe('camera-1', 'rotation.y', 10, 10); + + const t = store.getState().getInterpolatedTransform('camera-1', 5); + expect(t.rotation.y).toBeCloseTo(360, 5); + }); + // ─── getInterpolatedEffects ──────────────────────────────────────── it('getInterpolatedEffects: returns empty array for unknown clip', () => { diff --git a/tests/unit/gaussianLoaderOrientation.test.ts b/tests/unit/gaussianLoaderOrientation.test.ts index e646f29bc..90cfeb0ef 100644 --- a/tests/unit/gaussianLoaderOrientation.test.ts +++ b/tests/unit/gaussianLoaderOrientation.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from 'vitest'; import { applyCanonicalBasisCorrection } from '../../src/engine/gaussian/loaders/normalize'; -import { loadGaussianSplatAsset } from '../../src/engine/gaussian/loaders'; +import { loadGaussianSplatAsset, type GaussianSplatLoadProgress } from '../../src/engine/gaussian/loaders'; function createSplatFile(): File { const buffer = new ArrayBuffer(32); @@ -109,6 +109,19 @@ describe('gaussian splat loader orientation', () => { }); }); + it('reports progress while loading splat assets', async () => { + const events: GaussianSplatLoadProgress[] = []; + + await loadGaussianSplatAsset(createSplatFile(), 'splat', { + onProgress: (progress) => events.push(progress), + }); + + expect(events.some((event) => event.phase === 'reading')).toBe(true); + expect(events.some((event) => event.phase === 'parsing')).toBe(true); + expect(events.some((event) => event.phase === 'normalizing')).toBe(true); + expect(Math.max(...events.map((event) => event.percent ?? 0))).toBeGreaterThanOrEqual(0.96); + }); + it('does not fall back to legacy point-cloud PLY conversion', async () => { await expect(loadGaussianSplatAsset(createPointCloudPlyFile(), 'ply')).rejects.toThrow( /invalid file header|Missing required splat properties/i, diff --git a/tests/unit/keyframeInterpolation.test.ts b/tests/unit/keyframeInterpolation.test.ts index c620e6944..5cb8e6672 100644 --- a/tests/unit/keyframeInterpolation.test.ts +++ b/tests/unit/keyframeInterpolation.test.ts @@ -3,6 +3,7 @@ import { easingFunctions, PRESET_BEZIER, solveCubicBezierForX, + getShortestAngleDeltaDegrees, interpolateBezier, interpolateKeyframes, getInterpolatedClipTransform, @@ -149,6 +150,17 @@ describe('interpolateKeyframes', () => { expect(interpolateKeyframes(kfs, 'opacity', 1, 0)).toBeCloseTo(0.5, 5); }); + it('can interpolate angles across the shortest path', () => { + const kfs = [ + createMockKeyframe({ property: 'rotation.y', time: 0, value: 350, easing: 'linear' }), + createMockKeyframe({ property: 'rotation.y', time: 2, value: 10 }), + ]; + + expect(getShortestAngleDeltaDegrees(350, 10)).toBe(20); + expect(interpolateKeyframes(kfs, 'rotation.y', 1, 0, { angleMode: 'shortest' })).toBeCloseTo(360, 5); + expect(interpolateKeyframes(kfs, 'rotation.y', 2, 0, { angleMode: 'shortest' })).toBe(10); + }); + it('ease-in interpolation: midpoint < 0.5 of range', () => { const kfs = [ createMockKeyframe({ property: 'opacity', time: 0, value: 0, easing: 'ease-in' }), @@ -268,6 +280,17 @@ describe('getInterpolatedClipTransform', () => { expect(result.rotation.z).toBeCloseTo(180, 5); }); + it('keeps normal rotation interpolation by default but supports shortest rotation mode', () => { + const base = createMockTransform(); + const kfs: Keyframe[] = [ + createMockKeyframe({ property: 'rotation.y', time: 0, value: 350, easing: 'linear' }), + createMockKeyframe({ property: 'rotation.y', time: 2, value: 10 }), + ]; + + expect(getInterpolatedClipTransform(kfs, 1, base).rotation.y).toBeCloseTo(180, 5); + expect(getInterpolatedClipTransform(kfs, 1, base, { rotationMode: 'shortest' }).rotation.y).toBeCloseTo(360, 5); + }); + it('blendMode is always from baseTransform (not animatable)', () => { const base = createMockTransform({ blendMode: 'multiply' }); const result = getInterpolatedClipTransform([], 0, base); diff --git a/tests/unit/nestedCompRenderer.test.ts b/tests/unit/nestedCompRenderer.test.ts index 68351125c..cd710e338 100644 --- a/tests/unit/nestedCompRenderer.test.ts +++ b/tests/unit/nestedCompRenderer.test.ts @@ -2,7 +2,10 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import type { LayerRenderData } from '../../src/engine/core/types'; import { NestedCompRenderer } from '../../src/engine/render/NestedCompRenderer'; -import { resolveSharedSceneCamera } from '../../src/engine/scene/SceneCameraUtils'; +import { + getSharedSceneDefaultCameraDistance, + resolveRenderableSharedSceneCamera, +} from '../../src/engine/scene/SceneCameraUtils'; import { useMediaStore } from '../../src/stores/mediaStore'; import { useTimelineStore } from '../../src/stores/timeline'; @@ -200,7 +203,7 @@ describe('NestedCompRenderer shared-scene integration', () => { sourceWidth: 1920, sourceHeight: 1080, }]; - const expectedCamera = resolveSharedSceneCamera( + const expectedCamera = resolveRenderableSharedSceneCamera( { width: 1280, height: 720 }, 2, { @@ -260,4 +263,72 @@ describe('NestedCompRenderer shared-scene integration', () => { blendMode: 'screen', }); }); + + it('uses a renderable default camera for nested 3D video planes without a scene camera', () => { + const renderer = createRenderer(); + useMediaStore.setState({ + activeCompositionId: null, + compositions: [], + } as any); + useTimelineStore.setState({ + isPlaying: false, + isExporting: false, + clipKeyframes: new Map(), + tracks: [], + clips: [], + } as any); + + const layerData: LayerRenderData[] = [{ + layer: { + id: 'nested-video-plane', + name: 'Nested Video Plane', + sourceClipId: 'nested-video-plane-clip', + visible: true, + opacity: 1, + blendMode: 'normal', + effects: [], + position: { x: 0, y: 0, z: 0 }, + scale: { x: 1, y: 1, z: 1 }, + rotation: { x: 0, y: 0, z: 0 }, + is3D: true, + source: { + type: 'video', + videoElement: { + readyState: 4, + videoWidth: 1920, + videoHeight: 1080, + }, + }, + } as any, + isVideo: true, + externalTexture: null, + textureView: { label: 'nested-plane-view' } as any, + sourceWidth: 1920, + sourceHeight: 1080, + }]; + + (renderer as any).process3DLayersForNested( + layerData, + 1280, + 720, + 0, + 'nested-comp', + [], + [], + ); + + expect(mockNativeSceneRenderer.renderScene).toHaveBeenCalledTimes(1); + const [, layers3D, camera] = mockNativeSceneRenderer.renderScene.mock.calls[0]; + const defaultDistance = getSharedSceneDefaultCameraDistance(50); + expect(layers3D).toHaveLength(1); + expect(layers3D[0]).toMatchObject({ + kind: 'plane', + layerId: 'nested-video-plane', + clipId: 'nested-video-plane-clip', + }); + expect(camera.cameraPosition).toEqual({ x: 0, y: 0, z: defaultDistance }); + expect(camera.cameraTarget).toEqual({ x: 0, y: 0, z: 0 }); + expect(camera.viewMatrix[14]).toBeCloseTo(-defaultDistance); + expect(layerData[0]?.textureView).toEqual({ label: 'nested-shared-scene-view' }); + }); }); diff --git a/tests/unit/renderDispatcher.test.ts b/tests/unit/renderDispatcher.test.ts index 402321971..ef2abd297 100644 --- a/tests/unit/renderDispatcher.test.ts +++ b/tests/unit/renderDispatcher.test.ts @@ -1,5 +1,6 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import { RenderDispatcher } from '../../src/engine/render/RenderDispatcher'; +import { getSharedSceneDefaultCameraDistance } from '../../src/engine/scene/SceneCameraUtils'; import { useEngineStore } from '../../src/stores/engineStore'; import { useMediaStore } from '../../src/stores/mediaStore'; import { useTimelineStore } from '../../src/stores/timeline'; @@ -318,9 +319,11 @@ describe('RenderDispatcher empty playback hold', () => { expect(layers3D[0].worldMatrix[12]).toBeCloseTo(0.25); expect(layers3D[0].worldMatrix[13]).toBeCloseTo(-0.5); expect(layers3D[0].worldMatrix[14]).toBeCloseTo(3); + const defaultDistance = getSharedSceneDefaultCameraDistance(50); expect(camera.cameraPosition.x).toBeCloseTo(0); expect(camera.cameraPosition.y).toBeCloseTo(0); - expect(camera.cameraPosition.z).toBeCloseTo(0); + expect(camera.cameraPosition.z).toBeCloseTo(defaultDistance); + expect(camera.viewMatrix[14]).toBeCloseTo(-defaultDistance); expect(effectors).toHaveLength(1); expect(effectors[0]).toMatchObject({ clipId: 'effector-1', @@ -500,6 +503,60 @@ describe('RenderDispatcher empty playback hold', () => { expect(layerData[0]?.layer.blendMode).toBe('normal'); }); + it('uses a renderable default camera for 3D video planes', () => { + const { dispatcher, deps } = createDispatcher(false); + deps.sceneRenderer = { + isInitialized: true, + renderScene: vi.fn(() => ({ label: 'shared-scene-video-view' })), + }; + + const layerData = [ + { + layer: { + id: 'video-plane', + sourceClipId: 'video-plane-clip', + name: 'Video Plane', + visible: true, + opacity: 1, + blendMode: 'normal', + is3D: true, + position: { x: 0, y: 0, z: 0 }, + scale: { x: 1, y: 1, z: 1 }, + rotation: { x: 0, y: 0, z: 0 }, + source: { + type: 'video', + videoElement: { + readyState: 4, + videoWidth: 1920, + videoHeight: 1080, + }, + }, + }, + isVideo: true, + externalTexture: null, + textureView: { label: 'plane-layer-view' }, + sourceWidth: 1920, + sourceHeight: 1080, + }, + ] as any; + + (dispatcher as any).process3DLayers(layerData, {} as GPUDevice, 1920, 1080); + + expect(deps.sceneRenderer.renderScene).toHaveBeenCalledTimes(1); + const [, layers3D, camera] = deps.sceneRenderer.renderScene.mock.calls[0]; + const defaultDistance = getSharedSceneDefaultCameraDistance(50); + expect(layers3D).toHaveLength(1); + expect(layers3D[0]).toMatchObject({ + kind: 'plane', + layerId: 'video-plane', + clipId: 'video-plane-clip', + }); + expect(camera.cameraPosition).toEqual({ x: 0, y: 0, z: defaultDistance }); + expect(camera.cameraTarget).toEqual({ x: 0, y: 0, z: 0 }); + expect(camera.viewMatrix[14]).toBeCloseTo(-defaultDistance); + expect(layerData[0]?.textureView).toEqual({ label: 'shared-scene-video-view' }); + }); + it('routes 3D text plus native gaussian-splat scenes through the shared scene renderer once native assets are ready', () => { const { dispatcher, deps } = createDispatcher(false); deps.sceneRenderer = { diff --git a/tests/unit/sceneCameraUtils.test.ts b/tests/unit/sceneCameraUtils.test.ts index 3b40b7d87..c6ea9c1f0 100644 --- a/tests/unit/sceneCameraUtils.test.ts +++ b/tests/unit/sceneCameraUtils.test.ts @@ -5,7 +5,10 @@ import { resolveSharedSceneCamera, resolveSharedSceneCameraConfig, } from '../../src/engine/scene/SceneCameraUtils'; -import { resolveOrbitCameraPose } from '../../src/engine/gaussian/core/SplatCameraUtils'; +import { + resolveOrbitCameraPose, + resolveOrbitCameraTranslationForFixedEye, +} from '../../src/engine/gaussian/core/SplatCameraUtils'; import { useEngineStore } from '../../src/stores/engineStore'; import { useMediaStore } from '../../src/stores/mediaStore'; import { useTimelineStore } from '../../src/stores/timeline'; @@ -269,4 +272,92 @@ describe('SceneCameraUtils', () => { applyDefaultDistance: false, }); }); + + it('interpolates FPS look camera keyframes as world poses near vertical pitch', () => { + const viewport = { width: 1920, height: 1080 }; + const settings = { + nearPlane: 0.1, + farPlane: 1000, + fov: 60, + minimumDistance: getSharedSceneDefaultCameraDistance(60), + }; + const startTransform = { + position: { x: 0, y: 0, z: 0 }, + scale: { x: 1, y: 1, z: 0 }, + rotation: { x: 89.5, y: 0, z: 0 }, + }; + const endRotation = { x: 89.5, y: 180, z: 0 }; + const endTranslation = resolveOrbitCameraTranslationForFixedEye( + startTransform, + endRotation, + settings, + viewport, + ); + const endTransform = { + position: { + x: endTranslation.positionX, + y: endTranslation.positionY, + z: 0, + }, + scale: { + x: 1, + y: 1, + z: endTranslation.forwardOffset, + }, + rotation: endRotation, + }; + const startPose = resolveOrbitCameraPose(startTransform, settings, viewport); + const endPose = resolveOrbitCameraPose(endTransform, settings, viewport); + const cameraClip = { + id: 'vertical-fps-camera', + trackId: 'camera-track', + startTime: 0, + duration: 2, + transform: { + ...startTransform, + opacity: 1, + blendMode: 'normal', + }, + source: { + type: 'camera', + cameraSettings: { + fov: 60, + near: 0.1, + far: 1000, + }, + }, + }; + + const config = resolveSharedSceneCameraConfig(viewport, 1, { + sceneNavClipId: 'vertical-fps-camera', + tracks: [{ + id: 'camera-track', + type: 'video', + visible: true, + }], + clips: [cameraClip as any], + clipKeyframes: new Map([[ + 'vertical-fps-camera', + [ + { id: 'px0', clipId: 'vertical-fps-camera', property: 'position.x', time: 0, value: startTransform.position.x, easing: 'linear' }, + { id: 'px1', clipId: 'vertical-fps-camera', property: 'position.x', time: 2, value: endTransform.position.x, easing: 'linear' }, + { id: 'py0', clipId: 'vertical-fps-camera', property: 'position.y', time: 0, value: startTransform.position.y, easing: 'linear' }, + { id: 'py1', clipId: 'vertical-fps-camera', property: 'position.y', time: 2, value: endTransform.position.y, easing: 'linear' }, + { id: 'sz0', clipId: 'vertical-fps-camera', property: 'scale.z', time: 0, value: startTransform.scale.z, easing: 'linear' }, + { id: 'sz1', clipId: 'vertical-fps-camera', property: 'scale.z', time: 2, value: endTransform.scale.z, easing: 'linear' }, + { id: 'rx0', clipId: 'vertical-fps-camera', property: 'rotation.x', time: 0, value: startTransform.rotation.x, easing: 'linear' }, + { id: 'rx1', clipId: 'vertical-fps-camera', property: 'rotation.x', time: 2, value: endTransform.rotation.x, easing: 'linear' }, + { id: 'ry0', clipId: 'vertical-fps-camera', property: 'rotation.y', time: 0, value: startTransform.rotation.y, easing: 'linear' }, + { id: 'ry1', clipId: 'vertical-fps-camera', property: 'rotation.y', time: 2, value: endTransform.rotation.y, easing: 'linear' }, + ], + ]]), + }); + + expect(endPose.eye.x).toBeCloseTo(startPose.eye.x, 5); + expect(endPose.eye.y).toBeCloseTo(startPose.eye.y, 5); + expect(endPose.eye.z).toBeCloseTo(startPose.eye.z, 5); + expect(config.position.x).toBeCloseTo(startPose.eye.x, 5); + expect(config.position.y).toBeCloseTo(startPose.eye.y, 5); + expect(config.position.z).toBeCloseTo(startPose.eye.z, 5); + }); }); diff --git a/tests/unit/sceneNavFpsMoveSpeed.test.ts b/tests/unit/sceneNavFpsMoveSpeed.test.ts index 319c39f81..51fec118a 100644 --- a/tests/unit/sceneNavFpsMoveSpeed.test.ts +++ b/tests/unit/sceneNavFpsMoveSpeed.test.ts @@ -2,6 +2,7 @@ import { beforeEach, describe, expect, it } from 'vitest'; import { SCENE_NAV_FPS_MOVE_SPEED_STEPS, getSceneNavFpsMoveSpeedStepIndex, + selectActiveGaussianSplatLoadProgress, snapSceneNavFpsMoveSpeed, stepSceneNavFpsMoveSpeed, useEngineStore, @@ -42,4 +43,30 @@ describe('scene nav FPS movement speed', () => { useEngineStore.getState().setSceneNavFpsMoveSpeed(1.35); expect(useEngineStore.getState().sceneNavFpsMoveSpeed).toBe(1.5); }); + + it('tracks gaussian splat loading progress monotonically until cleared', () => { + useEngineStore.getState().setGaussianSplatLoadProgress({ + sceneKey: 'large-splat-scene', + fileName: 'large.splat', + phase: 'reading', + percent: 0.45, + }); + useEngineStore.getState().setGaussianSplatLoadProgress({ + sceneKey: 'large-splat-scene', + fileName: 'large.splat', + phase: 'parsing', + percent: 0.2, + }); + + const activeProgress = selectActiveGaussianSplatLoadProgress(useEngineStore.getState()); + expect(activeProgress).toMatchObject({ + sceneKey: 'large-splat-scene', + fileName: 'large.splat', + phase: 'parsing', + percent: 0.45, + }); + + useEngineStore.getState().clearGaussianSplatLoadProgress('large-splat-scene'); + expect(selectActiveGaussianSplatLoadProgress(useEngineStore.getState())).toBeNull(); + }); });