From f67bbc47ea289b617dda6caf71460a9199426ba6 Mon Sep 17 00:00:00 2001 From: Sportinger Date: Sun, 26 Apr 2026 20:05:56 +0200 Subject: [PATCH 1/4] Add scene gizmo overlay and stabilize splat effector --- src/App.css | 198 ++++++ src/components/preview/Preview.tsx | 21 +- src/components/preview/PreviewControls.tsx | 21 + src/components/preview/SceneObjectOverlay.tsx | 564 ++++++++++++++++++ .../preview/sceneObjectOverlayMath.ts | 222 +++++++ .../gaussian/core/GaussianSplatGpuRenderer.ts | 36 +- .../gaussianSplatGpuRendererUniforms.test.ts | 142 +++++ tests/unit/sceneObjectOverlayMath.test.ts | 102 ++++ 8 files changed, 1302 insertions(+), 4 deletions(-) create mode 100644 src/components/preview/SceneObjectOverlay.tsx create mode 100644 src/components/preview/sceneObjectOverlayMath.ts create mode 100644 tests/unit/sceneObjectOverlayMath.test.ts diff --git a/src/App.css b/src/App.css index d0e8bce0b..eecae387b 100644 --- a/src/App.css +++ b/src/App.css @@ -10038,6 +10038,34 @@ input[type="checkbox"] { color: var(--text-inverse); } +.preview-scene-toggle-btn { + display: inline-flex; + align-items: center; + justify-content: center; + width: 26px; + height: 24px; + padding: 0; + background: var(--preview-overlay-bg); + border: 1px solid var(--border-color); + border-radius: 4px; + color: var(--text-secondary); + cursor: pointer; + transition: all 0.15s; + backdrop-filter: blur(4px); +} + +.preview-scene-toggle-btn:hover { + background: var(--preview-overlay-bg-hover); + color: var(--text-primary); + border-color: var(--gray-300); +} + +.preview-scene-toggle-btn.active { + background: var(--accent); + border-color: var(--accent); + color: var(--text-inverse); +} + .preview-zoom-label { font-size: 11px; color: var(--text-secondary); @@ -10292,6 +10320,176 @@ input[type="checkbox"] { background: var(--danger); } +.preview-scene-object-overlay { + position: absolute; + left: 50%; + top: 50%; + transform: translate(-50%, -50%); + z-index: 14; + pointer-events: none; +} + +.preview-scene-object-handle { + position: absolute; + display: flex; + align-items: center; + justify-content: center; + width: 24px; + height: 24px; + padding: 0; + transform: translate(-50%, -50%); + color: #f6f8fb; + background: rgba(12, 15, 20, 0.82); + border: 1px solid rgba(255, 255, 255, 0.38); + border-radius: 50%; + box-shadow: 0 4px 14px rgba(0, 0, 0, 0.42); + cursor: pointer; + font-size: 10px; + font-weight: 700; + line-height: 1; + pointer-events: auto; + user-select: none; +} + +.preview-scene-object-handle:hover, +.preview-scene-object-handle.selected { + border-color: #ffffff; + background: #2563eb; +} + +.preview-scene-object-handle.kind-effector { + background: #7c2d12; +} + +.preview-scene-object-handle.kind-splat { + background: #166534; +} + +.preview-scene-object-handle.kind-model, +.preview-scene-object-handle.kind-plane { + background: #581c87; +} + +.preview-scene-object-handle.selected { + background: #2563eb; + box-shadow: 0 0 0 2px rgba(255, 255, 255, 0.92), 0 8px 20px rgba(0, 0, 0, 0.45); +} + +.preview-scene-gizmo-axis-layer { + position: absolute; + inset: 0; + pointer-events: none; +} + +.preview-scene-gizmo-axis { + position: absolute; + height: 22px; + margin: -11px 0 0 0; + padding: 0; + background: transparent; + border: 0; + cursor: grab; + pointer-events: auto; + transform-origin: 0 50%; +} + +.preview-scene-gizmo-axis:active { + cursor: grabbing; +} + +.preview-scene-gizmo-axis-line { + position: absolute; + left: 0; + right: 0; + top: 50%; + height: 3px; + border-radius: 999px; + background: currentColor; + transform: translateY(-50%); +} + +.preview-scene-gizmo-axis.mode-rotate .preview-scene-gizmo-axis-line { + background: repeating-linear-gradient( + 90deg, + currentColor 0, + currentColor 7px, + transparent 7px, + transparent 12px + ); +} + +.preview-scene-gizmo-axis.mode-scale .preview-scene-gizmo-axis-line { + height: 4px; +} + +.preview-scene-gizmo-end { + position: absolute; + right: -6px; + top: 50%; + width: 12px; + height: 12px; + background: #10141c; + border: 2px solid currentColor; + border-radius: 50%; + transform: translateY(-50%); +} + +.preview-scene-gizmo-label { + position: absolute; + transform: translate(-50%, -50%); + font-size: 11px; + font-weight: 800; + pointer-events: none; + text-shadow: + 0 1px 2px rgba(0, 0, 0, 0.9), + 0 -1px 2px rgba(0, 0, 0, 0.9); +} + +.preview-scene-gizmo-axis.axis-x, +.preview-scene-gizmo-label.axis-x { + color: #ef4444; +} + +.preview-scene-gizmo-axis.axis-y, +.preview-scene-gizmo-label.axis-y { + color: #22c55e; +} + +.preview-scene-gizmo-axis.axis-z, +.preview-scene-gizmo-label.axis-z { + color: #3b82f6; +} + +.preview-scene-gizmo-toolbar { + position: absolute; + display: flex; + gap: 3px; + padding: 3px; + background: rgba(8, 11, 16, 0.82); + border: 1px solid rgba(255, 255, 255, 0.18); + border-radius: 5px; + box-shadow: 0 8px 22px rgba(0, 0, 0, 0.36); + pointer-events: auto; +} + +.preview-scene-gizmo-toolbar button { + min-width: 44px; + height: 22px; + padding: 0 7px; + color: var(--text-secondary); + background: transparent; + border: 0; + border-radius: 3px; + cursor: pointer; + font-size: 10px; +} + +.preview-scene-gizmo-toolbar button:hover, +.preview-scene-gizmo-toolbar button.active { + color: #ffffff; + background: rgba(255, 255, 255, 0.15); +} + .preview-canvas-wrapper { transform-origin: center center; transition: transform 0.1s ease-out; diff --git a/src/components/preview/Preview.tsx b/src/components/preview/Preview.tsx index 0e3587470..84b2cb26b 100644 --- a/src/components/preview/Preview.tsx +++ b/src/components/preview/Preview.tsx @@ -27,6 +27,7 @@ import { SourceMonitor } from './SourceMonitor'; import { StatsOverlay } from './StatsOverlay'; import { PreviewControls } from './PreviewControls'; import { PreviewBottomControls } from './PreviewBottomControls'; +import { SceneObjectOverlay } from './SceneObjectOverlay'; import { useEditModeOverlay } from './useEditModeOverlay'; import { useLayerDrag } from './useLayerDrag'; import { useSAM2Store } from '../../stores/sam2Store'; @@ -105,7 +106,7 @@ export function Preview({ panelId, source, showTransparencyGrid }: PreviewProps) const sceneNavFpsMoveSpeed = useEngineStore(selectSceneNavFpsMoveSpeed); const activeSplatLoadProgress = useEngineStore(selectActiveGaussianSplatLoadProgress); const setSceneNavFpsMoveSpeed = useEngineStore((s) => s.setSceneNavFpsMoveSpeed); - const { clips, selectedClipIds, primarySelectedClipId, selectClip, updateClipTransform, maskEditMode, layers, selectedLayerId, selectLayer, updateLayer, tracks } = useTimelineStore(useShallow(s => ({ + const { clips, selectedClipIds, primarySelectedClipId, selectClip, updateClipTransform, maskEditMode, layers, selectedLayerId, selectLayer, updateLayer, tracks, isPlaying } = useTimelineStore(useShallow(s => ({ clips: s.clips, selectedClipIds: s.selectedClipIds, primarySelectedClipId: s.primarySelectedClipId, @@ -117,6 +118,7 @@ export function Preview({ panelId, source, showTransparencyGrid }: PreviewProps) selectLayer: s.selectLayer, updateLayer: s.updateLayer, tracks: s.tracks, + isPlaying: s.isPlaying, }))); const { compositions, activeCompositionId } = useMediaStore(useShallow(s => ({ compositions: s.compositions, @@ -335,6 +337,7 @@ export function Preview({ panelId, source, showTransparencyGrid }: PreviewProps) // Edit mode state const [editMode, setEditMode] = useState(false); + const [sceneObjectOverlayEnabled, setSceneObjectOverlayEnabled] = useState(true); const [viewZoom, setViewZoom] = useState(1); const [viewPan, setViewPan] = useState({ x: 0, y: 0 }); const [isPanning, setIsPanning] = useState(false); @@ -1174,6 +1177,7 @@ export function Preview({ panelId, source, showTransparencyGrid }: PreviewProps) const splatLoadPhaseLabel = activeSplatLoadProgress ? getSplatLoadPhaseLabel(activeSplatLoadProgress.phase) : ''; + const showSceneObjectOverlay = sceneObjectOverlayEnabled && isEditableSource && !isPlaying; return (
)} + {showSceneObjectOverlay && ( + + )} )}
diff --git a/src/components/preview/PreviewControls.tsx b/src/components/preview/PreviewControls.tsx index be7bc8e5c..592ca30f5 100644 --- a/src/components/preview/PreviewControls.tsx +++ b/src/components/preview/PreviewControls.tsx @@ -19,6 +19,8 @@ interface PreviewControlsProps { editMode: boolean; canEdit: boolean; setEditMode: (v: boolean) => void; + sceneObjectOverlayEnabled: boolean; + setSceneObjectOverlayEnabled: (v: boolean) => void; viewZoom: number; resetView: () => void; // Source selector @@ -45,6 +47,8 @@ export function PreviewControls({ editMode, canEdit, setEditMode, + sceneObjectOverlayEnabled, + setSceneObjectOverlayEnabled, viewZoom, resetView, source, @@ -116,6 +120,23 @@ export function PreviewControls({ > {editMode ? 'Edit On' : 'Edit'} bug + {canEdit && ( + + )} {editMode && canEdit && ( <> {Math.round(viewZoom * 100)}% diff --git a/src/components/preview/SceneObjectOverlay.tsx b/src/components/preview/SceneObjectOverlay.tsx new file mode 100644 index 000000000..3d7de7265 --- /dev/null +++ b/src/components/preview/SceneObjectOverlay.tsx @@ -0,0 +1,564 @@ +import { + useCallback, + useEffect, + useMemo, + useRef, + useState, + type CSSProperties, + type MouseEvent as ReactMouseEvent, + type PointerEvent as ReactPointerEvent, +} from 'react'; +import type { AnimatableProperty, ClipTransform, TimelineClip, TimelineTrack } from '../../types'; +import { engine } from '../../engine/WebGPUEngine'; +import { endBatch, startBatch } from '../../stores/historyStore'; +import { useTimelineStore } from '../../stores/timeline'; +import type { SceneViewport } from '../../engine/scene/types'; +import { + collectPreviewSceneObjects, + resolveAxisScreenHandle, + type PreviewSceneObject, + type SceneAxisScreenHandle, + type SceneGizmoAxis, + type SceneGizmoMode, +} from './sceneObjectOverlayMath'; + +interface SceneObjectOverlayProps { + clips: TimelineClip[]; + tracks: TimelineTrack[]; + selectedClipId: string | null; + selectClip: (id: string | null, addToSelection?: boolean, setPrimaryOnly?: boolean) => void; + canvasSize: { width: number; height: number }; + viewport: SceneViewport; + compositionId?: string | null; + sceneNavClipId?: string | null; + enabled: boolean; +} + +interface DragState { + clipId: string; + mode: SceneGizmoMode; + axis: SceneGizmoAxis; + transformSpace: PreviewSceneObject['transformSpace']; + startTransform: ClipTransform; + direction: { x: number; y: number }; + pixelsPerUnit: number; + viewport: SceneViewport; +} + +interface DragRuntime { + target: HTMLElement | null; + hasPointerLock: boolean; + accumulatedX: number; + accumulatedY: number; + lastClientX: number; + lastClientY: number; +} + +interface DisplaySceneObject extends PreviewSceneObject { + displayX: number; + displayY: number; +} + +const AXES: SceneGizmoAxis[] = ['x', 'y', 'z']; +const OVERLAY_REFRESH_MS = 125; + +const AXIS_LABELS: Record = { + x: 'X', + y: 'Y', + z: 'Z', +}; + +const MODE_LABELS: Record = { + move: 'Move', + rotate: 'Rotate', + scale: 'Scale', +}; + +function getObjectBadge(kind: PreviewSceneObject['kind']): string { + switch (kind) { + case 'effector': + return 'E'; + case 'splat': + return 'S'; + case 'model': + return 'M'; + case 'plane': + return '3D'; + } +} + +function resolveDisplayObjects( + objects: PreviewSceneObject[], + canvasSize: { width: number; height: number }, +): DisplaySceneObject[] { + const groups = new Map(); + for (const object of objects) { + if (!object.screen.visible) continue; + const key = `${Math.round(object.screen.x / 32)}:${Math.round(object.screen.y / 32)}`; + groups.set(key, [...(groups.get(key) ?? []), object]); + } + + return objects.map((object) => { + const key = `${Math.round(object.screen.x / 32)}:${Math.round(object.screen.y / 32)}`; + const group = groups.get(key) ?? [object]; + const index = group.findIndex((candidate) => candidate.clipId === object.clipId); + if (!object.screen.visible || group.length <= 1 || index < 0) { + return { ...object, displayX: object.screen.x, displayY: object.screen.y }; + } + + const angle = -Math.PI / 2 + (index * Math.PI * 2) / group.length; + const radius = 19; + return { + ...object, + displayX: Math.max(14, Math.min(canvasSize.width - 14, object.screen.x + Math.cos(angle) * radius)), + displayY: Math.max(14, Math.min(canvasSize.height - 14, object.screen.y + Math.sin(angle) * radius)), + }; + }); +} + +function cloneTransform(transform: ClipTransform): ClipTransform { + return { + opacity: transform.opacity, + blendMode: transform.blendMode, + position: { ...transform.position }, + scale: { ...transform.scale }, + rotation: { ...transform.rotation }, + }; +} + +function resolveTransformPropertyUpdates(transform: Partial): Array<[AnimatableProperty, number]> { + const updates: Array<[AnimatableProperty, number]> = []; + if (transform.opacity !== undefined) updates.push(['opacity', transform.opacity]); + if (transform.position) { + updates.push(['position.x', transform.position.x]); + updates.push(['position.y', transform.position.y]); + updates.push(['position.z', transform.position.z]); + } + if (transform.scale) { + updates.push(['scale.x', transform.scale.x]); + updates.push(['scale.y', transform.scale.y]); + if (transform.scale.z !== undefined) updates.push(['scale.z', transform.scale.z]); + } + if (transform.rotation) { + updates.push(['rotation.x', transform.rotation.x]); + updates.push(['rotation.y', transform.rotation.y]); + updates.push(['rotation.z', transform.rotation.z]); + } + return updates; +} + +function getAxisStyle(handle: SceneAxisScreenHandle): CSSProperties { + const length = Math.hypot(handle.end.x - handle.start.x, handle.end.y - handle.start.y); + const angle = Math.atan2(handle.end.y - handle.start.y, handle.end.x - handle.start.x); + return { + left: handle.start.x, + top: handle.start.y, + width: length, + transform: `rotate(${angle}rad)`, + }; +} + +function getDragSpeedMultiplier(event: MouseEvent): number { + if (event.ctrlKey) return 5; + if (event.altKey || event.shiftKey) return 0.1; + return 1; +} + +function applySceneObjectTransform(clipId: string, transform: Partial): void { + const store = useTimelineStore.getState(); + const updates = resolveTransformPropertyUpdates(transform); + const useKeyframePath = updates.some(([property]) => + store.hasKeyframes(clipId, property) || store.isRecording(clipId, property), + ); + + if (useKeyframePath) { + for (const [property, value] of updates) { + store.setPropertyValue(clipId, property, value); + } + } else { + store.updateClipTransform(clipId, transform); + } + engine.requestRender(); +} + +function applyDragTransform(drag: DragState, screenDistance: number): void { + const units = screenDistance / drag.pixelsPerUnit; + const start = drag.startTransform; + const axis = drag.axis; + + if (drag.mode === 'rotate') { + const degrees = screenDistance * 0.6; + applySceneObjectTransform(drag.clipId, { + rotation: { + x: start.rotation.x + (axis === 'x' ? degrees : 0), + y: start.rotation.y + (axis === 'y' ? degrees : 0), + z: start.rotation.z + (axis === 'z' ? degrees : 0), + }, + }); + return; + } + + if (drag.mode === 'scale') { + const scaleDelta = screenDistance / 90; + if (drag.transformSpace === 'effector') { + const next = Math.max(0.001, Math.max(start.scale.x, start.scale.y, start.scale.z ?? 1) + scaleDelta); + applySceneObjectTransform(drag.clipId, { + scale: { x: next, y: next, z: next }, + }); + return; + } + + applySceneObjectTransform(drag.clipId, { + scale: { + x: Math.max(0.001, start.scale.x + (axis === 'x' ? scaleDelta : 0)), + y: Math.max(0.001, start.scale.y + (axis === 'y' ? scaleDelta : 0)), + z: axis === 'z' + ? Math.max(0.001, (start.scale.z ?? 1) + scaleDelta) + : start.scale.z, + }, + }); + return; + } + + const aspect = drag.viewport.width / Math.max(1, drag.viewport.height); + const position = { ...start.position }; + if (axis === 'x') { + position.x += drag.transformSpace === 'effector' + ? units / aspect + : units; + } + if (axis === 'y') { + position.y += drag.transformSpace === 'effector' ? -units : units; + } + if (axis === 'z') { + position.z += units; + } + + applySceneObjectTransform(drag.clipId, { position }); +} + +export function SceneObjectOverlay({ + clips, + tracks, + selectedClipId, + selectClip, + canvasSize, + viewport, + compositionId, + sceneNavClipId, + enabled, +}: SceneObjectOverlayProps) { + const [mode, setMode] = useState('move'); + const [dragState, setDragState] = useState(null); + const [timelineSnapshotTick, setTimelineSnapshotTick] = useState(0); + const endedDragRef = useRef(false); + const overlayRef = useRef(null); + const dragRuntimeRef = useRef({ + target: null, + hasPointerLock: false, + accumulatedX: 0, + accumulatedY: 0, + lastClientX: 0, + lastClientY: 0, + }); + + const releasePointerLock = useCallback(() => { + const { target } = dragRuntimeRef.current; + if (target && document.pointerLockElement === target) { + document.exitPointerLock(); + } + dragRuntimeRef.current.hasPointerLock = false; + dragRuntimeRef.current.target = null; + }, []); + + const requestPointerLock = useCallback((target: HTMLElement, fallbackTarget?: HTMLElement) => { + if (!target.requestPointerLock) return; + + try { + const result = target.requestPointerLock(); + if (result && typeof (result as Promise).then === 'function') { + (result as Promise).then( + () => { + if (dragRuntimeRef.current.target === target) { + dragRuntimeRef.current.hasPointerLock = document.pointerLockElement === target; + } + }, + () => { + if (fallbackTarget && fallbackTarget !== target) { + dragRuntimeRef.current.target = fallbackTarget; + requestPointerLock(fallbackTarget); + } else if (dragRuntimeRef.current.target === target) { + dragRuntimeRef.current.hasPointerLock = false; + } + }, + ); + } else { + requestAnimationFrame(() => { + if (dragRuntimeRef.current.target === target) { + dragRuntimeRef.current.hasPointerLock = document.pointerLockElement === target; + } + }); + } + } catch { + if (fallbackTarget && fallbackTarget !== target) { + dragRuntimeRef.current.target = fallbackTarget; + requestPointerLock(fallbackTarget); + } else { + dragRuntimeRef.current.hasPointerLock = false; + } + } + }, []); + + useEffect(() => { + if (!enabled) return; + + const intervalId = window.setInterval(() => { + if (useTimelineStore.getState().isPlaying) return; + setTimelineSnapshotTick((tick) => (tick + 1) % 1000000); + }, OVERLAY_REFRESH_MS); + return () => window.clearInterval(intervalId); + }, [enabled]); + + const { camera, objects } = useMemo( + () => { + void timelineSnapshotTick; + const { clipKeyframes, playheadPosition } = useTimelineStore.getState(); + return collectPreviewSceneObjects({ + clips, + tracks, + clipKeyframes, + playheadPosition, + viewport, + canvasSize, + compositionId, + sceneNavClipId, + }); + }, + [canvasSize, clips, compositionId, sceneNavClipId, timelineSnapshotTick, tracks, viewport], + ); + + const selectedObject = useMemo( + () => objects.find((object) => object.clipId === selectedClipId) ?? null, + [objects, selectedClipId], + ); + const displayObjects = useMemo( + () => resolveDisplayObjects(objects, canvasSize), + [canvasSize, objects], + ); + + const axisHandles = useMemo(() => { + if (!selectedObject || !selectedObject.screen.visible) return []; + return AXES.map((axis) => resolveAxisScreenHandle(axis, selectedObject.worldPosition, camera, canvasSize)); + }, [camera, canvasSize, selectedObject]); + + const endDrag = useCallback(() => { + if (!dragState) return; + releasePointerLock(); + if (!endedDragRef.current) { + endedDragRef.current = true; + endBatch(); + } + setDragState(null); + }, [dragState, releasePointerLock]); + + useEffect(() => { + if (!dragState) return; + + const handlePointerLockChange = () => { + const { target } = dragRuntimeRef.current; + dragRuntimeRef.current.hasPointerLock = target !== null && document.pointerLockElement === target; + }; + + const handleMouseMove = (event: MouseEvent) => { + event.preventDefault(); + const runtime = dragRuntimeRef.current; + const pointerLockActive = runtime.target !== null && document.pointerLockElement === runtime.target; + runtime.hasPointerLock = pointerLockActive; + + let deltaX: number; + let deltaY: number; + if (pointerLockActive) { + deltaX = event.movementX; + deltaY = event.movementY; + } else { + deltaX = event.clientX - runtime.lastClientX; + deltaY = event.clientY - runtime.lastClientY; + runtime.lastClientX = event.clientX; + runtime.lastClientY = event.clientY; + } + + const speedMultiplier = getDragSpeedMultiplier(event); + runtime.accumulatedX += deltaX * speedMultiplier; + runtime.accumulatedY += deltaY * speedMultiplier; + + const screenDistance = + runtime.accumulatedX * dragState.direction.x + + runtime.accumulatedY * dragState.direction.y; + applyDragTransform(dragState, screenDistance); + }; + + const handleMouseUp = (event: MouseEvent) => { + event.preventDefault(); + endDrag(); + }; + + document.addEventListener('pointerlockchange', handlePointerLockChange); + document.addEventListener('pointerlockerror', handlePointerLockChange); + window.addEventListener('mousemove', handleMouseMove); + window.addEventListener('mouseup', handleMouseUp); + return () => { + document.removeEventListener('pointerlockchange', handlePointerLockChange); + document.removeEventListener('pointerlockerror', handlePointerLockChange); + window.removeEventListener('mousemove', handleMouseMove); + window.removeEventListener('mouseup', handleMouseUp); + releasePointerLock(); + }; + }, [dragState, endDrag, releasePointerLock]); + + useEffect(() => { + if (!enabled || !selectedObject) return; + + const handleKeyDown = (event: KeyboardEvent) => { + const target = event.target; + if (target instanceof HTMLInputElement || target instanceof HTMLTextAreaElement || target instanceof HTMLSelectElement) { + return; + } + if (event.altKey || event.ctrlKey || event.metaKey) return; + if (event.code === 'KeyW') { + event.preventDefault(); + setMode('move'); + } else if (event.code === 'KeyE') { + event.preventDefault(); + setMode('rotate'); + } else if (event.code === 'KeyR') { + event.preventDefault(); + setMode('scale'); + } + }; + + window.addEventListener('keydown', handleKeyDown); + return () => window.removeEventListener('keydown', handleKeyDown); + }, [enabled, selectedObject]); + + const handleObjectPointerDown = useCallback((event: ReactPointerEvent, object: PreviewSceneObject) => { + event.preventDefault(); + event.stopPropagation(); + selectClip(object.clipId, event.shiftKey); + }, [selectClip]); + + const handleAxisMouseDown = useCallback((event: ReactMouseEvent, handle: SceneAxisScreenHandle) => { + if (event.button !== 0) return; + if (!selectedObject) return; + const clip = clips.find((candidate) => candidate.id === selectedObject.clipId); + if (!clip) return; + + const lockTarget = overlayRef.current ?? document.body; + const fallbackTarget = event.currentTarget; + + event.preventDefault(); + event.stopPropagation(); + endedDragRef.current = false; + dragRuntimeRef.current = { + target: lockTarget, + hasPointerLock: false, + accumulatedX: 0, + accumulatedY: 0, + lastClientX: event.clientX, + lastClientY: event.clientY, + }; + requestPointerLock(lockTarget, fallbackTarget); + startBatch(`Scene ${mode}`); + setDragState({ + clipId: selectedObject.clipId, + mode, + axis: handle.axis, + transformSpace: selectedObject.transformSpace, + startTransform: cloneTransform(clip.transform), + direction: handle.direction, + pixelsPerUnit: handle.pixelsPerUnit, + viewport, + }); + }, [clips, mode, requestPointerLock, selectedObject, viewport]); + + if (!enabled || canvasSize.width <= 0 || canvasSize.height <= 0 || objects.length === 0) { + return null; + } + + return ( +
+ {selectedObject && selectedObject.screen.visible && ( + <> + {axisHandles.map((handle) => ( +
+ + + {AXIS_LABELS[handle.axis]} + +
+ ))} +
+ {(['move', 'rotate', 'scale'] as SceneGizmoMode[]).map((nextMode) => ( + + ))} +
+ + )} + + {displayObjects.map((object) => { + if (!object.screen.visible) return null; + const selected = object.clipId === selectedClipId; + return ( + + ); + })} +
+ ); +} diff --git a/src/components/preview/sceneObjectOverlayMath.ts b/src/components/preview/sceneObjectOverlayMath.ts new file mode 100644 index 000000000..f9af2c9a1 --- /dev/null +++ b/src/components/preview/sceneObjectOverlayMath.ts @@ -0,0 +1,222 @@ +import type { Keyframe, TimelineClip, TimelineTrack } from '../../types'; +import type { SceneCamera, SceneViewport, SceneVector3 } from '../../engine/scene/types'; +import { resolveRenderableSharedSceneCamera } from '../../engine/scene/SceneCameraUtils'; +import { resolveSceneClipTransform } from '../../engine/scene/SceneTimelineUtils'; + +export type SceneObjectKind = 'effector' | 'splat' | 'model' | 'plane'; +export type SceneObjectTransformSpace = 'world' | 'effector'; +export type SceneGizmoAxis = 'x' | 'y' | 'z'; +export type SceneGizmoMode = 'move' | 'rotate' | 'scale'; + +export interface PreviewSceneObject { + clipId: string; + name: string; + kind: SceneObjectKind; + transformSpace: SceneObjectTransformSpace; + worldPosition: SceneVector3; + screen: { + x: number; + y: number; + visible: boolean; + depth: number; + }; +} + +export interface SceneAxisScreenHandle { + axis: SceneGizmoAxis; + start: { x: number; y: number }; + end: { x: number; y: number }; + direction: { x: number; y: number }; + pixelsPerUnit: number; +} + +interface CollectPreviewSceneObjectsParams { + clips: TimelineClip[]; + tracks: TimelineTrack[]; + clipKeyframes: Map; + playheadPosition: number; + viewport: SceneViewport; + canvasSize: { width: number; height: number }; + compositionId?: string | null; + sceneNavClipId?: string | null; +} + +const AXIS_FALLBACKS: Record = { + x: { x: 1, y: 0 }, + y: { x: 0, y: -1 }, + z: { x: -0.72, y: -0.72 }, +}; + +function isClipActiveAtTime(clip: TimelineClip, timelineTime: number): boolean { + return timelineTime >= clip.startTime && timelineTime < clip.startTime + clip.duration; +} + +function resolveSceneObjectKind(clip: TimelineClip): SceneObjectKind | null { + const sourceType = clip.source?.type; + if (sourceType === 'splat-effector') return 'effector'; + if (sourceType === 'gaussian-splat') return 'splat'; + if (sourceType === 'model') return 'model'; + if (clip.is3D && sourceType && sourceType !== 'audio') return 'plane'; + return null; +} + +function multiplyMat4Vec4(matrix: Float32Array, vector: [number, number, number, number]): [number, number, number, number] { + const [x, y, z, w] = vector; + return [ + matrix[0] * x + matrix[4] * y + matrix[8] * z + matrix[12] * w, + matrix[1] * x + matrix[5] * y + matrix[9] * z + matrix[13] * w, + matrix[2] * x + matrix[6] * y + matrix[10] * z + matrix[14] * w, + matrix[3] * x + matrix[7] * y + matrix[11] * z + matrix[15] * w, + ]; +} + +export function projectWorldToCanvas( + point: SceneVector3, + camera: SceneCamera, + canvasSize: { width: number; height: number }, +): PreviewSceneObject['screen'] { + const viewPoint = multiplyMat4Vec4(camera.viewMatrix, [point.x, point.y, point.z, 1]); + const clipPoint = multiplyMat4Vec4(camera.projectionMatrix, viewPoint); + const w = clipPoint[3]; + if (Math.abs(w) < 0.000001) { + return { x: 0, y: 0, visible: false, depth: w }; + } + + const ndcX = clipPoint[0] / w; + const ndcY = clipPoint[1] / w; + const ndcZ = clipPoint[2] / w; + const visible = w > 0 && ndcZ >= -0.1 && ndcZ <= 1.1 && ndcX >= -1.2 && ndcX <= 1.2 && ndcY >= -1.2 && ndcY <= 1.2; + + return { + x: (ndcX * 0.5 + 0.5) * canvasSize.width, + y: (0.5 - ndcY * 0.5) * canvasSize.height, + visible, + depth: w, + }; +} + +function normalizeScreenVector(vector: { x: number; y: number }, fallback: { x: number; y: number }): { x: number; y: number; length: number } { + const length = Math.hypot(vector.x, vector.y); + if (length >= 8) { + return { x: vector.x / length, y: vector.y / length, length }; + } + + const fallbackLength = Math.hypot(fallback.x, fallback.y) || 1; + return { + x: fallback.x / fallbackLength, + y: fallback.y / fallbackLength, + length: 48, + }; +} + +export function resolveAxisScreenHandle( + axis: SceneGizmoAxis, + worldPosition: SceneVector3, + camera: SceneCamera, + canvasSize: { width: number; height: number }, +): SceneAxisScreenHandle { + const start = projectWorldToCanvas(worldPosition, camera, canvasSize); + const axisEndWorld = { + x: worldPosition.x + (axis === 'x' ? 1 : 0), + y: worldPosition.y + (axis === 'y' ? 1 : 0), + z: worldPosition.z + (axis === 'z' ? 1 : 0), + }; + const projectedEnd = projectWorldToCanvas(axisEndWorld, camera, canvasSize); + const normalized = normalizeScreenVector( + { x: projectedEnd.x - start.x, y: projectedEnd.y - start.y }, + AXIS_FALLBACKS[axis], + ); + const visualLength = Math.max(42, Math.min(78, normalized.length)); + + return { + axis, + start: { x: start.x, y: start.y }, + end: { + x: start.x + normalized.x * visualLength, + y: start.y + normalized.y * visualLength, + }, + direction: { x: normalized.x, y: normalized.y }, + pixelsPerUnit: Math.max(42, normalized.length), + }; +} + +function resolveClipWorldPosition( + kind: SceneObjectKind, + transform: TimelineClip['transform'], + viewport: SceneViewport, +): { position: SceneVector3; transformSpace: SceneObjectTransformSpace } { + const aspect = viewport.width / Math.max(1, viewport.height); + const halfWorldW = aspect; + + if (kind === 'effector') { + return { + transformSpace: 'effector', + position: { + x: transform.position.x * halfWorldW, + y: -transform.position.y, + z: transform.position.z, + }, + }; + } + + return { + transformSpace: 'world', + position: { + x: transform.position.x, + y: transform.position.y, + z: transform.position.z, + }, + }; +} + +export function collectPreviewSceneObjects({ + clips, + tracks, + clipKeyframes, + playheadPosition, + viewport, + canvasSize, + compositionId, + sceneNavClipId, +}: CollectPreviewSceneObjectsParams): { camera: SceneCamera; objects: PreviewSceneObject[] } { + const camera = resolveRenderableSharedSceneCamera(viewport, playheadPosition, { + clips, + tracks, + clipKeyframes, + compositionId, + sceneNavClipId, + }); + const visibleVideoTrackIds = new Set( + tracks + .filter((track) => track.type === 'video' && track.visible !== false) + .map((track) => track.id), + ); + const objects = clips + .filter((clip) => visibleVideoTrackIds.has(clip.trackId) && isClipActiveAtTime(clip, playheadPosition)) + .map((clip): PreviewSceneObject | null => { + const kind = resolveSceneObjectKind(clip); + if (!kind) return null; + + const transform = resolveSceneClipTransform( + clip, + playheadPosition - clip.startTime, + playheadPosition, + { clips, clipKeyframes }, + ); + const { position, transformSpace } = resolveClipWorldPosition(kind, transform, viewport); + const screen = projectWorldToCanvas(position, camera, canvasSize); + + return { + clipId: clip.id, + name: clip.name, + kind, + transformSpace, + worldPosition: position, + screen, + }; + }) + .filter((object): object is PreviewSceneObject => object !== null) + .toSorted((a, b) => b.screen.depth - a.screen.depth); + + return { camera, objects }; +} diff --git a/src/engine/gaussian/core/GaussianSplatGpuRenderer.ts b/src/engine/gaussian/core/GaussianSplatGpuRenderer.ts index f1eb3c66d..cad567bf2 100644 --- a/src/engine/gaussian/core/GaussianSplatGpuRenderer.ts +++ b/src/engine/gaussian/core/GaussianSplatGpuRenderer.ts @@ -48,6 +48,11 @@ interface SplatSceneGpuResources { sortedBindGroup: GPUBindGroup | null; workerSorter: SplatOrderSorter | null; workerSortedBindGroup: GPUBindGroup | null; + activeWorkerSortedBindGroup: { + dataBuffer: GPUBuffer; + orderBuffer: GPUBuffer; + bindGroup: GPUBindGroup; + } | null; } interface CameraUniformResource { @@ -289,6 +294,7 @@ export class GaussianSplatGpuRenderer { sortedBindGroup: null, workerSorter, workerSortedBindGroup, + activeWorkerSortedBindGroup: null, }); // Initialize sort pass for this scene's capacity (lazy init) @@ -428,7 +434,6 @@ export class GaussianSplatGpuRenderer { let hasValidatedCullResult = false; const canUseWorkerSort = !precise && sortFrequency !== 0 && - activeSplatBuffer === scene.splatBuffer && scene.workerSorter !== null && scene.workerSortedBindGroup !== null; let usedWorkerSort = false; @@ -542,8 +547,10 @@ export class GaussianSplatGpuRenderer { let renderBindGroup = scene.bindGroup; // default: identity indices + original data // Build the appropriate bind group based on which passes ran - if (canUseWorkerSort && scene.workerSortedBindGroup) { - renderBindGroup = scene.workerSortedBindGroup; + if (canUseWorkerSort && scene.workerSorter && scene.workerSortedBindGroup) { + renderBindGroup = activeSplatBuffer === scene.splatBuffer + ? scene.workerSortedBindGroup + : this.getActiveWorkerSortedBindGroup(scene, clipId, activeSplatBuffer, scene.workerSorter.orderBuffer); } else if (sortedIndexBuffer || cullIndexBuffer || activeSplatBuffer !== scene.splatBuffer) { const indexBuf = sortedIndexBuffer ?? cullIndexBuffer ?? scene.identityIndexBuffer; renderBindGroup = this.device.createBindGroup({ @@ -755,6 +762,29 @@ export class GaussianSplatGpuRenderer { return entry; } + private getActiveWorkerSortedBindGroup( + scene: SplatSceneGpuResources, + clipId: string, + dataBuffer: GPUBuffer, + orderBuffer: GPUBuffer, + ): GPUBindGroup { + const cached = scene.activeWorkerSortedBindGroup; + if (cached && cached.dataBuffer === dataBuffer && cached.orderBuffer === orderBuffer) { + return cached.bindGroup; + } + + const bindGroup = this.device!.createBindGroup({ + layout: this.splatDataBindGroupLayout!, + entries: [ + { binding: 0, resource: { buffer: dataBuffer } }, + { binding: 1, resource: { buffer: orderBuffer } }, + ], + label: `splat-worker-sorted-active-bind-group-${clipId}`, + }); + scene.activeWorkerSortedBindGroup = { dataBuffer, orderBuffer, bindGroup }; + return bindGroup; + } + /** Release effector buffer for a specific clip */ private releaseEffectorBuffer(clipId: string): void { const entry = this.effectorOutputBuffers.get(clipId); diff --git a/tests/unit/gaussianSplatGpuRendererUniforms.test.ts b/tests/unit/gaussianSplatGpuRendererUniforms.test.ts index d21273661..0273fb7ad 100644 --- a/tests/unit/gaussianSplatGpuRendererUniforms.test.ts +++ b/tests/unit/gaussianSplatGpuRendererUniforms.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from 'vitest'; import { GaussianSplatGpuRenderer } from '../../src/engine/gaussian/core/GaussianSplatGpuRenderer'; import type { SplatCameraParams } from '../../src/engine/gaussian/core/GaussianSplatGpuRenderer'; +import type { LocalSplatEffectorData } from '../../src/engine/native3d/passes/EffectorCompute'; type GaussianSplatGpuRendererTestAccess = GaussianSplatGpuRenderer & { device: GPUDevice; @@ -13,6 +14,11 @@ type GaussianSplatGpuRendererTestAccess = GaussianSplatGpuRenderer & { acquire: (width: number, height: number) => { texture: unknown; view: GPUTextureView }; }; sceneCache: Map; + effectorCompute: { + isInitialized: boolean; + prepareLocalSplatEffectors: ReturnType; + execute: ReturnType; + }; _initialized: boolean; createPipeline(): void; }; @@ -190,4 +196,140 @@ describe('GaussianSplatGpuRenderer camera uniforms', () => { expect(secondCameraBindGroup).toEqual({ label: 'splat-camera-bind-group-1' }); expect(firstCameraBindGroup).not.toBe(secondCameraBindGroup); }); + + it('keeps worker sort ordering when effectors render through an active splat buffer', () => { + const renderPasses: Array<{ setBindGroup: ReturnType }> = []; + const workerOrderBuffer = { label: 'worker-order-buffer' }; + const workerSorter = { + orderBuffer: workerOrderBuffer, + hasSortedOrder: true, + requestSort: vi.fn(), + applyPending: vi.fn(() => 1), + }; + const device = { + createBuffer: vi.fn((descriptor: { label?: string }) => ({ + label: descriptor.label, + destroy: vi.fn(), + })), + createBindGroup: vi.fn((descriptor: { label?: string }) => ({ + label: descriptor.label, + })), + queue: { + writeBuffer: vi.fn(), + }, + }; + const commandEncoder = { + beginRenderPass: vi.fn(() => { + const pass = { + setPipeline: vi.fn(), + setBindGroup: vi.fn(), + draw: vi.fn(), + end: vi.fn(), + }; + renderPasses.push(pass); + return pass; + }), + }; + const renderer = new GaussianSplatGpuRenderer() as unknown as GaussianSplatGpuRendererTestAccess; + const localEffectors: LocalSplatEffectorData[] = [{ + position: { x: 0, y: 0, z: 0 }, + axis: { x: 0, y: 0, z: 1 }, + radius: 1, + strength: 0.2, + falloff: 1, + speed: 1, + seed: 0, + time: 0, + mode: 0, + }]; + renderer.device = device; + renderer.pipeline = { label: 'pipeline' }; + renderer.pipelineWithDepth = null; + renderer.splatDataBindGroupLayout = { label: 'splat-layout' }; + renderer.cameraBindGroupLayout = { label: 'camera-layout' }; + renderer.renderTargetPool = { + resetFrame: vi.fn(), + acquire: vi.fn(() => ({ texture: { label: 'target' }, view: { label: 'target-view' } })), + }; + renderer.effectorCompute = { + isInitialized: true, + prepareLocalSplatEffectors: vi.fn(() => localEffectors), + execute: vi.fn(), + }; + renderer._initialized = true; + renderer.sceneCache.set('splat-a', { + ...makeScene({ label: 'identity-bind-group' }), + splatCount: 4, + workerSorter, + workerSortedBindGroup: { label: 'base-worker-sorted-bind-group' }, + }); + + renderer.renderToTexture( + 'splat-a', + makeCamera(), + { width: 1280, height: 720 }, + commandEncoder, + { + worldMatrix: makeWorldMatrix(1), + effectors: [{ + clipId: 'effector-1', + position: { x: 0, y: 0, z: 0 }, + rotation: { x: 0, y: 0, z: 0 }, + scale: { x: 1, y: 1, z: 1 }, + radius: 1, + mode: 'repel', + strength: 20, + falloff: 1, + speed: 1, + seed: 0, + time: 0, + }], + }, + ); + renderer.renderToTexture( + 'splat-a', + makeCamera(), + { width: 1280, height: 720 }, + commandEncoder, + { + worldMatrix: makeWorldMatrix(1), + effectors: [{ + clipId: 'effector-1', + position: { x: 0, y: 0, z: 0 }, + rotation: { x: 0, y: 0, z: 0 }, + scale: { x: 1, y: 1, z: 1 }, + radius: 1, + mode: 'repel', + strength: 20, + falloff: 1, + speed: 1, + seed: 0, + time: 0, + }], + }, + ); + + const activeWorkerBindGroupCalls = device.createBindGroup.mock.calls.filter( + ([descriptor]) => descriptor.label === 'splat-worker-sorted-active-bind-group-splat-a', + ); + expect(activeWorkerBindGroupCalls).toHaveLength(1); + const activeWorkerBindGroupDescriptor = activeWorkerBindGroupCalls[0]?.[0]; + expect(activeWorkerBindGroupDescriptor).toMatchObject({ + entries: [ + { binding: 0, resource: { buffer: { label: 'effector-output-splat-a' } } }, + { binding: 1, resource: { buffer: workerOrderBuffer } }, + ], + }); + expect(workerSorter.requestSort).toHaveBeenCalled(); + expect(workerSorter.applyPending).toHaveBeenCalledWith(device.queue); + + const splatDataBindGroup = renderPasses[0]?.setBindGroup.mock.calls.find( + ([slot]) => slot === 0, + )?.[1]; + expect(splatDataBindGroup).toEqual({ label: 'splat-worker-sorted-active-bind-group-splat-a' }); + const reusedSplatDataBindGroup = renderPasses[1]?.setBindGroup.mock.calls.find( + ([slot]) => slot === 0, + )?.[1]; + expect(reusedSplatDataBindGroup).toBe(splatDataBindGroup); + }); }); diff --git a/tests/unit/sceneObjectOverlayMath.test.ts b/tests/unit/sceneObjectOverlayMath.test.ts new file mode 100644 index 000000000..bae263042 --- /dev/null +++ b/tests/unit/sceneObjectOverlayMath.test.ts @@ -0,0 +1,102 @@ +import { describe, expect, it } from 'vitest'; +import type { TimelineClip, TimelineTrack } from '../../src/types'; +import { collectPreviewSceneObjects, projectWorldToCanvas } from '../../src/components/preview/sceneObjectOverlayMath'; +import { resolveRenderableSharedSceneCamera } from '../../src/engine/scene/SceneCameraUtils'; + +function makeClip(partial: Partial): TimelineClip { + return { + id: 'clip', + trackId: 'video-1', + name: 'Clip', + file: new File([], 'clip.dat'), + startTime: 0, + duration: 10, + inPoint: 0, + outPoint: 10, + source: { type: 'gaussian-splat' }, + transform: { + opacity: 1, + blendMode: 'normal', + position: { x: 0, y: 0, z: 0 }, + scale: { x: 1, y: 1 }, + rotation: { x: 0, y: 0, z: 0 }, + }, + effects: [], + masks: [], + ...partial, + }; +} + +const tracks: TimelineTrack[] = [{ + id: 'video-1', + name: 'Video 1', + type: 'video', + clips: [], + visible: true, + muted: false, + locked: false, +}]; + +describe('sceneObjectOverlayMath', () => { + it('projects world origin into the canvas', () => { + const camera = resolveRenderableSharedSceneCamera({ width: 1920, height: 1080 }, 0); + const screen = projectWorldToCanvas({ x: 0, y: 0, z: 0 }, camera, { width: 960, height: 540 }); + + expect(screen.visible).toBe(true); + expect(screen.x).toBeCloseTo(480, 0); + expect(screen.y).toBeCloseTo(270, 0); + }); + + it('collects active scene objects only from visible video tracks and skips cameras', () => { + const clips = [ + makeClip({ id: 'active-splat', name: 'Splat' }), + makeClip({ id: 'future-splat', startTime: 20 }), + makeClip({ + id: 'camera', + name: 'Camera', + source: { type: 'camera', cameraSettings: { fov: 50, near: 0.1, far: 1000 } }, + }), + ]; + + const { objects } = collectPreviewSceneObjects({ + clips, + tracks, + clipKeyframes: new Map(), + playheadPosition: 1, + viewport: { width: 1920, height: 1080 }, + canvasSize: { width: 960, height: 540 }, + }); + + expect(objects.map((object) => object.clipId).toSorted()).toEqual(['active-splat']); + }); + + it('maps effector transform into shared scene space', () => { + const clips = [ + makeClip({ + id: 'effector', + source: { type: 'splat-effector' }, + transform: { + opacity: 1, + blendMode: 'normal', + position: { x: 0.5, y: 0.25, z: 2 }, + scale: { x: 1, y: 1, z: 1 }, + rotation: { x: 0, y: 0, z: 0 }, + }, + }), + ]; + + const { objects } = collectPreviewSceneObjects({ + clips, + tracks, + clipKeyframes: new Map(), + playheadPosition: 1, + viewport: { width: 1920, height: 1080 }, + canvasSize: { width: 960, height: 540 }, + }); + + expect(objects[0]?.kind).toBe('effector'); + expect(objects[0]?.worldPosition.x).toBeCloseTo(0.8889, 3); + expect(objects[0]?.worldPosition.y).toBeCloseTo(-0.25, 3); + expect(objects[0]?.worldPosition.z).toBe(2); + }); +}); From a9336e8fceacdb71f6a92218f19e7a41d17942bf Mon Sep 17 00:00:00 2001 From: Sportinger Date: Sun, 26 Apr 2026 20:07:24 +0200 Subject: [PATCH 2/4] Add media panel board view --- docs/Features/Media-Panel.md | 28 +- src/App.css | 325 ++++++ src/components/panels/MediaPanel.tsx | 1039 ++++++++++++++++++- src/services/project/projectLoad.ts | 24 + src/services/project/projectSave.ts | 30 +- src/services/project/types/project.types.ts | 16 + src/stores/mediaStore/index.ts | 21 +- 7 files changed, 1445 insertions(+), 38 deletions(-) diff --git a/docs/Features/Media-Panel.md b/docs/Features/Media-Panel.md index 8046d1e40..09905b9cd 100644 --- a/docs/Features/Media-Panel.md +++ b/docs/Features/Media-Panel.md @@ -2,7 +2,7 @@ [← Back to Index](./README.md) -Import, organize, and manage media assets with folder structure, proxy generation, and dual view modes. +Import, organize, and manage media assets with folder structure, proxy generation, and three view modes. --- @@ -122,15 +122,15 @@ When supported (Chrome/Edge): ## View Modes -The panel supports two view modes through a single header toggle button that swaps between list and grid icons. The selected mode is persisted in `localStorage`. +The panel supports three view modes through the header mode control. The selected mode is persisted in `localStorage`. -### List View (default) +### Classic View (default) - Table layout with sortable, reorderable columns - Nested folder tree with expand/collapse arrows - Column headers for sorting and drag-to-reorder - Resizable name column (120px - 500px range, saved to localStorage) -### Grid View +### Icons View - Thumbnail grid with file names below each item - Folder navigation via breadcrumb bar - Double-click folders to navigate into them @@ -139,6 +139,21 @@ The panel supports two view modes through a single header toggle button that swa - Duration badge overlay on video and composition thumbnails - Item count badge on folder thumbnails +### Board View +- Freeform board canvas grouped by Media Panel folders +- Mouse wheel zooms around the cursor; left-dragging the board or a node pans the board +- Right-dragging nodes moves the selected board nodes; right-clicking opens the context menu +- Ctrl/right-drag starts a marquee selection and skips background cards +- Media, compositions, text, solids, meshes, cameras, and splat effectors appear as movable board nodes +- Node positions and the board viewport are saved into the project UI state, with `localStorage` as the live-session fallback +- Drag files or folders from the OS onto a group to import directly into that folder +- Drag existing Media Panel items onto groups to move them between folders +- Use the small handle on a node to drag that item to the timeline with the same drag payloads as Classic and Icons view +- The board toolbar and compact workspace context menu can create Text and Background items directly on the board +- The **AI** board action opens the existing AI Video panel; generated results still import through the normal Media Store path + +Board pan and node movement preview with imperative CSS transforms and commit final positions only on mouse-up. This avoids re-rendering the Media Panel and writing `localStorage` on every pointer move, which keeps board interaction responsive while heavy preview scenes or splat renders are active. + --- ## Source Thumbnail Cache @@ -285,7 +300,7 @@ Toggle proxy playback mode via `proxyEnabled` / `toggleProxyEnabled()`: - Click and drag on empty space in the item list to draw a selection rectangle - 4px movement threshold before marquee activates - Hold **Ctrl/Cmd** while marquee selecting to add to existing selection -- Works in both list and grid view modes +- Works in Classic and Icons view modes ### Label Colors 16 AE-style label colors assignable to any item (files, folders, compositions, text, solids): @@ -330,7 +345,7 @@ Right-click on items or empty space for context options. ## Media Properties -### Column Display (List View) +### Column Display (Classic View) The media list displays items in a table with the following columns: | Column | Description | Example | @@ -466,6 +481,7 @@ Media references are saved with the project file, while IndexedDB keeps the hand - File metadata (name, type, dimensions, duration, codec, etc.) - File handles (for reload on next session) - Folder structure +- Media Panel view mode, Board viewport, and Board node positions - Composition state with timeline data - Text items and solid items (via localStorage) - When present, `projectPath` points at the copied `Raw/` file and is used for automatic relinking diff --git a/src/App.css b/src/App.css index eecae387b..67bb74387 100644 --- a/src/App.css +++ b/src/App.css @@ -8617,6 +8617,25 @@ input[type="checkbox"] { background: var(--bg-hover); } +.media-view-segment { + display: inline-flex; + align-items: center; + gap: 2px; + padding: 2px; + border: 1px solid var(--border-color); + border-radius: 4px; + background: rgba(0, 0, 0, 0.12); +} + +.media-view-segment .media-view-toggle { + border: 0; + background: transparent; +} + +.media-view-segment .media-view-toggle.active { + background: var(--bg-hover); +} + /* Grid View */ .media-grid-wrapper { flex: 1; @@ -8768,6 +8787,312 @@ input[type="checkbox"] { z-index: 5; } +/* Media Board View */ +.media-board-wrapper { + display: flex; + flex-direction: column; + flex: 1; + min-height: 0; + overflow: hidden; + background: + linear-gradient(rgba(255, 255, 255, 0.035) 1px, transparent 1px), + linear-gradient(90deg, rgba(255, 255, 255, 0.035) 1px, transparent 1px), + var(--bg-primary); + background-size: 32px 32px; +} + +.media-board-toolbar { + display: flex; + align-items: center; + justify-content: space-between; + gap: 8px; + min-height: 34px; + padding: 5px 8px; + border-bottom: 1px solid var(--border-color); + background: var(--bg-tertiary); + flex-shrink: 0; +} + +.media-board-toolbar-title { + display: flex; + align-items: baseline; + gap: 8px; + min-width: 0; +} + +.media-board-toolbar-title span:first-child { + font-size: 12px; + font-weight: 600; + color: var(--text-primary); +} + +.media-board-toolbar-title span:last-child { + font-size: 11px; + color: var(--text-secondary); + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.media-board-toolbar-actions { + display: inline-flex; + align-items: center; + gap: 4px; + flex-shrink: 0; +} + +.media-board-canvas { + position: relative; + flex: 1; + min-height: 0; + overflow: hidden; + cursor: grab; + user-select: none; +} + +.media-board-canvas:active { + cursor: grabbing; +} + +.media-board-wrapper.board-interacting .media-board-node, +.media-board-wrapper.board-interacting .media-board-group { + transition: none !important; +} + +.media-board-wrapper.board-interacting .media-board-node { + will-change: transform; +} + +.media-board-wrapper.board-interacting .media-board-node-thumb img { + pointer-events: none; +} + +.media-board-canvas-inner { + position: absolute; + left: 0; + top: 0; + width: 1px; + height: 1px; + transform-origin: 0 0; +} + +.media-board-group { + position: absolute; + border: 1px solid rgba(255, 255, 255, 0.09); + border-radius: 6px; + background: rgba(255, 255, 255, 0.035); + box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.04); + pointer-events: auto; +} + +.media-board-group-header { + height: 42px; + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; + padding: 0 16px; + border-bottom: 1px solid rgba(255, 255, 255, 0.075); + color: var(--text-secondary); + font-size: 12px; +} + +.media-board-group-header span:first-child { + color: var(--text-primary); + font-weight: 600; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.media-board-group-header span:last-child { + min-width: 22px; + height: 18px; + border-radius: 9px; + display: inline-flex; + align-items: center; + justify-content: center; + background: rgba(255, 255, 255, 0.08); + color: var(--text-secondary); + font-size: 10px; + font-variant-numeric: tabular-nums; +} + +.media-board-node { + position: absolute; + display: flex; + flex-direction: column; + overflow: hidden; + border: 1px solid var(--border-color); + border-top: 3px solid var(--border-color); + border-radius: 6px; + background: var(--bg-secondary); + box-shadow: 0 8px 18px rgba(0, 0, 0, 0.22); + cursor: move; + transition: border-color 0.1s, box-shadow 0.1s, background 0.1s; + pointer-events: auto; +} + +.media-board-node:hover { + background: var(--bg-hover); + border-color: rgba(255, 255, 255, 0.18); +} + +.media-board-node.drag-preview { + box-shadow: 0 14px 30px rgba(0, 0, 0, 0.34); + z-index: 20; +} + +.media-board-node.selected { + border-color: var(--accent); + box-shadow: 0 0 0 2px rgba(45, 140, 235, 0.3), 0 10px 22px rgba(0, 0, 0, 0.26); +} + +.media-board-node.background { + background: rgba(18, 22, 28, 0.72); + border-style: solid; + box-shadow: inset 0 0 0 1px rgba(255, 255, 255, 0.035), 0 12px 28px rgba(0, 0, 0, 0.24); +} + +.media-board-node.no-file { + opacity: 0.55; +} + +.media-board-node-thumb { + position: relative; + height: 82px; + flex-shrink: 0; + background: var(--bg-primary); + overflow: hidden; +} + +.media-board-node-thumb img { + width: 100%; + height: 100%; + object-fit: cover; + display: block; +} + +.media-board-node-placeholder { + height: 100%; + display: flex; + align-items: center; + justify-content: center; + opacity: 0.74; +} + +.media-board-solid-preview, +.media-board-background-preview { + width: 100%; + height: 100%; +} + +.media-board-background-preview { + background-image: + linear-gradient(rgba(255, 255, 255, 0.075) 1px, transparent 1px), + linear-gradient(90deg, rgba(255, 255, 255, 0.075) 1px, transparent 1px); + background-size: 24px 24px; +} + +.media-board-text-preview { + height: 100%; + display: flex; + align-items: center; + padding: 10px; + background: rgba(255, 255, 255, 0.035); + font-size: 17px; + font-weight: 700; + line-height: 1.1; + overflow: hidden; + word-break: break-word; +} + +.media-board-duration, +.media-board-progress { + position: absolute; + right: 6px; + bottom: 5px; + padding: 1px 5px; + border-radius: 3px; + background: rgba(0, 0, 0, 0.72); + color: var(--text-primary); + font-size: 10px; + font-variant-numeric: tabular-nums; +} + +.media-board-progress { + left: 6px; + right: auto; + color: var(--accent); +} + +.media-board-node-timeline-drag { + position: absolute; + top: 5px; + right: 5px; + width: 22px; + height: 22px; + display: inline-flex; + align-items: center; + justify-content: center; + border-radius: 4px; + background: rgba(0, 0, 0, 0.58); + color: var(--text-primary); + cursor: grab; + opacity: 0; + transition: opacity 0.1s, background 0.1s; +} + +.media-board-node:hover .media-board-node-timeline-drag, +.media-board-node.selected .media-board-node-timeline-drag { + opacity: 1; +} + +.media-board-node-timeline-drag:hover { + background: rgba(45, 140, 235, 0.78); +} + +.media-board-node-body { + min-height: 0; + padding: 7px 8px 8px; +} + +.media-board-node-name { + color: var(--text-primary); + font-size: 12px; + font-weight: 500; + line-height: 1.2; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.media-board-node-meta { + display: flex; + align-items: center; + gap: 5px; + margin-top: 5px; + color: var(--text-muted); + font-size: 10px; + line-height: 1; + overflow: hidden; +} + +.media-board-node-meta span { + min-width: 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.media-board-marquee { + position: absolute; + border: 1px solid rgba(45, 140, 235, 0.72); + background: rgba(45, 140, 235, 0.16); + pointer-events: none; + z-index: 50; +} + .media-item { display: flex; align-items: center; diff --git a/src/components/panels/MediaPanel.tsx b/src/components/panels/MediaPanel.tsx index b5ca67da7..4964dd356 100644 --- a/src/components/panels/MediaPanel.tsx +++ b/src/components/panels/MediaPanel.tsx @@ -1,6 +1,6 @@ // Media Panel - Project browser like After Effects -import React, { useCallback, useRef, useState, useEffect } from 'react'; +import React, { useCallback, useMemo, useRef, useState, useEffect } from 'react'; import { Logger } from '../../services/logger'; import { FileTypeIcon } from './media/FileTypeIcon'; import { LABEL_COLORS, getLabelHex } from './media/labelColors'; @@ -13,7 +13,7 @@ import { collectDroppedMediaFiles, planDroppedMediaImports } from './media/dropI const log = Logger.create('MediaPanel'); import { useMediaStore } from '../../stores/mediaStore'; -import type { MediaFile, Composition, ProjectItem, SolidItem, CameraItem } from '../../stores/mediaStore'; +import type { MediaFile, Composition, ProjectItem, TextItem, SolidItem, CameraItem, MediaFolder } from '../../stores/mediaStore'; import { useTimelineStore } from '../../stores/timeline'; import { useDockStore } from '../../stores/dockStore'; import { useContextMenuPosition } from '../../hooks/useContextMenuPosition'; @@ -25,6 +25,52 @@ import { // Column definitions type ColumnId = 'label' | 'name' | 'duration' | 'resolution' | 'fps' | 'container' | 'codec' | 'audio' | 'bitrate' | 'size'; +type MediaPanelViewMode = 'classic' | 'icons' | 'board'; +type MediaBoardItem = Exclude; + +interface MediaBoardViewport { + zoom: number; + panX: number; + panY: number; +} + +interface MediaBoardNodeLayout { + x: number; + y: number; + width: number; + height: number; +} + +interface MediaBoardGroupLayout { + id: string | null; + name: string; + x: number; + y: number; + width: number; + height: number; + itemCount: number; +} + +interface MediaBoardNodePlacement { + item: MediaBoardItem; + layout: MediaBoardNodeLayout; + defaultLayout: MediaBoardNodeLayout; + groupId: string | null; +} + +interface MediaBoardMarquee { + startX: number; + startY: number; + currentX: number; + currentY: number; +} + +type MediaPanelContextMenu = { + x: number; + y: number; + itemId?: string; + mode?: 'default' | 'board-workspace'; +}; const COLUMN_LABELS_MAP: Record = { label: '●', @@ -41,6 +87,26 @@ const COLUMN_LABELS_MAP: Record = { const DEFAULT_COLUMN_ORDER: ColumnId[] = ['name', 'label', 'duration', 'resolution', 'fps', 'container', 'codec', 'audio', 'bitrate', 'size']; const STORAGE_KEY = 'media-panel-column-order'; +const VIEW_MODE_STORAGE_KEY = 'media-panel-view-mode'; +const BOARD_VIEWPORT_STORAGE_KEY = 'media-panel-board-viewport'; +const BOARD_LAYOUT_STORAGE_KEY = 'media-panel-board-layout'; +const MEDIA_PANEL_PROJECT_UI_LOADED_EVENT = 'media-panel-project-ui-loaded'; + +const DEFAULT_BOARD_VIEWPORT: MediaBoardViewport = { zoom: 0.82, panX: 32, panY: 28 }; +const MEDIA_BOARD_NODE_WIDTH = 156; +const MEDIA_BOARD_NODE_HEIGHT = 132; +const MEDIA_BOARD_NODE_GAP = 14; +const MEDIA_BOARD_GROUP_WIDTH = 732; +const MEDIA_BOARD_GROUP_HEADER_HEIGHT = 42; +const MEDIA_BOARD_GROUP_PADDING = 18; +const MEDIA_BOARD_GROUP_GAP = 72; +const MEDIA_BOARD_COLUMNS_PER_GROUP = 4; +const MEDIA_BOARD_PAN_ZOOM_MIN = 0.18; +const MEDIA_BOARD_PAN_ZOOM_MAX = 2.4; +const MEDIA_BOARD_DRAG_START_DISTANCE = 4; +const MEDIA_BOARD_BACKGROUND_COLOR = '#24313f'; +const MEDIA_BOARD_BACKGROUND_LAYOUT = { width: 680, height: 380 }; +const MEDIA_BOARD_TEXT_LAYOUT = { width: 280, height: 150 }; // Load column order from localStorage function loadColumnOrder(): ColumnId[] { @@ -67,6 +133,61 @@ function loadColumnOrder(): ColumnId[] { return DEFAULT_COLUMN_ORDER; } +function loadMediaPanelViewMode(): MediaPanelViewMode { + const stored = localStorage.getItem(VIEW_MODE_STORAGE_KEY); + if (stored === 'board') return 'board'; + if (stored === 'icons' || stored === 'grid') return 'icons'; + return 'classic'; +} + +function loadMediaBoardViewport(): MediaBoardViewport { + try { + const stored = localStorage.getItem(BOARD_VIEWPORT_STORAGE_KEY); + if (!stored) return DEFAULT_BOARD_VIEWPORT; + const parsed = JSON.parse(stored) as Partial; + const zoom = Number(parsed.zoom); + const panX = Number(parsed.panX); + const panY = Number(parsed.panY); + if (!Number.isFinite(zoom) || !Number.isFinite(panX) || !Number.isFinite(panY)) { + return DEFAULT_BOARD_VIEWPORT; + } + return { + zoom: Math.min(MEDIA_BOARD_PAN_ZOOM_MAX, Math.max(MEDIA_BOARD_PAN_ZOOM_MIN, zoom)), + panX, + panY, + }; + } catch { + return DEFAULT_BOARD_VIEWPORT; + } +} + +function loadMediaBoardLayout(): Record { + try { + const stored = localStorage.getItem(BOARD_LAYOUT_STORAGE_KEY); + if (!stored) return {}; + const parsed = JSON.parse(stored) as Record; + if (!parsed || typeof parsed !== 'object') return {}; + const valid: Record = {}; + Object.entries(parsed).forEach(([id, layout]) => { + const x = Number(layout?.x); + const y = Number(layout?.y); + const width = Number(layout?.width); + const height = Number(layout?.height); + if (id && Number.isFinite(x) && Number.isFinite(y) && Number.isFinite(width) && Number.isFinite(height)) { + valid[id] = { + x, + y, + width: Math.max(96, Math.min(1400, width)), + height: Math.max(84, Math.min(900, height)), + }; + } + }); + return valid; + } catch { + return {}; + } +} + function getProjectItemIconType(item: ProjectItem | undefined): string | undefined { if (!item || !('type' in item)) return undefined; if (item.type === 'model') { @@ -77,6 +198,31 @@ function getProjectItemIconType(item: ProjectItem | undefined): string | undefin return item.type; } +function getMediaBoardGroupName(folderId: string | null, folders: Array<{ id: string; name: string; parentId: string | null }>): string { + if (!folderId) return 'Root'; + const path: string[] = []; + let current = folders.find((folder) => folder.id === folderId); + while (current) { + path.unshift(current.name); + current = current.parentId ? folders.find((folder) => folder.id === current!.parentId) : undefined; + } + return path.length ? path.join(' / ') : 'Folder'; +} + +function getMediaBoardTypeLabel(item: MediaBoardItem): string { + if (item.type === 'composition') return 'Composition'; + if (item.type === 'gaussian-splat') return 'Splat'; + if (item.type === 'splat-effector') return 'Effector'; + if (isMediaBoardBackgroundItem(item)) return 'Background'; + if (item.type === 'solid') return 'Solid'; + if (item.type === 'model') return 'Model'; + return item.type.charAt(0).toUpperCase() + item.type.slice(1); +} + +function isMediaBoardBackgroundItem(item: MediaBoardItem): boolean { + return item.type === 'solid' && /^background\b/i.test(item.name.trim()); +} + export function MediaPanel() { // Reactive data - subscribe to specific values only const files = useMediaStore(state => state.files); @@ -118,6 +264,7 @@ export function MediaPanel() { showInExplorer, moveToFolder, createTextItem, + updateTextItem, getOrCreateTextFolder, removeTextItem, createSolidItem, @@ -138,10 +285,14 @@ export function MediaPanel() { const fileInputRef = useRef(null); const itemListRef = useRef(null); + const boardWrapperRef = useRef(null); + const boardCanvasRef = useRef(null); + const boardCanvasInnerRef = useRef(null); + const boardInteractionFrameRef = useRef(null); const [renamingId, setRenamingId] = useState(null); const [renameValue, setRenameValue] = useState(''); const renameTimerRef = useRef(null); - const [contextMenu, setContextMenu] = useState<{ x: number; y: number; itemId?: string } | null>(null); + const [contextMenu, setContextMenu] = useState(null); // Marquee selection state const [marquee, setMarquee] = useState<{ startX: number; startY: number; currentX: number; currentY: number } | null>(null); @@ -155,11 +306,14 @@ export function MediaPanel() { const [isExternalDragOver, setIsExternalDragOver] = useState(false); const [labelPickerItemId, setLabelPickerItemId] = useState(null); const [labelPickerPos, setLabelPickerPos] = useState<{ x: number; y: number } | null>(null); - const [viewMode, setViewMode] = useState<'list' | 'grid'>(() => { - return (localStorage.getItem('media-panel-view-mode') as 'list' | 'grid') || 'list'; - }); + const [viewMode, setViewMode] = useState(loadMediaPanelViewMode); // Grid view: current open folder (null = root) const [gridFolderId, setGridFolderId] = useState(null); + const [mediaBoardViewport, setMediaBoardViewport] = useState(loadMediaBoardViewport); + const [mediaBoardLayouts, setMediaBoardLayouts] = useState>(loadMediaBoardLayout); + const [mediaBoardMarquee, setMediaBoardMarquee] = useState(null); + const suppressMediaBoardContextMenuRef = useRef(false); + const suppressMediaBoardContextMenuTimerRef = useRef(null); // Column order state const [columnOrder, setColumnOrder] = useState(loadColumnOrder); @@ -175,6 +329,27 @@ export function MediaPanel() { localStorage.setItem(STORAGE_KEY, JSON.stringify(columnOrder)); }, [columnOrder]); + useEffect(() => { + localStorage.setItem(VIEW_MODE_STORAGE_KEY, viewMode); + }, [viewMode]); + + useEffect(() => { + localStorage.setItem(BOARD_VIEWPORT_STORAGE_KEY, JSON.stringify(mediaBoardViewport)); + }, [mediaBoardViewport]); + + useEffect(() => { + localStorage.setItem(BOARD_LAYOUT_STORAGE_KEY, JSON.stringify(mediaBoardLayouts)); + }, [mediaBoardLayouts]); + + useEffect(() => () => { + if (boardInteractionFrameRef.current !== null) { + window.cancelAnimationFrame(boardInteractionFrameRef.current); + } + if (suppressMediaBoardContextMenuTimerRef.current !== null) { + window.clearTimeout(suppressMediaBoardContextMenuTimerRef.current); + } + }, []); + // Column drag handlers const handleColumnDragStart = useCallback((e: React.DragEvent, columnId: ColumnId) => { e.stopPropagation(); @@ -469,8 +644,8 @@ export function MediaPanel() { // Handle double-click (open/expand) const handleItemDoubleClick = useCallback(async (item: ProjectItem) => { if ('isExpanded' in item) { - // It's a folder — in grid view navigate into it, in list view toggle expand - if (viewMode === 'grid') { + // Folders navigate in icon view and expand/collapse in the denser views. + if (viewMode === 'icons') { setGridFolderId(item.id); } else { toggleFolderExpanded(item.id); @@ -504,6 +679,33 @@ export function MediaPanel() { setContextMenu({ x: e.clientX, y: e.clientY, itemId }); }, [selectedIds, setSelection, addToSelection]); + const suppressNextMediaBoardContextMenu = useCallback(() => { + suppressMediaBoardContextMenuRef.current = true; + if (suppressMediaBoardContextMenuTimerRef.current !== null) { + window.clearTimeout(suppressMediaBoardContextMenuTimerRef.current); + } + suppressMediaBoardContextMenuTimerRef.current = window.setTimeout(() => { + suppressMediaBoardContextMenuRef.current = false; + suppressMediaBoardContextMenuTimerRef.current = null; + }, 600); + }, []); + + const consumeSuppressedMediaBoardContextMenu = useCallback(() => { + if (!suppressMediaBoardContextMenuRef.current) return false; + suppressMediaBoardContextMenuRef.current = false; + if (suppressMediaBoardContextMenuTimerRef.current !== null) { + window.clearTimeout(suppressMediaBoardContextMenuTimerRef.current); + suppressMediaBoardContextMenuTimerRef.current = null; + } + return true; + }, []); + + const handleMediaBoardWorkspaceContextMenu = useCallback((e: React.MouseEvent) => { + e.preventDefault(); + if (consumeSuppressedMediaBoardContextMenu()) return; + setContextMenu({ x: e.clientX, y: e.clientY, mode: 'board-workspace' }); + }, [consumeSuppressedMediaBoardContextMenu]); + const closeContextMenu = useCallback(() => { setContextMenu(null); }, []); @@ -580,10 +782,10 @@ export function MediaPanel() { closeContextMenu(); }, [selectedIds, files, compositions, folders, textItems, meshItems, cameraItems, splatEffectorItems, removeFile, removeComposition, removeFolder, removeTextItem, removeMeshItem, removeCameraItem, removeSplatEffectorItem, closeContextMenu]); - // Get the active parent folder (grid view: current open folder, list view: selected folder or null) + // Get the active parent folder (icons view: current open folder, classic/board view: selected folder or null) const getActiveParentId = useCallback((): string | null => { - if (viewMode === 'grid' && gridFolderId) return gridFolderId; - // In list view, if a single folder is selected, create inside it + if (viewMode === 'icons' && gridFolderId) return gridFolderId; + // In classic/board view, if a single folder is selected, create inside it if (selectedIds.length === 1) { const sel = folders.find(f => f.id === selectedIds[0]); if (sel) return sel.id; @@ -979,6 +1181,21 @@ export function MediaPanel() { }); const resizeRef = useRef<{ startX: number; startWidth: number } | null>(null); + useEffect(() => { + const handleProjectUiLoaded = () => { + setColumnOrder(loadColumnOrder()); + setViewMode(loadMediaPanelViewMode()); + setMediaBoardViewport(loadMediaBoardViewport()); + setMediaBoardLayouts(loadMediaBoardLayout()); + const storedNameWidth = localStorage.getItem('media-panel-name-width'); + setNameColumnWidth(storedNameWidth ? parseInt(storedNameWidth, 10) : 250); + setGridFolderId(null); + }; + + window.addEventListener(MEDIA_PANEL_PROJECT_UI_LOADED_EVENT, handleProjectUiLoaded); + return () => window.removeEventListener(MEDIA_PANEL_PROJECT_UI_LOADED_EVENT, handleProjectUiLoaded); + }, []); + // Save name column width useEffect(() => { localStorage.setItem('media-panel-name-width', String(nameColumnWidth)); @@ -1349,6 +1566,718 @@ export function MediaPanel() { splatEffectorItems.length ); + const mediaBoardItems = useMemo(() => ([ + ...files, + ...compositions, + ...textItems, + ...solidItems, + ...meshItems, + ...cameraItems, + ...splatEffectorItems, + ]), [files, compositions, textItems, solidItems, meshItems, cameraItems, splatEffectorItems]); + + const mediaBoardItemIds = useMemo(() => new Set(mediaBoardItems.map((item) => item.id)), [mediaBoardItems]); + + useEffect(() => { + setMediaBoardLayouts((current) => { + let changed = false; + const next: Record = {}; + Object.entries(current).forEach(([id, layout]) => { + if (mediaBoardItemIds.has(id)) { + next[id] = layout; + } else { + changed = true; + } + }); + return changed ? next : current; + }); + }, [mediaBoardItemIds]); + + const mediaBoardLayout = useMemo(() => { + const groupsByParent = new Map(); + groupsByParent.set(null, []); + folders.forEach((folder) => groupsByParent.set(folder.id, [])); + + mediaBoardItems.forEach((item) => { + const parentId = item.parentId ?? null; + if (!groupsByParent.has(parentId)) { + groupsByParent.set(parentId, []); + } + groupsByParent.get(parentId)!.push(item); + }); + + const orderedGroupIds: Array = [ + null, + ...folders + .map((folder) => folder.id) + .sort((leftId, rightId) => + getMediaBoardGroupName(leftId, folders).localeCompare(getMediaBoardGroupName(rightId, folders)) + ), + ].filter((groupId, index, ids) => ids.indexOf(groupId) === index); + + const visibleGroupIds = orderedGroupIds.filter((groupId) => { + const items = groupsByParent.get(groupId) ?? []; + return groupId === null || items.length > 0; + }); + + const groups: MediaBoardGroupLayout[] = []; + const placements: MediaBoardNodePlacement[] = []; + let rowY = 0; + let rowMaxHeight = 0; + let column = 0; + + visibleGroupIds.forEach((groupId) => { + const items = sortItems([...(groupsByParent.get(groupId) ?? [])]) as MediaBoardItem[]; + const rows = Math.max(1, Math.ceil(items.length / MEDIA_BOARD_COLUMNS_PER_GROUP)); + const height = MEDIA_BOARD_GROUP_HEADER_HEIGHT + + (MEDIA_BOARD_GROUP_PADDING * 2) + + (rows * MEDIA_BOARD_NODE_HEIGHT) + + ((rows - 1) * MEDIA_BOARD_NODE_GAP); + const x = column * (MEDIA_BOARD_GROUP_WIDTH + MEDIA_BOARD_GROUP_GAP); + const y = rowY; + const group: MediaBoardGroupLayout = { + id: groupId, + name: getMediaBoardGroupName(groupId, folders), + x, + y, + width: MEDIA_BOARD_GROUP_WIDTH, + height, + itemCount: items.length, + }; + groups.push(group); + + items.forEach((item, index) => { + const col = index % MEDIA_BOARD_COLUMNS_PER_GROUP; + const row = Math.floor(index / MEDIA_BOARD_COLUMNS_PER_GROUP); + const defaultLayout: MediaBoardNodeLayout = { + x: x + MEDIA_BOARD_GROUP_PADDING + (col * (MEDIA_BOARD_NODE_WIDTH + MEDIA_BOARD_NODE_GAP)), + y: y + MEDIA_BOARD_GROUP_HEADER_HEIGHT + MEDIA_BOARD_GROUP_PADDING + (row * (MEDIA_BOARD_NODE_HEIGHT + MEDIA_BOARD_NODE_GAP)), + width: MEDIA_BOARD_NODE_WIDTH, + height: MEDIA_BOARD_NODE_HEIGHT, + }; + placements.push({ + item, + defaultLayout, + groupId, + layout: mediaBoardLayouts[item.id] ?? defaultLayout, + }); + }); + + rowMaxHeight = Math.max(rowMaxHeight, height); + column += 1; + if (column >= 2) { + rowY += rowMaxHeight + MEDIA_BOARD_GROUP_GAP; + rowMaxHeight = 0; + column = 0; + } + }); + + return { groups, placements }; + }, [folders, mediaBoardItems, mediaBoardLayouts, sortItems]); + + const mediaBoardPlacementsById = useMemo(() => { + return new Map(mediaBoardLayout.placements.map((placement) => [placement.item.id, placement])); + }, [mediaBoardLayout.placements]); + + const screenToMediaBoard = useCallback((clientX: number, clientY: number) => { + const rect = boardCanvasRef.current?.getBoundingClientRect(); + if (!rect) return { x: 0, y: 0 }; + return { + x: (clientX - rect.left - mediaBoardViewport.panX) / mediaBoardViewport.zoom, + y: (clientY - rect.top - mediaBoardViewport.panY) / mediaBoardViewport.zoom, + }; + }, [mediaBoardViewport.panX, mediaBoardViewport.panY, mediaBoardViewport.zoom]); + + const getMediaBoardViewportCenter = useCallback(() => { + const rect = boardCanvasRef.current?.getBoundingClientRect(); + if (!rect) { + return screenToMediaBoard(window.innerWidth / 2, window.innerHeight / 2); + } + return screenToMediaBoard(rect.left + rect.width / 2, rect.top + rect.height / 2); + }, [screenToMediaBoard]); + + const placeMediaBoardItemAtCenter = useCallback((itemId: string, size: { width: number; height: number }) => { + const center = getMediaBoardViewportCenter(); + setMediaBoardLayouts((current) => ({ + ...current, + [itemId]: { + x: Math.round(center.x - size.width / 2), + y: Math.round(center.y - size.height / 2), + width: size.width, + height: size.height, + }, + })); + }, [getMediaBoardViewportCenter]); + + const openBoardAI = useCallback(() => { + useDockStore.getState().activatePanelType('ai-video'); + closeContextMenu(); + }, [closeContextMenu]); + + const handleNewBoardText = useCallback(() => { + const requestedText = window.prompt('Text eingeben', 'New Text'); + if (requestedText == null) return; + + const text = requestedText.trim() || 'Text'; + const itemId = createTextItem(text, getActiveParentId()); + updateTextItem(itemId, { name: text, text }); + placeMediaBoardItemAtCenter(itemId, MEDIA_BOARD_TEXT_LAYOUT); + setSelection([itemId]); + closeContextMenu(); + }, [closeContextMenu, createTextItem, getActiveParentId, placeMediaBoardItemAtCenter, setSelection, updateTextItem]); + + const handleNewBoardBackground = useCallback(() => { + const name = `Background ${solidItems.length + 1}`; + const itemId = createSolidItem(name, MEDIA_BOARD_BACKGROUND_COLOR, getActiveParentId()); + placeMediaBoardItemAtCenter(itemId, MEDIA_BOARD_BACKGROUND_LAYOUT); + setSelection([itemId]); + closeContextMenu(); + }, [closeContextMenu, createSolidItem, getActiveParentId, placeMediaBoardItemAtCenter, setSelection, solidItems.length]); + + const setMediaBoardPerformanceMode = useCallback((enabled: boolean) => { + boardWrapperRef.current?.classList.toggle('board-interacting', enabled); + }, []); + + const startMediaBoardPanGesture = useCallback((e: React.MouseEvent, options?: { clearSelectionOnTap?: boolean }) => { + e.preventDefault(); + closeContextMenu(); + + const startX = e.clientX; + const startY = e.clientY; + const startViewport = { ...mediaBoardViewport }; + let pendingViewport = startViewport; + let didPan = false; + + const schedulePreview = () => { + if (boardInteractionFrameRef.current !== null) return; + boardInteractionFrameRef.current = window.requestAnimationFrame(() => { + boardInteractionFrameRef.current = null; + const inner = boardCanvasInnerRef.current; + if (!inner) return; + inner.style.transform = `translate(${pendingViewport.panX}px, ${pendingViewport.panY}px) scale(${pendingViewport.zoom})`; + }); + }; + + const handleMouseMove = (moveEvent: MouseEvent) => { + const dx = moveEvent.clientX - startX; + const dy = moveEvent.clientY - startY; + const distance = Math.hypot(dx, dy); + if (!didPan && distance < MEDIA_BOARD_DRAG_START_DISTANCE) return; + + if (!didPan) { + didPan = true; + setMediaBoardPerformanceMode(true); + } + + pendingViewport = { + ...startViewport, + panX: startViewport.panX + dx, + panY: startViewport.panY + dy, + }; + schedulePreview(); + }; + + const handleMouseUp = () => { + if (boardInteractionFrameRef.current !== null) { + window.cancelAnimationFrame(boardInteractionFrameRef.current); + boardInteractionFrameRef.current = null; + } + setMediaBoardPerformanceMode(false); + + if (didPan) { + setMediaBoardViewport(pendingViewport); + } else if (options?.clearSelectionOnTap) { + setSelection([]); + } + + window.removeEventListener('mousemove', handleMouseMove); + window.removeEventListener('mouseup', handleMouseUp); + window.removeEventListener('blur', handleMouseUp); + }; + + window.addEventListener('mousemove', handleMouseMove); + window.addEventListener('mouseup', handleMouseUp); + window.addEventListener('blur', handleMouseUp); + }, [closeContextMenu, mediaBoardViewport, setMediaBoardPerformanceMode, setSelection]); + + const startMediaBoardMarqueeGesture = useCallback((e: React.MouseEvent, options?: { excludeBackgrounds?: boolean }) => { + const startPoint = screenToMediaBoard(e.clientX, e.clientY); + const startClientX = e.clientX; + const startClientY = e.clientY; + const initialSelection = e.ctrlKey || e.metaKey ? selectedIds : []; + const excludeBackgrounds = Boolean(options?.excludeBackgrounds); + let didSelect = false; + + const updateSelectionForRect = (rect: { left: number; right: number; top: number; bottom: number }) => { + const hitIds = mediaBoardLayout.placements + .filter(({ item, layout }) => { + if (excludeBackgrounds && isMediaBoardBackgroundItem(item)) return false; + const right = layout.x + layout.width; + const bottom = layout.y + layout.height; + return right > rect.left && layout.x < rect.right && bottom > rect.top && layout.y < rect.bottom; + }) + .map(({ item }) => item.id); + setSelection([...new Set([...initialSelection, ...hitIds])]); + }; + + const handleMouseMove = (moveEvent: MouseEvent) => { + const distance = Math.hypot(moveEvent.clientX - startClientX, moveEvent.clientY - startClientY); + if (!didSelect && distance < MEDIA_BOARD_DRAG_START_DISTANCE) return; + + didSelect = true; + closeContextMenu(); + const currentPoint = screenToMediaBoard(moveEvent.clientX, moveEvent.clientY); + const rect = { + left: Math.min(startPoint.x, currentPoint.x), + right: Math.max(startPoint.x, currentPoint.x), + top: Math.min(startPoint.y, currentPoint.y), + bottom: Math.max(startPoint.y, currentPoint.y), + }; + setMediaBoardMarquee({ + startX: startPoint.x, + startY: startPoint.y, + currentX: currentPoint.x, + currentY: currentPoint.y, + }); + updateSelectionForRect(rect); + }; + + const handleMouseUp = () => { + if (didSelect) { + suppressNextMediaBoardContextMenu(); + } + setMediaBoardMarquee(null); + window.removeEventListener('mousemove', handleMouseMove); + window.removeEventListener('mouseup', handleMouseUp); + window.removeEventListener('blur', handleMouseUp); + }; + + window.addEventListener('mousemove', handleMouseMove); + window.addEventListener('mouseup', handleMouseUp); + window.addEventListener('blur', handleMouseUp); + }, [closeContextMenu, mediaBoardLayout.placements, screenToMediaBoard, selectedIds, setSelection, suppressNextMediaBoardContextMenu]); + + const startMediaBoardNodeMoveGesture = useCallback((e: React.MouseEvent, item: MediaBoardItem) => { + const selectedMoveIds = selectedIds.includes(item.id) + ? selectedIds.filter((id) => mediaBoardItemIds.has(id)) + : [item.id]; + const startLayouts = selectedMoveIds.map((id) => { + const placement = mediaBoardPlacementsById.get(id); + return { + id, + layout: mediaBoardLayouts[id] ?? placement?.defaultLayout ?? placement?.layout, + }; + }).filter((entry): entry is { id: string; layout: MediaBoardNodeLayout } => !!entry.layout); + + if (startLayouts.length === 0) return; + + const startX = e.clientX; + const startY = e.clientY; + let didDrag = false; + let previewDx = 0; + let previewDy = 0; + + const clearPreview = () => { + startLayouts.forEach(({ id }) => { + const node = boardCanvasRef.current?.querySelector(`.media-board-node[data-item-id="${CSS.escape(id)}"]`); + if (!node) return; + node.style.transform = ''; + node.classList.remove('drag-preview'); + }); + }; + + const schedulePreview = () => { + if (boardInteractionFrameRef.current !== null) return; + boardInteractionFrameRef.current = window.requestAnimationFrame(() => { + boardInteractionFrameRef.current = null; + startLayouts.forEach(({ id }) => { + const node = boardCanvasRef.current?.querySelector(`.media-board-node[data-item-id="${CSS.escape(id)}"]`); + if (!node) return; + node.style.transform = `translate3d(${previewDx}px, ${previewDy}px, 0)`; + node.classList.add('drag-preview'); + }); + }); + }; + + const handleMouseMove = (moveEvent: MouseEvent) => { + const distance = Math.hypot(moveEvent.clientX - startX, moveEvent.clientY - startY); + if (!didDrag && distance < MEDIA_BOARD_DRAG_START_DISTANCE) return; + + if (!didDrag) { + didDrag = true; + suppressNextMediaBoardContextMenu(); + closeContextMenu(); + setMediaBoardPerformanceMode(true); + } + + previewDx = (moveEvent.clientX - startX) / mediaBoardViewport.zoom; + previewDy = (moveEvent.clientY - startY) / mediaBoardViewport.zoom; + schedulePreview(); + }; + + const handleMouseUp = () => { + if (boardInteractionFrameRef.current !== null) { + window.cancelAnimationFrame(boardInteractionFrameRef.current); + boardInteractionFrameRef.current = null; + } + clearPreview(); + setMediaBoardPerformanceMode(false); + + if (didDrag) { + setMediaBoardLayouts((current) => { + const next = { ...current }; + startLayouts.forEach(({ id, layout }) => { + next[id] = { + ...layout, + x: Math.round(layout.x + previewDx), + y: Math.round(layout.y + previewDy), + }; + }); + return next; + }); + } + + window.removeEventListener('mousemove', handleMouseMove); + window.removeEventListener('mouseup', handleMouseUp); + window.removeEventListener('blur', handleMouseUp); + }; + + window.addEventListener('mousemove', handleMouseMove); + window.addEventListener('mouseup', handleMouseUp); + window.addEventListener('blur', handleMouseUp); + }, [ + closeContextMenu, + mediaBoardItemIds, + mediaBoardLayouts, + mediaBoardPlacementsById, + mediaBoardViewport.zoom, + selectedIds, + setMediaBoardPerformanceMode, + suppressNextMediaBoardContextMenu, + ]); + + const handleMediaBoardWheel = useCallback((e: React.WheelEvent) => { + e.preventDefault(); + const rect = boardCanvasRef.current?.getBoundingClientRect(); + if (!rect) return; + + const zoomDelta = e.deltaY > 0 ? 0.9 : 1.1; + setMediaBoardViewport((current) => { + const nextZoom = Math.min( + MEDIA_BOARD_PAN_ZOOM_MAX, + Math.max(MEDIA_BOARD_PAN_ZOOM_MIN, current.zoom * zoomDelta), + ); + const cursorX = e.clientX - rect.left; + const cursorY = e.clientY - rect.top; + return { + zoom: nextZoom, + panX: cursorX - ((cursorX - current.panX) * (nextZoom / current.zoom)), + panY: cursorY - ((cursorY - current.panY) * (nextZoom / current.zoom)), + }; + }); + }, []); + + const handleMediaBoardMouseDown = useCallback((e: React.MouseEvent) => { + const target = e.target as HTMLElement; + if (target.closest('.media-board-node, button, input, .context-menu')) return; + + if (e.button === 2) { + startMediaBoardMarqueeGesture(e, { excludeBackgrounds: Boolean(e.ctrlKey || e.metaKey) }); + return; + } + + if (e.button !== 0 && e.button !== 1) return; + startMediaBoardPanGesture(e, { clearSelectionOnTap: e.button === 0 && !e.ctrlKey && !e.metaKey }); + }, [startMediaBoardMarqueeGesture, startMediaBoardPanGesture]); + + const handleMediaBoardNodeMouseDown = useCallback((e: React.MouseEvent, item: MediaBoardItem) => { + const target = e.target as HTMLElement; + if (target.closest('.media-board-node-timeline-drag, button, input')) return; + + if (e.button === 2) { + e.stopPropagation(); + if (e.ctrlKey || e.metaKey) { + startMediaBoardMarqueeGesture(e, { excludeBackgrounds: true }); + return; + } + if (!selectedIds.includes(item.id)) { + setSelection([item.id]); + } + startMediaBoardNodeMoveGesture(e, item); + return; + } + + if (e.button !== 0) return; + + e.stopPropagation(); + handleItemClick(item.id, e); + + if (!e.ctrlKey && !e.metaKey && !e.shiftKey) { + startMediaBoardPanGesture(e); + } + }, [ + handleItemClick, + setSelection, + selectedIds, + startMediaBoardMarqueeGesture, + startMediaBoardNodeMoveGesture, + startMediaBoardPanGesture, + ]); + + const handleMediaBoardDrop = useCallback(async (e: React.DragEvent) => { + e.preventDefault(); + e.stopPropagation(); + setIsExternalDragOver(false); + + if (e.dataTransfer.types.includes('application/x-media-panel-item')) { + const itemId = e.dataTransfer.getData('application/x-media-panel-item'); + if (itemId) { + const itemsToMove = selectedIds.includes(itemId) ? selectedIds : [itemId]; + moveToFolder(itemsToMove, null); + } + setDragOverFolderId(null); + setInternalDragId(null); + return; + } + + const point = screenToMediaBoard(e.clientX, e.clientY); + const targetGroup = mediaBoardLayout.groups.find((group) => ( + point.x >= group.x + && point.x <= group.x + group.width + && point.y >= group.y + && point.y <= group.y + group.height + )); + await handleExternalDropImport(e.dataTransfer, targetGroup?.id ?? null); + }, [handleExternalDropImport, mediaBoardLayout.groups, moveToFolder, screenToMediaBoard, selectedIds]); + + const handleMediaBoardGroupDrop = useCallback(async (e: React.DragEvent, groupId: string | null) => { + e.preventDefault(); + e.stopPropagation(); + + if (e.dataTransfer.types.includes('application/x-media-panel-item')) { + const itemId = e.dataTransfer.getData('application/x-media-panel-item'); + if (itemId) { + if (groupId) { + const draggedFolder = folders.find((folder) => folder.id === itemId); + if (draggedFolder) { + let parent = folders.find((folder) => folder.id === groupId); + while (parent) { + if (parent.id === itemId) { + setDragOverFolderId(null); + setInternalDragId(null); + return; + } + parent = parent.parentId ? folders.find((folder) => folder.id === parent!.parentId) : undefined; + } + } + } + const itemsToMove = selectedIds.includes(itemId) ? selectedIds : [itemId]; + moveToFolder(itemsToMove, groupId); + } + setDragOverFolderId(null); + setInternalDragId(null); + return; + } + + await handleExternalDropImport(e.dataTransfer, groupId); + setIsExternalDragOver(false); + }, [folders, handleExternalDropImport, moveToFolder, selectedIds]); + + const handleMediaBoardGroupDragOver = useCallback((e: React.DragEvent) => { + if (!e.dataTransfer.types.includes('application/x-media-panel-item') && !e.dataTransfer.types.includes('Files')) return; + e.preventDefault(); + e.stopPropagation(); + e.dataTransfer.dropEffect = e.dataTransfer.types.includes('application/x-media-panel-item') ? 'move' : 'copy'; + }, []); + + const resetMediaBoardLayout = useCallback(() => { + setMediaBoardLayouts({}); + setMediaBoardViewport(DEFAULT_BOARD_VIEWPORT); + }, []); + + const renderMediaBoardNode = (placement: MediaBoardNodePlacement) => { + const { item, layout } = placement; + const isSelected = selectedIds.includes(item.id); + const isMediaFile = isImportedMediaFileItem(item); + const mediaFile = isMediaFile ? item : null; + const isComp = item.type === 'composition'; + const comp = isComp ? (item as Composition) : null; + const isTextItem = item.type === 'text'; + const textItem = isTextItem ? (item as TextItem) : null; + const isSolidItem = item.type === 'solid'; + const solidItem = isSolidItem ? (item as SolidItem) : null; + const isBackgroundItem = isMediaBoardBackgroundItem(item); + const thumbUrl = mediaFile?.thumbnailUrl; + const duration = mediaFile?.duration || comp?.duration; + const importProgress = getItemImportProgress(item); + const labelHex = 'labelColor' in item ? getLabelHex(item.labelColor) : 'transparent'; + const title = buildGridTooltip(item, false, isComp); + const resolutionLabel = 'width' in item && 'height' in item && item.width && item.height + ? `${item.width}x${item.height}` + : comp + ? `${comp.width}x${comp.height}` + : null; + + return ( +
handleMediaBoardNodeMouseDown(e, item)} + onDoubleClick={() => { void handleItemDoubleClick(item); }} + onContextMenu={(e) => { + if (consumeSuppressedMediaBoardContextMenu()) { + e.preventDefault(); + return; + } + handleContextMenu(e, item.id); + }} + > +
+ {isBackgroundItem && solidItem ? ( +
+ ) : isSolidItem && solidItem ? ( +
+ ) : textItem ? ( +
+ {textItem.text} +
+ ) : thumbUrl ? ( + { void refreshFileUrls(mediaFile.id); } : undefined} + /> + ) : ( +
+ +
+ )} + {duration ? {formatDuration(duration)} : null} + {importProgress !== null ? {importProgress}% : null} + e.stopPropagation()} + onDragStart={(e) => handleDragStart(e, item)} + onDragEnd={handleDragEnd} + > + + +
+
+
{item.name}
+
+ {getMediaBoardTypeLabel(item)} + {resolutionLabel ? {resolutionLabel} : null} + {mediaFile?.codec ? {mediaFile.codec} : null} +
+
+
+ ); + }; + + const renderMediaBoardView = () => ( +
+
+
+ Board + {mediaBoardItems.length} assets in {mediaBoardLayout.groups.length} groups +
+
+ + + + +
+
+
{ + const target = e.target as HTMLElement; + if (!target.closest('.media-board-node')) handleMediaBoardWorkspaceContextMenu(e); + }} + onDragOver={(e) => { + if (e.dataTransfer.types.includes('Files')) { + setIsExternalDragOver(true); + } + e.preventDefault(); + e.dataTransfer.dropEffect = e.dataTransfer.types.includes('application/x-media-panel-item') ? 'move' : 'copy'; + }} + onDrop={handleMediaBoardDrop} + > +
+ {mediaBoardLayout.groups.map((group) => ( +
handleMediaBoardGroupDrop(e, group.id)} + > +
+ {group.name} + {group.itemCount} +
+
+ ))} + {mediaBoardLayout.placements.map(renderMediaBoardNode)} + {mediaBoardMarquee && (() => { + const left = Math.min(mediaBoardMarquee.startX, mediaBoardMarquee.currentX); + const top = Math.min(mediaBoardMarquee.startY, mediaBoardMarquee.currentY); + const width = Math.abs(mediaBoardMarquee.currentX - mediaBoardMarquee.startX); + const height = Math.abs(mediaBoardMarquee.currentY - mediaBoardMarquee.startY); + if (width < 2 && height < 2) return null; + return ( +
+ ); + })()} +
+
+
+ ); + // Grid view: items for current folder + breadcrumb path const gridItems = sortItems(getItemsByFolder(gridFolderId)); const gridBreadcrumb: Array<{ id: string | null; name: string }> = []; @@ -1384,21 +2313,38 @@ export function MediaPanel() { Project {totalItems} items
- +
+ + + +
{filesNeedReload && ( -
)} - - {sceneNavFpsMode - ? 'Click preview, hold LMB to look, WASD/QE move, MMB/RMB/Shift+LMB pan, wheel speed while moving/looking, wheel zoom otherwise. Dist = orbit distance.' - : 'Click preview, then WASD move, Q/E up-down, LMB orbit, MMB/RMB/Shift+LMB pan, wheel zoom. Dist = orbit distance.'} -
)} {!isCameraClip && ( -
+
{isLocked3D ? ( 3D @@ -243,7 +251,7 @@ export function TransformTab({ clipId, transform, speed = 1, is3D = false }: Tra
)} {supportsThreeDEffectorToggle && ( -
+
{canToggleThreeDEffectors && (