diff --git a/AGENTS.md b/AGENTS.md index 1715a6ae0..e10f49487 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -38,7 +38,28 @@ Die alten Claude-Commands unter `C:\Users\admin\.claude\commands\` sollen in Cod Wenn ein Skill nicht verfuegbar ist, direkt ueber lokale Skripte, MCP-Tools, HTTP-Bridge oder API arbeiten. Nicht an Claude-Command-Dateien haengen bleiben. -## 0.2 MasterSelects Debug Bridge +## 0.2 gstack Integration + +`gstack` ist fuer strukturierte Planung, Reviews, Browser-QA und Security-Checks verfuegbar. Global installieren, nicht ins Repo vendorisieren. + +Installationspfade: + +- Codex: `git clone --single-branch --depth 1 https://github.com/garrytan/gstack.git ~/gstack && cd ~/gstack && ./setup --host codex` +- Claude Code: `git clone --single-branch --depth 1 https://github.com/garrytan/gstack.git ~/.claude/skills/gstack && cd ~/.claude/skills/gstack && ./setup --team` +- Windows: Git Bash oder WSL verwenden; `bun` und `node` muessen installiert sein +- Nach der Codex-Installation Codex neu starten, damit neue Skills geladen werden + +Einsatzregeln: + +- `masterselects` bleibt erste Wahl fuer Timeline-, Preview-, Clip- und Debug-Bridge-Automation in der lokalen App +- `gstack-office-hours`, `gstack-autoplan`, `gstack-plan-eng-review`, `gstack-plan-design-review` und `gstack-plan-devex-review` fuer Discovery, Scope und Plan-Qualitaet +- `gstack-review` fuer unabhaengige Code-Reviews +- `gstack-investigate` fuer Root-Cause-Debugging statt Trial-and-Error-Fixes +- `gstack-cso` fuer Security-Reviews +- `gstack-qa`, `gstack-qa-only`, `gstack-browse`, `gstack-open-gstack-browser` und `gstack-setup-browser-cookies` fuer Browser-QA, Repros und auth-geschuetzte Flows +- `gstack-upgrade` verwenden, statt eine Repo-lokale gstack-Kopie zu pflegen + +## 0.3 MasterSelects Debug Bridge Fuer App-Debugging existieren lokale AI-Tools hinter `POST http://localhost:5173/api/ai-tools`. Voraussetzung: Dev-Server laeuft und die App ist im Browser geoeffnet. diff --git a/CLAUDE.md b/CLAUDE.md index ee13447af..574279df6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -30,6 +30,30 @@ Die `/masterselects` Skill stellt 4 Debug-Tools bereit, die über den HTTP Bridg --- +## 0.2 gstack Integration + +`gstack` ist fuer strukturierte Planung, Reviews, Browser-QA und Security-Checks verfuegbar. Global installieren, nicht ins Repo vendorisieren. + +Installationspfade: + +- Codex: `git clone --single-branch --depth 1 https://github.com/garrytan/gstack.git ~/gstack && cd ~/gstack && ./setup --host codex` +- Claude Code: `git clone --single-branch --depth 1 https://github.com/garrytan/gstack.git ~/.claude/skills/gstack && cd ~/.claude/skills/gstack && ./setup --team` +- Windows: Git Bash oder WSL verwenden; `bun` und `node` muessen installiert sein +- Nach der Codex-Installation Codex neu starten, damit neue Skills geladen werden + +Einsatzregeln: + +- `/masterselects` bleibt erste Wahl fuer Timeline-, Preview-, Clip- und Debug-Bridge-Automation in der lokalen App +- `gstack-office-hours`, `gstack-autoplan`, `gstack-plan-eng-review`, `gstack-plan-design-review` und `gstack-plan-devex-review` fuer Discovery, Scope und Plan-Qualitaet in Codex +- `/office-hours`, `/autoplan`, `/plan-eng-review`, `/plan-design-review` und `/plan-devex-review` fuer denselben Workflow in Claude Code +- `gstack-review` oder `/review` fuer unabhaengige Code-Reviews +- `gstack-investigate` oder `/investigate` fuer Root-Cause-Debugging statt Trial-and-Error-Fixes +- `gstack-cso` oder `/cso` fuer Security-Reviews +- `gstack-qa`, `gstack-qa-only`, `gstack-browse`, `gstack-open-gstack-browser`, `gstack-setup-browser-cookies` oder die entsprechenden Slash-Commands fuer Browser-QA, Repros und auth-geschuetzte Flows +- `gstack-upgrade` oder `/gstack-upgrade` verwenden, statt eine Repo-lokale gstack-Kopie zu pflegen + +--- + ## 1. Workflow (WICHTIG!) ### Branch-Regeln diff --git a/docs/Features/3D-Layers.md b/docs/Features/3D-Layers.md index 79115c6a6..815870d2c 100644 --- a/docs/Features/3D-Layers.md +++ b/docs/Features/3D-Layers.md @@ -13,10 +13,12 @@ Legacy gaussian-avatar support still exists in code for migration and old projec |---|---|---| | Per-layer 3D toggle | Stable | Any normal video/image layer can be switched between 2D and 3D. | | OBJ / glTF / GLB / FBX model import | Stable | Model clips are always 3D and render through Three.js. | +| Numbered GLB sequences | Stable | Numbered `.glb` frames import as a single 30fps model-sequence clip. | | Primitive mesh clips | Stable | Cube, sphere, plane, cylinder, torus, cone, and 3D text are created from the Media Panel. | | Scene camera clips | Stable | Timeline camera clips control the shared Three.js scene. | | Gaussian splat clips | Stable but specialized | Default route is shared-scene Three.js; native WebGPU rendering is optional. | -| Splat effector clips | Stable but specialized | They deform Three.js splats live at playback time. | +| Numbered PLY / SPLAT sequences | Stable | Numbered `.ply` / `.splat` frames import as one shared-scene gaussian-splat sequence clip. | +| 3D effector clips | Stable but specialized | They influence shared-scene 3D layers live at playback time. | | Gaussian avatar import | Legacy only | Import is blocked; existing projects may still expose blendshape editing. | | Temporal / particle splat settings | Experimental | Wired in the engine/export path, but not yet exposed as a dedicated properties tab. | @@ -34,7 +36,7 @@ Legacy gaussian-avatar support still exists in code for migration and old projec Three.js is used as the shared 3D scene for classic 3D layers and for the default gaussian-splat route. The native gaussian-splat renderer is a separate WebGPU path, enabled by the clip-level `useNativeRenderer` setting. -Camera clips and splat effectors only affect the shared Three.js scene. They do not drive the native gaussian-splat renderer. +Camera clips and 3D effectors only affect the shared Three.js scene. They do not drive the native gaussian-splat renderer. ## Stable 3D Features @@ -51,6 +53,8 @@ Camera clips and splat effectors only affect the shared Three.js scene. They do - Models are auto-centered and normalized to fit the viewport. - Default lighting is Ambient plus Directional lighting. - The Transform panel exposes a wireframe debug toggle for model clips. +- Numbered `.glb` files like `frame000000.glb`, `frame000001.glb`, `frame000002.glb` are grouped into one model-sequence asset during import. +- GLB sequences currently default to 30fps and use frame-based playback through the existing model clip path. ### Primitive Meshes and 3D Text @@ -90,11 +94,14 @@ The Transform tab also turns into camera-orbit controls for the active scene cam Gaussian splat clips are imported from `.ply` and `.splat` files. - Clips are created as `is3D: true`. -- The clip-level render tab exposes `useNativeRenderer`, `maxSplats`, `sortFrequency`, `splatScale`, `nearPlane`, and `farPlane`. +- The clip-level render tab exposes `useNativeRenderer`, `maxSplats`, `sortFrequency`, `splatScale`, `orientationPreset`, `nearPlane`, and `farPlane`. - The default renderer is the shared Three.js scene path. - Native WebGPU rendering is optional and off by default. -- The shared-scene route participates in scene cameras and splat effectors. +- Numbered `.ply` or `.splat` files like `scan000000.ply`, `scan000001.ply`, `scan000002.ply` are grouped into one gaussian-splat sequence asset during batch import. +- Gaussian-splat sequences currently stay on the shared Three.js scene path even if a clip was previously set to native render. +- The shared-scene route participates in scene cameras and 3D effectors. - The native route uses its own camera-style navigation controls in the Transform tab. +- A per-clip `3D Effector` toggle in the Transform tab lets you opt shared-scene splat/model layers in or out. Some gaussian-splat settings exist in the data model and export pipeline but are not yet surfaced as a full dedicated UI: @@ -104,16 +111,20 @@ Some gaussian-splat settings exist in the data model and export pipeline but are Those are wired through the renderer and export code, but they should still be treated as in-progress surface area. -## Splat Effectors +## 3D Effectors -Splat effector clips are timeline clips that only affect Three.js splats. +3D effector clips are non-rendering timeline clips that influence shared-scene 3D layers. - Modes: `repel`, `attract`, `swirl`, and `noise` - Controls: strength, falloff, speed, and seed - Transform scale acts as the effector radius - They do not render visible content on their own +- Shared-scene gaussian splats use the direct splat deformation path +- Shared-scene gaussian-splat sequences also use that direct splat deformation path frame by frame +- Models, primitive meshes, and 3D text receive object-level motion +- Native gaussian splats ignore 3D effectors because they do not run in the shared Three.js scene -This is a good example of a specialized 3D feature that is stable in the UI, but limited to the shared-scene splat path. +This is a shared-scene 3D feature that is stable in the UI, but it does not extend into the native gaussian-splat renderer. ## Legacy Gaussian Avatars @@ -133,7 +144,7 @@ If you see avatar-specific code paths in the renderer or AI tooling, treat them | Regular 2D clip | Transform, Effects, Masks, Transcript, Analysis | | Camera clip | Transform, Camera | | Gaussian splat clip | Transform, Gaussian, Effects, Masks, Transcript, Analysis | -| Splat effector clip | Transform, Effector, Effects, Masks, Transcript, Analysis | +| 3D effector clip | Transform, Effector, Effects, Masks, Transcript, Analysis | | 3D text clip | 3D Text, Transform, Effects, Masks | | Legacy gaussian avatar clip | Transform, Blendshapes | @@ -164,11 +175,11 @@ The Transform tab is context-sensitive: | `src/stores/timeline/meshClipSlice.ts` | Primitive mesh and 3D text clip creation | | `src/stores/timeline/cameraClipSlice.ts` | Timeline camera clip creation | | `src/stores/timeline/clip/addGaussianSplatClip.ts` | Gaussian splat clip creation | -| `src/stores/timeline/splatEffectorClipSlice.ts` | Splat effector clip creation | +| `src/stores/timeline/splatEffectorClipSlice.ts` | 3D effector clip creation | | `src/components/panels/properties/TransformTab.tsx` | Context-sensitive 3D transform and camera controls | | `src/components/panels/properties/GaussianSplatTab.tsx` | Gaussian splat render settings tab | | `src/components/panels/properties/CameraTab.tsx` | Scene camera settings tab | -| `src/components/panels/properties/SplatEffectorTab.tsx` | Splat effector settings tab | +| `src/components/panels/properties/SplatEffectorTab.tsx` | 3D effector settings tab | | `src/components/panels/properties/BlendshapesTab.tsx` | Legacy gaussian-avatar blendshapes tab | | `src/engine/featureFlags.ts` | 3D feature flags | @@ -178,10 +189,10 @@ The Transform tab is context-sensitive: |---|---|---| | `.obj` | Supported | Imported as a Three.js model clip. | | `.gltf` | Supported | Imported as a Three.js model clip. | -| `.glb` | Supported | Imported as a Three.js model clip. | +| `.glb` | Supported | Imported as a Three.js model clip. Numbered `.glb` frames are grouped into a model-sequence clip. | | `.fbx` | Supported | Imported as a Three.js model clip. | -| `.ply` | Supported | Gaussian splat import. | -| `.splat` | Supported | Gaussian splat import. | +| `.ply` | Supported | Gaussian splat import. Numbered `.ply` frames are grouped into a shared-scene sequence clip. | +| `.splat` | Supported | Gaussian splat import. Numbered `.splat` frames are grouped into a shared-scene sequence clip. | | `.ksplat` | Not yet supported | Parser stubs exist, but the file is rejected today. | | `.gsplat-zip` | Not yet supported | Parser stubs exist, but the file is rejected today. | | Gaussian avatar `.zip` | Legacy only | Import is blocked in the current product surface. | diff --git a/docs/Features/AI-Integration.md b/docs/Features/AI-Integration.md index 5fb0cfa4a..eb65137f5 100644 --- a/docs/Features/AI-Integration.md +++ b/docs/Features/AI-Integration.md @@ -29,29 +29,40 @@ GPT-powered editing with 79 exported tools across 15 exported definition groups, ### Features - Interactive chat interface -- Model selection dropdown +- Model selection badge with popover +- Shared hosted-credit pricing per model - Conversation history - Clear chat button - Auto-scrolling - Tool execution indicators +- Styled approval cards for mutating/sensitive tool calls +- Compact control pills below the prompt box +- Optional per-panel approval bypass toggle below the send button +- Thinking-effort selector for supported OpenAI chat models ### Available Models ``` -GPT-5.2, GPT-5.2 Pro -GPT-5.1, GPT-5.1 Codex, GPT-5.1 Codex Mini +GPT-5.4, GPT-5.4 Mini, GPT-5.4 Nano +GPT-5.3 Chat, GPT-5.3 Codex +GPT-5.2, GPT-5.2 Codex +GPT-5.1, GPT-5.1 Codex Mini GPT-5, GPT-5 Mini, GPT-5 Nano GPT-4.1, GPT-4.1 Mini, GPT-4.1 Nano GPT-4o, GPT-4o Mini -o3, o4-mini, o3-pro (reasoning) +o3, o4-mini, o3-mini ``` Default model: `gpt-5.1` +The dropdown is sourced from `src/shared/openAiModelCatalog.ts`. That same catalog also drives hosted chat billing in `functions/lib/modelPricing.ts`, so the UI label and the charged credits stay in sync. +The same catalog also marks which models support OpenAI reasoning effort controls, so the chat panel only shows the Thinking control when the selected model can use it. + ### Editor Mode -When enabled: +The AI chat panel always runs with editor tools enabled: - Includes timeline context in prompts - Uses the exported AI tool catalog from `src/services/aiTools/definitions` - The chat UI applies its own approval gate before calling mutating or sensitive tools +- Users can temporarily bypass those confirmations from the panel - AI can manipulate timeline directly The current tool surface is 79 exported tool definitions across 15 exported definition groups. Two dispatch gaps remain in the shared registry: @@ -92,6 +103,7 @@ That console surface is dev-only. The Vite dev bridge and the Native Helper HTTP - Active IN / OUT / REF assignments appear as removable color badges around the prompt box - Hovering a prompt-box badge strongly emphasizes the linked board node for as long as the badge is hovered - Nano Banana 2 accepts up to 14 ordered reference images; the composer labels them as `REF 1`, `REF 2`, ... so prompts can refer to them explicitly +- FlashBoard's Kie.ai catalog includes Kling 3.0, Seedance 2.0, and Nano Banana 2; Seedance 2.0 currently ships as a Board-only Kie option with text-to-video and image-to-video support - IN / OUT / REF outlines scale with zoom and use a stronger glow so references stay readable while navigating the board ### Current Backends @@ -100,7 +112,7 @@ The current AI Video stack is no longer best described as "PiAPI as one unified | Backend | Where it is used | Notes | |---------|------------------|-------| -| `Kie.ai` | Classic generator and FlashBoard | Current provider list comes from `getKieAiProviders()`; user-supplied key in Settings | +| `Kie.ai` | Classic generator and FlashBoard | Classic provider list comes from `getKieAiProviders()`; FlashBoard extends the Kie catalog with Seedance 2.0; user-supplied key in Settings | | `MasterSelects Cloud` | Classic generator and FlashBoard when hosted access is available | Hosted credits/account flow; board mode resolves to hosted Kling when no local Kie key is present | | `PiAPI` | Legacy compatibility and some catalog/pricing metadata | Still present in older history/key migration paths and FlashBoard pricing/catalog helpers, but not the primary runtime path the current panel describes | diff --git a/docs/Features/FlashBoard.md b/docs/Features/FlashBoard.md index f91cee2a5..98263ea35 100644 --- a/docs/Features/FlashBoard.md +++ b/docs/Features/FlashBoard.md @@ -13,7 +13,7 @@ FlashBoard is the AI canvas workspace behind the AI Video panel's Board mode. It FlashBoard is not a separate model backend. It is a workspace layer on top of the existing AI services: - `piapi` for the PiAPI catalog -- `kieai` for Kie.ai Kling 3.0 and Nano Banana 2 +- `kieai` for Kie.ai Kling 3.0, Seedance 2.0, and Nano Banana 2 - `cloud` for hosted Kling 3.0 and hosted Nano Banana 2 The AI Video panel switches into FlashBoard when the user selects Board mode. If the user has no Kie.ai key and is signed in to MasterSelects Cloud, the board uses the hosted cloud service scope. Otherwise it stays on Kie.ai. @@ -68,10 +68,13 @@ The board uses the shared catalog from `FlashBoardModelCatalog`: - PiAPI video providers from the shared PiAPI catalog - Kie.ai Kling 3.0 video +- Kie.ai Seedance 2.0 video - Kie.ai Nano Banana 2 image generation - Cloud Kling 3.0 video - Cloud Nano Banana 2 image generation +Seedance 2.0 is currently exposed only in FlashBoard on the Kie.ai path. The existing Classic AI Video flow remains narrower and continues to focus on the classic Kie provider list. + The classic AI Video flow is narrower: it currently exposes only the Kie.ai Kling 3.0 provider list, while FlashBoard exposes the richer catalog. --- diff --git a/docs/Features/Media-Panel.md b/docs/Features/Media-Panel.md index ef0eed5d5..7c6ccdb3d 100644 --- a/docs/Features/Media-Panel.md +++ b/docs/Features/Media-Panel.md @@ -39,6 +39,9 @@ The panel also accepts a few specialized asset types that flow into the timeline - `model` files: OBJ, glTF/GLB, FBX - `gaussian-splat` files: PLY, SPLAT +Numbered `.glb` files with a shared prefix are detected as a sequence during batch import. A folder containing `frame000000.glb`, `frame000001.glb`, `frame000002.glb` and so on will appear as one model asset instead of many separate files. +Numbered `.ply` or `.splat` files with a shared prefix are also detected as a sequence during batch import. A folder containing `scan000000.ply`, `scan000001.ply`, `scan000002.ply` and so on will appear as one gaussian-splat asset instead of many separate files. + Lottie imports are treated as first-class media items. `.json` files are only accepted when their contents actually match Lottie structure, so arbitrary JSON data is not misclassified as animation. ### Import Methods @@ -54,7 +57,7 @@ Click the **+ Add** button for creating new items: - **3D Text** - New 3D text mesh item - **Solid** - New solid color item (placed in auto-created "Solids" folder) - **Camera** - New camera item -- **Splat Effector** - New splat-effector item +- **3D Effector** - New shared-scene 3D effector item - **Mesh** ▶ - Submenu with 3D primitive meshes (placed in auto-created "Meshes" folder): - Cube, Sphere, Plane, Cylinder, Torus, Cone - Creates a `MeshItem` which can be dragged to the timeline as a 3D clip @@ -75,11 +78,13 @@ Imports use a two-phase approach: 1. **Phase 1 (instant):** A placeholder entry appears immediately in the panel with `isImporting: true`, showing file name and size 2. **Phase 2 (background):** Full processing runs in the background: - - Media info extraction (dimensions, duration, FPS, codec, bitrate, audio detection) - - Thumbnail generation (for video and image files) - - File hash calculation (for deduplication and proxy matching) - - Copy to project RAW folder when `copyMediaToProject` is enabled, or when the import is forced - - Existing proxy detection (by file hash) + - Media info extraction (dimensions, duration, FPS, codec, bitrate, audio detection) + - Thumbnail generation (for video and image files) + - File hash calculation (for deduplication and proxy matching) + - Copy to project RAW folder when `copyMediaToProject` is enabled, or when the import is forced + - Numbered GLB sequence grouping into a single 30fps model-sequence asset + - Numbered PLY/SPLAT sequence grouping into a single 30fps gaussian-splat sequence asset + - Existing proxy detection (by file hash) **Deduplication:** Files with matching name + size are automatically skipped. @@ -448,6 +453,7 @@ interface MediaFile { - Uses actual media duration - Audio-only files restricted to audio tracks - Files still importing or missing cannot be dragged to timeline +- Numbered gaussian-splat sequences always stay on the shared Three.js 3D renderer path after drop - Compositions cannot be dragged into themselves (active comp check) - Mesh items create 3D clips with `is3D: true` and `meshType` (rendered via Three.js) diff --git a/docs/Features/Timeline.md b/docs/Features/Timeline.md index b717da9d5..f87e4cb32 100644 --- a/docs/Features/Timeline.md +++ b/docs/Features/Timeline.md @@ -2,14 +2,14 @@ [<- Back to Index](./README.md) -The Timeline is the core editing interface for multi-track editing. It now covers video, audio, image, Lottie, text, solid, mesh, composition, camera, and splat-effector clips, with keyframe lanes, transitions, multicam grouping, pick-whip parenting, and slot-grid playback. +The Timeline is the core editing interface for multi-track editing. It now covers video, audio, image, Lottie, text, solid, mesh, composition, camera, and 3D-effector clips, with keyframe lanes, transitions, multicam grouping, pick-whip parenting, and slot-grid playback. --- ## Track Types ### Video Tracks -- Hold video, image, Lottie, text, solid, mesh, composition, camera, and splat-effector clips. +- Hold video, image, Lottie, text, solid, mesh, composition, camera, and 3D-effector clips. - Higher tracks render on top of lower tracks. - Expanded tracks can show keyframe property rows and curve editors. - Default layout starts with `Video 2` above `Video 1`. @@ -69,8 +69,8 @@ getTrackChildren() // Query child tracks - Nested compositions can be dropped from the media panel. - Double-click enters the nested comp for editing. -### Camera and Splat Effector -- Camera clips and splat-effector clips are first-class clip types in the store and copy/paste flow. +### Camera and 3D Effector +- Camera clips and 3D-effector clips are first-class clip types in the store and copy/paste flow. - Camera/native-gaussian clips expose camera-oriented property labels in the keyframe UI. ### YouTube Download diff --git a/functions/api/ai/chat.ts b/functions/api/ai/chat.ts index 314f7d6be..d6c7a3095 100644 --- a/functions/api/ai/chat.ts +++ b/functions/api/ai/chat.ts @@ -22,6 +22,7 @@ interface HostedChatRouteBody { idempotencyKey?: string; messages?: unknown; model?: string; + reasoning_effort?: string; stream?: boolean; } @@ -249,6 +250,7 @@ export const onRequest: AppRouteHandler = async (context: AppContext): Promise = { - // --- Tier: low (1 credit) --- - 'gpt-4.1-nano': { creditCost: 1, tier: 'low' }, - 'gpt-4.1-mini': { creditCost: 1, tier: 'low' }, - 'gpt-4o-mini': { creditCost: 1, tier: 'low' }, - 'gpt-5-nano': { creditCost: 1, tier: 'low' }, - 'gpt-5-mini': { creditCost: 1, tier: 'low' }, - 'gpt-5.1-codex-mini': { creditCost: 1, tier: 'low' }, - - // --- Tier: mid (3 credits) --- - 'o4-mini': { creditCost: 3, tier: 'mid' }, - 'o3-mini': { creditCost: 3, tier: 'mid' }, - - // --- Tier: high (5 credits) --- - 'gpt-4o': { creditCost: 5, tier: 'high' }, - 'gpt-4.1': { creditCost: 5, tier: 'high' }, - 'gpt-5': { creditCost: 5, tier: 'high' }, - 'gpt-5.1': { creditCost: 5, tier: 'high' }, - 'gpt-5.1-codex': { creditCost: 5, tier: 'high' }, - 'o3': { creditCost: 5, tier: 'high' }, - - // --- Tier: premium (8-50 credits) --- - 'gpt-5.2': { creditCost: 8, tier: 'premium' }, - 'gpt-5.2-pro': { creditCost: 10, tier: 'premium' }, - 'o3-pro': { creditCost: 50, tier: 'premium' }, -}; - -/** Default cost for unknown models */ -const DEFAULT_CREDIT_COST: ModelPricingEntry = { creditCost: 5, tier: 'high' }; - -/** Look up credit cost for a model. Unknown models default to 5 (high tier). */ +/** Look up credit cost for a model. Unknown models use the shared default. */ export function getModelCreditCost(model: string): number { - return (MODEL_PRICING[model] ?? DEFAULT_CREDIT_COST).creditCost; + return getOpenAiModelPricing(model).creditCost; } /** Look up full pricing entry for a model. */ export function getModelPricing(model: string): ModelPricingEntry { - return MODEL_PRICING[model] ?? DEFAULT_CREDIT_COST; + return getOpenAiModelPricing(model); } /** Get all known model pricing entries (for capabilities/UI). */ export function getAllModelPricing(): Record { - return { ...MODEL_PRICING }; + return Object.fromEntries( + OPENAI_MODEL_CATALOG.map((entry) => [ + entry.id, + { + creditCost: entry.creditCost, + tier: entry.tier, + }, + ]), + ); } + +export { DEFAULT_OPENAI_MODEL_PRICING }; diff --git a/functions/lib/providers/openai.ts b/functions/lib/providers/openai.ts index 6df92bd3b..c181523c3 100644 --- a/functions/lib/providers/openai.ts +++ b/functions/lib/providers/openai.ts @@ -23,6 +23,7 @@ export interface HostedChatRequest { max_tokens?: number; messages: HostedChatMessage[]; model: string; + reasoning_effort?: 'none' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh'; response_format?: Record; stream?: boolean; tool_choice?: unknown; @@ -38,6 +39,15 @@ export interface HostedChatCapabilities { streamingSupported: false; } +const SUPPORTED_REASONING_EFFORTS = new Set>([ + 'none', + 'minimal', + 'low', + 'medium', + 'high', + 'xhigh', +]); + function isRecord(value: unknown): value is Record { return typeof value === 'object' && value !== null; } @@ -132,12 +142,19 @@ export function normalizeHostedChatRequest(body: unknown): HostedChatRequest | n } const model = typeof body.model === 'string' && body.model.trim() ? body.model.trim() : 'gpt-4.1-mini'; + const reasoningEffortCandidate = + typeof body.reasoning_effort === 'string' + ? body.reasoning_effort.trim() as NonNullable + : undefined; return { max_completion_tokens: normalizeNumericValue(body.max_completion_tokens), max_tokens: normalizeNumericValue(body.max_tokens), messages, model, + reasoning_effort: reasoningEffortCandidate && SUPPORTED_REASONING_EFFORTS.has(reasoningEffortCandidate) + ? reasoningEffortCandidate + : undefined, response_format: isRecord(body.response_format) ? body.response_format : undefined, stream: body.stream === true, tool_choice: body.tool_choice, @@ -153,6 +170,7 @@ export async function runHostedChatCompletion(env: Env, request: HostedChatReque max_tokens: request.max_tokens, messages: request.messages, model: request.model, + reasoning_effort: request.reasoning_effort, response_format: request.response_format, stream: false, tool_choice: request.tool_choice, diff --git a/plan.md b/plan.md new file mode 100644 index 000000000..0cd58aa6b --- /dev/null +++ b/plan.md @@ -0,0 +1,359 @@ +# Issue 77 - Lottie / Rive Implementation Plan + +Date: 2026-04-15 +Status: Draft +Branch: `77-lottie-rive-animation-files-on-timeline` + +## Mission + +Add first-class timeline support for Lottie and prepare the repo for Rive without building a second render pipeline. + +The first shippable milestone is: + +- Lottie `.lottie` and Lottie JSON import +- Lottie clips on video tracks +- deterministic preview scrubbing +- deterministic nested composition preview +- deterministic export +- project save/load and relink +- media panel and timeline UX + +Rive is phase 2 on the same canvas-backed architecture. Do not let Rive block Lottie MVP. + +## Current Codebase Reality + +### Type gates + +- `src/stores/mediaStore/types.ts` does not know `lottie` or `rive`. +- `src/types/index.ts` does not know `lottie` or `rive` as clip source types. +- `src/services/project/types/media.types.ts` still restricts project media to `video | audio | image`. +- `src/services/project/types/composition.types.ts` and `src/types/index.ts` serialize explicit `sourceType` unions and must be updated. + +### Import and clip creation gates + +- `src/stores/timeline/helpers/mediaTypeHelpers.ts` only detects `video | audio | image | model | gaussian-splat | unknown`. +- `src/components/timeline/utils/fileTypeHelpers.ts` and `src/components/panels/media/dropImport.ts` only allow the currently known media types. +- `src/stores/mediaStore/helpers/importPipeline.ts` only extracts metadata and thumbnails for `video | audio | image`. +- `src/stores/mediaStore/slices/fileImportSlice.ts` only has placeholder/import flows for current media types plus gaussian splat. +- `src/stores/timeline/clipSlice.ts` only routes `video`, `audio`, `image`, `model`, and `gaussian-splat`. + +### Why this is feasible + +- Text and solid clips already prove that canvas-backed sources fit the render stack. +- `src/services/layerBuilder/LayerBuilderService.ts` already builds layers from `source.textCanvas`. +- `src/engine/render/LayerCollector.ts` already uploads `textCanvas` into GPU textures. +- `src/engine/export/ExportLayerBuilder.ts` already exports `textCanvas` layers. +- `src/services/thumbnailRenderer.ts` already renders thumbnails from `textCanvas`. +- `src/services/compositionRenderer.ts` already carries `textCanvas` through composition evaluation. + +### Real gaps that must be closed + +- Nested preview is incomplete. `LayerBuilderService.buildNestedClipLayer()` does not forward generic canvas-backed clips today. +- The paused-sync path in `src/components/timeline/hooks/useLayerSync.ts` only handles nested video/image clips. +- Save/load, relink, background slot playback, and clipboard restore all branch explicitly on known source types. +- UI files hardcode type badges and icons in `MediaPanel.tsx`, `FileTypeIcon.tsx`, `TimelineClip.tsx`, `useExternalDrop.ts`, and `properties/index.tsx`. + +## Frozen Decisions + +These decisions are not open for agent debate unless the user explicitly reopens them. + +1. Phase 1 is Lottie. Phase 2 is Rive. +2. Lottie runtime for MVP is `@lottiefiles/dotlottie-web`. +3. Do not use Lottie worker mode in MVP. Deterministic scrubbing and export matter more than max concurrency. +4. Rive must use the low-level web runtime, not the React wrapper. +5. Do not add a new compositor or WebGPU source path for Lottie/Rive in phase 1. +6. `TimelineClip.source.type` must become `lottie` or `rive`, but render `Layer.source` should continue to reuse the existing canvas path with `type: 'text'` plus `textCanvas`. +7. Do not rename `textCanvas` in this issue. The name is imperfect, but a generic rename would create unnecessary churn across the render stack. +8. `.json` files must be content-sniffed before they are treated as Lottie. Do not classify every JSON file as Lottie by extension alone. +9. Runtime cursors are clip-local. The same source reused at multiple times must not fight over one playback cursor. +10. Rive audio, data binding, and rich interactive state-machine inputs are out of scope for the first implementation. +11. Do not add new synthetic media-panel item types for Lottie or Rive. They are imported file types, not generated items like text/solid/camera. + +## Shared Contract To Land First + +This wave is serialized. One agent owns it. Nobody else edits these files until this wave lands. + +### Files + +- `src/types/index.ts` +- `src/stores/mediaStore/types.ts` +- `src/stores/timeline/types.ts` +- `src/services/project/types/media.types.ts` +- `src/services/project/types/composition.types.ts` +- new `src/types/vectorAnimation.ts` + +### Minimum contract + +Use one dedicated metadata object and one dedicated per-clip settings object. Do not spray new top-level optional fields everywhere. + +```ts +export interface VectorAnimationMetadata { + provider: 'lottie' | 'rive'; + width?: number; + height?: number; + fps?: number; + duration?: number; + totalFrames?: number; + animationNames?: string[]; + defaultAnimationName?: string; + artboardNames?: string[]; + stateMachineNames?: string[]; +} + +export interface VectorAnimationClipSettings { + loop: boolean; + endBehavior: 'hold' | 'clear' | 'loop'; + fit: 'contain' | 'cover' | 'fill'; + backgroundColor?: string; + animationName?: string; + artboard?: string; + stateMachineName?: string; +} +``` + +### Required contract updates + +- `MediaFile.type` includes `lottie` and `rive`. +- `TimelineClip.source.type` includes `lottie` and `rive`. +- `SerializableClip.sourceType` includes `lottie` and `rive`. +- `ProjectMediaFile.type` includes `lottie` and `rive`. +- `ProjectClip.sourceType` includes `lottie` and `rive`. +- `MediaFile` gets `vectorAnimation?: VectorAnimationMetadata`. +- `TimelineClip.source` gets `vectorAnimationSettings?: VectorAnimationClipSettings`. +- `SerializableClip` and `ProjectClip` get `vectorAnimationSettings?: VectorAnimationClipSettings`. + +### Acceptance + +- `npm run build` passes after the contract wave. +- No render behavior is changed yet. +- No agent edits shared union files in parallel after ownership is assigned. + +## Execution Order + +1. Land the shared contract wave. +2. Implement the Lottie runtime service and metadata extraction. +3. In parallel, implement import/clip creation, render/export wiring, persistence/reload, and UI. +4. Finish with tests and docs. +5. Start Rive only after Lottie is green. + +## Workstream 1 - Import And Clip Ingestion + +### Ownership + +- `src/stores/timeline/helpers/mediaTypeHelpers.ts` +- `src/components/timeline/utils/fileTypeHelpers.ts` +- `src/components/panels/media/dropImport.ts` +- `src/stores/mediaStore/helpers/importPipeline.ts` +- `src/stores/mediaStore/slices/fileImportSlice.ts` +- `src/stores/timeline/clipSlice.ts` +- new `src/stores/timeline/clip/addLottieClip.ts` +- optional stub `src/stores/timeline/clip/addRiveClip.ts` +- `src/components/timeline/hooks/useExternalDrop.ts` + +### Build + +- Add `.lottie` and `.riv` extension detection. +- Add async JSON sniffing for Lottie JSON in the import pipeline. +- Route imported Lottie files into a real `MediaFile.type === 'lottie'`. +- Create a Lottie clip placeholder with `source.type = 'lottie'`, `mediaFileId`, `naturalDuration`, `textCanvas`, and default `vectorAnimationSettings`. +- Keep Lottie on video tracks only. +- For `.json` desktop drop, allow a small async classification step before `addClip`. Do not force the old fully-sync fast path for these files. +- Do not attempt Rive runtime here. A stub clip factory is enough if it helps phase 2. + +### Acceptance + +- Importing a `.lottie` file creates a usable media panel item with dimensions and duration. +- Importing a Lottie JSON file works only when the JSON actually matches Lottie structure. +- Dragging a media panel Lottie item onto a video track creates a `lottie` clip. +- Dragging a Lottie file directly from desktop does not misclassify arbitrary JSON files. + +## Workstream 2 - Lottie Runtime And Canvas Lifecycle + +### Ownership + +- `package.json` +- `package-lock.json` +- new `src/services/vectorAnimation/LottieRuntimeManager.ts` +- new `src/services/vectorAnimation/lottieMetadata.ts` +- new `src/services/vectorAnimation/lottieJsonSniffer.ts` +- new `src/services/vectorAnimation/types.ts` +- `src/stores/mediaStore/slices/fileManageSlice.ts` + +### Build + +- Add `@lottiefiles/dotlottie-web`. +- Create a clip-keyed runtime manager that owns one hidden `HTMLCanvasElement` plus one Lottie runtime instance per active clip. +- Expose APIs to: + - load metadata from `File` + - ensure runtime for a clip + - render a clip at an exact timeline time + - update clip settings + - prune stale clip runtimes + - destroy all runtimes +- Rehydrate Lottie canvases on `reloadFile()` and `updateTimelineClips()`. +- Derive exact animation time from clip-local time plus `vectorAnimationSettings.loop` and `endBehavior`. +- Populate `clip.source.textCanvas` with the managed canvas so the rest of the stack can reuse it. + +### Acceptance + +- The same Lottie source can exist in two clips at different times without cursor fights. +- Scrubbing to the same frame twice produces the same canvas output. +- Reloading a missing file restores the Lottie canvas and clears `needsReload`. +- No free-running autoplay loop exists outside timeline time. + +## Workstream 3 - Render, Nested Preview, Export, And Thumbnails + +### Ownership + +- `src/services/layerBuilder/LayerBuilderService.ts` +- `src/components/timeline/hooks/useLayerSync.ts` +- `src/engine/export/ExportLayerBuilder.ts` +- `src/services/thumbnailRenderer.ts` +- `src/services/compositionRenderer.ts` +- touch `src/engine/render/LayerCollector.ts` only if strictly necessary + +### Build + +- Teach the main layer builder to recognize `clip.source.type === 'lottie'` and reuse `buildTextLayer()` or an equivalent canvas layer path. +- Before building layers, ask `LottieRuntimeManager` to render active Lottie clips at the current playhead time and prune stale runtimes. +- Extend nested preview in both `LayerBuilderService.buildNestedClipLayer()` and `useLayerSync` so nested clips with `textCanvas` are treated like renderable canvas clips, not ignored. +- Extend `ExportLayerBuilder` so Lottie clips and nested Lottie clips export through the existing canvas source path. +- Extend `thumbnailRenderer` and `compositionRenderer` conditions so canvas-backed `lottie` clips are not excluded just because their semantic source type is not `text`. + +### Acceptance + +- Lottie preview works while paused, while scrubbing, and during playback. +- A Lottie clip inside a nested composition is visible in preview. +- Export uses the same deterministic frame mapping as preview. +- Thumbnails can be generated from Lottie clips without a special export-only renderer. + +## Workstream 4 - Persistence, Restore, Background Playback, Clipboard + +### Ownership + +- `src/stores/timeline/serializationUtils.ts` +- `src/services/project/projectSave.ts` +- `src/services/project/projectLoad.ts` +- `src/services/layerPlaybackManager.ts` +- `src/services/slotDeckManager.ts` +- `src/stores/timeline/clipboardSlice.ts` +- `src/stores/timeline/clip/addCompClip.ts` + +### Build + +- Save `MediaFile.type === 'lottie'` plus `vectorAnimation` metadata into project files. +- Save `vectorAnimationSettings` on timeline clips and nested clips. +- Restore Lottie clips from project data with placeholder files when necessary, then hand them back to the runtime manager once the real file exists. +- Ensure nested composition hydration and clipboard paste preserve `sourceType === 'lottie'`. +- Extend background slot playback and composition preparation so canvas-backed Lottie clips are kept alive outside the active editor composition. + +### Acceptance + +- Save, close, reload, and relink preserve Lottie media and clip settings. +- Background composition playback does not silently drop Lottie clips. +- Copy/paste and nested comp duplication preserve Lottie type and settings. + +## Workstream 5 - UI, Icons, Properties + +### Ownership + +- `src/components/panels/MediaPanel.tsx` +- `src/components/panels/media/FileTypeIcon.tsx` +- `src/components/timeline/TimelineClip.tsx` +- `src/components/panels/properties/index.tsx` +- new `src/components/panels/properties/LottieTab.tsx` +- optional `src/components/panels/properties/RiveTab.tsx` +- `src/App.css` if styling is needed + +### Build + +- Add media and timeline icons for `lottie` and `rive`. +- Show semantic clip badges based on `source.type`. +- Add a Lottie properties tab with at least: + - loop toggle + - end behavior + - fit + - animation selection for multi-animation `.lottie` + - background color if the runtime exposes it cleanly +- Keep Lottie clips finite by default. Do not mark them as "infinite" like text/solid/camera. +- Do not add Media Panel "create new Lottie" menu items. Imported file UX is enough. + +### Acceptance + +- Imported Lottie items are visually distinct in the media panel and timeline. +- Properties edits update preview without re-importing the file. +- Multi-animation `.lottie` packages can switch the active animation from the properties panel. + +## Workstream 6 - Tests And Docs + +### Ownership + +- `tests/unit/importPipeline.test.ts` +- `tests/stores/timeline/clipSlice.test.ts` +- `tests/stores/mediaStore/fileManageSlice.test.ts` +- `tests/unit/layerBuilderService.test.ts` +- `tests/unit/exportLayerBuilder.test.ts` +- `tests/unit/serialization.test.ts` +- `tests/unit/projectMediaPersistence.test.ts` +- `tests/unit/mediaPanelDropImport.test.ts` +- `docs/Features/Media-Panel.md` +- `docs/Features/Timeline.md` +- add a focused feature doc if the existing docs become too noisy + +### Build + +- Add detection tests for `.lottie` and Lottie JSON sniffing. +- Add clip creation tests for `source.type === 'lottie'`. +- Add reload tests for Lottie relink. +- Add layer builder and export tests proving Lottie is routed through the canvas path. +- Add serialization and project persistence round-trip tests. +- Update feature docs with supported file types, limitations, and property behavior. + +### Acceptance + +- `npm run build` +- `npx vitest run tests/unit/importPipeline.test.ts tests/stores/timeline/clipSlice.test.ts tests/stores/mediaStore/fileManageSlice.test.ts tests/unit/layerBuilderService.test.ts tests/unit/exportLayerBuilder.test.ts tests/unit/serialization.test.ts tests/unit/projectMediaPersistence.test.ts tests/unit/mediaPanelDropImport.test.ts` + +## Rive Phase 2 + +Do not start this until Lottie is green. + +### Scope + +- Imported `.riv` files +- artboard selection +- one deterministic linear animation path first +- optional single state machine selection after the linear path works + +### Rules + +- Use the same canvas-backed render strategy as Lottie. +- Do not use `rive-react` as the core runtime. +- Cache parsed file data if helpful, but keep render cursors clip-local. +- Leave Rive audio and rich data binding out of the first pass. + +## Shared Hot Spots That Must Not Be Edited Casually In Parallel + +- `src/types/index.ts` +- `src/stores/mediaStore/types.ts` +- `src/services/project/types/media.types.ts` +- `src/services/project/types/composition.types.ts` +- `src/stores/timeline/serializationUtils.ts` +- `src/services/project/projectSave.ts` +- `src/services/project/projectLoad.ts` +- `src/services/layerBuilder/LayerBuilderService.ts` +- `src/components/timeline/hooks/useExternalDrop.ts` +- `package.json` +- `package-lock.json` + +## Definition Of Done + +Treat this issue as done only when all of the following are true: + +- A user can import `.lottie` and Lottie JSON files into the media panel. +- A user can drag a Lottie media item to a video track and get a real Lottie clip. +- Scrubbing, paused preview, playback, nested comps, thumbnails, and export all render deterministically. +- Save/load and relink preserve Lottie media and clip settings. +- The same Lottie source can be reused multiple times on the timeline at different times. +- Build passes and the targeted tests pass. +- Rive has a follow-up implementation path that reuses the same architecture instead of reopening the design from scratch. diff --git a/src/App.css b/src/App.css index 6964142f0..688e6c2a0 100644 --- a/src/App.css +++ b/src/App.css @@ -8443,6 +8443,11 @@ input[type="checkbox"] { background: rgba(45, 140, 235, 0.15); } +.media-grid-item.importing { + opacity: 0.55; + cursor: progress; +} + .media-grid-thumb { position: relative; aspect-ratio: 16 / 9; @@ -8492,6 +8497,20 @@ input[type="checkbox"] { text-align: center; } +.media-grid-import-badge { + position: absolute; + top: 3px; + left: 3px; + background: rgba(45, 140, 235, 0.18); + color: var(--accent); + font-size: 10px; + padding: 1px 5px; + border-radius: 8px; + font-variant-numeric: tabular-nums; + min-width: 28px; + text-align: center; +} + .media-grid-name { padding: 3px 6px; font-size: 11px; @@ -8555,6 +8574,7 @@ input[type="checkbox"] { .media-item.importing { opacity: 0.5; + cursor: progress; } .media-item.importing .media-col-name::after { @@ -8569,6 +8589,20 @@ input[type="checkbox"] { 50% { opacity: 1; } } +.media-item-import-progress { + display: inline-flex; + align-items: center; + justify-content: center; + min-width: 38px; + padding: 1px 5px; + margin-left: 6px; + border-radius: 999px; + background: rgba(45, 140, 235, 0.18); + color: var(--accent); + font-size: 10px; + font-variant-numeric: tabular-nums; +} + .media-folder-arrow { font-size: 8px; color: var(--text-secondary); diff --git a/src/changelog-data.json b/src/changelog-data.json index a19df638a..4821bcaee 100644 --- a/src/changelog-data.json +++ b/src/changelog-data.json @@ -1,4 +1,34 @@ [ + { + "date": "2026-04-20", + "type": "new", + "title": "FlashBoard Adds Seedance 2.0 via Kie.ai", + "description": "FlashBoard now exposes Seedance 2.0 in the model catalog and pricing flow, with the Kie.ai service path updated so text-to-video and sequence-oriented generation can be routed through the board UI.", + "section": "FlashBoard / AI Video", + "commits": [ + "324a5e81" + ] + }, + { + "date": "2026-04-20", + "type": "improve", + "title": "AI Chat Model Catalog and Landing Experience Were Refreshed", + "description": "The OpenAI model catalog, AI chat pricing/model wiring, and the dedicated landing page were updated together so the public entry flow and in-app AI surfaces stay aligned with the current product lineup.", + "section": "AI Chat / Landing", + "commits": [ + "324a5e81" + ] + }, + { + "date": "2026-04-20", + "type": "fix", + "title": "Overlaid PLY and Splat Sequences Stay in Sync During Playback", + "description": "Gaussian splat sequence playback now prewarms the upcoming per-frame Three.js runtimes instead of waiting for on-demand cache misses, which removes the small playback offset that could appear when `.ply` and `.splat` layers were stacked together.", + "section": "3D / Gaussian Splats", + "commits": [ + "196389f6" + ] + }, { "date": "2026-04-18", "type": "new", diff --git a/src/components/panels/AIChatPanel.css b/src/components/panels/AIChatPanel.css index 806a2426a..88fcf7ceb 100644 --- a/src/components/panels/AIChatPanel.css +++ b/src/components/panels/AIChatPanel.css @@ -81,87 +81,25 @@ background: var(--accent-hover); } -.ai-chat-header { - display: flex; - justify-content: space-between; - align-items: center; - padding: var(--sp-3) var(--sp-4); - background: var(--bg-tertiary); - border-bottom: 1px solid var(--border-color); - flex-shrink: 0; -} - -.ai-chat-header h2 { - margin: 0; - font-size: var(--font-xl); - font-weight: var(--font-semibold); -} - -.ai-chat-title-group { - display: flex; - align-items: center; - gap: var(--sp-2); -} - -.ai-access-chip { - display: inline-flex; - align-items: center; - padding: 2px 8px; - border-radius: var(--radius-full); - font-size: var(--font-xs); - text-transform: uppercase; - letter-spacing: var(--tracking-wide); - border: 1px solid var(--border-color); - color: var(--text-secondary); -} - -.ai-access-chip.hosted { - border-color: color-mix(in srgb, var(--accent) 45%, transparent); - color: var(--accent); -} - -.ai-access-chip.byo { - border-color: var(--border-color); -} - -.ai-access-chip.none { - color: var(--danger-light); - border-color: var(--danger-border); -} - -.ai-chat-controls { - display: flex; - gap: var(--sp-2); - align-items: center; -} - -.model-select { - background: var(--bg-secondary); - border: 1px solid var(--border-color); - border-radius: var(--radius-md); - color: var(--text-primary); - padding: var(--sp-1) var(--sp-2); - font-size: var(--font-md); - cursor: pointer; -} - -.model-select:hover { - border-color: var(--accent); -} - .btn-clear { - background: transparent; + background: var(--bg-secondary); border: 1px solid var(--border-color); - border-radius: var(--radius-md); + border-radius: var(--radius-lg); color: var(--text-secondary); - padding: var(--sp-1) var(--sp-2); - font-size: var(--font-sm); + padding: var(--sp-2) var(--sp-4); + font-size: var(--font-md); + font-weight: var(--font-medium); cursor: pointer; - transition: all var(--transition-normal); + transition: + background var(--transition-normal), + border-color var(--transition-normal), + color var(--transition-normal), + opacity var(--transition-normal); } .btn-clear:hover:not(:disabled) { - border-color: var(--accent); + background: color-mix(in srgb, var(--bg-secondary) 78%, white 8%); + border-color: color-mix(in srgb, var(--accent) 30%, var(--border-color)); color: var(--text-primary); } @@ -341,14 +279,25 @@ /* Input Area */ .ai-chat-input-area { - display: flex; - gap: var(--sp-2); padding: var(--sp-3) var(--sp-4); background: var(--bg-tertiary); border-top: 1px solid var(--border-color); flex-shrink: 0; } +.ai-chat-composer { + display: flex; + flex-direction: column; + gap: var(--sp-2); + width: 100%; +} + +.ai-chat-input-row { + display: flex; + gap: var(--sp-2); + width: 100%; +} + .ai-chat-input { flex: 1; background: var(--bg-secondary); @@ -358,7 +307,10 @@ padding: var(--sp-2) var(--sp-3); font-size: var(--font-lg); font-family: inherit; - resize: none; + min-height: 64px; + max-height: 240px; + resize: vertical; + overflow-y: auto; line-height: 1.4; } @@ -393,26 +345,163 @@ cursor: not-allowed; } -/* Editor Mode Toggle */ -.editor-mode-toggle { +.ai-chat-control-bar { display: flex; align-items: center; - gap: var(--sp-1); - cursor: pointer; - font-size: var(--font-sm); - color: var(--text-secondary); + justify-content: space-between; + gap: var(--sp-2); } -.editor-mode-toggle input { +.ai-chat-pill-group { + display: flex; + gap: 4px; + align-items: center; + position: relative; + flex-wrap: wrap; +} + +.ai-chat-pill { + appearance: none; + display: flex; + align-items: center; + gap: 4px; + padding: 4px 10px; + border: 1px solid rgba(255, 255, 255, 0.12); + border-radius: 16px; + background: rgba(255, 255, 255, 0.06); + color: var(--text-secondary); + font-size: 11px; cursor: pointer; + transition: + border-color var(--transition-normal), + background var(--transition-normal), + color var(--transition-normal), + opacity var(--transition-normal); + white-space: nowrap; +} + +.ai-chat-pill:hover { + background: rgba(255, 255, 255, 0.1); + color: var(--text-primary); } -.editor-mode-toggle .toggle-label { +.ai-chat-pill.active { + background: var(--accent); + color: var(--text-on-accent); + border-color: var(--accent); +} + +.ai-chat-toggle-pill { user-select: none; } -.editor-mode-toggle input:checked + .toggle-label { - color: var(--accent); +.ai-chat-toggle-pill input { + position: absolute; + opacity: 0; + pointer-events: none; +} + +.ai-chat-pill:disabled, +.ai-chat-toggle-pill:has(input:disabled) { + opacity: 0.55; + cursor: not-allowed; +} + +.ai-chat-toggle-dot { + width: 7px; + height: 7px; + border-radius: 50%; + background: currentColor; + opacity: 0.6; + flex-shrink: 0; +} + +.ai-chat-toggle-pill.active .ai-chat-toggle-dot { + opacity: 1; +} + +.ai-chat-popover { + position: absolute; + bottom: calc(100% + 6px); + left: 0; + min-width: 220px; + max-width: 320px; + padding: 10px; + border: 1px solid var(--border-color); + border-radius: 10px; + background: var(--bg-elevated, #2a2a3e); + box-shadow: 0 8px 24px rgba(0, 0, 0, 0.5); + z-index: 5; +} + +.ai-chat-popover-model { + min-width: 260px; +} + +.ai-chat-popover-title { + margin-bottom: 8px; + font-size: 11px; + font-weight: 600; + color: var(--text-primary); +} + +.ai-chat-popover-pills { + display: flex; + flex-wrap: wrap; + gap: 4px; +} + +.ai-chat-popover-pill { + appearance: none; + display: flex; + flex-direction: column; + align-items: flex-start; + gap: 2px; + padding: 5px 10px; + border: 1px solid rgba(255, 255, 255, 0.12); + border-radius: 14px; + background: rgba(255, 255, 255, 0.04); + color: var(--text-secondary); + font-size: 11px; + text-align: left; + cursor: pointer; + transition: + border-color var(--transition-normal), + background var(--transition-normal), + color var(--transition-normal); +} + +.ai-chat-popover-pill:hover { + background: rgba(255, 255, 255, 0.1); + color: var(--text-primary); +} + +.ai-chat-popover-pill.active { + background: var(--accent); + border-color: var(--accent); + color: var(--text-on-accent); +} + +.ai-chat-popover-pill-label { + color: inherit; +} + +.ai-chat-popover-pill-meta { + color: var(--text-muted); + font-size: 10px; + line-height: 1.3; +} + +.ai-chat-popover-pill.active .ai-chat-popover-pill-meta { + color: rgba(255, 255, 255, 0.82); +} + +.ai-chat-clear-bottom { + margin-left: auto; + align-self: center; + justify-content: center; + min-width: 0; + text-align: center; } /* Tool Calls Display */ @@ -502,6 +591,124 @@ animation: pulse 1.5s infinite; } +.ai-chat-message.tool-approval { + align-self: flex-start; + max-width: min(560px, 100%); + padding: 0; + background: transparent; +} + +.tool-approval-banner { + display: flex; + flex-direction: column; + gap: var(--sp-2); + padding: var(--sp-3); + background: + linear-gradient(180deg, color-mix(in srgb, var(--warning-subtle, var(--bg-tertiary)) 55%, var(--bg-tertiary)), var(--bg-tertiary)); + border: 1px solid color-mix(in srgb, var(--accent) 20%, var(--border-color)); + border-radius: var(--radius-xl); + box-shadow: 0 10px 24px rgba(0, 0, 0, 0.18); +} + +.tool-approval-header { + display: flex; + align-items: center; + justify-content: space-between; + gap: var(--sp-2); + flex-wrap: wrap; +} + +.tool-approval-label { + display: inline-flex; + align-items: center; + padding: 4px 10px; + border-radius: var(--radius-full); + background: color-mix(in srgb, var(--accent) 14%, transparent); + border: 1px solid color-mix(in srgb, var(--accent) 28%, transparent); + color: var(--accent); + font-size: var(--font-xs); + font-weight: var(--font-semibold); + text-transform: uppercase; + letter-spacing: var(--tracking-wide); +} + +.tool-approval-name { + color: var(--text-primary); + font-family: var(--font-mono); + font-size: var(--font-sm); + font-weight: var(--font-semibold); +} + +.tool-approval-copy { + margin: 0; + color: var(--text-secondary); + font-size: var(--font-sm); + line-height: 1.45; +} + +.tool-approval-args { + margin: 0; + padding: var(--sp-2); + border-radius: var(--radius-lg); + background: color-mix(in srgb, var(--bg-secondary) 92%, black 8%); + border: 1px solid var(--border-color); + color: var(--text-secondary); + font-size: var(--font-xs); + font-family: var(--font-mono); + white-space: pre-wrap; + word-break: break-word; + max-height: 220px; + overflow: auto; +} + +.tool-approval-buttons { + display: flex; + gap: var(--sp-2); + justify-content: flex-end; + flex-wrap: wrap; +} + +.btn-approve, +.btn-deny { + appearance: none; + border: 1px solid transparent; + border-radius: var(--radius-lg); + padding: var(--sp-2) var(--sp-3); + font-size: var(--font-sm); + font-weight: var(--font-semibold); + cursor: pointer; + transition: + background var(--transition-normal), + border-color var(--transition-normal), + color var(--transition-normal), + transform var(--transition-fast); +} + +.btn-approve:hover, +.btn-deny:hover { + transform: translateY(-1px); +} + +.btn-approve { + background: var(--accent); + color: var(--text-on-accent); +} + +.btn-approve:hover { + background: var(--accent-hover); +} + +.btn-deny { + background: transparent; + color: var(--text-secondary); + border-color: var(--border-color); +} + +.btn-deny:hover { + border-color: color-mix(in srgb, var(--danger-light) 45%, var(--border-color)); + color: var(--danger-light); +} + @keyframes pulse { 0%, 100% { opacity: 1; } 50% { opacity: 0.5; } @@ -635,3 +842,23 @@ .ai-chat-onboarding-dismiss:hover { background: var(--accent-hover); } + +@media (max-width: 640px) { + .ai-chat-input-row { + flex-direction: column; + } + + .btn-send { + width: 100%; + align-self: stretch; + } + + .tool-approval-buttons { + flex-direction: column; + } + + .btn-approve, + .btn-deny { + width: 100%; + } +} diff --git a/src/components/panels/AIChatPanel.tsx b/src/components/panels/AIChatPanel.tsx index 2707271f0..a83807438 100644 --- a/src/components/panels/AIChatPanel.tsx +++ b/src/components/panels/AIChatPanel.tsx @@ -5,35 +5,14 @@ import { useSettingsStore } from '../../stores/settingsStore'; import { useAccountStore } from '../../stores/accountStore'; import { AI_TOOLS, executeAITool, getQuickTimelineSummary, getToolPolicy } from '../../services/aiTools'; import { cloudAiService } from '../../services/cloudAiService'; +import { + DEFAULT_OPENAI_MODEL_ID, + OPENAI_CHAT_DROPDOWN_MODELS, + type OpenAiReasoningEffort, +} from '../../shared/openAiModelCatalog'; import type { ToolPolicyEntry } from '../../services/aiTools'; import './AIChatPanel.css'; -// Available OpenAI models with credit cost per request -const OPENAI_MODELS = [ - // GPT-5.2 series (newest - Dec 2025) - { id: 'gpt-5.2', name: 'GPT-5.2 (Thinking)', credits: 8 }, - { id: 'gpt-5.2-pro', name: 'GPT-5.2 Pro', credits: 10 }, - // GPT-5.1 series - { id: 'gpt-5.1', name: 'GPT-5.1', credits: 5 }, - { id: 'gpt-5.1-codex', name: 'GPT-5.1 Codex', credits: 5 }, - { id: 'gpt-5.1-codex-mini', name: 'GPT-5.1 Codex Mini', credits: 1 }, - // GPT-5 series - { id: 'gpt-5', name: 'GPT-5', credits: 5 }, - { id: 'gpt-5-mini', name: 'GPT-5 Mini', credits: 1 }, - { id: 'gpt-5-nano', name: 'GPT-5 Nano', credits: 1 }, - // Reasoning models - { id: 'o3', name: 'o3 (Reasoning)', credits: 5 }, - { id: 'o4-mini', name: 'o4-mini (Reasoning)', credits: 3 }, - { id: 'o3-pro', name: 'o3-pro (Deep Reasoning)', credits: 50 }, - // GPT-4.1 series - { id: 'gpt-4.1', name: 'GPT-4.1', credits: 5 }, - { id: 'gpt-4.1-mini', name: 'GPT-4.1 Mini', credits: 1 }, - { id: 'gpt-4.1-nano', name: 'GPT-4.1 Nano', credits: 1 }, - // GPT-4o series (legacy) - { id: 'gpt-4o', name: 'GPT-4o', credits: 5 }, - { id: 'gpt-4o-mini', name: 'GPT-4o Mini', credits: 1 }, -]; - // System prompt for editor mode const EDITOR_SYSTEM_PROMPT = `You are an AI video editing assistant with direct access to the timeline AND media panel. You can: @@ -115,10 +94,45 @@ interface PendingApproval { resolve: (approved: boolean) => void; } +type AiApprovalMode = 'auto' | 'confirm-destructive' | 'confirm-all-mutating'; +type ChatControlPopover = 'model' | 'reasoning' | null; + const MAX_TOOL_RESULT_MESSAGE_CHARS = 12000; const MAX_TOOL_RESULT_ARRAY_ITEMS = 20; const MAX_TOOL_RESULT_OBJECT_KEYS = 30; const MAX_TOOL_RESULT_STRING_CHARS = 1200; +const REASONING_OPTION_COPY: Record = { + none: { + compactLabel: 'none', + description: 'No extra reasoning step. Fastest and cheapest.', + label: 'None', + }, + minimal: { + compactLabel: 'min', + description: 'Very light reasoning for quick replies.', + label: 'Minimal', + }, + low: { + compactLabel: 'low', + description: 'Low reasoning effort with lower latency.', + label: 'Low', + }, + medium: { + compactLabel: 'med', + description: 'Balanced reasoning depth and speed.', + label: 'Medium', + }, + high: { + compactLabel: 'high', + description: 'Deeper reasoning for harder edits or planning.', + label: 'High', + }, + xhigh: { + compactLabel: 'xhigh', + description: 'Maximum available reasoning for supported models.', + label: 'XHigh', + }, +}; function truncateText(value: string, maxLength: number): string { if (value.length <= maxLength) { @@ -258,7 +272,7 @@ function sanitizeConversationHistory(messages: Message[]): Message[] { function shouldRequireConfirmation( policy: ToolPolicyEntry | undefined, - approvalMode: 'auto' | 'confirm-destructive' | 'confirm-all-mutating', + approvalMode: AiApprovalMode, ): boolean { if (!policy) return true; // unknown tools require confirmation if (approvalMode === 'auto') return false; @@ -310,14 +324,17 @@ export function AIChatPanel() { const [messages, setMessages] = useState([]); const [input, setInput] = useState(''); const [isLoading, setIsLoading] = useState(false); - const [model, setModel] = useState('gpt-5.1'); + const [model, setModel] = useState(DEFAULT_OPENAI_MODEL_ID); + const [reasoningEffort, setReasoningEffort] = useState('none'); const [error, setError] = useState(null); - const [editorMode, setEditorMode] = useState(true); // Enable tools by default + const [bypassToolApprovals, setBypassToolApprovals] = useState(false); + const [controlPopover, setControlPopover] = useState(null); const [currentToolAction, setCurrentToolAction] = useState(null); const [pendingApproval, setPendingApproval] = useState(null); const [onboardingClosing, setOnboardingClosing] = useState(false); const messagesEndRef = useRef(null); const inputRef = useRef(null); + const controlPopoverRef = useRef(null); // Auto-scroll to bottom when new messages arrive useEffect(() => { @@ -328,19 +345,57 @@ export function AIChatPanel() { const hasApiKey = !!apiKeys.openai; const accessMode: 'hosted' | 'byo' | 'none' = hasHostedAccess ? 'hosted' : hasApiKey ? 'byo' : 'none'; const hasAccess = accessMode !== 'none'; + const effectiveApprovalMode: AiApprovalMode = bypassToolApprovals ? 'auto' : aiApprovalMode; + const selectedModel = OPENAI_CHAT_DROPDOWN_MODELS.find((entry) => entry.id === model); + const supportedReasoningEfforts = selectedModel?.supportedReasoningEfforts; + const reasoningChipLabel = reasoningEffort ? REASONING_OPTION_COPY[reasoningEffort].compactLabel : null; + const modelChipLabel = selectedModel + ? `${selectedModel.label} · ${selectedModel.creditCost === 1 ? '1 cr' : `${selectedModel.creditCost} cr`}` + : model; + const bypassToggleHint = bypassToolApprovals + ? 'Tools run immediately in this panel' + : 'Keep confirmation prompts for tool actions'; + + useEffect(() => { + const availableReasoningEfforts = selectedModel?.supportedReasoningEfforts ?? []; + + if (!selectedModel || availableReasoningEfforts.length === 0) { + setReasoningEffort(null); + setControlPopover(null); + return; + } + + if (reasoningEffort && availableReasoningEfforts.includes(reasoningEffort)) { + return; + } + + setReasoningEffort(selectedModel.defaultReasoningEffort ?? availableReasoningEfforts[0] ?? null); + }, [reasoningEffort, selectedModel]); + + useEffect(() => { + if (!controlPopover) { + return; + } + + const handleOutsideClick = (event: MouseEvent) => { + if (controlPopoverRef.current && !controlPopoverRef.current.contains(event.target as Node)) { + setControlPopover(null); + } + }; + + window.addEventListener('mousedown', handleOutsideClick); + return () => window.removeEventListener('mousedown', handleOutsideClick); + }, [controlPopover]); // Build API messages from chat history const buildAPIMessages = useCallback((userContent: string): APIMessage[] => { const apiMessages: APIMessage[] = []; const safeMessages = sanitizeConversationHistory(messages); - // Add system prompt in editor mode - if (editorMode) { - apiMessages.push({ - role: 'system', - content: EDITOR_SYSTEM_PROMPT + getQuickTimelineSummary(), - }); - } + apiMessages.push({ + role: 'system', + content: EDITOR_SYSTEM_PROMPT + getQuickTimelineSummary(), + }); // Add conversation history for (const msg of safeMessages) { @@ -373,7 +428,7 @@ export function AIChatPanel() { apiMessages.push({ role: 'user', content: userContent }); return apiMessages; - }, [messages, editorMode]); + }, [messages]); // Call OpenAI API const callOpenAI = useCallback(async ( @@ -394,12 +449,13 @@ export function AIChatPanel() { : { max_tokens: 4096 }), }; - // Add tools in editor mode - if (editorMode) { - requestBody.tools = AI_TOOLS; - requestBody.tool_choice = 'auto'; + if (reasoningEffort) { + requestBody.reasoning_effort = reasoningEffort; } + requestBody.tools = AI_TOOLS; + requestBody.tool_choice = 'auto'; + if (accessMode === 'hosted') { if (idempotencyKey) { requestBody.idempotencyKey = idempotencyKey; @@ -423,7 +479,7 @@ export function AIChatPanel() { } return parseChatCompletionPayload(await response.json()); - }, [accessMode, model, editorMode, apiKeys.openai]); + }, [accessMode, model, reasoningEffort, apiKeys.openai]); // Send message to OpenAI (with tool calling loop) const sendMessage = useCallback(async () => { @@ -508,7 +564,7 @@ export function AIChatPanel() { // Check if this tool requires user confirmation const policy = getToolPolicy(toolCall.name); - const needsConfirmation = shouldRequireConfirmation(policy, aiApprovalMode); + const needsConfirmation = shouldRequireConfirmation(policy, effectiveApprovalMode); let result: { success: boolean; data?: unknown; error?: string }; @@ -573,7 +629,7 @@ export function AIChatPanel() { setIsLoading(false); setCurrentToolAction(null); } - }, [input, hasAccess, isLoading, buildAPIMessages, callOpenAI, aiApprovalMode, accessMode, loadAccountState]); + }, [input, hasAccess, isLoading, buildAPIMessages, callOpenAI, effectiveApprovalMode, accessMode, loadAccountState]); // Handle key press const handleKeyDown = useCallback((e: React.KeyboardEvent) => { @@ -631,45 +687,6 @@ export function AIChatPanel() { )} - {/* Header */} -
-
-

AI Editor

- - {accessMode === 'hosted' ? 'Cloud' : accessMode === 'byo' ? 'OpenAI key' : 'Locked'} - -
-
- - - -
-
- {/* Messages */}
{messages.length === 0 ? ( @@ -694,8 +711,8 @@ export function AIChatPanel() {
  • Downloads: "Search YouTube for nature footage and download it"

    • - The Tools toggle enables timeline editing. Turn it off for a normal chat. - Use the approval mode in Settings to control which actions need your confirmation. + AI chat runs directly against the editor tool surface. + Use the approval mode in Settings or the Auto tools switch below to control confirmations.