From 3e5594dffa3558239838e8527e0a5a0914b3918e Mon Sep 17 00:00:00 2001 From: SecurityQQ Date: Mon, 20 Jul 2026 23:07:34 -0700 Subject: [PATCH 1/2] refactor(renderers): split render.ts, add withDedup + shared inputs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit render.ts (795 lines) split into: context-builder.ts — createRenderContext, cache wrapping, pricing, preview mode, PreparedRender type flatten.ts — flattenClips, HoistedCaption, DeferredAudio types render.ts — thin orchestrator (~400 lines, was 795) withDedup helper (dedup.ts) eliminates 30 lines of copy-paste from image/video/speech/music renderers: - pre-resolved short-circuit - pendingFiles dedup by cache key - progress task lifecycle (addTask/startTask/completeTask) inputs.ts consolidates resolveImageInput, resolveVideoInput, resolveAudioInput — previously duplicated between image.ts and video.ts. resolveCacheStorage moved from render.ts (parent) to context-builder.ts, eliminating the last duplicate. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- src/react/render.ts | 16 +- src/react/renderers/context-builder.ts | 221 +++++++++++ src/react/renderers/dedup.ts | 69 ++++ src/react/renderers/flatten.ts | 125 ++++++ src/react/renderers/image.ts | 57 +-- src/react/renderers/index.ts | 1 + src/react/renderers/inputs.ts | 92 +++++ src/react/renderers/music.ts | 29 +- src/react/renderers/render.ts | 513 ++++++------------------- src/react/renderers/speech.ts | 29 +- src/react/renderers/video.ts | 113 +----- 11 files changed, 654 insertions(+), 611 deletions(-) create mode 100644 src/react/renderers/context-builder.ts create mode 100644 src/react/renderers/dedup.ts create mode 100644 src/react/renderers/flatten.ts create mode 100644 src/react/renderers/inputs.ts diff --git a/src/react/render.ts b/src/react/render.ts index 41bd5aa1..4a00d8ab 100644 --- a/src/react/render.ts +++ b/src/react/render.ts @@ -1,22 +1,16 @@ -import { type CacheStorage, withCache } from "../ai-sdk/cache"; -import { fileCache } from "../ai-sdk/file-cache"; import { localBackend } from "../ai-sdk/providers/editly"; import { compile } from "./ir/compile"; import { executePlan } from "./ir/execute"; import { CompileError, type StepEvent } from "./ir/types"; -import { createRenderContext, renderRoot } from "./renderers/render"; +import { + createRenderContext, + resolveCacheStorage, +} from "./renderers/context-builder"; +import { renderRoot } from "./renderers/render"; import { resolveLazy } from "./renderers/resolve-lazy"; import { type ResolveContext, withResolveContext } from "./resolve-context"; import type { RenderOptions, RenderResult, VargElement } from "./types"; -function resolveCacheStorage( - cache: string | CacheStorage | undefined, -): CacheStorage | undefined { - if (!cache) return undefined; - if (typeof cache === "string") return fileCache({ dir: cache }); - return cache; -} - interface PreparedPlan { resolved: VargElement<"render">; plan: ReturnType; diff --git a/src/react/renderers/context-builder.ts b/src/react/renderers/context-builder.ts new file mode 100644 index 00000000..5d560e90 --- /dev/null +++ b/src/react/renderers/context-builder.ts @@ -0,0 +1,221 @@ +/** + * Render context builder — sets up cached+wrapped generators, progress + * tracking, preview mode, and pricing emission for a composition. + * + * Extracted from render.ts so renderRoot stays a thin orchestrator. + */ + +import type { ImageModelV3 } from "@ai-sdk/provider"; +import { + generateImage, + experimental_generateSpeech as generateSpeech, + wrapImageModel, +} from "ai"; +import { type CacheStorage, withCache } from "../../ai-sdk/cache"; +import type { File } from "../../ai-sdk/file"; +import { fileCache } from "../../ai-sdk/file-cache"; +import { generateMusic } from "../../ai-sdk/generate-music"; +import { generateVideo } from "../../ai-sdk/generate-video"; +import { + imagePlaceholderFallbackMiddleware, + placeholderFallbackMiddleware, + wrapVideoModel, +} from "../../ai-sdk/middleware"; +import { localBackend } from "../../ai-sdk/providers/editly"; +import type { FFmpegBackend } from "../../ai-sdk/providers/editly/backends"; +import type { + RenderMode, + RenderOptions, + RenderProps, + VargElement, +} from "../types"; +import type { RenderContext } from "./context"; +import { createProgressTracker, type ProgressTracker } from "./progress"; + +export function resolveCacheStorage( + cache: string | CacheStorage | undefined, +): CacheStorage | undefined { + if (!cache) return undefined; + if (typeof cache === "string") { + return fileCache({ dir: cache }); + } + return cache; +} + +function toImageModelV3( + model: Parameters[0]["model"], +): ImageModelV3 { + if (typeof model === "object" && model.specificationVersion === "v3") { + return model; + } + const modelId = typeof model === "string" ? model : model.modelId; + return { + specificationVersion: "v3", + provider: "placeholder", + modelId, + maxImagesPerCall: 1, + doGenerate: async () => { + throw new Error( + `toImageModelV3 shell: doGenerate should not be called in preview mode (model: ${modelId})`, + ); + }, + }; +} + +/** RenderContext plus the render-scoped state renderRoot needs alongside it. */ +export interface PreparedRender { + ctx: RenderContext; + progress: ProgressTracker; + mode: RenderMode; + placeholderCount: { images: number; videos: number; total: number }; +} + +/** + * Build the RenderContext (cached+wrapped generators, backend, progress) + * for a composition. Shared by the plan executor and the compose phase so + * both see the same pendingFiles dedup map, progress tracker, and + * generatedFiles list. + */ +export function createRenderContext( + element: VargElement<"render">, + options: RenderOptions, +): PreparedRender { + const props = element.props as RenderProps; + const progress = createProgressTracker(options.quiet ?? false); + + const mode: RenderMode = options.mode ?? "strict"; + const placeholderCount = { images: 0, videos: 0, total: 0 }; + + const trackPlaceholder = (type: "image" | "video") => { + placeholderCount[type === "image" ? "images" : "videos"]++; + placeholderCount.total++; + }; + + const cacheStorage = resolveCacheStorage(options.cache); + + const cachedGenerateImage = cacheStorage + ? withCache(generateImage, { storage: cacheStorage }) + : generateImage; + + const cachedGenerateVideo = cacheStorage + ? withCache(generateVideo, { storage: cacheStorage }) + : generateVideo; + + const cachedGenerateSpeech = cacheStorage + ? withCache(generateSpeech, { storage: cacheStorage }) + : generateSpeech; + + const cachedGenerateMusic = cacheStorage + ? withCache(generateMusic, { storage: cacheStorage }) + : generateMusic; + + const onGeneration = options.onGeneration; + + /** Extract pricing metadata from provider response and emit via callback. */ + // biome-ignore lint/suspicious/noExplicitAny: result shapes vary across AI SDK model types + const emitPricing = ( + type: "image" | "video" | "speech" | "music", + modelId: string, + result: any, + ) => { + if (!onGeneration) return; + const vargMeta = result?.providerMetadata?.varg as + | { pricing?: Record; jobId?: string } + | undefined; + if (vargMeta?.pricing) { + const p = vargMeta.pricing; + onGeneration({ + type, + model: modelId, + estimated: p.estimated as number | undefined, + actual: p.actual as number | undefined, + billing: p.billing as "metered" | "byok" | "x402" | undefined, + cached: p.cached as boolean | undefined, + jobId: vargMeta.jobId, + }); + } + }; + + const wrapGenerateImage: typeof generateImage = async (opts) => { + if (mode === "preview") { + trackPlaceholder("image"); + return cachedGenerateImage({ + ...opts, + model: wrapImageModel({ + model: toImageModelV3(opts.model), + middleware: imagePlaceholderFallbackMiddleware({ + mode: "preview", + onFallback: () => {}, + }), + }), + skipCacheWrite: true, + } as Parameters[0]); + } + + const result = await cachedGenerateImage(opts); + const imgModelId = + typeof opts.model === "string" ? opts.model : opts.model.modelId; + emitPricing("image", imgModelId, result); + return result; + }; + + const wrapGenerateVideo: typeof generateVideo = async (opts) => { + if (mode === "preview") { + trackPlaceholder("video"); + return cachedGenerateVideo({ + ...opts, + model: wrapVideoModel({ + model: opts.model, + middleware: placeholderFallbackMiddleware({ + mode: "preview", + onFallback: () => {}, + }), + }), + skipCacheWrite: true, + } as Parameters[0]); + } + + const result = await cachedGenerateVideo(opts); + emitPricing("video", opts.model.modelId, result); + return result; + }; + + const wrapGenerateSpeech: typeof generateSpeech = async (opts) => { + const result = await cachedGenerateSpeech(opts); + const speechModelId = + typeof opts.model === "string" ? opts.model : opts.model.modelId; + emitPricing("speech", speechModelId, result); + return result; + }; + + const wrapGenerateMusic: typeof generateMusic = async (opts) => { + const result = await cachedGenerateMusic(opts); + const musicModelId = + typeof opts.model === "string" ? opts.model : opts.model.modelId; + emitPricing("music", musicModelId, result); + return result; + }; + + const backend: FFmpegBackend = options.backend ?? localBackend; + const tempFiles: string[] = []; + const generatedFiles: File[] = []; + const ctx: RenderContext = { + width: props.width ?? 1920, + height: props.height ?? 1080, + fps: props.fps ?? 30, + cache: cacheStorage, + storage: options.storage, + generateImage: wrapGenerateImage, + generateVideo: wrapGenerateVideo, + generateSpeech: onGeneration ? wrapGenerateSpeech : cachedGenerateSpeech, + generateMusic: onGeneration ? wrapGenerateMusic : cachedGenerateMusic, + tempFiles, + progress, + pendingFiles: new Map>(), + defaults: options.defaults, + backend, + generatedFiles, + }; + + return { ctx, progress, mode, placeholderCount }; +} diff --git a/src/react/renderers/dedup.ts b/src/react/renderers/dedup.ts new file mode 100644 index 00000000..83b0811a --- /dev/null +++ b/src/react/renderers/dedup.ts @@ -0,0 +1,69 @@ +/** + * Deduplication + progress tracking helpers for per-element renderers. + * + * Every generable renderer (image, video, speech, music, audio) follows + * the same pattern: check pre-resolved, compute cache key, check + * pendingFiles, track progress, generate, push to generatedFiles. + * These helpers eliminate that boilerplate. + */ + +import type { File } from "../../ai-sdk/file"; +import { ResolvedElement } from "../resolved-element"; +import type { VargElement } from "../types"; +import type { RenderContext } from "./context"; +import { addTask, completeTask, startTask } from "./progress"; +import { computeCacheKey } from "./utils"; + +/** GenerationType for progress tracking. */ +type GenType = "image" | "video" | "speech" | "music"; + +/** + * Run a generation function with deduplication and progress tracking. + * + * - Pre-resolved elements (awaited) short-circuit immediately. + * - Concurrent calls for the same element share a single promise via + * `ctx.pendingFiles` (dedup by cache key). + * - Progress task is created/completed around the generation call. + * + * @param element The VargElement being rendered. + * @param ctx The render context. + * @param genType Progress tracker type ("image" | "video" | "speech" | "music"). + * @param modelId Model ID for the progress label. + * @param fn The generation function — receives nothing, returns a File. + */ +export function withDedup( + element: VargElement, + ctx: RenderContext, + genType: GenType, + modelId: string, + fn: () => Promise, +): Promise { + // Pre-resolved (awaited) — reuse the pre-generated file. + if (element instanceof ResolvedElement) { + ctx.generatedFiles.push(element.meta.file); + return Promise.resolve(element.meta.file); + } + + // Dedup concurrent renders of the same element. + const cacheKeyStr = JSON.stringify(computeCacheKey(element)); + const pending = ctx.pendingFiles.get(cacheKeyStr); + if (pending) return pending; + + const promise = (async () => { + const taskId = ctx.progress + ? addTask(ctx.progress, genType, modelId) + : null; + if (taskId && ctx.progress) startTask(ctx.progress, taskId); + + try { + const file = await fn(); + ctx.generatedFiles.push(file); + return file; + } finally { + if (taskId && ctx.progress) completeTask(ctx.progress, taskId); + } + })(); + + ctx.pendingFiles.set(cacheKeyStr, promise); + return promise; +} diff --git a/src/react/renderers/flatten.ts b/src/react/renderers/flatten.ts new file mode 100644 index 00000000..3462a4d0 --- /dev/null +++ b/src/react/renderers/flatten.ts @@ -0,0 +1,125 @@ +/** + * Clip tree flattening — recursively flatten nested clips into leaf clips, + * hoist captions, and defer container-level audio. + * + * Extracted from render.ts so renderRoot stays a thin orchestrator. + */ + +import type { VargElement } from "../types"; + +export interface HoistedCaption { + element: VargElement<"captions">; + clipIndex: number; +} + +export interface DeferredAudio { + element: VargElement<"speech"> | VargElement<"music"> | VargElement<"audio">; + clipIndex: number; +} + +export interface FlattenResult { + clipElements: VargElement<"clip">[]; + hoistedCaptions: HoistedCaption[]; + deferredAudioElements: DeferredAudio[]; +} + +/** + * Recursively flatten nested clips into leaf clips. + * + * A "container clip" has child elements. Its non-clip children + * (Speech, Music, Captions) are hoisted/deferred to span the container's + * time region, which starts at the first leaf clip's timeline position. + * + * A "leaf clip" has no child elements — it contains visual/audio + * layers and is rendered directly by editly. + */ +export function flattenClips(rootChildren: VargElement[]): FlattenResult { + const clipElements: VargElement<"clip">[] = []; + const hoistedCaptions: HoistedCaption[] = []; + const deferredAudioElements: DeferredAudio[] = []; + let clipIndexCounter = 0; + + function flattenClip(clipElement: VargElement<"clip">): void { + const childClips: VargElement<"clip">[] = []; + const nonClipChildren: VargElement[] = []; + + for (const child of clipElement.children) { + if (!child || typeof child !== "object" || !("type" in child)) continue; + const el = child as VargElement; + if (el.type === "clip") { + childClips.push(el as VargElement<"clip">); + } else { + nonClipChildren.push(el); + } + } + + if (childClips.length === 0) { + // Leaf clip — hoist captions, keep everything else + const currentClipIndex = clipIndexCounter++; + const kept: typeof clipElement.children = []; + for (const el of nonClipChildren) { + if (el.type === "captions") { + hoistedCaptions.push({ + element: el as VargElement<"captions">, + clipIndex: currentClipIndex, + }); + } else { + kept.push(el); + } + } + clipElements.push({ + ...clipElement, + children: kept, + } as VargElement<"clip">); + return; + } + + // Container clip — has child clips. + const firstLeafClipIndex = clipIndexCounter; + + // Collect overlays from container level — inject into each child clip + const containerOverlays: VargElement[] = []; + for (const el of nonClipChildren) { + if (el.type === "overlay") { + containerOverlays.push(el); + } + } + + for (const childClip of childClips) { + if (containerOverlays.length > 0) { + childClip.children = [...childClip.children, ...containerOverlays]; + } + flattenClip(childClip); + } + + // Process remaining non-clip children at the container level + for (const el of nonClipChildren) { + if (el.type === "captions") { + hoistedCaptions.push({ + element: el as VargElement<"captions">, + clipIndex: firstLeafClipIndex, + }); + } else if ( + el.type === "speech" || + el.type === "music" || + el.type === "audio" + ) { + deferredAudioElements.push({ + element: el as + | VargElement<"speech"> + | VargElement<"music"> + | VargElement<"audio">, + clipIndex: firstLeafClipIndex, + }); + } + } + } + + for (const child of rootChildren) { + if (child.type === "clip") { + flattenClip(child as VargElement<"clip">); + } + } + + return { clipElements, hoistedCaptions, deferredAudioElements }; +} diff --git a/src/react/renderers/image.ts b/src/react/renderers/image.ts index 4ffd2a31..fae19c9a 100644 --- a/src/react/renderers/image.ts +++ b/src/react/renderers/image.ts @@ -1,34 +1,11 @@ import type { generateImage } from "ai"; import { File } from "../../ai-sdk/file"; import { ResolvedElement } from "../resolved-element"; -import type { - ImageInput, - ImagePrompt, - ImageProps, - VargElement, -} from "../types"; +import type { ImagePrompt, ImageProps, VargElement } from "../types"; import type { RenderContext } from "./context"; -import { addTask, completeTask, startTask } from "./progress"; -import { computeCacheKey, toFileUrl } from "./utils"; - -async function resolveImageInput( - input: ImageInput, - ctx: RenderContext, -): Promise { - if (input instanceof Uint8Array) { - return input; - } - if (typeof input === "string") { - const response = await fetch(toFileUrl(input)); - return new Uint8Array(await response.arrayBuffer()); - } - const file = await renderImage(input, ctx); - const data = await file.arrayBuffer(); - if (data instanceof ArrayBuffer) { - return new Uint8Array(data); - } - return data; -} +import { withDedup } from "./dedup"; +import { resolveImageInput } from "./inputs"; +import { computeCacheKey } from "./utils"; async function resolvePrompt( prompt: ImagePrompt, @@ -47,7 +24,6 @@ export async function renderImage( element: VargElement<"image">, ctx: RenderContext, ): Promise { - // If already resolved via `await Image(...)`, reuse the pre-generated file if (element instanceof ResolvedElement) { ctx.generatedFiles.push(element.meta.file); return element.meta.file; @@ -73,23 +49,12 @@ export async function renderImage( ); } + const modelId = typeof model === "string" ? model : model.modelId; const cacheKey = computeCacheKey(element); - const cacheKeyStr = JSON.stringify(cacheKey); - const pendingRender = ctx.pendingFiles.get(cacheKeyStr); - if (pendingRender) { - return pendingRender; - } - - const renderPromise = (async () => { + return withDedup(element, ctx, "image", modelId, async () => { const resolvedPrompt = await resolvePrompt(prompt, ctx); - const modelId = typeof model === "string" ? model : model.modelId; - const taskId = ctx.progress - ? addTask(ctx.progress, "image", modelId) - : null; - if (taskId && ctx.progress) startTask(ctx.progress, taskId); - const { images } = await ctx.generateImage({ model, prompt: resolvedPrompt, @@ -99,8 +64,6 @@ export async function renderImage( cacheKey, } as Parameters[0]); - if (taskId && ctx.progress) completeTask(ctx.progress, taskId); - const firstImage = images[0]; if (!firstImage?.uint8Array) { throw new Error("Image generation returned no image data"); @@ -123,12 +86,6 @@ export async function renderImage( await file.upload(ctx.storage); } - ctx.generatedFiles.push(file); - return file; - })(); - - ctx.pendingFiles.set(cacheKeyStr, renderPromise); - - return renderPromise; + }); } diff --git a/src/react/renderers/index.ts b/src/react/renderers/index.ts index e3e8c583..dfee3b85 100644 --- a/src/react/renderers/index.ts +++ b/src/react/renderers/index.ts @@ -1,6 +1,7 @@ export { renderCaptions } from "./captions"; export { renderClip } from "./clip"; export type { RenderContext } from "./context"; +export { createRenderContext, type PreparedRender } from "./context-builder"; export { renderImage } from "./image"; export { renderPackshot } from "./packshot"; export { diff --git a/src/react/renderers/inputs.ts b/src/react/renderers/inputs.ts new file mode 100644 index 00000000..f455ae2b --- /dev/null +++ b/src/react/renderers/inputs.ts @@ -0,0 +1,92 @@ +/** + * Shared input resolution helpers — convert string/Uint8Array/element + * inputs into raw bytes for generation prompts. + * + * Used by image.ts and video.ts (and indirectly by any renderer that + * accepts image/video/audio inputs). + */ + +import type { File } from "../../ai-sdk/file"; +import { ResolvedElement } from "../resolved-element"; +import type { ImageInput, VargElement } from "../types"; +import type { RenderContext } from "./context"; +import { renderImage } from "./image"; +import { renderSpeech } from "./speech"; +import { toFileUrl } from "./utils"; + +/** Resolve an image input (Uint8Array | path/URL | Image element) to bytes. */ +export async function resolveImageInput( + input: ImageInput, + ctx: RenderContext, +): Promise { + if (input instanceof Uint8Array) { + return input; + } + if (typeof input === "string") { + const response = await fetch(toFileUrl(input)); + return new Uint8Array(await response.arrayBuffer()); + } + const file = await renderImage(input, ctx); + const data = await file.arrayBuffer(); + if (data instanceof ArrayBuffer) { + return new Uint8Array(data); + } + return data; +} + +/** Resolve a video input (Uint8Array | path/URL | Video element) to bytes. */ +export async function resolveVideoInput( + input: Uint8Array | string | VargElement<"video"> | undefined, + ctx: RenderContext, +): Promise { + if (!input) return undefined; + if (input instanceof Uint8Array) return input; + if (typeof input === "string") { + const response = await fetch(toFileUrl(input)); + return new Uint8Array(await response.arrayBuffer()); + } + if (input.type === "video") { + const { renderVideo } = await import("./video"); + const file = await renderVideo(input, ctx); + return file.arrayBuffer(); + } + throw new Error( + `Unsupported video input type: ${(input as VargElement).type}`, + ); +} + +/** Resolve an audio input (Uint8Array | path/URL | speech/audio element) to bytes. */ +export async function resolveAudioInput( + input: + | Uint8Array + | string + | VargElement<"speech"> + | VargElement<"audio"> + | undefined, + ctx: RenderContext, +): Promise { + if (!input) return undefined; + if (input instanceof Uint8Array) return input; + if (typeof input === "string") { + const response = await fetch(toFileUrl(input)); + return new Uint8Array(await response.arrayBuffer()); + } + if ( + input instanceof ResolvedElement && + (input.type === "speech" || input.type === "audio") + ) { + return input.meta.file.arrayBuffer(); + } + if (input.type === "speech") { + const file = await renderSpeech(input as VargElement<"speech">, ctx); + return file.arrayBuffer(); + } + if (input.type === "audio") { + const { renderAudio } = await import("./audio"); + const file = await renderAudio(input as VargElement<"audio">, ctx); + return file.arrayBuffer(); + } + throw new Error( + `Unsupported audio input type: ${(input as VargElement).type}`, + ); +} diff --git a/src/react/renderers/music.ts b/src/react/renderers/music.ts index a3a85d7a..729c8ea9 100644 --- a/src/react/renderers/music.ts +++ b/src/react/renderers/music.ts @@ -3,14 +3,13 @@ import type { generateMusic } from "../../ai-sdk/generate-music"; import { ResolvedElement } from "../resolved-element"; import type { MusicProps, VargElement } from "../types"; import type { RenderContext } from "./context"; -import { addTask, completeTask, startTask } from "./progress"; +import { withDedup } from "./dedup"; import { computeCacheKey } from "./utils"; export async function renderMusic( element: VargElement<"music">, ctx: RenderContext, ): Promise { - // If already resolved via `await Music(...)`, reuse the pre-generated file if (element instanceof ResolvedElement) { ctx.generatedFiles.push(element.meta.file); return element.meta.file; @@ -24,22 +23,10 @@ export async function renderMusic( throw new Error("Music requires prompt and model (or set defaults.music)"); } + const modelId = model.modelId ?? "music"; const cacheKey = computeCacheKey(element); - const cacheKeyStr = JSON.stringify(cacheKey); - - // Deduplicate concurrent renders of the same music element - const pendingRender = ctx.pendingFiles.get(cacheKeyStr); - if (pendingRender) { - return pendingRender; - } - - const renderPromise = (async () => { - const modelId = model.modelId ?? "music"; - const taskId = ctx.progress - ? addTask(ctx.progress, "music", modelId) - : null; - if (taskId && ctx.progress) startTask(ctx.progress, taskId); + return withDedup(element, ctx, "music", modelId, async () => { const { audio } = await ctx.generateMusic({ model, prompt, @@ -47,8 +34,6 @@ export async function renderMusic( cacheKey, } as Parameters[0]); - if (taskId && ctx.progress) completeTask(ctx.progress, taskId); - if (!audio?.uint8Array) { throw new Error( `We couldn't generate the background music for this video (music model "${modelId}"). ` + @@ -70,12 +55,6 @@ export async function renderMusic( prompt, }); - ctx.generatedFiles.push(file); - return file; - })(); - - ctx.pendingFiles.set(cacheKeyStr, renderPromise); - - return renderPromise; + }); } diff --git a/src/react/renderers/render.ts b/src/react/renderers/render.ts index f3360bc1..dd75771e 100644 --- a/src/react/renderers/render.ts +++ b/src/react/renderers/render.ts @@ -1,21 +1,16 @@ -import type { ImageModelV3 } from "@ai-sdk/provider"; -import { - generateImage, - experimental_generateSpeech as generateSpeech, - wrapImageModel, -} from "ai"; +/** + * Compose phase: walk the (already materialized) tree, assemble the + * timeline, and run editly + captions burn-in. + * + * When called via `render()`, every generable element has been executed by + * the plan executor and carries `meta` — the per-element renderers inside + * only short-circuit on pre-resolved files. Calling renderRoot directly + * (without a prepared context) still works: renderers generate on demand, + * preserving the legacy recursive behavior. + */ + import pMap from "p-map"; -import { type CacheStorage, withCache } from "../../ai-sdk/cache"; -import type { File, File as VargFile } from "../../ai-sdk/file"; -import { fileCache } from "../../ai-sdk/file-cache"; -import { generateMusic } from "../../ai-sdk/generate-music"; -import { generateVideo } from "../../ai-sdk/generate-video"; -import { - imagePlaceholderFallbackMiddleware, - placeholderFallbackMiddleware, - wrapVideoModel, -} from "../../ai-sdk/middleware"; -import { editly, localBackend } from "../../ai-sdk/providers/editly"; +import { editly } from "../../ai-sdk/providers/editly"; import type { AudioTrack, Clip, @@ -27,7 +22,6 @@ import type { ClipProps, MusicProps, OverlayProps, - RenderMode, RenderOptions, RenderProps, RenderResult, @@ -38,17 +32,13 @@ import { audioMixVolume, renderAudio } from "./audio"; import { burnCaptions } from "./burn-captions"; import { type CaptionsResult, renderCaptions } from "./captions"; import { renderClip } from "./clip"; -import type { RenderContext } from "./context"; +import { createRenderContext, type PreparedRender } from "./context-builder"; import type { EmojiOverlay } from "./emoji"; +import { type FlattenResult, flattenClips } from "./flatten"; import { renderImage } from "./image"; import { mergeAssFiles, shiftAssTimestamps } from "./merge-ass"; import { renderMusic } from "./music"; -import { - addTask, - completeTask, - createProgressTracker, - startTask, -} from "./progress"; +import { addTask, completeTask, startTask } from "./progress"; import { renderSpeech } from "./speech"; import { resolvePath } from "./utils"; import { renderVideo } from "./video"; @@ -59,361 +49,37 @@ interface RenderedOverlay { isVideo: boolean; } -function resolveCacheStorage( - cache: string | CacheStorage | undefined, -): CacheStorage | undefined { - if (!cache) return undefined; - if (typeof cache === "string") { - return fileCache({ dir: cache }); - } - return cache; -} - -function toImageModelV3( - model: Parameters[0]["model"], -): ImageModelV3 { - if (typeof model === "object" && model.specificationVersion === "v3") { - return model; - } - // for string IDs and v2 models, create a shell that satisfies the type. - // in preview mode the middleware intercepts before doGenerate is called. - const modelId = typeof model === "string" ? model : model.modelId; - return { - specificationVersion: "v3", - provider: "placeholder", - modelId, - maxImagesPerCall: 1, - doGenerate: async () => { - throw new Error( - `toImageModelV3 shell: doGenerate should not be called in preview mode (model: ${modelId})`, - ); - }, - }; -} - -/** RenderContext plus the render-scoped state renderRoot needs alongside it. */ -export interface PreparedRender { - ctx: RenderContext; - progress: ReturnType; - mode: RenderMode; - placeholderCount: { images: number; videos: number; total: number }; -} - -/** - * Build the RenderContext (cached+wrapped generators, backend, progress) - * for a composition. Shared by the plan executor and the compose phase so - * both see the same pendingFiles dedup map, progress tracker, and - * generatedFiles list. - */ -export function createRenderContext( - element: VargElement<"render">, - options: RenderOptions, -): PreparedRender { - const props = element.props as RenderProps; - const progress = createProgressTracker(options.quiet ?? false); - - const mode: RenderMode = options.mode ?? "strict"; - const placeholderCount = { images: 0, videos: 0, total: 0 }; - - const trackPlaceholder = (type: "image" | "video") => { - placeholderCount[type === "image" ? "images" : "videos"]++; - placeholderCount.total++; - }; - - const cacheStorage = resolveCacheStorage(options.cache); - - const cachedGenerateImage = cacheStorage - ? withCache(generateImage, { storage: cacheStorage }) - : generateImage; - - const cachedGenerateVideo = cacheStorage - ? withCache(generateVideo, { storage: cacheStorage }) - : generateVideo; - - const cachedGenerateSpeech = cacheStorage - ? withCache(generateSpeech, { storage: cacheStorage }) - : generateSpeech; - - const cachedGenerateMusic = cacheStorage - ? withCache(generateMusic, { storage: cacheStorage }) - : generateMusic; - - const onGeneration = options.onGeneration; - - /** Extract pricing metadata from provider response and emit via callback. */ - // biome-ignore lint/suspicious/noExplicitAny: result shapes vary across AI SDK model types - const emitPricing = ( - type: "image" | "video" | "speech" | "music", - modelId: string, - result: any, - ) => { - if (!onGeneration) return; - const vargMeta = result?.providerMetadata?.varg as - | { pricing?: Record; jobId?: string } - | undefined; - if (vargMeta?.pricing) { - const p = vargMeta.pricing; - onGeneration({ - type, - model: modelId, - estimated: p.estimated as number | undefined, - actual: p.actual as number | undefined, - billing: p.billing as "metered" | "byok" | "x402" | undefined, - cached: p.cached as boolean | undefined, - jobId: vargMeta.jobId, - }); - } - }; - - const wrapGenerateImage: typeof generateImage = async (opts) => { - if (mode === "preview") { - trackPlaceholder("image"); - return cachedGenerateImage({ - ...opts, - model: wrapImageModel({ - model: toImageModelV3(opts.model), - middleware: imagePlaceholderFallbackMiddleware({ - mode: "preview", - onFallback: () => {}, - }), - }), - skipCacheWrite: true, - } as Parameters[0]); - } - - const result = await cachedGenerateImage(opts); - const imgModelId = - typeof opts.model === "string" ? opts.model : opts.model.modelId; - emitPricing("image", imgModelId, result); - return result; - }; - - const wrapGenerateVideo: typeof generateVideo = async (opts) => { - if (mode === "preview") { - trackPlaceholder("video"); - return cachedGenerateVideo({ - ...opts, - model: wrapVideoModel({ - model: opts.model, - middleware: placeholderFallbackMiddleware({ - mode: "preview", - onFallback: () => {}, - }), - }), - skipCacheWrite: true, - } as Parameters[0]); - } - - const result = await cachedGenerateVideo(opts); - emitPricing("video", opts.model.modelId, result); - return result; - }; - - const wrapGenerateSpeech: typeof generateSpeech = async (opts) => { - const result = await cachedGenerateSpeech(opts); - const speechModelId = - typeof opts.model === "string" ? opts.model : opts.model.modelId; - emitPricing("speech", speechModelId, result); - return result; - }; - - const wrapGenerateMusic: typeof generateMusic = async (opts) => { - const result = await cachedGenerateMusic(opts); - const musicModelId = - typeof opts.model === "string" ? opts.model : opts.model.modelId; - emitPricing("music", musicModelId, result); - return result; - }; - - const backend = options.backend ?? localBackend; - const tempFiles: string[] = []; - const generatedFiles: VargFile[] = []; - const ctx: RenderContext = { - width: props.width ?? 1920, - height: props.height ?? 1080, - fps: props.fps ?? 30, - cache: cacheStorage, - storage: options.storage, - generateImage: wrapGenerateImage, - generateVideo: wrapGenerateVideo, - generateSpeech: onGeneration ? wrapGenerateSpeech : cachedGenerateSpeech, - generateMusic: onGeneration ? wrapGenerateMusic : cachedGenerateMusic, - tempFiles, - progress, - pendingFiles: new Map>(), - defaults: options.defaults, - backend, - generatedFiles, - }; - - return { ctx, progress, mode, placeholderCount }; -} - /** - * Compose phase: walk the (already materialized) tree, assemble the - * timeline, and run editly + captions burn-in. - * - * When called via `render()`, every generable element has been executed by - * the plan executor and carries `meta` — the per-element renderers inside - * only short-circuit on pre-resolved files. Calling renderRoot directly - * (without a prepared context) still works: renderers generate on demand, - * preserving the legacy recursive behavior. + * Walk children, flatten clips, collect overlays/music/audio. */ -export async function renderRoot( +async function collectTopLevel( element: VargElement<"render">, - options: RenderOptions, - prepared?: PreparedRender, -): Promise { - const props = element.props as RenderProps; - const { ctx, progress, mode, placeholderCount } = - prepared ?? createRenderContext(element, options); - const generatedFiles = ctx.generatedFiles; - - const clipElements: VargElement<"clip">[] = []; + ctx: ReturnType["ctx"], +): Promise<{ + flatten: FlattenResult; + overlayElements: VargElement<"overlay">[]; + musicElements: VargElement<"music">[]; + audioTracks: AudioTrack[]; + hoistedCaptions: FlattenResult["hoistedCaptions"]; +}> { const overlayElements: VargElement<"overlay">[] = []; const musicElements: VargElement<"music">[] = []; const audioTracks: AudioTrack[] = []; - // --------------------------------------------------------------------------- - // Hoisted captions: track which clip each caption came from so we can apply - // the correct timeline offset when stitching audio and ASS files. - // --------------------------------------------------------------------------- - interface HoistedCaption { - element: VargElement<"captions">; - clipIndex: number; - } - const hoistedCaptions: HoistedCaption[] = []; - - // --------------------------------------------------------------------------- - // Deferred audio: speech/music at a container-clip level that needs to be - // offset to the container's start position in the timeline. - // We don't know the offset yet (clips haven't been laid out), so we record - // the clipIndex of the first leaf clip inside the container and resolve the - // offset later, after clipStartOffsets are computed. - // --------------------------------------------------------------------------- - interface DeferredAudio { - element: - | VargElement<"speech"> - | VargElement<"music"> - | VargElement<"audio">; - clipIndex: number; // first leaf clip of the container - } - const deferredAudioElements: DeferredAudio[] = []; - - // --------------------------------------------------------------------------- - // flattenClip: recursively flatten nested clips into leaf clips. - // - // A "container clip" has child elements. Its non-clip children - // (Speech, Music, Captions) are hoisted/deferred to span the container's - // time region, which starts at the first leaf clip's timeline position. - // - // A "leaf clip" has no child elements — it contains visual/audio - // layers and is rendered directly by editly. - // --------------------------------------------------------------------------- - let clipIndexCounter = 0; - - function flattenClip(clipElement: VargElement<"clip">): void { - const childClips: VargElement<"clip">[] = []; - const nonClipChildren: VargElement[] = []; - - for (const child of clipElement.children) { - if (!child || typeof child !== "object" || !("type" in child)) continue; - const el = child as VargElement; - if (el.type === "clip") { - childClips.push(el as VargElement<"clip">); - } else { - nonClipChildren.push(el); - } - } - - if (childClips.length === 0) { - // Leaf clip — hoist captions, keep everything else - const currentClipIndex = clipIndexCounter++; - const kept: typeof clipElement.children = []; - for (const el of nonClipChildren) { - if (el.type === "captions") { - hoistedCaptions.push({ - element: el as VargElement<"captions">, - clipIndex: currentClipIndex, - }); - } else { - kept.push(el); - } - } - clipElements.push({ - ...clipElement, - children: kept, - } as VargElement<"clip">); - return; - } - - // Container clip — has child clips. - // The first leaf clip's index is used as the anchor for audio/captions - // offsets (they all start at the container's position in the timeline). - const firstLeafClipIndex = clipIndexCounter; // before recursion increments it - - // Collect overlays from container level — these get injected into each - // child clip so the overlay appears across all inner clips. - const containerOverlays: VargElement[] = []; - for (const el of nonClipChildren) { - if (el.type === "overlay") { - containerOverlays.push(el); - } - } - - // Recurse into child clips, injecting container-level overlays - for (const childClip of childClips) { - if (containerOverlays.length > 0) { - childClip.children = [...childClip.children, ...containerOverlays]; - } - flattenClip(childClip); - } - - // Process remaining non-clip children at the container level - for (const el of nonClipChildren) { - if (el.type === "captions") { - hoistedCaptions.push({ - element: el as VargElement<"captions">, - clipIndex: firstLeafClipIndex, - }); - } else if ( - el.type === "speech" || - el.type === "music" || - el.type === "audio" - ) { - deferredAudioElements.push({ - element: el as - | VargElement<"speech"> - | VargElement<"music"> - | VargElement<"audio">, - clipIndex: firstLeafClipIndex, - }); - } - // overlay: already handled above (distributed to child clips) - // Image/Video at container level: not supported yet (would need - // background layer spanning all child clips — a future feature) - } - } - - // --------------------------------------------------------------------------- - // Process all children of - // --------------------------------------------------------------------------- + // Collect non-clip top-level children + const clipChildren: VargElement<"clip">[] = []; for (const child of element.children) { if (!child || typeof child !== "object" || !("type" in child)) continue; const childElement = child as VargElement; if (childElement.type === "clip") { - flattenClip(childElement as VargElement<"clip">); + clipChildren.push(childElement as VargElement<"clip">); } else if (childElement.type === "overlay") { overlayElements.push(childElement as VargElement<"overlay">); } else if (childElement.type === "captions") { - // Render-level captions — hoist with clipIndex 0 (start of timeline) - hoistedCaptions.push({ - element: childElement as VargElement<"captions">, - clipIndex: 0, - }); + // Render-level captions — hoist with clipIndex 0 + // (handled in flatten result, but we need to add it manually) } else if (childElement.type === "speech") { - // Render-level speech — immediate audio track (no offset needed) const file = childElement instanceof ResolvedElement ? childElement.meta.file @@ -425,7 +91,6 @@ export async function renderRoot( mixVolume: speechProps.volume ?? 1, }); } else if (childElement.type === "audio") { - // Render-level derived audio node — immediate audio track const file = await renderAudio(childElement as VargElement<"audio">, ctx); const path = await ctx.backend.resolvePath(file); audioTracks.push({ @@ -437,18 +102,44 @@ export async function renderRoot( } } - // Hoisted captions are processed AFTER clip timeline offsets are computed - // (see below) so that each caption's audio can be delayed to the correct - // clip start time and ASS timestamps can be shifted accordingly. + // Flatten clip tree + const flatten = flattenClips(clipChildren); + // Add render-level captions (clipIndex 0) + for (const child of element.children) { + if (!child || typeof child !== "object" || !("type" in child)) continue; + const childElement = child as VargElement; + if (childElement.type === "captions") { + flatten.hoistedCaptions.push({ + element: childElement as VargElement<"captions">, + clipIndex: 0, + }); + } + } + + return { + flatten, + overlayElements, + musicElements, + audioTracks, + hoistedCaptions: flatten.hoistedCaptions, + }; +} + +async function renderOverlays( + overlayElements: VargElement<"overlay">[], + ctx: ReturnType["ctx"], +): Promise<{ renderedOverlays: RenderedOverlay[]; audioTracks: AudioTrack[] }> { const renderedOverlays: RenderedOverlay[] = []; + const audioTracks: AudioTrack[] = []; + for (const overlay of overlayElements) { const overlayProps = overlay.props as OverlayProps; for (const child of overlay.children) { if (!child || typeof child !== "object" || !("type" in child)) continue; const childElement = child as VargElement; - let file: File | undefined; + let file: import("../../ai-sdk/file").File | undefined; const isVideo = childElement.type === "video"; if (childElement.type === "video") { @@ -471,9 +162,33 @@ export async function renderRoot( } } + return { renderedOverlays, audioTracks }; +} + +export async function renderRoot( + element: VargElement<"render">, + options: RenderOptions, + prepared?: PreparedRender, +): Promise { + const props = element.props as RenderProps; + const { ctx, progress, mode, placeholderCount } = + prepared ?? createRenderContext(element, options); + const generatedFiles = ctx.generatedFiles; + + // 1. Collect top-level children + flatten clip tree + const { flatten, overlayElements, musicElements, audioTracks } = + await collectTopLevel(element, ctx); + + // 2. Render overlays + const { renderedOverlays, audioTracks: overlayAudio } = await renderOverlays( + overlayElements, + ctx, + ); + audioTracks.push(...overlayAudio); + + // 3. Render clips in parallel const concurrency = options.concurrency === undefined ? 3 : options.concurrency; - if ( concurrency !== Number.POSITIVE_INFINITY && (!Number.isInteger(concurrency) || concurrency < 1) @@ -482,7 +197,7 @@ export async function renderRoot( } const clipResults = await pMap( - clipElements, + flatten.clipElements, async (clipElement, i) => { try { return { @@ -531,12 +246,13 @@ export async function renderRoot( return r.value; }); + // 4. Assemble timeline with clip start offsets + overlay layers const clips: Clip[] = []; const clipStartOffsets: number[] = []; let currentTime = 0; - for (let i = 0; i < clipElements.length; i++) { - const clipElement = clipElements[i]; + for (let i = 0; i < flatten.clipElements.length; i++) { + const clipElement = flatten.clipElements[i]; const clip = renderedClips[i]; if (!clipElement || !clip) { throw new Error(`Missing clip data at index ${i}`); @@ -564,19 +280,18 @@ export async function renderRoot( clips.push(clip); currentTime += clipDuration; - if (i < clipElements.length - 1 && clip.transition) { + if (i < flatten.clipElements.length - 1 && clip.transition) { currentTime -= clip.transition.duration ?? 0; } } const totalDuration = currentTime; - // --------------------------------------------------------------------------- - // Process deferred audio from container clips. - // Now that clipStartOffsets are known, we can resolve the audio elements - // and set the correct start offset in the timeline. - // --------------------------------------------------------------------------- - for (const { element: audioElement, clipIndex } of deferredAudioElements) { + // 5. Process deferred audio from container clips + for (const { + element: audioElement, + clipIndex, + } of flatten.deferredAudioElements) { const offset = clipStartOffsets[clipIndex] ?? 0; if (audioElement.type === "speech") { const file = @@ -622,16 +337,15 @@ export async function renderRoot( } } - // --------------------------------------------------------------------------- - // Process hoisted captions (from leaf clips, container clips, and Render - // level). Now that clipStartOffsets are known, each caption's audio can be - // delayed and ASS timestamps shifted to the correct timeline position. - // --------------------------------------------------------------------------- + // 6. Process hoisted captions const hoistedCaptionsResults: CaptionsResult[] = []; let mergedAssPath: string | undefined; - if (hoistedCaptions.length > 0) { - for (const { element: captionsElement, clipIndex } of hoistedCaptions) { + if (flatten.hoistedCaptions.length > 0) { + for (const { + element: captionsElement, + clipIndex, + } of flatten.hoistedCaptions) { const result = await renderCaptions(captionsElement, ctx); hoistedCaptionsResults.push(result); @@ -646,7 +360,8 @@ export async function renderRoot( // Merge ASS files: shift timestamps by each clip's start offset if (hoistedCaptionsResults.length === 1) { - const offset = clipStartOffsets[hoistedCaptions[0]!.clipIndex] ?? 0; + const offset = + clipStartOffsets[flatten.hoistedCaptions[0]!.clipIndex] ?? 0; const assPath = hoistedCaptionsResults[0]!.assPath; mergedAssPath = offset > 0 ? shiftAssTimestamps(assPath, offset) : assPath; @@ -656,7 +371,8 @@ export async function renderRoot( } else if (hoistedCaptionsResults.length > 1) { const segments = hoistedCaptionsResults.map((result, i) => ({ assPath: result.assPath, - timeOffset: clipStartOffsets[hoistedCaptions[i]!.clipIndex] ?? 0, + timeOffset: + clipStartOffsets[flatten.hoistedCaptions[i]!.clipIndex] ?? 0, styleSuffix: `_${i}`, })); mergedAssPath = mergeAssFiles(segments, ctx.width, ctx.height); @@ -664,7 +380,7 @@ export async function renderRoot( } } - // process music after clips so we know total duration for auto-trim + // 7. Process music after clips (need total duration for auto-trim) for (const musicElement of musicElements) { const musicProps = musicElement.props as MusicProps; const cutFrom = musicProps.cutFrom ?? 0; @@ -672,7 +388,7 @@ export async function renderRoot( musicProps.cutTo ?? (musicProps.duration !== undefined ? cutFrom + musicProps.duration - : totalDuration); // auto-trim to video length + : totalDuration); let path: string; if (musicProps.src) { @@ -693,10 +409,8 @@ export async function renderRoot( }); } - // Determine the ASS path to burn from merged hoisted captions. - const finalAssPath = mergedAssPath; - const hasCaptions = finalAssPath !== undefined; - + // 8. Run editly + const hasCaptions = mergedAssPath !== undefined; const tempOutPath = hasCaptions ? `/tmp/varg-pre-captions-${Date.now()}.mp4` : (options.output ?? `output/varg-${Date.now()}.mp4`); @@ -721,7 +435,8 @@ export async function renderRoot( let output = editlyResult.output; - if (hasCaptions && finalAssPath) { + // 9. Burn captions + if (hasCaptions && mergedAssPath) { const captionsTaskId = addTask(progress, "captions", "ffmpeg"); startTask(progress, captionsTaskId); @@ -738,7 +453,8 @@ export async function renderRoot( const allEmojiOverlays: EmojiOverlay[] = []; for (let i = 0; i < hoistedCaptionsResults.length; i++) { const result = hoistedCaptionsResults[i]!; - const offset = clipStartOffsets[hoistedCaptions[i]!.clipIndex] ?? 0; + const offset = + clipStartOffsets[flatten.hoistedCaptions[i]!.clipIndex] ?? 0; for (const overlay of result.emojiOverlays ?? []) { allEmojiOverlays.push( offset > 0 @@ -754,7 +470,7 @@ export async function renderRoot( output = await burnCaptions({ video: output, - assPath: finalAssPath, + assPath: mergedAssPath, outputPath: finalOutPath, backend: options.backend, verbose: options.verbose, @@ -769,6 +485,7 @@ export async function renderRoot( completeTask(progress, captionsTaskId); } + // 10. Read final output let finalBuffer: ArrayBuffer; if (output.type === "url") { const res = await fetch(output.url); diff --git a/src/react/renderers/speech.ts b/src/react/renderers/speech.ts index 2e9a9d9e..d0928493 100644 --- a/src/react/renderers/speech.ts +++ b/src/react/renderers/speech.ts @@ -3,14 +3,13 @@ import { File } from "../../ai-sdk/file"; import { ResolvedElement } from "../resolved-element"; import type { SpeechProps, VargElement } from "../types"; import type { RenderContext } from "./context"; -import { addTask, completeTask, startTask } from "./progress"; +import { withDedup } from "./dedup"; import { computeCacheKey, getTextContent } from "./utils"; export async function renderSpeech( element: VargElement<"speech">, ctx: RenderContext, ): Promise { - // If already resolved via `await Speech(...)`, reuse the pre-generated file if (element instanceof ResolvedElement) { ctx.generatedFiles.push(element.meta.file); return element.meta.file; @@ -28,22 +27,10 @@ export async function renderSpeech( throw new Error("Speech requires 'model' prop (or set defaults.speech)"); } + const modelId = typeof model === "string" ? model : model.modelId; const cacheKey = computeCacheKey(element); - const cacheKeyStr = JSON.stringify(cacheKey); - - // Deduplicate concurrent renders of the same speech element - const pendingRender = ctx.pendingFiles.get(cacheKeyStr); - if (pendingRender) { - return pendingRender; - } - - const renderPromise = (async () => { - const modelId = typeof model === "string" ? model : model.modelId; - const taskId = ctx.progress - ? addTask(ctx.progress, "speech", modelId) - : null; - if (taskId && ctx.progress) startTask(ctx.progress, taskId); + return withDedup(element, ctx, "speech", modelId, async () => { const { audio } = await ctx.generateSpeech({ model, text, @@ -51,8 +38,6 @@ export async function renderSpeech( cacheKey, } as Parameters[0]); - if (taskId && ctx.progress) completeTask(ctx.progress, taskId); - if (!audio?.uint8Array) { throw new Error( `We couldn't generate the voiceover for this scene (speech model "${modelId}"). ` + @@ -74,12 +59,6 @@ export async function renderSpeech( prompt: text, }); - ctx.generatedFiles.push(file); - return file; - })(); - - ctx.pendingFiles.set(cacheKeyStr, renderPromise); - - return renderPromise; + }); } diff --git a/src/react/renderers/video.ts b/src/react/renderers/video.ts index cfffbaf0..6d286e0e 100644 --- a/src/react/renderers/video.ts +++ b/src/react/renderers/video.ts @@ -1,87 +1,15 @@ import { File } from "../../ai-sdk/file"; import type { generateVideo } from "../../ai-sdk/generate-video"; import { ResolvedElement } from "../resolved-element"; -import type { - ImageInput, - VargElement, - VideoPrompt, - VideoProps, -} from "../types"; +import type { VargElement, VideoPrompt, VideoProps } from "../types"; import type { RenderContext } from "./context"; -import { renderImage } from "./image"; -import { addTask, completeTask, startTask } from "./progress"; -import { renderSpeech } from "./speech"; -import { computeCacheKey, toFileUrl } from "./utils"; - -async function resolveImageInput( - input: ImageInput, - ctx: RenderContext, -): Promise { - if (input instanceof Uint8Array) { - return input; - } - if (typeof input === "string") { - const response = await fetch(toFileUrl(input)); - return new Uint8Array(await response.arrayBuffer()); - } - const file = await renderImage(input, ctx); - return file.arrayBuffer(); -} - -async function resolveAudioInput( - input: - | Uint8Array - | string - | VargElement<"speech"> - | VargElement<"audio"> - | undefined, - ctx: RenderContext, -): Promise { - if (!input) return undefined; - if (input instanceof Uint8Array) return input; - if (typeof input === "string") { - const response = await fetch(toFileUrl(input)); - return new Uint8Array(await response.arrayBuffer()); - } - // Resolved speech/audio element — use pre-generated file directly - if ( - input instanceof ResolvedElement && - (input.type === "speech" || input.type === "audio") - ) { - return input.meta.file.arrayBuffer(); - } - if (input.type === "speech") { - const file = await renderSpeech(input as VargElement<"speech">, ctx); - return file.arrayBuffer(); - } - if (input.type === "audio") { - const { renderAudio } = await import("./audio"); - const file = await renderAudio(input as VargElement<"audio">, ctx); - return file.arrayBuffer(); - } - throw new Error( - `Unsupported audio input type: ${(input as VargElement).type}`, - ); -} - -async function resolveVideoInput( - input: Uint8Array | string | VargElement<"video"> | undefined, - ctx: RenderContext, -): Promise { - if (!input) return undefined; - if (input instanceof Uint8Array) return input; - if (typeof input === "string") { - const response = await fetch(toFileUrl(input)); - return new Uint8Array(await response.arrayBuffer()); - } - if (input.type === "video") { - const file = await renderVideo(input, ctx); - return file.arrayBuffer(); - } - throw new Error( - `Unsupported video input type: ${(input as VargElement).type}`, - ); -} +import { withDedup } from "./dedup"; +import { + resolveAudioInput, + resolveImageInput, + resolveVideoInput, +} from "./inputs"; +import { computeCacheKey } from "./utils"; async function resolvePrompt( prompt: VideoPrompt, @@ -164,23 +92,12 @@ export async function renderVideo( ); } + const modelId = typeof model === "string" ? model : model.modelId; const cacheKey = computeCacheKey(element); - const cacheKeyStr = JSON.stringify(cacheKey); - - const pendingRender = ctx.pendingFiles.get(cacheKeyStr); - if (pendingRender) { - return pendingRender; - } - const renderPromise = (async () => { + return withDedup(element, ctx, "video", modelId, async () => { const resolvedPrompt = await resolvePrompt(prompt, ctx); - const modelId = typeof model === "string" ? model : model.modelId; - const taskId = ctx.progress - ? addTask(ctx.progress, "video", modelId) - : null; - if (taskId && ctx.progress) startTask(ctx.progress, taskId); - const { video } = await ctx.generateVideo({ model, prompt: resolvedPrompt, @@ -190,8 +107,6 @@ export async function renderVideo( cacheKey, } as Parameters[0]); - if (taskId && ctx.progress) completeTask(ctx.progress, taskId); - const mediaType = video.mimeType ?? "video/mp4"; const promptText = typeof resolvedPrompt === "string" ? resolvedPrompt : resolvedPrompt.text; @@ -211,12 +126,6 @@ export async function renderVideo( await file.upload(ctx.storage); } - ctx.generatedFiles.push(file); - return file; - })(); - - ctx.pendingFiles.set(cacheKeyStr, renderPromise); - - return renderPromise; + }); } From 3de8e93dca19d6206d6743e9ba86d381d4331bdc Mon Sep 17 00:00:00 2001 From: SecurityQQ Date: Mon, 20 Jul 2026 23:10:11 -0700 Subject: [PATCH 2/2] refactor(captions): split into srt.ts + ass.ts + captions.ts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit captions.ts (743 lines) split into: srt.ts (88 lines) — formatTime, convertToSRT, parseSrt, SrtEntry ass.ts (306 lines) — SubtitleStyle, STYLE_PRESETS, convertSrtToAss, convertSrtToAssGrouped, formatAssTime, colorToAss, EntryEmojiData captions.ts (370 lines) — renderCaptions orchestrator + helpers (resolveSrtContent, buildEmojiOverlays) Each file has a single responsibility: srt.ts — SRT format <-> structured data ass.ts — SRT -> ASS conversion with styles + emoji data captions.ts — orchestration: transcription, font resolution, emoji overlays Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- src/react/renderers/ass.ts | 306 ++++++++++ src/react/renderers/captions.test.ts | 2 +- src/react/renderers/captions.ts | 815 +++++++-------------------- src/react/renderers/srt.ts | 88 +++ 4 files changed, 614 insertions(+), 597 deletions(-) create mode 100644 src/react/renderers/ass.ts create mode 100644 src/react/renderers/srt.ts diff --git a/src/react/renderers/ass.ts b/src/react/renderers/ass.ts new file mode 100644 index 00000000..3b541b7c --- /dev/null +++ b/src/react/renderers/ass.ts @@ -0,0 +1,306 @@ +/** + * ASS subtitle generation — convert SRT to ASS format with style presets, + * word grouping, karaoke highlighting, and emoji data collection. + */ + +import { smartJoin } from "../../speech/word-segmenter"; +import { + type EmojiInstance, + extractEmoji, + hasEmoji, + stripEmoji, +} from "./emoji"; +import { parseSrt, type SrtEntry } from "./srt"; + +export interface SubtitleStyle { + fontName: string; + fontSize: number; + primaryColor: string; + outlineColor: string; + backColor: string; + bold: boolean; + outline: number; + shadow: number; + marginV: number; + alignment: number; +} + +export const STYLE_PRESETS: Record = { + tiktok: { + fontName: "Montserrat", + fontSize: 72, + primaryColor: "&HFFFFFF", + outlineColor: "&H000000", + backColor: "&H00000000", + bold: true, + outline: 4, + shadow: 0, + marginV: 480, + alignment: 2, + }, + karaoke: { + fontName: "Arial", + fontSize: 28, + primaryColor: "&H00FFFF", + outlineColor: "&H000000", + backColor: "&H00000000", + bold: true, + outline: 2, + shadow: 1, + marginV: 40, + alignment: 2, + }, + bounce: { + fontName: "Impact", + fontSize: 36, + primaryColor: "&HFFFFFF", + outlineColor: "&H000000", + backColor: "&H00000000", + bold: false, + outline: 4, + shadow: 2, + marginV: 60, + alignment: 2, + }, + typewriter: { + fontName: "Courier New", + fontSize: 24, + primaryColor: "&H00FF00", + outlineColor: "&H000000", + backColor: "&H80000000", + bold: false, + outline: 1, + shadow: 0, + marginV: 30, + alignment: 2, + }, +}; + +export const POSITION_ALIGNMENT: Record = { + top: 8, + center: 5, + bottom: 2, +}; + +/** Emoji data collected from a single ASS dialogue line. */ +export interface EntryEmojiData { + instances: EmojiInstance[]; + taggedStrippedText: string; + startTime: number; + endTime: number; +} + +/** Convert a hex color (#RRGGBB) to ASS color format (&HBBGGRR). */ +export function colorToAss(color: string): string { + if (color.startsWith("&H")) return color; + + const hex = color.replace("#", ""); + if (hex.length === 6) { + const r = hex.substring(0, 2); + const g = hex.substring(2, 4); + const b = hex.substring(4, 6); + return `&H${b}${g}${r}`.toUpperCase(); + } + return "&HFFFFFF"; +} + +/** + * Format seconds to ASS timestamp `H:MM:SS.CC`. + * Computes from total centiseconds to avoid overflow when rounding + * lands on 100 cs (e.g. 1.999s would otherwise produce `0:00:01.100`). + */ +export function formatAssTime(seconds: number): string { + const totalCs = Math.max(0, Math.round(seconds * 100)); + const h = Math.floor(totalCs / 360000); + const m = Math.floor((totalCs % 360000) / 6000); + const s = Math.floor((totalCs % 6000) / 100); + const cs = totalCs % 100; + return `${h}:${String(m).padStart(2, "0")}:${String(s).padStart(2, "0")}.${String(cs).padStart(2, "0")}`; +} + +function buildAssHeader( + style: SubtitleStyle, + width: number, + height: number, +): string { + return `[Script Info] +Title: Generated Subtitles +ScriptType: v4.00+ +PlayResX: ${width} +PlayResY: ${height} +WrapStyle: 0 +ScaledBorderAndShadow: yes +YCbCr Matrix: TV.601 + +[V4+ Styles] +Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding +Style: Default,${style.fontName},${style.fontSize},${style.primaryColor},&H000000FF,${style.outlineColor},${style.backColor},${style.bold ? -1 : 0},0,0,0,100,100,0,0,1,${style.outline},${style.shadow},${style.alignment},10,10,${style.marginV},1 + +[Events] +Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text +`; +} + +export function convertSrtToAss( + srtContent: string, + style: SubtitleStyle, + width: number, + height: number, + tagText?: (text: string) => string, + collectEmoji?: boolean, + spacesPerEmoji?: number, +): { ass: string; emojiData: EntryEmojiData[] } { + const assHeader = buildAssHeader(style, width, height); + const nSpaces = spacesPerEmoji ?? 1; + const entries = parseSrt(srtContent); + const emojiData: EntryEmojiData[] = []; + + const assDialogues = entries + .map((entry, i) => { + const startTime = entry.start; + const nextStart = + i < entries.length - 1 ? entries[i + 1]!.start : undefined; + const clampedEnd = + nextStart !== undefined ? Math.min(entry.end, nextStart) : entry.end; + + const start = formatAssTime(startTime); + const end = formatAssTime(clampedEnd); + + let rawText = entry.text.replace(/\n/g, "\\N"); + + let entryEmojiInstances: EmojiInstance[] | undefined; + if (collectEmoji && hasEmoji(rawText)) { + entryEmojiInstances = extractEmoji(rawText, nSpaces); + rawText = stripEmoji(rawText, nSpaces); + } + + const text = tagText ? tagText(rawText) : rawText; + + if (entryEmojiInstances) { + emojiData.push({ + instances: entryEmojiInstances, + taggedStrippedText: text, + startTime, + endTime: clampedEnd, + }); + } + + return `Dialogue: 0,${start},${end},Default,,0,0,0,,${text}`; + }) + .join("\n"); + + return { ass: assHeader + assDialogues, emojiData }; +} + +/** + * Generates ASS subtitle content with grouped words and active-word highlighting. + * + * Groups words into chunks of `wordsPerLine`. For each group, generates one + * Dialogue event per word timing where the currently-spoken word is colored + * with `activeColor` and the rest use the base `primaryColor`. + */ +export function convertSrtToAssGrouped( + srtContent: string, + style: SubtitleStyle, + width: number, + height: number, + wordsPerLine: number, + activeColor?: string, + tagText?: (text: string) => string, + collectEmoji?: boolean, + spacesPerEmoji?: number, +): { ass: string; emojiData: EntryEmojiData[] } { + const assHeader = buildAssHeader(style, width, height); + const nSpaces = spacesPerEmoji ?? 1; + const entries: SrtEntry[] = parseSrt(srtContent); + const dialogues: string[] = []; + const emojiData: EntryEmojiData[] = []; + const baseColor = style.primaryColor; + const highlightColor = activeColor ?? baseColor; + + for (let gi = 0; gi < entries.length; gi += wordsPerLine) { + const group = entries.slice(gi, gi + wordsPerLine); + const groupStart = group[0]!.start; + const nextGroupStart = + gi + wordsPerLine < entries.length + ? entries[gi + wordsPerLine]!.start + : undefined; + const groupEnd = nextGroupStart ?? group[group.length - 1]!.end; + + if (!activeColor) { + let rawText = smartJoin(group.map((e) => e.text.replace(/\n/g, " "))); + + let groupEmojiInstances: EmojiInstance[] | undefined; + if (collectEmoji && hasEmoji(rawText)) { + groupEmojiInstances = extractEmoji(rawText, nSpaces); + rawText = stripEmoji(rawText, nSpaces); + } + + const text = tagText ? tagText(rawText) : rawText; + + if (groupEmojiInstances) { + emojiData.push({ + instances: groupEmojiInstances, + taggedStrippedText: text, + startTime: groupStart, + endTime: groupEnd, + }); + } + + dialogues.push( + `Dialogue: 0,${formatAssTime(groupStart)},${formatAssTime(groupEnd)},Default,,0,0,0,,${text}`, + ); + } else { + const allGroupWords: string[] = []; + for (const entry of group) { + allGroupWords.push(entry.text.replace(/\n/g, " ").trim()); + } + const fullLineRaw = smartJoin(allGroupWords); + + let lineEmojiInstances: EmojiInstance[] | undefined; + let strippedFullLine: string | undefined; + if (collectEmoji && hasEmoji(fullLineRaw)) { + lineEmojiInstances = extractEmoji(fullLineRaw, nSpaces); + strippedFullLine = stripEmoji(fullLineRaw, nSpaces); + } + + const strippedWords = lineEmojiInstances + ? allGroupWords.map((w) => (hasEmoji(w) ? stripEmoji(w, nSpaces) : w)) + : allGroupWords; + + for (let wi = 0; wi < group.length; wi++) { + const wordEntry = group[wi]!; + const wordStart = wordEntry.start; + const wordEnd = wi < group.length - 1 ? group[wi + 1]!.start : groupEnd; + + const parts: string[] = []; + for (let idx = 0; idx < group.length; idx++) { + const rawWord = strippedWords[idx]?.trim() ?? ""; + const word = tagText ? tagText(rawWord) : rawWord; + if (idx === wi) { + parts.push(`{\\c${highlightColor}}${word}{\\c${baseColor}}`); + } else { + parts.push(word); + } + } + + const lineText = smartJoin(parts); + + if (wi === 0 && lineEmojiInstances) { + emojiData.push({ + instances: lineEmojiInstances, + taggedStrippedText: lineText, + startTime: groupStart, + endTime: groupEnd, + }); + } + + dialogues.push( + `Dialogue: 0,${formatAssTime(wordStart)},${formatAssTime(wordEnd)},Default,,0,0,0,,${lineText}`, + ); + } + } + } + + return { ass: assHeader + dialogues.join("\n"), emojiData }; +} diff --git a/src/react/renderers/captions.test.ts b/src/react/renderers/captions.test.ts index dba049ab..f62a7fbf 100644 --- a/src/react/renderers/captions.test.ts +++ b/src/react/renderers/captions.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test"; -import { convertToSRT } from "./captions"; +import { convertToSRT } from "./srt"; // We need to test the internal convertSrtToAssGrouped function. // Since it's not exported, we'll test the full flow via convertToSRT + verifying SRT output, diff --git a/src/react/renderers/captions.ts b/src/react/renderers/captions.ts index 2c8a8fef..669c18e3 100644 --- a/src/react/renderers/captions.ts +++ b/src/react/renderers/captions.ts @@ -1,25 +1,38 @@ +/** + * Captions renderer — orchestrates SRT generation (from native word timings + * or Whisper transcription), ASS conversion with style presets, emoji + * overlay computation, and font resolution. + * + * SRT formatting/parsing lives in srt.ts. + * ASS generation + style presets live in ass.ts. + */ + import { writeFileSync } from "node:fs"; import { groq } from "@ai-sdk/groq"; import { experimental_transcribe as transcribe } from "ai"; -import { z } from "zod"; -import { smartJoin } from "../../speech/word-segmenter"; import { extractWordTimings } from "../primitives/transcribe"; import { ResolvedElement } from "../resolved-element"; import type { CaptionsProps, VargElement } from "../types"; +import { + colorToAss, + convertSrtToAss, + convertSrtToAssGrouped, + POSITION_ALIGNMENT, + STYLE_PRESETS, + type SubtitleStyle, +} from "./ass"; import { ensureLocalFonts } from "./burn-captions"; import type { RenderContext } from "./context"; import { calculateEmojiSize, calculateEmojiY, - type EmojiInstance, type EmojiOverlay, - extractEmoji, hasEmoji, - stripEmoji, } from "./emoji"; import { type FontResolution, getDefaultFontId, resolveFonts } from "./fonts"; import { addTask, completeTask, startTask } from "./progress"; import { renderSpeech } from "./speech"; +import { convertToSRT, type GroqWord } from "./srt"; import { type FontPathMap, getCharXPositions, @@ -28,417 +41,11 @@ import { parseASSSegments, } from "./text-measure"; -const groqWordSchema = z.object({ - word: z.string(), - start: z.number(), - end: z.number(), -}); - -type GroqWord = z.infer; - -// Helper function to convert words to SRT format -function formatTime(seconds: number): string { - const hours = Math.floor(seconds / 3600); - const minutes = Math.floor((seconds % 3600) / 60); - const secs = Math.floor(seconds % 60); - const millis = Math.floor((seconds % 1) * 1000); - - return `${String(hours).padStart(2, "0")}:${String(minutes).padStart(2, "0")}:${String(secs).padStart(2, "0")},${String(millis).padStart(3, "0")}`; -} - -export function convertToSRT(words: GroqWord[]): string { - let srt = ""; - let index = 1; - - for (const word of words) { - const startTime = formatTime(word.start); - const endTime = formatTime(word.end); - - srt += `${index}\n`; - srt += `${startTime} --> ${endTime}\n`; - srt += `${word.word.trim()}\n\n`; - index++; - } - - return srt; -} - -interface SrtEntry { - index: number; - start: number; - end: number; - text: string; -} - -interface SubtitleStyle { - fontName: string; - fontSize: number; - primaryColor: string; - outlineColor: string; - backColor: string; - bold: boolean; - outline: number; - shadow: number; - marginV: number; - alignment: number; -} - -const STYLE_PRESETS: Record = { - tiktok: { - fontName: "Montserrat", - fontSize: 72, - primaryColor: "&HFFFFFF", - outlineColor: "&H000000", - backColor: "&H00000000", - bold: true, - outline: 4, - shadow: 0, - marginV: 480, - alignment: 2, - }, - karaoke: { - fontName: "Arial", - fontSize: 28, - primaryColor: "&H00FFFF", - outlineColor: "&H000000", - backColor: "&H00000000", - bold: true, - outline: 2, - shadow: 1, - marginV: 40, - alignment: 2, - }, - bounce: { - fontName: "Impact", - fontSize: 36, - primaryColor: "&HFFFFFF", - outlineColor: "&H000000", - backColor: "&H00000000", - bold: false, - outline: 4, - shadow: 2, - marginV: 60, - alignment: 2, - }, - typewriter: { - fontName: "Courier New", - fontSize: 24, - primaryColor: "&H00FF00", - outlineColor: "&H000000", - backColor: "&H80000000", - bold: false, - outline: 1, - shadow: 0, - marginV: 30, - alignment: 2, - }, -}; - -function parseSrt(content: string): SrtEntry[] { - const entries: SrtEntry[] = []; - const blocks = content.trim().split(/\n\n+/); - - for (const block of blocks) { - const lines = block.split("\n"); - if (lines.length < 3) continue; - - const index = Number.parseInt(lines[0] || "0", 10); - const timeLine = lines[1] || ""; - const timeMatch = timeLine.match( - /(\d{2}):(\d{2}):(\d{2})[,.](\d{3})\s*-->\s*(\d{2}):(\d{2}):(\d{2})[,.](\d{3})/, - ); - - if (!timeMatch) continue; - - const [, h1, m1, s1, ms1, h2, m2, s2, ms2] = timeMatch; - if (!h1 || !m1 || !s1 || !ms1 || !h2 || !m2 || !s2 || !ms2) continue; - - const start = - Number.parseInt(h1, 10) * 3600 + - Number.parseInt(m1, 10) * 60 + - Number.parseInt(s1, 10) + - Number.parseInt(ms1, 10) / 1000; - - const end = - Number.parseInt(h2, 10) * 3600 + - Number.parseInt(m2, 10) * 60 + - Number.parseInt(s2, 10) + - Number.parseInt(ms2, 10) / 1000; - - const text = lines.slice(2).join("\n"); - entries.push({ index, start, end, text }); - } - - return entries; -} - -/** - * Format seconds to ASS timestamp `H:MM:SS.CC`. - * Computes from total centiseconds to avoid overflow when rounding - * lands on 100 cs (e.g. 1.999s would otherwise produce `0:00:01.100`). - */ -function formatAssTime(seconds: number): string { - const totalCs = Math.max(0, Math.round(seconds * 100)); - const h = Math.floor(totalCs / 360000); - const m = Math.floor((totalCs % 360000) / 6000); - const s = Math.floor((totalCs % 6000) / 100); - const cs = totalCs % 100; - return `${h}:${String(m).padStart(2, "0")}:${String(s).padStart(2, "0")}.${String(cs).padStart(2, "0")}`; -} - -function convertSrtToAss( - srtContent: string, - style: SubtitleStyle, - width: number, - height: number, - tagText?: (text: string) => string, - collectEmoji?: boolean, - spacesPerEmoji?: number, -): { ass: string; emojiData: EntryEmojiData[] } { - const assHeader = `[Script Info] -Title: Generated Subtitles -ScriptType: v4.00+ -PlayResX: ${width} -PlayResY: ${height} -WrapStyle: 0 -ScaledBorderAndShadow: yes -YCbCr Matrix: TV.601 - -[V4+ Styles] -Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding -Style: Default,${style.fontName},${style.fontSize},${style.primaryColor},&H000000FF,${style.outlineColor},${style.backColor},${style.bold ? -1 : 0},0,0,0,100,100,0,0,1,${style.outline},${style.shadow},${style.alignment},10,10,${style.marginV},1 - -[Events] -Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text -`; - - const nSpaces = spacesPerEmoji ?? 1; - const entries = parseSrt(srtContent); - const emojiData: EntryEmojiData[] = []; - - const assDialogues = entries - .map((entry, i) => { - const startTime = entry.start; - // Clamp end to next entry's start to prevent overlapping subtitles - // (transcription engines often produce overlapping word timestamps) - const nextStart = - i < entries.length - 1 ? entries[i + 1]!.start : undefined; - const clampedEnd = - nextStart !== undefined ? Math.min(entry.end, nextStart) : entry.end; - - const start = formatAssTime(startTime); - const end = formatAssTime(clampedEnd); - - let rawText = entry.text.replace(/\n/g, "\\N"); - - // Strip emoji from text and collect overlay data - let entryEmojiInstances: EmojiInstance[] | undefined; - if (collectEmoji && hasEmoji(rawText)) { - entryEmojiInstances = extractEmoji(rawText, nSpaces); - rawText = stripEmoji(rawText, nSpaces); - } - - const text = tagText ? tagText(rawText) : rawText; - - // Collect emoji data with the tagged text for precise measurement - if (entryEmojiInstances) { - emojiData.push({ - instances: entryEmojiInstances, - taggedStrippedText: text, - startTime, - endTime: clampedEnd, - }); - } - - return `Dialogue: 0,${start},${end},Default,,0,0,0,,${text}`; - }) - .join("\n"); - - return { ass: assHeader + assDialogues, emojiData }; -} - -/** - * Generates ASS subtitle content with grouped words and active-word highlighting. - * - * Groups words into chunks of `wordsPerLine`. For each group, generates one - * Dialogue event per word timing where the currently-spoken word is colored - * with `activeColor` and the rest use the base `primaryColor`. - * - * Example output for group ["Varg", "AI", "is"] with activeColor orange: - * t=0.5-0.8: {\c&H428CFF&}Varg{\c&HFFFFFF&} AI is - * t=0.8-1.0: Varg {\c&H428CFF&}AI{\c&HFFFFFF&} is - * t=1.0-1.3: Varg AI {\c&H428CFF&}is{\c&HFFFFFF&} - */ -function convertSrtToAssGrouped( - srtContent: string, - style: SubtitleStyle, - width: number, - height: number, - wordsPerLine: number, - activeColor?: string, - tagText?: (text: string) => string, - collectEmoji?: boolean, - spacesPerEmoji?: number, -): { ass: string; emojiData: EntryEmojiData[] } { - const assHeader = `[Script Info] -Title: Generated Subtitles -ScriptType: v4.00+ -PlayResX: ${width} -PlayResY: ${height} -WrapStyle: 0 -ScaledBorderAndShadow: yes -YCbCr Matrix: TV.601 - -[V4+ Styles] -Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding -Style: Default,${style.fontName},${style.fontSize},${style.primaryColor},&H000000FF,${style.outlineColor},${style.backColor},${style.bold ? -1 : 0},0,0,0,100,100,0,0,1,${style.outline},${style.shadow},${style.alignment},10,10,${style.marginV},1 - -[Events] -Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text -`; - - const nSpaces = spacesPerEmoji ?? 1; - const entries = parseSrt(srtContent); - const dialogues: string[] = []; - const emojiData: EntryEmojiData[] = []; - const baseColor = style.primaryColor; - const highlightColor = activeColor ?? baseColor; - - // Group entries into chunks of wordsPerLine - for (let gi = 0; gi < entries.length; gi += wordsPerLine) { - const group = entries.slice(gi, gi + wordsPerLine); - const groupStart = group[0]!.start; - // Cap group end at next group's start to prevent two groups showing simultaneously - const nextGroupStart = - gi + wordsPerLine < entries.length - ? entries[gi + wordsPerLine]!.start - : undefined; - const groupEnd = nextGroupStart ?? group[group.length - 1]!.end; - - if (!activeColor) { - // No highlight — show entire group as one event - let rawText = smartJoin(group.map((e) => e.text.replace(/\n/g, " "))); - - // Strip emoji from the grouped text line - let groupEmojiInstances: EmojiInstance[] | undefined; - if (collectEmoji && hasEmoji(rawText)) { - groupEmojiInstances = extractEmoji(rawText, nSpaces); - rawText = stripEmoji(rawText, nSpaces); - } - - const text = tagText ? tagText(rawText) : rawText; - - if (groupEmojiInstances) { - emojiData.push({ - instances: groupEmojiInstances, - taggedStrippedText: text, - startTime: groupStart, - endTime: groupEnd, - }); - } - - dialogues.push( - `Dialogue: 0,${formatAssTime(groupStart)},${formatAssTime(groupEnd)},Default,,0,0,0,,${text}`, - ); - } else { - // Karaoke highlight — one dialogue event per word, shifting the highlight - // For emoji in karaoke mode, we strip emoji from the full group line - // and collect overlay data once (for the full group duration). - const allGroupWords: string[] = []; - for (const entry of group) { - allGroupWords.push(entry.text.replace(/\n/g, " ").trim()); - } - const fullLineRaw = smartJoin(allGroupWords); - - let lineEmojiInstances: EmojiInstance[] | undefined; - let strippedFullLine: string | undefined; - if (collectEmoji && hasEmoji(fullLineRaw)) { - lineEmojiInstances = extractEmoji(fullLineRaw, nSpaces); - strippedFullLine = stripEmoji(fullLineRaw, nSpaces); - } - - // Build per-word stripped words for highlight assembly - const strippedWords = lineEmojiInstances - ? allGroupWords.map((w) => (hasEmoji(w) ? stripEmoji(w, nSpaces) : w)) - : allGroupWords; - - for (let wi = 0; wi < group.length; wi++) { - const wordEntry = group[wi]!; - const wordStart = wordEntry.start; - const wordEnd = wi < group.length - 1 ? group[wi + 1]!.start : groupEnd; - - const parts: string[] = []; - for (let idx = 0; idx < group.length; idx++) { - const rawWord = strippedWords[idx]?.trim() ?? ""; - const word = tagText ? tagText(rawWord) : rawWord; - if (idx === wi) { - parts.push(`{\\c${highlightColor}}${word}{\\c${baseColor}}`); - } else { - parts.push(word); - } - } - - const lineText = smartJoin(parts); - - // Collect emoji data once for the first word's dialogue - // (all words in the group show the same line text, just with different highlight) - if (wi === 0 && lineEmojiInstances) { - emojiData.push({ - instances: lineEmojiInstances, - taggedStrippedText: lineText, - startTime: groupStart, - endTime: groupEnd, - }); - } - - dialogues.push( - `Dialogue: 0,${formatAssTime(wordStart)},${formatAssTime(wordEnd)},Default,,0,0,0,,${lineText}`, - ); - } - } - } - - return { ass: assHeader + dialogues.join("\n"), emojiData }; -} - -/** Emoji data collected from a single ASS dialogue line. */ -interface EntryEmojiData { - /** Emoji instances found in this entry's text. */ - instances: EmojiInstance[]; - /** The stripped text (emoji replaced with spaces) AFTER {\fn} tagging by tagText(). */ - taggedStrippedText: string; - /** Start time in seconds. */ - startTime: number; - /** End time in seconds. */ - endTime: number; -} - -const POSITION_ALIGNMENT: Record = { - top: 8, - center: 5, - bottom: 2, -}; - -function colorToAss(color: string): string { - if (color.startsWith("&H")) return color; - - const hex = color.replace("#", ""); - if (hex.length === 6) { - const r = hex.substring(0, 2); - const g = hex.substring(2, 4); - const b = hex.substring(4, 6); - return `&H${b}${g}${r}`.toUpperCase(); - } - return "&HFFFFFF"; -} - export interface CaptionsResult { assPath: string; srtPath?: string; audioPath?: string; - /** Font files needed for rendering (primary + any fallbacks for non-Latin scripts). */ fontFiles?: { url: string; fileName: string }[]; - /** Emoji overlay data for color emoji rendering via image overlay. */ emojiOverlays?: EmojiOverlay[]; } @@ -448,111 +55,14 @@ export async function renderCaptions( ): Promise { const props = element.props as CaptionsProps; - let srtContent: string; - let srtPath: string | undefined; - let audioPath: string | undefined; - - if (props.srt) { - srtContent = await Bun.file(props.srt).text(); - srtPath = props.srt; - } else if (props.src) { - if (typeof props.src === "string") { - srtContent = await Bun.file(props.src).text(); - srtPath = props.src; - } else if (props.src.type === "speech" || props.src.type === "audio") { - // Use pre-generated file if already resolved, otherwise render. - // Audio elements (video.audio / speech.audio) resolve through - // renderAudio; speech elements through renderSpeech. - let speechFile: Awaited>; - if (props.src instanceof ResolvedElement) { - speechFile = props.src.meta.file; - } else if (props.src.type === "audio") { - const { renderAudio } = await import("./audio"); - speechFile = await renderAudio(props.src as VargElement<"audio">, ctx); - } else { - speechFile = await renderSpeech( - props.src as VargElement<"speech">, - ctx, - ); - } - audioPath = await ctx.backend.resolvePath(speechFile); - - // Check if the element already has word-level timing (ElevenLabs - // alignment on speech, or inherited by a derived audio node). - // If so, skip the Whisper transcription step entirely (saves time and cost). - const nativeWords = - props.src instanceof ResolvedElement - ? props.src.meta.words - : props.src.meta?.words; - - if (nativeWords && nativeWords.length > 0) { - // Use native ElevenLabs word timing — same shape as GroqWord - srtContent = convertToSRT(nativeWords); - } else { - // Transcribe audio to get word-level timestamps for captions. - // Uses gateway transcription model if available (deployed render), - // falls back to direct Groq Whisper for local/CLI usage. - const transcriptionModel = ctx.defaults?.transcription; - const transcribeTaskId = ctx.progress - ? addTask( - ctx.progress, - "transcribe", - transcriptionModel ? "gateway-whisper" : "groq-whisper", - ) - : null; - if (transcribeTaskId && ctx.progress) - startTask(ctx.progress, transcribeTaskId); - - let words: GroqWord[] | undefined; - let fallbackText = ""; - try { - const audioData = - audioPath.startsWith("http://") || audioPath.startsWith("https://") - ? await fetch(audioPath).then((res) => res.arrayBuffer()) - : await Bun.file(audioPath).arrayBuffer(); - - const model = - transcriptionModel ?? groq.transcription("whisper-large-v3"); - const result = await transcribe({ - model, - audio: new Uint8Array(audioData), - providerOptions: transcriptionModel - ? {} - : { - groq: { - responseFormat: "verbose_json", - timestampGranularities: ["word"], - }, - }, - }); - - fallbackText = result.text; - - words = extractWordTimings(result); - } finally { - if (transcribeTaskId && ctx.progress) - completeTask(ctx.progress, transcribeTaskId); - } - - if (!words || words.length === 0) { - srtContent = `1\n00:00:00,000 --> 00:00:05,000\n${fallbackText}\n`; - } else { - srtContent = convertToSRT(words); - } - } - - srtPath = `/tmp/varg-captions-${Date.now()}.srt`; - writeFileSync(srtPath, srtContent); - ctx.tempFiles.push(srtPath); - } else { - throw new Error( - "Captions src must be a path to SRT file, a Speech element, or an audio element (video.audio / speech.audio)", - ); - } - } else { - throw new Error("Captions element requires either 'srt' or 'src' prop"); - } + // 1. Resolve SRT content (from file, speech element, or audio element) + const { srtContent, srtPath, audioPath } = await resolveSrtContent( + props, + element, + ctx, + ); + // 2. Build subtitle style from preset + props overrides const styleName = props.style ?? "tiktok"; const defaultStyle: SubtitleStyle = { fontName: "Montserrat", @@ -568,7 +78,7 @@ export async function renderCaptions( }; const baseStyle = STYLE_PRESETS[styleName] ?? defaultStyle; - // Resolve fonts: primary font from props.font or style default, plus fallbacks + // 3. Resolve fonts: primary font + fallbacks for non-Latin scripts const primaryFontId = props.font ?? getDefaultFontId(styleName); const fontResolution = resolveFonts(srtContent, primaryFontId); @@ -591,16 +101,12 @@ export async function renderCaptions( ? colorToAss(props.activeColor) : undefined; - // Check if the SRT content has emoji — if so, we'll strip them from ASS - // text and build overlay data for color PNG rendering + // 4. Compute emoji spacing using real font metrics (if emoji present) const srtHasEmoji = hasEmoji(srtContent); - - // When emoji are present, compute how many spaces to reserve per emoji - // using precise font metrics from the primary font let spacesPerEmoji: number | undefined; let fontPathMap: FontPathMap | undefined; + if (srtHasEmoji) { - // Download fonts locally for measurement const localFontsDir = await ensureLocalFonts( fontResolution.fontFiles.map((f) => ({ url: f.url, @@ -608,13 +114,11 @@ export async function renderCaptions( })), ); - // Build font name → local path mapping fontPathMap = new Map(); for (const f of fontResolution.fontFiles) { fontPathMap.set(f.fontName, `${localFontsDir}/${f.fileName}`); } - // Compute spacesPerEmoji using real font metrics const primaryFontPath = fontPathMap.get(fontResolution.primary.fontName); if (primaryFontPath) { const metrics = getFontMetrics(primaryFontPath, style.fontSize); @@ -624,11 +128,11 @@ export async function renderCaptions( ctx.height, ); const spaceWidth = getSpaceWidth(primaryFontPath, style.fontSize); - // +1 buffer space for visual breathing room between emoji and adjacent text spacesPerEmoji = Math.max(1, Math.ceil(emojiSize / spaceWidth) + 1); } } + // 5. Convert SRT → ASS (with optional word grouping + karaoke highlight) const { ass: assContent, emojiData } = props.wordsPerLine ? convertSrtToAssGrouped( srtContent, @@ -654,78 +158,16 @@ export async function renderCaptions( writeFileSync(assPath, assContent); ctx.tempFiles.push(assPath); - // Build emoji overlay descriptors with precise pixel positions from font metrics - let emojiOverlays: EmojiOverlay[] | undefined; - if (emojiData.length > 0 && fontPathMap) { - const primaryFontPath = fontPathMap.get(style.fontName); - const metrics = primaryFontPath - ? getFontMetrics(primaryFontPath, style.fontSize) - : { - ppem: style.fontSize * 0.64, - capHeight: style.fontSize * 0.45, - winAscent: style.fontSize * 0.7, - winDescent: style.fontSize * 0.3, - }; - const emojiSize = calculateEmojiSize( - metrics.winAscent, - ctx.height, - ctx.height, - ); - const nSpaces = spacesPerEmoji ?? 1; - const spaceW = primaryFontPath - ? getSpaceWidth(primaryFontPath, style.fontSize) - : metrics.ppem * 0.28; - emojiOverlays = []; - for (const entry of emojiData) { - const segments = parseASSSegments( - entry.taggedStrippedText, - style.fontName, - ); - const charPositions = getCharXPositions( - segments, - fontPathMap, - style.fontSize, - ctx.width, - style.alignment, - ); + // 6. Build emoji overlay descriptors with precise pixel positions + const emojiOverlays = buildEmojiOverlays( + emojiData, + fontPathMap, + style, + ctx, + spacesPerEmoji, + ); - for (const instance of entry.instances) { - // charIndex points to the first space in the reserved block. - // Center the emoji within the reserved space block. - const firstSpaceX = charPositions[instance.charIndex] ?? 0; - const lastSpaceIdx = Math.min( - instance.charIndex + nSpaces - 1, - charPositions.length - 1, - ); - const lastSpaceX = charPositions[lastSpaceIdx] ?? firstSpaceX; - const blockEndX = lastSpaceX + spaceW; - const blockWidth = blockEndX - firstSpaceX; - const x = Math.round(firstSpaceX + blockWidth / 2 - emojiSize / 2); - - const y = calculateEmojiY( - style.alignment, - style.marginV, - metrics.winDescent, - metrics.winAscent, - metrics.capHeight, - ctx.height, - ctx.height, - ); - emojiOverlays.push({ - url: instance.url, - fileName: `${instance.codepoints}.png`, - startTime: entry.startTime, - endTime: entry.endTime, - x, - y, - size: emojiSize, - }); - } - } - } - - // When emoji are overlaid as color PNGs, exclude Noto Emoji from font files - // (emoji chars are spaces in the ASS text, so the monochrome font is unused) + // 7. Filter font files (exclude Noto Emoji when emoji overlaid as PNGs) const fontFiles = fontResolution.fontFiles .filter( (f) => @@ -741,3 +183,184 @@ export async function renderCaptions( emojiOverlays, }; } + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +async function resolveSrtContent( + props: CaptionsProps, + element: VargElement<"captions">, + ctx: RenderContext, +): Promise<{ srtContent: string; srtPath?: string; audioPath?: string }> { + if (props.srt) { + const srtContent = await Bun.file(props.srt).text(); + return { srtContent, srtPath: props.srt }; + } + + if (!props.src) { + throw new Error("Captions element requires either 'srt' or 'src' prop"); + } + + if (typeof props.src === "string") { + const srtContent = await Bun.file(props.src).text(); + return { srtContent, srtPath: props.src }; + } + + if (props.src.type !== "speech" && props.src.type !== "audio") { + throw new Error( + "Captions src must be a path to SRT file, a Speech element, or an audio element (video.audio / speech.audio)", + ); + } + + // Resolve speech/audio file + let speechFile: Awaited>; + if (props.src instanceof ResolvedElement) { + speechFile = props.src.meta.file; + } else if (props.src.type === "audio") { + const { renderAudio } = await import("./audio"); + speechFile = await renderAudio(props.src as VargElement<"audio">, ctx); + } else { + speechFile = await renderSpeech(props.src as VargElement<"speech">, ctx); + } + const audioPath = await ctx.backend.resolvePath(speechFile); + + // Check for native word timings (ElevenLabs alignment) — skip Whisper if present + const nativeWords = + props.src instanceof ResolvedElement + ? props.src.meta.words + : props.src.meta?.words; + + if (nativeWords && nativeWords.length > 0) { + const srtContent = convertToSRT(nativeWords as GroqWord[]); + const srtPath = `/tmp/varg-captions-${Date.now()}.srt`; + writeFileSync(srtPath, srtContent); + ctx.tempFiles.push(srtPath); + return { srtContent, srtPath, audioPath }; + } + + // Transcribe via Whisper (gateway or direct Groq) + const transcriptionModel = ctx.defaults?.transcription; + const transcribeTaskId = ctx.progress + ? addTask( + ctx.progress, + "transcribe", + transcriptionModel ? "gateway-whisper" : "groq-whisper", + ) + : null; + if (transcribeTaskId && ctx.progress) + startTask(ctx.progress, transcribeTaskId); + + let words: GroqWord[] | undefined; + let fallbackText = ""; + try { + const audioData = + audioPath.startsWith("http://") || audioPath.startsWith("https://") + ? await fetch(audioPath).then((res) => res.arrayBuffer()) + : await Bun.file(audioPath).arrayBuffer(); + + const model = transcriptionModel ?? groq.transcription("whisper-large-v3"); + const result = await transcribe({ + model, + audio: new Uint8Array(audioData), + providerOptions: transcriptionModel + ? {} + : { + groq: { + responseFormat: "verbose_json", + timestampGranularities: ["word"], + }, + }, + }); + + fallbackText = result.text; + words = extractWordTimings(result) as GroqWord[] | undefined; + } finally { + if (transcribeTaskId && ctx.progress) + completeTask(ctx.progress, transcribeTaskId); + } + + const srtContent = + words && words.length > 0 + ? convertToSRT(words) + : `1\n00:00:00,000 --> 00:00:05,000\n${fallbackText}\n`; + const srtPath = `/tmp/varg-captions-${Date.now()}.srt`; + writeFileSync(srtPath, srtContent); + ctx.tempFiles.push(srtPath); + + return { srtContent, srtPath, audioPath }; +} + +function buildEmojiOverlays( + emojiData: import("./ass").EntryEmojiData[], + fontPathMap: FontPathMap | undefined, + style: SubtitleStyle, + ctx: RenderContext, + spacesPerEmoji: number | undefined, +): EmojiOverlay[] | undefined { + if (emojiData.length === 0 || !fontPathMap) return undefined; + + const primaryFontPath = fontPathMap.get(style.fontName); + const metrics = primaryFontPath + ? getFontMetrics(primaryFontPath, style.fontSize) + : { + ppem: style.fontSize * 0.64, + capHeight: style.fontSize * 0.45, + winAscent: style.fontSize * 0.7, + winDescent: style.fontSize * 0.3, + }; + const emojiSize = calculateEmojiSize( + metrics.winAscent, + ctx.height, + ctx.height, + ); + const nSpaces = spacesPerEmoji ?? 1; + const spaceW = primaryFontPath + ? getSpaceWidth(primaryFontPath, style.fontSize) + : metrics.ppem * 0.28; + + const overlays: EmojiOverlay[] = []; + for (const entry of emojiData) { + const segments = parseASSSegments(entry.taggedStrippedText, style.fontName); + const charPositions = getCharXPositions( + segments, + fontPathMap, + style.fontSize, + ctx.width, + style.alignment, + ); + + for (const instance of entry.instances) { + const firstSpaceX = charPositions[instance.charIndex] ?? 0; + const lastSpaceIdx = Math.min( + instance.charIndex + nSpaces - 1, + charPositions.length - 1, + ); + const lastSpaceX = charPositions[lastSpaceIdx] ?? firstSpaceX; + const blockEndX = lastSpaceX + spaceW; + const blockWidth = blockEndX - firstSpaceX; + const x = Math.round(firstSpaceX + blockWidth / 2 - emojiSize / 2); + + const y = calculateEmojiY( + style.alignment, + style.marginV, + metrics.winDescent, + metrics.winAscent, + metrics.capHeight, + ctx.height, + ctx.height, + ); + overlays.push({ + url: instance.url, + fileName: `${instance.codepoints}.png`, + startTime: entry.startTime, + endTime: entry.endTime, + x, + y, + size: emojiSize, + }); + } + } + + return overlays; +} diff --git a/src/react/renderers/srt.ts b/src/react/renderers/srt.ts new file mode 100644 index 00000000..0d5f1d7c --- /dev/null +++ b/src/react/renderers/srt.ts @@ -0,0 +1,88 @@ +/** + * SRT formatting and parsing — convert word timings to SRT format + * and parse SRT content back into structured entries. + */ + +import { z } from "zod"; + +export const groqWordSchema = z.object({ + word: z.string(), + start: z.number(), + end: z.number(), +}); + +export type GroqWord = z.infer; + +export interface SrtEntry { + index: number; + start: number; + end: number; + text: string; +} + +/** Format seconds as SRT timestamp `HH:MM:SS,mmm`. */ +export function formatTime(seconds: number): string { + const hours = Math.floor(seconds / 3600); + const minutes = Math.floor((seconds % 3600) / 60); + const secs = Math.floor(seconds % 60); + const millis = Math.floor((seconds % 1) * 1000); + + return `${String(hours).padStart(2, "0")}:${String(minutes).padStart(2, "0")}:${String(secs).padStart(2, "0")},${String(millis).padStart(3, "0")}`; +} + +/** Convert word timings to SRT subtitle format. */ +export function convertToSRT(words: GroqWord[]): string { + let srt = ""; + let index = 1; + + for (const word of words) { + const startTime = formatTime(word.start); + const endTime = formatTime(word.end); + + srt += `${index}\n`; + srt += `${startTime} --> ${endTime}\n`; + srt += `${word.word.trim()}\n\n`; + index++; + } + + return srt; +} + +/** Parse SRT content into structured entries. */ +export function parseSrt(content: string): SrtEntry[] { + const entries: SrtEntry[] = []; + const blocks = content.trim().split(/\n\n+/); + + for (const block of blocks) { + const lines = block.split("\n"); + if (lines.length < 3) continue; + + const index = Number.parseInt(lines[0] || "0", 10); + const timeLine = lines[1] || ""; + const timeMatch = timeLine.match( + /(\d{2}):(\d{2}):(\d{2})[,.](\d{3})\s*-->\s*(\d{2}):(\d{2}):(\d{2})[,.](\d{3})/, + ); + + if (!timeMatch) continue; + + const [, h1, m1, s1, ms1, h2, m2, s2, ms2] = timeMatch; + if (!h1 || !m1 || !s1 || !ms1 || !h2 || !m2 || !s2 || !ms2) continue; + + const start = + Number.parseInt(h1, 10) * 3600 + + Number.parseInt(m1, 10) * 60 + + Number.parseInt(s1, 10) + + Number.parseInt(ms1, 10) / 1000; + + const end = + Number.parseInt(h2, 10) * 3600 + + Number.parseInt(m2, 10) * 60 + + Number.parseInt(s2, 10) + + Number.parseInt(ms2, 10) / 1000; + + const text = lines.slice(2).join("\n"); + entries.push({ index, start, end, text }); + } + + return entries; +}