diff --git a/apps/web/src/app/api/convert-video/route.ts b/apps/web/src/app/api/convert-video/route.ts new file mode 100644 index 000000000..17b909d45 --- /dev/null +++ b/apps/web/src/app/api/convert-video/route.ts @@ -0,0 +1,144 @@ +import { type NextRequest, NextResponse } from "next/server"; +import { writeFile, readFile, rm, mkdtemp } from "fs/promises"; +import { spawn } from "child_process"; +import path from "path"; +import os from "os"; + +const GENERIC_CONVERSION_ERROR = "Video conversion failed."; + +function sanitizeContentDispositionName({ + rawName, +}: { + rawName: string; +}): string { + // Strip characters that can break the Content-Disposition header (quotes, + // backslashes, CR/LF, control bytes) and replace non-ASCII so the + // "filename=" parameter stays unambiguous across clients. + const stripped = rawName + // eslint-disable-next-line no-control-regex + .replace(/[\\"\r\n\x00-\x1f]/g, "_") + .replace(/[^\x20-\x7e]/g, "_") + .slice(0, 200); + return stripped.length > 0 ? stripped : "video"; +} + +export async function POST(request: NextRequest) { + let workDir: string | null = null; + + try { + const formData = await request.formData(); + const file = formData.get("file") as File | null; + + if (!file || !(file instanceof File)) { + return NextResponse.json( + { error: "No file provided" }, + { status: 400 }, + ); + } + + workDir = await mkdtemp(path.join(os.tmpdir(), "opencut-convert-")); + const ext = path.extname(file.name) || ".mp4"; + const inputPath = path.join(workDir, `input${ext}`); + const outputPath = path.join(workDir, "output.mp4"); + + const buffer = Buffer.from(await file.arrayBuffer()); + await writeFile(inputPath, buffer); + + await runFfmpeg({ + inputPath, + outputPath, + }); + + const convertedBuffer = await readFile(outputPath); + + const safeName = sanitizeContentDispositionName({ + rawName: path.basename(file.name, ext), + }); + + return new Response(new Uint8Array(convertedBuffer), { + status: 200, + headers: { + "Content-Type": "video/mp4", + "Content-Disposition": `attachment; filename="${safeName}.mp4"`, + }, + }); + } catch (error) { + console.error("Video conversion error:", error); + return NextResponse.json( + { error: GENERIC_CONVERSION_ERROR }, + { status: 500 }, + ); + } finally { + if (workDir) { + await rm(workDir, { recursive: true, force: true }).catch(() => {}); + } + } +} + +function runFfmpeg({ + inputPath, + outputPath, +}: { + inputPath: string; + outputPath: string; +}): Promise { + return new Promise((resolve, reject) => { + const proc = spawn("ffmpeg", [ + "-i", + inputPath, + "-c:v", + "libx264", + "-profile:v", + "baseline", + "-level", + "3.1", + "-preset", + "veryfast", + "-crf", + "23", + "-pix_fmt", + "yuv420p", + "-g", + "60", + "-keyint_min", + "60", + "-sc_threshold", + "0", + "-movflags", + "+faststart", + "-tag:v", + "avc1", + "-c:a", + "aac", + "-ac", + "2", + "-ar", + "48000", + "-b:a", + "128k", + "-y", + outputPath, + ]); + + let stderr = ""; + proc.stderr.on("data", (data) => { + stderr += data.toString(); + }); + + proc.on("close", (code) => { + if (code === 0) { + resolve(); + } else { + reject( + new Error( + `ffmpeg exited with code ${code}. stderr: ${stderr.slice(-500)}`, + ), + ); + } + }); + + proc.on("error", (err) => { + reject(err); + }); + }); +} diff --git a/apps/web/src/core/managers/audio-manager.ts b/apps/web/src/core/managers/audio-manager.ts index d0716aadc..780421a20 100644 --- a/apps/web/src/core/managers/audio-manager.ts +++ b/apps/web/src/core/managers/audio-manager.ts @@ -20,6 +20,10 @@ import { Input, type WrappedAudioBuffer, } from "mediabunny"; +import { + type AudioBufferSinkLike, + FallbackAudioBufferSink, +} from "@/services/audio-cache/fallback-audio-buffer-sink"; export class AudioManager { private audioContext: AudioContext | null = null; @@ -30,7 +34,7 @@ export class AudioManager { private lookaheadSeconds = 2; private scheduleIntervalMs = 500; private clips: AudioClipSource[] = []; - private sinks = new Map(); + private sinks = new Map(); private inputs = new Map(); private activeClipIds = new Set(); private clipIterators = new Map< @@ -267,7 +271,42 @@ export class AudioManager { const iterator = sink.buffers(sourceStartTime); this.clipIterators.set(clip.id, iterator); - let consecutiveDroppedBufferCount = 0; + + try { + await this.consumeClipIterator({ + clip, + clipEnd, + iterator, + audioContext, + sessionId, + initialDroppedCount: 0, + }); + } catch (error) { + console.warn( + `[AudioManager] clip iterator failed for ${clip.id}; aborting playback for this clip.`, + error, + ); + this.clipIterators.delete(clip.id); + this.activeClipIds.delete(clip.id); + } + } + + private async consumeClipIterator({ + clip, + clipEnd, + iterator, + audioContext, + sessionId, + initialDroppedCount, + }: { + clip: AudioClipSource; + clipEnd: number; + iterator: AsyncGenerator; + audioContext: AudioContext; + sessionId: number; + initialDroppedCount: number; + }): Promise { + let consecutiveDroppedBufferCount = initialDroppedCount; for await (const { buffer, timestamp } of iterator) { if (!this.editor.playback.getIsPlaying()) return; @@ -605,7 +644,18 @@ export class AudioManager { return null; } - const sink = new AudioBufferSink(audioTrack); + const canDecode = await safeCanDecode({ audioTrack }); + const sink: AudioBufferSinkLike = canDecode + ? new AudioBufferSink(audioTrack) + : await FallbackAudioBufferSink.create({ + file: clip.file, + audioContext, + }); + if (!canDecode) { + console.warn( + "[AudioManager] WebCodecs unavailable; decoding clip audio via decodeAudioData.", + ); + } const chunks: AudioBuffer[] = []; let totalSamples = 0; @@ -673,12 +723,13 @@ export class AudioManager { clip, }: { clip: AudioClipSource; - }): Promise { + }): Promise { const existingSink = this.sinks.get(clip.sourceKey); if (existingSink) return existingSink; + let input: Input | null = null; try { - const input = new Input({ + input = new Input({ source: new BlobSource(clip.file), formats: ALL_FORMATS, }); @@ -688,13 +739,47 @@ export class AudioManager { return null; } + const canDecode = await safeCanDecode({ audioTrack }); + if (!canDecode) { + input.dispose(); + input = null; + const audioContext = this.ensureAudioContext(); + if (!audioContext) { + return null; + } + const fallback = await FallbackAudioBufferSink.create({ + file: clip.file, + audioContext, + }); + this.sinks.set(clip.sourceKey, fallback); + console.warn( + "[AudioManager] WebCodecs unavailable; using decodeAudioData fallback for audio clip.", + ); + return fallback; + } + const sink = new AudioBufferSink(audioTrack); this.inputs.set(clip.sourceKey, input); this.sinks.set(clip.sourceKey, sink); return sink; } catch (error) { + if (input) { + input.dispose(); + } console.warn("Failed to initialize audio sink:", error); return null; } } } + +async function safeCanDecode({ + audioTrack, +}: { + audioTrack: { canDecode: () => Promise }; +}): Promise { + try { + return await audioTrack.canDecode(); + } catch { + return false; + } +} diff --git a/apps/web/src/media/audio.ts b/apps/web/src/media/audio.ts index 7abaf90e7..29dd3cce4 100644 --- a/apps/web/src/media/audio.ts +++ b/apps/web/src/media/audio.ts @@ -20,6 +20,10 @@ import { canTrackHaveAudio } from "@/timeline"; import { mediaSupportsAudio } from "@/media/media-utils"; import { getSourceTimeAtClipTime, renderRetimedBuffer } from "@/retime"; import { Input, ALL_FORMATS, BlobSource, AudioBufferSink } from "mediabunny"; +import { + type AudioBufferSinkLike, + FallbackAudioBufferSink, +} from "@/services/audio-cache/fallback-audio-buffer-sink"; import { TICKS_PER_SECOND } from "@/wasm"; import { computeRmsBuckets, @@ -271,7 +275,23 @@ async function resolveAudioBufferForAsset({ const audioTrack = await input.getPrimaryAudioTrack(); if (!audioTrack) return null; - const sink = new AudioBufferSink(audioTrack); + let audioTrackCanDecode = false; + try { + audioTrackCanDecode = await audioTrack.canDecode(); + } catch { + audioTrackCanDecode = false; + } + const sink: AudioBufferSinkLike = audioTrackCanDecode + ? new AudioBufferSink(audioTrack) + : await FallbackAudioBufferSink.create({ + file: asset.file, + audioContext, + }); + if (!audioTrackCanDecode) { + console.warn( + "[audio] WebCodecs unavailable; decoding asset via decodeAudioData fallback.", + ); + } const targetSampleRate = audioContext.sampleRate; const chunks: AudioBuffer[] = []; diff --git a/apps/web/src/media/ffmpeg-convert.ts b/apps/web/src/media/ffmpeg-convert.ts new file mode 100644 index 000000000..0bc76c038 --- /dev/null +++ b/apps/web/src/media/ffmpeg-convert.ts @@ -0,0 +1,22 @@ +export async function convertVideoToH264({ + file, +}: { + file: File; +}): Promise { + const formData = new FormData(); + formData.append("file", file); + + const response = await fetch("/api/convert-video", { + method: "POST", + body: formData, + }); + + if (!response.ok) { + const errorData = await response.json().catch(() => ({})); + throw new Error(errorData.error || `Conversion failed: ${response.status}`); + } + + const blob = await response.blob(); + const baseName = file.name.replace(/\.[^.]+$/, ""); + return new File([blob], `${baseName}.mp4`, { type: "video/mp4" }); +} diff --git a/apps/web/src/media/processing.ts b/apps/web/src/media/processing.ts index 34ccadbc2..a2107b0bc 100644 --- a/apps/web/src/media/processing.ts +++ b/apps/web/src/media/processing.ts @@ -6,6 +6,7 @@ import type { MediaAsset } from "@/media/types"; import { readVideoFile } from "./mediabunny"; import type { VideoFileData } from "./mediabunny"; import { renderThumbnailDataUrl } from "./thumbnail"; +import { convertVideoToH264 } from "./ffmpeg-convert"; export interface ProcessedMediaAsset extends Omit {} @@ -132,8 +133,49 @@ export async function processMediaAssets({ width = result.width; height = result.height; } else if (fileType === "video") { + let currentFile = file; + let currentUrl = url; try { - const videoData = await readVideoFile({ file }); + let videoData = await readVideoFile({ file: currentFile }); + + if (!videoData.canDecode) { + toast.info(`Converting ${file.name} to a compatible format…`); + const convertedFile = await convertVideoToH264({ + file: currentFile, + }); + + const convertedStorageCheck = await storageService.canStoreFile({ + size: convertedFile.size, + }); + if (!convertedStorageCheck.canStore) { + throw new Error( + getStorageLimitDescription({ + fileSize: convertedFile.size, + availableBytes: convertedStorageCheck.availableBytes, + }), + ); + } + + URL.revokeObjectURL(currentUrl); + currentFile = convertedFile; + currentUrl = URL.createObjectURL(currentFile); + videoData = await readVideoFile({ file: currentFile }); + + if (!videoData.canDecode) { + // Conversion produced a file the browser still can't decode + // via mediabunny/WebCodecs. The HTMLVideoElement fallback in + // VideoCache will handle preview rendering, so warn but + // continue importing. + toast.warning(`${file.name} converted, preview may be limited`, { + description: getUnsupportedVideoDescription({ + codec: videoData.codec, + }), + }); + } else { + toast.success(`${file.name} converted successfully`); + } + } + duration = videoData.duration; width = videoData.width; height = videoData.height; @@ -143,13 +185,27 @@ export async function processMediaAssets({ hasAudio = videoData.hasAudio; thumbnailUrl = videoData.thumbnailUrl ?? undefined; - if (!videoData.canDecode) { - toast.error(`Can't preview ${file.name}`, { - description: getUnsupportedVideoDescription({ - codec: videoData.codec, - }), - }); + processedAssets.push({ + name: currentFile.name, + type: fileType, + file: currentFile, + url: currentUrl, + thumbnailUrl, + duration, + width, + height, + fps, + hasAudio, + }); + + await new Promise((resolve) => setTimeout(resolve, 0)); + + completed += 1; + if (onProgress) { + const percent = Math.round((completed / total) * 100); + onProgress({ progress: percent }); } + continue; } catch (error) { const message = error instanceof Error @@ -159,6 +215,8 @@ export async function processMediaAssets({ toast.error(`Couldn't process ${file.name}`, { description: message, }); + URL.revokeObjectURL(currentUrl); + continue; } } else if (fileType === "audio") { duration = await getMediaDuration({ file }); diff --git a/apps/web/src/services/audio-cache/__tests__/fallback-audio-buffer-sink.test.ts b/apps/web/src/services/audio-cache/__tests__/fallback-audio-buffer-sink.test.ts new file mode 100644 index 000000000..6f1815145 --- /dev/null +++ b/apps/web/src/services/audio-cache/__tests__/fallback-audio-buffer-sink.test.ts @@ -0,0 +1,308 @@ +import { + afterEach, + beforeEach, + describe, + expect, + test, +} from "bun:test"; +import { + FallbackAudioBufferSink, + isWebAudioDecoderAvailable, + sliceAudioBuffer, +} from "../fallback-audio-buffer-sink"; + +type AudioBufferLike = { + length: number; + sampleRate: number; + numberOfChannels: number; + duration: number; + getChannelData: (channel: number) => Float32Array; + copyToChannel: ( + source: Float32Array, + channel: number, + startInChannel?: number, + ) => void; +}; + +function createTestAudioBuffer({ + durationSeconds = 1, + sampleRate = 8, + channelCount = 2, +}: { + durationSeconds?: number; + sampleRate?: number; + channelCount?: number; +} = {}): AudioBufferLike { + const length = Math.round(durationSeconds * sampleRate); + const channels = Array.from( + { length: channelCount }, + (_, channelIndex) => + new Float32Array( + Array.from({ length }, (_, i) => channelIndex + 1 + i * 0.01), + ), + ); + return { + length, + sampleRate, + numberOfChannels: channelCount, + duration: durationSeconds, + getChannelData(channel: number) { + return channels[channel]; + }, + // eslint-disable-next-line opencut/prefer-object-params -- mirrors the AudioBuffer.copyToChannel Web API signature so this fake behaves like the real one + copyToChannel( + source: Float32Array, + channel: number, + startInChannel = 0, + ) { + channels[channel].set(source, startInChannel); + }, + }; +} + +function installFakeAudioBuffer(): { restore: () => void } { + const root = globalThis as { AudioBuffer?: unknown }; + const original = root.AudioBuffer; + class FakeAudioBuffer implements AudioBufferLike { + length: number; + sampleRate: number; + numberOfChannels: number; + duration: number; + private channels: Float32Array[]; + + constructor(opts: { + length: number; + numberOfChannels: number; + sampleRate: number; + }) { + this.length = opts.length; + this.sampleRate = opts.sampleRate; + this.numberOfChannels = opts.numberOfChannels; + this.duration = opts.length / opts.sampleRate; + this.channels = Array.from( + { length: opts.numberOfChannels }, + () => new Float32Array(opts.length), + ); + } + + getChannelData(channel: number): Float32Array { + return this.channels[channel]; + } + + // eslint-disable-next-line opencut/prefer-object-params -- mirrors AudioBuffer.copyToChannel Web API signature + copyToChannel( + source: Float32Array, + channel: number, + startInChannel = 0, + ): void { + this.channels[channel].set(source, startInChannel); + } + } + root.AudioBuffer = FakeAudioBuffer; + return { + restore() { + if (original === undefined) { + delete root.AudioBuffer; + } else { + root.AudioBuffer = original; + } + }, + }; +} + +describe("isWebAudioDecoderAvailable", () => { + const root = globalThis as { AudioDecoder?: unknown }; + let original: unknown; + + beforeEach(() => { + original = root.AudioDecoder; + }); + + afterEach(() => { + if (original === undefined) { + delete root.AudioDecoder; + } else { + root.AudioDecoder = original; + } + }); + + test("returns false when AudioDecoder undefined", () => { + delete root.AudioDecoder; + expect(isWebAudioDecoderAvailable()).toBe(false); + }); + + test("returns true when AudioDecoder defined", () => { + root.AudioDecoder = class {}; + expect(isWebAudioDecoderAvailable()).toBe(true); + }); +}); + +describe("sliceAudioBuffer", () => { + let fakeAudioBuffer: ReturnType; + + beforeEach(() => { + fakeAudioBuffer = installFakeAudioBuffer(); + }); + + afterEach(() => { + fakeAudioBuffer.restore(); + }); + + test("slices to the requested range", () => { + const buffer = createTestAudioBuffer({ + durationSeconds: 1, + sampleRate: 8, + channelCount: 1, + }); + const sliced = sliceAudioBuffer({ + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the fake duck-typed AudioBuffer is sufficient for this unit test + buffer: buffer as unknown as AudioBuffer, + start: 0.25, + end: 0.75, + }); + expect(sliced.length).toBe(4); + expect(sliced.numberOfChannels).toBe(1); + expect(sliced.sampleRate).toBe(8); + const channel = sliced.getChannelData(0); + expect(channel[0]).toBeCloseTo(1 + 0.02, 5); + expect(channel[3]).toBeCloseTo(1 + 0.05, 5); + }); + + test("clamps out-of-range start/end", () => { + const buffer = createTestAudioBuffer({ + durationSeconds: 0.5, + sampleRate: 8, + channelCount: 1, + }); + const sliced = sliceAudioBuffer({ + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the fake duck-typed AudioBuffer is sufficient for this unit test + buffer: buffer as unknown as AudioBuffer, + start: -1, + end: 5, + }); + expect(sliced.length).toBe(buffer.length); + }); + + test("preserves channel count", () => { + const buffer = createTestAudioBuffer({ + durationSeconds: 1, + sampleRate: 8, + channelCount: 2, + }); + const sliced = sliceAudioBuffer({ + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the fake duck-typed AudioBuffer is sufficient for this unit test + buffer: buffer as unknown as AudioBuffer, + start: 0, + end: 0.5, + }); + expect(sliced.numberOfChannels).toBe(2); + }); +}); + +describe("FallbackAudioBufferSink.buffers", () => { + let fakeAudioBuffer: ReturnType; + + beforeEach(() => { + fakeAudioBuffer = installFakeAudioBuffer(); + }); + + afterEach(() => { + fakeAudioBuffer.restore(); + }); + + test("yields a single buffer covering the full range by default", async () => { + const decoded = createTestAudioBuffer({ + durationSeconds: 1, + sampleRate: 8, + channelCount: 1, + }); + const sink = new FallbackAudioBufferSink( + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the fake duck-typed AudioBuffer is sufficient for this unit test + decoded as unknown as AudioBuffer, + ); + const yielded = []; + for await (const wrapped of sink.buffers()) { + yielded.push(wrapped); + } + expect(yielded).toHaveLength(1); + expect(yielded[0].timestamp).toBe(0); + expect(yielded[0].duration).toBeCloseTo(1, 5); + }); + + test("yields a slice when startTimestamp/endTimestamp provided", async () => { + const decoded = createTestAudioBuffer({ + durationSeconds: 2, + sampleRate: 8, + channelCount: 1, + }); + const sink = new FallbackAudioBufferSink( + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the fake duck-typed AudioBuffer is sufficient for this unit test + decoded as unknown as AudioBuffer, + ); + const yielded = []; + for await (const wrapped of sink.buffers(0.5, 1.5)) { + yielded.push(wrapped); + } + expect(yielded).toHaveLength(1); + expect(yielded[0].timestamp).toBe(0.5); + expect(yielded[0].duration).toBeCloseTo(1, 5); + }); + + test("treats NaN/Infinity startTimestamp as 0", async () => { + const decoded = createTestAudioBuffer({ + durationSeconds: 1, + sampleRate: 8, + channelCount: 1, + }); + const sink = new FallbackAudioBufferSink( + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the fake duck-typed AudioBuffer is sufficient for this unit test + decoded as unknown as AudioBuffer, + ); + const yielded = []; + for await (const wrapped of sink.buffers(Number.NaN)) { + yielded.push(wrapped); + } + expect(yielded).toHaveLength(1); + expect(yielded[0].timestamp).toBe(0); + expect(yielded[0].duration).toBeCloseTo(1, 5); + }); + + test("yields nothing when start >= end (or beyond duration)", async () => { + const decoded = createTestAudioBuffer({ + durationSeconds: 1, + sampleRate: 8, + channelCount: 1, + }); + const sink = new FallbackAudioBufferSink( + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the fake duck-typed AudioBuffer is sufficient for this unit test + decoded as unknown as AudioBuffer, + ); + + const yielded = []; + for await (const wrapped of sink.buffers(2)) { + yielded.push(wrapped); + } + expect(yielded).toHaveLength(0); + + const yielded2 = []; + for await (const wrapped of sink.buffers(0.4, 0.4)) { + yielded2.push(wrapped); + } + expect(yielded2).toHaveLength(0); + }); + + test("exposes sampleRate/numberOfChannels/duration getters", () => { + const decoded = createTestAudioBuffer({ + durationSeconds: 3, + sampleRate: 16, + channelCount: 2, + }); + const sink = new FallbackAudioBufferSink( + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the fake duck-typed AudioBuffer is sufficient for this unit test + decoded as unknown as AudioBuffer, + ); + expect(sink.sampleRate).toBe(16); + expect(sink.numberOfChannels).toBe(2); + expect(sink.duration).toBe(3); + }); +}); diff --git a/apps/web/src/services/audio-cache/fallback-audio-buffer-sink.ts b/apps/web/src/services/audio-cache/fallback-audio-buffer-sink.ts new file mode 100644 index 000000000..931a2b8d6 --- /dev/null +++ b/apps/web/src/services/audio-cache/fallback-audio-buffer-sink.ts @@ -0,0 +1,135 @@ +import type { WrappedAudioBuffer } from "mediabunny"; + +export interface AudioBufferSinkLike { + buffers( + startTimestamp?: number, + endTimestamp?: number, + ): AsyncGenerator; +} + +export function isWebAudioDecoderAvailable(): boolean { + return ( + typeof (globalThis as { AudioDecoder?: unknown }).AudioDecoder !== + "undefined" + ); +} + +/** + * AudioBufferSink implementation backed by AudioContext.decodeAudioData. + * + * Used when mediabunny's AudioSampleSink cannot decode the track because the + * environment lacks the WebCodecs AudioDecoder API. AudioContext.decodeAudioData + * uses the system's native audio decoders, mirroring the strategy used by + * HTMLVideoElementSink for video. The entire compressed audio is decoded + * upfront and exposed through a buffers() iterator that yields a single + * AudioBuffer slice covering the requested range. + */ +export class FallbackAudioBufferSink implements AudioBufferSinkLike { + constructor(private readonly decoded: AudioBuffer) {} + + static async create({ + file, + audioContext, + }: { + file: File; + audioContext: BaseAudioContext; + }): Promise { + const arrayBuffer = await file.arrayBuffer(); + const decoded = await decodeArrayBuffer({ arrayBuffer, audioContext }); + return new FallbackAudioBufferSink(decoded); + } + + get sampleRate(): number { + return this.decoded.sampleRate; + } + + get numberOfChannels(): number { + return this.decoded.numberOfChannels; + } + + get duration(): number { + return this.decoded.duration; + } + + // eslint-disable-next-line opencut/prefer-object-params -- mirrors mediabunny's AudioBufferSink.buffers(start, end) so consumers can swap sinks transparently + async *buffers( + startTimestamp = 0, + endTimestamp?: number, + ): AsyncGenerator { + const totalDuration = this.decoded.duration; + const safeStart = Number.isFinite(startTimestamp) ? startTimestamp : 0; + const start = Math.max(0, Math.min(safeStart, totalDuration)); + const end = + typeof endTimestamp === "number" && Number.isFinite(endTimestamp) + ? Math.min(endTimestamp, totalDuration) + : totalDuration; + if (start >= end) return; + + const sliced = sliceAudioBuffer({ + buffer: this.decoded, + start, + end, + }); + + yield { + buffer: sliced, + timestamp: start, + duration: sliced.duration, + }; + } +} + +function decodeArrayBuffer({ + arrayBuffer, + audioContext, +}: { + arrayBuffer: ArrayBuffer; + audioContext: BaseAudioContext; +}): Promise { + return new Promise((resolve, reject) => { + audioContext.decodeAudioData( + arrayBuffer, + (buffer) => resolve(buffer), + (err) => + reject( + err instanceof Error + ? err + : new Error(`decodeAudioData failed: ${String(err)}`), + ), + ); + }); +} + +export function sliceAudioBuffer({ + buffer, + start, + end, +}: { + buffer: AudioBuffer; + start: number; + end: number; +}): AudioBuffer { + const sampleRate = buffer.sampleRate; + const startSample = Math.max( + 0, + Math.min(buffer.length, Math.floor(start * sampleRate)), + ); + const endSample = Math.max( + startSample, + Math.min(buffer.length, Math.ceil(end * sampleRate)), + ); + const length = Math.max(1, endSample - startSample); + + const target = new AudioBuffer({ + length, + numberOfChannels: Math.max(1, buffer.numberOfChannels), + sampleRate, + }); + for (let channel = 0; channel < buffer.numberOfChannels; channel++) { + const channelView = buffer + .getChannelData(channel) + .subarray(startSample, endSample); + target.copyToChannel(channelView, channel, 0); + } + return target; +} diff --git a/apps/web/src/services/video-cache/__tests__/html-video-element-sink.test.ts b/apps/web/src/services/video-cache/__tests__/html-video-element-sink.test.ts new file mode 100644 index 000000000..275bad544 --- /dev/null +++ b/apps/web/src/services/video-cache/__tests__/html-video-element-sink.test.ts @@ -0,0 +1,259 @@ +import { + afterEach, + beforeEach, + describe, + expect, + test, +} from "bun:test"; +import { + clampFallbackFps, + HTMLVideoElementSink, + isWebCodecsAvailable, + type VideoElementHandle, +} from "../html-video-element-sink"; + +type FakeCanvas = { + width: number; + height: number; + getContext: (kind: string) => FakeCanvasContext | null; +}; + +type FakeCanvasContext = { + drawImage: (...args: unknown[]) => void; + drawnCount: number; +}; + +function installFakeDom(): { + createdCanvases: FakeCanvas[]; + restore: () => void; +} { + const createdCanvases: FakeCanvas[] = []; + const originalDocument = ( + globalThis as { document?: unknown } + ).document; + + const fakeDocument = { + createElement(tag: string) { + if (tag !== "canvas") { + throw new Error(`unexpected createElement(${tag}) in test`); + } + const ctx: FakeCanvasContext = { + drawnCount: 0, + drawImage() { + ctx.drawnCount += 1; + }, + }; + const canvas: FakeCanvas = { + width: 0, + height: 0, + getContext(kind: string) { + return kind === "2d" ? ctx : null; + }, + }; + createdCanvases.push(canvas); + return canvas; + }, + }; + + (globalThis as { document?: unknown }).document = fakeDocument; + + return { + createdCanvases, + restore() { + (globalThis as { document?: unknown }).document = originalDocument; + }, + }; +} + +function createFakeHandle({ + duration = 2, + videoWidth = 640, + videoHeight = 360, +}: { + duration?: number; + videoWidth?: number; + videoHeight?: number; +} = {}): VideoElementHandle & { + seekCalls: number[]; + drawCalls: number; + disposed: boolean; +} { + const handle = { + seekCalls: [] as number[], + drawCalls: 0, + disposed: false, + duration, + videoWidth, + videoHeight, + async seek(time: number) { + handle.seekCalls.push(time); + }, + drawTo() { + handle.drawCalls += 1; + }, + dispose() { + handle.disposed = true; + }, + }; + return handle; +} + +describe("clampFallbackFps", () => { + test("returns default for undefined/NaN/zero/negative", () => { + expect(clampFallbackFps(undefined)).toBe(30); + expect(clampFallbackFps(Number.NaN)).toBe(30); + expect(clampFallbackFps(0)).toBe(30); + expect(clampFallbackFps(-5)).toBe(30); + }); + + test("rounds and clamps within [1, 60]", () => { + expect(clampFallbackFps(24)).toBe(24); + expect(clampFallbackFps(29.4)).toBe(29); + expect(clampFallbackFps(120)).toBe(60); + expect(clampFallbackFps(0.4)).toBe(1); + }); +}); + +describe("isWebCodecsAvailable", () => { + const root = globalThis as { VideoDecoder?: unknown }; + let originalVideoDecoder: unknown; + + beforeEach(() => { + originalVideoDecoder = root.VideoDecoder; + }); + + afterEach(() => { + if (originalVideoDecoder === undefined) { + delete root.VideoDecoder; + } else { + root.VideoDecoder = originalVideoDecoder; + } + }); + + test("false when VideoDecoder is undefined", () => { + delete root.VideoDecoder; + expect(isWebCodecsAvailable()).toBe(false); + }); + + test("true when VideoDecoder is defined", () => { + root.VideoDecoder = class {}; + expect(isWebCodecsAvailable()).toBe(true); + }); +}); + +describe("HTMLVideoElementSink.canvases", () => { + let dom: ReturnType; + + beforeEach(() => { + dom = installFakeDom(); + }); + + afterEach(() => { + dom.restore(); + }); + + test("yields frames stepping by 1/fps from startTimestamp", async () => { + const handle = createFakeHandle({ duration: 0.25 }); + const sink = new HTMLVideoElementSink({ handle, fps: 10 }); // step = 0.1s + + const frames: Array<{ timestamp: number; duration: number }> = []; + for await (const frame of sink.canvases(0)) { + frames.push({ + timestamp: frame.timestamp, + duration: frame.duration, + }); + } + + expect(frames).toHaveLength(3); + expect(frames[0].timestamp).toBeCloseTo(0, 5); + expect(frames[1].timestamp).toBeCloseTo(0.1, 5); + expect(frames[2].timestamp).toBeCloseTo(0.2, 5); + expect(frames[0].duration).toBeCloseTo(0.1, 5); + expect(handle.seekCalls).toEqual([0, 0.1, 0.2]); + expect(handle.drawCalls).toBe(3); + }); + + test("respects endTimestamp", async () => { + const handle = createFakeHandle({ duration: 5 }); + const sink = new HTMLVideoElementSink({ handle, fps: 10 }); + + const frames: number[] = []; + for await (const frame of sink.canvases(0, 0.2)) { + frames.push(frame.timestamp); + } + + expect(frames).toHaveLength(2); + expect(frames[0]).toBeCloseTo(0, 5); + expect(frames[1]).toBeCloseTo(0.1, 5); + }); + + test("dispose stops the iterator early", async () => { + const handle = createFakeHandle({ duration: 10 }); + const sink = new HTMLVideoElementSink({ handle, fps: 10 }); + + const collected: number[] = []; + for await (const frame of sink.canvases(0)) { + collected.push(frame.timestamp); + if (collected.length === 2) sink.dispose(); + } + + expect(collected).toHaveLength(2); + expect(handle.disposed).toBe(true); + }); + + test("returns no frames when start >= duration", async () => { + const handle = createFakeHandle({ duration: 1 }); + const sink = new HTMLVideoElementSink({ handle, fps: 30 }); + + const frames: number[] = []; + for await (const frame of sink.canvases(2)) { + frames.push(frame.timestamp); + } + expect(frames).toHaveLength(0); + expect(handle.seekCalls).toHaveLength(0); + }); + + test("aborts cleanly when seek throws", async () => { + const handle = createFakeHandle({ duration: 1 }); + handle.seek = async () => { + throw new Error("boom"); + }; + const sink = new HTMLVideoElementSink({ handle, fps: 30 }); + + const frames: number[] = []; + for await (const frame of sink.canvases(0)) { + frames.push(frame.timestamp); + } + expect(frames).toHaveLength(0); + }); + + test("clamps invalid fps passed to constructor to the default", async () => { + const handle = createFakeHandle({ duration: 0.05 }); + // Passing 0 (negative, or NaN) used to leak through to 1/fps and produce + // invalid frame stepping. The constructor must reject these and fall back + // to the default FPS. + const sink = new HTMLVideoElementSink({ handle, fps: 0 }); + const frames = []; + for await (const frame of sink.canvases(0)) { + frames.push(frame); + } + expect(frames).toHaveLength(2); + expect(frames[0].duration).toBeCloseTo(1 / 30, 5); + }); + + test("frame canvas matches video dimensions", async () => { + const handle = createFakeHandle({ + duration: 0.02, + videoWidth: 1280, + videoHeight: 720, + }); + const sink = new HTMLVideoElementSink({ handle, fps: 30 }); + const frames = []; + for await (const frame of sink.canvases(0)) { + frames.push(frame); + } + expect(frames).toHaveLength(1); + expect(frames[0].canvas.width).toBe(1280); + expect(frames[0].canvas.height).toBe(720); + }); +}); diff --git a/apps/web/src/services/video-cache/html-video-element-sink.ts b/apps/web/src/services/video-cache/html-video-element-sink.ts new file mode 100644 index 000000000..ae525d91f --- /dev/null +++ b/apps/web/src/services/video-cache/html-video-element-sink.ts @@ -0,0 +1,249 @@ +import type { WrappedCanvas } from "mediabunny"; + +const DEFAULT_FALLBACK_FPS = 30; +const MAX_FALLBACK_FPS = 60; +const MIN_FALLBACK_FPS = 1; + +export interface FrameSink { + canvases( + startTimestamp?: number, + endTimestamp?: number, + ): AsyncGenerator; +} + +export interface VideoElementHandle { + readonly duration: number; + readonly videoWidth: number; + readonly videoHeight: number; + seek(time: number): Promise; + drawTo(target: { + canvas: HTMLCanvasElement | OffscreenCanvas; + ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D; + }): void; + dispose(): void; +} + +export function isWebCodecsAvailable(): boolean { + return typeof (globalThis as { VideoDecoder?: unknown }).VideoDecoder !== + "undefined"; +} + +export function clampFallbackFps(requested: number | undefined): number { + if ( + requested === undefined || + !Number.isFinite(requested) || + requested <= 0 + ) { + return DEFAULT_FALLBACK_FPS; + } + const rounded = Math.round(requested); + if (rounded < MIN_FALLBACK_FPS) return MIN_FALLBACK_FPS; + if (rounded > MAX_FALLBACK_FPS) return MAX_FALLBACK_FPS; + return rounded; +} + +export async function createHTMLVideoElementHandle({ + file, +}: { + file: File; +}): Promise { + const objectUrl = URL.createObjectURL(file); + const video = document.createElement("video"); + video.preload = "auto"; + video.muted = true; + video.playsInline = true; + video.crossOrigin = "anonymous"; + video.src = objectUrl; + + try { + await waitForLoadedMetadata({ video }); + } catch (error) { + URL.revokeObjectURL(objectUrl); + throw error; + } + + let disposed = false; + + return { + get duration() { + return video.duration; + }, + get videoWidth() { + return video.videoWidth; + }, + get videoHeight() { + return video.videoHeight; + }, + async seek(time) { + if (disposed) { + throw new Error("VideoElementHandle disposed"); + } + await seekVideoElement({ video, time }); + }, + drawTo({ canvas, ctx }) { + ctx.drawImage(video, 0, 0, canvas.width, canvas.height); + }, + dispose() { + if (disposed) return; + disposed = true; + try { + video.pause(); + } catch { + // ignore + } + video.removeAttribute("src"); + try { + video.load(); + } catch { + // ignore + } + URL.revokeObjectURL(objectUrl); + }, + }; +} + +function waitForLoadedMetadata({ + video, +}: { + video: HTMLVideoElement; +}): Promise { + return new Promise((resolve, reject) => { + const cleanup = () => { + video.removeEventListener("loadedmetadata", onLoaded); + video.removeEventListener("error", onError); + }; + const onLoaded = () => { + cleanup(); + resolve(); + }; + const onError = () => { + cleanup(); + const msg = video.error?.message ?? "unknown error"; + reject(new Error(`HTMLVideoElement load failed: ${msg}`)); + }; + video.addEventListener("loadedmetadata", onLoaded); + video.addEventListener("error", onError); + }); +} + +function seekVideoElement({ + video, + time, +}: { + video: HTMLVideoElement; + time: number; +}): Promise { + return new Promise((resolve, reject) => { + const cleanup = () => { + video.removeEventListener("seeked", onSeeked); + video.removeEventListener("error", onError); + }; + const onSeeked = () => { + cleanup(); + resolve(); + }; + const onError = () => { + cleanup(); + reject(new Error("HTMLVideoElement seek failed")); + }; + video.addEventListener("seeked", onSeeked); + video.addEventListener("error", onError); + try { + video.currentTime = time; + } catch (err) { + cleanup(); + reject(err instanceof Error ? err : new Error(String(err))); + } + }); +} + +export class HTMLVideoElementSink implements FrameSink { + private disposed = false; + private readonly handle: VideoElementHandle; + private readonly fps: number; + + constructor({ + handle, + fps, + }: { + handle: VideoElementHandle; + fps?: number; + }) { + this.handle = handle; + this.fps = clampFallbackFps(fps); + } + + static async create({ + file, + fps, + }: { + file: File; + fps?: number; + }): Promise { + const handle = await createHTMLVideoElementHandle({ file }); + return new HTMLVideoElementSink({ handle, fps: clampFallbackFps(fps) }); + } + + // eslint-disable-next-line opencut/prefer-object-params -- mirrors mediabunny's CanvasSink.canvases(start, end) so VideoCache can use either sink transparently + async *canvases( + startTimestamp = 0, + endTimestamp?: number, + ): AsyncGenerator { + const step = 1 / this.fps; + const duration = this.handle.duration; + const end = + typeof endTimestamp === "number" && Number.isFinite(endTimestamp) + ? Math.min(endTimestamp, duration) + : duration; + let t = Math.max(0, startTimestamp); + + while (t < end && !this.disposed) { + try { + await this.handle.seek(t); + } catch (error) { + console.warn("[HTMLVideoElementSink] seek failed:", error); + return; + } + if (this.disposed) return; + + const canvas = createFrameCanvas({ + width: this.handle.videoWidth, + height: this.handle.videoHeight, + }); + const ctx = canvas.getContext("2d"); + if (!ctx) { + console.warn( + "[HTMLVideoElementSink] 2d context unavailable, aborting iterator", + ); + return; + } + this.handle.drawTo({ canvas, ctx }); + + yield { + canvas, + timestamp: t, + duration: step, + }; + t += step; + } + } + + dispose(): void { + if (this.disposed) return; + this.disposed = true; + this.handle.dispose(); + } +} + +function createFrameCanvas({ + width, + height, +}: { + width: number; + height: number; +}): HTMLCanvasElement { + const canvas = document.createElement("canvas"); + canvas.width = Math.max(1, Math.round(width)); + canvas.height = Math.max(1, Math.round(height)); + return canvas; +} diff --git a/apps/web/src/services/video-cache/service.ts b/apps/web/src/services/video-cache/service.ts index cad4fc318..9f6dc8819 100644 --- a/apps/web/src/services/video-cache/service.ts +++ b/apps/web/src/services/video-cache/service.ts @@ -5,10 +5,15 @@ import { CanvasSink, type WrappedCanvas, } from "mediabunny"; +import { + type FrameSink, + HTMLVideoElementSink, + isWebCodecsAvailable, +} from "./html-video-element-sink"; interface VideoSinkData { - input: Input; - sink: CanvasSink; + sink: FrameSink; + dispose: () => void; iterator: AsyncGenerator | null; currentFrame: WrappedCanvas | null; nextFrame: WrappedCanvas | null; @@ -275,7 +280,15 @@ export class VideoCache { const canDecode = await videoTrack.canDecode(); if (!canDecode) { - throw new Error("Video codec not supported for decoding"); + if (isWebCodecsAvailable()) { + throw new Error("Video codec not supported for decoding"); + } + // WebCodecs API absent (e.g. browsers without VideoDecoder support). + // Fall back to an HTMLVideoElement-backed sink which uses the + // system's native video decoders via the