feat(media): support browsers without WebCodecs via FFmpeg + HTML5 fallbacks

apps/web/src/app/api/convert-video/route.ts

@@ -0,0 +1,144 @@
+import { type NextRequest, NextResponse } from "next/server";
+import { writeFile, readFile, rm, mkdtemp } from "fs/promises";
+import { spawn } from "child_process";
+import path from "path";
+import os from "os";
+
+const GENERIC_CONVERSION_ERROR = "Video conversion failed.";
+
+function sanitizeContentDispositionName({
+	rawName,
+}: {
+	rawName: string;
+}): string {
+	// Strip characters that can break the Content-Disposition header (quotes,
+	// backslashes, CR/LF, control bytes) and replace non-ASCII so the
+	// "filename=" parameter stays unambiguous across clients.
+	const stripped = rawName
+		// eslint-disable-next-line no-control-regex
+		.replace(/[\\"\r\n\x00-\x1f]/g, "_")
+		.replace(/[^\x20-\x7e]/g, "_")
+		.slice(0, 200);
+	return stripped.length > 0 ? stripped : "video";
+}
+
+export async function POST(request: NextRequest) {
+	let workDir: string | null = null;
+
+	try {
+		const formData = await request.formData();
+		const file = formData.get("file") as File | null;
+
+		if (!file || !(file instanceof File)) {
+			return NextResponse.json(
+				{ error: "No file provided" },
+				{ status: 400 },
+			);
+		}
+
+		workDir = await mkdtemp(path.join(os.tmpdir(), "opencut-convert-"));
+		const ext = path.extname(file.name) || ".mp4";
+		const inputPath = path.join(workDir, `input${ext}`);
+		const outputPath = path.join(workDir, "output.mp4");
+
+		const buffer = Buffer.from(await file.arrayBuffer());
+		await writeFile(inputPath, buffer);
+
+		await runFfmpeg({
+			inputPath,
+			outputPath,
+		});
+
+		const convertedBuffer = await readFile(outputPath);
+
+		const safeName = sanitizeContentDispositionName({
+			rawName: path.basename(file.name, ext),
+		});
+
+		return new Response(new Uint8Array(convertedBuffer), {
+			status: 200,
+			headers: {
+				"Content-Type": "video/mp4",
+				"Content-Disposition": `attachment; filename="${safeName}.mp4"`,
+			},
+		});
+	} catch (error) {
+		console.error("Video conversion error:", error);
+		return NextResponse.json(
+			{ error: GENERIC_CONVERSION_ERROR },
+			{ status: 500 },
+		);
+	} finally {
+		if (workDir) {
+			await rm(workDir, { recursive: true, force: true }).catch(() => {});
+		}
+	}
+}
+
+function runFfmpeg({
+	inputPath,
+	outputPath,
+}: {
+	inputPath: string;
+	outputPath: string;
+}): Promise<void> {
+	return new Promise((resolve, reject) => {
+		const proc = spawn("ffmpeg", [
+			"-i",
+			inputPath,
+			"-c:v",
+			"libx264",
+			"-profile:v",
+			"baseline",
+			"-level",
+			"3.1",
+			"-preset",
+			"veryfast",
+			"-crf",
+			"23",
+			"-pix_fmt",
+			"yuv420p",
+			"-g",
+			"60",
+			"-keyint_min",
+			"60",
+			"-sc_threshold",
+			"0",
+			"-movflags",
+			"+faststart",
+			"-tag:v",
+			"avc1",
+			"-c:a",
+			"aac",
+			"-ac",
+			"2",
+			"-ar",
+			"48000",
+			"-b:a",
+			"128k",
+			"-y",
+			outputPath,
+		]);
+
+		let stderr = "";
+		proc.stderr.on("data", (data) => {
+			stderr += data.toString();
+		});
+
+		proc.on("close", (code) => {
+			if (code === 0) {
+				resolve();
+			} else {
+				reject(
+					new Error(
+						`ffmpeg exited with code ${code}. stderr: ${stderr.slice(-500)}`,
+					),
+				);
+			}
+		});
+
+		proc.on("error", (err) => {
+			reject(err);
+		});
+	});
+}

apps/web/src/core/managers/audio-manager.ts

@@ -20,6 +20,10 @@ import {
 	Input,
 	type WrappedAudioBuffer,
 } from "mediabunny";
+import {
+	type AudioBufferSinkLike,
+	FallbackAudioBufferSink,
+} from "@/services/audio-cache/fallback-audio-buffer-sink";
 
 export class AudioManager {
 	private audioContext: AudioContext | null = null;
@@ -30,7 +34,7 @@ export class AudioManager {
 	private lookaheadSeconds = 2;
 	private scheduleIntervalMs = 500;
 	private clips: AudioClipSource[] = [];
-	private sinks = new Map<string, AudioBufferSink>();
+	private sinks = new Map<string, AudioBufferSinkLike>();
 	private inputs = new Map<string, Input>();
 	private activeClipIds = new Set<string>();
 	private clipIterators = new Map<
@@ -267,7 +271,42 @@ export class AudioManager {
 
 		const iterator = sink.buffers(sourceStartTime);
 		this.clipIterators.set(clip.id, iterator);
-		let consecutiveDroppedBufferCount = 0;
+
+		try {
+			await this.consumeClipIterator({
+				clip,
+				clipEnd,
+				iterator,
+				audioContext,
+				sessionId,
+				initialDroppedCount: 0,
+			});
+		} catch (error) {
+			console.warn(
+				`[AudioManager] clip iterator failed for ${clip.id}; aborting playback for this clip.`,
+				error,
+			);
+			this.clipIterators.delete(clip.id);
+			this.activeClipIds.delete(clip.id);
+		}
+	}
+
+	private async consumeClipIterator({
+		clip,
+		clipEnd,
+		iterator,
+		audioContext,
+		sessionId,
+		initialDroppedCount,
+	}: {
+		clip: AudioClipSource;
+		clipEnd: number;
+		iterator: AsyncGenerator<WrappedAudioBuffer, void, unknown>;
+		audioContext: AudioContext;
+		sessionId: number;
+		initialDroppedCount: number;
+	}): Promise<void> {
+		let consecutiveDroppedBufferCount = initialDroppedCount;
 
 		for await (const { buffer, timestamp } of iterator) {
 			if (!this.editor.playback.getIsPlaying()) return;
@@ -605,7 +644,18 @@ export class AudioManager {
 				return null;
 			}
 
-			const sink = new AudioBufferSink(audioTrack);
+			const canDecode = await safeCanDecode({ audioTrack });
+			const sink: AudioBufferSinkLike = canDecode
+				? new AudioBufferSink(audioTrack)
+				: await FallbackAudioBufferSink.create({
+						file: clip.file,
+						audioContext,
+					});
+			if (!canDecode) {
+				console.warn(
+					"[AudioManager] WebCodecs unavailable; decoding clip audio via decodeAudioData.",
+				);
+			}
 			const chunks: AudioBuffer[] = [];
 			let totalSamples = 0;
 
@@ -673,12 +723,13 @@ export class AudioManager {
 		clip,
 	}: {
 		clip: AudioClipSource;
-	}): Promise<AudioBufferSink | null> {
+	}): Promise<AudioBufferSinkLike | null> {
 		const existingSink = this.sinks.get(clip.sourceKey);
 		if (existingSink) return existingSink;
 
+		let input: Input | null = null;
 		try {
-			const input = new Input({
+			input = new Input({
 				source: new BlobSource(clip.file),
 				formats: ALL_FORMATS,
 			});
@@ -688,13 +739,47 @@ export class AudioManager {
 				return null;
 			}
 
+			const canDecode = await safeCanDecode({ audioTrack });
+			if (!canDecode) {
+				input.dispose();
+				input = null;
+				const audioContext = this.ensureAudioContext();
+				if (!audioContext) {
+					return null;
+				}
+				const fallback = await FallbackAudioBufferSink.create({
+					file: clip.file,
+					audioContext,
+				});
+				this.sinks.set(clip.sourceKey, fallback);
+				console.warn(
+					"[AudioManager] WebCodecs unavailable; using decodeAudioData fallback for audio clip.",
+				);
+				return fallback;
+			}
+
 			const sink = new AudioBufferSink(audioTrack);
 			this.inputs.set(clip.sourceKey, input);
 			this.sinks.set(clip.sourceKey, sink);
 			return sink;
 		} catch (error) {
+			if (input) {
+				input.dispose();
+			}
 			console.warn("Failed to initialize audio sink:", error);
 			return null;
 		}
 	}
 }
+
+async function safeCanDecode({
+	audioTrack,
+}: {
+	audioTrack: { canDecode: () => Promise<boolean> };
+}): Promise<boolean> {
+	try {
+		return await audioTrack.canDecode();
+	} catch {
+		return false;
+	}
+}

apps/web/src/media/audio.ts

@@ -20,6 +20,10 @@ import { canTrackHaveAudio } from "@/timeline";
 import { mediaSupportsAudio } from "@/media/media-utils";
 import { getSourceTimeAtClipTime, renderRetimedBuffer } from "@/retime";
 import { Input, ALL_FORMATS, BlobSource, AudioBufferSink } from "mediabunny";
+import {
+	type AudioBufferSinkLike,
+	FallbackAudioBufferSink,
+} from "@/services/audio-cache/fallback-audio-buffer-sink";
 import { TICKS_PER_SECOND } from "@/wasm";
 import {
 	computeRmsBuckets,
@@ -271,7 +275,23 @@ async function resolveAudioBufferForAsset({
 		const audioTrack = await input.getPrimaryAudioTrack();
 		if (!audioTrack) return null;
 
-		const sink = new AudioBufferSink(audioTrack);
+		let audioTrackCanDecode = false;
+		try {
+			audioTrackCanDecode = await audioTrack.canDecode();
+		} catch {
+			audioTrackCanDecode = false;
+		}
+		const sink: AudioBufferSinkLike = audioTrackCanDecode
+			? new AudioBufferSink(audioTrack)
+			: await FallbackAudioBufferSink.create({
+					file: asset.file,
+					audioContext,
+				});
+		if (!audioTrackCanDecode) {
+			console.warn(
+				"[audio] WebCodecs unavailable; decoding asset via decodeAudioData fallback.",
+			);
+		}
 		const targetSampleRate = audioContext.sampleRate;
 
 		const chunks: AudioBuffer[] = [];

apps/web/src/media/ffmpeg-convert.ts

@@ -0,0 +1,22 @@
+export async function convertVideoToH264({
+	file,
+}: {
+	file: File;
+}): Promise<File> {
+	const formData = new FormData();
+	formData.append("file", file);
+
+	const response = await fetch("/api/convert-video", {
+		method: "POST",
+		body: formData,
+	});
+
+	if (!response.ok) {
+		const errorData = await response.json().catch(() => ({}));
+		throw new Error(errorData.error || `Conversion failed: ${response.status}`);
+	}
+
+	const blob = await response.blob();
+	const baseName = file.name.replace(/\.[^.]+$/, "");
+	return new File([blob], `${baseName}.mp4`, { type: "video/mp4" });
+}