Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 8 additions & 7 deletions README.ja.md
Original file line number Diff line number Diff line change
Expand Up @@ -150,13 +150,14 @@ auto mode を使っていて、それでも実行が拒否される場合は、`
| `motion-render build <work> [out.mp4]` | ナレーション・BGM・効果音を合成し、映像と合わせて MP4 にする。どれかが足りないとき、HTML の映像が配置のチェックに通らないときは止まる |
| `motion-render --version` | 版を表示する |

| 環境変数 | 効果 |
| ----------------------- | ------------------------------------------------------------ |
| `GEMINI_API_KEY` | ナレーションと BGM の生成に必要 |
| `ELEVENLABS_API_KEY` | 効果音の生成に必要(無ければ効果音を付けない) |
| `MOTION_RENDER_OUT_DIR` | `build` の出力先を省略したときの保存先 |
| `CHROMIUM_PATH` | 同梱の Chromium の代わりに、手元の Chrome や Chromium を使う |
| `FFMPEG_PATH` | 同梱の ffmpeg の代わりに、手元の ffmpeg を使う |
| 環境変数 | 効果 |
| ----------------------- | ------------------------------------------------------------------------------------------ |
| `GEMINI_API_KEY` | ナレーションと BGM の生成に必要 |
| `ELEVENLABS_API_KEY` | 効果音の生成に必要(無ければ効果音を付けない) |
| `MOTION_RENDER_OUT_DIR` | `build` の出力先を省略したときの保存先 |
| `CHROMIUM_PATH` | 同梱の Chromium の代わりに、手元の Chrome や Chromium を使う |
| `FFMPEG_PATH` | 同梱の ffmpeg の代わりに、手元の ffmpeg を使う |
| `MOTION_RENDER_PAGES` | 同時にコマを描くブラウザの数(既定: コア数の半分、最大 8)。マシンが混んでいるときは減らす |

## 困ったとき

Expand Down
15 changes: 8 additions & 7 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -150,13 +150,14 @@ Each video uses one work directory (`<work>`). Its layout and the `script.json`
| `motion-render build <work> [out.mp4]` | Mixes narration, BGM and sound effects and renders the MP4. Stops if any is missing, or if an HTML film fails the layout check |
| `motion-render --version` | Prints the version |

| Environment variable | Effect |
| ----------------------- | ---------------------------------------------------------- |
| `GEMINI_API_KEY` | Required for narration and BGM |
| `ELEVENLABS_API_KEY` | Required for sound effects (none are added without it) |
| `MOTION_RENDER_OUT_DIR` | Default output directory when `build` gets no output path |
| `CHROMIUM_PATH` | Use your own Chrome or Chromium instead of the bundled one |
| `FFMPEG_PATH` | Use your own ffmpeg instead of the bundled one |
| Environment variable | Effect |
| ----------------------- | ------------------------------------------------------------------------------------------------------- |
| `GEMINI_API_KEY` | Required for narration and BGM |
| `ELEVENLABS_API_KEY` | Required for sound effects (none are added without it) |
| `MOTION_RENDER_OUT_DIR` | Default output directory when `build` gets no output path |
| `CHROMIUM_PATH` | Use your own Chrome or Chromium instead of the bundled one |
| `FFMPEG_PATH` | Use your own ffmpeg instead of the bundled one |
| `MOTION_RENDER_PAGES` | Browsers that draw frames at once (default: half the cores, at most 8). Lower it if the machine is busy |

## Troubleshooting

Expand Down
70 changes: 40 additions & 30 deletions skills/motion-render/scripts/audio.ts
Original file line number Diff line number Diff line change
@@ -1,14 +1,18 @@
// ナレーション(Gemini TTS)と BGM(Lyria)を作り、ffmpeg で 1 本の音声に合成する。
import { existsSync, readFileSync, writeFileSync, mkdirSync } from "node:fs";
import { join, dirname } from "node:path";
import { spawn, spawnSync } from "node:child_process";
import { basename, join, dirname } from "node:path";
import { spawn, spawnSync, type ChildProcess } from "node:child_process";
import { createRequire } from "node:module";
import { errorMessage, field, readJson } from "./json.ts";
import { limit } from "./limit.ts";
import type { Script } from "./work.ts";

const API = "https://generativelanguage.googleapis.com/v1beta/models";
export const TIMEOUT_MS = 90_000; // a TTS request once hung for 15 minutes without this
const RETRIES = 3;
// An audio job (mixing a few minutes of sound, measuring a level) takes ffmpeg seconds. One still
// running after this is stuck, not slow.
export const FFMPEG_MS = 300_000;

interface Audio {
mimeType: string;
Expand Down Expand Up @@ -60,40 +64,46 @@ export function findFfmpeg(): string {
throw new Error("ffmpeg not found: install ffmpeg or set FFMPEG_PATH");
}

export function runFfmpeg(args: string[]): Promise<void> {
// Resolves with ffmpeg's exit code once it has exited.
export function exited(ff: ChildProcess): Promise<number | null> {
return new Promise((ok, fail) => {
const ff = spawn(findFfmpeg(), ["-y", "-loglevel", "error", ...args], {
stdio: ["ignore", "inherit", "inherit"],
});
ff.on("error", fail);
ff.on("close", (code) =>
code === 0 ? ok() : fail(new Error(`ffmpeg exited with code ${code}`))
);
ff.on("close", ok);
});
}

// Average and peak level of an audio file in dBFS (ffmpeg's volumedetect). Silence reads as -91 dB or lower.
export function volumeStats(file: string): Promise<{ mean: number; max: number }> {
return new Promise((ok, fail) => {
const args = ["-hide_banner", "-nostats", "-i", file, "-af", "volumedetect", "-f", "null", "-"];
const ff = spawn(findFfmpeg(), args, { stdio: ["ignore", "ignore", "pipe"] });
let log = "";
ff.stderr?.on("data", (d: Buffer) => (log += d.toString()));
ff.on("error", fail);
ff.on("close", (code) => {
const level = (name: string): number => {
const v = new RegExp(`${name}: (-?[\\d.]+|-inf) dB`).exec(log)?.[1];
return v === "-inf" ? -120 : Number(v ?? NaN);
};
const mean = level("mean_volume"),
max = level("max_volume");
if (code !== 0 || !Number.isFinite(mean) || !Number.isFinite(max))
return fail(
new Error(`could not measure the level of ${file} (ffmpeg exited with ${code})`)
);
ok({ mean, max });
});
// Waits for ffmpeg to exit, killing it if it runs past ms.
function finish(ff: ChildProcess, what: string, ms: number): Promise<number | null> {
return limit(exited(ff), ms, what, () => ff.kill("SIGKILL"));
}

export async function runFfmpeg(args: string[], ms = FFMPEG_MS): Promise<void> {
const ff = spawn(findFfmpeg(), ["-y", "-loglevel", "error", ...args], {
stdio: ["ignore", "inherit", "inherit"],
});
const code = await finish(ff, `ffmpeg writing ${basename(args.at(-1) ?? "")}`, ms);
if (code !== 0) throw new Error(`ffmpeg exited with code ${code}`);
}

// Average and peak level of an audio file in dBFS (ffmpeg's volumedetect). Silence reads as -91 dB or lower.
export async function volumeStats(
file: string,
ms = FFMPEG_MS
): Promise<{ mean: number; max: number }> {
const args = ["-hide_banner", "-nostats", "-i", file, "-af", "volumedetect", "-f", "null", "-"];
const ff = spawn(findFfmpeg(), args, { stdio: ["ignore", "ignore", "pipe"] });
let log = "";
ff.stderr?.on("data", (d: Buffer) => (log += d.toString()));
const code = await finish(ff, `ffmpeg measuring ${basename(file)}`, ms);
const level = (name: string): number => {
const v = new RegExp(`${name}: (-?[\\d.]+|-inf) dB`).exec(log)?.[1];
return v === "-inf" ? -120 : Number(v ?? NaN);
};
const mean = level("mean_volume"),
max = level("max_volume");
if (code !== 0 || !Number.isFinite(mean) || !Number.isFinite(max))
throw new Error(`could not measure the level of ${file} (ffmpeg exited with ${code})`);
return { mean, max };
}

// Runs `attempt` up to RETRIES times, logging each failure under `label`, and rethrows the last error.
Expand Down
40 changes: 24 additions & 16 deletions skills/motion-render/scripts/cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,12 @@ import { spawn } from "node:child_process";
import { once } from "node:events";
import { parseArgs } from "node:util";
import { fileURLToPath } from "node:url";
import { openFilm, checkFilmDir, isHtmlFilm, type Film } from "./render.ts";
import { openFilm, checkFilmDir, isHtmlFilm, capturePages, type Film } from "./render.ts";
import { inOrder } from "./inorder.ts";
import { checkLayout, formatIssue } from "./layout.ts";
import { writeKit, writeKitTiming } from "./kit.ts";
import { narrate, bgm, bgmFile, mix, findFfmpeg, encodeArgs } from "./audio.ts";
import { narrate, bgm, bgmFile, mix, findFfmpeg, encodeArgs, exited, FFMPEG_MS } from "./audio.ts";
import { limit } from "./limit.ts";
import { loadScript, writeTiming, loadTiming, placeOnFilm, placeSfx } from "./work.ts";
import { checkSfx, makeSfx, narrationLevel, previewSfx, sfxTrack } from "./sfx.ts";
import { prepare } from "./store.ts";
Expand Down Expand Up @@ -39,7 +41,8 @@ env:
ELEVENLABS_API_KEY required by sfx
MOTION_RENDER_OUT_DIR default output directory for build
CHROMIUM_PATH use this Chrome/Chromium instead of the bundled one
FFMPEG_PATH use this ffmpeg instead of the bundled one`;
FFMPEG_PATH use this ffmpeg instead of the bundled one
MOTION_RENDER_PAGES browsers that draw frames at once (default: half the cores, at most 8)`;

function die(msg: string, code = 1): never {
console.error(`motion-render: ${msg}`);
Expand Down Expand Up @@ -68,7 +71,7 @@ async function stills(filmDir: string, outDir: string, list: string): Promise<vo
const times = list.split(",").map(Number);
if (times.some((t) => !Number.isFinite(t))) die(`times must be numbers: ${list}`);
mkdirSync(outDir, { recursive: true });
const film = await openFilm(filmDir);
const film = await openFilm(filmDir, { pages: capturePages() });
try {
for (const t of times)
writeFileSync(join(outDir, `t_${t.toFixed(2)}.png`), await film.frame(t));
Expand All @@ -83,7 +86,7 @@ async function stills(filmDir: string, outDir: string, list: string): Promise<vo
async function check(filmDir: string): Promise<void> {
if (!isHtmlFilm(filmDir))
die("check works on html films only (Motion films draw in their own player)");
const film = await openFilm(filmDir);
const film = await openFilm(filmDir, { pages: capturePages() });
let n: number;
try {
n = await reportLayout(film);
Expand Down Expand Up @@ -124,27 +127,31 @@ async function runSfx(work: string): Promise<void> {
console.log(`made ${made} new, listen to all of them: ${preview.file}`);
}

// Pipes rendered frames straight into ffmpeg (no temp files) and muxes the mixed audio.
// ffmpeg takes a frame in milliseconds. One that has not taken it after this is stuck.
const FRAME_MS = 60_000;

// Pipes rendered frames straight into ffmpeg (no temp files) and muxes the mixed audio. The whole
// encode runs as long as capture does, so the limits are per frame and for the final mux.
async function encode(film: Film, fps: number, audio: string, out: string): Promise<number> {
const count = Math.round(film.meta.duration * fps);
const args = encodeArgs(fps, audio, out);
const ff = spawn(findFfmpeg(), args, { stdio: ["pipe", "inherit", "inherit"] });
const finished = new Promise<void>((ok, fail) => {
ff.on("error", fail);
ff.on("close", (code) =>
code === 0 ? ok() : fail(new Error(`ffmpeg exited with code ${code}`))
);
});
const kill = () => ff.kill("SIGKILL");
const exit = exited(ff);
ff.stdin.on("error", () => {}); // EPIPE when ffmpeg dies early; its exit code reports the cause.
try {
for (let i = 0; i < count; i++) {
if (!ff.stdin.write(await film.frame(i / fps))) await once(ff.stdin, "drain");
let i = 0;
for await (const png of inOrder(count, film.pages, (k) => film.frame(k / fps))) {
if (!ff.stdin.write(png))
await limit(once(ff.stdin, "drain"), FRAME_MS, `ffmpeg taking frame ${i}`, kill);
if (i % Math.max(1, Math.round(count / 10)) === 0) console.error(`frames: ${i}/${count}`);
i++;
}
} finally {
ff.stdin.end();
}
await finished;
const code = await limit(exit, FFMPEG_MS, "ffmpeg finishing the MP4", kill);
if (code !== 0) throw new Error(`ffmpeg exited with code ${code}`);
return count;
}

Expand All @@ -163,7 +170,8 @@ async function build(work: string, out: string, fps: number): Promise<void> {
checkFilmDir(filmDir);
if (isHtmlFilm(filmDir)) writeKitTiming(work);

const film = await openFilm(filmDir);
const film = await openFilm(filmDir, { pages: capturePages() });
console.error(`capture: ${film.pages} pages`);
try {
if (await reportLayout(film))
throw new Error(
Expand Down
23 changes: 23 additions & 0 deletions skills/motion-render/scripts/inorder.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
// Yields get(0) … get(count - 1) in order while keeping up to `ahead` of them in flight, so the
// pages of a film draw frames in parallel and the frames still come out in film order. At most
// `ahead` results are held at once.
export async function* inOrder<T>(
count: number,
ahead: number,
get: (i: number) => Promise<T>
): AsyncGenerator<T> {
const queue: Promise<T>[] = [];
let next = 0;
for (let i = 0; i < count; i++) {
while (next < count && queue.length < Math.max(1, ahead)) {
const p = get(next++);
// Awaited in order below. This only keeps a later one that fails while an earlier one is
// still awaited from being reported as unhandled.
p.catch(() => {});
queue.push(p);
}
const p = queue.shift();
if (!p) return;
yield await p;
}
}
20 changes: 12 additions & 8 deletions skills/motion-render/scripts/layout.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
// Only what stays still is reported, so a chip passing over a card on its way is not a mistake.
import type { Film } from "./render.ts";
import { isRecord } from "./json.ts";
import { inOrder } from "./inorder.ts";

// A thing that paints on screen at one moment (snapshot.js). Coordinates are CSS px in the frame.
export interface LayoutBox {
Expand Down Expand Up @@ -228,15 +229,18 @@ export function parseBoxes(raw: unknown): LayoutBox[] {
return raw.map(parseBox);
}

// Samples the whole film every `step` seconds. A finding needs two samples in a row, so at 0.1 s a
// pose held for 0.2 s is reported. Motion films draw inside their own player and cannot be
// inspected, so they get null.
// Samples the whole film every `step` seconds, on all of the film's pages at once, and judges the
// samples in film order. A finding needs two samples in a row, so at 0.1 s a pose held for 0.2 s is
// reported. Motion films draw inside their own player and cannot be inspected, so they get null.
export async function checkLayout(film: Film, step = 0.1): Promise<LayoutIssue[] | null> {
if (!film.inspect) return null;
const samples: LayoutSample[] = [];
for (let i = 0; i * step < film.meta.duration; i++) {
const inspect = film.inspect;
if (!inspect) return null;
const count = Math.ceil(film.meta.duration / step - 1e-9);
const sample = (i: number): Promise<LayoutSample> => {
const t = Math.round(i * step * 1000) / 1000;
samples.push({ t, boxes: await film.inspect(t) });
}
return inspect(t).then((boxes) => ({ t, boxes }));
};
const samples: LayoutSample[] = [];
for await (const s of inOrder(count, film.pages, sample)) samples.push(s);
return findLayoutIssues(samples);
}
20 changes: 20 additions & 0 deletions skills/motion-render/scripts/limit.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
// Time limits for the steps that can stall without an error: calls into the film page and ffmpeg.
// A stalled step then fails with its name instead of leaving the command waiting forever.

// Fails with "<what> did not finish in <n>s" if p has not settled within ms. `onTimeout` runs first,
// to stop what is stuck (kill ffmpeg, for example).
export function limit<T>(
p: Promise<T>,
ms: number,
what: string,
onTimeout?: () => void
): Promise<T> {
let timer: NodeJS.Timeout | undefined;
const late = new Promise<never>((_, fail) => {
timer = setTimeout(() => {
onTimeout?.();
fail(new Error(`${what} did not finish in ${ms / 1000}s`));
}, ms);
});
return Promise.race([p, late]).finally(() => clearTimeout(timer));
}
Loading
Loading