From 74eeb6df36803bd6ded8c74cca298006f6103bac Mon Sep 17 00:00:00 2001 From: Zach Fleeman Date: Fri, 6 Nov 2026 10:28:02 -0700 Subject: [PATCH] Reuse the first pass's stats when a retry re-encodes --- README.md | 2 ++ ffmpeg4discord/__main__.py | 54 ++++++++++++++++++++------------------ ffmpeg4discord/twopass.py | 25 +++++++++++++++--- tests/test_main.py | 13 +++++++++ tests/test_twopass.py | 32 ++++++++++++++++++++++ 5 files changed, 97 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 6be346d..d7332af 100644 --- a/README.md +++ b/README.md @@ -212,6 +212,8 @@ Different codecs have different trade-offs in terms of quality, speed, file size Hardware encoders (NVENC and VideoToolbox) don't support ffmpeg's two-pass encoding, so `ff4d` encodes them in a single pass. This is faster, but the output size is less precise, so `ff4d` may need an extra retry to get under your target. +When an output lands over the target, `ff4d` encodes again at a lower bitrate. Software codecs skip the first pass on these retries and reuse its stats, since only the bitrate changed. On a 10-second 1080p60 game clip, that made a retry about 25% faster with `vp9`, 40% faster with `x264`, and twice as fast with `x265`. + - **`vp9` (VP9, CPU, `.webm`)** Produces `.webm` video. VP9 can save about 20–50% bitrate compared to H.264 (`x264`) at similar quality. It is well supported in modern browsers (especially for web streaming) but may have more limited support in older devices, legacy players, and some hardware decoders. [FFmpeg trac link](https://trac.ffmpeg.org/wiki/Encode/VP9) diff --git a/ffmpeg4discord/__main__.py b/ffmpeg4discord/__main__.py index c7fd5b9..4ab9dff 100644 --- a/ffmpeg4discord/__main__.py +++ b/ffmpeg4discord/__main__.py @@ -5,6 +5,7 @@ import math import re import shutil +import tempfile import threading import time import webbrowser @@ -49,30 +50,32 @@ def twopass_loop( """ target_filesize = settings.target_filesize - while True: - result = twopass.run(settings, on_pass=on_pass) - - # check if the filesize is within the target - if result.output_filesize < target_filesize or approx: - break - - print( - red( - f""" - The output file size ({round(result.output_filesize, 2)}MB) is still above the target of {target_filesize}MB. - Restarting... - """ + # One stats folder for every attempt. A retry only lowers the bitrate, so it reuses the first attempt's first pass. + with tempfile.TemporaryDirectory(prefix="ff4d-", ignore_cleanup_errors=True) as log_dir: + attempt = 1 + while True: + result = twopass.run(settings, on_pass=on_pass, log_dir=log_dir, reuse_first_pass=attempt > 1) + + # check if the filesize is within the target + if result.output_filesize < target_filesize or approx: + return result + + print( + red( + f""" + The output file size ({round(result.output_filesize, 2)}MB) is still above the target of {target_filesize}MB. + Restarting... + """ + ) ) - ) - Path(result.output_filename).unlink() - - # lower the target size for the next run, which lowers the bitrate. - # scale by how far off we are, but always cut by at least MIN_RETRY_STEP so an output - # that lands just over the target doesn't retry at a near-identical bitrate - adjustment_ratio = min(target_filesize / result.output_filesize, 1 - MIN_RETRY_STEP) - settings = replace(settings, target_filesize=max(settings.target_filesize * adjustment_ratio, 0.1)) + Path(result.output_filename).unlink() - return result + # lower the target size for the next run, which lowers the bitrate. + # scale by how far off we are, but always cut by at least MIN_RETRY_STEP so an output + # that lands just over the target doesn't retry at a near-identical bitrate + adjustment_ratio = min(target_filesize / result.output_filesize, 1 - MIN_RETRY_STEP) + settings = replace(settings, target_filesize=max(settings.target_filesize * adjustment_ratio, 0.1)) + attempt += 1 def open_browser(port: int) -> None: @@ -174,8 +177,9 @@ def run(self, twopass: TwoPass, settings: EncodeSettings, approx: bool) -> None: self.state = "failed" def start_pass(self, pass_name: str, progress_file: Path) -> None: - """TwoPass.run() calls this before each pass. A first or single pass means a new attempt has started.""" - if pass_name in ("first", "single"): + """TwoPass.run() calls this before each pass. Each attempt starts with a first or single pass, except a retry + that reuses the first pass's stats: it starts with a second pass right after the last attempt's.""" + if pass_name != "second" or self.pass_name == "second": self.attempt += 1 self.progress_file = progress_file self.pass_started = time.monotonic() @@ -184,7 +188,7 @@ def start_pass(self, pass_name: str, progress_file: Path) -> None: def remove_temp_folder(self) -> None: """ Deletes a running encode's temp folder. The encode thread is a daemon, so when Ctrl+C stops the server, - the TemporaryDirectory in TwoPass.run() never gets to clean up after itself. + the TemporaryDirectory in twopass_loop() never gets to clean up after itself. """ if self.state != "running" or self.progress_file is None: return diff --git a/ffmpeg4discord/twopass.py b/ffmpeg4discord/twopass.py index 7451eb0..2e771e3 100644 --- a/ffmpeg4discord/twopass.py +++ b/ffmpeg4discord/twopass.py @@ -8,6 +8,7 @@ import string import sys import tempfile +from contextlib import nullcontext from dataclasses import dataclass from datetime import datetime from pathlib import Path @@ -666,7 +667,12 @@ def _apply_audio_filters(self, ffinput, settings: EncodeSettings): ) def run( - self, settings: EncodeSettings, dry_run: bool = False, on_pass: Optional[Callable[[str, Path], None]] = None + self, + settings: EncodeSettings, + dry_run: bool = False, + on_pass: Optional[Callable[[str, Path], None]] = None, + log_dir: Optional[str] = None, + reuse_first_pass: bool = False, ) -> EncodeResult: """ Perform the CPU-intensive encoding job @@ -674,6 +680,10 @@ def run( :param dry_run: build the ffmpeg commands without running them :param on_pass: called with the pass name ("first", "second", or "single") and a file that ffmpeg writes its progress to, just before each pass starts. The web UI uses it for its progress bar. + :param log_dir: the folder for the two-pass stats and progress files. Without one, run() makes a temp folder + and deletes it when the encode ends. + :param reuse_first_pass: skip the first pass and use the stats an earlier run() left in `log_dir`. Only do this + when nothing but `settings.target_filesize` has changed, since the stats don't depend on the bitrate. :return: the output file, its size, and the ffmpeg commands """ @@ -723,8 +733,12 @@ def build_passes(pass1: dict, pass2: dict) -> tuple: if dry_run: return EncodeResult(output_filename, 0, commands) - # A new folder for each encode, so two encodes at once can't read or delete each other's stats - with tempfile.TemporaryDirectory(prefix="ff4d-", ignore_cleanup_errors=True) as log_dir: + # Without a folder from the caller, a new temp folder for each encode, so two encodes at once can't share stats + if log_dir: + folder = nullcontext(log_dir) # the caller deletes it + else: + folder = tempfile.TemporaryDirectory(prefix="ff4d-", ignore_cleanup_errors=True) + with folder as log_dir: first_pass, second_pass = build_passes( self._with_pass_log(params["pass1"], settings.codec, log_dir), self._with_pass_log(params["pass2"], settings.codec, log_dir), @@ -735,11 +749,14 @@ def with_logging(ffoutput, pass_name: str): args = ["-loglevel", loglevel, "-stats"] if on_pass: progress_file = Path(log_dir, f"{pass_name}-progress.txt") + progress_file.unlink(missing_ok=True) # a retry's progress shouldn't start with the last one's args += ["-progress", str(progress_file)] on_pass(pass_name, progress_file) return ffoutput.global_args(*args) - if first_pass is not None: + if first_pass is not None and reuse_first_pass: + print("Reusing the first pass's stats") + elif first_pass is not None: print("Performing first pass") run_pass(with_logging(first_pass, "first"), "first", capture_stdout=True) diff --git a/tests/test_main.py b/tests/test_main.py index 72525cb..66a34c3 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -87,6 +87,16 @@ def test_twopass_loop_cuts_target_by_min_step(twopass, no_cleanup): assert settings.target_filesize == 8 # the caller's settings stay the same +def test_twopass_loop_reuses_the_first_pass_on_retries(twopass, no_cleanup): + twopass.run.side_effect = [result(15), result(12), result(9)] + mainmod.twopass_loop(twopass, EncodeSettings(target_filesize=10)) + calls = twopass.run.call_args_list + assert [c.kwargs["reuse_first_pass"] for c in calls] == [False, True, True] + folders = {c.kwargs["log_dir"] for c in calls} + assert len(folders) == 1 # every attempt shares one stats folder + assert not Path(folders.pop()).exists() # and it's deleted when the loop ends + + def test_success_message(): message = mainmod.success_message(result(4.567)) assert message == f"Your compressed video file (4.57MB) is located at {Path('output.mp4').resolve()}" @@ -592,6 +602,9 @@ def fake_loop(twopass, settings, approx, on_pass): (["first", "second"], 30, 100, "Second pass: 100%"), (["single"], 15, 50, "Encoding: 50%"), (["first", "second", "first"], 3, 5, "Attempt 2, since the last one was over the target. First pass: 10%"), + # a retry that reuses the first pass's stats starts with the second pass + (["first", "second", "second"], 15, 75, "Attempt 2, since the last one was over the target. Second pass: 50%"), + (["single", "single"], 15, 50, "Attempt 2, since the last one was over the target. Encoding: 50%"), (["first"], 45, 50, "First pass: 100%"), # out_time can run a little past the clip's end ], ) diff --git a/tests/test_twopass.py b/tests/test_twopass.py index 9a81fbd..af43c1b 100644 --- a/tests/test_twopass.py +++ b/tests/test_twopass.py @@ -511,6 +511,38 @@ def failing_pass(ffoutput, pass_name, hint="", **run_kwargs): assert not folders[0].exists() +def test_run_uses_the_callers_stats_folder_and_leaves_it(make_twopass, passes, tmp_path): + make_twopass().run(make_settings(), log_dir=str(tmp_path)) + assert [p.args[p.args.index("-passlogfile") + 1] for p in passes] == [str(tmp_path / "ffmpeg2pass")] * 2 + assert tmp_path.exists() # the caller deletes it + + +@pytest.mark.parametrize("codec, flag", [("x264", "-passlogfile"), ("vp9", "-passlogfile"), ("x265", "-x265-params")]) +def test_reusing_the_first_pass_runs_only_the_second(make_twopass, passes, tmp_path, codec, flag): + tp = make_twopass() + tp.run(make_settings(codec=codec), log_dir=str(tmp_path)) + tp.run(make_settings(codec=codec, target_filesize=8), log_dir=str(tmp_path), reuse_first_pass=True) + assert [p.name for p in passes] == ["first", "second", "second"] + # the retry reads the stats the first attempt's first pass wrote + stats = [p.args[p.args.index(flag) + 1].split("stats=")[-1] for p in passes] + assert stats[0] == stats[2] + + +def test_reusing_the_first_pass_does_not_change_hardware_codecs(make_twopass, passes, tmp_path): + make_twopass().run(make_settings(codec="h264_nvenc"), log_dir=str(tmp_path), reuse_first_pass=True) + assert [p.name for p in passes] == ["single"] + + +def test_a_retry_starts_with_an_empty_progress_file(make_twopass, passes, tmp_path): + stale = tmp_path / "second-progress.txt" + stale.write_text("out_time_us=10000000\n") # left by the last attempt's second pass + seen = [] + make_twopass().run( + make_settings(), log_dir=str(tmp_path), reuse_first_pass=True, on_pass=lambda name, f: seen.append(f.exists()) + ) + assert seen == [False] + + def test_x265_keeps_its_stats_in_x265_params(make_twopass, passes): make_twopass().run(make_settings(codec="x265")) first = passes[0].args[passes[0].args.index("-x265-params") + 1]