From 83e9969d05a9c461726dad0c6afe351e98d74e2d Mon Sep 17 00:00:00 2001 From: Stoupy51 Date: Sat, 9 May 2026 18:00:58 +0200 Subject: [PATCH 1/5] feat(output): Implement incremental save functionality for packs (fixes #512) --- src/beet/contrib/output.py | 90 +++++++++++++++++++++++++++++++++++--- 1 file changed, 84 insertions(+), 6 deletions(-) diff --git a/src/beet/contrib/output.py b/src/beet/contrib/output.py index 48657e91f..453264d6a 100644 --- a/src/beet/contrib/output.py +++ b/src/beet/contrib/output.py @@ -6,31 +6,109 @@ ] -from typing import Optional +import filecmp +import os +from pathlib import Path from beet import Context, ListOption, PluginOptions, configurable from beet.core.utils import FileSystemPath, log_time_scope +from beet.library.base import Pack, PackFile +from beet.library.utils import list_files as list_dir_files class OutputOptions(PluginOptions): - directory: Optional[ListOption[FileSystemPath]] = None + directory: ListOption[FileSystemPath] | None = None + incremental: bool = False def beet_default(ctx: Context): ctx.require(output) +def incremental_save(pack: Pack, output_path: Path) -> None: + """ Save a pack incrementally: delete removed files, write new/changed files, skip unchanged. """ + # Build expected set: posix-style relative paths -> PackFile + expected: dict[str, PackFile] = dict(pack.list_files()) + + # Build disk set: posix-style relative paths currently on disk + disk_set: set[str] = set() + if output_path.is_dir(): + for rel in list_dir_files(output_path): + disk_set.add(rel.as_posix()) + + # Delete files that are no longer present in the pack + deleted_files: set[str] = disk_set - expected.keys() + for rel_path in deleted_files: + disk_path: Path = output_path / rel_path + disk_path.unlink(missing_ok=True) + + # Remove empty directories left after deletions (bottom-up) + for dirpath, _dirnames, _filenames in os.walk(output_path, topdown=False): + dir_obj = Path(dirpath) + if dir_obj == output_path: + continue + try: + dir_obj.rmdir() # only succeeds if empty + except OSError: + pass # not empty - leave it + + # Ensure the root output directory exists + output_path.mkdir(parents=True, exist_ok=True) + + # For each expected file, compare with disk and write if new/changed + for rel_path, pack_file in expected.items(): + disk_path: Path = output_path / rel_path + + # New file, write directly without comparison + if not disk_path.exists(): + disk_path.parent.mkdir(parents=True, exist_ok=True) + pack_file.dump(output_path, rel_path) + + # Existing file: compare before writing + else: + changed: bool = True + + # Fast path: if the pack file still points directly to a source path, avoid loading content into memory + if ( + pack_file.source_path is not None + and pack_file.source_start is None + and pack_file.source_stop is None + ): + changed = not filecmp.cmp(pack_file.source_path, disk_path, shallow=False) + else: + # Standard path: use beet's own content equality + try: + existing_file: PackFile = type(pack_file)(source_path=disk_path) + changed = not pack_file.content_equal(existing_file) + except Exception: + changed = True # Fallback to overwrite on any error + + if changed: + pack_file.dump(output_path, rel_path) + + @configurable(validator=OutputOptions) def output(ctx: Context, opts: OutputOptions): - """Plugin that outputs the data pack and the resource pack in a local directory.""" + """ Plugin that outputs the data pack and the resource pack in a local directory. """ if opts.directory is None: return - paths = [ctx.directory / path for path in opts.directory.entries()] - packs = list(filter(None, ctx.packs)) + # Check both opts and ctx.meta.output for incremental flag + incremental: bool = opts.incremental + if not incremental: + meta_opts = ctx.meta.get("output") + if isinstance(meta_opts, dict): + incremental = bool(meta_opts.get("incremental", False)) + + paths: list[Path] = [ctx.directory / path for path in opts.directory.entries()] + packs: list[Pack] = list(filter(None, ctx.packs)) if paths and packs: with log_time_scope("Output files."): for pack in packs: for path in paths: - pack.save(path, overwrite=True) + if incremental and not pack.zipped and pack.name is not None: + incremental_save(pack, Path(path) / pack.name) + else: + pack.save(path, overwrite=True) + From 5febfcb95bded4b11a62c9ce1ed2127bce0b5fad Mon Sep 17 00:00:00 2001 From: Stoupy51 Date: Sat, 9 May 2026 18:32:25 +0200 Subject: [PATCH 2/5] fix(output): Update incremental option handling and ensure directory management --- src/beet/contrib/output.py | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/src/beet/contrib/output.py b/src/beet/contrib/output.py index 453264d6a..471b2975d 100644 --- a/src/beet/contrib/output.py +++ b/src/beet/contrib/output.py @@ -18,7 +18,7 @@ class OutputOptions(PluginOptions): directory: ListOption[FileSystemPath] | None = None - incremental: bool = False + incremental: bool | None = None def beet_default(ctx: Context): @@ -53,6 +53,8 @@ def incremental_save(pack: Pack, output_path: Path) -> None: pass # not empty - leave it # Ensure the root output directory exists + if output_path.exists() and not output_path.is_dir(): + output_path.unlink() output_path.mkdir(parents=True, exist_ok=True) # For each expected file, compare with disk and write if new/changed @@ -67,21 +69,20 @@ def incremental_save(pack: Pack, output_path: Path) -> None: # Existing file: compare before writing else: changed: bool = True - - # Fast path: if the pack file still points directly to a source path, avoid loading content into memory - if ( - pack_file.source_path is not None - and pack_file.source_start is None - and pack_file.source_stop is None - ): - changed = not filecmp.cmp(pack_file.source_path, disk_path, shallow=False) - else: - # Standard path: use beet's own content equality - try: + try: + # Fast path: if the pack file still points directly to a source path, avoid loading content into memory + if ( + pack_file.source_path is not None + and pack_file.source_start is None + and pack_file.source_stop is None + ): + changed = not filecmp.cmp(pack_file.source_path, disk_path, shallow=False) + else: + # Standard path: use beet's own content equality existing_file: PackFile = type(pack_file)(source_path=disk_path) changed = not pack_file.content_equal(existing_file) - except Exception: - changed = True # Fallback to overwrite on any error + except Exception: + changed = True # Fallback to overwrite on any error if changed: pack_file.dump(output_path, rel_path) @@ -94,8 +95,8 @@ def output(ctx: Context, opts: OutputOptions): return # Check both opts and ctx.meta.output for incremental flag - incremental: bool = opts.incremental - if not incremental: + incremental: bool | None = opts.incremental + if incremental is None: meta_opts = ctx.meta.get("output") if isinstance(meta_opts, dict): incremental = bool(meta_opts.get("incremental", False)) From 803f4212ebdb8caa38267ff68337a3439bf1ba1a Mon Sep 17 00:00:00 2001 From: Stoupy51 Date: Sat, 9 May 2026 21:25:37 +0200 Subject: [PATCH 3/5] fix(output): Fixed serialized issues with incremental mode --- src/beet/contrib/output.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/beet/contrib/output.py b/src/beet/contrib/output.py index 471b2975d..19e278577 100644 --- a/src/beet/contrib/output.py +++ b/src/beet/contrib/output.py @@ -78,9 +78,13 @@ def incremental_save(pack: Pack, output_path: Path) -> None: ): changed = not filecmp.cmp(pack_file.source_path, disk_path, shallow=False) else: - # Standard path: use beet's own content equality - existing_file: PackFile = type(pack_file)(source_path=disk_path) - changed = not pack_file.content_equal(existing_file) + # Standard path: compare the **exact** serialized output against disk content. + serialized: str | bytes = pack_file.ensure_serialized() + if isinstance(serialized, str): + encoding: str = getattr(pack_file, "encoding", None) or "utf-8" + changed = disk_path.read_text(encoding=encoding) != serialized + else: + changed = disk_path.read_bytes() != serialized except Exception: changed = True # Fallback to overwrite on any error From 8f4d70126182ae6d49ebe40d82cf1179b8ca19d1 Mon Sep 17 00:00:00 2001 From: Stoupy51 Date: Fri, 14 Aug 2026 14:46:09 +0200 Subject: [PATCH 4/5] perf(output): Optimized list_files() function with precomputing the relative root --- src/beet/library/utils.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/beet/library/utils.py b/src/beet/library/utils.py index 9b158c02d..de6d2fcbd 100644 --- a/src/beet/library/utils.py +++ b/src/beet/library/utils.py @@ -17,9 +17,14 @@ def list_files(directory: FileSystemPath) -> Iterator[Path]: - for root, _, files in os.walk(directory): + # Slicing the prefix off the string os.walk already built is several times faster + # than rebuilding a Path per file just to have relative_to take it apart again + base: str = os.fspath(directory) + prefix_length: int = len(base) + (0 if base.endswith(("/", os.sep)) else 1) + for root, _, files in os.walk(base): + relative_root = root[prefix_length:] for filename in files: - yield Path(root, filename).relative_to(directory) + yield Path(relative_root, filename) def list_origin(origin: FileOrigin) -> List[PurePath]: From a300747da9dbb6e8e34765ec4f3ddf79343d7903 Mon Sep 17 00:00:00 2001 From: Stoupy51 Date: Fri, 14 Aug 2026 15:13:50 +0200 Subject: [PATCH 5/5] perf(output): Skip reading output files back with a written-files manifest --- src/beet/contrib/output.py | 187 ++++++++++++++++++++++++++----------- 1 file changed, 134 insertions(+), 53 deletions(-) diff --git a/src/beet/contrib/output.py b/src/beet/contrib/output.py index 19e278577..eb951543f 100644 --- a/src/beet/contrib/output.py +++ b/src/beet/contrib/output.py @@ -7,13 +7,17 @@ import filecmp +import hashlib +import json import os from pathlib import Path from beet import Context, ListOption, PluginOptions, configurable from beet.core.utils import FileSystemPath, log_time_scope from beet.library.base import Pack, PackFile -from beet.library.utils import list_files as list_dir_files + +ManifestEntry = tuple[int, int, str] +""" Size, modification time in nanoseconds and content signature of a file this plugin wrote. """ class OutputOptions(PluginOptions): @@ -25,71 +29,146 @@ def beet_default(ctx: Context): ctx.require(output) -def incremental_save(pack: Pack, output_path: Path) -> None: - """ Save a pack incrementally: delete removed files, write new/changed files, skip unchanged. """ - # Build expected set: posix-style relative paths -> PackFile - expected: dict[str, PackFile] = dict(pack.list_files()) +def scan_directory(directory: Path) -> dict[str, tuple[int, int]]: + """ Return the size and modification time of every file below the directory, efficiently. + One `scandir` pass answers both "which files are there" and "have they changed". """ + result: dict[str, tuple[int, int]] = {} - # Build disk set: posix-style relative paths currently on disk - disk_set: set[str] = set() - if output_path.is_dir(): - for rel in list_dir_files(output_path): - disk_set.add(rel.as_posix()) + def walk(path: str, prefix: str) -> None: + with os.scandir(path) as entries: + for entry in entries: + if entry.is_dir(follow_symlinks=False): + walk(entry.path, f"{prefix}{entry.name}/") + else: + stat = entry.stat() + result[f"{prefix}{entry.name}"] = (stat.st_size, stat.st_mtime_ns) - # Delete files that are no longer present in the pack - deleted_files: set[str] = disk_set - expected.keys() - for rel_path in deleted_files: - disk_path: Path = output_path / rel_path - disk_path.unlink(missing_ok=True) + if directory.is_dir(): + walk(str(directory), "") + return result - # Remove empty directories left after deletions (bottom-up) - for dirpath, _dirnames, _filenames in os.walk(output_path, topdown=False): - dir_obj = Path(dirpath) - if dir_obj == output_path: - continue - try: - dir_obj.rmdir() # only succeeds if empty - except OSError: - pass # not empty - leave it + +def prune_empty_directories(directory: Path) -> None: + """ Remove every directory left empty below the given root, deepest first. """ + for dirpath, _dirnames, _filenames in os.walk(directory, topdown=False): + if dirpath != str(directory): + try: + os.rmdir(dirpath) + except OSError: + pass # Not empty, leave it alone + + +def content_signature(pack_file: PackFile) -> str: + """ Return a signature that changes whenever the bytes the pack file would write change. + + A file still backed by an untouched source is signed from that source's stat, so unchanged assets + never have to be loaded into memory at all. + + Args: + pack_file (PackFile): The file to sign. + Returns: + str: Either `source::` or `sha1:`. + """ + if ( + pack_file.source_path is not None + and pack_file.source_start is None + and pack_file.source_stop is None + ): + stat = os.stat(pack_file.source_path) + return f"source:{stat.st_size}:{stat.st_mtime_ns}" + + serialized: str | bytes = pack_file.ensure_serialized() + if isinstance(serialized, str): + serialized = serialized.encode(getattr(pack_file, "encoding", None) or "utf-8") + return f"sha1:{hashlib.sha1(serialized).hexdigest()}" + + +def file_differs(pack_file: PackFile, disk_path: Path) -> bool: + """ Return whether the file on disk differs from what the pack file would write, by reading both. """ + try: + if ( + pack_file.source_path is not None + and pack_file.source_start is None + and pack_file.source_stop is None + ): + return not filecmp.cmp(pack_file.source_path, disk_path, shallow=False) + + serialized: str | bytes = pack_file.ensure_serialized() + if isinstance(serialized, str): + encoding: str = getattr(pack_file, "encoding", None) or "utf-8" + return disk_path.read_text(encoding=encoding) != serialized + return disk_path.read_bytes() != serialized + except Exception: + return True + + +def load_manifest(manifest_path: Path | None) -> dict[str, ManifestEntry]: + """ Read the record of what the previous build wrote, treating any problem as an empty record. """ + if manifest_path is None: + return {} + try: + raw = json.loads(manifest_path.read_text("utf-8")) + return { + key: (int(value[0]), int(value[1]), str(value[2])) + for key, value in raw.items() + if len(value) == 3 + } + except (OSError, ValueError, TypeError, KeyError, IndexError): + return {} + + +def incremental_save( + pack: Pack, output_path: Path, manifest_path: Path | None = None +) -> None: + """ Save a pack incrementally: delete removed files, write new/changed files, skip unchanged. + + Args: + pack (Pack): The pack to write out. + output_path (Path): The directory the pack is written to. + manifest_path (Path | None): Where to keep the record, or None to compare against the disk every time. + """ + expected: dict[str, PackFile] = dict(pack.list_files()) + manifest: dict[str, ManifestEntry] = load_manifest(manifest_path) + on_disk: dict[str, tuple[int, int]] = scan_directory(output_path) + + # Delete files that are no longer part of the pack, and the directories that leaves empty + deleted_files: set[str] = on_disk.keys() - expected.keys() + for rel_path in deleted_files: + (output_path / rel_path).unlink(missing_ok=True) + if deleted_files: + prune_empty_directories(output_path) # Ensure the root output directory exists if output_path.exists() and not output_path.is_dir(): output_path.unlink() output_path.mkdir(parents=True, exist_ok=True) - # For each expected file, compare with disk and write if new/changed + written: dict[str, ManifestEntry] = {} for rel_path, pack_file in expected.items(): + signature: str = content_signature(pack_file) + recorded: ManifestEntry | None = manifest.get(rel_path) + current: tuple[int, int] | None = on_disk.get(rel_path) + + # Untouched since we wrote it and still holding the same content, nothing to do + if recorded is not None and current == recorded[:2] and recorded[2] == signature: + written[rel_path] = recorded + continue + disk_path: Path = output_path / rel_path - # New file, write directly without comparison - if not disk_path.exists(): - disk_path.parent.mkdir(parents=True, exist_ok=True) - pack_file.dump(output_path, rel_path) + # No record to trust but the file is there, so fall back to comparing the bytes + if recorded is None and current is not None and not file_differs(pack_file, disk_path): + written[rel_path] = (*current, signature) + continue - # Existing file: compare before writing - else: - changed: bool = True - try: - # Fast path: if the pack file still points directly to a source path, avoid loading content into memory - if ( - pack_file.source_path is not None - and pack_file.source_start is None - and pack_file.source_stop is None - ): - changed = not filecmp.cmp(pack_file.source_path, disk_path, shallow=False) - else: - # Standard path: compare the **exact** serialized output against disk content. - serialized: str | bytes = pack_file.ensure_serialized() - if isinstance(serialized, str): - encoding: str = getattr(pack_file, "encoding", None) or "utf-8" - changed = disk_path.read_text(encoding=encoding) != serialized - else: - changed = disk_path.read_bytes() != serialized - except Exception: - changed = True # Fallback to overwrite on any error + disk_path.parent.mkdir(parents=True, exist_ok=True) + pack_file.dump(output_path, rel_path) + stat = disk_path.stat() + written[rel_path] = (stat.st_size, stat.st_mtime_ns, signature) - if changed: - pack_file.dump(output_path, rel_path) + if manifest_path is not None: + manifest_path.parent.mkdir(parents=True, exist_ok=True) + manifest_path.write_text(json.dumps(written), "utf-8") @configurable(validator=OutputOptions) @@ -110,10 +189,12 @@ def output(ctx: Context, opts: OutputOptions): if paths and packs: with log_time_scope("Output files."): + cache = ctx.cache["output_incremental"] for pack in packs: for path in paths: if incremental and not pack.zipped and pack.name is not None: - incremental_save(pack, Path(path) / pack.name) + target: Path = Path(path) / pack.name + incremental_save(pack, target, cache.get_path(f"manifest:{target}")) else: pack.save(path, overwrite=True)