Exclude re-downloadable catalog voices from backups

piper/DOCS.md

@@ -135,9 +135,18 @@ interface, because that directory is mounted read-only. Delete those files
 directly instead.
 
 Because voices can be added and removed here, the set you end up with is your
-own and is included in the app's backups — including uploads, which exist
-nowhere else. Only the large re-downloadable caches are left out: the OmniVoice
-model weights and the downloads for the Japanese and Thai options.
+own, and an uploaded voice exists nowhere else, so uploads are included in the
+app's backups. What can be fetched again is not: voices downloaded from Piper's
+catalog are stored in `/data/catalog_voices` (and any catalog voices already in
+`/data` from earlier versions are moved there when the app starts) and left out
+of backups, as are the OmniVoice model weights and the downloads for the
+Japanese and Thai options. A restore downloads the catalog voices it needs
+again, on demand.
+
+A voice in `/data` is only moved to `/data/catalog_voices` when the file is byte
+for byte the catalog's, so a voice of your own that borrows a catalog name — a
+fine-tune exported as `en_US-lessac-medium.onnx`, say — keeps its place in
+`/data` and in backups.
 
 ## Custom Voices
 

piper/config.yaml

@@ -10,11 +10,11 @@ arch:
 init: false
 discovery:
   - wyoming
-# Voice models ("*.onnx") used to be excluded, back when /data held nothing but
-# re-downloadable copies of catalog voices. The web interface now uploads and
-# deletes voices there, so which models are present is the user's own choice and
-# has to survive a backup -- an uploaded voice exists nowhere else. Only the
-# genuinely disposable caches below are skipped.
+# Voice models ("*.onnx") used to be excluded wholesale, back when /data held
+# nothing but re-downloadable copies of catalog voices. The web interface now
+# uploads and deletes voices there, so which models are present is the user's
+# own choice and has to survive a backup -- an uploaded voice exists nowhere
+# else. Only what can be fetched again is skipped.
 backup_exclude:
   # OmniVoice weights, 632 MB. The 1.5 MB graph beside them is kept, but the
   # backend needs both files, so a restore without the weights just downloads
@@ -24,6 +24,10 @@ backup_exclude:
   - "*/hub"
   # Wheel cache for the optional packs, rebuilt on demand.
   - "*/uv-cache"
+  # Voice models from Piper's catalog, downloaded into /data/catalog_voices
+  # (and moved there from /data at startup when their md5 matches the
+  # catalog's); uploads stay in /data and stay in backups.
+  - "*/catalog_voices"
 map:
   - share
 ingress: true

piper/rootfs/etc/s6-overlay/s6-rc.d/piper/run

@@ -11,6 +11,12 @@
 # <language>/<name>/ directory instead.
 readonly omnivoice_voices_dir='/data/omnivoice_voices'
 
+# Voice models from Piper's catalog are kept apart from uploaded ones, so
+# config.yaml can leave the re-downloadable half out of backups. Uploads still
+# go to --download-dir (/data), while catalog downloads are directed here and
+# any catalog copies already in /data are moved here at startup.
+readonly catalog_voices_dir='/data/catalog_voices'
+
 # Ingress proxies to the container's address rather than to loopback, so the web
 # UI has to bind every interface -- which also puts it in reach of every other
 # app on the same Docker network. It has no authentication of its own, so the
@@ -33,6 +39,17 @@ fi
 
 mkdir -p "${omnivoice_voices_dir}"
 
+# Move any catalog voices already in /data (from <= 2.5.2, or uploaded
+# unmodified through the web UI) before Piper starts, so a voice is never moved
+# out from under it. Never fatal: an unsorted voice costs backup space and
+# nothing else.
+if ! sorted_voices="$(python3 /usr/local/bin/sort_catalog_voices.py \
+    /data "${catalog_voices_dir}" 2>&1)"; then
+    bashio::log.warning "Could not sort catalog voices: ${sorted_voices}"
+elif bashio::var.has_value "${sorted_voices}"; then
+    bashio::log.info "${sorted_voices}"
+fi
+
 backend="$(bashio::config 'backend')"
 if [[ "${backend}" == 'omnivoice' ]]; then
     if ! have_module 'omnivoice'; then
@@ -89,11 +106,13 @@ done
 # cloning voices even while Piper is the active backend.
 #
 # shellcheck disable=SC2068
-exec python3 -m wyoming_piper \
+exec python3 /usr/local/bin/sort_catalog_voices.py \
+    --run "${catalog_voices_dir}" \
     --uri 'tcp://0.0.0.0:10200' \
     --zeroconf \
     --sentence-silence "$(bashio::config 'sentence_silence')" \
     --data-dir /data \
+    --data-dir "${catalog_voices_dir}" \
     --data-dir /share/piper \
     --download-dir /data \
     --omnivoice-ref-dir "${omnivoice_voices_dir}" \

piper/rootfs/usr/local/bin/sort_catalog_voices.py

@@ -0,0 +1,153 @@
+"""Keep re-downloadable catalog voice models out of the backed-up part of /data.
+
+Catalog voices and uploaded ones would otherwise sit side by side: by default
+wyoming-piper downloads catalog models into --download-dir, and the web
+interface uploads custom voices there too. Only the uploads are irreplaceable,
+while a single catalog voice is 20-110 MB of every backup the app is part of,
+for something a restore fetches again on demand.
+
+Catalog models live in /data/catalog_voices instead, which config.yaml excludes
+from backups and which is handed back to wyoming-piper as another --data-dir:
+
+1. At startup, sort_catalog_voices() moves any catalog voices already in /data
+   (from versions before this split, or uploaded unmodified through the web UI)
+   into /data/catalog_voices. A file is moved only when its md5 matches what the
+   catalog lists for that name, so a custom voice that reuses a catalog name --
+   a fine-tune exported as en_US-lessac-medium.onnx, say -- stays in /data and
+   stays in backups. Names are checked before hashing, so a custom voice under a
+   custom name reads no bytes at all.
+
+2. At runtime (--run), wyoming_piper.download.ensure_voice_exists is directed to
+   download missing catalog voices straight into /data/catalog_voices while
+   --download-dir remains /data. That keeps web UI uploads (and voices.json and
+   HF_HOME) in /data so uploads are always backed up, while voices downloaded at
+   startup or on demand while the app runs never land in /data in the first
+   place.
+"""
+
+import argparse
+import sys
+from pathlib import Path
+from typing import Any, Dict, Iterable, List, Set, Union
+
+import wyoming_piper.download as piper_download
+from wyoming_piper.download import get_voices
+from wyoming_piper.file_hash import get_file_hash
+
+_ensure_voice_exists = piper_download.ensure_voice_exists
+
+
+def catalog_digests(download_dir: Path) -> Dict[str, Set[str]]:
+    """Map file name to the md5 digests the catalog publishes for that name.
+
+    A set rather than one digest: the same file is listed under a voice and
+    under each of its aliases, and nothing guarantees those agree.
+    """
+    digests: Dict[str, Set[str]] = {}
+
+    # update_voices is left off: this must not depend on the network, and the
+    # Piper service downloads the current voices.json moments later anyway.
+    for voice in get_voices(download_dir).values():
+        for file_path, file_info in voice.get("files", {}).items():
+            digest = file_info.get("md5_digest")
+            if digest:
+                digests.setdefault(Path(file_path).name, set()).add(digest)
+
+    return digests
+
+
+def sort_catalog_voices(data_dir: Path, catalog_dir: Path) -> None:
+    """Move catalog voice pairs whose md5 matches the catalog to catalog_dir."""
+    digests = catalog_digests(data_dir)
+
+    moved: List[str] = []
+    moved_bytes = 0
+
+    for onnx_path in sorted(data_dir.glob("*.onnx")):
+        # A voice is the model and its config together. Moving one without the
+        # other would hide the voice from both directories.
+        pair = [onnx_path, onnx_path.with_name(f"{onnx_path.name}.json")]
+
+        if not all(path.is_file() and not path.is_symlink() for path in pair):
+            continue
+
+        if not all(path.name in digests for path in pair):
+            continue
+
+        if not all(get_file_hash(path) in digests[path.name] for path in pair):
+            continue
+
+        catalog_dir.mkdir(parents=True, exist_ok=True)
+        for path in pair:
+            moved_bytes += path.stat().st_size
+            # Same filesystem, so this is a rename: the voice is in exactly one
+            # of the two directories at every point, never half-copied.
+            path.replace(catalog_dir / path.name)
+
+        moved.append(onnx_path.name[: -len(".onnx")])
+
+    if moved:
+        print(
+            f"Moved {len(moved)} re-downloadable voice(s), "
+            f"{moved_bytes / (1024 * 1024):.0f} MiB, to {catalog_dir}, "
+            f"which is left out of backups: {', '.join(moved)}"
+        )
+
+
+def _has_voice_pair(directory: Path, name: str) -> bool:
+    """Return True if directory has a non-empty .onnx and .onnx.json for name."""
+    for suffix in (".onnx", ".onnx.json"):
+        path = directory / f"{name}{suffix}"
+        if not path.is_file() or path.stat().st_size == 0:
+            return False
+    return True
+
+
+def run_piper(catalog_dir: Path, piper_args: List[str]) -> None:
+    """Run wyoming-piper with catalog downloads directed to catalog_dir."""
+
+    def ensure_voice_exists(
+        name: str,
+        data_dirs: Iterable[Union[str, Path]],
+        _download_dir: Union[str, Path],
+        voices_info: Dict[str, Any],
+    ) -> None:
+        """Download missing catalog voices into catalog_dir."""
+        if "files" in voices_info.get(name, {}):
+            # find_voice() requires both files in the same data_dir. Only check
+            # directories that already have a complete pair so a stray single
+            # file in /data cannot split a download across two directories.
+            complete_dirs = [
+                d for d in data_dirs if _has_voice_pair(Path(d), name)
+            ]
+            data_dirs = complete_dirs or [catalog_dir]
+
+        _ensure_voice_exists(name, data_dirs, catalog_dir, voices_info)
+
+    import wyoming_piper.__main__ as piper_main
+    import wyoming_piper.handler as piper_handler
+
+    piper_download.ensure_voice_exists = ensure_voice_exists
+    piper_handler.ensure_voice_exists = ensure_voice_exists
+    piper_main.ensure_voice_exists = ensure_voice_exists
+
+    sys.argv = [sys.argv[0], *piper_args]
+    piper_main.run()
+
+
+def main() -> None:
+    """Entry point for sorting catalog voices or launching wyoming-piper."""
+    if len(sys.argv) >= 3 and sys.argv[1] == "--run":
+        run_piper(Path(sys.argv[2]), sys.argv[3:])
+        return
+
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("data_dir", help="Directory voices are uploaded into")
+    parser.add_argument("catalog_dir", help="Directory to move catalog voices to")
+    args = parser.parse_args()
+
+    sort_catalog_voices(Path(args.data_dir), Path(args.catalog_dir))
+
+
+if __name__ == "__main__":
+    main()