From e362b92532afe3ea03912199f9f0fe32d6bc8d4f Mon Sep 17 00:00:00 2001 From: tronikos Date: Sun, 27 Sep 2026 18:06:42 -0700 Subject: [PATCH] piper: Exclude re-downloadable catalog voices from backups 2.5.2 stopped excluding "*.onnx" from backups, because the new web interface can upload voices and an uploaded voice exists nowhere else. That is right for uploads, but it also put catalog voices back into every backup: a single model is 20-110 MB, and the add-on downloads it again from HuggingFace whenever it is missing. An install with two voices adds over 130 MB to each backup for files that a restore would fetch anyway. wyoming-piper writes downloads and uploads to the same --download-dir, so the two cannot be separated by configuration. Instead, sort them at startup: a voice whose md5 matches what the catalog publishes for that name is moved to /data/catalog_voices, which config.yaml excludes, and that directory is passed as another --data-dir so the voice keeps working and is not downloaded again. After a restore it is simply absent, which is the state of a fresh install, and it is downloaded on demand. Matching on the hash rather than the name keeps a custom voice that reuses a catalog name - a fine-tune exported as en_US-lessac-medium.onnx, say - in /data and in backups. Names are checked before hashing, so a custom voice under a custom name is never read. --download-dir stays /data, so a voice downloaded while the add-on runs is moved only at the next start. That way the failure mode is one larger backup rather than an upload that is missing from one. --- piper/CHANGELOG.md | 9 ++ piper/DOCS.md | 15 ++- piper/config.yaml | 17 ++-- piper/rootfs/etc/s6-overlay/s6-rc.d/piper/run | 17 ++++ .../usr/local/bin/sort_catalog_voices.py | 96 +++++++++++++++++++ 5 files changed, 145 insertions(+), 9 deletions(-) create mode 100644 piper/rootfs/usr/local/bin/sort_catalog_voices.py diff --git a/piper/CHANGELOG.md b/piper/CHANGELOG.md index a44e8a729..0b30ca47c 100644 --- a/piper/CHANGELOG.md +++ b/piper/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## 2.5.3 + +- Voice models downloaded from Piper's catalog are left out of backups again. + They are moved to `/data/catalog_voices` at startup and excluded from there, + which keeps a catalog voice (20-110 MB each) out of every backup while + uploaded voices, which exist nowhere else, stay in. A model is only moved + when its md5 matches the catalog's, so a custom voice that reuses a catalog + name is still backed up + ## 2.5.2 - Upgrade to `wyoming-piper` 2.5.2 diff --git a/piper/DOCS.md b/piper/DOCS.md index 8d3a3ab7a..ae9e11363 100644 --- a/piper/DOCS.md +++ b/piper/DOCS.md @@ -135,9 +135,18 @@ interface, because that directory is mounted read-only. Delete those files directly instead. Because voices can be added and removed here, the set you end up with is your -own and is included in the app's backups — including uploads, which exist -nowhere else. Only the large re-downloadable caches are left out: the OmniVoice -model weights and the downloads for the Japanese and Thai options. +own, and an uploaded voice exists nowhere else, so uploads are included in the +app's backups. What can be fetched again is not: a voice from Piper's catalog +is moved to `/data/catalog_voices` when the app starts and left out of backups +from then on, as are the OmniVoice model weights and the downloads for the +Japanese and Thai options. A restore downloads the catalog voices it needs +again, on demand. + +The move only happens when the file is byte for byte the catalog's, so a voice +of your own that borrows a catalog name — a fine-tune exported as +`en_US-lessac-medium.onnx`, say — keeps its place in backups. A voice already +downloaded while the app is running is moved at the next restart, so one backup +can still include it. ## Custom Voices diff --git a/piper/config.yaml b/piper/config.yaml index 0da7732cd..5e228107d 100644 --- a/piper/config.yaml +++ b/piper/config.yaml @@ -1,5 +1,5 @@ --- -version: 2.5.2 +version: 2.5.3 slug: piper name: Piper description: Text-to-speech with Piper @@ -10,11 +10,11 @@ arch: init: false discovery: - wyoming -# Voice models ("*.onnx") used to be excluded, back when /data held nothing but -# re-downloadable copies of catalog voices. The web interface now uploads and -# deletes voices there, so which models are present is the user's own choice and -# has to survive a backup -- an uploaded voice exists nowhere else. Only the -# genuinely disposable caches below are skipped. +# Voice models ("*.onnx") used to be excluded wholesale, back when /data held +# nothing but re-downloadable copies of catalog voices. The web interface now +# uploads and deletes voices there, so which models are present is the user's +# own choice and has to survive a backup -- an uploaded voice exists nowhere +# else. Only what can be fetched again is skipped. backup_exclude: # OmniVoice weights, 632 MB. The 1.5 MB graph beside them is kept, but the # backend needs both files, so a restore without the weights just downloads @@ -24,6 +24,11 @@ backup_exclude: - "*/hub" # Wheel cache for the optional packs, rebuilt on demand. - "*/uv-cache" + # Voice models that came from the catalog, moved here at startup by + # sort_catalog_voices.py precisely so they can be excluded. A model is only + # put here when its md5 matches the catalog's, which makes it a copy of a + # file that is still published; uploads stay in /data and stay in backups. + - "*/catalog_voices" map: - share ingress: true diff --git a/piper/rootfs/etc/s6-overlay/s6-rc.d/piper/run b/piper/rootfs/etc/s6-overlay/s6-rc.d/piper/run index 75f910114..708044fd3 100755 --- a/piper/rootfs/etc/s6-overlay/s6-rc.d/piper/run +++ b/piper/rootfs/etc/s6-overlay/s6-rc.d/piper/run @@ -11,6 +11,13 @@ # // directory instead. readonly omnivoice_voices_dir='/data/omnivoice_voices' +# Voice models that came from the catalog are kept apart from uploaded ones, so +# config.yaml can leave the re-downloadable half out of backups. wyoming-piper +# writes both kinds to --download-dir, which stays /data: an upload that is not +# backed up yet is a file that can be lost, while a catalog voice that is still +# in /data only makes the next backup bigger until it is sorted below. +readonly catalog_voices_dir='/data/catalog_voices' + # Ingress proxies to the container's address rather than to loopback, so the web # UI has to bind every interface -- which also puts it in reach of every other # app on the same Docker network. It has no authentication of its own, so the @@ -33,6 +40,15 @@ fi mkdir -p "${omnivoice_voices_dir}" +# Before Piper starts, so a voice is never moved out from under it. Never +# fatal: an unsorted voice costs backup space and nothing else. +if ! sorted_voices="$(python3 /usr/local/bin/sort_catalog_voices.py \ + /data "${catalog_voices_dir}" 2>&1)"; then + bashio::log.warning "Could not sort catalog voices: ${sorted_voices}" +elif bashio::var.has_value "${sorted_voices}"; then + bashio::log.info "${sorted_voices}" +fi + backend="$(bashio::config 'backend')" if [[ "${backend}" == 'omnivoice' ]]; then if ! have_module 'omnivoice'; then @@ -94,6 +110,7 @@ exec python3 -m wyoming_piper \ --zeroconf \ --sentence-silence "$(bashio::config 'sentence_silence')" \ --data-dir /data \ + --data-dir "${catalog_voices_dir}" \ --data-dir /share/piper \ --download-dir /data \ --omnivoice-ref-dir "${omnivoice_voices_dir}" \ diff --git a/piper/rootfs/usr/local/bin/sort_catalog_voices.py b/piper/rootfs/usr/local/bin/sort_catalog_voices.py new file mode 100644 index 000000000..970e97364 --- /dev/null +++ b/piper/rootfs/usr/local/bin/sort_catalog_voices.py @@ -0,0 +1,96 @@ +"""Move re-downloadable voice models out of the backed-up part of /data. + +Catalog voices and uploaded ones sit side by side: wyoming-piper downloads +into --download-dir, and the web interface uploads there too. Only the uploads +are irreplaceable, while a single catalog voice is 20-110 MB of every backup +the add-on is part of, for something a restore fetches again on demand. + +So the catalog copies are moved into a subdirectory that config.yaml leaves +out of backups, and that subdirectory is handed back to wyoming-piper as +another --data-dir. Nothing is re-downloaded now; after a restore the +subdirectory is simply absent, which is the state of a fresh install. + +A file is moved only when its md5 matches what the catalog lists for that +name, so a voice that merely borrows a catalog name -- a fine-tune exported as +en_US-lessac-medium.onnx, say -- stays in /data and stays in backups. Names +are checked before hashing, so the common case of a custom voice under a +custom name reads no bytes at all. + +This runs at startup because the add-on does not control where wyoming-piper +writes: a voice downloaded while the add-on is running stays in /data until +the next start and is moved then. +""" + +import argparse +from pathlib import Path +from typing import Dict, List, Set + +from wyoming_piper.download import get_voices +from wyoming_piper.file_hash import get_file_hash + + +def catalog_digests(download_dir: Path) -> Dict[str, Set[str]]: + """Map file name to the md5 digests the catalog publishes for that name. + + A set rather than one digest: the same file is listed under a voice and + under each of its aliases, and nothing guarantees those agree. + """ + digests: Dict[str, Set[str]] = {} + + # update_voices is left off: this must not depend on the network, and the + # Piper service downloads the current voices.json moments later anyway. + for voice in get_voices(download_dir).values(): + for file_path, file_info in voice.get("files", {}).items(): + digest = file_info.get("md5_digest") + if digest: + digests.setdefault(Path(file_path).name, set()).add(digest) + + return digests + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("data_dir", help="Directory voices are downloaded into") + parser.add_argument("catalog_dir", help="Directory to move catalog voices to") + args = parser.parse_args() + + data_dir = Path(args.data_dir) + catalog_dir = Path(args.catalog_dir) + digests = catalog_digests(data_dir) + + moved: List[str] = [] + moved_bytes = 0 + + for onnx_path in sorted(data_dir.glob("*.onnx")): + # A voice is the model and its config together. Moving one without the + # other would hide the voice from both directories. + pair = [onnx_path, onnx_path.with_name(f"{onnx_path.name}.json")] + + if not all(path.is_file() and not path.is_symlink() for path in pair): + continue + + if not all(path.name in digests for path in pair): + continue + + if not all(get_file_hash(path) in digests[path.name] for path in pair): + continue + + catalog_dir.mkdir(parents=True, exist_ok=True) + for path in pair: + moved_bytes += path.stat().st_size + # Same filesystem, so this is a rename: the voice is in exactly one + # of the two directories at every point, never half-copied. + path.replace(catalog_dir / path.name) + + moved.append(onnx_path.name[: -len(".onnx")]) + + if moved: + print( + f"Moved {len(moved)} re-downloadable voice(s), " + f"{moved_bytes / (1024 * 1024):.0f} MiB, to {catalog_dir}, " + f"which is left out of backups: {', '.join(moved)}" + ) + + +if __name__ == "__main__": + main()