diff --git a/piper/CHANGELOG.md b/piper/CHANGELOG.md index a44e8a729..0b30ca47c 100644 --- a/piper/CHANGELOG.md +++ b/piper/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## 2.5.3 + +- Voice models downloaded from Piper's catalog are left out of backups again. + They are moved to `/data/catalog_voices` at startup and excluded from there, + which keeps a catalog voice (20-110 MB each) out of every backup while + uploaded voices, which exist nowhere else, stay in. A model is only moved + when its md5 matches the catalog's, so a custom voice that reuses a catalog + name is still backed up + ## 2.5.2 - Upgrade to `wyoming-piper` 2.5.2 diff --git a/piper/DOCS.md b/piper/DOCS.md index 8d3a3ab7a..ae9e11363 100644 --- a/piper/DOCS.md +++ b/piper/DOCS.md @@ -135,9 +135,18 @@ interface, because that directory is mounted read-only. Delete those files directly instead. Because voices can be added and removed here, the set you end up with is your -own and is included in the app's backups — including uploads, which exist -nowhere else. Only the large re-downloadable caches are left out: the OmniVoice -model weights and the downloads for the Japanese and Thai options. +own, and an uploaded voice exists nowhere else, so uploads are included in the +app's backups. What can be fetched again is not: a voice from Piper's catalog +is moved to `/data/catalog_voices` when the app starts and left out of backups +from then on, as are the OmniVoice model weights and the downloads for the +Japanese and Thai options. A restore downloads the catalog voices it needs +again, on demand. + +The move only happens when the file is byte for byte the catalog's, so a voice +of your own that borrows a catalog name — a fine-tune exported as +`en_US-lessac-medium.onnx`, say — keeps its place in backups. A voice already +downloaded while the app is running is moved at the next restart, so one backup +can still include it. ## Custom Voices diff --git a/piper/config.yaml b/piper/config.yaml index 0da7732cd..5e228107d 100644 --- a/piper/config.yaml +++ b/piper/config.yaml @@ -1,5 +1,5 @@ --- -version: 2.5.2 +version: 2.5.3 slug: piper name: Piper description: Text-to-speech with Piper @@ -10,11 +10,11 @@ arch: init: false discovery: - wyoming -# Voice models ("*.onnx") used to be excluded, back when /data held nothing but -# re-downloadable copies of catalog voices. The web interface now uploads and -# deletes voices there, so which models are present is the user's own choice and -# has to survive a backup -- an uploaded voice exists nowhere else. Only the -# genuinely disposable caches below are skipped. +# Voice models ("*.onnx") used to be excluded wholesale, back when /data held +# nothing but re-downloadable copies of catalog voices. The web interface now +# uploads and deletes voices there, so which models are present is the user's +# own choice and has to survive a backup -- an uploaded voice exists nowhere +# else. Only what can be fetched again is skipped. backup_exclude: # OmniVoice weights, 632 MB. The 1.5 MB graph beside them is kept, but the # backend needs both files, so a restore without the weights just downloads @@ -24,6 +24,11 @@ backup_exclude: - "*/hub" # Wheel cache for the optional packs, rebuilt on demand. - "*/uv-cache" + # Voice models that came from the catalog, moved here at startup by + # sort_catalog_voices.py precisely so they can be excluded. A model is only + # put here when its md5 matches the catalog's, which makes it a copy of a + # file that is still published; uploads stay in /data and stay in backups. + - "*/catalog_voices" map: - share ingress: true diff --git a/piper/rootfs/etc/s6-overlay/s6-rc.d/piper/run b/piper/rootfs/etc/s6-overlay/s6-rc.d/piper/run index 75f910114..708044fd3 100755 --- a/piper/rootfs/etc/s6-overlay/s6-rc.d/piper/run +++ b/piper/rootfs/etc/s6-overlay/s6-rc.d/piper/run @@ -11,6 +11,13 @@ # // directory instead. readonly omnivoice_voices_dir='/data/omnivoice_voices' +# Voice models that came from the catalog are kept apart from uploaded ones, so +# config.yaml can leave the re-downloadable half out of backups. wyoming-piper +# writes both kinds to --download-dir, which stays /data: an upload that is not +# backed up yet is a file that can be lost, while a catalog voice that is still +# in /data only makes the next backup bigger until it is sorted below. +readonly catalog_voices_dir='/data/catalog_voices' + # Ingress proxies to the container's address rather than to loopback, so the web # UI has to bind every interface -- which also puts it in reach of every other # app on the same Docker network. It has no authentication of its own, so the @@ -33,6 +40,15 @@ fi mkdir -p "${omnivoice_voices_dir}" +# Before Piper starts, so a voice is never moved out from under it. Never +# fatal: an unsorted voice costs backup space and nothing else. +if ! sorted_voices="$(python3 /usr/local/bin/sort_catalog_voices.py \ + /data "${catalog_voices_dir}" 2>&1)"; then + bashio::log.warning "Could not sort catalog voices: ${sorted_voices}" +elif bashio::var.has_value "${sorted_voices}"; then + bashio::log.info "${sorted_voices}" +fi + backend="$(bashio::config 'backend')" if [[ "${backend}" == 'omnivoice' ]]; then if ! have_module 'omnivoice'; then @@ -94,6 +110,7 @@ exec python3 -m wyoming_piper \ --zeroconf \ --sentence-silence "$(bashio::config 'sentence_silence')" \ --data-dir /data \ + --data-dir "${catalog_voices_dir}" \ --data-dir /share/piper \ --download-dir /data \ --omnivoice-ref-dir "${omnivoice_voices_dir}" \ diff --git a/piper/rootfs/usr/local/bin/sort_catalog_voices.py b/piper/rootfs/usr/local/bin/sort_catalog_voices.py new file mode 100644 index 000000000..970e97364 --- /dev/null +++ b/piper/rootfs/usr/local/bin/sort_catalog_voices.py @@ -0,0 +1,96 @@ +"""Move re-downloadable voice models out of the backed-up part of /data. + +Catalog voices and uploaded ones sit side by side: wyoming-piper downloads +into --download-dir, and the web interface uploads there too. Only the uploads +are irreplaceable, while a single catalog voice is 20-110 MB of every backup +the add-on is part of, for something a restore fetches again on demand. + +So the catalog copies are moved into a subdirectory that config.yaml leaves +out of backups, and that subdirectory is handed back to wyoming-piper as +another --data-dir. Nothing is re-downloaded now; after a restore the +subdirectory is simply absent, which is the state of a fresh install. + +A file is moved only when its md5 matches what the catalog lists for that +name, so a voice that merely borrows a catalog name -- a fine-tune exported as +en_US-lessac-medium.onnx, say -- stays in /data and stays in backups. Names +are checked before hashing, so the common case of a custom voice under a +custom name reads no bytes at all. + +This runs at startup because the add-on does not control where wyoming-piper +writes: a voice downloaded while the add-on is running stays in /data until +the next start and is moved then. +""" + +import argparse +from pathlib import Path +from typing import Dict, List, Set + +from wyoming_piper.download import get_voices +from wyoming_piper.file_hash import get_file_hash + + +def catalog_digests(download_dir: Path) -> Dict[str, Set[str]]: + """Map file name to the md5 digests the catalog publishes for that name. + + A set rather than one digest: the same file is listed under a voice and + under each of its aliases, and nothing guarantees those agree. + """ + digests: Dict[str, Set[str]] = {} + + # update_voices is left off: this must not depend on the network, and the + # Piper service downloads the current voices.json moments later anyway. + for voice in get_voices(download_dir).values(): + for file_path, file_info in voice.get("files", {}).items(): + digest = file_info.get("md5_digest") + if digest: + digests.setdefault(Path(file_path).name, set()).add(digest) + + return digests + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("data_dir", help="Directory voices are downloaded into") + parser.add_argument("catalog_dir", help="Directory to move catalog voices to") + args = parser.parse_args() + + data_dir = Path(args.data_dir) + catalog_dir = Path(args.catalog_dir) + digests = catalog_digests(data_dir) + + moved: List[str] = [] + moved_bytes = 0 + + for onnx_path in sorted(data_dir.glob("*.onnx")): + # A voice is the model and its config together. Moving one without the + # other would hide the voice from both directories. + pair = [onnx_path, onnx_path.with_name(f"{onnx_path.name}.json")] + + if not all(path.is_file() and not path.is_symlink() for path in pair): + continue + + if not all(path.name in digests for path in pair): + continue + + if not all(get_file_hash(path) in digests[path.name] for path in pair): + continue + + catalog_dir.mkdir(parents=True, exist_ok=True) + for path in pair: + moved_bytes += path.stat().st_size + # Same filesystem, so this is a rename: the voice is in exactly one + # of the two directories at every point, never half-copied. + path.replace(catalog_dir / path.name) + + moved.append(onnx_path.name[: -len(".onnx")]) + + if moved: + print( + f"Moved {len(moved)} re-downloadable voice(s), " + f"{moved_bytes / (1024 * 1024):.0f} MiB, to {catalog_dir}, " + f"which is left out of backups: {', '.join(moved)}" + ) + + +if __name__ == "__main__": + main()