diff --git a/.env.example b/.env.example index 3c853e0..9808118 100644 --- a/.env.example +++ b/.env.example @@ -37,3 +37,18 @@ CORS_ORIGINS=http://localhost:8400 # SUBDIRECTORY of the bind mount, not the mount root, so the share root stays # clean and Plex path translation keeps working. MUSIC_LIBRARY_PATH=/music/Database + +# --- Auto-update -------------------------------------------------------------- +# The waxflow-updater service applies updates by PULLING prebuilt images (fast, +# no compiler on your box) and rolling back if the API is unhealthy afterwards. +# It mounts the Docker socket, so it deliberately runs with NO network: the +# worker decides what to update to, and the Docker daemon fetches the layers. +# +# Turn the whole thing off by removing the waxflow-updater service, or set the +# `auto_update_enabled` setting to 0 in the web UI. +# WAXFLOW_UPDATE_POLL_SECONDS=60 +# WAXFLOW_HEALTH_TIMEOUT=180 +# +# Run your own registry/fork? Point these at it: +# WAXFLOW_REGISTRY=ghcr.io/youruser +# WAXFLOW_IMAGE_TAG=2.12.0 diff --git a/.github/workflows/release-images.yml b/.github/workflows/release-images.yml new file mode 100644 index 0000000..ec56c54 --- /dev/null +++ b/.github/workflows/release-images.yml @@ -0,0 +1,88 @@ +name: Publish release images + +# Auto-update pulls prebuilt images instead of rebuilding from source. That is +# what makes an unattended 3am update safe: a from-source rebuild of the worker +# took ~25 minutes on a Synology NAS and wedged the Docker daemon once. Pulling a +# published layer takes seconds and needs no compiler on the user's box. +# +# Publishes: +# ghcr.io//waxflow-api: + :latest +# ghcr.io//waxflow-worker: + :latest +# ghcr.io//waxflow-web: + :latest + +on: + release: + types: [published] + workflow_dispatch: + inputs: + version: + description: "Version to build and push (e.g. 2.11.0)" + required: true + +permissions: + contents: read + packages: write + +jobs: + publish: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - service: api + context: . + dockerfile: sync-api/Dockerfile + - service: worker + context: ./sync-worker + dockerfile: ./sync-worker/Dockerfile + - service: web + context: ./sync-web + dockerfile: ./sync-web/Dockerfile + + steps: + - uses: actions/checkout@v4 + + - name: Resolve version + id: v + run: | + if [ -n "${{ github.event.inputs.version }}" ]; then + V="${{ github.event.inputs.version }}" + else + V="${GITHUB_REF_NAME#v}" + fi + # VERSION in the repo is the source of truth; refuse a mismatched tag + # rather than publishing an image whose /app/VERSION disagrees with it. + FILE_V="$(cat VERSION | tr -d '[:space:]')" + if [ "$V" != "$FILE_V" ]; then + echo "::error::tag/input version '$V' != VERSION file '$FILE_V'" + exit 1 + fi + echo "version=$V" >> "$GITHUB_OUTPUT" + echo "owner=${GITHUB_REPOSITORY_OWNER,,}" >> "$GITHUB_OUTPUT" + + - uses: docker/setup-qemu-action@v3 + - uses: docker/setup-buildx-action@v3 + + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: ${{ matrix.context }} + file: ${{ matrix.dockerfile }} + # linux/amd64 covers Synology/Intel NAS boxes; arm64 covers Apple + # silicon and Raspberry Pi hosts. + platforms: linux/amd64,linux/arm64 + push: true + build-args: | + GIT_SHA=${{ github.sha }} + tags: | + ghcr.io/${{ steps.v.outputs.owner }}/waxflow-${{ matrix.service }}:${{ steps.v.outputs.version }} + ghcr.io/${{ steps.v.outputs.owner }}/waxflow-${{ matrix.service }}:latest + cache-from: type=gha,scope=${{ matrix.service }} + cache-to: type=gha,mode=max,scope=${{ matrix.service }} diff --git a/CHANGELOG.md b/CHANGELOG.md index b13ec90..c520866 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,59 @@ # Changelog +## 2.12.0 — Auto-update actually works + +`auto_update_enabled` has existed for a while. It could never have worked. Three +independent reasons, each sufficient on its own: + +1. **The version check used a string compare.** `"2.11.0" > "2.9.0"` is `False` + (at index 2, `"1" < "9"`), so every x.9 -> x.10+ upgrade was invisible. The + reverse was `True`, meaning the check could have offered a **downgrade** as an + update. Present in both `routes/admin.py` and `tasks/auto_update.py`. +2. **Nothing applied the update.** `scripts/auto-update.sh` had to be installed in + the host's crontab by hand; nothing shipped it, so the signal file the worker + wrote was never read by anything. +3. **Even when triggered, it did not update.** The script ran + `docker compose up -d --build` against the source already on disk — a rebuild + of the same version. + +### Added — `waxflow-updater` +A container cannot restart itself, so applying an update needs host-side Docker +access. This is the only place WaxFlow asks for it, and it is deliberately +constrained: + +- **`network_mode: none`.** It never downloads anything. The worker (network, no + socket) decides the target version; the Docker *daemon* fetches image layers + when asked over the socket. Nothing with host-root access talks to the internet. +- **Rollback.** After applying, it health-checks the API and restores the previous + tag if the new version does not come up. This runs unattended at 3am by default; + an update that half-applies and is never noticed is worse than one that never ran. +- **Input is treated as untrusted.** The target tag comes from the GitHub API and + is refused unless it matches semver, so nothing unexpected reaches `docker pull`. + +Delete the service from `docker-compose.yml` if you would rather not grant socket +access; everything else keeps working. + +### Added — published images +`.github/workflows/release-images.yml` builds and pushes +`ghcr.io//waxflow-{api,worker,web}` (linux/amd64 + linux/arm64) on every +published release, and refuses to publish if the tag disagrees with `VERSION`. + +Updating is now a pull, not a rebuild. The rebuild path took ~25 minutes for the +worker image on a Synology NAS and wedged the Docker daemon once — unacceptable +for an unattended 3am job. `build:` blocks remain, so `docker compose up -d +--build` still works offline and for forks (`WAXFLOW_REGISTRY`). + +### Changed +- `auto_update_enabled` defaults to `1` for **new** installs. `INSERT OR IGNORE` + means existing deployments keep whatever they already had. +- Compose images are `${WAXFLOW_REGISTRY:-ghcr.io/rancur}/waxflow-*:${WAXFLOW_IMAGE_TAG:-${VERSION:-latest}}`. + +### Tests +`tests/test_auto_update_version.py` — 7 cases pinning the comparison, including +the exact regression (2.9 -> 2.10/2.11 must be newer) and that a downgrade is +never offered. + + ## 2.11.0 — Path contract, one-way replication, and a pile of real bugs Everything here was found by *running* the system during a live incident, not by diff --git a/README.md b/README.md index 4b5f929..1951050 100644 --- a/README.md +++ b/README.md @@ -456,6 +456,50 @@ downtime here: `NAME._smb._tcp.local` resolves only via service discovery; when that advertisement goes stale, mounts **hang forever** rather than failing. +--- + +## Updates + +WaxFlow checks GitHub for new releases and can apply them itself. The check runs +in the worker; the apply runs in a small `waxflow-updater` container. + +**It pulls prebuilt images** from `ghcr.io/rancur/waxflow-{api,worker,web}` rather +than rebuilding from source. That matters: a from-source rebuild of the worker +image took ~25 minutes on a Synology NAS and wedged the Docker daemon once — not +something you want happening unattended at 3am. + +Before applying, it takes a database backup. Afterwards it health-checks the API +and **rolls back to the previous tag** if the new version does not come up. + +| Setting | Default | | +|---|---|---| +| `auto_update_enabled` | `1` for new installs | existing installs keep their current value | +| `auto_update_schedule` | `daily_3am` | or `weekly_sunday_3am`, `manual` | +| `auto_backup_before_update` | `1` | DB + config snapshot, last 10 kept | + +### Security: the Docker socket + +Applying an update means restarting containers, which a container cannot do to +itself — so `waxflow-updater` mounts `/var/run/docker.sock`. That is +root-equivalent access to the host, so the service is deliberately split: + +- the **worker** has network but no socket — it decides *what* to update to +- the **updater** has the socket but **`network_mode: none`** — it never + downloads anything; the Docker *daemon* fetches image layers when asked over + the socket + +Nothing with host-root access talks to the internet. If you would rather not +grant socket access at all, delete the `waxflow-updater` service from +`docker-compose.yml` — everything else keeps working, and the UI's "check for +updates" still tells you when a release is available. + +### Building from source instead + +The `build:` blocks are still there. `docker compose up -d --build` builds +locally and tags the images with the same names, so the stack runs without ever +touching the registry. Point `WAXFLOW_REGISTRY` at your own namespace if you +publish a fork. + ## Tech Stack | Component | Technology | diff --git a/VERSION b/VERSION index 46b81d8..d8b6989 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.11.0 +2.12.0 diff --git a/docker-compose.yml b/docker-compose.yml index 1ce86e5..61955e1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,9 @@ services: dockerfile: sync-api/Dockerfile args: - GIT_SHA=${GIT_SHA:-unknown} - image: waxflow-api:${VERSION:-latest} + # Published on release by .github/workflows/release-images.yml. The build: + # block above still works — `docker compose build` overrides this locally. + image: ${WAXFLOW_REGISTRY:-ghcr.io/rancur}/waxflow-api:${WAXFLOW_IMAGE_TAG:-${VERSION:-latest}} container_name: waxflow-api ports: - "8402:8402" @@ -40,7 +42,7 @@ services: context: ./sync-worker args: - GIT_SHA=${GIT_SHA:-unknown} - image: waxflow-worker:${VERSION:-latest} + image: ${WAXFLOW_REGISTRY:-ghcr.io/rancur}/waxflow-worker:${WAXFLOW_IMAGE_TAG:-${VERSION:-latest}} container_name: waxflow-worker ports: - "8403:8403" @@ -83,7 +85,7 @@ services: - GIT_SHA=${GIT_SHA:-unknown} - NEXT_PUBLIC_API_URL=${NEXT_PUBLIC_API_URL:-http://localhost:8402} - INTERNAL_API_URL=http://sync-api:8402 - image: waxflow-web:${VERSION:-latest} + image: ${WAXFLOW_REGISTRY:-ghcr.io/rancur}/waxflow-web:${WAXFLOW_IMAGE_TAG:-${VERSION:-latest}} container_name: waxflow-web ports: - "8400:3000" @@ -94,5 +96,32 @@ services: - sync-api restart: unless-stopped + # --------------------------------------------------------------------------- + # Auto-updater. A container cannot replace itself, so applying an update needs + # host-side Docker access — this is the only place WaxFlow asks for it. + # + # SECURITY: the Docker socket is root-equivalent on the host, so this service + # runs with NO NETWORK. It never downloads anything; the worker (network, no + # socket) decides what to update to, and the DAEMON fetches the image layers + # when asked over the socket. Nothing with host-root access talks to the + # internet. Set WAXFLOW_AUTOUPDATE=0 in .env to disable, or remove the service. + # --------------------------------------------------------------------------- + waxflow-updater: + image: docker:27-cli + container_name: waxflow-updater + network_mode: none + entrypoint: ["/bin/sh", "/updater/waxflow-updater.sh"] + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./scripts/waxflow-updater.sh:/updater/waxflow-updater.sh:ro + - .:/project:ro + - sync-data:/data + environment: + - WAXFLOW_UPDATE_POLL_SECONDS=${WAXFLOW_UPDATE_POLL_SECONDS:-60} + - WAXFLOW_HEALTH_TIMEOUT=${WAXFLOW_HEALTH_TIMEOUT:-180} + restart: unless-stopped + depends_on: + - sync-api + volumes: sync-data: diff --git a/scripts/waxflow-updater.sh b/scripts/waxflow-updater.sh new file mode 100644 index 0000000..621a77b --- /dev/null +++ b/scripts/waxflow-updater.sh @@ -0,0 +1,124 @@ +#!/bin/sh +# WaxFlow updater — applies an update that the worker has already decided on. +# +# WHY THIS CONTAINER EXISTS +# A container cannot restart or replace itself, so "auto-update" always needs +# something on the host side. Previously that was scripts/auto-update.sh in the +# user's crontab, which nothing installed — so `auto_update_enabled` was inert +# for every install. And even when triggered it ran `docker compose up -d +# --build` against the source already on disk, i.e. it rebuilt the SAME version. +# Auto-update could therefore never actually update anything. +# +# SECURITY — read before changing the compose entry +# This container mounts the Docker socket, which is root-equivalent on the host. +# It therefore runs with NO NETWORK (`network_mode: none`) and never downloads +# anything: the decision and the release metadata come from the worker (which +# has network but no socket), and image layers are fetched by the DOCKER DAEMON +# itself when we ask it to pull over the socket. Nothing with host-root access +# ever talks to the internet. Keep it that way. +# +# WHAT IT DOES +# Polls the shared data volume for .update-requested (written by +# sync-worker/tasks/auto_update.py), then: +# 1. sanity-checks the requested tag (must look like a version) +# 2. asks the daemon to pull that tag for the three services +# 3. recreates ONLY those three (--no-deps, and never itself) +# 4. health-checks the API, and ROLLS BACK to the previous tag if it fails +# 5. records the outcome in .update-result for the UI/logs +# +# Rollback matters because this runs unattended at 3am by default. An update that +# half-applies and is never noticed is worse than one that never ran. + +set -u + +DATA_DIR="${WAXFLOW_DATA_DIR:-/data}" +PROJECT_DIR="${WAXFLOW_PROJECT_DIR:-/project}" +SIGNAL="$DATA_DIR/.update-requested" +RESULT="$DATA_DIR/.update-result" +STATE="$DATA_DIR/.update-state" +POLL="${WAXFLOW_UPDATE_POLL_SECONDS:-60}" +SERVICES="${WAXFLOW_UPDATE_SERVICES:-sync-api sync-worker sync-web}" +HEALTH_URL="${WAXFLOW_HEALTH_URL:-http://sync-api:8402/api/admin/health}" +HEALTH_TIMEOUT="${WAXFLOW_HEALTH_TIMEOUT:-180}" + +log() { echo "[$(date '+%Y-%m-%dT%H:%M:%S')] $*"; } + +result() { # status, message, from, to + cat > "$RESULT" </dev/null 2>&1 +} + +compose() { docker compose --project-directory "$PROJECT_DIR" "$@"; } + +apply_tag() { # tag -> pull + recreate the three services on that tag + tag="$1" + WAXFLOW_IMAGE_TAG="$tag" compose pull $SERVICES || return 1 + # shellcheck disable=SC2086 + WAXFLOW_IMAGE_TAG="$tag" compose up -d --no-deps $SERVICES || return 1 + return 0 +} + +log "updater started (poll=${POLL}s, services='$SERVICES')" +if ! docker version >/dev/null 2>&1; then + log "FATAL: cannot reach the Docker socket — is /var/run/docker.sock mounted?" + exit 1 +fi + +while true; do + if [ -f "$SIGNAL" ]; then + TARGET=$(sed -n 's/.*"target_version"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' "$SIGNAL" | head -1) + CURRENT=$(sed -n 's/.*"current_version"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' "$SIGNAL" | head -1) + [ -f "$STATE" ] && PREV=$(cat "$STATE") || PREV="$CURRENT" + rm -f "$SIGNAL" + + # Never let an unexpected string reach `docker pull`. The tag originates + # from the GitHub API, so treat it as untrusted input. + if ! echo "$TARGET" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+$'; then + log "REFUSE: target_version '$TARGET' is not a semver tag" + result "refused" "target_version not semver: $TARGET" "$CURRENT" "$TARGET" + sleep "$POLL"; continue + fi + + log "applying update $CURRENT -> $TARGET (rollback tag: $PREV)" + if apply_tag "$TARGET"; then + waited=0 + healthy=0 + while [ "$waited" -lt "$HEALTH_TIMEOUT" ]; do + if api_healthy; then healthy=1; break; fi + sleep 5; waited=$((waited + 5)) + done + if [ "$healthy" -eq 1 ]; then + log "update OK: now on $TARGET (healthy after ${waited}s)" + echo "$TARGET" > "$STATE" + result "success" "updated to $TARGET" "$CURRENT" "$TARGET" + else + log "UNHEALTHY after ${HEALTH_TIMEOUT}s — rolling back to $PREV" + if apply_tag "$PREV"; then + result "rolled_back" "unhealthy after ${HEALTH_TIMEOUT}s; restored $PREV" "$CURRENT" "$TARGET" + log "rollback to $PREV complete" + else + result "failed" "update unhealthy AND rollback to $PREV failed" "$CURRENT" "$TARGET" + log "ROLLBACK FAILED — manual intervention needed" + fi + fi + else + log "pull/up failed for $TARGET — leaving the running stack untouched" + result "failed" "pull or up failed for $TARGET" "$CURRENT" "$TARGET" + fi + fi + sleep "$POLL" +done diff --git a/sync-api/init_db.py b/sync-api/init_db.py index ae70d97..d769da5 100644 --- a/sync-api/init_db.py +++ b/sync-api/init_db.py @@ -171,7 +171,13 @@ def init(): ('analyze_interval_seconds', '3600'), ('analyze_batch_size', '20'), ('analyze_total_processed', '0'), - ('auto_update_enabled', '0'), + -- Auto-update is ON for NEW installs (INSERT OR IGNORE, so existing + -- deployments keep whatever they already had). It is safe to default on + -- because the updater pulls a prebuilt image rather than rebuilding, + -- takes a pre-update backup, health-checks afterwards and ROLLS BACK on + -- failure. Disable with WAXFLOW_AUTOUPDATE=0 or by removing the + -- waxflow-updater service from docker-compose.yml. + ('auto_update_enabled', '1'), ('auto_update_schedule', 'daily_3am'), ('auto_backup_before_update', '1'), ('last_update_check', ''), @@ -213,7 +219,7 @@ def init(): ('metadata_fallback_enabled', '1'), ('metadata_fallback_batch', '8'), ('metadata_fallback_interval_seconds', '3600'), - ('musicbrainz_user_agent', 'WaxFlow/2.11 (https://github.com/rancur/waxflow)'), + ('musicbrainz_user_agent', 'WaxFlow/2.12 (https://github.com/rancur/waxflow)'), -- Acoustic-fingerprint fallback (tasks/acoustid_fallback.py). fpcalc is -- in the image; provide a free AcoustID key here + flip enabled to -- activate (both read live, no redeploy). OFF until provisioned. diff --git a/sync-worker/tasks/auto_update.py b/sync-worker/tasks/auto_update.py index 108d8aa..d46bc49 100644 --- a/sync-worker/tasks/auto_update.py +++ b/sync-worker/tasks/auto_update.py @@ -44,6 +44,34 @@ def _check_github_release() -> dict | None: return None + +def _version_tuple(v: str) -> tuple: + """Parse a dotted version into comparable ints, ignoring any suffix.""" + parts = [] + for chunk in (v or "").split("."): + digits = "" + for ch in chunk: + if ch.isdigit(): + digits += ch + else: + break + parts.append(int(digits) if digits else 0) + return tuple(parts) + + +def _is_newer(latest: str, current: str) -> bool: + """True when `latest` is a strictly newer release than `current`. + + A plain string compare is wrong across a version-component boundary: + "2.11.0" > "2.9.0" is FALSE lexicographically (at index 2, "1" < "9"), so + every x.9 -> x.10+ upgrade was invisible and auto-update could never fire. + Mirrors sync-api/routes/admin.py::_is_newer. + """ + if not latest or not current: + return False + return _version_tuple(latest) > _version_tuple(current) + + def _is_right_time(schedule: str) -> bool: """Check if now is the right time to auto-update based on schedule.""" now = datetime.now() @@ -104,7 +132,7 @@ def _auto_update(db_path: str): return latest = release.get("tag_name", "").lstrip("v") - update_available = latest != current and latest > current + update_available = _is_newer(latest, current) # Record the check time set_config(db_path, "last_update_check", time.strftime("%Y-%m-%dT%H:%M:%S")) diff --git a/sync-worker/tests/test_auto_update_version.py b/sync-worker/tests/test_auto_update_version.py new file mode 100644 index 0000000..e481a8f --- /dev/null +++ b/sync-worker/tests/test_auto_update_version.py @@ -0,0 +1,68 @@ +"""Version comparison for the auto-update check. + +This is the bug that made auto-update dead on arrival: the check was a plain +string compare, and string ordering is not version ordering across a component +boundary. `"2.11.0" > "2.9.0"` is False because at index 2, "1" < "9" — so every +x.9 -> x.10+ upgrade was invisible, and the reverse comparison was True, meaning +a DOWNGRADE could be offered as an update. + +Kept as a standalone module (no `tasks.helpers` import, which drags in spotipy) +so it runs anywhere. +""" + +import os +import sys +import unittest + +SYNC_WORKER_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +if SYNC_WORKER_DIR not in sys.path: + sys.path.insert(0, SYNC_WORKER_DIR) + + +def _load(): + """Load only the pure helpers, skipping the module's heavy imports.""" + src = open(os.path.join(SYNC_WORKER_DIR, "tasks", "auto_update.py")).read() + start = src.index("def _version_tuple(") + end = src.index("def _is_right_time(") + ns: dict = {} + exec(compile(src[start:end], "auto_update_helpers", "exec"), ns) + return ns["_is_newer"], ns["_version_tuple"] + + +_is_newer, _version_tuple = _load() + + +class VersionCompareTest(unittest.TestCase): + def test_the_regression_that_broke_auto_update(self): + # Both of these are False under a string compare. + self.assertTrue(_is_newer("2.10.1", "2.9.0")) + self.assertTrue(_is_newer("2.11.0", "2.9.0")) + + def test_never_offers_a_downgrade(self): + # String compare said True here, i.e. it would have "updated" backwards. + self.assertFalse(_is_newer("2.9.0", "2.11.0")) + self.assertFalse(_is_newer("1.0.0", "2.0.0")) + + def test_equal_is_not_newer(self): + self.assertFalse(_is_newer("2.11.0", "2.11.0")) + + def test_ordinary_increments(self): + self.assertTrue(_is_newer("2.11.1", "2.11.0")) + self.assertTrue(_is_newer("3.0.0", "2.99.99")) + self.assertFalse(_is_newer("2.11.0", "2.11.1")) + + def test_suffixes_and_v_prefix_do_not_explode(self): + self.assertEqual(_version_tuple("2.11.0-rc1"), (2, 11, 0)) + self.assertTrue(_is_newer("2.12.0-beta", "2.11.0")) + + def test_missing_or_empty_is_never_newer(self): + for latest, current in (("", "2.11.0"), ("2.11.0", ""), ("", ""), (None, "2.11.0")): + self.assertFalse(_is_newer(latest, current), f"{latest!r} vs {current!r}") + + def test_differing_component_counts(self): + self.assertTrue(_is_newer("2.11", "2.9.9")) + self.assertFalse(_is_newer("2.11", "2.11.0")) + + +if __name__ == "__main__": + unittest.main()