Fix LLM consensus (Groq models, vote window) and make deploy/backup workflows safe

.github/workflows/backup.yml

@@ -1,49 +1,55 @@
 name: Backup database
 
+# Production database = PostgreSQL `metadata_engine` in the shared `hbrl-postgres` container
+# (moved off SQLite on 2026-09-17). The previous version of this job snapshotted the retired
+# SQLite file with the `sqlite3` CLI, which is not installed on the VPS: it failed every day
+# (2026-09-14 .. 2026-09-21) and no backup was ever written.
+#
+# The dump stays ON THE VPS. It is deliberately NOT uploaded as a GitHub artifact: this repository
+# is public and a dump holds user data (emails, password hashes, credits). An off-site copy has to
+# go to private, encrypted storage (restic - see the HRL handbook, decision D-1).
+
 on:
   schedule:
     - cron: "0 3 * * *"   # daily 03:00 UTC
   workflow_dispatch:
 
+concurrency:
+  group: backup-db
+  cancel-in-progress: false
+
 jobs:
   backup:
     runs-on: ubuntu-latest
     steps:
-      - name: Snapshot + rotate on the VPS
+      - name: pg_dump on the VPS + verify + rotate
         uses: appleboy/ssh-action@v1.2.2
         with:
           host: ${{ secrets.VPS_HOST }}
           username: ${{ secrets.VPS_USER }}
           port: ${{ secrets.VPS_PORT }}
           key: ${{ secrets.VPS_SSH_KEY }}
-          command_timeout: 5m
+          command_timeout: 10m
           script: |
-            set -e
+            set -euo pipefail
             cd /srv/hbrl/Metadata-Engine
             mkdir -p data/backups
             STAMP=$(date -u +%Y%m%d-%H%M%S)
-            # consistent copy even while the app is running
-            sqlite3 data/music_metadata.db ".backup 'data/backups/mme-$STAMP.db'"
-            gzip -f "data/backups/mme-$STAMP.db"
-            # keep the 14 most recent
-            ls -1t data/backups/mme-*.db.gz | tail -n +15 | xargs -r rm --
-            ln -sf "mme-$STAMP.db.gz" data/backups/latest.db.gz
-            ls -lh data/backups/ | tail -5
+            OUT="data/backups/mme-pg-$STAMP.dump"
 
-      - name: Pull latest snapshot to the runner
-        uses: appleboy/scp-action@v0.1.7
-        with:
-          host: ${{ secrets.VPS_HOST }}
-          username: ${{ secrets.VPS_USER }}
-          port: ${{ secrets.VPS_PORT }}
-          key: ${{ secrets.VPS_SSH_KEY }}
-          source: /srv/hbrl/Metadata-Engine/data/backups/latest.db.gz
-          target: ./_backup
-          strip_components: 5
+            # Custom format (compressed, restorable with pg_restore). No -t on docker exec:
+            # a TTY would corrupt the binary stream. Written to a .partial file first.
+            docker exec hbrl-postgres pg_dump -U hbrl_admin -d metadata_engine -Fc > "$OUT.partial"
 
-      - name: Keep an off-site copy as an artifact (90-day retention)
-        uses: actions/upload-artifact@v4
-        with:
-          name: db-backup-${{ github.run_id }}
-          path: ./_backup/latest.db.gz
-          retention-days: 90
+            # Verify before it counts as a backup: non-trivial size and a readable table of contents
+            SIZE=$(stat -c %s "$OUT.partial")
+            if [ "$SIZE" -lt 10000 ]; then echo "FAIL: dump is only $SIZE bytes"; rm -f "$OUT.partial"; exit 1; fi
+            docker exec -i hbrl-postgres pg_restore --list < "$OUT.partial" | grep -q 'TABLE DATA' || { echo "FAIL: dump has no table data"; rm -f "$OUT.partial"; exit 1; }
+            mv "$OUT.partial" "$OUT"
+            chmod 600 "$OUT"
+            ln -sf "mme-pg-$STAMP.dump" data/backups/latest-pg.dump
+
+            # Rotation: keep the 14 most recent dumps of THIS job (mme-pg-*); nothing else is touched.
+            ls -1t data/backups/mme-pg-*.dump | tail -n +15 | xargs -r rm --
+            echo "backup OK: $OUT ($SIZE bytes)"
+            ls -lh data/backups/ | tail -6

.github/workflows/ci.yml

@@ -34,6 +34,13 @@ jobs:
       - name: Install system deps
         run: sudo apt-get update && sudo apt-get install -y ffmpeg libsndfile1
       - working-directory: backend
+        env:
+          # The app creates CERT_DIR at import time (default /data/certificates, not creatable on a
+          # runner) and refuses to start without SECRET_KEY; without these every test module failed
+          # at collection ("PermissionError: /data") - backend CI was red on main since 2026-09-02.
+          CERT_DIR: ${{ runner.temp }}/certificates
+          SECRET_KEY: ci-${{ github.run_id }}-${{ github.sha }}
+          DATABASE_URL: sqlite:///${{ runner.temp }}/ci.db
         run: |
           pip install -r requirements.txt
           pytest -q --no-header

.github/workflows/deploy.yml

@@ -5,6 +5,11 @@ on:
     branches:
       - main
 
+# Never run two deploys at once (a second push while a build is running would race on the VPS).
+concurrency:
+  group: deploy-vps
+  cancel-in-progress: false
+
 jobs:
   # Fast gate — a broken push must not reach the VPS. Full test suites
   # (vitest, pytest) run separately in ci.yml and don't block the deploy.
@@ -54,24 +59,34 @@ jobs:
           # trimming the CUDA wheels from requirements.txt. Headroom, not tight.
           command_timeout: 30m
           script: |
-            set -e
+            set -euo pipefail
             cd /srv/hbrl/Metadata-Engine
             # Plain progress: BuildKit's default TTY renderer streams
             # carriage-return redraws that break the SSH action mid-build.
             export BUILDKIT_PROGRESS=plain
             export DOCKER_BUILDKIT=1
 
-            # Backup existing env if present (docker-compose reads root .env)
-            if [ -f .env ]; then
-              cp .env .env.backup.$(date +%F-%H%M%S)
-            fi
+            # Production compose file: Postgres (hbrl-postgres / metadata_engine) over the
+            # external hbrl-db network. vps.docker-compose.yml is only a deprecated alias now.
+            COMPOSE_FILE="docker-compose.yml"
+            DB_RE='^DATABASE_URL=postgres(ql)?(\+[a-z0-9]+)?://[^@[:space:]]+@hbrl-postgres:5432/metadata_engine'
+
+            # Guard 1 (fail closed, before anything is touched): the VPS .env OWNS DATABASE_URL and
+            # must already point at production Postgres. The GitHub secret of the same name is stale
+            # (SQLite) and is deliberately NOT written below - on 2026-09-21 it switched prod to SQLite.
+            test -f .env || { echo "FAIL: .env is missing on the VPS - refusing to deploy"; exit 1; }
+            OLD_DB_LINE="$(grep -m1 '^DATABASE_URL=' .env || true)"
+            printf '%s\n' "$OLD_DB_LINE" | grep -Eq "$DB_RE" || { echo "FAIL: DATABASE_URL in the VPS .env is not production Postgres (hbrl-postgres/metadata_engine) - refusing to deploy"; exit 1; }
+
+            # Backup existing env (docker-compose reads root .env)
+            cp -p .env ".env.backup.$(date +%F-%H%M%S)"
 
-            # Write root .env from GitHub Secrets (no secrets in repo)
+            # Write the env from GitHub Secrets into .env.new (no secrets in repo).
             # NOTE: every heredoc body line below must stay indented to match
             # this YAML block scalar (12 spaces) or the workflow file fails
             # to parse entirely (GitHub reports it as a run with zero jobs).
-            # The sed strips that indentation back out before it reaches .env.
-            cat <<'EOF' | sed -e 's/^[[:space:]]*//' > .env
+            # The sed strips that indentation back out before it reaches .env.new.
+            cat <<'EOF' | sed -e 's/^[[:space:]]*//' > .env.new
             GEMINI_API_KEY=${{ secrets.GEMINI_API_KEY }}
             GROQ_API_KEY=${{ secrets.GROQ_API_KEY }}
             MISTRAL_API_KEY=${{ secrets.MISTRAL_API_KEY }}
@@ -104,8 +119,6 @@ jobs:
             API_CLIENT_ID=${{ secrets.API_CLIENT_ID }}
             API_CLIENT_SECRET=${{ secrets.API_CLIENT_SECRET }}
 
-            DATABASE_URL=${{ secrets.DATABASE_URL }}
-
             APP_BASE_URL=https://app-metadata.hardbanrecordslab.online
             SMTP_HOST=${{ secrets.SMTP_HOST }}
             SMTP_PORT=${{ secrets.SMTP_PORT }}
@@ -115,18 +128,39 @@ jobs:
             SENTRY_DSN=${{ secrets.SENTRY_DSN }}
             EOF
 
-            # Pull latest code and rebuild containers
-            git pull origin main
-
-            # Use VPS compose file (binds 127.0.0.1:8888, mounts .env)
-            COMPOSE_FILE="vps.docker-compose.yml"
-            if command -v docker >/dev/null 2>&1 && docker compose version >/dev/null 2>&1; then
-              docker compose -f "$COMPOSE_FILE" down
-              docker compose -f "$COMPOSE_FILE" up -d --build
-            elif command -v docker-compose >/dev/null 2>&1; then
-              docker-compose -f "$COMPOSE_FILE" down
-              docker-compose -f "$COMPOSE_FILE" up -d --build
-            else
-              echo "Docker Compose not found on VPS" >&2
+            # Re-attach the production DATABASE_URL, re-check it, then swap the file in atomically.
+            printf '%s\n' "$OLD_DB_LINE" >> .env.new
+            grep -Eq "$DB_RE" .env.new || { echo "FAIL: assembled .env lost the Postgres DATABASE_URL - keeping the old .env"; rm -f .env.new; exit 1; }
+            chmod --reference=.env .env.new
+            mv .env.new .env
+
+            # Code: fast-forward only. Fails (instead of silently losing or mixing commits) if the
+            # VPS checkout ever diverges from origin/main again.
+            git fetch origin main
+            git checkout main
+            git merge --ff-only origin/main
+
+            # Keep the running image as a rollback target, then build + swap. No `compose down`:
+            # `up -d --build` builds first and only then replaces the container, so a failed build
+            # leaves the old container serving.
+            docker tag music-metadata-engine:latest music-metadata-engine:previous 2>/dev/null || true
+            docker compose -f "$COMPOSE_FILE" up -d --build
+
+            # Health-gated: container healthy AND still on production Postgres, else roll back.
+            healthy=0
+            status=unknown
+            for i in $(seq 1 30); do
+              status="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' metadata-backend 2>/dev/null || echo missing)"
+              if [ "$status" = "healthy" ]; then healthy=1; break; fi
+              sleep 10
+            done
+            CONTAINER_DB="$( (docker exec metadata-backend printenv DATABASE_URL 2>/dev/null | sed -E 's#(://[^:]+:)[^@]+@#\1***@#') || true )"
+            echo "container health: $status | DATABASE_URL: ${CONTAINER_DB:-<none>}"
+            if [ "$healthy" != "1" ] || ! printf '%s' "$CONTAINER_DB" | grep -Eq '@hbrl-postgres:5432/metadata_engine'; then
+              echo "FAIL: post-deploy verification failed - rolling back to the previous image"
+              docker tag music-metadata-engine:previous music-metadata-engine:latest
+              docker compose -f "$COMPOSE_FILE" up -d --no-build
               exit 1
             fi
+            curl -fsS http://127.0.0.1:8888/api/health/diagnose | grep -q '"llm"' || echo "WARN: /api/health/diagnose has no llm block"
+            echo "Deploy OK"

README.md

@@ -47,7 +47,7 @@ Full documentation lives in **[`docs/`](docs/README.md)**.
 
 | Layer | Tech |
 |---|---|
-| Backend | FastAPI (Python 3.10), SQLAlchemy, SQLite |
+| Backend | FastAPI (Python 3.10), SQLAlchemy, PostgreSQL (shared `hbrl-postgres`; SQLite only for local dev) |
 | Audio | Essentia, Librosa, openai-whisper (CPU), FFmpeg + Chromaprint |
 | AI | Groq, Google Gemini, OpenRouter |
 | Frontend | React 19, TypeScript, Vite, Tailwind CSS |
@@ -80,8 +80,10 @@ API docs: `http://localhost:8888/docs` (FastAPI Swagger).
 ## Deploy
 
 Push to `main` → GitHub Actions (`.github/workflows/deploy.yml`) SSHes to the
-VPS, writes `.env` from **GitHub repo Secrets**, and runs
-`docker compose -f vps.docker-compose.yml up -d --build`.
+VPS, refreshes `.env` from **GitHub repo Secrets** (except `DATABASE_URL`, which the
+VPS `.env` owns), fast-forwards the checkout and runs
+`docker compose -f docker-compose.yml up -d --build`, then verifies health and the
+Postgres connection and rolls back to the previous image if either check fails.
 
 Everything about env vars, secrets, the VPS layout and operational procedures
 is in **[`docs/OPERATIONS.md`](docs/OPERATIONS.md)**.

backend/app/routes/generative.py

@@ -10,6 +10,7 @@
 import httpx
 import asyncio
 from urllib.parse import quote
+from app.services.groq_models import groq_create
 
 from app.config import settings
 from app.dependencies import get_user_and_check_quota
@@ -36,12 +37,12 @@ class RefineFieldRequest(BaseModel):
 
 # --- Helper for Groq JSON response ---
 async def call_groq_json(
-    prompt: str, model_name: str = "llama-3.3-70b-versatile"
+    prompt: str, model_name: str = None
 ) -> Dict[str, Any]:
     try:
         client = get_groq_client()
-        response = client.chat.completions.create(
-            model=model_name,
+        response = groq_create(
+            client, "pro", model=model_name,
             messages=[
                 {
                     "role": "system",

backend/app/routes/health.py

@@ -1,5 +1,6 @@
 from fastapi import APIRouter
 from app.config import settings
+from app.services.groq_models import groq_model
 import os
 
 router = APIRouter(prefix="/health", tags=["health"])
@@ -23,7 +24,9 @@ async def diagnose_system():
         },
         "llm": {
             "openrouter_paid_allowed": os.getenv("OPENROUTER_ALLOW_PAID", "").strip().lower() not in ("0", "false", "no", "off"),
-            "vote_models": (os.getenv("OPENROUTER_VOTE_MODELS", "").strip() or "default (gemini-2.5-flash-lite, deepseek-v4-flash, gemma-4-31b-it)"),
+            "vote_models": (os.getenv("OPENROUTER_VOTE_MODELS", "").strip() or "default (gemini-2.5-flash-lite, mistral-small-24b-instruct-2501, gemma-4-31b-it)"),
+            "vote_timeout_sec": os.getenv("OPENROUTER_VOTE_TIMEOUT_SEC", "").strip() or "default (75)",
+            "groq_models": {"pro": groq_model("pro"), "flash": groq_model("flash")},
         },
         "system": {"os": os.name, "cwd": os.getcwd()},
     }

backend/app/routes/test_analysis_timeout.py

@@ -70,7 +70,9 @@ async def slow_analyze_fresh_track(*args, **kwargs):
         job = session.query(db.Job).filter(db.Job.id == "job-timeout").first()
         assert job is not None
         assert job.status == "error"
-        assert job.error == "TIMEOUT_1s"
+        # process_analysis reports str(e) or, when the message is empty (asyncio.TimeoutError), the
+        # exception type name; there was never a "TIMEOUT_<n>s" code in the app (nor in the frontend).
+        assert job.error == "TimeoutError"
     finally:
         session.close()
 

backend/app/services/fresh_track_analyzer.py

@@ -10,12 +10,14 @@
 """
 
 import asyncio
+import os
 import time
 from typing import Dict, Any
 import logging
 
 from .deep_audio_analyzer import DeepAudioAnalyzer
 from .llm_ensemble import LLMEnsemble
+from .groq_models import groq_create
 
 logger = logging.getLogger(__name__)
 
@@ -87,7 +89,13 @@ async def analyze_fresh_track(
             # The DSP layer can eat the whole budget on a busy VPS (66-170 s observed). The LLM
             # vote is cheap and quick (~4-40 s), so it always gets a guaranteed window instead of
             # being skipped - skipping silently produced template-only results ("0 LLMs").
-            remaining = max(remaining, 45.0)
+            # The window must cover the OpenRouter vote timeout (75s by default, see llm_ensemble); a
+            # 45s window cut the vote off and gave "1 LLMs"/"0 LLMs" results. Env: LLM_MIN_WINDOW_SEC.
+            try:
+                llm_min_window = float(os.getenv("LLM_MIN_WINDOW_SEC", "").strip() or 90.0)
+            except ValueError:
+                llm_min_window = 90.0
+            remaining = max(remaining, llm_min_window)
 
             # RELAXED LIMIT: Only skip if less than 3 seconds (was 5)
             # This gives LLM a chance even in tight scenarios
@@ -444,11 +452,11 @@ async def _analyze_lyrics_with_llm(self, lyrics: str) -> Dict:
 }}"""
 
             response = await asyncio.to_thread(
-                client.chat.completions.create,
-                model="llama-3.3-70b-versatile",
+                groq_create,
+                client, "pro",
                 messages=[{"role": "user", "content": prompt}],
                 temperature=0.3,
-                max_tokens=300
+                max_tokens=300,
             )
 
             import json