From 175f9f3e53548d54e5a3ad975846e9187b9b6f72 Mon Sep 17 00:00:00 2001
From: Hardban Records Lab <hardbanrecordslab.pl@gmail.com>
Date: Mon, 21 Sep 2026 20:10:38 +0200
Subject: [PATCH 1/7] fix(llm): replace retired Groq models, widen OpenRouter
 vote window, stop logging DB password

Verified on the VPS on 2026-09-21 (analyses returned "1 LLMs" / "0 LLMs"):

* Groq no longer serves llama-3.3-70b-versatile / llama-3.1-8b-instant for our key
  (HTTP 404 model_not_found). Model ids now come from services/groq_models.py
  (GROQ_MODEL_PRO=openai/gpt-oss-120b, GROQ_MODEL_FLASH=openai/gpt-oss-20b, both overridable);
  gpt-oss reasoning models get reasoning_effort=low and extra max_tokens headroom.
* Pro mode got a single vote: deepseek-v4-flash (23-45s) and gemma (17-19s) regularly missed the
  40s vote timeout, and the whole LLM step was additionally capped at ~44s. Vote lineup is now
  gemini-2.5-flash-lite / mistral-small-24b / gemma-4-31b (measured 3-19s), vote timeout 75s
  (OPENROUTER_VOTE_TIMEOUT_SEC) and the guaranteed LLM window 90s (LLM_MIN_WINDOW_SEC).
* entrypoint.sh printed DATABASE_URL including the password to the container logs; it is masked now.
* /api/health/diagnose also reports the Groq models and the vote timeout.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 backend/app/routes/generative.py             |  7 ++-
 backend/app/routes/health.py                 |  5 +-
 backend/app/services/fresh_track_analyzer.py | 16 ++++--
 backend/app/services/groq_models.py          | 39 ++++++++++++++
 backend/app/services/groq_whisper.py         |  9 ++--
 backend/app/services/llm_ensemble.py         | 55 ++++++++++++--------
 backend/entrypoint.sh                        |  3 +-
 backend/tests/test_groq_models.py            | 46 ++++++++++++++++
 8 files changed, 148 insertions(+), 32 deletions(-)
 create mode 100644 backend/app/services/groq_models.py
 create mode 100644 backend/tests/test_groq_models.py

diff --git a/backend/app/routes/generative.py b/backend/app/routes/generative.py
index 0f3b96f..9d6161d 100644
--- a/backend/app/routes/generative.py
+++ b/backend/app/routes/generative.py
@@ -10,6 +10,7 @@
 import httpx
 import asyncio
 from urllib.parse import quote
+from app.services.groq_models import groq_model, groq_extra, groq_budget
 
 from app.config import settings
 from app.dependencies import get_user_and_check_quota
@@ -36,9 +37,10 @@ class RefineFieldRequest(BaseModel):
 
 # --- Helper for Groq JSON response ---
 async def call_groq_json(
-    prompt: str, model_name: str = "llama-3.3-70b-versatile"
+    prompt: str, model_name: str = None
 ) -> Dict[str, Any]:
     try:
+        model_name = model_name or groq_model("pro")
         client = get_groq_client()
         response = client.chat.completions.create(
             model=model_name,
@@ -50,8 +52,9 @@ async def call_groq_json(
                 {"role": "user", "content": prompt},
             ],
             temperature=0.3,
-            max_tokens=2000,
+            max_tokens=groq_budget(model_name, 2000),
             response_format={"type": "json_object"},
+            **groq_extra(model_name),
         )
         
         result_text = response.choices[0].message.content
diff --git a/backend/app/routes/health.py b/backend/app/routes/health.py
index fd6c7cf..d794770 100644
--- a/backend/app/routes/health.py
+++ b/backend/app/routes/health.py
@@ -1,5 +1,6 @@
 from fastapi import APIRouter
 from app.config import settings
+from app.services.groq_models import groq_model
 import os
 
 router = APIRouter(prefix="/health", tags=["health"])
@@ -23,7 +24,9 @@ async def diagnose_system():
         },
         "llm": {
             "openrouter_paid_allowed": os.getenv("OPENROUTER_ALLOW_PAID", "").strip().lower() not in ("0", "false", "no", "off"),
-            "vote_models": (os.getenv("OPENROUTER_VOTE_MODELS", "").strip() or "default (gemini-2.5-flash-lite, deepseek-v4-flash, gemma-4-31b-it)"),
+            "vote_models": (os.getenv("OPENROUTER_VOTE_MODELS", "").strip() or "default (gemini-2.5-flash-lite, mistral-small-24b-instruct-2501, gemma-4-31b-it)"),
+            "vote_timeout_sec": os.getenv("OPENROUTER_VOTE_TIMEOUT_SEC", "").strip() or "default (75)",
+            "groq_models": {"pro": groq_model("pro"), "flash": groq_model("flash")},
         },
         "system": {"os": os.name, "cwd": os.getcwd()},
     }
diff --git a/backend/app/services/fresh_track_analyzer.py b/backend/app/services/fresh_track_analyzer.py
index 6797992..92cc5e1 100644
--- a/backend/app/services/fresh_track_analyzer.py
+++ b/backend/app/services/fresh_track_analyzer.py
@@ -10,12 +10,14 @@
 """
 
 import asyncio
+import os
 import time
 from typing import Dict, Any
 import logging
 
 from .deep_audio_analyzer import DeepAudioAnalyzer
 from .llm_ensemble import LLMEnsemble
+from .groq_models import groq_model, groq_extra, groq_budget
 
 logger = logging.getLogger(__name__)
 
@@ -87,7 +89,13 @@ async def analyze_fresh_track(
             # The DSP layer can eat the whole budget on a busy VPS (66-170 s observed). The LLM
             # vote is cheap and quick (~4-40 s), so it always gets a guaranteed window instead of
             # being skipped - skipping silently produced template-only results ("0 LLMs").
-            remaining = max(remaining, 45.0)
+            # The window must cover the OpenRouter vote timeout (75s by default, see llm_ensemble); a
+            # 45s window cut the vote off and gave "1 LLMs"/"0 LLMs" results. Env: LLM_MIN_WINDOW_SEC.
+            try:
+                llm_min_window = float(os.getenv("LLM_MIN_WINDOW_SEC", "").strip() or 90.0)
+            except ValueError:
+                llm_min_window = 90.0
+            remaining = max(remaining, llm_min_window)
 
             # RELAXED LIMIT: Only skip if less than 3 seconds (was 5)
             # This gives LLM a chance even in tight scenarios
@@ -443,12 +451,14 @@ async def _analyze_lyrics_with_llm(self, lyrics: str) -> Dict:
   "genre_hints": ["genre based on lyrical style"]
 }}"""
             
+            lyrics_model = groq_model("pro")
             response = await asyncio.to_thread(
                 client.chat.completions.create,
-                model="llama-3.3-70b-versatile",
+                model=lyrics_model,
                 messages=[{"role": "user", "content": prompt}],
                 temperature=0.3,
-                max_tokens=300
+                max_tokens=groq_budget(lyrics_model, 300),
+                **groq_extra(lyrics_model),
             )
             
             import json
diff --git a/backend/app/services/groq_models.py b/backend/app/services/groq_models.py
new file mode 100644
index 0000000..1dd3702
--- /dev/null
+++ b/backend/app/services/groq_models.py
@@ -0,0 +1,39 @@
+"""Single place that decides which Groq chat model each call site uses.
+
+Groq retired ``llama-3.3-70b-versatile`` and ``llama-3.1-8b-instant`` for our key (HTTP 404
+``model_not_found``, verified 2026-09-21 with GET /openai/v1/models), so every Groq call silently
+failed. Model names now live here and can be overridden without a code change:
+
+    GROQ_MODEL_PRO    (default openai/gpt-oss-120b)  - 'pro' mode, metadata merge, lyrics
+    GROQ_MODEL_FLASH  (default openai/gpt-oss-20b)   - 'flash' mode
+
+gpt-oss models are reasoning models: reasoning tokens are counted against ``max_tokens`` and the
+answer comes back empty when the budget is too small, so ``groq_extra`` pins a low reasoning effort
+and ``groq_budget`` adds headroom on top of the visible-answer budget the call site asks for.
+"""
+import os
+
+DEFAULT_PRO = "openai/gpt-oss-120b"
+DEFAULT_FLASH = "openai/gpt-oss-20b"
+REASONING_HEADROOM = 1500
+
+
+def groq_model(model_preference: str = "pro") -> str:
+    """Model id for a mode ('flash' -> fast/cheap model, anything else -> pro model)."""
+    if model_preference == "flash":
+        return os.getenv("GROQ_MODEL_FLASH", "").strip() or DEFAULT_FLASH
+    return os.getenv("GROQ_MODEL_PRO", "").strip() or DEFAULT_PRO
+
+
+def _is_reasoning(model: str) -> bool:
+    return model.startswith("openai/gpt-oss")
+
+
+def groq_extra(model: str) -> dict:
+    """Extra create() kwargs for a model (low reasoning effort for gpt-oss, nothing otherwise)."""
+    return {"extra_body": {"reasoning_effort": "low"}} if _is_reasoning(model) else {}
+
+
+def groq_budget(model: str, visible_tokens: int) -> int:
+    """max_tokens to request so a reasoning model still has room for its visible answer."""
+    return visible_tokens + REASONING_HEADROOM if _is_reasoning(model) else visible_tokens
diff --git a/backend/app/services/groq_whisper.py b/backend/app/services/groq_whisper.py
index d45a8f7..f254811 100644
--- a/backend/app/services/groq_whisper.py
+++ b/backend/app/services/groq_whisper.py
@@ -7,6 +7,7 @@
 import logging
 from typing import Dict, Any, Optional
 from app.config import settings
+from app.services.groq_models import groq_model, groq_extra, groq_budget
 
 logger = logging.getLogger(__name__)
 
@@ -226,15 +227,17 @@ def safe_s(v):
 
         logger.info("Sending metadata prompt to Groq (length: %d)", len(prompt))
 
+        merge_model = groq_model("pro")
         response = client.chat.completions.create(
-            model="llama-3.3-70b-versatile",
+            model=merge_model,
             messages=[
                 {"role": "system", "content": "You are a professional music metadata API. Always respond with valid JSON only. Never include markdown or explanations outside the JSON."},
                 {"role": "user", "content": prompt},
             ],
             temperature=0.25,
-            max_tokens=2500,
+            max_tokens=groq_budget(merge_model, 2500),
             response_format={"type": "json_object"},
+            **groq_extra(merge_model),
         )
 
         result_text = response.choices[0].message.content
@@ -278,7 +281,7 @@ async def full_pipeline(file_path: str, transcribe: bool = True) -> Dict[str, An
         audio_analysis = results[0]
         transcription = results[1].get("text", "") if (transcribe and len(results) > 1) else None
 
-        logger.info("Step 2: Groq Cloud LLM (Llama 3.3 70B)...")
+        logger.info("Step 2: Groq Cloud LLM (%s)...", groq_model("pro"))
         metadata = await GroqWhisperService.generate_metadata(
             audio_analysis=audio_analysis,
             transcription=transcription,
diff --git a/backend/app/services/llm_ensemble.py b/backend/app/services/llm_ensemble.py
index c3f5326..7f15586 100644
--- a/backend/app/services/llm_ensemble.py
+++ b/backend/app/services/llm_ensemble.py
@@ -15,6 +15,7 @@
 import logging
 import json
 from .standards import MAIN_GENRES, SUB_GENRES, MOODS, INSTRUMENTATION, VOCAL_STYLES
+from .groq_models import groq_model, groq_extra, groq_budget
 
 logger = logging.getLogger(__name__)
 
@@ -98,7 +99,7 @@ async def stream_description(
                 return ""
 
             client = Groq(api_key=groq_key)
-            model = "llama-3.3-70b-versatile" if model_preference == "pro" else "llama-3.1-8b-instant"
+            model = groq_model(model_preference)
 
             rhythm   = audio_features.get("rhythm",   {})
             harmonic = audio_features.get("harmonic", {})
@@ -139,8 +140,9 @@ async def stream_description(
                 model=model,
                 messages=[{"role": "user", "content": desc_prompt}],
                 temperature=0.6,
-                max_tokens=600,
+                max_tokens=groq_budget(model, 600),
                 stream=True,
+                **groq_extra(model),
             )
 
             for chunk in stream:
@@ -453,7 +455,7 @@ def _validate_and_refine_classification(self, raw_result: Dict, audio_features:
     
     async def _groq_classify(self, context: str, system_prompt: str = None, retries: int = 3, model_preference: str = 'flash') -> Dict:
         """
-        Groq: Llama 3.1 8B Instant (flash) / Llama 3.3 70B Versatile (pro)
+        Groq: model per mode from groq_models.py (GROQ_MODEL_FLASH / GROQ_MODEL_PRO)
         """
         if not self.groq_key:
             return {'error': 'no_api_key'}
@@ -463,8 +465,8 @@ async def _groq_classify(self, context: str, system_prompt: str = None, retries:
         
         # Select model based on mode preference
         is_flash = model_preference == 'flash'
-        groq_model = "llama-3.1-8b-instant" if is_flash else "llama-3.3-70b-versatile"
-        groq_max_tokens = 800 if is_flash else 1000
+        model_name = groq_model('flash' if is_flash else 'pro')
+        groq_max_tokens = groq_budget(model_name, 800 if is_flash else 1000)
         
         # INCREASE VARIETY: Use job_id as part of the seed or just increase temperature
         # Temperature 0.7 allows for creative variety while keeping structure
@@ -485,11 +487,12 @@ async def _groq_classify(self, context: str, system_prompt: str = None, retries:
         for attempt in range(retries):
             try:
                 response = client.chat.completions.create(
-                    model=groq_model,
+                    model=model_name,
                     messages=messages,
                     temperature=temperature,
                     max_tokens=groq_max_tokens,
-                    response_format={"type": "json_object"}
+                    response_format={"type": "json_object"},
+                    **groq_extra(model_name),
                 )
 
                 
@@ -499,7 +502,7 @@ async def _groq_classify(self, context: str, system_prompt: str = None, retries:
                     
                 result = json.loads(content)
                 result['llm_source'] = 'groq'
-                result['_groq_model'] = groq_model
+                result['_groq_model'] = model_name
                 return result
             except Exception as e:
                 logger.warning(f"Groq attempt {attempt+1} failed: {e}")
@@ -547,23 +550,30 @@ async def _groq_classify(self, context: str, system_prompt: str = None, retries:
         "openai/gpt-oss-120b",                        # ~$0.037/$0.170 per M — strong but ~9s, last resort
     ]
 
-    # OpenRouter-first vote (paid but very cheap, ~$0.0011 for a 3-model vote,
-    # i.e. ~4,000 analyses per $5). Chosen 2026-09-21 after a live test of 7
-    # candidates on the real prompt: all returned clean JSON, but only
-    # gemini-2.5-flash-lite was consistently fast (~3s); deepseek-v4-flash
-    # (~$0.00024/call) and gemma-4-31b-it (~$0.00038/call) are slower (15-40s).
-    # Order matters: 'flash' uses the first two (both reliably < 25s), 'pro'
-    # adds deepseek (a different family, sometimes slow - it is simply dropped
-    # from the vote if it misses the timeout).
+    # OpenRouter-first vote (paid but very cheap, ~$0.001 for a 3-model vote,
+    # i.e. thousands of analyses per $5). Lineup re-measured 2026-09-21 from the VPS on the real
+    # prompt, 3 concurrent calls per model: gemini-2.5-flash-lite 3-5s, gemma-4-31b-it 17-19s,
+    # mistral-small-24b 15-18s (all stable), gpt-oss-120b 4-36s (erratic), deepseek-v4-flash
+    # 23-45s (usually past the old 40s cut-off, so pro mode kept getting only 1 vote) and
+    # gpt-oss-20b (cut JSON mid-string). deepseek was therefore dropped from the default.
+    # Order matters: 'flash' uses the first two, 'pro' all three - three different providers.
     # Override with OPENROUTER_VOTE_MODELS="a/b,c/d,e/f" (comma separated).
     OPENROUTER_VOTE_MODELS = [
         "google/gemini-2.5-flash-lite",
+        "mistralai/mistral-small-24b-instruct-2501",
         "google/gemma-4-31b-it",
-        "deepseek/deepseek-v4-flash",
     ]
-    # How long the vote waits for the slowest model before using whatever
-    # already answered (a slow model must never discard the fast ones).
-    OPENROUTER_VOTE_TIMEOUT_SEC = 40.0
+    # How long the vote waits for the slowest model before using whatever already answered
+    # (a slow model must never discard the fast ones). Override with OPENROUTER_VOTE_TIMEOUT_SEC.
+    # Must stay below the LLM window granted by fresh_track_analyzer (LLM_MIN_WINDOW_SEC).
+    OPENROUTER_VOTE_TIMEOUT_SEC = 75.0
+
+    def _openrouter_vote_timeout(self) -> float:
+        import os
+        try:
+            return float(os.getenv("OPENROUTER_VOTE_TIMEOUT_SEC", "").strip() or self.OPENROUTER_VOTE_TIMEOUT_SEC)
+        except ValueError:
+            return float(self.OPENROUTER_VOTE_TIMEOUT_SEC)
 
     @staticmethod
     def _openrouter_paid_allowed() -> bool:
@@ -586,11 +596,12 @@ async def _openrouter_vote(self, user_prompt: str, model_preference: str) -> Lis
             asyncio.ensure_future(self._openrouter_classify(user_prompt, system_prompt=MUSIC_EXPERT_SYSTEM_PROMPT, models=[m]))
             for m in models
         ]
-        done, pending = await asyncio.wait(tasks, timeout=self.OPENROUTER_VOTE_TIMEOUT_SEC)
+        vote_timeout = self._openrouter_vote_timeout()
+        done, pending = await asyncio.wait(tasks, timeout=vote_timeout)
         for t in pending:
             t.cancel()
         if pending:
-            logger.warning("OpenRouter vote: %d/%d models did not answer within %.0fs", len(pending), len(tasks), self.OPENROUTER_VOTE_TIMEOUT_SEC)
+            logger.warning("OpenRouter vote: %d/%d models did not answer within %.0fs", len(pending), len(tasks), vote_timeout)
         results = []
         for t in done:
             try:
diff --git a/backend/entrypoint.sh b/backend/entrypoint.sh
index 7b69363..d65127e 100644
--- a/backend/entrypoint.sh
+++ b/backend/entrypoint.sh
@@ -4,7 +4,8 @@ set -e
 echo "--- RUNTIME DEBUG INFO ---"
 echo "Current User: $(whoami)"
 echo "Current Dir: $(pwd)"
-echo "DATABASE_URL Env: $DATABASE_URL"
+# never log the password: mask the userinfo part of the URL
+echo "DATABASE_URL Env: $(printf '%s' "$DATABASE_URL" | sed -E 's#(://[^:]+:)[^@]+@#\1***@#')"
 echo "--------------------------"
 
 # Force any necessary migrations or checks
diff --git a/backend/tests/test_groq_models.py b/backend/tests/test_groq_models.py
new file mode 100644
index 0000000..e2d6396
--- /dev/null
+++ b/backend/tests/test_groq_models.py
@@ -0,0 +1,46 @@
+"""Groq model selection (llama-3.x were retired -> 404) and the OpenRouter vote timeout override."""
+from app.services.groq_models import groq_model, groq_extra, groq_budget, DEFAULT_PRO, DEFAULT_FLASH
+from app.services.llm_ensemble import LLMEnsemble
+
+
+def test_defaults_are_not_the_retired_llama_models(monkeypatch):
+    monkeypatch.delenv("GROQ_MODEL_PRO", raising=False)
+    monkeypatch.delenv("GROQ_MODEL_FLASH", raising=False)
+    assert groq_model("pro") == DEFAULT_PRO == "openai/gpt-oss-120b"
+    assert groq_model("flash") == DEFAULT_FLASH == "openai/gpt-oss-20b"
+    assert "llama" not in groq_model("pro") and "llama" not in groq_model("flash")
+
+
+def test_env_override_and_empty_env_falls_back(monkeypatch):
+    monkeypatch.setenv("GROQ_MODEL_PRO", "qwen/qwen3.8-27b")
+    monkeypatch.setenv("GROQ_MODEL_FLASH", "  ")            # blank (e.g. unset GitHub variable) -> default
+    assert groq_model("pro") == "qwen/qwen3.8-27b"
+    assert groq_model("flash") == DEFAULT_FLASH
+
+
+def test_reasoning_models_get_low_effort_and_token_headroom():
+    assert groq_extra("openai/gpt-oss-120b") == {"extra_body": {"reasoning_effort": "low"}}
+    assert groq_budget("openai/gpt-oss-120b", 300) > 300
+    assert groq_extra("qwen/qwen3.8-27b") == {}
+    assert groq_budget("qwen/qwen3.8-27b", 300) == 300
+
+
+def test_vote_timeout_default_env_override_and_bad_value(monkeypatch):
+    ens = LLMEnsemble(openrouter_key="k")
+    monkeypatch.delenv("OPENROUTER_VOTE_TIMEOUT_SEC", raising=False)
+    assert ens._openrouter_vote_timeout() == LLMEnsemble.OPENROUTER_VOTE_TIMEOUT_SEC == 75.0
+    monkeypatch.setenv("OPENROUTER_VOTE_TIMEOUT_SEC", "120")
+    assert ens._openrouter_vote_timeout() == 120.0
+    monkeypatch.setenv("OPENROUTER_VOTE_TIMEOUT_SEC", "oops")
+    assert ens._openrouter_vote_timeout() == 75.0
+    monkeypatch.setenv("OPENROUTER_VOTE_TIMEOUT_SEC", "")
+    ens.OPENROUTER_VOTE_TIMEOUT_SEC = 0.3                   # instance override (used by test_openrouter_vote)
+    assert ens._openrouter_vote_timeout() == 0.3
+
+
+def test_default_vote_lineup_has_no_slow_deepseek(monkeypatch):
+    monkeypatch.delenv("OPENROUTER_VOTE_MODELS", raising=False)
+    ens = LLMEnsemble(openrouter_key="k")
+    models = ens._openrouter_vote_models("pro")
+    assert len(models) == 3 and len(set(models)) == 3
+    assert not any("deepseek" in m for m in models)

From cf386ee6417a794e35939c3c6874b8a5279bd7f5 Mon Sep 17 00:00:00 2001
From: Hardban Records Lab <hardbanrecordslab.pl@gmail.com>
Date: Mon, 21 Sep 2026 20:58:07 +0200
Subject: [PATCH 2/7] ci(deploy): stop the workflow from switching production
 to SQLite; add guards and rollback

On 2026-09-21 "Deploy to VPS" rewrote the VPS .env from GitHub Secrets (DATABASE_URL=sqlite:///...)
and then failed on `git pull`; production actually runs docker-compose.yml on PostgreSQL
(hbrl-postgres/metadata_engine), while the workflow deployed vps.docker-compose.yml, which pinned
DATABASE_URL to SQLite. A successful run would have started the app on an empty SQLite file.

* Deploy docker-compose.yml (the production file) instead of vps.docker-compose.yml.
* The VPS .env owns DATABASE_URL: the deploy refuses to run unless it already points at
  hbrl-postgres/metadata_engine, does not write the stale GitHub secret, carries the value over and
  re-checks the assembled file before swapping it in (.env.new -> .env).
* git: fetch + checkout main + merge --ff-only instead of `git pull` (fails instead of mixing commits).
* No `compose down`: a failed build no longer takes the service down. The running image is tagged
  music-metadata-engine:previous first.
* Health-gated: wait for `healthy`, verify the container's DATABASE_URL still points at Postgres,
  otherwise roll back to :previous and fail the run.
* concurrency group so two deploys never overlap.
* vps.docker-compose.yml becomes a deprecated alias (`include: docker-compose.yml`); its resolved
  config was checked to be identical to docker-compose.yml.

The script was exercised against stubbed docker/git/curl in 7 scenarios (happy path, SQLite in .env,
missing .env, container on SQLite, never healthy, no fast-forward, first DATABASE_URL line SQLite).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 .github/workflows/deploy.yml | 80 +++++++++++++++++++++++++-----------
 vps.docker-compose.yml       | 52 ++++-------------------
 2 files changed, 66 insertions(+), 66 deletions(-)

diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
index 9b853b7..19df60a 100644
--- a/.github/workflows/deploy.yml
+++ b/.github/workflows/deploy.yml
@@ -5,6 +5,11 @@ on:
     branches:
       - main
 
+# Never run two deploys at once (a second push while a build is running would race on the VPS).
+concurrency:
+  group: deploy-vps
+  cancel-in-progress: false
+
 jobs:
   # Fast gate — a broken push must not reach the VPS. Full test suites
   # (vitest, pytest) run separately in ci.yml and don't block the deploy.
@@ -54,24 +59,34 @@ jobs:
           # trimming the CUDA wheels from requirements.txt. Headroom, not tight.
           command_timeout: 30m
           script: |
-            set -e
+            set -euo pipefail
             cd /srv/hbrl/Metadata-Engine
             # Plain progress: BuildKit's default TTY renderer streams
             # carriage-return redraws that break the SSH action mid-build.
             export BUILDKIT_PROGRESS=plain
             export DOCKER_BUILDKIT=1
 
-            # Backup existing env if present (docker-compose reads root .env)
-            if [ -f .env ]; then
-              cp .env .env.backup.$(date +%F-%H%M%S)
-            fi
+            # Production compose file: Postgres (hbrl-postgres / metadata_engine) over the
+            # external hbrl-db network. vps.docker-compose.yml is only a deprecated alias now.
+            COMPOSE_FILE="docker-compose.yml"
+            DB_RE='^DATABASE_URL=postgres(ql)?(\+[a-z0-9]+)?://[^@[:space:]]+@hbrl-postgres:5432/metadata_engine'
+
+            # Guard 1 (fail closed, before anything is touched): the VPS .env OWNS DATABASE_URL and
+            # must already point at production Postgres. The GitHub secret of the same name is stale
+            # (SQLite) and is deliberately NOT written below - on 2026-09-21 it switched prod to SQLite.
+            test -f .env || { echo "FAIL: .env is missing on the VPS - refusing to deploy"; exit 1; }
+            OLD_DB_LINE="$(grep -m1 '^DATABASE_URL=' .env || true)"
+            printf '%s\n' "$OLD_DB_LINE" | grep -Eq "$DB_RE" || { echo "FAIL: DATABASE_URL in the VPS .env is not production Postgres (hbrl-postgres/metadata_engine) - refusing to deploy"; exit 1; }
+
+            # Backup existing env (docker-compose reads root .env)
+            cp -p .env ".env.backup.$(date +%F-%H%M%S)"
 
-            # Write root .env from GitHub Secrets (no secrets in repo)
+            # Write the env from GitHub Secrets into .env.new (no secrets in repo).
             # NOTE: every heredoc body line below must stay indented to match
             # this YAML block scalar (12 spaces) or the workflow file fails
             # to parse entirely (GitHub reports it as a run with zero jobs).
-            # The sed strips that indentation back out before it reaches .env.
-            cat <<'EOF' | sed -e 's/^[[:space:]]*//' > .env
+            # The sed strips that indentation back out before it reaches .env.new.
+            cat <<'EOF' | sed -e 's/^[[:space:]]*//' > .env.new
             GEMINI_API_KEY=${{ secrets.GEMINI_API_KEY }}
             GROQ_API_KEY=${{ secrets.GROQ_API_KEY }}
             MISTRAL_API_KEY=${{ secrets.MISTRAL_API_KEY }}
@@ -104,8 +119,6 @@ jobs:
             API_CLIENT_ID=${{ secrets.API_CLIENT_ID }}
             API_CLIENT_SECRET=${{ secrets.API_CLIENT_SECRET }}
 
-            DATABASE_URL=${{ secrets.DATABASE_URL }}
-
             APP_BASE_URL=https://app-metadata.hardbanrecordslab.online
             SMTP_HOST=${{ secrets.SMTP_HOST }}
             SMTP_PORT=${{ secrets.SMTP_PORT }}
@@ -115,18 +128,39 @@ jobs:
             SENTRY_DSN=${{ secrets.SENTRY_DSN }}
             EOF
 
-            # Pull latest code and rebuild containers
-            git pull origin main
-
-            # Use VPS compose file (binds 127.0.0.1:8888, mounts .env)
-            COMPOSE_FILE="vps.docker-compose.yml"
-            if command -v docker >/dev/null 2>&1 && docker compose version >/dev/null 2>&1; then
-              docker compose -f "$COMPOSE_FILE" down
-              docker compose -f "$COMPOSE_FILE" up -d --build
-            elif command -v docker-compose >/dev/null 2>&1; then
-              docker-compose -f "$COMPOSE_FILE" down
-              docker-compose -f "$COMPOSE_FILE" up -d --build
-            else
-              echo "Docker Compose not found on VPS" >&2
+            # Re-attach the production DATABASE_URL, re-check it, then swap the file in atomically.
+            printf '%s\n' "$OLD_DB_LINE" >> .env.new
+            grep -Eq "$DB_RE" .env.new || { echo "FAIL: assembled .env lost the Postgres DATABASE_URL - keeping the old .env"; rm -f .env.new; exit 1; }
+            chmod --reference=.env .env.new
+            mv .env.new .env
+
+            # Code: fast-forward only. Fails (instead of silently losing or mixing commits) if the
+            # VPS checkout ever diverges from origin/main again.
+            git fetch origin main
+            git checkout main
+            git merge --ff-only origin/main
+
+            # Keep the running image as a rollback target, then build + swap. No `compose down`:
+            # `up -d --build` builds first and only then replaces the container, so a failed build
+            # leaves the old container serving.
+            docker tag music-metadata-engine:latest music-metadata-engine:previous 2>/dev/null || true
+            docker compose -f "$COMPOSE_FILE" up -d --build
+
+            # Health-gated: container healthy AND still on production Postgres, else roll back.
+            healthy=0
+            status=unknown
+            for i in $(seq 1 30); do
+              status="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' metadata-backend 2>/dev/null || echo missing)"
+              if [ "$status" = "healthy" ]; then healthy=1; break; fi
+              sleep 10
+            done
+            CONTAINER_DB="$( (docker exec metadata-backend printenv DATABASE_URL 2>/dev/null | sed -E 's#(://[^:]+:)[^@]+@#\1***@#') || true )"
+            echo "container health: $status | DATABASE_URL: ${CONTAINER_DB:-<none>}"
+            if [ "$healthy" != "1" ] || ! printf '%s' "$CONTAINER_DB" | grep -Eq '@hbrl-postgres:5432/metadata_engine'; then
+              echo "FAIL: post-deploy verification failed - rolling back to the previous image"
+              docker tag music-metadata-engine:previous music-metadata-engine:latest
+              docker compose -f "$COMPOSE_FILE" up -d --no-build
               exit 1
             fi
+            curl -fsS http://127.0.0.1:8888/api/health/diagnose | grep -q '"llm"' || echo "WARN: /api/health/diagnose has no llm block"
+            echo "Deploy OK"
diff --git a/vps.docker-compose.yml b/vps.docker-compose.yml
index 2683808..cabf99a 100644
--- a/vps.docker-compose.yml
+++ b/vps.docker-compose.yml
@@ -1,43 +1,9 @@
-version: '3.8'
-
-services:
-  backend:
-    build:
-      context: .
-      dockerfile: Dockerfile
-    image: music-metadata-engine:latest
-    container_name: metadata-backend
-    restart: always
-    env_file:
-      - .env
-    environment:
-      # Intentionally pinned to SQLite: this overrides any DATABASE_URL set in
-      # .env (Docker Compose `environment` wins over `env_file` for the same
-      # key). Production data currently lives in this SQLite file. Do not
-      # change this to a Postgres URL without first migrating the data in
-      # /data/music_metadata.db — flipping this silently would make the app
-      # start against an empty/different database.
-      - DATABASE_URL=sqlite:////data/music_metadata.db
-    ports:
-      - "127.0.0.1:8888:7860"
-    volumes:
-      - ./data:/data
-      - ./logs:/home/user/app/logs
-      - ./temp_uploads:/home/user/app/temp_uploads
-      - ./.env:/home/user/app/.env
-    deploy:
-      resources:
-        limits:
-          cpus: '6.0'
-          memory: 10G
-    networks:
-      - hbrl-network
-    healthcheck:
-      test: [ "CMD", "curl", "-f", "http://localhost:7860/api/health/" ]
-      interval: 30s
-      timeout: 10s
-      retries: 3
-
-networks:
-  hbrl-network:
-    external: true
+# DEPRECATED (2026-09-21) - do not use directly. Production runs from docker-compose.yml
+# (Postgres hbrl-postgres / metadata_engine, over the external hbrl-db network).
+#
+# This file used to pin DATABASE_URL=sqlite:////data/music_metadata.db and mount ./.env. That pin
+# silently switched production back to an empty SQLite file when the "Deploy to VPS" workflow
+# ran on 2026-09-21. It is kept only as an alias so old commands and links keep working
+# without flipping the database; delete it once nothing references it.
+include:
+  - docker-compose.yml

From ce36ef396acd08258de07b504e8e4b69f04d8717 Mon Sep 17 00:00:00 2001
From: Hardban Records Lab <hardbanrecordslab.pl@gmail.com>
Date: Mon, 21 Sep 2026 20:58:07 +0200
Subject: [PATCH 3/7] ci(backup): back up the PostgreSQL database instead of a
 retired SQLite file; fix docs

backup.yml called the `sqlite3` CLI, which is not installed on the VPS: it failed every day
(2026-09-14 .. 2026-09-21), the backups directory is empty and no artifact was ever uploaded.
It also targeted data/music_metadata.db, which production no longer uses (PostgreSQL since 2026-09-17).

* pg_dump -Fc of metadata_engine inside hbrl-postgres -> data/backups/mme-pg-<stamp>.dump on the VPS,
  written as .partial, verified (size + pg_restore --list), then renamed; mode 600; keeps the last 14
  dumps of this job only.
* No GitHub artifact upload: the repository is public and a dump holds user data.
* README / ARCHITECTURE / OPERATIONS now describe the real setup (PostgreSQL, docker-compose.yml, guarded
  deploy with rollback, pg_dump backup + restore steps) instead of SQLite + vps.docker-compose.yml.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 .github/workflows/backup.yml | 58 +++++++++++++++------------
 README.md                    |  8 ++--
 docs/ARCHITECTURE.md         | 24 +++++++-----
 docs/OPERATIONS.md           | 76 ++++++++++++++++++++----------------
 4 files changed, 93 insertions(+), 73 deletions(-)

diff --git a/.github/workflows/backup.yml b/.github/workflows/backup.yml
index f59dcaa..8987d35 100644
--- a/.github/workflows/backup.yml
+++ b/.github/workflows/backup.yml
@@ -1,49 +1,55 @@
 name: Backup database
 
+# Production database = PostgreSQL `metadata_engine` in the shared `hbrl-postgres` container
+# (moved off SQLite on 2026-09-17). The previous version of this job snapshotted the retired
+# SQLite file with the `sqlite3` CLI, which is not installed on the VPS: it failed every day
+# (2026-09-14 .. 2026-09-21) and no backup was ever written.
+#
+# The dump stays ON THE VPS. It is deliberately NOT uploaded as a GitHub artifact: this repository
+# is public and a dump holds user data (emails, password hashes, credits). An off-site copy has to
+# go to private, encrypted storage (restic - see the HRL handbook, decision D-1).
+
 on:
   schedule:
     - cron: "0 3 * * *"   # daily 03:00 UTC
   workflow_dispatch:
 
+concurrency:
+  group: backup-db
+  cancel-in-progress: false
+
 jobs:
   backup:
     runs-on: ubuntu-latest
     steps:
-      - name: Snapshot + rotate on the VPS
+      - name: pg_dump on the VPS + verify + rotate
         uses: appleboy/ssh-action@v1.2.2
         with:
           host: ${{ secrets.VPS_HOST }}
           username: ${{ secrets.VPS_USER }}
           port: ${{ secrets.VPS_PORT }}
           key: ${{ secrets.VPS_SSH_KEY }}
-          command_timeout: 5m
+          command_timeout: 10m
           script: |
-            set -e
+            set -euo pipefail
             cd /srv/hbrl/Metadata-Engine
             mkdir -p data/backups
             STAMP=$(date -u +%Y%m%d-%H%M%S)
-            # consistent copy even while the app is running
-            sqlite3 data/music_metadata.db ".backup 'data/backups/mme-$STAMP.db'"
-            gzip -f "data/backups/mme-$STAMP.db"
-            # keep the 14 most recent
-            ls -1t data/backups/mme-*.db.gz | tail -n +15 | xargs -r rm --
-            ln -sf "mme-$STAMP.db.gz" data/backups/latest.db.gz
-            ls -lh data/backups/ | tail -5
+            OUT="data/backups/mme-pg-$STAMP.dump"
 
-      - name: Pull latest snapshot to the runner
-        uses: appleboy/scp-action@v0.1.7
-        with:
-          host: ${{ secrets.VPS_HOST }}
-          username: ${{ secrets.VPS_USER }}
-          port: ${{ secrets.VPS_PORT }}
-          key: ${{ secrets.VPS_SSH_KEY }}
-          source: /srv/hbrl/Metadata-Engine/data/backups/latest.db.gz
-          target: ./_backup
-          strip_components: 5
+            # Custom format (compressed, restorable with pg_restore). No -t on docker exec:
+            # a TTY would corrupt the binary stream. Written to a .partial file first.
+            docker exec hbrl-postgres pg_dump -U hbrl_admin -d metadata_engine -Fc > "$OUT.partial"
 
-      - name: Keep an off-site copy as an artifact (90-day retention)
-        uses: actions/upload-artifact@v4
-        with:
-          name: db-backup-${{ github.run_id }}
-          path: ./_backup/latest.db.gz
-          retention-days: 90
+            # Verify before it counts as a backup: non-trivial size and a readable table of contents
+            SIZE=$(stat -c %s "$OUT.partial")
+            if [ "$SIZE" -lt 10000 ]; then echo "FAIL: dump is only $SIZE bytes"; rm -f "$OUT.partial"; exit 1; fi
+            docker exec -i hbrl-postgres pg_restore --list < "$OUT.partial" | grep -q 'TABLE DATA' || { echo "FAIL: dump has no table data"; rm -f "$OUT.partial"; exit 1; }
+            mv "$OUT.partial" "$OUT"
+            chmod 600 "$OUT"
+            ln -sf "mme-pg-$STAMP.dump" data/backups/latest-pg.dump
+
+            # Rotation: keep the 14 most recent dumps of THIS job (mme-pg-*); nothing else is touched.
+            ls -1t data/backups/mme-pg-*.dump | tail -n +15 | xargs -r rm --
+            echo "backup OK: $OUT ($SIZE bytes)"
+            ls -lh data/backups/ | tail -6
diff --git a/README.md b/README.md
index d2513bf..670a31b 100644
--- a/README.md
+++ b/README.md
@@ -47,7 +47,7 @@ Full documentation lives in **[`docs/`](docs/README.md)**.
 
 | Layer | Tech |
 |---|---|
-| Backend | FastAPI (Python 3.10), SQLAlchemy, SQLite |
+| Backend | FastAPI (Python 3.10), SQLAlchemy, PostgreSQL (shared `hbrl-postgres`; SQLite only for local dev) |
 | Audio | Essentia, Librosa, openai-whisper (CPU), FFmpeg + Chromaprint |
 | AI | Groq, Google Gemini, OpenRouter |
 | Frontend | React 19, TypeScript, Vite, Tailwind CSS |
@@ -80,8 +80,10 @@ API docs: `http://localhost:8888/docs` (FastAPI Swagger).
 ## Deploy
 
 Push to `main` → GitHub Actions (`.github/workflows/deploy.yml`) SSHes to the
-VPS, writes `.env` from **GitHub repo Secrets**, and runs
-`docker compose -f vps.docker-compose.yml up -d --build`.
+VPS, refreshes `.env` from **GitHub repo Secrets** (except `DATABASE_URL`, which the
+VPS `.env` owns), fast-forwards the checkout and runs
+`docker compose -f docker-compose.yml up -d --build`, then verifies health and the
+Postgres connection and rolls back to the previous image if either check fails.
 
 Everything about env vars, secrets, the VPS layout and operational procedures
 is in **[`docs/OPERATIONS.md`](docs/OPERATIONS.md)**.
diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md
index 889ac58..f1a1306 100644
--- a/docs/ARCHITECTURE.md
+++ b/docs/ARCHITECTURE.md
@@ -10,7 +10,7 @@ Browser (React SPA, Vercel)                VPS (Docker)
   app-metadata.hardbanrecordslab.online      metadata.hardbanrecordslab.online
   ├─ essentia.js (WASM) — in-browser DSP     └─ Nginx ──► metadata-backend container
   ├─ jsmediatags — read existing tags             FastAPI (app.main:app), 1 worker
-  └─ /api/* ──proxied──► VPS                       ├─ SQLite  /data/music_metadata.db
+  └─ /api/* ──proxied──► VPS                       ├─ PostgreSQL  hbrl-postgres / metadata_engine
                                                    ├─ Essentia + Librosa (server DSP)
                                                    ├─ LLM ensemble (Groq/Gemini/OpenRouter)
                                                    ├─ WeasyPrint (certificate PDF)
@@ -18,7 +18,7 @@ Browser (React SPA, Vercel)                VPS (Docker)
 ```
 
 - One FastAPI container. No microservices, no worker cluster, no message
-  queue — the "job queue" is a `jobs` table in SQLite plus an in-process
+  queue — the "job queue" is a `jobs` table in the database plus an in-process
   async task.
 - The frontend is a **routerless SPA**: `index.tsx` renders `<App/>` for every
   path except `/verify/<id>` (certificate verification page). Vercel rewrites
@@ -47,7 +47,7 @@ Routers are mounted under both `/api` and `/auth` prefixes (`main.py`) — the
 `/auth`-prefixed copies exist for the Vercel proxy path; treat `/api/*` as
 canonical.
 
-## Data model (`app/db.py`, SQLite)
+## Data model (`app/db.py`, PostgreSQL in prod)
 
 `User`, `Job`, `CreditPurchase`, `RedeemCode`, `RedeemedCode`,
 `AnalysisHistory`, `Certificate`, `VerificationEvent`.
@@ -129,13 +129,17 @@ table (unique `stripe_session_id`) for idempotency. Redeem codes
 ## Deploy
 
 Push to `main` → `.github/workflows/deploy.yml`:
-`appleboy/ssh-action@v1.2.2` SSHes to the VPS, writes
-`/srv/hbrl/Metadata-Engine/.env` from **GitHub repo Secrets**, then
-`docker compose -f vps.docker-compose.yml up -d --build`
-(`BUILDKIT_PROGRESS=plain`, `command_timeout: 30m`). `vps.docker-compose.yml`
-pins `DATABASE_URL=sqlite:////data/music_metadata.db` regardless of what the
-`.env` says, and mounts `./data`, `./logs`, `./.env`. Full operational detail:
-[OPERATIONS.md](OPERATIONS.md).
+`appleboy/ssh-action@v1.2.2` SSHes to the VPS, refreshes
+`/srv/hbrl/Metadata-Engine/.env` from **GitHub repo Secrets** (never
+`DATABASE_URL` — the VPS `.env` owns it and the deploy refuses to run unless it
+points at `hbrl-postgres/metadata_engine`), fast-forwards the checkout, then
+`docker compose -f docker-compose.yml up -d --build`
+(`BUILDKIT_PROGRESS=plain`, `command_timeout: 30m`). The container joins the
+external `hbrl-db` network to reach Postgres. After the swap the job waits for
+`healthy`, re-checks the container's `DATABASE_URL` and rolls back to the
+`music-metadata-engine:previous` image on failure. `vps.docker-compose.yml` is a
+deprecated alias (`include: docker-compose.yml`) — it no longer pins SQLite.
+Full operational detail: [OPERATIONS.md](OPERATIONS.md).
 
 ## Notable constraints
 
diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md
index 7656551..bcd4231 100644
--- a/docs/OPERATIONS.md
+++ b/docs/OPERATIONS.md
@@ -9,18 +9,20 @@ Everything needed to run, deploy and maintain the Service.
 | Frontend (React SPA) | Vercel — `app-metadata.hardbanrecordslab.online` |
 | Backend (FastAPI) | VPS `84.247.162.167`, Docker container `metadata-backend`, host `127.0.0.1:8888` → container `7860` |
 | Public API | `metadata.hardbanrecordslab.online/api` (Nginx → the container) |
-| Database | **SQLite** at `/data/music_metadata.db` inside the container, bind-mounted from `/srv/hbrl/Metadata-Engine/data/` |
+| Database | **PostgreSQL** `metadata_engine` in the shared `hbrl-postgres` container (external Docker network `hbrl-db`); `DATABASE_URL` lives only in the VPS `.env` |
 | Repo path on VPS | `/srv/hbrl/Metadata-Engine` |
 
 Vercel's `frontend/vercel.json` proxies `/api/*` to the VPS and rewrites
 everything else to `index.html`.
 
 > The container also lives among other services on that box (WordPress,
-> AzuraCast, a CMLP platform, Vault, Postgres instances). Those Postgres
-> databases belong to other services — **this app uses SQLite**, despite any
-> Postgres URL a `.env` example might show. `vps.docker-compose.yml` pins
-> `DATABASE_URL=sqlite:////data/music_metadata.db` via `environment:`, which
-> overrides `env_file`.
+> AzuraCast, a CMLP platform, Vault, Postgres instances). This app uses the
+> `metadata_engine` database of the shared `hbrl-postgres` instance (migrated from
+> SQLite on 2026-09-17; the old `data/music_metadata.db` is a stale leftover).
+> Production runs from **`docker-compose.yml`**. The former
+> `vps.docker-compose.yml` pinned `DATABASE_URL` to SQLite, which switched
+> production to an empty database on 2026-09-21 — it is now only a deprecated
+> alias of `docker-compose.yml`.
 
 ## Deploy pipeline
 
@@ -28,14 +30,21 @@ everything else to `index.html`.
 
 1. `appleboy/ssh-action@v1.2.2` SSHes to the VPS (`VPS_HOST` / `VPS_USER` /
    `VPS_PORT` / `VPS_SSH_KEY` secrets).
-2. Backs up the current `.env`, then rewrites `/srv/hbrl/Metadata-Engine/.env`
-   from **GitHub repo Secrets** (heredoc — keep its 12-space indentation or
-   the workflow fails to parse).
-3. `git pull origin main`.
-4. `export BUILDKIT_PROGRESS=plain` (the old TTY progress renderer broke the
-   SSH action), then
-   `docker compose -f vps.docker-compose.yml up -d --build`
-   (`command_timeout: 30m`).
+2. **Guard:** refuses to run unless the VPS `.env` has a `DATABASE_URL` pointing
+   at `hbrl-postgres:5432/metadata_engine`.
+3. Backs up the current `.env`, then rewrites it from **GitHub repo Secrets**
+   (heredoc — keep its 12-space indentation or the workflow fails to parse).
+   `DATABASE_URL` is **not** taken from the secrets: the value from the old
+   `.env` is carried over and re-checked.
+4. `git fetch` + `git checkout main` + `git merge --ff-only origin/main` (fails
+   instead of mixing commits if the VPS checkout diverged).
+5. Tags the running image as `music-metadata-engine:previous`, then
+   `export BUILDKIT_PROGRESS=plain` (the old TTY progress renderer broke the
+   SSH action) and `docker compose -f docker-compose.yml up -d --build`
+   (`command_timeout: 30m`; no `down` — a failed build leaves the old container up).
+6. **Verify:** waits up to 5 min for `healthy` and checks that the container's
+   `DATABASE_URL` is still `hbrl-postgres/metadata_engine`; otherwise it restores
+   `music-metadata-engine:previous` and fails the run.
 
 Check runs at **GitHub → Actions → "Deploy to VPS"**. "Re-run jobs" replays
 the *old* workflow file — to pick up a workflow change, push a fresh commit.
@@ -44,8 +53,8 @@ Manual deploy (when the pipeline is down):
 ```bash
 ssh root@84.247.162.167
 cd /srv/hbrl/Metadata-Engine
-git pull origin main
-docker compose -f vps.docker-compose.yml up -d --build
+git fetch origin main && git checkout main && git merge --ff-only origin/main
+docker compose -f docker-compose.yml up -d --build
 ```
 
 ## Environment variables
@@ -55,7 +64,7 @@ Full annotated list: **`backend/.env.example`**. Summary:
 | Variable | Used by | Notes |
 |---|---|---|
 | `SECRET_KEY` | `security.py` | JWT signing — **app won't start without it**. `openssl rand -hex 32`. Also accepts `JWT_SECRET`. |
-| `DATABASE_URL` | `db.py` | Overridden to SQLite by compose in prod. |
+| `DATABASE_URL` | `db.py` | Prod: `postgresql://…@hbrl-postgres:5432/metadata_engine`, set **only** in the VPS `.env` (not from GitHub Secrets). Unset → local SQLite. |
 | `CORS_ORIGINS` | `config.py` | Explicit origin list. |
 | `GROQ_API_KEY` | ensemble, Whisper | Required for analysis. |
 | `GEMINI_API_KEY` | ensemble, proxy | Required for analysis. |
@@ -120,27 +129,26 @@ Idempotency: the webhook keys on `credit_purchases.stripe_session_id`.
 
 ## Backups
 
-The whole state is `/srv/hbrl/Metadata-Engine/data/` (the SQLite DB + uploads).
+State = the `metadata_engine` database in `hbrl-postgres` + `/srv/hbrl/Metadata-Engine/data/`
+(uploaded certificates, stale SQLite leftover).
 
 **Automated:** `.github/workflows/backup.yml` runs daily at 03:00 UTC (and on
-manual dispatch). It takes a consistent `sqlite3 .backup` snapshot into
-`data/backups/mme-<stamp>.db.gz` on the VPS (keeps the last 14), then pulls the
-latest to GitHub as a workflow **artifact** with 90-day retention — an
-off-site copy. It reuses the `VPS_*` secrets.
-
-**Ad-hoc full copy:**
-```bash
-ssh root@84.247.162.167 \
-  "tar czf /root/mme-backup-$(date +%F).tar.gz -C /srv/hbrl/Metadata-Engine data"
-```
-
-**Restore:** stop the container, replace `data/music_metadata.db` with a
-`gunzip`-ed snapshot, start it again.
+manual dispatch). It runs `pg_dump -Fc metadata_engine` inside `hbrl-postgres`,
+verifies the dump (size + `pg_restore --list`), stores it as
+`data/backups/mme-pg-<stamp>.dump` on the VPS (mode 600, keeps the last 14 of
+these, `latest-pg.dump` points at the newest). It is **not** uploaded to GitHub:
+the repo is public and the dump holds user data. There is no off-site copy yet
+(restic — handbook decision D-1). Before 2026-09-21 this job snapshotted the
+retired SQLite file with a `sqlite3` CLI that is not installed on the VPS, so it
+failed daily and no backup existed.
+
+**Restore** (try it on a scratch database first if unsure):
 ```bash
 cd /srv/hbrl/Metadata-Engine
-docker compose -f vps.docker-compose.yml down
-gunzip -c data/backups/latest.db.gz > data/music_metadata.db
-docker compose -f vps.docker-compose.yml up -d
+docker compose -f docker-compose.yml stop backend
+docker exec -i hbrl-postgres pg_restore -U hbrl_admin -d metadata_engine --clean --if-exists \
+  < data/backups/latest-pg.dump
+docker compose -f docker-compose.yml start backend
 ```
 
 The deploy also drops timestamped `.env.backup.*` files in the repo dir — prune

From cfb3cadfd65a17f59b3529f3f8fad5e4e51b1121 Mon Sep 17 00:00:00 2001
From: Hardban Records Lab <hardbanrecordslab.pl@gmail.com>
Date: Mon, 21 Sep 2026 22:20:01 +0200
Subject: [PATCH 4/7] ci: give the backend test job the env the app needs at
 import time

Backend CI has been red on main since 2026-09-02: certificate_pdf.py runs os.makedirs(CERT_DIR)
at import (default /data/certificates) and security.py raises without SECRET_KEY, so every test
module failed at collection with "PermissionError: /data". Point CERT_DIR and a throw-away SQLite
DATABASE_URL at the runner temp dir and derive a per-run SECRET_KEY.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 .github/workflows/ci.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 25b3e33..8f6d7fd 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -34,6 +34,13 @@ jobs:
       - name: Install system deps
         run: sudo apt-get update && sudo apt-get install -y ffmpeg libsndfile1
       - working-directory: backend
+        env:
+          # The app creates CERT_DIR at import time (default /data/certificates, not creatable on a
+          # runner) and refuses to start without SECRET_KEY; without these every test module failed
+          # at collection ("PermissionError: /data") - backend CI was red on main since 2026-09-02.
+          CERT_DIR: ${{ runner.temp }}/certificates
+          SECRET_KEY: ci-${{ github.run_id }}-${{ github.sha }}
+          DATABASE_URL: sqlite:///${{ runner.temp }}/ci.db
         run: |
           pip install -r requirements.txt
           pytest -q --no-header

From 81d53395312f81d1efefd1ca87a979b6e0abe6cc Mon Sep 17 00:00:00 2001
From: Hardban Records Lab <hardbanrecordslab.pl@gmail.com>
Date: Mon, 21 Sep 2026 22:23:16 +0200
Subject: [PATCH 5/7] test: put backend/ on sys.path via pytest.ini so plain
 `pytest` (as run in CI) can import `app`

CI runs `pytest -q` from backend/, which - unlike `python -m pytest` - does not add the working
directory to sys.path, so every test module failed with "No module named 'app'" (hidden behind the
/data PermissionError until the previous commit). `pythonpath = .` (pytest >= 7) fixes it.
Reproduced locally with the cwd removed from sys.path: 12x "No module named 'app'" before, none after.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 backend/pytest.ini | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/backend/pytest.ini b/backend/pytest.ini
index 9c25131..64db1fd 100644
--- a/backend/pytest.ini
+++ b/backend/pytest.ini
@@ -1,5 +1,7 @@
 [pytest]
 asyncio_mode = auto
+# plain `pytest` (as run in CI) does not put backend/ on sys.path, so `import app` failed
+pythonpath = .
 testpaths = tests app/routes
 python_files = test_*.py
 python_classes = Test*

From 112725851151da6c15c37c97ab78ff9e5e536bf7 Mon Sep 17 00:00:00 2001
From: Hardban Records Lab <hardbanrecordslab.pl@gmail.com>
Date: Mon, 21 Sep 2026 22:35:45 +0200
Subject: [PATCH 6/7] test: update four stale backend tests that CI could not
 reach until now

With CI able to import the app, four tests that had been failing unnoticed since before 2026-09-21
showed up. None is a production regression:

* test_analysis_limit (2 tests) asserted the pre-1be3f48 behaviour "tight budget => LLM skipped /
  ~5s window". Since 1be3f48 the LLM always gets a guaranteed window (LLM_MIN_WINDOW_SEC), so the
  tests now cover the real contract: a fast LLM answer is used even with no DSP budget left, and a
  slow LLM is cut off at the window with the DSP fallback and target_met == False.
* test_analysis_timeout expected job.error == "TIMEOUT_1s"; that code exists nowhere in the app or the
  frontend - process_analysis reports the exception type name for an empty message ("TimeoutError").
* test_ai_proxy::test_config_keys required a real Gemini/Groq key in the environment; it now skips
  when none is configured (CI).

Verified locally (libraries missing on the dev machine stubbed): 12 passed, 1 skipped.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 backend/app/routes/test_analysis_timeout.py |  4 +-
 backend/tests/test_ai_proxy.py              |  2 +
 backend/tests/test_analysis_limit.py        | 56 ++++++++++++---------
 3 files changed, 36 insertions(+), 26 deletions(-)

diff --git a/backend/app/routes/test_analysis_timeout.py b/backend/app/routes/test_analysis_timeout.py
index f6cb321..c736ff0 100644
--- a/backend/app/routes/test_analysis_timeout.py
+++ b/backend/app/routes/test_analysis_timeout.py
@@ -70,7 +70,9 @@ async def slow_analyze_fresh_track(*args, **kwargs):
         job = session.query(db.Job).filter(db.Job.id == "job-timeout").first()
         assert job is not None
         assert job.status == "error"
-        assert job.error == "TIMEOUT_1s"
+        # process_analysis reports str(e) or, when the message is empty (asyncio.TimeoutError), the
+        # exception type name; there was never a "TIMEOUT_<n>s" code in the app (nor in the frontend).
+        assert job.error == "TimeoutError"
     finally:
         session.close()
 
diff --git a/backend/tests/test_ai_proxy.py b/backend/tests/test_ai_proxy.py
index 9ba09cb..3ca57db 100644
--- a/backend/tests/test_ai_proxy.py
+++ b/backend/tests/test_ai_proxy.py
@@ -20,4 +20,6 @@ def test_ai_proxy_gemini_format():
 
 def test_config_keys():
     """Check if Gemini key is available in environment"""
+    if settings.GEMINI_API_KEY is None and settings.GROQ_API_KEY is None:
+        pytest.skip("no LLM API key configured in this environment (e.g. CI)")
     assert settings.GEMINI_API_KEY is not None or settings.GROQ_API_KEY is not None
diff --git a/backend/tests/test_analysis_limit.py b/backend/tests/test_analysis_limit.py
index 0115170..98154c1 100644
--- a/backend/tests/test_analysis_limit.py
+++ b/backend/tests/test_analysis_limit.py
@@ -1,14 +1,16 @@
 """Time-budget enforcement in FreshTrackAnalyzer.analyze_fresh_track().
 
-Was previously broken in two ways: (1) it called the real file hasher on a
-literal "test.mp3" path that doesn't exist, crashing before the mocked
-components were ever exercised; (2) its only assertion (target_met == True)
-assumed the analyzer always stays under budget, which contradicts the
-analyzer's own deliberately-relaxed behavior (see the "RELAXED LIMIT"
-comment in fresh_track_analyzer.py) of still giving the LLM a real ~5s
-window even when the remaining budget is tighter than that — so for some
-budgets it is *supposed* to overrun, and is supposed to report that
-honestly via target_met == False. This file now tests both cases.
+Contract (since the OpenRouter-first consensus, commit 1be3f48): the DSP layer can eat the whole time
+budget on a busy VPS, so the LLM step is ALWAYS given a guaranteed window
+(``LLM_MIN_WINDOW_SEC``, default 90 s, never less than 5 s) instead of being skipped - skipping it
+produced template-only "0 LLMs" results. Consequently:
+
+* a fast LLM answer is used even when the DSP layer left no budget, and
+* a slow/unresponsive LLM is cut off at the window and the offline DSP fallback is used; total time
+  then overruns ``time_budget`` and ``target_met`` honestly reports False.
+
+The two tests below used to assert the pre-1be3f48 behaviour (LLM skipped when the budget is tight),
+which is why they failed once CI could import the app at all.
 """
 import pytest
 from unittest.mock import patch
@@ -28,6 +30,12 @@ async def _slow_audio_analysis(*args, **kwargs):
     return dict(MOCK_AUDIO_FEATURES)
 
 
+async def _fast_llm_consensus(*args, **kwargs):
+    import asyncio
+    await asyncio.sleep(0.3)
+    return {'mainGenre': 'Test Genre', 'confidence': 0.9}
+
+
 async def _slow_llm_consensus(*args, **kwargs):
     import asyncio
     await asyncio.sleep(10)  # stands in for a slow/unresponsive LLM
@@ -35,15 +43,16 @@ async def _slow_llm_consensus(*args, **kwargs):
 
 
 @pytest.mark.asyncio
-async def test_tight_budget_skips_llm_and_stays_under_target():
-    """remaining < 3s after the DSP layer: Layer 2 is skipped outright, so
-    the analyzer finishes well under budget and target_met is True."""
+async def test_tight_budget_still_gives_the_llm_its_guaranteed_window(monkeypatch):
+    """DSP layer leaves ~2s of a 4s budget: the LLM is NOT skipped, its (fast) answer is used and the
+    whole analysis still finishes inside the budget."""
     import time
 
+    monkeypatch.setenv("LLM_MIN_WINDOW_SEC", "20")
     analyzer = FreshTrackAnalyzer()
     with patch('app.utils.hash_generator.generate_file_hash', return_value='0' * 64), \
          patch.object(analyzer.audio_analyzer, 'extract_all_features', side_effect=_slow_audio_analysis), \
-         patch.object(analyzer.llm_ensemble, 'consensus_classification', side_effect=_slow_llm_consensus):
+         patch.object(analyzer.llm_ensemble, 'consensus_classification', side_effect=_fast_llm_consensus):
 
         start = time.time()
         result = await analyzer.analyze_fresh_track("test.mp3", time_budget=4)
@@ -51,30 +60,27 @@ async def test_tight_budget_skips_llm_and_stays_under_target():
 
     assert duration < 4
     assert result['_tech_meta']['target_met'] is True
-    # The slow LLM must never have been awaited to completion — genre came
-    # from the offline DSP fallback, not the mocked 10s call.
-    assert result['mainGenre'] != 'Test Genre'
+    assert result['mainGenre'] == 'Test Genre'  # the LLM answer was used, not the DSP fallback
 
 
 @pytest.mark.asyncio
-async def test_moderate_budget_still_gives_llm_a_real_window_even_if_it_overruns():
-    """remaining >= 3s: the analyzer deliberately gives the LLM a ~5s
-    window rather than an unrealistically short one, even if that pushes
-    total time past time_budget. This is intentional (see the "RELAXED
-    LIMIT" comment) — the regression to guard is that target_met correctly
-    reports False when this happens, not that it never happens."""
+async def test_llm_slower_than_its_window_is_cut_off_and_falls_back_to_dsp(monkeypatch):
+    """The LLM window is bounded (here: the 5s floor): a 10s LLM is cut off, the offline DSP fallback
+    is used and, since that overruns the nominal budget, target_met correctly reports False."""
     import time
 
+    # window >= 3s (below that the analyzer skips the LLM outright), so llm_timeout = max(5, 4 - 1) = the 5s floor
+    monkeypatch.setenv("LLM_MIN_WINDOW_SEC", "4")
     analyzer = FreshTrackAnalyzer()
     with patch('app.utils.hash_generator.generate_file_hash', return_value='0' * 64), \
          patch.object(analyzer.audio_analyzer, 'extract_all_features', side_effect=_slow_audio_analysis), \
          patch.object(analyzer.llm_ensemble, 'consensus_classification', side_effect=_slow_llm_consensus):
 
         start = time.time()
-        result = await analyzer.analyze_fresh_track("test.mp3", time_budget=6)
+        result = await analyzer.analyze_fresh_track("test.mp3", time_budget=3)
         duration = time.time() - start
 
-    assert duration > 6  # overran the nominal budget, as designed
-    assert duration < 10  # but bounded by the ~5s LLM timeout, not the full 10s mock sleep
+    assert duration > 3   # overran the nominal budget, as designed
+    assert duration < 10  # but bounded by the LLM window, not the full 10s mock sleep
     assert result['_tech_meta']['target_met'] is False
     assert result['mainGenre'] != 'Test Genre'

From 6f5e7737642ec35c6dcc38e45dc3926547a7f14e Mon Sep 17 00:00:00 2001
From: Hardban Records Lab <hardbanrecordslab.pl@gmail.com>
Date: Mon, 21 Sep 2026 22:42:28 +0200
Subject: [PATCH 7/7] fix(llm): fall back to other Groq models on 429/404
 (per-model daily token quota)

Groq's free tier gives each model its own daily token quota; openai/gpt-oss-120b (200k tokens/day)
was exhausted within hours by the library batch, after which the description stream and the Groq
metadata merge failed with HTTP 429 for every track - so results would have been inconsistent
(some with the Groq merge, some without).

* groq_models.groq_create(): one wrapper around chat.completions.create that walks
  primary -> GROQ_MODEL_FALLBACKS (default qwen/qwen3.8-27b, openai/gpt-oss-20b). Only 429 (limit)
  and 404 (retired model) move on; every other error is raised unchanged. Per-model reasoning
  settings and token headroom are applied inside the wrapper.
* All five Groq call sites (description stream, _groq_classify, metadata merge, lyrics, generative
  proxy) use it.
* Unit tests with a fake client: order/dedup/env, 429 -> next model, 404 -> next, 500 raised,
  all exhausted -> last error.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 backend/app/routes/generative.py             | 10 ++-
 backend/app/services/fresh_track_analyzer.py | 10 ++-
 backend/app/services/groq_models.py          | 44 ++++++++++++-
 backend/app/services/groq_whisper.py         | 10 ++-
 backend/app/services/llm_ensemble.py         | 20 +++---
 backend/tests/test_groq_models.py            | 68 ++++++++++++++++++++
 6 files changed, 130 insertions(+), 32 deletions(-)

diff --git a/backend/app/routes/generative.py b/backend/app/routes/generative.py
index 9d6161d..0845948 100644
--- a/backend/app/routes/generative.py
+++ b/backend/app/routes/generative.py
@@ -10,7 +10,7 @@
 import httpx
 import asyncio
 from urllib.parse import quote
-from app.services.groq_models import groq_model, groq_extra, groq_budget
+from app.services.groq_models import groq_create
 
 from app.config import settings
 from app.dependencies import get_user_and_check_quota
@@ -40,10 +40,9 @@ async def call_groq_json(
     prompt: str, model_name: str = None
 ) -> Dict[str, Any]:
     try:
-        model_name = model_name or groq_model("pro")
         client = get_groq_client()
-        response = client.chat.completions.create(
-            model=model_name,
+        response = groq_create(
+            client, "pro", model=model_name,
             messages=[
                 {
                     "role": "system",
@@ -52,9 +51,8 @@ async def call_groq_json(
                 {"role": "user", "content": prompt},
             ],
             temperature=0.3,
-            max_tokens=groq_budget(model_name, 2000),
+            max_tokens=2000,
             response_format={"type": "json_object"},
-            **groq_extra(model_name),
         )
         
         result_text = response.choices[0].message.content
diff --git a/backend/app/services/fresh_track_analyzer.py b/backend/app/services/fresh_track_analyzer.py
index 92cc5e1..b7f232a 100644
--- a/backend/app/services/fresh_track_analyzer.py
+++ b/backend/app/services/fresh_track_analyzer.py
@@ -17,7 +17,7 @@
 
 from .deep_audio_analyzer import DeepAudioAnalyzer
 from .llm_ensemble import LLMEnsemble
-from .groq_models import groq_model, groq_extra, groq_budget
+from .groq_models import groq_create
 
 logger = logging.getLogger(__name__)
 
@@ -451,14 +451,12 @@ async def _analyze_lyrics_with_llm(self, lyrics: str) -> Dict:
   "genre_hints": ["genre based on lyrical style"]
 }}"""
             
-            lyrics_model = groq_model("pro")
             response = await asyncio.to_thread(
-                client.chat.completions.create,
-                model=lyrics_model,
+                groq_create,
+                client, "pro",
                 messages=[{"role": "user", "content": prompt}],
                 temperature=0.3,
-                max_tokens=groq_budget(lyrics_model, 300),
-                **groq_extra(lyrics_model),
+                max_tokens=300,
             )
             
             import json
diff --git a/backend/app/services/groq_models.py b/backend/app/services/groq_models.py
index 1dd3702..dcb2ab3 100644
--- a/backend/app/services/groq_models.py
+++ b/backend/app/services/groq_models.py
@@ -4,17 +4,27 @@
 ``model_not_found``, verified 2026-09-21 with GET /openai/v1/models), so every Groq call silently
 failed. Model names now live here and can be overridden without a code change:
 
-    GROQ_MODEL_PRO    (default openai/gpt-oss-120b)  - 'pro' mode, metadata merge, lyrics
-    GROQ_MODEL_FLASH  (default openai/gpt-oss-20b)   - 'flash' mode
+    GROQ_MODEL_PRO        (default openai/gpt-oss-120b)  - 'pro' mode, metadata merge, lyrics
+    GROQ_MODEL_FLASH      (default openai/gpt-oss-20b)   - 'flash' mode
+    GROQ_MODEL_FALLBACKS  (default qwen/qwen3.8-27b,openai/gpt-oss-20b) - tried in order when the
+                          primary model answers 429 (rate/daily-token limit) or 404 (retired)
+
+Groq's free tier has a per-MODEL daily token quota (200k tokens/day on gpt-oss-120b, hit within hours
+by the library batch on 2026-09-21), so every call goes through ``groq_create`` which walks the chain.
 
 gpt-oss models are reasoning models: reasoning tokens are counted against ``max_tokens`` and the
 answer comes back empty when the budget is too small, so ``groq_extra`` pins a low reasoning effort
 and ``groq_budget`` adds headroom on top of the visible-answer budget the call site asks for.
 """
+import logging
 import os
 
+logger = logging.getLogger(__name__)
+
 DEFAULT_PRO = "openai/gpt-oss-120b"
 DEFAULT_FLASH = "openai/gpt-oss-20b"
+DEFAULT_FALLBACKS = "qwen/qwen3.8-27b,openai/gpt-oss-20b"
+RETRYABLE_STATUS = (404, 429)
 REASONING_HEADROOM = 1500
 
 
@@ -37,3 +47,33 @@ def groq_extra(model: str) -> dict:
 def groq_budget(model: str, visible_tokens: int) -> int:
     """max_tokens to request so a reasoning model still has room for its visible answer."""
     return visible_tokens + REASONING_HEADROOM if _is_reasoning(model) else visible_tokens
+
+
+def groq_chain(model_preference: str = "pro", model: str = None) -> list:
+    """Primary model first (or the explicitly requested ``model``), then the fallbacks, de-duplicated."""
+    raw = os.getenv("GROQ_MODEL_FALLBACKS")
+    fallbacks = [m.strip() for m in (DEFAULT_FALLBACKS if raw is None else raw).split(",") if m.strip()]
+    chain = []
+    for m in [model or groq_model(model_preference)] + fallbacks:
+        if m not in chain:
+            chain.append(m)
+    return chain
+
+
+def groq_create(client, model_preference: str = "pro", *, max_tokens: int, model: str = None, **kwargs):
+    """``client.chat.completions.create`` with model fallback.
+
+    ``max_tokens`` is the visible-answer budget (reasoning headroom is added per model). Only a
+    rate/quota limit (429) or a retired model (404) moves on to the next model; anything else is raised
+    unchanged. With streaming the error surfaces from ``create`` itself, before the first token.
+    """
+    last = None
+    for name in groq_chain(model_preference, model):
+        try:
+            return client.chat.completions.create(model=name, max_tokens=groq_budget(name, max_tokens), **groq_extra(name), **kwargs)
+        except Exception as e:  # groq.RateLimitError / NotFoundError expose .status_code
+            if getattr(e, "status_code", None) not in RETRYABLE_STATUS:
+                raise
+            logger.warning("Groq model %s unavailable (HTTP %s) - trying the next model", name, getattr(e, "status_code", "?"))
+            last = e
+    raise last
diff --git a/backend/app/services/groq_whisper.py b/backend/app/services/groq_whisper.py
index f254811..08719bd 100644
--- a/backend/app/services/groq_whisper.py
+++ b/backend/app/services/groq_whisper.py
@@ -7,7 +7,7 @@
 import logging
 from typing import Dict, Any, Optional
 from app.config import settings
-from app.services.groq_models import groq_model, groq_extra, groq_budget
+from app.services.groq_models import groq_model, groq_create
 
 logger = logging.getLogger(__name__)
 
@@ -227,17 +227,15 @@ def safe_s(v):
 
         logger.info("Sending metadata prompt to Groq (length: %d)", len(prompt))
 
-        merge_model = groq_model("pro")
-        response = client.chat.completions.create(
-            model=merge_model,
+        response = groq_create(
+            client, "pro",
             messages=[
                 {"role": "system", "content": "You are a professional music metadata API. Always respond with valid JSON only. Never include markdown or explanations outside the JSON."},
                 {"role": "user", "content": prompt},
             ],
             temperature=0.25,
-            max_tokens=groq_budget(merge_model, 2500),
+            max_tokens=2500,
             response_format={"type": "json_object"},
-            **groq_extra(merge_model),
         )
 
         result_text = response.choices[0].message.content
diff --git a/backend/app/services/llm_ensemble.py b/backend/app/services/llm_ensemble.py
index 7f15586..941b192 100644
--- a/backend/app/services/llm_ensemble.py
+++ b/backend/app/services/llm_ensemble.py
@@ -15,7 +15,7 @@
 import logging
 import json
 from .standards import MAIN_GENRES, SUB_GENRES, MOODS, INSTRUMENTATION, VOCAL_STYLES
-from .groq_models import groq_model, groq_extra, groq_budget
+from .groq_models import groq_create
 
 logger = logging.getLogger(__name__)
 
@@ -99,7 +99,6 @@ async def stream_description(
                 return ""
 
             client = Groq(api_key=groq_key)
-            model = groq_model(model_preference)
 
             rhythm   = audio_features.get("rhythm",   {})
             harmonic = audio_features.get("harmonic", {})
@@ -136,13 +135,12 @@ async def stream_description(
 """
 
             accumulated = []
-            stream = client.chat.completions.create(
-                model=model,
+            stream = groq_create(
+                client, model_preference,
                 messages=[{"role": "user", "content": desc_prompt}],
                 temperature=0.6,
-                max_tokens=groq_budget(model, 600),
+                max_tokens=600,
                 stream=True,
-                **groq_extra(model),
             )
 
             for chunk in stream:
@@ -465,8 +463,7 @@ async def _groq_classify(self, context: str, system_prompt: str = None, retries:
         
         # Select model based on mode preference
         is_flash = model_preference == 'flash'
-        model_name = groq_model('flash' if is_flash else 'pro')
-        groq_max_tokens = groq_budget(model_name, 800 if is_flash else 1000)
+        groq_max_tokens = 800 if is_flash else 1000
         
         # INCREASE VARIETY: Use job_id as part of the seed or just increase temperature
         # Temperature 0.7 allows for creative variety while keeping structure
@@ -486,13 +483,12 @@ async def _groq_classify(self, context: str, system_prompt: str = None, retries:
         
         for attempt in range(retries):
             try:
-                response = client.chat.completions.create(
-                    model=model_name,
+                response = groq_create(
+                    client, 'flash' if is_flash else 'pro',
                     messages=messages,
                     temperature=temperature,
                     max_tokens=groq_max_tokens,
                     response_format={"type": "json_object"},
-                    **groq_extra(model_name),
                 )
 
                 
@@ -502,7 +498,7 @@ async def _groq_classify(self, context: str, system_prompt: str = None, retries:
                     
                 result = json.loads(content)
                 result['llm_source'] = 'groq'
-                result['_groq_model'] = model_name
+                result['_groq_model'] = getattr(response, 'model', None)
                 return result
             except Exception as e:
                 logger.warning(f"Groq attempt {attempt+1} failed: {e}")
diff --git a/backend/tests/test_groq_models.py b/backend/tests/test_groq_models.py
index e2d6396..95ec670 100644
--- a/backend/tests/test_groq_models.py
+++ b/backend/tests/test_groq_models.py
@@ -44,3 +44,71 @@ def test_default_vote_lineup_has_no_slow_deepseek(monkeypatch):
     models = ens._openrouter_vote_models("pro")
     assert len(models) == 3 and len(set(models)) == 3
     assert not any("deepseek" in m for m in models)
+
+
+# ---- fallback chain (Groq has per-model daily token quotas: 429 must move on to the next model) ----
+import pytest
+from app.services.groq_models import groq_chain, groq_create
+
+
+class _Err(Exception):
+    def __init__(self, status):
+        super().__init__("HTTP %s" % status)
+        self.status_code = status
+
+
+class _FakeClient:
+    """client.chat.completions.create(...) that fails with the given status for the listed models."""
+    def __init__(self, fail):
+        self.fail, self.calls = fail, []
+        outer = self
+
+        class _C:
+            @staticmethod
+            def create(**kw):
+                outer.calls.append(kw)
+                if kw["model"] in outer.fail:
+                    raise _Err(outer.fail[kw["model"]])
+                return {"model": kw["model"], "kw": kw}
+
+        class _Chat:
+            completions = _C
+
+        self.chat = _Chat
+
+
+def test_chain_order_dedup_and_env(monkeypatch):
+    monkeypatch.delenv("GROQ_MODEL_PRO", raising=False)
+    monkeypatch.delenv("GROQ_MODEL_FALLBACKS", raising=False)
+    assert groq_chain("pro") == ["openai/gpt-oss-120b", "qwen/qwen3.8-27b", "openai/gpt-oss-20b"]
+    assert groq_chain("flash") == ["openai/gpt-oss-20b", "qwen/qwen3.8-27b"]          # duplicate dropped
+    assert groq_chain("pro", model="x/y")[0] == "x/y"
+    monkeypatch.setenv("GROQ_MODEL_FALLBACKS", "")
+    assert groq_chain("pro") == ["openai/gpt-oss-120b"]                                  # empty = no fallbacks
+
+
+def test_429_moves_to_the_next_model_and_budget_is_per_model(monkeypatch):
+    monkeypatch.delenv("GROQ_MODEL_FALLBACKS", raising=False)
+    c = _FakeClient({"openai/gpt-oss-120b": 429})
+    out = groq_create(c, "pro", max_tokens=300, messages=[])
+    assert out["model"] == "qwen/qwen3.8-27b"
+    assert [k["model"] for k in c.calls] == ["openai/gpt-oss-120b", "qwen/qwen3.8-27b"]
+    assert c.calls[0]["max_tokens"] > 300 and c.calls[0]["extra_body"] == {"reasoning_effort": "low"}   # gpt-oss: headroom + low effort
+    assert c.calls[1]["max_tokens"] == 300 and "extra_body" not in c.calls[1]                            # qwen: as asked
+
+
+def test_404_also_falls_through_and_other_errors_are_raised(monkeypatch):
+    monkeypatch.delenv("GROQ_MODEL_FALLBACKS", raising=False)
+    c = _FakeClient({"openai/gpt-oss-120b": 404, "qwen/qwen3.8-27b": 429})
+    assert groq_create(c, "pro", max_tokens=10, messages=[])["model"] == "openai/gpt-oss-20b"
+    with pytest.raises(_Err) as ei:
+        groq_create(_FakeClient({"openai/gpt-oss-120b": 500}), "pro", max_tokens=10, messages=[])
+    assert ei.value.status_code == 500                                                                   # 500 is not swallowed
+
+
+def test_all_models_exhausted_raises_the_last_error(monkeypatch):
+    monkeypatch.delenv("GROQ_MODEL_FALLBACKS", raising=False)
+    c = _FakeClient({m: 429 for m in ("openai/gpt-oss-120b", "qwen/qwen3.8-27b", "openai/gpt-oss-20b")})
+    with pytest.raises(_Err) as ei:
+        groq_create(c, "pro", max_tokens=10, messages=[])
+    assert ei.value.status_code == 429 and len(c.calls) == 3