feat: deepeval offline suite

Author: inesaranab

Dockerfile

@@ -17,7 +17,12 @@ COPY app ./app
 RUN useradd -m screening-user
 USER screening-user
 
-RUN python -c "from app.adapters.guard_classifier import ClassifierGuardrail; ClassifierGuardrail()"
+# Presidio recognizers (incl. GLiNER) lazy-load on first `.analyze()` call, not
+# at construction, so building the guardrail alone doesn't fetch GLiNER's
+# weights. Run an actual scrub so every model — spaCy, GLiNER, the injection
+# classifier — is downloaded and cached into the image before HF_HUB_OFFLINE
+# is set below; otherwise the first real request fails offline.
+RUN python -c "import asyncio; from app.adapters.guard_classifier import ClassifierGuardrail; asyncio.run(ClassifierGuardrail().scrub('warmup'))"
 EXPOSE 8000
 
 ENV HF_HUB_OFFLINE=1

README.md

@@ -61,7 +61,7 @@ The **injection** (*manipulation* — text that tries to hijack the model's inst
 ## Run the evals
 
 ```bash
-uv run pytest -m "not live and not prod"   # deterministic — no model, no network. Run these in CI.
+uv run pytest -m "not live and not prod and not quality"   # deterministic — no model, no network. Run these in CI.
 uv run pytest -m live                      # hits the real Presidio + GLiNER + injection classifier (+ a live LLM)
 uv run pytest -m prod --run-prod           # hits the deployed prod endpoint (needs az login; opt-in on purpose)
 

evals/metrics.py

@@ -13,18 +13,24 @@
 
 from evals.judge import PortkeyJudge
 
+# One judge shared by every metric below. PortkeyJudge already caches its HTTP
+# clients per instance (see evals/judge.py) specifically so they're built once,
+# not once per call — instantiating a separate PortkeyJudge per metric would
+# undercut that by opening a separate client pool per metric instead.
+_JUDGE = PortkeyJudge()
+
 # truthfulness; truthful claims / total claims
 # a claim = factual assertion (checked against transcript for truth)
-FAITHFULNESS = FaithfulnessMetric(threshold=1.0, model=PortkeyJudge(), async_mode=False)
+FAITHFULNESS = FaithfulnessMetric(threshold=1.0, model=_JUDGE, async_mode=False)
 
 # on-topic-ness: relevant statements / total statements
-RELEVANCY = AnswerRelevancyMetric(threshold=0.7, model=PortkeyJudge(), async_mode=False)
+RELEVANCY = AnswerRelevancyMetric(threshold=0.7, model=_JUDGE, async_mode=False)
 
 # prejudice
-BIAS = BiasMetric(threshold=0.0, model=PortkeyJudge(), async_mode=False)
+BIAS = BiasMetric(threshold=0.0, model=_JUDGE, async_mode=False)
 
 # judge splits the output into statements scores each one, 1 means nothing leaked
-PII = PIILeakageMetric(threshold=1.0, model=PortkeyJudge(), async_mode=False)
+PII = PIILeakageMetric(threshold=1.0, model=_JUDGE, async_mode=False)
 
 
 # Deep Acyclic Graph: decision tree to write.
@@ -55,7 +61,7 @@
     # the `_used_to_justify` follow-up decision-dead; 0.5 is what makes it mean
     # something — merely mentioning passes, justifying the score with it fails.
     threshold=0.5,
-    model=PortkeyJudge(),
+    model=_JUDGE,
     async_mode=False,
 )