Add LLM translation eval suite and page cost tooling

.gitignore

@@ -17,3 +17,8 @@ apps/guide/settings/local.py
 .coverage
 .ruff_cache
 .idea/
+
+# Inspect AI eval logs (browse with `just eval-view`)
+prompts/evals/translations-inspect_ai/logs/
+# Generated glossaries (regenerate with `just eval-glossary <lang>`)
+prompts/evals/translations-inspect_ai/glossary/

apps/core/translator.py

@@ -1,9 +1,15 @@
-from django.utils.translation import get_language_info
+import re
+from html import unescape
+
+from django.utils.translation import get_language_info, trans_real
 from django.utils.translation import gettext_lazy as _
 from wagtail_ai.agents.base import get_llm_service as _get_llm_service
 from wagtail_localize.machine_translators.base import BaseMachineTranslator
 from wagtail_localize.strings import StringValue
 
+# Text inside <b>/<i> tags — most often a Wagtail admin UI label.
+UI_LABEL_RE = re.compile(r"<(b|i)\b[^>]*>(.*?)</\1>", re.IGNORECASE | re.DOTALL)
+
 
 class LLMTranslator(BaseMachineTranslator):
     display_name = _("LLM Translator")
@@ -22,6 +28,13 @@ class LLMTranslator(BaseMachineTranslator):
         "attributes.\n"
         "- Preserve any inline tags, whitespace, and punctuation exactly."
     )
+    default_glossary_prompt = (
+        "\n- Text inside <b> or <i> tags is usually a label from the Wagtail "
+        "admin interface. When a term appears in the glossary below, use the "
+        "official translation exactly as given; otherwise translate it "
+        "naturally and consistently.\n"
+        "Glossary (official Wagtail admin translations):\n{glossary}"
+    )
 
     def __init__(self, options):
         super().__init__(options)
@@ -37,15 +50,52 @@ def get_prompt_context(self, source_locale, target_locale):
             "target_language": get_language_info(target_locale.language_code)["name"],
         }
 
-    def get_system_prompt(self, source_locale, target_locale):
+    def get_ui_terms(self, strings):
+        """Unique <b>/<i> label texts across the strings, in order."""
+        terms = []
+        for string in strings:
+            for _tag, text in UI_LABEL_RE.findall(string.get_translatable_html()):
+                term = unescape(re.sub(r"<[^>]+>", "", text)).strip()
+                if term and term not in terms:
+                    terms.append(term)
+        return terms
+
+    def get_glossary(self, target_locale, terms):
+        """Map each term to its official admin translation, where one exists.
+
+        Looks the terms up in Django's merged translation catalog for the
+        target language, which includes Wagtail's own admin translations.
+        """
+        catalog = trans_real.translation(target_locale.language_code)._catalog
+        glossary = {}
+        for term in terms:
+            translated = catalog.get(term)
+            if isinstance(translated, str) and translated.strip():
+                glossary[term] = translated
+        return glossary
+
+    def get_system_prompt(self, source_locale, target_locale, glossary=None):
         template = self.options.get("SYSTEM_PROMPT", self.default_system_prompt)
-        return template.format(**self.get_prompt_context(source_locale, target_locale))
+        prompt = template.format(
+            **self.get_prompt_context(source_locale, target_locale)
+        )
+        if glossary:
+            glossary_template = self.options.get(
+                "GLOSSARY_PROMPT", self.default_glossary_prompt
+            )
+            lines = "\n".join(
+                f"- {source} = {translated}" for source, translated in glossary.items()
+            )
+            prompt += glossary_template.format(glossary=lines)
+        return prompt
 
-    def get_messages(self, source_locale, target_locale, string):
+    def get_messages(self, source_locale, target_locale, string, glossary=None):
         return [
             {
                 "role": "system",
-                "content": self.get_system_prompt(source_locale, target_locale),
+                "content": self.get_system_prompt(
+                    source_locale, target_locale, glossary=glossary
+                ),
             },
             {
                 "role": "user",
@@ -61,9 +111,12 @@ def get_llm_service(self, alias):
 
     def translate(self, source_locale, target_locale, strings):
         llm_service = self.get_llm_service(self.provider_alias)
+        glossary = self.get_glossary(target_locale, self.get_ui_terms(strings))
         results = {}
         for string in strings:
-            messages = self.get_messages(source_locale, target_locale, string)
+            messages = self.get_messages(
+                source_locale, target_locale, string, glossary=glossary
+            )
             response = llm_service.completion(
                 messages=messages, temperature=self.temperature
             )

justfile

@@ -3,6 +3,8 @@
 
 # Load environment variables from a `.env` file, if present.
 set dotenv-load
+# Pass recipe arguments through as "$@" so quoted arguments survive.
+set positional-arguments
 
 # List all the justfile recipes.
 help:
@@ -73,6 +75,26 @@ makemessages:
 compilemessages:
     cd apps && uv run python ../manage.py compilemessages
 
+# Evaluate translation quality of candidate LLMs with Inspect AI (Scaleway).
+eval-translations *ARGS:
+    ./prompts/evals/translations-inspect_ai/translation_task.py "$@"
+
+# Export Wagtail admin UI translations as eval glossaries, per language code.
+eval-glossary *LANGS="ar":
+    uv run python prompts/evals/translations-inspect_ai/export_glossary.py {{ LANGS }}
+
+# Run the Promptfoo translation eval across all candidate models (Scaleway).
+eval-promptfoo *ARGS:
+    npx --yes promptfoo@latest eval -c prompts/evals/translations/translations.yaml "$@"
+
+# Browse Promptfoo translation eval results.
+eval-promptfoo-view:
+    npx --yes promptfoo@latest view --yes
+
+# Browse translation eval results in the Inspect viewer.
+eval-view:
+    uvx --from inspect-ai --python 3.12 inspect view --log-dir prompts/evals/translations-inspect_ai/logs
+
 # Build the Docker image.
 docker-build:
     docker compose build

prompts/evals/translations-inspect_ai/README.md

@@ -0,0 +1,90 @@
+# Translation eval with Inspect AI
+
+Prototype replacement for the custom eval suite in `scripts/eval/`, built on
+[Inspect AI](https://inspect.aisi.org.uk/). Inspect provides the runner
+(parallelism, retries, rate limiting), multi-model comparison, structured eval
+logs, and a results UI — the only project code is `translation_task.py`:
+the task definition, a stdlib-only rule scorer, and an LLM-judge scorer.
+
+It evaluates the translator behaviour defined in `apps/core/translator.py`
+(same system prompt, glossary injection, temperature 0) without needing Django
+or the database.
+
+## Setup
+
+The script is self-contained: a [PEP 723](https://peps.python.org/pep-0723/)
+inline-metadata script with a `uv run` shebang, so `uv` resolves `inspect-ai`
+and `openai` on first run — nothing to install beyond `uv`.
+
+Candidate models and the judge are declared at the top of
+`translation_task.py` (`CANDIDATE_MODELS`, `JUDGE_MODEL`). Scaleway's
+Generative APIs are OpenAI Chat Completions compatible and addressed via
+Inspect's `openai-api/scaleway/<model>` naming; credentials are picked up from
+the standard `SCW_SECRET_KEY` and `SCW_DEFAULT_PROJECT_ID` environment
+variables (e.g. from `.env`, which `just` loads automatically).
+
+## UI label glossary
+
+UI labels in `<b>`/`<i>` tags must reuse the official Wagtail admin interface
+translations — e.g. Icelandic `Collection` = `Safn` — so guide content and the
+interface always use the same terms. `apps/core/translator.py` reads these
+live from Django's merged translation catalog (which includes Wagtail's own
+`.po` files) and injects a per-batch glossary into the system prompt. The eval
+mirrors this exactly, from a JSON export of the same catalog:
+
+```sh
+just eval-glossary ar        # writes prompts/evals/translations-inspect_ai/glossary/ar.json
+just eval-glossary ar is fr  # one file per language code
+```
+
+Labels without an official translation are translated freely.
+
+## Run
+
+```sh
+just eval-translations                          # all candidates, Arabic
+just eval-translations --limit 2                # quick smoke run
+just eval-translations --target-language Icelandic --lang-code is
+just eval-translations --models openai-api/scaleway/qwen3.6-35b-a3b
+just eval-translations --judge openai-api/scaleway/mistral-medium-3.5-128b
+```
+
+Then browse per-sample transcripts, scores, and token usage:
+
+```sh
+just eval-view
+```
+
+Logs land in `prompts/evals/translations-inspect_ai/logs/` (gitignored; `.eval` files are also
+readable programmatically with `inspect_ai.log.read_eval_log`).
+
+## What's scored
+
+-   `rule_checks` — deterministic, stdlib `html.parser` only: HTML tag/attribute
+    structure, glossary compliance for `<b>`/`<i>` labels (position-wise, exact
+    official translation), no truncation, no reasoning bleed. Reported per rule
+    (accuracy ± stderr).
+-   `model_graded_qa` — LLM judge (`glm-5.2` by default) grading
+    accuracy/fluency/rule compliance as C/P/I with partial credit. The judge
+    prompt includes the same per-segment glossary (via sample metadata). The
+    judge is deliberately not one of the candidates, to avoid self-preference.
+
+## Notes / differences from the custom suite
+
+-   The old "keep `<b>`/`<i>` text in English" rule is replaced by glossary
+    compliance, matching the translator's actual behaviour.
+-   Grading is per-candidate (C/P/I) rather than pairwise A/B/C ranking. Inspect
+    shuffles nothing because there is no position to bias; comparison happens
+    across eval runs in the viewer. Pairwise ranking is possible but needs a
+    custom scorer again — start without it.
+-   Rules that don't apply to a sample (e.g. no glossary term in the source)
+    count as a pass rather than being excluded from the denominator.
+-   The dataset is `prompts/evals/translations/segments.yaml`, shared with the
+    Promptfoo eval: segments extracted verbatim from real guide content
+    (`prompts/content/en/how-to-guides/manage-documents.md` and
+    `prompts/content/en/releases/new-in-wagtail-7-4.md`), in the HTML form the
+    translator receives from wagtail-localize. Each entry is a Promptfoo test
+    case (`description`, `vars.text`, `metadata.id`/`metadata.source`);
+    `translation_task.py` maps them to Inspect Samples. Regenerate from live
+    page content with `export_segments.py` (see its docstring), or grow the
+    file as regressions are found.

prompts/evals/translations-inspect_ai/export_glossary.py

@@ -0,0 +1,49 @@
+"""
+Export merged Django/Wagtail translation catalogs as glossary JSON files.
+
+The output is the same catalog `apps/core/translator.py` reads live via
+Django's translation machinery — this build step just makes it available to
+the Inspect eval, which runs outside the project environment.
+
+Usage (from the project root, one file per language code):
+
+    uv run python prompts/evals/translations-inspect_ai/export_glossary.py ar is fr
+    # or: just eval-glossary ar is fr
+
+Output: prompts/evals/translations-inspect_ai/glossary/<lang>.json  ({"Collection": "Safn", ...})
+"""
+
+import json
+import os
+import sys
+from pathlib import Path
+
+# Make the project root importable when run by path.
+sys.path.insert(0, str(Path(__file__).parents[3]))
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "apps.guide.settings.test")
+
+import django
+
+django.setup()
+
+from django.utils.translation import trans_real
+
+OUT_DIR = Path(__file__).parent / "glossary"
+OUT_DIR.mkdir(exist_ok=True)
+
+for lang in sys.argv[1:] or ["ar"]:
+    catalog = trans_real.translation(lang)._catalog
+    entries = {}
+    for key in catalog.keys():
+        # Skip plural entries (tuple keys) and contextual entries (ctx\x04msgid).
+        if not isinstance(key, str) or "\x04" in key:
+            continue
+        value = catalog.get(key)
+        if isinstance(value, str) and value.strip():
+            entries[key] = value
+    path = OUT_DIR / f"{lang}.json"
+    path.write_text(
+        json.dumps(entries, ensure_ascii=False, indent=1, sort_keys=True) + "\n",
+        encoding="utf-8",
+    )
+    print(f"{path}: {len(entries)} entries")

prompts/evals/translations-inspect_ai/export_segments.py

@@ -0,0 +1,50 @@
+"""
+Export real translatable segments in the shared eval dataset format.
+
+Optional — a hand-picked prompts/evals/translations/segments.yaml (extracted
+from real guide content) is checked in. Use this to regenerate it from actual
+page content (the output is a JSON array, which is valid YAML):
+
+    docker compose exec -T web python manage.py shell -c \
+        "exec(open('prompts/evals/translations-inspect_ai/export_segments.py').read())" \
+        > prompts/evals/translations/segments.yaml
+"""
+
+import json
+import os
+
+from wagtail.models import Locale, Page
+from wagtail_localize.models import TranslationSource
+
+PAGE_IDS = [int(p) for p in os.environ.get("EVAL_PAGE_IDS", "10,66").split(",")]
+
+cases = []
+en = Locale.objects.get(language_code="en")
+for pid in PAGE_IDS:
+    page = Page.objects.filter(id=pid, locale=en).first() or (
+        Page.objects.filter(
+            translation_key=Page.objects.get(id=pid).translation_key, locale=en
+        ).first()
+    )
+    if not page:
+        continue
+    source = TranslationSource.objects.filter(
+        object_id=page.translation_key, locale=en
+    ).first()
+    if not source:
+        continue
+    for seg in source.stringsegment_set.select_related("string", "context").order_by(
+        "order"
+    ):
+        cases.append(
+            {
+                "description": f"{page.title} — {seg.context.path if seg.context_id else 'segment'}",
+                "metadata": {
+                    "id": f"page{page.id}-{seg.string_id}",
+                    "source": f"page {page.id}: {page.title}",
+                },
+                "vars": {"text": seg.string.as_value().get_translatable_html()},
+            }
+        )
+
+print(json.dumps(cases, indent=2, ensure_ascii=False))