diff --git a/.gitignore b/.gitignore index f6f8f9e9..12140ded 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,8 @@ apps/guide/settings/local.py .coverage .ruff_cache .idea/ + +# Inspect AI eval logs (browse with `just eval-view`) +prompts/evals/translations-inspect_ai/logs/ +# Generated glossaries (regenerate with `just eval-glossary `) +prompts/evals/translations-inspect_ai/glossary/ diff --git a/apps/core/translator.py b/apps/core/translator.py index c666e04c..b6686c33 100644 --- a/apps/core/translator.py +++ b/apps/core/translator.py @@ -1,9 +1,15 @@ -from django.utils.translation import get_language_info +import re +from html import unescape + +from django.utils.translation import get_language_info, trans_real from django.utils.translation import gettext_lazy as _ from wagtail_ai.agents.base import get_llm_service as _get_llm_service from wagtail_localize.machine_translators.base import BaseMachineTranslator from wagtail_localize.strings import StringValue +# Text inside / tags — most often a Wagtail admin UI label. +UI_LABEL_RE = re.compile(r"<(b|i)\b[^>]*>(.*?)", re.IGNORECASE | re.DOTALL) + class LLMTranslator(BaseMachineTranslator): display_name = _("LLM Translator") @@ -22,6 +28,13 @@ class LLMTranslator(BaseMachineTranslator): "attributes.\n" "- Preserve any inline tags, whitespace, and punctuation exactly." ) + default_glossary_prompt = ( + "\n- Text inside or tags is usually a label from the Wagtail " + "admin interface. When a term appears in the glossary below, use the " + "official translation exactly as given; otherwise translate it " + "naturally and consistently.\n" + "Glossary (official Wagtail admin translations):\n{glossary}" + ) def __init__(self, options): super().__init__(options) @@ -37,15 +50,52 @@ def get_prompt_context(self, source_locale, target_locale): "target_language": get_language_info(target_locale.language_code)["name"], } - def get_system_prompt(self, source_locale, target_locale): + def get_ui_terms(self, strings): + """Unique / label texts across the strings, in order.""" + terms = [] + for string in strings: + for _tag, text in UI_LABEL_RE.findall(string.get_translatable_html()): + term = unescape(re.sub(r"<[^>]+>", "", text)).strip() + if term and term not in terms: + terms.append(term) + return terms + + def get_glossary(self, target_locale, terms): + """Map each term to its official admin translation, where one exists. + + Looks the terms up in Django's merged translation catalog for the + target language, which includes Wagtail's own admin translations. + """ + catalog = trans_real.translation(target_locale.language_code)._catalog + glossary = {} + for term in terms: + translated = catalog.get(term) + if isinstance(translated, str) and translated.strip(): + glossary[term] = translated + return glossary + + def get_system_prompt(self, source_locale, target_locale, glossary=None): template = self.options.get("SYSTEM_PROMPT", self.default_system_prompt) - return template.format(**self.get_prompt_context(source_locale, target_locale)) + prompt = template.format( + **self.get_prompt_context(source_locale, target_locale) + ) + if glossary: + glossary_template = self.options.get( + "GLOSSARY_PROMPT", self.default_glossary_prompt + ) + lines = "\n".join( + f"- {source} = {translated}" for source, translated in glossary.items() + ) + prompt += glossary_template.format(glossary=lines) + return prompt - def get_messages(self, source_locale, target_locale, string): + def get_messages(self, source_locale, target_locale, string, glossary=None): return [ { "role": "system", - "content": self.get_system_prompt(source_locale, target_locale), + "content": self.get_system_prompt( + source_locale, target_locale, glossary=glossary + ), }, { "role": "user", @@ -61,9 +111,12 @@ def get_llm_service(self, alias): def translate(self, source_locale, target_locale, strings): llm_service = self.get_llm_service(self.provider_alias) + glossary = self.get_glossary(target_locale, self.get_ui_terms(strings)) results = {} for string in strings: - messages = self.get_messages(source_locale, target_locale, string) + messages = self.get_messages( + source_locale, target_locale, string, glossary=glossary + ) response = llm_service.completion( messages=messages, temperature=self.temperature ) diff --git a/justfile b/justfile index 182dd3c3..99afb127 100644 --- a/justfile +++ b/justfile @@ -3,6 +3,8 @@ # Load environment variables from a `.env` file, if present. set dotenv-load +# Pass recipe arguments through as "$@" so quoted arguments survive. +set positional-arguments # List all the justfile recipes. help: @@ -73,6 +75,26 @@ makemessages: compilemessages: cd apps && uv run python ../manage.py compilemessages +# Evaluate translation quality of candidate LLMs with Inspect AI (Scaleway). +eval-translations *ARGS: + ./prompts/evals/translations-inspect_ai/translation_task.py "$@" + +# Export Wagtail admin UI translations as eval glossaries, per language code. +eval-glossary *LANGS="ar": + uv run python prompts/evals/translations-inspect_ai/export_glossary.py {{ LANGS }} + +# Run the Promptfoo translation eval across all candidate models (Scaleway). +eval-promptfoo *ARGS: + npx --yes promptfoo@latest eval -c prompts/evals/translations/translations.yaml "$@" + +# Browse Promptfoo translation eval results. +eval-promptfoo-view: + npx --yes promptfoo@latest view --yes + +# Browse translation eval results in the Inspect viewer. +eval-view: + uvx --from inspect-ai --python 3.12 inspect view --log-dir prompts/evals/translations-inspect_ai/logs + # Build the Docker image. docker-build: docker compose build diff --git a/prompts/evals/translations-inspect_ai/README.md b/prompts/evals/translations-inspect_ai/README.md new file mode 100644 index 00000000..8aac2791 --- /dev/null +++ b/prompts/evals/translations-inspect_ai/README.md @@ -0,0 +1,90 @@ +# Translation eval with Inspect AI + +Prototype replacement for the custom eval suite in `scripts/eval/`, built on +[Inspect AI](https://inspect.aisi.org.uk/). Inspect provides the runner +(parallelism, retries, rate limiting), multi-model comparison, structured eval +logs, and a results UI — the only project code is `translation_task.py`: +the task definition, a stdlib-only rule scorer, and an LLM-judge scorer. + +It evaluates the translator behaviour defined in `apps/core/translator.py` +(same system prompt, glossary injection, temperature 0) without needing Django +or the database. + +## Setup + +The script is self-contained: a [PEP 723](https://peps.python.org/pep-0723/) +inline-metadata script with a `uv run` shebang, so `uv` resolves `inspect-ai` +and `openai` on first run — nothing to install beyond `uv`. + +Candidate models and the judge are declared at the top of +`translation_task.py` (`CANDIDATE_MODELS`, `JUDGE_MODEL`). Scaleway's +Generative APIs are OpenAI Chat Completions compatible and addressed via +Inspect's `openai-api/scaleway/` naming; credentials are picked up from +the standard `SCW_SECRET_KEY` and `SCW_DEFAULT_PROJECT_ID` environment +variables (e.g. from `.env`, which `just` loads automatically). + +## UI label glossary + +UI labels in ``/`` tags must reuse the official Wagtail admin interface +translations — e.g. Icelandic `Collection` = `Safn` — so guide content and the +interface always use the same terms. `apps/core/translator.py` reads these +live from Django's merged translation catalog (which includes Wagtail's own +`.po` files) and injects a per-batch glossary into the system prompt. The eval +mirrors this exactly, from a JSON export of the same catalog: + +```sh +just eval-glossary ar # writes prompts/evals/translations-inspect_ai/glossary/ar.json +just eval-glossary ar is fr # one file per language code +``` + +Labels without an official translation are translated freely. + +## Run + +```sh +just eval-translations # all candidates, Arabic +just eval-translations --limit 2 # quick smoke run +just eval-translations --target-language Icelandic --lang-code is +just eval-translations --models openai-api/scaleway/qwen3.6-35b-a3b +just eval-translations --judge openai-api/scaleway/mistral-medium-3.5-128b +``` + +Then browse per-sample transcripts, scores, and token usage: + +```sh +just eval-view +``` + +Logs land in `prompts/evals/translations-inspect_ai/logs/` (gitignored; `.eval` files are also +readable programmatically with `inspect_ai.log.read_eval_log`). + +## What's scored + +- `rule_checks` — deterministic, stdlib `html.parser` only: HTML tag/attribute + structure, glossary compliance for ``/`` labels (position-wise, exact + official translation), no truncation, no reasoning bleed. Reported per rule + (accuracy ± stderr). +- `model_graded_qa` — LLM judge (`glm-5.2` by default) grading + accuracy/fluency/rule compliance as C/P/I with partial credit. The judge + prompt includes the same per-segment glossary (via sample metadata). The + judge is deliberately not one of the candidates, to avoid self-preference. + +## Notes / differences from the custom suite + +- The old "keep ``/`` text in English" rule is replaced by glossary + compliance, matching the translator's actual behaviour. +- Grading is per-candidate (C/P/I) rather than pairwise A/B/C ranking. Inspect + shuffles nothing because there is no position to bias; comparison happens + across eval runs in the viewer. Pairwise ranking is possible but needs a + custom scorer again — start without it. +- Rules that don't apply to a sample (e.g. no glossary term in the source) + count as a pass rather than being excluded from the denominator. +- The dataset is `prompts/evals/translations/segments.yaml`, shared with the + Promptfoo eval: segments extracted verbatim from real guide content + (`prompts/content/en/how-to-guides/manage-documents.md` and + `prompts/content/en/releases/new-in-wagtail-7-4.md`), in the HTML form the + translator receives from wagtail-localize. Each entry is a Promptfoo test + case (`description`, `vars.text`, `metadata.id`/`metadata.source`); + `translation_task.py` maps them to Inspect Samples. Regenerate from live + page content with `export_segments.py` (see its docstring), or grow the + file as regressions are found. diff --git a/prompts/evals/translations-inspect_ai/export_glossary.py b/prompts/evals/translations-inspect_ai/export_glossary.py new file mode 100644 index 00000000..e985710c --- /dev/null +++ b/prompts/evals/translations-inspect_ai/export_glossary.py @@ -0,0 +1,49 @@ +""" +Export merged Django/Wagtail translation catalogs as glossary JSON files. + +The output is the same catalog `apps/core/translator.py` reads live via +Django's translation machinery — this build step just makes it available to +the Inspect eval, which runs outside the project environment. + +Usage (from the project root, one file per language code): + + uv run python prompts/evals/translations-inspect_ai/export_glossary.py ar is fr + # or: just eval-glossary ar is fr + +Output: prompts/evals/translations-inspect_ai/glossary/.json ({"Collection": "Safn", ...}) +""" + +import json +import os +import sys +from pathlib import Path + +# Make the project root importable when run by path. +sys.path.insert(0, str(Path(__file__).parents[3])) +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "apps.guide.settings.test") + +import django + +django.setup() + +from django.utils.translation import trans_real + +OUT_DIR = Path(__file__).parent / "glossary" +OUT_DIR.mkdir(exist_ok=True) + +for lang in sys.argv[1:] or ["ar"]: + catalog = trans_real.translation(lang)._catalog + entries = {} + for key in catalog.keys(): + # Skip plural entries (tuple keys) and contextual entries (ctx\x04msgid). + if not isinstance(key, str) or "\x04" in key: + continue + value = catalog.get(key) + if isinstance(value, str) and value.strip(): + entries[key] = value + path = OUT_DIR / f"{lang}.json" + path.write_text( + json.dumps(entries, ensure_ascii=False, indent=1, sort_keys=True) + "\n", + encoding="utf-8", + ) + print(f"{path}: {len(entries)} entries") diff --git a/prompts/evals/translations-inspect_ai/export_segments.py b/prompts/evals/translations-inspect_ai/export_segments.py new file mode 100644 index 00000000..ecc33686 --- /dev/null +++ b/prompts/evals/translations-inspect_ai/export_segments.py @@ -0,0 +1,50 @@ +""" +Export real translatable segments in the shared eval dataset format. + +Optional — a hand-picked prompts/evals/translations/segments.yaml (extracted +from real guide content) is checked in. Use this to regenerate it from actual +page content (the output is a JSON array, which is valid YAML): + + docker compose exec -T web python manage.py shell -c \ + "exec(open('prompts/evals/translations-inspect_ai/export_segments.py').read())" \ + > prompts/evals/translations/segments.yaml +""" + +import json +import os + +from wagtail.models import Locale, Page +from wagtail_localize.models import TranslationSource + +PAGE_IDS = [int(p) for p in os.environ.get("EVAL_PAGE_IDS", "10,66").split(",")] + +cases = [] +en = Locale.objects.get(language_code="en") +for pid in PAGE_IDS: + page = Page.objects.filter(id=pid, locale=en).first() or ( + Page.objects.filter( + translation_key=Page.objects.get(id=pid).translation_key, locale=en + ).first() + ) + if not page: + continue + source = TranslationSource.objects.filter( + object_id=page.translation_key, locale=en + ).first() + if not source: + continue + for seg in source.stringsegment_set.select_related("string", "context").order_by( + "order" + ): + cases.append( + { + "description": f"{page.title} — {seg.context.path if seg.context_id else 'segment'}", + "metadata": { + "id": f"page{page.id}-{seg.string_id}", + "source": f"page {page.id}: {page.title}", + }, + "vars": {"text": seg.string.as_value().get_translatable_html()}, + } + ) + +print(json.dumps(cases, indent=2, ensure_ascii=False)) diff --git a/prompts/evals/translations-inspect_ai/translation_task.py b/prompts/evals/translations-inspect_ai/translation_task.py new file mode 100755 index 00000000..83d893ce --- /dev/null +++ b/prompts/evals/translations-inspect_ai/translation_task.py @@ -0,0 +1,378 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.12,<3.13" +# dependencies = ["inspect-ai", "openai", "pyyaml"] +# /// +""" +Inspect AI eval for the LLM translator in apps/core/translator.py. + +Replaces the hand-rolled collect/check/judge/summarize pipeline: Inspect +provides the runner, parallelism, retries, per-model comparison, log storage, +and a results viewer (`inspect view`). This file only defines the task, a +deterministic rule scorer, and an LLM-judge scorer. + +Candidate and judge models are declared in CANDIDATE_MODELS / JUDGE_MODEL +below. Credentials come from the standard Scaleway environment variables +(SCW_SECRET_KEY, SCW_DEFAULT_PROJECT_ID), e.g. via the project `.env`. + +UI labels in / tags are held to the official Wagtail admin translations +(e.g. Icelandic "Collection" = "Safn"), mirroring the glossary injection in +apps/core/translator.py. Export the glossary for a language first: + + just eval-glossary ar + +Run it directly (uv resolves the inline dependencies): + + ./prompts/evals/translations-inspect_ai/translation_task.py + ./prompts/evals/translations-inspect_ai/translation_task.py --limit 2 --target-language Icelandic --lang-code is + +or via just: + + just eval-translations + just eval-view +""" + +import json +import re +from html import unescape +from html.parser import HTMLParser +from pathlib import Path + +import yaml +from inspect_ai import Task, task +from inspect_ai.dataset import MemoryDataset, Sample +from inspect_ai.model import ChatMessageSystem, GenerateConfig +from inspect_ai.scorer import ( + Score, + Target, + accuracy, + model_graded_qa, + scorer, + stderr, +) +from inspect_ai.solver import TaskState, generate, solver + +# Scaleway Generative APIs (OpenAI Chat Completions compatible), accessed +# through Inspect's `openai-api//` naming. +CANDIDATE_MODELS = [ + "openai-api/scaleway/deepseek-v4-flash-0731", + "openai-api/scaleway/mistral-medium-3.5-128b", + "openai-api/scaleway/gemma-4-26b-a4b-it", +] +JUDGE_MODEL = "openai-api/scaleway/glm-5.2" + +LOG_DIR = str(Path(__file__).parent / "logs") +GLOSSARY_DIR = Path(__file__).parent / "glossary" +# Segment dataset shared with the Promptfoo eval (Promptfoo test-case shape). +SEGMENTS_PATH = Path(__file__).parents[1] / "translations" / "segments.yaml" + + +def load_dataset(): + cases = yaml.safe_load(SEGMENTS_PATH.read_text(encoding="utf-8")) + return MemoryDataset( + [ + Sample( + id=case["metadata"]["id"], + input=case["vars"]["text"], + metadata={"description": case["description"], **case["metadata"]}, + ) + for case in cases + ] + ) + + +try: + # Single source of truth when run inside the project environment. + from apps.core.translator import LLMTranslator + + SYSTEM_PROMPT = str(LLMTranslator.default_system_prompt) + GLOSSARY_PROMPT = str(LLMTranslator.default_glossary_prompt) +except Exception: # noqa: BLE001 - any Django/Wagtail setup failure + # Fallback copies for running outside Django. Keep in sync with + # apps/core/translator.py. + SYSTEM_PROMPT = ( + "You are a professional translator translating text from " + "{source_language} to {target_language}.\n" + "Translate only the text and keep its structure intact.\n" + "Reply with just the translated text and no wrapper, explanation, or " + "code fence of any kind.\n" + "- Only standard HTML inline tags are allowed: a, abbr, acronym, b, " + "code, em, i, strong, br.\n" + "- tags may keep only their id attribute; other tags must have no " + "attributes.\n" + "- Preserve any inline tags, whitespace, and punctuation exactly." + ) + GLOSSARY_PROMPT = ( + "\n- Text inside or tags is usually a label from the Wagtail " + "admin interface. When a term appears in the glossary below, use the " + "official translation exactly as given; otherwise translate it " + "naturally and consistently.\n" + "Glossary (official Wagtail admin translations):\n{glossary}" + ) + +UI_LABEL_RE = re.compile(r"<(b|i)\b[^>]*>(.*?)", re.IGNORECASE | re.DOTALL) + + +def ui_terms(html: str) -> list[str]: + """Unique / label texts, in order — as in LLMTranslator.""" + terms = [] + for _tag, text in UI_LABEL_RE.findall(html or ""): + term = unescape(re.sub(r"<[^>]+>", "", text)).strip() + if term and term not in terms: + terms.append(term) + return terms + + +def load_glossary(lang_code: str) -> dict[str, str]: + path = GLOSSARY_DIR / f"{lang_code}.json" + if not path.exists(): + print( + f"WARNING: no glossary at {path} — UI label consistency will not " + f"be prompted or scored. Generate it with: just eval-glossary {lang_code}" + ) + return {} + return json.loads(path.read_text(encoding="utf-8")) + + +# --- Solver: per-sample system prompt with glossary, as in production -------- + + +@solver +def translator_prompt(system_prompt: str, glossary: dict[str, str]): + """Prepend the translator system prompt, plus the glossary entries for the + UI labels present in this sample (mirrors LLMTranslator.get_messages).""" + + async def solve(state: TaskState, generate): + matched = { + term: glossary[term] + for term in ui_terms(state.input_text) + if term in glossary + } + lines = "\n".join(f"- {src} = {dst}" for src, dst in matched.items()) + # Stash for the judge template ({glossary} via sample metadata). + state.metadata["glossary"] = lines or "(none for this segment)" + prompt = system_prompt + if matched: + prompt += GLOSSARY_PROMPT.format(glossary=lines) + state.messages.insert(0, ChatMessageSystem(content=prompt)) + return state + + return solve + + +# --- Deterministic rule scorer (stdlib only, no bs4) ------------------------- + + +class _TagCollector(HTMLParser): + """Collects (tag, attrs) in document order and text inside /.""" + + def __init__(self): + super().__init__(convert_charrefs=True) + self.tags = [] + self.bold_texts = [] + self._bold_depth = 0 + self._bold_buf = [] + + def handle_starttag(self, tag, attrs): + self.tags.append((tag, tuple(sorted(attrs)))) + if tag in ("b", "i"): + self._bold_depth += 1 + + def handle_startendtag(self, tag, attrs): + self.tags.append((tag, tuple(sorted(attrs)))) + + def handle_endtag(self, tag): + if tag in ("b", "i") and self._bold_depth: + self._bold_depth -= 1 + if not self._bold_depth: + self.bold_texts.append("".join(self._bold_buf).strip()) + self._bold_buf = [] + + def handle_data(self, data): + if self._bold_depth: + self._bold_buf.append(data) + + +def _parse(html: str) -> _TagCollector: + collector = _TagCollector() + collector.feed(html or "") + collector.close() + return collector + + +BLEED_RE = re.compile( + r"^```|^(sure|here is|let me|certainly|of course|translation)\b" + r"|(here is the translation|let me translate|the translation is)", + re.IGNORECASE, +) +TRUNC_MIN_RATIO = 0.25 + + +@scorer(metrics={"*": [accuracy(), stderr()]}) +def rule_checks(glossary: dict[str, str]): + """Format rules from the translator system prompt, checked mechanically. + + html_tags same tag sequence + attributes as the source + glossary / labels use the official Wagtail admin translation + no_trunc output not suspiciously short vs source + no_bleed no code fences or commentary preamble + + Checks that don't apply to a sample (e.g. no glossary term in the source) + score 1. + """ + + async def score(state: TaskState, target: Target) -> Score: + source = state.input_text + candidate = (state.output.completion or "").strip() + src = _parse(source) + cand = _parse(candidate) + + # Position-wise comparison of / labels against the glossary. + expected = [ + (i, text, glossary[text]) + for i, text in enumerate(src.bold_texts) + if text in glossary + ] + if not expected: + glossary_ok = True + elif len(cand.bold_texts) != len(src.bold_texts): + glossary_ok = False + else: + glossary_ok = all( + cand.bold_texts[i] == official for i, _text, official in expected + ) + + checks = { + # Strict equality both ways: also fails when the model invents + # tags in a plain-prose source (a known failure mode). + "html_tags": src.tags == cand.tags, + "glossary": glossary_ok, + "no_trunc": len(candidate) >= TRUNC_MIN_RATIO * len(source), + "no_bleed": bool(candidate) and not BLEED_RE.search(candidate[:200]), + } + failures = [name for name, passed in checks.items() if not passed] + return Score( + value={name: 1 if passed else 0 for name, passed in checks.items()}, + answer=candidate, + explanation="all rules pass" if not failures else f"FAIL: {failures}", + ) + + return score + + +# --- LLM judge scorer -------------------------------------------------------- + +JUDGE_TEMPLATE = """ +You are an expert reviewer of {target_language} machine translation quality +for a software user guide. + +SOURCE (English, may contain inline HTML such as ): +{question} + +CANDIDATE {target_language} TRANSLATION: +{answer} + +Requirements the translation must meet: +1. HTML tags and their attributes match the source exactly — same tags, same + order, no additions. +2. Text inside / tags is a UI label from the Wagtail admin interface. + Where an official admin translation exists, it must be used exactly. + Official translations for the labels in this segment: +{glossary} + Labels without an official translation are translated naturally and + consistently. +3. Plain prose and link text are translated into fluent, formal + {target_language}, with meaning fully preserved (no omissions, additions, + or mistranslations). +4. The output contains only the translation — no commentary, reasoning, or + code fences — and is complete (not truncated). +5. Terminology is consistent: one term per concept. + +{instructions} +""" + +JUDGE_INSTRUCTIONS = """ +Grade the candidate: +- C: meets all requirements; accurate and fluent. +- P: understandable and structurally intact, but with minor fluency, + terminology, or accuracy issues. +- I: violates a structural requirement (HTML, official UI label translations, + extra commentary, truncation) or materially mistranslates. + +First reason briefly about each requirement, then finish with exactly one +line: GRADE: C, GRADE: P, or GRADE: I. +""" + + +@task +def translation_eval( + target_language: str = "Arabic", + lang_code: str = "ar", + source_language: str = "English", + judge_model: str = JUDGE_MODEL, +): + glossary = load_glossary(lang_code) + system_prompt = SYSTEM_PROMPT.format( + source_language=source_language, target_language=target_language + ) + return Task( + dataset=load_dataset(), + solver=[ + translator_prompt(system_prompt, glossary), + generate(), + ], + scorer=[ + rule_checks(glossary), + model_graded_qa( + template=JUDGE_TEMPLATE.replace("{target_language}", target_language), + instructions=JUDGE_INSTRUCTIONS, + partial_credit=True, + model=judge_model, + ), + ], + config=GenerateConfig(temperature=0), + ) + + +def main(): + import argparse + import os + + # Map the standard Scaleway env vars (e.g. from `.env`) to the ones + # Inspect's openai-api provider reads. + if os.environ.get("SCW_SECRET_KEY"): + os.environ.setdefault("SCALEWAY_API_KEY", os.environ["SCW_SECRET_KEY"]) + if os.environ.get("SCW_DEFAULT_PROJECT_ID"): + os.environ.setdefault( + "SCALEWAY_BASE_URL", + f"https://api.scaleway.ai/{os.environ['SCW_DEFAULT_PROJECT_ID']}/v1", + ) + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--models", + default=",".join(CANDIDATE_MODELS), + help="comma-separated candidate models", + ) + parser.add_argument("--judge", default=JUDGE_MODEL, help="judge model") + parser.add_argument("--target-language", default="Arabic") + parser.add_argument("--lang-code", default="ar", help="glossary language code") + parser.add_argument("--limit", type=int, default=None, help="max samples") + args = parser.parse_args() + + from inspect_ai import eval as inspect_eval + + inspect_eval( + translation_eval( + target_language=args.target_language, + lang_code=args.lang_code, + judge_model=args.judge, + ), + model=args.models.split(","), + limit=args.limit, + log_dir=LOG_DIR, + ) + + +if __name__ == "__main__": + main() diff --git a/prompts/evals/translations/check_format.py b/prompts/evals/translations/check_format.py new file mode 100644 index 00000000..9f317c26 --- /dev/null +++ b/prompts/evals/translations/check_format.py @@ -0,0 +1,166 @@ +"""Deterministic output-format assertions for the LLM translator eval. + +These mirror the rules in apps/core/translator.py's system prompt: reply with +only the translated text, keep inline tags and structure intact, and keep + ids while stripping other attributes. + +Each function is referenced from promptfooconfig.yaml as +file://check_format.py: and receives (output, context). +""" + +import re +import sys +from html.parser import HTMLParser +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent)) + +from glossary_util import matched_glossary + +ALLOWED_TAGS = { + "a", + "abbr", + "acronym", + "b", + "br", + "code", + "em", + "i", + "strong", +} + +BLEED_PATTERNS = [ + r"^```", + r"^(sure|here is|let me|certainly|of course|the translation is|translated text|translation)\b", +] +BLEED_RE = re.compile("|".join(BLEED_PATTERNS), re.IGNORECASE) + + +class TagCollector(HTMLParser): + def __init__(self): + super().__init__(convert_charrefs=True) + self.tags = [] + self.bold_texts = [] + self._bold_depth = 0 + self._bold_buf = [] + + def handle_starttag(self, tag, attrs): + self.tags.append((tag, dict(attrs))) + if tag in ("b", "i"): + self._bold_depth += 1 + + def handle_startendtag(self, tag, attrs): + self.tags.append((tag, dict(attrs))) + + def handle_endtag(self, tag): + if tag in ("b", "i") and self._bold_depth: + self._bold_depth -= 1 + if not self._bold_depth: + self.bold_texts.append("".join(self._bold_buf).strip()) + self._bold_buf = [] + + def handle_data(self, data): + if self._bold_depth: + self._bold_buf.append(data) + + +def parse_html(html): + parser = TagCollector() + parser.feed(html or "") + return parser + + +def _pass(reason): + return {"pass": True, "score": 1, "reason": reason} + + +def _fail(reason): + return {"pass": False, "score": 0, "reason": reason} + + +def no_wrapper(output, context): + if not output: + return _fail("empty output") + stripped = output.strip() + if BLEED_RE.search(stripped[:200]): + return _fail(f"reasoning/wrapper bleed detected: {stripped[:60]!r}") + if stripped.startswith(('"', "'", "\u201c")) and stripped.endswith( + ('"', "'", "\u201d") + ): + return _fail(f"output wrapped in quotes: {stripped[:60]!r}") + source = context["vars"].get("text", "") + if "\n" not in source and "\n" in stripped: + return _fail( + f"output has multiple lines but source is single-line: {stripped[:60]!r}" + ) + return _pass("no wrapper, quote, or bleed") + + +def allowed_tags_only(output, context): + found = {tag for tag, _ in parse_html(output).tags} + disallowed = found - ALLOWED_TAGS + if disallowed: + return _fail(f"disallowed tags: {sorted(disallowed)}") + return _pass(f"tags within allowed set: {sorted(found)}") + + +def tags_preserved(output, context): + source = context["vars"].get("text", "") + src = [(tag, attrs.get("id")) for tag, attrs in parse_html(source).tags] + out = [(tag, attrs.get("id")) for tag, attrs in parse_html(output).tags] + # Strict equality both ways: also fails when the model invents tags in a + # plain-prose source (a known failure mode). + if src != out: + return _fail(f"tags differ: source={src} output={out}") + return _pass("tag sequence and ids preserved") + + +def attributes_clean(output, context): + for tag, attrs in parse_html(output).tags: + if tag == "a": + extra = set(attrs) - {"id"} + if extra: + return _fail(f" has non-id attributes: {sorted(extra)}") + elif attrs: + return _fail(f"<{tag}> has attributes: {attrs}") + return _pass("no attributes beyond ") + + +def not_truncated(output, context): + source = context["vars"].get("text", "") + stripped = (output or "").strip() + if len(stripped) < 0.25 * len(source): + return _fail( + f"output suspiciously short ({len(stripped)} chars vs {len(source)} source)" + ) + return _pass("output length plausible") + + +def glossary_compliant(output, context): + """/ labels use the official Wagtail admin translation, in place.""" + source = context["vars"].get("text", "") + lang_code = context["vars"].get("lang_code", "") + expected = matched_glossary(source, lang_code) + if not expected: + return _pass("no glossary terms in this segment") + src_bold = parse_html(source).bold_texts + out_bold = parse_html(output).bold_texts + if len(out_bold) != len(src_bold): + return _fail( + f"/ label count differs: source={src_bold} output={out_bold}" + ) + for i, term in enumerate(src_bold): + if term in expected and out_bold[i] != expected[term]: + return _fail( + f"label {term!r} should be {expected[term]!r}, got {out_bold[i]!r}" + ) + return _pass(f"official translations used: {expected}") + + +def translated(output, context): + source = context["vars"].get("text", "") + if not output: + return _fail("empty output") + if output.strip() == source.strip(): + return _fail("output is identical to source (not translated)") + return _pass("output differs from source") diff --git a/prompts/evals/translations/glossary_util.py b/prompts/evals/translations/glossary_util.py new file mode 100644 index 00000000..0a3476bd --- /dev/null +++ b/prompts/evals/translations/glossary_util.py @@ -0,0 +1,42 @@ +"""Shared glossary helpers for the Promptfoo translation eval. + +Reads the Wagtail admin translation glossaries exported by +`just eval-glossary ` (prompts/evals/translations-inspect_ai/glossary/.json), +and mirrors the UI label extraction in apps/core/translator.py. +""" + +import json +import re +from functools import cache +from html import unescape +from pathlib import Path + +GLOSSARY_DIR = Path(__file__).parents[1] / "translations-inspect_ai" / "glossary" + +UI_LABEL_RE = re.compile(r"<(b|i)\b[^>]*>(.*?)", re.IGNORECASE | re.DOTALL) + + +def ui_terms(html): + """Unique / label texts, in order — as in LLMTranslator.""" + terms = [] + for _tag, text in UI_LABEL_RE.findall(html or ""): + term = unescape(re.sub(r"<[^>]+>", "", text)).strip() + if term and term not in terms: + terms.append(term) + return terms + + +@cache +def load_glossary(lang_code): + path = GLOSSARY_DIR / f"{lang_code}.json" + if not path.exists(): + raise FileNotFoundError( + f"No glossary at {path} — generate it with: just eval-glossary {lang_code}" + ) + return json.loads(path.read_text(encoding="utf-8")) + + +def matched_glossary(text, lang_code): + """Official admin translations for the UI labels present in `text`.""" + glossary = load_glossary(lang_code) + return {term: glossary[term] for term in ui_terms(text) if term in glossary} diff --git a/prompts/evals/translations/glossary_var.py b/prompts/evals/translations/glossary_var.py new file mode 100644 index 00000000..c331914e --- /dev/null +++ b/prompts/evals/translations/glossary_var.py @@ -0,0 +1,26 @@ +"""Promptfoo dynamic var: the glossary lines for a test's text + language. + +Referenced from translations.yaml as `glossary: file://./glossary_var.py`. +The resolved value is used by the prompt function and the llm-rubric grader. +""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent)) + +from glossary_util import matched_glossary + +# Promptfoo rejects empty dynamic vars, so segments with no glossary terms get +# an explicit marker. translation_prompt.py skips the glossary section for it. +NO_GLOSSARY = "(none for this segment)" + + +def get_var(var_name, prompt, other_vars): + matched = matched_glossary( + other_vars.get("text", ""), other_vars.get("lang_code", "") + ) + lines = "\n".join( + f"- {source} = {translated}" for source, translated in matched.items() + ) + return {"output": lines or NO_GLOSSARY} diff --git a/prompts/evals/translations/segments.yaml b/prompts/evals/translations/segments.yaml new file mode 100644 index 00000000..aa48e82f --- /dev/null +++ b/prompts/evals/translations/segments.yaml @@ -0,0 +1,101 @@ +# Shared translation eval segments — single source of truth for both the +# Promptfoo eval (referenced from translations.yaml via `tests: file://`) +# and the Inspect AI eval (loaded by translations-inspect_ai/translation_task.py). +# +# Each entry is a Promptfoo test case (description, vars, metadata); Inspect +# maps it to a Sample (id=metadata.id, input=vars.text, metadata=metadata). +# +# Segments are extracted verbatim from real guide content, as the +# wagtail-localize HTML segments the translator receives: markdown links +# become and **bold** UI labels become . + +- description: Plain prose, no markup (page intro) + metadata: + id: manage-documents-intro + source: prompts/content/en/how-to-guides/manage-documents.md + vars: + text: 'Documents such as PDFs can be managed from the Documents interface, available in the left-hand menu.' + +- description: Link with id — link text translated + metadata: + id: manage-documents-link + source: prompts/content/en/how-to-guides/manage-documents.md + vars: + text: 'You can manage documents such as PDF files from the Documents interface.' + +- description: Bold UI label with official translation, plus a link + metadata: + id: manage-documents-sidebar + source: prompts/content/en/how-to-guides/manage-documents.md + vars: + text: 'To access the Documents interface, click Documents in the Sidebar.' + +- description: Bold label with official translation, repeated term in prose + metadata: + id: manage-documents-add + source: prompts/content/en/how-to-guides/manage-documents.md + vars: + text: 'Add documents by clicking Add documents at the top right of the documents interface.' + +- description: Multi-sentence segment with one bold label (Collection) + metadata: + id: manage-documents-collection + source: prompts/content/en/how-to-guides/manage-documents.md + vars: + text: 'Search for previously added documents by entering the title of the document in the search bar. The results automatically update as you type. You can also filter the results by Collection by selecting a collection from the Collection dropdown preceding the documents list.' + +- description: Bold label (Save) and link at end of a long sentence + metadata: + id: manage-documents-save + source: prompts/content/en/how-to-guides/manage-documents.md + vars: + text: 'Once you are done editing a document, save your edits by clicking Save at the bottom of the edit screen.' + +- description: Bold label (Delete) with official translation + metadata: + id: manage-documents-delete + source: prompts/content/en/how-to-guides/manage-documents.md + vars: + text: 'Then click Delete from the action bar displayed at the bottom of the Documents interface.' + +- description: Repeated link text plus compound label (Delete document) + metadata: + id: manage-documents-delete-document + source: prompts/content/en/how-to-guides/manage-documents.md + vars: + text: 'You can also delete a document and its record from the edit screen when editing a document by clicking Delete document at the bottom of the edit screen.' + +- description: Short warning callout + metadata: + id: manage-documents-warning + source: prompts/content/en/how-to-guides/manage-documents.md + vars: + text: 'Warning: Deleted documents cannot be recovered.' + +- description: Long plain prose with product terminology + metadata: + id: new-in-7-4-autosave + source: prompts/content/en/releases/new-in-wagtail-7-4.md + vars: + text: 'The autosave and concurrent editing notifications introduced in Wagtail 7.3 have been refined based on user feedback. Race conditions and brief network outages are handled more gracefully, the indicator messaging and alignment have been polished, and you can now distinguish which other users are active or idle while looking at the same page or snippet.' + +- description: Bold label without an official translation — translated freely + metadata: + id: new-in-7-4-add-block + source: prompts/content/en/releases/new-in-wagtail-7-4.md + vars: + text: 'Block-based content now lists block groups in the order developers defined them, making the Add block menu more predictable.' + +- description: Bold multi-word label without an official translation + metadata: + id: new-in-7-4-workflow + source: prompts/content/en/releases/new-in-wagtail-7-4.md + vars: + text: "The Submit to workflow menu now uses the workflow's name when creating a new page." + +- description: Long segment with two links and proper nouns kept intact + metadata: + id: new-in-7-4-security + source: prompts/content/en/releases/new-in-wagtail-7-4.md + vars: + text: 'Wagtail recently underwent an independent code security audit commissioned by the Interministerial Digital Directorate (DINUM) of France.' diff --git a/prompts/evals/translations/translation_prompt.py b/prompts/evals/translations/translation_prompt.py new file mode 100644 index 00000000..61400f0d --- /dev/null +++ b/prompts/evals/translations/translation_prompt.py @@ -0,0 +1,47 @@ +"""Promptfoo prompt function for the LLM translator eval. + +Builds the same messages as LLMTranslator.get_messages in +apps/core/translator.py: the translator system prompt, plus a glossary section +when the segment contains UI labels with official Wagtail admin translations. +Keep the prompt strings in sync with apps/core/translator.py. +""" + +# Keep in sync with LLMTranslator.default_system_prompt. +SYSTEM_PROMPT = ( + "You are a professional translator translating text from " + "{source_language} to {target_language}.\n" + "Translate only the text and keep its structure intact.\n" + "Reply with just the translated text and no wrapper, explanation, or " + "code fence of any kind.\n" + "- Only standard HTML inline tags are allowed: a, abbr, acronym, b, " + "code, em, i, strong, br.\n" + "- tags may keep only their id attribute; other tags must have no " + "attributes.\n" + "- Preserve any inline tags, whitespace, and punctuation exactly." +) + +# Keep in sync with LLMTranslator.default_glossary_prompt. +GLOSSARY_PROMPT = ( + "\n- Text inside or tags is usually a label from the Wagtail " + "admin interface. When a term appears in the glossary below, use the " + "official translation exactly as given; otherwise translate it " + "naturally and consistently.\n" + "Glossary (official Wagtail admin translations):\n{glossary}" +) + + +def prompt(context): + variables = context["vars"] + system = SYSTEM_PROMPT.format( + source_language=variables.get("source_language", "English"), + target_language=variables["target_language"], + ) + glossary = (variables.get("glossary") or "").strip() + if glossary.startswith("(none"): + glossary = "" + if glossary: + system += GLOSSARY_PROMPT.format(glossary=glossary) + return [ + {"role": "system", "content": system}, + {"role": "user", "content": variables["text"]}, + ] diff --git a/prompts/evals/translations/translations.yaml b/prompts/evals/translations/translations.yaml new file mode 100644 index 00000000..bf5bf0c9 --- /dev/null +++ b/prompts/evals/translations/translations.yaml @@ -0,0 +1,97 @@ +# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json +# LLM translator eval — Promptfoo counterpart of prompts/evals/translations-inspect_ai/. +# +# Prerequisite: export the glossaries once per language: +# just eval-glossary is de fr pt-br ar +# Run: +# just eval-promptfoo # full run: 5 languages x 10 tests x 3 models +# just eval-promptfoo -n 3 # smoke run: first 3 tests per scenario +# just eval-promptfoo-view # browse results +description: LLM translator quality — evals the system prompt and glossary injection in apps/core/translator.py + +prompts: + # Python prompt function mirroring LLMTranslator.get_messages, including the + # per-segment glossary of official Wagtail admin translations. + - file://./translation_prompt.py:prompt + +providers: + - id: openai:chat:deepseek-v4-flash-0731 + config: &scaleway + apiBaseUrl: 'https://api.scaleway.ai/{{ env.SCW_DEFAULT_PROJECT_ID }}/v1' + apiKey: '{{ env.SCW_SECRET_KEY }}' + temperature: 0 + # Reasoning models spend output tokens on thinking before the answer; + # the default max_tokens truncates their visible output. + max_tokens: 4096 + showThinking: false + - id: openai:chat:mistral-medium-3.5-128b + config: *scaleway + - id: openai:chat:gemma-4-26b-a4b-it + config: *scaleway + +defaultTest: + vars: + source_language: English + # Dynamic var: glossary lines for this test's text + lang_code, used by + # both the prompt function and the llm-rubric grader below. + glossary: file://./glossary_var.py + assert: + # Deterministic format rules, as in the Inspect eval's rule_checks scorer. + - type: python + value: file://./check_format.py:no_wrapper + - type: python + value: file://./check_format.py:allowed_tags_only + - type: python + value: file://./check_format.py:tags_preserved + - type: python + value: file://./check_format.py:attributes_clean + - type: python + value: file://./check_format.py:not_truncated + - type: python + value: file://./check_format.py:glossary_compliant + - type: python + value: file://./check_format.py:translated + # LLM judge, as in the Inspect eval's model_graded_qa scorer. + - type: llm-rubric + provider: + id: openai:chat:glm-5.2 + config: *scaleway + value: | + You are an expert reviewer of {{ target_language }} machine + translation quality for a software user guide. The source English + text (may contain inline HTML) is: + + {{ text }} + + Requirements the translation must meet: + 1. HTML tags and their attributes match the source exactly — same + tags, same order, no additions. + 2. Text inside / tags is a UI label from the Wagtail admin + interface. Where an official admin translation exists, it must be + used exactly. Official translations for the labels in this segment: + {{ glossary }} + Labels without an official translation are translated naturally + and consistently. + 3. Plain prose and link text are translated into fluent, formal + {{ target_language }}, with meaning fully preserved (no omissions, + additions, or mistranslations). + 4. The output contains only the translation — no commentary, + reasoning, or code fences — and is complete (not truncated). + 5. Terminology is consistent: one term per concept. + + Fail the output if it violates a structural requirement (HTML, + official UI label translations, extra commentary, truncation) or + materially mistranslates. Score 0.5 but pass for minor fluency, + terminology, or accuracy issues. Score 1 if it meets all + requirements. + +scenarios: + - config: + - vars: { target_language: Icelandic, lang_code: is } + - vars: { target_language: German, lang_code: de } + - vars: { target_language: French, lang_code: fr } + - vars: { target_language: Brazilian Portuguese, lang_code: pt-br } + - vars: { target_language: Arabic, lang_code: ar } + # Shared with the Inspect AI eval — segments extracted from real guide + # content (see segments.yaml for sources). + tests: file://./segments.yaml diff --git a/ruff.toml b/ruff.toml index 49c23d28..8d78032d 100644 --- a/ruff.toml +++ b/ruff.toml @@ -21,4 +21,9 @@ select = [ "YTT", # flake8-2020 ] +# Standalone management/CLI scripts: they print to the console and import +# Django models after the shell bootstrap, so those rules don't apply. +[lint.per-file-ignores] +"prompts/evals/**/*.py" = ["T20", "E402"] + fixable = ["C4", "E", "F", "I", "UP"]