diff --git a/MASTER_PLAN.md b/MASTER_PLAN.md index 5603408..93d0b1e 100644 --- a/MASTER_PLAN.md +++ b/MASTER_PLAN.md @@ -362,3 +362,12 @@ on ours. License check before every download. `.aiassistant` blobs, 5MB wasm). These are mined feature-by-feature from the tag with tests during Phases 1–2, not merged wholesale. Mining checklist lives here until each item ships or is explicitly rejected. +- 2026-07-05 — Phase 1 golden harness landed: 24 translation fixtures + 6 + reconstruction fixtures, word-order-aware scorer (`tests/golden/scoring.py`), CI + gate (criticals must pass 100%). First run caught and fixed four real grammar bugs + ('need'→'NE' stemming, false FINISH from -s/-ing forms, 'walking'→'WALKE', + RIGHTS→RIGHT library-sign mangling) and one vocabulary hole (NEED absent from the + avatar — added as a review-flagged placeholder, conf 2). **Phase 3 vocabulary + debt (measured):** 16 transformer signs the avatar cannot play — BOOK, BUY, FEEL, + HE, LEARN, LIKE, LIVE, MAKE, MILK, MY, NAME, PHONE, SOUTH AFRICA, TEACH, THINK, + YOUR. Record real signer data for these (plus CALL, HERE) before expanding further. diff --git a/sasl_transformer/grammar_rules.py b/sasl_transformer/grammar_rules.py index e2f78c9..825b89b 100644 --- a/sasl_transformer/grammar_rules.py +++ b/sasl_transformer/grammar_rules.py @@ -259,6 +259,26 @@ "came": "come", "saw": "see", "seen": "see", + # Common silent-e progressives — suffix stripping cannot recover the 'e' + # (walking→walk is right, but making→mak is not), so map them explicitly. + # NOTE: these are progressive (-ing) forms, NOT past tense; the caller + # must not derive a FINISH marker from them. + "making": "make", + "taking": "take", + "coming": "come", + "having": "have", + "giving": "give", + "writing": "write", + "living": "live", + "moving": "move", + "using": "use", + "losing": "lose", + "choosing": "choose", + "driving": "drive", + "hoping": "hope", + "closing": "close", + "sharing": "share", + "leaving": "leave", "took": "take", "taken": "take", "gave": "give", diff --git a/sasl_transformer/transformer.py b/sasl_transformer/transformer.py index 70c5e70..4aae414 100644 --- a/sasl_transformer/transformer.py +++ b/sasl_transformer/transformer.py @@ -398,9 +398,16 @@ def _translate_with_rules( question_markers.append(clean.upper()) continue - # Convert verbs to base form + # Convert verbs to base form. Only -ed forms and irregular past + # verbs signal past tense — stripping plural/3rd-person 's' or + # progressive '-ing' must NOT add a FINISH aspect marker + # (was: 'My chest hurts' → '... HURT FINISH'). base = self._to_base_form(clean) - if base != clean: + if ( + base != clean + and not clean.endswith("ing") # progressives are not past tense + and (clean.endswith("ed") or clean in IRREGULAR_VERB_BASE_FORMS) + ): has_past_tense = True content_words.append(base.upper()) @@ -468,23 +475,38 @@ def _to_base_form(self, word: str) -> str: if clean in IRREGULAR_VERB_BASE_FORMS: return IRREGULAR_VERB_BASE_FORMS[clean] + # If the word as-typed is already a known sign, never mangle it — + # suffix stripping would turn the library sign RIGHTS into RIGHT + # (not a sign) and silently downgrade it to fingerspelling. + if self._sign_library.has_sign(clean.upper()): + return clean + # Regular verb suffix stripping if clean.endswith("ing"): - # running → run (double consonant) stem = clean[:-3] - if len(stem) >= 2 and stem[-1] == stem[-2]: + if len(stem) < 2: + return clean + # running → run (double consonant) + if stem[-1] == stem[-2]: return stem[:-1] - # driving → drive (silent e) - if stem and stem[-1] not in "aeiou": + # Prefer whichever candidate the sign library actually knows; + # otherwise the bare stem is the safer fingerspell (walking→walk, + # not 'walke'). Silent-e verbs are handled by the irregular map. + if self._sign_library.has_sign(stem.upper()): + return stem + if self._sign_library.has_sign((stem + "e").upper()): return stem + "e" - return stem if stem else clean + return stem if clean.endswith("ed"): stem = clean[:-2] - if stem and stem[-1] == stem[-2]: - return stem[:-1] - if not stem: + # 'need'/'feed'/'speed' are not past tenses — the 'ed' belongs to + # the stem. Require a plausible stem length so we never emit + # fragments like 'ne' (was: 'I need help' → 'I NE HELP'). + if len(stem) < 3 or clean.endswith("eed"): return clean + if len(stem) >= 2 and stem[-1] == stem[-2]: + return stem[:-1] return stem if clean.endswith("ies"): diff --git a/signs_library.js b/signs_library.js index b0913c5..a7e4e6d 100644 --- a/signs_library.js +++ b/signs_library.js @@ -1374,6 +1374,15 @@ const SIGN_LIBRARY = { IL.sh,IL.el,IL.wr, NL, {j:'R_el', ax:'x', amp:0.18, freq:2.0}), + // PLACEHOLDER pending Deaf review (MASTER_PLAN D15/D16): NEED shares the + // downward X-hand family with MUST, signed softer/single motion. Added + // because NEED is the highest-frequency clinic verb and was missing from + // the avatar vocabulary entirely (caught by the golden fixture harness). + 'NEED': sign('NEED','X-hand bends downward, single soft motion','Right X-hand dips downward once — need (placeholder, review required)',2, + {x:-0.45,y:0,z:-0.22},{x:-0.60,y:0,z:0},{x:0,y:0,z:0}, HS.xhand, + IL.sh,IL.el,IL.wr, NL, + {j:'R_el', ax:'x', amp:0.12, freq:1.2}), + 'VERY': sign('VERY','Both V-hands spread apart','V-hands separate from each other — intensifier',4, {x:-0.52,y:0,z:-0.35},{x:-0.28,y:0,z:0},{x:0,y:0,z:0}, HS.vhand, {x:-0.52,y:0,z:0.35},{x:-0.28,y:0,z:0},{x:0,y:0,z:0}, HS.vhand, diff --git a/tests/golden/__init__.py b/tests/golden/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/golden/scoring.py b/tests/golden/scoring.py new file mode 100644 index 0000000..a97ee44 --- /dev/null +++ b/tests/golden/scoring.py @@ -0,0 +1,142 @@ +""" +Deterministic scorer for the golden translation fixtures (MASTER_PLAN Phase 1). + +Scores a gloss-token sequence against a fixture case on five axes: + +1. required-sign recall — every expected sign present +2. forbidden signs — banned glosses absent +3. marker preservation — WILL/MUST/CAN/FINISH survive +4. word order — [before, after] pairs hold (the SASL axis a + bag-of-signs metric cannot see) +5. avatar-library compliance — unknown-sign count vs the signs the avatar + can actually play (signs_library.js is ground + truth, NOT the transformer's JSON library) + +No network, no Ollama, no Node — the JS library is parsed with a regex. +""" + +import json +import re +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +SIGNS_LIBRARY_JS = REPO_ROOT / "signs_library.js" +GOLDEN_DIR = Path(__file__).resolve().parent + +# Matches library entries like 'HELLO': sign( or "I'M FINE": signWithFrames( +_SIGN_KEY_RE = re.compile(r"""^\s*(['"])([A-Z][A-Z0-9 '\-]*)\1\s*:\s*sign""", re.MULTILINE) + + +def load_known_signs(js_path=SIGNS_LIBRARY_JS): + """Return the set of sign names the avatar can actually play.""" + text = Path(js_path).read_text(encoding="utf-8") + return {m.group(2) for m in _SIGN_KEY_RE.finditer(text)} + + +def load_cases(filename): + with open(GOLDEN_DIR / filename, encoding="utf-8") as fh: + return json.load(fh)["cases"] + + +def _contains_sign(glosses, name): + """True if `name` appears in the gloss sequence. + + Multi-word library signs (e.g. 'THANK YOU') match either as a single + token or as a consecutive run of tokens. + """ + if name in glosses: + return True + words = name.split(" ") + if len(words) > 1: + for i in range(len(glosses) - len(words) + 1): + if glosses[i:i + len(words)] == words: + return True + return False + + +def _first_index(glosses, name): + """Index of the first occurrence of `name` (multi-word aware); -1 if absent.""" + words = name.split(" ") + if len(words) == 1: + return glosses.index(name) if name in glosses else -1 + for i in range(len(glosses) - len(words) + 1): + if glosses[i:i + len(words)] == words: + return i + return glosses.index(name) if name in glosses else -1 + + +def score_case(case, glosses): + """Score one fixture case against a produced gloss sequence. + + Returns a dict with per-axis results and an overall `passed` flag. + A `critical` case passes only when every axis is clean. + """ + required = case.get("expected_required_signs", []) + forbidden = case.get("expected_forbidden_signs", []) + markers = case.get("expected_markers", []) + order_pairs = case.get("expected_order", []) + max_unknown = case.get("max_unknown_signs", 0) + allow_fs = case.get("allow_fingerspell", False) + + missing_required = [s for s in required if not _contains_sign(glosses, s)] + forbidden_hits = [s for s in forbidden if _contains_sign(glosses, s)] + missing_markers = [m for m in markers if not _contains_sign(glosses, m)] + + order_violations = [] + for before, after in order_pairs: + i, j = _first_index(glosses, before), _first_index(glosses, after) + if i >= 0 and j >= 0 and i >= j: + order_violations.append([before, after]) + + known = load_known_signs() + covered = set() + for name in known: + idx = _first_index(glosses, name) + if idx >= 0: + covered.update(range(idx, idx + len(name.split(" ")))) + unknown_signs = [g for i, g in enumerate(glosses) if i not in covered] + unknown_ok = len(unknown_signs) <= max_unknown if allow_fs else len(unknown_signs) <= max_unknown + + recall = 1.0 if not required else (len(required) - len(missing_required)) / len(required) + + passed = ( + not missing_required + and not forbidden_hits + and not missing_markers + and not order_violations + and unknown_ok + ) + + return { + "id": case["id"], + "priority": case.get("priority", "normal"), + "passed": passed, + "required_recall": round(recall, 3), + "missing_required": missing_required, + "forbidden_hits": forbidden_hits, + "missing_markers": missing_markers, + "order_violations": order_violations, + "unknown_signs": unknown_signs, + "glosses": glosses, + } + + +def summarize(results): + """Aggregate a result list into the baseline report shape.""" + total = len(results) + passed = sum(1 for r in results if r["passed"]) + critical = [r for r in results if r["priority"] == "critical"] + critical_passed = sum(1 for r in critical if r["passed"]) + mean_recall = round(sum(r["required_recall"] for r in results) / total, 3) if total else 1.0 + return { + "cases_total": total, + "cases_passed": passed, + "critical_total": len(critical), + "critical_passed": critical_passed, + "mean_required_recall": mean_recall, + "failures": [ + {k: r[k] for k in ("id", "missing_required", "forbidden_hits", + "missing_markers", "order_violations", "unknown_signs", "glosses")} + for r in results if not r["passed"] + ], + } diff --git a/tests/golden/sign_reconstruction_cases.json b/tests/golden/sign_reconstruction_cases.json new file mode 100644 index 0000000..8af947d --- /dev/null +++ b/tests/golden/sign_reconstruction_cases.json @@ -0,0 +1,60 @@ +{ + "description": "Golden deaf→hearing sign-reconstruction fixtures (MASTER_PLAN Phase 1). Scored at INTENT level against the deterministic fallback (simple_signs_to_english) — English fluency may vary, meaning may not.", + "review_status": "draft — awaiting SASL interpreter review (D15/D16)", + "cases": [ + { + "id": "recon-help-001", + "group": "medical", + "priority": "critical", + "input_signs": ["ME", "NEED", "HELP"], + "expected_text_contains": ["help"], + "forbidden_text_contains": ["fine", "no help", "not"], + "notes": "Urgent help intent must survive; wording may vary." + }, + { + "id": "recon-hungry-001", + "group": "daily", + "priority": "high", + "input_signs": ["ME", "HUNGRY"], + "expected_text_contains": ["hungry"], + "forbidden_text_contains": ["not"], + "notes": "State report." + }, + { + "id": "recon-doctor-where-001", + "group": "medical", + "priority": "critical", + "input_signs": ["DOCTOR", "WHERE"], + "expected_text_contains": ["doctor", "where"], + "forbidden_text_contains": [], + "notes": "Location question about a doctor — both concepts must appear." + }, + { + "id": "recon-water-001", + "group": "daily", + "priority": "high", + "input_signs": ["ME", "WANT", "WATER"], + "expected_text_contains": ["water"], + "forbidden_text_contains": ["no water"], + "notes": "Request for water. KNOWN QUIRK: fallback currently phrases WANT as 'need' — intent preserved, flagged for interpreter review." + }, + { + "id": "recon-yes-001", + "group": "daily", + "priority": "high", + "input_signs": ["YES"], + "expected_text_contains": ["yes"], + "forbidden_text_contains": ["no"], + "notes": "Single-sign answer must never invert." + }, + { + "id": "recon-single-help-001", + "group": "emergency", + "priority": "critical", + "input_signs": ["HELP"], + "expected_text_contains": ["help"], + "forbidden_text_contains": ["not", "fine"], + "notes": "Bare HELP is an emergency signal — must expand to a help request, never soften." + } + ] +} diff --git a/tests/golden/translation_cases.json b/tests/golden/translation_cases.json new file mode 100644 index 0000000..99a9044 --- /dev/null +++ b/tests/golden/translation_cases.json @@ -0,0 +1,318 @@ +{ + "description": "Golden English→SASL translation fixtures (MASTER_PLAN Phase 1). Scored by tests/golden/scoring.py against the deterministic rules pipeline. Every expectation was verified against actual pipeline output on 2026-07-05 — cases the rules cannot yet handle are priority 'exploratory' with the gap named in notes, never faked.", + "review_status": "draft — awaiting SASL interpreter review (D15/D16)", + "cases": [ + { + "id": "medical-help-001", + "group": "medical", + "priority": "critical", + "input_text": "I need help. My chest hurts.", + "expected_required_signs": ["I", "NEED", "HELP", "HURT"], + "expected_forbidden_signs": ["NE", "FINISH"], + "expected_markers": [], + "expected_order": [], + "allow_fingerspell": true, + "max_unknown_signs": 2, + "notes": "Safety-critical. Regression: 'need' was mangled to 'NE' by the -ed stripper; present-tense 'hurts' must not emit FINISH. CHEST/MY fingerspell." + }, + { + "id": "medical-doctor-001", + "group": "medical", + "priority": "critical", + "input_text": "Tomorrow I will go to the doctor.", + "expected_required_signs": ["TOMORROW", "I", "WILL", "GO", "DOCTOR"], + "expected_forbidden_signs": [], + "expected_markers": ["WILL"], + "expected_order": [["TOMORROW", "GO"], ["TOMORROW", "DOCTOR"]], + "allow_fingerspell": false, + "max_unknown_signs": 0, + "notes": "Time-first ordering plus WILL modal preservation." + }, + { + "id": "medical-child-001", + "group": "medical", + "priority": "critical", + "input_text": "My child is sick.", + "expected_required_signs": ["CHILD", "SICK"], + "expected_forbidden_signs": ["FINISH"], + "expected_markers": [], + "expected_order": [], + "allow_fingerspell": true, + "max_unknown_signs": 1, + "notes": "Present-state description; MY fingerspells." + }, + { + "id": "medical-medicine-001", + "group": "medical", + "priority": "critical", + "input_text": "I need medicine.", + "expected_required_signs": ["I", "NEED", "MEDICINE"], + "expected_forbidden_signs": ["NE"], + "expected_markers": [], + "expected_order": [], + "allow_fingerspell": false, + "max_unknown_signs": 0, + "notes": "Core clinic phrase; everything is in the avatar library." + }, + { + "id": "emergency-ambulance-001", + "group": "emergency", + "priority": "critical", + "input_text": "Please call an ambulance now.", + "expected_required_signs": ["NOW", "PLEASE", "AMBULANCE"], + "expected_forbidden_signs": [], + "expected_markers": [], + "expected_order": [["NOW", "AMBULANCE"]], + "allow_fingerspell": true, + "max_unknown_signs": 1, + "notes": "Time word NOW fronted; CALL fingerspells (not in avatar library)." + }, + { + "id": "emergency-stop-001", + "group": "emergency", + "priority": "critical", + "input_text": "Stop!", + "expected_required_signs": ["STOP"], + "expected_forbidden_signs": [], + "expected_markers": [], + "expected_order": [], + "allow_fingerspell": false, + "max_unknown_signs": 0, + "notes": "Single-sign urgency must survive untouched." + }, + { + "id": "modal-must-001", + "group": "modal", + "priority": "critical", + "input_text": "You must wait here.", + "expected_required_signs": ["YOU", "MUST", "WAIT"], + "expected_forbidden_signs": [], + "expected_markers": ["MUST"], + "expected_order": [], + "allow_fingerspell": true, + "max_unknown_signs": 1, + "notes": "MUST must never be dropped as filler; HERE fingerspells." + }, + { + "id": "modal-can-001", + "group": "modal", + "priority": "critical", + "input_text": "Can you help me?", + "expected_required_signs": ["CAN", "YOU", "HELP"], + "expected_forbidden_signs": [], + "expected_markers": ["CAN"], + "expected_order": [], + "allow_fingerspell": true, + "max_unknown_signs": 1, + "notes": "CAN modal preservation. ME maps to a transformer variant but is not an avatar-library key." + }, + { + "id": "aspect-finish-001", + "group": "aspect", + "priority": "critical", + "input_text": "I finished the work yesterday.", + "expected_required_signs": ["YESTERDAY", "I", "FINISH", "WORK"], + "expected_forbidden_signs": [], + "expected_markers": ["FINISH"], + "expected_order": [["YESTERDAY", "WORK"]], + "allow_fingerspell": false, + "max_unknown_signs": 0, + "notes": "Past completion: FINISH aspect + time-first." + }, + { + "id": "aspect-past-001", + "group": "aspect", + "priority": "critical", + "input_text": "Yesterday I was sick.", + "expected_required_signs": ["YESTERDAY", "I", "SICK", "FINISH"], + "expected_forbidden_signs": [], + "expected_markers": ["FINISH"], + "expected_order": [["YESTERDAY", "SICK"], ["SICK", "FINISH"]], + "allow_fingerspell": false, + "max_unknown_signs": 0, + "notes": "Irregular past ('was') must still produce the FINISH marker, placed after content." + }, + { + "id": "greeting-hello-001", + "group": "greeting", + "priority": "critical", + "input_text": "Hello, how are you?", + "expected_required_signs": ["HELLO", "YOU", "HOW"], + "expected_forbidden_signs": [], + "expected_markers": [], + "expected_order": [["HELLO", "HOW"], ["YOU", "HOW"]], + "allow_fingerspell": false, + "max_unknown_signs": 0, + "notes": "Question word moves to the end — the SASL signature." + }, + { + "id": "question-where-001", + "group": "question", + "priority": "critical", + "input_text": "Where is the hospital?", + "expected_required_signs": ["HOSPITAL", "WHERE"], + "expected_forbidden_signs": [], + "expected_markers": [], + "expected_order": [["HOSPITAL", "WHERE"]], + "allow_fingerspell": false, + "max_unknown_signs": 0, + "notes": "Topic first, WH-question last." + }, + { + "id": "question-name-001", + "group": "question", + "priority": "high", + "input_text": "What is your name?", + "expected_required_signs": ["WHAT"], + "expected_forbidden_signs": [], + "expected_markers": [], + "expected_order": [["NAME", "WHAT"]], + "allow_fingerspell": true, + "max_unknown_signs": 2, + "notes": "YOUR/NAME are transformer-known but not avatar-library keys — they fingerspell. WHAT must be last." + }, + { + "id": "present-no-finish-001", + "group": "regression", + "priority": "critical", + "input_text": "It hurts here.", + "expected_required_signs": ["HURT"], + "expected_forbidden_signs": ["FINISH"], + "expected_markers": [], + "expected_order": [], + "allow_fingerspell": true, + "max_unknown_signs": 2, + "notes": "Regression: plural/3rd-person -s stripping must not mark past tense." + }, + { + "id": "rights-signs-001", + "group": "rights", + "priority": "critical", + "input_text": "I have rights.", + "expected_required_signs": ["I", "RIGHTS"], + "expected_forbidden_signs": ["RIGHT"], + "expected_markers": [], + "expected_order": [], + "allow_fingerspell": false, + "max_unknown_signs": 0, + "notes": "Regression: the library sign RIGHTS must not be singularized into a non-sign." + }, + { + "id": "progressive-001", + "group": "regression", + "priority": "critical", + "input_text": "She is walking home.", + "expected_required_signs": ["WALK", "HOME"], + "expected_forbidden_signs": ["WALKE", "FINISH"], + "expected_markers": [], + "expected_order": [], + "allow_fingerspell": true, + "max_unknown_signs": 1, + "notes": "Regression: -ing stripping must not invent 'WALKE' or mark past tense." + }, + { + "id": "daily-hungry-001", + "group": "daily", + "priority": "high", + "input_text": "I am hungry.", + "expected_required_signs": ["I", "HUNGRY"], + "expected_forbidden_signs": ["AM", "FINISH"], + "expected_markers": [], + "expected_order": [], + "allow_fingerspell": false, + "max_unknown_signs": 0, + "notes": "Copula dropped, state sign kept." + }, + { + "id": "daily-water-001", + "group": "daily", + "priority": "high", + "input_text": "I want water.", + "expected_required_signs": ["I", "WANT", "WATER"], + "expected_forbidden_signs": [], + "expected_markers": [], + "expected_order": [], + "allow_fingerspell": false, + "max_unknown_signs": 0, + "notes": "KNOWN LIMITATION: rules keep SVO order here; true SASL would prefer WATER WANT (SOV). Order pair intentionally not asserted — revisit with the SASL interpreter (D15)." + }, + { + "id": "rights-police-001", + "group": "rights", + "priority": "high", + "input_text": "The police will help you.", + "expected_required_signs": ["POLICE", "WILL", "HELP", "YOU"], + "expected_forbidden_signs": [], + "expected_markers": ["WILL"], + "expected_order": [], + "allow_fingerspell": false, + "max_unknown_signs": 0, + "notes": "Article dropped, WILL preserved." + }, + { + "id": "question-when-001", + "group": "question", + "priority": "high", + "input_text": "When can I go home?", + "expected_required_signs": ["CAN", "I", "GO", "HOME", "WHEN"], + "expected_forbidden_signs": [], + "expected_markers": ["CAN"], + "expected_order": [["GO", "WHEN"], ["HOME", "WHEN"]], + "allow_fingerspell": false, + "max_unknown_signs": 0, + "notes": "WH-word to the end even with a modal present." + }, + { + "id": "unknown-fingerspell-001", + "group": "unknown_word", + "priority": "high", + "input_text": "The xylophone is broken.", + "expected_required_signs": ["XYLOPHONE"], + "expected_forbidden_signs": [], + "expected_markers": [], + "expected_order": [], + "allow_fingerspell": true, + "max_unknown_signs": 3, + "notes": "Unknown noun survives verbatim for fingerspelling — never dropped, never replaced." + }, + { + "id": "hostile-input-001", + "group": "hostile_input", + "priority": "critical", + "input_text": "", + "expected_required_signs": [], + "expected_forbidden_signs": [], + "expected_markers": [], + "expected_order": [], + "allow_fingerspell": true, + "max_unknown_signs": 99, + "notes": "Must not crash and must not echo executable markup as a crash trace. Output content unconstrained." + }, + { + "id": "rights-interpreter-001", + "group": "rights", + "priority": "exploratory", + "input_text": "My employer refused an interpreter.", + "expected_required_signs": [], + "expected_forbidden_signs": [], + "expected_markers": ["FINISH"], + "expected_order": [], + "allow_fingerspell": true, + "max_unknown_signs": 99, + "notes": "KNOWN GAP: EMPLOYER/INTERPRETER/REFUSE are not avatar-library signs, and 'refused' stems to 'REFUS'. Tracked for Phase 3 vocabulary expansion + stemmer dictionary. Only the past marker is asserted." + }, + { + "id": "greeting-thanks-001", + "group": "greeting", + "priority": "high", + "input_text": "Thank you very much.", + "expected_required_signs": ["THANK YOU", "VERY"], + "expected_forbidden_signs": [], + "expected_markers": [], + "expected_order": [], + "allow_fingerspell": true, + "max_unknown_signs": 2, + "notes": "Multi-word library sign THANK YOU must be matched as one unit by the splitter." + } + ] +} diff --git a/tests/test_golden_reconstruction.py b/tests/test_golden_reconstruction.py new file mode 100644 index 0000000..25d94fd --- /dev/null +++ b/tests/test_golden_reconstruction.py @@ -0,0 +1,30 @@ +""" +Golden sign-reconstruction gate (MASTER_PLAN Phase 1). + +Runs deaf→hearing fixtures through the deterministic fallback +(simple_signs_to_english) and asserts INTENT preservation: expected concepts +present, forbidden inversions absent. Fluency is not scored — meaning is. + +Run with: pytest tests/test_golden_reconstruction.py -v +""" + +import pytest + +from backend.services.sign_reconstruction import simple_signs_to_english + +from tests.golden import scoring + +CASES = scoring.load_cases("sign_reconstruction_cases.json") + + +@pytest.mark.parametrize("case", CASES, ids=[c["id"] for c in CASES]) +def test_reconstruction_case(case): + text = simple_signs_to_english(case["input_signs"]).lower() + + assert text.strip(), f"{case['id']}: empty output for valid sign input" + + missing = [w for w in case["expected_text_contains"] if w.lower() not in text] + assert not missing, f"{case['id']}: intent words {missing} missing from {text!r}" + + inversions = [w for w in case["forbidden_text_contains"] if w.lower() in text] + assert not inversions, f"{case['id']}: forbidden phrase {inversions} present in {text!r}" diff --git a/tests/test_golden_translation.py b/tests/test_golden_translation.py new file mode 100644 index 0000000..e02f0dd --- /dev/null +++ b/tests/test_golden_translation.py @@ -0,0 +1,63 @@ +""" +Golden translation gate (MASTER_PLAN Phase 1 — the Definition of Working). + +Runs every fixture in tests/golden/translation_cases.json through the +DETERMINISTIC rules pipeline (no Ollama, no network) and scores required +signs, forbidden signs, markers, word order, and avatar-library compliance. + +Gates: +- every `critical` case must pass fully +- `high` cases must pass fully (they were verified against real output) +- `exploratory` cases only assert their named expectations (known gaps) +- the aggregate baseline is printed so regressions are visible in CI logs + +Run with: pytest tests/test_golden_translation.py -v +""" + +import pytest + +from sasl_transformer.models import TranslationRequest +from sasl_transformer.transformer import SASLTransformer + +from tests.golden import scoring + +CASES = scoring.load_cases("translation_cases.json") + + +@pytest.fixture(scope="module") +def transformer(): + return SASLTransformer() + + +def _translate(transformer, text): + response = transformer.translate_with_rules(text, TranslationRequest(english_text=text)) + return [tok.gloss for tok in response.tokens] + + +@pytest.mark.parametrize("case", CASES, ids=[c["id"] for c in CASES]) +def test_golden_case(case, transformer): + glosses = _translate(transformer, case["input_text"]) + result = scoring.score_case(case, glosses) + assert result["passed"], ( + f"{case['id']} failed: missing={result['missing_required']} " + f"forbidden={result['forbidden_hits']} markers={result['missing_markers']} " + f"order={result['order_violations']} unknown={result['unknown_signs']} " + f"glosses={result['glosses']}" + ) + + +def test_baseline_summary(transformer, capsys): + """Not a gate — prints the rules-only baseline so CI logs carry the number to beat.""" + results = [ + scoring.score_case(case, _translate(transformer, case["input_text"])) + for case in CASES + ] + summary = scoring.summarize(results) + with capsys.disabled(): + print( + f"\n[GOLDEN BASELINE] rules-only: {summary['cases_passed']}/{summary['cases_total']} passed, " + f"critical {summary['critical_passed']}/{summary['critical_total']}, " + f"mean required-sign recall {summary['mean_required_recall']}" + ) + # The floor: criticals may never regress below full pass. + assert summary["critical_passed"] == summary["critical_total"]