fix(observe): confidence bands stop flagging legible text as 'low'

simdrive/CHANGELOG.md

@@ -57,6 +57,27 @@
   stack frames as basenames, version, OS) — no messages, paths, or PII — and
   never sent anywhere. Disable with `HEKA_CRASH_SINK_DISABLED=1`.
 
+### Fixed — `observe` confidence bands no longer flag legible on-screen text as `low`
+
+- **The english-likeness fence now uses the host dictionary, not a ~370-word
+  seed.** `observe` clamps a mark's `confidence`/`confidence_band` down when its
+  text fails a dictionary check — the fence that catches stylized-cover
+  gibberish OCR'd as plausible fake words. But the built-in wordlist was tiny
+  and hand-maintained, so everyday UI copy that simply wasn't in it —
+  "Discard", "Download", "Sending your ticket", "Support will reply within 1
+  business day." — was mislabeled `low` at a perfect OCR score. A band that
+  flags correct reads as low-quality is one agents learn to ignore. simdrive now
+  unions the system word list (`/usr/share/dict/words`, ~235k words on macOS and
+  most Linux) with the curated seed, so real words pass and only genuine
+  gibberish stays `low`. Hosts without a system list fall back to the seed
+  unchanged — no hard dependency.
+- **Typographic punctuation no longer trips the charset gate.** Sentences using
+  an em-dash, ellipsis, or curly apostrophe — "I haven't seen exactly that
+  before —", "OK — let's start fresh.", "Looking into this…" — were rejected
+  before the dictionary check ever ran and clamped to `low`. Those characters
+  are real text; the fence now accepts them while still rejecting OCR-noise
+  symbols. Gibberish separated by an em-dash still lands `low`.
+
 ## [1.0.0b12] — 2026-06-23
 
 ### Fixed — host-AX `set_text` / `perform_accessibility_action` now reach app content on iOS 26

simdrive/src/simdrive/som.py

@@ -99,6 +99,50 @@
 )
 
 
+def _load_dictionary() -> frozenset[str]:
+    """The effective wordlist the english-likeness fence checks against.
+
+    The curated `_ENGLISH_WORDS` seed above is a *portable fallback*, not the
+    real dictionary: a hand-maintained ~370-word list loses to real UI copy
+    faster than words can be added (F#18 was the third such whack-a-mole —
+    'Wi-Fi', 'Bluetooth', 'General' all clamped legible system text to 'low').
+    On any host that ships a system word list (macOS `/usr/share/dict/words`
+    → web2 with ~235k entries; most Linux with the `words` package), union it
+    in so plainly-English strings — "Discard", "Sending your ticket…",
+    "Reference", "Support will reply within 1 business day." — stop failing the
+    dictionary gate and getting mislabeled 'low'.
+
+    Adding real words can never let gibberish pass the fence (gibberish stays
+    absent from the dict), so this only removes false negatives. Best-effort and
+    portable: if no system list exists (minimal CI containers), the seed alone
+    is used and behavior degrades to the pre-existing gate rather than erroring.
+    """
+    words = set(_ENGLISH_WORDS)
+    for path in ("/usr/share/dict/words", "/usr/share/dict/web2"):
+        try:
+            with open(path, encoding="utf-8", errors="ignore") as fh:
+                for line in fh:
+                    tok = line.strip().lower()
+                    if len(tok) >= 2 and tok.isalpha():
+                        words.add(tok)
+            break
+        except OSError:
+            continue
+    # Modern app/UI compounds the 1934 web2 corpus predates but that appear
+    # constantly in real screens. (web2 has 'audiobook' via the seed but not
+    # 'download'.) Real words only — safe to add.
+    words.update({
+        "download", "downloads", "upload", "uploads", "login", "logout",
+        "signin", "signout", "username", "wifi", "online", "offline",
+        "email", "app", "apps", "reset", "resets", "resync", "rescan",
+    })
+    return frozenset(words)
+
+
+# Loaded once at import. The seed stays the fallback; this is what the fence uses.
+_DICTIONARY: frozenset[str] = _load_dictionary()
+
+
 # v0.3.0a3 — semantic-name → likely-OCR-misread lookup for icon glyphs.
 # OCR rasterizes the search magnifying glass as "Q/" / "Q." / "O" depending
 # on resolution. Stable IDs catch it for replay against the same screen, but
@@ -127,7 +171,12 @@
 }
 
 
-_ALLOWED_PUNCT = set("-'.,!?:")
+# Straight ASCII punctuation PLUS the typographic characters real UI copy uses
+# constantly: em/en dashes, the ellipsis, and curly quotes. Without these the
+# charset fence rejected plainly-English sentences before they ever reached the
+# dictionary check — "I haven't seen exactly that before —" and "OK — let's
+# start fresh." both clamped to 'low' purely on the em-dash / U+2019 apostrophe.
+_ALLOWED_PUNCT = set("-'.,!?:") | set("—–…’‘“”")
 _VOWELS = set("aeiouAEIOU")
 _TOKEN_RE = re.compile(r"\s+")
 
@@ -143,7 +192,7 @@ def _is_english_like_token(token: str) -> bool:
         return True  # punctuation-only fragment
     if len(cleaned) <= 3:
         return True
-    return cleaned in _ENGLISH_WORDS
+    return cleaned in _DICTIONARY
 
 
 def _english_likeness(text: str) -> bool:

simdrive/tests/test_b5_domain_c_text_targeting.py

@@ -527,3 +527,66 @@ def test_low_confidence_non_english_mark_stays_low(self):
         assert m.confidence_band == "low", (
             "Non-English OCR reads at any confidence should remain 'low' (regression guard)"
         )
+
+
+class TestSystemDictionaryFence:
+    """The english-likeness fence is backed by the host word list (unioned with
+    the curated seed), not the ~370-word seed alone.
+
+    Regression from a Palace triage-bot sim-drive: legible UI copy —
+    'Discard', 'Sending your ticket…', 'Support will reply within 1 business
+    day.' — was clamped to 'low' at raw 1.0 because the words were absent from
+    the tiny seed, and em-dash / ellipsis / curly-quote strings were rejected by
+    the charset gate before the dictionary ever ran. A confidence band that
+    flags perfectly-correct reads as 'low' is worse than none — agents can't
+    trust it. This guards both fixes at once.
+    """
+
+    def test_dictionary_is_superset_of_curated_seed(self):
+        """Whatever the host, the effective dictionary must never be smaller
+        than the curated fallback (portability floor)."""
+        from simdrive.som import _DICTIONARY, _ENGLISH_WORDS
+        assert _ENGLISH_WORDS <= _DICTIONARY
+        # On a dev/CI host with /usr/share/dict/words this is vastly larger;
+        # even without it the seed keeps the gate working.
+        assert len(_DICTIONARY) >= len(_ENGLISH_WORDS)
+
+    def test_common_ui_words_not_low(self):
+        """Everyday English UI vocabulary absent from the old seed must land
+        'high' at raw 1.0, not 'low'."""
+        from simdrive.som import Mark
+        for text in ("Discard", "Download", "Sending your ticket",
+                     "Support will reply within 1 business day.",
+                     "Describe what's happening"):
+            m = Mark(id=1, x=0, y=0, w=200, h=40, text=text,
+                     confidence=1.0, raw_confidence=1.0)
+            assert m.confidence_band != "low", (
+                f"legible UI copy {text!r} clamped to 'low' — dictionary fence "
+                f"false-negative (got band={m.confidence_band!r})"
+            )
+
+    def test_typographic_punctuation_survives_charset_gate(self):
+        """Em-dash, ellipsis, and curly apostrophes are real text, not OCR
+        noise — sentences using them must reach the dictionary check, not get
+        rejected by the charset fence and clamped to 'low'."""
+        from simdrive.som import Mark
+        for text in ("I haven’t seen exactly that before —",
+                     "OK — let’s start fresh.",
+                     "Looking into this…"):
+            m = Mark(id=1, x=0, y=0, w=300, h=40, text=text,
+                     confidence=1.0, raw_confidence=1.0)
+            assert m.confidence_band != "low", (
+                f"typographic-punctuation string {text!r} rejected by the "
+                f"charset gate (got band={m.confidence_band!r})"
+            )
+
+    def test_gibberish_with_typographic_punct_still_low(self):
+        """Widening the charset must not let stylized-cover gibberish through:
+        an em-dash between fake words stays 'low'."""
+        from simdrive.som import Mark
+        m = Mark(id=1, x=0, y=0, w=300, h=40,
+                 text="Sary — liotex canxz zqxw",
+                 confidence=1.0, raw_confidence=1.0)
+        assert m.confidence_band == "low", (
+            "gibberish must stay 'low' even with allowed typographic punctuation"
+        )