From e3934ab388cee3e099ca822b8f16fa7aad136fd0 Mon Sep 17 00:00:00 2001 From: Jiri Puc Date: Wed, 29 Jul 2026 22:50:37 +0200 Subject: [PATCH] fix: make D32 union grounding token-tolerant Resolve the tolerant-matching backlog item exposed by the conv-26 E2 07h ledger: 144 grounding rejections, dominated by prompt-mandated attribution and decontextualization scaffolding. Keep all content and numeric tokens source-bound, skip empty additions, and record failed tokens. --- decisions.md | 52 +++++-- plan/designs/e2_e3_claims_relations_design.md | 46 +++++- src/rememberstack/workers/e1.py | 20 +-- src/rememberstack/workers/e2.py | 146 ++++++++++++++++-- .../workers/test_claimify_loss_ledger.py | 142 ++++++++++++++++- 5 files changed, 363 insertions(+), 43 deletions(-) diff --git a/decisions.md b/decisions.md index 89d6521d..f742c5be 100644 --- a/decisions.md +++ b/decisions.md @@ -670,12 +670,12 @@ claiming row — refined 2026-07-18). **Decision.** A claim stores both a standalone `claim_text` and a verbatim `source_span` + character offsets, plus an `added_context[]` list naming each added substring's bundle source. Acceptance layers, cheapest first: (1) deterministic **anchor** — the source span is a real slice of the chunk; (2) -deterministic **window-membership** — every added substring verbatim-exists in the union of -source-derived bundle elements, while its declared source tag is advisory provenance (rejects -fabrication, not mislabeling); (3) an in-call **entailment self-verdict** (incl. the "*X said* Y -entails *X said Y*, not *Y*" rule); (4) a **sampled independent** entailment audit (never per-claim). -Replaces the verbatim-substring gate, which is incompatible with decontextualization. No external -knowledge. +deterministic **window-membership** — every content token in non-empty added text must occur in the +union of source-derived bundle elements, while only a closed set of functional scaffolding tokens may +be absent and the declared source tag remains advisory provenance (rejects fabrication, not +mislabeling); (3) an in-call **entailment self-verdict** (incl. the "*X said* Y entails *X said Y*, +not *Y*" rule); (4) a **sampled independent** entailment audit (never per-claim). Replaces the +verbatim-substring gate, which is incompatible with decontextualization. No external knowledge. **Context.** A decontextualized claim is a rewrite, so it is never a verbatim substring; grounding must be provenance + entailment, as every surveyed decompose-then-verify system does. (C6.) @@ -690,6 +690,39 @@ tagged them `header` (258) or `prefix` (99), and only 27 claims from 19 document **Section summaries remain outside the union (the stored prefix, though LLM-derived, is a designed union member per D79's accepted second-order channel); D79 consumption rules are unchanged.** +**Amendment (2026-07-29, token-tolerant union grounding).** Layer 2 checks an addition at token +grain instead of requiring its whole connective phrase to occur verbatim. It tokenizes Unicode words +and punctuation and splits possessives (`Caroline's` → `caroline` + `'s`). Every token must either +occur case-insensitively at a word boundary somewhere in the same source-derived union or belong to +this closed functional allowlist: + +- attribution scaffolding: `said`, `says`, `saying`, `asked`, `asks`, `told`, `tells`, `mentioned`, + `mentions`, `wrote`, `writes`, `according`; +- pure function words: `that`, `the`, `a`, `an`, `of`, `to`, `in`, `on`, `at`, `and`, `or`, `is`, + `was`, `were`, `be`, `been`, `she`, `he`, `they`, `her`, `his`, `their`, `it`, `its`, `this`, + `these`, `those`, `with`, `for`, `as`, `by`, `from`; +- punctuation: `,`, `.`, `:`, `;`, straight or curly single/double quote tokens, and `'s`. + +Empty or whitespace-only additions are no-ops. **Numeric tokens are never allowlisted**: a number +such as `2022` must occur in the union or the addition is rejected, preserving #158's rule that a +computed date cannot enter claim text through `added_context`. Proper names and every other content +noun, verb, or adjective likewise always require a union match; `Melanie` therefore passes beside +the colon in a `Melanie:` speaker label, while an absent `Paris`, `pride`, or `parade` does not. +The preserved invariant is that **every content token of every addition remains traceable verbatim +(ignoring case) to source-derived bundle text**. The allowlist can supply grammar and +attribution/decontextualization connective tissue, never outside facts. + +This amendment resolves a measured contradiction in the conv-26 GLM-5.2 E2 07h loss ledger: +of 144 `grounding_rejected` decisions, the dominant class (about 40–60%) was scaffolding the prompt +itself mandates — preserve attribution ("X said Y") and resolve pronouns/possessives — rejected only +because exact connective strings such as `said` (13 rows), `Caroline said` (9), `said she` (7), or +`Caroline's` (5) did not occur whole in a bundle element. Thirteen empty additions were also rejected +despite adding nothing. Gold facts died as a result: the source turn `Melanie: Yeah, I painted that +lake sunrise last year!` yielded a correct attributed claim, then layer 2 rejected +`Melanie said, ` because the source used a speaker-label colon. Token-grain matching admits that +prompt-required scaffolding without weakening content or numeric traceability. Rejections still drop +the claim and are ledgered; `edit_detail.failed_tokens` names the tokens that failed. + **Refined by D65 (media).** For media-derived documents grounding is **two hops**: the anchor (layer 1) proves the claim derives from the *representation* (document.md); it cannot prove the ASR heard or the VLM saw correctly. The layer-4 sampled audit therefore becomes @@ -890,9 +923,10 @@ files and K pages link to each other, in both directions, as consumers — never `claim_valid_from` / `claim_valid_until`, plus a `claim_valid_precision` (year/quarter/day/…/open/ unknown) and a `claim_valid_kind` (proposition-validity vs. event-time vs. measurement-period). It is the structured form of the date decontextualization already resolves into the claim text ("launched -*in 2024*"), emitted in the same E2 call and **grounded** by the existing window-membership check (the -date must verbatim-exist in the bundle, D32). It is **evidence about *when***, epistemically identical -to `claim_text` (evidence about *what*) and `source_span` (evidence about *where in the source*). +*in 2024*"), emitted in the same E2 call. Date text introduced through `added_context` remains +**grounded** by D32 — every numeric token must exist in the source-derived union — while #158 governs +computed structured dates separately. It is **evidence about *when***, epistemically identical to +`claim_text` (evidence about *what*) and `source_span` (evidence about *where in the source*). Adjudicated, current-fact validity stays **exclusively on relations** (`valid_from`/`valid_until` + `invalidated_at`, D3). diff --git a/plan/designs/e2_e3_claims_relations_design.md b/plan/designs/e2_e3_claims_relations_design.md index 282fd7e8..0ae99320 100644 --- a/plan/designs/e2_e3_claims_relations_design.md +++ b/plan/designs/e2_e3_claims_relations_design.md @@ -147,13 +147,45 @@ decontextualizations, while names present verbatim in TARGET CHUNK turn lines we **Section summaries and every other LLM-orientation text remain excluded from the union**, so the fact-injection defense is unchanged and D79 consumption rules are unchanged. +**Amendment (2026-07-29, token-tolerant union grounding):** the membership unit is now a token, +not the whole added connective phrase. E2 tokenizes Unicode words and punctuation, splitting a +possessive such as `Caroline's` into `caroline` and `'s`. A token passes when the same token occurs +case-insensitively at a word boundary in any source-derived union element, or when it belongs to this +closed functional allowlist: + +- attribution scaffolding: `said`, `says`, `saying`, `asked`, `asks`, `told`, `tells`, `mentioned`, + `mentions`, `wrote`, `writes`, `according`; +- pure function words: `that`, `the`, `a`, `an`, `of`, `to`, `in`, `on`, `at`, `and`, `or`, `is`, + `was`, `were`, `be`, `been`, `she`, `he`, `they`, `her`, `his`, `their`, `it`, `its`, `this`, + `these`, `those`, `with`, `for`, `as`, `by`, `from`; +- punctuation: `,`, `.`, `:`, `;`, straight or curly single/double quote tokens, and `'s`. + +Empty or whitespace-only additions add no information and are skipped. Numeric tokens are **never** +allowlisted: `in 2022` fails when `2022` is absent from the union and passes only when `2022` occurs +there. All proper names and other content nouns, verbs, and adjectives obey the same source-only +rule. Thus `Melanie said, ` passes against a `Melanie:` speaker label (`said` and punctuation are +functional; `Melanie` has a word-boundary match), while `in Paris` fails when `Paris` is absent. +Every content token of every accepted addition therefore remains traceable verbatim, ignoring case, +to source-derived bundle text; token tolerance creates no outside-knowledge channel. + +The reason is a measured prompt/gate contradiction. The prompt requires attributed claims to stay +attributed and pronouns/possessives to be decontextualized, but whole-string membership rejected the +resulting grammar because source transcripts use forms such as speaker-label colons rather than +`X said`. In the conv-26 GLM-5.2 E2 07h ledger, 144 claims were grounding-rejected and this mandated +scaffolding was the dominant class (about 40–60%); 13 empty additions were rejected too. A gold claim +from `Melanie: Yeah, I painted that lake sunrise last year!` died solely because its addition was +`Melanie said, `. Failed additions still reject and drop the claim, and their +`grounding_rejected.edit_detail` now includes `failed_tokens` for diagnosis. + Acceptance layers four checks, cheapest first: 1. **Anchor** (deterministic): the `source_span` must be a real, in-bounds slice of the target chunk — a check the model cannot talk its way past. -2. **Window-membership** (deterministic): every *added* substring must verbatim-exist somewhere in - the source-derived union above. The attribution tag is advisory. A claim that invents "in San - Francisco" with no union member containing it is rejected. +2. **Window-membership** (deterministic): tokenize every non-empty addition; every content and numeric + token must occur in the source-derived union above, while only the closed functional allowlist may + supply absent scaffolding. The attribution tag is advisory. A claim that invents `in San + Francisco` with neither content token in the union is rejected and records `san` and `francisco` + as its failed tokens. 3. **Entailment self-verdict** (in-call, ~free): the model asserts the chunk + bundle entail the claim; includes the rule that "*X said* Y" entails "X said Y", not "Y". 4. **Sampled independent audit** (offline, not per-claim): a separate judge re-checks a sample, because @@ -169,9 +201,9 @@ Acceptance layers four checks, cheapest first: at the converter's disclosed precision — `media_design.md` §4.) So in the example, `"Project Atlas launched last year"` is accepted: its anchor is the verbatim "It -launched last year", and "Project Atlas" (→ neighbour) is the only `added_context` entry. The resolved -bounds are emitted only in the structured valid-time fields and do not enter the membership gate. -The attributed stance is grounded separately. +launched last year", and "Project Atlas" (→ neighbour) is the only `added_context` entry. Both content +tokens occur in the source-derived union. The resolved bounds are emitted only in the structured +valid-time fields and do not enter the membership gate. The attributed stance is grounded separately. ### 3.4 Nothing is silently lost (D33, D35) @@ -198,7 +230,7 @@ lands a claim is indistinguishable from a keep that produced a claim the gate re | `decision_type` | When written | `source_span` | `edit_detail` | |---|---|---|---| | `claimify_omitted` | A kept Selection span for which Claimify returned **no claim at all** (the model simply skipped it). One row per dead keep. | The Selection span. | null | -| `grounding_rejected` | A Claimify-returned claim rejected by a D32 gate. One row per rejected claim. | The claim's returned `source_span` (even if not findable in the chunk). | `{"gate": "span_not_found" \| "outside_kept_ranges" \| "added_context_unverified", "claim_span": }`; for `added_context_unverified` also `{"kind": ..., "text": , "searched_elements": [...]}`. | +| `grounding_rejected` | A Claimify-returned claim rejected by a D32 gate. One row per rejected claim. | The claim's returned `source_span` (even if not findable in the chunk). | `{"gate": "span_not_found" \| "outside_kept_ranges" \| "added_context_unverified", "claim_span": }`; for `added_context_unverified` also `{"kind": ..., "text": , "searched_elements": [...], "failed_tokens": [...]}`. | **Invariant — every kept span is accounted for end-to-end.** Two independent rules (revised 2026-07-27 after review — one keep can decompose into several returned claims with mixed fates, diff --git a/src/rememberstack/workers/e1.py b/src/rememberstack/workers/e1.py index 1e64fae0..8ec63b70 100644 --- a/src/rememberstack/workers/e1.py +++ b/src/rememberstack/workers/e1.py @@ -57,16 +57,18 @@ 07b pins temperature=0.0 — generation parameters are part of provenance. 07c adds D79's bounded current-generation section-summary orientation.""" -E2_EXTRACTOR_VERSION: Final = f"e2-extract-2026.07h:union-grounding-1:temporal-anchor-1:{SECTION_ORIENTATION_VERSION}" +E2_EXTRACTOR_VERSION: Final = f"e2-extract-2026.07i:token-union-grounding-1:temporal-anchor-1:{SECTION_ORIENTATION_VERSION}" """The extractor generation baked into extraction_input_hash (D56); the E2 -stage (WP-1.3) binds its handler to this same constant. 07h requires relative -temporal expressions to resolve against an in-document absolute anchor into -structured D41 valid-time while claim text keeps the source wording (#158); -07g makes D32 layer-2 grounding union-based across source-derived bundle texts -with advisory source tags; 07f adds D79 summary orientation to the bundle -without making summaries hash or grounding inputs; 07e ledgers Claimify -omissions and grounding-gate rejections on the D33 transcript (#161); 07d -pinned temperature=0.0 on the Selection call (Claimify already carried it).""" +stage (WP-1.3) binds its handler to this same constant. 07i makes D32 layer-2 +union grounding token-tolerant for closed functional scaffolding while keeping +every content and numeric token source-bound; 07h requires relative temporal +expressions to resolve against an in-document absolute anchor into structured +D41 valid-time while claim text keeps the source wording (#158); 07g makes D32 +layer-2 grounding union-based across source-derived bundle texts with advisory +source tags; 07f adds D79 summary orientation to the bundle without making +summaries hash or grounding inputs; 07e ledgers Claimify omissions and +grounding-gate rejections on the D33 transcript (#161); 07d pinned +temperature=0.0 on the Selection call (Claimify already carried it).""" _PREFIX_PROMPT_TEMPLATE: Final = ( "In one sentence, state where this passage sits in the document — " diff --git a/src/rememberstack/workers/e2.py b/src/rememberstack/workers/e2.py index bc65de24..a9a5967a 100644 --- a/src/rememberstack/workers/e2.py +++ b/src/rememberstack/workers/e2.py @@ -4,8 +4,9 @@ drop — drops and flags go to the D33 ledger), then one fused call decontextualizes, decomposes, and self-grounds the keeps. The deterministic grounding gate (D32 layers 1-2) accepts a claim only if its verbatim source -span anchors inside the chunk and every added substring exists in the union of -the bundle's source-derived texts. The model's source tag is advisory provenance, +span anchors inside the chunk and every content token in added text exists in +the union of the bundle's source-derived texts. A closed set of functional +scaffolding tokens is permitted; the model's source tag is advisory provenance, not an acceptance boundary. Every kept span ends in accepted claim(s), grounding_rejected row(s), or a claimify_omitted row so Claimify-stage losses are never silent (#161). @@ -17,6 +18,7 @@ from datetime import UTC from enum import StrEnum import logging +import re from typing import Final from uuid import UUID from uuid import uuid4 @@ -63,6 +65,70 @@ # keeps the ledger row small without hiding the gate identity (#161). _LEDGER_SPAN_MAX: Final = 512 +_ADDED_CONTEXT_TOKEN_RE: Final = re.compile(r"\w+|['’][sS](?!\w)|[^\w\s]") +_WORD_TOKEN_RE: Final = re.compile(r"\w+") + +_ADDED_CONTEXT_FUNCTIONAL_ALLOWLIST: Final = frozenset( + { + "said", + "says", + "saying", + "asked", + "asks", + "told", + "tells", + "mentioned", + "mentions", + "wrote", + "writes", + "according", + "that", + "the", + "a", + "an", + "of", + "to", + "in", + "on", + "at", + "and", + "or", + "is", + "was", + "were", + "be", + "been", + "she", + "he", + "they", + "her", + "his", + "their", + "it", + "its", + "this", + "these", + "those", + "with", + "for", + "as", + "by", + "from", + ",", + ".", + ":", + ";", + '"', + "'", + "“", + "”", + "‘", + "’", + "'s", + } +) +"""Closed non-content vocabulary tolerated by D32 layer-2 token membership.""" + _SELECTION_PROMPT: Final = """You are the Selection stage of a claim extractor. Judge every proposition in the TARGET CHUNK: keep statements making a specific, verifiable proposition (state, event, decision, quantity, policy, relationship). @@ -385,6 +451,7 @@ class GroundingRejection: kind: str | None = None text: str | None = None searched_elements: tuple[str, ...] = () + failed_tokens: tuple[str, ...] = () def _grounded_claim( @@ -402,16 +469,19 @@ def _grounded_claim( Layer 1 (anchor): the source span must be a real in-bounds slice of the target chunk, and must overlap a span Selection kept — the fused call can - never resurrect a dropped proposition. Layer 2 (window membership): every - added substring must verbatim-exist in the union of source-derived bundle - texts. The model's ``source_kind`` is preserved as advisory provenance but - cannot reject a grounded addition by being wrong. Section summaries are - excluded from this union (the stored prefix, though LLM text, is a - designed union member — D79's accepted second-order channel). A failed check - returns which gate fired so the D33 ledger can record - ``grounding_rejected`` (#161). Semantic invention behind a real span is - layer-3/4 territory: the in-call self-verdict is stored advisory, and the - sampled independent audit owns the honest measurement. + never resurrect a dropped proposition. Layer 2 (window membership): + tokenize each non-empty addition, then require every content token to + appear case-insensitively at a word boundary in the source-derived bundle + union. Only the closed functional allowlist may supply absent scaffolding; + numeric tokens are never allowlisted. The model's ``source_kind`` is + preserved as advisory provenance but cannot reject a grounded addition by + being wrong. Section summaries are excluded from this union (the stored + prefix, though LLM text, is a designed union member — D79's accepted + second-order channel). A failed check returns which gate fired and which + tokens failed so the D33 ledger can record ``grounding_rejected`` (#161). + Semantic invention behind a real span is layer-3/4 territory: the in-call + self-verdict is stored advisory, and the sampled independent audit owns the + honest measurement. """ claim_span = candidate.source_span anchor_at = document_md.find(claim_span, chunk.char_start, chunk.char_end) @@ -432,13 +502,19 @@ def _grounded_claim( source=source, chunks=chunks, index=index, document_md=document_md ) for added in candidate.added_context: - if not any(added.text in text for _, text in grounding_elements): + if not added.text.strip(): + continue + failed_tokens = _failed_added_context_tokens( + text=added.text, grounding_elements=grounding_elements + ) + if failed_tokens: return GroundingRejection( gate=GroundingGate.ADDED_CONTEXT_UNVERIFIED, claim_span=claim_span, kind=added.source_kind, text=added.text, searched_elements=tuple(name for name, _ in grounding_elements), + failed_tokens=failed_tokens, ) valid_from, valid_until, valid_precision, valid_kind = _parse_claim_valid_time( candidate=candidate @@ -636,6 +712,49 @@ def _source_grounding_elements( return tuple(elements) +def _failed_added_context_tokens( + *, text: str, grounding_elements: tuple[tuple[str, str], ...] +) -> tuple[str, ...]: + """Return normalized addition tokens that fail D32 layer-2 membership. + + All numeric tokens must appear in the source union even if the functional + allowlist is later edited incorrectly. Repeated failures are reported once, + in first-seen order, for compact and useful decision-ledger diagnostics. + """ + failed: list[str] = [] + for token in _added_context_tokens(text): + if _token_in_grounding_union( + token=token, grounding_elements=grounding_elements + ): + continue + if any(character.isnumeric() for character in token): + failed.append(token) + elif token not in _ADDED_CONTEXT_FUNCTIONAL_ALLOWLIST: + failed.append(token) + return tuple(dict.fromkeys(failed)) + + +def _added_context_tokens(text: str) -> tuple[str, ...]: + """Tokenize Unicode words and punctuation, keeping possessive ``'s`` whole.""" + return tuple( + "'s" if token.casefold() == "’s" else token.casefold() + for token in _ADDED_CONTEXT_TOKEN_RE.findall(text) + ) + + +def _token_in_grounding_union( + *, token: str, grounding_elements: tuple[tuple[str, str], ...] +) -> bool: + """Case-insensitive source membership with word boundaries for word tokens.""" + if _WORD_TOKEN_RE.fullmatch(token): + token_pattern = re.compile( + rf"(? str: """The deterministic document header shared by every chunk's bundle.""" modified = source.source_modified_at or source.published_at @@ -769,6 +888,7 @@ def _grounding_rejected_decision( edit_detail["kind"] = _truncate_for_ledger(rejection.kind or "") edit_detail["text"] = _truncate_for_ledger(rejection.text or "") edit_detail["searched_elements"] = list(rejection.searched_elements) + edit_detail["failed_tokens"] = list(rejection.failed_tokens) return DecisionRecord( decision_id=uuid4(), deployment_id=source.deployment_id, diff --git a/src/tests/workers/test_claimify_loss_ledger.py b/src/tests/workers/test_claimify_loss_ledger.py index bbc3b574..d2561983 100644 --- a/src/tests/workers/test_claimify_loss_ledger.py +++ b/src/tests/workers/test_claimify_loss_ledger.py @@ -10,6 +10,8 @@ from typing import TYPE_CHECKING from uuid import UUID +import pytest + from rememberstack.adapters.testing import FakeModelProvider from rememberstack.adapters.testing import NoopCostMeter from rememberstack.model import AddedContext @@ -104,11 +106,13 @@ def _chunk(*, document_md: str = _DOC_MD) -> ChunkForEmbedding: ) -def _kept_ranges_for(*spans: str) -> tuple[tuple[int, int], ...]: +def _kept_ranges_for( + *spans: str, document_md: str = _DOC_MD +) -> tuple[tuple[int, int], ...]: """Absolute ranges for the given verbatim spans inside the fixture doc.""" ranges: list[tuple[int, int]] = [] for span in spans: - at = _DOC_MD.find(span) + at = document_md.find(span) assert at >= 0, span ranges.append((at, at + len(span))) return tuple(ranges) @@ -118,18 +122,19 @@ def _ground( *, candidate: CandidateClaim, kept_spans: tuple[str, ...] = (_KEEP_LAUNCH, _KEEP_STANCE), + document_md: str = _DOC_MD, ) -> object: """Run the grounding gate against the fixture document and keeps.""" - chunk = _chunk() + chunk = _chunk(document_md=document_md) return _grounded_claim( candidate=candidate, source=_source(), chunk=chunk, chunks=(chunk,), index=0, - document_md=_DOC_MD, + document_md=document_md, flagged_spans=set(), - kept_ranges=_kept_ranges_for(*kept_spans), + kept_ranges=_kept_ranges_for(*kept_spans, document_md=document_md), ) @@ -197,6 +202,7 @@ def test_gate_added_context_unverified_writes_grounding_rejected() -> None: assert result.gate is GroundingGate.ADDED_CONTEXT_UNVERIFIED assert result.kind == "neighbour" assert result.text == "in San Francisco" + assert result.failed_tokens == ("san", "francisco") decision = _grounding_rejected_decision( source=_source(), chunk=_chunk(), rejection=result @@ -209,9 +215,108 @@ def test_gate_added_context_unverified_writes_grounding_rejected() -> None: "kind": "neighbour", "text": "in San Francisco", "searched_elements": _SEARCHED_SOURCE_ELEMENTS, + "failed_tokens": ["san", "francisco"], } +def test_attribution_scaffolding_passes_across_speaker_colon() -> None: + """Prompt-mandated attribution survives when the speaker is a turn label.""" + document_md = "Melanie: Yeah, I painted that lake sunrise last year!" + result = _ground( + candidate=CandidateClaim( + claim_text='Melanie said, "I painted that lake sunrise last year!"', + source_span="Yeah, I painted that lake sunrise last year!", + added_context=( + AddedContext(text='Melanie said, "', source_kind="neighbour"), + ), + entailment_self_verdict=True, + is_attributed=True, + ), + kept_spans=(document_md,), + document_md=document_md, + ) + + assert isinstance(result, ClaimRecord) + + +@pytest.mark.parametrize("addition", ["Caroline said that", "Caroline's"]) +def test_functional_and_possessive_scaffolding_passes(addition: str) -> None: + """Names stay source-bound while attribution words and possessives pass.""" + result = _ground( + candidate=CandidateClaim( + claim_text=f"{addition} the launch happened.", + source_span="I went to the launch.", + added_context=(AddedContext(text=addition, source_kind="header"),), + entailment_self_verdict=True, + is_attributed=True, + ), + kept_spans=(_KEEP_CAROLINE,), + ) + + assert isinstance(result, ClaimRecord) + + +@pytest.mark.parametrize( + ("addition", "failed_tokens"), + [("at the pride parade", ("pride", "parade")), ("in Paris", ("paris",))], +) +def test_invented_content_tokens_still_fail( + addition: str, failed_tokens: tuple[str, ...] +) -> None: + """Functional glue cannot carry an invented noun through layer 2.""" + result = _ground( + candidate=CandidateClaim( + claim_text=f"Project Atlas launched {addition}.", + source_span="Project Atlas launched in 2024", + added_context=(AddedContext(text=addition, source_kind="neighbour"),), + entailment_self_verdict=True, + ) + ) + + assert isinstance(result, GroundingRejection) + assert result.gate is GroundingGate.ADDED_CONTEXT_UNVERIFIED + assert result.failed_tokens == failed_tokens + + +def test_numeric_token_requires_source_union_membership() -> None: + """No functional allowance can introduce an absent computed year.""" + candidate = CandidateClaim( + claim_text="Project Atlas launched in 2022.", + source_span="Project Atlas launched in 2024", + added_context=(AddedContext(text="in 2022", source_kind="header"),), + entailment_self_verdict=True, + ) + + absent = _ground(candidate=candidate) + assert isinstance(absent, GroundingRejection) + assert absent.failed_tokens == ("2022",) + + document_md = _DOC_MD + "The archive covers 2022.\n" + present = _ground(candidate=candidate, document_md=document_md) + assert isinstance(present, ClaimRecord) + + +@pytest.mark.parametrize( + "addition", + [ + AddedContext.model_construct(text="", source_kind="neighbour"), + AddedContext(text=" \t", source_kind="neighbour"), + ], +) +def test_empty_added_context_is_a_no_op(addition: AddedContext) -> None: + """Empty model output neither rejects nor removes an otherwise valid claim.""" + result = _ground( + candidate=CandidateClaim( + claim_text="Project Atlas launched in 2024.", + source_span="Project Atlas launched in 2024", + added_context=(addition,), + entailment_self_verdict=True, + ) + ) + + assert isinstance(result, ClaimRecord) + + def test_target_chunk_addition_with_wrong_header_label_is_accepted() -> None: """Production regression: an in-turn speaker name survives a wrong tag.""" result = _ground( @@ -386,6 +491,31 @@ def test_handler_rejection_suppresses_omission_only_for_its_keep() -> None: assert rows[DecisionType.CLAIMIFY_OMITTED] == [_KEEP_STANCE] +def test_handler_empty_added_context_survives_without_rejection() -> None: + """Whitespace-only additions are no-ops, not grounding ledger losses.""" + recorder = _run_extract( + selection={"candidates": [{"source_span": _KEEP_LAUNCH, "outcome": "keep"}]}, + claimify={ + "claims": [ + { + "claim_text": "Project Atlas launched in 2024.", + "source_span": "Project Atlas launched in 2024", + "added_context": [{"text": " ", "source_kind": "neighbour"}], + "entailment_self_verdict": True, + } + ] + }, + ) + + assert [claim.claim_text for claim in recorder.claims] == [ + "Project Atlas launched in 2024." + ] + assert not any( + decision.decision_type is DecisionType.GROUNDING_REJECTED + for decision in recorder.decisions + ) + + def test_handler_rejects_summary_only_added_context_fact_injection() -> None: """Summary text is visible orientation but cannot ground an addition.""" recorder = _run_extract( @@ -416,6 +546,7 @@ def test_handler_rejects_summary_only_added_context_fact_injection() -> None: "kind": "summary", "text": "Project Orion", "searched_elements": _SEARCHED_SOURCE_ELEMENTS, + "failed_tokens": ["orion"], } assert not any( decision.decision_type is DecisionType.CLAIMIFY_OMITTED @@ -552,6 +683,7 @@ def test_summary_text_fails_membership_under_every_legal_kind() -> None: assert result.kind == kind assert result.text == text assert list(result.searched_elements) == _SEARCHED_SOURCE_ELEMENTS + assert result.failed_tokens == (text.split()[-1].casefold(),) def test_handler_union_grounding_preserves_loss_ledger_balance() -> None: