diff --git a/.env.example b/.env.example index adca29af..f6d719ac 100644 --- a/.env.example +++ b/.env.example @@ -37,3 +37,8 @@ REMEMBERSTACK_OPENROUTER_API_KEY=replace-before-real-use # reproducible run; do not use a rotating router such as openrouter/free. # REMEMBERSTACK_E2_EXTRACT_MODEL=nvidia/nemotron-3-super-120b-a12b:free # REMEMBERSTACK_E3_NORMALIZE_MODEL=nvidia/nemotron-3-super-120b-a12b:free +# D79 structure seats: STRUCTURER names only the string-anchor fallback +# proposer; the routine skeleton path is deterministic and calls no model. +# REMEMBERSTACK_STRUCTURER_MODEL=openai/gpt-5.6-luna +# REMEMBERSTACK_SKELETON_CHECK_MODEL=z-ai/glm-4.7-flash +# REMEMBERSTACK_ROLE_MODEL=z-ai/glm-4.7-flash diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8cc24335..e0188a48 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -166,7 +166,7 @@ jobs: trap - EXIT test "$(docker compose --env-file .env.example exec -T postgres \ psql -U rememberstack -d rememberstack -Atc \ - 'SELECT version_num FROM alembic_version')" = 'p1_03_0018' + 'SELECT version_num FROM alembic_version')" = 'p1_04_0019' test "$(docker compose --env-file .env.example exec -T postgres \ psql -U rememberstack -d rememberstack -Atc \ 'SELECT count(*) FROM deployments')" = '1' diff --git a/compose.yaml b/compose.yaml index 65f07fcb..dd4826b1 100644 --- a/compose.yaml +++ b/compose.yaml @@ -23,7 +23,10 @@ x-app: &app REMEMBERSTACK_SELFHOST_DEPLOYMENT_SLUG: ${REMEMBERSTACK_SELFHOST_DEPLOYMENT_SLUG} REMEMBERSTACK_SELFHOST_DEPLOYMENT_NAME: ${REMEMBERSTACK_SELFHOST_DEPLOYMENT_NAME} REMEMBERSTACK_SELFHOST_API_PORT: "8000" + # D79: STRUCTURER now names only the anchor-proposal fallback seat. REMEMBERSTACK_STRUCTURER_MODEL: ${REMEMBERSTACK_STRUCTURER_MODEL:-openai/gpt-5.6-luna} + REMEMBERSTACK_SKELETON_CHECK_MODEL: ${REMEMBERSTACK_SKELETON_CHECK_MODEL:-z-ai/glm-4.7-flash} + REMEMBERSTACK_ROLE_MODEL: ${REMEMBERSTACK_ROLE_MODEL:-z-ai/glm-4.7-flash} REMEMBERSTACK_E1_EMBEDDING_MODEL: ${REMEMBERSTACK_E1_EMBEDDING_MODEL:-qwen/qwen3-embedding-8b} REMEMBERSTACK_E1_PREFIX_MODEL: ${REMEMBERSTACK_E1_PREFIX_MODEL:-openai/gpt-5.6-luna} REMEMBERSTACK_E2_EXTRACT_MODEL: ${REMEMBERSTACK_E2_EXTRACT_MODEL:-openai/gpt-5.6-luna} diff --git a/plan/designs/e0_files_design.md b/plan/designs/e0_files_design.md index badd4c31..b1827342 100644 --- a/plan/designs/e0_files_design.md +++ b/plan/designs/e0_files_design.md @@ -279,6 +279,9 @@ unchanged): ``` parse candidate skeleton → compute + persist stats → density / oversized-leaf gates + (a gate demotion enters the SAME fallback → TERMINAL-check sequence below — + every fallback-produced tree faces the terminal judge, however the + document got demoted; clarified 2026-07-28 with Wave-1 implementation) → eligibility (≥ MIN_CHECK_SECTIONS non-root sections; below it: outcome not_run_short, skeleton accepted) → check call → coherent: keep parsed skeleton diff --git a/src/rememberstack/adapters/openrouter.py b/src/rememberstack/adapters/openrouter.py index d5aadc7b..cae61b2e 100644 --- a/src/rememberstack/adapters/openrouter.py +++ b/src/rememberstack/adapters/openrouter.py @@ -25,6 +25,7 @@ from rememberstack.model import ProviderAccountingError from rememberstack.model import ProviderCallError from rememberstack.model import ProviderCallUsage +from rememberstack.model import ProviderInvalidResponseError from rememberstack.model import StructuredResponseModel ResponseT = TypeVar("ResponseT", bound=StructuredResponseModel) @@ -120,6 +121,12 @@ class OpenRouterProviderError(ProviderCallError): """OpenRouter returned an error or an unusable response body.""" +class OpenRouterInvalidResponseError( + OpenRouterProviderError, ProviderInvalidResponseError +): + """OpenRouter completed a generation without a schema-valid output.""" + + class OpenRouterModelProvider: """Structured generations and embeddings over the OpenRouter HTTP API.""" @@ -162,7 +169,7 @@ def generate( try: decoded = json.loads(content) except json.JSONDecodeError as err: - raise OpenRouterProviderError( + raise OpenRouterInvalidResponseError( f"{response_type.__name__}: completion content is not JSON" f" ({_content_fingerprint(content=content)})", usage=usage, @@ -170,7 +177,7 @@ def generate( try: output = response_type.model_validate(decoded) except ValidationError as error: - raise OpenRouterProviderError( + raise OpenRouterInvalidResponseError( f"completion body failed {response_type.__name__} validation", usage=usage, ) from error @@ -204,7 +211,7 @@ def _completion_text( ) content = _completion_content(body=body) if content is None: - raise OpenRouterProviderError( + raise OpenRouterInvalidResponseError( f"{response_type.__name__}: provider returned no completion" f" content ({_completion_diagnosis(body=body)})", usage=usage, diff --git a/src/rememberstack/core/__init__.py b/src/rememberstack/core/__init__.py index 862cc07e..636bdd38 100644 --- a/src/rememberstack/core/__init__.py +++ b/src/rememberstack/core/__init__.py @@ -3,7 +3,9 @@ from rememberstack.core.blockizer import block_hash from rememberstack.core.blockizer import blockize from rememberstack.core.blockizer import BLOCKIZER_VERSION +from rememberstack.core.blockizer import blocks_from_sidecar from rememberstack.core.blockizer import normalized_block_text +from rememberstack.core.blockizer import normalized_heading_title from rememberstack.core.chunker import chunk_content_hash from rememberstack.core.chunker import CHUNKER_VERSION from rememberstack.core.chunker import chunker_version @@ -61,6 +63,18 @@ from rememberstack.core.section_snap import snap_sections from rememberstack.core.storage_routing import HOT_MIME_PREFIXES from rememberstack.core.storage_routing import storage_class_for +from rememberstack.core.structure_skeleton import analyze_skeleton +from rememberstack.core.structure_skeleton import deterministic_section_role +from rememberstack.core.structure_skeleton import LONG_TITLE +from rememberstack.core.structure_skeleton import MAX_FALLBACK_DEPTH +from rememberstack.core.structure_skeleton import MIN_CHECK_SECTIONS +from rememberstack.core.structure_skeleton import parse_heading_skeleton +from rememberstack.core.structure_skeleton import resolve_fallback_skeleton +from rememberstack.core.structure_skeleton import skeleton_hash +from rememberstack.core.structure_skeleton import SKELETON_PARSER_VERSION +from rememberstack.core.structure_skeleton import SKELETON_STATS_VERSION +from rememberstack.core.structure_skeleton import SkeletonAnalysis +from rememberstack.core.structure_skeleton import TINY_FLOOR __all__ = ( "BLOCKIZER_VERSION", @@ -88,12 +102,26 @@ "PredicateSignatureDefinition", "block_hash", "blockize", + "blocks_from_sidecar", "normalized_block_text", + "normalized_heading_title", "HOT_MIME_PREFIXES", "storage_class_for", "source_identity_hash", "SECTION_ROLES", "snap_sections", + "analyze_skeleton", + "deterministic_section_role", + "LONG_TITLE", + "MAX_FALLBACK_DEPTH", + "MIN_CHECK_SECTIONS", + "parse_heading_skeleton", + "resolve_fallback_skeleton", + "SKELETON_PARSER_VERSION", + "SKELETON_STATS_VERSION", + "skeleton_hash", + "SkeletonAnalysis", + "TINY_FLOOR", "DEFAULT_EVIDENCE_COUNT_WEIGHT", "DEFAULT_GRAPH_DISTANCE_WEIGHT", "DEFAULT_RRF_K", diff --git a/src/rememberstack/core/blockizer.py b/src/rememberstack/core/blockizer.py index ccec8d18..7a0a0ee5 100644 --- a/src/rememberstack/core/blockizer.py +++ b/src/rememberstack/core/blockizer.py @@ -18,8 +18,15 @@ from rememberstack.model import Block from rememberstack.model import BlockType -BLOCKIZER_VERSION: Final = "blockizer-2026.07:markdown-it-py-4:gfm-tables" -"""Pins the parser library generation and enabled-extension set (e1 §2).""" +BLOCKIZER_VERSION: Final = ( + "blockizer-2026.07b:markdown-it-py-4:gfm-tables:heading-metadata" +) +"""Pins the parser profile and D79 heading metadata contract. + +The ``07b`` bump changes only ``blocks.json`` metadata. Segmentation, offsets, +normalization for ``block_hash``, and therefore the block/chunk grid are byte +identical to ``blockizer-2026.07:markdown-it-py-4:gfm-tables``. +""" _ATOMIC_CONTAINERS: Final = { "table_open": ("table_close", BlockType.TABLE), @@ -28,6 +35,26 @@ _LIST_OPENERS: Final = frozenset({"bullet_list_open", "ordered_list_open"}) +def blocks_from_sidecar( + *, blocks_doc: dict[str, object], document_md: str +) -> tuple[Block, ...]: + """Load a blocks.json sidecar, re-blockizing when its version is stale. + + The sidecar is a CACHE of the pinned blockizer's output, never a second + source of truth: on a version mismatch (e.g. pre-``07b`` sidecars without + heading metadata) the blocks are re-derived from ``document.md`` instead + of failing validation. The ``07`` → ``07b`` bump is grid-byte-identical — + the recompute changes metadata only — so STRUCTURE/CHUNK work queued or + retried across the upgrade keeps flowing instead of dead-lettering on a + permanently stale representation. Strictness stays on the produce side. + """ + if blocks_doc.get("blockizer_version") == BLOCKIZER_VERSION: + payloads = blocks_doc["blocks"] + assert isinstance(payloads, list) + return tuple(Block.model_validate(payload) for payload in payloads) + return blockize(document_md=document_md) + + def blockize(*, document_md: str) -> tuple[Block, ...]: """Derive the deterministic block sequence from a document.md rendering. @@ -63,6 +90,12 @@ def normalized_block_text(*, raw: str) -> str: return " ".join(composed.split()) +def normalized_heading_title(*, title: str) -> str: + """D79 title normalization: NFKC → casefold → whitespace collapse.""" + compatible = unicodedata.normalize("NFKC", title) + return " ".join(compatible.casefold().split()) + + def block_hash(*, raw: str) -> str: """The block identity: sha256 over the normalized text.""" digest = hashlib.sha256(normalized_block_text(raw=raw).encode("utf-8")) @@ -130,12 +163,16 @@ def _emit( ), ) if token.type == "heading_open": + inline = tokens[index + 1] + heading_title = _heading_text(token=inline) return 3, _block_from_lines( token=token, block_type=BlockType.HEADING, document_md=document_md, line_offsets=line_offsets, ordinal=ordinal, + heading_level=int(token.tag.removeprefix("h")), + heading_title=heading_title, ) if token.type == "paragraph_open": return 3, _block_from_lines( @@ -172,6 +209,8 @@ def _block_from_lines( document_md: str, line_offsets: tuple[int, ...], ordinal: int, + heading_level: int | None = None, + heading_title: str | None = None, ) -> Block: """Build the block record from a token's source line map.""" if token.map is None: @@ -186,4 +225,31 @@ def _block_from_lines( char_start=char_start, char_end=char_start + len(raw), block_hash=block_hash(raw=raw), + heading_level=heading_level, + heading_title=heading_title, + normalized_title=( + normalized_heading_title(title=heading_title) + if heading_title is not None + else None + ), ) + + +def _heading_text(*, token: Token) -> str: + """Render a heading inline token to plain title text without re-tokenizing. + + The canonical markdown-it parse already produced child tokens. Reusing + them preserves D57's single-tokenizer rule while removing emphasis/link + markup from the title used by the parser, stats, and deterministic roles. + """ + if token.children is None: + return token.content.strip() + pieces: list[str] = [] + for child in token.children: + if child.type in {"text", "code_inline"}: + pieces.append(child.content) + elif child.type == "image": + pieces.append(child.content) + elif child.type in {"softbreak", "hardbreak"}: + pieces.append(" ") + return " ".join("".join(pieces).split()) diff --git a/src/rememberstack/core/chunker.py b/src/rememberstack/core/chunker.py index 2028f654..4a887bb1 100644 --- a/src/rememberstack/core/chunker.py +++ b/src/rememberstack/core/chunker.py @@ -22,9 +22,16 @@ from rememberstack.model import PackedChunk from rememberstack.model import SectionSpan -CHUNKER_VERSION: Final = "e1-chunker-2026.07b:whitespace-tokens:anchored:owner-runs" -"""Pins the packing algorithm and the token counter; the full packing -generation additionally encodes the parameter values — see `chunker_version`.""" +CHUNKER_VERSION: Final = ( + "e1-chunker-2026.07c:whitespace-tokens:anchored:owner-runs" + ":blockizer-heading-metadata" +) +"""Pins packing plus its blockizer-contract generation. + +The ``07c`` bump follows D12/D57 because ``blocks.json`` gained D79 heading +metadata. Packing behavior and emitted chunk-grid bytes are unchanged from +``e1-chunker-2026.07b:whitespace-tokens:anchored:owner-runs``. +""" class ChunkerParams(BaseModel): diff --git a/src/rememberstack/core/structure_skeleton.py b/src/rememberstack/core/structure_skeleton.py new file mode 100644 index 00000000..2103a80f --- /dev/null +++ b/src/rememberstack/core/structure_skeleton.py @@ -0,0 +1,667 @@ +"""D79 deterministic skeleton parsing, fallback-anchor disposal, and stats.""" + +from __future__ import annotations + +from collections import Counter +from collections import defaultdict +from collections.abc import Iterable +from dataclasses import dataclass +import hashlib +import json +import math +import re +from typing import Final +import unicodedata + +from rememberstack.core.blockizer import normalized_heading_title +from rememberstack.model import Block +from rememberstack.model import BlockType +from rememberstack.model import FallbackAnchor +from rememberstack.model import SkeletonStats +from rememberstack.model import SnappedSection + +MIN_CHECK_SECTIONS: Final = 3 +"""Minimum non-root section count for a D79 checker call.""" + +TINY_FLOOR: Final = 80 +"""Direct-body characters below this count as tiny in stats version 1.""" + +LONG_TITLE: Final = 120 +"""Normalized-title length reported as long and the checker line cap.""" + +MAX_FALLBACK_DEPTH: Final = 16 +"""Preserves the D57 snap's pathological-recursion ceiling.""" + +SKELETON_STATS_VERSION: Final = "e0-skeleton-stats-2026.07:d79-v2:min3-tiny80-long120" +"""d79-v2: level_jump_count uses raw heading levels ONLY — pairs missing a +raw level are excluded and an all-anchor tree reports null, never a +structurally fake zero from materialized tree depth.""" +"""Pins every formula, zero case, normalization, and named floor below.""" + +SKELETON_PARSER_VERSION: Final = ( + "e0-skeleton-parser-2026.07:d79-heading-stack:block-grid-v1" +) +"""The deterministic heading-stack skeleton producer generation.""" + +_ARABIC_NUMBER = re.compile(r"^(\d+(?:\.\d+)*)(?=$|[\s.)])") +_ROMAN_NUMBER = re.compile(r"^([IVXLCDM]+)(?=$|[\s.)])", re.IGNORECASE) +_ALPHA_NUMBER = re.compile(r"^([A-Z])\.(?=$|\s)", re.IGNORECASE) + + +@dataclass(frozen=True) +class SkeletonAnalysis: + """Stats plus the shared unmasked measurements used by routing/rendering.""" + + stats: SkeletonStats + direct_body_chars: dict[str, int] + leaf_span_chars: dict[str, int] + heading_density: float + oversized_leaf_ratio: float + + +@dataclass +class _HeadingNode: + """Mutable construction node; persistence receives frozen sections.""" + + block: Block + children: list[_HeadingNode] + block_end: int = -1 + + +@dataclass +class _AnchorNode: + """One resolved fallback anchor on the block grid.""" + + proposal: FallbackAnchor + block_start: int + block_end: int + children: list[_AnchorNode] + + +def parse_heading_skeleton( + *, blocks: tuple[Block, ...], title: str | None, markdown_chars: int +) -> tuple[SnappedSection, ...]: + """Build a deterministic tree from canonical heading blocks. + + A heading's parent is the nearest preceding heading with a strictly lower + raw markdown level. Skipped levels therefore add one materialized edge, + not placeholder sections. Each span begins at its heading block and ends + immediately before the next heading at the same or a lower level. + """ + root = _root_section(blocks=blocks, title=title, markdown_chars=markdown_chars) + headings = tuple(block for block in blocks if block.type is BlockType.HEADING) + if not headings: + return (root,) + + roots: list[_HeadingNode] = [] + stack: list[_HeadingNode] = [] + nodes: list[_HeadingNode] = [] + for heading in headings: + level = _required_heading_level(block=heading) + while stack and _required_heading_level(block=stack[-1].block) >= level: + stack.pop() + node = _HeadingNode(block=heading, children=[]) + if stack: + stack[-1].children.append(node) + else: + roots.append(node) + stack.append(node) + nodes.append(node) + + last_block = len(blocks) - 1 + for index, node in enumerate(nodes): + level = _required_heading_level(block=node.block) + node.block_end = last_block + for following in nodes[index + 1 :]: + if _required_heading_level(block=following.block) <= level: + node.block_end = following.block.ordinal - 1 + break + + output = [root] + _materialize_heading_nodes( + nodes=roots, parent_path="0", blocks=blocks, output=output + ) + return tuple(output) + + +def resolve_fallback_skeleton( + *, + proposed: tuple[FallbackAnchor, ...], + blocks: tuple[Block, ...], + document_md: str, + title: str | None, +) -> tuple[SnappedSection, ...]: + """Resolve exact anchors and deterministically partition on the block grid. + + Missing anchors disappear and their children are retried in the enclosing + parent. Duplicate text is resolved by ``occurrence_index``. Siblings are + ordered by their resolved block and tile to the next sibling; same-block + siblings collapse with the first proposal adopting later children. + """ + root = _root_section(blocks=blocks, title=title, markdown_chars=len(document_md)) + if not blocks or not proposed: + return (root,) + nodes = _resolve_anchor_level( + proposed=proposed, + parent_start=0, + parent_end=len(blocks) - 1, + blocks=blocks, + document_md=document_md, + depth=1, + ) + output = [root] + _materialize_anchor_nodes( + nodes=nodes, parent_path="0", blocks=blocks, output=output + ) + return tuple(output) + + +def analyze_skeleton( + *, + sections: tuple[SnappedSection, ...], + blocks: tuple[Block, ...], + markdown_chars: int, +) -> SkeletonAnalysis: + """Compute the normative D79 stats and shared route-gate measurements.""" + candidates = sections[1:] + count = len(candidates) + children = _children_by_parent(sections=candidates) + direct_body = { + section.node_path: _direct_body_size( + section=section, children=children.get(section.node_path, ()), blocks=blocks + ) + for section in candidates + } + leaf_spans = { + section.node_path: section.char_end - section.char_start + for section in candidates + if not children.get(section.node_path) + } + heading_density = _safe_ratio(count * 10_000, markdown_chars) + oversized_leaf_ratio = _safe_ratio( + max(leaf_spans.values(), default=0), markdown_chars + ) + + if count < MIN_CHECK_SECTIONS: + stats = SkeletonStats( + stats_version=SKELETON_STATS_VERSION, + section_count=count, + duplicate_title_ratio=None, + max_title_multiplicity=None, + sibling_duplicate_ratio=None, + level_jump_count=None, + numbering_coverage=None, + numbering_inversions=None, + numbering_scheme_switches=None, + tiny_section_ratio=None, + zero_direct_body_ratio=None, + oversized_leaf_ratio=None, + heading_density=None, + title_length_p50=None, + title_length_p95=None, + long_title_ratio=None, + low_letter_ratio=None, + empty_title_ratio=None, + max_sibling_fanout=None, + ) + return SkeletonAnalysis( + stats=stats, + direct_body_chars=direct_body, + leaf_span_chars=leaf_spans, + heading_density=heading_density, + oversized_leaf_ratio=oversized_leaf_ratio, + ) + + titles = tuple(section.normalized_title for section in candidates) + multiplicities = Counter(titles) + sibling_duplicates = sum( + len(siblings) - len({section.normalized_title for section in siblings}) + for siblings in children.values() + ) + # Raw markdown levels only (§4.1): anchor-derived sections carry no + # heading_level, and materialized tree depth changes by at most one per + # step — feeding it in would report a structurally fake zero. Pairs with + # a missing side are excluded; a tree with no raw levels at all reports + # level_jump_count null rather than a number the data cannot support. + raw_levels = tuple(section.heading_level for section in candidates) + level_jumps: int | None = ( + sum( + max(0, right - left - 1) + for left, right in zip(raw_levels, raw_levels[1:], strict=False) + if left is not None and right is not None + ) + if any(level is not None for level in raw_levels) + else None + ) + numbering = tuple(_leading_number(title=title) for title in titles) + inversions, switches = _numbering_run_stats( + children=children, + numbering_by_path=dict( + zip((section.node_path for section in candidates), numbering, strict=True) + ), + ) + lengths = tuple(len(title) for title in titles) + stats = SkeletonStats( + stats_version=SKELETON_STATS_VERSION, + section_count=count, + duplicate_title_ratio=1.0 - (len(multiplicities) / count), + max_title_multiplicity=max(multiplicities.values(), default=0), + sibling_duplicate_ratio=sibling_duplicates / count, + level_jump_count=level_jumps, + numbering_coverage=sum(item is not None for item in numbering) / count, + numbering_inversions=inversions, + numbering_scheme_switches=switches, + tiny_section_ratio=sum(size < TINY_FLOOR for size in direct_body.values()) + / count, + zero_direct_body_ratio=sum(size == 0 for size in direct_body.values()) / count, + oversized_leaf_ratio=oversized_leaf_ratio, + heading_density=heading_density, + title_length_p50=_nearest_rank(values=lengths, percentile=0.50), + title_length_p95=_nearest_rank(values=lengths, percentile=0.95), + long_title_ratio=sum(length > LONG_TITLE for length in lengths) / count, + low_letter_ratio=sum(_letter_ratio(title=title) < 0.5 for title in titles) + / count, + empty_title_ratio=sum(not title for title in titles) / count, + max_sibling_fanout=max( + (len(siblings) for siblings in children.values()), default=0 + ), + ) + return SkeletonAnalysis( + stats=stats, + direct_body_chars=direct_body, + leaf_span_chars=leaf_spans, + heading_density=heading_density, + oversized_leaf_ratio=oversized_leaf_ratio, + ) + + +def skeleton_hash(*, sections: tuple[SnappedSection, ...]) -> str: + """Hash skeleton geometry/title metadata, deliberately excluding roles.""" + payload = [ + { + "node_path": section.node_path, + "parent_path": section.parent_path, + "title": section.title, + "normalized_title": section.normalized_title, + "heading_level": section.heading_level, + "block_start": section.block_start, + "block_end": section.block_end, + "char_start": section.char_start, + "char_end": section.char_end, + } + for section in sections + ] + encoded = json.dumps( + payload, ensure_ascii=False, separators=(",", ":"), sort_keys=True + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def deterministic_section_role(*, normalized_title: str) -> str | None: + """Resolve unambiguous title-only roles before the bounded classifier.""" + exact = { + "abstract": "abstract", + "summary": "abstract", + "introduction": "introduction", + "background": "introduction", + "methods": "methods", + "methodology": "methods", + "materials and methods": "methods", + "results": "results", + "findings": "results", + "discussion": "discussion", + "conclusion": "conclusion", + "conclusions": "conclusion", + "references": "references", + "bibliography": "references", + "works cited": "references", + "table of contents": "nav", + "contents": "nav", + "navigation": "nav", + "tables": "table", + "figures": "figure_caption", + "figure captions": "figure_caption", + "legal": "legal", + "legal notice": "legal", + "terms and conditions": "legal", + "boilerplate": "boilerplate", + # Media wrapper headings have fixed, explicit roles. + "transcript": "body", + "audio transcript": "body", + "video transcript": "body", + "acoustic events": "body", + "visible text (ocr)": "body", + "shot notes": "body", + "image description": "figure_caption", + "visual description": "figure_caption", + } + if normalized_title in exact: + return exact[normalized_title] + if normalized_title == "appendix" or normalized_title.startswith("appendix "): + return "appendix" + if normalized_title == "table" or normalized_title.startswith("table "): + return "table" + if normalized_title == "figure" or normalized_title.startswith("figure "): + return "figure_caption" + return None + + +def _root_section( + *, blocks: tuple[Block, ...], title: str | None, markdown_chars: int +) -> SnappedSection: + return SnappedSection( + node_path="0", + parent_path=None, + title=title or "", + role="body", + block_start=0, + block_end=len(blocks) - 1, + char_start=0, + char_end=markdown_chars, + summary="", + ordinal=0, + heading_level=None, + normalized_title=normalized_heading_title(title=title or ""), + ) + + +def _materialize_heading_nodes( + *, + nodes: Iterable[_HeadingNode], + parent_path: str, + blocks: tuple[Block, ...], + output: list[SnappedSection], +) -> None: + for sibling_index, node in enumerate(nodes): + path = f"{parent_path}.{sibling_index}" + block = node.block + output.append( + SnappedSection( + node_path=path, + parent_path=parent_path, + title=block.heading_title or "", + role="body", + block_start=block.ordinal, + block_end=node.block_end, + char_start=block.char_start, + char_end=blocks[node.block_end].char_end, + summary="", + ordinal=len(output), + heading_level=_required_heading_level(block=block), + normalized_title=block.normalized_title or "", + ) + ) + _materialize_heading_nodes( + nodes=node.children, parent_path=path, blocks=blocks, output=output + ) + + +def _resolve_anchor_level( + *, + proposed: tuple[FallbackAnchor, ...], + parent_start: int, + parent_end: int, + blocks: tuple[Block, ...], + document_md: str, + depth: int, +) -> list[_AnchorNode]: + if depth >= MAX_FALLBACK_DEPTH: + return [] + candidates: list[tuple[int, int, FallbackAnchor]] = [] + emission_index = 0 + + def collect(items: tuple[FallbackAnchor, ...]) -> None: + nonlocal emission_index + for item in items: + current_emission = emission_index + emission_index += 1 + resolved = _resolve_anchor( + anchor=item.anchor, + occurrence_index=item.occurrence_index, + parent_start=parent_start, + parent_end=parent_end, + blocks=blocks, + document_md=document_md, + ) + if resolved is None: + collect(item.children) + else: + candidates.append((resolved, current_emission, item)) + + collect(proposed) + candidates.sort(key=lambda item: item[:2]) + deduped: list[tuple[int, FallbackAnchor]] = [] + for start, _, proposal in candidates: + if deduped and deduped[-1][0] == start: + prior_start, prior = deduped[-1] + deduped[-1] = ( + prior_start, + prior.model_copy( + update={"children": (*prior.children, *proposal.children)} + ), + ) + else: + deduped.append((start, proposal)) + + result: list[_AnchorNode] = [] + for index, (start, proposal) in enumerate(deduped): + end = deduped[index + 1][0] - 1 if index + 1 < len(deduped) else parent_end + if end < start: + continue + children = _resolve_anchor_level( + proposed=proposal.children, + parent_start=start, + parent_end=end, + blocks=blocks, + document_md=document_md, + depth=depth + 1, + ) + result.append( + _AnchorNode( + proposal=proposal, block_start=start, block_end=end, children=children + ) + ) + return result + + +def _resolve_anchor( + *, + anchor: str, + occurrence_index: int, + parent_start: int, + parent_end: int, + blocks: tuple[Block, ...], + document_md: str, +) -> int | None: + occurrences: list[int] = [] + for ordinal in range(parent_start, parent_end + 1): + block = blocks[ordinal] + raw = document_md[block.char_start : block.char_end] + cursor = 0 + while True: + found = raw.find(anchor, cursor) + if found < 0: + break + occurrences.append(ordinal) + cursor = found + max(1, len(anchor)) + if occurrence_index >= len(occurrences): + return None + return occurrences[occurrence_index] + + +def _materialize_anchor_nodes( + *, + nodes: Iterable[_AnchorNode], + parent_path: str, + blocks: tuple[Block, ...], + output: list[SnappedSection], +) -> None: + for sibling_index, node in enumerate(nodes): + path = f"{parent_path}.{sibling_index}" + normalized = normalized_heading_title(title=node.proposal.anchor) + anchor_block = blocks[node.block_start] + output.append( + SnappedSection( + node_path=path, + parent_path=parent_path, + title=node.proposal.anchor, + role="body", + block_start=node.block_start, + block_end=node.block_end, + char_start=anchor_block.char_start, + char_end=blocks[node.block_end].char_end, + summary="", + ordinal=len(output), + heading_level=( + anchor_block.heading_level + if anchor_block.type is BlockType.HEADING + else None + ), + normalized_title=normalized, + ) + ) + _materialize_anchor_nodes( + nodes=node.children, parent_path=path, blocks=blocks, output=output + ) + + +def _children_by_parent( + *, sections: tuple[SnappedSection, ...] +) -> dict[str, tuple[SnappedSection, ...]]: + grouped: defaultdict[str, list[SnappedSection]] = defaultdict(list) + for section in sections: + if section.parent_path is not None: + grouped[section.parent_path].append(section) + return { + parent: tuple(sorted(children, key=lambda item: item.block_start)) + for parent, children in grouped.items() + } + + +def _direct_body_size( + *, + section: SnappedSection, + children: tuple[SnappedSection, ...], + blocks: tuple[Block, ...], +) -> int: + excluded = {section.block_start} + for child in children: + excluded.update(range(child.block_start, child.block_end + 1)) + return sum( + blocks[ordinal].char_end - blocks[ordinal].char_start + for ordinal in range(section.block_start, section.block_end + 1) + if ordinal not in excluded and 0 <= ordinal < len(blocks) + ) + + +def _numbering_run_stats( + *, + children: dict[str, tuple[SnappedSection, ...]], + numbering_by_path: dict[str, tuple[str, tuple[int, ...]] | None], +) -> tuple[int, int]: + inversions = 0 + switches = 0 + for siblings in children.values(): + previous: tuple[str, tuple[int, ...]] | None = None + for sibling in siblings: + current = numbering_by_path[sibling.node_path] + if current is None: + previous = None + continue + if previous is not None: + if previous[0] == current[0]: + if current[1] < previous[1]: + inversions += 1 + else: + switches += 1 + previous = current + return inversions, switches + + +def _leading_number(title: str) -> tuple[str, tuple[int, ...]] | None: + stripped = title.lstrip() + arabic = _ARABIC_NUMBER.match(stripped) + if arabic is not None: + return "arabic_dotted", tuple(int(part) for part in arabic.group(1).split(".")) + alpha = _ALPHA_NUMBER.match(stripped) + if alpha is not None and alpha.group(1).casefold() not in { + "i", + "v", + "x", + "l", + "c", + "d", + "m", + }: + return "alpha", (ord(alpha.group(1).casefold()) - ord("a") + 1,) + roman = _ROMAN_NUMBER.match(stripped) + if roman is not None: + value = _roman_value(token=roman.group(1).upper()) + if value is not None: + return "roman", (value,) + if alpha is not None: + return "alpha", (ord(alpha.group(1).casefold()) - ord("a") + 1,) + return None + + +def _roman_value(*, token: str) -> int | None: + values = {"I": 1, "V": 5, "X": 10, "L": 50, "C": 100, "D": 500, "M": 1000} + total = 0 + previous = 0 + for character in reversed(token): + value = values[character] + if value < previous: + total -= value + else: + total += value + previous = value + canonical = _to_roman(value=total) + return total if canonical == token else None + + +def _to_roman(*, value: int) -> str: + parts: list[str] = [] + for number, symbol in ( + (1000, "M"), + (900, "CM"), + (500, "D"), + (400, "CD"), + (100, "C"), + (90, "XC"), + (50, "L"), + (40, "XL"), + (10, "X"), + (9, "IX"), + (5, "V"), + (4, "IV"), + (1, "I"), + ): + while value >= number: + parts.append(symbol) + value -= number + return "".join(parts) + + +def _nearest_rank(*, values: tuple[int, ...], percentile: float) -> float: + ordered = sorted(values) + index = max(0, math.ceil(percentile * len(ordered)) - 1) + return float(ordered[index]) + + +def _letter_ratio(*, title: str) -> float: + if not title: + return 0.0 + letters = sum( + unicodedata.category(character).startswith("L") for character in title + ) + return letters / len(title) + + +def _safe_ratio(numerator: int, denominator: int) -> float: + return numerator / denominator if denominator > 0 else 0.0 + + +def _required_heading_level(*, block: Block) -> int: + if block.heading_level is None: + raise ValueError(f"heading block {block.ordinal} has no raw level") + return block.heading_level diff --git a/src/rememberstack/model/__init__.py b/src/rememberstack/model/__init__.py index 1e1e984c..346e93f0 100644 --- a/src/rememberstack/model/__init__.py +++ b/src/rememberstack/model/__init__.py @@ -231,6 +231,7 @@ from rememberstack.model.model_provider import ProviderAccountingError from rememberstack.model.model_provider import ProviderCallError from rememberstack.model.model_provider import ProviderCallUsage +from rememberstack.model.model_provider import ProviderInvalidResponseError from rememberstack.model.model_provider import StructuredResponseModel from rememberstack.model.mounts import PublishedMounts from rememberstack.model.object_store import ObjectAlreadyExistsError @@ -295,11 +296,20 @@ from rememberstack.model.retrieval_spikes import RetrievalSpikeMeasurement from rememberstack.model.retrieval_spikes import RetrievalSpikeName from rememberstack.model.retrieval_spikes import RetrievalSpikeReport +from rememberstack.model.sections import FallbackAnchor +from rememberstack.model.sections import FallbackStructureResponse from rememberstack.model.sections import PersistedSectionTree from rememberstack.model.sections import ProposedSection +from rememberstack.model.sections import RoleAssignment +from rememberstack.model.sections import RoleClassificationResponse from rememberstack.model.sections import SectionTreeRecord +from rememberstack.model.sections import SkeletonCheckOutcome +from rememberstack.model.sections import SkeletonCheckRecord +from rememberstack.model.sections import SkeletonCheckResponse +from rememberstack.model.sections import SkeletonStats +from rememberstack.model.sections import SkeletonVerdict from rememberstack.model.sections import SnappedSection -from rememberstack.model.sections import StructureResponse +from rememberstack.model.sections import StructureRouteTag from rememberstack.model.telemetry import TelemetryAttribute from rememberstack.model.telemetry import TelemetryEvent @@ -389,6 +399,8 @@ "FactLabelResponse", "FactResult", "FactSupport", + "FallbackAnchor", + "FallbackStructureResponse", "ForgetError", "ForgetInProgressError", "ForgetManifest", @@ -447,6 +459,7 @@ "ProviderAccountingError", "ProviderCallError", "ProviderCallUsage", + "ProviderInvalidResponseError", "PublishedMounts", "QueueRoute", "PageRef", @@ -576,9 +589,16 @@ "ScopeInterestsRuleParams", "PersistedSectionTree", "ProposedSection", + "RoleAssignment", + "RoleClassificationResponse", "SectionTreeRecord", + "SkeletonCheckOutcome", + "SkeletonCheckRecord", + "SkeletonCheckResponse", + "SkeletonStats", + "SkeletonVerdict", "SnappedSection", - "StructureResponse", + "StructureRouteTag", "SourceItem", "SourceRecord", "StructureSource", diff --git a/src/rememberstack/model/blocks.py b/src/rememberstack/model/blocks.py index 92c49f25..d46a9479 100644 --- a/src/rememberstack/model/blocks.py +++ b/src/rememberstack/model/blocks.py @@ -5,6 +5,7 @@ from pydantic import BaseModel from pydantic import ConfigDict from pydantic import Field +from pydantic import model_validator class BlockType(StrEnum): @@ -28,3 +29,20 @@ class Block(BaseModel): char_start: int = Field(ge=0) char_end: int = Field(ge=0) block_hash: str + heading_level: int | None = Field(default=None, ge=1, le=6) + heading_title: str | None = None + normalized_title: str | None = None + + @model_validator(mode="after") + def heading_metadata_matches_type(self) -> "Block": + """Heading tokens alone carry the D79 parser metadata.""" + metadata = (self.heading_level, self.heading_title, self.normalized_title) + if self.type is BlockType.HEADING and any(value is None for value in metadata): + raise ValueError( + "heading blocks require level, title, and normalized title" + ) + if self.type is not BlockType.HEADING and any( + value is not None for value in metadata + ): + raise ValueError("non-heading blocks cannot carry heading metadata") + return self diff --git a/src/rememberstack/model/documents.py b/src/rememberstack/model/documents.py index e6c034cb..1aec5c7f 100644 --- a/src/rememberstack/model/documents.py +++ b/src/rememberstack/model/documents.py @@ -111,6 +111,7 @@ class StructureSource(BaseModel): blocks_uri: str markdown_uri: str title: str | None + source_kind: str class SyntheticRootRecord(BaseModel): diff --git a/src/rememberstack/model/model_provider.py b/src/rememberstack/model/model_provider.py index 5fedf686..465bbfc0 100644 --- a/src/rememberstack/model/model_provider.py +++ b/src/rememberstack/model/model_provider.py @@ -43,6 +43,10 @@ def __init__(self, message: str, *, usage: ProviderCallUsage | None = None) -> N self.usage = usage +class ProviderInvalidResponseError(ProviderCallError): + """A completed generation returned no value valid for the response schema.""" + + class GeneratedResponse(BaseModel, Generic[ResponseT]): """A validated structured output paired with its provider accounting.""" diff --git a/src/rememberstack/model/sections.py b/src/rememberstack/model/sections.py index a09d5266..84975212 100644 --- a/src/rememberstack/model/sections.py +++ b/src/rememberstack/model/sections.py @@ -1,15 +1,10 @@ -"""Section-structure values (D39/D57): the LLM's proposal and the snapped truth. - -Two deliberately different trust levels share this module. `ProposedSection` / -`StructureResponse` are the structurer LLM's raw output — free-hand character -spans that may overlap, gap, nest wrongly, or point outside the document; they -are never persisted. `SnappedSection` is what the deterministic snap -(`core/section_snap.py`) makes of them: a well-formed partition on the block -grid, the only form that reaches `document_sections`. -""" +"""D79 section skeleton, checker, role, and immutable-generation values.""" from __future__ import annotations +from decimal import Decimal +from enum import StrEnum +from typing import Literal from uuid import UUID from pydantic import BaseModel @@ -19,7 +14,12 @@ class ProposedSection(BaseModel): - """One LLM-proposed section span (pre-snap): untrusted free-hand geometry.""" + """A span proposal accepted only by the generic D57 snap primitive. + + D79 removed this shape from the E0 model boundary. It remains an internal + value for the total snap algorithm and its regression corpus; no provider + response contains offsets anymore. + """ model_config = ConfigDict(frozen=True, extra="ignore") @@ -31,13 +31,137 @@ class ProposedSection(BaseModel): children: tuple[ProposedSection, ...] = () -class StructureResponse(BaseModel): - """The structurer's structured output: a proposed tree + placement hint.""" +class FallbackAnchor(BaseModel): + """One fallback-proposed exact block-contained anchor. - model_config = ConfigDict(frozen=True, extra="ignore") + ``occurrence_index`` is zero-based among exact occurrences inside the + enclosing parent's block range. Geometry is deliberately absent. + """ + + model_config = ConfigDict(frozen=True, extra="forbid") + + anchor: str = Field(min_length=1) + occurrence_index: int = Field(ge=0) + children: tuple[FallbackAnchor, ...] = () + + +class FallbackStructureResponse(BaseModel): + """The D79 fallback seat's closed anchor-only response.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + sections: tuple[FallbackAnchor, ...] = () + + +class SkeletonVerdict(StrEnum): + """The checker can express exactly one primary skeleton-quality verdict.""" + + COHERENT = "coherent" + INCOHERENT_REPEATED_BOILERPLATE = "incoherent_repeated_boilerplate" + INCOHERENT_HEADING_SEQUENCE = "incoherent_heading_sequence" + INCOHERENT_JUNK_TITLES = "incoherent_junk_titles" + INCOHERENT_OVER_FRAGMENTED = "incoherent_over_fragmented" + + +class SkeletonCheckResponse(BaseModel): + """One closed enum and no explanation, confidence, or proposal fields.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + verdict: Literal[ + "coherent", + "incoherent_repeated_boilerplate", + "incoherent_heading_sequence", + "incoherent_junk_titles", + "incoherent_over_fragmented", + ] + + +class SkeletonCheckOutcome(StrEnum): + """Persisted checker outcome: verdicts plus explicit non-verdict states.""" + + NOT_RUN_SHORT = "not_run_short" + COHERENT = "coherent" + INCOHERENT_REPEATED_BOILERPLATE = "incoherent_repeated_boilerplate" + INCOHERENT_HEADING_SEQUENCE = "incoherent_heading_sequence" + INCOHERENT_JUNK_TITLES = "incoherent_junk_titles" + INCOHERENT_OVER_FRAGMENTED = "incoherent_over_fragmented" + PROVIDER_ERROR = "provider_error" + INVALID_RESPONSE = "invalid_response" + + +class StructureRouteTag(StrEnum): + """Why one immutable skeleton generation has the shape it carries.""" - sections: tuple[ProposedSection, ...] = () - placement: str = "" + PARSER = "parser" + PARSER_DEMOTED_CHECK = "parser_demoted_check" + FALLBACK_DENSITY = "fallback_density" + FALLBACK_LEAF = "fallback_leaf" + FALLBACK_AFTER_CHECK = "fallback_after_check" + SYNTHETIC_AFTER_CHECK = "synthetic_after_check" + LEGACY = "legacy" + + +class SkeletonStats(BaseModel): + """The exact versioned D79 sanity-check stat schema. + + ``section_count`` remains populated so eligibility is auditable. Every + formula field is null when it is below ``MIN_CHECK_SECTIONS``. + """ + + model_config = ConfigDict(frozen=True, extra="forbid") + + stats_version: str + section_count: int = Field(ge=0) + duplicate_title_ratio: float | None + max_title_multiplicity: int | None + sibling_duplicate_ratio: float | None + level_jump_count: int | None + numbering_coverage: float | None + numbering_inversions: int | None + numbering_scheme_switches: int | None + tiny_section_ratio: float | None + zero_direct_body_ratio: float | None + oversized_leaf_ratio: float | None + heading_density: float | None + title_length_p50: float | None + title_length_p95: float | None + long_title_ratio: float | None + low_letter_ratio: float | None + empty_title_ratio: float | None + max_sibling_fanout: int | None + + +class RoleAssignment(BaseModel): + """One title-only classifier assignment keyed to an existing node path.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + node_path: str = Field(min_length=1) + role: Literal[ + "body", + "abstract", + "introduction", + "results", + "methods", + "discussion", + "conclusion", + "references", + "appendix", + "table", + "figure_caption", + "nav", + "boilerplate", + "legal", + ] + + +class RoleClassificationResponse(BaseModel): + """The bounded title-only role seat's closed response.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + assignments: tuple[RoleAssignment, ...] = () class SnappedSection(BaseModel): @@ -59,6 +183,38 @@ class SnappedSection(BaseModel): char_end: int = Field(ge=0) summary: str ordinal: int = Field(ge=0) + heading_level: int | None = Field(default=None, ge=1, le=6) + normalized_title: str = "" + + +class SkeletonCheckRecord(BaseModel): + """One append-only D52 checker record; never stores completion text.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + check_id: UUID + processing_id: UUID + deployment_id: UUID + doc_id: UUID + version_id: UUID + representation_id: UUID + candidate_skeleton_hash: str + stats_version: str + stats: SkeletonStats + sampled_input_hash: str | None + """Null when no prompt was ever rendered (not_run_short) — a hash of an + unsent input would be bookkeeping fiction.""" + check_outcome: SkeletonCheckOutcome + checker_component_version: str + checker_model: str + checker_model_hash: str + checker_prompt_hash: str + checker_schema_hash: str + provider_failure: dict[str, str | int | float | bool | None] | None + tokens_in: int | None + tokens_out: int | None + cost_usd: Decimal | None + latency_ms: int | None class SectionTreeRecord(BaseModel): @@ -77,10 +233,25 @@ class SectionTreeRecord(BaseModel): doc_id: UUID version_id: UUID representation_id: UUID + structure_generation_id: UUID sections: tuple[SnappedSection, ...] = Field(min_length=1) placement_path: str | None structurer_name: str structurer_version: str + skeleton_version: str + skeleton_hash: str + skeleton_producer_family: str + skeleton_check_version: str | None + roles_version: str | None + summary_version: str | None = None + placement_version: str | None = None + selecting_check_id: UUID | None + route_tag: StructureRouteTag + candidate_skeleton_hash: str + stats_version: str + stats: SkeletonStats + pageindex_uri: str + make_current: bool = True @model_validator(mode="after") def _tree_is_connected(self) -> SectionTreeRecord: @@ -118,3 +289,17 @@ class PersistedSectionTree(BaseModel): sections: tuple[SnappedSection, ...] = Field(min_length=1) placement_path: str | None structurer_version: str + structure_generation_id: UUID + pageindex_uri: str + skeleton_version: str + skeleton_hash: str + skeleton_producer_family: str + skeleton_check_version: str | None + roles_version: str | None + summary_version: str | None + placement_version: str | None + selecting_check_id: UUID | None + route_tag: StructureRouteTag + candidate_skeleton_hash: str + stats_version: str + stats: SkeletonStats diff --git a/src/rememberstack/profiles/selfhost.py b/src/rememberstack/profiles/selfhost.py index 21553459..ad42538a 100644 --- a/src/rememberstack/profiles/selfhost.py +++ b/src/rememberstack/profiles/selfhost.py @@ -387,6 +387,8 @@ def _handler(self, *, stage: PipelineStage) -> StageHandler: from rememberstack.workers import NormalizeRelationsHandler from rememberstack.workers import P1Settings from rememberstack.workers import ReconcileHandler + from rememberstack.workers import RoleSettings + from rememberstack.workers import SkeletonCheckSettings from rememberstack.workers import StructureHandler from rememberstack.workers import StructurerSettings @@ -413,6 +415,8 @@ def _handler(self, *, stage: PipelineStage) -> StageHandler: artifact_store=self._artifact_store, model_provider=self._model_provider, settings=StructurerSettings.model_validate({}), + check_settings=SkeletonCheckSettings.model_validate({}), + role_settings=RoleSettings.model_validate({}), ) if stage is PipelineStage.CHUNK: return ChunkHandler( @@ -591,9 +595,13 @@ def _model_bindings() -> dict[str, str]: from rememberstack.workers import E2Settings from rememberstack.workers import E3Settings from rememberstack.workers import P1Settings + from rememberstack.workers import RoleSettings + from rememberstack.workers import SkeletonCheckSettings from rememberstack.workers import StructurerSettings structurer = StructurerSettings.model_validate({}) + skeleton_check = SkeletonCheckSettings.model_validate({}) + roles = RoleSettings.model_validate({}) e1 = E1Settings.model_validate({}) e2 = E2Settings.model_validate({}) e3 = E3Settings.model_validate({}) @@ -602,7 +610,9 @@ def _model_bindings() -> dict[str, str]: p1 = P1Settings.model_validate({}) openrouter = OpenRouterSettings.model_validate({}) return { - "structure": structurer.model, + "structure_fallback": structurer.model, + "skeleton_check": skeleton_check.model, + "section_role": roles.model, "chunk_embedding": e1.embedding_model, "context_prefix": e1.prefix_model, "claim_extraction": e2.extract_model, diff --git a/src/rememberstack/spine/catalog_contract.py b/src/rememberstack/spine/catalog_contract.py index 073eed1c..4341b012 100644 --- a/src/rememberstack/spine/catalog_contract.py +++ b/src/rememberstack/spine/catalog_contract.py @@ -87,7 +87,9 @@ "section_role", "selection_drop_reason", "selection_outcome", + "skeleton_check_outcome", "snapshot_status", + "structure_route_tag", "subscription_status", "versioning_mode", ) @@ -106,7 +108,9 @@ "deployments", "document_crossrefs", "document_representations", + "document_skeleton_checks", "document_sections", + "document_structure_generations", "document_versions", "documents", "entities", @@ -238,6 +242,8 @@ "ix_sections_doc", "ix_sections_parent", "ix_sections_role", + "ix_skeleton_checks_representation", + "ix_structure_generations_representation", "ux_kae_link", "ux_kquarantine_open_artifact", "ux_kwatch", @@ -297,7 +303,7 @@ def lane_is_valid(*, stage: str, lane: str | None) -> bool: "predicate_signatures", "predicates", ) -EXPECTED_CONSTRAINT_COUNTS: Final = {"c": 40, "f": 113, "p": 60, "u": 30, "x": 1} +EXPECTED_CONSTRAINT_COUNTS: Final = {"c": 47, "f": 124, "p": 62, "u": 31, "x": 1} DECISION_OBJECTS: Final = { "D1": ("pipeline_component_versions",), "D2": ("claims", "relations", "relation_evidence"), @@ -317,6 +323,7 @@ def lane_is_valid(*, stage: str, lane: str | None) -> bool: "D68": ("deployments", "ix_entities_name_trgm"), "D69": ("v_graph_relates",), "D74": ("forget_manifests", "ix_forget_content_guard"), + "D79": ("document_structure_generations", "document_skeleton_checks"), } diff --git a/src/rememberstack/spine/chunk_catalog.py b/src/rememberstack/spine/chunk_catalog.py index 116678bf..6a81a6b0 100644 --- a/src/rememberstack/spine/chunk_catalog.py +++ b/src/rememberstack/spine/chunk_catalog.py @@ -161,9 +161,12 @@ def record_embeddings(self, *, updates: tuple[EmbeddingUpdate, ...]) -> None: _SELECT_SECTIONS = text( """ - SELECT section_id, node_path, role, block_start, block_end - FROM document_sections - WHERE representation_id = :representation_id + SELECT s.section_id, s.node_path, s.role, s.block_start, s.block_end + FROM document_sections s + JOIN document_representations r + ON r.representation_id = s.representation_id + AND r.current_structure_generation_id = s.structure_generation_id + WHERE s.representation_id = :representation_id ORDER BY string_to_array(node_path, '.')::int[] """ ) diff --git a/src/rememberstack/spine/document_catalog.py b/src/rememberstack/spine/document_catalog.py index 78be2e10..92e0cf6c 100644 --- a/src/rememberstack/spine/document_catalog.py +++ b/src/rememberstack/spine/document_catalog.py @@ -1,14 +1,18 @@ -"""The E0 document catalog: lineage, version, representation, and section writes. +"""The E0 document catalog: lineage, representation, and D79 generations. Spine-owned SQL for the D36 sub-worker chain over the D55 lineage model: `record_upload` lands content + lineage + version rows and enqueues convert atomically; `record_representation` lands one immutable conversion output -(D65); `record_synthetic_root` completes the chain — section row, live -representation pointer, version/lineage currency — in one transaction. +(D65); immutable section generations carry an explicit current pointer and +skeleton checks append independently under D52. """ +from decimal import Decimal +import json +from uuid import NAMESPACE_URL from uuid import UUID from uuid import uuid4 +from uuid import uuid5 from sqlalchemy import text from sqlalchemy.engine import Connection @@ -25,7 +29,10 @@ from rememberstack.model import RepresentationNotFoundError from rememberstack.model import RepresentationRecord from rememberstack.model import SectionTreeRecord +from rememberstack.model import SkeletonCheckRecord +from rememberstack.model import SkeletonStats from rememberstack.model import SnappedSection +from rememberstack.model import StructureRouteTag from rememberstack.model import StructureSource from rememberstack.model import SyntheticRootRecord from rememberstack.model import UploadRecord @@ -209,19 +216,45 @@ def structure_source(self, *, representation_id: UUID) -> StructureSource: return StructureSource.model_validate(dict(row)) def record_synthetic_root(self, *, record: SyntheticRootRecord) -> None: - """Complete the E0 chain with the single full-span root (D39). + """Compatibility helper for callers that explicitly request a root. - The degenerate section tree: one ``role=body`` root covering every - block — what a short document (or a degraded structurer) gets. An - empty document yields the empty range ``0..-1`` on the inclusive - block grid (D57) and a zero-width char span. + The D79 worker uses ``record_section_tree`` directly. This helper wraps + older internal callers in one deterministic legacy generation. """ + generation_id = uuid5( + NAMESPACE_URL, + "rememberstack:legacy-synthetic:" + f"{record.representation_id}:{record.structurer_version}", + ) + stats = SkeletonStats( + stats_version="legacy-synthetic", + section_count=0, + duplicate_title_ratio=None, + max_title_multiplicity=None, + sibling_duplicate_ratio=None, + level_jump_count=None, + numbering_coverage=None, + numbering_inversions=None, + numbering_scheme_switches=None, + tiny_section_ratio=None, + zero_direct_body_ratio=None, + oversized_leaf_ratio=None, + heading_density=None, + title_length_p50=None, + title_length_p95=None, + long_title_ratio=None, + low_letter_ratio=None, + empty_title_ratio=None, + max_sibling_fanout=None, + ) + digest = str(generation_id).replace("-", "") self.record_section_tree( record=SectionTreeRecord( deployment_id=record.deployment_id, doc_id=record.doc_id, version_id=record.version_id, representation_id=record.representation_id, + structure_generation_id=generation_id, sections=( SnappedSection( node_path="0", @@ -234,26 +267,82 @@ def record_synthetic_root(self, *, record: SyntheticRootRecord) -> None: char_end=record.markdown_chars, summary="", ordinal=0, + normalized_title=(record.title or "").casefold(), ), ), placement_path=None, structurer_name="synthetic_root", structurer_version=record.structurer_version, + skeleton_version=record.structurer_version, + skeleton_hash=digest, + skeleton_producer_family="N/A", + skeleton_check_version=None, + roles_version=record.structurer_version, + selecting_check_id=None, + route_tag=StructureRouteTag.LEGACY, + candidate_skeleton_hash=digest, + stats_version=stats.stats_version, + stats=stats, + pageindex_uri=f"legacy://pageindex/{generation_id}.json", + ) + ) + + def record_skeleton_check(self, *, record: SkeletonCheckRecord) -> None: + """Append one checker record without ever storing completion text.""" + payload = record.model_dump(mode="json") + payload["stats"] = json.dumps( + payload["stats"], ensure_ascii=False, separators=(",", ":"), sort_keys=True + ) + failure = payload["provider_failure"] + payload["provider_failure"] = ( + json.dumps( + failure, ensure_ascii=False, separators=(",", ":"), sort_keys=True + ) + if failure is not None + else None + ) + with self._engine.begin() as connection: + connection.execute(_INSERT_SKELETON_CHECK, payload) + + def skeleton_checks( + self, *, representation_id: UUID + ) -> tuple[SkeletonCheckRecord, ...]: + """Read a representation's append-only checks in insertion order.""" + with self._engine.connect() as connection: + rows = ( + connection.execute( + _SELECT_SKELETON_CHECKS, {"representation_id": representation_id} + ) + .mappings() + .all() ) + return tuple( + SkeletonCheckRecord.model_validate( + { + **dict(row), + "stats": dict(row["stats"]), + "cost_usd": ( + Decimal(row["cost_usd"]) + if row["cost_usd"] is not None + else None + ), + } + ) + for row in rows ) def record_section_tree(self, *, record: SectionTreeRecord) -> PersistedSectionTree: """Complete the E0 chain for one representation in one transaction (D39/D54). - Inserts the section rows (root first, parents resolved by path), - marks the representation ready, swaps the version's live-reading + Inserts one immutable generation and its rows (root first, parents + resolved by path), marks the representation ready, swaps the version's live-reading pointer, and moves the lineage's current-version pointer — so currency flips only when the chain is whole. The walking skeleton's chain ends at structure; when the E1/E2 stages land, this flip moves with the chain's end (D54's rule is "after conversion→E1→E2 completes"). Every statement is idempotent for a retried attempt - (section rows conflict on ``(version_id, node_path)`` and keep their - first-written ids), the pointer swap never overwrites a different + (a generation-id conflict skips the entire proposed tree and keeps the + first-written generation), the pointer swap never overwrites a different live representation, and the currency pointer only moves FORWARD by version number — a delayed older version completing after a newer one must not drag the lineage back to stale content. @@ -263,78 +352,100 @@ def record_section_tree(self, *, record: SectionTreeRecord) -> PersistedSectionT artifacts (the sidecar) must be built from it, not from the input. """ with self._engine.begin() as connection: + generation_payload = record.model_dump(mode="json") + generation_payload["stats"] = json.dumps( + generation_payload["stats"], + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + inserted_generation = connection.execute( + _INSERT_STRUCTURE_GENERATION, generation_payload + ).scalar_one_or_none() ids_by_path: dict[str, UUID] = {} - for section in record.sections: - parent_id = ( - ids_by_path.get(section.parent_path) - if section.parent_path is not None - else None - ) - section_id = connection.execute( - _INSERT_SECTION, - { - "section_id": uuid4(), - "deployment_id": record.deployment_id, - "doc_id": record.doc_id, - "version_id": record.version_id, - "representation_id": record.representation_id, - "parent_section_id": parent_id, - "node_path": section.node_path, - "block_start": section.block_start, - "block_end": section.block_end, - "title": section.title or None, - "role": section.role, - "char_start": section.char_start, - "char_end": section.char_end, - "ordinal": section.ordinal, - "summary": section.summary or None, - "placement_path": ( - record.placement_path - if section.parent_path is None - else None - ), - "structurer_version": record.structurer_version, - }, - ).scalar_one_or_none() - if section_id is None: # a retry: the first attempt's row won + if inserted_generation is not None: + for section in record.sections: + parent_id = ( + ids_by_path[section.parent_path] + if section.parent_path is not None + else None + ) section_id = connection.execute( - _SELECT_SECTION_BY_PATH, + _INSERT_SECTION, { + "section_id": uuid4(), + "deployment_id": record.deployment_id, + "doc_id": record.doc_id, "version_id": record.version_id, + "representation_id": record.representation_id, + "structure_generation_id": record.structure_generation_id, + "parent_section_id": parent_id, "node_path": section.node_path, + "block_start": section.block_start, + "block_end": section.block_end, + "title": section.title or None, + "role": section.role, + "char_start": section.char_start, + "char_end": section.char_end, + "ordinal": section.ordinal, + "heading_level": section.heading_level, + "normalized_title": section.normalized_title, + "summary": section.summary or None, + "placement_path": ( + record.placement_path + if section.parent_path is None + else None + ), + "structurer_version": record.structurer_version, }, ).scalar_one() - ids_by_path[section.node_path] = section_id - connection.execute( - _MARK_REPRESENTATION_READY, - { - "representation_id": record.representation_id, - "structurer_name": record.structurer_name, - "structurer_version": record.structurer_version, - }, - ) - connection.execute( - _MARK_VERSION_READY, - { - "version_id": record.version_id, - "representation_id": record.representation_id, - }, - ) - connection.execute( - _MARK_LINEAGE_CURRENT, - {"doc_id": record.doc_id, "version_id": record.version_id}, - ) - connection.execute( # the lineage pointer moved: older versions - _SUPERSEDE_PRIOR_VERSIONS, # are superseded as of now (D55) - {"doc_id": record.doc_id, "version_id": record.version_id}, - ) + ids_by_path[section.node_path] = section_id persisted = ( connection.execute( - _SELECT_SECTION_TREE, {"version_id": record.version_id} + _SELECT_SECTION_TREE, + {"structure_generation_id": record.structure_generation_id}, ) .mappings() .all() ) + if not persisted: + raise RuntimeError( + "structure generation exists without its root section" + ) + generation = ( + connection.execute( + _SELECT_STRUCTURE_GENERATION, + {"structure_generation_id": record.structure_generation_id}, + ) + .mappings() + .one() + ) + if record.make_current: + connection.execute( + _MARK_REPRESENTATION_READY, + { + "representation_id": record.representation_id, + "structure_generation_id": record.structure_generation_id, + "pageindex_uri": generation["pageindex_uri"], + "structurer_name": generation["route_tag"], + "structurer_version": persisted[0]["structurer_version"], + }, + ) + connection.execute( + _MARK_VERSION_READY, + { + "version_id": record.version_id, + "representation_id": record.representation_id, + }, + ) + connection.execute( + _MARK_LINEAGE_CURRENT, + {"doc_id": record.doc_id, "version_id": record.version_id}, + ) + connection.execute( # the lineage pointer moved: older versions + _SUPERSEDE_PRIOR_VERSIONS, # are superseded as of now (D55) + {"doc_id": record.doc_id, "version_id": record.version_id}, + ) return PersistedSectionTree( sections=tuple( SnappedSection( @@ -352,11 +463,27 @@ def record_section_tree(self, *, record: SectionTreeRecord) -> PersistedSectionT char_end=row["char_end"], summary=row["summary"] or "", ordinal=row["ordinal"], + heading_level=row["heading_level"], + normalized_title=row["normalized_title"], ) for row in persisted ), placement_path=persisted[0]["placement_path"], structurer_version=persisted[0]["structurer_version"] or "", + structure_generation_id=generation["structure_generation_id"], + pageindex_uri=generation["pageindex_uri"], + skeleton_version=generation["skeleton_version"], + skeleton_hash=generation["skeleton_hash"], + skeleton_producer_family=generation["skeleton_producer_family"], + skeleton_check_version=generation["skeleton_check_version"], + roles_version=generation["roles_version"], + summary_version=generation["summary_version"], + placement_version=generation["placement_version"], + selecting_check_id=generation["selecting_check_id"], + route_tag=generation["route_tag"], + candidate_skeleton_hash=generation["candidate_skeleton_hash"], + stats_version=generation["stats_version"], + stats=SkeletonStats.model_validate(generation["stats"]), ) @@ -510,7 +637,7 @@ def _lineage_locked(*, connection: Connection, record: UploadRecord) -> UUID: _SELECT_STRUCTURE_SOURCE = text( """ SELECT r.deployment_id, v.doc_id, r.version_id, r.representation_id, - r.blocks_uri, r.markdown_uri, d.title + r.blocks_uri, r.markdown_uri, d.title, d.source_kind FROM document_representations r JOIN document_versions v ON v.version_id = r.version_id JOIN documents d ON d.doc_id = v.doc_id @@ -518,28 +645,90 @@ def _lineage_locked(*, connection: Connection, record: UploadRecord) -> UUID: """ ) +_INSERT_SKELETON_CHECK = text( + """ + INSERT INTO document_skeleton_checks ( + check_id, processing_id, deployment_id, doc_id, version_id, + representation_id, candidate_skeleton_hash, stats_version, stats, + sampled_input_hash, check_outcome, checker_component_version, + checker_model, checker_model_hash, checker_prompt_hash, + checker_schema_hash, provider_failure, tokens_in, tokens_out, + cost_usd, latency_ms + ) VALUES ( + :check_id, :processing_id, :deployment_id, :doc_id, :version_id, + :representation_id, :candidate_skeleton_hash, :stats_version, + CAST(:stats AS jsonb), :sampled_input_hash, + CAST(:check_outcome AS skeleton_check_outcome), + :checker_component_version, :checker_model, :checker_model_hash, + :checker_prompt_hash, :checker_schema_hash, + CAST(:provider_failure AS jsonb), :tokens_in, :tokens_out, + :cost_usd, :latency_ms + ) + """ +) + +_SELECT_SKELETON_CHECKS = text( + """ + SELECT check_id, processing_id, deployment_id, doc_id, version_id, + representation_id, candidate_skeleton_hash, stats_version, stats, + sampled_input_hash, check_outcome::text AS check_outcome, + checker_component_version, checker_model, checker_model_hash, + checker_prompt_hash, checker_schema_hash, provider_failure, + tokens_in, tokens_out, cost_usd, latency_ms + FROM document_skeleton_checks + WHERE representation_id = :representation_id + ORDER BY checked_at, check_id + """ +) + +_INSERT_STRUCTURE_GENERATION = text( + """ + INSERT INTO document_structure_generations ( + structure_generation_id, deployment_id, doc_id, version_id, + representation_id, skeleton_version, skeleton_hash, + skeleton_producer_family, skeleton_check_version, roles_version, + summary_version, placement_version, selecting_check_id, route_tag, + candidate_skeleton_hash, stats_version, stats, pageindex_uri + ) VALUES ( + :structure_generation_id, :deployment_id, :doc_id, :version_id, + :representation_id, :skeleton_version, :skeleton_hash, + :skeleton_producer_family, :skeleton_check_version, :roles_version, + :summary_version, :placement_version, :selecting_check_id, + CAST(:route_tag AS structure_route_tag), :candidate_skeleton_hash, + :stats_version, CAST(:stats AS jsonb), :pageindex_uri + ) + ON CONFLICT (structure_generation_id) DO NOTHING + RETURNING structure_generation_id + """ +) + _INSERT_SECTION = text( """ INSERT INTO document_sections ( section_id, deployment_id, doc_id, version_id, representation_id, - parent_section_id, node_path, block_start, block_end, + structure_generation_id, parent_section_id, node_path, block_start, block_end, title, role, char_start, char_end, ordinal, - summary, placement_path, structurer_version + heading_level, normalized_title, summary, placement_path, structurer_version ) VALUES ( :section_id, :deployment_id, :doc_id, :version_id, :representation_id, - :parent_section_id, :node_path, :block_start, :block_end, + :structure_generation_id, :parent_section_id, :node_path, :block_start, :block_end, :title, CAST(:role AS section_role), :char_start, :char_end, :ordinal, - :summary, :placement_path, :structurer_version + :heading_level, :normalized_title, :summary, :placement_path, :structurer_version ) - ON CONFLICT (version_id, node_path) DO NOTHING + ON CONFLICT (structure_generation_id, node_path) DO NOTHING RETURNING section_id """ ) -_SELECT_SECTION_BY_PATH = text( +_SELECT_STRUCTURE_GENERATION = text( """ - SELECT section_id FROM document_sections - WHERE version_id = :version_id AND node_path = :node_path + SELECT structure_generation_id, skeleton_version, skeleton_hash, + skeleton_producer_family, skeleton_check_version, roles_version, + summary_version, placement_version, selecting_check_id, + route_tag::text AS route_tag, candidate_skeleton_hash, + stats_version, stats, pageindex_uri + FROM document_structure_generations + WHERE structure_generation_id = :structure_generation_id """ ) @@ -547,9 +736,9 @@ def _lineage_locked(*, connection: Connection, record: UploadRecord) -> UUID: """ SELECT node_path, title, role::text AS role, block_start, block_end, char_start, char_end, summary, ordinal, placement_path, - structurer_version + structurer_version, heading_level, normalized_title FROM document_sections - WHERE version_id = :version_id + WHERE structure_generation_id = :structure_generation_id ORDER BY ordinal """ ) @@ -560,8 +749,11 @@ def _lineage_locked(*, connection: Connection, record: UploadRecord) -> UUID: SET structurer_name = :structurer_name, structurer_version = :structurer_version, section_index_version = :structurer_version, + current_structure_generation_id = :structure_generation_id, + pageindex_uri = :pageindex_uri, status = 'ready' - WHERE representation_id = :representation_id AND status = 'structuring' + WHERE representation_id = :representation_id + AND status IN ('structuring', 'ready') """ ) diff --git a/src/rememberstack/spine/forget.py b/src/rememberstack/spine/forget.py index 72f97cda..7e23c98c 100644 --- a/src/rememberstack/spine/forget.py +++ b/src/rememberstack/spine/forget.py @@ -947,6 +947,11 @@ def _text_values( ) uri(object_key) WHERE version.deployment_id = :deployment_id AND version.doc_id = :doc_id UNION ALL + SELECT generation.pageindex_uri + FROM document_structure_generations generation + WHERE generation.deployment_id = :deployment_id + AND generation.doc_id = :doc_id + UNION ALL SELECT compilation.session_transcript_uri FROM knowledge_compilations compilation WHERE compilation.deployment_id = :deployment_id @@ -1364,6 +1369,23 @@ def _text_values( WHERE deployment_id = :deployment_id AND doc_id = :doc_id """ ), + # D79 audit tables are erased EXPLICITLY, never via implied cascade: + # generations first (their selecting_check_id references checks with + # default NO ACTION), then checks, then representations. A future FK + # edit must not silently strand titles/stats/failure envelopes of a + # forgotten document. + text( + """ + DELETE FROM document_structure_generations + WHERE deployment_id = :deployment_id AND doc_id = :doc_id + """ + ), + text( + """ + DELETE FROM document_skeleton_checks + WHERE deployment_id = :deployment_id AND doc_id = :doc_id + """ + ), text( """ DELETE FROM document_representations @@ -1491,6 +1513,12 @@ def _text_values( SELECT 1 FROM document_sections WHERE deployment_id = :deployment_id AND doc_id = :doc_id UNION ALL + SELECT 1 FROM document_structure_generations + WHERE deployment_id = :deployment_id AND doc_id = :doc_id + UNION ALL + SELECT 1 FROM document_skeleton_checks + WHERE deployment_id = :deployment_id AND doc_id = :doc_id + UNION ALL SELECT 1 FROM chunks WHERE deployment_id = :deployment_id AND doc_id = :doc_id UNION ALL diff --git a/src/rememberstack/spine/migrations/versions/p1_04_0019_d79_structure_generations.py b/src/rememberstack/spine/migrations/versions/p1_04_0019_d79_structure_generations.py new file mode 100644 index 00000000..b1fbce54 --- /dev/null +++ b/src/rememberstack/spine/migrations/versions/p1_04_0019_d79_structure_generations.py @@ -0,0 +1,225 @@ +"""Add D79 immutable structure generations and append-only skeleton checks.""" + +from collections.abc import Sequence + +from alembic import op + +from rememberstack.spine.migrations._helpers import apply_ddl +from rememberstack.spine.migrations._helpers import drop_tables +from rememberstack.spine.migrations._helpers import drop_types + +revision: str = "p1_04_0019" +down_revision: str | None = "p1_03_0018" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +_DDL = r"""CREATE TYPE skeleton_check_outcome AS ENUM ( + 'not_run_short', + 'coherent', + 'incoherent_repeated_boilerplate', + 'incoherent_heading_sequence', + 'incoherent_junk_titles', + 'incoherent_over_fragmented', + 'provider_error', + 'invalid_response' +); +CREATE TYPE structure_route_tag AS ENUM ( + 'parser', + 'parser_demoted_check', + 'fallback_density', + 'fallback_leaf', + 'fallback_after_check', + 'synthetic_after_check', + 'legacy' +); + +CREATE TABLE document_skeleton_checks ( + check_id uuid PRIMARY KEY, + processing_id uuid NOT NULL, -- LOGICAL FK → processing_state; cost-attribution attempt + deployment_id uuid NOT NULL REFERENCES deployments, + doc_id uuid NOT NULL, + version_id uuid NOT NULL, + representation_id uuid NOT NULL, + candidate_skeleton_hash text NOT NULL, + stats_version text NOT NULL, + stats jsonb NOT NULL, + sampled_input_hash text, -- null: not_run_short renders no prompt (#165 review) + check_outcome skeleton_check_outcome NOT NULL, + checker_component_version text NOT NULL, + checker_model text NOT NULL, + checker_model_hash text NOT NULL, + checker_prompt_hash text NOT NULL, + checker_schema_hash text NOT NULL, + provider_failure jsonb, -- metadata envelope only; completion text is forbidden + tokens_in integer, + tokens_out integer, + cost_usd numeric(18,8), + latency_ms integer, + checked_at timestamptz NOT NULL DEFAULT now(), + FOREIGN KEY (deployment_id, doc_id) REFERENCES documents (deployment_id, doc_id) ON DELETE CASCADE, + FOREIGN KEY (deployment_id, version_id) REFERENCES document_versions (deployment_id, version_id) ON DELETE CASCADE, + FOREIGN KEY (deployment_id, representation_id) REFERENCES document_representations (deployment_id, representation_id) ON DELETE CASCADE, + CHECK (jsonb_typeof(stats) = 'object'), + CHECK (provider_failure IS NULL OR jsonb_typeof(provider_failure) = 'object'), + CHECK ((tokens_in IS NULL) = (tokens_out IS NULL)), + CHECK ((tokens_in IS NULL) = (cost_usd IS NULL)), + CHECK ((tokens_in IS NULL) = (latency_ms IS NULL)) +); +COMMENT ON TABLE document_skeleton_checks IS + 'D79/D52 append-only per-document skeleton checks. Every call/non-run is auditable; provider_failure is metadata-only and never contains completion text.'; +CREATE INDEX ix_skeleton_checks_representation + ON document_skeleton_checks (representation_id, checked_at DESC); + +CREATE TABLE document_structure_generations ( + structure_generation_id uuid PRIMARY KEY, + deployment_id uuid NOT NULL REFERENCES deployments, + doc_id uuid NOT NULL, + version_id uuid NOT NULL, + representation_id uuid NOT NULL, + skeleton_version text NOT NULL, + skeleton_hash text NOT NULL, + skeleton_producer_family text NOT NULL, -- deterministic parser/synthetic = N/A (D53) + skeleton_check_version text, -- Wave 1 checker slot + roles_version text, -- Wave 1 role-pass slot; null on demoted pre-role candidates + summary_version text, -- Wave 2 slot; null in Wave 1 + placement_version text, -- Wave 2 slot; null in Wave 1 + selecting_check_id uuid REFERENCES document_skeleton_checks, + route_tag structure_route_tag NOT NULL, + candidate_skeleton_hash text NOT NULL, + stats_version text NOT NULL, + stats jsonb NOT NULL, + pageindex_uri text, + created_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (representation_id, structure_generation_id), + FOREIGN KEY (deployment_id, doc_id) REFERENCES documents (deployment_id, doc_id) ON DELETE CASCADE, + FOREIGN KEY (deployment_id, version_id) REFERENCES document_versions (deployment_id, version_id) ON DELETE CASCADE, + FOREIGN KEY (deployment_id, representation_id) REFERENCES document_representations (deployment_id, representation_id) ON DELETE CASCADE, + CHECK (jsonb_typeof(stats) = 'object') +); +COMMENT ON TABLE document_structure_generations IS + 'D79 immutable five-slot provenance: skeleton, skeleton_check, roles, summaries, placement. A representation points to the current generation; old trees and sidecars remain addressable.'; +CREATE INDEX ix_structure_generations_representation + ON document_structure_generations (representation_id, created_at DESC); + +ALTER TABLE document_representations + ADD COLUMN current_structure_generation_id uuid; + +ALTER TABLE document_sections + ADD COLUMN structure_generation_id uuid, + ADD COLUMN heading_level smallint, + ADD COLUMN normalized_title text NOT NULL DEFAULT '', + ADD CHECK (heading_level IS NULL OR heading_level BETWEEN 1 AND 6); +""" + +_BACKFILL = r"""INSERT INTO document_structure_generations ( + structure_generation_id, deployment_id, doc_id, version_id, representation_id, + skeleton_version, skeleton_hash, skeleton_producer_family, + skeleton_check_version, roles_version, summary_version, placement_version, + selecting_check_id, route_tag, candidate_skeleton_hash, stats_version, stats, + pageindex_uri +) +SELECT + gen_random_uuid(), r.deployment_id, v.doc_id, r.version_id, r.representation_id, + coalesce(r.structurer_version, 'legacy-e0-structure'), + encode(digest(r.representation_id::text || '-legacy-skeleton', 'sha256'), 'hex'), + 'legacy-unknown', NULL, coalesce(r.structurer_version, 'legacy-e0-role'), NULL, NULL, + NULL, 'legacy', + encode(digest(r.representation_id::text || '-legacy-skeleton', 'sha256'), 'hex'), + 'legacy', jsonb_build_object( + 'stats_version', 'legacy', + 'section_count', (SELECT greatest(count(*) - 1, 0) + FROM document_sections s + WHERE s.representation_id = r.representation_id) + ), + r.pageindex_uri +FROM document_representations r +JOIN document_versions v ON v.version_id = r.version_id +WHERE EXISTS ( + SELECT 1 FROM document_sections s + WHERE s.representation_id = r.representation_id +); + +UPDATE document_sections s +SET structure_generation_id = g.structure_generation_id +FROM document_structure_generations g +WHERE g.representation_id = s.representation_id + AND g.route_tag = 'legacy'; + +UPDATE document_representations r +SET current_structure_generation_id = g.structure_generation_id +FROM document_structure_generations g +WHERE g.representation_id = r.representation_id + AND g.route_tag = 'legacy'; +""" + +_CONSTRAINTS = r"""ALTER TABLE document_sections + ALTER COLUMN structure_generation_id SET NOT NULL; +ALTER TABLE document_sections + DROP CONSTRAINT document_sections_version_id_node_path_key; +ALTER TABLE document_sections + ADD CONSTRAINT uq_sections_generation_path + UNIQUE (structure_generation_id, node_path); +ALTER TABLE document_sections + ADD CONSTRAINT fk_sections_structure_generation + FOREIGN KEY (structure_generation_id) + REFERENCES document_structure_generations (structure_generation_id) + ON DELETE CASCADE; +ALTER TABLE document_representations + ADD CONSTRAINT fk_docreps_current_structure_generation + FOREIGN KEY (representation_id, current_structure_generation_id) + REFERENCES document_structure_generations + (representation_id, structure_generation_id); +""" + + +def upgrade() -> None: + """Add generations/checks and wrap every existing section tree as legacy.""" + apply_ddl(sql=_DDL) + op.execute(_BACKFILL) + apply_ddl(sql=_CONSTRAINTS) + + +def downgrade() -> None: + """Keep current trees, then remove the D79 provenance surface. + + Effectively one-way on real deployments: non-current generations and all + check records are dropped (honest data loss), and restoring the legacy + ``UNIQUE (version_id, node_path)`` constraint fails outright when the + blockizer bump has produced two representations for one version — each + carries a root row. Downgrade exists for CI round-trips, not operations. + """ + op.execute( + """ + DELETE FROM document_sections s + USING document_representations r + WHERE s.representation_id = r.representation_id + AND s.structure_generation_id <> r.current_structure_generation_id + """ + ) + op.execute( + "ALTER TABLE document_representations" + " DROP CONSTRAINT fk_docreps_current_structure_generation" + ) + op.execute( + "ALTER TABLE document_sections DROP CONSTRAINT fk_sections_structure_generation" + ) + op.execute( + "ALTER TABLE document_sections DROP CONSTRAINT uq_sections_generation_path" + ) + op.execute( + "ALTER TABLE document_sections" + " ADD CONSTRAINT document_sections_version_id_node_path_key" + " UNIQUE (version_id, node_path)" + ) + op.execute( + "ALTER TABLE document_sections DROP COLUMN normalized_title," + " DROP COLUMN heading_level, DROP COLUMN structure_generation_id" + ) + op.execute( + "ALTER TABLE document_representations" + " DROP COLUMN current_structure_generation_id" + ) + drop_tables( + table_names=("document_structure_generations", "document_skeleton_checks") + ) + drop_types(type_names=("structure_route_tag", "skeleton_check_outcome")) diff --git a/src/rememberstack/spine/projection.py b/src/rememberstack/spine/projection.py index 1e8f31a5..1a676dba 100644 --- a/src/rememberstack/spine/projection.py +++ b/src/rememberstack/spine/projection.py @@ -573,7 +573,9 @@ def latest_snapshot( LEFT JOIN content_objects c ON c.deployment_id = v.deployment_id AND c.content_hash = v.content_hash LEFT JOIN document_sections s - ON s.version_id = v.version_id AND s.node_path = '0' + ON s.representation_id = r.representation_id + AND s.structure_generation_id = r.current_structure_generation_id + AND s.node_path = '0' WHERE d.deployment_id = :deployment_id AND d.deleted_at IS NULL ORDER BY d.doc_id """ diff --git a/src/rememberstack/workers/__init__.py b/src/rememberstack/workers/__init__.py index 264d402b..cf3dcd2f 100644 --- a/src/rememberstack/workers/__init__.py +++ b/src/rememberstack/workers/__init__.py @@ -7,7 +7,12 @@ from rememberstack.workers.base import Worker from rememberstack.workers.e0 import ConvertHandler from rememberstack.workers.e0 import E0_CONVERT_VERSION +from rememberstack.workers.e0 import E0_ROLE_VERSION +from rememberstack.workers.e0 import E0_SKELETON_CHECK_VERSION +from rememberstack.workers.e0 import E0_SKELETON_VERSION from rememberstack.workers.e0 import E0_STRUCTURE_VERSION +from rememberstack.workers.e0 import RoleSettings +from rememberstack.workers.e0 import SkeletonCheckSettings from rememberstack.workers.e0 import StructureHandler from rememberstack.workers.e0 import StructurerSettings from rememberstack.workers.e0 import UPLOAD_SOURCE_KIND @@ -118,6 +123,9 @@ "P1Settings", "P1_EMBED_CLAIMS_VERSION", "E0_CONVERT_VERSION", + "E0_ROLE_VERSION", + "E0_SKELETON_CHECK_VERSION", + "E0_SKELETON_VERSION", "E0_STRUCTURE_VERSION", "HandlerOutcome", "HandlerRegistry", @@ -146,6 +154,8 @@ "ReconcileHandler", "RECONCILE_VERSION", "StructureHandler", + "RoleSettings", + "SkeletonCheckSettings", "StructurerSettings", "SyncCycleRunner", "SyncSettings", diff --git a/src/rememberstack/workers/e0.py b/src/rememberstack/workers/e0.py index 74653a26..732d4214 100644 --- a/src/rememberstack/workers/e0.py +++ b/src/rememberstack/workers/e0.py @@ -6,15 +6,13 @@ (`///…`, D37/D65); Postgres carries only the index. -The structure stage runs the full D39 route when a model provider is -composed: the structurer LLM proposes a section tree + placement hint from -`document.md`, the deterministic snap (`core/section_snap.py`) normalizes it -onto the block grid, and the tree lands as `document_sections` rows plus the -`pageindex.json` sidecar. Every degradation path — no provider, a short -document, a failed or malformed LLM call — falls back to the synthetic root: -a document never fails structuring. +The D79 structure stage parses canonical heading blocks, checks eligible +skeletons with a bounded judge, uses an anchor-only LLM fallback for deficient +trees, assigns title-only roles, and appends immutable generations. A document +still never fails structuring. """ +from collections.abc import Iterable from datetime import datetime import hashlib import json @@ -30,17 +28,29 @@ from pydantic_settings import BaseSettings from pydantic_settings import SettingsConfigDict +from rememberstack.core import analyze_skeleton from rememberstack.core import blockize from rememberstack.core import BLOCKIZER_VERSION +from rememberstack.core import blocks_from_sidecar from rememberstack.core import ConversionRouter +from rememberstack.core import deterministic_section_role +from rememberstack.core import LONG_TITLE +from rememberstack.core import MAX_FALLBACK_DEPTH +from rememberstack.core import MIN_CHECK_SECTIONS +from rememberstack.core import parse_heading_skeleton +from rememberstack.core import resolve_fallback_skeleton from rememberstack.core import SECTION_ROLES -from rememberstack.core import snap_sections +from rememberstack.core import skeleton_hash +from rememberstack.core import SKELETON_PARSER_VERSION +from rememberstack.core import SKELETON_STATS_VERSION +from rememberstack.core import SkeletonAnalysis from rememberstack.core import storage_class_for from rememberstack.model import Block from rememberstack.model import ClaimedWork from rememberstack.model import ConversionError from rememberstack.model import DocumentUpload from rememberstack.model import EnqueueWork +from rememberstack.model import FallbackStructureResponse from rememberstack.model import IngestedVersion from rememberstack.model import ModelRequest from rememberstack.model import NonRetryableHandlerError @@ -50,10 +60,16 @@ from rememberstack.model import ProcessingLane from rememberstack.model import ProviderAccountingError from rememberstack.model import ProviderCallError +from rememberstack.model import ProviderCallUsage +from rememberstack.model import ProviderInvalidResponseError from rememberstack.model import RepresentationRecord +from rememberstack.model import RoleClassificationResponse from rememberstack.model import SectionTreeRecord +from rememberstack.model import SkeletonCheckOutcome +from rememberstack.model import SkeletonCheckRecord +from rememberstack.model import SkeletonCheckResponse from rememberstack.model import SnappedSection -from rememberstack.model import StructureResponse +from rememberstack.model import StructureRouteTag from rememberstack.model import StructureSource from rememberstack.model import UnroutableMimeError from rememberstack.model import UploadRecord @@ -67,9 +83,38 @@ E0_CONVERT_VERSION: Final = "e0-convert-2026.07" """The convert sub-worker's component version (D12 idempotency key member).""" -E0_STRUCTURE_VERSION: Final = "e0-structure-2026.07c:temp0-1" -"""The structure stage's component version (D39): LLM route + snap algorithm. -07c pins temperature=0.0 — generation parameters are part of provenance.""" +E0_STRUCTURE_VERSION: Final = "e0-structure-2026.07d:d79-wave1" +"""The aggregate Wave-1 generation identity. + +Maps old ``e0-structure-2026.07c:temp0-1`` (one-shot offsets/tree/roles/ +summaries/placement) to the D79 split: deterministic/anchor skeleton, sanity +check, and role pass. Summary and placement generation slots remain empty. +""" + +E0_SKELETON_VERSION: Final = ( + f"e0-skeleton-2026.07a:d79:{SKELETON_PARSER_VERSION}" + f":anchor-v1-depth{MAX_FALLBACK_DEPTH}" +) +"""Skeleton contract: canonical heading stack plus exact-anchor fallback.""" + +SKELETON_CHECK_PROMPT_CEILING: Final = 16_000 +"""Hard total checker prompt ceiling, pinned by the checker version.""" + +E0_SKELETON_CHECK_VERSION: Final = ( + f"e0-skeleton-check-2026.07a:{SKELETON_STATS_VERSION}" + f":sample-v2:enum-v1:ceiling{SKELETON_CHECK_PROMPT_CEILING}" +) +"""Checker prompt/sampling/schema generation. sample-v2: the greedy fit went +arithmetic (conservative digit/newline padding), so boundary selections can +differ from the per-trial re-render greedy it replaced.""" + +ROLE_PROMPT_CEILING: Final = 12_000 +"""Hard total title-only role prompt ceiling, pinned by the role version.""" + +E0_ROLE_VERSION: Final = ( + f"e0-role-2026.07a:title-rules-v1:classifier-v1:ceiling{ROLE_PROMPT_CEILING}" +) +"""Deterministic normalized-title rules plus bounded title-only classifier.""" UPLOAD_SOURCE_KIND: Final = "upload" """The one-shot upload connector's source kind (D55 lineage identity).""" @@ -309,7 +354,9 @@ def handle(self, *, work: ClaimedWork, meter: CostMeterPort) -> HandlerOutcome: "blockizer_version": BLOCKIZER_VERSION, "block_count": len(blocks), "markdown_chars": len(result.document_md), - "blocks": [block.model_dump(mode="json") for block in blocks], + "blocks": [ + block.model_dump(mode="json", exclude_none=True) for block in blocks + ], } ) manifest_bytes = _json_bytes( @@ -389,14 +436,13 @@ def _structure_follow_up( class StructurerSettings(BaseSettings): - """The full structure route's knobs (D39/D70 port defaults; D22 numbers). - - The model seat follows the D70 principle — a per-deployment port - configuration, defaulting to the extraction tier. The thresholds are - starting points to be measured: a document below ``min_blocks_for_llm`` - is served by the synthetic root alone (the LLM cannot improve a - three-paragraph note), and the prompt reads at most - ``max_prompt_chars`` of `document.md`. + """The D79 fallback structure-proposal seat and deterministic gate knobs. + + ``REMEMBERSTACK_STRUCTURER_*`` is retained for deployment compatibility + but now names only the anchor-proposal fallback. ``min_blocks_for_llm`` is + accepted as a deprecated compatibility value; parser/check routing never + reads it. Gate metrics come from the same computation persisted as D79 + stats. These operational starting points are not stat-to-verdict rules. """ model_config = SettingsConfigDict(env_prefix="REMEMBERSTACK_STRUCTURER_") @@ -404,40 +450,84 @@ class StructurerSettings(BaseSettings): model: str = Field(default="openai/gpt-5.6-luna") min_blocks_for_llm: int = Field(default=8, ge=1) max_prompt_chars: int = Field(default=200_000, ge=1_000) + min_heading_density_per_10k: float = Field(default=0.25, ge=0) + max_oversized_leaf_ratio: float = Field(default=0.75, ge=0, le=1) + + +class SkeletonCheckSettings(BaseSettings): + """The independent D79 skeleton-check seat.""" + model_config = SettingsConfigDict(env_prefix="REMEMBERSTACK_SKELETON_CHECK_") -_STRUCTURE_PROMPT: Final = """You are a document structurer. Read the document \ -and propose its hierarchical section tree (a table of contents with spans). + model: str = Field(default="z-ai/glm-4.7-flash") + + +class RoleSettings(BaseSettings): + """The narrow bounded title-only role classifier seat.""" + + model_config = SettingsConfigDict(env_prefix="REMEMBERSTACK_ROLE_") + + model: str = Field(default="z-ai/glm-4.7-flash") + + +_FALLBACK_PROMPT: Final = """You propose section starts for a document whose \ +markdown headings were insufficient. Return a hierarchical tree of anchors. + +Each node has exactly: +- anchor: an EXACT non-empty substring contained wholly in one rendered block +- occurrence_index: zero-based occurrence of that exact substring inside the \ +enclosing parent's block range +- children: nested nodes using the same shape + +Never return character offsets, block ordinals, roles, summaries, or placement. +If no reliable internal sections exist, return an empty sections list. + +Document title: {title} +Source kind: {source_kind} +Rendered blocks ({included} included, {omitted} omitted): +{blocks}""" -Rules: -- Return sections as a JSON tree: each node has "title", "role", \ -"char_start", "char_end", "summary" (one line), and "children" (nested nodes). -- Character offsets index into EXACTLY the document text below (0-based; \ -char_end is exclusive). Top-level sections should cover the document in order. -- "role" must be one of: {roles}. -- Also return "placement": a proposed corpus path for this document, e.g. \ -"/finance/annual-reports/2023/" — where it would live in an ideal directory \ -tree of a whole document collection. Advisory only. -- Do not invent sections a short flat document does not have; return an empty \ -"sections" list if the document has no internal structure worth naming. +_CHECK_INSTRUCTION: Final = """Judge whether this document SECTION SKELETON \ +is structurally coherent. You see statistics and heading lines only, never \ +section body text. Return exactly one primary verdict: +coherent | incoherent_repeated_boilerplate | \ +incoherent_heading_sequence | incoherent_junk_titles | \ +incoherent_over_fragmented. +Do not propose structure and do not add a reason or confidence.""" +_CHECK_PROMPT_TEMPLATE: Final = """{instruction} Document title: {title} +Source kind: {source_kind} +Stats: +{stats} +Section lines: {included} included, {omitted} omitted +{lines}""" -DOCUMENT: -{document}""" +_ROLE_INSTRUCTION: Final = """Assign one section role from this closed set: +{roles} +Use only the provided title. Return assignments keyed by the exact node_path. +Do not summarize or inspect body content.""" + +_CHECK_PROMPT_HASH: Final = hashlib.sha256( + _CHECK_PROMPT_TEMPLATE.format( + instruction=_CHECK_INSTRUCTION, + title="{capped_document_title}", + source_kind="{capped_source_kind}", + stats="{canonical_stats_json}", + included="{included_count}", + omitted="{omitted_count}", + lines="{ordered_sampled_lines}", + ).encode("utf-8") +).hexdigest() +_CHECK_SCHEMA_HASH: Final = hashlib.sha256( + json.dumps( + SkeletonCheckResponse.model_json_schema(), separators=(",", ":"), sort_keys=True + ).encode("utf-8") +).hexdigest() class StructureHandler: - """The structure stage (D39): the full PageIndex-style route, snap-guarded. - - With a composed model provider and a long-enough document, the LLM - proposes the section tree and placement hint; the deterministic snap - normalizes it onto the block grid; rows + `pageindex.json` sidecar land - and the chain completes — representation ready, live-reading pointer - set, lineage currency moved (D54). Without a provider — or when the - call fails or returns nothing usable — the document gets the synthetic - root: structuring never fails a document. - """ + """D79 parse → stats/gates → check/fallback → roles → immutable generation.""" def __init__( self, @@ -446,53 +536,144 @@ def __init__( artifact_store: ObjectStorePort, model_provider: ModelProviderPort | None = None, settings: StructurerSettings | None = None, + check_settings: SkeletonCheckSettings | None = None, + role_settings: RoleSettings | None = None, ) -> None: - """Bind the handler to its catalog, artifacts bucket, and model seat.""" + """Bind the handler to all three independently versioned seats.""" self._catalog = catalog self._artifact_store = artifact_store self._model_provider = model_provider self._settings = settings or StructurerSettings() + self._check_settings = check_settings or SkeletonCheckSettings() + self._role_settings = role_settings or RoleSettings() def handle(self, *, work: ClaimedWork, meter: CostMeterPort) -> HandlerOutcome: - """Structure one representation and flip currency.""" + """Run the acyclic D79 state machine and flip currency once.""" source = self._catalog.structure_source( representation_id=_payload_uuid(work=work, field="representation_id") ) blocks_doc = json.loads( self._artifact_store.read_bytes(key=ObjectKey(source.blocks_uri)) ) - blocks = tuple( - Block.model_validate(payload) for payload in blocks_doc["blocks"] - ) - response = self._propose(source=source, block_count=len(blocks), meter=meter) - proposed = response.sections if response is not None else () - placement = (response.placement or None) if response is not None else None - sections = snap_sections( - proposed=proposed, + markdown = self._artifact_store.read_bytes( + key=ObjectKey(source.markdown_uri) + ).decode("utf-8") + blocks = blocks_from_sidecar(blocks_doc=blocks_doc, document_md=markdown) + parsed = parse_heading_skeleton( blocks=blocks, title=source.title, markdown_chars=blocks_doc["markdown_chars"], ) - structurer_name = "pageindex_llm" if len(sections) > 1 else "synthetic_root" - persisted = self._catalog.record_section_tree( - record=SectionTreeRecord( - deployment_id=source.deployment_id, - doc_id=source.doc_id, - version_id=source.version_id, - representation_id=source.representation_id, - sections=sections, - placement_path=placement, - structurer_name=structurer_name, - structurer_version=E0_STRUCTURE_VERSION, - ) + parsed_analysis = analyze_skeleton( + sections=parsed, blocks=blocks, markdown_chars=len(markdown) ) - # the sidecar is derived from what the database actually persisted — - # a retry that lost to an earlier attempt must not write a fresher - # LLM proposal beside rows carrying the first one (Codex review): - self._write_sidecar( + parsed_hash = skeleton_hash(sections=parsed) + + route: StructureRouteTag + selected = parsed + selection_analysis = parsed_analysis + selection_candidate_hash = parsed_hash + selecting_check_id: UUID | None = None + check_version: str | None = None + configured_check_version = _skeleton_check_version( + settings=self._check_settings + ) + producer_family = "N/A" + + if parsed_analysis.heading_density < self._settings.min_heading_density_per_10k: + ( + route, + selected, + selection_analysis, + selection_candidate_hash, + selecting_check_id, + producer_family, + ) = self._fallback_with_terminal( + source=source, + work=work, + blocks=blocks, + markdown=markdown, + meter=meter, + parsed_root=parsed[0], + kept_route=StructureRouteTag.FALLBACK_DENSITY, + ) + check_version = configured_check_version + elif ( + parsed_analysis.oversized_leaf_ratio + > self._settings.max_oversized_leaf_ratio + ): + ( + route, + selected, + selection_analysis, + selection_candidate_hash, + selecting_check_id, + producer_family, + ) = self._fallback_with_terminal( + source=source, + work=work, + blocks=blocks, + markdown=markdown, + meter=meter, + parsed_root=parsed[0], + kept_route=StructureRouteTag.FALLBACK_LEAF, + ) + check_version = configured_check_version + else: + initial_outcome, initial_check_id = self._check( + source=source, + work=work, + sections=parsed, + analysis=parsed_analysis, + meter=meter, + terminal=False, + ) + selecting_check_id = initial_check_id + check_version = configured_check_version + if _is_incoherent(outcome=initial_outcome): + self._persist_generation( + source=source, + sections=parsed, + route=StructureRouteTag.PARSER_DEMOTED_CHECK, + analysis=parsed_analysis, + candidate_skeleton_hash=parsed_hash, + selecting_check_id=initial_check_id, + check_version=configured_check_version, + roles_version=None, + producer_family="N/A", + make_current=False, + ) + ( + route, + selected, + selection_analysis, + selection_candidate_hash, + selecting_check_id, + producer_family, + ) = self._fallback_with_terminal( + source=source, + work=work, + blocks=blocks, + markdown=markdown, + meter=meter, + parsed_root=parsed[0], + kept_route=StructureRouteTag.FALLBACK_AFTER_CHECK, + ) + else: + route = StructureRouteTag.PARSER + + with_roles = self._assign_roles(sections=selected, meter=meter) + self._persist_generation( source=source, - sections=persisted.sections, - placement=persisted.placement_path, + sections=with_roles, + route=route, + analysis=selection_analysis, + candidate_skeleton_hash=selection_candidate_hash, + selecting_check_id=selecting_check_id, + check_version=check_version, + roles_version=_role_version(settings=self._role_settings), + producer_family=producer_family, + make_current=True, ) return HandlerOutcome( follow_up=( @@ -512,66 +693,624 @@ def handle(self, *, work: ClaimedWork, meter: CostMeterPort) -> HandlerOutcome: ) ) - def _propose( - self, *, source: StructureSource, block_count: int, meter: CostMeterPort - ) -> StructureResponse | None: - """Ask the structurer LLM for a tree; every failure degrades to None.""" + def _fallback_with_terminal( + self, + *, + source: StructureSource, + work: ClaimedWork, + blocks: tuple[Block, ...], + markdown: str, + meter: CostMeterPort, + parsed_root: SnappedSection, + kept_route: StructureRouteTag, + ) -> tuple[ + StructureRouteTag, tuple[SnappedSection, ...], SkeletonAnalysis, str, UUID, str + ]: + """Every fallback tree faces the terminal judge (§4.1 state machine). + + The plausible-wrong-tree failure mode does not care which gate demoted + the document, so density/leaf demotions get the same terminal check as + incoherent-verdict demotions. Terminal incoherence degrades to the + synthetic root — honest no-structure over a plausible-wrong tree; the + check-record chain preserves which gate started the demotion. The + returned analysis/candidate hash always describe the SELECTED tree — + generation rows describe what is persisted, check records describe the + candidates they judged. + """ + fallback, producer_family = self._fallback( + source=source, blocks=blocks, markdown=markdown, meter=meter + ) + fallback_analysis = analyze_skeleton( + sections=fallback, blocks=blocks, markdown_chars=len(markdown) + ) + terminal_outcome, terminal_check_id = self._check( + source=source, + work=work, + sections=fallback, + analysis=fallback_analysis, + meter=meter, + terminal=True, + ) + if _is_incoherent(outcome=terminal_outcome): + synthetic = (parsed_root,) + return ( + StructureRouteTag.SYNTHETIC_AFTER_CHECK, + synthetic, + analyze_skeleton( + sections=synthetic, blocks=blocks, markdown_chars=len(markdown) + ), + skeleton_hash(sections=synthetic), + terminal_check_id, + "N/A", + ) + return ( + kept_route, + fallback, + fallback_analysis, + skeleton_hash(sections=fallback), + terminal_check_id, + producer_family, + ) + + def _fallback( + self, + *, + source: StructureSource, + blocks: tuple[Block, ...], + markdown: str, + meter: CostMeterPort, + ) -> tuple[tuple[SnappedSection, ...], str]: + """Return the resolved proposal and actual family; failures are N/A roots.""" if self._model_provider is None: - return None - if block_count < self._settings.min_blocks_for_llm: - return None # a short document: the synthetic root serves it - markdown = self._artifact_store.read_bytes( - key=ObjectKey(source.markdown_uri) - ).decode("utf-8") - prompt = _STRUCTURE_PROMPT.format( - roles=", ".join(sorted(SECTION_ROLES)), - title=source.title or "(untitled)", - document=markdown[: self._settings.max_prompt_chars], + return ( + resolve_fallback_skeleton( + proposed=(), blocks=blocks, document_md=markdown, title=source.title + ), + "N/A", + ) + prompt = _render_fallback_prompt( + source=source, + blocks=blocks, + markdown=markdown, + ceiling=self._settings.max_prompt_chars, ) try: generated = self._model_provider.generate( request=ModelRequest( model=self._settings.model, prompt=prompt, temperature=0.0 ), - response_type=StructureResponse, + response_type=FallbackStructureResponse, ) except ProviderAccountingError: - raise # budget enforcement must never degrade missing usage to zero + raise except ProviderCallError as error: if error.usage is not None: meter.record( - call_key="structure_failure", - tier="failed_response", + call_key="structure_fallback_failure", + tier="fallback_failed_response", usage=error.usage, ) - return None + return ( + resolve_fallback_skeleton( + proposed=(), blocks=blocks, document_md=markdown, title=source.title + ), + "N/A", + ) except Exception: # noqa: BLE001 — a document never fails structuring - return None - meter.record(call_key="structure", tier="structure", usage=generated.usage) - return generated.output + return ( + resolve_fallback_skeleton( + proposed=(), blocks=blocks, document_md=markdown, title=source.title + ), + "N/A", + ) + meter.record( + call_key="structure_fallback", + tier="structure_fallback", + usage=generated.usage, + ) + return ( + resolve_fallback_skeleton( + proposed=generated.output.sections, + blocks=blocks, + document_md=markdown, + title=source.title, + ), + _model_family(model=generated.usage.model_name), + ) - def _write_sidecar( + def _check( self, *, source: StructureSource, sections: tuple[SnappedSection, ...], - placement: str | None, + analysis: SkeletonAnalysis, + work: ClaimedWork, + meter: CostMeterPort, + terminal: bool, + ) -> tuple[SkeletonCheckOutcome, UUID]: + """Append one explicit checker outcome; failures are fail-open.""" + outcome = SkeletonCheckOutcome.NOT_RUN_SHORT + usage: ProviderCallUsage | None = None + failure: dict[str, str | int | float | bool | None] | None = None + # A prompt no call will ever send gets no render and no hash — a + # not_run_short record with a sampled_input_hash would bookkeep an + # input that never existed (and rendering is the expensive part). + sampled_hash: str | None = None + if len(sections) - 1 >= MIN_CHECK_SECTIONS: + prompt = _render_check_prompt( + source=source, sections=sections, analysis=analysis + ) + sampled_hash = hashlib.sha256(prompt.encode("utf-8")).hexdigest() + if self._model_provider is None: + outcome = SkeletonCheckOutcome.PROVIDER_ERROR + failure = {"error_type": "provider_unavailable", "has_usage": False} + else: + try: + generated = self._model_provider.generate( + request=ModelRequest( + model=self._check_settings.model, + prompt=prompt, + temperature=0.0, + ), + response_type=SkeletonCheckResponse, + ) + usage = generated.usage + outcome = SkeletonCheckOutcome(generated.output.verdict) + except ProviderAccountingError: + raise + except ( + ProviderInvalidResponseError, + # ValueError covers pydantic ValidationError AND the + # off-enum verdict coercion from a non-OpenRouter + # provider — both are invalid responses, not outages + ValueError, + ) as error: + outcome = SkeletonCheckOutcome.INVALID_RESPONSE + usage = getattr(error, "usage", None) or usage + failure = _failure_envelope(error=error, usage=usage) + except ProviderCallError as error: + outcome = SkeletonCheckOutcome.PROVIDER_ERROR + usage = error.usage + failure = _failure_envelope(error=error, usage=usage) + except Exception as error: # noqa: BLE001 - guard is fail-open + outcome = SkeletonCheckOutcome.PROVIDER_ERROR + failure = _failure_envelope(error=error, usage=None) + + call_key = "skeleton_check_terminal" if terminal else "skeleton_check" + if usage is not None: + meter.record(call_key=call_key, tier="skeleton_check", usage=usage) + check_id = uuid4() + checker_model = ( + usage.model_name if usage is not None else self._check_settings.model + ) + self._catalog.record_skeleton_check( + record=SkeletonCheckRecord( + check_id=check_id, + processing_id=work.processing_id, + deployment_id=source.deployment_id, + doc_id=source.doc_id, + version_id=source.version_id, + representation_id=source.representation_id, + candidate_skeleton_hash=skeleton_hash(sections=sections), + stats_version=SKELETON_STATS_VERSION, + stats=analysis.stats, + sampled_input_hash=sampled_hash, + check_outcome=outcome, + checker_component_version=E0_SKELETON_CHECK_VERSION, + checker_model=checker_model, + checker_model_hash=hashlib.sha256( + checker_model.encode("utf-8") + ).hexdigest(), + checker_prompt_hash=_CHECK_PROMPT_HASH, + checker_schema_hash=_CHECK_SCHEMA_HASH, + provider_failure=failure, + tokens_in=usage.tokens_in if usage is not None else None, + tokens_out=usage.tokens_out if usage is not None else None, + cost_usd=usage.cost_usd if usage is not None else None, + latency_ms=usage.latency_ms if usage is not None else None, + ) + ) + return outcome, check_id + + def _assign_roles( + self, *, sections: tuple[SnappedSection, ...], meter: CostMeterPort + ) -> tuple[SnappedSection, ...]: + """Rules first, one bounded title-only call second, explicit body last.""" + roles: dict[str, str] = {"0": "body"} + undecided: list[SnappedSection] = [] + for section in sections[1:]: + role = deterministic_section_role(normalized_title=section.normalized_title) + if role is None: + undecided.append(section) + else: + roles[section.node_path] = role + + included, prompt = _render_role_prompt(sections=tuple(undecided)) + if included and self._model_provider is not None: + try: + generated = self._model_provider.generate( + request=ModelRequest( + model=self._role_settings.model, prompt=prompt, temperature=0.0 + ), + response_type=RoleClassificationResponse, + ) + meter.record( + call_key="section_roles", + tier="title_classifier", + usage=generated.usage, + ) + allowed = {section.node_path for section in included} + for assignment in generated.output.assignments: + if ( + assignment.node_path in allowed + and assignment.node_path not in roles + ): + roles[assignment.node_path] = assignment.role + except ProviderAccountingError: + raise + except ProviderCallError as error: + if error.usage is not None: + meter.record( + call_key="section_roles_failure", + tier="title_classifier_failed_response", + usage=error.usage, + ) + except Exception: # noqa: BLE001 - undecided titles become body + pass + return tuple( + section.model_copy(update={"role": roles.get(section.node_path, "body")}) + for section in sections + ) + + def _persist_generation( + self, + *, + source: StructureSource, + sections: tuple[SnappedSection, ...], + route: StructureRouteTag, + analysis: SkeletonAnalysis, + candidate_skeleton_hash: str, + selecting_check_id: UUID | None, + check_version: str | None, + roles_version: str | None, + producer_family: str, + make_current: bool, ) -> None: - """Write the reproducible `pageindex.json` next to `document.md` (D39).""" - sidecar_key = source.blocks_uri.rsplit("/", 1)[0] + "/pageindex.json" + """Append one generation, then write its versioned sidecar. + + The generation identity is seeded from the SELECTED tree and route — + never from the checker version. Both halves of the D79 rule follow + from one seed: a checker bump over an unchanged route+tree derives + the same id (ON CONFLICT no-op — no minted generation, no pointer + churn), while a genuine route flip or a different fallback tree + changes the seed and appends a real generation. Role version stays + in the seed because roles change the section rows themselves. + """ + skeleton_version = _skeleton_version(settings=self._settings) + generation_id = uuid5( + NAMESPACE_URL, + "rememberstack:structure:" + f"{source.representation_id}:{E0_STRUCTURE_VERSION}:" + f"{route.value}:{skeleton_hash(sections=sections)}:" + f"{skeleton_version}:{roles_version or 'none'}", + ) + sidecar_key = ( + source.blocks_uri.rsplit("/", 1)[0] + + f"/structure/{generation_id}/pageindex.json" + ) + persisted = self._catalog.record_section_tree( + record=SectionTreeRecord( + deployment_id=source.deployment_id, + doc_id=source.doc_id, + version_id=source.version_id, + representation_id=source.representation_id, + structure_generation_id=generation_id, + sections=sections, + placement_path=None, + structurer_name=route.value, + structurer_version=E0_STRUCTURE_VERSION, + skeleton_version=skeleton_version, + skeleton_hash=skeleton_hash(sections=sections), + skeleton_producer_family=producer_family, + skeleton_check_version=check_version, + roles_version=roles_version, + selecting_check_id=selecting_check_id, + route_tag=route, + candidate_skeleton_hash=candidate_skeleton_hash, + stats_version=SKELETON_STATS_VERSION, + stats=analysis.stats, + pageindex_uri=sidecar_key, + make_current=make_current, + ) + ) payload = _json_bytes( payload={ - "structurer_version": E0_STRUCTURE_VERSION, - "placement": placement, - "sections": [section.model_dump(mode="json") for section in sections], + "structure_generation_id": str(persisted.structure_generation_id), + "structurer_version": persisted.structurer_version, + "generations": { + "skeleton": persisted.skeleton_version, + "skeleton_check": persisted.skeleton_check_version, + "roles": persisted.roles_version, + "summary": persisted.summary_version, + "placement": persisted.placement_version, + }, + "route_tag": persisted.route_tag.value, + "selecting_check_id": ( + str(persisted.selecting_check_id) + if persisted.selecting_check_id is not None + else None + ), + "candidate_skeleton_hash": persisted.candidate_skeleton_hash, + "stats_version": persisted.stats_version, + "stats": persisted.stats.model_dump(mode="json"), + "placement": persisted.placement_path, + "sections": [ + section.model_dump(mode="json") for section in persisted.sections + ], } ) try: self._artifact_store.write_bytes( - key=ObjectKey(sidecar_key), content=payload + key=ObjectKey(persisted.pageindex_uri), content=payload ) except ObjectAlreadyExistsError: - pass # a retried attempt replays; the first write is the truth + pass + + +def _render_fallback_prompt( + *, source: StructureSource, blocks: tuple[Block, ...], markdown: str, ceiling: int +) -> str: + """Render whole block excerpts under the compatibility prompt ceiling.""" + rendered: list[str] = [] + for block in blocks: + raw = markdown[block.char_start : block.char_end] + line = f"[{block.type.value}]\n{raw}" + trial = "\n\n".join((*rendered, line)) + prompt = _FALLBACK_PROMPT.format( + title=(source.title or "(untitled)")[:LONG_TITLE], + source_kind=source.source_kind[:LONG_TITLE], + included=len(rendered) + 1, + omitted=len(blocks) - len(rendered) - 1, + blocks=trial, + ) + if len(prompt) > ceiling: + break + rendered.append(line) + return _FALLBACK_PROMPT.format( + title=(source.title or "(untitled)")[:LONG_TITLE], + source_kind=source.source_kind[:LONG_TITLE], + included=len(rendered), + omitted=len(blocks) - len(rendered), + blocks="\n\n".join(rendered), + ) + + +def _render_check_prompt( + *, + source: StructureSource, + sections: tuple[SnappedSection, ...], + analysis: SkeletonAnalysis, +) -> str: + """Anomaly-first deterministic sampling, rendered back in document order. + + Linear, not quadratic: every candidate line renders exactly once, the + static envelope (stats block, header) renders exactly once, and the + greedy fit tracks lengths arithmetically — the previous shape re-rendered + the full prompt per candidate, ~5s of pure CPU on a 6,000-heading + document. The fit is conservative (worst-case count digits, a newline + per line), so the final render can only land under the ceiling; the + assert is the belt to that suspenders. + """ + candidates = sections[1:] + priorities = _anomaly_priority(sections=candidates, analysis=analysis) + line_for = tuple( + _check_line( + section=section, + direct_body_chars=analysis.direct_body_chars[section.node_path], + ) + for section in candidates + ) + stats_json = json.dumps( + analysis.stats.model_dump(mode="json"), + ensure_ascii=False, + indent=2, + sort_keys=True, + ) + + def compose(ordered: tuple[int, ...]) -> str: + return _CHECK_PROMPT_TEMPLATE.format( + instruction=_CHECK_INSTRUCTION, + title=(source.title or "(untitled)")[:LONG_TITLE], + source_kind=source.source_kind[:LONG_TITLE], + stats=stats_json, + included=len(ordered), + omitted=len(candidates) - len(ordered), + lines="\n".join(line_for[index] for index in ordered), + ) + + overhead = len(compose(())) + budget = SKELETON_CHECK_PROMPT_CEILING - overhead + selected: set[int] = set() + used = 0 + for index in priorities: + cost = len(line_for[index]) + 1 # +1: the joining newline, worst case + if used + cost <= budget: + selected.add(index) + used += cost + rendered = compose(tuple(sorted(selected))) + if len(rendered) > SKELETON_CHECK_PROMPT_CEILING: + raise AssertionError("checker prompt base exceeds its versioned hard ceiling") + return rendered + + +def _anomaly_priority( + *, sections: tuple[SnappedSection, ...], analysis: SkeletonAnalysis +) -> tuple[int, ...]: + """Repeated titles, jumps, tiny bodies, large leaves, then head/tail.""" + priorities: list[int] = [] + + def add(indexes: Iterable[int]) -> None: + for index in indexes: + if index not in priorities: + priorities.append(index) + + multiplicity: dict[str, int] = {} + for section in sections: + multiplicity[section.normalized_title] = ( + multiplicity.get(section.normalized_title, 0) + 1 + ) + add( + tuple( + index + for index, section in sorted( + enumerate(sections), + key=lambda item: (-multiplicity[item[1].normalized_title], item[0]), + ) + if multiplicity[section.normalized_title] > 1 + )[:8] + ) + jumps = sorted( + ( + ( + # raw levels only — a pair missing a raw level is no jump, + # matching the d79-v2 stat rule (tree depth would fake zero + # or invent jumps that steer the sample) + max(0, right.heading_level - left.heading_level - 1) + if left.heading_level is not None and right.heading_level is not None + else 0, + index + 1, + ) + for index, (left, right) in enumerate( + zip(sections, sections[1:], strict=False) + ) + ), + key=lambda item: (-item[0], item[1]), + ) + add(tuple(index for jump, index in jumps if jump > 0)[:8]) + add( + tuple( + index + for index, _ in sorted( + enumerate(sections), + key=lambda item: ( + analysis.direct_body_chars[item[1].node_path], + item[0], + ), + ) + )[:8] + ) + add( + tuple( + index + for index, section in sorted( + enumerate(sections), + key=lambda item: ( + -analysis.leaf_span_chars.get(item[1].node_path, -1), + item[0], + ), + ) + if section.node_path in analysis.leaf_span_chars + )[:8] + ) + add(range(min(8, len(sections)))) + add(range(max(0, len(sections) - 8), len(sections))) + add(range(len(sections))) + return tuple(priorities) + + +def _check_line(*, section: SnappedSection, direct_body_chars: int) -> str: + title = section.title + if len(title) > LONG_TITLE: + title = title[: LONG_TITLE - 1] + "…" + # anchor-derived sections carry no raw level; "null" tells the judge the + # truth instead of a tree depth masquerading as one + level = "null" if section.heading_level is None else str(section.heading_level) + return ( + f"(level={level}, " + f"title={json.dumps(title, ensure_ascii=False)}, " + f"direct_body_chars={direct_body_chars})" + ) + + +def _render_role_prompt( + *, sections: tuple[SnappedSection, ...] +) -> tuple[tuple[SnappedSection, ...], str]: + included: list[SnappedSection] = [] + instruction = _ROLE_INSTRUCTION.format(roles=", ".join(sorted(SECTION_ROLES))) + for section in sections: + line = ( + f"{section.node_path}\t" + f"{json.dumps(section.title[:LONG_TITLE], ensure_ascii=False)}" + ) + trial = "\n".join( + ( + instruction, + f"Title lines: {len(included) + 1} included," + f" {len(sections) - len(included) - 1} omitted", + *( + f"{item.node_path}\t" + f"{json.dumps(item.title[:LONG_TITLE], ensure_ascii=False)}" + for item in included + ), + line, + ) + ) + if len(trial) > ROLE_PROMPT_CEILING: + break + included.append(section) + prompt = "\n".join( + ( + instruction, + f"Title lines: {len(included)} included," + f" {len(sections) - len(included)} omitted", + *( + f"{item.node_path}\t" + f"{json.dumps(item.title[:LONG_TITLE], ensure_ascii=False)}" + for item in included + ), + ) + ) + return tuple(included), prompt + + +def _failure_envelope( + *, error: Exception, usage: ProviderCallUsage | None +) -> dict[str, str | int | float | bool | None]: + """Metadata-only failure fingerprint; never persist exception/completion text.""" + return { + "error_type": type(error).__name__, + "error_fingerprint": hashlib.sha256(str(error).encode("utf-8")).hexdigest(), + "has_usage": usage is not None, + } + + +def _is_incoherent(*, outcome: SkeletonCheckOutcome) -> bool: + return outcome.value.startswith("incoherent_") + + +def _model_family(*, model: str) -> str: + return model.split("/", 1)[0] if "/" in model else model + + +def _skeleton_version(*, settings: StructurerSettings) -> str: + return ( + f"{E0_SKELETON_VERSION}" + f":density{settings.min_heading_density_per_10k:g}" + f":leaf{settings.max_oversized_leaf_ratio:g}" + f":fallback-model-{hashlib.sha256(settings.model.encode()).hexdigest()[:12]}" + ) + + +def _role_version(*, settings: RoleSettings) -> str: + """Bind the independently configured classifier model into role provenance.""" + model_hash = hashlib.sha256(settings.model.encode("utf-8")).hexdigest()[:12] + return f"{E0_ROLE_VERSION}:model-{model_hash}" + + +def _skeleton_check_version(*, settings: SkeletonCheckSettings) -> str: + """Bind the checker seat model into its generation slot.""" + model_hash = hashlib.sha256(settings.model.encode("utf-8")).hexdigest()[:12] + return f"{E0_SKELETON_CHECK_VERSION}:model-{model_hash}" def _payload_uuid(*, work: ClaimedWork, field: str) -> UUID: diff --git a/src/rememberstack/workers/e1.py b/src/rememberstack/workers/e1.py index f66ac1c1..86a842f5 100644 --- a/src/rememberstack/workers/e1.py +++ b/src/rememberstack/workers/e1.py @@ -16,12 +16,12 @@ from pydantic_settings import BaseSettings from pydantic_settings import SettingsConfigDict +from rememberstack.core import blocks_from_sidecar from rememberstack.core import CHUNKER_VERSION from rememberstack.core import chunker_version from rememberstack.core import ChunkerParams from rememberstack.core import extraction_input_hash from rememberstack.core import pack_blocks -from rememberstack.model import Block from rememberstack.model import CarryForwardSource from rememberstack.model import ChunkForEmbedding from rememberstack.model import ChunkRecord @@ -113,7 +113,7 @@ def handle(self, *, work: ClaimedWork, meter: CostMeterPort) -> HandlerOutcome: blocks_doc = json.loads( self._artifact_store.read_bytes(key=ObjectKey(source.blocks_uri)) ) - blocks = tuple(Block.model_validate(block) for block in blocks_doc["blocks"]) + blocks = blocks_from_sidecar(blocks_doc=blocks_doc, document_md=document_md) packed = pack_blocks( blocks=blocks, sections=source.sections, diff --git a/src/tests/adapters/test_openrouter.py b/src/tests/adapters/test_openrouter.py index 7d131c6b..96744acb 100644 --- a/src/tests/adapters/test_openrouter.py +++ b/src/tests/adapters/test_openrouter.py @@ -17,11 +17,13 @@ from rememberstack.model import ClaimifyResponse from rememberstack.model import EmbeddingRequest from rememberstack.model import FactLabelResponse +from rememberstack.model import FallbackStructureResponse from rememberstack.model import ModelRequest from rememberstack.model import NormalizationResponse from rememberstack.model import ProviderAccountingError +from rememberstack.model import RoleClassificationResponse from rememberstack.model import SelectionResponse -from rememberstack.model import StructureResponse +from rememberstack.model import SkeletonCheckResponse class _Answer(BaseModel): @@ -297,7 +299,14 @@ def post(*, path: str, payload: dict[str, object]) -> dict[str, object]: @pytest.mark.parametrize( - "response_type", (StructureResponse, SelectionResponse, ClaimifyResponse) + "response_type", + ( + FallbackStructureResponse, + SkeletonCheckResponse, + RoleClassificationResponse, + SelectionResponse, + ClaimifyResponse, + ), ) def test_strict_schema_closes_every_nested_object_and_removes_defaults( response_type: type[BaseModel], diff --git a/src/tests/blockizer_corpus/expected_hashes.json b/src/tests/blockizer_corpus/expected_hashes.json index c1551587..130f7f78 100644 --- a/src/tests/blockizer_corpus/expected_hashes.json +++ b/src/tests/blockizer_corpus/expected_hashes.json @@ -1,5 +1,5 @@ { - "blockizer_version": "blockizer-2026.07:markdown-it-py-4:gfm-tables", + "blockizer_version": "blockizer-2026.07b:markdown-it-py-4:gfm-tables:heading-metadata", "parser": "markdown-it-py==4.2.0", "blocks": [ { diff --git a/src/tests/core/test_blockizer_corpus.py b/src/tests/core/test_blockizer_corpus.py index 6c40e71c..6fda68e4 100644 --- a/src/tests/core/test_blockizer_corpus.py +++ b/src/tests/core/test_blockizer_corpus.py @@ -1,13 +1,18 @@ """The blockizer golden corpus: locked hashes per BLOCKIZER_VERSION (WP-0.7, D57).""" +import hashlib import json from pathlib import Path +from uuid import UUID from markdown_it import __version__ as markdown_it_version from rememberstack.core import block_hash from rememberstack.core import blockize from rememberstack.core import BLOCKIZER_VERSION +from rememberstack.core import ChunkerParams +from rememberstack.core import pack_blocks +from rememberstack.model import SectionSpan _CORPUS = Path(__file__).resolve().parents[1] / "blockizer_corpus" @@ -28,6 +33,51 @@ def test_seed_document_hash_sequence_is_locked() -> None: assert observed == expected["blocks"] +def test_heading_metadata_reuses_the_canonical_parse_and_normalizes_nfkc() -> None: + """D79 metadata is raw-level + NFKC/casefold/whitespace title only.""" + source = "### ＦＯＯ *Bär*\n\nBody.\n" + heading, body = blockize(document_md=source) + assert heading.heading_level == 3 + assert heading.heading_title == "ＦＯＯ Bär" + assert heading.normalized_title == "foo bär" + assert body.heading_level is None + assert body.normalized_title is None + + +def test_chunk_work_identity_tracks_the_metadata_only_blockizer_bump() -> None: + """The version bumps while the golden document's packed grid bytes stay locked.""" + from rememberstack.core import CHUNKER_VERSION + from rememberstack.workers import E1_CHUNK_VERSION + + assert E1_CHUNK_VERSION == CHUNKER_VERSION + source = (_CORPUS / "seed_mixed.md").read_text() + blocks = blockize(document_md=source) + chunks = pack_blocks( + blocks=blocks, + sections=( + SectionSpan( + section_id=UUID("00000000-0000-0000-0000-000000000079"), + node_path="0", + role="body", + block_start=0, + block_end=len(blocks) - 1, + ), + ), + document_md=source, + params=ChunkerParams( + token_budget=20, anchor_modulus=24, anchor_min_gap_tokens=200 + ), + ) + grid_bytes = json.dumps( + [chunk.model_dump(mode="json") for chunk in chunks], + sort_keys=True, + separators=(",", ":"), + ).encode() + assert hashlib.sha256(grid_bytes).hexdigest() == ( + "a8f22df40a4492ca66bc87b7b744d508c9dbcd72e386548bded0040b37f402ff" + ) + + def test_offsets_slice_the_source_exactly() -> None: """Every block's offsets are a real slice of document.md (the grounding chain).""" source = (_CORPUS / "seed_mixed.md").read_text() @@ -77,3 +127,42 @@ def test_nested_list_items_stay_inside_their_parent_block() -> None: assert [block.type.value for block in blocks] == ["list_item", "list_item"] parent_raw = source[blocks[0].char_start : blocks[0].char_end] assert "nested child" in parent_raw + + +def test_packing_is_byte_identical_under_different_heading_metadata() -> None: + """The chunk grid depends on segmentation and hashes, never on heading + metadata: repacking with every heading's metadata replaced yields + byte-identical chunks. This is the pre-vs-post-07b equivalence proof the + single hardcoded digest cannot give (review).""" + source = (_CORPUS / "seed_mixed.md").read_text() + blocks = blockize(document_md=source) + # model_copy skips validation, letting us fake arbitrary metadata drift + mutated = tuple( + block.model_copy( + update={"heading_title": "REPLACED", "normalized_title": "replaced"} + ) + if block.heading_title is not None + else block + for block in blocks + ) + sections = ( + SectionSpan( + section_id=UUID("00000000-0000-0000-0000-000000000079"), + node_path="0", + role="body", + block_start=0, + block_end=len(blocks) - 1, + ), + ) + params = ChunkerParams( + token_budget=20, anchor_modulus=24, anchor_min_gap_tokens=200 + ) + original = pack_blocks( + blocks=blocks, sections=sections, document_md=source, params=params + ) + repacked = pack_blocks( + blocks=mutated, sections=sections, document_md=source, params=params + ) + assert [chunk.model_dump(mode="json") for chunk in original] == [ + chunk.model_dump(mode="json") for chunk in repacked + ] diff --git a/src/tests/core/test_section_snap.py b/src/tests/core/test_section_snap.py index 90925849..33a79d93 100644 --- a/src/tests/core/test_section_snap.py +++ b/src/tests/core/test_section_snap.py @@ -248,6 +248,8 @@ def test_the_catalog_refuses_a_disconnected_tree() -> None: from pydantic import ValidationError from rememberstack.model import SectionTreeRecord + from rememberstack.model import SkeletonStats + from rememberstack.model import StructureRouteTag root = SnappedSection( node_path="0", @@ -274,13 +276,46 @@ def test_the_catalog_refuses_a_disconnected_tree() -> None: ordinal=1, ) with pytest.raises(ValidationError, match="before its parent"): + stats = SkeletonStats( + stats_version="test", + section_count=0, + duplicate_title_ratio=None, + max_title_multiplicity=None, + sibling_duplicate_ratio=None, + level_jump_count=None, + numbering_coverage=None, + numbering_inversions=None, + numbering_scheme_switches=None, + tiny_section_ratio=None, + zero_direct_body_ratio=None, + oversized_leaf_ratio=None, + heading_density=None, + title_length_p50=None, + title_length_p95=None, + long_title_ratio=None, + low_letter_ratio=None, + empty_title_ratio=None, + max_sibling_fanout=None, + ) SectionTreeRecord( deployment_id=uuid4(), doc_id=uuid4(), version_id=uuid4(), representation_id=uuid4(), + structure_generation_id=uuid4(), sections=(root, orphan), placement_path=None, structurer_name="pageindex_llm", structurer_version="test", + skeleton_version="test", + skeleton_hash="hash", + skeleton_producer_family="N/A", + skeleton_check_version=None, + roles_version="test", + selecting_check_id=None, + route_tag=StructureRouteTag.LEGACY, + candidate_skeleton_hash="hash", + stats_version="test", + stats=stats, + pageindex_uri="test/pageindex.json", ) diff --git a/src/tests/core/test_structure_skeleton.py b/src/tests/core/test_structure_skeleton.py new file mode 100644 index 00000000..02c6ee92 --- /dev/null +++ b/src/tests/core/test_structure_skeleton.py @@ -0,0 +1,264 @@ +"""D79 parser, fallback-anchor, and normative-stat contract proofs.""" + +from math import isclose + +from rememberstack.core import analyze_skeleton +from rememberstack.core import blockize +from rememberstack.core import MIN_CHECK_SECTIONS +from rememberstack.core import parse_heading_skeleton +from rememberstack.core import resolve_fallback_skeleton +from rememberstack.model import FallbackAnchor + + +def _parse(source: str): + blocks = blockize(document_md=source) + sections = parse_heading_skeleton( + blocks=blocks, title="Document", markdown_chars=len(source) + ) + return blocks, sections + + +def test_nested_flat_and_skipped_heading_levels_assign_parents() -> None: + """Raw levels select the nearest prior lower level; no placeholders exist.""" + source = "\n\n".join( + ( + "# One", + "body", + "### Skipped child", + "body", + "## Sibling child", + "body", + "# Two", + "body", + ) + ) + _, sections = _parse(source) + + assert [(s.title, s.parent_path, s.heading_level) for s in sections[1:]] == [ + ("One", "0", 1), + ("Skipped child", "0.0", 3), + ("Sibling child", "0.0", 2), + ("Two", "0", 1), + ] + one, skipped, sibling, two = sections[1:] + assert one.block_end == two.block_start - 1 + assert skipped.block_end == sibling.block_start - 1 + + flat = _parse("# A\n\nbody\n\n# B\n\nbody\n\n# C\n")[1] + assert [section.parent_path for section in flat[1:]] == ["0", "0", "0"] + + +def test_no_headings_and_heading_only_documents_are_total() -> None: + no_heading_blocks, no_heading = _parse("A paragraph.\n\nAnother paragraph.\n") + assert len(no_heading) == 1 + assert no_heading[0].block_end == len(no_heading_blocks) - 1 + + heading_blocks, heading_only = _parse("# A\n\n## B\n\n### C\n") + assert [section.title for section in heading_only] == ["Document", "A", "B", "C"] + assert heading_only[-1].block_start == heading_only[-1].block_end + assert heading_only[0].block_end == len(heading_blocks) - 1 + + +def test_duplicate_titles_are_preserved_as_distinct_heading_occurrences() -> None: + _, sections = _parse("# Report\n\n## Summary\n\n## Summary\n") + summaries = [section for section in sections if section.title == "Summary"] + assert len(summaries) == 2 + assert summaries[0].node_path != summaries[1].node_path + assert summaries[0].block_end < summaries[1].block_start + + +def test_five_heading_design_review_example_uses_raw_levels_not_tree_depth() -> None: + """The five-heading review case includes both a skip and a level reset.""" + source = "\n\n".join(("## A", "#### A.1", "### A.2", "# B", "### B.1")) + _, sections = _parse(source) + assert [(s.title, s.parent_path) for s in sections[1:]] == [ + ("A", "0"), + ("A.1", "0.0"), + ("A.2", "0.0"), + ("B", "0"), + ("B.1", "0.1"), + ] + analysis = analyze_skeleton( + sections=sections, + blocks=blockize(document_md=source), + markdown_chars=len(source), + ) + assert analysis.stats.level_jump_count == 2 # 2→4 contributes 1; 1→3 contributes 1 + + +def test_stats_duplicate_and_sibling_formulas_are_hand_computed() -> None: + source = "# Chapter\n\n## Summary\n\n## Summary\n\n# Summary\n" + blocks, sections = _parse(source) + stats = analyze_skeleton( + sections=sections, blocks=blocks, markdown_chars=len(source) + ).stats + + assert stats.section_count == 4 + assert stats.duplicate_title_ratio == 0.5 # 1 - 2 distinct / 4 + assert stats.max_title_multiplicity == 3 + assert stats.sibling_duplicate_ratio == 0.25 # one duplicate / four headings + assert stats.max_sibling_fanout == 2 + + +def test_stats_numbering_runs_jumps_density_leaf_and_body_sizes() -> None: + source = "\n\n".join( + ( + "# 1. Chapter", + "A" * 100, + "### 3. Item", + "B" * 10, + "## 2. Item", + "### A. Alpha", + "C" * 90, + "### B. Beta", + ) + ) + blocks, sections = _parse(source) + analysis = analyze_skeleton( + sections=sections, blocks=blocks, markdown_chars=len(source) + ) + stats = analysis.stats + + assert stats.section_count == 5 + assert stats.level_jump_count == 1 # raw 1→3 contributes one skipped level + assert stats.numbering_coverage == 1.0 + # The parser makes raw H3/H2 peers under H1, so 3→2 is one real inversion. + assert stats.numbering_inversions == 1 + # A/B are one alpha run; the separate arabic runs never switch in-place. + assert stats.numbering_scheme_switches == 0 + assert isclose(stats.heading_density or 0, 50_000 / len(source)) + expected_leaf = max( + section.char_end - section.char_start + for section in sections[1:] + if not any(child.parent_path == section.node_path for child in sections[1:]) + ) + assert isclose(stats.oversized_leaf_ratio or 0, expected_leaf / len(source)) + assert analysis.direct_body_chars["0.0"] == 100 + assert analysis.direct_body_chars["0.0.0"] == 10 + assert analysis.direct_body_chars["0.0.1.0"] == 90 + assert stats.tiny_section_ratio == 3 / 5 + assert stats.zero_direct_body_ratio == 2 / 5 + + +def test_numbering_inversion_and_scheme_switch_count_same_parent_runs_only() -> None: + source = "# Root\n\n## 3. Third\n\n## 2. Second\n\n## A. Alpha\n\n## B. Beta\n" + blocks, sections = _parse(source) + stats = analyze_skeleton( + sections=sections, blocks=blocks, markdown_chars=len(source) + ).stats + assert stats.numbering_coverage == 4 / 5 # unnumbered Root + assert stats.numbering_inversions == 1 + assert stats.numbering_scheme_switches == 1 + + +def test_title_shape_nearest_rank_and_character_formulas() -> None: + long = "A" * 121 + source = f"# {long}\n\n## 123\n\n##\n" + blocks, sections = _parse(source) + stats = analyze_skeleton( + sections=sections, blocks=blocks, markdown_chars=len(source) + ).stats + assert stats.title_length_p50 == 3.0 + assert stats.title_length_p95 == 121.0 + assert stats.long_title_ratio == 1 / 3 + assert stats.low_letter_ratio == 2 / 3 + assert stats.empty_title_ratio == 1 / 3 + + +def test_short_and_empty_stats_have_exact_null_zero_cases() -> None: + for source in ("", "plain body", "# Only\n\nbody"): + blocks, sections = _parse(source) + analysis = analyze_skeleton( + sections=sections, blocks=blocks, markdown_chars=len(source) + ) + dumped = analysis.stats.model_dump() + assert dumped["section_count"] < MIN_CHECK_SECTIONS + assert all( + value is None + for key, value in dumped.items() + if key not in {"stats_version", "section_count"} + ) + if not source: + assert analysis.heading_density == 0 + assert analysis.oversized_leaf_ratio == 0 + + +def test_fallback_occurrence_index_resolves_duplicates() -> None: + source = "Intro marker\n\nRepeat marker\n\nRepeat marker\n\nEnd marker\n" + blocks = blockize(document_md=source) + sections = resolve_fallback_skeleton( + proposed=( + FallbackAnchor(anchor="Repeat marker", occurrence_index=1, children=()), + ), + blocks=blocks, + document_md=source, + title="Fallback", + ) + assert len(sections) == 2 + assert sections[1].block_start == 2 + + +def test_unresolved_fallback_anchor_degrades_children_to_enclosing_parent() -> None: + source = "Intro marker\n\nMiddle marker\n\nEnd marker\n" + blocks = blockize(document_md=source) + sections = resolve_fallback_skeleton( + proposed=( + FallbackAnchor( + anchor="missing", + occurrence_index=0, + children=( + FallbackAnchor( + anchor="End marker", occurrence_index=0, children=() + ), + ), + ), + ), + blocks=blocks, + document_md=source, + title="Fallback", + ) + assert [(section.title, section.parent_path) for section in sections[1:]] == [ + ("End marker", "0") + ] + + +def test_same_block_fallback_boundaries_adopt_later_children() -> None: + """D57 tie collapse keeps one boundary without erasing the later subtree.""" + source = "Alpha and Beta\n\nChild marker\n\nEnd marker\n" + blocks = blockize(document_md=source) + sections = resolve_fallback_skeleton( + proposed=( + FallbackAnchor(anchor="Alpha", occurrence_index=0, children=()), + FallbackAnchor( + anchor="Beta", + occurrence_index=0, + children=( + FallbackAnchor( + anchor="Child marker", occurrence_index=0, children=() + ), + ), + ), + ), + blocks=blocks, + document_md=source, + title="Fallback", + ) + assert [(section.title, section.parent_path) for section in sections[1:]] == [ + ("Alpha", "0"), + ("Child marker", "0.0"), + ] + + +def test_fallback_nesting_preserves_the_snap_depth_ceiling() -> None: + source = "\n\n".join(f"Marker {index}" for index in range(20)) + blocks = blockize(document_md=source) + proposal = FallbackAnchor(anchor="Marker 19", occurrence_index=0, children=()) + for index in range(18, -1, -1): + proposal = FallbackAnchor( + anchor=f"Marker {index}", occurrence_index=0, children=(proposal,) + ) + sections = resolve_fallback_skeleton( + proposed=(proposal,), blocks=blocks, document_md=source, title="Deep fallback" + ) + assert len(sections) == 16 # root plus the D57 maximum of 15 nested nodes + assert sections[-1].node_path.count(".") == 15 diff --git a/src/tests/spine/test_forget_catalog.py b/src/tests/spine/test_forget_catalog.py index 581950d3..74dd4be5 100644 --- a/src/tests/spine/test_forget_catalog.py +++ b/src/tests/spine/test_forget_catalog.py @@ -167,6 +167,7 @@ def _restore_pre_forget_postgres(*, engine: Engine) -> None: ) with engine.begin() as connection: _seed_documents(connection=connection) + _seed_structure_provenance(connection=connection) _seed_evidence(connection=connection) _seed_knowledge_and_residuals(connection=connection) @@ -251,6 +252,57 @@ def test_readiness_rehonors_manifest_after_old_postgres_restore( _assert_scrubbed_and_control_survives(engine=seeded_engine) +_TARGET_CHECK_ID = UUID("75000000-0000-0000-0000-0000000000c1") +_TARGET_GENERATION_ID = UUID("75000000-0000-0000-0000-0000000000c2") + + +def _seed_structure_provenance(*, connection: Connection) -> None: + """A D79 check record + generation for the target doc: hard-forget must + erase both EXPLICITLY (review MAJOR — never via implied cascade).""" + connection.execute( + text( + "INSERT INTO document_skeleton_checks (" + " check_id, processing_id, deployment_id, doc_id, version_id," + " representation_id, candidate_skeleton_hash, stats_version, stats," + " sampled_input_hash, check_outcome, checker_component_version," + " checker_model, checker_model_hash, checker_prompt_hash," + " checker_schema_hash" + ") VALUES (" + " :check_id, :check_id, :d, :doc, :version, :representation," + " 'hash', 'stats-v', CAST('{}' AS jsonb), NULL, 'coherent'," + " 'check-v', 'model', 'mh', 'ph', 'sh')" + ), + { + "check_id": _TARGET_CHECK_ID, + "d": _DEPLOYMENT_ID, + "doc": _TARGET_DOC_ID, + "version": _TARGET_VERSION_ID, + "representation": _TARGET_REPRESENTATION_ID, + }, + ) + connection.execute( + text( + "INSERT INTO document_structure_generations (" + " structure_generation_id, deployment_id, doc_id, version_id," + " representation_id, skeleton_version, skeleton_hash," + " skeleton_producer_family, selecting_check_id, route_tag," + " candidate_skeleton_hash, stats_version, stats" + ") VALUES (" + " :generation_id, :d, :doc, :version, :representation," + " 'skeleton-v', 'hash', 'N/A', :check_id, 'parser', 'hash'," + " 'stats-v', CAST('{}' AS jsonb))" + ), + { + "generation_id": _TARGET_GENERATION_ID, + "d": _DEPLOYMENT_ID, + "doc": _TARGET_DOC_ID, + "version": _TARGET_VERSION_ID, + "representation": _TARGET_REPRESENTATION_ID, + "check_id": _TARGET_CHECK_ID, + }, + ) + + def _seed_documents(*, connection: Connection) -> None: """Seed two complete E0/E1 lineages and an inbound source-bearing crossref.""" for content_hash, raw_uri in ( @@ -953,6 +1005,16 @@ def _assert_scrubbed_and_control_survives(*, engine: Engine) -> None: _count(connection, "claim_extraction_decisions", "doc_id", _TARGET_DOC_ID) == 0 ) + assert ( + _count( + connection, "document_structure_generations", "doc_id", _TARGET_DOC_ID + ) + == 0 + ) + assert ( + _count(connection, "document_skeleton_checks", "doc_id", _TARGET_DOC_ID) + == 0 + ) assert _count(connection, "grounding_audits", "claim_id", _TARGET_CLAIM_ID) == 0 assert ( _count(connection, "relations", "relation_id", _EXCLUSIVE_RELATION_ID) == 0 diff --git a/src/tests/spine/test_migrations.py b/src/tests/spine/test_migrations.py index 82cc9f12..99cda0cd 100644 --- a/src/tests/spine/test_migrations.py +++ b/src/tests/spine/test_migrations.py @@ -97,13 +97,16 @@ def test_revision_graph_is_one_linear_structural_chain() -> None: "p7_02_0016", "p7_05_0017", "p1_03_0018", + "p1_04_0019", ) assert len(script.get_heads()) == 1 migration_source = "\n".join( path.read_text(encoding="utf-8") for path in sorted(_VERSIONS.glob("p*_*.py")) ).lower() - assert "insert into" not in migration_source + # D79's structural migration performs the one required legacy-generation + # backfill; no deployment/bootstrap seed DML belongs in this chain. + assert migration_source.count("insert into") == 1 assert "bootstrap_deployment" not in migration_source @@ -221,6 +224,156 @@ def test_claim_citation_coordinate_migration_deduplicates_real_rows() -> None: command.upgrade(config=config, revision="head") +def test_d79_migration_backfills_existing_tree_as_legacy_generation() -> None: + """Existing first-write section rows gain one immutable legacy wrapper/current pointer.""" + database_url = _database_url() + config = _alembic_config(database_url=database_url) + command.downgrade(config=config, revision="base") + command.upgrade(config=config, revision="p1_03_0018") + deployment_id = uuid4() + doc_id = uuid4() + version_id = uuid4() + representation_id = uuid4() + section_id = uuid4() + engine = create_engine(database_url) + try: + with engine.begin() as connection: + connection.execute( + text( + "INSERT INTO deployments (deployment_id, slug, name, raw_bucket," + " artifacts_bucket, corpusfs_bucket)" + " VALUES (:deployment, 'd79-backfill', 'D79 backfill'," + " 'mem://raw', 'mem://artifacts', 'mem://corpusfs')" + ), + {"deployment": deployment_id}, + ) + connection.execute( + text( + "INSERT INTO content_objects (deployment_id, content_hash, mime," + " byte_size, raw_uri) VALUES" + " (:deployment, 'legacy-content', 'text/markdown', 1, 'raw')" + ), + {"deployment": deployment_id}, + ) + connection.execute( + text( + "INSERT INTO documents (doc_id, deployment_id, source_kind," + " source_ref, title) VALUES" + " (:doc, :deployment, 'test', 'legacy', 'Legacy')" + ), + {"doc": doc_id, "deployment": deployment_id}, + ) + connection.execute( + text( + "INSERT INTO document_versions (version_id, deployment_id, doc_id," + " content_hash, version_no, status) VALUES" + " (:version, :deployment, :doc, 'legacy-content', 1, 'ready')" + ), + {"version": version_id, "deployment": deployment_id, "doc": doc_id}, + ) + connection.execute( + text( + "INSERT INTO document_representations (representation_id," + " deployment_id, version_id, route, structurer_name," + " structurer_version, pageindex_uri, status) VALUES" + " (:representation, :deployment, :version, 'passthrough'," + " 'pageindex_llm', 'e0-structure-2026.07c:temp0-1'," + " 'legacy/pageindex.json', 'ready')" + ), + { + "representation": representation_id, + "deployment": deployment_id, + "version": version_id, + }, + ) + connection.execute( + text( + "INSERT INTO document_sections (section_id, deployment_id, doc_id," + " version_id, representation_id, node_path, block_start, block_end," + " title, role, char_start, char_end, ordinal, structurer_version)" + " VALUES (:section, :deployment, :doc, :version, :representation," + " '0', 0, 0, 'Legacy', 'body', 0, 1, 0," + " 'e0-structure-2026.07c:temp0-1')" + ), + { + "section": section_id, + "deployment": deployment_id, + "doc": doc_id, + "version": version_id, + "representation": representation_id, + }, + ) + connection.execute( + text( + "INSERT INTO document_sections (section_id, deployment_id, doc_id," + " version_id, representation_id, node_path, block_start, block_end," + " title, role, char_start, char_end, ordinal, structurer_version)" + " VALUES (:section, :deployment, :doc, :version, :representation," + " '0.1', 0, 0, 'Legacy Child', 'body', 0, 1, 1," + " 'e0-structure-2026.07c:temp0-1')" + ), + { + "section": uuid4(), + "deployment": deployment_id, + "doc": doc_id, + "version": version_id, + "representation": representation_id, + }, + ) + + command.upgrade(config=config, revision="head") + with engine.connect() as connection: + row = ( + connection.execute( + text( + "SELECT g.structure_generation_id, g.route_tag::text AS route," + " g.skeleton_version, g.skeleton_producer_family," + " g.roles_version, g.summary_version," + " g.placement_version, g.pageindex_uri," + " r.current_structure_generation_id" + " FROM document_structure_generations g" + " JOIN document_representations r" + " ON r.representation_id = g.representation_id" + " WHERE g.representation_id = :representation" + ), + {"representation": representation_id}, + ) + .mappings() + .one() + ) + assert row["route"] == "legacy" + assert row["skeleton_version"] == "e0-structure-2026.07c:temp0-1" + assert row["skeleton_producer_family"] == "legacy-unknown" + assert row["roles_version"] == "e0-structure-2026.07c:temp0-1" + assert row["summary_version"] is None + assert row["placement_version"] is None + assert row["pageindex_uri"] == "legacy/pageindex.json" + assert row["current_structure_generation_id"] == row["structure_generation_id"] + with engine.connect() as connection: + sections = ( + connection.execute( + text( + "SELECT node_path, structure_generation_id, normalized_title" + " FROM document_sections" + " WHERE representation_id = :representation ORDER BY node_path" + ), + {"representation": representation_id}, + ) + .mappings() + .all() + ) + # a MULTI-section legacy tree wraps under ONE generation (review gap) + assert [section["node_path"] for section in sections] == ["0", "0.1"] + assert {section["structure_generation_id"] for section in sections} == { + row["structure_generation_id"] + } + assert {section["normalized_title"] for section in sections} == {""} + finally: + engine.dispose() + command.downgrade(config=config, revision="base") + command.upgrade(config=config, revision="head") + + def test_postgresql_fresh_downgrade_reupgrade_mutation_and_noop_lifecycle() -> None: """Exercise the complete PostgreSQL 16+ lifecycle and negative catalog proof.""" database_url = _database_url() @@ -236,7 +389,7 @@ def test_postgresql_fresh_downgrade_reupgrade_mutation_and_noop_lifecycle() -> N "observation_evidence": 64, "relation_evidence": 64, } - assert len(fresh_inventory.tables) == 60 + assert len(fresh_inventory.tables) == 62 assert fresh_inventory.empty_tables == ( "deployments", "entity_types", @@ -263,5 +416,5 @@ def test_postgresql_fresh_downgrade_reupgrade_mutation_and_noop_lifecycle() -> N head_before_noop = _head_revision(database_url=database_url) command.upgrade(config=config, revision="head") head_after_noop = _head_revision(database_url=database_url) - assert head_before_noop == head_after_noop == "p1_03_0018" + assert head_before_noop == head_after_noop == "p1_04_0019" assert _inventory(database_url=database_url) == restored_inventory diff --git a/src/tests/workers/test_d79_structure_route.py b/src/tests/workers/test_d79_structure_route.py new file mode 100644 index 00000000..0aa9d556 --- /dev/null +++ b/src/tests/workers/test_d79_structure_route.py @@ -0,0 +1,508 @@ +"""Unit-speed proofs of the acyclic D79 checker/fallback routing state machine.""" + +from collections.abc import Iterator +import json +from pathlib import Path +from uuid import UUID +from uuid import uuid4 + +import pytest + +from rememberstack.adapters.selfhost import LocalFSObjectStore +from rememberstack.adapters.testing import FakeModelProvider +from rememberstack.adapters.testing import NoopCostMeter +from rememberstack.core import analyze_skeleton +from rememberstack.core import blockize +from rememberstack.core import BLOCKIZER_VERSION +from rememberstack.core import parse_heading_skeleton +from rememberstack.core import skeleton_hash +from rememberstack.model import ClaimedWork +from rememberstack.model import ObjectKey +from rememberstack.model import PersistedSectionTree +from rememberstack.model import PipelineStage +from rememberstack.model import ProcessingLane +from rememberstack.model import ProcessingTarget +from rememberstack.model import ProviderCallError +from rememberstack.model import SectionTreeRecord +from rememberstack.model import SkeletonCheckOutcome +from rememberstack.model import SkeletonCheckRecord +from rememberstack.model import StructureRouteTag +from rememberstack.model import StructureSource +from rememberstack.workers import E0_STRUCTURE_VERSION +from rememberstack.workers import RoleSettings +from rememberstack.workers import SkeletonCheckSettings +from rememberstack.workers import StructureHandler +from rememberstack.workers import StructurerSettings +from rememberstack.workers.e0 import _render_check_prompt +from rememberstack.workers.e0 import SKELETON_CHECK_PROMPT_CEILING + +_DEPLOYMENT = UUID("70000000-0000-0000-0000-000000000001") +_DOC = UUID("70000000-0000-0000-0000-000000000002") +_VERSION = UUID("70000000-0000-0000-0000-000000000003") +_REPRESENTATION = UUID("70000000-0000-0000-0000-000000000004") + +_SOURCE = "\n\n".join( + ("# Section A", "A" * 100, "# Section B", "B" * 100, "# Section C", "C" * 100) +) + + +class _Catalog: + """Capture checker and generation appends without replacing route logic.""" + + def __init__(self) -> None: + self.checks: list[SkeletonCheckRecord] = [] + self.generations: list[SectionTreeRecord] = [] + + def structure_source(self, *, representation_id: UUID) -> StructureSource: + assert representation_id == _REPRESENTATION + return StructureSource( + deployment_id=_DEPLOYMENT, + doc_id=_DOC, + version_id=_VERSION, + representation_id=_REPRESENTATION, + blocks_uri="doc/blocks.json", + markdown_uri="doc/document.md", + title="Routing proof", + source_kind="upload", + ) + + def record_skeleton_check(self, *, record: SkeletonCheckRecord) -> None: + self.checks.append(record) + + def record_section_tree(self, *, record: SectionTreeRecord) -> PersistedSectionTree: + self.generations.append(record) + return PersistedSectionTree( + sections=record.sections, + placement_path=record.placement_path, + structurer_version=record.structurer_version, + structure_generation_id=record.structure_generation_id, + pageindex_uri=record.pageindex_uri, + skeleton_version=record.skeleton_version, + skeleton_hash=record.skeleton_hash, + skeleton_producer_family=record.skeleton_producer_family, + skeleton_check_version=record.skeleton_check_version, + roles_version=record.roles_version, + summary_version=record.summary_version, + placement_version=record.placement_version, + selecting_check_id=record.selecting_check_id, + route_tag=record.route_tag, + candidate_skeleton_hash=record.candidate_skeleton_hash, + stats_version=record.stats_version, + stats=record.stats, + ) + + +def _work() -> ClaimedWork: + return ClaimedWork( + processing_id=uuid4(), + deployment_id=_DEPLOYMENT, + target_kind=ProcessingTarget.DOCUMENT_VERSION, + target_id=_VERSION, + stage=PipelineStage.STRUCTURE, + component_version=E0_STRUCTURE_VERSION, + content_hash="content", + lane=ProcessingLane.STEADY, + attempt=1, + payload={ + "version_id": str(_VERSION), + "representation_id": str(_REPRESENTATION), + }, + ) + + +def _router(verdicts: Iterator[str]): + def route(prompt: str, response_type: str) -> dict[str, object]: + del prompt + if response_type == "SkeletonCheckResponse": + return {"verdict": next(verdicts)} + if response_type == "FallbackStructureResponse": + return { + "sections": [ + {"anchor": "Section A", "occurrence_index": 0, "children": []}, + {"anchor": "Section B", "occurrence_index": 0, "children": []}, + {"anchor": "Section C", "occurrence_index": 0, "children": []}, + ] + } + if response_type == "RoleClassificationResponse": + return {"assignments": []} + raise AssertionError(response_type) + + return route + + +def _run( + *, + tmp_path: Path, + provider: object | None, + source: str = _SOURCE, + min_density: float = 0, + max_leaf: float = 1, +) -> _Catalog: + store = LocalFSObjectStore(root=tmp_path) + blocks = blockize(document_md=source) + store.write_bytes(key=ObjectKey("doc/document.md"), content=source.encode()) + store.write_bytes( + key=ObjectKey("doc/blocks.json"), + content=json.dumps( + { + "blockizer_version": BLOCKIZER_VERSION, + "markdown_chars": len(source), + "blocks": [block.model_dump(mode="json") for block in blocks], + } + ).encode(), + ) + catalog = _Catalog() + handler = StructureHandler( + catalog=catalog, # type: ignore[arg-type] + artifact_store=store, + model_provider=provider, # type: ignore[arg-type] + settings=StructurerSettings( + min_heading_density_per_10k=min_density, max_oversized_leaf_ratio=max_leaf + ), + check_settings=SkeletonCheckSettings(model="checker/fake"), + role_settings=RoleSettings(model="roles/fake"), + ) + handler.handle(work=_work(), meter=NoopCostMeter()) + return catalog + + +def test_not_run_short_is_persisted_without_a_provider(tmp_path: Path) -> None: + catalog = _run(tmp_path=tmp_path, provider=None, source="# A\n\nbody\n\n# B\n") + assert [record.check_outcome for record in catalog.checks] == ["not_run_short"] + assert catalog.generations[-1].route_tag is StructureRouteTag.PARSER + + +def test_coherent_keeps_the_parser_tree(tmp_path: Path) -> None: + provider = FakeModelProvider(generate_router=_router(iter(("coherent",)))) + catalog = _run(tmp_path=tmp_path, provider=provider) + assert [record.check_outcome for record in catalog.checks] == ["coherent"] + assert catalog.generations[-1].route_tag is StructureRouteTag.PARSER + assert catalog.generations[-1].skeleton_producer_family == "N/A" + + +@pytest.mark.parametrize( + "verdict", + ( + "incoherent_repeated_boilerplate", + "incoherent_heading_sequence", + "incoherent_junk_titles", + "incoherent_over_fragmented", + ), +) +def test_every_incoherent_verdict_demotes_then_terminally_accepts_fallback( + tmp_path: Path, verdict: str +) -> None: + provider = FakeModelProvider(generate_router=_router(iter((verdict, "coherent")))) + catalog = _run(tmp_path=tmp_path, provider=provider) + assert [record.check_outcome for record in catalog.checks] == [verdict, "coherent"] + assert [record.route_tag for record in catalog.generations] == [ + StructureRouteTag.PARSER_DEMOTED_CHECK, + StructureRouteTag.FALLBACK_AFTER_CHECK, + ] + + +def test_provider_error_is_explicit_and_fail_open(tmp_path: Path) -> None: + class DeadProvider: + def generate(self, *, request: object, response_type: object) -> object: + raise ProviderCallError("offline") + + def embed(self, *, request: object) -> object: + raise NotImplementedError + + catalog = _run(tmp_path=tmp_path, provider=DeadProvider()) + assert [record.check_outcome for record in catalog.checks] == ["provider_error"] + failure = catalog.checks[0].provider_failure + assert failure is not None + assert failure["error_type"] == "ProviderCallError" + assert failure["has_usage"] is False + assert len(str(failure["error_fingerprint"])) == 64 + assert catalog.generations[-1].route_tag is StructureRouteTag.PARSER + + +def test_invalid_response_is_explicit_and_fail_open(tmp_path: Path) -> None: + provider = FakeModelProvider( + generate_router=lambda prompt, response_type: ( + {"verdict": "not-an-enum"} + if response_type == "SkeletonCheckResponse" + else {"assignments": []} + ) + ) + catalog = _run(tmp_path=tmp_path, provider=provider) + assert [record.check_outcome for record in catalog.checks] == ["invalid_response"] + assert catalog.generations[-1].route_tag is StructureRouteTag.PARSER + + +def test_terminal_incoherence_yields_one_synthetic_root_and_stops( + tmp_path: Path, +) -> None: + provider = FakeModelProvider( + generate_router=_router( + iter(("incoherent_junk_titles", "incoherent_over_fragmented")) + ) + ) + catalog = _run(tmp_path=tmp_path, provider=provider) + assert len(catalog.checks) == 2 + final = catalog.generations[-1] + assert final.route_tag is StructureRouteTag.SYNTHETIC_AFTER_CHECK + assert len(final.sections) == 1 + assert final.sections[0].node_path == "0" + + +@pytest.mark.parametrize("terminal_failure", ("provider_error", "invalid_response")) +def test_terminal_provider_failures_are_explicit_and_keep_the_fallback( + tmp_path: Path, terminal_failure: str +) -> None: + check_calls = 0 + + def router(prompt: str, response_type: str) -> dict[str, object]: + nonlocal check_calls + del prompt + if response_type == "SkeletonCheckResponse": + check_calls += 1 + if check_calls == 1: + return {"verdict": "incoherent_junk_titles"} + if terminal_failure == "provider_error": + raise ProviderCallError("terminal offline") + return {"verdict": "not-an-enum"} + if response_type == "FallbackStructureResponse": + return { + "sections": [ + {"anchor": "Section A", "occurrence_index": 0, "children": []}, + {"anchor": "Section B", "occurrence_index": 0, "children": []}, + {"anchor": "Section C", "occurrence_index": 0, "children": []}, + ] + } + if response_type == "RoleClassificationResponse": + return {"assignments": []} + raise AssertionError(response_type) + + catalog = _run( + tmp_path=tmp_path, provider=FakeModelProvider(generate_router=router) + ) + assert [record.check_outcome for record in catalog.checks] == [ + "incoherent_junk_titles", + terminal_failure, + ] + assert catalog.generations[-1].route_tag is StructureRouteTag.FALLBACK_AFTER_CHECK + assert len(catalog.generations[-1].sections) == 4 + + +@pytest.mark.parametrize( + ("source", "min_density", "max_leaf", "route"), + ( + ("One block without headings.\n", 1, 1, StructureRouteTag.FALLBACK_DENSITY), + ("# Wrapper\n\n" + "body " * 100, 0, 0.5, StructureRouteTag.FALLBACK_LEAF), + ), +) +def test_shared_density_and_leaf_metrics_drive_direct_fallback_routes( + tmp_path: Path, + source: str, + min_density: float, + max_leaf: float, + route: StructureRouteTag, +) -> None: + provider = FakeModelProvider( + generate_router=lambda prompt, response_type: ( + {"sections": []} + if response_type == "FallbackStructureResponse" + else {"assignments": []} + ) + ) + catalog = _run( + tmp_path=tmp_path, + provider=provider, + source=source, + min_density=min_density, + max_leaf=max_leaf, + ) + # Every fallback tree faces the terminal judge (§4.1) — gate demotions + # included. These fallback trees are root-only, so the terminal check is + # not_run_short: recorded, fail-open, route tag kept. + assert [check.check_outcome for check in catalog.checks] == [ + SkeletonCheckOutcome.NOT_RUN_SHORT + ] + generation = catalog.generations[-1] + assert generation.route_tag is route + assert generation.selecting_check_id == catalog.checks[0].check_id + # Generation rows describe the SELECTED tree; check records describe the + # candidates they judged (review convention). + assert generation.candidate_skeleton_hash == generation.skeleton_hash + parsed = parse_heading_skeleton( + blocks=blockize(document_md=source), + title="Routing proof", + markdown_chars=len(source), + ) + if route is StructureRouteTag.FALLBACK_LEAF: + assert generation.skeleton_hash != skeleton_hash(sections=parsed) + + +def test_checker_prompt_is_hard_capped_and_samples_mid_document_anomalies() -> None: + headings = [f"# Heading {index} {'x' * 100}" for index in range(500)] + headings[249] = "# MID TEMPLATE DUPLICATE" + headings[250] = "# MID TEMPLATE DUPLICATE" + source = "\n\n".join(headings) + blocks = blockize(document_md=source) + sections = parse_heading_skeleton( + blocks=blocks, title="Large", markdown_chars=len(source) + ) + analysis = analyze_skeleton( + sections=sections, blocks=blocks, markdown_chars=len(source) + ) + prompt = _render_check_prompt( + source=StructureSource( + deployment_id=_DEPLOYMENT, + doc_id=_DOC, + version_id=_VERSION, + representation_id=_REPRESENTATION, + blocks_uri="blocks", + markdown_uri="markdown", + title="Large", + source_kind="upload", + ), + sections=sections, + analysis=analysis, + ) + assert len(prompt) <= SKELETON_CHECK_PROMPT_CEILING + assert "omitted" in prompt + assert "MID TEMPLATE DUPLICATE" in prompt + + +def test_roles_use_rules_then_title_classifier_with_body_as_failure_default( + tmp_path: Path, +) -> None: + source = "# Abstract\n\nbody\n\n# Bibliography\n\nbody\n\n# Mystery\n\nbody\n" + + def router(prompt: str, response_type: str) -> dict[str, object]: + del prompt + if response_type == "SkeletonCheckResponse": + return {"verdict": "coherent"} + if response_type == "RoleClassificationResponse": + return {"assignments": [{"node_path": "0.2", "role": "methods"}]} + raise AssertionError(response_type) + + catalog = _run( + tmp_path=tmp_path, + provider=FakeModelProvider(generate_router=router), + source=source, + ) + final = catalog.generations[-1] + assert [section.role for section in final.sections] == [ + "body", + "abstract", + "references", + "methods", + ] + + failed = _run(tmp_path=tmp_path / "failed", provider=None, source=source) + assert [section.role for section in failed.generations[-1].sections] == [ + "body", + "abstract", + "references", + "body", + ] + + +def test_checker_bump_does_not_mint_a_new_generation_when_route_holds( + tmp_path: Path, +) -> None: + """The generation id is seeded from route + selected tree, never the + checker version (review: a checker bump must not churn generations or + re-chunk when the route does not flip).""" + source = "# A\n\nbody\n\n# B\n" + store = LocalFSObjectStore(root=tmp_path) + blocks = blockize(document_md=source) + store.write_bytes(key=ObjectKey("doc/document.md"), content=source.encode()) + store.write_bytes( + key=ObjectKey("doc/blocks.json"), + content=json.dumps( + { + "blockizer_version": BLOCKIZER_VERSION, + "markdown_chars": len(source), + "blocks": [block.model_dump(mode="json") for block in blocks], + } + ).encode(), + ) + catalog = _Catalog() + for checker_model in ("checker/generation-one", "checker/generation-two"): + StructureHandler( + catalog=catalog, # type: ignore[arg-type] + artifact_store=store, + model_provider=None, + settings=StructurerSettings( + min_heading_density_per_10k=0, max_oversized_leaf_ratio=1 + ), + check_settings=SkeletonCheckSettings(model=checker_model), + role_settings=RoleSettings(model="roles/fake"), + ).handle(work=_work(), meter=NoopCostMeter()) + assert len(catalog.generations) == 2 # append-only recorder sees both runs + assert ( + len({record.structure_generation_id for record in catalog.generations}) == 1 + ) # ...but they derive the SAME id: ON CONFLICT makes the second a no-op + assert len(catalog.checks) == 2 # while every check outcome is ledgered + + +def test_role_classifier_failure_defaults_to_body(tmp_path: Path) -> None: + """A role-classifier provider failure must degrade to body, not retry-loop.""" + + def _router(prompt: str, response_type: str) -> dict[str, object]: + if response_type == "RoleClassificationResponse": + raise ProviderCallError("role seat down") + if response_type == "SkeletonCheckResponse": + return {"verdict": "coherent"} + return {"sections": []} + + catalog = _run( + tmp_path=tmp_path, + provider=FakeModelProvider(generate_router=_router), + source=( + "# Zephyr Blorptangle\n\nbody\n\n# Quixotic Framblewort\n\nbody\n\n" + "# Crenulated Sprocketry\n\nbody\n\n# Vorpal Mimsy\n\nbody\n" + ), + ) + generation = catalog.generations[-1] + assert generation.route_tag is StructureRouteTag.PARSER + roles = {section.node_path: section.role for section in generation.sections} + assert set(roles.values()) == {"body"} + + +def test_stale_blocks_sidecar_reblockizes_instead_of_dead_lettering( + tmp_path: Path, +) -> None: + """A pre-metadata blocks.json (old blockizer version, heading blocks + without level/title) structures fine: the sidecar is a cache, and a + version mismatch re-derives blocks from document.md (review MAJOR).""" + source = "# A\n\nbody\n\n# B\n\nmore body\n" + store = LocalFSObjectStore(root=tmp_path) + legacy_blocks = [ + { + key: value + for key, value in block.model_dump(mode="json").items() + if key not in ("heading_level", "heading_title", "normalized_title") + } + for block in blockize(document_md=source) + ] + store.write_bytes(key=ObjectKey("doc/document.md"), content=source.encode()) + store.write_bytes( + key=ObjectKey("doc/blocks.json"), + content=json.dumps( + { + "blockizer_version": "blockizer-2026.07:markdown-it-py-4:gfm-tables", + "markdown_chars": len(source), + "blocks": legacy_blocks, + } + ).encode(), + ) + catalog = _Catalog() + StructureHandler( + catalog=catalog, # type: ignore[arg-type] + artifact_store=store, + model_provider=None, + settings=StructurerSettings( + min_heading_density_per_10k=0, max_oversized_leaf_ratio=1 + ), + check_settings=SkeletonCheckSettings(model="checker/fake"), + role_settings=RoleSettings(model="roles/fake"), + ).handle(work=_work(), meter=NoopCostMeter()) + generation = catalog.generations[-1] + assert generation.route_tag is StructureRouteTag.PARSER + assert [section.title for section in generation.sections[1:]] == ["A", "B"] diff --git a/src/tests/workers/test_e0_chain.py b/src/tests/workers/test_e0_chain.py index 2c74e83f..e285be13 100644 --- a/src/tests/workers/test_e0_chain.py +++ b/src/tests/workers/test_e0_chain.py @@ -9,6 +9,7 @@ import json from pathlib import Path from uuid import UUID +from uuid import uuid4 from alembic import command from alembic.config import Config @@ -34,7 +35,9 @@ from rememberstack.model import ProcessingTarget from rememberstack.model import RunResultOutcome from rememberstack.model import SectionTreeRecord +from rememberstack.model import SkeletonStats from rememberstack.model import SnappedSection +from rememberstack.model import StructureRouteTag from rememberstack.spine import DeploymentBootstrapper from rememberstack.spine import DocumentCatalog from rememberstack.spine import ForgetCatalog @@ -432,8 +435,13 @@ def test_stale_structure_never_overwrites_the_live_representation(rig: _E0Rig) - ) assert after["current_representation_id"] == live section = rig.row( - sql="SELECT representation_id FROM document_sections" - " WHERE version_id = :version_id", + sql="SELECT s.representation_id FROM document_sections s" + " JOIN document_representations r" + " ON r.representation_id = s.representation_id" + " AND r.current_structure_generation_id = s.structure_generation_id" + " JOIN document_versions v" + " ON v.current_representation_id = r.representation_id" + " WHERE s.version_id = :version_id AND s.node_path = '0'", params={"version_id": ingested.version_id}, ) assert section["representation_id"] == live @@ -492,31 +500,11 @@ def _structure_worker(rig: _E0Rig, provider: object) -> Worker: def test_full_structure_route_persists_the_snapped_tree(rig: _E0Rig) -> None: - """WP-3.3: the LLM proposal lands as a snapped multi-section tree — rows - with parent links and sanitized roles, placement on the root, and the - pageindex.json sidecar next to document.md.""" - findings = _STRUCTURED_SOURCE.index("## Findings") - recommendations = _STRUCTURED_SOURCE.index("## Recommendations") + """D79 parser rows, deterministic roles, null placement, versioned sidecar.""" provider = FakeModelProvider( generate_payloads={ - "StructureResponse": { - "placement": "/surveys/field-reports/", - "sections": [ - { - "title": "Findings", - "role": "results", - "char_start": findings, - "char_end": recommendations, - "summary": "What the survey found.", - }, - { - "title": "Recommendations", - "role": "ACTION_ITEMS", # invented: must degrade to body - "char_start": recommendations, - "char_end": len(_STRUCTURED_SOURCE), - }, - ], - } + "SkeletonCheckResponse": {"verdict": "coherent"}, + "RoleClassificationResponse": {"assignments": []}, } ) ingested = rig.ingestor.ingest( @@ -557,31 +545,35 @@ def test_full_structure_route_persists_the_snapped_tree(rig: _E0Rig) -> None: ), {"version_id": ingested.version_id}, ).scalar_one() - assert [row["node_path"] for row in sections] == ["0", "0.0", "0.1"] - root, first, second = sections - assert root["placement_path"] == "/surveys/field-reports/" - assert first["parent_section_id"] == root["section_id"] - assert second["parent_section_id"] == root["section_id"] - assert first["role"] == "results" - assert second["role"] == "body" # the invented role degraded - assert first["block_end"] == second["block_start"] - 1 # tiled partition - assert structurer == "pageindex_llm" + assert [row["node_path"] for row in sections] == ["0", "0.0", "0.0.0", "0.0.1"] + root, report, findings, recommendations = sections + assert root["placement_path"] is None + assert report["parent_section_id"] == root["section_id"] + assert findings["parent_section_id"] == report["section_id"] + assert recommendations["parent_section_id"] == report["section_id"] + assert findings["role"] == "results" + assert recommendations["role"] == "body" + assert findings["block_end"] == recommendations["block_start"] - 1 + assert structurer == "parser" representation = rig.row( - sql="SELECT blocks_uri FROM document_representations" + sql="SELECT blocks_uri, pageindex_uri FROM document_representations" " WHERE version_id = :version_id", params={"version_id": ingested.version_id}, ) - sidecar_key = str(representation["blocks_uri"]).rsplit("/", 1)[0] sidecar = json.loads( - rig.artifact_store.read_bytes(key=ObjectKey(f"{sidecar_key}/pageindex.json")) + rig.artifact_store.read_bytes( + key=ObjectKey(str(representation["pageindex_uri"])) + ) ) - assert sidecar["placement"] == "/surveys/field-reports/" - assert len(sidecar["sections"]) == 3 + assert sidecar["placement"] is None + assert sidecar["generations"]["summary"] is None + assert sidecar["generations"]["placement"] is None + assert len(sidecar["sections"]) == 4 -def test_failed_structurer_degrades_to_the_synthetic_root(rig: _E0Rig) -> None: - """A dead model seat never fails a document — the root serves it.""" +def test_failed_checker_is_explicit_and_fail_open_on_the_parser(rig: _E0Rig) -> None: + """A dead checker is provider_error, never false coherence or fallback.""" class _DeadProvider: def generate(self, *, request: object, response_type: object) -> object: @@ -607,11 +599,30 @@ def embed(self, *, request: object) -> object: ).outcome assert outcome is RunResultOutcome.SUCCEEDED section = rig.row( - sql="SELECT node_path, role::text AS role FROM document_sections" + sql="SELECT count(*) AS count FROM document_sections s" + " JOIN document_representations r" + " ON r.current_structure_generation_id = s.structure_generation_id" + " WHERE s.version_id = :version_id", + params={"version_id": ingested.version_id}, + ) + assert section == {"count": 4} + check = rig.row( + sql="SELECT check_outcome::text AS outcome" + " FROM document_skeleton_checks WHERE version_id = :version_id", + params={"version_id": ingested.version_id}, + ) + assert check == {"outcome": "provider_error"} + representation = rig.row( + sql="SELECT representation_id FROM document_representations" " WHERE version_id = :version_id", params={"version_id": ingested.version_id}, ) - assert section == {"node_path": "0", "role": "body"} + checks = rig.catalog.skeleton_checks( + representation_id=representation["representation_id"] # type: ignore[arg-type] + ) + assert len(checks) == 1 + assert checks[0].check_outcome == "provider_error" + assert checks[0].provider_failure is not None def test_retried_tree_write_returns_the_first_attempts_truth(rig: _E0Rig) -> None: @@ -633,6 +644,28 @@ def test_retried_tree_write_returns_the_first_attempts_truth(rig: _E0Rig) -> Non params={"version_id": ingested.version_id}, ) representation_id = representation["representation_id"] + generation_id = uuid4() + stats = SkeletonStats( + stats_version="test", + section_count=0, + duplicate_title_ratio=None, + max_title_multiplicity=None, + sibling_duplicate_ratio=None, + level_jump_count=None, + numbering_coverage=None, + numbering_inversions=None, + numbering_scheme_switches=None, + tiny_section_ratio=None, + zero_direct_body_ratio=None, + oversized_leaf_ratio=None, + heading_density=None, + title_length_p50=None, + title_length_p95=None, + long_title_ratio=None, + low_letter_ratio=None, + empty_title_ratio=None, + max_sibling_fanout=None, + ) def _record(title: str) -> SectionTreeRecord: return SectionTreeRecord( @@ -640,6 +673,7 @@ def _record(title: str) -> SectionTreeRecord: doc_id=ingested.doc_id, version_id=ingested.version_id, representation_id=representation_id, # type: ignore[arg-type] + structure_generation_id=generation_id, sections=( SnappedSection( node_path="0", @@ -652,15 +686,52 @@ def _record(title: str) -> SectionTreeRecord: char_end=len(_STRUCTURED_SOURCE), summary="", ordinal=0, + normalized_title=title, ), ), placement_path=f"/{title}/", structurer_name="pageindex_llm", structurer_version="test-structurer", + skeleton_version="test-skeleton", + skeleton_hash="first-hash", + skeleton_producer_family="N/A", + skeleton_check_version=None, + roles_version="test-role", + selecting_check_id=None, + route_tag=StructureRouteTag.LEGACY, + candidate_skeleton_hash="first-hash", + stats_version="test", + stats=stats, + pageindex_uri="test/pageindex.json", ) first = rig.catalog.record_section_tree(record=_record("first")) - retry = rig.catalog.record_section_tree(record=_record("second")) + retry_input = _record("second") + retry_input = retry_input.model_copy( + update={ + "sections": ( + *retry_input.sections, + SnappedSection( + node_path="0.0", + parent_path="0", + title="retry-only child", + role="body", + block_start=1, + block_end=8, + char_start=1, + char_end=len(_STRUCTURED_SOURCE), + summary="", + ordinal=1, + heading_level=1, + normalized_title="retry-only child", + ), + ), + "skeleton_hash": "retry-hash", + "candidate_skeleton_hash": "retry-hash", + } + ) + retry = rig.catalog.record_section_tree(record=retry_input) assert first.sections[0].title == "first" - assert retry.sections[0].title == "first" # the first attempt's row won + assert [section.title for section in retry.sections] == ["first"] assert retry.placement_path == "/first/" + assert retry.skeleton_hash == "first-hash" diff --git a/src/tests/workers/test_p3_corpusfs.py b/src/tests/workers/test_p3_corpusfs.py index 8f2142c1..3c820f10 100644 --- a/src/tests/workers/test_p3_corpusfs.py +++ b/src/tests/workers/test_p3_corpusfs.py @@ -90,6 +90,7 @@ def _document( ) -> UUID: """One lineage with a ready version, representation, and root section.""" doc_id, version_id, representation_id = uuid4(), uuid4(), uuid4() + structure_generation_id = uuid4() connection.execute( # type: ignore[attr-defined] text( "INSERT INTO documents (doc_id, deployment_id, source_kind," @@ -135,14 +136,37 @@ def _document( "md": f"{doc_id}/document.md", }, ) + connection.execute( # type: ignore[attr-defined] + text( + "INSERT INTO document_structure_generations" + " (structure_generation_id, deployment_id, doc_id, version_id," + " representation_id, skeleton_version, skeleton_hash," + " skeleton_producer_family, roles_version, summary_version," + " placement_version, route_tag, candidate_skeleton_hash," + " stats_version, stats)" + " VALUES (:generation, :d, :doc, :v, :r, 'test-skeleton'," + " :hash, 'N/A', 'test-role', 'test-summary', 'test-placement'," + " 'parser', :hash, 'test-stats'," + " jsonb_build_object('stats_version', 'test-stats'," + " 'section_count', 0))" + ), + { + "generation": structure_generation_id, + "d": _DEPLOYMENT_ID, + "doc": doc_id, + "v": version_id, + "r": representation_id, + "hash": f"test-skeleton-{doc_id}", + }, + ) connection.execute( # type: ignore[attr-defined] text( "INSERT INTO document_sections (section_id, deployment_id, doc_id," - " version_id, representation_id, node_path, block_start, block_end," - " title, role, char_start, char_end, ordinal, summary," + " version_id, representation_id, structure_generation_id, node_path," + " block_start, block_end, title, role, char_start, char_end, ordinal, summary," " placement_path, structurer_version)" - " VALUES (:s, :d, :doc, :v, :r, '0', 0, 1, :title, 'body', 0, 10," - " 0, :summary, :placement, 'test')" + " VALUES (:s, :d, :doc, :v, :r, :generation, '0', 0, 1, :title," + " 'body', 0, 10, 0, :summary, :placement, 'test')" ), { "s": uuid4(), @@ -150,6 +174,7 @@ def _document( "doc": doc_id, "v": version_id, "r": representation_id, + "generation": structure_generation_id, "title": title, "summary": summary, "placement": placement, @@ -162,6 +187,14 @@ def _document( ), {"r": representation_id, "v": version_id}, ) + connection.execute( # type: ignore[attr-defined] + text( + "UPDATE document_representations" + " SET current_structure_generation_id = :generation" + " WHERE representation_id = :r" + ), + {"generation": structure_generation_id, "r": representation_id}, + ) connection.execute( # type: ignore[attr-defined] text("UPDATE documents SET current_version_id = :v WHERE doc_id = :doc"), {"v": version_id, "doc": doc_id}, @@ -442,3 +475,63 @@ def test_pathological_titles_and_refs_are_contained( assert len(field_lines) == 1 assert "attacker-controlled" not in field_lines[0] assert "\\n" in hostile # the newline was escaped, not emitted raw + + +def test_non_current_generations_never_leak_into_the_tree( + corpus: _Corpus, tmp_path: Path +) -> None: + """P3 reads ONLY the representation's current generation: a second, + non-current generation with a poison section must leave the built tree + byte-identical (review: nothing proved the join excludes history).""" + before, _ = _build(corpus, tmp_path / "before") + with corpus.engine.begin() as connection: + representation = connection.execute( + text( + "SELECT r.representation_id, r.version_id, v.doc_id" + " FROM document_representations r" + " JOIN document_versions v ON v.version_id = r.version_id LIMIT 1" + ) + ).one() + stale_generation = uuid4() + connection.execute( + text( + "INSERT INTO document_structure_generations (" + " structure_generation_id, deployment_id, doc_id, version_id," + " representation_id, skeleton_version, skeleton_hash," + " skeleton_producer_family, route_tag, candidate_skeleton_hash," + " stats_version, stats) VALUES (" + " :generation, :d, :doc, :v, :r, 'stale-v', 'stale-hash', 'N/A'," + " 'parser', 'stale-hash', 'stale-stats', CAST('{}' AS jsonb))" + ), + { + "generation": stale_generation, + "d": _DEPLOYMENT_ID, + "doc": representation.doc_id, + "v": representation.version_id, + "r": representation.representation_id, + }, + ) + connection.execute( + text( + "INSERT INTO document_sections (section_id, deployment_id, doc_id," + " version_id, representation_id, structure_generation_id, node_path," + " block_start, block_end, title, role, char_start, char_end," + " ordinal, structurer_version)" + " VALUES (:s, :d, :doc, :v, :r, :generation, '0', 0, 1," + " 'POISON NON-CURRENT SECTION', 'body', 0, 10, 0, 'stale-v')" + ), + { + "s": uuid4(), + "d": _DEPLOYMENT_ID, + "doc": representation.doc_id, + "v": representation.version_id, + "r": representation.representation_id, + "generation": stale_generation, + }, + ) + after, _ = _build(corpus, tmp_path / "after") + assert before.paths() == after.paths() + for path in sorted(after.paths()): + content = after.read(path) + assert "POISON" not in content + assert after.read(path) == before.read(path) diff --git a/website/src/app/docs/deployment/page.mdx b/website/src/app/docs/deployment/page.mdx index 9a032ded..df569a67 100644 --- a/website/src/app/docs/deployment/page.mdx +++ b/website/src/app/docs/deployment/page.mdx @@ -16,6 +16,15 @@ canonical plus P2 retrieval recipes. The RememberStack, `rememberstack`, and `remember` names used here are final. Do not expose this stack to untrusted traffic. +Model seats are per-deployment environment bindings. The document-structure +stage uses three: `REMEMBERSTACK_SKELETON_CHECK_MODEL` (the bounded skeleton +sanity judge, default `z-ai/glm-4.7-flash`), `REMEMBERSTACK_ROLE_MODEL` (the +title-only section-role classifier, default `z-ai/glm-4.7-flash`), and +`REMEMBERSTACK_STRUCTURER_MODEL` — which now names **only** the string-anchor +fallback proposer for documents whose headings cannot be parsed or whose +parsed tree fails the sanity check; the routine path is deterministic and +calls no model. + ## Start the stack Docker Engine with Compose v2 is required. diff --git a/website/src/app/docs/project-status/page.mdx b/website/src/app/docs/project-status/page.mdx index 8adcdfa6..0a335ccb 100644 --- a/website/src/app/docs/project-status/page.mdx +++ b/website/src/app/docs/project-status/page.mdx @@ -21,7 +21,7 @@ RememberStack is being built **design-first**: the complete system — requireme - **Phase 4, complete** — the projections: derived, rebuildable read surfaces. The knowledge graph rebuilds whole from PostgreSQL on every cycle (zero drift by construction, entity merges become no-ops), validates before publishing — a merge cycle or a row-count mismatch aborts the snapshot loudly — and ships as an immutable snapshot that read-only readers hot-swap to. Agents query it through a typed `graph` primitive: neighborhoods and shortest paths, filtered as-of a point in world or system time, with explicit truncation markers instead of silent top-k caps and typed answers when there is nothing to return. Community detection, PageRank, k-core, and connected components run natively on each fresh snapshot and write back to PostgreSQL. And the corpus filesystem is browsable: a generated directory tree where one index file tells an agent what every document in a directory is about, canonical per-document and per-entity paths never move across rebuilds, and the original files sit deliberately off the navigation path behind audited access. - **Phase 5, complete** — retrieval complete. The full zero-LLM primitive set is implemented (`fuse`, `rerank`, `transcript`, `delta`, `pages_about`, enumerated `aggregate`, and a streaming batch `scan`, alongside resolve/lookup/search/graph); recipes are registry rows whose declared grain is enforced mechanically; and the self-accounting envelope surfaces contradictions, withdrawn support, mixed-grain parts, identity regime, belief horizons, freshness, and typed negatives. API, CLI, and MCP all render the same deployment registry. The consumption skill is guarded by the repeatable S58 cold-agent eval, while the retrieval spike battery records filtered Lance search at 10 million rows and graph pagination at a 100,000-edge hub. The client-first `rememberstack` base wheel contains the typed SDK, remote `remember` CLI and MCP, lineage-aware E0 ingest, and typed remote connector-management commands plus their deployment composition port, without server dependencies; `[server]`, `[connectors-watched-directory]`, and `[k]` name the heavier install surfaces. - **Phase 6, complete** — Plane K now includes the deterministic control plane and crash-safe single-committer driver, exact rule routing and staleness, deterministic fact-sheet and agent-written prose bands, planner/reflection decisions with quarantine and adoption, and authored-page frontmatter, citations, watches, review flags, and debounced workflow dispatch. D73 removed the proposed K3 tier: personal or organizational principles are authored K2 content, supported by compiled scope pages and never machine-promoted or rewritten. -- **Phase 3, complete** — the evidence lifecycle: documents that change. Watched sources poll as recorded sync cycles (a local-directory watcher ships; revision and content no-ops, debounce, source-deletion detection); an edited document becomes a new version of its lineage and the full structure route (an LLM-proposed section tree normalized by a deterministic snap) re-reads it; unchanged chunks reuse their prior claims, prefixes, and vectors so cost is proportional to the edit (measured hit rate 0.79 on the spike corpus); reconciliation transitions testimony currency on an append-only ledger, recounts evidence by distinct current lineages, closes solely-supported facts when the source withdrew them (at the sync-cycle barrier, so a moved section is a support swap — never a retract flicker) and flags them for review when only the toolchain changed; deletion removes a document's contribution uniformly while keeping its claims as history; and a standing `lifecycle` eval suite guards the cache/ledger/count invariants with planted regression canaries. Those lifecycle events now feed the graph, corpus filesystem, and Plane-K control plane described above. +- **Phase 3, complete** — the evidence lifecycle: documents that change. Watched sources poll as recorded sync cycles (a local-directory watcher ships; revision and content no-ops, debounce, source-deletion detection); an edited document becomes a new version of its lineage and the full structure route (a deterministic heading-parsed section tree, sanity-checked by a bounded flash-class judge, with a string-anchored LLM fallback for deficient trees) re-reads it; unchanged chunks reuse their prior claims, prefixes, and vectors so cost is proportional to the edit (measured hit rate 0.79 on the spike corpus); reconciliation transitions testimony currency on an append-only ledger, recounts evidence by distinct current lineages, closes solely-supported facts when the source withdrew them (at the sync-cycle barrier, so a moved section is a support swap — never a retract flicker) and flags them for review when only the toolchain changed; deletion removes a document's contribution uniformly while keeping its claims as history; and a standing `lifecycle` eval suite guards the cache/ledger/count invariants with planted regression canaries. Those lifecycle events now feed the graph, corpus filesystem, and Plane-K control plane described above. - **Phase 7, complete** — operational correctness now includes resumable initial-load and version-bump backfill on the same work ledger, a reproducible PostgreSQL scale battery for the designed partitions, indexes, hubs, and batching invariants, and optional per-deployment/stage/lane cost ceilings. Budget exhaustion parks healthy work durably without consuming an attempt; the local CLI reports current spend, remaining budget, tier attribution, and parked work from the same authoritative rows. Typed worker telemetry preserves complete exception chains, while bounded CLI inspection exposes pipeline/DLQ state, poison targets, P2/P3 pointers, and currency-ledger drift; an explicit one-row replay grants only additional attempts, and rebuild drills call the production P2/P3 builders. Hard-forget now purges every active library-controlled surface and replays a separately durable, content-free manifest before serving after a restore. Portability is deliberately smaller than a backup product: operators move PostgreSQL, objects, and Git with native tools; the library defines the fail-closed restore order and rebuilds P1/P2/P3. The fresh-deployment Compose profile now runs PostgreSQL, MinIO, the API, and all ten implemented continuous E/P1 routes, with explicit one-shot P2/P3 publication and machine-verifiable per-version readiness. Release engineering enforces one semantic version across PyPI, the shared API/worker GHCR image, and Compose; CI measures cold start and drills a schema downgrade followed by migrations-before-process-restart. The public [`v0.1.0`](https://github.com/writeitai/remember-stack/releases/tag/v0.1.0) release proved the complete tag workflow across PyPI, GHCR, and GitHub Releases; trusted publishing, protected release tags, and the required repository-native bounded-CLA check are active. Timings remain measurements rather than hosted SLAs, and no export/import CLI, dashboard, billing policy, HA topology, backup scheduler, or control plane enters the library. ## The build order