D79 Wave 1: the structure stage — deterministic skeleton, sanity check, anchor fallback (#165)

.env.example

@@ -37,3 +37,8 @@ REMEMBERSTACK_OPENROUTER_API_KEY=replace-before-real-use
 # reproducible run; do not use a rotating router such as openrouter/free.
 # REMEMBERSTACK_E2_EXTRACT_MODEL=nvidia/nemotron-3-super-120b-a12b:free
 # REMEMBERSTACK_E3_NORMALIZE_MODEL=nvidia/nemotron-3-super-120b-a12b:free
+# D79 structure seats: STRUCTURER names only the string-anchor fallback
+# proposer; the routine skeleton path is deterministic and calls no model.
+# REMEMBERSTACK_STRUCTURER_MODEL=openai/gpt-5.6-luna
+# REMEMBERSTACK_SKELETON_CHECK_MODEL=z-ai/glm-4.7-flash
+# REMEMBERSTACK_ROLE_MODEL=z-ai/glm-4.7-flash

.github/workflows/ci.yml

@@ -166,7 +166,7 @@ jobs:
           trap - EXIT
           test "$(docker compose --env-file .env.example exec -T postgres \
             psql -U rememberstack -d rememberstack -Atc \
-            'SELECT version_num FROM alembic_version')" = 'p1_03_0018'
+            'SELECT version_num FROM alembic_version')" = 'p1_04_0019'
           test "$(docker compose --env-file .env.example exec -T postgres \
             psql -U rememberstack -d rememberstack -Atc \
             'SELECT count(*) FROM deployments')" = '1'

compose.yaml

@@ -23,7 +23,10 @@ x-app: &app
     REMEMBERSTACK_SELFHOST_DEPLOYMENT_SLUG: ${REMEMBERSTACK_SELFHOST_DEPLOYMENT_SLUG}
     REMEMBERSTACK_SELFHOST_DEPLOYMENT_NAME: ${REMEMBERSTACK_SELFHOST_DEPLOYMENT_NAME}
     REMEMBERSTACK_SELFHOST_API_PORT: "8000"
+    # D79: STRUCTURER now names only the anchor-proposal fallback seat.
     REMEMBERSTACK_STRUCTURER_MODEL: ${REMEMBERSTACK_STRUCTURER_MODEL:-openai/gpt-5.6-luna}
+    REMEMBERSTACK_SKELETON_CHECK_MODEL: ${REMEMBERSTACK_SKELETON_CHECK_MODEL:-z-ai/glm-4.7-flash}
+    REMEMBERSTACK_ROLE_MODEL: ${REMEMBERSTACK_ROLE_MODEL:-z-ai/glm-4.7-flash}
     REMEMBERSTACK_E1_EMBEDDING_MODEL: ${REMEMBERSTACK_E1_EMBEDDING_MODEL:-qwen/qwen3-embedding-8b}
     REMEMBERSTACK_E1_PREFIX_MODEL: ${REMEMBERSTACK_E1_PREFIX_MODEL:-openai/gpt-5.6-luna}
     REMEMBERSTACK_E2_EXTRACT_MODEL: ${REMEMBERSTACK_E2_EXTRACT_MODEL:-openai/gpt-5.6-luna}

plan/designs/e0_files_design.md

@@ -279,6 +279,9 @@ unchanged):
 
   ```
   parse candidate skeleton → compute + persist stats → density / oversized-leaf gates
+    (a gate demotion enters the SAME fallback → TERMINAL-check sequence below —
+    every fallback-produced tree faces the terminal judge, however the
+    document got demoted; clarified 2026-07-28 with Wave-1 implementation)
     → eligibility (≥ MIN_CHECK_SECTIONS non-root sections; below it: outcome
       not_run_short, skeleton accepted)
     → check call → coherent: keep parsed skeleton

src/rememberstack/adapters/openrouter.py

@@ -25,6 +25,7 @@
 from rememberstack.model import ProviderAccountingError
 from rememberstack.model import ProviderCallError
 from rememberstack.model import ProviderCallUsage
+from rememberstack.model import ProviderInvalidResponseError
 from rememberstack.model import StructuredResponseModel
 
 ResponseT = TypeVar("ResponseT", bound=StructuredResponseModel)
@@ -120,6 +121,12 @@ class OpenRouterProviderError(ProviderCallError):
     """OpenRouter returned an error or an unusable response body."""
 
 
+class OpenRouterInvalidResponseError(
+    OpenRouterProviderError, ProviderInvalidResponseError
+):
+    """OpenRouter completed a generation without a schema-valid output."""
+
+
 class OpenRouterModelProvider:
     """Structured generations and embeddings over the OpenRouter HTTP API."""
 
@@ -162,15 +169,15 @@ def generate(
         try:
             decoded = json.loads(content)
         except json.JSONDecodeError as err:
-            raise OpenRouterProviderError(
+            raise OpenRouterInvalidResponseError(
                 f"{response_type.__name__}: completion content is not JSON"
                 f" ({_content_fingerprint(content=content)})",
                 usage=usage,
             ) from err
         try:
             output = response_type.model_validate(decoded)
         except ValidationError as error:
-            raise OpenRouterProviderError(
+            raise OpenRouterInvalidResponseError(
                 f"completion body failed {response_type.__name__} validation",
                 usage=usage,
             ) from error
@@ -204,7 +211,7 @@ def _completion_text(
         )
         content = _completion_content(body=body)
         if content is None:
-            raise OpenRouterProviderError(
+            raise OpenRouterInvalidResponseError(
                 f"{response_type.__name__}: provider returned no completion"
                 f" content ({_completion_diagnosis(body=body)})",
                 usage=usage,

src/rememberstack/core/__init__.py

@@ -3,7 +3,9 @@
 from rememberstack.core.blockizer import block_hash
 from rememberstack.core.blockizer import blockize
 from rememberstack.core.blockizer import BLOCKIZER_VERSION
+from rememberstack.core.blockizer import blocks_from_sidecar
 from rememberstack.core.blockizer import normalized_block_text
+from rememberstack.core.blockizer import normalized_heading_title
 from rememberstack.core.chunker import chunk_content_hash
 from rememberstack.core.chunker import CHUNKER_VERSION
 from rememberstack.core.chunker import chunker_version
@@ -61,6 +63,18 @@
 from rememberstack.core.section_snap import snap_sections
 from rememberstack.core.storage_routing import HOT_MIME_PREFIXES
 from rememberstack.core.storage_routing import storage_class_for
+from rememberstack.core.structure_skeleton import analyze_skeleton
+from rememberstack.core.structure_skeleton import deterministic_section_role
+from rememberstack.core.structure_skeleton import LONG_TITLE
+from rememberstack.core.structure_skeleton import MAX_FALLBACK_DEPTH
+from rememberstack.core.structure_skeleton import MIN_CHECK_SECTIONS
+from rememberstack.core.structure_skeleton import parse_heading_skeleton
+from rememberstack.core.structure_skeleton import resolve_fallback_skeleton
+from rememberstack.core.structure_skeleton import skeleton_hash
+from rememberstack.core.structure_skeleton import SKELETON_PARSER_VERSION
+from rememberstack.core.structure_skeleton import SKELETON_STATS_VERSION
+from rememberstack.core.structure_skeleton import SkeletonAnalysis
+from rememberstack.core.structure_skeleton import TINY_FLOOR
 
 __all__ = (
     "BLOCKIZER_VERSION",
@@ -88,12 +102,26 @@
     "PredicateSignatureDefinition",
     "block_hash",
     "blockize",
+    "blocks_from_sidecar",
     "normalized_block_text",
+    "normalized_heading_title",
     "HOT_MIME_PREFIXES",
     "storage_class_for",
     "source_identity_hash",
     "SECTION_ROLES",
     "snap_sections",
+    "analyze_skeleton",
+    "deterministic_section_role",
+    "LONG_TITLE",
+    "MAX_FALLBACK_DEPTH",
+    "MIN_CHECK_SECTIONS",
+    "parse_heading_skeleton",
+    "resolve_fallback_skeleton",
+    "SKELETON_PARSER_VERSION",
+    "SKELETON_STATS_VERSION",
+    "skeleton_hash",
+    "SkeletonAnalysis",
+    "TINY_FLOOR",
     "DEFAULT_EVIDENCE_COUNT_WEIGHT",
     "DEFAULT_GRAPH_DISTANCE_WEIGHT",
     "DEFAULT_RRF_K",

src/rememberstack/core/blockizer.py

@@ -18,8 +18,15 @@
 from rememberstack.model import Block
 from rememberstack.model import BlockType
 
-BLOCKIZER_VERSION: Final = "blockizer-2026.07:markdown-it-py-4:gfm-tables"
-"""Pins the parser library generation and enabled-extension set (e1 §2)."""
+BLOCKIZER_VERSION: Final = (
+    "blockizer-2026.07b:markdown-it-py-4:gfm-tables:heading-metadata"
+)
+"""Pins the parser profile and D79 heading metadata contract.
+
+The ``07b`` bump changes only ``blocks.json`` metadata. Segmentation, offsets,
+normalization for ``block_hash``, and therefore the block/chunk grid are byte
+identical to ``blockizer-2026.07:markdown-it-py-4:gfm-tables``.
+"""
 
 _ATOMIC_CONTAINERS: Final = {
     "table_open": ("table_close", BlockType.TABLE),
@@ -28,6 +35,26 @@
 _LIST_OPENERS: Final = frozenset({"bullet_list_open", "ordered_list_open"})
 
 
+def blocks_from_sidecar(
+    *, blocks_doc: dict[str, object], document_md: str
+) -> tuple[Block, ...]:
+    """Load a blocks.json sidecar, re-blockizing when its version is stale.
+
+    The sidecar is a CACHE of the pinned blockizer's output, never a second
+    source of truth: on a version mismatch (e.g. pre-``07b`` sidecars without
+    heading metadata) the blocks are re-derived from ``document.md`` instead
+    of failing validation. The ``07`` → ``07b`` bump is grid-byte-identical —
+    the recompute changes metadata only — so STRUCTURE/CHUNK work queued or
+    retried across the upgrade keeps flowing instead of dead-lettering on a
+    permanently stale representation. Strictness stays on the produce side.
+    """
+    if blocks_doc.get("blockizer_version") == BLOCKIZER_VERSION:
+        payloads = blocks_doc["blocks"]
+        assert isinstance(payloads, list)
+        return tuple(Block.model_validate(payload) for payload in payloads)
+    return blockize(document_md=document_md)
+
+
 def blockize(*, document_md: str) -> tuple[Block, ...]:
     """Derive the deterministic block sequence from a document.md rendering.
 
@@ -63,6 +90,12 @@ def normalized_block_text(*, raw: str) -> str:
     return " ".join(composed.split())
 
 
+def normalized_heading_title(*, title: str) -> str:
+    """D79 title normalization: NFKC → casefold → whitespace collapse."""
+    compatible = unicodedata.normalize("NFKC", title)
+    return " ".join(compatible.casefold().split())
+
+
 def block_hash(*, raw: str) -> str:
     """The block identity: sha256 over the normalized text."""
     digest = hashlib.sha256(normalized_block_text(raw=raw).encode("utf-8"))
@@ -130,12 +163,16 @@ def _emit(
             ),
         )
     if token.type == "heading_open":
+        inline = tokens[index + 1]
+        heading_title = _heading_text(token=inline)
         return 3, _block_from_lines(
             token=token,
             block_type=BlockType.HEADING,
             document_md=document_md,
             line_offsets=line_offsets,
             ordinal=ordinal,
+            heading_level=int(token.tag.removeprefix("h")),
+            heading_title=heading_title,
         )
     if token.type == "paragraph_open":
         return 3, _block_from_lines(
@@ -172,6 +209,8 @@ def _block_from_lines(
     document_md: str,
     line_offsets: tuple[int, ...],
     ordinal: int,
+    heading_level: int | None = None,
+    heading_title: str | None = None,
 ) -> Block:
     """Build the block record from a token's source line map."""
     if token.map is None:
@@ -186,4 +225,31 @@ def _block_from_lines(
         char_start=char_start,
         char_end=char_start + len(raw),
         block_hash=block_hash(raw=raw),
+        heading_level=heading_level,
+        heading_title=heading_title,
+        normalized_title=(
+            normalized_heading_title(title=heading_title)
+            if heading_title is not None
+            else None
+        ),
     )
+
+
+def _heading_text(*, token: Token) -> str:
+    """Render a heading inline token to plain title text without re-tokenizing.
+
+    The canonical markdown-it parse already produced child tokens. Reusing
+    them preserves D57's single-tokenizer rule while removing emphasis/link
+    markup from the title used by the parser, stats, and deterministic roles.
+    """
+    if token.children is None:
+        return token.content.strip()
+    pieces: list[str] = []
+    for child in token.children:
+        if child.type in {"text", "code_inline"}:
+            pieces.append(child.content)
+        elif child.type == "image":
+            pieces.append(child.content)
+        elif child.type in {"softbreak", "hardbreak"}:
+            pieces.append(" ")
+    return " ".join("".join(pieces).split())

src/rememberstack/core/chunker.py

@@ -22,9 +22,16 @@
 from rememberstack.model import PackedChunk
 from rememberstack.model import SectionSpan
 
-CHUNKER_VERSION: Final = "e1-chunker-2026.07b:whitespace-tokens:anchored:owner-runs"
-"""Pins the packing algorithm and the token counter; the full packing
-generation additionally encodes the parameter values — see `chunker_version`."""
+CHUNKER_VERSION: Final = (
+    "e1-chunker-2026.07c:whitespace-tokens:anchored:owner-runs"
+    ":blockizer-heading-metadata"
+)
+"""Pins packing plus its blockizer-contract generation.
+
+The ``07c`` bump follows D12/D57 because ``blocks.json`` gained D79 heading
+metadata. Packing behavior and emitted chunk-grid bytes are unchanged from
+``e1-chunker-2026.07b:whitespace-tokens:anchored:owner-runs``.
+"""
 
 
 class ChunkerParams(BaseModel):