D79 Wave 2: bottom-up summaries + placement (#165)

.env.example

@@ -42,3 +42,4 @@ REMEMBERSTACK_OPENROUTER_API_KEY=replace-before-real-use
 # REMEMBERSTACK_STRUCTURER_MODEL=openai/gpt-5.6-luna
 # REMEMBERSTACK_SKELETON_CHECK_MODEL=z-ai/glm-4.7-flash
 # REMEMBERSTACK_ROLE_MODEL=z-ai/glm-4.7-flash
+# REMEMBERSTACK_SUMMARY_MODEL=z-ai/glm-4.7-flash

compose.yaml

@@ -27,6 +27,7 @@ x-app: &app
     REMEMBERSTACK_STRUCTURER_MODEL: ${REMEMBERSTACK_STRUCTURER_MODEL:-openai/gpt-5.6-luna}
     REMEMBERSTACK_SKELETON_CHECK_MODEL: ${REMEMBERSTACK_SKELETON_CHECK_MODEL:-z-ai/glm-4.7-flash}
     REMEMBERSTACK_ROLE_MODEL: ${REMEMBERSTACK_ROLE_MODEL:-z-ai/glm-4.7-flash}
+    REMEMBERSTACK_SUMMARY_MODEL: ${REMEMBERSTACK_SUMMARY_MODEL:-z-ai/glm-4.7-flash}
     REMEMBERSTACK_E1_EMBEDDING_MODEL: ${REMEMBERSTACK_E1_EMBEDDING_MODEL:-qwen/qwen3-embedding-8b}
     REMEMBERSTACK_E1_PREFIX_MODEL: ${REMEMBERSTACK_E1_PREFIX_MODEL:-openai/gpt-5.6-luna}
     REMEMBERSTACK_E2_EXTRACT_MODEL: ${REMEMBERSTACK_E2_EXTRACT_MODEL:-openai/gpt-5.6-luna}

plan/designs/e0_files_design.md

@@ -382,10 +382,20 @@ unchanged):
   bounded: a leaf larger than the summary-call token ceiling is sharded at block grain and
   reduced; a parent call reads its **own direct blocks plus its children's one-liners** (a
   chapter's preamble is content too) with balanced fan-in when children are many. Calls are
-  parallel and cached per section; the cache key is the ordered constituent block hashes,
-  child-summary hashes, model, prompt, generation parameters, and summarizer version. This is
-  the call shape where cheap models are reliable: bounded context, no arithmetic, no giant
-  output.
+  parallel and cached per section; the cache key hashes the ordered rendered block strings,
+  rendered child lines (path + capped title + summary), model, prompt, generation parameters,
+  and summarizer version. This is the call shape where cheap models are reliable: bounded
+  context, no arithmetic, no giant output. The versioned token estimate is whitespace-based;
+  the companion character ceiling is the hard bound for whitespace-poor/CJK/minified input,
+  and every rendered call must fit both. **Degradation convention:** `summary_version` is
+  non-null only when every section
+  in the generation has a summary; any failed section call leaves that section null (and any
+  ancestor that cannot receive all child one-liners null), may preserve successful summaries
+  in independent subtrees, and makes the generation's `summary_version` null. Because root
+  placement is emitted by the same complete reduction, `placement_version` is non-null iff
+  `summary_version` and root `placement_path` are non-null; otherwise both placement fields
+  are null. Thus a partially useful tree is persisted without mislabeling a degraded run as a
+  complete summary generation.
 - **Placement rides the root reduction.** The one-shot call also produced the D39 placement
   hint; that responsibility moves to the **document-level (root) summary call**, which sees
   exactly what placement needs — the title, source kind, and the child one-liners. On the

src/rememberstack/model/__init__.py

@@ -302,6 +302,8 @@
 from rememberstack.model.sections import ProposedSection
 from rememberstack.model.sections import RoleAssignment
 from rememberstack.model.sections import RoleClassificationResponse
+from rememberstack.model.sections import RootSummaryPlacementResponse
+from rememberstack.model.sections import SectionSummaryResponse
 from rememberstack.model.sections import SectionTreeRecord
 from rememberstack.model.sections import SkeletonCheckOutcome
 from rememberstack.model.sections import SkeletonCheckRecord
@@ -591,7 +593,9 @@
     "ProposedSection",
     "RoleAssignment",
     "RoleClassificationResponse",
+    "RootSummaryPlacementResponse",
     "SectionTreeRecord",
+    "SectionSummaryResponse",
     "SkeletonCheckOutcome",
     "SkeletonCheckRecord",
     "SkeletonCheckResponse",

src/rememberstack/model/sections.py

@@ -164,6 +164,23 @@ class RoleClassificationResponse(BaseModel):
     assignments: tuple[RoleAssignment, ...] = ()
 
 
+class SectionSummaryResponse(BaseModel):
+    """One section summary and no auxiliary prose."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    summary: str = Field(min_length=1)
+
+
+class RootSummaryPlacementResponse(BaseModel):
+    """The root reduction's closed two-field summary + D39 placement shape."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    summary: str = Field(min_length=1)
+    placement_path: str = Field(min_length=1)
+
+
 class SnappedSection(BaseModel):
     """One well-formed section after the deterministic snap (block coordinates).
 
@@ -181,7 +198,7 @@ class SnappedSection(BaseModel):
     block_end: int = Field(ge=-1)
     char_start: int = Field(ge=0)
     char_end: int = Field(ge=0)
-    summary: str
+    summary: str | None
     ordinal: int = Field(ge=0)
     heading_level: int | None = Field(default=None, ge=1, le=6)
     normalized_title: str = ""

src/rememberstack/profiles/selfhost.py

@@ -391,6 +391,7 @@ def _handler(self, *, stage: PipelineStage) -> StageHandler:
         from rememberstack.workers import SkeletonCheckSettings
         from rememberstack.workers import StructureHandler
         from rememberstack.workers import StructurerSettings
+        from rememberstack.workers import SummarySettings
 
         documents = DocumentCatalog(engine=self._engine)
         chunks = ChunkCatalog(engine=self._engine)
@@ -417,6 +418,7 @@ def _handler(self, *, stage: PipelineStage) -> StageHandler:
                 settings=StructurerSettings.model_validate({}),
                 check_settings=SkeletonCheckSettings.model_validate({}),
                 role_settings=RoleSettings.model_validate({}),
+                summary_settings=SummarySettings.model_validate({}),
             )
         if stage is PipelineStage.CHUNK:
             return ChunkHandler(
@@ -598,10 +600,12 @@ def _model_bindings() -> dict[str, str]:
     from rememberstack.workers import RoleSettings
     from rememberstack.workers import SkeletonCheckSettings
     from rememberstack.workers import StructurerSettings
+    from rememberstack.workers import SummarySettings
 
     structurer = StructurerSettings.model_validate({})
     skeleton_check = SkeletonCheckSettings.model_validate({})
     roles = RoleSettings.model_validate({})
+    summaries = SummarySettings.model_validate({})
     e1 = E1Settings.model_validate({})
     e2 = E2Settings.model_validate({})
     e3 = E3Settings.model_validate({})
@@ -613,6 +617,7 @@ def _model_bindings() -> dict[str, str]:
         "structure_fallback": structurer.model,
         "skeleton_check": skeleton_check.model,
         "section_role": roles.model,
+        "section_summary": summaries.model,
         "chunk_embedding": e1.embedding_model,
         "context_prefix": e1.prefix_model,
         "claim_extraction": e2.extract_model,

src/rememberstack/spine/document_catalog.py

@@ -7,6 +7,7 @@
 skeleton checks append independently under D52.
 """
 
+from collections.abc import Sequence
 from decimal import Decimal
 import json
 from uuid import NAMESPACE_URL
@@ -17,6 +18,7 @@
 from sqlalchemy import text
 from sqlalchemy.engine import Connection
 from sqlalchemy.engine import Engine
+from sqlalchemy.engine import RowMapping
 
 from rememberstack.model import ConvertSource
 from rememberstack.model import DocumentVersionNotFoundError
@@ -215,6 +217,41 @@ def structure_source(self, *, representation_id: UUID) -> StructureSource:
             )
         return StructureSource.model_validate(dict(row))
 
+    def current_section_tree(
+        self, *, representation_id: UUID
+    ) -> PersistedSectionTree | None:
+        """Load the representation's current immutable structure generation."""
+        with self._engine.connect() as connection:
+            generation = (
+                connection.execute(
+                    _SELECT_CURRENT_STRUCTURE_GENERATION,
+                    {"representation_id": representation_id},
+                )
+                .mappings()
+                .one_or_none()
+            )
+            if generation is None:
+                return None
+            persisted = (
+                connection.execute(
+                    _SELECT_SECTION_TREE,
+                    {"structure_generation_id": generation["structure_generation_id"]},
+                )
+                .mappings()
+                .all()
+            )
+        if not persisted:
+            raise RuntimeError("current structure generation has no root section")
+        return _persisted_tree(generation=generation, sections=persisted)
+
+    def summary_cache_sidecars(self, *, doc_id: UUID) -> tuple[str, ...]:
+        """All prior sidecars that may contain keyed summary cache entries."""
+        with self._engine.connect() as connection:
+            rows = connection.execute(
+                _SELECT_SUMMARY_CACHE_SIDECARS, {"doc_id": doc_id}
+            ).scalars()
+            return tuple(str(uri) for uri in rows)
+
     def record_synthetic_root(self, *, record: SyntheticRootRecord) -> None:
         """Compatibility helper for callers that explicitly request a root.
 
@@ -446,45 +483,7 @@ def record_section_tree(self, *, record: SectionTreeRecord) -> PersistedSectionT
                     _SUPERSEDE_PRIOR_VERSIONS,  # are superseded as of now (D55)
                     {"doc_id": record.doc_id, "version_id": record.version_id},
                 )
-        return PersistedSectionTree(
-            sections=tuple(
-                SnappedSection(
-                    node_path=row["node_path"],
-                    parent_path=(
-                        row["node_path"].rsplit(".", 1)[0]
-                        if "." in row["node_path"]
-                        else None
-                    ),
-                    title=row["title"] or "",
-                    role=row["role"],
-                    block_start=row["block_start"],
-                    block_end=row["block_end"],
-                    char_start=row["char_start"],
-                    char_end=row["char_end"],
-                    summary=row["summary"] or "",
-                    ordinal=row["ordinal"],
-                    heading_level=row["heading_level"],
-                    normalized_title=row["normalized_title"],
-                )
-                for row in persisted
-            ),
-            placement_path=persisted[0]["placement_path"],
-            structurer_version=persisted[0]["structurer_version"] or "",
-            structure_generation_id=generation["structure_generation_id"],
-            pageindex_uri=generation["pageindex_uri"],
-            skeleton_version=generation["skeleton_version"],
-            skeleton_hash=generation["skeleton_hash"],
-            skeleton_producer_family=generation["skeleton_producer_family"],
-            skeleton_check_version=generation["skeleton_check_version"],
-            roles_version=generation["roles_version"],
-            summary_version=generation["summary_version"],
-            placement_version=generation["placement_version"],
-            selecting_check_id=generation["selecting_check_id"],
-            route_tag=generation["route_tag"],
-            candidate_skeleton_hash=generation["candidate_skeleton_hash"],
-            stats_version=generation["stats_version"],
-            stats=SkeletonStats.model_validate(generation["stats"]),
-        )
+        return _persisted_tree(generation=generation, sections=persisted)
 
 
 def _lineage_locked(*, connection: Connection, record: UploadRecord) -> UUID:
@@ -732,6 +731,31 @@ def _lineage_locked(*, connection: Connection, record: UploadRecord) -> UUID:
     """
 )
 
+_SELECT_CURRENT_STRUCTURE_GENERATION = text(
+    """
+    SELECT g.structure_generation_id, g.skeleton_version, g.skeleton_hash,
+           g.skeleton_producer_family, g.skeleton_check_version, g.roles_version,
+           g.summary_version, g.placement_version, g.selecting_check_id,
+           g.route_tag::text AS route_tag, g.candidate_skeleton_hash,
+           g.stats_version, g.stats, g.pageindex_uri
+    FROM document_representations r
+    JOIN document_structure_generations g
+      ON g.representation_id = r.representation_id
+     AND g.structure_generation_id = r.current_structure_generation_id
+    WHERE r.representation_id = :representation_id
+    """
+)
+
+_SELECT_SUMMARY_CACHE_SIDECARS = text(
+    """
+    SELECT pageindex_uri
+    FROM document_structure_generations
+    WHERE doc_id = :doc_id
+      AND pageindex_uri IS NOT NULL
+    ORDER BY created_at DESC, structure_generation_id
+    """
+)
+
 _SELECT_SECTION_TREE = text(
     """
     SELECT node_path, title, role::text AS role, block_start, block_end,
@@ -743,6 +767,54 @@ def _lineage_locked(*, connection: Connection, record: UploadRecord) -> UUID:
     """
 )
 
+
+def _persisted_tree(
+    *, generation: RowMapping, sections: Sequence[RowMapping]
+) -> PersistedSectionTree:
+    """Materialize one immutable generation from SQL mapping rows."""
+    generation_row = generation
+    section_rows = sections
+    return PersistedSectionTree(
+        sections=tuple(
+            SnappedSection(
+                node_path=row["node_path"],
+                parent_path=(
+                    row["node_path"].rsplit(".", 1)[0]
+                    if "." in row["node_path"]
+                    else None
+                ),
+                title=row["title"] or "",
+                role=row["role"],
+                block_start=row["block_start"],
+                block_end=row["block_end"],
+                char_start=row["char_start"],
+                char_end=row["char_end"],
+                summary=row["summary"],
+                ordinal=row["ordinal"],
+                heading_level=row["heading_level"],
+                normalized_title=row["normalized_title"],
+            )
+            for row in section_rows
+        ),
+        placement_path=section_rows[0]["placement_path"],
+        structurer_version=section_rows[0]["structurer_version"] or "",
+        structure_generation_id=generation_row["structure_generation_id"],
+        pageindex_uri=generation_row["pageindex_uri"] or "",
+        skeleton_version=generation_row["skeleton_version"],
+        skeleton_hash=generation_row["skeleton_hash"],
+        skeleton_producer_family=generation_row["skeleton_producer_family"],
+        skeleton_check_version=generation_row["skeleton_check_version"],
+        roles_version=generation_row["roles_version"],
+        summary_version=generation_row["summary_version"],
+        placement_version=generation_row["placement_version"],
+        selecting_check_id=generation_row["selecting_check_id"],
+        route_tag=generation_row["route_tag"],
+        candidate_skeleton_hash=generation_row["candidate_skeleton_hash"],
+        stats_version=generation_row["stats_version"],
+        stats=SkeletonStats.model_validate(generation_row["stats"]),
+    )
+
+
 _MARK_REPRESENTATION_READY = text(
     """
     UPDATE document_representations

src/rememberstack/workers/__init__.py

@@ -17,6 +17,9 @@
 from rememberstack.workers.e0 import StructurerSettings
 from rememberstack.workers.e0 import UPLOAD_SOURCE_KIND
 from rememberstack.workers.e0 import UploadIngestor
+from rememberstack.workers.e0_summary import E0_PLACEMENT_VERSION
+from rememberstack.workers.e0_summary import E0_SUMMARY_VERSION
+from rememberstack.workers.e0_summary import SummarySettings
 from rememberstack.workers.e1 import ChunkHandler
 from rememberstack.workers.e1 import E1_CHUNK_VERSION
 from rememberstack.workers.e1 import E1_EMBED_VERSION
@@ -127,6 +130,8 @@
     "E0_SKELETON_CHECK_VERSION",
     "E0_SKELETON_VERSION",
     "E0_STRUCTURE_VERSION",
+    "E0_PLACEMENT_VERSION",
+    "E0_SUMMARY_VERSION",
     "HandlerOutcome",
     "HandlerRegistry",
     "ForgetKnowledgeRebuilder",
@@ -157,6 +162,7 @@
     "RoleSettings",
     "SkeletonCheckSettings",
     "StructurerSettings",
+    "SummarySettings",
     "SyncCycleRunner",
     "SyncSettings",
     "UPLOAD_SOURCE_KIND",