Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions decisions.md
Original file line number Diff line number Diff line change
Expand Up @@ -2836,3 +2836,13 @@ shared tokenizer) with heading metadata (raw level, normalized title) exposed un
`blockizer_version` bump — never a second parallel parse. Scope fence: the check judges the
tree, not the text under it — intra-section reading-order scrambles are the conversion-layer
track (D38, issue #168). Design detail: `plan/designs/e0_files_design.md` §4.1.

**Amendment (2026-07-29, Wave-3 review of #165): the summary→prefix second-order channel.**
Summaries feed the E1 prefix input (as this decision mandates) and the stored prefix is a
quotable `added_context` element, so a summary-informed prefix is a bounded second-order path
by which non-neighbouring content can reach the grounding surface. Adjudicated: status quo on
quotability, widening of reach; accepted as layer-3/4 audit territory with an output-constraining
prefix instruction (describe location only, never restate a summary's assertions). Also
recorded: carried prefixes keep their summary generation until content re-chunks or the
prefixer version bumps — the no-fan-out corollary. Detail: `plan/designs/e0_files_design.md`
§4.1 consumption bullet.
15 changes: 14 additions & 1 deletion plan/designs/e0_files_design.md
Original file line number Diff line number Diff line change
Expand Up @@ -419,7 +419,20 @@ unchanged):
so a re-summarization — including an ancestor summary changing because a sibling leaf
was edited — never silently invalidates or re-extracts unchanged chunks. Better
summaries improve *future* extractions; they never fan out into document-wide
reprocessing.
reprocessing. Corollary (2026-07-29, Wave-3 review): the same no-fan-out rule means
better summaries reach the **prefix** only through re-chunked content or a
prefixer-version bump — a carried prefix keeps the summary generation it was written
under indefinitely, by design.

**Accepted second-order channel (2026-07-29, Wave-3 review).** Because summaries feed the
E1 prefix input and the stored prefix is itself a quotable `added_context` element, a
summary-informed prefix is a bounded second-order path by which content from
NON-neighbouring sections (a bottom-up parent one-liner distills sibling subtrees) can
reach the grounding surface — a reach the `_neighbour_text` same-scope rule deliberately
forbids for direct quotation. Status quo on *quotability* (the prefix was always LLM
text), a widening of *reach*. Accepted because the prefix's honesty was always layer-3/4
audit territory; mitigated in-prompt by instructing the prefixer to use summaries only to
DESCRIBE WHERE the passage sits, never to restate a summary's factual assertions.
- **Provenance and migration.** `structurer_version` splits into generations: skeleton
(deterministic parser + fallback contract), **skeleton check** (the sanity-check bullet
above), role pass, summary seat, and placement — each hash-stamped per D12. Structure/summary generations are immutable with a current pointer
Expand Down
5 changes: 5 additions & 0 deletions src/rememberstack/model/chunks.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ class SectionSpan(BaseModel):
role: str
block_start: int = Field(ge=0)
block_end: int = Field(ge=-1)
summary: str | None = None


class ChunkSource(BaseModel):
Expand Down Expand Up @@ -101,6 +102,10 @@ class ChunkForEmbedding(BaseModel):
extraction_input_hash: str
section_role: str
section_path: str
section_id: UUID | None = None
"""The section row the chunk was cut under — the cross-generation guard
input for summary orientation (optional: legacy constructors omit it and
the guard simply does not arm)."""
context_prefix: str | None
prefixer_version: str | None

Expand Down
5 changes: 3 additions & 2 deletions src/rememberstack/spine/chunk_catalog.py
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,8 @@ def record_embeddings(self, *, updates: tuple[EmbeddingUpdate, ...]) -> None:

_SELECT_SECTIONS = text(
"""
SELECT s.section_id, s.node_path, s.role, s.block_start, s.block_end
SELECT s.section_id, s.node_path, s.role, s.block_start, s.block_end,
s.summary
FROM document_sections s
JOIN document_representations r
ON r.representation_id = s.representation_id
Expand Down Expand Up @@ -200,7 +201,7 @@ def record_embeddings(self, *, updates: tuple[EmbeddingUpdate, ...]) -> None:
"""
SELECT c.chunk_id, c.doc_id, c.version_id, c.ordinal,
c.char_start, c.char_end, c.context_prefix, c.prefixer_version,
c.chunk_content_hash, c.extraction_input_hash,
c.chunk_content_hash, c.extraction_input_hash, c.section_id,
s.role AS section_role, s.node_path AS section_path
FROM chunks c
JOIN document_sections s ON s.section_id = c.section_id
Expand Down
62 changes: 49 additions & 13 deletions src/rememberstack/workers/e1.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,27 +43,35 @@
from rememberstack.ports.p1_index import ChunkIndexPort
from rememberstack.spine.chunk_catalog import ChunkCatalog
from rememberstack.workers.base import HandlerOutcome
from rememberstack.workers.section_orientation import render_section_orientation
from rememberstack.workers.section_orientation import SECTION_ORIENTATION_VERSION

E1_CHUNK_VERSION: Final = CHUNKER_VERSION
"""The chunk stage's component version IS the chunker version (D12/D58)."""

E1_EMBED_VERSION: Final = "e1-embed-2026.07"
"""The embed stage's component version (model identity rides settings/stamps)."""

E1_PREFIXER_VERSION: Final = "e1-prefix-2026.07b:temp0-1"
E1_PREFIXER_VERSION: Final = f"e1-prefix-2026.07c:temp0-1:{SECTION_ORIENTATION_VERSION}"
"""The context-prefix call's prompt generation (D58; conventional mode, D63).
07b pins temperature=0.0 — generation parameters are part of provenance."""
07b pins temperature=0.0 — generation parameters are part of provenance.
07c adds D79's bounded current-generation section-summary orientation."""

E2_EXTRACTOR_VERSION: Final = "e2-extract-2026.07e:loss-ledger-1"
E2_EXTRACTOR_VERSION: Final = (
f"e2-extract-2026.07f:loss-ledger-1:{SECTION_ORIENTATION_VERSION}"
)
"""The extractor generation baked into extraction_input_hash (D56); the E2
stage (WP-1.3) binds its handler to this same constant. 07e ledgers Claimify
omissions and grounding-gate rejections on the D33 transcript (#161); 07d
pinned temperature=0.0 on the Selection call (Claimify already carried it)."""
stage (WP-1.3) binds its handler to this same constant. 07f adds D79 summary
orientation to the bundle without making summaries hash or grounding inputs;
07e ledgers Claimify omissions and grounding-gate rejections on the D33
transcript (#161); 07d pinned temperature=0.0 on the Selection call (Claimify
already carried it)."""

_PREFIX_PROMPT_TEMPLATE: Final = (
"In one sentence, state where this passage sits in the document — "
"document title, section, and what surrounds it. Passage from "
"{title!r}, section path {section_path}, chunk {ordinal}:\n\n{head}"
"{title!r}, section path {section_path}, chunk {ordinal}"
"{section_orientation}:\n\n{head}"
)


Expand Down Expand Up @@ -317,12 +325,7 @@ def _resolve_prefix(
if carried is not None:
return carried.context_prefix
head = document_md[chunk.char_start : chunk.char_end][:400]
prompt = _PREFIX_PROMPT_TEMPLATE.format(
title=source.title or "untitled",
section_path=chunk.section_path,
ordinal=chunk.ordinal,
head=head,
)
prompt = _prefix_prompt(source=source, chunk=chunk, head=head)
response = self._model_provider.generate(
request=ModelRequest(
model=self._settings.prefix_model, prompt=prompt, temperature=0.0
Expand All @@ -335,6 +338,39 @@ def _resolve_prefix(
return response.output.prefix


def _prefix_prompt(*, source: ChunkSource, chunk: ChunkForEmbedding, head: str) -> str:
"""Render the prefix input; a degraded generation contributes zero bytes.

The instruction constrains the prefixer's OUTPUT, not just its reading:
summaries are abstractive LLM text, and the stored prefix is a quotable
added_context element downstream — a prefix that restates a summary's
claim would launder second-order content into the grounding surface
(review finding; accepted residual is location-description only).
"""
orientation = render_section_orientation(
sections=source.sections,
target_path=chunk.section_path,
target_section_id=chunk.section_id,
)
section_orientation = (
""
if orientation is None
else (
"\nSECTION SUMMARIES (background only — use them to describe"
" WHERE the passage sits; never restate or assert a fact from a"
" summary in your sentence):\n"
f"{orientation}"
)
)
return _PREFIX_PROMPT_TEMPLATE.format(
title=source.title or "untitled",
section_path=chunk.section_path,
ordinal=chunk.ordinal,
section_orientation=section_orientation,
head=head,
)


def _chunk_record(
*,
source: ChunkSource,
Expand Down
16 changes: 13 additions & 3 deletions src/rememberstack/workers/e2.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@
from rememberstack.workers.base import HandlerOutcome
from rememberstack.workers.e1 import E2_EXTRACTOR_VERSION
from rememberstack.workers.e3 import E3_NORMALIZER_VERSION
from rememberstack.workers.section_orientation import render_section_orientation

_logger = logging.getLogger(__name__)

Expand All @@ -73,6 +74,7 @@
source_span must be a verbatim substring of the target chunk. Report one
outcome per candidate, exactly one of: {outcomes}. The drop_* values carry the
reason in the value itself; there is no separate reason field.
SECTION SUMMARIES are orientation only and are never quotable source text.

{bundle}"""

Expand All @@ -85,9 +87,10 @@
candidate. For each claim return: claim_text (standalone), source_span (the
verbatim chunk substring it derives from), added_context (every substring you
ADDED, each tagged header|neighbour|prefix with the exact text as it appears
in that bundle element), entailment_self_verdict (does chunk+bundle entail the
claim), is_attributed. When the source states or implies WHEN a fact holds or
happened, resolve relative dates USING ONLY THE BUNDLE (as with
in that bundle element; SECTION SUMMARIES are orientation only, never quotable
and never an added_context source), entailment_self_verdict (does chunk+bundle
entail the claim), is_attributed. When the source states or implies WHEN a fact
holds or happened, resolve relative dates USING ONLY THE BUNDLE (as with
decontextualization) and emit valid_kind, valid_from_iso, valid_until_iso, and
valid_precision. Use ISO-8601 dates (YYYY-MM-DD) or datetimes WITH an explicit
offset or Z; never emit a datetime without an offset. Otherwise leave
Expand Down Expand Up @@ -547,9 +550,16 @@ def _bundle_text(
) -> str:
"""Assemble the D31 context bundle for one target chunk."""
chunk = chunks[index]
summaries = render_section_orientation(
sections=source.sections,
target_path=chunk.section_path,
target_section_id=chunk.section_id,
)
return (
f"DOCUMENT HEADER: {_header_text(source=source)}\n"
f"SECTION: path {chunk.section_path}, role {chunk.section_role}\n"
"SECTION SUMMARIES (orientation only; never quote as source):\n"
f"{summaries or '(none)'}\n"
f"CONTEXT PREFIX: {chunk.context_prefix or '(none)'}\n"
f"PREVIOUS CHUNK:\n{_neighbour_text(chunks=chunks, index=index - 1, document_md=document_md, section_path=chunk.section_path)}\n"
f"NEXT CHUNK:\n{_neighbour_text(chunks=chunks, index=index + 1, document_md=document_md, section_path=chunk.section_path)}\n"
Expand Down
98 changes: 98 additions & 0 deletions src/rememberstack/workers/section_orientation.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
"""Bounded D79 section-summary orientation shared by E1 and E2."""

from typing import Final
from uuid import UUID

from rememberstack.model import SectionSpan

SECTION_ORIENTATION_MAX_CHARS: Final = 2_048
"""Hard cap for the complete target + ancestor summary rendering."""

SECTION_ORIENTATION_VERSION: Final = (
"d79-section-orientation-v1:"
f"max-chars{SECTION_ORIENTATION_MAX_CHARS}:target-first:unicode-ellipsis"
)
"""Pins ordering and truncation semantics for E1/E2 provenance versions."""


def render_section_orientation(
*,
sections: tuple[SectionSpan, ...],
target_path: str,
target_section_id: UUID | None = None,
) -> str | None:
"""Render non-null target/ancestor summaries under one hard character cap.

The target is first so the most local orientation cannot be crowded out by
a deep ancestor chain. Ancestors then run nearest-first. Missing summaries
contribute no line; an entirely degraded generation returns ``None``.

``target_section_id`` is the cross-generation guard (review finding): the
chunk grid carries the section ids of the generation it was cut under,
while ``sections`` is always the CURRENT generation. When the caller
passes the chunk's section id and the current generation's row at
``target_path`` is a different section, the whole rendering degrades to
``None`` — a path that merely coincides across a skeleton regeneration
must not attach another section's summaries. Rendering semantics for
matching generations are unchanged, so the contract version holds.
"""
by_path = {section.node_path: section for section in sections}
target_row = by_path.get(target_path)
if (
target_section_id is not None
and target_row is not None
and target_row.section_id != target_section_id
):
return None
paths = _target_and_ancestor_paths(target_path=target_path)
lines = tuple(
f"{'TARGET' if index == 0 else 'ANCESTOR'} {path}: {summary}"
for index, path in enumerate(paths)
if (section := by_path.get(path)) is not None
and (summary := _one_line(section.summary)) is not None
)
if not lines:
return None
return _bounded_lines(lines=lines)


def _target_and_ancestor_paths(*, target_path: str) -> tuple[str, ...]:
"""Return the target followed by its materialized-path ancestors."""
paths = [target_path]
while "." in paths[-1]:
paths.append(paths[-1].rsplit(".", 1)[0])
return tuple(paths)


def _one_line(value: str | None) -> str | None:
"""Normalize defensive legacy values; D79 outputs are already one-line."""
if value is None:
return None
normalized = " ".join(value.split())
return normalized or None


def _bounded_lines(*, lines: tuple[str, ...]) -> str:
"""Join full lines until the cap, ellipsizing the final fitting fragment."""
rendered: list[str] = []
used = 0
for line in lines:
separator_chars = 1 if rendered else 0
remaining = SECTION_ORIENTATION_MAX_CHARS - used - separator_chars
if remaining <= 0:
break
if len(line) <= remaining:
rendered.append(line)
used += separator_chars + len(line)
continue
rendered.append(_ellipsize(value=line, max_chars=remaining))
break
return "\n".join(rendered)


def _ellipsize(*, value: str, max_chars: int) -> str:
if len(value) <= max_chars:
return value
if max_chars <= 1:
return "…"[:max_chars]
return value[: max_chars - 1] + "…"
Loading
Loading