From d1fd44bf47c3d1a736846942d4f088b584374d59 Mon Sep 17 00:00:00 2001 From: Jiri Puc Date: Fri, 24 Jul 2026 01:33:35 +0200 Subject: [PATCH 1/6] feat: compose full pipeline and LoCoMo system benchmark --- .env.example | 9 +- .github/workflows/ci.yml | 47 +- README.md | 7 +- benchmarks/locomo/README.md | 69 +- benchmarks/locomo/cli.py | 23 +- benchmarks/locomo/model.py | 60 +- benchmarks/locomo/protocol.py | 99 ++- benchmarks/locomo/runner.py | 554 ++++++++++----- compose.yaml | 76 ++ decisions.md | 96 ++- plan/analysis/locomo_benchmark_analysis.md | 450 ++++-------- plan/designs/locomo_benchmark_design.md | 670 +++++------------- plan/plans/phase-8-benchmarks.md | 20 +- src/rememberstack/adapters/openrouter.py | 18 +- src/rememberstack/client.py | 8 + src/rememberstack/core/recipe_linter.py | 3 + src/rememberstack/model/__init__.py | 10 + src/rememberstack/model/client.py | 53 ++ src/rememberstack/model/model_provider.py | 9 + src/rememberstack/profiles/selfhost.py | 339 ++++++++- src/rememberstack/spine/__init__.py | 6 + src/rememberstack/spine/readiness.py | 153 ++++ src/rememberstack/spine/recipes.py | 103 ++- src/rememberstack/surfaces/http_api.py | 42 ++ src/rememberstack/surfaces/recipe_executor.py | 36 +- src/rememberstack/surfaces/recipe_surface.py | 39 +- src/rememberstack/surfaces/sdk.py | 18 + src/rememberstack/workers/base.py | 22 + src/rememberstack/workers/e0.py | 9 + src/rememberstack/workers/e1.py | 44 +- src/rememberstack/workers/e2.py | 41 +- src/rememberstack/workers/e3.py | 95 +-- src/rememberstack/workers/p2.py | 13 +- src/rememberstack/workers/reconcile.py | 27 +- src/tests/adapters/test_openrouter.py | 44 +- src/tests/benchmarks/test_locomo_protocol.py | 100 ++- src/tests/benchmarks/test_locomo_runner.py | 358 ++++++---- src/tests/packaging/test_client_wheel.py | 3 +- src/tests/profiles/test_selfhost_profile.py | 54 ++ src/tests/spine/test_pipeline_readiness.py | 217 ++++++ src/tests/spine/test_work_ledger.py | 61 ++ src/tests/surfaces/test_client_sdk.py | 31 + src/tests/surfaces/test_recipe_registry.py | 37 + src/tests/surfaces/test_retrieval_api.py | 12 + src/tests/workers/test_e3_chain.py | 54 ++ .../workers/test_lifecycle_reconciliation.py | 6 +- src/tests/workers/test_p2_rebuild.py | 8 +- website/src/app/docs/deployment/page.mdx | 53 +- website/src/app/docs/project-status/page.mdx | 2 +- website/src/app/docs/reference/api/page.mdx | 13 + 50 files changed, 2864 insertions(+), 1457 deletions(-) create mode 100644 src/rememberstack/spine/readiness.py create mode 100644 src/tests/profiles/test_selfhost_profile.py create mode 100644 src/tests/spine/test_pipeline_readiness.py diff --git a/.env.example b/.env.example index dfd7824b..74f4f63c 100644 --- a/.env.example +++ b/.env.example @@ -11,6 +11,11 @@ REMEMBERSTACK_SELFHOST_DEPLOYMENT_SLUG=local REMEMBERSTACK_SELFHOST_DEPLOYMENT_NAME=Local memory REMEMBERSTACK_SELFHOST_API_PORT=8000 -# Required to start the provider adapter. The short Markdown smoke path makes no -# provider request; replace this value before processing a real corpus. +# Required to start the provider adapter. Replace this value before processing +# a corpus; the complete Compose pipeline makes extraction and embedding calls. REMEMBERSTACK_OPENROUTER_API_KEY=replace-before-real-use + +# Optional benchmark/deployment model overrides. Keep explicit model IDs for a +# reproducible run; do not use a rotating router such as openrouter/free. +# REMEMBERSTACK_E2_EXTRACT_MODEL=nvidia/nemotron-3-super-120b-a12b:free +# REMEMBERSTACK_E3_NORMALIZE_MODEL=nvidia/nemotron-3-super-120b-a12b:free diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e2f7fd5a..9b8424d0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,28 +53,49 @@ jobs: raise RuntimeError("MinIO replaced an immutable object key") PY - - name: Ingest and observe the E0 chain + - name: Prove the zero-cost full continuous pipeline run: | - curl --fail --data-binary $'# Hello\n\nCompose smoke.\n' \ - 'http://localhost:8000/ingest?filename=smoke.md&mime=text%2Fmarkdown' + running_services="$( + docker compose --env-file .env.example ps --status running --services + )" + for service in \ + api worker-convert worker-structure worker-chunk worker-embed-chunk \ + worker-extract-claims worker-normalize-relations \ + worker-adjudicate-supersession worker-embed-claim worker-reconcile \ + worker-label-relation; do + printf '%s\n' "$running_services" | grep -qx "$service" + done + ingested="$( + curl --fail --data-binary '' \ + 'http://localhost:8000/ingest?filename=empty.md&mime=text%2Fmarkdown' + )" + version_id="$( + printf '%s' "$ingested" | + docker compose --env-file .env.example exec -T api \ + python -c 'import json, sys; print(json.load(sys.stdin)["version_id"])' + )" observed='' - for attempt in {1..30}; do + for attempt in {1..60}; do observed="$(docker compose --env-file .env.example exec -T postgres \ psql -U rememberstack -d rememberstack -Atc \ - "SELECT count(*) FILTER (WHERE stage='convert' AND status='succeeded'), count(*) FILTER (WHERE stage='structure' AND status='succeeded'), count(*) FILTER (WHERE stage='chunk' AND status='pending') FROM processing_state")" - if [ "$observed" = '1|1|1' ]; then + "SELECT count(*), count(*) FILTER (WHERE status='succeeded'), count(DISTINCT stage) FROM processing_state WHERE target_id='$version_id'")" + if [ "$observed" = '10|10|10' ]; then break fi sleep 1 done - test "$observed" = '1|1|1' + test "$observed" = '10|10|10' test "$(docker compose --env-file .env.example exec -T postgres \ - psql -U rememberstack -d rememberstack -Atc 'SELECT count(*) FROM cost_ledger')" = '0' + psql -U rememberstack -d rememberstack -Atc \ + 'SELECT count(*) FROM cost_ledger')" = '0' - name: Stage an upgrade from the prior schema run: | docker compose --env-file .env.example stop \ - api worker-convert worker-structure + api worker-convert worker-structure worker-chunk worker-embed-chunk \ + worker-extract-claims worker-normalize-relations \ + worker-adjudicate-supersession worker-embed-claim worker-reconcile \ + worker-label-relation docker compose --env-file .env.example run --rm --no-deps \ --entrypoint alembic setup downgrade p7_02_0016 test "$(docker compose --env-file .env.example exec -T postgres \ @@ -116,7 +137,7 @@ jobs: docker compose "${compose_args[@]}" ps --status running --services )" test -z "$(printf '%s\n' "$running_services" \ - | grep -E '^(api|worker-convert|worker-structure)$' || true)" + | grep -E '^(api|worker-(convert|structure|chunk|embed-chunk|extract-claims|normalize-relations|adjudicate-supersession|embed-claim|reconcile|label-relation))$' || true)" test "$(docker compose "${compose_args[@]}" exec -T postgres \ psql -U rememberstack -d rememberstack -Atc \ 'SELECT version_num FROM alembic_version')" = 'p7_02_0016' @@ -133,7 +154,11 @@ jobs: running_services="$( docker compose "${compose_args[@]}" ps --status running --services )" - for service in api worker-convert worker-structure; do + for service in \ + api worker-convert worker-structure worker-chunk worker-embed-chunk \ + worker-extract-claims worker-normalize-relations \ + worker-adjudicate-supersession worker-embed-claim worker-reconcile \ + worker-label-relation; do printf '%s\n' "$running_services" | grep -qx "$service" done curl --fail http://localhost:8000/healthz diff --git a/README.md b/README.md index 10779dd5..4681d649 100644 --- a/README.md +++ b/README.md @@ -11,10 +11,11 @@ at a million documents. > **Pre-release software.** Phases 0–7 are implemented and tested. The public > [`v0.1.0`](https://github.com/writeitai/remember-stack/releases/tag/v0.1.0) release is available > from PyPI and GHCR; release automation, trusted publishing, tag protection, and bounded -> contributor-agreement enforcement are in place. The fresh-deployment Docker Compose skeleton +> contributor-agreement enforcement are in place. The fresh-deployment Docker Compose profile > is documented under -> [Self-host deployment](website/src/app/docs/deployment/page.mdx); it proves PostgreSQL, MinIO, -> API ingestion, and the first two E0 worker stages, not a production rollout. The build follows +> [Self-host deployment](website/src/app/docs/deployment/page.mdx); it composes PostgreSQL, +> MinIO, the API, all ten continuous E/P1 routes, and explicit P2/P3 publication for one +> deployment. It remains pre-release software, not a production rollout. The build follows > [plan/plans/roadmap.md](plan/plans/roadmap.md). ## TL;DR diff --git a/benchmarks/locomo/README.md b/benchmarks/locomo/README.md index 57225490..5b204848 100644 --- a/benchmarks/locomo/README.md +++ b/benchmarks/locomo/README.md @@ -1,17 +1,16 @@ -# RS-LoCoMo-v1 setup +# RS-LoCoMo-Full-v1 setup -This directory implements the reviewed `RS-LoCoMo-v1 J@30` protocol. It does not contain the -LoCoMo data and does not auto-download it. The data is CC BY-NC 4.0; confirm the intended use -before obtaining it from the -[official repository](https://github.com/snap-research/locomo/tree/3eb6f2c585f5e1699204e3c3bdf7adc5c28cb376/data). +This directory contains the unshipped full-system LoCoMo adapter. It does not vendor or +auto-download LoCoMo. Supply the exact pinned `locomo10.json` only after confirming its +CC BY-NC 4.0 terms. -Install the repository and the two deterministic-scorer dependencies: +Install the repository plus deterministic scorer dependencies: ```bash uv sync --extra benchmark ``` -The safe first command is local: +The safe first command is local and makes no API or model call: ```bash uv run --extra benchmark python -m benchmarks.locomo prepare \ @@ -20,27 +19,35 @@ uv run --extra benchmark python -m benchmarks.locomo prepare \ --output .benchmark-runs/locomo-smoke ``` -`prepare` checks the exact pinned SHA, validates the committed eight-question smoke manifest, -renders session Markdown, and writes a call/document plan. It does not contact RememberStack or -an evaluator. - -Do not proceed to `ingest`, `answer`, or `judge` until the owner walkthrough in -[`locomo_benchmark_design.md`](../../plan/designs/locomo_benchmark_design.md#12-pre-run-checklist). -Those stages require `--execute` plus exact isolation/readiness acknowledgements and explicit -document, question, call, and shared evaluator-cost ceilings. - -Question and call ceilings are run-absolute, not allowances for one sample invocation. -`--max-questions` must therefore cover the complete prepared tier (8 for smoke, 200 for -development, or 1,540 for publication), while reader and judge call ceilings include calls -already checkpointed for earlier samples. The evaluator-cost ceiling likewise covers the shared -reader-plus-judge ledger for the entire run. - -The local ledger records provider usage attached to successfully parsed, checkpointed calls. -Provider-billed calls that fail before usable accounting is returned are not reconstructable, and -a process death after a response but before its atomic checkpoint may repeat that one call. -Provider/account hard limits are therefore the hard monetary boundary; the harness ceilings are -additional fail-closed operational guards. - -The released Compose profile is not yet a valid target: it wires only `convert` and `structure`, -not the complete claim-indexing path. Use of a complete isolated deployment is a pre-run -prerequisite, not something this harness silently constructs. +The harness validates the pinned bytes, renders session documents, and fingerprints the +eight-question smoke plan. Do not run remote stages until reviewing +[`locomo_benchmark_design.md`](../../plan/designs/locomo_benchmark_design.md). + +The stock Compose deployment now includes all ten continuous E/P1 workers. After ingesting one +isolated conversation and waiting for them to settle, publish the aggregate projections once: + +```bash +docker compose --profile operations run --rm projections +``` + +The `answer` command then calls the public readiness endpoint. It refuses to run unless every +requested version completed the exact composed stage generations and both P2/P3 builds began +after that work completed. It also refuses a changed public recipe catalog. There is no manual “index ready” +acknowledgement. + +Readiness also records the API process's current non-secret model configuration for operator +review. Those values are not processing-time provenance; freeze one Compose environment for the +run and retain the provider/cost artifacts. + +The primary protocol uses a bounded answer agent over normal public recipes, not hard-coded claim +search. Limits are run-absolute: allow up to nine agent calls per selected question and one judge +call per answer. The shared evaluator-cost value is a reported-spend stop threshold: a completed +call can cross it, is recorded, and stops the run. Use the provider account cap as the hard +monetary boundary. If that leaves later questions unanswered, they remain visible as zero-scored +missing records; resuming them requires an explicitly higher threshold. + +P3 is built and freshness-checked as part of the ordinary deployment, but the remote recipe +agent has no filesystem mount. This protocol therefore does not attribute answer quality to P3 +navigation. A future mount-enabled protocol needs a new fingerprint and name. + +No real benchmark has been run as part of the setup implementation. diff --git a/benchmarks/locomo/cli.py b/benchmarks/locomo/cli.py index 6af790d5..3375cc57 100644 --- a/benchmarks/locomo/cli.py +++ b/benchmarks/locomo/cli.py @@ -1,4 +1,4 @@ -"""Command line for the deliberately staged RS-LoCoMo-v1 harness.""" +"""Command line for the deliberately staged full-system LoCoMo harness.""" from __future__ import annotations @@ -51,10 +51,9 @@ def main(argv: list[str] | None = None) -> int: run_dir=args.run, sample_id=args.sample, max_questions=args.max_questions, - max_reader_calls=args.max_reader_calls, + max_agent_calls=args.max_agent_calls, max_evaluator_cost_usd=args.max_evaluator_cost_usd, execute=args.execute, - index_ready_confirmation=args.confirm_index_ready, client=client, provider=provider, ) @@ -89,7 +88,7 @@ def _provider() -> OpenRouterModelProvider: def _positive_decimal(value: str) -> Decimal: - """Parse one strictly positive CLI monetary ceiling.""" + """Parse one strictly positive reported-spend stop threshold.""" try: parsed = Decimal(value) except InvalidOperation as error: @@ -104,7 +103,7 @@ def _parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="python -m benchmarks.locomo", description=( - "RS-LoCoMo-v1: prepare is local; ingest/answer/judge require " + "RS-LoCoMo-Full-v1: prepare is local; ingest/answer/judge require " "explicit execution acknowledgements" ), ) @@ -128,7 +127,7 @@ def _parser() -> argparse.ArgumentParser: ingest.add_argument("--confirm-isolated-deployment") answer = commands.add_parser( - "answer", help="retrieve and call the frozen reader for one sample" + "answer", help="run the bounded public-recipe answer agent for one sample" ) _run_and_sample(answer) answer.add_argument( @@ -138,19 +137,20 @@ def _parser() -> argparse.ArgumentParser: help="run-absolute authorization; must cover the prepared tier item count", ) answer.add_argument( - "--max-reader-calls", + "--max-agent-calls", type=int, required=True, - help="run-absolute ceiling over reader calls already recorded plus new calls", + help="run-absolute ceiling over answer-agent model calls; recipe calls" + " have a separate per-question cap", ) answer.add_argument( "--max-evaluator-cost-usd", type=_positive_decimal, required=True, - help="run-absolute shared reader-plus-judge reported-cost ceiling", + help="run-absolute shared reported-spend stop threshold; use a provider" + " account cap as the hard monetary boundary", ) answer.add_argument("--execute", action="store_true") - answer.add_argument("--confirm-index-ready") judge = commands.add_parser( "judge", help="call the frozen judge for one sample's answers" @@ -166,7 +166,8 @@ def _parser() -> argparse.ArgumentParser: "--max-evaluator-cost-usd", type=_positive_decimal, required=True, - help="run-absolute shared reader-plus-judge reported-cost ceiling", + help="run-absolute shared reported-spend stop threshold; use a provider" + " account cap as the hard monetary boundary", ) judge.add_argument("--execute", action="store_true") diff --git a/benchmarks/locomo/model.py b/benchmarks/locomo/model.py index 5e4cd2bf..57d3008d 100644 --- a/benchmarks/locomo/model.py +++ b/benchmarks/locomo/model.py @@ -1,4 +1,4 @@ -"""Typed dataset, protocol, checkpoint, and result values for RS-LoCoMo-v1.""" +"""Typed values for the full-system RS-LoCoMo-Full-v1 protocol.""" from __future__ import annotations @@ -13,6 +13,8 @@ from pydantic import Field from pydantic import model_validator +from rememberstack.model import Envelope +from rememberstack.model import PipelineReadinessReport from rememberstack.model import ProviderCallUsage NonEmpty = Annotated[str, Field(min_length=1)] @@ -20,7 +22,7 @@ RetainedCategory = Literal[1, 2, 3, 4] Tier = Literal["smoke", "development", "publication"] FailureKind = Literal[ - "retrieval", "reader", "judge", "accounting", "invalid_response", "missing" + "readiness", "tool", "reader", "judge", "accounting", "invalid_response", "missing" ] @@ -104,7 +106,7 @@ class QuestionManifest(FrozenModel): class RunConfiguration(FrozenModel): """Immutable identity of one prepared benchmark run.""" - protocol_name: Literal["RS-LoCoMo-v1"] = "RS-LoCoMo-v1" + protocol_name: Literal["RS-LoCoMo-Full-v1"] = "RS-LoCoMo-Full-v1" adapter_version: NonEmpty prepared_at: datetime repository_revision: NonEmpty @@ -117,15 +119,18 @@ class RunConfiguration(FrozenModel): documents_sha256: NonEmpty item_count: int = Field(ge=1) sample_ids: Annotated[tuple[NonEmpty, ...], Field(min_length=1)] - top_k: Literal[30] = 30 - reader_model: Literal["openai/gpt-4o-mini"] = "openai/gpt-4o-mini" + max_tool_calls_per_question: Literal[8] = 8 + max_agent_calls_per_question: Literal[9] = 9 + knowledge_mode: Literal["not_composed"] = "not_composed" + answer_agent_model: Literal["openai/gpt-4o-mini"] = "openai/gpt-4o-mini" judge_model: Literal["openai/gpt-4o-mini"] = "openai/gpt-4o-mini" - reader_temperature: float = Field(default=0.0, ge=0, le=2) + answer_agent_temperature: float = Field(default=0.0, ge=0, le=2) judge_temperature: float = Field(default=0.0, ge=0, le=2) judge_repetitions: Literal[1] = 1 - reader_prompt_sha256: NonEmpty + tool_catalog_sha256: NonEmpty + answer_prompt_sha256: NonEmpty judge_prompt_sha256: NonEmpty - reader_schema_sha256: NonEmpty + answer_schema_sha256: NonEmpty judge_schema_sha256: NonEmpty protocol_fingerprint: NonEmpty @@ -182,10 +187,32 @@ class BenchmarkFailure(FrozenModel): message: NonEmpty -class ReaderOutput(FrozenModel): - """The complete strict reader response: one non-empty answer.""" +class AnswerAgentStep(FrozenModel): + """One bounded answer-agent decision: call a public recipe or finish.""" - answer: NonEmpty + action: Literal["tool", "answer"] + tool_name: str | None = None + arguments: dict[str, object] = Field(default_factory=dict) + answer: str | None = None + + @model_validator(mode="after") + def require_one_action_shape(self) -> "AnswerAgentStep": + """Make tool and answer decisions mutually exclusive and complete.""" + if self.action == "tool": + if not self.tool_name or self.answer is not None: + raise ValueError("a tool step requires tool_name and no answer") + elif not self.answer or self.tool_name is not None or self.arguments: + raise ValueError("an answer step requires only a non-empty answer") + return self + + +class ToolCallRecord(FrozenModel): + """One ordinary public recipe call and its complete response envelope.""" + + name: NonEmpty + arguments: dict[str, object] + latency_ms: int = Field(ge=0) + response: Envelope class JudgeOutput(FrozenModel): @@ -195,7 +222,7 @@ class JudgeOutput(FrozenModel): class AnswerRecord(FrozenModel): - """Retrieval plus reader outcome for one manifest item.""" + """Bounded public-tool trace plus answer-agent outcome for one item.""" item_id: NonEmpty sample_id: NonEmpty @@ -204,10 +231,12 @@ class AnswerRecord(FrozenModel): gold_answer: str gold_evidence: tuple[str, ...] claims: tuple[RetrievedClaim, ...] = () + tool_calls: tuple[ToolCallRecord, ...] = () dropped_by_hydration: int = Field(default=0, ge=0) retrieval_succeeded: bool retrieval_latency_ms: int = Field(ge=0) reader_called: bool + agent_call_count: int = Field(default=0, ge=0) reader_latency_ms: int | None = Field(default=None, ge=0) generated_answer: str | None = None reader_usage: ProviderCallUsage | None = None @@ -224,6 +253,8 @@ def require_answer_xor_failure(self) -> "AnswerRecord": raise ValueError("a generated answer requires a reader call") if self.reader_usage is not None and not self.reader_called: raise ValueError("reader usage requires a reader call") + if self.reader_called != (self.agent_call_count > 0): + raise ValueError("reader_called must match agent_call_count") return self @@ -253,6 +284,7 @@ class RunState(BaseModel): model_config = ConfigDict(extra="forbid") ingests: dict[str, IngestRecord] = Field(default_factory=dict) + readiness: dict[str, PipelineReadinessReport] = Field(default_factory=dict) answers: dict[str, AnswerRecord] = Field(default_factory=dict) judges: dict[str, JudgeRecord] = Field(default_factory=dict) evaluator_cost_usd: Decimal = Field(default=Decimal(0), ge=Decimal(0)) @@ -283,7 +315,7 @@ class SessionDiagnosticSummary(FrozenModel): class RunSummary(FrozenModel): """Publication-ready local aggregate with no hidden denominator.""" - protocol_name: Literal["RS-LoCoMo-v1"] = "RS-LoCoMo-v1" + protocol_name: Literal["RS-LoCoMo-Full-v1"] = "RS-LoCoMo-Full-v1" protocol_fingerprint: NonEmpty tier: Tier questions: int = Field(ge=1) @@ -293,7 +325,7 @@ class RunSummary(FrozenModel): categories: tuple[CategorySummary, ...] session_diagnostic: SessionDiagnosticSummary failures: dict[str, int] - reader_calls: int = Field(ge=0) + answer_agent_calls: int = Field(ge=0) judge_calls: int = Field(ge=0) tokens_in: int = Field(ge=0) tokens_out: int = Field(ge=0) diff --git a/benchmarks/locomo/protocol.py b/benchmarks/locomo/protocol.py index 3506f2c3..9111b4d6 100644 --- a/benchmarks/locomo/protocol.py +++ b/benchmarks/locomo/protocol.py @@ -1,4 +1,4 @@ -"""Pure RS-LoCoMo-v1 rendering, prompts, diagnostics, and scoring.""" +"""Pure full-system LoCoMo rendering, prompts, diagnostics, and scoring.""" from __future__ import annotations @@ -12,31 +12,66 @@ from nltk.stem import PorterStemmer import regex +from benchmarks.locomo.model import AnswerAgentStep from benchmarks.locomo.model import JudgeOutput from benchmarks.locomo.model import LoCoMoSample from benchmarks.locomo.model import LoCoMoSession -from benchmarks.locomo.model import ReaderOutput from benchmarks.locomo.model import RetainedCategory -from benchmarks.locomo.model import RetrievedClaim - -PROTOCOL_NAME: Final = "RS-LoCoMo-v1" -ADAPTER_VERSION: Final = "locomo-adapter-2026.07" -TOP_K: Final = 30 -READER_MODEL: Final = "openai/gpt-4o-mini" +from benchmarks.locomo.model import ToolCallRecord +from rememberstack.model import ToolDescriptor + +PROTOCOL_NAME: Final = "RS-LoCoMo-Full-v1" +ADAPTER_VERSION: Final = "locomo-full-adapter-2026.07" +MAX_TOOL_CALLS: Final = 8 +MAX_AGENT_CALLS: Final = 9 +EXPECTED_TOOL_CATALOG_SHA256: Final = ( + "96061015e1681877015027a8f4eccad0ceebec53d6d8dbade346ec0934a9024a" +) +EXPECTED_PIPELINE_STAGES: Final = ( + "convert", + "structure", + "chunk", + "embed_chunk", + "extract_claims", + "normalize_relations", + "adjudicate_supersession", + "embed_claim", + "reconcile", + "label_relation", +) +EXPECTED_PROJECTION_PLANES: Final = ("P2_graph", "P3_corpusfs") +ANSWER_AGENT_MODEL: Final = "openai/gpt-4o-mini" JUDGE_MODEL: Final = "openai/gpt-4o-mini" TEMPERATURE: Final = 0.0 -READER_PROMPT_TEMPLATE: Final = """Answer the question using only the ranked conversation memories below. -Use memory timestamps when present. Resolve relative time references to the -corresponding date, month, or year. If memories conflict, prefer the most recent -one. Do not confuse people mentioned in a memory with the conversation speakers. -If the memories do not contain the answer, answer "Unknown". -Return only a concise answer of at most six words. +ANSWER_AGENT_PROMPT_TEMPLATE: Final = """You answer a question using one ordinary +RememberStack deployment. You may call only the public recipe tools listed +below. Work as a normal memory agent: + +1. Orient: resolve names and inspect compiled/corpus or graph orientation when + useful. +2. Verify: query current fact tools for what holds now. +3. Audit: use evidence/hydration tools when wording, time, attribution, or + conflicts matter. + +Respect every response envelope's grain, negative, freshness, truncation, and +dropped_by_hydration fields. Evidence says what a source asserted; it is not +automatically current fact. Use timestamps to resolve relative dates. Do not +confuse people mentioned in a memory with the conversation speakers. Never use +outside knowledge. If the deployment does not contain the answer, finish with +"Unknown". A final answer must be concise and at most six words. + +Return one structured step: either action="tool" with one listed tool_name and +arguments, or action="answer" with the final answer. Never invent a tool. -Ranked memories: -{memories} +PUBLIC TOOLS: +{tools} -Question: {question}""" +TOOL TRACE SO FAR: +{trace} + +QUESTION: +{question}""" JUDGE_PROMPT_TEMPLATE: Final = """Classify the generated answer to the question as CORRECT or WRONG against the gold answer. Be generous about concise paraphrases that identify the same topic. @@ -90,14 +125,28 @@ def render_session(*, sample: LoCoMoSample, session: LoCoMoSession) -> str: return "\n".join(lines) + "\n" -def render_reader_prompt(*, question: str, claims: tuple[RetrievedClaim, ...]) -> str: - """Render rank-only claim text; no gold or reconstructed metadata.""" - memories = ( - "\n".join(f"[{claim.rank}] {claim.claim_text}" for claim in claims) - if claims - else "(none)" +def render_answer_agent_prompt( + *, + question: str, + tools: tuple[ToolDescriptor, ...], + trace: tuple[ToolCallRecord, ...], +) -> str: + """Render the frozen public tool catalog and trace, never gold annotations.""" + tool_payload = json.dumps( + [tool.model_dump(mode="json") for tool in tools], + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + trace_payload = json.dumps( + [record.model_dump(mode="json") for record in trace], + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + return ANSWER_AGENT_PROMPT_TEMPLATE.format( + tools=tool_payload, trace=trace_payload or "[]", question=question ) - return READER_PROMPT_TEMPLATE.format(memories=memories, question=question) def render_judge_prompt( @@ -114,7 +163,7 @@ def prompt_sha256(*, template: str) -> str: return hashlib.sha256(template.encode()).hexdigest() -def schema_sha256(*, model: type[ReaderOutput] | type[JudgeOutput]) -> str: +def schema_sha256(*, model: type[AnswerAgentStep] | type[JudgeOutput]) -> str: """Hash a canonical strict-output JSON schema.""" canonical = json.dumps( model.model_json_schema(), diff --git a/benchmarks/locomo/runner.py b/benchmarks/locomo/runner.py index 000ff304..1e86beea 100644 --- a/benchmarks/locomo/runner.py +++ b/benchmarks/locomo/runner.py @@ -1,4 +1,4 @@ -"""Staged, guarded, checkpointed execution for RS-LoCoMo-v1.""" +"""Staged, guarded execution for the full-system LoCoMo protocol.""" from __future__ import annotations @@ -25,6 +25,7 @@ from benchmarks.locomo.dataset import load_manifest from benchmarks.locomo.dataset import manifest_bytes_hash from benchmarks.locomo.dataset import validate_manifest +from benchmarks.locomo.model import AnswerAgentStep from benchmarks.locomo.model import AnswerRecord from benchmarks.locomo.model import BenchmarkFailure from benchmarks.locomo.model import CategorySummary @@ -36,33 +37,37 @@ from benchmarks.locomo.model import LoCoMoQuestion from benchmarks.locomo.model import PreparedDocument from benchmarks.locomo.model import QuestionManifest -from benchmarks.locomo.model import ReaderOutput from benchmarks.locomo.model import RetainedCategory from benchmarks.locomo.model import RetrievedClaim from benchmarks.locomo.model import RunConfiguration from benchmarks.locomo.model import RunState from benchmarks.locomo.model import RunSummary from benchmarks.locomo.model import SessionDiagnosticSummary +from benchmarks.locomo.model import ToolCallRecord from benchmarks.locomo.protocol import ADAPTER_VERSION +from benchmarks.locomo.protocol import ANSWER_AGENT_MODEL +from benchmarks.locomo.protocol import ANSWER_AGENT_PROMPT_TEMPLATE +from benchmarks.locomo.protocol import EXPECTED_PIPELINE_STAGES +from benchmarks.locomo.protocol import EXPECTED_PROJECTION_PLANES +from benchmarks.locomo.protocol import EXPECTED_TOOL_CATALOG_SHA256 from benchmarks.locomo.protocol import JUDGE_MODEL from benchmarks.locomo.protocol import JUDGE_PROMPT_TEMPLATE +from benchmarks.locomo.protocol import MAX_AGENT_CALLS +from benchmarks.locomo.protocol import MAX_TOOL_CALLS from benchmarks.locomo.protocol import official_f1 from benchmarks.locomo.protocol import prompt_sha256 from benchmarks.locomo.protocol import PROTOCOL_NAME -from benchmarks.locomo.protocol import READER_MODEL -from benchmarks.locomo.protocol import READER_PROMPT_TEMPLATE +from benchmarks.locomo.protocol import render_answer_agent_prompt from benchmarks.locomo.protocol import render_judge_prompt -from benchmarks.locomo.protocol import render_reader_prompt from benchmarks.locomo.protocol import render_session from benchmarks.locomo.protocol import schema_sha256 from benchmarks.locomo.protocol import session_diagnostic from benchmarks.locomo.protocol import TEMPERATURE -from benchmarks.locomo.protocol import TOP_K from rememberstack.adapters.openrouter import OpenRouterProviderError -from rememberstack.model import Grain from rememberstack.model import ModelRequest from rememberstack.model import ProviderAccountingError from rememberstack.model import ProviderCallUsage +from rememberstack.model import ToolDescriptor from rememberstack.ports import ModelProviderPort from rememberstack.surfaces.sdk import MemoryApiError from rememberstack.surfaces.sdk import MemoryClient @@ -111,15 +116,18 @@ def prepare_run(*, dataset_path: Path, tier: str, output: Path) -> RunConfigurat "documents_sha256": _models_hash(values=documents), "item_count": len(manifest.item_ids), "sample_ids": sample_ids, - "top_k": TOP_K, - "reader_model": READER_MODEL, + "max_tool_calls_per_question": MAX_TOOL_CALLS, + "max_agent_calls_per_question": MAX_AGENT_CALLS, + "knowledge_mode": "not_composed", + "answer_agent_model": ANSWER_AGENT_MODEL, "judge_model": JUDGE_MODEL, - "reader_temperature": TEMPERATURE, + "answer_agent_temperature": TEMPERATURE, "judge_temperature": TEMPERATURE, "judge_repetitions": 1, - "reader_prompt_sha256": prompt_sha256(template=READER_PROMPT_TEMPLATE), + "tool_catalog_sha256": EXPECTED_TOOL_CATALOG_SHA256, + "answer_prompt_sha256": prompt_sha256(template=ANSWER_AGENT_PROMPT_TEMPLATE), "judge_prompt_sha256": prompt_sha256(template=JUDGE_PROMPT_TEMPLATE), - "reader_schema_sha256": schema_sha256(model=ReaderOutput), + "answer_schema_sha256": schema_sha256(model=AnswerAgentStep), "judge_schema_sha256": schema_sha256(model=JudgeOutput), } configuration = RunConfiguration( @@ -212,10 +220,9 @@ def answer_sample( run_dir: Path, sample_id: str, max_questions: int, - max_reader_calls: int, + max_agent_calls: int, max_evaluator_cost_usd: Decimal, execute: bool, - index_ready_confirmation: str | None, client: MemoryClient, provider: ModelProviderPort, ) -> tuple[AnswerRecord, ...]: @@ -225,8 +232,8 @@ def answer_sample( context=context, execute=execute, sample_id=sample_id, - confirmation=index_ready_confirmation, - confirmation_name="confirm-index-ready", + confirmation=sample_id, + confirmation_name="sample", ) questions = _sample_questions(context=context, sample_id=sample_id) if max_questions < context.configuration.item_count: @@ -235,16 +242,58 @@ def answer_sample( f"{context.configuration.item_count}" ) _require_sample_ingested(context=context, sample_id=sample_id) + version_ids = tuple( + record.version_id + for record in context.state.ingests.values() + if record.sample_id == sample_id + ) + readiness = client.pipeline_readiness( + version_ids=version_ids, require_projections=True + ) + reported_versions = {version.version_id for version in readiness.versions} + complete_versions = all( + version.ready + and tuple(stage.stage for stage in version.stages) == EXPECTED_PIPELINE_STAGES + and all(stage.status in {"succeeded", "skipped"} for stage in version.stages) + for version in readiness.versions + ) + reported_planes = tuple(projection.plane for projection in readiness.projections) + complete_projections = reported_planes == EXPECTED_PROJECTION_PLANES and all( + projection.ready for projection in readiness.projections + ) + if ( + not readiness.ready + or reported_versions != set(version_ids) + or not complete_versions + or not complete_projections + ): + raise ExecutionGuardError( + "the deployment did not report the exact completed" + " RS-LoCoMo-Full-v1 pipeline and fresh P2/P3 projections" + ) + prior_readiness = context.state.readiness.get(sample_id) + if prior_readiness is not None and prior_readiness != readiness: + raise ExecutionGuardError( + "deployment readiness fingerprint changed after it was checkpointed" + ) + context.state.readiness[sample_id] = readiness + _save_state(run_dir=run_dir, state=context.state) + tools = client.recipes() + if _models_hash(values=tools) != context.configuration.tool_catalog_sha256: + raise ExecutionGuardError( + "deployment recipe catalog differs from the prepared protocol" + ) remaining = tuple( question for question in questions if question.item_id not in context.state.answers ) - called = sum(record.reader_called for record in context.state.answers.values()) - if called + len(remaining) > max_reader_calls: + called = sum(record.agent_call_count for record in context.state.answers.values()) + worst_case = called + len(remaining) * MAX_AGENT_CALLS + if worst_case > max_agent_calls: raise ExecutionGuardError( - f"max-reader-calls {max_reader_calls} cannot cover at most " - f"{called + len(remaining)} run calls" + f"max-agent-calls {max_agent_calls} cannot cover at most" + f" {worst_case} run calls" ) _require_cost_ceiling( spent=context.state.evaluator_cost_usd, ceiling=max_evaluator_cost_usd @@ -259,9 +308,10 @@ def answer_sample( question=question, client=client, provider=provider, + tools=tools, doc_sessions=doc_sessions, state=context.state, - max_reader_calls=max_reader_calls, + max_agent_calls=max_agent_calls, max_evaluator_cost_usd=max_evaluator_cost_usd, ) context.state.answers[question.item_id] = record @@ -424,8 +474,8 @@ def summarize_run(*, run_dir: Path) -> RunSummary: ), ), failures=dict(sorted(failures.items())), - reader_calls=sum( - record.reader_called for record in context.state.answers.values() + answer_agent_calls=sum( + record.agent_call_count for record in context.state.answers.values() ), judge_calls=sum( record.model_called for record in context.state.judges.values() @@ -538,7 +588,7 @@ def _validate_run( ) -> None: """Recompute immutable run identity before any local or remote stage.""" if configuration.dataset_sha256 != DATASET_SHA256: - raise BenchmarkRunError("run dataset hash is not RS-LoCoMo-v1") + raise BenchmarkRunError("run dataset hash is not RS-LoCoMo-Full-v1") if item_ids_hash(item_ids=manifest.item_ids) != manifest.item_ids_sha256: raise BenchmarkRunError("run manifest item hash changed") if manifest_bytes_hash(manifest=manifest) != configuration.manifest_sha256: @@ -548,14 +598,14 @@ def _validate_run( if manifest.tier != configuration.tier: raise BenchmarkRunError("run manifest tier changed") if configuration.dataset_commit != DATASET_COMMIT: - raise BenchmarkRunError("run dataset commit is not RS-LoCoMo-v1") + raise BenchmarkRunError("run dataset commit is not RS-LoCoMo-Full-v1") if configuration.adapter_version != ADAPTER_VERSION: raise BenchmarkRunError("run adapter version differs from current code") if ( - configuration.reader_temperature != TEMPERATURE + configuration.answer_agent_temperature != TEMPERATURE or configuration.judge_temperature != TEMPERATURE ): - raise BenchmarkRunError("run temperature differs from RS-LoCoMo-v1") + raise BenchmarkRunError("run temperature differs from RS-LoCoMo-Full-v1") if _models_hash(values=documents) != configuration.documents_sha256: raise BenchmarkRunError("prepared document manifest changed") if len(questions) != configuration.item_count: @@ -579,23 +629,27 @@ def _validate_run( "documents_sha256": configuration.documents_sha256, "item_count": configuration.item_count, "sample_ids": configuration.sample_ids, - "top_k": configuration.top_k, - "reader_model": configuration.reader_model, + "max_tool_calls_per_question": configuration.max_tool_calls_per_question, + "max_agent_calls_per_question": configuration.max_agent_calls_per_question, + "knowledge_mode": configuration.knowledge_mode, + "answer_agent_model": configuration.answer_agent_model, "judge_model": configuration.judge_model, - "reader_temperature": configuration.reader_temperature, + "answer_agent_temperature": configuration.answer_agent_temperature, "judge_temperature": configuration.judge_temperature, "judge_repetitions": configuration.judge_repetitions, - "reader_prompt_sha256": configuration.reader_prompt_sha256, + "tool_catalog_sha256": configuration.tool_catalog_sha256, + "answer_prompt_sha256": configuration.answer_prompt_sha256, "judge_prompt_sha256": configuration.judge_prompt_sha256, - "reader_schema_sha256": configuration.reader_schema_sha256, + "answer_schema_sha256": configuration.answer_schema_sha256, "judge_schema_sha256": configuration.judge_schema_sha256, } if _canonical_hash(base) != configuration.protocol_fingerprint: raise BenchmarkRunError("run protocol fingerprint changed") current_hashes = { - "reader_prompt_sha256": prompt_sha256(template=READER_PROMPT_TEMPLATE), + "tool_catalog_sha256": EXPECTED_TOOL_CATALOG_SHA256, + "answer_prompt_sha256": prompt_sha256(template=ANSWER_AGENT_PROMPT_TEMPLATE), "judge_prompt_sha256": prompt_sha256(template=JUDGE_PROMPT_TEMPLATE), - "reader_schema_sha256": schema_sha256(model=ReaderOutput), + "answer_schema_sha256": schema_sha256(model=AnswerAgentStep), "judge_schema_sha256": schema_sha256(model=JudgeOutput), } for field, actual in current_hashes.items(): @@ -756,128 +810,231 @@ def _answer_one( question: LoCoMoQuestion, client: MemoryClient, provider: ModelProviderPort, + tools: tuple[ToolDescriptor, ...], doc_sessions: dict[UUID, str], state: RunState, - max_reader_calls: int, + max_agent_calls: int, max_evaluator_cost_usd: Decimal, ) -> AnswerRecord: - """Retrieve and invoke the reader once, returning one terminal record.""" - started = time.monotonic_ns() - try: - envelope = client.search_claims(query=question.question, k=TOP_K) - except MemoryApiError as error: - return _failed_answer( - question=question, - kind="retrieval", - message=str(error), - retrieval_latency_ms=_elapsed_ms(started), - retrieval_succeeded=False, - reader_called=False, - ) - retrieval_latency = _elapsed_ms(started) - if envelope.grain is not Grain.EVIDENCE: - return _failed_answer( - question=question, - kind="invalid_response", - message=f"claim search returned grain {envelope.grain}", - retrieval_latency_ms=retrieval_latency, - retrieval_succeeded=False, - reader_called=False, - ) - if len(envelope.evidence) > TOP_K or any( - not claim.is_current_testimony for claim in envelope.evidence - ): - return _failed_answer( - question=question, - kind="invalid_response", - message=("claim search exceeded top-k or returned non-current testimony"), - retrieval_latency_ms=retrieval_latency, - retrieval_succeeded=False, - reader_called=False, - ) - claims = tuple( - RetrievedClaim( - rank=rank, - claim_id=claim.claim_id, - doc_id=claim.doc_id, - chunk_id=claim.chunk_id, - claim_text=claim.claim_text, - source_span=claim.source_span, - char_start=claim.char_start, - char_end=claim.char_end, - is_attributed=claim.is_attributed, - is_current_testimony=claim.is_current_testimony, - session_id=doc_sessions.get(claim.doc_id), - ) - for rank, claim in enumerate(envelope.evidence, start=1) - ) - called = sum(record.reader_called for record in state.answers.values()) - if called >= max_reader_calls: - raise ExecutionGuardError("reader call ceiling reached before next call") - _require_cost_before_call( - spent=state.evaluator_cost_usd, ceiling=max_evaluator_cost_usd - ) - prompt = render_reader_prompt(question=question.question, claims=claims) - reader_started = time.monotonic_ns() - try: - response = provider.generate( - request=ModelRequest( - model=READER_MODEL, prompt=prompt, temperature=TEMPERATURE - ), - response_type=ReaderOutput, - ) - except ProviderAccountingError as error: - return _failed_answer( - question=question, - kind="accounting", - message=str(error), - retrieval_latency_ms=retrieval_latency, - retrieval_succeeded=True, - reader_called=True, - reader_latency_ms=_elapsed_ms(reader_started), - claims=claims, - dropped_by_hydration=envelope.dropped_by_hydration, + """Let a bounded agent choose ordinary public recipes, then answer.""" + tool_names = {tool.name for tool in tools} + trace: list[ToolCallRecord] = [] + usages: list[ProviderCallUsage] = [] + agent_latency_ms = 0 + tool_latency_ms = 0 + prior_calls = sum(record.agent_call_count for record in state.answers.values()) + for _ in range(MAX_AGENT_CALLS): + if prior_calls + len(usages) >= max_agent_calls: + raise ExecutionGuardError( + "answer-agent call ceiling reached before next call" + ) + _require_cost_before_call( + spent=state.evaluator_cost_usd, ceiling=max_evaluator_cost_usd ) - except ValidationError as error: - return _failed_answer( - question=question, - kind="invalid_response", - message=str(error), - retrieval_latency_ms=retrieval_latency, - retrieval_succeeded=True, - reader_called=True, - reader_latency_ms=_elapsed_ms(reader_started), - claims=claims, - dropped_by_hydration=envelope.dropped_by_hydration, + prompt = render_answer_agent_prompt( + question=question.question, tools=tools, trace=tuple(trace) ) - except OpenRouterProviderError as error: - return _failed_answer( - question=question, - kind="reader", - message=str(error), - retrieval_latency_ms=retrieval_latency, - retrieval_succeeded=True, - reader_called=True, - reader_latency_ms=_elapsed_ms(reader_started), - claims=claims, - dropped_by_hydration=envelope.dropped_by_hydration, + started = time.monotonic_ns() + try: + response = provider.generate( + request=ModelRequest( + model=ANSWER_AGENT_MODEL, prompt=prompt, temperature=TEMPERATURE + ), + response_type=AnswerAgentStep, + ) + except ProviderAccountingError as error: + return _failed_answer( + question=question, + kind="accounting", + message=str(error), + retrieval_latency_ms=tool_latency_ms, + retrieval_succeeded=bool(trace), + agent_call_count=len(usages) + 1, + reader_latency_ms=agent_latency_ms + _elapsed_ms(started), + claims=_claims_from_trace( + trace=tuple(trace), doc_sessions=doc_sessions + ), + tool_calls=tuple(trace), + usages=tuple(usages), + ) + except ValidationError as error: + return _failed_answer( + question=question, + kind="invalid_response", + message=str(error), + retrieval_latency_ms=tool_latency_ms, + retrieval_succeeded=bool(trace), + agent_call_count=len(usages) + 1, + reader_latency_ms=agent_latency_ms + _elapsed_ms(started), + claims=_claims_from_trace( + trace=tuple(trace), doc_sessions=doc_sessions + ), + tool_calls=tuple(trace), + usages=tuple(usages), + ) + except OpenRouterProviderError as error: + if error.usage is not None: + usages.append(error.usage) + state.evaluator_cost_usd += error.usage.cost_usd + return _failed_answer( + question=question, + kind="reader", + message=str(error), + retrieval_latency_ms=tool_latency_ms, + retrieval_succeeded=bool(trace), + agent_call_count=( + len(usages) if error.usage is not None else len(usages) + 1 + ), + reader_latency_ms=agent_latency_ms + _elapsed_ms(started), + claims=_claims_from_trace( + trace=tuple(trace), doc_sessions=doc_sessions + ), + tool_calls=tuple(trace), + usages=tuple(usages), + ) + agent_latency_ms += _elapsed_ms(started) + usages.append(response.usage) + state.evaluator_cost_usd += response.usage.cost_usd + if state.evaluator_cost_usd > max_evaluator_cost_usd: + return _failed_answer( + question=question, + kind="accounting", + message=( + f"reported evaluator spend {state.evaluator_cost_usd} crossed" + f" stop threshold {max_evaluator_cost_usd}" + ), + retrieval_latency_ms=tool_latency_ms, + retrieval_succeeded=bool(trace), + agent_call_count=len(usages), + reader_latency_ms=agent_latency_ms, + claims=_claims_from_trace( + trace=tuple(trace), doc_sessions=doc_sessions + ), + tool_calls=tuple(trace), + usages=tuple(usages), + ) + step = response.output + if step.action == "answer": + if not trace: + return _failed_answer( + question=question, + kind="invalid_response", + message="answer agent finished without consulting RememberStack", + retrieval_latency_ms=tool_latency_ms, + retrieval_succeeded=False, + agent_call_count=len(usages), + reader_latency_ms=agent_latency_ms, + tool_calls=tuple(trace), + usages=tuple(usages), + ) + answer = step.answer or "" + if len(answer.split()) > 6: + return _failed_answer( + question=question, + kind="invalid_response", + message="answer agent exceeded the six-word answer limit", + retrieval_latency_ms=tool_latency_ms, + retrieval_succeeded=True, + agent_call_count=len(usages), + reader_latency_ms=agent_latency_ms, + claims=_claims_from_trace( + trace=tuple(trace), doc_sessions=doc_sessions + ), + tool_calls=tuple(trace), + usages=tuple(usages), + ) + claims = _claims_from_trace(trace=tuple(trace), doc_sessions=doc_sessions) + return AnswerRecord( + item_id=question.item_id, + sample_id=question.sample_id, + category=_retained_category(question=question), + question=question.question, + gold_answer=question.answer or "", + gold_evidence=question.evidence, + claims=claims, + tool_calls=tuple(trace), + dropped_by_hydration=sum( + call.response.dropped_by_hydration for call in trace + ), + retrieval_succeeded=True, + retrieval_latency_ms=tool_latency_ms, + reader_called=True, + agent_call_count=len(usages), + reader_latency_ms=agent_latency_ms, + generated_answer=answer, + reader_usage=_aggregate_usage(usages=tuple(usages)), + ) + if step.tool_name not in tool_names: + return _failed_answer( + question=question, + kind="invalid_response", + message=f"answer agent requested unknown tool {step.tool_name!r}", + retrieval_latency_ms=tool_latency_ms, + retrieval_succeeded=bool(trace), + agent_call_count=len(usages), + reader_latency_ms=agent_latency_ms, + claims=_claims_from_trace( + trace=tuple(trace), doc_sessions=doc_sessions + ), + tool_calls=tuple(trace), + usages=tuple(usages), + ) + if len(trace) >= MAX_TOOL_CALLS: + return _failed_answer( + question=question, + kind="invalid_response", + message="answer agent exceeded the per-question tool-call limit", + retrieval_latency_ms=tool_latency_ms, + retrieval_succeeded=True, + agent_call_count=len(usages), + reader_latency_ms=agent_latency_ms, + claims=_claims_from_trace( + trace=tuple(trace), doc_sessions=doc_sessions + ), + tool_calls=tuple(trace), + usages=tuple(usages), + ) + tool_started = time.monotonic_ns() + try: + envelope = client.run_recipe( + name=step.tool_name or "", arguments=step.arguments + ) + except MemoryApiError as error: + return _failed_answer( + question=question, + kind="tool", + message=str(error), + retrieval_latency_ms=tool_latency_ms + _elapsed_ms(tool_started), + retrieval_succeeded=False, + agent_call_count=len(usages), + reader_latency_ms=agent_latency_ms, + claims=_claims_from_trace( + trace=tuple(trace), doc_sessions=doc_sessions + ), + tool_calls=tuple(trace), + usages=tuple(usages), + ) + latency = _elapsed_ms(tool_started) + tool_latency_ms += latency + trace.append( + ToolCallRecord( + name=step.tool_name or "", + arguments=step.arguments, + latency_ms=latency, + response=envelope, + ) ) - state.evaluator_cost_usd += response.usage.cost_usd - return AnswerRecord( - item_id=question.item_id, - sample_id=question.sample_id, - category=_retained_category(question=question), - question=question.question, - gold_answer=question.answer or "", - gold_evidence=question.evidence, - claims=claims, - dropped_by_hydration=envelope.dropped_by_hydration, - retrieval_succeeded=True, - retrieval_latency_ms=retrieval_latency, - reader_called=True, - reader_latency_ms=_elapsed_ms(reader_started), - generated_answer=response.output.answer, - reader_usage=response.usage, + return _failed_answer( + question=question, + kind="invalid_response", + message="answer agent exhausted its step budget without a final answer", + retrieval_latency_ms=tool_latency_ms, + retrieval_succeeded=bool(trace), + agent_call_count=len(usages), + reader_latency_ms=agent_latency_ms, + claims=_claims_from_trace(trace=tuple(trace), doc_sessions=doc_sessions), + tool_calls=tuple(trace), + usages=tuple(usages), ) @@ -919,7 +1076,18 @@ def _judge_one( latency_ms=_elapsed_ms(started), failure=_failure(kind="accounting", message=str(error)), ) - except (OpenRouterProviderError, ValidationError) as error: + except OpenRouterProviderError as error: + if error.usage is not None: + state.evaluator_cost_usd += error.usage.cost_usd + return JudgeRecord( + item_id=question.item_id, + label="WRONG", + model_called=True, + usage=error.usage, + latency_ms=_elapsed_ms(started), + failure=_failure(kind="judge", message=str(error)), + ) + except ValidationError as error: return JudgeRecord( item_id=question.item_id, label="WRONG", @@ -928,6 +1096,21 @@ def _judge_one( failure=_failure(kind="judge", message=str(error)), ) state.evaluator_cost_usd += response.usage.cost_usd + if state.evaluator_cost_usd > max_evaluator_cost_usd: + return JudgeRecord( + item_id=question.item_id, + label="WRONG", + model_called=True, + usage=response.usage, + latency_ms=_elapsed_ms(started), + failure=_failure( + kind="accounting", + message=( + f"reported evaluator spend {state.evaluator_cost_usd} crossed" + f" stop threshold {max_evaluator_cost_usd}" + ), + ), + ) return JudgeRecord( item_id=question.item_id, label=response.output.label, @@ -944,10 +1127,11 @@ def _failed_answer( message: str, retrieval_latency_ms: int, retrieval_succeeded: bool, - reader_called: bool, + agent_call_count: int, reader_latency_ms: int | None = None, claims: tuple[RetrievedClaim, ...] = (), - dropped_by_hydration: int = 0, + tool_calls: tuple[ToolCallRecord, ...] = (), + usages: tuple[ProviderCallUsage, ...] = (), ) -> AnswerRecord: """Build a bounded terminal failure without erasing retrieval evidence.""" return AnswerRecord( @@ -958,15 +1142,69 @@ def _failed_answer( gold_answer=question.answer or "", gold_evidence=question.evidence, claims=claims, - dropped_by_hydration=dropped_by_hydration, + tool_calls=tool_calls, + dropped_by_hydration=sum( + call.response.dropped_by_hydration for call in tool_calls + ), retrieval_succeeded=retrieval_succeeded, retrieval_latency_ms=retrieval_latency_ms, - reader_called=reader_called, + reader_called=agent_call_count > 0, + agent_call_count=agent_call_count, reader_latency_ms=reader_latency_ms, + reader_usage=_aggregate_usage(usages=usages) if usages else None, failure=_failure(kind=kind, message=message), ) +def _claims_from_trace( + *, trace: tuple[ToolCallRecord, ...], doc_sessions: dict[UUID, str] +) -> tuple[RetrievedClaim, ...]: + """Collect first-seen evidence claims from every public tool response.""" + seen: set[UUID] = set() + claims: list[RetrievedClaim] = [] + for call in trace: + evidence = [ + *call.response.evidence, + *(item for part in call.response.parts for item in part.evidence), + ] + for claim in evidence: + if claim.claim_id in seen: + continue + seen.add(claim.claim_id) + claims.append( + RetrievedClaim( + rank=len(claims) + 1, + claim_id=claim.claim_id, + doc_id=claim.doc_id, + chunk_id=claim.chunk_id, + claim_text=claim.claim_text, + source_span=claim.source_span, + char_start=claim.char_start, + char_end=claim.char_end, + is_attributed=claim.is_attributed, + is_current_testimony=claim.is_current_testimony, + session_id=doc_sessions.get(claim.doc_id), + ) + ) + return tuple(claims) + + +def _aggregate_usage(*, usages: tuple[ProviderCallUsage, ...]) -> ProviderCallUsage: + """Collapse one question's same-model agent calls for durable accounting.""" + if not usages: + raise ValueError("cannot aggregate an empty usage tuple") + model_names = {usage.model_name for usage in usages} + if len(model_names) != 1: + raise BenchmarkRunError("one answer-agent trace used multiple models") + return ProviderCallUsage( + model_name=usages[0].model_name, + tokens_in=sum(usage.tokens_in for usage in usages), + tokens_out=sum(usage.tokens_out for usage in usages), + cost_usd=sum((usage.cost_usd for usage in usages), start=Decimal(0)), + latency_ms=sum(usage.latency_ms for usage in usages), + ) + + def _failure(*, kind: FailureKind, message: str) -> BenchmarkFailure: """Normalize an external error into a bounded durable failure.""" bounded = " ".join(message.split())[:500] or "unspecified failure" @@ -986,21 +1224,21 @@ def _sample_questions( def _require_cost_ceiling(*, spent: Decimal, ceiling: Decimal) -> None: - """Require a positive run ceiling no lower than persisted spend.""" + """Require a positive reported-spend threshold no lower than persisted spend.""" if ceiling <= 0: raise ExecutionGuardError("max-evaluator-cost-usd must be positive") if ceiling < spent: raise ExecutionGuardError( - f"cost ceiling {ceiling} is below already recorded spend {spent}" + f"cost threshold {ceiling} is below already recorded spend {spent}" ) def _require_cost_before_call(*, spent: Decimal, ceiling: Decimal) -> None: - """Stop before a provider call once the run ceiling is reached.""" + """Stop before a provider call once the reported-spend threshold is reached.""" _require_cost_ceiling(spent=spent, ceiling=ceiling) if spent >= ceiling: raise ExecutionGuardError( - f"evaluator spend {spent} has reached run ceiling {ceiling}" + f"evaluator spend {spent} has reached run threshold {ceiling}" ) diff --git a/compose.yaml b/compose.yaml index 4ee01f20..4b06f46e 100644 --- a/compose.yaml +++ b/compose.yaml @@ -15,6 +15,18 @@ x-app: &app REMEMBERSTACK_SELFHOST_DEPLOYMENT_SLUG: ${REMEMBERSTACK_SELFHOST_DEPLOYMENT_SLUG} REMEMBERSTACK_SELFHOST_DEPLOYMENT_NAME: ${REMEMBERSTACK_SELFHOST_DEPLOYMENT_NAME} REMEMBERSTACK_SELFHOST_API_PORT: "8000" + REMEMBERSTACK_STRUCTURER_MODEL: ${REMEMBERSTACK_STRUCTURER_MODEL:-openai/gpt-5.6-luna} + REMEMBERSTACK_E1_EMBEDDING_MODEL: ${REMEMBERSTACK_E1_EMBEDDING_MODEL:-qwen/qwen3-embedding-8b} + REMEMBERSTACK_E1_PREFIX_MODEL: ${REMEMBERSTACK_E1_PREFIX_MODEL:-openai/gpt-5.6-luna} + REMEMBERSTACK_E2_EXTRACT_MODEL: ${REMEMBERSTACK_E2_EXTRACT_MODEL:-openai/gpt-5.6-luna} + REMEMBERSTACK_E3_NORMALIZE_MODEL: ${REMEMBERSTACK_E3_NORMALIZE_MODEL:-openai/gpt-5.6-luna} + REMEMBERSTACK_OBS_SMALL_MODEL: ${REMEMBERSTACK_OBS_SMALL_MODEL:-openai/gpt-5.6-luna} + REMEMBERSTACK_OBS_FRONTIER_MODEL: ${REMEMBERSTACK_OBS_FRONTIER_MODEL:-openai/gpt-5.6-sol} + REMEMBERSTACK_OBS_EMBEDDING_MODEL: ${REMEMBERSTACK_OBS_EMBEDDING_MODEL:-qwen/qwen3-embedding-8b} + REMEMBERSTACK_ADJUDICATOR_SMALL_MODEL: ${REMEMBERSTACK_ADJUDICATOR_SMALL_MODEL:-openai/gpt-5.6-luna} + REMEMBERSTACK_ADJUDICATOR_FRONTIER_MODEL: ${REMEMBERSTACK_ADJUDICATOR_FRONTIER_MODEL:-openai/gpt-5.6-sol} + REMEMBERSTACK_P1_EMBEDDING_MODEL: ${REMEMBERSTACK_P1_EMBEDDING_MODEL:-qwen/qwen3-embedding-8b} + REMEMBERSTACK_P1_LABEL_MODEL: ${REMEMBERSTACK_P1_LABEL_MODEL:-openai/gpt-5.6-luna} volumes: - app-state:/var/lib/rememberstack - forget-manifests:/var/lib/rememberstack/forget-manifests @@ -91,6 +103,70 @@ services: setup: condition: service_completed_successfully + worker-chunk: + <<: *app + command: ["worker", "--stage", "chunk"] + depends_on: + setup: + condition: service_completed_successfully + + worker-embed-chunk: + <<: *app + command: ["worker", "--stage", "embed_chunk"] + depends_on: + setup: + condition: service_completed_successfully + + worker-extract-claims: + <<: *app + command: ["worker", "--stage", "extract_claims"] + depends_on: + setup: + condition: service_completed_successfully + + worker-normalize-relations: + <<: *app + command: ["worker", "--stage", "normalize_relations"] + depends_on: + setup: + condition: service_completed_successfully + + worker-adjudicate-supersession: + <<: *app + command: ["worker", "--stage", "adjudicate_supersession"] + depends_on: + setup: + condition: service_completed_successfully + + worker-embed-claim: + <<: *app + command: ["worker", "--stage", "embed_claim"] + depends_on: + setup: + condition: service_completed_successfully + + worker-reconcile: + <<: *app + command: ["worker", "--stage", "reconcile"] + depends_on: + setup: + condition: service_completed_successfully + + worker-label-relation: + <<: *app + command: ["worker", "--stage", "label_relation"] + depends_on: + setup: + condition: service_completed_successfully + + projections: + <<: *app + command: ["project", "--plane", "all"] + profiles: ["operations"] + depends_on: + setup: + condition: service_completed_successfully + volumes: app-state: forget-manifests: diff --git a/decisions.md b/decisions.md index a911c10c..271e15ad 100644 --- a/decisions.md +++ b/decisions.md @@ -2629,43 +2629,63 @@ environment, and protected `v*` tag ruleset are already configured. The first GH the container package; making that package public is the only post-publish owner action. WP-7.6 may proceed to its first tagged artifact proof after CLA activation. -## D78. LoCoMo uses a named, fingerprinted protocol rather than a context-free headline - -**Decision.** The first competitive benchmark is **`RS-LoCoMo-v1 J@30`** over the exact pinned -LoCoMo ten-conversation file, categories 1–4, 30 unified current-testimony claims, one frozen -`gpt-4o-mini` reader, one frozen `gpt-4o-mini` judge pass, and the official deterministic LoCoMo -F1 as a secondary metric. Prompts, strict output schemas, dataset/manifests, rendered documents, -models, temperature, adapter version, and repository revision are hashed into the run -fingerprint. Failures and missing records remain in the full manifest denominator. - -Each conversation runs in an isolated deployment because the public claims query has no -source/document filter. One conversation session becomes one immutable Markdown source document; -turn timestamps and IDs remain explicit, while generated image captions/search queries are -labelled as derived and summary annotations are never ingested. A coarse session-grain evidence -diagnostic is reported with malformed-field coverage; the system does not claim exact turn -Recall@k without a generic returned source locator. - -The adapter is unshipped repository tooling around the public SDK. It does not download or vendor -the CC BY-NC dataset, own deployment creation/deletion, add a benchmark-only query surface, or -become a general benchmark framework. Remote ingest, answer, and judge stages require explicit -execution acknowledgements and run-absolute call/cost ceilings. The released Compose skeleton -does not provide the full claim-indexing path, so a complete isolated deployment is an explicit -pre-run prerequisite. Implementation and synthetic checks do not satisfy WP-8.2's end-to-end -acceptance; an owner-authorized real smoke is still required. +## D78. LoCoMo measures the ordinary OSS query system, not a claims-only shortcut + +**Decision.** The first competitive benchmark is **`RS-LoCoMo-Full-v1`** over the exact pinned +LoCoMo ten-conversation file and categories 1–4. Each conversation is an isolated deployment; +each session is one immutable Markdown source. The deployment processes every document through +the ten implemented continuous E/P1 routes, then publishes fresh P2 and P3 projections. + +Questions are answered by a bounded `gpt-4o-mini` agent using only the deployment's ordinary +registry-rendered public recipe tools. The agent may resolve entities, read current relations and +observations, search/hydrate evidence, inspect timelines and transcripts, discover K pages, and +traverse P2. It has at most eight tool calls and nine model calls per question. Every request, +response envelope, model binding, component version, latency, usage, and failure is recorded. +One frozen `gpt-4o-mini` judge pass supplies the primary accuracy metric; official deterministic +LoCoMo F1 is secondary. Gold answers and evidence never enter retrieval or answer context. + +The API, not an operator assertion, proves readiness: every exact expected stage generation for +the requested versions is terminal and P2/P3 builds began after their latest terminal stage. The +answer command checkpoints that report and refuses a changed tool catalog. Prompts, schemas, +tool catalog, dataset/manifests, rendered documents, models, adapter, and repository revision +are fingerprinted. Failures and missing records remain in the full denominator. + +The former `RS-LoCoMo-v1 J@30` hard-coded `search_claims(k=30)` path is not the primary +RememberStack benchmark. It bypasses the truth, graph, recipe, envelope, and agent-consumption +logic the OSS package is designed to provide. A claims-only result may return as a separately +named diagnostic/ablation, never as the full-system headline. + +Plane K is disclosed rather than simulated. The stock self-host profile has no reproducible K +planner/writer runtime or seeded routing rules; a `pages_about` negative is honest but is not K +coverage. A K-enabled run requires explicit repository/runtime/routing fingerprints, K +settlement in readiness, and a new protocol version. + +P3 is built and freshness-checked, but this remote recipe-agent protocol has no filesystem +mount and therefore makes no claim that P3 navigation improved its answers. A mount-enabled +answer protocol receives a new name and fingerprint. + +The adapter remains unshipped repository tooling. It does not vendor or download the CC BY-NC +dataset, own deployment creation/destruction, expose benchmark-only queries, or become a general +benchmark framework. Run-absolute call limits and a reported-spend stop threshold remain +mandatory; provider account limits are the hard monetary boundary. Implementation and +synthetic checks do not satisfy WP-8.2 acceptance; an owner-authorized real smoke is still +required. **Context.** Published “LoCoMo scores” use materially different datasets, ingestion units, -retrieval depths, readers, judges, prompts, and repetition counts. In particular, a historical -Mem0-lineage `k=30` may retrieve 30 memories per speaker, while this protocol retrieves 30 claims -total. A bare score would invite false comparison. The named protocol retains the recognizable -J-score lineage while making every consequential asymmetry visible and reproducible. - -**Rejected alternatives.** Calling the result a standard LoCoMo score; adopting a moving vendor -harness or benchmark-tuned judge; mixing all conversations in one deployment; feeding gold -evidence to the reader or judge; exact-turn retrieval claims without a locator; automatic -deployment destruction; dataset vendoring; and a plugin/DSL/dashboard framework before a second -adapter proves shared machinery is needed. - -**Consequences.** WP-8.2 starts with this one adapter. Vendor-reported numbers are contextual only -until WP-8.3 reruns matched baselines under the same fingerprint. A changed top-k, model, prompt, -schema, judge repetition, or ingestion mapping is a separately named protocol, not an override -flag on `RS-LoCoMo-v1`. +retrieval depths, answer models, judges, prompts, and repetition counts. More importantly, a +fixed claims retrieval benchmark would measure only one RememberStack projection after the owner +explicitly chose to evaluate the full OSS memory logic. A named, traced agent protocol preserves +comparability without reducing the product to dense RAG. + +**Rejected alternatives.** Publishing the claims-only `J@30` score as full-system; a +benchmark-specific SQL/search tool; one service per unused stage enum; treating reconciliation +alone as readiness; requiring an operator to type “index ready”; rebuilding P2/P3 after every +document; pretending an empty K plane ran; mixing conversations; gold leakage; automatic +deployment destruction; dataset vendoring; and an orchestration/dashboard framework. + +**Consequences.** Compose runs the ten real continuous routes by default and offers one explicit +P2/P3 build command. Fact labeling follows adjudication and reconciliation, while readiness +joins it with claim embedding. The API/query writer share `P1Settings`. WP-8.2 continues with +this protocol; matched baseline runs in WP-8.3 must use the same public tool budget and +fingerprint. Any changed tool inventory, call budget, model, prompt, schema, judge repetition, +ingestion mapping, or K mode is a separately named protocol. diff --git a/plan/analysis/locomo_benchmark_analysis.md b/plan/analysis/locomo_benchmark_analysis.md index 7fc01942..54723d21 100644 --- a/plan/analysis/locomo_benchmark_analysis.md +++ b/plan/analysis/locomo_benchmark_analysis.md @@ -1,332 +1,172 @@ -# WP-8.2 — LoCoMo benchmark analysis +# WP-8.2 — LoCoMo full-system benchmark analysis **Date:** 2026-07-23 -**Status:** approved protocol analysis; no benchmark run has been performed - -**Scope:** the first external benchmark adapter, deliberately not a general benchmark -framework +**Status:** approved analysis; implementation prepared; no real benchmark run performed ## Recommendation -Implement a small repository-native LoCoMo harness around the public `MemoryClient` SDK and -report a precisely named **`RS-LoCoMo-v1 J@30`** score: +The primary RememberStack LoCoMo result must exercise the ordinary OSS memory system, not a +benchmark-specific claim-search shortcut. Use the named protocol **`RS-LoCoMo-Full-v1`**: -- the pinned `locomo10.json` release at LoCoMo commit +- exact pinned `locomo10.json` bytes from commit `3eb6f2c585f5e1699204e3c3bdf7adc5c28cb376`; -- local dataset SHA-256 - `79fa87e90f04081343b8c8debecb80a9a6842b76a7aa537dc9fdf651ea698ff4`; -- categories 1–4 only: multi-hop, temporal, open-domain, and single-hop; -- 1,540 questions for publication, with smaller committed smoke and development manifests; +- SHA-256 `79fa87e90f04081343b8c8debecb80a9a6842b76a7aa537dc9fdf651ea698ff4`; +- categories 1–4 and the committed 8/200/1,540-question tiers; - one isolated RememberStack deployment per conversation; - one source document per conversation session; -- claim search at `k=30`; -- one frozen `openai/gpt-4o-mini` reader call per question; -- one frozen `openai/gpt-4o-mini` judge call per generated answer; and -- the official deterministic LoCoMo F1 calculation as a secondary metric over the same - answers. - -The `J@30` name is intentional. A bare “LoCoMo score” would hide retrieval depth and imply a -standardization that does not exist. The run artifact must retain the dataset, prompt, manifest, -model, adapter, and repository revisions so a number can be compared only to a matching -protocol. - -Do not run the benchmark as part of this work package. Implement and verify parsing, rendering, -manifests, scoring, cost guards, checkpointing, and command boundaries with synthetic data only. -The real ingest, retrieval, reader, and judge stages remain behind explicit execution flags for -an owner-reviewed run. - -## What LoCoMo actually contains - -The [official LoCoMo repository](https://github.com/snap-research/locomo) and -[ACL 2024 paper](https://aclanthology.org/2024.acl-long.747/) define a long-conversation -benchmark built from multi-session dialogues. The selected public release contains ten -conversations. Each conversation contains: - -- two named speakers; -- ordered sessions with a human-readable timestamp; -- turns carrying `speaker`, `dia_id`, and `text`; -- optional image URL, generated image caption, and image-search query fields; and -- questions carrying a gold answer, evidence dialog IDs, and a numeric category. - -A read-only audit of the exact pinned file produced: - -| Property | Pinned value | -|---|---:| -| Conversations | 10 | -| Sessions | 272 | -| Dialogue turns | 5,882 | -| Turns with image-related derived fields | 1,226 | -| All questions | 1,986 | -| Category 1 — multi-hop | 282 | -| Category 2 — temporal | 321 | -| Category 3 — open-domain | 96 | -| Category 4 — single-hop | 841 | -| Category 5 — adversarial | 446 | -| Retained categories 1–4 | **1,540** | - -The dataset is [CC BY-NC 4.0](https://github.com/snap-research/locomo/blob/main/LICENSE.txt). -It must not be vendored into this Apache-2.0 repository. A user supplies a local copy and the -harness checks its hash before doing anything else. The non-commercial restriction needs an -owner/legal assessment before a commercial benchmark use; code implementation does not waive -that obligation. - -### Dataset irregularities that must stay visible - -Six evidence entries in the pinned release are not one exact `D:` dialog ID. -Examples include combined strings such as `D8:6; D9:17`, whitespace-separated IDs, and a typo -such as `D:11:26`. A scorer must not silently invent corrected ground truth. - -The v1 harness can still report a clearly labelled **coarse session-recall diagnostic**: -extract syntactically valid dialog IDs, reduce them to session IDs, and compare them with the -sessions of retrieved claims. It must report coverage and malformed evidence count beside the -metric. Exact turn Recall@k is not reported because the public claim-search result does not carry -a stable LoCoMo turn locator. Adding benchmark-specific turn IDs to the core retrieval contract -would be scope creep. - -## Why published LoCoMo numbers are not one standard - -LoCoMo supplies data and deterministic QA scoring, but the conversational-memory “J score” used -by vendors comes from later evaluation harnesses. Important protocol choices vary: - -- included categories; -- dataset revision and subset; -- ingestion unit and whether memories are split by speaker; -- retrieval depth; -- answer model and prompt; -- judge model, prompt, temperature, and repetition count; -- whether evidence is shown to the reader or judge; -- whether failures remain in the denominator; and -- whether the score is deterministic F1, LLM-judge accuracy, or an aggregate. - -The [Mem0 paper](https://arxiv.org/abs/2504.19413) established the recognizable -categories-1–4 J-score lineage and reported Mem0, Mem0g, and contextual competitor results with -F1, BLEU-1, and an LLM judge. Its appendix uses a concise memory-answering prompt and a generous -correct/wrong judge. An -[open reproduction](https://github.com/memodb-io/memobase/tree/358c16bbc6d687937d79bc2f984a11c3be8da901/docs/experiments/locomo-benchmark) -exposes that prompt lineage and a default retrieval depth of 30. - -That historical open harness applies `top_k=30` separately to each speaker store and concatenates -the results, so it may send roughly 60 memories to the reader. `RS-LoCoMo-v1 J@30` retrieves 30 -claims total from one unified deployment. The identical number therefore does not mean an -identical context budget. The lineage harness also reports a simple overlap F1 distinct from the -official stemmed, category-aware LoCoMo F1 used here; those F1 numbers must not be compared -directly. - -The current [Mem0 memory-benchmarks repository](https://github.com/mem0ai/memory-benchmarks) -uses materially different defaults: newer models, multiple top-k settings, an automatically -downloaded moving dataset, and a much longer, benchmark-specific permissive judge prompt. Its -headline is useful context, but not a matched comparator for the historical paper protocol or -for `RS-LoCoMo-v1`. - -Therefore: - -1. Use the historical concise prompt lineage, not the current vendor-tuned prompt. -2. Pin every protocol input and publish raw per-question records. -3. Call the result `RS-LoCoMo-v1 J@30`, not “the standard LoCoMo score.” -4. Put mismatched vendor-reported numbers in a contextual table only. -5. Reproduce baselines under this exact harness in WP-8.3 before making matched claims. - -The LoCoMo paper's prose list order and the numeric category IDs used by the released JSON and -official `evaluation.py` are easy to conflate. `RS-LoCoMo-v1` always reports the dataset/scorer -IDs: 1 multi-hop, 2 temporal, 3 open-domain, and 4 single-hop. - -## Protocol choices - -### Retained questions and tiers - -Category 5 is excluded, matching the widely reported memory QA setup and WP-8.1. Failures, -timeouts, parse errors, missing answers, and missing judge results remain in the denominator as -zero. - -| Tier | Questions | Use | -|---|---:|---| -| Smoke | 8 from `conv-26`, two from every retained category | Contract/debug only; never a headline | -| Development | 200, exactly 20 from each conversation and stratified by available category | Bounded iteration | -| Publication | all 1,540 category 1–4 questions | Deliberate headline run | - -LoCoMo has no native QA identifier. The adapter assigns stable positional IDs of the form -`conv-26/qa/0000` against the pinned bytes. All selected IDs are committed in manifests, and -their expected count and hash are checked before execution. Selection never changes after -scores are observed. - -### Ingestion mapping - -Use **one Markdown document per session**, not one document per conversation and not one -document per turn. - -Each turn is rendered as its own paragraph: +- the complete implemented ten-route E/P1 lifecycle; +- fresh P2 graph and P3 corpus projections after ingestion; +- a bounded answer agent that sees only the question and the deployment's ordinary public + recipe catalog; +- at most eight public tool calls and nine answer-agent model calls per question; +- frozen `openai/gpt-4o-mini` answer-agent and judge seats at temperature zero; +- judge accuracy as the primary metric and official deterministic LoCoMo F1 as secondary; and +- complete tool traces, response envelopes, model identities, component versions, costs, and + failures in the run artifacts. + +The former fixed `search_claims(k=30)` reader measured a useful claims-channel ablation, but it +did not measure the full retrieval logic. It bypassed entity resolution, current relation and +observation reads, graph traversal, hydration, recipes, typed negatives, and the grain/freshness +contract. It must not be published as the primary RememberStack result or retain the `J@30` +headline after the protocol changes. + +## What “full system” means here + +It means the complete normal ingestion and interactive retrieval path relevant to answering +LoCoMo: ```text -[D1:3 | 1:56 pm on 8 May, 2023] Caroline: ... +upload + -> convert -> structure -> chunk -> embed_chunk -> extract_claims + -> normalize_relations + -> embed_claim + -> adjudicate_supersession -> reconcile -> label_relation + -> build P2 + P3 + -> public recipe tools + -> bounded answer agent ``` -When present, `blip_caption` and `query` follow the turn with explicit labels such as -`Dataset-provided derived image caption`. The image URL is metadata, not fetched. The source is -always transparent about generated visual text; it never presents a caption as a human-authored -message. +It does not mean forcing unrelated operational scenarios into every QA item. Backfill, restore, +hard-forget, deletion, connector polling, and migration drills belong in WP-8.5's capability +suite. -The session document choice is the smallest faithful mapping: +Plane K is disclosed separately. The OSS K implementation requires routing rules, a knowledge +repository, and a reproducible planner/writer runtime. The stock Compose profile does not yet +provide those inputs. `pages_about` remains an honest public tool and returns a typed +`known_empty` when no K page exists, but `RS-LoCoMo-Full-v1` must not claim that K synthesis was +exercised. Adding K later creates a separately fingerprinted protocol. -- it preserves session timestamps and dialog IDs; -- 272 documents are materially cheaper than 5,882 turn documents; -- turns remain distinct blocks for the normal RememberStack chunker; -- it exercises the public document-ingestion boundary; and -- it avoids a benchmark-only chunking path. +P3 is likewise disclosed precisely: the stock deployment builds it and readiness proves the +build followed ingestion, but the remote recipe agent has no filesystem mount. The score does +not measure or claim P3 navigation. Adding a mount-enabled answer harness creates a separately +fingerprinted protocol rather than smuggling a benchmark-only file API into the OSS surface. -Source lineage is immutable: +## Why the existing Compose profile was insufficient -```text -source_kind = "locomo" -source_ref = "//" -versioning_mode = "snapshot" -source_version_ref = "" -``` +The work ledger already implemented ten continuous document-version handlers, but the released +Compose profile ran only `convert` and `structure`. `chunk` was deliberately left pending. +Adding containers for every `PipelineStage` enum would also be wrong: several enum stages are +fused into implemented handlers, while others have no runtime handler. + +The actual continuous routes are: + +| Route | Work performed | +|---|---| +| `convert` | immutable Markdown/block representation | +| `structure` | PageIndex-style tree and placement | +| `chunk` | deterministic packed chunks | +| `embed_chunk` | context prefixes and chunk vectors | +| `extract_claims` | claim extraction plus grounding gates | +| `normalize_relations` | entity resolution, relations, observation adjudication | +| `adjudicate_supersession` | relation lifecycle decisions | +| `embed_claim` | P1 claims channel | +| `reconcile` | testimony currency, support recount, lifecycle events | +| `label_relation` | post-lifecycle relation/observation labels and P1 facts channel | -The dataset gives no timezone, so `source_modified_at` is omitted rather than falsely declaring -UTC. The literal session timestamp remains in every turn. +P2 and P3 are aggregate rebuilds, not per-document queue handlers. They run once after the +selected ingestion set has settled. -### Conversation isolation +## Correctness issues found during review -All ten conversations must not share one deployment. `MemoryClient.search_claims()` scopes to a -deployment and currently has no document/source filter. A combined corpus could retrieve another -conversation's answer and inflate or corrupt the score. +### Missing fan-out join and fact-label race -The simplest honest boundary is: +Normalization previously enqueued supersession, claim embedding, and fact labeling in parallel, +while reconciliation followed only supersession. A fact label could therefore be indexed before +supersession changed its status, and “reconcile succeeded” did not mean the other terminal +branches had completed. + +The smallest correct ordering is: ```text -clean isolated deployment for one conversation - -> ingest that conversation's sessions - -> answer its selected questions - -> retain local artifacts - -> move to the next isolated deployment +normalize + ├─ embed_claim + └─ adjudicate_supersession -> reconcile -> label_relation ``` -The harness does not delete Docker volumes, databases, or cloud resources. Destructive lifecycle -ownership remains with the operator. Execution commands require an explicit sample-specific -isolation acknowledgement and record it, but cannot prove an external deployment is clean -through the current public API. +Readiness joins both terminal branches by checking all ten exact component generations. This +keeps claim embedding parallel but prevents fact labels from racing lifecycle state. + +### Writer/query model drift + +The API hard-coded the Qwen embedding model while P1 writers used `P1Settings`. A deployment +override could write vectors with one model and query them with another. The self-host API and +writers now load the same `REMEMBERSTACK_P1_EMBEDDING_MODEL` setting, and readiness reports all +current non-secret serving-process model bindings. This is configuration evidence, not +processing-time provenance; a benchmark deployment must keep one frozen Compose environment. + +### Manual readiness was not evidence + +The old harness accepted `--confirm-index-ready `. That proved only that an operator +typed a string. The normal API now exposes a read-only readiness report for bounded version IDs: +every expected stage/version must be terminal and P2/P3 builds must have begun after the latest +terminal stage. The harness checkpoints the report before any question is answered. + +## Public retrieval protocol + +The answer agent receives the current registry-rendered recipe descriptors, not internal Python +objects or database access. The stock self-host inventory includes: + +- `resolve_entity`; +- current relations and observations; +- entity timeline; +- verbatim and hybrid claims; +- relation explanation/hydration; +- identity transcript and change feed; +- K page discovery; +- P2 neighborhood and shortest-path tools. + +Every call is executed through `MemoryClient.run_recipe()`. The trace stores the tool name, +arguments, latency, and complete typed envelope. The agent must make at least one tool call, +cannot call an unlisted tool, and must finish in at most six words. Gold answers and gold +evidence never enter retrieval or answer-agent prompts. + +## Dataset and comparability + +The selected LoCoMo release contains ten conversations, 272 sessions, 5,882 turns, and 1,986 +questions. Categories 1–4 contribute 1,540 scored questions; category 5 is excluded to match the +common conversational-memory QA setup. The dataset is CC BY-NC 4.0 and is not vendored. + +Published “LoCoMo scores” are not one protocol: dataset revisions, ingestion units, top-k, +answer models, judge prompts, judge repetitions, and failure denominators differ. Therefore a +RememberStack result is comparable only with its full fingerprint. Vendor numbers remain +contextual until WP-8.3 reruns matched baselines. + +## Cost and reproducibility + +The maximum answer-side call count for `N` questions is `9N` answer-agent calls plus `N` judge +calls; actual tool and model counts are recorded. Ingestion calls remain governed by the normal +deployment ledger. A reproducible run pins explicit model IDs; rotating routers such as +`openrouter/free` are forbidden. A named free model that supports the required structured output +may be used for ingestion, but its exact ID belongs in the readiness artifact and produces a +distinct result configuration. + +## Scope guardrails -### Retrieval and answer generation - -For each question: - -1. call public `search_claims(query=question, k=30)`; -2. retain rank order, claim text, source span, and provenance IDs in the raw artifact; -3. render only each returned `claim_text` as `[] ` in the frozen reader prompt; -4. call `openai/gpt-4o-mini` once with temperature zero and a strict answer schema; and -5. checkpoint before advancing to the next question. - -Ground-truth answers and evidence IDs are never included in retrieval or reader context. -Retrieved claims from both speakers remain in one ranked list because RememberStack is a unified -memory backend; inventing two user stores would misrepresent it. - -An empty evidence envelope, including `known_empty`, is a successful retrieval with an empty -memory list. It still receives one reader call and, if the reader returns a valid answer, one -judge call. Only transport, API, or response-validation errors are retrieval failures. The -harness does not reconstruct timestamps or dialog text from gold data: if claim extraction did -not preserve the timestamp in `claim_text`, that loss is part of the measured system behavior. - -### Scoring - -The primary score is the percentage of retained questions labelled `CORRECT` by one frozen -`openai/gpt-4o-mini` judge call at temperature zero. The judge sees only the question, gold -answer, and generated answer. It never sees gold evidence. - -One judge pass is deliberate. The historical Mem0 paper used repeated judging to report -variance; WP-8.1 instead binds one frozen judge and a later fixed audit sample so publication -cost is bounded. The run artifact states `judge_repetitions=1`, so it is not represented as an -exact reproduction of a ten-repeat result. - -The secondary F1 metric reproduces the official LoCoMo normalization, category-1 comma-split -multi-answer handling, and Porter stemming over the same answer records. It costs no additional -model calls. - -Coarse session evidence recall is diagnostic only. It is not blended into J or F1. - -## Calls, costs, and execution safety - -For `N` selected questions after ingestion: - -| Operation | Calls | -|---|---:| -| RememberStack retrieval | `N` | -| Frozen reader | at most `N` | -| Frozen judge | at most `N` | - -That is 400 reader/judge calls for development and 3,080 for publication, before any retry. -RememberStack ingestion also performs model extraction and embedding; its exact call count -depends on normal chunking and extraction output and must be constrained by the deployment's -existing cost ledger rather than guessed by the harness. - -The harness enforces: - -- no dataset auto-download; -- exact dataset and manifest hashes; -- separate `prepare`, `ingest`, `answer`, `judge`, and `summarize` commands; -- an explicit execution acknowledgement on every API/model stage; -- a declared maximum question count before any remote call; -- declared reader and judge call limits; -- one explicit run-absolute evaluator cost ceiling over reader plus judge calls, with the shared - persisted spend checkpointed from reported provider usage after every call; -- resume from immutable run configuration and per-question checkpoints; and -- refusal when an existing run's protocol fingerprint differs. - -A provider-reported cost ceiling can stop subsequent calls but may overshoot by one call because -the charge is known only after the response. The harness ledger covers usage returned with -successfully parsed calls; a billed call that fails before usable accounting is returned is not -reconstructable. A process death after a response but before its atomic checkpoint can also cause -that call to repeat on resume. Provider/account hard limits and the deployment cost ledger are -therefore the true hard monetary boundary. The pre-run checklist must verify both. -Before each provider call the harness refuses when shared reader-plus-judge spend is already at -the run ceiling. A resumed or later stage must declare a ceiling no lower than spend already -recorded; the flag never resets the ledger or creates a per-command allowance. - -`prepare` reports ingestion units (sessions, turns, and bytes), not a fabricated ingestion price. -The ingestion/build cost row required by WP-8.1 comes from the deployment cost ledger after the -run; the harness can preflight and enforce only its directly owned reader/judge calls. - -## Current deployment prerequisite - -The released Compose profile is intentionally a fresh-deployment skeleton. It wires API -ingestion plus `convert` and `structure`, but not the complete E1 chunk/embed, E2 extraction, and -P1 claim-index worker path required by `search_claims`. - -The benchmark harness must not conceal this by: - -- importing repositories and handlers directly; -- constructing a benchmark-only in-process system; -- adding a benchmark-specific query endpoint; or -- claiming an end-to-end sample passed against the released Compose profile. - -The implemented adapter can target any complete RememberStack deployment through the public SDK. -Before the first run, the presumptive path is to finish and review the ordinary self-host -composition. A complete deployment from the parallel cloud project could also exercise the same -public SDK, but it must not become the only working path. Deployment completion is a pre-run -prerequisite, not hidden scope inside the LoCoMo adapter. - -## Rejected expansions - -- A general benchmark plugin system, dataset DSL, workflow engine, or dashboard. -- Automatic dataset download or vendoring non-commercial data. -- A new benchmark-only retrieval API or document filter. -- One deployment containing all conversations. -- One deployment created and destroyed automatically by the harness. -- Feeding gold evidence to the reader or judge. -- Adopting a current vendor's long benchmark-tuned judge as a neutral standard. -- Exact turn-recall claims without a stable returned turn locator. -- Adding LongMemEval, baselines, or Phase-8 metrics infrastructure in this first adapter. -- Wiring the full production deployment topology under the guise of benchmark setup. - -## Owner review before any real run - -The setup review should explicitly confirm: - -1. use of the CC BY-NC dataset is appropriate for the intended run; -2. `RS-LoCoMo-v1 J@30` is the desired public protocol label; -3. `gpt-4o-mini` remains the reader and judge for comparison continuity; -4. one judge pass is acceptable; -5. the deployment used for each sample is complete and isolated; -6. deployment-side ingestion/retrieval budgets and provider-side hard limits are active; -7. smoke is run first, then development, then publication only after inspecting artifacts; and -8. no result is published without its full protocol fingerprint and limitations. +- No benchmark-only SQL or search endpoint. +- No gold evidence in retrieval or answer context. +- No automatic dataset download or vendoring. +- No deployment creation, reset, or deletion in the harness. +- No real LoCoMo, API, OpenRouter, answer-agent, or judge call during implementation tests. +- No K claim unless a reproducible K runtime and artifacts are actually present. +- Claims-only retrieval may return later as a clearly labelled diagnostic, never the headline. diff --git a/plan/designs/locomo_benchmark_design.md b/plan/designs/locomo_benchmark_design.md index b6f5d49e..31e74f71 100644 --- a/plan/designs/locomo_benchmark_design.md +++ b/plan/designs/locomo_benchmark_design.md @@ -1,557 +1,241 @@ -# LoCoMo benchmark adapter design +# LoCoMo full-system benchmark design -> **Status:** approved binding design for WP-8.2; the setup is implemented and may be tested with -> synthetic fixtures, but no real LoCoMo/API/model run is authorized by this document. +> **Status:** binding setup for WP-8.2. Implementation and synthetic tests are allowed; no real +> LoCoMo/API/provider run is authorized by this document. -## 1. Scope and acceptance boundary +## 1. Acceptance boundary -This design implements one adapter for the pinned LoCoMo ten-conversation dataset. It is -repository tooling that consumes the public `MemoryClient` SDK. It does not change query -semantics, own deployment lifecycle, or introduce a general benchmark framework. +The adapter is repository tooling around the public `MemoryClient`. Before the owner walkthrough: -Implementation acceptance before owner review is: +- exact dataset and manifests validate locally; +- the stock self-host profile composes all ten continuous handlers; +- P2/P3 can be built explicitly over the same stores the API reads; +- readiness is machine-verifiable through the public API; +- the answer agent uses only registry-rendered public recipes; +- all tool calls, envelopes, model usage, costs, and failures checkpoint; +- pure and synthetic tests pass; and +- no real benchmark or provider call occurs. -- the exact dataset schema and SHA are validated; -- smoke, development, and publication item manifests are committed and self-checking; -- sessions render deterministically to lineage-aware Markdown uploads; -- remote stages are visibly separated and execution-guarded; -- retrieval, reader, judge, deterministic F1, coarse session recall, failure accounting, - checkpointing, and aggregation are implemented; -- pure and synthetic tests are green; -- no real LoCoMo file is ingested; -- no deployment query is made; -- no reader or judge model is called; and -- the released Compose limitation is documented as a pre-run prerequisite. +WP-8.2 remains in progress until an owner-authorized eight-question smoke finishes. -The WP-8.2 plan row remains `in progress` until an owner-authorized smoke run completes end to -end against a complete isolated deployment. - -## 2. Location and dependencies - -Keep the harness outside the shipped `rememberstack` wheel: +## 2. Fixed protocol ```text -benchmarks/ - __init__.py - locomo/ - __init__.py - __main__.py - cli.py - dataset.py - model.py - protocol.py - runner.py - manifests/ - smoke.json - development.json - publication.json +protocol RS-LoCoMo-Full-v1 +dataset commit 3eb6f2c585f5e1699204e3c3bdf7adc5c28cb376 +dataset SHA-256 79fa87e90f04081343b8c8debecb80a9a6842b76a7aa537dc9fdf651ea698ff4 +categories 1, 2, 3, 4 +answer-agent model openai/gpt-4o-mini +answer temperature 0 +max tool calls/question 8 +max agent calls/question 9 +judge model openai/gpt-4o-mini +judge temperature 0 +judge repetitions 1 +primary metric judge accuracy +secondary metric official LoCoMo F1 +diagnostic coarse evidence-session recall ``` -Tests remain under `src/tests/benchmarks/`. Pyright includes `benchmarks/` so the research -harness receives the same standard type checking as library code. Implementation updates -`[tool.pyright].include`, the repository lint/type commands, and CI invocations from `src/` to -`src/ benchmarks/`; otherwise placing the harness outside the wheel would silently exempt it -from the enforced checks. Pytest keeps its tests under `src/tests/` and adds the repository root -to `pythonpath` so those tests can import the unshipped top-level package. - -The harness may import public SDK/model values and the existing provider port/adapter from -`rememberstack`; library code never imports `benchmarks`. This preserves the wheel's -client-first surface and avoids putting a research CLI under the stable `remember` command. - -Add only the scorer dependencies needed to reproduce official F1: - -```toml -[project.optional-dependencies] -benchmark = ["nltk>=3.9", "regex>=2024.11.6"] -``` +The tool catalog hash, prompt and schema hashes, adapter and repository revisions, manifests, +rendered documents, model identities, and component generations are stored. A change creates a +new protocol version. -No orchestration framework, dataframe library, tokenizer download, or vendor benchmark package -is added. `nltk.PorterStemmer` requires no corpus download. The development dependency group -also carries these two packages so the committed benchmark tests run in the ordinary locked CI -environment; the optional extra is the end-user install surface. +## 3. Ingestion mapping -Run locally as: +Each conversation runs in a clean isolated deployment. Each session is one immutable Markdown +document. Every turn is rendered: ```text -uv run --extra benchmark python -m benchmarks.locomo +[D1:3 | 1:56 pm on 8 May, 2023] Caroline: ... ``` -## 3. Fixed protocol +Image URLs are not fetched. Dataset captions and image queries are included only with explicit +derived-data labels. Session summaries and event summaries are never ingested. ```text -protocol name RS-LoCoMo-v1 -dataset commit 3eb6f2c585f5e1699204e3c3bdf7adc5c28cb376 -dataset SHA-256 79fa87e90f04081343b8c8debecb80a9a6842b76a7aa537dc9fdf651ea698ff4 -categories 1, 2, 3, 4 -retrieval target current testimony claims -top-k 30 -reader model openai/gpt-4o-mini -reader temperature 0 -judge model openai/gpt-4o-mini -judge temperature 0 -judge repetitions 1 -primary metric judge accuracy (J@30) -secondary metric official LoCoMo F1 -diagnostic coarse evidence-session recall and complete-session success +source_kind locomo +source_ref // +versioning_mode snapshot +source_version_ref ``` -Changing any fixed value creates a different protocol name/version. CLI flags do not override -these values in v1. This removes a large configuration matrix and prevents accidental -apples-to-oranges output. - -The only run selections are the committed tier and one sample ID for isolated execution. - -## 4. Typed boundaries - -All external JSON and all durable harness artifacts are Pydantic models with `extra="forbid"`. -The minimum types are: - -### Dataset - -- `LoCoMoTurn`: speaker, dialog ID, text, and optional image-derived fields. -- `LoCoMoQuestion`: stable derived ID, question, stringified gold answer, evidence tuple, and - category `Literal[1, 2, 3, 4, 5]`. -- `LoCoMoSession`: numeric session ordinal, dialog prefix, literal timestamp, and ordered turns. -- `LoCoMoSample`: official sample ID, two speakers, ordered sessions, and ordered questions. -- `LoCoMoDataset`: the ten ordered samples plus validated aggregate counts. +The dataset has no timezone, so its literal timestamp stays in text and `source_modified_at` is +omitted. -The six top-level sample fields are `sample_id`, `conversation`, `qa`, `observation`, -`session_summary`, and `event_summary`. Speakers and dynamic session fields are nested inside -`conversation`: `speaker_a`, `speaker_b`, list-valued `session_`, and -`session__date_time`. +## 4. Runtime composition -The pinned file contains orphan timestamp keys for removed/non-selected sessions (for example, -timestamp ordinals beyond the 19 list-valued sessions in `conv-26`). The parser discovers -sessions from list-valued `session_` fields only, requires a matching timestamp for each -discovered session, and ignores orphan timestamp keys. It rejects: +### Continuous services -- missing session timestamps; -- duplicate sample or dialog IDs; -- a session whose turn dialog prefix disagrees with its ordinal; -- retained questions with null answers; -- unexpected categories; -- aggregate counts that differ from the pinned release; and -- any file whose bytes do not match the pinned SHA. - -Gold answers are JSON strings or integers. After rejecting `None` for retained categories, the -parser canonicalizes them with `str(answer)`. Many excluded category-5 rows omit `answer` and -carry `adversarial_answer` instead; the boundary accepts that official field but never selects or -scores it. - -Unused official `observation`, `session_summary`, and `event_summary` fields are accepted at the -sample boundary but never ingested. They are generated benchmark annotations and would leak -post-hoc summaries into the tested memory backend. - -### Manifests and run state - -- `QuestionManifest`: tier, dataset revision/hash, exact ordered IDs, count, and IDs hash. -- `RunConfiguration`: protocol constants, tier, sample, manifest hash, repository revision, - prompt hashes, adapter version, and isolation acknowledgement. -- `PreparedDocument`: session identity, rendered content hash, filename, source metadata. -- `IngestRecord`: document identity plus the public SDK's returned lineage/version IDs. -- `RetrievedClaim`: rank and the public evidence-grain fields. -- `AnswerRecord`: question, gold answer, claims, generated answer or typed failure, timing, and - provider usage. -- `JudgeRecord`: label or typed failure plus provider usage. -- `RunSummary`: denominators, J@30, F1, per-category results, session-recall coverage, failures, - calls, tokens, costs, and protocol fingerprint. - -Decimal monetary values serialize as strings. Times are UTC. IDs are never inferred from display -text. - -## 5. Manifest construction - -Stable question IDs are positional against the pinned bytes: +`docker compose up` starts one process for each implemented steady route: ```text -/qa/ +convert +structure +chunk +embed_chunk +extract_claims +normalize_relations +adjudicate_supersession +embed_claim +reconcile +label_relation ``` -The checked-in manifests are canonical data, not generated at execution time. - -- Smoke selects the first two retained questions in each category from `conv-26`, preserving - original order in the final list. -- Development allocates exactly 20 questions per conversation. Within each conversation it - gives every available retained category at least one position, allocates remaining positions - by largest remainder against that conversation's retained category distribution, and takes - the earliest items in each category. Ties resolve by category number. The final manifest is in - dataset order. -- Publication contains every category-1–4 ID in dataset order. - -A maintenance-only manifest generator may reproduce these files, but normal commands only load -and validate them. The generator refuses any dataset hash other than the fixed v1 hash. - -## 6. Deterministic session rendering - -For each sample, sessions sort by numeric ordinal and render as UTF-8 Markdown with LF line -endings: +All use the same deployment ID, PostgreSQL ledger, MinIO stores, OpenRouter adapter, and Lance +root. One route per process preserves the existing queue/rate-limit design; no workflow engine +is introduced. -```markdown -# LoCoMo conv-26 — session D1 +### Aggregate projections -Participants: Caroline and Melanie +P2 and P3 rebuild after all selected session versions are E/P1-ready: -Dataset timestamp: 1:56 pm on 8 May, 2023 (timezone unspecified) - -[D1:1 | 1:56 pm on 8 May, 2023] Caroline: Hey Mel! ... - -[D1:2 | 1:56 pm on 8 May, 2023] Melanie: ... - -Dataset-provided derived image caption for D1:2: ... -Dataset-provided derived image search query for D1:2: ... +```bash +docker compose --profile operations run --rm projections ``` -Rules: +The one-shot service builds P2 into the snapshot bucket and P3 into the corpusfs bucket. It does +not run on every document and does not remain resident. -- preserve official text verbatim after the fixed prefix; -- repeat the literal timestamp on every turn so independently extracted claims retain temporal - context; -- do not fetch or ingest `img_url`; -- label caption/query fields as derived; -- ignore the `re-download` implementation flag; -- end with one newline; and -- hash the exact bytes before upload. +P3 publication is a deployment-integrity requirement in this protocol, not an answer channel. +The remote `MemoryClient` answer agent cannot browse a local P3 mount, and the ordinary recipe +registry has no filesystem operation. Results must not attribute answer quality to P3 +navigation. A future mount-enabled LoCoMo harness is a separately named protocol. -Uploads use `text/markdown`, snapshot versioning, fixed dataset revision, and stable session -source refs. Re-preparing the same bytes is a no-op; changed rendering changes the protocol -fingerprint. +### Plane K -## 7. Command/state machine +The benchmark records that the stock profile has no K planner/writer runtime. `pages_about` +remains available and honest, but an empty result is not reported as K coverage. A later K-enabled +LoCoMo run needs explicit routing rules, repository/runtime fingerprints, K settlement in +readiness, and a new protocol name. -Commands are intentionally staged: +## 5. Lifecycle ordering and readiness -### `prepare` +Normalization fans out as: ```text -python -m benchmarks.locomo prepare - --dataset /absolute/path/locomo10.json - --tier smoke|development|publication - --output /absolute/path/to/new-run-directory +normalize_relations + ├── embed_claim + └── adjudicate_supersession + └── reconcile + └── label_relation ``` -Local only. It validates the data and manifest, records `git rev-parse HEAD`, writes immutable -configuration, and renders the selected samples' session documents. It refuses a non-empty output -directory. +This ensures labels enter P1 only after supersession and testimony reconciliation. A +no-claims document still creates the no-op terminal rows, so readiness has one deterministic +shape. -### `ingest` +`POST /readiness?require_projections=true` receives a bounded JSON list of version IDs. The +response contains: -```text -python -m benchmarks.locomo ingest - --run /absolute/path/to/run - --sample conv-26 - --max-documents 19 - --execute - --confirm-isolated-deployment conv-26 -``` +- every expected stage and exact component version; +- its status and completion time; +- P2/P3 version and publication time; +- a Boolean requiring every stage to be `succeeded`/`skipped`; +- a Boolean requiring both projection builds to begin after the latest requested terminal stage; +- every non-secret ingestion/query model binding. -Remote and state-changing. It requires exact confirmation that the configured API points at a -clean deployment dedicated to the named sample. `--max-documents` must cover the already prepared -count before the first API call. It uploads each session through `MemoryClient.ingest()` and -atomically checkpoints each response. +The answer command refuses a false report and checkpoints a true one. The old +`--confirm-index-ready` flag is removed. -Re-running is allowed only when stored content hashes and public no-op/version results remain -consistent. The command never creates, resets, or deletes a deployment. +## 6. Public tool surface -### Readiness pause +The self-host setup seeds the normal canonical recipes plus P2 recipes. `resolve_entity` is +canonical because UUID-addressed fact tools otherwise cannot be used by a remote recipe-only +agent. P2 recipes are seeded only by profiles that compose `GraphQueries`. -There is no benchmark polling loop. The deployment owns asynchronous pipeline processing. The -operator verifies that all documents reached claim indexing using deployment operations, then -starts answering. The absence of a public per-version completion endpoint is stated in the -pre-run checklist. +The protocol hashes the exact descriptor list returned by `GET /recipes` and refuses a mismatch. +This prevents an added, removed, or changed tool from silently changing the benchmark. -### `answer` +No benchmark tool reads Postgres, Lance, MinIO, or internal handlers directly. -```text -python -m benchmarks.locomo answer - --run /absolute/path/to/run - --sample conv-26 - --max-questions 8 - --max-reader-calls 8 - --max-evaluator-cost-usd 1.00 - --execute - --confirm-index-ready conv-26 -``` +## 7. Answer loop -Remote. It validates that every prepared session has a successful ingest record. Before the first -call, the maximum question and reader-call limits must cover the remaining work. For every item it: +For each question: -1. retrieves 30 claims through the public SDK; -2. maps returned document IDs to ingested session IDs for the coarse diagnostic; -3. records returned count and `dropped_by_hydration`; -4. checkpoints only transport/API/response-validation failure without calling the reader, or - calls the reader once, including when the successful envelope contains zero claims or a - `known_empty` negative; -5. records response/failure, latency, usage, and cumulative evaluator cost atomically. +1. Render the frozen answer-agent prompt with question, public tool descriptors, and prior trace. +2. Ask for strict `AnswerAgentStep`. +3. For `action="tool"`, validate the name against the catalog and call + `MemoryClient.run_recipe()`. +4. Append arguments, latency, and the complete envelope. +5. For `action="answer"`, require at least one tool call and at most six words. +6. Stop at eight tools or nine model calls; exhaustion is a visible wrong, not a retry. +7. Checkpoint the terminal answer or failure. -Existing records with the same fingerprint are skipped. A failure record is terminal for that -run and remains in the denominator; an explicit new run is required to change protocol or retry -policy. +The agent is instructed to orient, verify current facts, and audit evidence while respecting +grain, validity, freshness, truncation, typed negatives, and hydration drops. It receives no gold +answer, evidence IDs, summaries, or outside retrieval. -`--max-questions` is a run-absolute authorization watermark and must cover the complete prepared -tier: 8 for smoke, 200 for development, or 1,540 for publication. `--max-reader-calls` also counts -reader calls already checkpointed for earlier sample commands; it is not a fresh allowance for -the named sample. +Evidence claims found anywhere in the trace are de-duplicated in first-seen order for the coarse +session diagnostic. This diagnostic remains separate from the primary score. -### `judge` +## 8. Commands -```text -python -m benchmarks.locomo judge - --run /absolute/path/to/run - --sample conv-26 - --max-judge-calls 8 - --max-evaluator-cost-usd 2.00 - --execute -``` - -Remote. It judges successful generated answers once. Retrieval/reader failures receive a local -wrong result without a model call. Judge call failures are recorded as wrong. The same cumulative -evaluator cost ledger is checked after every provider response. - -`--max-judge-calls` is likewise run-absolute and includes calls checkpointed for earlier samples. - -`--max-evaluator-cost-usd` is a **run-absolute** ceiling over a single persisted reader-plus-judge -ledger, never a new allowance for the current command. Before a provider call, the command refuses -when recorded spend is already at the ceiling. After a successful response it adds the exact -reported `usage.cost_usd` and checkpoints, so one completed call may overshoot. A resumed or later -stage must pass a ceiling greater than or equal to spend already recorded. For example, a smoke -run may authorize at most `$1` through `answer` and then raise the total run ceiling to `$2` -through `judge`; it has not authorized `$1` independently to each stage. +Local preparation: -The ledger accounts only usage returned with successfully parsed calls. A provider-billed call -that fails before usable accounting is returned depends on the provider/account hard limit, not -this reconstructed ledger. Atomic checkpoints prevent partial state, but the provider port has no -idempotency key: process death after a response and before its checkpoint can repeat that call on -resume. Resume skips every successfully checkpointed record. - -### `summarize` - -```text -python -m benchmarks.locomo summarize --run /absolute/path/to/run +```bash +uv run --extra benchmark python -m benchmarks.locomo prepare \ + --dataset /absolute/path/locomo10.json \ + --tier smoke \ + --output .benchmark-runs/locomo-smoke ``` -Local only. It includes every manifest item. Missing sample files, answer records, or judge records -become explicit failures and zeros, never a reduced denominator. A smoke/development/publication -summary states its tier prominently. - -## 8. Prompts and provider contract - -Prompts live as the following exact named templates in `protocol.py`, with this design, source -lineage, and SHA-256 hashes in the run configuration. They are concise original adaptations of -the unified-memory (`ANSWER_PROMPT_ZEP`) and judge prompts in the -[memobase LoCoMo reproduction](https://github.com/memodb-io/memobase/tree/358c16bbc6d687937d79bc2f984a11c3be8da901/docs/experiments/locomo-benchmark), -which in turn identifies the Mem0 evaluation lineage. They are not the dual-user-bank prompt and -not a verbatim reproduction. +Per isolated sample: -The exact reader template is: - -```text -Answer the question using only the ranked conversation memories below. -Use memory timestamps when present. Resolve relative time references to the -corresponding date, month, or year. If memories conflict, prefer the most recent -one. Do not confuse people mentioned in a memory with the conversation speakers. -If the memories do not contain the answer, answer "Unknown". -Return only a concise answer of at most six words. - -Ranked memories: -{memories} - -Question: {question} -``` - -`memories` is the rank-ordered join of the public evidence results: - -```text -[1] -[2] -... -``` - -For zero returned claims it is exactly `(none)`. `source_span`, session identity, document title, -gold answer, and gold evidence are retained in the raw result where applicable but are not -interpolated into the reader prompt. In particular, the harness does not restore a timestamp that -the memory pipeline failed to preserve. - -The exact judge template is: - -```text -Classify the generated answer to the question as CORRECT or WRONG against the -gold answer. Be generous about concise paraphrases that identify the same topic. -For time questions, accept equivalent formats or relative expressions only when -they denote the same date or time period. Extra wording does not make an otherwise -correct answer wrong. A missing, unknown, contradictory, or different answer is -WRONG. - -Question: {question} -Gold answer: {gold_answer} -Generated answer: {generated_answer} -``` - -The judge does not produce or store a rationale. Its strict output model is: - -```text -label: Literal["CORRECT", "WRONG"] -``` - -The reader's strict output model is equally binding: - -```text -ReaderOutput: - answer: non-empty str -``` - -It has no rationale or other field. `AnswerRecord.generated_answer` is exactly -`ReaderOutput.answer`. A missing, empty, or invalid `answer` is an `invalid_response` reader -failure and stays in the denominator. - -This is a disclosed `RS-LoCoMo-v1` adaptation of the generous Mem0-lineage judge, not a claim -that the prompt bytes or repetition protocol match the Mem0 paper. - -The strict structured reader response is also deliberate. The historical dual-speaker lineage -prompt requested free-text step-by-step reasoning and judged that entire completion; this -protocol returns only a terse structured answer from one unified memory list. Both the different -context budget (30 unified claims rather than up to 30 memories per speaker) and output shape are -part of the protocol fingerprint and prevent direct comparison to a historical `k=30` number. - -`ModelRequest` gains an optional bounded `temperature`; the OpenRouter adapter forwards it only -when present. Existing callers remain unchanged, while the benchmark binds temperature zero -instead of relying on a provider default. - -Gold answers and evidence are prohibited from the reader prompt. Retrieved memories are -prohibited from the judge prompt. Tests assert both separations. - -The protocol fingerprint includes SHA-256 hashes of both prompt templates and both strict JSON -schemas. Prompt rendering performs one `str.format` pass with all named fields supplied after the -memory lines are assembled. Python does not re-scan substituted values, so braces in a question -or claim remain literal. Memory lines join with one `\n` and no trailing blank line. - -## 9. Scoring details - -### J@30 - -```text -J@30 = 100 * count(CORRECT) / count(manifest questions) -``` - -Missing/failed answer or judge records are `WRONG`. Report overall and per category with integer -numerators and denominators. - -### Official deterministic F1 - -Reproduce the official scorer: - -- lowercase; -- remove commas and ASCII punctuation; -- remove tokens `a`, `an`, `the`, and `and`; -- normalize whitespace; -- Porter-stem whitespace tokens; -- category 3 uses the gold substring before the first semicolon; -- categories 2–4 use token F1; and -- category 1 splits both generated and gold answers on commas, scores each gold part against its - best generated part, and averages. - -A failed/missing generated answer scores zero. Report the arithmetic mean overall and per -category. Do not add BLEU-1 merely because another paper reported it; it adds no decision value -to this first harness. +```bash +uv run --extra benchmark python -m benchmarks.locomo ingest \ + --run .benchmark-runs/locomo-smoke \ + --sample conv-26 \ + --max-documents 19 \ + --execute \ + --confirm-isolated-deployment conv-26 -### Coarse session evidence diagnostic +docker compose --profile operations run --rm projections -Find `D:` substrings in gold evidence fields, while separately marking any field -whose entire value is not one exact dialog ID as malformed. Reduce the valid matches and the -retrieved claim documents to session IDs. The pinned file has six malformed evidence fields. +uv run --extra benchmark python -m benchmarks.locomo answer \ + --run .benchmark-runs/locomo-smoke \ + --sample conv-26 \ + --max-questions 8 \ + --max-agent-calls 72 \ + --max-evaluator-cost-usd 1.00 \ + --execute -```text -session_recall = |gold_sessions ∩ retrieved_sessions| / |gold_sessions| -complete_session_success = gold_sessions ⊆ retrieved_sessions +uv run --extra benchmark python -m benchmarks.locomo judge \ + --run .benchmark-runs/locomo-smoke \ + --sample conv-26 \ + --max-judge-calls 8 \ + --max-evaluator-cost-usd 2.00 \ + --execute ``` -Report: - -- scorable questions; -- questions with malformed/unparseable evidence; -- mean session recall; -- complete-session success; and -- the warning “session-grain diagnostic; not turn Recall@k.” - -## 10. Persistence and failure behavior - -Run files are canonical JSON with sorted keys. Every state update writes a temporary sibling, -flushes and fsyncs it, then replaces the destination. A crash leaves either the previous complete -file or the next complete file. - -The run configuration is immutable after `prepare`. Remote commands recalculate: - -- dataset file hash; -- selected manifest hash; -- rendered document hashes; -- prompt hashes; -- protocol fingerprint; and -- current repository revision. - -The repository may be dirty during development, but a real run refuses a dirty worktree and a -revision different from the prepared revision. - -Expected per-item remote failures are recorded by stable class (`retrieval`, `reader`, -`judge`, `accounting`, or `invalid_response`) and a bounded message. Programming errors and -interrupts stop the process rather than being converted into benchmark losses. - -Secrets, authorization headers, full provider bodies, and environment values are never written. - -## 11. Tests that are allowed before owner review - -All tests use tiny synthetic fixtures, `httpx.MockTransport`, and the existing fake model -provider. They do not open the real pinned dataset path or network. - -Required tests: - -1. dataset shape, aggregate, duplicate, timestamp, and hash rejection; -2. stable IDs and manifest selection on synthetic data; -3. deterministic Markdown rendering and explicit derived-image labels; -4. no summary/observation annotation leakage; -5. prompt separation: no gold in reader and no retrieved context in judge; -6. successful zero-claim and `known_empty` retrieval each call the reader once with memories - exactly `(none)`, consume reader-call/cost allowance, and are not retrieval failures, while - transport/API/validation failures do not call the reader; -7. the reader uses the exact strict `ReaderOutput` schema; missing/empty answers become typed - reader failures in the denominator; -8. official F1 examples, including category-1 comma splitting and category-3 semicolon handling; -9. malformed evidence coverage and coarse session recall; -10. explicit execution and isolation/readiness guards; -11. document/question/call/shared-run-cost limit refusal before the next remote call; -12. checkpoint/resume without duplicating checkpointed successful calls or resetting shared - evaluator spend; -13. failures and missing records remain in the denominator; -14. protocol/prompt/schema/hash mismatch refusal; -15. SDK-only ingest/retrieval calls through mock transports; and -16. bounded `ModelRequest.temperature` forwarding without changing existing callers. - -Allowed verification commands are targeted pytest, Ruff, Pyright, and import-linter. The command -help and a synthetic `prepare` may run. Real `ingest`, `answer`, and `judge` commands may not. - -## 12. Pre-run checklist - -The owner walkthrough must resolve these in order: - -1. Confirm CC BY-NC use. -2. Inspect the checked-in protocol, prompts, and manifests. -3. Choose a complete deployment path; the released Compose skeleton is insufficient. -4. Provision one clean deployment for the first smoke conversation. -5. Set deployment ingestion/retrieval budget caps and provider account hard limits. -6. Set `REMEMBERSTACK_*` SDK and OpenRouter credentials through typed settings. -7. Run `prepare` and inspect its call/document plan. -8. Run `ingest` for `conv-26`. -9. Verify all session documents reached claim indexing outside the harness. -10. Run `answer`, inspect returned claims and failures, then run `judge`. -11. Inspect the smoke summary before authorizing development. -12. Repeat the same gate before any publication run. - -## 13. Deferred work - -- Complete self-host worker composition and a public per-version pipeline-status surface. -- Exact turn-level evidence recall through a generic source-locator contract. -- Matched BM25, dense-RAG, Mem0, and Graphiti adapters (WP-8.3). -- Shared latency/cost artifact machinery across benchmarks (WP-8.4). -- Capability benchmark and publication report (WP-8.5/8.6). -- More LoCoMo models, top-k sweeps, judge repetitions, or prompt variants. - -These are not latent hooks in the v1 code. They are separate work packages that may reuse the -small run-manifest/result shapes only after a second real benchmark demonstrates the need. +The limits are run-absolute across resumed sample commands. The harness never creates or destroys +the deployment. + +## 9. State and failure rules + +`run.json`, manifests, and rendered document hashes are immutable. `state.json` is atomically +replaced after each ingestion, readiness checkpoint, answer, and judge. + +Transport errors, invalid tool decisions, schema failures, provider accounting failures, step +exhaustion, and missing records remain explicit and score zero. Successfully parsed provider +usage is added to the shared answer/judge ledger. A call that crosses the CLI reported-spend +threshold is recorded as a failure and stops the run. Later unanswered items remain explicit +zero-scored missing records unless the operator resumes with an explicitly higher threshold. +Provider-side account limits remain the hard monetary boundary because a process can die after +billing but before checkpointing. + +## 10. Pre-run checklist + +- Clean git revision equals `run.json`. +- Local dataset hash and manifest validate. +- One fresh deployment is dedicated to exactly one conversation. +- Explicit ingestion model IDs are set; no rotating model router. +- All ten workers are running. +- Every prepared session has an ingest record. +- P2/P3 one-shot build completed. +- Public readiness is true; current serving-process model bindings are reviewed as + configuration, not processing-time provenance. +- Public recipe catalog hash matches. +- Account/provider hard limits and the CLI reported-spend stop threshold are acceptable. +- No claim is made that K ran. +- Raw artifacts and failures will be retained for publication review. diff --git a/plan/plans/phase-8-benchmarks.md b/plan/plans/phase-8-benchmarks.md index 6fff8649..54ad41a8 100644 --- a/plan/plans/phase-8-benchmarks.md +++ b/plan/plans/phase-8-benchmarks.md @@ -5,9 +5,10 @@ Distinct from the internal D22 harness: that answers "are we correct against our set"; this answers "are we better than the alternatives on shared ground", plus "what can we do that they cannot". -**Entry gates:** Phases 1–3 + 5 done (ingestion, lifecycle, full retrieval); Phase 6 optional -per benchmark (K helps orientation-style tasks). Each run records its actual cost and explicit -run cap; no deployment-owner budget is an OSS benchmark gate (D60). +**Entry gates:** Phases 1–5 done (ingestion, lifecycle, projections, full retrieval). A benchmark +records which Plane-K runtime actually ran; an empty/unconfigured K plane is never represented as +coverage. Each run records its actual cost and explicit run cap; no deployment-owner budget is +an OSS benchmark gate (D60). **Exit criteria:** a published methodology + results document (reproducible runs, pinned versions, honest losses included); the capability benchmark demonstrates the differentiators end to end. @@ -37,7 +38,7 @@ than expanding WP-8.2. The reusable prompt for independent external research is ## WP-8.2 LoCoMo setup -The first adapter is the reviewed `RS-LoCoMo-v1 J@30` protocol: +The first adapter is the reviewed `RS-LoCoMo-Full-v1` protocol: - analysis and comparability limits: [`locomo_benchmark_analysis.md`](../analysis/locomo_benchmark_analysis.md); @@ -45,8 +46,9 @@ The first adapter is the reviewed `RS-LoCoMo-v1 J@30` protocol: [`locomo_benchmark_design.md`](../designs/locomo_benchmark_design.md); and - unshipped repository harness: `benchmarks/locomo/`. -Its smoke, development, and publication manifests pin 8, 200, and 1,540 question IDs. The setup -is implemented behind explicit remote execution guards, but no real ingest, query, reader, judge, -or score run has occurred. WP-8.2 remains in progress until the owner reviews the setup and an -eight-question smoke completes against a full isolated deployment. The released Compose skeleton -does not yet wire the required claim-indexing stages. +Its smoke, development, and publication manifests pin 8, 200, and 1,540 question IDs. Compose +now runs the complete ten-route continuous lifecycle and exposes a one-shot P2/P3 build. The +answer harness verifies exact stage/projection readiness and lets a bounded agent choose the +ordinary public recipe tools; the former claims-only J@30 path is not the headline. No real +ingest, query, answer-agent, judge, or score run has occurred. WP-8.2 remains in progress until +the owner reviews the setup and an eight-question smoke completes against an isolated deployment. diff --git a/src/rememberstack/adapters/openrouter.py b/src/rememberstack/adapters/openrouter.py index 570fc2f1..4700f36e 100644 --- a/src/rememberstack/adapters/openrouter.py +++ b/src/rememberstack/adapters/openrouter.py @@ -9,6 +9,7 @@ import httpx from pydantic import Field +from pydantic import ValidationError from pydantic_settings import BaseSettings from pydantic_settings import SettingsConfigDict @@ -17,6 +18,7 @@ from rememberstack.model import GeneratedResponse from rememberstack.model import ModelRequest from rememberstack.model import ProviderAccountingError +from rememberstack.model import ProviderCallError from rememberstack.model import ProviderCallUsage from rememberstack.model import StructuredResponseModel @@ -33,7 +35,7 @@ class OpenRouterSettings(BaseSettings): timeout_s: float = Field(default=120.0, gt=0) -class OpenRouterProviderError(Exception): +class OpenRouterProviderError(ProviderCallError): """OpenRouter returned an error or an unusable response body.""" @@ -79,9 +81,15 @@ def generate( decoded = json.loads(content) except (KeyError, IndexError, TypeError, json.JSONDecodeError) as err: raise OpenRouterProviderError( - f"unusable completion body for {response_type.__name__}" + f"unusable completion body for {response_type.__name__}", usage=usage ) from err - output = response_type.model_validate(decoded) + try: + output = response_type.model_validate(decoded) + except ValidationError as error: + raise OpenRouterProviderError( + f"completion body failed {response_type.__name__} validation", + usage=usage, + ) from error return GeneratedResponse(output=output, usage=usage) def embed(self, *, request: EmbeddingRequest) -> EmbeddingResponse: @@ -102,7 +110,9 @@ def embed(self, *, request: EmbeddingRequest) -> EmbeddingResponse: vectors=tuple(tuple(item["embedding"]) for item in ordered), usage=usage ) except (KeyError, TypeError, ValueError) as err: - raise OpenRouterProviderError("unusable embeddings body") from err + raise OpenRouterProviderError( + "unusable embeddings body", usage=usage + ) from err def _post(self, *, path: str, payload: dict[str, object]) -> dict[str, Any]: """POST one JSON request; non-2xx responses become typed errors.""" diff --git a/src/rememberstack/client.py b/src/rememberstack/client.py index 8d68db45..1f956cb4 100644 --- a/src/rememberstack/client.py +++ b/src/rememberstack/client.py @@ -3,7 +3,11 @@ from rememberstack.model.client import ConnectorCreate from rememberstack.model.client import ConnectorDescriptor from rememberstack.model.client import ConnectorNotFoundError +from rememberstack.model.client import PipelineReadinessReport +from rememberstack.model.client import PipelineStageReadiness +from rememberstack.model.client import ProjectionReadiness from rememberstack.model.client import ToolDescriptor +from rememberstack.model.client import VersionPipelineReadiness from rememberstack.surfaces.sdk import ClientSettings from rememberstack.surfaces.sdk import MemoryApiError from rememberstack.surfaces.sdk import MemoryClient @@ -15,5 +19,9 @@ "ConnectorNotFoundError", "MemoryApiError", "MemoryClient", + "PipelineReadinessReport", + "PipelineStageReadiness", + "ProjectionReadiness", "ToolDescriptor", + "VersionPipelineReadiness", ) diff --git a/src/rememberstack/core/recipe_linter.py b/src/rememberstack/core/recipe_linter.py index 17277ace..50b587bc 100644 --- a/src/rememberstack/core/recipe_linter.py +++ b/src/rememberstack/core/recipe_linter.py @@ -52,6 +52,7 @@ class _OpSpec: # or count live-but-expired rows — so it can never sit in a `current_facts` # recipe. `fuse` returns an evidence-grade ranking (candidates to hydrate). _OPS: dict[str, _OpSpec] = { + "resolve": _OpSpec(Grain.FACT, validity_filtered=False), "lookup_relations": _OpSpec(Grain.FACT, validity_filtered=True), "lookup_observations": _OpSpec(Grain.FACT, validity_filtered=True), "aggregate": _OpSpec(Grain.FACT, validity_filtered=False), @@ -60,6 +61,8 @@ class _OpSpec: "transcript": _OpSpec(Grain.COMPOSITE, validity_filtered=False), "delta": _OpSpec(Grain.COMPOSITE, validity_filtered=False), "pages_about": _OpSpec(Grain.COMPILED, validity_filtered=False), + "graph_neighborhood": _OpSpec(Grain.FACT, validity_filtered=True), + "graph_path": _OpSpec(Grain.FACT, validity_filtered=True), "fuse": _OpSpec(Grain.EVIDENCE, validity_filtered=False, min_inputs=1), } diff --git a/src/rememberstack/model/__init__.py b/src/rememberstack/model/__init__.py index 4b09c869..ceaf7a43 100644 --- a/src/rememberstack/model/__init__.py +++ b/src/rememberstack/model/__init__.py @@ -41,7 +41,11 @@ from rememberstack.model.client import ConnectorCreate from rememberstack.model.client import ConnectorDescriptor from rememberstack.model.client import ConnectorNotFoundError +from rememberstack.model.client import PipelineReadinessReport +from rememberstack.model.client import PipelineStageReadiness +from rememberstack.model.client import ProjectionReadiness from rememberstack.model.client import ToolDescriptor +from rememberstack.model.client import VersionPipelineReadiness from rememberstack.model.clustering import ClusterConfig from rememberstack.model.clustering import MergeProposal from rememberstack.model.clustering import NeighborhoodReport @@ -220,6 +224,7 @@ from rememberstack.model.model_provider import GeneratedResponse from rememberstack.model.model_provider import ModelRequest from rememberstack.model.model_provider import ProviderAccountingError +from rememberstack.model.model_provider import ProviderCallError from rememberstack.model.model_provider import ProviderCallUsage from rememberstack.model.model_provider import StructuredResponseModel from rememberstack.model.mounts import PublishedMounts @@ -420,7 +425,9 @@ "PackedChunk", "PerimeterCredential", "PipelineComponent", + "PipelineReadinessReport", "PipelineStage", + "PipelineStageReadiness", "PipelineRouteStatus", "PoisonTargetRecord", "PoisonTargetReport", @@ -428,7 +435,9 @@ "ProcessingStatus", "ProcessingTarget", "ProjectionSnapshotState", + "ProjectionReadiness", "ProviderAccountingError", + "ProviderCallError", "ProviderCallUsage", "PublishedMounts", "QueueRoute", @@ -580,6 +589,7 @@ "UnroutableMimeError", "UploadRecord", "Validity", + "VersionPipelineReadiness", "WorkLedgerError", "WorkNotDeadLetterError", "WorkNotFoundError", diff --git a/src/rememberstack/model/client.py b/src/rememberstack/model/client.py index a0ae53d0..55b44a1b 100644 --- a/src/rememberstack/model/client.py +++ b/src/rememberstack/model/client.py @@ -1,5 +1,6 @@ """Client-surface values that are safe in the dependency-light base install.""" +from datetime import datetime from typing import Literal from typing import Self from uuid import UUID @@ -40,6 +41,58 @@ class ToolDescriptor(BaseModel): answer_intent: str +class PipelineStageReadiness(BaseModel): + """One expected document-version stage at the public readiness boundary.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + stage: str + component_version: str + status: Literal[ + "missing", "pending", "running", "succeeded", "failed", "dead_letter", "skipped" + ] + finished_at: datetime | None = None + + +class VersionPipelineReadiness(BaseModel): + """The complete expected continuous pipeline state for one version.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + version_id: UUID + ready: bool + stages: tuple[PipelineStageReadiness, ...] + + +class ProjectionReadiness(BaseModel): + """Whether one aggregate projection began after the requested E work.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + plane: Literal["P2_graph", "P3_corpusfs"] + ready: bool + version: str | None = None + built_at: datetime | None = None + published_at: datetime | None = None + + +class PipelineReadinessReport(BaseModel): + """Machine-verifiable E/P readiness for a bounded set of versions.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + ready: bool + versions: tuple[VersionPipelineReadiness, ...] + projections: tuple[ProjectionReadiness, ...] + model_bindings: dict[str, str] = Field( + default_factory=dict, + description=( + "Current non-secret serving-process configuration; this is not" + " processing-time provenance for the requested versions." + ), + ) + + class ConnectorCreate(BaseModel): """Deployment-side connector configuration sent by a client. diff --git a/src/rememberstack/model/model_provider.py b/src/rememberstack/model/model_provider.py index 6c0d0d79..5fedf686 100644 --- a/src/rememberstack/model/model_provider.py +++ b/src/rememberstack/model/model_provider.py @@ -34,6 +34,15 @@ class ProviderCallUsage(BaseModel): latency_ms: int = Field(ge=0) +class ProviderCallError(Exception): + """A provider call failed after it may already have reported billable usage.""" + + def __init__(self, message: str, *, usage: ProviderCallUsage | None = None) -> None: + """Keep parsed usage available to the worker's authoritative meter.""" + super().__init__(message) + self.usage = usage + + class GeneratedResponse(BaseModel, Generic[ResponseT]): """A validated structured output paired with its provider accounting.""" diff --git a/src/rememberstack/profiles/selfhost.py b/src/rememberstack/profiles/selfhost.py index 5a34bf43..1c5e735d 100644 --- a/src/rememberstack/profiles/selfhost.py +++ b/src/rememberstack/profiles/selfhost.py @@ -29,14 +29,27 @@ from rememberstack.spine import DeploymentBootstrapper from rememberstack.spine import RecipeRegistry from rememberstack.spine import seed_canonical_recipes +from rememberstack.spine import seed_graph_recipes from rememberstack.spine.settings import load_database_settings if TYPE_CHECKING: from fastapi import FastAPI from rememberstack.adapters.selfhost import SelfHostWorkerLoop - -_SUPPORTED_WORKER_STAGES = (PipelineStage.CONVERT, PipelineStage.STRUCTURE) + from rememberstack.workers import StageHandler + +_SUPPORTED_WORKER_STAGES = ( + PipelineStage.CONVERT, + PipelineStage.STRUCTURE, + PipelineStage.CHUNK, + PipelineStage.EMBED_CHUNK, + PipelineStage.EXTRACT_CLAIMS, + PipelineStage.NORMALIZE_RELATIONS, + PipelineStage.ADJUDICATE_SUPERSESSION, + PipelineStage.EMBED_CLAIM, + PipelineStage.RECONCILE, + PipelineStage.LABEL_RELATION, +) class SelfHostSettings(BaseSettings): @@ -53,7 +66,10 @@ class SelfHostSettings(BaseSettings): raw_bucket_name: str = Field(default="remember-raw", min_length=1) artifacts_bucket_name: str = Field(default="remember-artifacts", min_length=1) corpusfs_bucket_name: str = Field(default="remember-corpusfs", min_length=1) + snapshot_bucket_name: str = Field(default="remember-snapshots", min_length=1) lance_root: Path = Path("/var/lib/rememberstack/lance") + projection_work_root: Path = Path("/var/lib/rememberstack/projection-work") + graph_cache_root: Path = Path("/var/lib/rememberstack/graph-cache") forget_manifest_root: Path = Path("/var/lib/rememberstack/forget-manifests") migration_config: Path = Path("alembic.ini") api_host: str = "0.0.0.0" @@ -88,7 +104,7 @@ def ensure_ready(self, *, deployment_id: UUID) -> tuple[UUID, ...]: class SelfHostProfile: - """Compose the existing API and E0 workers over PostgreSQL, MinIO, and Lance.""" + """Compose the complete continuous E/P1 path plus aggregate P2/P3 builds.""" def __init__( self, @@ -98,6 +114,7 @@ def __init__( raw_store: MinIOObjectStore, artifact_store: MinIOObjectStore, corpusfs_store: MinIOObjectStore, + snapshot_store: MinIOObjectStore, model_provider: OpenRouterModelProvider, ) -> None: """Retain one dependency graph for an API, setup, or worker process.""" @@ -106,6 +123,7 @@ def __init__( self._raw_store = raw_store self._artifact_store = artifact_store self._corpusfs_store = corpusfs_store + self._snapshot_store = snapshot_store self._model_provider = model_provider @classmethod @@ -127,6 +145,9 @@ def from_settings(cls) -> Self: corpusfs_store=MinIOObjectStore( bucket=profile_settings.corpusfs_bucket_name, settings=minio_settings ), + snapshot_store=MinIOObjectStore( + bucket=profile_settings.snapshot_bucket_name, settings=minio_settings + ), model_provider=OpenRouterModelProvider( settings=OpenRouterSettings.model_validate({}) ), @@ -146,7 +167,10 @@ def setup(self) -> None: self._raw_store.ensure_bucket() self._artifact_store.ensure_bucket() self._corpusfs_store.ensure_bucket() + self._snapshot_store.ensure_bucket() self._settings.forget_manifest_root.mkdir(parents=True, exist_ok=True) + self._settings.projection_work_root.mkdir(parents=True, exist_ok=True) + self._settings.graph_cache_root.mkdir(parents=True, exist_ok=True) DeploymentBootstrapper(engine=self._engine).bootstrap_deployment( deployment_input=DeploymentBootstrapInput( deployment_id=self._settings.deployment_id, @@ -162,23 +186,42 @@ def setup(self) -> None: registry=RecipeRegistry(engine=self._engine), deployment_id=self._settings.deployment_id, ) + seed_graph_recipes( + registry=RecipeRegistry(engine=self._engine), + deployment_id=self._settings.deployment_id, + ) def api(self) -> FastAPI: """Build the existing HTTP surface over this self-host dependency graph.""" from rememberstack.adapters.selfhost.lance import LanceChunkIndex from rememberstack.spine import DocumentCatalog from rememberstack.spine import ForgetCatalog + from rememberstack.spine import PipelineReadinessCatalog + from rememberstack.spine import ProjectionCatalog from rememberstack.surfaces import build_api + from rememberstack.surfaces import GraphQueries from rememberstack.surfaces import QueryEngine from rememberstack.surfaces import RecipeExecutor from rememberstack.surfaces import RecipeSurface + from rememberstack.workers import GraphSnapshotReader + from rememberstack.workers import P1Settings from rememberstack.workers.e0 import UploadIngestor + p1_settings = P1Settings.model_validate({}) + projection_catalog = ProjectionCatalog(engine=self._engine) + graph_queries = GraphQueries( + reader=GraphSnapshotReader( + catalog=projection_catalog, + snapshot_store=self._snapshot_store, + deployment_id=self._settings.deployment_id, + cache_dir=self._settings.graph_cache_root, + ) + ) query_engine = QueryEngine( engine=self._engine, search_index=LanceChunkIndex(root=self._settings.lance_root), model_provider=self._model_provider, - embedding_model="qwen/qwen3-embedding-8b", + embedding_model=p1_settings.embedding_model, ) app = build_api( engine=query_engine, @@ -191,7 +234,9 @@ def api(self) -> FastAPI: ), surface=RecipeSurface( registry=RecipeRegistry(engine=self._engine), - executor=RecipeExecutor(query_engine=query_engine), + executor=RecipeExecutor( + query_engine=query_engine, graph_queries=graph_queries + ), deployment_id=self._settings.deployment_id, ), ingest=UploadIngestor( @@ -199,6 +244,12 @@ def api(self) -> FastAPI: raw_store=self._raw_store, admission=ForgetCatalog(engine=self._engine), ), + pipeline_readiness=PipelineReadinessCatalog( + engine=self._engine, + expected_components=_expected_components(), + projections=projection_catalog, + model_bindings=_model_bindings(), + ), ) @app.get("/healthz", include_in_schema=False) @@ -211,50 +262,28 @@ def healthz() -> dict[str, str]: return app def worker_loop(self, *, stage: PipelineStage) -> SelfHostWorkerLoop: - """Build one E0 route's ordinary LISTEN/NOTIFY worker loop.""" + """Build one continuous route's ordinary LISTEN/NOTIFY worker loop.""" + from rememberstack.adapters.selfhost import JsonLineTelemetry from rememberstack.adapters.selfhost import SelfHostTaskQueue from rememberstack.adapters.selfhost import SelfHostWorkerLoop from rememberstack.adapters.selfhost import TokenBucket - from rememberstack.core import ConversionRouter - from rememberstack.core import MarkdownPassthroughConverter from rememberstack.model import ProcessingLane - from rememberstack.spine import DocumentCatalog from rememberstack.spine import WorkLedger from rememberstack.spine import WorkLedgerSettings - from rememberstack.workers import ConvertHandler from rememberstack.workers import HandlerRegistry - from rememberstack.workers import StructureHandler from rememberstack.workers import Worker if stage not in _SUPPORTED_WORKER_STAGES: - raise ValueError(f"the Compose skeleton has no handler for stage {stage}") - catalog = DocumentCatalog(engine=self._engine) + raise ValueError(f"the self-host profile has no handler for stage {stage}") registry = HandlerRegistry() - if stage is PipelineStage.CONVERT: - registry.register( - stage=stage, - handler=ConvertHandler( - catalog=catalog, - raw_store=self._raw_store, - artifact_store=self._artifact_store, - router=ConversionRouter( - routes={"text/markdown": MarkdownPassthroughConverter()} - ), - ), - ) - else: - registry.register( - stage=stage, - handler=StructureHandler( - catalog=catalog, - artifact_store=self._artifact_store, - model_provider=self._model_provider, - ), - ) + registry.register(stage=stage, handler=self._handler(stage=stage)) ledger = WorkLedger(engine=self._engine, settings=WorkLedgerSettings()) return SelfHostWorkerLoop( worker=Worker( - ledger=ledger, registry=registry, queue=SelfHostTaskQueue(ledger=ledger) + ledger=ledger, + registry=registry, + queue=SelfHostTaskQueue(ledger=ledger), + telemetry=JsonLineTelemetry(), ), deployment_id=self._settings.deployment_id, stage=stage, @@ -268,11 +297,178 @@ def worker_loop(self, *, stage: PipelineStage) -> SelfHostWorkerLoop: ) def run_worker(self, *, stage: PipelineStage) -> None: - """Run one configured E0 route until the process is stopped or fails.""" + """Run one configured continuous route until stopped or failed.""" loop = self.worker_loop(stage=stage) while True: loop.run_for(duration_s=self._settings.worker_session_s) + def run_projection(self, *, plane: str) -> dict[str, object]: + """Build P2, P3, or both once after continuous ingestion settles.""" + from rememberstack.spine import ForgetCatalog + from rememberstack.spine import ProjectionCatalog + from rememberstack.workers import CorpusFsBuilder + from rememberstack.workers import GraphRebuildWorker + + ForgetCatalog(engine=self._engine).assert_available( + deployment_id=self._settings.deployment_id + ) + catalog = ProjectionCatalog(engine=self._engine) + reports: dict[str, object] = {} + if plane in {"p2", "all"}: + reports["p2"] = GraphRebuildWorker( + catalog=catalog, snapshot_store=self._snapshot_store + ).rebuild( + deployment_id=self._settings.deployment_id, + workdir=self._settings.projection_work_root, + ) + if plane in {"p3", "all"}: + reports["p3"] = CorpusFsBuilder( + catalog=catalog, snapshot_store=self._corpusfs_store + ).build(deployment_id=self._settings.deployment_id) + if not reports: + raise ValueError(f"unknown projection plane {plane!r}") + return reports + + def _handler(self, *, stage: PipelineStage) -> StageHandler: + """Compose exactly one implemented stage handler for one worker process.""" + from rememberstack.adapters.selfhost.lance import LanceChunkIndex + from rememberstack.core import chunker_version + from rememberstack.core import ChunkerParams + from rememberstack.core import ConversionRouter + from rememberstack.core import MarkdownPassthroughConverter + from rememberstack.model import ResolverConfig + from rememberstack.spine import CascadeResolver + from rememberstack.spine import ChunkCatalog + from rememberstack.spine import ClaimCatalog + from rememberstack.spine import DocumentCatalog + from rememberstack.spine import EntityRegistry + from rememberstack.spine import FactCatalog + from rememberstack.spine import LifecycleCatalog + from rememberstack.spine import ObservationAdjudicator + from rememberstack.spine import ObservationSettings + from rememberstack.spine import RESOLVER_VERSION + from rememberstack.spine import ReviewQueue + from rememberstack.spine import SupersessionAdjudicator + from rememberstack.spine import SupersessionSettings + from rememberstack.workers import AdjudicateSupersessionHandler + from rememberstack.workers import ChunkHandler + from rememberstack.workers import ConvertHandler + from rememberstack.workers import E1Settings + from rememberstack.workers import E2Settings + from rememberstack.workers import E3Settings + from rememberstack.workers import EmbedChunksHandler + from rememberstack.workers import EmbedClaimsHandler + from rememberstack.workers import ExtractClaimsHandler + from rememberstack.workers import LabelFactsHandler + from rememberstack.workers import NormalizeRelationsHandler + from rememberstack.workers import P1Settings + from rememberstack.workers import ReconcileHandler + from rememberstack.workers import StructureHandler + from rememberstack.workers import StructurerSettings + + documents = DocumentCatalog(engine=self._engine) + chunks = ChunkCatalog(engine=self._engine) + claims = ClaimCatalog(engine=self._engine) + facts = FactCatalog(engine=self._engine) + index = LanceChunkIndex(root=self._settings.lance_root) + params = ChunkerParams() + chunk_generation = chunker_version(params=params) + p1_settings = P1Settings.model_validate({}) + if stage is PipelineStage.CONVERT: + return ConvertHandler( + catalog=documents, + raw_store=self._raw_store, + artifact_store=self._artifact_store, + router=ConversionRouter( + routes={"text/markdown": MarkdownPassthroughConverter()} + ), + ) + if stage is PipelineStage.STRUCTURE: + return StructureHandler( + catalog=documents, + artifact_store=self._artifact_store, + model_provider=self._model_provider, + settings=StructurerSettings.model_validate({}), + ) + if stage is PipelineStage.CHUNK: + return ChunkHandler( + catalog=chunks, artifact_store=self._artifact_store, params=params + ) + if stage is PipelineStage.EMBED_CHUNK: + return EmbedChunksHandler( + catalog=chunks, + artifact_store=self._artifact_store, + model_provider=self._model_provider, + chunk_index=index, + settings=E1Settings.model_validate({}), + params=params, + ) + if stage is PipelineStage.EXTRACT_CLAIMS: + return ExtractClaimsHandler( + catalog=claims, + chunk_catalog=chunks, + artifact_store=self._artifact_store, + model_provider=self._model_provider, + settings=E2Settings.model_validate({}), + chunker_version=chunk_generation, + ) + if stage is PipelineStage.NORMALIZE_RELATIONS: + observation_settings = ObservationSettings.model_validate({}) + return NormalizeRelationsHandler( + claim_catalog=claims, + chunk_catalog=chunks, + registry=EntityRegistry(engine=self._engine), + resolver=CascadeResolver( + engine=self._engine, + entity_index=index, + model_provider=self._model_provider, + config=ResolverConfig(resolver_version=RESOLVER_VERSION), + embedding_model=observation_settings.embedding_model, + small_model=observation_settings.small_model, + frontier_model=observation_settings.frontier_model, + ), + facts=facts, + observation_adjudicator=ObservationAdjudicator( + engine=self._engine, + model_provider=self._model_provider, + settings=observation_settings, + ), + model_provider=self._model_provider, + settings=E3Settings.model_validate({}), + chunker_version=chunk_generation, + ) + if stage is PipelineStage.ADJUDICATE_SUPERSESSION: + return AdjudicateSupersessionHandler( + adjudicator=SupersessionAdjudicator( + engine=self._engine, + model_provider=self._model_provider, + settings=SupersessionSettings.model_validate({}), + ) + ) + if stage is PipelineStage.EMBED_CLAIM: + return EmbedClaimsHandler( + claim_catalog=claims, + chunk_catalog=chunks, + model_provider=self._model_provider, + claim_index=index, + settings=p1_settings, + chunker_version=chunk_generation, + ) + if stage is PipelineStage.RECONCILE: + return ReconcileHandler( + catalog=LifecycleCatalog(engine=self._engine), + review_queue=ReviewQueue(engine=self._engine), + chunker_version=chunk_generation, + ) + if stage is PipelineStage.LABEL_RELATION: + return LabelFactsHandler( + facts=facts, + model_provider=self._model_provider, + fact_index=index, + settings=p1_settings, + ) + raise ValueError(f"the self-host profile has no handler for stage {stage}") + def create_api() -> FastAPI: """Uvicorn factory for the self-host API process.""" @@ -280,17 +476,21 @@ def create_api() -> FastAPI: def main(argv: list[str] | None = None) -> int: - """Run setup, API, or one E0 worker process for Docker Compose.""" + """Run setup, API, one continuous worker, or an aggregate projection.""" parser = argparse.ArgumentParser(description="rememberstack self-host profile") subparsers = parser.add_subparsers(dest="command", required=True) subparsers.add_parser("setup", help="migrate and bootstrap the deployment") subparsers.add_parser("api", help="serve the deployment HTTP API") - worker = subparsers.add_parser("worker", help="run one E0 worker route") + worker = subparsers.add_parser("worker", help="run one continuous worker route") worker.add_argument( "--stage", choices=tuple(stage.value for stage in _SUPPORTED_WORKER_STAGES), required=True, ) + projection = subparsers.add_parser( + "project", help="build aggregate projections once" + ) + projection.add_argument("--plane", choices=("p2", "p3", "all"), required=True) args = parser.parse_args(argv) settings = SelfHostSettings.model_validate({}) if args.command == "api": @@ -308,6 +508,9 @@ def main(argv: list[str] | None = None) -> int: if args.command == "setup": profile.setup() return 0 + if args.command == "project": + print(profile.run_projection(plane=args.plane)) + return 0 profile.run_worker(stage=PipelineStage(args.stage)) return 0 finally: @@ -320,5 +523,65 @@ def _psycopg_url() -> str: return url.set(drivername="postgresql").render_as_string(hide_password=False) +def _expected_components() -> dict[PipelineStage, str]: + """The exact ten continuous generations composed by this profile.""" + from rememberstack.spine import ADJUDICATOR_VERSION + from rememberstack.workers import E0_CONVERT_VERSION + from rememberstack.workers import E0_STRUCTURE_VERSION + from rememberstack.workers import E1_CHUNK_VERSION + from rememberstack.workers import E1_EMBED_VERSION + from rememberstack.workers import E2_EXTRACTOR_VERSION + from rememberstack.workers import E3_NORMALIZER_VERSION + from rememberstack.workers import FACT_LABEL_VERSION + from rememberstack.workers import P1_EMBED_CLAIMS_VERSION + from rememberstack.workers import RECONCILE_VERSION + + return { + PipelineStage.CONVERT: E0_CONVERT_VERSION, + PipelineStage.STRUCTURE: E0_STRUCTURE_VERSION, + PipelineStage.CHUNK: E1_CHUNK_VERSION, + PipelineStage.EMBED_CHUNK: E1_EMBED_VERSION, + PipelineStage.EXTRACT_CLAIMS: E2_EXTRACTOR_VERSION, + PipelineStage.NORMALIZE_RELATIONS: E3_NORMALIZER_VERSION, + PipelineStage.ADJUDICATE_SUPERSESSION: ADJUDICATOR_VERSION, + PipelineStage.EMBED_CLAIM: P1_EMBED_CLAIMS_VERSION, + PipelineStage.RECONCILE: RECONCILE_VERSION, + PipelineStage.LABEL_RELATION: FACT_LABEL_VERSION, + } + + +def _model_bindings() -> dict[str, str]: + """Non-secret provider model identities used by the composed pipeline.""" + from rememberstack.spine import ObservationSettings + from rememberstack.spine import SupersessionSettings + from rememberstack.workers import E1Settings + from rememberstack.workers import E2Settings + from rememberstack.workers import E3Settings + from rememberstack.workers import P1Settings + from rememberstack.workers import StructurerSettings + + structurer = StructurerSettings.model_validate({}) + e1 = E1Settings.model_validate({}) + e2 = E2Settings.model_validate({}) + e3 = E3Settings.model_validate({}) + observations = ObservationSettings.model_validate({}) + supersession = SupersessionSettings.model_validate({}) + p1 = P1Settings.model_validate({}) + return { + "structure": structurer.model, + "chunk_embedding": e1.embedding_model, + "context_prefix": e1.prefix_model, + "claim_extraction": e2.extract_model, + "relation_normalization": e3.normalize_model, + "entity_observation_embedding": observations.embedding_model, + "observation_small": observations.small_model, + "observation_frontier": observations.frontier_model, + "supersession_small": supersession.small_model, + "supersession_frontier": supersession.frontier_model, + "p1_embedding": p1.embedding_model, + "fact_label": p1.label_model, + } + + if __name__ == "__main__": sys.exit(main()) diff --git a/src/rememberstack/spine/__init__.py b/src/rememberstack/spine/__init__.py index b7990d7c..1f74b841 100644 --- a/src/rememberstack/spine/__init__.py +++ b/src/rememberstack/spine/__init__.py @@ -31,9 +31,12 @@ from rememberstack.spine.operations import OperationalCatalog from rememberstack.spine.operations import OperationalSettings from rememberstack.spine.projection import ProjectionCatalog +from rememberstack.spine.readiness import PipelineReadinessCatalog from rememberstack.spine.recipes import CANONICAL_RECIPES +from rememberstack.spine.recipes import GRAPH_RECIPES from rememberstack.spine.recipes import RecipeRegistry from rememberstack.spine.recipes import seed_canonical_recipes +from rememberstack.spine.recipes import seed_graph_recipes from rememberstack.spine.resolver import CascadeResolver from rememberstack.spine.resolver import RESOLVER_VERSION from rememberstack.spine.resolver import seed_resolver_version @@ -76,9 +79,12 @@ "KnowledgeControlPlane", "KnowledgeDispatchUnavailableError", "ProjectionCatalog", + "PipelineReadinessCatalog", "CANONICAL_RECIPES", + "GRAPH_RECIPES", "RecipeRegistry", "seed_canonical_recipes", + "seed_graph_recipes", "ReviewQueue", "seed_resolver_version", "T0_RESOLVER_VERSION", diff --git a/src/rememberstack/spine/readiness.py b/src/rememberstack/spine/readiness.py new file mode 100644 index 00000000..06b0382a --- /dev/null +++ b/src/rememberstack/spine/readiness.py @@ -0,0 +1,153 @@ +"""Machine-verifiable readiness for the ordinary self-host pipeline. + +The work ledger remains authoritative. This read model checks the exact +component generations composed by a profile for each requested document +version, then verifies that P2 and P3 builds began after those terminal +E-stage rows. Publication time alone is insufficient: an older build can +finish after newer document work. This read does not execute work or hide +failures. +""" + +from collections.abc import Mapping +from datetime import datetime +from uuid import UUID + +from sqlalchemy import bindparam +from sqlalchemy import text +from sqlalchemy.engine import Engine + +from rememberstack.model import PipelineReadinessReport +from rememberstack.model import PipelineStage +from rememberstack.model import PipelineStageReadiness +from rememberstack.model import ProjectionReadiness +from rememberstack.model import VersionPipelineReadiness +from rememberstack.spine.projection import ProjectionCatalog + +_PLANES = ("P2_graph", "P3_corpusfs") + + +class PipelineReadinessCatalog: + """Read exact per-version stage and aggregate-projection completion.""" + + def __init__( + self, + *, + engine: Engine, + expected_components: Mapping[PipelineStage, str], + projections: ProjectionCatalog, + model_bindings: Mapping[str, str] | None = None, + ) -> None: + """Bind the spine and the component generations this process serves.""" + self._engine = engine + self._expected = tuple(expected_components.items()) + self._projections = projections + self._model_bindings = dict(model_bindings or {}) + + def inspect( + self, + *, + deployment_id: UUID, + version_ids: tuple[UUID, ...], + require_projections: bool, + ) -> PipelineReadinessReport: + """Return readiness without mutating or waiting for the pipeline.""" + version_ids = tuple(dict.fromkeys(version_ids)) + if not version_ids: + raise ValueError("pipeline readiness requires at least one version_id") + with self._engine.connect() as connection: + rows = ( + connection.execute( + _VERSION_WORK, + {"deployment_id": deployment_id, "version_ids": version_ids}, + ) + .mappings() + .all() + ) + by_key = { + ( + UUID(str(row["target_id"])), + PipelineStage(str(row["stage"])), + str(row["component_version"]), + ): row + for row in rows + } + versions: list[VersionPipelineReadiness] = [] + terminal_at = None + for version_id in version_ids: + stages: list[PipelineStageReadiness] = [] + for stage, component_version in self._expected: + row = by_key.get((version_id, stage, component_version)) + status = "missing" if row is None else str(row["status"]) + finished_at = None if row is None else row["finished_at"] + stages.append( + PipelineStageReadiness.model_validate( + { + "stage": stage.value, + "component_version": component_version, + "status": status, + "finished_at": finished_at, + } + ) + ) + if finished_at is not None and ( + terminal_at is None or finished_at > terminal_at + ): + terminal_at = finished_at + versions.append( + VersionPipelineReadiness( + version_id=version_id, + ready=all( + item.status in {"succeeded", "skipped"} + and item.finished_at is not None + for item in stages + ), + stages=tuple(stages), + ) + ) + projection_states: list[ProjectionReadiness] = [] + for plane in _PLANES: + latest = self._projections.latest_snapshot( + deployment_id=deployment_id, plane=plane + ) + raw_built_at = None if latest is None else latest["built_at"] + built_at = raw_built_at if isinstance(raw_built_at, datetime) else None + raw_published_at = None if latest is None else latest["published_at"] + published_at = ( + raw_published_at if isinstance(raw_published_at, datetime) else None + ) + fresh = ( + latest is not None + and built_at is not None + and published_at is not None + and terminal_at is not None + and built_at >= terminal_at + ) + projection_states.append( + ProjectionReadiness( + plane=plane, + ready=fresh, + version=None if latest is None else str(latest["version"]), + built_at=built_at, + published_at=published_at, + ) + ) + versions_ready = all(version.ready for version in versions) + projections_ready = all(item.ready for item in projection_states) + return PipelineReadinessReport( + ready=versions_ready and (projections_ready or not require_projections), + versions=tuple(versions), + projections=tuple(projection_states), + model_bindings=self._model_bindings, + ) + + +_VERSION_WORK = text( + """ + SELECT target_id, stage::text AS stage, component_version, + status::text AS status, finished_at + FROM processing_state + WHERE deployment_id = :deployment_id + AND target_kind = 'document_version' + AND target_id IN :version_ids + """ +).bindparams(bindparam("version_ids", expanding=True)) diff --git a/src/rememberstack/spine/recipes.py b/src/rememberstack/spine/recipes.py index 58ebf198..2d89c525 100644 --- a/src/rememberstack/spine/recipes.py +++ b/src/rememberstack/spine/recipes.py @@ -148,6 +148,23 @@ def _recipe_from_row(row: RowMapping) -> Recipe: # exactly its chain. Adding a query pattern is adding an entry here. # ───────────────────────────────────────────────────────────────────────── CANONICAL_RECIPES: tuple[Recipe, ...] = ( + Recipe( + name="resolve_entity", + description="Resolve a name to ranked current entity candidates before" + " using UUID-addressed fact or graph tools. Returns every exact-name" + " candidate rather than silently guessing.", + parameters={ + "name": {"type": "string", "required": True}, + "entity_type": {"type": "string", "required": False}, + }, + chain=( + RecipeStep( + op="resolve", bind={"name": "name", "entity_type": "entity_type"} + ), + ), + output_grain=Grain.FACT, + answer_intent=RecipeAnswerIntent.ORIENTATION, + ), Recipe( name="relation_current", description="Current relations matching a subject and optional predicate" @@ -198,11 +215,18 @@ def _recipe_from_row(row: RowMapping) -> Recipe: " (S6). Evidence grain — never a current-fact answer (the D41 bar).", parameters={ "query": {"type": "string", "required": True}, - "k": {"type": "integer", "required": False, "default": 10}, + "k": { + "type": "integer", + "required": False, + "default": 10, + "minimum": 1, + "maximum": 30, + }, }, chain=(RecipeStep(op="search_claims", bind={"query": "query", "k": "k"}),), output_grain=Grain.EVIDENCE, answer_intent=RecipeAnswerIntent.ASSERTION_HISTORY, + version=2, ), Recipe( name="claims_hybrid_rrf", @@ -210,7 +234,13 @@ def _recipe_from_row(row: RowMapping) -> Recipe: " orderings by reciprocal-rank fusion (S46). Evidence grain.", parameters={ "query": {"type": "string", "required": True}, - "k": {"type": "integer", "required": False, "default": 10}, + "k": { + "type": "integer", + "required": False, + "default": 10, + "minimum": 1, + "maximum": 30, + }, }, chain=( RecipeStep(op="search_claims", bind={"query": "query", "k": "k"}), @@ -219,6 +249,7 @@ def _recipe_from_row(row: RowMapping) -> Recipe: ), output_grain=Grain.EVIDENCE, answer_intent=RecipeAnswerIntent.ASSERTION_HISTORY, + version=2, ), Recipe( name="explain", @@ -264,6 +295,67 @@ def _recipe_from_row(row: RowMapping) -> Recipe: ), ) +GRAPH_RECIPES: tuple[Recipe, ...] = ( + Recipe( + name="graph_neighborhood", + description="Current P2 graph neighborhood around an entity, ranked by" + " distance and carrying explicit truncation metadata.", + parameters={ + "entity_id": {"type": "uuid", "required": True}, + "hops": { + "type": "integer", + "required": False, + "default": 2, + "minimum": 1, + "maximum": 4, + }, + "limit": { + "type": "integer", + "required": False, + "default": 30, + "minimum": 1, + "maximum": 50, + }, + }, + chain=( + RecipeStep( + op="graph_neighborhood", + bind={"entity_id": "entity_id", "hops": "hops", "limit": "limit"}, + ), + ), + output_grain=Grain.FACT, + answer_intent=RecipeAnswerIntent.ORIENTATION, + ), + Recipe( + name="graph_path", + description="Current shortest P2 paths between two resolved entities," + " with every traversed fact edge returned for inspection.", + parameters={ + "from_entity_id": {"type": "uuid", "required": True}, + "to_entity_id": {"type": "uuid", "required": True}, + "max_hops": { + "type": "integer", + "required": False, + "default": 4, + "minimum": 1, + "maximum": 6, + }, + }, + chain=( + RecipeStep( + op="graph_path", + bind={ + "from_entity_id": "from_entity_id", + "to_entity_id": "to_entity_id", + "max_hops": "max_hops", + }, + ), + ), + output_grain=Grain.FACT, + answer_intent=RecipeAnswerIntent.ORIENTATION, + ), +) + def seed_canonical_recipes(*, registry: RecipeRegistry, deployment_id: UUID) -> int: """Register the canonical recipe set into a deployment (idempotent). @@ -274,3 +366,10 @@ def seed_canonical_recipes(*, registry: RecipeRegistry, deployment_id: UUID) -> for recipe in CANONICAL_RECIPES: registry.register(deployment_id=deployment_id, recipe=recipe) return len(CANONICAL_RECIPES) + + +def seed_graph_recipes(*, registry: RecipeRegistry, deployment_id: UUID) -> int: + """Seed P2 recipes only for profiles that actually compose graph queries.""" + for recipe in GRAPH_RECIPES: + registry.register(deployment_id=deployment_id, recipe=recipe) + return len(GRAPH_RECIPES) diff --git a/src/rememberstack/surfaces/http_api.py b/src/rememberstack/surfaces/http_api.py index 09506f86..94591f53 100644 --- a/src/rememberstack/surfaces/http_api.py +++ b/src/rememberstack/surfaces/http_api.py @@ -17,6 +17,7 @@ from datetime import datetime from datetime import timedelta from typing import Annotated +from typing import Final from typing import Literal from typing import Protocol from uuid import UUID @@ -38,6 +39,7 @@ from rememberstack.model import ForgetInProgressError from rememberstack.model import IngestedVersion from rememberstack.model import PerimeterCredential +from rememberstack.model import PipelineReadinessReport from rememberstack.model import ToolDescriptor from rememberstack.ports.auth import AuthPerimeterPort from rememberstack.surfaces.query_engine import QueryEngine @@ -47,6 +49,9 @@ from rememberstack.surfaces.recipe_surface import RecipeSurface from rememberstack.surfaces.recipe_surface import UnknownRecipeError +PIPELINE_READINESS_VERSION_LIMIT: Final = 1_000 +"""Maximum document versions in one read-only readiness inspection.""" + class IngestPort(Protocol): """The E0 ingest operations the HTTP surface may expose.""" @@ -103,6 +108,18 @@ def ensure_ready(self, *, deployment_id: UUID) -> tuple[UUID, ...]: ... +class PipelineReadinessPort(Protocol): + """Inspect ordinary per-version and aggregate projection completion.""" + + def inspect( + self, + *, + deployment_id: UUID, + version_ids: tuple[UUID, ...], + require_projections: bool, + ) -> PipelineReadinessReport: ... + + def build_api( *, engine: QueryEngine, @@ -113,6 +130,7 @@ def build_api( auth: AuthPerimeterPort | None = None, ingest: IngestPort | None = None, connectors: ConnectorManagementPort | None = None, + pipeline_readiness: PipelineReadinessPort | None = None, ) -> FastAPI: """Build one deployment's query API over a composed engine. @@ -212,10 +230,34 @@ def hydrate_relation(relation_id: UUID) -> Envelope: _mount_ingest(app=app, ingest=ingest, deployment_id=deployment_id) if connectors is not None: _mount_connectors(app=app, connectors=connectors, deployment_id=deployment_id) + if pipeline_readiness is not None: + _mount_pipeline_readiness( + app=app, readiness=pipeline_readiness, deployment_id=deployment_id + ) return app +def _mount_pipeline_readiness( + *, app: FastAPI, readiness: PipelineReadinessPort, deployment_id: UUID +) -> None: + """Expose a normal, read-only completion boundary for bounded versions.""" + + @app.post("/readiness", response_model=PipelineReadinessReport) + def pipeline_readiness( + version_ids: Annotated[ + list[UUID], Body(min_length=1, max_length=PIPELINE_READINESS_VERSION_LIMIT) + ], + require_projections: bool = True, + ) -> PipelineReadinessReport: + """Inspect exact E generations and optionally fresh P2/P3 snapshots.""" + return readiness.inspect( + deployment_id=deployment_id, + version_ids=tuple(version_ids), + require_projections=require_projections, + ) + + def _mount_recipes(*, app: FastAPI, surface: RecipeSurface) -> None: """Add the registry-rendered recipe endpoints to the app (D50).""" diff --git a/src/rememberstack/surfaces/recipe_executor.py b/src/rememberstack/surfaces/recipe_executor.py index d72993b9..bbcf8c6c 100644 --- a/src/rememberstack/surfaces/recipe_executor.py +++ b/src/rememberstack/surfaces/recipe_executor.py @@ -20,6 +20,7 @@ from rememberstack.model import Envelope from rememberstack.model import Recipe from rememberstack.model import RecipeStep +from rememberstack.surfaces.graph_queries import GraphQueries from rememberstack.surfaces.query_engine import QueryEngine @@ -30,9 +31,12 @@ class RecipeExecutionError(Exception): class RecipeExecutor: """Replay a recipe's frozen chain over the zero-LLM query primitives.""" - def __init__(self, *, query_engine: QueryEngine) -> None: - """Bind the executor to the query engine whose primitives it composes.""" + def __init__( + self, *, query_engine: QueryEngine, graph_queries: GraphQueries | None = None + ) -> None: + """Bind the ordinary primitives and an optional P2 graph surface.""" self._engine = query_engine + self._graph = graph_queries def execute( self, *, deployment_id: UUID, recipe: Recipe, arguments: dict[str, object] @@ -80,6 +84,10 @@ def _run_step( return self._engine.fuse( rankings=[rankings[index] for index in step.inputs], **kwargs ) + if step.op == "graph_neighborhood": + return self._graph_step(op=step.op, kwargs=kwargs) + if step.op == "graph_path": + return self._graph_step(op=step.op, kwargs=kwargs) handler = _SINGLE_OP_HANDLERS.get(step.op) if handler is None: raise RecipeExecutionError( @@ -87,6 +95,16 @@ def _run_step( ) return handler(self._engine, deployment_id, kwargs) + def _graph_step(self, *, op: str, kwargs: dict[str, Any]) -> Envelope: + """Run a P2 operation only when the deployment composed P2 queries.""" + if self._graph is None: + raise RecipeExecutionError( + f"recipe op {op!r} requires a composed P2 graph query surface" + ) + if op == "graph_neighborhood": + return self._graph.neighborhood(**kwargs) + return self._graph.path(**kwargs) + def _ranking_of(envelope: Envelope) -> list[UUID]: """The ordered ids of an envelope's payload — what `fuse` consumes. @@ -120,6 +138,13 @@ def _lookup_relations( return engine.lookup_relations(deployment_id=deployment_id, **kwargs) +def _resolve( + engine: QueryEngine, deployment_id: UUID, kwargs: dict[str, Any] +) -> Envelope: + """The `resolve` op.""" + return engine.resolve(deployment_id=deployment_id, **kwargs) + + def _lookup_observations( engine: QueryEngine, deployment_id: UUID, kwargs: dict[str, Any] ) -> Envelope: @@ -170,6 +195,7 @@ def _pages_about( _SINGLE_OP_HANDLERS = { + "resolve": _resolve, "lookup_relations": _lookup_relations, "lookup_observations": _lookup_observations, "aggregate": _aggregate, @@ -180,6 +206,10 @@ def _pages_about( "pages_about": _pages_about, } -EXECUTABLE_OPS = frozenset(_SINGLE_OP_HANDLERS) | {"fuse"} +EXECUTABLE_OPS = frozenset(_SINGLE_OP_HANDLERS) | { + "fuse", + "graph_neighborhood", + "graph_path", +} """Every op the executor can run. Kept equal to the linter's `KNOWN_OPS` (a test enforces it), so no chain ever lints clean only to fail at execution.""" diff --git a/src/rememberstack/surfaces/recipe_surface.py b/src/rememberstack/surfaces/recipe_surface.py index 9441a450..43ea5084 100644 --- a/src/rememberstack/surfaces/recipe_surface.py +++ b/src/rememberstack/surfaces/recipe_surface.py @@ -114,14 +114,9 @@ def descriptors(self) -> tuple[ToolDescriptor, ...]: advertises exactly that: a deployment with v1 and v2 both active shows one `relation_current`, whose schema is the one that will execute. """ - seen: set[str] = set() - descriptors: list[ToolDescriptor] = [] - for recipe in self._registry.active(deployment_id=self._deployment_id): - if recipe.name in seen: # active() is name, version DESC — first wins - continue - seen.add(recipe.name) - descriptors.append(_descriptor(recipe)) - return tuple(descriptors) + return recipe_descriptors( + recipes=self._registry.active(deployment_id=self._deployment_id) + ) def run(self, *, name: str, arguments: dict[str, object]) -> Envelope: """Run one recipe by name over coerced arguments. @@ -142,6 +137,18 @@ def run(self, *, name: str, arguments: dict[str, object]) -> Envelope: ) +def recipe_descriptors(*, recipes: tuple[Recipe, ...]) -> tuple[ToolDescriptor, ...]: + """Render the latest recipe per name into the shared public tool contract.""" + seen: set[str] = set() + descriptors: list[ToolDescriptor] = [] + for recipe in recipes: + if recipe.name in seen: + continue + seen.add(recipe.name) + descriptors.append(_descriptor(recipe)) + return tuple(descriptors) + + def _descriptor(recipe: Recipe) -> ToolDescriptor: """Render one recipe as a JSON-Schema-carrying tool descriptor. @@ -155,7 +162,7 @@ def _descriptor(recipe: Recipe) -> ToolDescriptor: for name, spec in recipe.parameters.items(): declared = spec if isinstance(spec, dict) else {} rendered = dict(_TYPE_SCHEMA.get(str(declared.get("type")), {"type": "string"})) - for facet in ("default", "enum", "description"): + for facet in ("default", "enum", "description", "minimum", "maximum"): if facet in declared: rendered[facet] = declared[facet] properties[name] = rendered @@ -216,4 +223,18 @@ def _coerce_arguments( f"argument {name!r} of recipe {recipe.name!r} is not a valid" f" {declared.get('type', 'string')}: {value!r}" ) from error + coerced_value = coerced[name] + if isinstance(coerced_value, int): + minimum = declared.get("minimum") + maximum = declared.get("maximum") + if isinstance(minimum, int) and coerced_value < minimum: + raise InvalidArgumentError( + f"argument {name!r} of recipe {recipe.name!r} must be at" + f" least {minimum}" + ) + if isinstance(maximum, int) and coerced_value > maximum: + raise InvalidArgumentError( + f"argument {name!r} of recipe {recipe.name!r} must be at" + f" most {maximum}" + ) return coerced diff --git a/src/rememberstack/surfaces/sdk.py b/src/rememberstack/surfaces/sdk.py index 07b54f80..baa84808 100644 --- a/src/rememberstack/surfaces/sdk.py +++ b/src/rememberstack/surfaces/sdk.py @@ -24,6 +24,7 @@ from rememberstack.model.client import ConnectorCreate from rememberstack.model.client import ConnectorDescriptor +from rememberstack.model.client import PipelineReadinessReport from rememberstack.model.client import ToolDescriptor from rememberstack.model.documents import IngestedVersion from rememberstack.model.envelope import Envelope @@ -163,6 +164,23 @@ def hydrate_relation(self, *, relation_id: UUID) -> Envelope: endpoint=f"GET /hydrate/relation/{relation_id}", ) + def pipeline_readiness( + self, *, version_ids: tuple[UUID, ...], require_projections: bool = True + ) -> PipelineReadinessReport: + """Inspect exact continuous-stage and aggregate-projection readiness.""" + if not version_ids: + raise ValueError("pipeline readiness requires at least one version_id") + return _validated( + PipelineReadinessReport, + self._json( + "POST", + "/readiness", + params={"require_projections": str(require_projections).lower()}, + json_body=[str(version_id) for version_id in version_ids], + ), + endpoint="POST /readiness", + ) + def ingest( self, source: bytes | Path, diff --git a/src/rememberstack/workers/base.py b/src/rememberstack/workers/base.py index 55fd1f29..9218e816 100644 --- a/src/rememberstack/workers/base.py +++ b/src/rememberstack/workers/base.py @@ -25,6 +25,7 @@ from rememberstack.model import NonRetryableHandlerError from rememberstack.model import PipelineStage from rememberstack.model import ProcessingLane +from rememberstack.model import ProviderCallError from rememberstack.model import ProviderCallUsage from rememberstack.model import QueueRoute from rememberstack.model import RecordCall @@ -215,6 +216,7 @@ def run_one( outcome=RunResultOutcome.DEAD_LETTERED, ) except Exception as exception: + self._record_failed_provider_usage(meter=meter, exception=exception) _logger.exception( "retryable failure in stage %s for %s", claimed.stage, @@ -271,6 +273,26 @@ def run_one( processing_id=claimed.processing_id, outcome=RunResultOutcome.SUCCEEDED ) + @staticmethod + def _record_failed_provider_usage( + *, meter: CostMeterPort, exception: Exception + ) -> None: + """Meter a billable call whose response failed after usage was parsed.""" + if not isinstance(exception, ProviderCallError) or exception.usage is None: + return + try: + meter.record( + call_key="provider_failure", + tier="failed_response", + usage=exception.usage, + ) + except Exception as accounting_exception: # pragma: no cover - DB outage path + exception.add_note( + "failed to persist usage from the failed provider response:" + f" {accounting_exception}" + ) + _logger.exception("failed to meter a usage-bearing provider exception") + def _export_event(self, *, event: TelemetryEvent) -> None: """Export one completed transition when telemetry is configured.""" if self._telemetry is not None: diff --git a/src/rememberstack/workers/e0.py b/src/rememberstack/workers/e0.py index 121c5700..551e3c95 100644 --- a/src/rememberstack/workers/e0.py +++ b/src/rememberstack/workers/e0.py @@ -49,6 +49,7 @@ from rememberstack.model import PipelineStage from rememberstack.model import ProcessingLane from rememberstack.model import ProviderAccountingError +from rememberstack.model import ProviderCallError from rememberstack.model import RepresentationRecord from rememberstack.model import SectionTreeRecord from rememberstack.model import SnappedSection @@ -533,6 +534,14 @@ def _propose( ) except ProviderAccountingError: raise # budget enforcement must never degrade missing usage to zero + except ProviderCallError as error: + if error.usage is not None: + meter.record( + call_key="structure_failure", + tier="failed_response", + usage=error.usage, + ) + return None except Exception: # noqa: BLE001 — a document never fails structuring return None meter.record(call_key="structure", tier="structure", usage=generated.usage) diff --git a/src/rememberstack/workers/e1.py b/src/rememberstack/workers/e1.py index 22ebd354..02cfcfd3 100644 --- a/src/rememberstack/workers/e1.py +++ b/src/rememberstack/workers/e1.py @@ -179,7 +179,10 @@ def handle(self, *, work: ClaimedWork, meter: CostMeterPort) -> HandlerOutcome: chunker_version=self._chunker_version, ) if not chunks: - return HandlerOutcome() # an empty document has nothing to index + # Empty is a successful, explicit pipeline state. Continue through + # the no-op extraction/normalization branches so readiness has the + # same terminal shape as a non-empty document. + return _extract_follow_up(work=work, source=source) document_md = self._artifact_store.read_bytes( key=ObjectKey(source.markdown_uri) ).decode("utf-8") @@ -256,23 +259,7 @@ def handle(self, *, work: ClaimedWork, meter: CostMeterPort) -> HandlerOutcome: for chunk, prefix in zip(chunks, prefixes, strict=True) ) ) - return HandlerOutcome( - follow_up=( - EnqueueWork( - deployment_id=work.deployment_id, - target_kind=work.target_kind, - target_id=work.target_id, - stage=PipelineStage.EXTRACT_CLAIMS, - component_version=E2_EXTRACTOR_VERSION, - content_hash=work.content_hash, - lane=work.lane, - payload={ - "version_id": str(source.version_id), - "representation_id": str(source.representation_id), - }, - ), - ) - ) + return _extract_follow_up(work=work, source=source) def _carried_vectors( self, @@ -410,6 +397,27 @@ def _embed_follow_up(*, work: ClaimedWork, source: ChunkSource) -> HandlerOutcom ) +def _extract_follow_up(*, work: ClaimedWork, source: ChunkSource) -> HandlerOutcome: + """Chain extraction even when a representation produced zero chunks.""" + return HandlerOutcome( + follow_up=( + EnqueueWork( + deployment_id=work.deployment_id, + target_kind=work.target_kind, + target_id=work.target_id, + stage=PipelineStage.EXTRACT_CLAIMS, + component_version=E2_EXTRACTOR_VERSION, + content_hash=work.content_hash, + lane=work.lane, + payload={ + "version_id": str(source.version_id), + "representation_id": str(source.representation_id), + }, + ), + ) + ) + + def _isoformat_or_empty(*, value: datetime | None) -> str: """Render an optional datetime deterministically for the reuse key.""" return "" if value is None else value.isoformat() diff --git a/src/rememberstack/workers/e2.py b/src/rememberstack/workers/e2.py index 5a3ebfe3..1666020a 100644 --- a/src/rememberstack/workers/e2.py +++ b/src/rememberstack/workers/e2.py @@ -113,7 +113,7 @@ def handle(self, *, work: ClaimedWork, meter: CostMeterPort) -> HandlerOutcome: chunker_version=self._chunker_version, ) if not chunks: - return HandlerOutcome() + return _normalize_follow_up(work=work, source=source) document_md = self._artifact_store.read_bytes( key=ObjectKey(source.markdown_uri) ).decode("utf-8") @@ -131,23 +131,7 @@ def handle(self, *, work: ClaimedWork, meter: CostMeterPort) -> HandlerOutcome: document_md=document_md, meter=meter, ) - return HandlerOutcome( - follow_up=( - EnqueueWork( - deployment_id=work.deployment_id, - target_kind=work.target_kind, - target_id=work.target_id, - stage=PipelineStage.NORMALIZE_RELATIONS, - component_version=E3_NORMALIZER_VERSION, - content_hash=work.content_hash, - lane=work.lane, - payload={ - "version_id": str(source.version_id), - "representation_id": str(source.representation_id), - }, - ), - ) - ) + return _normalize_follow_up(work=work, source=source) def _reuse_prior_extraction( self, *, source: ChunkSource, chunk: ChunkForEmbedding @@ -466,6 +450,27 @@ def _empty_extraction_marker( ) +def _normalize_follow_up(*, work: ClaimedWork, source: ChunkSource) -> HandlerOutcome: + """Continue an extracted version even when it contains no chunks.""" + return HandlerOutcome( + follow_up=( + EnqueueWork( + deployment_id=work.deployment_id, + target_kind=work.target_kind, + target_id=work.target_id, + stage=PipelineStage.NORMALIZE_RELATIONS, + component_version=E3_NORMALIZER_VERSION, + content_hash=work.content_hash, + lane=work.lane, + payload={ + "version_id": str(source.version_id), + "representation_id": str(source.representation_id), + }, + ), + ) + ) + + def _selection_decisions( *, source: ChunkSource, chunk: ChunkForEmbedding, selection: SelectionResponse ) -> tuple[DecisionRecord, ...]: diff --git a/src/rememberstack/workers/e3.py b/src/rememberstack/workers/e3.py index 0f90ca11..1ba3e061 100644 --- a/src/rememberstack/workers/e3.py +++ b/src/rememberstack/workers/e3.py @@ -37,7 +37,6 @@ from rememberstack.spine.supersession import ADJUDICATOR_VERSION from rememberstack.spine.supersession import SupersessionAdjudicator from rememberstack.workers.base import HandlerOutcome -from rememberstack.workers.p1 import FACT_LABEL_VERSION from rememberstack.workers.p1 import P1_EMBED_CLAIMS_VERSION from rememberstack.workers.reconcile import RECONCILE_VERSION @@ -121,22 +120,14 @@ def handle(self, *, work: ClaimedWork, meter: CostMeterPort) -> HandlerOutcome: chunk_ids=tuple(chunk.chunk_id for chunk in chunks) ) if not claims: - # a version whose extraction yielded nothing still completes its - # basis change: the chain must reach reconciliation (Codex - # review — a living replacement of pure boilerplate would - # otherwise never supersede the old claims) + # A version whose extraction yielded nothing still completes both + # terminal branches. Reconciliation is required for a living + # replacement of pure boilerplate, while the no-op embed row makes + # per-version readiness mechanically identical to the non-empty + # path. return HandlerOutcome( - follow_up=( - EnqueueWork( - deployment_id=work.deployment_id, - target_kind=work.target_kind, - target_id=work.target_id, - stage=PipelineStage.ADJUDICATE_SUPERSESSION, - component_version=ADJUDICATOR_VERSION, - content_hash=work.content_hash, - lane=work.lane, - payload={**(work.payload or {}), "relation_ids": []}, - ), + follow_up=self._terminal_branches( + work=work, doc_id=source.doc_id, relation_ids=() ) ) deployment_id = work.deployment_id @@ -172,40 +163,49 @@ def handle(self, *, work: ClaimedWork, meter: CostMeterPort) -> HandlerOutcome: call_key=f"observation:{entity_id}", ) return HandlerOutcome( - follow_up=( - EnqueueWork( - deployment_id=work.deployment_id, - target_kind=work.target_kind, - target_id=work.target_id, - stage=PipelineStage.ADJUDICATE_SUPERSESSION, - component_version=ADJUDICATOR_VERSION, - content_hash=work.content_hash, - lane=work.lane, - payload={**(work.payload or {}), "relation_ids": created_relations}, - ), - EnqueueWork( - deployment_id=work.deployment_id, - target_kind=work.target_kind, - target_id=work.target_id, - stage=PipelineStage.EMBED_CLAIM, - component_version=P1_EMBED_CLAIMS_VERSION, - content_hash=work.content_hash, - lane=work.lane, - payload=dict(work.payload or {}), - ), - EnqueueWork( - deployment_id=work.deployment_id, - target_kind=work.target_kind, - target_id=work.target_id, - stage=PipelineStage.LABEL_RELATION, - component_version=FACT_LABEL_VERSION, - content_hash=work.content_hash, - lane=work.lane, - payload={**(work.payload or {}), "doc_id": str(source.doc_id)}, - ), + follow_up=self._terminal_branches( + work=work, doc_id=source.doc_id, relation_ids=tuple(created_relations) ) ) + @staticmethod + def _terminal_branches( + *, work: ClaimedWork, doc_id: UUID, relation_ids: tuple[str, ...] + ) -> tuple[EnqueueWork, ...]: + """Start the lifecycle and claim-index branches after normalization. + + Fact labeling deliberately does not fan out here. It follows + reconciliation, after supersession and lifecycle state have settled, + so the P1 facts channel cannot race ahead with a pre-adjudication + status. Readiness joins this branch with ``embed_claim``. + """ + return ( + EnqueueWork( + deployment_id=work.deployment_id, + target_kind=work.target_kind, + target_id=work.target_id, + stage=PipelineStage.ADJUDICATE_SUPERSESSION, + component_version=ADJUDICATOR_VERSION, + content_hash=work.content_hash, + lane=work.lane, + payload={ + **(work.payload or {}), + "doc_id": str(doc_id), + "relation_ids": list(relation_ids), + }, + ), + EnqueueWork( + deployment_id=work.deployment_id, + target_kind=work.target_kind, + target_id=work.target_id, + stage=PipelineStage.EMBED_CLAIM, + component_version=P1_EMBED_CLAIMS_VERSION, + content_hash=work.content_hash, + lane=work.lane, + payload=dict(work.payload or {}), + ), + ) + def _normalize_claim( self, *, @@ -428,6 +428,7 @@ def handle(self, *, work: ClaimedWork, meter: CostMeterPort) -> HandlerOutcome: payload={ "version_id": version_id, "representation_id": representation_id, + "doc_id": payload.get("doc_id"), }, ), ) diff --git a/src/rememberstack/workers/p2.py b/src/rememberstack/workers/p2.py index 7f4b0060..5c2d85a5 100644 --- a/src/rememberstack/workers/p2.py +++ b/src/rememberstack/workers/p2.py @@ -21,6 +21,7 @@ import json from pathlib import Path import shutil +from threading import Lock from typing import Final from uuid import UUID @@ -393,6 +394,7 @@ def __init__( self._published_at: datetime | None = None self._database: ladybug.Database | None = None self._connection: ladybug.Connection | None = None + self._refresh_lock = Lock() @property def version(self) -> str | None: @@ -406,6 +408,11 @@ def published_at(self) -> datetime | None: def refresh(self) -> bool: """Serve the latest published snapshot; True when a swap happened.""" + with self._refresh_lock: + return self._refresh_locked() + + def _refresh_locked(self) -> bool: + """Download and swap while serializing concurrent refresh attempts.""" latest = self._catalog.latest_snapshot( deployment_id=self._deployment_id, plane="P2_graph" ) @@ -452,8 +459,7 @@ def refresh(self) -> bool: def connection(self) -> ladybug.Connection: """The read-only connection to the served snapshot.""" - if self._connection is None: - self.refresh() + self.refresh() if self._connection is None: raise RuntimeError("no published P2 snapshot exists yet") return self._connection @@ -466,8 +472,7 @@ def fresh_connection(self) -> ladybug.Connection: connection must not be able to break the read permanently. Same read-only database, so it never sees uncommitted or torn data. """ - if self._database is None: - self.refresh() + self.refresh() if self._database is None: raise RuntimeError("no published P2 snapshot exists yet") return ladybug.Connection(self._database) diff --git a/src/rememberstack/workers/reconcile.py b/src/rememberstack/workers/reconcile.py index 5dd5a886..708bda46 100644 --- a/src/rememberstack/workers/reconcile.py +++ b/src/rememberstack/workers/reconcile.py @@ -26,13 +26,16 @@ from rememberstack.core import ChunkerParams from rememberstack.model import ClaimedWork from rememberstack.model import CurrencyTransition +from rememberstack.model import EnqueueWork from rememberstack.model import NonRetryableHandlerError +from rememberstack.model import PipelineStage from rememberstack.model import ReconciliationDelta from rememberstack.ports.cost_meter import CostMeterPort from rememberstack.spine.lifecycle import LifecycleCatalog from rememberstack.spine.review import ReviewQueue from rememberstack.workers.base import HandlerOutcome from rememberstack.workers.e1 import E2_EXTRACTOR_VERSION +from rememberstack.workers.p1 import FACT_LABEL_VERSION RECONCILE_VERSION = "reconcile-2026.07" """The reconcile stage's component version (D12 idempotency key member).""" @@ -185,7 +188,29 @@ def handle(self, *, work: ClaimedWork, meter: CostMeterPort) -> HandlerOutcome: flags_raised=flags, ), ) - return HandlerOutcome() + doc_id = context["doc_id"] + if not isinstance(doc_id, UUID): + raise NonRetryableHandlerError( + f"version {version_id} reconciliation context has no doc_id" + ) + return HandlerOutcome( + follow_up=( + EnqueueWork( + deployment_id=work.deployment_id, + target_kind=work.target_kind, + target_id=work.target_id, + stage=PipelineStage.LABEL_RELATION, + component_version=FACT_LABEL_VERSION, + content_hash=work.content_hash, + lane=work.lane, + payload={ + "version_id": str(version_id), + "representation_id": str(representation_id), + "doc_id": str(doc_id), + }, + ), + ) + ) def _flag_transcription_only( self, diff --git a/src/tests/adapters/test_openrouter.py b/src/tests/adapters/test_openrouter.py index 20078e72..d5042aca 100644 --- a/src/tests/adapters/test_openrouter.py +++ b/src/tests/adapters/test_openrouter.py @@ -5,12 +5,13 @@ from pydantic import BaseModel from pydantic import Field -from pydantic import ValidationError import pytest from rememberstack.adapters import OpenRouterModelProvider +from rememberstack.adapters import OpenRouterProviderError from rememberstack.adapters import OpenRouterSettings from rememberstack.adapters.openrouter import _usage +from rememberstack.model import EmbeddingRequest from rememberstack.model import ModelRequest from rememberstack.model import ProviderAccountingError @@ -88,10 +89,10 @@ def post(*, path: str, payload: dict[str, object]) -> dict[str, object]: assert observed["temperature"] == 0.0 -def test_generation_preserves_structured_output_validation_error( +def test_generation_preserves_usage_on_structured_output_validation_error( monkeypatch: pytest.MonkeyPatch, ) -> None: - """Keep valid JSON with an invalid schema distinct from malformed bodies.""" + """A billable invalid schema carries its already parsed provider usage.""" provider = OpenRouterModelProvider(settings=OpenRouterSettings(api_key="test-key")) def post(*, path: str, payload: dict[str, object]) -> dict[str, object]: @@ -105,7 +106,7 @@ def post(*, path: str, payload: dict[str, object]) -> dict[str, object]: monkeypatch.setattr(provider, "_post", post) try: - with pytest.raises(ValidationError): + with pytest.raises(OpenRouterProviderError) as raised: provider.generate( request=ModelRequest( model="openai/gpt-4o-mini", prompt="Where?", temperature=0 @@ -114,3 +115,38 @@ def post(*, path: str, payload: dict[str, object]) -> dict[str, object]: ) finally: provider._client.close() + + assert raised.value.usage is not None + assert raised.value.usage.tokens_in == 3 + assert raised.value.usage.tokens_out == 1 + + +def test_embedding_preserves_usage_when_the_vector_body_is_unusable( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A malformed billable embedding remains attributable to its worker.""" + provider = OpenRouterModelProvider(settings=OpenRouterSettings(api_key="test-key")) + + def post(*, path: str, payload: dict[str, object]) -> dict[str, object]: + assert path == "/embeddings" + assert payload + return { + "model": "qwen/qwen3-embedding-8b", + "usage": {"prompt_tokens": 4, "cost": "0.000004"}, + "data": [{"index": 0, "embedding": []}], + } + + monkeypatch.setattr(provider, "_post", post) + try: + with pytest.raises(OpenRouterProviderError) as raised: + provider.embed( + request=EmbeddingRequest( + model="qwen/qwen3-embedding-8b", texts=("memory",) + ) + ) + finally: + provider._client.close() + + assert raised.value.usage is not None + assert raised.value.usage.tokens_in == 4 + assert raised.value.usage.cost_usd == Decimal("0.000004") diff --git a/src/tests/benchmarks/test_locomo_protocol.py b/src/tests/benchmarks/test_locomo_protocol.py index 0a8026f4..dbecaa47 100644 --- a/src/tests/benchmarks/test_locomo_protocol.py +++ b/src/tests/benchmarks/test_locomo_protocol.py @@ -1,19 +1,31 @@ -"""Pure RS-LoCoMo-v1 rendering, prompt, diagnostic, and scorer proofs.""" +"""Pure full-system LoCoMo rendering, prompt, diagnostic, and scorer proofs.""" -from uuid import uuid4 +from datetime import datetime +from datetime import timezone +import hashlib +import json from benchmarks.locomo.model import LoCoMoQuestion from benchmarks.locomo.model import LoCoMoSample from benchmarks.locomo.model import LoCoMoSession from benchmarks.locomo.model import LoCoMoTurn -from benchmarks.locomo.model import RetrievedClaim +from benchmarks.locomo.model import ToolCallRecord +from benchmarks.locomo.protocol import EXPECTED_TOOL_CATALOG_SHA256 from benchmarks.locomo.protocol import official_f1 +from benchmarks.locomo.protocol import render_answer_agent_prompt from benchmarks.locomo.protocol import render_judge_prompt -from benchmarks.locomo.protocol import render_reader_prompt from benchmarks.locomo.protocol import render_session from benchmarks.locomo.protocol import session_diagnostic import pytest +from rememberstack.model import Envelope +from rememberstack.model import Freshness +from rememberstack.model import Grain +from rememberstack.model import ToolDescriptor +from rememberstack.spine import CANONICAL_RECIPES +from rememberstack.spine import GRAPH_RECIPES +from rememberstack.surfaces.recipe_surface import recipe_descriptors + def test_session_render_preserves_turns_and_discloses_derived_visual_text() -> None: session = LoCoMoSession( @@ -57,28 +69,70 @@ def test_session_render_preserves_turns_and_discloses_derived_visual_text() -> N assert rendered.endswith("\n") -def test_reader_uses_only_ranked_claim_text_and_braces_stay_literal() -> None: - claim = _claim(rank=1, text="Literal {question}; timestamp 1 May 2023") +def test_answer_agent_prompt_contains_only_public_tools_trace_and_question() -> None: + tool = ToolDescriptor( + name="claims_verbatim", + description="What sources asserted", + input_schema={"type": "object"}, + output_grain="evidence", + answer_intent="assertion_history", + ) + trace = ( + ToolCallRecord( + name=tool.name, + arguments={"query": "Literal {question}"}, + latency_ms=1, + response=Envelope( + grain=Grain.EVIDENCE, + freshness=Freshness( + pg_live_ts=datetime(2026, 7, 23, tzinfo=timezone.utc) + ), + ), + ), + ) - prompt = render_reader_prompt(question="What about {memories}?", claims=(claim,)) + prompt = render_answer_agent_prompt( + question="What about {tools}?", tools=(tool,), trace=trace + ) - assert "[1] Literal {question}; timestamp 1 May 2023" in prompt - assert "What about {memories}?" in prompt - assert claim.source_span not in prompt + assert '"name":"claims_verbatim"' in prompt + assert "Literal {question}" in prompt + assert "What about {tools}?" in prompt assert "gold answer" not in prompt.lower() -def test_empty_reader_context_is_exactly_none() -> None: - prompt = render_reader_prompt(question="Unknown?", claims=()) - assert "Ranked memories:\n(none)\n\nQuestion: Unknown?" in prompt +def test_empty_answer_agent_trace_is_explicit_json_array() -> None: + prompt = render_answer_agent_prompt(question="Unknown?", tools=(), trace=()) + assert "TOOL TRACE SO FAR:\n[]" in prompt + + +def test_frozen_tool_catalog_hash_matches_stock_full_system_recipes() -> None: + recipes = tuple( + sorted((*CANONICAL_RECIPES, *GRAPH_RECIPES), key=lambda recipe: recipe.name) + ) + descriptors = recipe_descriptors(recipes=recipes) + canonical = json.dumps( + [ + descriptor.model_dump(mode="json", exclude_none=False) + for descriptor in descriptors + ], + default=str, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + + assert hashlib.sha256(canonical.encode()).hexdigest() == ( + EXPECTED_TOOL_CATALOG_SHA256 + ) -def test_judge_never_receives_retrieved_context() -> None: +def test_judge_never_receives_tool_trace() -> None: prompt = render_judge_prompt( question="Where?", gold_answer="Prague", generated_answer="Prague" ) assert "Gold answer: Prague" in prompt - assert "Ranked memories" not in prompt + assert "TOOL TRACE" not in prompt assert "source_span" not in prompt @@ -116,19 +170,3 @@ def test_session_diagnostic_keeps_valid_ids_and_discloses_malformed_fields() -> assert diagnostic.malformed_fields == 2 assert diagnostic.recall == pytest.approx(2 / 3) assert diagnostic.complete is False - - -def _claim(*, rank: int, text: str) -> RetrievedClaim: - return RetrievedClaim( - rank=rank, - claim_id=uuid4(), - doc_id=uuid4(), - chunk_id=uuid4(), - claim_text=text, - source_span="SECRET VERBATIM SOURCE", - char_start=0, - char_end=22, - is_attributed=False, - is_current_testimony=True, - session_id="D1", - ) diff --git a/src/tests/benchmarks/test_locomo_runner.py b/src/tests/benchmarks/test_locomo_runner.py index 8f2fe2e8..ebd6e1aa 100644 --- a/src/tests/benchmarks/test_locomo_runner.py +++ b/src/tests/benchmarks/test_locomo_runner.py @@ -1,4 +1,4 @@ -"""Synthetic-only remote-boundary, cost, and denominator proofs.""" +"""Synthetic full-system tool-loop, readiness, cost, and denominator proofs.""" from __future__ import annotations @@ -15,6 +15,7 @@ from benchmarks.locomo.dataset import DATASET_COMMIT from benchmarks.locomo.dataset import DATASET_SHA256 from benchmarks.locomo.dataset import item_ids_hash +from benchmarks.locomo.model import AnswerAgentStep from benchmarks.locomo.model import JudgeOutput from benchmarks.locomo.model import LoCoMoDataset from benchmarks.locomo.model import LoCoMoQuestion @@ -22,9 +23,9 @@ from benchmarks.locomo.model import LoCoMoSession from benchmarks.locomo.model import LoCoMoTurn from benchmarks.locomo.model import QuestionManifest -from benchmarks.locomo.model import ReaderOutput from benchmarks.locomo.model import RunState -from benchmarks.locomo.protocol import READER_MODEL +from benchmarks.locomo.protocol import ANSWER_AGENT_MODEL +from benchmarks.locomo.protocol import EXPECTED_PIPELINE_STAGES from benchmarks.locomo.runner import _answer_one from benchmarks.locomo.runner import _judge_one from benchmarks.locomo.runner import answer_sample @@ -45,110 +46,90 @@ from rememberstack.model import GeneratedResponse from rememberstack.model import Grain from rememberstack.model import ModelRequest -from rememberstack.model import Negative -from rememberstack.model import NegativeKind from rememberstack.model import ProviderCallUsage from rememberstack.model import StructuredResponseModel +from rememberstack.model import ToolDescriptor +from rememberstack.spine import CANONICAL_RECIPES +from rememberstack.spine import GRAPH_RECIPES +from rememberstack.surfaces.recipe_surface import recipe_descriptors from rememberstack.surfaces.sdk import MemoryClient ResponseT = TypeVar("ResponseT", bound=StructuredResponseModel) -@pytest.mark.parametrize("known_empty", (False, True)) -def test_successful_empty_retrieval_still_calls_reader_once(known_empty: bool) -> None: - envelope = _empty_envelope(known_empty=known_empty) - client, raw_client = _memory_client(envelope=envelope) - provider = FakeModelProvider(generate_payload={"answer": "Unknown"}) - state = RunState() +def test_agent_calls_public_recipe_then_answers() -> None: + client, raw_client = _memory_client() + provider = FakeModelProvider(generate_router=_tool_then_answer) try: answer = _answer_one( question=_question(), client=client, provider=provider, + tools=(_tool(),), doc_sessions={}, - state=state, - max_reader_calls=1, + state=RunState(), + max_agent_calls=9, max_evaluator_cost_usd=Decimal("1"), ) finally: raw_client.close() assert answer.retrieval_succeeded is True - assert answer.reader_called is True - assert answer.failure is None - assert answer.generated_answer == "Unknown" - assert provider.generated_prompts[0].count("(none)") == 1 - + assert answer.generated_answer == "Prague" + assert answer.agent_call_count == 2 + assert [call.name for call in answer.tool_calls] == ["claims_verbatim"] + assert len(provider.generated_prompts) == 2 -def test_transport_failure_does_not_call_reader() -> None: - def fail(request: httpx.Request) -> httpx.Response: - raise httpx.ConnectError("offline", request=request) - raw_client = httpx.Client( - base_url="http://memory.test", transport=httpx.MockTransport(fail) +def test_answer_without_consulting_memory_is_rejected() -> None: + client, raw_client = _memory_client() + provider = FakeModelProvider( + generate_payload={ + "action": "answer", + "tool_name": None, + "arguments": {}, + "answer": "Prague", + } ) - provider = FakeModelProvider(generate_payload={"answer": "must not run"}) - try: - answer = _answer_one( - question=_question(), - client=MemoryClient(client=raw_client), - provider=provider, - doc_sessions={}, - state=RunState(), - max_reader_calls=1, - max_evaluator_cost_usd=Decimal("1"), - ) - finally: - raw_client.close() - - assert answer.retrieval_succeeded is False - assert answer.reader_called is False - assert answer.failure is not None - assert answer.failure.kind == "retrieval" - assert provider.generated_prompts == [] - - -def test_invalid_empty_reader_answer_is_a_terminal_reader_failure() -> None: - client, raw_client = _memory_client(envelope=_empty_envelope()) - provider = FakeModelProvider(generate_payload={"answer": ""}) try: answer = _answer_one( question=_question(), client=client, provider=provider, + tools=(_tool(),), doc_sessions={}, state=RunState(), - max_reader_calls=1, + max_agent_calls=9, max_evaluator_cost_usd=Decimal("1"), ) finally: raw_client.close() - assert answer.generated_answer is None - assert answer.reader_called is True assert answer.failure is not None assert answer.failure.kind == "invalid_response" + assert answer.agent_call_count == 1 -def test_reader_and_judge_share_one_run_absolute_cost_ceiling() -> None: - client, raw_client = _memory_client(envelope=_empty_envelope()) - provider = _CostProvider(cost=Decimal("0.60")) +def test_agent_and_judge_share_one_run_absolute_cost_threshold() -> None: + client, raw_client = _memory_client() + provider = _CostProvider(cost=Decimal("0.30")) state = RunState() try: answer = _answer_one( question=_question(), client=client, provider=provider, + tools=(_tool(),), doc_sessions={}, state=state, - max_reader_calls=1, + max_agent_calls=9, max_evaluator_cost_usd=Decimal("0.60"), ) finally: raw_client.close() assert state.evaluator_cost_usd == Decimal("0.60") - with pytest.raises(ExecutionGuardError, match="reached run ceiling"): + with pytest.raises(ExecutionGuardError, match="reached run threshold"): _judge_one( question=_question(), answer=answer, @@ -157,21 +138,38 @@ def test_reader_and_judge_share_one_run_absolute_cost_ceiling() -> None: max_judge_calls=1, max_evaluator_cost_usd=Decimal("0.60"), ) - assert provider.models == [READER_MODEL] + assert provider.models == [ANSWER_AGENT_MODEL, ANSWER_AGENT_MODEL] + +def test_a_call_that_crosses_the_cost_threshold_is_recorded_then_stops() -> None: + client, raw_client = _memory_client() + provider = _CostProvider(cost=Decimal("0.70")) + state = RunState() + try: + answer = _answer_one( + question=_question(), + client=client, + provider=provider, + tools=(_tool(),), + doc_sessions={}, + state=state, + max_agent_calls=9, + max_evaluator_cost_usd=Decimal("0.60"), + ) + finally: + raw_client.close() -def test_model_request_temperature_is_bounded() -> None: - assert ModelRequest(model="m", prompt="p", temperature=0).temperature == 0 - with pytest.raises(ValueError): - ModelRequest(model="m", prompt="p", temperature=-0.1) - with pytest.raises(ValueError): - ModelRequest(model="m", prompt="p", temperature=2.1) + assert answer.failure is not None + assert answer.failure.kind == "accounting" + assert answer.reader_usage is not None + assert answer.reader_usage.cost_usd == Decimal("0.70") + assert state.evaluator_cost_usd == Decimal("0.70") + assert provider.models == [ANSWER_AGENT_MODEL] -def test_staged_mock_run_resumes_without_duplicate_checkpointed_calls( +def test_staged_mock_run_checks_readiness_and_resumes( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - """The staged contract executes only against synthetic local boundaries.""" _patch_prepared_inputs(monkeypatch=monkeypatch) run_dir = tmp_path / "run" prepare_run(dataset_path=tmp_path / "synthetic.json", tier="smoke", output=run_dir) @@ -179,12 +177,7 @@ def test_staged_mock_run_resumes_without_duplicate_checkpointed_calls( base_url="http://memory.test", transport=httpx.MockTransport(_run_transport) ) client = MemoryClient(client=raw_client) - provider = FakeModelProvider( - generate_payloads={ - "ReaderOutput": {"answer": "Prague"}, - "JudgeOutput": {"label": "CORRECT"}, - } - ) + provider = FakeModelProvider(generate_router=_tool_answer_and_judge) try: ingests = ingest_sample( run_dir=run_dir, @@ -198,10 +191,9 @@ def test_staged_mock_run_resumes_without_duplicate_checkpointed_calls( run_dir=run_dir, sample_id="conv-test", max_questions=1, - max_reader_calls=1, + max_agent_calls=9, max_evaluator_cost_usd=Decimal("1"), execute=True, - index_ready_confirmation="conv-test", client=client, provider=provider, ) @@ -209,10 +201,9 @@ def test_staged_mock_run_resumes_without_duplicate_checkpointed_calls( run_dir=run_dir, sample_id="conv-test", max_questions=1, - max_reader_calls=1, + max_agent_calls=9, max_evaluator_cost_usd=Decimal("1"), execute=True, - index_ready_confirmation="conv-test", client=client, provider=provider, ) @@ -238,38 +229,34 @@ def test_staged_mock_run_resumes_without_duplicate_checkpointed_calls( assert len(ingests) == 1 assert first_answers == second_answers assert first_judges == second_judges - assert len(provider.generated_prompts) == 2 + assert len(provider.generated_prompts) == 3 summary = summarize_run(run_dir=run_dir) - assert summary.questions == 1 assert summary.judge_correct == 1 - assert summary.judge_percent == 100 assert summary.official_f1 == 1 + assert summary.answer_agent_calls == 2 -def test_missing_records_remain_in_full_manifest_denominator( +def test_readiness_flag_cannot_hide_an_incomplete_pipeline_report( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: + """The protocol verifies the report structure instead of trusting one bool.""" _patch_prepared_inputs(monkeypatch=monkeypatch) run_dir = tmp_path / "run" prepare_run(dataset_path=tmp_path / "synthetic.json", tier="smoke", output=run_dir) - summary = summarize_run(run_dir=run_dir) - - assert summary.questions == 1 - assert summary.judge_correct == 0 - assert summary.official_f1 == 0 - assert summary.failures == {"missing_answer": 1, "missing_judge": 1} - + def incomplete_readiness(request: httpx.Request) -> httpx.Response: + if request.method == "POST" and request.url.path == "/readiness": + return httpx.Response( + 200, json={"ready": True, "versions": [], "projections": []} + ) + return _run_transport(request) -def test_upstream_failure_gets_local_wrong_without_fake_judge_failure( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - _patch_prepared_inputs(monkeypatch=monkeypatch) - run_dir = tmp_path / "run" - prepare_run(dataset_path=tmp_path / "synthetic.json", tier="smoke", output=run_dir) - ingest_raw = httpx.Client( - base_url="http://memory.test", transport=httpx.MockTransport(_run_transport) + raw_client = httpx.Client( + base_url="http://memory.test", + transport=httpx.MockTransport(incomplete_readiness), ) + client = MemoryClient(client=raw_client) + provider = FakeModelProvider(generate_router=_tool_answer_and_judge) try: ingest_sample( run_dir=run_dir, @@ -277,46 +264,38 @@ def test_upstream_failure_gets_local_wrong_without_fake_judge_failure( max_documents=1, execute=True, isolated_deployment_confirmation="conv-test", - client=MemoryClient(client=ingest_raw), - ) - finally: - ingest_raw.close() - failed_raw = httpx.Client( - base_url="http://memory.test", - transport=httpx.MockTransport( - lambda _request: httpx.Response(503, text="offline") - ), - ) - provider = FakeModelProvider(generate_payload={"answer": "must not run"}) - try: - answer_sample( - run_dir=run_dir, - sample_id="conv-test", - max_questions=1, - max_reader_calls=1, - max_evaluator_cost_usd=Decimal("1"), - execute=True, - index_ready_confirmation="conv-test", - client=MemoryClient(client=failed_raw), - provider=provider, + client=client, ) + with pytest.raises(ExecutionGuardError, match="exact completed"): + answer_sample( + run_dir=run_dir, + sample_id="conv-test", + max_questions=1, + max_agent_calls=9, + max_evaluator_cost_usd=Decimal("1"), + execute=True, + client=client, + provider=provider, + ) finally: - failed_raw.close() - judges = judge_sample( - run_dir=run_dir, - sample_id="conv-test", - max_judge_calls=1, - max_evaluator_cost_usd=Decimal("1"), - execute=True, - provider=provider, - ) + raw_client.close() + + assert provider.generated_prompts == [] + + +def test_missing_records_remain_in_full_manifest_denominator( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _patch_prepared_inputs(monkeypatch=monkeypatch) + run_dir = tmp_path / "run" + prepare_run(dataset_path=tmp_path / "synthetic.json", tier="smoke", output=run_dir) summary = summarize_run(run_dir=run_dir) - assert judges[0].model_called is False - assert judges[0].failure is None + + assert summary.questions == 1 assert summary.judge_correct == 0 - assert summary.failures == {"answer_retrieval": 1} - assert provider.generated_prompts == [] + assert summary.official_f1 == 0 + assert summary.failures == {"missing_answer": 1, "missing_judge": 1} def test_protocol_mutation_is_rejected( @@ -327,7 +306,7 @@ def test_protocol_mutation_is_rejected( prepare_run(dataset_path=tmp_path / "synthetic.json", tier="smoke", output=run_dir) run_path = run_dir / "run.json" payload = json.loads(run_path.read_text(encoding="utf-8")) - payload["reader_prompt_sha256"] = "0" * 64 + payload["answer_prompt_sha256"] = "0" * 64 run_path.write_text(json.dumps(payload), encoding="utf-8") with pytest.raises(BenchmarkRunError, match="fingerprint"): @@ -340,7 +319,7 @@ def test_remote_stage_requires_explicit_execution( _patch_prepared_inputs(monkeypatch=monkeypatch) run_dir = tmp_path / "run" prepare_run(dataset_path=tmp_path / "synthetic.json", tier="smoke", output=run_dir) - client, raw_client = _memory_client(envelope=_empty_envelope()) + client, raw_client = _memory_client() try: with pytest.raises(ExecutionGuardError, match="--execute"): ingest_sample( @@ -355,28 +334,55 @@ def test_remote_stage_requires_explicit_execution( raw_client.close() -def _memory_client(*, envelope: Envelope) -> tuple[MemoryClient, httpx.Client]: - def respond(_request: httpx.Request) -> httpx.Response: - return httpx.Response(200, json=envelope.model_dump(mode="json")) - +def _memory_client() -> tuple[MemoryClient, httpx.Client]: raw = httpx.Client( - base_url="http://memory.test", transport=httpx.MockTransport(respond) + base_url="http://memory.test", + transport=httpx.MockTransport( + lambda request: ( + httpx.Response(200, json=_empty_envelope().model_dump(mode="json")) + if request.url.path.startswith("/recipe/") + else httpx.Response(404, text="unexpected") + ) + ), ) return MemoryClient(client=raw), raw -def _empty_envelope(*, known_empty: bool = False) -> Envelope: +def _empty_envelope() -> Envelope: return Envelope( grain=Grain.EVIDENCE, freshness=Freshness(pg_live_ts=datetime(2026, 7, 23, tzinfo=timezone.utc)), - negative=( - Negative(kind=NegativeKind.KNOWN_EMPTY, explanation="No current claims.") - if known_empty - else None - ), ) +def _tool() -> ToolDescriptor: + return ToolDescriptor( + name="claims_verbatim", + description="What sources asserted", + input_schema={"type": "object"}, + output_grain="evidence", + answer_intent="assertion_history", + ) + + +def _tool_then_answer(prompt: str, type_name: str) -> dict[str, object]: + assert type_name == "AnswerAgentStep" + if "TOOL TRACE SO FAR:\n[]" in prompt: + return { + "action": "tool", + "tool_name": "claims_verbatim", + "arguments": {"query": "Where?"}, + "answer": None, + } + return {"action": "answer", "tool_name": None, "arguments": {}, "answer": "Prague"} + + +def _tool_answer_and_judge(prompt: str, type_name: str) -> dict[str, object]: + if type_name == "JudgeOutput": + return {"label": "CORRECT"} + return _tool_then_answer(prompt, type_name) + + def _question() -> LoCoMoQuestion: return LoCoMoQuestion( item_id="conv-test/qa/0000", @@ -394,17 +400,32 @@ class _CostProvider: def __init__(self, *, cost: Decimal) -> None: self.cost = cost self.models: list[str] = [] + self.answer_calls = 0 def generate( self, *, request: ModelRequest, response_type: type[ResponseT] ) -> GeneratedResponse[ResponseT]: self.models.append(request.model) - payload: dict[str, str] - if response_type is ReaderOutput: - payload = {"answer": "Prague"} + if response_type is AnswerAgentStep: + self.answer_calls += 1 + payload = ( + { + "action": "tool", + "tool_name": "claims_verbatim", + "arguments": {"query": "Where?"}, + "answer": None, + } + if self.answer_calls == 1 + else { + "action": "answer", + "tool_name": None, + "arguments": {}, + "answer": "Prague", + } + ) elif response_type is JudgeOutput: payload = {"label": "CORRECT"} - else: # pragma: no cover - the test provider supports only protocol calls + else: # pragma: no cover raise AssertionError(response_type) return GeneratedResponse( output=response_type.model_validate(payload), @@ -473,6 +494,51 @@ def _run_transport(request: httpx.Request) -> httpx.Response: "created": True, }, ) - if request.method == "GET" and request.url.path == "/search/claims": + if request.method == "POST" and request.url.path == "/readiness": + return httpx.Response(200, json=_complete_readiness_payload()) + if request.method == "GET" and request.url.path == "/recipes": + return httpx.Response( + 200, json=[tool.model_dump(mode="json") for tool in _stock_tools()] + ) + if request.method == "POST" and request.url.path.startswith("/recipe/"): return httpx.Response(200, json=_empty_envelope().model_dump(mode="json")) return httpx.Response(404, text="unexpected synthetic request") + + +def _complete_readiness_payload() -> dict[str, object]: + timestamp = "2026-07-23T12:00:00Z" + return { + "ready": True, + "versions": [ + { + "version_id": "57000000-0000-0000-0000-000000000003", + "ready": True, + "stages": [ + { + "stage": stage, + "component_version": f"test-{stage}-v1", + "status": "succeeded", + "finished_at": timestamp, + } + for stage in EXPECTED_PIPELINE_STAGES + ], + } + ], + "projections": [ + { + "plane": plane, + "ready": True, + "version": "test-v1", + "built_at": timestamp, + "published_at": timestamp, + } + for plane in ("P2_graph", "P3_corpusfs") + ], + } + + +def _stock_tools() -> tuple[ToolDescriptor, ...]: + recipes = tuple( + sorted((*CANONICAL_RECIPES, *GRAPH_RECIPES), key=lambda recipe: recipe.name) + ) + return recipe_descriptors(recipes=recipes) diff --git a/src/tests/packaging/test_client_wheel.py b/src/tests/packaging/test_client_wheel.py index 12f8b4bb..698e1a0c 100644 --- a/src/tests/packaging/test_client_wheel.py +++ b/src/tests/packaging/test_client_wheel.py @@ -104,8 +104,9 @@ def test_fresh_base_wheel_queries_and_ingests_over_http( "-c", ( "import importlib.util; " - "from rememberstack.client import MemoryClient; " + "from rememberstack.client import MemoryClient, PipelineReadinessReport; " "assert MemoryClient.__name__ == 'MemoryClient'; " + "assert PipelineReadinessReport.__name__ == 'PipelineReadinessReport'; " "assert importlib.util.find_spec('sqlalchemy') is None" ), ], diff --git a/src/tests/profiles/test_selfhost_profile.py b/src/tests/profiles/test_selfhost_profile.py new file mode 100644 index 00000000..ceee540d --- /dev/null +++ b/src/tests/profiles/test_selfhost_profile.py @@ -0,0 +1,54 @@ +"""The stock self-host profile exposes the complete implemented runtime shape.""" + +from pathlib import Path +import re + +from rememberstack.model import PipelineStage +from rememberstack.profiles.selfhost import _expected_components +from rememberstack.profiles.selfhost import _SUPPORTED_WORKER_STAGES + +_ROOT = Path(__file__).resolve().parents[3] + + +def test_selfhost_composes_every_implemented_continuous_route() -> None: + """Ten real handlers run; enum-only/fused stages do not get dummy workers.""" + assert _SUPPORTED_WORKER_STAGES == ( + PipelineStage.CONVERT, + PipelineStage.STRUCTURE, + PipelineStage.CHUNK, + PipelineStage.EMBED_CHUNK, + PipelineStage.EXTRACT_CLAIMS, + PipelineStage.NORMALIZE_RELATIONS, + PipelineStage.ADJUDICATE_SUPERSESSION, + PipelineStage.EMBED_CLAIM, + PipelineStage.RECONCILE, + PipelineStage.LABEL_RELATION, + ) + assert tuple(_expected_components()) == _SUPPORTED_WORKER_STAGES + + +def test_enum_only_and_fused_stages_are_not_advertised_as_workers() -> None: + """A stage enum is not proof that an independently runnable handler exists.""" + assert { + PipelineStage.GROUND_CLAIMS, + PipelineStage.RESOLVE_ENTITIES, + PipelineStage.ADJUDICATE_OBSERVATIONS, + PipelineStage.EMBED_RELATION, + PipelineStage.EMBED_OBSERVATION, + PipelineStage.LABEL_OBSERVATION, + PipelineStage.CROSSREF, + PipelineStage.REFRESH_PROFILE, + }.isdisjoint(_SUPPORTED_WORKER_STAGES) + + +def test_compose_wires_the_exact_supported_worker_set_and_projection_job() -> None: + """Keep deployable Compose wiring in lockstep with the executable profile.""" + compose = (_ROOT / "compose.yaml").read_text(encoding="utf-8") + composed_stages = tuple( + PipelineStage(value) + for value in re.findall(r'command: \["worker", "--stage", "([^"]+)"\]', compose) + ) + + assert composed_stages == _SUPPORTED_WORKER_STAGES + assert 'profiles: ["operations"]' in compose + assert 'command: ["project", "--plane", "all"]' in compose diff --git a/src/tests/spine/test_pipeline_readiness.py b/src/tests/spine/test_pipeline_readiness.py new file mode 100644 index 00000000..43ecd4dc --- /dev/null +++ b/src/tests/spine/test_pipeline_readiness.py @@ -0,0 +1,217 @@ +"""Public readiness is derived from exact work and projection rows.""" + +from collections.abc import Iterator +from datetime import datetime +from datetime import timedelta +from datetime import UTC +from pathlib import Path +from uuid import UUID +from uuid import uuid4 + +from alembic import command +from alembic.config import Config +from pydantic import ValidationError +import pytest +from sqlalchemy import create_engine +from sqlalchemy import text +from sqlalchemy.engine import Engine + +from rememberstack.model import DeploymentBootstrapInput +from rememberstack.model import PipelineStage +from rememberstack.spine import DeploymentBootstrapper +from rememberstack.spine import PipelineReadinessCatalog +from rememberstack.spine import ProjectionCatalog +from rememberstack.spine.settings import load_database_settings + +_ROOT = Path(__file__).resolve().parents[3] +_DEPLOYMENT_ID = UUID("59000000-0000-0000-0000-000000000001") + + +@pytest.fixture(scope="module") +def database_engine() -> Iterator[Engine]: + """Apply structural head against the real PostgreSQL acceptance database.""" + try: + database_url = load_database_settings().sqlalchemy_url() + except ValidationError: + pytest.skip("REMEMBERSTACK_DATABASE_URL is required for readiness proofs") + config = Config(str(_ROOT / "alembic.ini")) + config.set_main_option("sqlalchemy.url", database_url) + command.downgrade(config=config, revision="base") + command.upgrade(config=config, revision="head") + engine = create_engine(database_url) + try: + yield engine + finally: + engine.dispose() + + +@pytest.fixture() +def ready_rows(database_engine: Engine) -> tuple[Engine, UUID]: + """One version with two exact succeeded generations and fresh P2/P3.""" + with database_engine.begin() as connection: + connection.execute(text("TRUNCATE TABLE deployments CASCADE")) + DeploymentBootstrapper(engine=database_engine).bootstrap_deployment( + deployment_input=DeploymentBootstrapInput( + deployment_id=_DEPLOYMENT_ID, + slug="readiness", + name="Readiness", + default_language="en", + raw_bucket="mem://raw", + artifacts_bucket="mem://artifacts", + corpusfs_bucket="mem://corpusfs", + ) + ) + version_id = uuid4() + finished = datetime.now(tz=UTC) - timedelta(minutes=1) + with database_engine.begin() as connection: + for stage, component in (("convert", "convert-v1"), ("structure", "struct-v1")): + connection.execute( + text( + "INSERT INTO processing_state (processing_id, deployment_id," + " target_kind, target_id, stage, component_version, content_hash," + " lane, status, attempts, finished_at)" + " VALUES (:p, :d, 'document_version', :v," + " CAST(:s AS pipeline_stage), :c, 'hash', 'steady'," + " 'succeeded', 1, :finished)" + ), + { + "p": uuid4(), + "d": _DEPLOYMENT_ID, + "v": version_id, + "s": stage, + "c": component, + "finished": finished, + }, + ) + for plane in ("P2_graph", "P3_corpusfs"): + connection.execute( + text( + "INSERT INTO projection_snapshots (snapshot_id, deployment_id," + " plane, version, gcs_uri, status, is_latest, published_at)" + " VALUES (:p, :d, CAST(:plane AS projection_plane), 'v1'," + " 'mem://snapshot', 'published', true, now())" + ), + {"p": uuid4(), "d": _DEPLOYMENT_ID, "plane": plane}, + ) + return database_engine, version_id + + +def test_exact_terminal_stages_and_fresh_projections_are_ready( + ready_rows: tuple[Engine, UUID], +) -> None: + engine, version_id = ready_rows + report = PipelineReadinessCatalog( + engine=engine, + expected_components={ + PipelineStage.CONVERT: "convert-v1", + PipelineStage.STRUCTURE: "struct-v1", + }, + projections=ProjectionCatalog(engine=engine), + model_bindings={"claim_extraction": "model-v1"}, + ).inspect( + deployment_id=_DEPLOYMENT_ID, + version_ids=(version_id,), + require_projections=True, + ) + + assert report.ready is True + assert report.versions[0].ready is True + assert all(projection.ready for projection in report.projections) + assert report.model_bindings == {"claim_extraction": "model-v1"} + + +def test_a_missing_exact_generation_is_not_ready( + ready_rows: tuple[Engine, UUID], +) -> None: + engine, version_id = ready_rows + report = PipelineReadinessCatalog( + engine=engine, + expected_components={ + PipelineStage.CONVERT: "convert-v1", + PipelineStage.STRUCTURE: "different-generation", + }, + projections=ProjectionCatalog(engine=engine), + ).inspect( + deployment_id=_DEPLOYMENT_ID, + version_ids=(version_id,), + require_projections=True, + ) + + assert report.ready is False + assert report.versions[0].stages[1].status == "missing" + + +def test_a_projection_started_before_terminal_work_is_not_fresh( + ready_rows: tuple[Engine, UUID], +) -> None: + engine, version_id = ready_rows + with engine.begin() as connection: + terminal_at = connection.execute( + text( + "SELECT max(finished_at) FROM processing_state" + " WHERE deployment_id = :deployment_id AND target_id = :version_id" + ), + {"deployment_id": _DEPLOYMENT_ID, "version_id": version_id}, + ).scalar_one() + connection.execute( + text( + "UPDATE projection_snapshots SET built_at = :built_at," + " published_at = now()" + " WHERE deployment_id = :deployment_id AND plane = 'P2_graph'" + ), + { + "built_at": terminal_at - timedelta(seconds=1), + "deployment_id": _DEPLOYMENT_ID, + }, + ) + + report = PipelineReadinessCatalog( + engine=engine, + expected_components={ + PipelineStage.CONVERT: "convert-v1", + PipelineStage.STRUCTURE: "struct-v1", + }, + projections=ProjectionCatalog(engine=engine), + ).inspect( + deployment_id=_DEPLOYMENT_ID, + version_ids=(version_id,), + require_projections=True, + ) + + assert report.ready is False + assert report.projections[0].plane == "P2_graph" + assert report.projections[0].ready is False + + +def test_terminal_status_without_a_completion_timestamp_fails_closed( + ready_rows: tuple[Engine, UUID], +) -> None: + """Projection freshness requires a timestamp from every terminal stage.""" + engine, version_id = ready_rows + with engine.begin() as connection: + connection.execute( + text( + "UPDATE processing_state SET finished_at = NULL" + " WHERE deployment_id = :deployment_id" + " AND target_id = :version_id AND stage = 'structure'" + ), + {"deployment_id": _DEPLOYMENT_ID, "version_id": version_id}, + ) + + report = PipelineReadinessCatalog( + engine=engine, + expected_components={ + PipelineStage.CONVERT: "convert-v1", + PipelineStage.STRUCTURE: "struct-v1", + }, + projections=ProjectionCatalog(engine=engine), + ).inspect( + deployment_id=_DEPLOYMENT_ID, + version_ids=(version_id,), + require_projections=True, + ) + + assert report.ready is False + assert report.versions[0].ready is False + assert report.versions[0].stages[1].status == "succeeded" + assert report.versions[0].stages[1].finished_at is None diff --git a/src/tests/spine/test_work_ledger.py b/src/tests/spine/test_work_ledger.py index 1f5d1700..d47afdf9 100644 --- a/src/tests/spine/test_work_ledger.py +++ b/src/tests/spine/test_work_ledger.py @@ -30,6 +30,8 @@ from rememberstack.model import PipelineStage from rememberstack.model import ProcessingLane from rememberstack.model import ProcessingTarget +from rememberstack.model import ProviderCallError +from rememberstack.model import ProviderCallUsage from rememberstack.model import RecordCall from rememberstack.model import RunResultOutcome from rememberstack.model import UnknownStageHandlerError @@ -442,6 +444,65 @@ def handle(self, *, work: ClaimedWork, meter: CostMeterPort) -> HandlerOutcome: raise RuntimeError(f"deliberate failure for {work.processing_id}") +class _BillableFailingHandler: + """A provider failure whose malformed output still carries billed usage.""" + + def handle(self, *, work: ClaimedWork, meter: CostMeterPort) -> HandlerOutcome: + """Expose usage to the worker error boundary without recording it here.""" + del work, meter + raise ProviderCallError( + "malformed provider response", + usage=ProviderCallUsage( + model_name="provider/model", + tokens_in=12, + tokens_out=3, + cost_usd=Decimal("0.25"), + latency_ms=8, + ), + ) + + +def test_worker_meters_usage_bearing_provider_failure_before_retry( + database_engine: Engine, ledger: WorkLedger +) -> None: + """A malformed billable response cannot disappear from route budgets.""" + registry = HandlerRegistry() + registry.register( + stage=PipelineStage.EXTRACT_CLAIMS, handler=_BillableFailingHandler() + ) + worker = Worker(ledger=ledger, registry=registry) + enqueued = ledger.enqueue(work=_work()) + + result = worker.run_one( + deployment_id=_DEPLOYMENT_ID, + stage=PipelineStage.EXTRACT_CLAIMS, + lane=ProcessingLane.STEADY, + ) + + assert result.outcome is RunResultOutcome.RETRY_SCHEDULED + with database_engine.connect() as connection: + row = ( + connection.execute( + text( + "SELECT call_key, tier, model_name, tokens_in, tokens_out," + " cost_usd FROM cost_ledger" + " WHERE processing_id = :processing_id" + ), + {"processing_id": enqueued.processing_id}, + ) + .mappings() + .one() + ) + assert dict(row) == { + "call_key": "provider_failure", + "tier": "failed_response", + "model_name": "provider/model", + "tokens_in": 12, + "tokens_out": 3, + "cost_usd": Decimal("0.250000"), + } + + def test_demo_no_op_worker_chain_retry_and_dead_letter( database_engine: Engine, ledger: WorkLedger ) -> None: diff --git a/src/tests/surfaces/test_client_sdk.py b/src/tests/surfaces/test_client_sdk.py index e16a307b..c5886fc0 100644 --- a/src/tests/surfaces/test_client_sdk.py +++ b/src/tests/surfaces/test_client_sdk.py @@ -20,6 +20,7 @@ from rememberstack.client import ConnectorNotFoundError from rememberstack.client import MemoryApiError from rememberstack.client import MemoryClient +from rememberstack.client import PipelineReadinessReport from rememberstack.model import DocumentUpload from rememberstack.model import IngestedVersion from rememberstack.surfaces import build_api @@ -29,6 +30,7 @@ from rememberstack.surfaces.remote_mcp import serve_mcp_stdio _DEPLOYMENT_ID = UUID("57000000-0000-0000-0000-000000000001") +_VERSION_ID = UUID("57000000-0000-0000-0000-000000000003") class _OpenBoundary: @@ -41,6 +43,23 @@ def ensure_ready(self, *, deployment_id: UUID) -> tuple[UUID, ...]: def assert_available(self, *, deployment_id: UUID) -> None: assert deployment_id == _DEPLOYMENT_ID + def inspect( + self, + *, + deployment_id: UUID, + version_ids: tuple[UUID, ...], + require_projections: bool, + ) -> PipelineReadinessReport: + assert deployment_id == _DEPLOYMENT_ID + assert version_ids == (_VERSION_ID,) + assert require_projections is True + return PipelineReadinessReport( + ready=True, + versions=(), + projections=(), + model_bindings={"claim_extraction": "model-v1"}, + ) + class _Ingest: """Record which E0 entry point the HTTP API selects.""" @@ -124,6 +143,7 @@ def client_surface() -> tuple[MemoryClient, _Ingest, _Connectors]: readiness=boundary, ingest=ingest, connectors=connectors, + pipeline_readiness=boundary, ) return MemoryClient(client=TestClient(app)), ingest, connectors @@ -181,6 +201,17 @@ def test_sdk_manages_connectors_remotely( client.connector_status(connector_id=uuid4()) +def test_sdk_reads_machine_verifiable_pipeline_readiness( + client_surface: tuple[MemoryClient, _Ingest, _Connectors], +) -> None: + client, _, _ = client_surface + + report = client.pipeline_readiness(version_ids=(_VERSION_ID,)) + + assert report.ready is True + assert report.model_bindings == {"claim_extraction": "model-v1"} + + def test_sdk_validates_lineage_pair_and_maps_api_failures( client_surface: tuple[MemoryClient, _Ingest, _Connectors], ) -> None: diff --git a/src/tests/surfaces/test_recipe_registry.py b/src/tests/surfaces/test_recipe_registry.py index 0839cecc..7c5e6d85 100644 --- a/src/tests/surfaces/test_recipe_registry.py +++ b/src/tests/surfaces/test_recipe_registry.py @@ -332,6 +332,41 @@ def test_seeding_is_idempotent_and_round_trips_every_chain(corpus: _Corpus) -> N assert by_name[canonical.name].answer_intent == canonical.answer_intent +def test_seeding_upgrades_a_changed_recipe_instead_of_masking_it( + corpus: _Corpus, +) -> None: + """A v1 row must not hide the bounded v2 public parameter schema.""" + registry = RecipeRegistry(engine=corpus.engine) + current = next( + recipe for recipe in CANONICAL_RECIPES if recipe.name == "claims_verbatim" + ) + registry.register( + deployment_id=_DEPLOYMENT_ID, + recipe=current.model_copy( + update={ + "version": 1, + "parameters": { + "query": {"type": "string", "required": True}, + "k": {"type": "integer", "required": False, "default": 10}, + }, + } + ), + ) + + seed_canonical_recipes(registry=registry, deployment_id=_DEPLOYMENT_ID) + + active = registry.by_name(deployment_id=_DEPLOYMENT_ID, name="claims_verbatim") + assert active is not None + assert active.version == 2 + assert active.parameters["k"] == { + "type": "integer", + "required": False, + "default": 10, + "minimum": 1, + "maximum": 30, + } + + # --- a recipe ≡ its chain -------------------------------------------------- @@ -342,6 +377,7 @@ def test_every_recipe_equals_its_hand_composed_chain(corpus: _Corpus) -> None: executor = RecipeExecutor(query_engine=engine) alice, acme = corpus.ids["Alice"], corpus.ids["Acme"] arguments: dict[str, dict[str, object]] = { + "resolve_entity": {"name": "Alice"}, "relation_current": {"subject_entity_id": alice, "predicate": "works_for"}, "observation_current": {"entity_id": acme}, "entity_timeline": {"entity_id": alice}, @@ -354,6 +390,7 @@ def test_every_recipe_equals_its_hand_composed_chain(corpus: _Corpus) -> None: } # direct hand-composition of each recipe's chain, primitive by primitive direct = { + "resolve_entity": engine.resolve(deployment_id=_DEPLOYMENT_ID, name="Alice"), "relation_current": engine.lookup_relations( deployment_id=_DEPLOYMENT_ID, subject_entity_id=alice, predicate="works_for" ), diff --git a/src/tests/surfaces/test_retrieval_api.py b/src/tests/surfaces/test_retrieval_api.py index e8b96d1f..36c2f4ed 100644 --- a/src/tests/surfaces/test_retrieval_api.py +++ b/src/tests/surfaces/test_retrieval_api.py @@ -42,9 +42,11 @@ from rememberstack.spine import EntityRegistry from rememberstack.spine import FactCatalog from rememberstack.spine import ForgetCatalog +from rememberstack.spine import LifecycleCatalog from rememberstack.spine import ObservationAdjudicator from rememberstack.spine import ObservationSettings from rememberstack.spine import RESOLVER_VERSION +from rememberstack.spine import ReviewQueue from rememberstack.spine import SupersessionAdjudicator from rememberstack.spine import SupersessionSettings from rememberstack.spine import WorkLedger @@ -66,6 +68,7 @@ from rememberstack.workers import LabelFactsHandler from rememberstack.workers import NormalizeRelationsHandler from rememberstack.workers import P1Settings +from rememberstack.workers import ReconcileHandler from rememberstack.workers import StructureHandler from rememberstack.workers import UploadIngestor from rememberstack.workers import Worker @@ -308,6 +311,14 @@ def __init__(self, *, engine: Engine, root: Path) -> None: chunker_version=generation, ), ) + registry.register( + stage=PipelineStage.RECONCILE, + handler=ReconcileHandler( + catalog=LifecycleCatalog(engine=engine), + review_queue=ReviewQueue(engine=engine), + chunker_version=generation, + ), + ) registry.register( stage=PipelineStage.LABEL_RELATION, handler=LabelFactsHandler( @@ -352,6 +363,7 @@ def build_corpus(self) -> None: PipelineStage.NORMALIZE_RELATIONS, PipelineStage.ADJUDICATE_SUPERSESSION, PipelineStage.EMBED_CLAIM, + PipelineStage.RECONCILE, PipelineStage.LABEL_RELATION, ): outcome = self.worker.run_one( diff --git a/src/tests/workers/test_e3_chain.py b/src/tests/workers/test_e3_chain.py index 70381343..58b64d32 100644 --- a/src/tests/workers/test_e3_chain.py +++ b/src/tests/workers/test_e3_chain.py @@ -35,9 +35,11 @@ from rememberstack.spine import EntityRegistry from rememberstack.spine import FactCatalog from rememberstack.spine import ForgetCatalog +from rememberstack.spine import LifecycleCatalog from rememberstack.spine import ObservationAdjudicator from rememberstack.spine import ObservationSettings from rememberstack.spine import RESOLVER_VERSION +from rememberstack.spine import ReviewQueue from rememberstack.spine import SupersessionAdjudicator from rememberstack.spine import SupersessionSettings from rememberstack.spine import WorkLedger @@ -56,6 +58,7 @@ from rememberstack.workers import LabelFactsHandler from rememberstack.workers import NormalizeRelationsHandler from rememberstack.workers import P1Settings +from rememberstack.workers import ReconcileHandler from rememberstack.workers import StructureHandler from rememberstack.workers import UploadIngestor from rememberstack.workers import Worker @@ -315,6 +318,14 @@ def __init__(self, *, engine: Engine, root: Path) -> None: registry.register( stage=PipelineStage.LABEL_RELATION, handler=self.label_handler ) + registry.register( + stage=PipelineStage.RECONCILE, + handler=ReconcileHandler( + catalog=LifecycleCatalog(engine=engine), + review_queue=ReviewQueue(engine=engine), + chunker_version=chunker_version(params=_PARAMS), + ), + ) self.worker = Worker(ledger=ledger, registry=registry) def run_chain(self) -> None: @@ -328,6 +339,7 @@ def run_chain(self) -> None: PipelineStage.NORMALIZE_RELATIONS, PipelineStage.ADJUDICATE_SUPERSESSION, PipelineStage.EMBED_CLAIM, + PipelineStage.RECONCILE, PipelineStage.LABEL_RELATION, ): outcome = self.worker.run_one( @@ -421,6 +433,48 @@ def test_same_fact_twice_is_one_relation_with_lineage_distinct_count( ] +def test_empty_document_completes_the_same_terminal_pipeline_without_model_calls( + rig: _E3Rig, +) -> None: + """An empty but valid memory is terminal, not permanently unready.""" + ingested = rig.ingestor.ingest( + deployment_id=_DEPLOYMENT_ID, + upload=DocumentUpload(filename="empty.md", mime="text/markdown", content=b""), + ) + + rig.run_chain() + + with rig.engine.connect() as connection: + rows = connection.execute( + text( + "SELECT stage::text AS stage, status::text AS status" + " FROM processing_state" + " WHERE deployment_id = :deployment_id" + " AND target_id = :version_id" + ), + {"deployment_id": _DEPLOYMENT_ID, "version_id": ingested.version_id}, + ).all() + + assert {stage for stage, _status in rows} == { + stage.value + for stage in ( + PipelineStage.CONVERT, + PipelineStage.STRUCTURE, + PipelineStage.CHUNK, + PipelineStage.EMBED_CHUNK, + PipelineStage.EXTRACT_CLAIMS, + PipelineStage.NORMALIZE_RELATIONS, + PipelineStage.ADJUDICATE_SUPERSESSION, + PipelineStage.EMBED_CLAIM, + PipelineStage.RECONCILE, + PipelineStage.LABEL_RELATION, + ) + } + assert len(rows) == 10 + assert {status for _stage, status in rows} == {"succeeded"} + assert rig.provider.generated_prompts == [] + + def test_rerunning_normalization_replays_without_model_calls(rig: _E3Rig) -> None: """D7 replay: a second normalize pass reads mentions and never re-calls.""" rig.ingestor.ingest( diff --git a/src/tests/workers/test_lifecycle_reconciliation.py b/src/tests/workers/test_lifecycle_reconciliation.py index 5d29655d..d74a1a9e 100644 --- a/src/tests/workers/test_lifecycle_reconciliation.py +++ b/src/tests/workers/test_lifecycle_reconciliation.py @@ -101,8 +101,8 @@ PipelineStage.NORMALIZE_RELATIONS, PipelineStage.ADJUDICATE_SUPERSESSION, PipelineStage.EMBED_CLAIM, - PipelineStage.LABEL_RELATION, PipelineStage.RECONCILE, + PipelineStage.LABEL_RELATION, ) _TARGET_PATTERN = re.compile(r"TARGET CHUNK:\n(.+)") _TABLES = ( @@ -754,7 +754,9 @@ def test_interrupted_reconcile_completes_on_retry(rig: _LifecycleRig) -> None: # drain everything EXCEPT reconcile, so its work row sits queued while True: progressed = False - for stage in _STAGES[:-1]: + for stage in ( + stage for stage in _STAGES if stage is not PipelineStage.RECONCILE + ): outcome = rig.worker.run_one( deployment_id=_DEPLOYMENT_ID, stage=stage, lane=ProcessingLane.STEADY ).outcome diff --git a/src/tests/workers/test_p2_rebuild.py b/src/tests/workers/test_p2_rebuild.py index 2dec43b1..ecbf5a78 100644 --- a/src/tests/workers/test_p2_rebuild.py +++ b/src/tests/workers/test_p2_rebuild.py @@ -322,8 +322,7 @@ def test_validation_gate_aborts_on_a_merge_cycle( def test_reader_hot_swaps_to_a_newer_snapshot(corpus: _Corpus, tmp_path: Path) -> None: - """The reader serves v1, keeps serving through a rebuild, and swaps to - v2 on refresh — old snapshots remain point-in-time artifacts.""" + """An ordinary connection observes v2 without an API-process restart.""" worker, reader, _ = _rig(corpus.engine, tmp_path) first = worker.rebuild(deployment_id=_DEPLOYMENT_ID, workdir=tmp_path / "work") assert reader.refresh() is True @@ -333,10 +332,9 @@ def test_reader_hot_swaps_to_a_newer_snapshot(corpus: _Corpus, tmp_path: Path) - with corpus.engine.begin() as connection: # the corpus grows _seed_entity(connection, entity_id=uuid4(), name="Newcomer") second = worker.rebuild(deployment_id=_DEPLOYMENT_ID, workdir=tmp_path / "work") - assert reader.version == first["version"] # stable until asked - assert reader.refresh() is True - assert reader.version == second["version"] + assert reader.version == first["version"] # stable until the next read nodes = _scalar(reader.connection(), "MATCH (e:Entity) RETURN count(*)") + assert reader.version == second["version"] assert nodes == 4 # the newcomer arrived with the swap diff --git a/website/src/app/docs/deployment/page.mdx b/website/src/app/docs/deployment/page.mdx index c7032eee..9a032ded 100644 --- a/website/src/app/docs/deployment/page.mdx +++ b/website/src/app/docs/deployment/page.mdx @@ -1,24 +1,18 @@ export const metadata = { title: "Self-host Deployment", description: - "Run the fresh-deployment Docker Compose skeleton with PostgreSQL, MinIO, the API, and E0 workers.", + "Run one complete self-host deployment with PostgreSQL, MinIO, the API, continuous workers, and explicit projections.", }; # Self-host Deployment -The repository ships a Docker Compose skeleton for a **fresh, single-deployment** -self-host. It starts PostgreSQL, MinIO, the HTTP API, and separate conversion and -structure workers. The one-shot `setup` service applies every migration before -the API or workers start, provisions the three object buckets, bootstraps the -deployment, and seeds the canonical retrieval recipes. - -This is a pre-release infrastructure proof. It processes a Markdown upload -through conversion and deterministic structure, then leaves its chunk work -pending. Later pipeline workers are not yet wired into this quickstart. The -release workflow and tag-pinned Compose contract exist and trusted publishing is -configured. The bounded contributor agreement is enforced as a required `main` -check. The public `v0.1.0` release is available from PyPI, GHCR, and GitHub -Releases. +The repository ships Docker Compose for a **fresh, single-deployment** self-host. +It starts PostgreSQL, MinIO, the HTTP API, and one process for each of the ten +implemented continuous document routes: conversion, structure, chunking, chunk +embedding, claim extraction, normalization, supersession, claim embedding, +reconciliation, and fact labeling. The one-shot `setup` service applies every +migration, provisions object buckets, bootstraps the deployment, and seeds +canonical plus P2 retrieval recipes. The RememberStack, `rememberstack`, and `remember` names used here are final. Do not expose this stack to untrusted traffic. @@ -38,9 +32,9 @@ also ships this Compose file pinned to The example file contains local-only PostgreSQL and MinIO credentials. Replace every secret before using the stack outside an isolated development machine. -It also contains an OpenRouter placeholder so the provider adapter can be -constructed without making a model request. Replace it with a real key before -processing a corpus; the short smoke upload below makes no LLM or embedding call. +It also contains an OpenRouter placeholder. Replace it before processing a +corpus: the complete pipeline makes extraction and embedding calls. Pin explicit +model IDs for reproducible work; do not use a rotating free-model router. Check the API and the deployment-seeded recipe registry: @@ -49,7 +43,7 @@ curl --fail http://localhost:8000/healthz curl --fail http://localhost:8000/recipes ``` -## Ingest the smoke document +## Ingest a document Create a short Markdown document and push it through the ordinary E0 API: @@ -60,20 +54,33 @@ curl --fail \ 'http://localhost:8000/ingest?filename=remember-smoke.md&mime=text%2Fmarkdown' ``` -The response names the deployment, document, and version. Both E0 workers use -the normal PostgreSQL work ledger: `convert` and `structure` become `succeeded`, -and the next `chunk` row remains visibly `pending` because this skeleton does not -pretend the later worker topology is composed. +The response names the deployment, document, and version. Every worker uses the +normal PostgreSQL work ledger. Inspect its exact state: ```bash docker compose exec postgres psql -U rememberstack -d rememberstack -c \ - "SELECT stage, status FROM processing_state ORDER BY created_at" + "SELECT stage, status FROM processing_state ORDER BY enqueued_at" ``` The original bytes and derived Markdown/sidecars are in the MinIO buckets. The MinIO console is available at `http://localhost:9001` with the credentials from `.env`. +## Publish aggregate projections + +P2 and P3 are whole-corpus rebuilds, not per-document workers. After the selected +ingestion set has settled, publish both once: + +```bash +docker compose --profile operations run --rm projections +``` + +The API reads the same MinIO snapshot stores. `POST /readiness` can then verify a +bounded list of document version IDs: all ten exact component generations must +be terminal, and both projection builds must begin after the latest terminal stage. +The response also records the API process's current non-secret model bindings for +configuration review; these are not processing-time provenance. + ## Stop and reset Stop containers while retaining state: diff --git a/website/src/app/docs/project-status/page.mdx b/website/src/app/docs/project-status/page.mdx index 871c0c3e..8adcdfa6 100644 --- a/website/src/app/docs/project-status/page.mdx +++ b/website/src/app/docs/project-status/page.mdx @@ -22,7 +22,7 @@ RememberStack is being built **design-first**: the complete system — requireme - **Phase 5, complete** — retrieval complete. The full zero-LLM primitive set is implemented (`fuse`, `rerank`, `transcript`, `delta`, `pages_about`, enumerated `aggregate`, and a streaming batch `scan`, alongside resolve/lookup/search/graph); recipes are registry rows whose declared grain is enforced mechanically; and the self-accounting envelope surfaces contradictions, withdrawn support, mixed-grain parts, identity regime, belief horizons, freshness, and typed negatives. API, CLI, and MCP all render the same deployment registry. The consumption skill is guarded by the repeatable S58 cold-agent eval, while the retrieval spike battery records filtered Lance search at 10 million rows and graph pagination at a 100,000-edge hub. The client-first `rememberstack` base wheel contains the typed SDK, remote `remember` CLI and MCP, lineage-aware E0 ingest, and typed remote connector-management commands plus their deployment composition port, without server dependencies; `[server]`, `[connectors-watched-directory]`, and `[k]` name the heavier install surfaces. - **Phase 6, complete** — Plane K now includes the deterministic control plane and crash-safe single-committer driver, exact rule routing and staleness, deterministic fact-sheet and agent-written prose bands, planner/reflection decisions with quarantine and adoption, and authored-page frontmatter, citations, watches, review flags, and debounced workflow dispatch. D73 removed the proposed K3 tier: personal or organizational principles are authored K2 content, supported by compiled scope pages and never machine-promoted or rewritten. - **Phase 3, complete** — the evidence lifecycle: documents that change. Watched sources poll as recorded sync cycles (a local-directory watcher ships; revision and content no-ops, debounce, source-deletion detection); an edited document becomes a new version of its lineage and the full structure route (an LLM-proposed section tree normalized by a deterministic snap) re-reads it; unchanged chunks reuse their prior claims, prefixes, and vectors so cost is proportional to the edit (measured hit rate 0.79 on the spike corpus); reconciliation transitions testimony currency on an append-only ledger, recounts evidence by distinct current lineages, closes solely-supported facts when the source withdrew them (at the sync-cycle barrier, so a moved section is a support swap — never a retract flicker) and flags them for review when only the toolchain changed; deletion removes a document's contribution uniformly while keeping its claims as history; and a standing `lifecycle` eval suite guards the cache/ledger/count invariants with planted regression canaries. Those lifecycle events now feed the graph, corpus filesystem, and Plane-K control plane described above. -- **Phase 7, complete** — operational correctness now includes resumable initial-load and version-bump backfill on the same work ledger, a reproducible PostgreSQL scale battery for the designed partitions, indexes, hubs, and batching invariants, and optional per-deployment/stage/lane cost ceilings. Budget exhaustion parks healthy work durably without consuming an attempt; the local CLI reports current spend, remaining budget, tier attribution, and parked work from the same authoritative rows. Typed worker telemetry preserves complete exception chains, while bounded CLI inspection exposes pipeline/DLQ state, poison targets, P2/P3 pointers, and currency-ledger drift; an explicit one-row replay grants only additional attempts, and rebuild drills call the production P2/P3 builders. Hard-forget now purges every active library-controlled surface and replays a separately durable, content-free manifest before serving after a restore. Portability is deliberately smaller than a backup product: operators move PostgreSQL, objects, and Git with native tools; the library defines the fail-closed restore order and rebuilds P1/P2/P3. A fresh-deployment Compose skeleton now wires PostgreSQL, MinIO, the API, and the conversion/structure workers through the real profile. Release engineering enforces one semantic version across PyPI, the shared API/worker GHCR image, and Compose; CI measures cold start and drills a schema downgrade followed by migrations-before-process-restart. The public [`v0.1.0`](https://github.com/writeitai/remember-stack/releases/tag/v0.1.0) release proved the complete tag workflow across PyPI, GHCR, and GitHub Releases; trusted publishing, protected release tags, and the required repository-native bounded-CLA check are active. Timings remain measurements rather than hosted SLAs, and no export/import CLI, dashboard, billing policy, HA topology, backup scheduler, or control plane enters the library. +- **Phase 7, complete** — operational correctness now includes resumable initial-load and version-bump backfill on the same work ledger, a reproducible PostgreSQL scale battery for the designed partitions, indexes, hubs, and batching invariants, and optional per-deployment/stage/lane cost ceilings. Budget exhaustion parks healthy work durably without consuming an attempt; the local CLI reports current spend, remaining budget, tier attribution, and parked work from the same authoritative rows. Typed worker telemetry preserves complete exception chains, while bounded CLI inspection exposes pipeline/DLQ state, poison targets, P2/P3 pointers, and currency-ledger drift; an explicit one-row replay grants only additional attempts, and rebuild drills call the production P2/P3 builders. Hard-forget now purges every active library-controlled surface and replays a separately durable, content-free manifest before serving after a restore. Portability is deliberately smaller than a backup product: operators move PostgreSQL, objects, and Git with native tools; the library defines the fail-closed restore order and rebuilds P1/P2/P3. The fresh-deployment Compose profile now runs PostgreSQL, MinIO, the API, and all ten implemented continuous E/P1 routes, with explicit one-shot P2/P3 publication and machine-verifiable per-version readiness. Release engineering enforces one semantic version across PyPI, the shared API/worker GHCR image, and Compose; CI measures cold start and drills a schema downgrade followed by migrations-before-process-restart. The public [`v0.1.0`](https://github.com/writeitai/remember-stack/releases/tag/v0.1.0) release proved the complete tag workflow across PyPI, GHCR, and GitHub Releases; trusted publishing, protected release tags, and the required repository-native bounded-CLA check are active. Timings remain measurements rather than hosted SLAs, and no export/import CLI, dashboard, billing policy, HA topology, backup scheduler, or control plane enters the library. ## The build order diff --git a/website/src/app/docs/reference/api/page.mdx b/website/src/app/docs/reference/api/page.mdx index 04cacd93..4ca3a32a 100644 --- a/website/src/app/docs/reference/api/page.mdx +++ b/website/src/app/docs/reference/api/page.mdx @@ -70,6 +70,9 @@ curl -X POST "$REMEMBERSTACK_API_URL/recipe/relation_current" \ ``` An unknown recipe is **404**; a missing required argument is **422**. +The stock self-host adds `resolve_entity` so recipe-only agents can obtain UUIDs, +plus `graph_neighborhood` and `graph_path` because that profile composes P2. +Profiles without P2 do not seed the graph recipes. ## Client writes @@ -86,6 +89,15 @@ must be supplied together. `source_modified_at`, `source_version_ref`, and bytes under the same identity append a document version; identical bytes are an idempotent no-op. +### `POST /readiness` + +When composed, accepts a JSON list of document-version UUIDs and returns the +exact expected stage generations and their states. With +`require_projections=true` (the default), readiness also requires both P2 and +P3 builds to have begun after the latest requested terminal stage. The response +includes non-secret model IDs, making writer/query configuration auditable. +This endpoint inspects work; it does not wait, retry, or trigger a rebuild. + ### Connector management | Method & path | Operation | @@ -122,6 +134,7 @@ with MemoryClient.from_settings() as memory: source_ref="workspace/project", source_version_ref="revision-17", ) + ready = memory.pipeline_readiness(version_ids=(landed.version_id,)) ``` The SDK also exposes typed `resolve`, `search_claims`, `hydrate_relation`, and From 4aace5e10f2c788ddeafa88bd737bf0f7f1e1d91 Mon Sep 17 00:00:00 2001 From: Jiri Puc Date: Fri, 24 Jul 2026 06:47:17 +0200 Subject: [PATCH 2/6] fix: exercise full Compose pipeline in CI --- .github/workflows/ci.yml | 15 +++++++++++++-- Dockerfile | 3 ++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9b8424d0..44413273 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,7 +53,7 @@ jobs: raise RuntimeError("MinIO replaced an immutable object key") PY - - name: Prove the zero-cost full continuous pipeline + - name: Prove the zero-cost full pipeline run: | running_services="$( docker compose --env-file .env.example ps --status running --services @@ -66,7 +66,8 @@ jobs: printf '%s\n' "$running_services" | grep -qx "$service" done ingested="$( - curl --fail --data-binary '' \ + curl --fail --header 'Content-Type: application/octet-stream' \ + --data-binary ' ' \ 'http://localhost:8000/ingest?filename=empty.md&mime=text%2Fmarkdown' )" version_id="$( @@ -88,6 +89,16 @@ jobs: test "$(docker compose --env-file .env.example exec -T postgres \ psql -U rememberstack -d rememberstack -Atc \ 'SELECT count(*) FROM cost_ledger')" = '0' + docker compose --env-file .env.example --profile operations \ + run --rm projections + readiness="$( + curl --fail --header 'Content-Type: application/json' \ + --data "[\"$version_id\"]" \ + 'http://localhost:8000/readiness?require_projections=true' + )" + printf '%s' "$readiness" | + docker compose --env-file .env.example exec -T api python -c \ + 'import json, sys; report = json.load(sys.stdin); assert report["ready"]; assert all(item["ready"] for item in report["projections"])' - name: Stage an upgrade from the prior schema run: | diff --git a/Dockerfile b/Dockerfile index 3a95a0db..4614330f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,8 @@ WORKDIR /app COPY pyproject.toml uv.lock README.md LICENSE alembic.ini ./ RUN addgroup --system app \ - && adduser --system --ingroup app app \ + && adduser --system --ingroup app \ + --home /var/lib/rememberstack --no-create-home app \ && mkdir -p /var/lib/rememberstack/forget-manifests \ && chown -R app:app /var/lib/rememberstack \ && uv sync --locked --no-dev --extra server --no-install-project From 87735426ade128ed214ef7dccfb17b749af0d358 Mon Sep 17 00:00:00 2001 From: Jiri Puc Date: Fri, 24 Jul 2026 14:23:41 +0200 Subject: [PATCH 3/6] fix: normalize strict provider schemas --- src/rememberstack/adapters/openrouter.py | 27 ++++++++- src/tests/adapters/test_openrouter.py | 70 ++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 1 deletion(-) diff --git a/src/rememberstack/adapters/openrouter.py b/src/rememberstack/adapters/openrouter.py index 4700f36e..660cb56d 100644 --- a/src/rememberstack/adapters/openrouter.py +++ b/src/rememberstack/adapters/openrouter.py @@ -64,7 +64,7 @@ def generate( "json_schema": { "name": response_type.__name__, "strict": True, - "schema": response_type.model_json_schema(), + "schema": _strict_json_schema(response_type), }, }, } @@ -125,6 +125,31 @@ def _post(self, *, path: str, payload: dict[str, object]) -> dict[str, Any]: return response.json() +def _strict_json_schema(response_type: type[StructuredResponseModel]) -> dict[str, Any]: + """Adapt Pydantic defaults to the strict schema subset used by OpenAI routes.""" + schema = response_type.model_json_schema() + _require_all_object_properties(schema) + return schema + + +def _require_all_object_properties(node: object) -> None: + """Make every declared property required and remove unsupported defaults.""" + if isinstance(node, list): + for item in node: + _require_all_object_properties(item) + return + if not isinstance(node, dict): + return + + node.pop("default", None) + properties = node.get("properties") + if isinstance(properties, dict): + node["required"] = list(properties) + node["additionalProperties"] = False + for value in node.values(): + _require_all_object_properties(value) + + def _usage( *, body: dict[str, Any], requested_model: str, latency_ms: int ) -> ProviderCallUsage: diff --git a/src/tests/adapters/test_openrouter.py b/src/tests/adapters/test_openrouter.py index d5042aca..5eb46d13 100644 --- a/src/tests/adapters/test_openrouter.py +++ b/src/tests/adapters/test_openrouter.py @@ -10,10 +10,14 @@ from rememberstack.adapters import OpenRouterModelProvider from rememberstack.adapters import OpenRouterProviderError from rememberstack.adapters import OpenRouterSettings +from rememberstack.adapters.openrouter import _strict_json_schema from rememberstack.adapters.openrouter import _usage from rememberstack.model import EmbeddingRequest from rememberstack.model import ModelRequest +from rememberstack.model import NormalizationResponse from rememberstack.model import ProviderAccountingError +from rememberstack.model import SelectionResponse +from rememberstack.model import StructureResponse class _Answer(BaseModel): @@ -89,6 +93,72 @@ def post(*, path: str, payload: dict[str, object]) -> dict[str, object]: assert observed["temperature"] == 0.0 +def test_generation_uses_strict_schema_for_defaulted_response_fields( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """OpenAI-backed routes require every property, even when Pydantic has defaults.""" + provider = OpenRouterModelProvider(settings=OpenRouterSettings(api_key="test-key")) + observed_schema: dict[str, object] = {} + + def post(*, path: str, payload: dict[str, object]) -> dict[str, object]: + assert path == "/chat/completions" + response_format = payload["response_format"] + assert isinstance(response_format, dict) + json_schema = response_format["json_schema"] + assert isinstance(json_schema, dict) + schema = json_schema["schema"] + assert isinstance(schema, dict) + observed_schema.update(schema) + return { + "model": "openai/strict-model", + "usage": {"prompt_tokens": 3, "completion_tokens": 1, "cost": "0"}, + "choices": [{"message": {"content": '{"relations":[],"observations":[]}'}}], + } + + monkeypatch.setattr(provider, "_post", post) + try: + provider.generate( + request=ModelRequest(model="openai/strict-model", prompt="Normalize"), + response_type=NormalizationResponse, + ) + finally: + provider._client.close() + + required = observed_schema["required"] + assert isinstance(required, list) + assert set(required) == {"relations", "observations"} + assert observed_schema["additionalProperties"] is False + properties = observed_schema["properties"] + assert isinstance(properties, dict) + assert "default" not in properties["relations"] + assert "default" not in properties["observations"] + + +@pytest.mark.parametrize("response_type", (StructureResponse, SelectionResponse)) +def test_strict_schema_closes_every_nested_object_and_removes_defaults( + response_type: type[BaseModel], +) -> None: + """Recursive and nullable production schemas remain strict at every depth.""" + schema = _strict_json_schema(response_type) + + def assert_strict(node: object) -> None: + if isinstance(node, list): + for item in node: + assert_strict(item) + return + if not isinstance(node, dict): + return + assert "default" not in node + properties = node.get("properties") + if isinstance(properties, dict): + assert set(node["required"]) == set(properties) + assert node["additionalProperties"] is False + for value in node.values(): + assert_strict(value) + + assert_strict(schema) + + def test_generation_preserves_usage_on_structured_output_validation_error( monkeypatch: pytest.MonkeyPatch, ) -> None: From 58da1c09c227e23eebcda44bcecb4b334a08aadb Mon Sep 17 00:00:00 2001 From: Jiri Puc Date: Fri, 24 Jul 2026 15:08:03 +0200 Subject: [PATCH 4/6] feat: pin OpenRouter embedding provider --- .env.example | 2 + compose.yaml | 1 + src/rememberstack/adapters/openrouter.py | 24 ++++++-- src/rememberstack/profiles/selfhost.py | 2 + src/tests/adapters/test_openrouter.py | 68 +++++++++++++++++++++ src/tests/profiles/test_selfhost_profile.py | 19 ++++++ 6 files changed, 112 insertions(+), 4 deletions(-) diff --git a/.env.example b/.env.example index 74f4f63c..119e6fe3 100644 --- a/.env.example +++ b/.env.example @@ -14,6 +14,8 @@ REMEMBERSTACK_SELFHOST_API_PORT=8000 # Required to start the provider adapter. Replace this value before processing # a corpus; the complete Compose pipeline makes extraction and embedding calls. REMEMBERSTACK_OPENROUTER_API_KEY=replace-before-real-use +# Optional exact OpenRouter embedding-provider slug; unset keeps automatic routing. +# REMEMBERSTACK_OPENROUTER_EMBEDDING_PROVIDER=nebius # Optional benchmark/deployment model overrides. Keep explicit model IDs for a # reproducible run; do not use a rotating router such as openrouter/free. diff --git a/compose.yaml b/compose.yaml index 4b06f46e..b9371f89 100644 --- a/compose.yaml +++ b/compose.yaml @@ -11,6 +11,7 @@ x-app: &app REMEMBERSTACK_MINIO_ACCESS_KEY: ${REMEMBERSTACK_MINIO_ACCESS_KEY} REMEMBERSTACK_MINIO_SECRET_KEY: ${REMEMBERSTACK_MINIO_SECRET_KEY} REMEMBERSTACK_OPENROUTER_API_KEY: ${REMEMBERSTACK_OPENROUTER_API_KEY} + REMEMBERSTACK_OPENROUTER_EMBEDDING_PROVIDER: ${REMEMBERSTACK_OPENROUTER_EMBEDDING_PROVIDER:-} REMEMBERSTACK_SELFHOST_DEPLOYMENT_ID: ${REMEMBERSTACK_SELFHOST_DEPLOYMENT_ID} REMEMBERSTACK_SELFHOST_DEPLOYMENT_SLUG: ${REMEMBERSTACK_SELFHOST_DEPLOYMENT_SLUG} REMEMBERSTACK_SELFHOST_DEPLOYMENT_NAME: ${REMEMBERSTACK_SELFHOST_DEPLOYMENT_NAME} diff --git a/src/rememberstack/adapters/openrouter.py b/src/rememberstack/adapters/openrouter.py index 660cb56d..4fb1ffa6 100644 --- a/src/rememberstack/adapters/openrouter.py +++ b/src/rememberstack/adapters/openrouter.py @@ -9,6 +9,7 @@ import httpx from pydantic import Field +from pydantic import field_validator from pydantic import ValidationError from pydantic_settings import BaseSettings from pydantic_settings import SettingsConfigDict @@ -33,6 +34,15 @@ class OpenRouterSettings(BaseSettings): api_key: str = Field(min_length=1) base_url: str = Field(default="https://openrouter.ai/api/v1") timeout_s: float = Field(default=120.0, gt=0) + embedding_provider: str | None = None + + @field_validator("embedding_provider", mode="before") + @classmethod + def normalize_optional_provider(cls, value: object) -> object: + """Treat Compose's empty optional value as no provider pin.""" + if not isinstance(value, str): + return value + return value.strip() or None class OpenRouterProviderError(ProviderCallError): @@ -95,10 +105,16 @@ def generate( def embed(self, *, request: EmbeddingRequest) -> EmbeddingResponse: """One embeddings call for the caller's batch.""" started_ns = time.monotonic_ns() - body = self._post( - path="/embeddings", - payload={"model": request.model, "input": list(request.texts)}, - ) + payload: dict[str, object] = { + "model": request.model, + "input": list(request.texts), + } + if self._settings.embedding_provider: + payload["provider"] = { + "only": [self._settings.embedding_provider], + "allow_fallbacks": False, + } + body = self._post(path="/embeddings", payload=payload) usage = _usage( body=body, requested_model=request.model, diff --git a/src/rememberstack/profiles/selfhost.py b/src/rememberstack/profiles/selfhost.py index 1c5e735d..701eb90b 100644 --- a/src/rememberstack/profiles/selfhost.py +++ b/src/rememberstack/profiles/selfhost.py @@ -567,6 +567,7 @@ def _model_bindings() -> dict[str, str]: observations = ObservationSettings.model_validate({}) supersession = SupersessionSettings.model_validate({}) p1 = P1Settings.model_validate({}) + openrouter = OpenRouterSettings.model_validate({}) return { "structure": structurer.model, "chunk_embedding": e1.embedding_model, @@ -580,6 +581,7 @@ def _model_bindings() -> dict[str, str]: "supersession_frontier": supersession.frontier_model, "p1_embedding": p1.embedding_model, "fact_label": p1.label_model, + "openrouter_embedding_provider": openrouter.embedding_provider or "auto", } diff --git a/src/tests/adapters/test_openrouter.py b/src/tests/adapters/test_openrouter.py index 5eb46d13..0d06c40a 100644 --- a/src/tests/adapters/test_openrouter.py +++ b/src/tests/adapters/test_openrouter.py @@ -200,6 +200,7 @@ def test_embedding_preserves_usage_when_the_vector_body_is_unusable( def post(*, path: str, payload: dict[str, object]) -> dict[str, object]: assert path == "/embeddings" assert payload + assert "provider" not in payload return { "model": "qwen/qwen3-embedding-8b", "usage": {"prompt_tokens": 4, "cost": "0.000004"}, @@ -220,3 +221,70 @@ def post(*, path: str, payload: dict[str, object]) -> dict[str, object]: assert raised.value.usage is not None assert raised.value.usage.tokens_in == 4 assert raised.value.usage.cost_usd == Decimal("0.000004") + + +def test_embedding_pins_configured_provider_without_fallback( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A benchmark can freeze one embedding provider instead of load balancing.""" + provider = OpenRouterModelProvider( + settings=OpenRouterSettings(api_key="test-key", embedding_provider="nebius") + ) + observed: dict[str, object] = {} + + def post(*, path: str, payload: dict[str, object]) -> dict[str, object]: + assert path == "/embeddings" + observed.update(payload) + return { + "model": "qwen/qwen3-embedding-8b", + "usage": {"prompt_tokens": 2, "cost": "0.000001"}, + "data": [{"index": 0, "embedding": [0.1, 0.2]}], + } + + monkeypatch.setattr(provider, "_post", post) + try: + response = provider.embed( + request=EmbeddingRequest(model="qwen/qwen3-embedding-8b", texts=("memory",)) + ) + finally: + provider._client.close() + + assert observed["provider"] == {"only": ["nebius"], "allow_fallbacks": False} + assert response.vectors == ((0.1, 0.2),) + + +@pytest.mark.parametrize("configured", (None, "", " ")) +def test_empty_embedding_provider_is_unset(configured: str | None) -> None: + """Compose's empty optional value must preserve automatic provider routing.""" + settings = OpenRouterSettings(api_key="test-key", embedding_provider=configured) + + assert settings.embedding_provider is None + + +def test_embedding_provider_pin_is_not_forwarded_to_generation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Embedding routing must not constrain independently hosted chat models.""" + provider = OpenRouterModelProvider( + settings=OpenRouterSettings(api_key="test-key", embedding_provider="nebius") + ) + + def post(*, path: str, payload: dict[str, object]) -> dict[str, object]: + assert path == "/chat/completions" + assert "provider" not in payload + return { + "model": "deepseek/deepseek-v4-flash", + "usage": {"prompt_tokens": 2, "completion_tokens": 1, "cost": "0"}, + "choices": [{"message": {"content": '{"answer":"Prague"}'}}], + } + + monkeypatch.setattr(provider, "_post", post) + try: + response = provider.generate( + request=ModelRequest(model="deepseek/deepseek-v4-flash", prompt="Where?"), + response_type=_Answer, + ) + finally: + provider._client.close() + + assert response.output.answer == "Prague" diff --git a/src/tests/profiles/test_selfhost_profile.py b/src/tests/profiles/test_selfhost_profile.py index ceee540d..939f1aa9 100644 --- a/src/tests/profiles/test_selfhost_profile.py +++ b/src/tests/profiles/test_selfhost_profile.py @@ -3,8 +3,11 @@ from pathlib import Path import re +import pytest + from rememberstack.model import PipelineStage from rememberstack.profiles.selfhost import _expected_components +from rememberstack.profiles.selfhost import _model_bindings from rememberstack.profiles.selfhost import _SUPPORTED_WORKER_STAGES _ROOT = Path(__file__).resolve().parents[3] @@ -52,3 +55,19 @@ def test_compose_wires_the_exact_supported_worker_set_and_projection_job() -> No assert composed_stages == _SUPPORTED_WORKER_STAGES assert 'profiles: ["operations"]' in compose assert 'command: ["project", "--plane", "all"]' in compose + + +@pytest.mark.parametrize( + ("configured", "reported"), (("nebius", "nebius"), ("", "auto")) +) +def test_model_bindings_report_embedding_provider_without_secrets( + monkeypatch: pytest.MonkeyPatch, configured: str, reported: str +) -> None: + """Readiness fingerprints the routing choice without exposing credentials.""" + monkeypatch.setenv("REMEMBERSTACK_OPENROUTER_API_KEY", "test-key") + monkeypatch.setenv("REMEMBERSTACK_OPENROUTER_EMBEDDING_PROVIDER", configured) + + bindings = _model_bindings() + + assert bindings["openrouter_embedding_provider"] == reported + assert "test-key" not in bindings.values() From 88046be571a12d08bbd944733a85e0345516455f Mon Sep 17 00:00:00 2001 From: Jiri Puc Date: Fri, 24 Jul 2026 15:55:52 +0200 Subject: [PATCH 5/6] fix: make OpenRouter reasoning effort explicit --- .env.example | 5 +++ compose.yaml | 1 + src/rememberstack/adapters/openrouter.py | 12 +++++-- src/rememberstack/profiles/selfhost.py | 1 + src/tests/adapters/test_openrouter.py | 39 +++++++++++++++++++++ src/tests/profiles/test_selfhost_profile.py | 13 +++++++ 6 files changed, 68 insertions(+), 3 deletions(-) diff --git a/.env.example b/.env.example index 119e6fe3..4ec90c70 100644 --- a/.env.example +++ b/.env.example @@ -16,6 +16,11 @@ REMEMBERSTACK_SELFHOST_API_PORT=8000 REMEMBERSTACK_OPENROUTER_API_KEY=replace-before-real-use # Optional exact OpenRouter embedding-provider slug; unset keeps automatic routing. # REMEMBERSTACK_OPENROUTER_EMBEDDING_PROVIDER=nebius +# Optional global reasoning effort for every chat-generation call. Allowed: +# none, minimal, low, medium, high, xhigh, max. Unset uses the model default; +# "none" reduces extraction latency but can reduce adjudication quality, and +# models that require reasoning may reject it. +# REMEMBERSTACK_OPENROUTER_REASONING_EFFORT=none # Optional benchmark/deployment model overrides. Keep explicit model IDs for a # reproducible run; do not use a rotating router such as openrouter/free. diff --git a/compose.yaml b/compose.yaml index b9371f89..423ae986 100644 --- a/compose.yaml +++ b/compose.yaml @@ -12,6 +12,7 @@ x-app: &app REMEMBERSTACK_MINIO_SECRET_KEY: ${REMEMBERSTACK_MINIO_SECRET_KEY} REMEMBERSTACK_OPENROUTER_API_KEY: ${REMEMBERSTACK_OPENROUTER_API_KEY} REMEMBERSTACK_OPENROUTER_EMBEDDING_PROVIDER: ${REMEMBERSTACK_OPENROUTER_EMBEDDING_PROVIDER:-} + REMEMBERSTACK_OPENROUTER_REASONING_EFFORT: ${REMEMBERSTACK_OPENROUTER_REASONING_EFFORT:-} REMEMBERSTACK_SELFHOST_DEPLOYMENT_ID: ${REMEMBERSTACK_SELFHOST_DEPLOYMENT_ID} REMEMBERSTACK_SELFHOST_DEPLOYMENT_SLUG: ${REMEMBERSTACK_SELFHOST_DEPLOYMENT_SLUG} REMEMBERSTACK_SELFHOST_DEPLOYMENT_NAME: ${REMEMBERSTACK_SELFHOST_DEPLOYMENT_NAME} diff --git a/src/rememberstack/adapters/openrouter.py b/src/rememberstack/adapters/openrouter.py index 4fb1ffa6..3b923854 100644 --- a/src/rememberstack/adapters/openrouter.py +++ b/src/rememberstack/adapters/openrouter.py @@ -5,6 +5,7 @@ import json import time from typing import Any +from typing import Literal from typing import TypeVar import httpx @@ -35,11 +36,14 @@ class OpenRouterSettings(BaseSettings): base_url: str = Field(default="https://openrouter.ai/api/v1") timeout_s: float = Field(default=120.0, gt=0) embedding_provider: str | None = None + reasoning_effort: ( + Literal["none", "minimal", "low", "medium", "high", "xhigh", "max"] | None + ) = None - @field_validator("embedding_provider", mode="before") + @field_validator("embedding_provider", "reasoning_effort", mode="before") @classmethod - def normalize_optional_provider(cls, value: object) -> object: - """Treat Compose's empty optional value as no provider pin.""" + def normalize_optional_string(cls, value: object) -> object: + """Treat Compose's empty optional values as unset.""" if not isinstance(value, str): return value return value.strip() or None @@ -80,6 +84,8 @@ def generate( } if request.temperature is not None: payload["temperature"] = request.temperature + if self._settings.reasoning_effort is not None: + payload["reasoning"] = {"effort": self._settings.reasoning_effort} body = self._post(path="/chat/completions", payload=payload) usage = _usage( body=body, diff --git a/src/rememberstack/profiles/selfhost.py b/src/rememberstack/profiles/selfhost.py index 701eb90b..f6866c56 100644 --- a/src/rememberstack/profiles/selfhost.py +++ b/src/rememberstack/profiles/selfhost.py @@ -582,6 +582,7 @@ def _model_bindings() -> dict[str, str]: "p1_embedding": p1.embedding_model, "fact_label": p1.label_model, "openrouter_embedding_provider": openrouter.embedding_provider or "auto", + "openrouter_reasoning_effort": openrouter.reasoning_effort or "auto", } diff --git a/src/tests/adapters/test_openrouter.py b/src/tests/adapters/test_openrouter.py index 0d06c40a..37796347 100644 --- a/src/tests/adapters/test_openrouter.py +++ b/src/tests/adapters/test_openrouter.py @@ -93,6 +93,44 @@ def post(*, path: str, payload: dict[str, object]) -> dict[str, object]: assert observed["temperature"] == 0.0 +@pytest.mark.parametrize( + ("configured", "expected"), + ((None, None), ("", None), (" ", None), ("none", {"effort": "none"})), +) +def test_generation_forwards_configured_reasoning_effort( + monkeypatch: pytest.MonkeyPatch, + configured: str | None, + expected: dict[str, str] | None, +) -> None: + """A deployment can disable unnecessary reasoning without changing its model.""" + provider = OpenRouterModelProvider( + settings=OpenRouterSettings.model_validate( + {"api_key": "test-key", "reasoning_effort": configured} + ) + ) + observed: dict[str, object] = {} + + def post(*, path: str, payload: dict[str, object]) -> dict[str, object]: + observed.update(payload) + assert path == "/chat/completions" + return { + "model": "deepseek/deepseek-v4-flash", + "usage": {"prompt_tokens": 3, "completion_tokens": 1, "cost": "0"}, + "choices": [{"message": {"content": '{"answer":"Prague"}'}}], + } + + monkeypatch.setattr(provider, "_post", post) + try: + provider.generate( + request=ModelRequest(model="deepseek/deepseek-v4-flash", prompt="Where?"), + response_type=_Answer, + ) + finally: + provider._client.close() + + assert observed.get("reasoning") == expected + + def test_generation_uses_strict_schema_for_defaulted_response_fields( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -235,6 +273,7 @@ def test_embedding_pins_configured_provider_without_fallback( def post(*, path: str, payload: dict[str, object]) -> dict[str, object]: assert path == "/embeddings" observed.update(payload) + assert "reasoning" not in payload return { "model": "qwen/qwen3-embedding-8b", "usage": {"prompt_tokens": 2, "cost": "0.000001"}, diff --git a/src/tests/profiles/test_selfhost_profile.py b/src/tests/profiles/test_selfhost_profile.py index 939f1aa9..11e2542c 100644 --- a/src/tests/profiles/test_selfhost_profile.py +++ b/src/tests/profiles/test_selfhost_profile.py @@ -71,3 +71,16 @@ def test_model_bindings_report_embedding_provider_without_secrets( assert bindings["openrouter_embedding_provider"] == reported assert "test-key" not in bindings.values() + + +@pytest.mark.parametrize(("configured", "reported"), (("none", "none"), ("", "auto"))) +def test_model_bindings_report_reasoning_effort( + monkeypatch: pytest.MonkeyPatch, configured: str, reported: str +) -> None: + """Readiness fingerprints the configured generation reasoning policy.""" + monkeypatch.setenv("REMEMBERSTACK_OPENROUTER_API_KEY", "test-key") + monkeypatch.setenv("REMEMBERSTACK_OPENROUTER_REASONING_EFFORT", configured) + + bindings = _model_bindings() + + assert bindings["openrouter_reasoning_effort"] == reported From cea2209009fdab6946121ca3d0ef0c4d5610bf24 Mon Sep 17 00:00:00 2001 From: Jiri Puc Date: Fri, 24 Jul 2026 16:41:24 +0200 Subject: [PATCH 6/6] fix: constrain selection drop reasons --- src/rememberstack/model/__init__.py | 2 + src/rememberstack/model/claims.py | 29 +++++++++- src/rememberstack/workers/e1.py | 2 +- src/rememberstack/workers/e2.py | 13 ++++- src/tests/test_port_model_values.py | 56 +++++++++++++++++++ src/tests/workers/test_e2_chain.py | 18 ++++++ .../workers/test_lifecycle_reconciliation.py | 7 ++- src/tests/workers/test_reuse_lifecycle.py | 3 +- 8 files changed, 120 insertions(+), 10 deletions(-) diff --git a/src/rememberstack/model/__init__.py b/src/rememberstack/model/__init__.py index ceaf7a43..9875fe96 100644 --- a/src/rememberstack/model/__init__.py +++ b/src/rememberstack/model/__init__.py @@ -36,6 +36,7 @@ from rememberstack.model.claims import ObservationForEmbedding from rememberstack.model.claims import OtherPredicateGrammarError from rememberstack.model.claims import SelectionCandidate +from rememberstack.model.claims import SelectionDropReason from rememberstack.model.claims import SelectionResponse from rememberstack.model.claims import SelectionVerdict from rememberstack.model.client import ConnectorCreate @@ -468,6 +469,7 @@ "S58Answer", "SectionSpan", "SelectionCandidate", + "SelectionDropReason", "SelectionResponse", "SelectionVerdict", "LifecycleReport", diff --git a/src/rememberstack/model/claims.py b/src/rememberstack/model/claims.py index 54ade0c9..748eca10 100644 --- a/src/rememberstack/model/claims.py +++ b/src/rememberstack/model/claims.py @@ -7,6 +7,7 @@ from pydantic import BaseModel from pydantic import ConfigDict from pydantic import Field +from pydantic import model_validator _NonEmpty = Annotated[str, Field(min_length=1)] @@ -19,6 +20,21 @@ class SelectionVerdict(StrEnum): DROP = "drop" +class SelectionDropReason(StrEnum): + """The exact D31 vocabulary persisted by PostgreSQL.""" + + OPINION = "opinion" + ADVICE = "advice" + HYPOTHETICAL = "hypothetical" + GENERIC = "generic" + QUESTION = "question" + INTRO = "intro" + CONCLUSION = "conclusion" + NO_INFO = "no_info" + AMBIGUOUS = "ambiguous" + REFERENCES_BOILERPLATE = "references_boilerplate" + + class SelectionCandidate(BaseModel): """One proposition Selection judged inside the target chunk.""" @@ -26,9 +42,18 @@ class SelectionCandidate(BaseModel): source_span: _NonEmpty verdict: SelectionVerdict - drop_reason: str | None = None + drop_reason: SelectionDropReason | None = None protected_class: str | None = None + @model_validator(mode="after") + def validate_drop_reason(self) -> "SelectionCandidate": + """Require one controlled reason exactly when Selection drops a span.""" + if self.verdict is SelectionVerdict.DROP and self.drop_reason is None: + raise ValueError("drop candidates require drop_reason") + if self.verdict is not SelectionVerdict.DROP and self.drop_reason is not None: + raise ValueError("kept candidates cannot carry drop_reason") + return self + class SelectionResponse(BaseModel): """The Selection call's structured output: every judged candidate.""" @@ -109,7 +134,7 @@ class DecisionRecord(BaseModel): claim_id: UUID | None decision_type: DecisionType source_span: str | None - reason: str | None + reason: SelectionDropReason | None edit_detail: dict[str, object] | None protected_class: str | None extractor_version: _NonEmpty diff --git a/src/rememberstack/workers/e1.py b/src/rememberstack/workers/e1.py index 02cfcfd3..67983c9d 100644 --- a/src/rememberstack/workers/e1.py +++ b/src/rememberstack/workers/e1.py @@ -53,7 +53,7 @@ E1_PREFIXER_VERSION: Final = "e1-prefix-2026.07" """The context-prefix call's prompt generation (D58; conventional mode, D63).""" -E2_EXTRACTOR_VERSION: Final = "e2-extract-2026.07" +E2_EXTRACTOR_VERSION: Final = "e2-extract-2026.07b:drop-reason-enum-1" """The extractor generation baked into extraction_input_hash (D56); the E2 stage (WP-1.3) binds its handler to this same constant.""" diff --git a/src/rememberstack/workers/e2.py b/src/rememberstack/workers/e2.py index 1666020a..ee057276 100644 --- a/src/rememberstack/workers/e2.py +++ b/src/rememberstack/workers/e2.py @@ -31,6 +31,7 @@ from rememberstack.model import ObjectKey from rememberstack.model import PipelineStage from rememberstack.model import SelectionCandidate +from rememberstack.model import SelectionDropReason from rememberstack.model import SelectionResponse from rememberstack.model import SelectionVerdict from rememberstack.ports.cost_meter import CostMeterPort @@ -44,6 +45,8 @@ _logger = logging.getLogger(__name__) +_DROP_REASONS: Final = "|".join(reason.value for reason in SelectionDropReason) + _SELECTION_PROMPT: Final = """You are the Selection stage of a claim extractor. Judge every proposition in the TARGET CHUNK: keep statements making a specific, verifiable proposition (state, event, decision, quantity, policy, relationship). @@ -52,7 +55,9 @@ stance ("X said/believes/opposes Y") is a KEEP. Never-drop classes even if phrased opinionatedly: quantities, dates, named-entity+predicate, change-of-state. When unsure, prefer keep_flagged over drop. Each candidate's -source_span must be a verbatim substring of the target chunk. +source_span must be a verbatim substring of the target chunk. For a DROP, +drop_reason must be exactly one of: {drop_reasons}. For KEEP or KEEP_FLAGGED, +it must be null. {bundle}""" @@ -184,7 +189,9 @@ def _extract_chunk( selection_call = self._model_provider.generate( request=ModelRequest( model=self._settings.extract_model, - prompt=_SELECTION_PROMPT.format(bundle=bundle), + prompt=_SELECTION_PROMPT.format( + drop_reasons=_DROP_REASONS, bundle=bundle + ), ), response_type=SelectionResponse, ) @@ -443,7 +450,7 @@ def _empty_extraction_marker( claim_id=None, decision_type=DecisionType.SELECTION_DROP, source_span=None, - reason="no_info", + reason=SelectionDropReason.NO_INFO, edit_detail=None, protected_class=None, extractor_version=E2_EXTRACTOR_VERSION, diff --git a/src/tests/test_port_model_values.py b/src/tests/test_port_model_values.py index ad53c6df..11923edb 100644 --- a/src/tests/test_port_model_values.py +++ b/src/tests/test_port_model_values.py @@ -13,6 +13,8 @@ from rememberstack.model import PerimeterCredential from rememberstack.model import ProviderCallUsage from rememberstack.model import PublishedMounts +from rememberstack.model import SelectionDropReason +from rememberstack.model import SelectionResponse class _Output(BaseModel): @@ -86,3 +88,57 @@ def test_perimeter_credential_redacts_secret_bytes() -> None: ) assert "must-not-appear" not in repr(credential) + + +def test_selection_drop_reason_matches_the_database_vocabulary() -> None: + """Reject provider prose before a selection decision reaches PostgreSQL.""" + valid = SelectionResponse.model_validate( + { + "candidates": [ + { + "source_span": "How are you?", + "verdict": "drop", + "drop_reason": "question", + "protected_class": None, + } + ] + } + ) + + assert valid.candidates[0].drop_reason is SelectionDropReason.QUESTION + with pytest.raises(ValidationError): + SelectionResponse.model_validate( + { + "candidates": [ + { + "source_span": "How are you?", + "verdict": "drop", + "drop_reason": "question (the speaker asks a question)", + "protected_class": None, + } + ] + } + ) + + +@pytest.mark.parametrize( + ("verdict", "drop_reason"), + (("drop", None), ("keep", "question"), ("keep_flagged", "advice")), +) +def test_selection_drop_reason_is_present_only_for_drops( + verdict: str, drop_reason: str | None +) -> None: + """Keep the decision transcript complete without attaching false drop reasons.""" + with pytest.raises(ValidationError): + SelectionResponse.model_validate( + { + "candidates": [ + { + "source_span": "A statement.", + "verdict": verdict, + "drop_reason": drop_reason, + "protected_class": None, + } + ] + } + ) diff --git a/src/tests/workers/test_e2_chain.py b/src/tests/workers/test_e2_chain.py index 1ad9f655..3990aafb 100644 --- a/src/tests/workers/test_e2_chain.py +++ b/src/tests/workers/test_e2_chain.py @@ -26,6 +26,7 @@ from rememberstack.model import PipelineStage from rememberstack.model import ProcessingLane from rememberstack.model import RunResultOutcome +from rememberstack.model import SelectionDropReason from rememberstack.spine import ChunkCatalog from rememberstack.spine import ClaimCatalog from rememberstack.spine import DeploymentBootstrapper @@ -130,6 +131,23 @@ def database_engine() -> Iterator[Engine]: engine.dispose() +def test_selection_drop_reasons_match_postgres(database_engine: Engine) -> None: + """Keep the provider schema vocabulary identical to the persisted enum.""" + with database_engine.connect() as connection: + labels = tuple( + connection.execute( + text( + "SELECT enumlabel FROM pg_enum" + " JOIN pg_type ON pg_type.oid = pg_enum.enumtypid" + " WHERE pg_type.typname = 'selection_drop_reason'" + " ORDER BY enumsortorder" + ) + ).scalars() + ) + + assert labels == tuple(reason.value for reason in SelectionDropReason) + + @pytest.fixture(autouse=True) def bootstrapped_deployment(database_engine: Engine) -> None: """Give every proof a fresh deployment and empty partitioned tables.""" diff --git a/src/tests/workers/test_lifecycle_reconciliation.py b/src/tests/workers/test_lifecycle_reconciliation.py index d74a1a9e..c47c695d 100644 --- a/src/tests/workers/test_lifecycle_reconciliation.py +++ b/src/tests/workers/test_lifecycle_reconciliation.py @@ -72,6 +72,7 @@ from rememberstack.workers import CycleFinalizer from rememberstack.workers import DeletionService from rememberstack.workers import E1Settings +from rememberstack.workers import E2_EXTRACTOR_VERSION from rememberstack.workers import E2Settings from rememberstack.workers import E3Settings from rememberstack.workers import EmbedChunksHandler @@ -874,8 +875,8 @@ def test_lifecycle_suite_passes_on_healthy_state_and_records_the_run( assert report.quiescent # the chain is drained: full checks ran assert report.violations == {} assert report.canary_failures == () - assert "e2-extract-2026.07" in report.flag_rate_by_extractor - assert report.flag_rate_by_extractor["e2-extract-2026.07"]["flag_rate"] == 0.0 + assert E2_EXTRACTOR_VERSION in report.flag_rate_by_extractor + assert report.flag_rate_by_extractor[E2_EXTRACTOR_VERSION]["flag_rate"] == 0.0 recorded = rig.scalar("SELECT passed FROM eval_runs WHERE suite = 'lifecycle'") assert recorded is True @@ -1004,4 +1005,4 @@ def test_restore_support_plants_a_canary_the_pack_rechecks(rig: _LifecycleRig) - assert not regressed.passed # the canary blocks the regressing version flag_rates = flag_rate_by_extractor(engine=rig.engine, deployment_id=_DEPLOYMENT_ID) - assert flag_rates["e2-extract-2026.07"]["flags_raised"] == 1.0 + assert flag_rates[E2_EXTRACTOR_VERSION]["flags_raised"] == 1.0 diff --git a/src/tests/workers/test_reuse_lifecycle.py b/src/tests/workers/test_reuse_lifecycle.py index 3382d8f6..fde7f729 100644 --- a/src/tests/workers/test_reuse_lifecycle.py +++ b/src/tests/workers/test_reuse_lifecycle.py @@ -46,6 +46,7 @@ from rememberstack.workers import ChunkHandler from rememberstack.workers import ConvertHandler from rememberstack.workers import E1Settings +from rememberstack.workers import E2_EXTRACTOR_VERSION from rememberstack.workers import E2Settings from rememberstack.workers import EmbedChunksHandler from rememberstack.workers import ExtractClaimsHandler @@ -424,7 +425,7 @@ def test_extractor_bump_re_extracts_reused_chunks(rig: _ReuseRig) -> None: ) ).scalar_one() # a chunk holding only re-attached links assert rig.claim_catalog.chunk_already_extracted( - chunk_id=reused_chunk, extractor_version="e2-extract-2026.07" + chunk_id=reused_chunk, extractor_version=E2_EXTRACTOR_VERSION ) assert not rig.claim_catalog.chunk_already_extracted( chunk_id=reused_chunk, extractor_version="e2-extract-9999.01"