Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,12 @@ REMEMBERSTACK_OPENROUTER_API_KEY=replace-before-real-use
# "none" reduces extraction latency but can reduce adjudication quality, and
# models that require reasoning may reject it.
# REMEMBERSTACK_OPENROUTER_REASONING_EFFORT=none
# Optional per-model overrides as a JSON object of model-id → effort. A model's
# entry wins over the global pin above; models absent from the map fall back to
# the global pin (or the model default when that is also unset). Same allowed
# effort literals. Example: pin flash models to none while keeping high for a
# reasoning model.
# REMEMBERSTACK_OPENROUTER_REASONING_EFFORT_MAP={"z-ai/glm-4.7-flash":"none","openai/gpt-5.6-luna":"high"}

# Optional benchmark/deployment model overrides. Keep explicit model IDs for a
# reproducible run; do not use a rotating router such as openrouter/free.
Expand Down
2 changes: 1 addition & 1 deletion benchmarks/locomo/README.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# RS-LoCoMo-Full-v4 setup
# RS-LoCoMo-Full-v5 setup

This directory contains the unshipped full-system LoCoMo adapter. It does not vendor or
auto-download LoCoMo. Supply the exact pinned `locomo10.json` only after confirming its
Expand Down
2 changes: 1 addition & 1 deletion benchmarks/locomo/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ def _parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
prog="python -m benchmarks.locomo",
description=(
"RS-LoCoMo-Full-v4: prepare is local; ingest/answer/judge require "
"RS-LoCoMo-Full-v5: prepare is local; ingest/answer/judge require "
"explicit execution acknowledgements"
),
)
Expand Down
6 changes: 3 additions & 3 deletions benchmarks/locomo/model.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
"""Typed values for the full-system RS-LoCoMo-Full-v4 protocol."""
"""Typed values for the full-system RS-LoCoMo-Full-v5 protocol."""

from __future__ import annotations

Expand Down Expand Up @@ -108,7 +108,7 @@ class QuestionManifest(FrozenModel):
class RunConfiguration(FrozenModel):
"""Immutable identity of one prepared benchmark run."""

protocol_name: Literal["RS-LoCoMo-Full-v4"] = "RS-LoCoMo-Full-v4"
protocol_name: Literal["RS-LoCoMo-Full-v5"] = "RS-LoCoMo-Full-v5"
adapter_version: NonEmpty
prepared_at: datetime
repository_revision: NonEmpty
Expand Down Expand Up @@ -350,7 +350,7 @@ class SessionDiagnosticSummary(FrozenModel):
class RunSummary(FrozenModel):
"""Publication-ready local aggregate with no hidden denominator."""

protocol_name: Literal["RS-LoCoMo-Full-v4"] = "RS-LoCoMo-Full-v4"
protocol_name: Literal["RS-LoCoMo-Full-v5"] = "RS-LoCoMo-Full-v5"
protocol_fingerprint: NonEmpty
tier: Tier
questions: int = Field(ge=1)
Expand Down
4 changes: 2 additions & 2 deletions benchmarks/locomo/protocol.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,12 @@
from benchmarks.locomo.model import ToolCallRecord
from rememberstack.model import ToolDescriptor

PROTOCOL_NAME: Final = "RS-LoCoMo-Full-v4"
PROTOCOL_NAME: Final = "RS-LoCoMo-Full-v5"
ADAPTER_VERSION: Final = "locomo-full-adapter-2026.07"
MAX_TOOL_CALLS: Final = 8
MAX_AGENT_CALLS: Final = 9
EXPECTED_TOOL_CATALOG_SHA256: Final = (
"f0492cedb5a648816511d99ff35c96862bd9299be186e08bd27a68de76012953"
"34d2069ae37fabf033d2b1f0fae2ed9e7c1ad5c3f6e1fd14b2971344cd89c3a6"
)
EXPECTED_PIPELINE_STAGES: Final = (
"convert",
Expand Down
8 changes: 4 additions & 4 deletions benchmarks/locomo/runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -354,7 +354,7 @@ def answer_sample(
):
raise ExecutionGuardError(
"the deployment did not report the exact completed"
" RS-LoCoMo-Full-v4 pipeline and fresh P2/P3 projections"
" RS-LoCoMo-Full-v5 pipeline and fresh P2/P3 projections"
)
_require_serving_revision(context=context, readiness=readiness)
prior_readiness = context.state.readiness.get(sample_id)
Expand Down Expand Up @@ -674,7 +674,7 @@ def _validate_run(
) -> None:
"""Recompute immutable run identity before any local or remote stage."""
if configuration.dataset_sha256 != DATASET_SHA256:
raise BenchmarkRunError("run dataset hash is not RS-LoCoMo-Full-v4")
raise BenchmarkRunError("run dataset hash is not RS-LoCoMo-Full-v5")
if item_ids_hash(item_ids=manifest.item_ids) != manifest.item_ids_sha256:
raise BenchmarkRunError("run manifest item hash changed")
if manifest_bytes_hash(manifest=manifest) != configuration.manifest_sha256:
Expand All @@ -684,14 +684,14 @@ def _validate_run(
if manifest.tier != configuration.tier:
raise BenchmarkRunError("run manifest tier changed")
if configuration.dataset_commit != DATASET_COMMIT:
raise BenchmarkRunError("run dataset commit is not RS-LoCoMo-Full-v4")
raise BenchmarkRunError("run dataset commit is not RS-LoCoMo-Full-v5")
if configuration.adapter_version != ADAPTER_VERSION:
raise BenchmarkRunError("run adapter version differs from current code")
if (
configuration.answer_agent_temperature != TEMPERATURE
or configuration.judge_temperature != TEMPERATURE
):
raise BenchmarkRunError("run temperature differs from RS-LoCoMo-Full-v4")
raise BenchmarkRunError("run temperature differs from RS-LoCoMo-Full-v5")
if _models_hash(values=documents) != configuration.documents_sha256:
raise BenchmarkRunError("prepared document manifest changed")
if len(questions) != configuration.item_count:
Expand Down
1 change: 1 addition & 0 deletions compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ x-app: &app
REMEMBERSTACK_OPENROUTER_API_KEY: ${REMEMBERSTACK_OPENROUTER_API_KEY}
REMEMBERSTACK_OPENROUTER_EMBEDDING_PROVIDER: ${REMEMBERSTACK_OPENROUTER_EMBEDDING_PROVIDER:-}
REMEMBERSTACK_OPENROUTER_REASONING_EFFORT: ${REMEMBERSTACK_OPENROUTER_REASONING_EFFORT:-}
REMEMBERSTACK_OPENROUTER_REASONING_EFFORT_MAP: ${REMEMBERSTACK_OPENROUTER_REASONING_EFFORT_MAP:-}
REMEMBERSTACK_SELFHOST_DEPLOYMENT_ID: ${REMEMBERSTACK_SELFHOST_DEPLOYMENT_ID}
REMEMBERSTACK_SELFHOST_DEPLOYMENT_SLUG: ${REMEMBERSTACK_SELFHOST_DEPLOYMENT_SLUG}
REMEMBERSTACK_SELFHOST_DEPLOYMENT_NAME: ${REMEMBERSTACK_SELFHOST_DEPLOYMENT_NAME}
Expand Down
10 changes: 9 additions & 1 deletion plan/designs/locomo_benchmark_design.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ WP-8.2 remains in progress until an owner-authorized eight-question smoke finish
## 2. Fixed protocol

```text
protocol RS-LoCoMo-Full-v4
protocol RS-LoCoMo-Full-v5
dataset commit 3eb6f2c585f5e1699204e3c3bdf7adc5c28cb376
dataset SHA-256 79fa87e90f04081343b8c8debecb80a9a6842b76a7aa537dc9fdf651ea698ff4
categories 1, 2, 3, 4
Expand Down Expand Up @@ -53,6 +53,14 @@ tool+arguments retries; switch tools after a useless result; try a claims
search before "Unknown"). Prompt, tool-catalog hash, and descriptors all
changed, so the protocol version bumps. No v3 score is comparable.

**v4 → v5 (2026-07-27, issue #156):** the `identity_as_of` recipe descriptor
became honest about its recent-first transcript bound (default 40 rows,
truncation signalled in the envelope) and gained an optional `limit`
parameter so a truncated history is recoverable through the public surface.
The tool-catalog hash changed, so the protocol version bumps — the v4 smoke
scores (glm-4.7-flash arm) were taken against the pre-truncation catalog and
are not directly comparable.

### 2.1 Why v2+ uses a stronger judge

`RS-LoCoMo-Full-v1` used `openai/gpt-4o-mini` for both the answer agent and the judge. The
Expand Down
14 changes: 13 additions & 1 deletion plan/designs/retrieval_design.md
Original file line number Diff line number Diff line change
Expand Up @@ -117,12 +117,24 @@ never trigger anything** — all K/E triggering originates from writes).
| `fuse` | result_sets → RRF-merged set | reciprocal-rank fusion of parallel channels (D9), exposed as an operator so *agent-composed* channel sets fuse the same way recipes do | S46 |
| `rerank` | candidates × signal — graph_distance(focal), evidence_count, cross_encoder (flagged) | the D9 rerankers as explicit, inspectable stages | S46, S48 |
| `hydrate` | ids, depth: record \| evidence \| sources \| bytes, locator? | the §2 confirmation hop + progressive deepening: record → evidence rows + claims → documents → GCS handles. At `depth=bytes` an optional **source locator** (D65) scopes the fetch to a time interval / region, returning a seekable, codec-aware segment (§7 — unmounted parity for media) | S5, S59, all |
| `transcript` | relation \| observation \| entity \| k_page → its decision history | adjudications, resolution decisions, compile provenance — the audit trail as a first-class query ("why do we believe…") | S8, S32, S35 |
| `transcript` | relation \| observation \| entity \| k_page → its decision history (recent-first bound; see amendment below) | adjudications, resolution decisions, compile provenance — the audit trail as a first-class query ("why do we believe…") | S8, S32, S35 |
| `delta` | since T, scope?, kinds? → changed evidence / pages | the change feed as a query (new / capped / invalidated / recompiled) | S13, S14, S30 |
| `pages_about` | entity \| key → K pages (+ freshness/flags) | **the K routing index read backwards**: the rule-key inverted index built for write-side routing doubles as the reader's discovery index — which pages exist about X, mechanically | S31, S45 |
| `aggregate` | enumerated forms: count / group-by-predicate / group-by-object / timeline(entity) / delta-top-entities(since T) / typed-absence(type, predicate) | see §9 — enumerated, not general | S12, S26–S28, S30, S40 |
| `scan` | filter → stream | the batch surface (§9): full exports for compilers, auditors, external analytics — same zero-LLM reads, streaming contract, separate resource pool from interactive | S53 |

**Amendment (2026-07-27, issue #156).** `transcript` (and the `identity_as_of`
recipe that is one fixed chain over it) is **recent-first bounded** by a
measurable default (`DEFAULT_TRANSCRIPT_LIMIT`, starting point 40 rows), with
the envelope's existing truncation marker set when the cap applies. Long
entity resolution logs were returning hundreds of rows and blowing agent
reader contexts; S18 already forbids silent caps, so the bound is disclosed
rather than unbounded-by-default. Callers that need a larger window pass an
explicit `limit` — on the Python surface and as the recipe's optional `limit`
parameter (recipe v3), so a truncated history is recoverable through the
public surface too. This is the same cap-with-signal pattern as `delta`, not
a new mechanism.

**Temporal parameters — composed, not special-cased (S15/S16).** Every primitive that touches
validity accepts `valid_at` (world time: "what held at T") and `believed_at` (system time:
"what did we believe at T"; caps `ingested_at`/`invalidated_at` across *all* channels at once —
Expand Down
75 changes: 70 additions & 5 deletions src/rememberstack/adapters/openrouter.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,9 @@
import json
import time
from typing import Any
from typing import Final
from typing import Literal
from typing import TypeAlias
from typing import TypeVar

import httpx
Expand All @@ -27,6 +29,15 @@

ResponseT = TypeVar("ResponseT", bound=StructuredResponseModel)

ReasoningEffort: TypeAlias = Literal[
"none", "minimal", "low", "medium", "high", "xhigh", "max"
]
"""Allowed OpenRouter reasoning-effort values (global pin or per-model map)."""

_ALLOWED_REASONING_EFFORTS: Final[frozenset[str]] = frozenset(
("none", "minimal", "low", "medium", "high", "xhigh", "max")
)


class StrictSchemaError(ValueError):
"""A response model cannot be expressed under strict structured output.
Expand All @@ -46,9 +57,15 @@ class OpenRouterSettings(BaseSettings):
base_url: str = Field(default="https://openrouter.ai/api/v1")
timeout_s: float = Field(default=120.0, gt=0)
embedding_provider: str | None = None
reasoning_effort: (
Literal["none", "minimal", "low", "medium", "high", "xhigh", "max"] | None
) = None
reasoning_effort: ReasoningEffort | None = None
reasoning_effort_map: dict[str, ReasoningEffort] | None = None
"""Optional per-model effort overrides as a JSON object env var
(`REMEMBERSTACK_OPENROUTER_REASONING_EFFORT_MAP`, e.g.
`{"z-ai/glm-4.7-flash":"none","openai/gpt-5.6-luna":"high"}`). A model's
entry wins over the global `reasoning_effort` for requests to that model;
absent entries fall back to the global pin (or the model default when the
global pin is also unset). Values must be one of the allowed effort
literals."""

@field_validator("embedding_provider", "reasoning_effort", mode="before")
@classmethod
Expand All @@ -58,6 +75,46 @@ def normalize_optional_string(cls, value: object) -> object:
return value
return value.strip() or None

@field_validator("reasoning_effort_map", mode="before")
@classmethod
def parse_reasoning_effort_map(cls, value: object) -> object:
"""Parse the JSON object env form and reject unknown effort literals.

Compose often supplies empty strings for unset optionals; treat those
as unset. A JSON string is accepted because pydantic-settings may hand
the raw env value through before typed decoding on some paths.
"""
if value is None:
return None
if isinstance(value, str):
stripped = value.strip()
if not stripped:
return None
try:
value = json.loads(stripped)
except json.JSONDecodeError as error:
raise ValueError(
"reasoning_effort_map must be a JSON object of model-id → effort"
) from error
if not isinstance(value, dict):
raise ValueError(
"reasoning_effort_map must be a JSON object of model-id → effort"
)
parsed: dict[str, str] = {}
for model_id, effort in value.items():
if not isinstance(model_id, str) or not model_id.strip():
raise ValueError(
"reasoning_effort_map keys must be non-empty model id strings"
)
if not isinstance(effort, str) or effort not in _ALLOWED_REASONING_EFFORTS:
raise ValueError(
f"reasoning_effort_map[{model_id!r}]={effort!r} is not an"
f" allowed effort"
f" ({', '.join(sorted(_ALLOWED_REASONING_EFFORTS))})"
)
parsed[model_id.strip()] = effort
return parsed or None


class OpenRouterProviderError(ProviderCallError):
"""OpenRouter returned an error or an unusable response body."""
Expand Down Expand Up @@ -94,8 +151,9 @@ def generate(
}
if request.temperature is not None:
payload["temperature"] = request.temperature
if self._settings.reasoning_effort is not None:
payload["reasoning"] = {"effort": self._settings.reasoning_effort}
effort = self._reasoning_effort_for(model=request.model)
if effort is not None:
payload["reasoning"] = {"effort": effort}

content, usage = self._completion_text(
payload=payload, response_type=response_type, started_ns=started_ns
Expand All @@ -118,6 +176,13 @@ def generate(
) from error
return GeneratedResponse(output=output, usage=usage)

def _reasoning_effort_for(self, *, model: str) -> ReasoningEffort | None:
"""Resolve effort for one model: per-model map entry, else global pin."""
mapped = self._settings.reasoning_effort_map
if mapped is not None and model in mapped:
return mapped[model]
return self._settings.reasoning_effort

def _completion_text(
self,
*,
Expand Down
2 changes: 2 additions & 0 deletions src/rememberstack/adapters/testing/model_provider.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,13 @@ def __init__(
self._generate_router = generate_router
self.embedded_texts: list[str] = []
self.generated_prompts: list[str] = []
self.generated_requests: list[ModelRequest] = []

def generate[ResponseT: StructuredResponseModel](
self, *, request: ModelRequest, response_type: type[ResponseT]
) -> GeneratedResponse[ResponseT]:
"""Return the canned payload validated as the caller's declared type."""
self.generated_requests.append(request)
self.generated_prompts.append(request.prompt)
if callable(self._generate_router):
output = response_type.model_validate(
Expand Down
8 changes: 8 additions & 0 deletions src/rememberstack/profiles/selfhost.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
from __future__ import annotations

import argparse
import json
from pathlib import Path
import sys
from typing import Self
Expand Down Expand Up @@ -615,6 +616,13 @@ def _model_bindings() -> dict[str, str]:
"fact_label": p1.label_model,
"openrouter_embedding_provider": openrouter.embedding_provider or "auto",
"openrouter_reasoning_effort": openrouter.reasoning_effort or "auto",
# Canonical (sorted-key) form so the effective per-model effort policy
# is part of measurement provenance, not hidden behind the global pin.
"openrouter_reasoning_effort_map": json.dumps(
openrouter.reasoning_effort_map, sort_keys=True
)
if openrouter.reasoning_effort_map
else "unset",
}


Expand Down
13 changes: 9 additions & 4 deletions src/rememberstack/spine/observation_adjudication.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,9 @@
from rememberstack.ports.cost_meter import CostMeterPort
from rememberstack.ports.model_provider import ModelProviderPort

OBSERVATION_ADJUDICATOR_VERSION: Final = "obs-adjudicator-2026.07"
"""The observation adjudicator generation (D12; replayed on rebuild, D7)."""
OBSERVATION_ADJUDICATOR_VERSION: Final = "obs-adjudicator-2026.07b:temp0-1"
"""The observation adjudicator generation (D12; replayed on rebuild, D7).
07b pins temperature=0.0 — generation parameters are part of provenance."""

_VERDICT_PROMPT: Final = """You adjudicate observations for a memory system.
Both statements are believed facts about the SAME entity:
Expand Down Expand Up @@ -503,7 +504,9 @@ def _ladder(
"""Small-model verdict, escalating to frontier below the floor."""
prompt = _VERDICT_PROMPT.format(existing=existing, new=new)
verdict_call = self._model_provider.generate(
request=ModelRequest(model=self._settings.small_model, prompt=prompt),
request=ModelRequest(
model=self._settings.small_model, prompt=prompt, temperature=0.0
),
response_type=ObservationVerdict,
)
if meter is not None:
Expand All @@ -516,7 +519,9 @@ def _ladder(
if verdict.confidence >= self._settings.confidence_floor:
return verdict, "small_model"
frontier_call = self._model_provider.generate(
request=ModelRequest(model=self._settings.frontier_model, prompt=prompt),
request=ModelRequest(
model=self._settings.frontier_model, prompt=prompt, temperature=0.0
),
response_type=ObservationVerdict,
)
if meter is not None:
Expand Down
18 changes: 12 additions & 6 deletions src/rememberstack/spine/recipes.py
Original file line number Diff line number Diff line change
Expand Up @@ -271,20 +271,26 @@ def _recipe_from_row(row: RowMapping) -> Recipe:
),
Recipe(
name="identity_as_of",
description="An entity's identity history — how its mentions resolved"
" and every merge it took part in (S61). Composite grain, audit."
" As-of regime resolution, not a biography or fact timeline.",
parameters={"entity_id": {"type": "uuid", "required": True}},
description="An entity's identity history — recent resolution"
" decisions and merges (S61). Composite grain, audit. As-of regime"
" resolution, not a biography or fact timeline. Keeps the newest"
" `limit` rows (default 40), returned oldest-to-newest; the envelope"
" signals truncation when older history was cut. Pass a larger"
" `limit` to widen the window.",
parameters={
"entity_id": {"type": "uuid", "required": True},
"limit": {"type": "integer", "required": False, "minimum": 1},
},
chain=(
RecipeStep(
op="transcript",
settings={"subject_kind": "entity"},
bind={"subject_id": "entity_id"},
bind={"subject_id": "entity_id", "limit": "limit"},
),
),
output_grain=Grain.COMPOSITE,
answer_intent=RecipeAnswerIntent.AUDIT,
version=2,
version=3,
),
Recipe(
name="changed_since",
Expand Down
Loading
Loading