From 646931c93e3551219e0d4b2f8e438e44e07a02fd Mon Sep 17 00:00:00 2001 From: Gorkem Date: Sun, 12 Jul 2026 19:21:25 +0300 Subject: [PATCH 1/3] fix(reflection): route mapped reflection rows through admission control The distiller's four mapped headings (User model deltas, Agent model deltas, Lessons & pitfalls, Decisions) become ordinary durable memory rows via bulkStore with no admission gate, no gating knob, and no audit trail; reflectionStoreToLanceDB gates only the document writer. A contaminated or hallucinated distillate line therefore landed directly as a confirmed memory. Gate each mapped row through the same AdmissionController instance the smart extractor uses (new SmartExtractor accessors), scoring legacy mapped categories under the smart registers admission priors are keyed by: user/agent model deltas as preferences, lessons as cases, decisions as events. Rejected rows are skipped with the audit reason logged; admitted rows persist the audit (tagged provenance memory-reflection-mapped) in their metadata when auditMetadata is on. Behavior is unchanged when admission control is disabled, and the gate fails open on infra errors so a transient store or LLM failure cannot suppress reflection persistence. Also add a grounding rule to the distiller prompt (in-fiction or test-frame claims must never appear under the four mapped headings), export buildReflectionPrompt for prompt-content tests, and mark the caller-less mapped-rows reader in reflection-store with a comment. Wire the new regression suite into the local npm test chain and the core-regression CI group. --- index.ts | 29 ++- package.json | 2 +- scripts/ci-test-manifest.mjs | 1 + src/reflection-mapped-admission.ts | 113 +++++++++++ src/reflection-mapped-metadata.ts | 2 + src/reflection-store.ts | 4 + src/smart-extractor.ts | 14 ++ .../reflection-mapped-rows-admission.test.mjs | 180 ++++++++++++++++++ 8 files changed, 343 insertions(+), 2 deletions(-) create mode 100644 src/reflection-mapped-admission.ts create mode 100644 test/reflection-mapped-rows-admission.test.mjs diff --git a/index.ts b/index.ts index 74a1e46c..2bd8faf4 100644 --- a/index.ts +++ b/index.ts @@ -67,6 +67,7 @@ import { } from "./src/reflection-slices.js"; import { createReflectionEventId } from "./src/reflection-event-store.js"; import { buildReflectionMappedMetadata } from "./src/reflection-mapped-metadata.js"; +import { gateMappedReflectionEntry } from "./src/reflection-mapped-admission.js"; import { createMemoryCLI } from "./cli.js"; import { isNoise } from "./src/noise-filter.js"; import { normalizeAutoCaptureText } from "./src/auto-capture-cleanup.js"; @@ -1455,7 +1456,7 @@ async function ensureDailyLogFile(dailyPath: string, dateStr: string): Promise api.logger.warn(msg), + }); + if (!mappedGate.admit) { + api.logger.info( + `memory-reflection: admission rejected mapped row heading=${JSON.stringify(mapped.heading)} provenance=memory-reflection-mapped: ${mappedGate.reason ?? "no reason"}`, + ); + continue; + } + const importance = mapped.category === "decision" ? 0.85 : 0.8; const baseMetadata = buildReflectionMappedMetadata({ mappedItem: mapped, @@ -4939,6 +4963,9 @@ const memoryLanceDBProPlugin = { }); // embed heading in metadata JSON so it survives bulkStore round-trip to LanceDB baseMetadata._reflectionHeading = mapped.heading; + if (mappedGate.auditJson) { + baseMetadata.admission_audit = mappedGate.auditJson; + } const metadata = JSON.stringify(baseMetadata); mappedEntries.push({ diff --git a/package.json b/package.json index b8deb2d4..193cf1a3 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "skills/**/*.md" ], "scripts": { - "test": "node test/embedder-error-hints.test.mjs && node --test test/embedder-max-input-chars.test.mjs && node test/cjk-recursion-regression.test.mjs && node test/extraction-prompt-structural-noise.test.mjs && node test/i18n-memory-triggers.test.mjs && node test/migrate-legacy-schema.test.mjs && node --test test/config-session-strategy-migration.test.mjs && node --test test/scope-access-undefined.test.mjs && node --test test/reflection-bypass-hook.test.mjs && node --test test/smart-extractor-scope-filter.test.mjs && node --test test/store-empty-scope-filter.test.mjs && node --test test/recall-text-cleanup.test.mjs && node test/update-consistency-lancedb.test.mjs && node --test test/strip-envelope-metadata.test.mjs && node test/cli-smoke.mjs && node test/functional-e2e.mjs && node --test test/per-agent-auto-recall.test.mjs && node test/retriever-rerank-regression.mjs && node test/smart-memory-lifecycle.mjs && node test/smart-extractor-branches.mjs && node --test test/smart-extractor-noise-gating.test.mjs && node test/memory-capability-runtime.test.mjs && node --test test/startup-health-diagnostics.test.mjs && node test/corpus-indexer.test.mjs && node --test test/regex-fallback-bulk-store.test.mjs && node test/plugin-manifest-regression.mjs && node --test test/dreaming-engine.test.mjs && node --test test/session-summary-before-reset.test.mjs && node --test test/sync-plugin-version.test.mjs && node test/smart-metadata-v2.mjs && node test/vector-search-cosine.test.mjs && node test/context-support-e2e.mjs && node test/temporal-facts.test.mjs && node test/memory-update-supersede.test.mjs && node test/memory-update-metadata-refresh.test.mjs && node test/memory-upgrader-diagnostics.test.mjs && node --test test/llm-api-key-client.test.mjs && node --test test/llm-oauth-client.test.mjs && node --test test/cli-oauth-login.test.mjs && node --test test/workflow-fork-guards.test.mjs && node --test test/clawteam-scope.test.mjs && node --test test/cross-process-lock.test.mjs && node --test test/preference-slots.test.mjs && node test/is-latest-auto-supersede.test.mjs && node --test test/temporal-awareness.test.mjs && node --test test/command-reflection-guard.test.mjs && node --test test/tier1-counters.test.mjs && node --test test/startup-check-timeout.test.mjs && node --test test/memory-subsession-prompt-hooks.test.mjs && node --test test/read-consistency-interval.test.mjs && node --test test/reflection-distiller-hook-skip.test.mjs && node --test test/register-scope-dedup.test.mjs && node --test test/raw-run-distiller-hooks.test.mjs", + "test": "node test/embedder-error-hints.test.mjs && node --test test/embedder-max-input-chars.test.mjs && node test/cjk-recursion-regression.test.mjs && node test/extraction-prompt-structural-noise.test.mjs && node test/i18n-memory-triggers.test.mjs && node test/migrate-legacy-schema.test.mjs && node --test test/config-session-strategy-migration.test.mjs && node --test test/scope-access-undefined.test.mjs && node --test test/reflection-bypass-hook.test.mjs && node --test test/smart-extractor-scope-filter.test.mjs && node --test test/store-empty-scope-filter.test.mjs && node --test test/recall-text-cleanup.test.mjs && node test/update-consistency-lancedb.test.mjs && node --test test/strip-envelope-metadata.test.mjs && node test/cli-smoke.mjs && node test/functional-e2e.mjs && node --test test/per-agent-auto-recall.test.mjs && node test/retriever-rerank-regression.mjs && node test/smart-memory-lifecycle.mjs && node test/smart-extractor-branches.mjs && node --test test/smart-extractor-noise-gating.test.mjs && node test/memory-capability-runtime.test.mjs && node --test test/startup-health-diagnostics.test.mjs && node test/corpus-indexer.test.mjs && node --test test/regex-fallback-bulk-store.test.mjs && node test/plugin-manifest-regression.mjs && node --test test/dreaming-engine.test.mjs && node --test test/session-summary-before-reset.test.mjs && node --test test/sync-plugin-version.test.mjs && node test/smart-metadata-v2.mjs && node test/vector-search-cosine.test.mjs && node test/context-support-e2e.mjs && node test/temporal-facts.test.mjs && node test/memory-update-supersede.test.mjs && node test/memory-update-metadata-refresh.test.mjs && node test/memory-upgrader-diagnostics.test.mjs && node --test test/llm-api-key-client.test.mjs && node --test test/llm-oauth-client.test.mjs && node --test test/cli-oauth-login.test.mjs && node --test test/workflow-fork-guards.test.mjs && node --test test/clawteam-scope.test.mjs && node --test test/cross-process-lock.test.mjs && node --test test/preference-slots.test.mjs && node test/is-latest-auto-supersede.test.mjs && node --test test/temporal-awareness.test.mjs && node --test test/command-reflection-guard.test.mjs && node --test test/tier1-counters.test.mjs && node --test test/startup-check-timeout.test.mjs && node --test test/memory-subsession-prompt-hooks.test.mjs && node --test test/read-consistency-interval.test.mjs && node --test test/reflection-distiller-hook-skip.test.mjs && node --test test/register-scope-dedup.test.mjs && node --test test/raw-run-distiller-hooks.test.mjs && node --test test/reflection-mapped-rows-admission.test.mjs", "test:cli-smoke": "node scripts/run-ci-tests.mjs --group cli-smoke", "test:core-regression": "node scripts/run-ci-tests.mjs --group core-regression", "test:storage-and-schema": "node scripts/run-ci-tests.mjs --group storage-and-schema", diff --git a/scripts/ci-test-manifest.mjs b/scripts/ci-test-manifest.mjs index 324e30ae..f0a73c50 100644 --- a/scripts/ci-test-manifest.mjs +++ b/scripts/ci-test-manifest.mjs @@ -104,6 +104,7 @@ export const CI_TEST_MANIFEST = [ { group: "core-regression", runner: "node", file: "test/register-scope-dedup.test.mjs", args: ["--test"] }, // Reflection distiller sub-run must request raw-run semantics (skip foreign before_prompt_build hooks) { group: "core-regression", runner: "node", file: "test/raw-run-distiller-hooks.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/reflection-mapped-rows-admission.test.mjs", args: ["--test"] }, ]; export function getEntriesForGroup(group) { diff --git a/src/reflection-mapped-admission.ts b/src/reflection-mapped-admission.ts new file mode 100644 index 00000000..bf334bd0 --- /dev/null +++ b/src/reflection-mapped-admission.ts @@ -0,0 +1,113 @@ +/** + * Admission gating for reflection writer-1 (mapped rows). + * + * The reflection distiller's mapped sections (User model deltas, Agent model + * deltas, Lessons & pitfalls, Decisions) become ordinary durable memory rows + * via bulkStore. Historically they bypassed admission control entirely, so a + * contaminated or hallucinated distillate line landed as a confirmed memory + * with no gate and no audit trail. This module routes each mapped row through + * the same AdmissionController evaluation extraction candidates get, while + * preserving passthrough behavior when admission control is disabled. + */ + +import type { AdmissionEvaluation } from "./admission-control.js"; +import type { CandidateMemory, MemoryCategory } from "./memory-categories.js"; + +/** + * Admission typePriors are keyed by the six smart registers, but mapped rows + * carry legacy store categories. Score them under the smart register that + * matches their shape: user-model/agent-model deltas are preference-shaped + * statements about the human or the assistant ("preference"), lessons are + * symptom/cause/fix/prevention pairs ("fact" here, cases-shaped), and + * decisions are episodic records of something decided ("events"). + */ +export function mapReflectionMappedCategoryToSmartRegister( + category: string, +): MemoryCategory { + switch (category) { + case "preference": + return "preferences"; + case "fact": + return "cases"; + case "decision": + return "events"; + default: + return "events"; + } +} + +export interface MappedReflectionAdmissionGate { + evaluate(params: { + candidate: CandidateMemory; + candidateVector: number[]; + conversationText: string; + scopeFilter: string[]; + }): Promise; +} + +export interface MappedReflectionGateResult { + admit: boolean; + reason?: string; + /** Serialized audit (with provenance) to persist in the row's metadata. */ + auditJson?: string; +} + +/** + * Gate one mapped reflection row through admission control. + * + * - No controller (admission disabled or smart extraction off): passthrough, + * identical to the historical behavior. + * - Controller reject: the row is dropped; the caller logs the reason. + * - Controller pass: the row proceeds; when `attachAudit` is set the audit + * record (tagged with provenance "memory-reflection-mapped") is returned + * for persistence alongside the row. + * - Infra failure inside evaluation: fail open with a reason, so a transient + * store/LLM error cannot silently suppress reflection persistence. + */ +export async function gateMappedReflectionEntry(params: { + admissionController: MappedReflectionAdmissionGate | null; + attachAudit: boolean; + text: string; + category: string; + heading: string; + vector: number[]; + reflectionText: string; + scopeFilter: string[]; + warnLog?: (msg: string) => void; +}): Promise { + if (!params.admissionController) { + return { admit: true }; + } + + const smartCategory = mapReflectionMappedCategoryToSmartRegister(params.category); + let evaluation: AdmissionEvaluation; + try { + evaluation = await params.admissionController.evaluate({ + candidate: { + category: smartCategory, + abstract: params.text, + overview: `## ${params.heading}`, + content: params.text, + }, + candidateVector: params.vector, + conversationText: params.reflectionText, + scopeFilter: params.scopeFilter, + }); + } catch (err) { + params.warnLog?.( + `memory-reflection: mapped-row admission evaluation failed, admitting without audit: ${String(err)}`, + ); + return { admit: true, reason: "admission evaluation failed open" }; + } + + if (evaluation.decision === "reject") { + return { admit: false, reason: evaluation.audit.reason }; + } + + return { + admit: true, + auditJson: params.attachAudit + ? JSON.stringify({ ...evaluation.audit, provenance: "memory-reflection-mapped" }) + : undefined, + }; +} diff --git a/src/reflection-mapped-metadata.ts b/src/reflection-mapped-metadata.ts index e4085055..7b821fcb 100644 --- a/src/reflection-mapped-metadata.ts +++ b/src/reflection-mapped-metadata.ts @@ -27,6 +27,8 @@ export interface ReflectionMappedMetadata { sourceReflectionPath?: string; // Issue #680: heading stored in entry for bulkStore filtering recovery _reflectionHeading?: string; + /** Serialized admission audit when the row passed through admission control. */ + admission_audit?: string; } export interface ReflectionMappedDecayDefaults { diff --git a/src/reflection-store.ts b/src/reflection-store.ts index 482def40..fdaed5bb 100644 --- a/src/reflection-store.ts +++ b/src/reflection-store.ts @@ -611,6 +611,10 @@ export interface ReflectionMappedSlices { decision: string[]; } +// NOTE: this reader currently has no production callers (only tests import +// it). Mapped rows written by the reflection writer are consumed exclusively +// through generic semantic recall today; the dedicated injection path this +// reader was built for was never wired up. Kept for that planned wiring. export function loadReflectionMappedRowsFromEntries(params: LoadReflectionMappedRowsParams): ReflectionMappedSlices { const now = Number.isFinite(params.now) ? Number(params.now) : Date.now(); const maxAgeMs = Number.isFinite(params.maxAgeMs) diff --git a/src/smart-extractor.ts b/src/smart-extractor.ts index e92c41b3..f29f3773 100644 --- a/src/smart-extractor.ts +++ b/src/smart-extractor.ts @@ -340,6 +340,20 @@ export class SmartExtractor { : null; } + /** + * Expose the admission controller so sibling write paths (reflection + * mapped rows) gate through the same instance and config as extraction + * candidates. Null when admission control is disabled. + */ + getAdmissionController(): AdmissionController | null { + return this.admissionController; + } + + /** Whether admitted entries should carry the admission audit in metadata. */ + shouldPersistAdmissionAudit(): boolean { + return this.persistAdmissionAudit; + } + /** * Notify the onPersisted sink (e.g. markdown mirror) after a successful * create or merge. Fire-and-forget from the caller's perspective: awaited diff --git a/test/reflection-mapped-rows-admission.test.mjs b/test/reflection-mapped-rows-admission.test.mjs new file mode 100644 index 00000000..860bf779 --- /dev/null +++ b/test/reflection-mapped-rows-admission.test.mjs @@ -0,0 +1,180 @@ +/** + * Regression tests for reflection writer-1 (mapped rows) admission routing. + * + * The distiller's mapped sections (User model deltas, Agent model deltas, + * Lessons & pitfalls, Decisions) become durable memory rows via bulkStore. + * They historically bypassed admission control entirely. These tests cover + * the new gate: category mapping onto the smart registers admission priors + * are keyed by, admit/deny paths, admission-disabled passthrough, fail-open + * on infra errors, provenance in the persisted audit, and the distiller + * prompt's new grounding rule. + * + * Fixtures are entirely synthetic; no real fleet data. + */ + +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import jitiFactory from "jiti"; + +const testDir = path.dirname(fileURLToPath(import.meta.url)); +const pluginSdkStubPath = path.resolve(testDir, "helpers", "openclaw-plugin-sdk-stub.mjs"); +const jiti = jitiFactory(import.meta.url, { + interopDefault: true, + alias: { + "openclaw/plugin-sdk": pluginSdkStubPath, + }, +}); + +const { + mapReflectionMappedCategoryToSmartRegister, + gateMappedReflectionEntry, +} = jiti("../src/reflection-mapped-admission.ts"); +const { AdmissionController, ADMISSION_CONTROL_PRESETS } = jiti("../src/admission-control.ts"); + +const REFLECTION_TEXT = [ + "## User model deltas (about the human)", + "- Operator prefers streaming test reporters for long suites.", + "## Decisions (durable)", + "- Decision: keep the deploy branch cut from a fresh master.", +].join("\n"); + +describe("mapReflectionMappedCategoryToSmartRegister", () => { + it("maps legacy mapped categories onto the smart registers admission priors use", () => { + assert.equal(mapReflectionMappedCategoryToSmartRegister("preference"), "preferences"); + assert.equal(mapReflectionMappedCategoryToSmartRegister("fact"), "cases"); + assert.equal(mapReflectionMappedCategoryToSmartRegister("decision"), "events"); + assert.equal(mapReflectionMappedCategoryToSmartRegister("unknown-legacy"), "events", "unknown categories take the lowest-prior durable-free register"); + }); +}); + +describe("gateMappedReflectionEntry", () => { + const baseParams = { + text: "Operator prefers streaming test reporters for long suites.", + category: "preference", + heading: "User model deltas (about the human)", + vector: [1, 0, 0], + reflectionText: REFLECTION_TEXT, + scopeFilter: ["global"], + }; + + it("passes rows through untouched when admission control is disabled (null controller)", async () => { + const result = await gateMappedReflectionEntry({ + ...baseParams, + admissionController: null, + attachAudit: true, + }); + assert.deepEqual(result, { admit: true }, "disabled admission must preserve the historical ungated behavior"); + }); + + it("drops rows the controller rejects, surfacing the audit reason", async () => { + const controller = { + async evaluate() { + return { + decision: "reject", + audit: { decision: "reject", reason: "Admission rejected (0.100 < 0.450)." }, + }; + }, + }; + const result = await gateMappedReflectionEntry({ + ...baseParams, + admissionController: controller, + attachAudit: true, + }); + assert.equal(result.admit, false); + assert.match(result.reason, /Admission rejected/); + }); + + it("admits passing rows and tags the persisted audit with mapped-row provenance", async () => { + let seenCandidate = null; + const controller = { + async evaluate(params) { + seenCandidate = params.candidate; + return { + decision: "pass_to_dedup", + audit: { decision: "pass_to_dedup", reason: "Admission passed (0.800)." }, + }; + }, + }; + const result = await gateMappedReflectionEntry({ + ...baseParams, + admissionController: controller, + attachAudit: true, + }); + assert.equal(result.admit, true); + const audit = JSON.parse(result.auditJson); + assert.equal(audit.provenance, "memory-reflection-mapped"); + assert.equal(seenCandidate.category, "preferences", "the controller must score under the mapped smart register"); + assert.equal(seenCandidate.content, baseParams.text); + }); + + it("omits the audit when auditMetadata persistence is off", async () => { + const controller = { + async evaluate() { + return { decision: "pass_to_dedup", audit: { decision: "pass_to_dedup", reason: "ok" } }; + }, + }; + const result = await gateMappedReflectionEntry({ + ...baseParams, + admissionController: controller, + attachAudit: false, + }); + assert.equal(result.admit, true); + assert.equal(result.auditJson, undefined); + }); + + it("fails open when the admission evaluation throws (infra error must not suppress reflection rows)", async () => { + const warnings = []; + const controller = { + async evaluate() { + throw new Error("vector store unavailable"); + }, + }; + const result = await gateMappedReflectionEntry({ + ...baseParams, + admissionController: controller, + attachAudit: true, + warnLog: (msg) => warnings.push(msg), + }); + assert.equal(result.admit, true); + assert.match(result.reason, /failed open/); + assert.equal(warnings.length, 1); + }); + + it("integrates with a real AdmissionController end to end (admit path)", async () => { + const store = { async vectorSearch() { return []; } }; + const llm = { + async completeJson(_prompt, mode) { + return mode === "admission-utility" ? { utility: 0.9, reason: "useful" } : null; + }, + }; + const controller = new AdmissionController(store, llm, ADMISSION_CONTROL_PRESETS.balanced); + + const result = await gateMappedReflectionEntry({ + ...baseParams, + admissionController: controller, + attachAudit: true, + }); + assert.equal(result.admit, true); + const audit = JSON.parse(result.auditJson); + assert.equal(audit.provenance, "memory-reflection-mapped"); + assert.equal(audit.decision, "pass_to_dedup"); + }); +}); + +describe("buildReflectionPrompt grounding discipline", () => { + it("instructs the distiller to keep in-fiction claims out of the mapped (durable) sections", () => { + const { buildReflectionPrompt } = jiti("../index.ts"); + assert.equal(typeof buildReflectionPrompt, "function", "buildReflectionPrompt must be exported for prompt-content tests"); + const prompt = buildReflectionPrompt("conversation text", 4000, []); + + assert.match(prompt, /roleplay/i); + assert.match(prompt, /not real/i); + assert.match( + prompt, + /must NEVER appear under Decisions \(durable\), User model deltas, Agent model deltas, or Lessons & pitfalls/, + "the rule must name the four mapped sections verbatim", + ); + }); +}); From 29ec57c1cff3d15c4b37a0a5cdb29da78ef0c13f Mon Sep 17 00:00:00 2001 From: Gorkem Date: Sun, 12 Jul 2026 19:21:25 +0300 Subject: [PATCH 2/3] docs: add reflection write-path findings notes Dead reader, cross-writer double-store, memory_layer parity question, and the auto-capture watermark pointer, kept as PR discussion material. --- docs/notes/reflection-write-path-findings.md | 68 ++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 docs/notes/reflection-write-path-findings.md diff --git a/docs/notes/reflection-write-path-findings.md b/docs/notes/reflection-write-path-findings.md new file mode 100644 index 00000000..ebd0238c --- /dev/null +++ b/docs/notes/reflection-write-path-findings.md @@ -0,0 +1,68 @@ +# Reflection write-path findings + +Findings from a code-plus-trace audit of the reflection persistence paths. +Kept here as discussion material; each item states what the code does today +and what remains open. + +## Two writers, different gates + +The reflection distiller output is persisted by two independent writers: + +1. **Writer 1 (mapped rows).** Four headings of the reflection markdown + (User model deltas, Agent model deltas, Lessons & pitfalls, Decisions) + are parsed into individual rows (`extractInjectableReflectionMappedMemoryItems`) + and bulk-stored as ordinary memories with `metadata.type = + "memory-reflection-mapped"`. Until the accompanying change, this path had + no admission gate and no gating knob; `reflectionStoreToLanceDB` gates + only writer 2. Mapped rows now route through the same AdmissionController + as extraction candidates when admission control is enabled (passthrough + when disabled, fail-open on infra errors). Scoring category mapping: + `preference` (user/agent model deltas) scores as `preferences`, `fact` + (lessons) as `cases`, `decision` as `events`. + +2. **Writer 2 (reflection documents).** `storeReflectionToLanceDB` persists + sliced reflection documents under category `reflection`, gated by the + `reflectionStoreToLanceDB` config knob. Unchanged by this work. + +## Dead reader + +`loadReflectionMappedRowsFromEntries` (reflection-store) was built to read +mapped rows back for a dedicated injection path, but it has no production +callers (only a test imports it). The only living read path for mapped rows +is generic semantic recall. A code comment now marks this. Open question: +either wire the dedicated reader or consider retiring mapped rows in favor +of writer 2 plus recall. + +## Cross-writer double-store + +"Decisions (durable)" bullets can be persisted twice: once by writer 1 as a +`decision` row, and once inside writer 2's document slices (invariant +fallback). There is no cross-lane dedup between the two writers. Any future +dedup would need to compare across `metadata.type` values. + +## memory_layer parity question + +The auto-recall governance filter drops entries with `memory_layer` in +`{archive, reflection}`, but only the dreaming engine sets layer +`reflection`. Mapped rows carry layer semantics of ordinary confirmed +memories, so they pass recall filters that reflection-layer content does +not. If mapped rows are conceptually reflection output, they may deserve +the same layer tag; if they are conceptually ordinary memories, the +admission gate added here is the appropriate control. Left as a design +question. + +## Auto-capture watermark + +The per-session auto-capture cursor (`autoCaptureSeenTextCount`) is reset +to zero after every successful smart extraction, which makes the next +capture of the same session re-read the whole history instead of the +delta. Analysis and fix (recording the consumed length instead) are on the +dedicated watermark fix branch. + +## Residual gap + +The mapped-row admission gate reuses the SmartExtractor's controller +instance. When smart extraction is disabled while admission control is +enabled, mapped rows still pass ungated (no controller exists to borrow). +This mirrors the pre-change behavior and is called out here for +completeness. From d871c11daab028a80e17ba4a30cece6a0da53b11 Mon Sep 17 00:00:00 2001 From: Gorkem Date: Sun, 12 Jul 2026 19:21:25 +0300 Subject: [PATCH 3/3] chore: rebuild dist for reflection mapped-rows admission routing --- dist/index.js | 26 ++++++++- dist/src/reflection-mapped-admission.js | 76 +++++++++++++++++++++++++ dist/src/reflection-store.js | 4 ++ dist/src/smart-extractor.js | 12 ++++ 4 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 dist/src/reflection-mapped-admission.js diff --git a/dist/index.js b/dist/index.js index 6d3e2655..6e24fb41 100644 --- a/dist/index.js +++ b/dist/index.js @@ -37,6 +37,7 @@ import { parseReflectionMetadata } from "./src/reflection-metadata.js"; import { extractReflectionLearningGovernanceCandidates, extractInjectableReflectionMappedMemoryItems, isRecallUsed, } from "./src/reflection-slices.js"; import { createReflectionEventId } from "./src/reflection-event-store.js"; import { buildReflectionMappedMetadata } from "./src/reflection-mapped-metadata.js"; +import { gateMappedReflectionEntry } from "./src/reflection-mapped-admission.js"; import { createMemoryCLI } from "./cli.js"; import { isNoise } from "./src/noise-filter.js"; import { normalizeAutoCaptureText } from "./src/auto-capture-cleanup.js"; @@ -1008,7 +1009,7 @@ async function ensureDailyLogFile(dailyPath, dateStr) { await writeFile(dailyPath, `# ${dateStr}\n\n`, "utf-8"); } } -function buildReflectionPrompt(conversation, maxInputChars, toolErrorSignals = []) { +export function buildReflectionPrompt(conversation, maxInputChars, toolErrorSignals = []) { const clipped = conversation.slice(-maxInputChars); const errorHints = toolErrorSignals.length > 0 ? toolErrorSignals @@ -1039,6 +1040,7 @@ function buildReflectionPrompt(conversation, maxInputChars, toolErrorSignals = [ "- Do not wrap one bullet across multiple lines.", "- If a bullet section is empty, write exactly: '- (none captured)'", "- Do not paste raw transcript.", + "- Grounding: treat claims made inside roleplay, games, fiction, hypotheticals, or test/simulation frames as not real. Such content may be summarized in Context or Open loops, but must NEVER appear under Decisions (durable), User model deltas, Agent model deltas, or Lessons & pitfalls \u2014 those sections become durable memory rows.", "- Do not invent Logged timestamps, ids, file paths, commit hashes, session ids, or storage metadata unless they already appear in the input.", "- If secrets/tokens/passwords appear, keep them as [REDACTED].", "", @@ -3848,6 +3850,25 @@ const memoryLanceDBProPlugin = { if (existing.length > 0 && existing[0].score > 0.95) { continue; } + // Writer-1 admission routing: mapped rows previously bypassed + // admission control entirely. Gate each row through the same + // AdmissionController as extraction candidates; passthrough when + // admission control (or smart extraction) is disabled. + const mappedGate = await gateMappedReflectionEntry({ + admissionController: smartExtractor?.getAdmissionController() ?? null, + attachAudit: smartExtractor?.shouldPersistAdmissionAudit() ?? false, + text: mapped.text, + category: mapped.category, + heading: mapped.heading, + vector, + reflectionText, + scopeFilter: [targetScope], + warnLog: (msg) => api.logger.warn(msg), + }); + if (!mappedGate.admit) { + api.logger.info(`memory-reflection: admission rejected mapped row heading=${JSON.stringify(mapped.heading)} provenance=memory-reflection-mapped: ${mappedGate.reason ?? "no reason"}`); + continue; + } const importance = mapped.category === "decision" ? 0.85 : 0.8; const baseMetadata = buildReflectionMappedMetadata({ mappedItem: mapped, @@ -3862,6 +3883,9 @@ const memoryLanceDBProPlugin = { }); // embed heading in metadata JSON so it survives bulkStore round-trip to LanceDB baseMetadata._reflectionHeading = mapped.heading; + if (mappedGate.auditJson) { + baseMetadata.admission_audit = mappedGate.auditJson; + } const metadata = JSON.stringify(baseMetadata); mappedEntries.push({ text: mapped.text, diff --git a/dist/src/reflection-mapped-admission.js b/dist/src/reflection-mapped-admission.js new file mode 100644 index 00000000..b29c6b22 --- /dev/null +++ b/dist/src/reflection-mapped-admission.js @@ -0,0 +1,76 @@ +/** + * Admission gating for reflection writer-1 (mapped rows). + * + * The reflection distiller's mapped sections (User model deltas, Agent model + * deltas, Lessons & pitfalls, Decisions) become ordinary durable memory rows + * via bulkStore. Historically they bypassed admission control entirely, so a + * contaminated or hallucinated distillate line landed as a confirmed memory + * with no gate and no audit trail. This module routes each mapped row through + * the same AdmissionController evaluation extraction candidates get, while + * preserving passthrough behavior when admission control is disabled. + */ +/** + * Admission typePriors are keyed by the six smart registers, but mapped rows + * carry legacy store categories. Score them under the smart register that + * matches their shape: user-model/agent-model deltas are preference-shaped + * statements about the human or the assistant ("preference"), lessons are + * symptom/cause/fix/prevention pairs ("fact" here, cases-shaped), and + * decisions are episodic records of something decided ("events"). + */ +export function mapReflectionMappedCategoryToSmartRegister(category) { + switch (category) { + case "preference": + return "preferences"; + case "fact": + return "cases"; + case "decision": + return "events"; + default: + return "events"; + } +} +/** + * Gate one mapped reflection row through admission control. + * + * - No controller (admission disabled or smart extraction off): passthrough, + * identical to the historical behavior. + * - Controller reject: the row is dropped; the caller logs the reason. + * - Controller pass: the row proceeds; when `attachAudit` is set the audit + * record (tagged with provenance "memory-reflection-mapped") is returned + * for persistence alongside the row. + * - Infra failure inside evaluation: fail open with a reason, so a transient + * store/LLM error cannot silently suppress reflection persistence. + */ +export async function gateMappedReflectionEntry(params) { + if (!params.admissionController) { + return { admit: true }; + } + const smartCategory = mapReflectionMappedCategoryToSmartRegister(params.category); + let evaluation; + try { + evaluation = await params.admissionController.evaluate({ + candidate: { + category: smartCategory, + abstract: params.text, + overview: `## ${params.heading}`, + content: params.text, + }, + candidateVector: params.vector, + conversationText: params.reflectionText, + scopeFilter: params.scopeFilter, + }); + } + catch (err) { + params.warnLog?.(`memory-reflection: mapped-row admission evaluation failed, admitting without audit: ${String(err)}`); + return { admit: true, reason: "admission evaluation failed open" }; + } + if (evaluation.decision === "reject") { + return { admit: false, reason: evaluation.audit.reason }; + } + return { + admit: true, + auditJson: params.attachAudit + ? JSON.stringify({ ...evaluation.audit, provenance: "memory-reflection-mapped" }) + : undefined, + }; +} diff --git a/dist/src/reflection-store.js b/dist/src/reflection-store.js index a44785b3..ca3bae34 100644 --- a/dist/src/reflection-store.js +++ b/dist/src/reflection-store.js @@ -441,6 +441,10 @@ function resolveLegacyDeriveBaseWeight(metadata) { } return 1; } +// NOTE: this reader currently has no production callers (only tests import +// it). Mapped rows written by the reflection writer are consumed exclusively +// through generic semantic recall today; the dedicated injection path this +// reader was built for was never wired up. Kept for that planned wiring. export function loadReflectionMappedRowsFromEntries(params) { const now = Number.isFinite(params.now) ? Number(params.now) : Date.now(); const maxAgeMs = Number.isFinite(params.maxAgeMs) diff --git a/dist/src/smart-extractor.js b/dist/src/smart-extractor.js index aa0fcb85..65ec09b3 100644 --- a/dist/src/smart-extractor.js +++ b/dist/src/smart-extractor.js @@ -197,6 +197,18 @@ export class SmartExtractor { ? new AdmissionController(this.store, this.llm, config.admissionControl, this.debugLog) : null; } + /** + * Expose the admission controller so sibling write paths (reflection + * mapped rows) gate through the same instance and config as extraction + * candidates. Null when admission control is disabled. + */ + getAdmissionController() { + return this.admissionController; + } + /** Whether admitted entries should carry the admission audit in metadata. */ + shouldPersistAdmissionAudit() { + return this.persistAdmissionAudit; + } /** * Notify the onPersisted sink (e.g. markdown mirror) after a successful * create or merge. Fire-and-forget from the caller's perspective: awaited