From 09f7a67eae37db04bd11fd1f1439bb22e3d1bbbb Mon Sep 17 00:00:00 2001 From: Thomas <155702229+MakerViking@users.noreply.github.com> Date: Fri, 16 Oct 2026 01:50:52 +0200 Subject: [PATCH 1/3] =?UTF-8?q?feat(learn):=20co-activation=20edges,=20pha?= =?UTF-8?q?se=201=20=E2=80=94=20record,=20don't=20rank?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two memories opened in the same stretch of work are probably about the same work, and nothing in the store noticed. `record_opened` now pairs the opened memory with every other live memory opened in the last 30 minutes and strengthens a derived `co_recalled` edge between them (Hebbian, rate 0.15 toward 1.0, one fire per pair per UTC day, 30-day half-life applied lazily on read, at most 32 edges per node, weakest evicted first). Phase 1 deliberately does not touch ranking: an unmeasured scoring change is how the per-file-cap experiment lost 0.070 nDCG. The edges are written so there is something real to measure. - co-opened, not co-shown: `shown` fires for every hit of every recall - memory<->memory only; pairs with an authored edge are skipped - canonical orientation (src = lower rowid), so one pair is one row - `link` (CLI and MCP) refuses rel=co_recalled: hand-authoring one forges usage and permanently masks the real signal for that pair - excluded from sync and export: derived, per-machine, regenerable - `edges_of` skips a rel this binary doesn't know instead of failing the whole query, so a newer store can't break an older `get` - `recall --linked` shows authored edges, not derived ones; the full record lists "often recalled with"; dashboard counts them on their own row --- crates/mimir-cli/src/commands.rs | 88 ++++- crates/mimir-cli/src/dashboard.rs | 8 + crates/mimir-cli/src/graph_viz.rs | 85 ++++- crates/mimir-cli/src/main.rs | 4 +- crates/mimir-cli/src/mcp.rs | 65 +++- crates/mimir-cli/src/report.rs | 8 +- crates/mimir-core/src/format.rs | 31 ++ crates/mimir-core/src/import.rs | 52 ++- crates/mimir-core/src/learn.rs | 565 ++++++++++++++++++++++++++++- crates/mimir-core/src/model.rs | 6 + crates/mimir-core/src/replicate.rs | 95 ++++- crates/mimir-core/src/store.rs | 28 +- 12 files changed, 1002 insertions(+), 33 deletions(-) diff --git a/crates/mimir-cli/src/commands.rs b/crates/mimir-cli/src/commands.rs index 7487b59..c35c559 100644 --- a/crates/mimir-cli/src/commands.rs +++ b/crates/mimir-cli/src/commands.rs @@ -1268,7 +1268,17 @@ pub fn recall( ); } if linked && !json { - for edge in store::edges_of(&mimir.conn, hit.node.id)?.iter().take(4) { + // `edges_of` is an unordered `src=?1 OR dst=?1`, so the four slots + // go to whatever the table hands back first. A node can carry 32 + // derived co_recall edges and only one or two authored ones, which + // is exactly the wrong way round for a flag whose whole job is to + // show the links somebody drew. The "often recalled with" line in + // the full record is where the derived ones belong. + for edge in store::edges_of(&mimir.conn, hit.node.id)? + .iter() + .filter(|e| e.rel != Rel::CoRecalled) + .take(4) + { let other_id = if edge.src == hit.node.id { edge.dst } else { @@ -1815,9 +1825,11 @@ pub fn reproject(json: bool, reference: &str, project: Option, global: b Ok(()) } -pub fn link(a: &str, b: &str, rel: &str) -> Result<()> { - let mimir = Mimir::open()?; - let rel: Rel = rel.parse()?; +/// The relations `mimir link` will not author, and why. Lives apart from +/// `link` because `link` opens the store before it ever looks at the +/// relation, and a refusal that needs a database on disk to exercise is a +/// refusal nobody tests. +fn reject_unauthorable_rel(rel: Rel) -> Result<()> { // See the same guard in the MCP `link` tool: the `supersedes` edge and the // `superseded_by` column are independent, and only the column hides a node // from recall. Allowing it here would silently record a retirement that @@ -1829,6 +1841,26 @@ pub fn link(a: &str, b: &str, rel: &str) -> Result<()> { ` instead (it sets both)." ); } + // `co_recalled` is not a claim anybody makes: the store writes it for + // itself when it observes two memories opened together. Hand-authoring one + // forges a usage record for a co-recall that never happened — and because + // the observer skips any pair that already has an authored edge, it would + // also suppress the real edge if that pair ever does get recalled together. + if rel == Rel::CoRecalled { + bail!( + "`--rel co_recalled` is the store's own observation of memories recalled \ + together — writing one by hand forges usage that never happened, and it \ + would stop the store recording the real co-recall for that pair. Use \ + `link --rel relates` for a link you actually mean." + ); + } + Ok(()) +} + +pub fn link(a: &str, b: &str, rel: &str) -> Result<()> { + let mimir = Mimir::open()?; + let rel: Rel = rel.parse()?; + reject_unauthorable_rel(rel)?; let src = resolve_any(&mimir, a)?; let dst = resolve_any(&mimir, b)?; store::link(&mimir.conn, src.id, dst.id, rel, 1.0)?; @@ -1964,10 +1996,20 @@ pub fn link_scan(dry_run: bool, all_projects: bool) -> Result<()> { // Every edge, once. The inner loop asks "is this pair already linked?" // per candidate, and re-reading the table per project turned that into // the dominant cost on a store with a hundred thousand symbols. + // + // The set is rel-agnostic, so a derived co_recall edge in it would read as + // "already linked" and cost the pair its real `about` edge, silently. Today + // co_recalled only joins memory to memory and this scan links memory to + // symbol, so the two cannot actually collide — the exclusion is here so + // that stays true if the observer ever learns to relate other kinds. let mut existing: std::collections::HashSet<(i64, i64)> = Default::default(); { - let mut stmt = mimir.conn.prepare("SELECT src, dst FROM edge")?; - let mut rows = stmt.query([])?; + let mut stmt = mimir + .conn + .prepare("SELECT src, dst FROM edge WHERE rel != ?1")?; + // Bound rather than inlined so the stored spelling comes from the enum + // and cannot drift away from it in a string literal here. + let mut rows = stmt.query([Rel::CoRecalled.as_str()])?; while let Some(r) = rows.next()? { let (s, d): (i64, i64) = (r.get(0)?, r.get(1)?); existing.insert((s, d)); @@ -2835,3 +2877,37 @@ mod link_scan_hint_tests { assert!(hint.contains("45d ago"), "{hint}"); } } + +#[cfg(test)] +mod link_rel_guard_tests { + use super::reject_unauthorable_rel; + use mimir_core::model::Rel; + + /// A hand-written `co_recalled` edge is a forged usage record, and it + /// would also cost the pair its real derived edge later (the observer + /// skips pairs that already carry an authored one). The message has to + /// leave the user somewhere to go, so it names the relation they meant. + #[test] + fn co_recalled_is_refused_and_points_at_relates() { + let err = reject_unauthorable_rel(Rel::CoRecalled) + .expect_err("co_recalled must not be authorable") + .to_string(); + assert!(err.contains("co_recalled"), "{err}"); + assert!(err.contains("relates"), "{err}"); + } + + /// The pre-existing refusal, kept honest now that it shares a function. + #[test] + fn supersedes_is_still_refused() { + assert!(reject_unauthorable_rel(Rel::Supersedes).is_err()); + } + + /// Everything a person can legitimately draw still passes — a guard + /// that over-refuses is worse than the forgery it prevents. + #[test] + fn authored_relations_pass() { + for rel in [Rel::Relates, Rel::About, Rel::Links, Rel::Mentions] { + assert!(reject_unauthorable_rel(rel).is_ok(), "{rel} was refused"); + } + } +} diff --git a/crates/mimir-cli/src/dashboard.rs b/crates/mimir-cli/src/dashboard.rs index b64783d..20bd748 100644 --- a/crates/mimir-cli/src/dashboard.rs +++ b/crates/mimir-cli/src/dashboard.rs @@ -174,6 +174,12 @@ fn render(mimir: &Mimir) -> Result { let imports = q1("SELECT count(*) FROM edge WHERE rel='imports'")?; let links = q1("SELECT count(*) FROM edge WHERE rel IN ('about','relates','links','mentions')")?; + // Counted separately and shown on its own row rather than added to `links` + // above: these are edges the store derived from watching itself, so adding + // them to a number a reader takes as authored work would quietly turn that + // row into machine activity. Hiding them entirely was the other option and + // is worse — the rows exist and take space, so they should be legible. + let co_recalled = q1("SELECT count(*) FROM edge WHERE rel='co_recalled'")?; let embedded = q1("SELECT count(*) FROM embedding")?; // Kind list mirrors `embed::embed_pending`'s eligibility filter exactly — // drawing from the shared constant means a kind added there (e.g. the @@ -439,6 +445,7 @@ fn render(mimir: &Mimir) -> Result { .replace("{{CALLS_H}}", &fmt_n(calls_heuristic)) .replace("{{IMPORTS}}", &fmt_n(imports)) .replace("{{LINKS}}", &fmt_n(links)) + .replace("{{CO_RECALLED}}", &fmt_n(co_recalled)) .replace("{{EMBEDDED}}", &fmt_n(embedded)) .replace("{{COVERAGE}}", &format!("{coverage:.0}")) .replace("{{MODEL}}", &esc(&mimir.config.embedding.model)) @@ -689,6 +696,7 @@ footer span{margin-left:auto}
calls heuristic{{CALLS_H}}
import edges{{IMPORTS}}
memory↔code links{{LINKS}}
+
co-recalled (observed){{CO_RECALLED}}
embedded{{EMBEDDED}} ({{COVERAGE}}%)
model{{MODEL}}
superseded{{SUPERSEDED}}
diff --git a/crates/mimir-cli/src/graph_viz.rs b/crates/mimir-cli/src/graph_viz.rs index 0d1cf46..5d57bba 100644 --- a/crates/mimir-cli/src/graph_viz.rs +++ b/crates/mimir-cli/src/graph_viz.rs @@ -8,6 +8,7 @@ use std::collections::{HashMap, HashSet}; use anyhow::{Context, Result}; +use mimir_core::model::Rel; use mimir_core::Mimir; use serde_json::json; @@ -72,9 +73,22 @@ fn render(mimir: &Mimir, project_id: i64, project: &str, max_nodes: usize) -> Re // One pass over the edge table serves both the degree map here and the // included-set filtering at the end (it was two full scans before). + // + // co_recalled is excluded at the query, not at render time, because this + // scan also feeds `degree` — which sets node radius and orders the symbol + // budget below. Dropping those edges later would still leave the picture + // reshaped by links nobody can see. And they are dense memory↔memory + // springs the layout would visibly bunch. The alternative — draw them and + // let the viewer switch them off — doesn't exist here: the legend is built + // from node KINDS, not rels, so there is no per-rel toggle and a wrong + // default could not be undone at view time. Excluding is reversible. + // + // The spelling comes from the enum, not a literal, so a rename there + // cannot leave this filter silently matching nothing. + let co_recalled = Rel::CoRecalled.as_str(); let all_edges: Vec<(i64, i64, String)> = { - let mut stmt = conn.prepare("SELECT src, dst, rel FROM edge")?; - let rows = stmt.query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))?; + let mut stmt = conn.prepare("SELECT src, dst, rel FROM edge WHERE rel <> ?1")?; + let rows = stmt.query_map([co_recalled], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))?; rows.collect::>()? }; let mut degree: HashMap = HashMap::new(); @@ -157,12 +171,21 @@ fn render(mimir: &Mimir, project_id: i64, project: &str, max_nodes: usize) -> Re if !mem_ids.is_empty() { // Chunked: the id list appears twice in the query, and SQLite caps // bound parameters at 32766 — a big store would abort otherwise. + // + // Both halves exclude co_recalled. This expansion has no project + // filter, and co-activation crosses projects by construction (a global + // memory opened next to a project one), so without the rel test it + // would pull foreign-project memories in as first-class nodes and + // spend the budget computed below on them. Interpolated rather than + // bound because the id list already fills the anonymous `?` slots + // twice; it is the enum's own constant, not user input. for chunk in mem_ids.chunks(8_000) { let placeholders = vec!["?"; chunk.len()].join(","); let sql = format!( "SELECT {COLS} FROM node WHERE {live} AND id IN ( - SELECT dst FROM edge WHERE src IN ({placeholders}) - UNION SELECT src FROM edge WHERE dst IN ({placeholders}))" + SELECT dst FROM edge WHERE rel <> '{co_recalled}' AND src IN ({placeholders}) + UNION + SELECT src FROM edge WHERE rel <> '{co_recalled}' AND dst IN ({placeholders}))" ); let params: Vec<&dyn rusqlite::ToSql> = chunk .iter() @@ -637,3 +660,57 @@ for(const k of kinds){ "#; + +#[cfg(test)] +mod tests { + use super::*; + use mimir_core::Mimir; + + fn node(m: &Mimir, uid: &str, kind: &str, title: &str, project: Option) -> i64 { + m.conn + .execute( + "INSERT INTO node (uid, kind, project_id, title, created_at, updated_at) + VALUES (?1, ?2, ?3, ?4, 1, 1)", + rusqlite::params![uid, kind, project, title], + ) + .unwrap(); + m.conn.last_insert_rowid() + } + + fn edge(m: &Mimir, src: i64, dst: i64, rel: &str) { + m.conn + .execute( + "INSERT INTO edge (src, dst, rel, created_at) VALUES (?1, ?2, ?3, 1)", + rusqlite::params![src, dst, rel], + ) + .unwrap(); + } + + /// co_recalled must be invisible to the viz in every way it could show: + /// as a drawn edge, as a node the expansion dragged in from another + /// project, as degree (radius / budget order), and in the HUD count. + #[test] + fn co_recalled_edges_never_reach_the_viz() { + let m = Mimir::open_in_memory().unwrap(); + let proj = node(&m, "p:here", "project", "here", None); + let other = node(&m, "p:elsewhere", "project", "elsewhere", None); + let mine = node(&m, "m:mine", "memory", "MINE", Some(proj)); + let doc = node(&m, "d:doc", "doc", "DOC", None); + let foreign = node(&m, "m:foreign", "memory", "FOREIGNMEM", Some(other)); + + edge(&m, mine, doc, "about"); + // Written through the enum so a rename can't turn this into a test + // that inserts a rel nothing filters and then passes anyway. + edge(&m, mine, foreign, Rel::CoRecalled.as_str()); + + let out = render(&m, proj, "here", 500).unwrap(); + assert!(!out.contains(Rel::CoRecalled.as_str()), "{out}"); + assert!(!out.contains("FOREIGNMEM"), "{out}"); + // The HUD has to agree with what is drawn: project + memory + doc, + // and the one authored edge. + assert!(out.contains("3 nodes · 1 edges"), "{out}"); + // Degree feeds node radius and the symbol budget, so a co_recalled + // edge counted there would deform the layout even while unrendered. + assert!(!out.contains("\"deg\":2"), "{out}"); + } +} diff --git a/crates/mimir-cli/src/main.rs b/crates/mimir-cli/src/main.rs index 5bb1b79..4ce875f 100644 --- a/crates/mimir-cli/src/main.rs +++ b/crates/mimir-cli/src/main.rs @@ -300,7 +300,9 @@ enum Command { Link { a: Option, b: Option, - /// links|about|relates|mentions|describes|supersedes… + /// links|about|relates|mentions|describes|supersedes|summarizes| + /// derived_from. `co_recalled` is not in the vocabulary here: the + /// store writes those for itself from what you actually recall. #[arg(long, default_value = "relates")] rel: String, /// Auto-link memories to code symbols their text mentions diff --git a/crates/mimir-cli/src/mcp.rs b/crates/mimir-cli/src/mcp.rs index 8220b85..6a85e38 100644 --- a/crates/mimir-cli/src/mcp.rs +++ b/crates/mimir-cli/src/mcp.rs @@ -160,7 +160,9 @@ pub struct LinkArgs { pub a: String, /// Target node reference. pub b: String, - /// Relation: relates | about | links | mentions | supersedes (default relates). + /// Relation: relates | about | links | mentions (default relates). + /// Refused here: `supersedes` (use the `supersede` tool) and + /// `co_recalled` (the store derives those from observed usage). #[serde(default)] pub rel: Option, } @@ -473,7 +475,7 @@ impl MimirServer { } #[tool( - description = "Link two nodes (memory/doc/code) so context surfaces together. Note: `supersedes` is NOT accepted here — use the `supersede` tool, which also retires the old node." + description = "Link two nodes (memory/doc/code) so context surfaces together. Note: `supersedes` is NOT accepted here — use the `supersede` tool, which also retires the old node. `co_recalled` is NOT accepted either — the store writes those itself from observed usage, so a hand-authored one forges a co-activation that never happened; use `relates`." )] async fn link(&self, Parameters(args): Parameters) -> String { self.blocking(move |m| { @@ -496,6 +498,23 @@ impl MimirServer { .into(), ))); } + // `co_recalled` is not a claim, it is an observation the store made + // about its own usage: two memories opened in the same stretch of + // work. Hand-authoring one forges a usage record that never + // happened. Worse, `learn::record_coactivation` deliberately skips + // any pair that already carries an authored edge, so the forgery is + // not a harmless white lie — it permanently suppresses the genuine + // co-activation signal for that pair. + if rel == Rel::CoRecalled { + return Err(engine_err(mimir_core::error::Error::Invalid( + "link does not accept rel=\"co_recalled\": the store derives those \ + from observed co-recall and refuses to draw one for a pair that \ + already has an authored edge, so hand-authoring one fakes usage \ + that never happened AND blocks the real signal for that pair \ + forever. Use rel=\"relates\" for a link you actually mean." + .into(), + ))); + } let src = store::resolve_ref(&m.conn, &args.a).map_err(engine_err)?; let dst = store::resolve_ref(&m.conn, &args.b).map_err(engine_err)?; store::link(&m.conn, src.id, dst.id, rel, 1.0).map_err(engine_err)?; @@ -1931,6 +1950,48 @@ mod tests { assert!(ok.contains("relates"), "relates should still link: {ok}"); } + /// `link` must refuse `co_recalled`. It is derived from observed usage, and + /// the store skips drawing a real one for any pair that already has an + /// authored edge — so a hand-written one would not just fake a co-recall, + /// it would permanently mask the genuine one for that pair. + #[tokio::test(flavor = "multi_thread")] + async fn link_refuses_the_co_recalled_rel() { + let (_dir, s) = server(); + let a = s + .remember(Parameters(remember_args("first fact here", "note"))) + .await; + let b = s + .remember(Parameters(remember_args("second fact here", "note"))) + .await; + let (a, b) = ( + a.split_whitespace().nth(1).unwrap().to_string(), + b.split_whitespace().nth(1).unwrap().to_string(), + ); + + let refused = s + .link(Parameters(LinkArgs { + a: a.clone(), + b: b.clone(), + rel: Some("co_recalled".into()), + })) + .await; + assert!( + refused.contains("co_recalled") && refused.contains("co-recall"), + "expected a derived-edge refusal, got: {refused}" + ); + + // The neighbouring rel still works — this is a targeted refusal, not + // a broken link tool. + let ok = s + .link(Parameters(LinkArgs { + a, + b, + rel: Some("relates".into()), + })) + .await; + assert!(ok.contains("relates"), "relates should still link: {ok}"); + } + /// `remember` with an unparseable `expires_in` must fail loudly rather /// than storing a memory whose requested deadline was silently dropped. #[tokio::test(flavor = "multi_thread")] diff --git a/crates/mimir-cli/src/report.rs b/crates/mimir-cli/src/report.rs index f8cbd89..cd67116 100644 --- a/crates/mimir-cli/src/report.rs +++ b/crates/mimir-cli/src/report.rs @@ -61,11 +61,17 @@ pub fn report(json: bool) -> Result<()> { ("doc chunks indexed", node_counts("chunk")?), ("symbols extracted", node_counts("symbol")?), ( + // `count(*) FROM edge` would fold in the store's own co_recall + // observations, and this row is labelled as work somebody did: a + // week of heavy recalling would read as a week of heavy linking. + // The predicate lives in WHERE rather than inside the four period + // sums so the all-time column is filtered the same way — filtering + // only the periods leaves a total that no column adds up to. "links created", counts( "SELECT COALESCE(sum(created_at>=?1),0), COALESCE(sum(created_at>=?2),0), COALESCE(sum(created_at>=?3),0), COALESCE(sum(created_at>=?4),0), - count(*) FROM edge", + count(*) FROM edge WHERE rel <> 'co_recalled'", )?, ), ( diff --git a/crates/mimir-core/src/format.rs b/crates/mimir-core/src/format.rs index 041cf56..75f6e80 100644 --- a/crates/mimir-core/src/format.rs +++ b/crates/mimir-core/src/format.rs @@ -127,6 +127,12 @@ const STEM_PREFIX: usize = 5; /// simply head-truncate, which is what this code replaced anyway. const MATCH_SCAN_CHARS: usize = 20_000; +/// Co-activation partners listed on a full record. The store may hold up to +/// 32 per node; the point of the line is "this belongs with those", which the +/// strongest few make, and a longer list would cost more context than the +/// record it annotates. +const COACTIVATION_SHOWN: usize = 5; + /// Content-bearing query stems, sharing `tokens`/`is_stopword` with the FTS /// leg so the snippet can't drift from what actually matched. fn query_stems(query: &str) -> Vec { @@ -395,6 +401,13 @@ pub fn full_record( out.push_str(&format!("\ntags: {}", node.tags_text)); } for edge in store::edges_of(conn, node.id)? { + // Co-activation is not an authored relation and does not belong in + // the arrow list beside edges a person or the extractor meant. It + // gets its own line below, ranked and capped; left here it would + // print up to 32 unordered rows and bury the real links. + if edge.rel == crate::model::Rel::CoRecalled { + continue; + } let (arrow, other_id) = if edge.src == node.id { ("→", edge.dst) } else { @@ -409,6 +422,24 @@ pub fn full_record( )); } } + // What the store noticed on its own: memories repeatedly opened in the + // same stretch of work as this one. Labelled as observed, because every + // other line in this record is something somebody asserted. + let partners = crate::learn::coactivation_partners(conn, node.id, crate::model::now_unix())?; + if !partners.is_empty() { + let shown = partners + .iter() + .take(COACTIVATION_SHOWN) + .filter_map(|(id, _)| store::get_node(conn, *id).ok()) + .map(|other| short_uid(other.kind, &other.uid)) + .collect::>(); + if !shown.is_empty() { + out.push_str(&format!( + "\noften recalled with: {} (observed co-recall, not an authored link)", + shown.join(", ") + )); + } + } // Only annotate memories, and only when there is something to say. A // "grounded" badge on a doc chunk is noise (of course it is — it *is* // the artifact), and stamping "ungrounded" on every ordinary note would diff --git a/crates/mimir-core/src/import.rs b/crates/mimir-core/src/import.rs index ac1289c..b1f498f 100644 --- a/crates/mimir-core/src/import.rs +++ b/crates/mimir-core/src/import.rs @@ -251,8 +251,8 @@ pub fn qmd_collections(yml: &str) -> Vec<(String, String)> { // ---------- Export ---------- -/// Everything (except embeddings, which are derived) as JSONL: -/// node lines then edge lines, suitable for backup or migration. +/// Everything (except embeddings and co-recall edges, which are derived) as +/// JSONL: node lines then edge lines, suitable for backup or migration. pub fn export_jsonl(conn: &Connection, out: &mut dyn std::io::Write) -> Result { use crate::store::{row_to_node, NODE_COLS}; let mut count = 0usize; @@ -276,10 +276,19 @@ pub fn export_jsonl(conn: &Connection, out: &mut dyn std::io::Write) -> Result 'co_recalled' ORDER BY e.id", )?; let edges: Vec<(String, String, String, f64)> = stmt .query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)))? @@ -394,4 +403,41 @@ mod tests { assert_eq!(lines[2]["type"], "edge"); assert_eq!(lines[2]["rel"], "relates"); } + + #[test] + fn export_omits_co_recall_edges() { + let conn = db::open_in_memory().unwrap(); + let mk = |t: &str| { + crate::store::insert_node(&conn, { + let mut n = crate::model::NewNode::new(crate::model::Kind::Memory); + n.title = Some(t.into()); + n.body = Some(t.into()); + n + }) + .unwrap() + }; + let a = mk("a"); + let b = mk("b"); + let c = mk("c"); + crate::store::link(&conn, a.id, b.id, crate::model::Rel::Relates, 1.0).unwrap(); + // Written with raw SQL on purpose: `store::link` is the authoring door + // and nothing outside `learn` is supposed to walk through it with this + // rel, so the test forges the row the way the store itself does rather + // than depending on that door staying open. + conn.execute( + "INSERT INTO edge (src, dst, rel, weight, created_at) + VALUES (?1, ?2, 'co_recalled', 0.9, 0)", + [a.id, c.id], + ) + .unwrap(); + let mut buf = Vec::new(); + // 3 nodes + the authored edge; the derived one must not be in the file + // at all — a consumer that simply ignores an unknown rel is not enough, + // because the export is also a backup a human reads. + let count = export_jsonl(&conn, &mut buf).unwrap(); + assert_eq!(count, 4); + let out = String::from_utf8(buf).unwrap(); + assert!(!out.contains("co_recalled"), "{out}"); + assert!(out.contains("\"rel\":\"relates\""), "{out}"); + } } diff --git a/crates/mimir-core/src/learn.rs b/crates/mimir-core/src/learn.rs index b29ad27..fc7c0e1 100644 --- a/crates/mimir-core/src/learn.rs +++ b/crates/mimir-core/src/learn.rs @@ -5,7 +5,7 @@ //! (auto, +0.3), explicit `mark` (±1.0); both capped at one effect per //! node per day so loops can't pump a memory. -use rusqlite::Connection; +use rusqlite::{Connection, OptionalExtension}; use crate::error::Result; use crate::model::{now_unix, Kind, Node}; @@ -161,6 +161,14 @@ pub fn record_opened(conn: &Connection, node_id: i64) -> Result<()> { if first_today { adjust_strength(conn, node_id, OPENED_BONUS)?; } + // Co-activation is an observation ABOUT this open, not a part of it: if + // the edge write fails, the open still happened and the caller must not + // be told it didn't. Warned rather than propagated — two of the callers + // (`peek`, `index::file_slice`) discard this Result, so an error returned + // from here would go nowhere quietly, which is worse than a log line. + if let Err(e) = record_coactivation(conn, node_id, now_unix()) { + tracing::warn!(node_id, error = %e, "co-activation edges not recorded"); + } Ok(()) } @@ -180,11 +188,253 @@ pub fn apply_mark(conn: &Connection, node_id: i64, useful: bool) -> Result ) } +// --------------------------------------------------------------------------- +// Co-activation: the only edges the store draws for itself. +// --------------------------------------------------------------------------- +// +// Two memories opened in the same stretch of work are probably about the same +// stretch of work. Nothing else in the store notices that: `relates` edges are +// authored, `about` edges point at code, and recall treats every hit as +// independent. This records the observation and nothing more — phase 1 does +// not touch ranking, because an unmeasured scoring change is how the +// per-file-cap experiment lost 0.070 nDCG on an idea that sounded just as +// reasonable as this one. The edges are written now so there is something real +// to measure later. +// +// The signal is co-*opened*, not co-shown. `shown` fires for all eight hits of +// every recall — twenty-eight pairs a query, almost all of them noise. An open +// means an agent pulled the body, which is evidence rather than a guess. + +/// Hebbian learning rate: a fire moves the weight `rate` of the way to 1.0. +/// The target is inside the legal range, so the update cannot overshoot and +/// needs no clamp — `w + r(1-w)` is a convex combination of two in-range +/// values. Reaching the visible threshold takes five co-opens. +pub const COACTIVATION_RATE: f64 = 0.15; + +/// Half-life of an un-refreshed co-activation, in days. Applied lazily on +/// read (like [`effective_strength`]) rather than by a sweep, so there is no +/// background job and no write amplification on a table with one row per pair. +pub const COACTIVATION_HALF_LIFE_DAYS: f64 = 30.0; + +/// Effective weight a pair must reach before anything displays it. Below this +/// the edge exists but is still a rumour: one shared session is a coincidence. +pub const COACTIVATION_VISIBLE: f64 = 0.5; + +/// How close together two opens must fall to count as the same stretch of +/// work. Long enough to span a detour, short enough that yesterday's session +/// is not "the same stretch". +const COACTIVATION_WINDOW_SECS: i64 = 30 * 60; + +/// Most co-activation edges kept per node, weakest evicted first. Without a +/// cap, a memory opened in every session pairs with the entire store and its +/// edges stop carrying information. +const COACTIVATION_MAX_PER_NODE: usize = 32; + +/// Co-activation weight after time decay: `weight · 2^(−age/half_life)`, +/// age counted from the last fire. Same shape as [`effective_strength`]; a +/// pair that stops co-occurring fades instead of standing forever. +pub fn coactivation_weight(stored: f64, last_fired_at: i64, now: i64) -> f64 { + let age_days = (now - last_fired_at).max(0) as f64 / 86_400.0; + stored * 2f64.powf(-age_days / COACTIVATION_HALF_LIFE_DAYS) +} + +/// Start of `now`'s UTC day. Mirrors [`capped_today`]'s boundary, using +/// `rem_euclid` so a pre-epoch timestamp floors instead of rounding toward +/// zero. The cap it feeds is per PAIR, which is why `capped_today` itself +/// cannot be reused: that one asks whether a node was opened today, and both +/// nodes of a pair can be opened repeatedly without the pair ever firing. +fn midnight(now: i64) -> i64 { + now - now.rem_euclid(86_400) +} + +/// Is this node a live memory? Co-activation is memory↔memory only. +/// `record_opened` also fires for symbols (`graph node`, `peek`) and for file +/// nodes (`index::file_slice`), and "this function was read near that note" +/// is a different claim with a different meaning — the code graph already +/// records what code relates to what code. +fn is_live_memory(conn: &Connection, node_id: i64) -> Result { + let n: i64 = conn.query_row( + "SELECT count(*) FROM node + WHERE id = ?1 AND kind = 'memory' + AND deleted_at IS NULL AND superseded_by IS NULL", + [node_id], + |r| r.get(0), + )?; + Ok(n > 0) +} + +/// Does a pair already have an edge somebody meant? Authored edges are +/// written in either orientation (`supersede` writes new→old, `link` writes +/// as given), so both directions are checked, and every rel counts: if a +/// person already said how these two relate, the store has nothing to add. +fn has_authored_edge(conn: &Connection, a: i64, b: i64) -> Result { + let n: i64 = conn.query_row( + "SELECT count(*) FROM edge + WHERE rel <> 'co_recalled' + AND ((src = ?1 AND dst = ?2) OR (src = ?2 AND dst = ?1))", + [a, b], + |r| r.get(0), + )?; + Ok(n > 0) +} + +/// Strengthen one pair, or create it at `COACTIVATION_RATE`. Returns false if +/// the pair already fired today. +/// +/// Endpoints are stored canonically (`src` = the lower rowid) because +/// `UNIQUE(src, dst, rel)` is directional: inserting the same pair both ways +/// round would make two rows that the cap, the eviction and the authored-edge +/// check would each see as unrelated. +/// +/// This cannot go through `store::link`, which overwrites `weight` on +/// conflict and never writes `meta` — it would reset the accumulated weight +/// and drop the timestamp the decay is measured from. `link_call` in +/// mimir-graph is the precedent for an edge writer that carries its own meta. +fn fire_pair(conn: &Connection, a: i64, b: i64, now: i64) -> Result { + let (src, dst) = (a.min(b), a.max(b)); + let existing: Option<(i64, f64, Option)> = conn + .query_row( + "SELECT id, weight, json_extract(meta, '$.last_fired_at') FROM edge + WHERE src = ?1 AND dst = ?2 AND rel = 'co_recalled'", + [src, dst], + |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)), + ) + .optional()?; + + match existing { + Some((id, weight, last_fired_at)) => { + let last_fired_at = last_fired_at.unwrap_or(0); + if last_fired_at >= midnight(now) { + return Ok(false); + } + let decayed = coactivation_weight(weight, last_fired_at, now); + let next = decayed + COACTIVATION_RATE * (1.0 - decayed); + conn.execute( + "UPDATE edge SET weight = ?2, + meta = json_set(meta, '$.last_fired_at', ?3) + WHERE id = ?1", + rusqlite::params![id, next, now], + )?; + } + None => { + conn.execute( + "INSERT INTO edge (src, dst, rel, weight, meta, created_at) + VALUES (?1, ?2, 'co_recalled', ?3, json_object('last_fired_at', ?4), ?4)", + rusqlite::params![src, dst, COACTIVATION_RATE, now], + )?; + } + } + Ok(true) +} + +/// Trim `node_id` back to [`COACTIVATION_MAX_PER_NODE`] co-activation edges, +/// weakest first. +/// +/// Ranking happens in Rust over the (at most ~33) candidate rows rather than +/// in SQL, because the comparison is on the DECAYED weight: ordering by the +/// stored column would keep a strong-but-stale pair over a fresh one, which +/// is the opposite of what the decay is for. `id` breaks ties, since the +/// same-day cap makes exact float ties common. +fn evict_beyond_cap(conn: &Connection, node_id: i64, now: i64) -> Result<()> { + let mut stmt = conn.prepare( + "SELECT id, weight, COALESCE(json_extract(meta, '$.last_fired_at'), created_at) + FROM edge WHERE rel = 'co_recalled' AND (src = ?1 OR dst = ?1)", + )?; + let mut rows: Vec<(i64, f64)> = stmt + .query_map([node_id], |r| { + let id: i64 = r.get(0)?; + let weight: f64 = r.get(1)?; + let last_fired_at: i64 = r.get(2)?; + Ok((id, coactivation_weight(weight, last_fired_at, now))) + })? + .collect::>()?; + if rows.len() <= COACTIVATION_MAX_PER_NODE { + return Ok(()); + } + rows.sort_by(|a, b| a.1.total_cmp(&b.1).then(a.0.cmp(&b.0))); + for (id, _) in &rows[..rows.len() - COACTIVATION_MAX_PER_NODE] { + conn.execute("DELETE FROM edge WHERE id = ?1", [id])?; + } + Ok(()) +} + +/// Pair `node_id` with every other live memory opened in the last +/// [`COACTIVATION_WINDOW_SECS`], strengthening one edge per pair per day. +/// Returns how many pairs fired. +/// +/// Call this AFTER the node's own `opened` row is written: the window is read +/// from `recall_event`, so ordering decides whether two opens in the same +/// instant see each other. +pub fn record_coactivation(conn: &Connection, node_id: i64, now: i64) -> Result { + if !is_live_memory(conn, node_id)? { + return Ok(0); + } + let mut stmt = conn.prepare( + "SELECT DISTINCT e.node_id FROM recall_event e + JOIN node n ON n.id = e.node_id + WHERE e.event = 'opened' AND e.at >= ?2 AND e.node_id <> ?1 + AND n.kind = 'memory' + AND n.deleted_at IS NULL AND n.superseded_by IS NULL", + )?; + let partners: Vec = stmt + .query_map( + rusqlite::params![node_id, now - COACTIVATION_WINDOW_SECS], + |r| r.get(0), + )? + .collect::>()?; + + let mut fired = Vec::new(); + for partner in partners { + if has_authored_edge(conn, node_id, partner)? { + continue; + } + if fire_pair(conn, node_id, partner, now)? { + fired.push(partner); + } + } + if !fired.is_empty() { + evict_beyond_cap(conn, node_id, now)?; + } + Ok(fired.len()) +} + +/// The co-activation partners of `node_id` that have earned display, strongest +/// first: effective weight at or above [`COACTIVATION_VISIBLE`], dead and +/// retired endpoints dropped. Returns `(node_id, effective_weight)`. +/// +/// Endpoints are filtered here rather than at write time because `forget` and +/// `supersede` are soft — neither deletes the row, so `ON DELETE CASCADE` +/// never fires and edges to retired memories survive indefinitely. +pub fn coactivation_partners(conn: &Connection, node_id: i64, now: i64) -> Result> { + let mut stmt = conn.prepare( + "SELECT CASE WHEN e.src = ?1 THEN e.dst ELSE e.src END, + e.weight, + COALESCE(json_extract(e.meta, '$.last_fired_at'), e.created_at) + FROM edge e + JOIN node n ON n.id = CASE WHEN e.src = ?1 THEN e.dst ELSE e.src END + WHERE e.rel = 'co_recalled' AND (e.src = ?1 OR e.dst = ?1) + AND n.deleted_at IS NULL AND n.superseded_by IS NULL", + )?; + let mut partners: Vec<(i64, f64)> = stmt + .query_map([node_id], |r| { + let other: i64 = r.get(0)?; + let weight: f64 = r.get(1)?; + let last_fired_at: i64 = r.get(2)?; + Ok((other, coactivation_weight(weight, last_fired_at, now))) + })? + .collect::>>()? + .into_iter() + .filter(|(_, w)| *w >= COACTIVATION_VISIBLE) + .collect(); + partners.sort_by(|a, b| b.1.total_cmp(&a.1).then(a.0.cmp(&b.0))); + Ok(partners) +} + #[cfg(test)] mod tests { use super::*; use crate::db; - use crate::model::NewNode; + use crate::model::{Actor, NewNode, Rel}; fn memory(conn: &Connection, subkind: &str) -> Node { let mut new = NewNode::new(Kind::Memory); @@ -318,4 +568,315 @@ mod tests { // damp ~= 1 - alpha/2, not clamped by the floor at these numbers. assert!((stale - (1.0 - 0.4 / 2.0)).abs() < 0.01, "stale={stale}"); } + + // ----------------------------------------------------------------- + // Co-activation + // ----------------------------------------------------------------- + + /// A fixed synthetic clock, never `now_unix()`. Both day boundaries in + /// this feature (`midnight`, and `capped_today` behind `record_opened`) + /// are derived from the timestamp itself, so a test that planted an + /// event at "sixty seconds ago" would land the two ends of a pair on + /// opposite sides of UTC midnight whenever CI happened to run at 00:00. + /// This instant sits 80_000s into its own day — far enough in that a + /// full window can be subtracted without crossing back into yesterday. + const T: i64 = 1_700_000_000; + + /// Plant an `opened` row directly. `record_opened` stamps `now_unix()` + /// and would put the event outside any synthetic window. + fn opened_at(conn: &Connection, node_id: i64, at: i64) { + conn.execute( + "INSERT INTO recall_event (node_id, event, at) VALUES (?1, 'opened', ?2)", + rusqlite::params![node_id, at], + ) + .unwrap(); + } + + /// Write a co-activation edge with a chosen weight and age, for the + /// cases that would otherwise need dozens of simulated days to reach. + fn seed_co_edge(conn: &Connection, a: i64, b: i64, weight: f64, last_fired_at: i64) { + conn.execute( + "INSERT INTO edge (src, dst, rel, weight, meta, created_at) + VALUES (?1, ?2, 'co_recalled', ?3, json_object('last_fired_at', ?4), ?4)", + rusqlite::params![a.min(b), a.max(b), weight, last_fired_at], + ) + .unwrap(); + } + + /// Every co-activation edge in the store, insertion order, as + /// `(src, dst, weight, last_fired_at)`. Read raw rather than through + /// `store::edges_of` so the stored orientation is visible: `edges_of` + /// returns both directions of the same row identically. + fn co_edges(conn: &Connection) -> Vec<(i64, i64, f64, i64)> { + let mut stmt = conn + .prepare( + "SELECT src, dst, weight, json_extract(meta, '$.last_fired_at') + FROM edge WHERE rel = 'co_recalled' ORDER BY id", + ) + .unwrap(); + stmt.query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?))) + .unwrap() + .collect::>() + .unwrap() + } + + /// The other end of every co-activation edge touching `node_id`, + /// unfiltered — unlike `coactivation_partners`, which hides the weak + /// and the retired and so cannot show what the cap actually kept. + fn co_partner_ids(conn: &Connection, node_id: i64) -> Vec { + let mut stmt = conn + .prepare( + "SELECT CASE WHEN src = ?1 THEN dst ELSE src END FROM edge + WHERE rel = 'co_recalled' AND (src = ?1 OR dst = ?1) ORDER BY id", + ) + .unwrap(); + stmt.query_map([node_id], |r| r.get(0)) + .unwrap() + .collect::>() + .unwrap() + } + + fn retire(conn: &Connection, id: i64, by: i64) { + conn.execute( + "UPDATE node SET superseded_by = ?2 WHERE id = ?1", + rusqlite::params![id, by], + ) + .unwrap(); + } + + #[test] + fn a_pair_in_the_window_gets_one_edge_stored_the_same_way_round_either_way() { + // "Opened second" is whichever node `record_coactivation` is called + // for. The row must not encode that: `UNIQUE(src, dst, rel)` is + // directional, so an orientation that followed call order would let + // one pair exist as two rows the cap and the eviction see as + // unrelated. + for lower_opened_first in [true, false] { + let conn = db::open_in_memory().unwrap(); + let lower = memory(&conn, "note"); + let higher = memory(&conn, "note"); + assert!(lower.id < higher.id, "insertion order fixes the rowids"); + let (earlier, later) = if lower_opened_first { + (lower.id, higher.id) + } else { + (higher.id, lower.id) + }; + opened_at(&conn, earlier, T - 60); + let fired = record_coactivation(&conn, later, T).unwrap(); + assert_eq!(fired, 1, "one partner in the window"); + + let edges = co_edges(&conn); + assert_eq!(edges.len(), 1, "exactly one edge: {edges:?}"); + let (src, dst, weight, last_fired_at) = edges[0]; + assert_eq!( + (src, dst), + (lower.id, higher.id), + "src must be the lower rowid whichever end was opened second: {edges:?}" + ); + assert!( + (weight - COACTIVATION_RATE).abs() < 1e-9, + "a new pair starts at the learning rate, got {weight}" + ); + assert_eq!(last_fired_at, T, "decay is measured from this stamp"); + } + } + + #[test] + fn opens_further_apart_than_the_window_never_pair() { + let conn = db::open_in_memory().unwrap(); + let a = memory(&conn, "note"); + let b = memory(&conn, "note"); + opened_at(&conn, a.id, T - COACTIVATION_WINDOW_SECS - 1); + let fired = record_coactivation(&conn, b.id, T).unwrap(); + assert_eq!( + fired, 0, + "one second past the window is a different session" + ); + assert!(co_edges(&conn).is_empty(), "{:?}", co_edges(&conn)); + + // The other side of the same second. Without this the assertion + // above would also pass for a query that never matches anything. + let conn = db::open_in_memory().unwrap(); + let a = memory(&conn, "note"); + let b = memory(&conn, "note"); + opened_at(&conn, a.id, T - COACTIVATION_WINDOW_SECS); + let fired = record_coactivation(&conn, b.id, T).unwrap(); + assert_eq!(fired, 1, "the window edge itself still counts"); + } + + #[test] + fn a_pair_fires_once_a_day_and_compounds_on_the_decayed_weight() { + let conn = db::open_in_memory().unwrap(); + let a = memory(&conn, "note"); + let b = memory(&conn, "note"); + opened_at(&conn, a.id, T - 60); + assert_eq!(record_coactivation(&conn, b.id, T).unwrap(), 1); + + // Same day, same partner still inside the window: the pair is + // capped, so nothing about the row may move — not the weight and + // not the stamp the decay is measured from. + let again = record_coactivation(&conn, b.id, T + 60).unwrap(); + assert_eq!(again, 0, "second fire of the day is a no-op"); + let (_, _, weight, last_fired_at) = co_edges(&conn)[0]; + assert!( + (weight - COACTIVATION_RATE).abs() < 1e-9, + "weight must not move on a capped fire, got {weight}" + ); + assert_eq!(last_fired_at, T, "nor may the stamp be refreshed"); + + // Exactly one day later. Decay first, then the Hebbian step on the + // decayed value: + // decayed = 0.15 · 2^(-1/30) = 0.14657399526513687 + // next = decayed + 0.15(1-decayed) = 0.27458789597536637 + let tomorrow = T + 86_400; + opened_at(&conn, a.id, tomorrow - 60); + assert_eq!(record_coactivation(&conn, b.id, tomorrow).unwrap(), 1); + let (_, _, weight, last_fired_at) = co_edges(&conn)[0]; + assert!( + (weight - 0.274_587_895_975_366_37).abs() < 1e-12, + "a day-old 0.15 pair must land on 0.2745878959753664, got {weight}" + ); + assert_eq!(last_fired_at, tomorrow); + assert_eq!(co_edges(&conn).len(), 1, "still one row, not a second one"); + } + + #[test] + fn an_authored_edge_suppresses_the_pair_in_either_orientation() { + // `link` writes as given and `supersede` writes new→old, so the + // authored edge can point either way; both must silence the pair. + for authored_low_to_high in [true, false] { + let conn = db::open_in_memory().unwrap(); + let a = memory(&conn, "note"); + let b = memory(&conn, "note"); + let (src, dst) = if authored_low_to_high { + (a.id, b.id) + } else { + (b.id, a.id) + }; + store::link(&conn, src, dst, Rel::Relates, 1.0).unwrap(); + opened_at(&conn, a.id, T - 60); + + let fired = record_coactivation(&conn, b.id, T).unwrap(); + assert_eq!(fired, 0, "authored {src}->{dst} must suppress the pair"); + assert!( + co_edges(&conn).is_empty(), + "no co_recalled row may be added beside an authored one: {:?}", + co_edges(&conn) + ); + } + } + + #[test] + fn only_live_memories_pair() { + let conn = db::open_in_memory().unwrap(); + let live = memory(&conn, "note"); + let symbol = store::insert_node(&conn, NewNode::new(Kind::Symbol)).unwrap(); + let deleted = memory(&conn, "note"); + store::soft_delete(&conn, deleted.id, Actor::Human, None).unwrap(); + let superseded = memory(&conn, "note"); + retire(&conn, superseded.id, live.id); + + for id in [symbol.id, deleted.id, superseded.id] { + opened_at(&conn, id, T - 60); + } + let fired = record_coactivation(&conn, live.id, T).unwrap(); + assert_eq!( + fired, 0, + "a symbol, a forgotten memory and a retired one are not partners" + ); + + // And the same rule read from the other end: the node being opened + // is the one that isn't a live memory. + opened_at(&conn, live.id, T - 60); + let fired = record_coactivation(&conn, symbol.id, T).unwrap(); + assert_eq!(fired, 0, "a symbol never co-activates with a memory"); + assert!(co_edges(&conn).is_empty(), "{:?}", co_edges(&conn)); + } + + #[test] + fn partners_are_strongest_first_above_the_threshold_and_drop_dead_ends() { + let conn = db::open_in_memory().unwrap(); + let hub = memory(&conn, "note"); + let strong = memory(&conn, "note"); + let mid = memory(&conn, "note"); + let exact = memory(&conn, "note"); + let weak = memory(&conn, "note"); + let faded = memory(&conn, "note"); + let deleted = memory(&conn, "note"); + let superseded = memory(&conn, "note"); + + seed_co_edge(&conn, hub.id, mid.id, 0.6, T); + seed_co_edge(&conn, hub.id, strong.id, 0.9, T); + seed_co_edge(&conn, hub.id, exact.id, COACTIVATION_VISIBLE, T); + seed_co_edge(&conn, hub.id, weak.id, COACTIVATION_VISIBLE - 0.01, T); + // Two half-lives of silence takes a once-strong pair to 0.225. + seed_co_edge(&conn, hub.id, faded.id, 0.9, T - 60 * 86_400); + seed_co_edge(&conn, hub.id, deleted.id, 1.0, T); + seed_co_edge(&conn, hub.id, superseded.id, 1.0, T); + store::soft_delete(&conn, deleted.id, Actor::Human, None).unwrap(); + retire(&conn, superseded.id, hub.id); + + let got = coactivation_partners(&conn, hub.id, T).unwrap(); + let ids: Vec = got.iter().map(|(id, _)| *id).collect(); + assert_eq!( + ids, + vec![strong.id, mid.id, exact.id], + "strongest first, threshold inclusive, soft-deleted and retired ends gone: {got:?}" + ); + assert!( + (got[0].1 - 0.9).abs() < 1e-9, + "the effective weight is returned, not the stored one: {got:?}" + ); + } + + #[test] + fn the_per_node_cap_evicts_the_weakest_by_decayed_weight() { + let conn = db::open_in_memory().unwrap(); + let hub = memory(&conn, "note"); + + // One short of the cap, all fired in the same session. + let fresh: Vec = (0..COACTIVATION_MAX_PER_NODE - 1) + .map(|_| memory(&conn, "note").id) + .collect(); + for id in &fresh { + opened_at(&conn, *id, T - 60); + } + let fired = record_coactivation(&conn, hub.id, T).unwrap(); + assert_eq!(fired, COACTIVATION_MAX_PER_NODE - 1, "all pair at once"); + + // A pair with a big stored weight that has not fired in eight + // years: 0.9 · 2^(-100) ≈ 7e-31, far below a fresh 0.15. Created + // last on purpose, so it holds the HIGHEST edge id — if eviction + // ranked by the stored column, or fell through to the id + // tiebreaker, this row would be the one that survived. + let stale = memory(&conn, "note"); + seed_co_edge(&conn, hub.id, stale.id, 0.9, T - 3000 * 86_400); + assert_eq!( + co_partner_ids(&conn, hub.id).len(), + COACTIVATION_MAX_PER_NODE + ); + + // One more fresh pair takes the node one over the cap. + let newcomer = memory(&conn, "note"); + opened_at(&conn, newcomer.id, T - 60); + let fired = record_coactivation(&conn, hub.id, T).unwrap(); + assert_eq!(fired, 1, "the ones already fired today must not fire again"); + + let kept = co_partner_ids(&conn, hub.id); + assert_eq!(kept.len(), COACTIVATION_MAX_PER_NODE, "cap holds: {kept:?}"); + assert!( + !kept.contains(&stale.id), + "the strong-but-stale pair is the weakest once decayed: {kept:?}" + ); + assert!( + kept.contains(&newcomer.id), + "the pair that just fired must not be the one evicted: {kept:?}" + ); + for id in &fresh { + assert!( + kept.contains(id), + "fresh pair {id} evicted instead: {kept:?}" + ); + } + } } diff --git a/crates/mimir-core/src/model.rs b/crates/mimir-core/src/model.rs index 9ec6cbc..9356ccd 100644 --- a/crates/mimir-core/src/model.rs +++ b/crates/mimir-core/src/model.rs @@ -126,6 +126,11 @@ str_enum!(MutationOp { Review => "review", }); +// Edge relations. All are authored — by a person, by the code-graph +// extractor, or by an importer — except `CoRecalled`, which the store +// observes for itself (see `learn::record_coactivation`). That one is +// derived, per-machine and regenerable, so it is excluded from replication +// and from export rather than shipped as if it were a claim anyone made. str_enum!(Rel { Links => "links", Mentions => "mentions", @@ -140,6 +145,7 @@ str_enum!(Rel { Imports => "imports", Defines => "defines", MemberOf => "member_of", + CoRecalled => "co_recalled", }); /// Where a query or capture applies. diff --git a/crates/mimir-core/src/replicate.rs b/crates/mimir-core/src/replicate.rs index 0aaf62d..f952aba 100644 --- a/crates/mimir-core/src/replicate.rs +++ b/crates/mimir-core/src/replicate.rs @@ -12,7 +12,9 @@ //! one second resolve by arrival order (one is silently kept, the other //! dropped). Tombstones ride along as `deleted_at`. Local-only signals //! (strength, access_count, recall_event) and embeddings are never synced; a -//! peer recomputes embeddings locally. +//! peer recomputes embeddings locally. `co_recalled` edges are local-only for +//! the same reason and are filtered in BOTH directions — see `changes_since` +//! and the edge loop in `apply_changes`. use rusqlite::{params, Connection, OptionalExtension, Row, TransactionBehavior}; use serde::{Deserialize, Serialize}; @@ -86,8 +88,9 @@ pub fn project_is_syncable(meta: &serde_json::Value) -> bool { } /// "Changed since `since`" = syncable memories whose latest activity -/// (max of updated_at and deleted_at) is newer than `since`, plus all edges -/// among synced memories (sparse; shipped whole and upserted idempotently). +/// (max of updated_at and deleted_at) is newer than `since`, plus all +/// *authored* edges among synced memories (sparse; shipped whole and upserted +/// idempotently). pub fn changes_since(conn: &Connection, since: i64) -> Result { let mut stmt = conn.prepare(&format!( "SELECT m.uid, m.subkind, m.title, m.body, m.tags_text, m.pinned, m.content_hash, m.meta, @@ -124,6 +127,15 @@ pub fn changes_since(conn: &Connection, since: i64) -> Result { // Edges between two syncable memories (each endpoint global, or in a // sync-enabled keyed project). + // + // `co_recalled` is excluded because it is not a claim anyone authored: it + // is what THIS install observed about its own opens, per-machine and + // regenerable from local `recall_event` rows. Shipping it would forge that + // history on the peer — and its Hebbian weight and `meta.last_fired_at` + // are meaningless there, since the wire carries neither the meta nor the + // peer's own decay clock. Filtered here in SQL rather than in the Rust + // map, so the rel never reaches the batch at all (and `snapshot`, which is + // the file transport's whole payload, can't leak it either). let mut estmt = conn.prepare( "SELECT s.uid, d.uid, e.rel, e.weight, e.created_at FROM edge e @@ -132,6 +144,7 @@ pub fn changes_since(conn: &Connection, since: i64) -> Result { LEFT JOIN node sp ON sp.id = s.project_id AND sp.kind='project' LEFT JOIN node dp ON dp.id = d.project_id AND dp.kind='project' WHERE s.kind='memory' AND d.kind='memory' + AND e.rel <> 'co_recalled' AND (s.project_id IS NULL OR (json_extract(sp.meta,'$.sync')=1 AND json_extract(sp.meta,'$.portable_key') IS NOT NULL)) AND (d.project_id IS NULL OR (json_extract(dp.meta,'$.sync')=1 @@ -269,6 +282,16 @@ pub fn apply_changes(conn: &Connection, batch: &SyncBatch) -> Result let src = store::get_node_by_uid(&tx, &e.src_uid).ok(); let dst = store::get_node_by_uid(&tx, &e.dst_uid).ok(); match (src, dst, e.rel.parse::()) { + // A co-activation edge is an observation the receiving store makes + // about its OWN opens, so an inbound one is a claim about a history + // that never happened here. Rejected at the door rather than left + // to the sender's filter: a peer on an older or unfiltered build + // still ships them, and `store::link` would write the peer's weight + // with no `meta.last_fired_at` — which reads back as never-decayed + // and could push a fabricated partner over the visibility floor. + // Counted as skipped, like a dangling endpoint: it is a row this + // store declined, not an error that should fail the batch. + (_, _, Ok(Rel::CoRecalled)) => stats.edges_skipped += 1, (Some(s), Some(d), Ok(rel)) => { store::link(&tx, s.id, d.id, rel, e.weight)?; stats.edges_upserted += 1; @@ -614,6 +637,72 @@ mod tests { assert_eq!(s.edges_upserted, 1); } + #[test] + fn coactivation_edge_stays_local() { + let a = db::open_in_memory().unwrap(); + let x = mem(&a, "node x"); + let y = mem(&a, "node y"); + // Control: an authored edge between a DIFFERENT pair, which must still + // ship. Without it, a bug that dropped every edge would pass this test. + // It has to be another pair — `record_coactivation` never fires for a + // pair that already has an authored edge, so the two cannot coexist. + let p = mem(&a, "node p"); + let q = mem(&a, "node q"); + store::link(&a, p.id, q.id, Rel::Relates, 1.0).unwrap(); + + // Earn the co_recalled edge the way the store does — two opens inside + // the window — rather than hand-writing the row, so this test also + // fails if the real writer ever starts producing something else. + crate::learn::record_opened(&a, x.id).unwrap(); + crate::learn::record_opened(&a, y.id).unwrap(); + let local: i64 = a + .query_row( + "SELECT count(*) FROM edge WHERE rel='co_recalled'", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(local, 1, "co-activation edge exists locally to begin with"); + + let batch = snapshot(&a).unwrap(); + let rels: Vec<&str> = batch.edges.iter().map(|e| e.rel.as_str()).collect(); + assert!( + !rels.contains(&"co_recalled"), + "derived edge must not ship: {rels:?}" + ); + assert_eq!(rels, vec!["relates"], "authored edge still ships: {rels:?}"); + } + + #[test] + fn inbound_coactivation_edge_is_rejected() { + let a = db::open_in_memory().unwrap(); + let x = mem(&a, "node x"); + let y = mem(&a, "node y"); + let mut batch = snapshot(&a).unwrap(); + // A peer running an unfiltered (older or newer) build: its nodes are + // fine, its co-activation edge is not ours to hold. + batch.edges.push(SyncEdge { + src_uid: x.uid.clone(), + dst_uid: y.uid.clone(), + rel: "co_recalled".into(), + weight: 0.9, + created_at: 1, + }); + + let b = db::open_in_memory().unwrap(); + let s = apply_changes(&b, &batch).unwrap(); + assert_eq!(s.edges_upserted, 0, "{s:?}"); + assert_eq!(s.edges_skipped, 1, "{s:?}"); + let landed: i64 = b + .query_row( + "SELECT count(*) FROM edge WHERE rel='co_recalled'", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(landed, 0, "peer's co-activation edge must not land"); + } + #[test] fn soft_delete_bumps_updated_at() { let conn = db::open_in_memory().unwrap(); diff --git a/crates/mimir-core/src/store.rs b/crates/mimir-core/src/store.rs index 3d540ea..485c4dd 100644 --- a/crates/mimir-core/src/store.rs +++ b/crates/mimir-core/src/store.rs @@ -367,6 +367,15 @@ pub fn link(conn: &Connection, src: i64, dst: i64, rel: Rel, weight: f64) -> Res Ok(()) } +/// Every edge touching `node_id`, in either direction. +/// +/// A row whose `rel` this binary does not recognise is SKIPPED, not fatal. +/// The previous behaviour turned an unknown rel into a +/// `FromSqlConversionFailure` that failed the whole query, which meant a +/// store written by a newer build could make `get` explode on an older one — +/// and the two do coexist, because the CLI and the MCP server are installed +/// separately and upgrade at different times. Dropping the row degrades one +/// annotation; failing the query loses the whole record. pub fn edges_of(conn: &Connection, node_id: i64) -> Result> { let mut stmt = conn.prepare( "SELECT id, src, dst, rel, weight, meta, created_at FROM edge @@ -376,24 +385,21 @@ pub fn edges_of(conn: &Connection, node_id: i64) -> Result> { .query_map([node_id], |row| { let rel_text: String = row.get("rel")?; let meta_text: String = row.get("meta")?; - Ok(Edge { + let Ok(rel) = rel_text.parse::() else { + return Ok(None); + }; + Ok(Some(Edge { id: row.get("id")?, src: row.get("src")?, dst: row.get("dst")?, - rel: rel_text.parse().map_err(|e: Error| { - rusqlite::Error::FromSqlConversionFailure( - 0, - rusqlite::types::Type::Text, - Box::new(e), - ) - })?, + rel, weight: row.get("weight")?, meta: serde_json::from_str(&meta_text).unwrap_or(serde_json::Value::Null), created_at: row.get("created_at")?, - }) + })) })? - .collect::>()?; - Ok(edges) + .collect::>>>()?; + Ok(edges.into_iter().flatten().collect()) } /// Find or create the project node for a filesystem root. From 9df1682adb492ab82f83dc1bb882c00e8b81a0e9 Mon Sep 17 00:00:00 2001 From: Thomas <155702229+MakerViking@users.noreply.github.com> Date: Fri, 16 Oct 2026 01:53:00 +0200 Subject: [PATCH 2/3] feat(recall): weak-match labels and the miss ledger MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A ranked list always has a #1, so the fused score cannot tell "the answer" from "the least-bad thing in the store". Each recall hit is now labelled by evidence: `strong` when it contains the query's words AND both the lexical and semantic legs rank it in their own top 10 (or, with no semantic leg, covers every content word; or names a symbol exactly), otherwise `weak` — one word on the compact line (CLI and MCP), `"strength"` in --json. Calibrated against the labelled retrieval corpus with the real model (eval::strength, k=8): strong 23/42 relevant, weak 3/142, 0/6 unanswerable probes passed as answered. Measured and rejected: a middle `likely` tier (0/8 relevant), semantic rank alone as evidence (let 6/6 probes through), half-the-words coverage (relevant-but-weak 3 -> 10), and 5-char prefix stemming (11 more irrelevant strongs, 0 relevant). `mimir misses`: a recall page with no strong hit is recorded — query text, count, last asked — most asked first; a later strong answer to the same question resolves it, and a miss after that reopens it. `mimir report` gains "unanswered queries". Explicit recalls only, never the inject hook. Per-machine (not synced, not exported), never stores a query the secrets scanner flags, ages out with event_retention_days, `[learn] log_misses = false` to disable. Schema v13: recall_miss. --- CHANGELOG.md | 29 +++ README.md | 33 +++ crates/mimir-cli/src/commands.rs | 121 ++++++++- crates/mimir-cli/src/main.rs | 23 ++ crates/mimir-cli/src/mcp.rs | 95 +++++++- crates/mimir-cli/src/report.rs | 16 ++ crates/mimir-core/src/config.rs | 11 + crates/mimir-core/src/db/migrations.rs | 47 ++++ crates/mimir-core/src/db/mod.rs | 5 + crates/mimir-core/src/eval/mod.rs | 1 + crates/mimir-core/src/eval/strength.rs | 263 ++++++++++++++++++++ crates/mimir-core/src/format.rs | 48 +++- crates/mimir-core/src/lib.rs | 23 +- crates/mimir-core/src/misses.rs | 324 +++++++++++++++++++++++++ crates/mimir-core/src/strength.rs | 304 +++++++++++++++++++++++ 15 files changed, 1320 insertions(+), 23 deletions(-) create mode 100644 crates/mimir-core/src/eval/strength.rs create mode 100644 crates/mimir-core/src/misses.rs create mode 100644 crates/mimir-core/src/strength.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b81446..73753e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,35 @@ All notable changes are documented here. Versions follow semver; the CLI, the `mimir-mem` crate, and the on-disk schema move together. +## [Unreleased] + +### Added + +- **Recall says when a hit is weak.** A ranked list always has a #1, so the + score cannot tell "the answer" from "the least-bad thing in the store". + Each hit is now labelled by evidence instead: `strong` when it contains the + query's words and both the lexical and semantic legs rank it in their own + top 10, otherwise `weak` — one word on the compact line (CLI and MCP), + `"strength"` in `--json`. Calibrated against the labelled retrieval corpus + with the real model: weak hits are relevant ~2% of the time, strong ~55%. + A three-tier version (with `likely`) and two stricter matching rules were + measured and rejected; the numbers are in `strength.rs`, and + `eval::strength` gates the rule in `cargo test`. +- **`mimir misses` — what agents asked that the store couldn't answer.** A + recall with no strong hit is recorded (query text, count, last asked), most + asked first; the row resolves itself when the same question later gets a + strong answer, and reopens if that answer goes away. `mimir report` gains an + "unanswered queries" row. Explicit recalls only — the inject hook never + feeds it. Per-machine (not synced or exported), never stores a query the + secrets scanner flags, ages out with `[learn] event_retention_days`, and + `[learn] log_misses = false` turns it off. + +### Changed + +- **Schema v13** adds the `recall_miss` table. As with every schema bump, a + store opened by this binary is refused by older ones (`SchemaTooNew`): + reinstall, then restart running `mimir mcp` servers. + ## [0.17.1] - 2026-10-15 ### Fixed diff --git a/README.md b/README.md index 38fcca1..3bf8bd8 100644 --- a/README.md +++ b/README.md @@ -271,6 +271,39 @@ mimir link --scan --all-projects # ~600 links on a mature store `mimir doctor` reminds you if you have code graphs and have never scanned, and goes quiet once you have. +### Weak matches and the miss ledger: what the store couldn't answer + +Every ranked list has a #1, including the list for a question the store knows +nothing about. So recall labels each hit by the *evidence* behind it rather +than its score: a hit is `strong` when it contains the query's words **and** +both the lexical and the semantic search rank it in their own top 10. +Anything else gets one word on its line: + +``` +m:46MBBK [decision 10-15 weak] Release process: bump versions, update CHANGELOG… +``` + +Measured on the labelled retrieval corpus with the real embedding model, a +`weak` hit is relevant about 2% of the time and a `strong` one about 55% +(`eval::strength`; numbers and the two rules tried and rejected are in +`crates/mimir-core/src/strength.rs`). `--json` carries `"strength"` on every +hit. + +A recall whose page has no `strong` hit is a **miss**, and misses are kept: + +```sh +mimir misses # most-asked unanswered queries, this project + global +mimir misses --all # every project +mimir misses --dismiss 12 # it was noise, not a gap +``` + +That list is what to write down next. A row resolves itself the next time the +same question gets a strong answer, and `mimir report` counts what's open. +Only explicit recalls (CLI and MCP) feed it — never the per-prompt inject +hook. The query text stays on this machine (not synced, not exported), a query +the secrets scanner flags is never written, and rows age out with +`[learn] event_retention_days`. Turn it off with `[learn] log_misses = false`. + ### The fast path (recommended setup) Three commands get you the lowest-latency configuration Mimir has — measured diff --git a/crates/mimir-cli/src/commands.rs b/crates/mimir-cli/src/commands.rs index c35c559..0c18f6f 100644 --- a/crates/mimir-cli/src/commands.rs +++ b/crates/mimir-cli/src/commands.rs @@ -1201,9 +1201,27 @@ pub fn recall( as_of, text, }; - let mut hits = mimir.search_with(&query, rerank)?; + let (mut hits, legs) = mimir.search_with_evidence(&query, rerank)?; + let mut strengths = mimir_core::strength::classify_page(&hits, &query.text, &legs); + // Judged on the page as retrieved, before `--min-score` trims it: a + // threshold the caller chose is not evidence the store lacks an answer. + // A time-sliced view (`--since`, `--as-of`) is not a question to the + // store as it stands, so it never feeds the ledger either. + if mimir.config.learn.log_misses && query.since.is_none() && as_of.is_none() { + mimir_core::misses::observe( + &mimir.conn, + &query.text, + mimir_core::misses::scope_project(&query.scope), + &strengths, + mimir_core::model::now_unix(), + ); + } if let Some(min) = min_score { - hits.retain(|hit| hit.score >= min); + (hits, strengths) = hits + .into_iter() + .zip(strengths) + .filter(|(hit, _)| hit.score >= min) + .unzip(); } let query_hash = blake3::hash(query.text.as_bytes()); @@ -1247,10 +1265,11 @@ pub fn recall( &hits.iter().map(|h| h.node.id).collect::>(), ) .unwrap_or_default(); - for hit in &hits { + for (hit, strength) in hits.iter().zip(&strengths) { if json { let mut value = node_json(&hit.node, &projects); value["score"] = serde_json::json!(hit.score); + value["strength"] = serde_json::json!(strength.as_str()); println!("{value}"); } else if full { print_full(&hit.node, &mimir, &projects)?; @@ -1263,7 +1282,10 @@ pub fn recall( &projects, mimir.config.output.snippet_chars, Some(&query.text), - stale.contains(&hit.node.id) + mimir_core::format::HitMarks { + stale_grounding: stale.contains(&hit.node.id), + weak_match: *strength == mimir_core::strength::MatchStrength::Weak, + } ) ); } @@ -1521,6 +1543,85 @@ pub fn grounding(stale_only: bool, limit: usize) -> Result<()> { Ok(()) } +/// The miss ledger: recall queries that came back with no strong match. +pub fn misses( + dismiss: Option, + all: bool, + resolved: bool, + limit: usize, + json: bool, +) -> Result<()> { + let mimir = Mimir::open()?; + if let Some(id) = dismiss { + if !mimir_core::misses::dismiss(&mimir.conn, id)? { + return Err(anyhow!("no miss #{id} (see `mimir misses`)")); + } + println!("dismissed miss #{id}"); + return Ok(()); + } + let project = if all { + None + } else { + mimir_core::misses::scope_project(&read_scope(&mimir, false, false)?) + }; + let rows = mimir_core::misses::list(&mimir.conn, project, resolved, limit)?; + let projects = store::project_titles(&mimir.conn)?; + if json { + for m in &rows { + println!( + "{}", + serde_json::json!({ + "id": m.id, + "query": m.query, + "project": m.project_id.and_then(|id| projects.get(&id)), + "hits": m.hits, + "count": m.count, + "first_at": m.first_at, + "last_at": m.last_at, + "resolved_at": m.resolved_at, + }) + ); + } + return Ok(()); + } + if rows.is_empty() { + println!("no misses — every recall logged here found a strong match"); + if !mimir.config.learn.log_misses { + println!("(logging is off: `[learn] log_misses = false`)"); + } + return Ok(()); + } + println!( + "{:>5} {:>5} {:<6} {:<12} query", + "id", "asked", "last", "project" + ); + for m in &rows { + let project = m + .project_id + .and_then(|id| projects.get(&id)) + .map_or("-", String::as_str); + let state = match (m.resolved_at, m.hits) { + (Some(_), _) => " (answered since)", + (None, 0) => " (nothing came back)", + (None, _) => "", + }; + println!( + "{:>5} {:>5} {:<6} {:<12} {}{state}", + m.id, + m.count, + mimir_core::format::month_day(m.last_at), + project, + m.query + ); + } + println!( + "\nEach of these got no strong match: nothing that both shares the query's \ + words and ranks near the top of both search legs.\nWrite the memory that \ + answers it (`mimir remember`), or `mimir misses --dismiss ` if it was noise." + ); + Ok(()) +} + pub fn refusals(limit: usize) -> Result<()> { let mimir = Mimir::open()?; let rows = mimir_core::secrets::refusals(&mimir.conn, limit)?; @@ -2436,7 +2537,13 @@ fn line_q( snippet_chars: usize, query: Option<&str>, ) -> String { - line_with_grounding(node, projects, snippet_chars, query, false) + line_with_grounding( + node, + projects, + snippet_chars, + query, + mimir_core::format::HitMarks::default(), + ) } /// [`line_for_query`] plus the stale-grounding marker. Separate so the many @@ -2449,13 +2556,13 @@ fn line_with_grounding( projects: &HashMap, snippet_chars: usize, query: Option<&str>, - stale: bool, + marks: mimir_core::format::HitMarks, ) -> String { let project = node .project_id .and_then(|id| projects.get(&id)) .map(String::as_str); - mimir_core::format::agent_line_for_query(node, project, snippet_chars, query, stale) + mimir_core::format::agent_line_for_query(node, project, snippet_chars, query, marks) } fn print_full(node: &Node, mimir: &Mimir, projects: &HashMap) -> Result<()> { diff --git a/crates/mimir-cli/src/main.rs b/crates/mimir-cli/src/main.rs index 4ce875f..022ae34 100644 --- a/crates/mimir-cli/src/main.rs +++ b/crates/mimir-cli/src/main.rs @@ -373,6 +373,23 @@ enum Command { #[arg(long, default_value_t = 20)] limit: usize, }, + /// Recall queries the store had no answer for: what agents keep asking + /// that nobody has written down. Most-asked first. A row resolves itself + /// when a later recall of the same question finds a strong match. + Misses { + /// Remove this row (from the list): noise, not a gap. + #[arg(long)] + dismiss: Option, + /// Every project's misses, not just this one's and the global ones. + #[arg(long)] + all: bool, + /// Include questions a later recall answered. + #[arg(long)] + resolved: bool, + /// How many to list. + #[arg(long, default_value_t = 20)] + limit: usize, + }, /// What the secret guard turned away — fingerprints and counts, never /// the refused values. Refusals { @@ -973,6 +990,12 @@ fn run(cli: Cli) -> anyhow::Result<()> { commands::mark(&reference, useful) } Command::Consolidate { dry_run } => commands::consolidate(dry_run), + Command::Misses { + dismiss, + all, + resolved, + limit, + } => commands::misses(dismiss, all, resolved, limit, cli.json), Command::Refusals { limit } => commands::refusals(limit), Command::Audit { reference, limit } => { commands::audit(reference.as_deref(), limit, cli.json) diff --git a/crates/mimir-cli/src/mcp.rs b/crates/mimir-cli/src/mcp.rs index 6a85e38..dedf86a 100644 --- a/crates/mimir-cli/src/mcp.rs +++ b/crates/mimir-cli/src/mcp.rs @@ -8,10 +8,11 @@ use std::sync::{Arc, Mutex}; use anyhow::Result; -use mimir_core::format::{agent_line, agent_line_for_query}; +use mimir_core::format::{agent_line, agent_line_for_query, HitMarks}; use mimir_core::memory::{self, RememberOutcome}; use mimir_core::model::{short_uid, Kind, MemoryType, Rel, Scope}; use mimir_core::search::SearchQuery; +use mimir_core::strength::MatchStrength; use mimir_core::{store, Mimir}; use rmcp::handler::server::router::tool::ToolRouter; use rmcp::handler::server::wrapper::Parameters; @@ -240,7 +241,7 @@ impl MimirServer { } #[tool( - description = "Search memories, docs and code in one ranked list (hybrid BM25+semantic). ~25-token line per hit." + description = "Search memories, docs and code in one ranked list (hybrid BM25+semantic). ~25-token line per hit; `weak` = little evidence it matches the query." )] async fn recall(&self, Parameters(args): Parameters) -> String { let project_id = self.project_id; @@ -280,7 +281,21 @@ impl MimirServer { include_superseded: args.include_superseded, as_of, }; - let hits = m.search_with(&query, args.rerank).map_err(engine_err)?; + let (hits, legs) = m + .search_with_evidence(&query, args.rerank) + .map_err(engine_err)?; + let strengths = mimir_core::strength::classify_page(&hits, &query.text, &legs); + // Same rule as the CLI: a time-sliced view is not a question to + // the store as it stands, so `as_of` never feeds the ledger. + if m.config.learn.log_misses && query.as_of.is_none() { + mimir_core::misses::observe( + &m.conn, + &query.text, + mimir_core::misses::scope_project(&query.scope), + &strengths, + mimir_core::model::now_unix(), + ); + } if hits.is_empty() { return Ok("no results".into()); } @@ -295,7 +310,7 @@ impl MimirServer { let ids: Vec = hits.iter().map(|h| h.node.id).collect(); let stale = mimir_core::grounding::stale_ids(&m.conn, &ids).unwrap_or_default(); let mut out = Vec::new(); - for hit in &hits { + for (hit, strength) in hits.iter().zip(&strengths) { let project = hit .node .project_id @@ -306,7 +321,10 @@ impl MimirServer { project, m.config.output.snippet_chars, Some(&query.text), - stale.contains(&hit.node.id), + HitMarks { + stale_grounding: stale.contains(&hit.node.id), + weak_match: *strength == MatchStrength::Weak, + }, )); } Ok(out.join("\n")) @@ -1908,6 +1926,73 @@ mod tests { assert!(body.contains("SCRAM"), "get: {body}"); } + /// The agent surface carries both new signals: a hit with no evidence + /// behind it says `weak` on its line, and a page with nothing better + /// lands in the miss ledger — while a real answer to the same question + /// later resolves it. + #[tokio::test(flavor = "multi_thread")] + async fn recall_marks_weak_hits_and_feeds_the_miss_ledger() { + let (_dir, s) = server(); + s.remember(Parameters(remember_args( + "SCRAM auth rejects non-ASCII passwords", + "gotcha", + ))) + .await; + let recall = |query: &str| { + s.recall(Parameters(RecallArgs { + query: query.into(), + kind: None, + limit: None, + all_projects: true, + rerank: false, + include_superseded: false, + as_of: None, + })) + }; + let open_misses = || { + let engine = s.engine.clone(); + move || { + let m = engine.lock().unwrap(); + mimir_core::misses::list(&m.conn, None, false, 10) + .unwrap() + .into_iter() + .map(|miss| miss.query) + .collect::>() + } + }; + + // No embedder in tests, so this is the lexical-only path. Nothing + // shares a word: an empty page, which is a miss. + assert_eq!(recall("sourdough hydration ratio").await, "no results"); + // One shared word of three: the hit comes back, marked weak. + let partial = recall("passwords reset flow").await; + assert!( + partial.contains("SCRAM") && partial.contains(" weak]"), + "{partial}" + ); + assert_eq!( + open_misses()(), + vec!["sourdough hydration ratio", "passwords reset flow"] + ); + + s.remember(Parameters(remember_args( + "Passwords reset flow sends a magic link, never a temporary password", + "decision", + ))) + .await; + let answered = recall("passwords reset flow").await; + let first = answered.lines().next().unwrap_or_default(); + assert!( + first.contains("magic link") && !first.contains(" weak]"), + "{answered}" + ); + assert_eq!( + open_misses()(), + vec!["sourdough hydration ratio"], + "a strong answer resolves its own miss and no other" + ); + } + /// `link` must refuse `supersedes`. The edge and the `superseded_by` /// column are independent and only the column suppresses recall, so /// accepting the rel here records something that reads as a retirement diff --git a/crates/mimir-cli/src/report.rs b/crates/mimir-cli/src/report.rs index cd67116..5473af1 100644 --- a/crates/mimir-cli/src/report.rs +++ b/crates/mimir-cli/src/report.rs @@ -85,6 +85,16 @@ pub fn report(json: bool) -> Result<()> { FROM recall_event WHERE query_hash IS NOT NULL", )?, ), + // Distinct questions last missed in the period and still open — the + // count a person can act on, not how many times agents retried. + ( + "unanswered queries", + counts( + "SELECT COALESCE(sum(last_at>=?1),0), COALESCE(sum(last_at>=?2),0), + COALESCE(sum(last_at>=?3),0), COALESCE(sum(last_at>=?4),0), count(*) + FROM recall_miss WHERE resolved_at IS NULL", + )?, + ), ("results shown", event_counts("shown")?), ("memories opened", event_counts("opened")?), ("marked useful", event_counts("useful")?), @@ -169,6 +179,12 @@ pub fn report(json: bool) -> Result<()> { if let Some(ts) = last_consolidate { println!("last consolidation {}", date(ts)); } + if rows + .iter() + .any(|(label, vals)| *label == "unanswered queries" && vals[4] > 0) + { + println!("unanswered queries `mimir misses` lists them, most-asked first"); + } Ok(()) } diff --git a/crates/mimir-core/src/config.rs b/crates/mimir-core/src/config.rs index 7c500f5..7fc0f16 100644 --- a/crates/mimir-core/src/config.rs +++ b/crates/mimir-core/src/config.rs @@ -278,12 +278,20 @@ pub struct LearnConfig { /// /// [`effective_retention_days`]: LearnConfig::effective_retention_days pub event_retention_days: u32, + /// Keep the miss ledger (`mimir misses`): the text of explicit recall + /// queries that came back with no `strong` hit, so the gaps agents keep + /// hitting can be written down. On by default — the text stays on this + /// machine (never synced or exported), a query the secrets scanner flags + /// is never written, and rows age out on `event_retention_days`. `false` + /// stops new rows; existing ones remain until pruned or dismissed. + pub log_misses: bool, } impl Default for LearnConfig { fn default() -> Self { LearnConfig { event_retention_days: 180, + log_misses: true, } } } @@ -653,6 +661,7 @@ mod learn_config_tests { fn zero_means_never_prune() { let cfg = LearnConfig { event_retention_days: 0, + ..LearnConfig::default() }; assert_eq!(cfg.effective_retention_days(), None); } @@ -666,6 +675,7 @@ mod learn_config_tests { fn below_floor_is_clamped_up_to_the_floor() { let cfg = LearnConfig { event_retention_days: 7, + ..LearnConfig::default() }; assert_eq!(cfg.effective_retention_days(), Some(60)); } @@ -674,6 +684,7 @@ mod learn_config_tests { fn exactly_at_the_floor_is_not_clamped_further() { let cfg = LearnConfig { event_retention_days: 60, + ..LearnConfig::default() }; assert_eq!(cfg.effective_retention_days(), Some(60)); } diff --git a/crates/mimir-core/src/db/migrations.rs b/crates/mimir-core/src/db/migrations.rs index 090e450..0fb09f6 100644 --- a/crates/mimir-core/src/db/migrations.rs +++ b/crates/mimir-core/src/db/migrations.rs @@ -316,6 +316,38 @@ CREATE TABLE review ( ); CREATE UNIQUE INDEX review_finding ON review(kind, node_id, COALESCE(other_id, -1)); CREATE INDEX review_open ON review(at) WHERE disposition IS NULL; +"#, + // Miss ledger: what agents asked for that the store could not answer. + // + // `recall_event` keys on node_id, so a recall that found nothing wrote + // nothing — the store could say which memories were useful and never + // which ones were missing, and the second is the list worth acting on. + // A miss is a page with no `strong` hit (`strength::is_miss`). + // + // Unlike every other ledger here this keeps the query TEXT, because a + // hash of a question nobody can read back answers nothing. Two guards + // make that safe to hold: a query the secrets scanner flags is never + // written (`misses::observe`), and the table is per-machine — not + // replicated, not exported — like `recall_event` itself. + // + // One row per (normalised query, project): asking again bumps `count`, + // which is the ranking signal; a later `strong` answer to the same query + // stamps `resolved_at`, so the list empties itself as memories get + // written. + r#" +CREATE TABLE recall_miss ( + id INTEGER PRIMARY KEY, + query TEXT NOT NULL, + query_key BLOB NOT NULL, + project_id INTEGER, + hits INTEGER NOT NULL, + count INTEGER NOT NULL DEFAULT 1, + first_at INTEGER NOT NULL, + last_at INTEGER NOT NULL, + resolved_at INTEGER +); +CREATE UNIQUE INDEX recall_miss_key ON recall_miss(query_key, COALESCE(project_id, -1)); +CREATE INDEX recall_miss_open ON recall_miss(last_at) WHERE resolved_at IS NULL; "#, ]; @@ -470,6 +502,21 @@ mod tests { 1, "upgraded store must be able to write the mutation ledger" ); + // And the miss ledger (v13), whose upsert targets an expression + // index — the shape most likely to work on a fresh store and fail on + // an upgraded one. + crate::misses::observe( + &conn, + "a question from after the upgrade", + None, + &[crate::strength::MatchStrength::Weak], + 1_700_000_000, + ); + assert_eq!( + crate::misses::counts(&conn, 0).unwrap(), + (1, 0), + "upgraded store must be able to write the miss ledger" + ); } #[test] diff --git a/crates/mimir-core/src/db/mod.rs b/crates/mimir-core/src/db/mod.rs index 91b10d9..7085064 100644 --- a/crates/mimir-core/src/db/mod.rs +++ b/crates/mimir-core/src/db/mod.rs @@ -127,6 +127,11 @@ fn prune_events_once(conn: &Connection, days: u32) { Ok(_) => {} Err(err) => tracing::warn!(%err, "recall_event prune failed"), } + match crate::misses::prune(conn, cutoff) { + Ok(n) if n > 0 => tracing::info!(rows = n, days, "pruned old recall_miss rows"), + Ok(_) => {} + Err(err) => tracing::warn!(%err, "recall_miss prune failed"), + } } /// One `injection_log`/`session_state` prune pass, `SESSION_STATE_RETENTION_DAYS` diff --git a/crates/mimir-core/src/eval/mod.rs b/crates/mimir-core/src/eval/mod.rs index d6d4ffe..f96fb9a 100644 --- a/crates/mimir-core/src/eval/mod.rs +++ b/crates/mimir-core/src/eval/mod.rs @@ -28,6 +28,7 @@ pub mod brief; pub mod fixtures; pub mod forget; +pub mod strength; use std::collections::HashSet; diff --git a/crates/mimir-core/src/eval/strength.rs b/crates/mimir-core/src/eval/strength.rs new file mode 100644 index 0000000..ce20385 --- /dev/null +++ b/crates/mimir-core/src/eval/strength.rs @@ -0,0 +1,263 @@ +//! Calibration of the recall match-strength label (`strength::classify`) +//! against the labelled retrieval corpus. +//! +//! A label is only worth its token if it means something, so this asks the +//! two questions an agent relies on it for: +//! +//! 1. **Does `sure` mean relevant more often than `weak` does?** Every top-k +//! hit of every [`fixtures::Question`] is labelled and checked against +//! the question's relevant set, giving P(relevant | label). +//! 2. **Does a miss mean the store had nothing?** The answerable questions +//! should almost never come back as a miss, and the [`UNANSWERABLE`] +//! probes — questions the corpus says nothing about — always should. +//! +//! Hermetic mode runs in plain `cargo test` and gates the ordering; the +//! real-model numbers are the honest ones and are printed by the ignored +//! test under `--features eval`. + +use std::collections::HashSet; + +use rusqlite::Connection; + +use super::fixtures; +use crate::error::Result; +use crate::model::Scope; +use crate::search::{self, SearchQuery}; +use crate::strength::{self, MatchStrength}; + +/// Questions nothing in the corpus answers. Each has its own topic anchor so +/// the hermetic query vector lands near no fixture's cluster — the semantic +/// leg still returns a top-k (it always does), which is exactly the case the +/// label must not mistake for evidence. +pub const UNANSWERABLE: &[(&str, &str)] = &[ + ( + "how do I configure TLS certificates for a kubernetes ingress", + "probe_k8s", + ), + ( + "what hydration ratio works best for sourdough bread", + "probe_bread", + ), + ( + "which terraform provider version pins the aws region", + "probe_tf", + ), + ( + "how should I tune the garbage collector in the JVM", + "probe_jvm", + ), + ( + "what is the airspeed velocity of an unladen swallow", + "probe_swallow", + ), + ("how do I center a div with flexbox in css", "probe_css"), +]; + +/// `(relevant, total)` per label, indexed by `MatchStrength as usize`. +#[derive(Debug, Default, Clone)] +pub struct Calibration { + pub by_label: [(usize, usize); 2], + /// Answerable questions whose page came back as a miss. + pub false_misses: Vec<&'static str>, + pub answerable: usize, + /// Unanswerable probes whose page did NOT come back as a miss. + pub false_answers: Vec<&'static str>, + pub unanswerable: usize, +} + +impl Calibration { + pub fn precision(&self, label: MatchStrength) -> Option { + let (relevant, total) = self.by_label[label as usize]; + (total > 0).then(|| relevant as f64 / total as f64) + } + + pub fn print(&self, title: &str) { + println!("\n=== {title} ==="); + for label in [MatchStrength::Strong, MatchStrength::Weak] { + let (relevant, total) = self.by_label[label as usize]; + let p = self + .precision(label) + .map_or(" -".to_string(), |p| format!("{p:.3}")); + println!( + "{:<7} P(relevant)={p} ({relevant}/{total})", + label.as_str() + ); + } + println!( + "answerable questions reported as a miss: {}/{} {:?}", + self.false_misses.len(), + self.answerable, + self.false_misses + ); + println!( + "unanswerable probes NOT reported as a miss: {}/{} {:?}", + self.false_answers.len(), + self.unanswerable, + self.false_answers + ); + } +} + +fn query(text: &str, k: usize) -> SearchQuery { + let scoring = crate::config::ScoringConfig::default(); + SearchQuery { + text: text.to_string(), + scope: Scope::All, + kinds: vec![], + since: None, + limit: k, + strength_alpha: scoring.strength_alpha, + recency_alpha: scoring.recency_alpha, + type_prior_alpha: scoring.type_prior_alpha, + code_damp: scoring.code_damp, + include_superseded: false, + as_of: None, + } +} + +/// Label every top-`k` hit of every question (and probe) and tally. +/// `query_vec(text, topic)` produces the query embedding — synthetic in +/// hermetic mode, the real model otherwise. +pub fn calibrate( + conn: &Connection, + ids: &fixtures::FixtureIds, + k: usize, + model: &str, + mut query_vec: impl FnMut(&str, &str) -> Result>, +) -> Result { + let mut cache = None; + let mut cal = Calibration::default(); + for q in fixtures::questions() { + let qvec = query_vec(q.query, q.topic)?; + let (hits, legs) = search::search_hybrid_with_legs( + conn, + &query(q.query, k), + Some((model, &qvec)), + &mut cache, + )?; + let relevant: HashSet = fixtures::relevant_ids(ids, q); + let labels = strength::classify_page(&hits, q.query, &legs); + for (hit, label) in hits.iter().zip(&labels) { + let slot = &mut cal.by_label[*label as usize]; + slot.1 += 1; + if relevant.contains(&hit.node.id) { + slot.0 += 1; + } + } + cal.answerable += 1; + if strength::is_miss(&labels) { + cal.false_misses.push(q.query); + } + } + for (text, topic) in UNANSWERABLE { + let qvec = query_vec(text, topic)?; + let (hits, legs) = search::search_hybrid_with_legs( + conn, + &query(text, k), + Some((model, &qvec)), + &mut cache, + )?; + cal.unanswerable += 1; + if !strength::is_miss(&strength::classify_page(&hits, text, &legs)) { + cal.false_answers.push(text); + } + } + Ok(cal) +} + +pub fn run_hermetic(k: usize) -> Result { + let conn = crate::db::open_in_memory()?; + let ids = fixtures::insert_fixture_nodes(&conn)?; + fixtures::seed_synthetic_vectors(&conn, &ids)?; + calibrate(&conn, &ids, k, fixtures::SYNTHETIC_MODEL, |text, topic| { + Ok(fixtures::synthetic_query_vector(topic, text)) + }) +} + +/// Real-model counterpart: identical corpus, embedded with the configured +/// default model. `None` when the model isn't downloaded locally. +#[cfg(feature = "eval")] +pub fn run_real_model(k: usize) -> Result> { + use rusqlite::params; + + use crate::config::{Config, Paths}; + use crate::embed::{self, Embedder}; + + let paths = Paths::resolve()?; + let config = Config::load(&paths.config_file)?; + let model_name = config.embedding.model.clone(); + if !embed::model_ready(&paths, &model_name) { + println!("eval: model '{model_name}' not downloaded; skipping strength calibration"); + return Ok(None); + } + let mut embedder = Embedder::load(&paths, &model_name, &config.embedding.device, false)?; + let conn = crate::db::open_in_memory()?; + let ids = fixtures::insert_fixture_nodes(&conn)?; + for (node_id, text) in fixtures::fixture_texts(&ids) { + let vec = embedder.embed(vec![text])?.remove(0); + conn.execute( + "INSERT INTO embedding (node_id, model, dim, vec, content_hash) \ + VALUES (?1, ?2, ?3, ?4, x'00')", + params![ + node_id, + embedder.name, + embedder.dim as i64, + embed::to_blob(&vec) + ], + )?; + } + let model_name = embedder.name.clone(); + let cal = calibrate(&conn, &ids, k, &model_name, |text, _| { + Ok(embedder.embed(vec![text.to_string()])?.remove(0)) + })?; + Ok(Some(cal)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn hermetic_labels_are_ordered_and_misses_mean_nothing_matched() { + let cal = run_hermetic(8).unwrap(); + cal.print("strength calibration (hermetic, k=8)"); + let strong = cal + .precision(MatchStrength::Strong) + .expect("some hit is strong"); + let weak = cal + .precision(MatchStrength::Weak) + .expect("some hit is weak"); + // A label earns its word on the line only if it separates: the + // real-model run has strong at 25x the weak rate; 10x is the floor + // this hermetic gate holds the rule to. + assert!( + strong >= 10.0 * weak, + "strong must mean relevant far more often than weak: \ + strong {strong:.3}, weak {weak:.3}" + ); + // Three vague drift prompts ("how do we cut a release", "anything I + // should watch out for ...", "what should I know about past mistakes + // ...") share too few words with their answers to be anything but + // weak; the same three miss with the real model. More is the rule + // getting worse. + assert!( + cal.false_misses.len() <= 3, + "answerable questions reported as misses: {:?}", + cal.false_misses + ); + assert!( + cal.false_answers.is_empty(), + "a question nothing answers must be a miss: {:?}", + cal.false_answers + ); + } + + #[cfg(feature = "eval")] + #[test] + #[ignore] + fn real_model_strength_calibration() { + if let Some(cal) = run_real_model(8).unwrap() { + cal.print("strength calibration (real model, k=8)"); + } + } +} diff --git a/crates/mimir-core/src/format.rs b/crates/mimir-core/src/format.rs index 75f6e80..ed25c27 100644 --- a/crates/mimir-core/src/format.rs +++ b/crates/mimir-core/src/format.rs @@ -257,7 +257,18 @@ fn match_window(text: &str, stems: &[String], snippet_chars: usize) -> Option, snippet_chars: usize) -> String { - agent_line_for_query(node, project, snippet_chars, None, false) + agent_line_for_query(node, project, snippet_chars, None, HitMarks::default()) +} + +/// Per-hit facts a ranked recall knows about and a bare node does not, each +/// costing one word on the compact line when true and nothing when false. +#[derive(Debug, Clone, Copy, Default)] +pub struct HitMarks { + /// The memory's grounding link points at a symbol that is gone. + pub stale_grounding: bool, + /// Too little evidence that this hit matches the query at all + /// (`strength::MatchStrength::Weak`). + pub weak_match: bool, } /// [`agent_line`], but with the search query in hand so the snippet can be @@ -280,7 +291,7 @@ pub fn agent_line_for_query( project: Option<&str>, snippet_chars: usize, query: Option<&str>, - stale_grounding: bool, + marks: HitMarks, ) -> String { let id = short_uid(node.kind, &node.uid); let tag = node.subkind.as_deref().unwrap_or(node.kind.as_str()); @@ -333,8 +344,19 @@ pub fn agent_line_for_query( // ran `get` on that exact hit afterwards — which, in a recall loop, // is nobody. Costs a word on the hits that have it and nothing on the // rest. - let stale = if stale_grounding { " stale-link" } else { "" }; - let mut line = format!("{id} [{tag}{scope} {date}{uses}{doubt}{validity}{stale}] {title}"); + let stale = if marks.stale_grounding { + " stale-link" + } else { + "" + }; + // Same asymmetry again: `strong` is what a hit is supposed to be, so it + // goes unsaid; `weak` changes what an agent should do — check before + // acting, or treat the page as "the store doesn't know". Measured + // (`eval::strength`): a weak hit is relevant ~2% of the time, a strong + // one ~55%. + let weak = if marks.weak_match { " weak" } else { "" }; + let mut line = + format!("{id} [{tag}{scope} {date}{uses}{doubt}{validity}{stale}{weak}] {title}"); if let Some(body) = node.body.as_deref() { let flat = collapse_ws(body); let stems = query.map(query_stems).unwrap_or_default(); @@ -711,7 +733,13 @@ mod tests { "head truncation should miss the match: {blind}" ); - let aware = agent_line_for_query(&node, None, 120, Some("prompt cache breakpoints"), false); + let aware = agent_line_for_query( + &node, + None, + 120, + Some("prompt cache breakpoints"), + HitMarks::default(), + ); assert!( aware.contains("breakpoints"), "snippet should be centred on the match: {aware}" @@ -723,7 +751,13 @@ mod tests { fn query_aware_snippet_falls_back_when_nothing_matches() { let node = breadcrumb_chunk(); assert_eq!( - agent_line_for_query(&node, None, 120, Some("kubernetes ingress"), false), + agent_line_for_query( + &node, + None, + 120, + Some("kubernetes ingress"), + HitMarks::default() + ), agent_line(&node, None, 120), "a query with no lexical match must not change the line" ); @@ -734,7 +768,7 @@ mod tests { let node = breadcrumb_chunk(); // "compress" is inside the head window, which already shows it. assert_eq!( - agent_line_for_query(&node, None, 120, Some("compress"), false), + agent_line_for_query(&node, None, 120, Some("compress"), HitMarks::default()), agent_line(&node, None, 120), ); } diff --git a/crates/mimir-core/src/lib.rs b/crates/mimir-core/src/lib.rs index 8d9e76d..25f71da 100644 --- a/crates/mimir-core/src/lib.rs +++ b/crates/mimir-core/src/lib.rs @@ -22,6 +22,7 @@ pub mod index; pub mod inject; pub mod learn; pub mod memory; +pub mod misses; pub mod model; pub mod replicate; pub mod review; @@ -31,6 +32,7 @@ pub mod scope; pub mod search; pub mod secrets; pub mod store; +pub mod strength; pub mod tokens; use std::path::Path; @@ -310,6 +312,19 @@ impl Mimir { /// `[rerank] auto` and the per-query gates say it should — see /// `should_auto_rerank`. pub fn search_with(&mut self, query: &SearchQuery, rerank: bool) -> Result> { + Ok(self.search_with_evidence(query, rerank)?.0) + } + + /// [`Mimir::search_with`], plus the per-leg ranked id lists the hits were + /// fused from (`legs[0]` = FTS, `legs[1]` = vector iff a model ran) — + /// what `strength::classify` needs to say how much evidence stands behind + /// each hit. The legs are the pre-rerank retrieval: reranking reorders + /// the page, it does not change what either retriever found. + pub fn search_with_evidence( + &mut self, + query: &SearchQuery, + rerank: bool, + ) -> Result<(Vec, Vec>)> { let explicit = rerank; let rerank = rerank || self.should_auto_rerank(query); let mut fetch_query = query.clone(); @@ -319,10 +334,10 @@ impl Mimir { let query_vec = self.query_embedding(&query.text); let alpha = self.config.scoring.impression_alpha; - let mut hits = match query_vec { + let (mut hits, legs) = match query_vec { Some(v) => { let model = self.config.embedding.model.clone(); - search::search_hybrid_scored( + search::search_hybrid_with_legs_scored( &self.conn, &fetch_query, Some((&model, &v)), @@ -335,7 +350,7 @@ impl Mimir { // otherwise a cold/model-less process would silently bypass a // configured impression_alpha instead of degrading to BM25-only // *and* keeping the damp, same as `search_with_legs` does. - None => search::search_hybrid_scored( + None => search::search_hybrid_with_legs_scored( &self.conn, &fetch_query, None, @@ -369,7 +384,7 @@ impl Mimir { } } hits.truncate(query.limit); - Ok(hits) + Ok((hits, legs)) } /// Rank `docs` against `text`, best-first `(doc_index, score)`. Priority: diff --git a/crates/mimir-core/src/misses.rs b/crates/mimir-core/src/misses.rs new file mode 100644 index 0000000..75e2778 --- /dev/null +++ b/crates/mimir-core/src/misses.rs @@ -0,0 +1,324 @@ +//! The miss ledger: recall queries the store had no answer for. +//! +//! `recall_event` can say which memories were shown, opened and marked +//! useful. It cannot say what was *missing*, because a recall that found +//! nothing wrote no row — and "what do agents keep asking that nobody has +//! written down?" is the list that tells a person which memory to write +//! next. This keeps that list. +//! +//! A miss is a page with no `strong` hit ([`strength::is_miss`]): either +//! nothing came back, or everything that did is only there because a ranked +//! list always has something in it. Only *explicit* recalls feed it (CLI +//! `recall`, MCP `recall`). The per-prompt inject hook is deliberately not a +//! feeder: it runs on every prompt, most of which are not questions for the +//! store, and silence is its normal answer — logging those would mean +//! storing nearly every prompt anyone types. + +use rusqlite::{params, Connection, OptionalExtension}; + +use crate::error::Result; +use crate::model::Scope; +use crate::strength::{self, MatchStrength}; + +/// Longest query kept. Recall queries are a sentence; something much longer +/// is a pasted prompt or log, and the head says what it was about. +const MAX_QUERY_CHARS: usize = 300; + +/// One ledger row. +#[derive(Debug, Clone)] +pub struct Miss { + pub id: i64, + pub query: String, + pub project_id: Option, + /// How many (weak) hits the last miss returned; 0 = nothing at all. + pub hits: i64, + pub count: i64, + pub first_at: i64, + pub last_at: i64, + pub resolved_at: Option, +} + +/// The project a recall's scope belongs to, for keying. `All` and `Global` +/// both mean "not one project's question". +pub fn scope_project(scope: &Scope) -> Option { + match scope { + Scope::Project(id) => Some(*id), + Scope::All | Scope::Global => None, + } +} + +/// The comparison form: case and spacing are not part of the question, so +/// "WAL growth" and "wal growth" are one row with a count of two. +fn normalise(query: &str) -> String { + query + .split_whitespace() + .collect::>() + .join(" ") + .to_lowercase() +} + +/// Record what one explicit recall page says about the store: a miss bumps +/// (or reopens) the query's row, an answer resolves it. The single entry +/// point both recall surfaces call. +/// +/// Never writes a query the secrets scanner flags — the ledger keeps text, +/// and a secret pasted into a search box must not become the one copy of it +/// that outlives the session. Best-effort like the other ledgers: a failed +/// write is a warning, never a failed recall. +pub fn observe( + conn: &Connection, + query: &str, + project_id: Option, + strengths: &[MatchStrength], + now: i64, +) { + let normalised = normalise(query); + if normalised.is_empty() || crate::secrets::scan(query).is_some() { + return; + } + let key = blake3::hash(normalised.as_bytes()); + let res = if strength::is_miss(strengths) { + record( + conn, + query, + key.as_bytes(), + project_id, + strengths.len(), + now, + ) + } else { + resolve(conn, key.as_bytes(), project_id, now) + }; + if let Err(err) = res { + tracing::warn!(%err, "could not update the miss ledger"); + } +} + +fn record( + conn: &Connection, + query: &str, + key: &[u8], + project_id: Option, + hits: usize, + now: i64, +) -> Result<()> { + // Spacing collapsed, case kept: the row should read as the question was + // asked, minus the stray whitespace `normalise` already ignores. + let text: String = query + .split_whitespace() + .collect::>() + .join(" ") + .chars() + .take(MAX_QUERY_CHARS) + .collect(); + // A resolved row that misses again is reopened, not duplicated: the + // memory that answered it was deleted or superseded, and the question + // is open again with all of its history. + conn.execute( + "INSERT INTO recall_miss (query, query_key, project_id, hits, count, first_at, last_at) + VALUES (?1, ?2, ?3, ?4, 1, ?5, ?5) + ON CONFLICT(query_key, COALESCE(project_id, -1)) DO UPDATE SET + query = ?1, + hits = ?4, + count = count + 1, + last_at = ?5, + resolved_at = NULL", + params![text, key, project_id, hits as i64, now], + )?; + Ok(()) +} + +fn resolve(conn: &Connection, key: &[u8], project_id: Option, now: i64) -> Result<()> { + conn.execute( + "UPDATE recall_miss SET resolved_at = ?3 + WHERE query_key = ?1 AND COALESCE(project_id, -1) = COALESCE(?2, -1) + AND resolved_at IS NULL", + params![key, project_id, now], + )?; + Ok(()) +} + +/// Ledger rows, most-asked first, then most recent. `project_id = Some` +/// keeps that project's misses and the cross-project ones (the same reach a +/// recall from that project has); `None` keeps everything. +pub fn list( + conn: &Connection, + project_id: Option, + include_resolved: bool, + limit: usize, +) -> Result> { + let mut stmt = conn.prepare( + "SELECT id, query, project_id, hits, count, first_at, last_at, resolved_at + FROM recall_miss + WHERE (?1 IS NULL OR project_id IS NULL OR project_id = ?1) + AND (?2 OR resolved_at IS NULL) + ORDER BY count DESC, last_at DESC, id + LIMIT ?3", + )?; + let rows = stmt.query_map(params![project_id, include_resolved, limit as i64], |r| { + Ok(Miss { + id: r.get(0)?, + query: r.get(1)?, + project_id: r.get(2)?, + hits: r.get(3)?, + count: r.get(4)?, + first_at: r.get(5)?, + last_at: r.get(6)?, + resolved_at: r.get(7)?, + }) + })?; + Ok(rows.collect::>>()?) +} + +/// `(open, resolved)` among queries last missed since `since`. +pub fn counts(conn: &Connection, since: i64) -> Result<(i64, i64)> { + Ok(conn.query_row( + "SELECT COALESCE(SUM(resolved_at IS NULL), 0), COALESCE(SUM(resolved_at IS NOT NULL), 0) + FROM recall_miss WHERE last_at >= ?1", + [since], + |r| Ok((r.get(0)?, r.get(1)?)), + )?) +} + +/// Remove one row — a question that was noise, not a gap. +pub fn dismiss(conn: &Connection, id: i64) -> Result { + Ok(conn.execute("DELETE FROM recall_miss WHERE id = ?1", [id])? > 0) +} + +/// Drop rows nobody has asked about since `cutoff`. Runs with the +/// `recall_event` prune, on the same retention window. +pub fn prune(conn: &Connection, cutoff: i64) -> Result { + Ok(conn.execute("DELETE FROM recall_miss WHERE last_at < ?1", [cutoff])?) +} + +/// The row for `query` in `project_id`, if one exists. +pub fn find(conn: &Connection, query: &str, project_id: Option) -> Result> { + let key = blake3::hash(normalise(query).as_bytes()); + Ok(conn + .query_row( + "SELECT id, query, project_id, hits, count, first_at, last_at, resolved_at + FROM recall_miss + WHERE query_key = ?1 AND COALESCE(project_id, -1) = COALESCE(?2, -1)", + params![key.as_bytes(), project_id], + |r| { + Ok(Miss { + id: r.get(0)?, + query: r.get(1)?, + project_id: r.get(2)?, + hits: r.get(3)?, + count: r.get(4)?, + first_at: r.get(5)?, + last_at: r.get(6)?, + resolved_at: r.get(7)?, + }) + }, + ) + .optional()?) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db; + use MatchStrength::{Strong, Weak}; + + const T: i64 = 1_700_000_000; + + #[test] + fn a_repeated_miss_is_one_row_counted_twice_regardless_of_case_and_spacing() { + let conn = db::open_in_memory().unwrap(); + observe(&conn, "How do we rotate the hub token", None, &[Weak], T); + observe(&conn, "how do we rotate the HUB token", None, &[], T + 5); + let rows = list(&conn, None, false, 10).unwrap(); + assert_eq!(rows.len(), 1, "{rows:?}"); + assert_eq!(rows[0].count, 2); + assert_eq!(rows[0].hits, 0, "the latest miss came back empty"); + assert_eq!((rows[0].first_at, rows[0].last_at), (T, T + 5)); + } + + #[test] + fn a_strong_answer_resolves_and_a_later_miss_reopens() { + let conn = db::open_in_memory().unwrap(); + observe(&conn, "hub token rotation", None, &[Weak], T); + observe(&conn, "hub token rotation", None, &[Weak, Strong], T + 10); + assert!(list(&conn, None, false, 10).unwrap().is_empty(), "answered"); + let resolved = find(&conn, "hub token rotation", None).unwrap().unwrap(); + assert_eq!(resolved.resolved_at, Some(T + 10)); + + // The answering memory went away; the question is open again, with + // its history rather than as a stranger. + observe(&conn, "hub token rotation", None, &[Weak], T + 20); + let reopened = find(&conn, "hub token rotation", None).unwrap().unwrap(); + assert_eq!(reopened.resolved_at, None); + assert_eq!(reopened.count, 2); + assert_eq!(reopened.first_at, T); + } + + #[test] + fn an_answer_never_creates_a_row() { + let conn = db::open_in_memory().unwrap(); + observe(&conn, "wal growth", None, &[Strong], T); + assert!(find(&conn, "wal growth", None).unwrap().is_none()); + } + + #[test] + fn the_same_question_in_two_projects_is_two_gaps() { + let conn = db::open_in_memory().unwrap(); + observe(&conn, "release steps", Some(1), &[Weak], T); + observe(&conn, "release steps", Some(2), &[Weak], T); + observe(&conn, "release steps", Some(1), &[Strong], T + 1); + assert_eq!( + find(&conn, "release steps", Some(1)) + .unwrap() + .unwrap() + .resolved_at, + Some(T + 1) + ); + assert_eq!( + find(&conn, "release steps", Some(2)) + .unwrap() + .unwrap() + .resolved_at, + None + ); + // A project sees its own misses and the cross-project ones only. + observe(&conn, "global question", None, &[Weak], T); + let seen: Vec = list(&conn, Some(2), false, 10) + .unwrap() + .into_iter() + .map(|m| m.query) + .collect(); + assert_eq!(seen, vec!["release steps", "global question"], "{seen:?}"); + } + + #[test] + fn a_query_carrying_a_secret_is_never_written() { + let conn = db::open_in_memory().unwrap(); + let token = format!("ghp_{}", "a1B2c3D4e5F6g7H8i9J0k1L2m3N4o5P6q7R8"); + assert!( + crate::secrets::scan(&token).is_some(), + "fixture must trip the scanner" + ); + observe(&conn, &format!("why does {token} fail"), None, &[Weak], T); + let n: i64 = conn + .query_row("SELECT count(*) FROM recall_miss", [], |r| r.get(0)) + .unwrap(); + assert_eq!(n, 0); + } + + #[test] + fn long_queries_are_capped_and_prune_drops_the_unasked() { + let conn = db::open_in_memory().unwrap(); + observe(&conn, &"word ".repeat(200), None, &[Weak], T); + observe(&conn, "recent", None, &[Weak], T + 100); + let long = list(&conn, None, false, 10) + .unwrap() + .into_iter() + .find(|m| m.query != "recent") + .unwrap(); + assert!(long.query.chars().count() <= MAX_QUERY_CHARS); + assert_eq!(prune(&conn, T + 50).unwrap(), 1); + assert_eq!(list(&conn, None, true, 10).unwrap().len(), 1); + assert_eq!(counts(&conn, 0).unwrap(), (1, 0)); + } +} diff --git a/crates/mimir-core/src/strength.rs b/crates/mimir-core/src/strength.rs new file mode 100644 index 0000000..8dc8ae4 --- /dev/null +++ b/crates/mimir-core/src/strength.rs @@ -0,0 +1,304 @@ +//! How much evidence stands behind one recall hit: `strong` or `weak`. +//! +//! The fused score cannot answer that. RRF ranks by *position* in each leg, +//! so the best hit on a query the store knows nothing about scores exactly +//! what the best hit on a perfect match scores — every ranked list has a #1. +//! An agent reading the list cannot tell "this is the answer" from "this is +//! the least-bad thing in the store", and the second case is the one where +//! acting on the hit is a mistake. +//! +//! So the label is built from evidence that does not depend on rank alone, +//! the same two signals `inject::clears_floor` already trusts before it puts +//! a memory in front of an agent unasked: +//! +//! - **term overlap** — how many of the query's content words the hit's +//! title, body and tags actually contain; +//! - **leg agreement** — whether the hit sits in the lexical leg's own +//! top [`LEG_TOP_K`] *and* the semantic leg's, i.e. two independent +//! retrievers both put it near the top. +//! +//! What is deliberately NOT here: inject's rule 1 (only gotchas and +//! decisions). That rule is about whether a memory is worth injecting +//! unprompted, not about whether it matches. An explicit recall for a note +//! deserves an honest label for the note. +//! +//! **Two levels, not three, and that is measured.** A middle `likely` tier +//! (words present, legs disagreeing) was built first and checked against the +//! labelled corpus with the real embedding model (`eval::strength`, k=8): +//! `sure` 23/42 relevant, `likely` 0/8, `weak` 3/134. A tier that is right +//! less often than the one below it is noise with a reassuring name, so it +//! folded into `weak`. The same run is why the upper level says `strong` +//! rather than `sure`: 55% relevant is strong evidence — 25× the weak rate — +//! and not certainty. + +use std::collections::HashSet; + +use crate::inject::{LEG_TOP_K, MIN_OVERLAP}; +use crate::memory; +use crate::model::Kind; +use crate::search::{fts, Hit}; + +/// Evidence behind one hit. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum MatchStrength { + Weak, + Strong, +} + +impl MatchStrength { + pub fn as_str(self) -> &'static str { + match self { + MatchStrength::Weak => "weak", + MatchStrength::Strong => "strong", + } + } +} + +/// The query's content words: tokenized the way the inject floor tokenizes, +/// stopwords dropped. +/// +/// Exact tokens, deliberately unstemmed. The snippet window's 5-char prefix +/// stem (`format::STEM_PREFIX`) was tried here so "password" would meet +/// "passwords", and measured with the real model it found no relevant hit +/// the exact match missed (23 strong-and-relevant either way) while adding +/// 11 irrelevant strong ones — prefix collisions, precision 55% → 43%. +pub fn query_terms(query: &str) -> HashSet { + memory::tokens(query) + .into_iter() + .filter(|t| !fts::is_stopword(t)) + .collect() +} + +/// Label one hit. `legs` is what [`crate::Mimir::search_with_evidence`] +/// returns: `legs[0]` the lexical leg, `legs[1]` the semantic leg iff an +/// embedding model ran for this query. +/// +/// `strong` when the query names this symbol exactly, or when the hit +/// contains the query's words AND both legs rank it in their own top-K. +/// Without a semantic leg there is nothing to agree with, so covering +/// *every* content word (at least two) stands in for the agreement. +/// Everything else is `weak`: in the list because something had to be. +/// +/// Semantic rank alone is deliberately NOT evidence. The vector leg returns +/// a top-k for every query, nonsense included, so "in the semantic top 10" +/// is rank again, not relevance: counted on its own, it let all six +/// unanswerable calibration probes (`eval::strength::UNANSWERABLE`) through. +/// It still counts as agreement alongside matching words, where it confirms +/// rather than stands in for a match. The cost is that a pure paraphrase — +/// right answer, no shared words — reads `weak`; that is the honest reading +/// of the evidence this has, and why the label says "weak", not "wrong". +pub fn classify(hit: &Hit, terms: &HashSet, legs: &[Vec]) -> MatchStrength { + if terms.is_empty() { + return MatchStrength::Weak; + } + if names_symbol(hit, terms) { + return MatchStrength::Strong; + } + let id = hit.node.id; + let in_top = |leg: Option<&Vec>| { + leg.is_some_and(|leg| leg.iter().take(LEG_TOP_K).any(|other| *other == id)) + }; + let lexical = in_top(legs.first()); + let semantic = in_top(legs.get(1)); + + let hay = format!( + "{} {} {}", + hit.node.title.as_deref().unwrap_or(""), + hit.node.body.as_deref().unwrap_or(""), + hit.node.tags_text, + ); + let hay: HashSet = memory::tokens(&hay).into_iter().collect(); + let overlap = terms.iter().filter(|t| hay.contains(*t)).count(); + // A one-word query cannot reach MIN_OVERLAP; matching its one word is + // all the overlap it can show. + // + // A flat count rather than a share of the query, and that is measured + // too. "Half the query's words" was tried after two real-store hits came + // back `strong` on 2 of 5 shared words (`watchdog.sh` for a TLS + // certificate question, a UI row helper for a centring question — both + // sharing the query's two meaningful words, so partial topical matches + // rather than junk). With the real model it lifted strong's precision + // 55% → 73% but tripled what matters more, the relevant hits labelled + // weak (3 → 10), and doubled answerable questions reported as misses + // (3 → 6 of 23): natural questions carry words their answers never use. + // `weak` is the label that promises "safe to discount", so it is the one + // kept honest. + let covers = overlap >= MIN_OVERLAP.min(terms.len()); + + let agrees = if legs.len() > 1 { + lexical && semantic + } else { + lexical && overlap == terms.len() && overlap >= MIN_OVERLAP + }; + tracing::debug!( + node = id, + title = hit.node.title.as_deref().unwrap_or(""), + overlap, + terms = terms.len(), + lexical, + semantic, + "match evidence" + ); + if covers && agrees { + MatchStrength::Strong + } else { + MatchStrength::Weak + } +} + +/// A query that names a code symbol exactly (`retry_with_backoff`, +/// `Mimir::search_with`) and got that symbol back. The identifier leg exists +/// for this case and is not returned in `legs`, so it is recognised here +/// from the hit itself: the title's last path segment must be one of the +/// query's tokens, and must look like an identifier rather than a word — +/// otherwise a symbol called `open` would be `strong` for any query saying +/// "open". +fn names_symbol(hit: &Hit, terms: &HashSet) -> bool { + if hit.node.kind != Kind::Symbol { + return false; + } + let Some(title) = hit.node.title.as_deref() else { + return false; + }; + let name = title + .rsplit([':', '.']) + .next() + .unwrap_or(title) + .to_lowercase(); + let identifier_shaped = name.contains('_') || title.chars().any(|c| c.is_uppercase()); + identifier_shaped && terms.contains(&name) +} + +/// Label a whole page, in order. +pub fn classify_page(hits: &[Hit], query: &str, legs: &[Vec]) -> Vec { + let terms = query_terms(query); + hits.iter().map(|hit| classify(hit, &terms, legs)).collect() +} + +/// Did this page answer the query at all? An empty page, or one with no +/// `strong` hit, is a miss: the store had nothing that matched, it only had +/// something to rank. +pub fn is_miss(strengths: &[MatchStrength]) -> bool { + !strengths.contains(&MatchStrength::Strong) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::NewNode; + use crate::{db, store}; + + /// Hits keep their real rowids so the `legs` below can name them: a + /// fresh store numbers from 1, so the `id`-th insert has rowid `id`. + fn hit(id: i64, kind: Kind, title: &str, body: &str) -> Hit { + let conn = db::open_in_memory().unwrap(); + let mut node = None; + for _ in 0..id { + let mut new = NewNode::new(kind); + new.title = Some(title.into()); + new.body = Some(body.into()); + node = Some(store::insert_node(&conn, new).unwrap()); + } + let node = node.expect("id >= 1"); + assert_eq!(node.id, id); + Hit { node, score: 1.0 } + } + + fn terms(q: &str) -> HashSet { + query_terms(q) + } + + #[test] + fn both_legs_and_coverage_is_strong() { + let h = hit( + 1, + Kind::Memory, + "WAL file growth", + "checkpoint starvation keeps the wal large", + ); + let legs = vec![vec![1, 2], vec![3, 1]]; + assert_eq!( + classify(&h, &terms("why does the wal file grow"), &legs), + MatchStrength::Strong + ); + } + + #[test] + fn coverage_without_semantic_agreement_is_weak() { + let h = hit(1, Kind::Memory, "WAL file growth", "checkpoint starvation"); + let legs = vec![vec![1], vec![9, 8, 7]]; + assert_eq!(classify(&h, &terms("wal file"), &legs), MatchStrength::Weak); + } + + #[test] + fn semantic_rank_alone_is_not_evidence() { + // The vector leg has a top-k for every query; ranking first there + // with none of the query's words is what a nonsense query looks like. + let h = hit(1, Kind::Memory, "Connection pooling", "one writer"); + let semantic_only = vec![vec![], vec![1]]; + assert_eq!( + classify(&h, &terms("sqlite concurrency limits"), &semantic_only), + MatchStrength::Weak + ); + } + + #[test] + fn without_a_semantic_leg_only_full_coverage_is_strong() { + let h = hit(1, Kind::Memory, "WAL file growth", "checkpoint starvation"); + let lexical_only = vec![vec![1]]; + assert_eq!( + classify(&h, &terms("wal file growth"), &lexical_only), + MatchStrength::Strong + ); + assert_eq!( + classify(&h, &terms("wal file growth policy"), &lexical_only), + MatchStrength::Weak + ); + } + + #[test] + fn an_exact_identifier_is_strong_but_a_plain_word_symbol_is_not() { + let named = hit( + 1, + Kind::Symbol, + "retry_with_backoff", + "fn retry_with_backoff()", + ); + assert_eq!( + classify(&named, &terms("retry_with_backoff"), &[vec![]]), + MatchStrength::Strong + ); + let path = hit( + 2, + Kind::Symbol, + "Mimir::search_with", + "pub fn search_with()", + ); + assert_eq!( + classify(&path, &terms("what does search_with do"), &[vec![]]), + MatchStrength::Strong + ); + // `open` is a word, not an identifier: no shortcut. + let word = hit(3, Kind::Symbol, "open", "pub fn open()"); + assert_eq!( + classify(&word, &terms("open the file"), &[vec![]]), + MatchStrength::Weak + ); + } + + #[test] + fn a_stopword_only_query_has_no_evidence() { + let h = hit(1, Kind::Memory, "anything", "the a of"); + assert_eq!( + classify(&h, &terms("what is the"), &[vec![1], vec![1]]), + MatchStrength::Weak + ); + } + + #[test] + fn a_page_is_a_miss_only_when_nothing_is_strong() { + assert!(is_miss(&[])); + assert!(is_miss(&[MatchStrength::Weak, MatchStrength::Weak])); + assert!(!is_miss(&[MatchStrength::Weak, MatchStrength::Strong])); + } +} From ab0074474b84ce4b4162fcde004447b07ad07b3c Mon Sep 17 00:00:00 2001 From: Thomas <155702229+MakerViking@users.noreply.github.com> Date: Fri, 16 Oct 2026 01:59:18 +0200 Subject: [PATCH 3/3] feat(shadow): score candidate rankings on real recalls before they ship MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The retrieval eval proves a ranking change doesn't break the questions someone wrote down; it can't say whether the change helps the questions agents actually ask. Shadow mode runs a candidate beside the live ranking on every explicit recall (CLI and MCP), keeps its order only when it differs, and scores both against what the agent opened. - attribution: an `opened` is a verdict on the most recent page that showed that node, within 10 minutes — not on every page that listed it - scoreboard (`mimir shadow`, --json, --days): MRR of the opened hit, wins/losses/ties, exact two-sided sign test, and a verdict that needs 20 decided pages before it says anything - states plainly that the comparison favours live (agents open what the live order put on top): a lead is evidence, a deficit is not harm - never changes recall output; nothing promotes itself - first candidate, co-activation phase 2: score * (1 + 0.5 * w) for a memory with a visible co_recalled edge (w >= 0.5) to one opened in the last 30 minutes - its own table, not rows in recall_event, which several readers count whole; pruned with the recall_event retention - `[learn] shadow_ranking = false` turns it off Schema v14: shadow_rank. --- CHANGELOG.md | 15 +- README.md | 24 ++ crates/mimir-cli/src/commands.rs | 64 +++ crates/mimir-cli/src/main.rs | 9 + crates/mimir-cli/src/mcp.rs | 8 + crates/mimir-core/src/config.rs | 7 + crates/mimir-core/src/db/migrations.rs | 30 ++ crates/mimir-core/src/db/mod.rs | 5 + crates/mimir-core/src/learn.rs | 2 +- crates/mimir-core/src/lib.rs | 1 + crates/mimir-core/src/shadow.rs | 516 +++++++++++++++++++++++++ 11 files changed, 679 insertions(+), 2 deletions(-) create mode 100644 crates/mimir-core/src/shadow.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 73753e2..1c820ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,9 +26,22 @@ the `mimir-mem` crate, and the on-disk schema move together. secrets scanner flags, ages out with `[learn] event_retention_days`, and `[learn] log_misses = false` turns it off. +- **Co-activation edges (phase 1).** Two memories opened within 30 minutes + of each other strengthen a derived `co_recalled` edge (Hebbian, one fire + per pair per day, 30-day half-life, at most 32 per node). Recorded only — + ranking is untouched. Not synced or exported; `link` refuses to author one. +- **Shadow ranking — `mimir shadow`.** A candidate ranking runs beside the + live one on every explicit recall; where the orders differ both are kept, + and each later `opened` is scored against the page it came from (MRR of the + opened hit, wins/losses, exact sign test). Recall output never changes and + nothing promotes itself. First candidate: lift memories co-recalled with + one opened in the last 30 minutes. The report says plainly that the + comparison favours live. `[learn] shadow_ranking = false` to disable. + ### Changed -- **Schema v13** adds the `recall_miss` table. As with every schema bump, a +- **Schema v13** adds the `recall_miss` table, **v14** the `shadow_rank` + table. As with every schema bump, a store opened by this binary is refused by older ones (`SchemaTooNew`): reinstall, then restart running `mimir mcp` servers. diff --git a/README.md b/README.md index 3bf8bd8..9779ec1 100644 --- a/README.md +++ b/README.md @@ -304,6 +304,30 @@ hook. The query text stays on this machine (not synced, not exported), a query the secrets scanner flags is never written, and rows age out with `[learn] event_retention_days`. Turn it off with `[learn] log_misses = false`. +### Shadow ranking: test a ranking idea on real use before it ships + +The retrieval eval proves a ranking change doesn't break the questions +someone wrote down; it can't say whether the change helps the questions +agents actually ask. Shadow mode runs a candidate ranking beside the live one +on every recall, keeps its order when the two differ, and scores both on what +the agent opened from that page — while recall keeps returning the live +order. + +```sh +mimir shadow # candidate vs live: MRR of the opened hit, wins/losses, sign test +mimir shadow --days 30 +``` + +The one candidate today is phase 2 of co-activation: lift a memory that is +co-recalled with one you opened in the last 30 minutes. Two honest limits. +The comparison **favours live**, because agents open what they were shown +near the top, so a shadow lead is real evidence and a shadow deficit isn't +evidence of harm. And it needs data: a co-activation edge only counts once +the pair has been opened together on several separate days, so expect +nothing for the first week or two. Nothing promotes itself — a winning +candidate goes live as an ordinary ranking change, through the eval gate. +`[learn] shadow_ranking = false` turns it off. + ### The fast path (recommended setup) Three commands get you the lowest-latency configuration Mimir has — measured diff --git a/crates/mimir-cli/src/commands.rs b/crates/mimir-cli/src/commands.rs index 0c18f6f..f16faae 100644 --- a/crates/mimir-cli/src/commands.rs +++ b/crates/mimir-cli/src/commands.rs @@ -1231,6 +1231,14 @@ pub fn recall( .map(|(rank, hit)| (hit.node.id, rank as i64, hit.score)) .collect(); store::record_shown(&mimir.conn, query_hash.as_bytes(), &shown)?; + if mimir.config.learn.shadow_ranking { + mimir_core::shadow::observe( + &mimir.conn, + query_hash.as_bytes(), + &hits, + mimir_core::model::now_unix(), + ); + } if let Some(report) = mimir_core::consolidate::maybe_auto( &mimir.conn, &mimir.config.embedding.model, @@ -1622,6 +1630,62 @@ pub fn misses( Ok(()) } +/// Shadow-ranking scoreboard: each candidate against the live ranking. +pub fn shadow(days: Option, json: bool) -> Result<()> { + let mimir = Mimir::open()?; + let since = days.map_or(0, |d| mimir_core::model::now_unix() - i64::from(d) * 86_400); + let e = mimir_core::shadow::evaluate(&mimir.conn, mimir_core::shadow::COACTIVATION, since)?; + if json { + println!( + "{}", + serde_json::json!({ + "candidate": e.candidate, + "pages": e.pages, + "decided": e.decided, + "live_mrr": e.live_mrr, + "shadow_mrr": e.shadow_mrr, + "wins": e.wins, + "losses": e.losses, + "ties": e.ties, + "sign_test_p": e.sign_test_p(), + "verdict": e.verdict(), + }) + ); + return Ok(()); + } + println!( + "candidate: {} — boost a memory co-recalled with one opened in the last 30 min", + e.candidate + ); + println!( + "pages it would have reordered: {} decided by an open: {}", + e.pages, e.decided + ); + if e.decided > 0 { + println!( + "MRR of the opened hit live {:.3} shadow {:.3}", + e.live_mrr, e.shadow_mrr + ); + let p = e + .sign_test_p() + .map_or("-".to_string(), |p| format!("{p:.3}")); + println!( + "shadow ranked it higher {} · lower {} · same {} (sign test p = {p})", + e.wins, e.losses, e.ties + ); + } + println!("verdict: {}", e.verdict()); + if !mimir.config.learn.shadow_ranking { + println!("(shadow ranking is off: `[learn] shadow_ranking = false`)"); + } + println!( + "\nAgents open what they were shown near the top, and they were shown the live \ + order — so this comparison favours live.\nA shadow lead is real evidence; a \ + shadow deficit is not evidence of harm. Nothing goes live from here on its own." + ); + Ok(()) +} + pub fn refusals(limit: usize) -> Result<()> { let mimir = Mimir::open()?; let rows = mimir_core::secrets::refusals(&mimir.conn, limit)?; diff --git a/crates/mimir-cli/src/main.rs b/crates/mimir-cli/src/main.rs index 022ae34..d89a155 100644 --- a/crates/mimir-cli/src/main.rs +++ b/crates/mimir-cli/src/main.rs @@ -390,6 +390,14 @@ enum Command { #[arg(long, default_value_t = 20)] limit: usize, }, + /// How candidate ranking signals running in shadow compare with the live + /// ranking, scored on what agents actually opened. Nothing here changes + /// what recall returns. + Shadow { + /// Only pages from the last N days (default: everything kept). + #[arg(long)] + days: Option, + }, /// What the secret guard turned away — fingerprints and counts, never /// the refused values. Refusals { @@ -996,6 +1004,7 @@ fn run(cli: Cli) -> anyhow::Result<()> { resolved, limit, } => commands::misses(dismiss, all, resolved, limit, cli.json), + Command::Shadow { days } => commands::shadow(days, cli.json), Command::Refusals { limit } => commands::refusals(limit), Command::Audit { reference, limit } => { commands::audit(reference.as_deref(), limit, cli.json) diff --git a/crates/mimir-cli/src/mcp.rs b/crates/mimir-cli/src/mcp.rs index dedf86a..817685c 100644 --- a/crates/mimir-cli/src/mcp.rs +++ b/crates/mimir-cli/src/mcp.rs @@ -306,6 +306,14 @@ impl MimirServer { .map(|(rank, hit)| (hit.node.id, rank as i64, hit.score)) .collect(); store::record_shown(&m.conn, query_hash.as_bytes(), &shown).map_err(engine_err)?; + if m.config.learn.shadow_ranking { + mimir_core::shadow::observe( + &m.conn, + query_hash.as_bytes(), + &hits, + mimir_core::model::now_unix(), + ); + } let projects = store::project_titles(&m.conn).map_err(engine_err)?; let ids: Vec = hits.iter().map(|h| h.node.id).collect(); let stale = mimir_core::grounding::stale_ids(&m.conn, &ids).unwrap_or_default(); diff --git a/crates/mimir-core/src/config.rs b/crates/mimir-core/src/config.rs index 7fc0f16..e435d6b 100644 --- a/crates/mimir-core/src/config.rs +++ b/crates/mimir-core/src/config.rs @@ -285,6 +285,12 @@ pub struct LearnConfig { /// is never written, and rows age out on `event_retention_days`. `false` /// stops new rows; existing ones remain until pruned or dismissed. pub log_misses: bool, + /// Run candidate ranking signals in shadow beside the live ranking + /// (`mimir shadow`): each recall page gets the candidate's order too, + /// kept only when it differs, and later scored against what the agent + /// opened. Never changes what recall returns. Costs one indexed lookup + /// per recall, and a write only on pages the candidate would reorder. + pub shadow_ranking: bool, } impl Default for LearnConfig { @@ -292,6 +298,7 @@ impl Default for LearnConfig { LearnConfig { event_retention_days: 180, log_misses: true, + shadow_ranking: true, } } } diff --git a/crates/mimir-core/src/db/migrations.rs b/crates/mimir-core/src/db/migrations.rs index 0fb09f6..885cf48 100644 --- a/crates/mimir-core/src/db/migrations.rs +++ b/crates/mimir-core/src/db/migrations.rs @@ -348,6 +348,29 @@ CREATE TABLE recall_miss ( ); CREATE UNIQUE INDEX recall_miss_key ON recall_miss(query_key, COALESCE(project_id, -1)); CREATE INDEX recall_miss_open ON recall_miss(last_at) WHERE resolved_at IS NULL; +"#, + // Shadow ranking (`shadow`): a candidate ranking signal's order for a + // real recall page, kept only when it differs from the live order, so + // the two can later be scored against what the agent opened from that + // page. One row per hit of a differing page. + // + // Its own table rather than an `event = 'shadow'` row in `recall_event`: + // that ledger is the store's record of what actually happened, and + // several readers count it whole (`mimir report`'s top-recalled list, + // the dashboard's total). A counterfactual mixed into it would be + // counted as an impression by each of them, and by every reader written + // after them. + r#" +CREATE TABLE shadow_rank ( + id INTEGER PRIMARY KEY, + candidate TEXT NOT NULL, + query_hash BLOB NOT NULL, + at INTEGER NOT NULL, + node_id INTEGER NOT NULL, + live_rank INTEGER NOT NULL, + shadow_rank INTEGER NOT NULL +); +CREATE INDEX shadow_rank_at ON shadow_rank(candidate, at); "#, ]; @@ -517,6 +540,13 @@ mod tests { (1, 0), "upgraded store must be able to write the miss ledger" ); + assert_eq!( + crate::shadow::evaluate(&conn, crate::shadow::COACTIVATION, 0) + .unwrap() + .pages, + 0, + "upgraded store must be able to read the shadow ledger" + ); } #[test] diff --git a/crates/mimir-core/src/db/mod.rs b/crates/mimir-core/src/db/mod.rs index 7085064..bee7da6 100644 --- a/crates/mimir-core/src/db/mod.rs +++ b/crates/mimir-core/src/db/mod.rs @@ -127,6 +127,11 @@ fn prune_events_once(conn: &Connection, days: u32) { Ok(_) => {} Err(err) => tracing::warn!(%err, "recall_event prune failed"), } + match crate::shadow::prune(conn, cutoff) { + Ok(n) if n > 0 => tracing::info!(rows = n, days, "pruned old shadow_rank rows"), + Ok(_) => {} + Err(err) => tracing::warn!(%err, "shadow_rank prune failed"), + } match crate::misses::prune(conn, cutoff) { Ok(n) if n > 0 => tracing::info!(rows = n, days, "pruned old recall_miss rows"), Ok(_) => {} diff --git a/crates/mimir-core/src/learn.rs b/crates/mimir-core/src/learn.rs index fc7c0e1..055fa06 100644 --- a/crates/mimir-core/src/learn.rs +++ b/crates/mimir-core/src/learn.rs @@ -223,7 +223,7 @@ pub const COACTIVATION_VISIBLE: f64 = 0.5; /// How close together two opens must fall to count as the same stretch of /// work. Long enough to span a detour, short enough that yesterday's session /// is not "the same stretch". -const COACTIVATION_WINDOW_SECS: i64 = 30 * 60; +pub const COACTIVATION_WINDOW_SECS: i64 = 30 * 60; /// Most co-activation edges kept per node, weakest evicted first. Without a /// cap, a memory opened in every session pairs with the entire store and its diff --git a/crates/mimir-core/src/lib.rs b/crates/mimir-core/src/lib.rs index 25f71da..1464127 100644 --- a/crates/mimir-core/src/lib.rs +++ b/crates/mimir-core/src/lib.rs @@ -31,6 +31,7 @@ pub mod savings; pub mod scope; pub mod search; pub mod secrets; +pub mod shadow; pub mod store; pub mod strength; pub mod tokens; diff --git a/crates/mimir-core/src/shadow.rs b/crates/mimir-core/src/shadow.rs new file mode 100644 index 0000000..94317ec --- /dev/null +++ b/crates/mimir-core/src/shadow.rs @@ -0,0 +1,516 @@ +//! Shadow ranking: a candidate ranking signal run beside the live one, +//! logged, and scored against what agents went on to open — never shown. +//! +//! Every ranking change in this store has to clear the retrieval eval, and +//! the eval is a fixed, hand-labelled corpus: it can say a change does not +//! break the questions someone wrote down, not that it helps the questions +//! agents actually ask. Shadow mode is the other half. A candidate computes +//! its own order for each real recall page; when that order differs from the +//! live one, both are kept; when an agent then opens a hit from the page, +//! the two orders are scored on where they had put it. +//! +//! **The comparison is biased toward the live ranking, on purpose of +//! nobody's.** Agents open what they were shown near the top, and what they +//! were shown is the live order — so a candidate starts every page behind. +//! A shadow lead is therefore conservative evidence (it won despite the +//! bias); a shadow deficit is not evidence of harm. The report says so. +//! +//! **Nothing promotes itself.** A candidate that wins here goes live by a +//! human changing the ranking code, which then has to pass the retrieval +//! eval gate like any other ranking change. The shadow only produces the +//! evidence for that decision. +//! +//! The one candidate today is [`COACTIVATION`]: phase 2 of `learn`'s +//! co-activation edges. Phase 1 records which memories are opened together +//! and deliberately ranks nothing; this asks whether ranking on it would +//! have helped. + +use std::collections::HashMap; + +use rusqlite::{params, Connection}; + +use crate::error::Result; +use crate::learn::{self, COACTIVATION_WINDOW_SECS}; +use crate::model::Kind; +use crate::search::Hit; + +/// The co-activation candidate: a memory co-activated with something opened +/// in the current stretch of work is probably part of that work. +pub const COACTIVATION: &str = "coactivation"; + +/// How strongly a co-activation edge lifts a hit: `score · (1 + BOOST · w)` +/// with `w` the edge's effective weight, so a fully-formed edge (w = 1) is +/// worth half again the hit's score. A guess, and labelled as one — the +/// point of running it in shadow is to find out. +pub const BOOST: f64 = 0.5; + +/// How long after a page an `opened` event still counts as a verdict on that +/// page. An agent reads a recall and opens from it within seconds; ten +/// minutes covers a slow read without crediting the page with opens that +/// came from a later, different question. +pub const VERDICT_WINDOW_SECS: i64 = 10 * 60; + +/// Fewer decided pages than this and the report declines to call it. +pub const MIN_DECIDED: usize = 20; + +/// Memories "in play" right now — partners of anything opened in the last +/// [`COACTIVATION_WINDOW_SECS`] — mapped to the strongest visible edge +/// weight that put them there. +pub fn primed(conn: &Connection, now: i64) -> Result> { + let mut stmt = conn.prepare_cached( + "SELECT DISTINCT node_id FROM recall_event + WHERE event = 'opened' AND at >= ?1", + )?; + let opened: Vec = stmt + .query_map([now - COACTIVATION_WINDOW_SECS], |r| r.get(0))? + .collect::>()?; + let mut primed: HashMap = HashMap::new(); + for id in opened { + for (partner, weight) in learn::coactivation_partners(conn, id, now)? { + let slot = primed.entry(partner).or_default(); + *slot = slot.max(weight); + } + } + Ok(primed) +} + +/// The candidate's order for a live page: indices into `hits`, best first. +/// Only memories can be primed (the edges are memory↔memory); everything +/// else keeps its live score. Ties keep live order, so an unprimed page +/// comes back exactly as it went in. +pub fn coactivation_order(hits: &[Hit], primed: &HashMap) -> Vec { + let boosted: Vec = hits + .iter() + .map(|hit| { + let w = if hit.node.kind == Kind::Memory { + primed.get(&hit.node.id).copied().unwrap_or(0.0) + } else { + 0.0 + }; + hit.score * (1.0 + BOOST * w) + }) + .collect(); + let mut order: Vec = (0..hits.len()).collect(); + order.sort_by(|&a, &b| boosted[b].total_cmp(&boosted[a]).then(a.cmp(&b))); + order +} + +/// Run the candidates on one live page and keep the result if it differs. +/// The single entry point both recall surfaces call, right where they log +/// `shown`. Best-effort like the other ledgers: a failure here is a warning, +/// never a failed recall. +pub fn observe(conn: &Connection, query_hash: &[u8], hits: &[Hit], now: i64) { + if let Err(err) = observe_inner(conn, query_hash, hits, now) { + tracing::warn!(%err, "shadow ranking not recorded"); + } +} + +fn observe_inner(conn: &Connection, query_hash: &[u8], hits: &[Hit], now: i64) -> Result<()> { + if hits.len() < 2 { + return Ok(()); + } + let primed = primed(conn, now)?; + if primed.is_empty() { + return Ok(()); + } + let order = coactivation_order(hits, &primed); + if order.iter().enumerate().all(|(rank, &idx)| rank == idx) { + return Ok(()); + } + let tx = rusqlite::Transaction::new_unchecked(conn, rusqlite::TransactionBehavior::Immediate)?; + { + let mut stmt = tx.prepare_cached( + "INSERT INTO shadow_rank (candidate, query_hash, at, node_id, live_rank, shadow_rank) + VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + )?; + for (shadow_rank, &live_rank) in order.iter().enumerate() { + stmt.execute(params![ + COACTIVATION, + query_hash, + now, + hits[live_rank].node.id, + live_rank as i64, + shadow_rank as i64 + ])?; + } + } + tx.commit()?; + Ok(()) +} + +/// How one candidate has done against the live ranking. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct Evaluation { + pub candidate: String, + /// Pages where the candidate's order differed from the live one. + pub pages: usize, + /// …of which an agent then opened at least one hit. + pub decided: usize, + /// Mean reciprocal rank of the first opened hit, live vs candidate. + pub live_mrr: f64, + pub shadow_mrr: f64, + /// Decided pages where the candidate ranked the opened hit higher / + /// lower / the same. + pub wins: usize, + pub losses: usize, + pub ties: usize, +} + +impl Evaluation { + /// Two-sided exact sign-test p over wins vs losses (ties dropped). `None` + /// with nothing decided either way. + pub fn sign_test_p(&self) -> Option { + sign_test(self.wins, self.losses) + } + + /// One line a person can act on. + pub fn verdict(&self) -> &'static str { + if self.decided < MIN_DECIDED { + return "not enough evidence yet"; + } + match self.sign_test_p() { + Some(p) if p < 0.05 && self.wins > self.losses => { + "candidate AHEAD — worth a ranking change, then the eval gate" + } + Some(p) if p < 0.05 => { + "candidate behind — not evidence of harm (the comparison favours live)" + } + _ => "no significant difference", + } + } +} + +/// One recall page: its query hash and when it was shown. +type PageKey<'a> = (&'a [u8], i64); + +/// Score `candidate` over pages since `since`. +/// +/// An `opened` event is a verdict on exactly one page: the most recent one +/// that showed that node before the open (from the `shown` rows every recall +/// writes), and only if it came within [`VERDICT_WINDOW_SECS`]. A flat +/// "opened within ten minutes of the page" rule would credit one open to +/// every page that happened to list the node in that time — including the +/// ones the agent never read. +pub fn evaluate(conn: &Connection, candidate: &str, since: i64) -> Result { + struct Row { + hash: Vec, + at: i64, + node: i64, + live: i64, + shadow: i64, + } + let mut stmt = conn.prepare( + "SELECT query_hash, at, node_id, live_rank, shadow_rank FROM shadow_rank + WHERE candidate = ?1 AND at >= ?2", + )?; + let rows: Vec = stmt + .query_map(params![candidate, since], |r| { + Ok(Row { + hash: r.get(0)?, + at: r.get(1)?, + node: r.get(2)?, + live: r.get(3)?, + shadow: r.get(4)?, + }) + })? + .collect::>()?; + + // Which page each open was a verdict on: (node, query_hash, shown_at). + let mut verdicts: Vec<(i64, Vec, i64)> = Vec::new(); + let mut nodes: Vec = rows.iter().map(|r| r.node).collect(); + nodes.sort_unstable(); + nodes.dedup(); + let mut events = conn.prepare_cached( + "SELECT event, query_hash, at FROM recall_event + WHERE node_id = ?1 AND event IN ('shown', 'opened') AND at >= ?2 + ORDER BY at, id", + )?; + for node in nodes { + let mut last_shown: Option<(Vec, i64)> = None; + let mut events = events.query(params![node, since])?; + while let Some(r) = events.next()? { + let event: String = r.get(0)?; + let at: i64 = r.get(2)?; + if event == "shown" { + last_shown = r.get::<_, Option>>(1)?.map(|hash| (hash, at)); + } else if let Some((hash, shown_at)) = &last_shown { + if at - shown_at <= VERDICT_WINDOW_SECS { + verdicts.push((node, hash.clone(), *shown_at)); + } + } + } + } + + // `shown` and the shadow row are stamped by separate clock reads in the + // same recall, so they can straddle a second boundary. + let opened_from = |row: &Row| { + verdicts.iter().any(|(node, hash, at)| { + *node == row.node && *hash == row.hash && (at - row.at).abs() <= 1 + }) + }; + // Per page: the best (live, shadow) rank among its opened hits, `None` + // while nothing on it was opened. + let mut pages: HashMap> = HashMap::new(); + for row in &rows { + let page = pages.entry((row.hash.as_slice(), row.at)).or_default(); + if opened_from(row) { + *page = Some(match *page { + Some((live, shadow)) => (live.min(row.live), shadow.min(row.shadow)), + None => (row.live, row.shadow), + }); + } + } + + let mut eval = Evaluation { + candidate: candidate.to_string(), + pages: pages.len(), + ..Evaluation::default() + }; + let (mut live_sum, mut shadow_sum) = (0.0, 0.0); + for (live, shadow) in pages.values().flatten() { + eval.decided += 1; + live_sum += 1.0 / (*live as f64 + 1.0); + shadow_sum += 1.0 / (*shadow as f64 + 1.0); + match shadow.cmp(live) { + std::cmp::Ordering::Less => eval.wins += 1, + std::cmp::Ordering::Greater => eval.losses += 1, + std::cmp::Ordering::Equal => eval.ties += 1, + } + } + if eval.decided > 0 { + eval.live_mrr = live_sum / eval.decided as f64; + eval.shadow_mrr = shadow_sum / eval.decided as f64; + } + Ok(eval) +} + +/// Drop rows older than `cutoff`. Runs with the `recall_event` prune: a +/// shadow row is only ever judged against `opened` events, so it is useless +/// once those are gone. +pub fn prune(conn: &Connection, cutoff: i64) -> Result { + Ok(conn.execute("DELETE FROM shadow_rank WHERE at < ?1", [cutoff])?) +} + +/// Exact two-sided binomial sign test at p = 0.5, in log space so a few +/// thousand pages cannot underflow. +fn sign_test(wins: usize, losses: usize) -> Option { + let n = wins + losses; + if n == 0 { + return None; + } + let k = wins.min(losses); + let ln_half_n = n as f64 * 0.5f64.ln(); + let mut ln_choose = 0.0; // ln C(n, 0) + let mut tail = 0.0; + for i in 0..=k { + if i > 0 { + ln_choose += ((n - i + 1) as f64).ln() - (i as f64).ln(); + } + tail += (ln_choose + ln_half_n).exp(); + } + Some((2.0 * tail).min(1.0)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db; + use crate::model::NewNode; + use crate::store; + + fn memory(conn: &Connection, text: &str) -> crate::model::Node { + let mut new = NewNode::new(Kind::Memory); + new.subkind = Some("note".into()); + new.title = Some(text.into()); + new.body = Some(text.into()); + store::insert_node(conn, new).unwrap() + } + + fn page(nodes: &[&crate::model::Node]) -> Vec { + nodes + .iter() + .enumerate() + .map(|(i, n)| Hit { + node: (*n).clone(), + // Close scores, so a full-strength edge can reorder them. + score: 1.0 - 0.1 * i as f64, + }) + .collect() + } + + fn opened_at(conn: &Connection, node_id: i64, at: i64) { + conn.execute( + "INSERT INTO recall_event (node_id, event, at) VALUES (?1, 'opened', ?2)", + params![node_id, at], + ) + .unwrap(); + } + + fn co_edge(conn: &Connection, a: i64, b: i64, weight: f64, at: i64) { + conn.execute( + "INSERT INTO edge (src, dst, rel, weight, meta, created_at) + VALUES (?1, ?2, 'co_recalled', ?3, json_object('last_fired_at', ?4), ?4)", + params![a.min(b), a.max(b), weight, at], + ) + .unwrap(); + } + + /// What a real recall does: log `shown` for the page, then run the + /// shadow — with the synthetic clock, which `store::record_shown` (it + /// stamps `now_unix()`) cannot take. + fn show(conn: &Connection, hash: &[u8], hits: &[Hit], at: i64) { + for (rank, hit) in hits.iter().enumerate() { + conn.execute( + "INSERT INTO recall_event (node_id, event, query_hash, rank, at) + VALUES (?1, 'shown', ?2, ?3, ?4)", + params![hit.node.id, hash, rank as i64, at], + ) + .unwrap(); + } + observe(conn, hash, hits, at); + } + + fn shadow_rows(conn: &Connection) -> i64 { + conn.query_row("SELECT count(*) FROM shadow_rank", [], |r| r.get(0)) + .unwrap() + } + + const T: i64 = 1_700_000_000; + + #[test] + fn an_unprimed_page_keeps_live_order_and_writes_nothing() { + let conn = db::open_in_memory().unwrap(); + let (a, b, c) = (memory(&conn, "a"), memory(&conn, "b"), memory(&conn, "c")); + let hits = page(&[&a, &b, &c]); + assert_eq!(coactivation_order(&hits, &HashMap::new()), vec![0, 1, 2]); + observe(&conn, b"q", &hits, T); + assert_eq!(shadow_rows(&conn), 0); + } + + #[test] + fn a_partner_of_something_just_opened_moves_up_in_shadow_only() { + let conn = db::open_in_memory().unwrap(); + let (a, b, c, seed) = ( + memory(&conn, "a"), + memory(&conn, "b"), + memory(&conn, "c"), + memory(&conn, "seed"), + ); + co_edge(&conn, seed.id, c.id, 1.0, T); + opened_at(&conn, seed.id, T - 60); + let hits = page(&[&a, &b, &c]); + + let primed = primed(&conn, T).unwrap(); + assert_eq!(primed.get(&c.id).copied(), Some(1.0), "{primed:?}"); + // c: 0.8 · 1.5 = 1.2, ahead of a's 1.0. + assert_eq!(coactivation_order(&hits, &primed), vec![2, 0, 1]); + + observe(&conn, b"q", &hits, T); + let ranks: Vec<(i64, i64, i64)> = conn + .prepare("SELECT node_id, live_rank, shadow_rank FROM shadow_rank ORDER BY shadow_rank") + .unwrap() + .query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?))) + .unwrap() + .collect::>() + .unwrap(); + assert_eq!(ranks, vec![(c.id, 2, 0), (a.id, 0, 1), (b.id, 1, 2)]); + } + + #[test] + fn a_stale_or_weak_edge_primes_nothing() { + let conn = db::open_in_memory().unwrap(); + let (seed, weak, stale) = (memory(&conn, "s"), memory(&conn, "w"), memory(&conn, "o")); + co_edge(&conn, seed.id, weak.id, 0.3, T); + // Full weight, but five half-lives old: 1.0 · 2^-5 ≈ 0.03. + co_edge(&conn, seed.id, stale.id, 1.0, T - 150 * 86_400); + opened_at(&conn, seed.id, T - 60); + assert!(primed(&conn, T).unwrap().is_empty()); + // And an open outside the working window primes nothing either. + let conn = db::open_in_memory().unwrap(); + let (seed, partner) = (memory(&conn, "s"), memory(&conn, "p")); + co_edge(&conn, seed.id, partner.id, 1.0, T); + opened_at(&conn, seed.id, T - COACTIVATION_WINDOW_SECS - 1); + assert!(primed(&conn, T).unwrap().is_empty()); + } + + #[test] + fn each_open_is_a_verdict_on_the_page_it_came_from_only() { + let conn = db::open_in_memory().unwrap(); + let (a, b, c, seed) = ( + memory(&conn, "a"), + memory(&conn, "b"), + memory(&conn, "c"), + memory(&conn, "seed"), + ); + co_edge(&conn, seed.id, c.id, 1.0, T); + opened_at(&conn, seed.id, T - 60); + let hits = page(&[&a, &b, &c]); + + // Page 1: the agent opens c — the hit shadow promoted (live 2, + // shadow 0). A win. + show(&conn, b"q1", &hits, T); + opened_at(&conn, c.id, T + 30); + // Page 2, a minute later: the agent opens a (live 0, shadow 1). A + // loss — and it must not also count against page 1, which was still + // inside its window but was not where this open came from. + show(&conn, b"q2", &hits, T + 100); + opened_at(&conn, a.id, T + 140); + // Page 3: nothing opened. Undecided. + show(&conn, b"q3", &hits, T + 200); + + let e = evaluate(&conn, COACTIVATION, 0).unwrap(); + assert_eq!((e.pages, e.decided), (3, 2), "{e:?}"); + assert_eq!((e.wins, e.losses, e.ties), (1, 1, 0), "{e:?}"); + assert!((e.live_mrr - (1.0 / 3.0 + 1.0) / 2.0).abs() < 1e-9, "{e:?}"); + assert!((e.shadow_mrr - (1.0 + 0.5) / 2.0).abs() < 1e-9, "{e:?}"); + assert_eq!(e.verdict(), "not enough evidence yet"); + } + + #[test] + fn an_open_long_after_the_page_is_no_verdict() { + let conn = db::open_in_memory().unwrap(); + let (a, b, c, seed) = ( + memory(&conn, "a"), + memory(&conn, "b"), + memory(&conn, "c"), + memory(&conn, "seed"), + ); + co_edge(&conn, seed.id, c.id, 1.0, T); + opened_at(&conn, seed.id, T - 60); + show(&conn, b"q", &page(&[&a, &b, &c]), T); + opened_at(&conn, c.id, T + VERDICT_WINDOW_SECS + 1); + let e = evaluate(&conn, COACTIVATION, 0).unwrap(); + assert_eq!((e.pages, e.decided), (1, 0)); + assert_eq!(prune(&conn, T + 1).unwrap(), 3); + } + + #[test] + fn sign_test_matches_known_values() { + assert_eq!(sign_test(0, 0), None); + // 10 vs 0: 2 · 0.5^10. + assert!((sign_test(10, 0).unwrap() - 2.0 / 1024.0).abs() < 1e-12); + // 5 vs 5 is as balanced as it gets. + assert!((sign_test(5, 5).unwrap() - 1.0).abs() < 1e-12); + // 15 vs 5, two-sided: 0.04139… + assert!((sign_test(15, 5).unwrap() - 0.041_389_465).abs() < 1e-6); + // Large n must not underflow to 0 or NaN. + let p = sign_test(2_600, 2_400).unwrap(); + assert!(p > 0.0 && p < 0.01, "{p}"); + } + + #[test] + fn verdict_needs_enough_pages_and_significance() { + let mut e = Evaluation { + decided: MIN_DECIDED, + wins: 18, + losses: 2, + ..Evaluation::default() + }; + assert!(e.verdict().starts_with("candidate AHEAD")); + e.wins = 11; + e.losses = 9; + assert_eq!(e.verdict(), "no significant difference"); + e.decided = MIN_DECIDED - 1; + assert_eq!(e.verdict(), "not enough evidence yet"); + } +}