Graph revision

.gitignore

@@ -47,6 +47,10 @@ htmlcov/
 *.sqlite
 *.sqlite-journal
 test-results/
+
+# eval memory snapshots: PG dump + full Neo4j JSON export (~hundreds of MB
+# each, regenerable from a fresh ingest). Same rationale as results/.
+evals/snapshots/
 test_*.db
 *.log
 .persist

archive/README.md

@@ -0,0 +1,46 @@
+# Archive — superseded code
+
+Modules retired from the live tree because nothing in the current design
+(graph_version v7.x / v8: hypergraph ingest hooks, V7Retriever with rerank,
+gated auto_dream) reaches them. Preserved here — and in full per-version
+history at the `graph_revision_pre_squash` tag — rather than deleted.
+
+## legacy_graph/
+
+The pre-v7 graph generations and rejected v7 extraction variants:
+
+- `episodic_graph_manager.py`, `semantic_graph_manager.py`, `lightrag_merger.py`
+  — the v5 dual-graph ingest builders (Episode/EpisodicEntity/Concept labels).
+- `episodic_graph_retriever.py`, `semantic_graph_retriever.py`,
+  `lightrag_keyword_extractor.py`, `_graph_retriever_base.py` — the v5
+  retrieval pipeline (keyword-LLM call + per-graph retrievers + budget split).
+- `graph_memory_manager_v6.py`, `graph_retriever_v6.py` — the v6 lean entity
+  index.
+- `proposition_extractor.py` — v7.3 proposition ingest (rejected: collapsed the
+  edge structure).
+- `gliner_extractor.py` — v7.4 GLiNER extraction (superseded by direction D:
+  a span tagger cannot abstract concepts).
+
+The dispatcher and the episodic/semantic ingest hooks now warn-and-skip for
+graph versions outside the v7 family instead of running these.
+
+Also removed from live files (recoverable from git history, documented in
+`docs/graph_memory_v7/development_history.md`): the v7.7 PPR and v7.2 coverage
+retrieval branches and the v7.9 role-split rendering in `graph_retriever_v7.py`,
+and the rejected answerer gates (v7.11 consolidate tool, persona store,
+graph-search merge) in `evals/task_agent.py`. A second audited sweep also
+removed: the never-measured MIRIX_GRAPH_ROUTED_SEARCH gate and its helper in
+rest_api.py, the v5/v6 Neo4j DDL and neo4j_healthcheck, the LightRAG relation
+output surface (the live v7/v8 path reads entities only), four orphaned
+lightrag prompt symbols, token_tracker.set_phase, the vestigial dispatcher
+budget parameters, and EpisodicEventUpdate.source_refs.
+
+## evals/
+
+Eval-side scripts for rejected or superseded experiments: `build_persona.py`
+(persona store, QA-negative), `refine_hypergraph.py` (one-off corpus cleanup,
+superseded by ingest-time prevention + maintenance passes),
+`proposition_ingest.py` (v7.3 driver), `run_v4_sample0.sh`,
+`visualize_v6_graph.py`, `draw_hypergraph.py` (viz one-off). (`memory_snapshot.py` was initially archived here too,
+then restored — the MAB runner scripts call it for post-run snapshots; the
+"unreferenced" scan had only covered *.py, not *.sh.)

archive/evals/build_persona.py

@@ -0,0 +1,61 @@
+"""Build a compact, structured PERSONA profile for a LongMemEval user from the
+existing memory store (no re-ingest). Targets the preference-synthesis questions
+(Q7/Q11/Q44...) whose gold answers require grounding advice in the user's own
+history — hobbies, past successes, stated likes/dislikes, work/social situation.
+
+The persona is a pre-synthesized "who is this user" summary so the answerer does
+not have to reconstruct it from scattered search hits on open-ended "any tips?"
+questions. Written to persona_<user>.txt; TaskAgent injects it when MIRIX_PERSONA
+is set. LaMP-PAG / PersonaAgent style.
+"""
+import os
+import sys
+import psycopg2
+from openai import OpenAI
+
+USER = sys.argv[1] if len(sys.argv) > 1 else "longmem_s_0"
+PG_DB = os.environ.get("MIRIX_PG_DB") or os.environ.get("PG_DB", "mirix_lm114_pm")
+OUT = os.path.expanduser(f"~/MIRIX_eval/persona_{USER}.txt")
+
+conn = psycopg2.connect(host="localhost", port=5432, user="mirix", password="mirix", dbname=PG_DB)
+cur = conn.cursor()
+# User's own statements (episodic actor='user') + stable facts (semantic) = the
+# raw material for a persona. Assistant turns are advice, not the user's identity.
+cur.execute("SELECT occurred_at, summary FROM episodic_memory WHERE user_id=%s AND is_deleted=false "
+            "AND actor='user' ORDER BY occurred_at", (USER,))
+ep = cur.fetchall()
+cur.execute("SELECT name, summary FROM semantic_memory WHERE user_id=%s AND is_deleted=false", (USER,))
+sem = cur.fetchall()
+cur.close(); conn.close()
+
+lines = [f"[{str(o)[:10]}] {s}" for o, s in ep if s] + [f"{n}: {s}" for n, s in sem if n]
+context = "\n".join(lines)
+
+PROMPT = (
+    "Below are a user's own statements over time plus stable facts about them. Build a "
+    "concise PERSONA PROFILE that captures WHO THIS USER IS, so an assistant can give "
+    "personalized advice grounded in their history (not generic tips).\n\n"
+    "Organize by life domain (e.g. Food & Baking, Art & Creativity, Fitness, Work & "
+    "Career, Social & Relationships, Travel, Hobbies, Home & Pets, Reading, Finance). "
+    "For each domain that applies, capture in 1-3 tight bullets:\n"
+    "  - what they DO / are into (recurring activities, tools, brands they use)\n"
+    "  - notable PAST experiences / successes / projects (e.g. 'baked a lemon poppyseed "
+    "cake that went well', 'did a 30-day painting challenge')\n"
+    "  - stated PREFERENCES: likes, dislikes, constraints, what they value\n"
+    "Be specific and name concrete things (recipes, apps, accounts, places, people). "
+    "Skip domains with no signal. Keep the whole profile under ~450 words. Output plain "
+    "text with domain headers and bullets, no preamble.\n\n"
+    "USER STATEMENTS & FACTS:\n" + context
+)
+
+client = OpenAI(api_key=os.environ["OPENAI_API_KEY"])
+resp = client.chat.completions.create(
+    model="gpt-4.1-mini", temperature=0,
+    messages=[{"role": "user", "content": PROMPT}],
+)
+persona = resp.choices[0].message.content.strip()
+with open(OUT, "w", encoding="utf-8") as f:
+    f.write(persona)
+print(f"wrote {OUT}  ({len(persona)} chars, from {len(ep)} user-episodic + {len(sem)} semantic)")
+print("=" * 60)
+print(persona)

archive/evals/draw_hypergraph.py

@@ -0,0 +1,89 @@
+"""Render a v7.10 HYPERGRAPH neighborhood: V7Fact nodes (reified triples) as
+hyperedges linking a SUBJECT anchor + OBJECT anchor + the source MEMORY, stamped
+with role. Shows the distinctive shape — facts are first-class nodes, not edges.
+"""
+import os
+import textwrap
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+import matplotlib.patches as mpatches
+import networkx as nx
+from neo4j import GraphDatabase
+
+U = "longmem_s_0"
+d = GraphDatabase.driver("bolt://localhost:7687", auth=("neo4j", "mirix_neo4j_dev"))
+
+# Two thematically-clean memories (a travel/flight cluster) so the picture reads.
+SEED_MEMS = ["ep_DU49", "ep_EJ78"]
+rows = []
+with d.session() as s:
+    # Pick facts seeded from these memories, then show ALL their source memories so the
+    # post-refinement multi-citation structure (one fact, N citations) is visible.
+    q = """
+    MATCH (f:V7Fact {user_id:$u})-[:V7_FACT_FROM]->(seed:V7MemoryRef)
+    WHERE seed.memory_id IN $mids
+    MATCH (subj:V7Anchor)<-[:V7_FACT_SUBJECT]-(f)-[:V7_FACT_OBJECT]->(obj:V7Anchor)
+    MATCH (f)-[:V7_FACT_FROM]->(m:V7MemoryRef)
+    WHERE subj.name IS NOT NULL AND obj.name IS NOT NULL
+    RETURN subj.name AS s, f.predicate AS p, obj.name AS o, f.role AS role,
+           collect(DISTINCT m.memory_id) AS mids
+    """
+    for r in s.run(q, u=U, mids=SEED_MEMS):
+        for mid in r["mids"]:
+            rows.append((r["s"], r["p"], r["o"], r["role"] or "shared", mid))
+d.close()
+
+G = nx.Graph()
+kind = {}
+def short(t, w=16):
+    return "\n".join(textwrap.wrap(t, w))[:60]
+
+for s_, p, o, role, mid in rows:
+    # key the fact by its triple so a fact cited by N memories stays ONE node
+    fid = f"F::{s_}|{p}|{o}"
+    sN, oN, mN = f"A::{s_}", f"A::{o}", f"M::{mid}"
+    for n, k, lbl in [(sN, "anchor", short(s_)), (oN, "anchor", short(o)),
+                      (mN, "memory", mid), (fid, "fact", p)]:
+        if n not in G:
+            G.add_node(n, label=lbl); kind[n] = k
+    G.add_edge(fid, sN, etype="subj")
+    G.add_edge(fid, oN, etype="obj")
+    G.add_edge(fid, mN, etype="from")
+
+pos = nx.spring_layout(G, k=0.9, iterations=200, seed=7)
+plt.figure(figsize=(17, 12))
+
+col = {"anchor": "#4C78A8", "memory": "#E45756", "fact": "#F2C744"}
+size = {"anchor": 1500, "memory": 2600, "fact": 700}
+for k in col:
+    ns = [n for n in G if kind[n] == k]
+    nx.draw_networkx_nodes(G, pos, nodelist=ns, node_color=col[k], node_size=size[k],
+                           node_shape=("s" if k == "memory" else ("D" if k == "fact" else "o")),
+                           edgecolors="white", linewidths=1.5)
+ecol = {"subj": "#59A14F", "obj": "#B279A2", "from": "#BAB0AC"}
+for et, c in ecol.items():
+    es = [(a, b) for a, b, dd in G.edges(data=True) if dd["etype"] == et]
+    nx.draw_networkx_edges(G, pos, edgelist=es, edge_color=c, width=1.6, alpha=0.8)
+
+anchor_lbls = {n: G.nodes[n]["label"] for n in G if kind[n] == "anchor"}
+mem_lbls = {n: G.nodes[n]["label"] for n in G if kind[n] == "memory"}
+nx.draw_networkx_labels(G, pos, labels=anchor_lbls, font_size=7.5, font_color="white")
+nx.draw_networkx_labels(G, pos, labels=mem_lbls, font_size=9, font_color="white", font_weight="bold")
+fact_lbls = {n: G.nodes[n]["label"] for n in G if kind[n] == "fact"}
+nx.draw_networkx_labels(G, pos, labels=fact_lbls, font_size=6, font_color="#5a4a00")
+
+legend = [mpatches.Patch(color=col["anchor"], label="V7Anchor (entity/concept)"),
+          mpatches.Patch(color=col["fact"], label="V7Fact  (reified triple = HYPEREDGE)"),
+          mpatches.Patch(color=col["memory"], label="V7MemoryRef (source memory)"),
+          mpatches.Patch(color=ecol["subj"], label="FACT_SUBJECT"),
+          mpatches.Patch(color=ecol["obj"], label="FACT_OBJECT"),
+          mpatches.Patch(color=ecol["from"], label="FACT_FROM")]
+plt.legend(handles=legend, loc="upper left", fontsize=9, framealpha=0.9)
+plt.title(f"v7.10 hypergraph neighborhood — {len(rows)} V7Fact hyperedges from 2 memories "
+          f"(longmem_s_0)\nEach yellow diamond is a fact linking a subject + object anchor to its source memory",
+          fontsize=12)
+plt.axis("off"); plt.tight_layout()
+OUT = os.path.expanduser("~/MIRIX_eval/hypergraph_neighborhood.png")
+plt.savefig(OUT, dpi=130, bbox_inches="tight")
+print(f"wrote {OUT}  ({len(rows)} facts, {G.number_of_nodes()} nodes, {G.number_of_edges()} edges)")

archive/evals/proposition_ingest.py

@@ -0,0 +1,116 @@
+"""v7.3 ingest: build LongMemEval-S / MAB memory from ATOMIC PROPOSITIONS.
+
+ONE LLM call per chunk emits both the propositions and their typed entities. Each
+proposition becomes a fine-grained semantic_memory (embedded); its entities are
+handed straight to the V7 graph build, which consumes only name + entity_type.
+LightRAG is never invoked -- it was ~97% of graph-build time, and its relations
+and descriptions were discarded by v7 anyway.
+
+Env:
+  MIRIX_PG_DB           target DB (schema must exist -- start the server against it once)
+  MIRIX_ENABLE_GRAPH_MEMORY=true, MIRIX_GRAPH_VERSION=v7.3
+  SOURCE                HF metadata.source (default longmemeval_s*)
+  MAX_CHUNKS            optional cap for a smoke test
+Usage:  python evals/mab/proposition_ingest.py
+"""
+import asyncio
+import datetime
+import os
+import sys
+import time
+import types
+
+sys.path.insert(0, os.path.dirname(__file__))
+from sqlalchemy import text as sa_text
+
+import longmem_eval as L
+from mirix.constants import MAX_EMBEDDING_DIM
+from mirix.database.neo4j_client import get_neo4j_driver, init_neo4j_client
+from mirix.schemas.embedding_config import EmbeddingConfig
+from mirix.schemas.llm_config import LLMConfig
+from mirix.server.server import db_context
+from mirix.services._graph_common import embed_batch
+from mirix.services.graph_memory_manager_v7 import V7GraphManager
+from mirix.services.lightrag_extractor import ExtractedEntity
+from mirix.services.proposition_extractor import extract_propositions_with_entities
+from mirix.settings import settings
+
+EMB = EmbeddingConfig(
+    embedding_endpoint_type="openai", embedding_endpoint="https://api.openai.com/v1",
+    embedding_model="text-embedding-3-small", embedding_dim=1536, embedding_chunk_size=300)
+LLM = LLMConfig(model="gpt-4.1-mini", model_endpoint_type="openai",
+    model_endpoint="https://api.openai.com/v1", context_window=128000)
+AST = types.SimpleNamespace(embedding_config=EMB, llm_config=LLM)
+UID, ORG = "longmem_s_0", "mirix-eval-org"
+SOURCE = os.environ.get("SOURCE", "longmemeval_s*")
+MAXC = int(os.environ.get("MAX_CHUNKS", "0")) or None
+API_KEY = os.environ["OPENAI_API_KEY"]
+
+INSERT = sa_text(
+    "INSERT INTO semantic_memory (id,name,summary,details,source,source_refs,prior_values,"
+    "last_modify,created_at,is_deleted,user_id,organization_id,summary_embedding) VALUES "
+    "(:id,:nm,:sm,:dt,:src,'[]','[]','{}',:ts,false,:u,:o,CAST(:emb AS vector))")
+
+
+async def main():
+    await init_neo4j_client()
+    drv = get_neo4j_driver()
+    async with drv.session(database=settings.neo4j_database) as s:
+        # consume() so the delete actually completes before we start writing
+        await (await s.run("MATCH (n) DETACH DELETE n")).consume()
+        left = (await (await s.run("MATCH (n) RETURN count(n) AS c")).single())["c"]
+    print(f"neo4j wiped (remaining nodes: {left})", flush=True)
+
+    it = L.load_longmem_s(source=SOURCE, limit=1)[0]
+    chunks = L.parse_sessions(it["context"])
+    if MAXC:
+        chunks = chunks[:MAXC]
+    print(f"chunks={len(chunks)} source={SOURCE}", flush=True)
+
+    mgr = V7GraphManager()
+    total = n_ents = 0
+    t0 = time.time()
+    for ci, ch in enumerate(chunks, 1):
+        props = await extract_propositions_with_entities(ch["text"], api_key=API_KEY)
+        if not props:
+            print(f"chunk {ci}/{len(chunks)}: no propositions", flush=True)
+            continue
+
+        embs = await embed_batch([p.text for p in props], AST)
+        rows = []  # (memory_id, Proposition, embedding)
+        for p, e in zip(props, embs):
+            if e is None:
+                continue
+            total += 1
+            rows.append((f"prop_{total:06d}", p, e))
+
+        async with db_context() as session:
+            for mid, p, e in rows:
+                epad = str(list(e) + [0.0] * (MAX_EMBEDDING_DIM - len(e)))
+                await session.execute(INSERT, {
+                    "id": mid, "nm": p.text[:120], "sm": p.text, "dt": p.text,
+                    "src": f"chunk_{ci - 1}", "ts": datetime.datetime.utcnow(),
+                    "u": UID, "o": ORG, "emb": epad})
+            await session.commit()
+
+        for mid, p, _ in rows:
+            n_ents += len(p.entities)
+            try:
+                await mgr.process_memory(
+                    source_kind="semantic", source_id=mid, text=p.text, title=p.text[:80],
+                    summary=p.text, source_meta=None, agent_state=AST,
+                    organization_id=ORG, user_id=UID,
+                    entities=[ExtractedEntity(name=x.name, entity_type=x.type, description="")
+                              for x in p.entities])
+            except Exception as ex:  # noqa: BLE001
+                print(f"  graph fail {mid}: {str(ex)[:80]}", flush=True)
+
+        print(f"chunk {ci}/{len(chunks)}  props={total} entities={n_ents} "
+              f"(+{time.time() - t0:.0f}s)", flush=True)
+
+    print(f"DONE: {total} propositions, {n_ents} entity mentions, "
+          f"{time.time() - t0:.0f}s, {len(chunks)} LLM extraction calls", flush=True)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())