|
| 1 | +#!/usr/bin/env python3 |
| 2 | +"""Multi-hop relational recall benchmark for Cortex (black-box via MCP stdio). |
| 3 | +
|
| 4 | +Single-hop semantic recall (see recall_scale.py) measures whether a paraphrased |
| 5 | +query surfaces ONE matching memory. This benchmark measures a harder case: |
| 6 | +MULTI-HOP relational recall, where answering a query requires CHAINING facts |
| 7 | +across entities and the answer memory shares little or no surface text with the |
| 8 | +query. |
| 9 | +
|
| 10 | +Example chain (2-hop): |
| 11 | + store: "Marisol manages the Helios project." |
| 12 | + store: "The Helios project runs on the Aurora database." |
| 13 | + store: "The Aurora database is hosted in the Frankfurt datacenter." |
| 14 | + query: "Which datacenter does Marisol's project depend on?" |
| 15 | +The required hop memories are the Helios->Aurora and Aurora->Frankfurt links, |
| 16 | +which do NOT contain the word "Marisol" — pure vector similarity tends to miss |
| 17 | +them because the query's salient tokens never appear in the target memories. |
| 18 | +
|
| 19 | +We plant ~25 such chains, bury them among ~3000 plausible distractors (reusing |
| 20 | +the noisy-corpus style from recall_scale.py), then for each query check whether |
| 21 | +the REQUIRED hop memories appear in memory_search top-k. We report: |
| 22 | + - hop-recall@1/@5/@10: fraction of required hop memories that surfaced |
| 23 | + - chain-completable@10: fraction of queries where ALL required hops were in top-10 |
| 24 | + - latency p50/p95 |
| 25 | +
|
| 26 | +This is the baseline metric a future "graph-edge re-ranking" change must move. |
| 27 | +
|
| 28 | +Usage: BIN=~/.local/bin/cortex-mcp-server python3 bench/recall_multihop.py |
| 29 | +""" |
| 30 | +import json, os, subprocess, time, statistics, random, tempfile, shutil |
| 31 | + |
| 32 | +TMP = tempfile.mkdtemp(prefix="cortex-multihop-") |
| 33 | +DB = f"{TMP}/multihop.db" |
| 34 | +BIN = os.path.expanduser(os.environ.get("BIN", "~/.local/bin/cortex-mcp-server")) |
| 35 | +random.seed(20260613) # deterministic; NOT time/Math.random seeded |
| 36 | + |
| 37 | + |
| 38 | +def srv(db): |
| 39 | + e = {**os.environ, "RUST_LOG": "error", "CORTEX_NO_KEYCHAIN": "1"} |
| 40 | + p = subprocess.Popen([BIN, db], stdin=subprocess.PIPE, stdout=subprocess.PIPE, |
| 41 | + stderr=subprocess.DEVNULL, text=True, env=e) |
| 42 | + n = [0] |
| 43 | + |
| 44 | + def call(name, args): |
| 45 | + n[0] += 1 |
| 46 | + p.stdin.write(json.dumps({"jsonrpc": "2.0", "id": n[0], "method": "tools/call", |
| 47 | + "params": {"name": name, "arguments": args}}) + "\n") |
| 48 | + p.stdin.flush() |
| 49 | + r = json.loads(p.stdout.readline())["result"] |
| 50 | + return r.get("isError", False), r["content"][0]["text"] |
| 51 | + |
| 52 | + p.stdin.write('{"jsonrpc":"2.0","id":0,"method":"initialize","params":{}}\n') |
| 53 | + p.stdin.flush() |
| 54 | + p.stdout.readline() |
| 55 | + return p, call |
| 56 | + |
| 57 | + |
| 58 | +# Each chain: list of link memories (stored verbatim) + a query whose answer |
| 59 | +# requires the LATER links (which lack the query's head-entity token). |
| 60 | +# "hops" = the indices of link memories REQUIRED to answer the query. We treat |
| 61 | +# every link after the head mention as required, since those are the ones a |
| 62 | +# vector search of the query tends to miss. |
| 63 | +# |
| 64 | +# Structure: (links, query, required_idxs) |
| 65 | +# links[0] connects the head entity (named in the query) to a middle entity. |
| 66 | +# The query references the head entity but asks about the tail; the required |
| 67 | +# hop memories are the ones NOT containing the head entity's name. |
| 68 | +CHAINS = [ |
| 69 | + # ---- 2-hop ---- |
| 70 | + (["Marisol manages the Helios project.", |
| 71 | + "The Helios project runs on the Aurora database.", |
| 72 | + "The Aurora database is hosted in the Frankfurt datacenter."], |
| 73 | + "Which datacenter does Marisol's project depend on?", [1, 2]), |
| 74 | + (["Devon leads the Pegasus initiative.", |
| 75 | + "The Pegasus initiative is funded by the Northwind grant.", |
| 76 | + "The Northwind grant expires at the end of the fiscal year."], |
| 77 | + "When does the funding for Devon's initiative run out?", [1, 2]), |
| 78 | + (["Priya owns the Cobalt service.", |
| 79 | + "The Cobalt service writes logs to the Meridian bucket.", |
| 80 | + "The Meridian bucket is replicated to the Osaka region."], |
| 81 | + "Which region holds a copy of the logs from Priya's service?", [1, 2]), |
| 82 | + (["Tomas chairs the Atlas committee.", |
| 83 | + "The Atlas committee meets in the Lighthouse building.", |
| 84 | + "The Lighthouse building is on the Riverside campus."], |
| 85 | + "On which campus does Tomas's committee gather?", [1, 2]), |
| 86 | + (["Yuki maintains the Sable library.", |
| 87 | + "The Sable library depends on the Quartz runtime.", |
| 88 | + "The Quartz runtime is maintained by the platform team."], |
| 89 | + "Which team is responsible for what Yuki's library runs on?", [1, 2]), |
| 90 | + (["Bianca founded the Lumen startup.", |
| 91 | + "The Lumen startup leases space in the Cedar tower.", |
| 92 | + "The Cedar tower is managed by the Hawthorne property group."], |
| 93 | + "Who manages the building where Bianca's startup is located?", [1, 2]), |
| 94 | + (["Rashid coaches the Falcons squad.", |
| 95 | + "The Falcons squad trains at the Birchwood arena.", |
| 96 | + "The Birchwood arena was built by the Stonebridge firm."], |
| 97 | + "Which firm constructed the venue where Rashid's squad practices?", [1, 2]), |
| 98 | + (["Elena curates the Vermillion collection.", |
| 99 | + "The Vermillion collection is stored at the Glasshouse annex.", |
| 100 | + "The Glasshouse annex floods during the monsoon season."], |
| 101 | + "When is the location of Elena's collection at risk?", [1, 2]), |
| 102 | + (["Kwame directs the Solstice fund.", |
| 103 | + "The Solstice fund invests heavily in the Marlin venture.", |
| 104 | + "The Marlin venture operates out of the Harbor district."], |
| 105 | + "In which district does the main investment of Kwame's fund operate?", [1, 2]), |
| 106 | + (["Ingrid wrote the Nimbus proposal.", |
| 107 | + "The Nimbus proposal targets the Greenfield market.", |
| 108 | + "The Greenfield market is regulated by the Tideland authority."], |
| 109 | + "Which authority regulates the market in Ingrid's proposal?", [1, 2]), |
| 110 | + (["Hassan runs the Tundra pipeline.", |
| 111 | + "The Tundra pipeline feeds the Crimson warehouse.", |
| 112 | + "The Crimson warehouse is audited every quarter."], |
| 113 | + "How often is the destination of Hassan's pipeline audited?", [1, 2]), |
| 114 | + (["Noor leads the Zephyr team.", |
| 115 | + "The Zephyr team owns the Ironwood module.", |
| 116 | + "The Ironwood module was deprecated last spring."], |
| 117 | + "What happened to the module owned by Noor's team?", [1, 2]), |
| 118 | + (["Leon manages the Drift account.", |
| 119 | + "The Drift account is billed through the Vantage system.", |
| 120 | + "The Vantage system charges fees in euros."], |
| 121 | + "In which currency are charges applied to Leon's account?", [1, 2]), |
| 122 | + |
| 123 | + # ---- 3-hop ---- |
| 124 | + (["Sofia oversees the Comet program.", |
| 125 | + "The Comet program relies on the Basalt cluster.", |
| 126 | + "The Basalt cluster is powered by the Riverbend grid.", |
| 127 | + "The Riverbend grid draws from the Eastgate dam."], |
| 128 | + "Which dam ultimately supplies power to Sofia's program?", [1, 2, 3]), |
| 129 | + (["Mateo heads the Orchid division.", |
| 130 | + "The Orchid division ships through the Verde port.", |
| 131 | + "The Verde port connects to the Sandbar highway.", |
| 132 | + "The Sandbar highway crosses the Pinecrest county."], |
| 133 | + "Which county does the shipping route of Mateo's division pass through?", [1, 2, 3]), |
| 134 | + (["Aisha sponsors the Lattice scholarship.", |
| 135 | + "The Lattice scholarship is administered by the Brightwater foundation.", |
| 136 | + "The Brightwater foundation banks with the Sterling trust.", |
| 137 | + "The Sterling trust is headquartered in Zurich."], |
| 138 | + "Where is the bank of the foundation behind Aisha's scholarship based?", [1, 2, 3]), |
| 139 | + (["Felix architects the Onyx platform.", |
| 140 | + "The Onyx platform authenticates via the Kestrel gateway.", |
| 141 | + "The Kestrel gateway forwards to the Halcyon directory.", |
| 142 | + "The Halcyon directory syncs nightly with the legacy mainframe."], |
| 143 | + "What does the auth backend of Felix's platform sync with?", [1, 2, 3]), |
| 144 | + (["Greta launched the Pinnacle campaign.", |
| 145 | + "The Pinnacle campaign is tracked in the Cobblestone dashboard.", |
| 146 | + "The Cobblestone dashboard pulls from the Driftwood feed.", |
| 147 | + "The Driftwood feed updates only on weekdays."], |
| 148 | + "How frequently does the underlying data feed of Greta's campaign refresh?", [1, 2, 3]), |
| 149 | + (["Omar manages the Vesper rollout.", |
| 150 | + "The Vesper rollout depends on the Amber toolkit.", |
| 151 | + "The Amber toolkit is licensed from the Quill vendor.", |
| 152 | + "The Quill vendor sunsets support next year."], |
| 153 | + "When does support end for the toolkit behind Omar's rollout?", [1, 2, 3]), |
| 154 | + (["Lena directs the Cascade study.", |
| 155 | + "The Cascade study recruits from the Hollow clinic.", |
| 156 | + "The Hollow clinic partners with the Ridgeline university.", |
| 157 | + "The Ridgeline university is in the Maplewood town."], |
| 158 | + "In which town is the university partnered with the clinic for Lena's study?", [1, 2, 3]), |
| 159 | + (["Bruno owns the Slate franchise.", |
| 160 | + "The Slate franchise sources beans from the Acacia farm.", |
| 161 | + "The Acacia farm sits in the Verdant valley.", |
| 162 | + "The Verdant valley suffers droughts every few years."], |
| 163 | + "What climate problem affects the source region for Bruno's franchise?", [1, 2, 3]), |
| 164 | + (["Carmen leads the Beacon coalition.", |
| 165 | + "The Beacon coalition lobbies the Granite assembly.", |
| 166 | + "The Granite assembly convenes in the Capitol annex.", |
| 167 | + "The Capitol annex is closed for renovation."], |
| 168 | + "What is the status of the venue where the body lobbied by Carmen's coalition meets?", [1, 2, 3]), |
| 169 | + (["Viktor maintains the Cinder framework.", |
| 170 | + "The Cinder framework bundles the Pewter parser.", |
| 171 | + "The Pewter parser was forked from the Driftless project.", |
| 172 | + "The Driftless project is no longer maintained."], |
| 173 | + "What is the state of the upstream project behind Viktor's framework's parser?", [1, 2, 3]), |
| 174 | + (["Tara coordinates the Harbor festival.", |
| 175 | + "The Harbor festival is sponsored by the Goldleaf brewery.", |
| 176 | + "The Goldleaf brewery sources hops from the Willow estate.", |
| 177 | + "The Willow estate was sold to a foreign buyer."], |
| 178 | + "What recently happened to the hop supplier of Tara's festival sponsor?", [1, 2, 3]), |
| 179 | + (["Idris runs the Mosaic exchange.", |
| 180 | + "The Mosaic exchange clears trades through the Pillar bank.", |
| 181 | + "The Pillar bank reports to the Summit regulator.", |
| 182 | + "The Summit regulator imposed new rules this month."], |
| 183 | + "Which oversight body recently changed rules affecting Idris's exchange?", [1, 2, 3]), |
| 184 | +] |
| 185 | + |
| 186 | + |
| 187 | +def main(): |
| 188 | + # Build corpus: all chain link memories + heavy distractors, shuffled. |
| 189 | + chain_mems = [] # flat list of every link memory text |
| 190 | + for links, _q, _req in CHAINS: |
| 191 | + chain_mems.extend(links) |
| 192 | + |
| 193 | + corpus = list(chain_mems) |
| 194 | + |
| 195 | + # Plausible distractors echoing the same "entity does X / X relates to Y" |
| 196 | + # relational shape, so distractors are semantically adjacent to the chains. |
| 197 | + DISTRACT = [ |
| 198 | + "Carlos manages the Aurora reporting suite.", # collides on 'Aurora' |
| 199 | + "The Helios mascot appears at every company event.", # collides on 'Helios' |
| 200 | + "The Frankfurt office hosts the annual summit.", # collides on 'Frankfurt' |
| 201 | + "Nadia leads the Borealis project.", |
| 202 | + "The Borealis project runs on the Cascade database.", |
| 203 | + "The Cascade database lives in the Dublin datacenter.", |
| 204 | + "The Pegasus statue stands in the lobby.", |
| 205 | + "The platform team also owns the build system.", |
| 206 | + "The Harbor district is famous for its seafood.", |
| 207 | + "The Sterling award is given out each spring.", |
| 208 | + "Many startups lease space in the Cedar district downtown.", |
| 209 | + "The Riverbend cafe serves the best coffee nearby.", |
| 210 | + "The Quartz countertops were installed last year.", |
| 211 | + "The Falcons mascot is a large foam bird.", |
| 212 | + "The monsoon season brings heavy rain to the coast.", |
| 213 | + "The euro strengthened against the dollar last week.", |
| 214 | + "The mainframe migration is scheduled for next quarter.", |
| 215 | + "The drought relief fund opened applications today.", |
| 216 | + "The brewery tour runs every Saturday afternoon.", |
| 217 | + "The regulator published its annual report online.", |
| 218 | + ] |
| 219 | + corpus += DISTRACT * 3 |
| 220 | + |
| 221 | + TOPICS = ["the Q3 roadmap", "the migration", "a customer issue", "the offsite", |
| 222 | + "code review", "onboarding", "an incident", "pricing", "the refactor", |
| 223 | + "a contract", "the campaign", "a retro", "the budget", "hiring"] |
| 224 | + ENTS = ["Orion", "Vega", "Nova", "Delta", "Echo", "Sierra", "Tango", "Lima"] |
| 225 | + i = 0 |
| 226 | + while len(corpus) < 3000: |
| 227 | + ent = ENTS[i % len(ENTS)] |
| 228 | + corpus.append(f"On day {i % 365} the {ent} group handled {TOPICS[i % len(TOPICS)]}; note {i} for team {i % 6}.") |
| 229 | + i += 1 |
| 230 | + random.shuffle(corpus) |
| 231 | + |
| 232 | + p, call = srv(DB) |
| 233 | + stored = 0 |
| 234 | + for b in range(0, len(corpus), 100): |
| 235 | + err, out = call("memory_ingest_batch", |
| 236 | + {"items": [{"text": x, "channel": "multihop"} for x in corpus[b:b + 100]]}) |
| 237 | + stored += json.loads(out).get("stored", 0) |
| 238 | + |
| 239 | + err, stats = call("memory_stats", {}) |
| 240 | + st = json.loads(stats) |
| 241 | + print(f"ingested {stored} | index_size={st['index_size']} total={st['total']}\n") |
| 242 | + |
| 243 | + n2 = sum(1 for _l, _q, req in CHAINS if len(req) == 2) |
| 244 | + n3 = sum(1 for _l, _q, req in CHAINS if len(req) == 3) |
| 245 | + print(f"=== MULTI-HOP RELATIONAL RECALL ({len(CHAINS)} chains: {n2} 2-hop, {n3} 3-hop, ~{st['total']} memories) ===") |
| 246 | + |
| 247 | + # hop-recall: over EVERY required hop memory across all chains, what fraction |
| 248 | + # surfaced at rank 0 / <5 / <10. chain-completable: all required hops in top-10. |
| 249 | + hop_total = 0 |
| 250 | + hop_r1 = hop_r5 = hop_r10 = 0 |
| 251 | + chains_complete = 0 |
| 252 | + lat = [] |
| 253 | + incomplete = [] |
| 254 | + |
| 255 | + for links, query, req in CHAINS: |
| 256 | + t = time.time() |
| 257 | + err, out = call("memory_search", {"query": query, "limit": 10}) |
| 258 | + lat.append((time.time() - t) * 1000) |
| 259 | + res = json.loads(out).get("results", []) |
| 260 | + ranks = {} # required link idx -> rank (or -1) |
| 261 | + all_in_top10 = True |
| 262 | + for idx in req: |
| 263 | + target = links[idx] |
| 264 | + rk = next((r_i for r_i, r in enumerate(res) if r.get("text", "") == target), -1) |
| 265 | + ranks[idx] = rk |
| 266 | + hop_total += 1 |
| 267 | + if rk == 0: |
| 268 | + hop_r1 += 1 |
| 269 | + if 0 <= rk < 5: |
| 270 | + hop_r5 += 1 |
| 271 | + if 0 <= rk < 10: |
| 272 | + hop_r10 += 1 |
| 273 | + else: |
| 274 | + all_in_top10 = False |
| 275 | + if all_in_top10: |
| 276 | + chains_complete += 1 |
| 277 | + else: |
| 278 | + missed = [links[idx][:40] for idx in req if not (0 <= ranks[idx] < 10)] |
| 279 | + incomplete.append((query[:50], missed)) |
| 280 | + |
| 281 | + C = len(CHAINS) |
| 282 | + print(f"hop-recall@1 : {hop_r1}/{hop_total} = {hop_r1 / hop_total * 100:.0f}% (required hop memory at rank 0)") |
| 283 | + print(f"hop-recall@5 : {hop_r5}/{hop_total} = {hop_r5 / hop_total * 100:.0f}% (required hop memory in top-5)") |
| 284 | + print(f"hop-recall@10 : {hop_r10}/{hop_total} = {hop_r10 / hop_total * 100:.0f}% (required hop memory in top-10)") |
| 285 | + print(f"chain-completable@10 : {chains_complete}/{C} = {chains_complete / C * 100:.0f}% (ALL required hops in top-10)") |
| 286 | + print(f"latency: p50={statistics.median(lat):.1f}ms p95={sorted(lat)[min(int(0.95 * C), C - 1)]:.1f}ms") |
| 287 | + |
| 288 | + if incomplete: |
| 289 | + print(f"\nINCOMPLETE CHAINS (missing >=1 required hop in top-10), {len(incomplete)}:") |
| 290 | + for q, missed in incomplete: |
| 291 | + print(f" - q: {q}") |
| 292 | + for m in missed: |
| 293 | + print(f" missing hop: {m}") |
| 294 | + |
| 295 | + p.stdin.close() |
| 296 | + p.wait(timeout=10) |
| 297 | + |
| 298 | + |
| 299 | +if __name__ == "__main__": |
| 300 | + try: |
| 301 | + main() |
| 302 | + finally: |
| 303 | + shutil.rmtree(TMP, ignore_errors=True) |
0 commit comments