From c1d790680de4b8dd279c2a227a27bf687f85ab1e Mon Sep 17 00:00:00 2001 From: cyork Date: Sun, 21 Jun 2026 08:15:18 -0400 Subject: [PATCH] feat: provenance & traceability (#6) + remote provenance (#17) - ProvenanceStore: JSON-backed, tracks source/sourceUrl/sourceSystem/ingestedBy/confidence - remoteNodeId/remoteConnector fields for remote ingestion traceability (#17) - HTTP: POST/GET /provenance, DELETE /provenance/:id, GET /provenance/nodes, GET /provenance/nodes/:nodeId - MCP: record_provenance, get_provenance, list_remote_provenance Co-Authored-By: Claude Sonnet 4.6 --- agent-memory-mesh/src/config.ts | 3 + agent-memory-mesh/src/memory/provenance.ts | 93 ++++++++++++++++++++++ agent-memory-mesh/src/service/engine.ts | 31 +++++++- agent-memory-mesh/src/service/http.ts | 32 ++++++++ agent-memory-mesh/src/service/mcp.ts | 67 ++++++++++++++++ agent-memory-mesh/test/provenance.test.ts | 51 ++++++++++++ agent-memory-mesh/test/run-tests.ts | 2 + 7 files changed, 278 insertions(+), 1 deletion(-) create mode 100644 agent-memory-mesh/src/memory/provenance.ts create mode 100644 agent-memory-mesh/test/provenance.test.ts diff --git a/agent-memory-mesh/src/config.ts b/agent-memory-mesh/src/config.ts index 9c507dc..d01acd7 100644 --- a/agent-memory-mesh/src/config.ts +++ b/agent-memory-mesh/src/config.ts @@ -32,6 +32,8 @@ export interface MemoryConfig { * Leave empty for rule-based synthesis (no LLM required). */ consolidationModel: string; + /** JSON file path for provenance records. */ + provenancePath: string; } export function loadConfig(): MemoryConfig { @@ -50,5 +52,6 @@ export function loadConfig(): MemoryConfig { port: Number(env("MEMORY_PORT", "8377")), apiKey: env("MEMORY_API_KEY", ""), consolidationModel: env("CONSOLIDATION_MODEL", ""), + provenancePath: env("PROVENANCE_PATH") || join(base, "provenance.json"), }; } diff --git a/agent-memory-mesh/src/memory/provenance.ts b/agent-memory-mesh/src/memory/provenance.ts new file mode 100644 index 0000000..06d1b3c --- /dev/null +++ b/agent-memory-mesh/src/memory/provenance.ts @@ -0,0 +1,93 @@ +import { readFileSync, writeFileSync, existsSync } from "node:fs"; +import { randomUUID } from "node:crypto"; + +export type ProvenanceSource = "vault" | "indexer" | "work-memory" | "remote" | "manual"; + +export interface ProvenanceRecord { + id: string; + notePath: string; + source: ProvenanceSource; + sourceUrl?: string; + sourceSystem?: string; + ingestedAt: string; + ingestedBy?: string; + sessionId?: string; + confidence?: number; + remoteNodeId?: string; + remoteConnector?: string; + metadata?: Record; +} + +export interface ProvenanceFilter { + source?: ProvenanceSource; + since?: string; + remoteNodeId?: string; + notePath?: string; +} + +export class ProvenanceStore { + private records: ProvenanceRecord[] = []; + + constructor(private filePath: string) { + if (existsSync(filePath)) { + this.records = JSON.parse(readFileSync(filePath, "utf8")); + } + } + + private save(): void { + writeFileSync(this.filePath, JSON.stringify(this.records, null, 2)); + } + + record(entry: Omit): ProvenanceRecord { + const rec: ProvenanceRecord = { + ...entry, + id: randomUUID(), + ingestedAt: new Date().toISOString(), + }; + this.records.push(rec); + this.save(); + return rec; + } + + getByNotePath(notePath: string): ProvenanceRecord[] { + return this.records.filter((r) => r.notePath === notePath); + } + + list(filter?: ProvenanceFilter): ProvenanceRecord[] { + let results = this.records; + if (filter?.source) results = results.filter((r) => r.source === filter.source); + if (filter?.since) results = results.filter((r) => r.ingestedAt >= filter.since!); + if (filter?.remoteNodeId) results = results.filter((r) => r.remoteNodeId === filter.remoteNodeId); + if (filter?.notePath) results = results.filter((r) => r.notePath === filter.notePath); + return results; + } + + delete(id: string): boolean { + const before = this.records.length; + this.records = this.records.filter((r) => r.id !== id); + if (this.records.length < before) { + this.save(); + return true; + } + return false; + } + + listByRemoteNode(remoteNodeId: string): ProvenanceRecord[] { + return this.records.filter((r) => r.remoteNodeId === remoteNodeId); + } + + summaryByNode(): { remoteNodeId: string; count: number; lastSync: string }[] { + const byNode = new Map(); + for (const r of this.records) { + if (!r.remoteNodeId) continue; + const existing = byNode.get(r.remoteNodeId); + if (!existing) { + byNode.set(r.remoteNodeId, { count: 1, lastSync: r.ingestedAt }); + } else { + existing.count++; + if (r.ingestedAt > existing.lastSync) existing.lastSync = r.ingestedAt; + } + } + return [...byNode.entries()].map(([remoteNodeId, v]) => ({ remoteNodeId, ...v })); + } +} diff --git a/agent-memory-mesh/src/service/engine.ts b/agent-memory-mesh/src/service/engine.ts index 3e39b20..4f96ffc 100644 --- a/agent-memory-mesh/src/service/engine.ts +++ b/agent-memory-mesh/src/service/engine.ts @@ -17,8 +17,9 @@ import { Consolidator, type ConsolidationResult } from "../memory/consolidator.j import { ContextGraph, type ContextGraphEntity, type ContextGraphEdge, type EntityType, type NeighborResult } from "../memory/context-graph.js"; import { RetrievalPolicyStore, applyRecencyBoost, type RetrievalPolicy } from "../memory/retrieval-policy.js"; import { FeedbackStore, applyFeedbackScoring, type FeedbackSignal, type NoteFeedback } from "../memory/feedback.js"; +import { ProvenanceStore, type ProvenanceRecord, type ProvenanceFilter, type ProvenanceSource } from "../memory/provenance.js"; -export { type WorkMemoryEntry, type WorkMemoryQuery, type ConsolidationResult, type ContextGraphEntity, type ContextGraphEdge, type EntityType, type NeighborResult, type RetrievalPolicy, type FeedbackSignal, type NoteFeedback }; +export { type WorkMemoryEntry, type WorkMemoryQuery, type ConsolidationResult, type ContextGraphEntity, type ContextGraphEdge, type EntityType, type NeighborResult, type RetrievalPolicy, type FeedbackSignal, type NoteFeedback, type ProvenanceRecord, type ProvenanceFilter, type ProvenanceSource }; export interface WikiSummary { entityId: string; @@ -41,6 +42,7 @@ export class MemoryEngine { private graph: ContextGraph; private policies: RetrievalPolicyStore; private feedback: FeedbackStore; + private provenance: ProvenanceStore; private opened = false; constructor(private cfg: MemoryConfig) { @@ -58,6 +60,7 @@ export class MemoryEngine { }); this.policies = new RetrievalPolicyStore(cfg.policiesPath); this.feedback = new FeedbackStore(cfg.feedbackPath); + this.provenance = new ProvenanceStore(cfg.provenancePath); } /** Open the store lazily, sizing the table from a probe embedding the first time. */ @@ -267,6 +270,32 @@ export class MemoryEngine { return { processed: processedIds.length, signals }; } + // Provenance & traceability + + recordProvenance(entry: Omit): ProvenanceRecord { + return this.provenance.record(entry); + } + + getProvenance(notePath: string): ProvenanceRecord[] { + return this.provenance.getByNotePath(notePath); + } + + listProvenance(filter?: ProvenanceFilter): ProvenanceRecord[] { + return this.provenance.list(filter); + } + + deleteProvenance(id: string): boolean { + return this.provenance.delete(id); + } + + listProvenanceByNode(remoteNodeId: string): ProvenanceRecord[] { + return this.provenance.listByRemoteNode(remoteNodeId); + } + + provenanceSummaryByNode() { + return this.provenance.summaryByNode(); + } + private collectWikis(query: string, types?: EntityType[]): WikiSummary[] { const matches = this.graph.findByName(query); const candidates = types ? matches.filter((e) => types.includes(e.type)) : matches; diff --git a/agent-memory-mesh/src/service/http.ts b/agent-memory-mesh/src/service/http.ts index f1750c1..67bf06f 100644 --- a/agent-memory-mesh/src/service/http.ts +++ b/agent-memory-mesh/src/service/http.ts @@ -279,6 +279,38 @@ export function startHttp( return send(res, 200, { entity }); } + // Provenance & traceability endpoints + if (req.method === "POST" && url.pathname === "/provenance") { + const body = await readJson(req); + if (!body.notePath) return send(res, 400, { error: "notePath is required" }); + if (!body.source) return send(res, 400, { error: "source is required" }); + const record = engine.recordProvenance(body); + return send(res, 201, { record }); + } + if (req.method === "GET" && url.pathname === "/provenance/nodes") { + return send(res, 200, { nodes: engine.provenanceSummaryByNode() }); + } + const provenanceNodeMatch = url.pathname.match(/^\/provenance\/nodes\/([^/]+)$/); + if (req.method === "GET" && provenanceNodeMatch) { + const nodeId = decodeURIComponent(provenanceNodeMatch[1]); + return send(res, 200, { records: engine.listProvenanceByNode(nodeId) }); + } + if (req.method === "GET" && url.pathname === "/provenance") { + const q = url.searchParams; + const records = engine.listProvenance({ + source: (q.get("source") as any) ?? undefined, + since: q.get("since") ?? undefined, + remoteNodeId: q.get("remoteNodeId") ?? undefined, + notePath: q.get("notePath") ?? undefined, + }); + return send(res, 200, { records }); + } + const provenanceIdMatch = url.pathname.match(/^\/provenance\/([^/]+)$/); + if (req.method === "DELETE" && provenanceIdMatch) { + const ok = engine.deleteProvenance(decodeURIComponent(provenanceIdMatch[1])); + return send(res, ok ? 200 : 404, { ok }); + } + return send(res, 404, { error: "not found" }); } catch (err) { return send(res, 500, { error: String(err) }); diff --git a/agent-memory-mesh/src/service/mcp.ts b/agent-memory-mesh/src/service/mcp.ts index 6560f08..60591bf 100644 --- a/agent-memory-mesh/src/service/mcp.ts +++ b/agent-memory-mesh/src/service/mcp.ts @@ -359,6 +359,73 @@ export async function startMcpStdio(cfg: MemoryConfig, engine: MemoryEngine): Pr } ); + server.registerTool( + "record_provenance", + { + title: "Record provenance", + description: "Record provenance metadata for a vault note — who ingested it, from where, and when. Use after any indexing or ingestion operation.", + inputSchema: { + notePath: z.string().describe("Vault-relative note path."), + source: z.enum(["vault", "indexer", "work-memory", "remote", "manual"]).describe("Ingestion source type."), + sourceUrl: z.string().optional().describe("Original URL or file path."), + sourceSystem: z.string().optional().describe("Source system name, e.g. 'onedrive', 'local-fs'."), + ingestedBy: z.string().optional().describe("Agent ID or connector name."), + sessionId: z.string().optional(), + confidence: z.number().optional().describe("Confidence 0-1."), + remoteNodeId: z.string().optional().describe("Remote node ID for remote ingestion."), + remoteConnector: z.string().optional().describe("Remote connector name."), + metadata: z.record(z.unknown()).optional(), + }, + }, + async (input) => { + const record = engine.recordProvenance(input as any); + return { content: [{ type: "text", text: JSON.stringify(record, null, 2) }] }; + } + ); + + server.registerTool( + "get_provenance", + { + title: "Get note provenance", + description: "Get all provenance records for a vault note path.", + inputSchema: { + notePath: z.string().describe("Vault-relative note path."), + }, + }, + async ({ notePath }) => { + const records = engine.getProvenance(notePath); + const text = records.length + ? records + .map( + (r) => + `[${r.ingestedAt}] ${r.source}${r.ingestedBy ? ` by ${r.ingestedBy}` : ""}${r.sourceUrl ? ` from ${r.sourceUrl}` : ""}` + ) + .join("\n") + : "No provenance records found."; + return { content: [{ type: "text", text }] }; + } + ); + + server.registerTool( + "list_remote_provenance", + { + title: "List remote node provenance", + description: "List all provenance records from a specific remote ingestion node.", + inputSchema: { + remoteNodeId: z.string().describe("The remote node ID."), + }, + }, + async ({ remoteNodeId }) => { + const records = engine.listProvenanceByNode(remoteNodeId); + const text = records.length + ? records + .map((r) => `[${r.ingestedAt}] ${r.notePath} (${r.remoteConnector ?? "unknown connector"})`) + .join("\n") + : "No provenance records for this node."; + return { content: [{ type: "text", text }] }; + } + ); + const transport = new StdioServerTransport(); await server.connect(transport); console.error("[mcp] agent-memory-mesh stdio server ready"); diff --git a/agent-memory-mesh/test/provenance.test.ts b/agent-memory-mesh/test/provenance.test.ts new file mode 100644 index 0000000..b53d11b --- /dev/null +++ b/agent-memory-mesh/test/provenance.test.ts @@ -0,0 +1,51 @@ +import { ProvenanceStore } from "../src/memory/provenance.js"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { randomUUID } from "node:crypto"; + +export async function runProvenanceTests() { + const file = join(tmpdir(), `provenance-test-${randomUUID()}.json`); + const store = new ProvenanceStore(file); + + const rec = store.record({ notePath: "notes/foo.md", source: "vault", ingestedBy: "test-agent" }); + console.assert(rec.id.length > 0, "id generated"); + console.assert(rec.ingestedAt.length > 0, "ingestedAt set"); + console.assert(rec.notePath === "notes/foo.md", "notePath correct"); + console.assert(rec.source === "vault", "source correct"); + + const byPath = store.getByNotePath("notes/foo.md"); + console.assert(byPath.length === 1, `getByNotePath should return 1, got ${byPath.length}`); + console.assert(store.getByNotePath("notes/bar.md").length === 0, "unknown path returns 0"); + + store.record({ notePath: "notes/bar.md", source: "remote", remoteNodeId: "node-abc", remoteConnector: "onedrive" }); + + console.assert(store.list({ source: "vault" }).length === 1, "filter by source=vault"); + console.assert(store.list({ source: "remote" }).length === 1, "filter by source=remote"); + console.assert(store.list({ remoteNodeId: "node-abc" }).length === 1, "filter by remoteNodeId"); + + const future = new Date(Date.now() + 86400000).toISOString(); + console.assert(store.list({ since: future }).length === 0, "filter by future since returns 0"); + console.assert(store.list({ notePath: "notes/foo.md" }).length === 1, "filter by notePath"); + + const byNode = store.listByRemoteNode("node-abc"); + console.assert(byNode.length === 1, `listByRemoteNode should return 1, got ${byNode.length}`); + + // summaryByNode + store.record({ notePath: "notes/baz.md", source: "remote", remoteNodeId: "node-abc" }); + const summary = store.summaryByNode(); + const nodeEntry = summary.find((s) => s.remoteNodeId === "node-abc"); + console.assert(nodeEntry !== undefined, "node-abc in summary"); + console.assert(nodeEntry!.count === 2, `count should be 2, got ${nodeEntry!.count}`); + console.assert(nodeEntry!.lastSync.length > 0, "lastSync set"); + + // delete + console.assert(store.delete(rec.id) === true, "delete returns true"); + console.assert(store.delete("nonexistent-id") === false, "delete nonexistent returns false"); + console.assert(store.getByNotePath("notes/foo.md").length === 0, "deleted record gone"); + + // Persistence + const store2 = new ProvenanceStore(file); + console.assert(store2.list().length > 0, "records persist across reload"); + + console.log("[provenance] All tests passed"); +} diff --git a/agent-memory-mesh/test/run-tests.ts b/agent-memory-mesh/test/run-tests.ts index d1ca5c2..cc9ebae 100644 --- a/agent-memory-mesh/test/run-tests.ts +++ b/agent-memory-mesh/test/run-tests.ts @@ -5,6 +5,7 @@ import { runConsolidatorTests } from "./consolidator.test.js"; import { runContextGraphTests } from "./context-graph.test.js"; import { runRetrievalPolicyTests } from "./retrieval-policy.test.js"; import { runFeedbackTests } from "./feedback.test.js"; +import { runProvenanceTests } from "./provenance.test.js"; async function main() { let exitCode = 0; @@ -14,6 +15,7 @@ async function main() { { name: "ContextGraph", fn: runContextGraphTests }, { name: "RetrievalPolicy", fn: runRetrievalPolicyTests }, { name: "Feedback", fn: runFeedbackTests }, + { name: "Provenance", fn: runProvenanceTests }, ]; for (const suite of suites) {