From c131f029cbc81d8f9127e0328a02755bf46a9756 Mon Sep 17 00:00:00 2001 From: Zack Katz Date: Tue, 11 Aug 2026 16:34:31 -0400 Subject: [PATCH 01/33] fix(qdrant): index the `type` payload field so recall filters stop 400ing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `_vector_search` always builds its filter with `excluded_types=RECALL_EXCLUDED_TYPES`, and that setting defaults to `MetaPattern` — a non-empty value — so every vector search sends Qdrant a condition on the `type` payload field. `ensure_qdrant_collection` only ever indexed `tags` and `tag_prefixes`, and Qdrant rejects an entire search with 400 when a filter references an unindexed field: Bad request: Index required but not found for "type" of one of the following types: [keyword]. `_vector_search` catches that broadly and returns `[]`, so recall silently degrades to keyword-only rather than surfacing an error. On a live instance the symptom is `vector_search.matched: false` on every query, `vector_count: null` in `/health`, and semantically-phrased queries returning unrelated keyword collisions — with nothing in the response indicating that vector search is failing. Add `type` to the ensured indexes, collapsing the duplicated enum/string branches into one loop over a named tuple so the set of filtered fields stays visible in one place. Verified against a production instance (~25k points): creating the `type` payload index restored `vector_search.matched: true` and `match_type: "vector"` results immediately, with no other change. --- automem/stores/runtime_clients.py | 28 +++++++-------------- tests/test_vector_size_safety.py | 42 +++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 19 deletions(-) diff --git a/automem/stores/runtime_clients.py b/automem/stores/runtime_clients.py index b7ae939..61fb369 100644 --- a/automem/stores/runtime_clients.py +++ b/automem/stores/runtime_clients.py @@ -3,6 +3,11 @@ import os from typing import Any, Callable +# Qdrant rejects an entire search with 400 when a filter references an unindexed payload +# field, so every field recall filters on must be indexed. `type` backs RECALL_EXCLUDED_TYPES, +# which applies to all vector searches since that setting defaults to a non-empty value. +QDRANT_FILTERED_PAYLOAD_FIELDS = ("tags", "tag_prefixes", "type") + def init_falkordb( *, @@ -157,27 +162,12 @@ def ensure_qdrant_collection( return logger.info("Ensuring Qdrant payload indexes for collection '%s'", collection_name) - if payload_schema_type_enum: - state.qdrant.create_payload_index( - collection_name=collection_name, - field_name="tags", - field_schema=payload_schema_type_enum.KEYWORD, - ) - state.qdrant.create_payload_index( - collection_name=collection_name, - field_name="tag_prefixes", - field_schema=payload_schema_type_enum.KEYWORD, - ) - else: - state.qdrant.create_payload_index( - collection_name=collection_name, - field_name="tags", - field_schema="keyword", - ) + keyword_schema = payload_schema_type_enum.KEYWORD if payload_schema_type_enum else "keyword" + for field_name in QDRANT_FILTERED_PAYLOAD_FIELDS: state.qdrant.create_payload_index( collection_name=collection_name, - field_name="tag_prefixes", - field_schema="keyword", + field_name=field_name, + field_schema=keyword_schema, ) except ValueError: raise diff --git a/tests/test_vector_size_safety.py b/tests/test_vector_size_safety.py index 02bd80c..4485c6b 100644 --- a/tests/test_vector_size_safety.py +++ b/tests/test_vector_size_safety.py @@ -251,6 +251,48 @@ def bad_config(): class TestEnsureQdrantCollectionPayloadIndexes: + def _run_ensure(self, payload_schema_type_enum): + from automem.stores.runtime_clients import ensure_qdrant_collection + + qdrant = MagicMock() + qdrant.get_collections.return_value = SimpleNamespace( + collections=[SimpleNamespace(name="memories")] + ) + state = SimpleNamespace(qdrant=qdrant, effective_vector_size=None) + + ensure_qdrant_collection( + state=state, + logger=MagicMock(), + collection_name="memories", + vector_size_config=1024, + get_effective_vector_size_fn=lambda _client: (1024, "collection"), + vector_params_cls=MagicMock(), + distance_enum=SimpleNamespace(COSINE="Cosine"), + payload_schema_type_enum=payload_schema_type_enum, + ) + return qdrant + + def test_indexes_every_field_recall_filters_on(self): + """`type` backs RECALL_EXCLUDED_TYPES; without an index Qdrant 400s every search.""" + qdrant = self._run_ensure(SimpleNamespace(KEYWORD="keyword")) + + indexed = { + call.kwargs["field_name"] for call in qdrant.create_payload_index.call_args_list + } + assert indexed == {"tags", "tag_prefixes", "type"} + + def test_indexes_every_field_without_payload_schema_enum(self): + qdrant = self._run_ensure(None) + + indexed = { + call.kwargs["field_name"] for call in qdrant.create_payload_index.call_args_list + } + assert indexed == {"tags", "tag_prefixes", "type"} + assert all( + call.kwargs["field_schema"] == "keyword" + for call in qdrant.create_payload_index.call_args_list + ) + @patch.dict("os.environ", {"QDRANT_ENSURE_PAYLOAD_INDEXES": "false"}, clear=False) def test_can_skip_payload_indexes_for_restore_tuned_lab_collection(self): from automem.stores.runtime_clients import ensure_qdrant_collection From 92b98ec8cfe85cfcd6e5d939f61dfe059bcb7b3f Mon Sep 17 00:00:00 2001 From: Jack Arturo Date: Fri, 14 Aug 2026 13:40:34 -0400 Subject: [PATCH 02/33] docs(mcp): audit remote vs stdio transport parity Records where the streamable-HTTP/SSE bridge and the stdio package had drifted apart, and pins the short list of differences that stay intentional. Follow-up to #224, which fixed one symptom of the split. Co-Authored-By: Claude Opus 5 --- docs/MCP_TRANSPORT_PARITY.md | 136 +++++++++++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 docs/MCP_TRANSPORT_PARITY.md diff --git a/docs/MCP_TRANSPORT_PARITY.md b/docs/MCP_TRANSPORT_PARITY.md new file mode 100644 index 0000000..77fe71e --- /dev/null +++ b/docs/MCP_TRANSPORT_PARITY.md @@ -0,0 +1,136 @@ +# MCP Transport Parity + +AutoMem ships its MCP surface over two transports: + +| Transport | Implementation | Package / path | +| ---------- | ------------------------------------------- | --------------------------------------------- | +| **stdio** | `@verygoodplugins/mcp-automem` (TypeScript) | [verygoodplugins/mcp-automem](https://github.com/verygoodplugins/mcp-automem) | +| **remote** | Streamable HTTP + SSE bridge (ESM JS) | `mcp-sse-server/server.js` (this repo) | + +`mcp-automem`'s `server.json` publishes stdio, streamable-HTTP, and SSE as **one +server with one shared 6-tool array**. That is the contract: a client that picks +"AutoMem" out of the registry must get the same tools, the same request mapping, +and the same rendered output regardless of which transport it connected over. + +This document records the audit that established where the two had drifted, and +the short list of differences that remain intentional. + +## Why this exists + +[PR #224](https://github.com/verygoodplugins/automem/pull/224) fixed one symptom: +the remote bridge's compact recall block dropped the memory's stored date. An +agent replaying that text read a two-week-old itinerary as today's plan, because +nothing in the output said how old the memory was. AutoHub's parser already +looked for a `Created:` line — the remote transport simply never emitted one, +while the stdio package always had. + +That single missing line turned out to be one instance of a much wider split. A +manual schema sync had already been attempted once +(`d99b86d fix(mcp-sse): sync tool schemas for SSE/MCP parity (#104)`) and had +since drifted again, which is the case against maintaining two hand-synced +copies. + +## Audit: where the transports had diverged + +Snapshot taken against `mcp-automem` 0.15.0 and `mcp-sse-server` at +`4b5eaaf`. Both transports registered the same six tool names in the same order — +`store_memory`, `recall_memory`, `associate_memories`, `update_memory`, +`delete_memory`, `check_database_health`. Everything below that differed. + +### Tool definitions + +| Axis | stdio | remote | +| --------------------------------- | -------------------------------------------------------------------- | ----------------------------------------------------------- | +| `description` | Multi-paragraph: modes, "When to use", "Examples" | One line each | +| `title` | Present (and duplicated in `annotations.title`) | Absent | +| `outputSchema` | On all 6 | On none | +| `annotations` | `readOnlyHint`, `destructiveHint`, `idempotentHint`, `openWorldHint` | `readOnlyHint`, `destructiveHint` only | +| `_meta['anthropic/alwaysLoad']` | On `store_memory`, `recall_memory`, `associate_memories` | Absent | +| server `instructions` | Set (~450 chars) | Not set | +| `serverInfo` | `mcp-automem` / package version | `automem-mcp-sse` / `0.1.0`, disagreeing with its own `0.2.0` | + +### Capability missing on the remote transport + +- **`recall_memory`** — 38 params vs 27. Absent remotely: `memory_id` (ID-fetch + mode), `exhaustive` + `offset` (tag-enumeration mode), `exclude_tags`, + `current_only`, `state_mode`, `state_debug`, `recency_bias`, `min_score`, + `adaptive_floor`, `expand_respect_tags`. `limit` capped at 50 remotely vs 200 + on stdio. stdio declared defaults (`expansion_limit: 25`, `relation_limit: 5`, + `current_only: true`); remote declared none. +- **`store_memory`** — no `memories[]` batch mode, and no supersede mode + (`supersedes_memory_id` / `supersede_relation` / `supersede_reason`, a + four-call sequence with a compensating delete on partial failure). +- **`delete_memory`** — no bulk-delete-by-tag. +- **`associate_memories`** — none of the nine relation-specific properties + (`context`, `reason`, `pattern_type`, `confidence`, `resolution`, + `observations`, `timestamp`, `transformation`, `role`). `additionalProperties: + true` let them through unvalidated, but an agent reading the schema never + learned to send them. +- **`update_memory`** — remote advertised `embedding` and forwarded it; + `PATCH /memory/` (`automem/api/memory.py:789`) never reads it. Dead field. + +### Response rendering + +| Call | stdio | remote | +| ----------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------- | +| recall `text` | `1. [tags] (importance: raw) score=0.123 [match] relations=N`, ` ID:`, ` Created: … Updated: …`; summary-first, 400-char preview, 18k-token budget, trailer | `1. [tags] score=0.123`, ` ID:` — no date, no budget, no summary | +| recall `detailed` | 2-space indent, no numbering, `Created:` | no indent, numbered, `Timestamp:` | +| recall `items` | `[] ` blocks, no header | `Found N memories:` header block plus compact blocks | +| recall `json` | own structured envelope | raw upstream response | +| recall empty | `No memories found matching your query.` | `No memories found.` | +| `store_memory` | `Memory stored successfully!\n\nMemory ID: ` | `Memory stored: ` | +| `update_memory` | `Memory updated successfully!` | `Updated ` | +| `delete_memory` | `Memory deleted successfully!` | `Deleted ` | +| `check_database_health` | formatted block with a status emoji | `JSON.stringify(r)` | +| error | `Error: ` | `AutoMem error: (request_id: …)` | +| `structuredContent` | on all six successes | never | + +### Not a parity gap: `store_memory.id` + +Both transports advertised an `id` parameter, and +`automem/api/memory.py:475` mints a server-side UUID unconditionally ("Always +generate server-side UUID to prevent collision/overwrite attacks"). Neither +transport could honor it. Rather than copy the lie into parity, `id` was dropped +from the shared schema. + +## Accepted transport-level differences + +These are intentional and are **not** parity violations. The differential harness +allowlists exactly these and nothing else. + +| Difference | remote | stdio | Why | +| ---------------------- | ------------------------------------------------------------------------------------- | ---------------------------- | -------------------------------------------------------------------------------------------------------------------- | +| `serverInfo.name` | `automem-mcp-sse` | `mcp-automem` | Clients must be able to tell the transports apart. | +| Auth mechanisms | `Authorization: Bearer`, `X-API-Key` / `X-API-Token`, `?api_key=` / `?apiKey=` / `?api_token=` | `Authorization: Bearer` only | Browser and EventSource clients cannot set headers. | +| Error text suffix | ` (request_id: )` | none | Observability on a hosted service. Same code path — the bridge supplies a `requestIdProvider`, stdio does not. | +| Timeout / retry policy | `UPSTREAM_TIMEOUT_MS` (15 s), `UPSTREAM_MAX_RETRIES` (2) | 25 s, 3 | Tuned per deployment; injected via `AutoMemConfig`. | +| Transport surface | `/mcp`, `/mcp/sse`, `/mcp/messages`, `/health`, `/ready`, `/alexa` | stdio only | Not part of the MCP tool surface. | + +Everything else — tool names, order, `title`, `description`, `inputSchema`, +`outputSchema`, `annotations`, `_meta`, server `instructions`, and every byte of +rendered `tools/call` text — must be identical. + +## How parity is enforced + +A differential harness drives both transports against a single +docker-compose AutoMem and diffs them: + +```bash +make test-parity +``` + +It lives in `mcp-sse-server/parity/` with its entry point at +`mcp-sse-server/test/parity.test.js`, and it asserts three things: + +1. `tools/list` is deep-equal across transports after key-order normalization. +2. Server capabilities and `instructions` match; `serverInfo.name` is allowlisted + to differ. +3. A 19-scenario `tools/call` matrix renders identical text on both, after + redacting values that legitimately vary per run (UUIDs, timestamps, scores, + `query_time_ms`, and the per-transport tag namespace). + +The harness is gated on `AUTOMEM_RUN_PARITY_TESTS=1` because it needs a live +service. Without the gate it skips cleanly, which is what CI's `node-test` job +sees. `.github/workflows/mcp-parity.yml` runs it with the stack up on PRs that +touch `mcp-sse-server/**`, `automem/api/**`, or `app.py`, plus weekly as a drift +alarm against newly published `mcp-automem` versions. From ee844bb5673c3b739bbc06301dbeab691f3dc0e2 Mon Sep 17 00:00:00 2001 From: Jack Arturo Date: Fri, 14 Aug 2026 13:43:56 -0400 Subject: [PATCH 03/33] test(mcp): add gated cross-transport parity harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Connects one MCP client to each transport — the bridge in-process over streamable HTTP, the published stdio package as a child process — and diffs tools/list and the initialize result. Both point at the same live AutoMem, so any difference is the transports, not the data. Red as of this commit: remote omits _meta, title, outputSchema, two annotation hints, and server instructions, and its descriptions are one-liners. Pins @modelcontextprotocol/sdk to 1.20.0 so the new devDependency's newer SDK gets nested rather than hoisted over the deployed bridge's. Co-Authored-By: Claude Opus 5 --- mcp-sse-server/package-lock.json | 1502 ++++++++++++++++++++++++++-- mcp-sse-server/package.json | 5 +- mcp-sse-server/parity/clients.js | 68 ++ mcp-sse-server/parity/normalize.js | 46 + mcp-sse-server/test/parity.test.js | 46 + 5 files changed, 1563 insertions(+), 104 deletions(-) create mode 100644 mcp-sse-server/parity/clients.js create mode 100644 mcp-sse-server/parity/normalize.js create mode 100644 mcp-sse-server/test/parity.test.js diff --git a/mcp-sse-server/package-lock.json b/mcp-sse-server/package-lock.json index ac2750b..ddbac67 100644 --- a/mcp-sse-server/package-lock.json +++ b/mcp-sse-server/package-lock.json @@ -1,15 +1,392 @@ { "name": "automem-mcp-sse-server", - "version": "0.1.0", + "version": "0.2.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "automem-mcp-sse-server", - "version": "0.1.0", + "version": "0.2.0", "dependencies": { - "@modelcontextprotocol/sdk": "^1.20.0", + "@modelcontextprotocol/sdk": "1.20.0", "express": "^4.19.2" + }, + "devDependencies": { + "@verygoodplugins/mcp-automem": "^0.15.0" + } + }, + "node_modules/@hono/node-server": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-2.1.1.tgz", + "integrity": "sha512-ELuehkj5VCBdgEw9zs+ivkKwyzzUCSQuE96YmiPvn1ECBoZCczbFXJLeEGMTYjphP6gydh4pHMqEYPVMYUVgQg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "hono": "^4" + } + }, + "node_modules/@inquirer/ansi": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/@inquirer/ansi/-/ansi-2.0.7.tgz", + "integrity": "sha512-3eTuUO1vH2cZm2ZKHeQxnOqlTi9EfZDGgIe3BL3I4u+rJHocr9Fz86M4fjYABPvFnQG/gGK551HqDiIcETwU6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + } + }, + "node_modules/@inquirer/checkbox": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/@inquirer/checkbox/-/checkbox-5.2.1.tgz", + "integrity": "sha512-b6xmA/VlTe0ZgDQHDui+Nav470u7u49nRd8/iuhOcQPO9Ch7lGuogydhi2VOmNlZ+zXcM8IcPuNSwQcdJaF/kw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/ansi": "^2.0.7", + "@inquirer/core": "^11.2.1", + "@inquirer/figures": "^2.0.7", + "@inquirer/type": "^4.0.7" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/confirm": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/@inquirer/confirm/-/confirm-6.1.1.tgz", + "integrity": "sha512-eb8DBZcz/2qHWQda4rk2JiQk5h9QV/cVHi1yjt0f69WFZMRFn0sJTye3EAP8icut8UDMjQPsaH5KbcOogefrFQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/core": "^11.2.1", + "@inquirer/type": "^4.0.7" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/core": { + "version": "11.2.1", + "resolved": "https://registry.npmjs.org/@inquirer/core/-/core-11.2.1.tgz", + "integrity": "sha512-Qd6GJT1yVyrZZCfN8W2qKF5ApmqryXRhRKCuip8h01x2w/esJQ2XIYc6f9abMIHgKQdBfFTSOdbHRLAhuM09UA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/ansi": "^2.0.7", + "@inquirer/figures": "^2.0.7", + "@inquirer/type": "^4.0.7", + "cli-width": "^4.1.0", + "fast-wrap-ansi": "^0.2.0", + "mute-stream": "^3.0.0", + "signal-exit": "^4.1.0" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/editor": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/@inquirer/editor/-/editor-5.2.2.tgz", + "integrity": "sha512-ZRVd/oD+sYsUd5zVm0NflqEzlqfYCyHNsqkHl2oWXEUHs12tCbcSFi+wVFEvD8+LGRaMUsVrE7qeo6lSG/S1Vg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/core": "^11.2.1", + "@inquirer/external-editor": "^3.0.3", + "@inquirer/type": "^4.0.7" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/expand": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/@inquirer/expand/-/expand-5.1.1.tgz", + "integrity": "sha512-YmQpenjbFSHAK3sOd44puHh3V1KXXr+JiNpUztoSQ4drLh2rTVzTap/YtlAVu/5xavifIlBfNEzJ/neZJ1a/1g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/core": "^11.2.1", + "@inquirer/type": "^4.0.7" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/external-editor": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@inquirer/external-editor/-/external-editor-3.0.3.tgz", + "integrity": "sha512-6thf5I8q7lZwzGLAxPaaGEREEkZ3nyePPDQ1oyobblxmEE8mqTLguScP7pDjUTAibiyb4hfXl+qjUEJ+di/aNA==", + "dev": true, + "license": "MIT", + "dependencies": { + "chardet": "^2.1.1", + "iconv-lite": "^0.7.2" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/external-editor/node_modules/iconv-lite": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", + "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@inquirer/figures": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/@inquirer/figures/-/figures-2.0.7.tgz", + "integrity": "sha512-aJ8TBPOGB6f/2qziPfElISTCEd5XOYTFckA2SGjhNmiKzfK/u4ot3v0DUzGVdUnKjN10EqnnEPck36BkyfLnJw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + } + }, + "node_modules/@inquirer/input": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/@inquirer/input/-/input-5.1.2.tgz", + "integrity": "sha512-9K/DDBSQpOyZSkt6sOVP9Vo0TR7atX2kuILsUu0x3wVcVbe97lJwIJKMLdMw25tDYuXl/qp6erT0Xs1rfmcfZg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/core": "^11.2.1", + "@inquirer/type": "^4.0.7" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/number": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/@inquirer/number/-/number-4.1.1.tgz", + "integrity": "sha512-XF4IXAbPnGPgw0wsbC/i2tPcyfdZgDpUlhsqU0SfT4IRIGWha6Xm9VRgN5yYxJq+jnyXlfXI/nQ3ulfk0iEICA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/core": "^11.2.1", + "@inquirer/type": "^4.0.7" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/password": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/@inquirer/password/-/password-5.1.1.tgz", + "integrity": "sha512-3XBfF7DAsp5qeDsvN5Rd1HmbNokVvEQoUM0QLrRcybC9nX96w3Pbmu7qUsb3IT3J3jBvs2+mTXaKHOUsgHMLzg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/ansi": "^2.0.7", + "@inquirer/core": "^11.2.1", + "@inquirer/type": "^4.0.7" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/prompts": { + "version": "8.5.2", + "resolved": "https://registry.npmjs.org/@inquirer/prompts/-/prompts-8.5.2.tgz", + "integrity": "sha512-IYR/3C/paEVVQYQvdDlFZVjRCJVYHHON0XXMH91KO9GSxs0TdKYWlUdvfQl2EfAHDxUaN3IBffkE/BDTh5nJ6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/checkbox": "^5.2.1", + "@inquirer/confirm": "^6.1.1", + "@inquirer/editor": "^5.2.2", + "@inquirer/expand": "^5.1.1", + "@inquirer/input": "^5.1.2", + "@inquirer/number": "^4.1.1", + "@inquirer/password": "^5.1.1", + "@inquirer/rawlist": "^5.3.1", + "@inquirer/search": "^4.2.1", + "@inquirer/select": "^5.2.1" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/rawlist": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/@inquirer/rawlist/-/rawlist-5.3.1.tgz", + "integrity": "sha512-QqdTqQddL3qPX/PPrjobpsO25NZ4dWXgTLenrR445L2ptLEYE6Z+PD5c5CNDJNx4ugRgELAIpSIJxZaO2jJ2Og==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/core": "^11.2.1", + "@inquirer/type": "^4.0.7" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/search": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/@inquirer/search/-/search-4.2.1.tgz", + "integrity": "sha512-xJj8QWKRSrfKoBIITLZK61dD3zwo0Rz11fgDImku30/Oe81zMdIdGgrLY2h6RkJ+KZ/GhNYIRMKnH/62qBTA5g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/core": "^11.2.1", + "@inquirer/figures": "^2.0.7", + "@inquirer/type": "^4.0.7" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/select": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/@inquirer/select/-/select-5.2.1.tgz", + "integrity": "sha512-FlDndEUww8m7BfukO2nJa25vhD+H5jxxCv4oGioKqzyWz3nPHhhw4LKdYRSlXuAx7DsdWia7iyaBPKKS95Evfw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/ansi": "^2.0.7", + "@inquirer/core": "^11.2.1", + "@inquirer/figures": "^2.0.7", + "@inquirer/type": "^4.0.7" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/type": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/@inquirer/type/-/type-4.0.7.tgz", + "integrity": "sha512-t28inv14nMQ1PhKpsJPY+kEs/c00qzeCOS2gTNRyTjG5d6qsVA2fItxW4hkvGZ5lvanGLdtCzVIx5dwdRpN1+g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } } }, "node_modules/@modelcontextprotocol/sdk": { @@ -49,35 +426,53 @@ } }, "node_modules/@modelcontextprotocol/sdk/node_modules/body-parser": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.2.0.tgz", - "integrity": "sha512-02qvAaxv8tp7fBa/mw1ga98OGm+eCbqzJOKoRt70sLmfEEi+jyBYVTDGfCL/k06/4EMk/z01gCe7HoCH/f2LTg==", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.3.0.tgz", + "integrity": "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==", "license": "MIT", "dependencies": { "bytes": "^3.1.2", - "content-type": "^1.0.5", - "debug": "^4.4.0", - "http-errors": "^2.0.0", - "iconv-lite": "^0.6.3", + "content-type": "^2.0.0", + "debug": "^4.4.3", + "http-errors": "^2.0.1", + "iconv-lite": "^0.7.2", "on-finished": "^2.4.1", - "qs": "^6.14.0", - "raw-body": "^3.0.0", - "type-is": "^2.0.0" + "qs": "^6.15.2", + "raw-body": "^3.0.2", + "type-is": "^2.1.0" + }, + "engines": { + "node": ">=18" }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/body-parser/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "license": "MIT", "engines": { "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/@modelcontextprotocol/sdk/node_modules/content-disposition": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.0.0.tgz", - "integrity": "sha512-Au9nRL8VNUut/XSzbQA38+M78dzP4D+eqg3gfJHMIHHYa3bg067xj1KxMUWj+VULbiZMowKngFFbKczUrNJ1mg==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz", + "integrity": "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==", "license": "MIT", - "dependencies": { - "safe-buffer": "5.2.1" - }, "engines": { - "node": ">= 0.6" + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/@modelcontextprotocol/sdk/node_modules/cookie-signature": { @@ -107,18 +502,19 @@ } }, "node_modules/@modelcontextprotocol/sdk/node_modules/express": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/express/-/express-5.1.0.tgz", - "integrity": "sha512-DT9ck5YIRU+8GYzzU5kT3eHGA5iL+1Zd0EutOmTE9Dtk+Tvuzd23VBU+ec7HPNSTxXYO55gPV/hq4pSBJDjFpA==", + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", "license": "MIT", "dependencies": { "accepts": "^2.0.0", - "body-parser": "^2.2.0", + "body-parser": "^2.2.1", "content-disposition": "^1.0.0", "content-type": "^1.0.5", "cookie": "^0.7.1", "cookie-signature": "^1.2.1", "debug": "^4.4.0", + "depd": "^2.0.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "etag": "^1.8.1", @@ -149,9 +545,9 @@ } }, "node_modules/@modelcontextprotocol/sdk/node_modules/finalhandler": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.0.tgz", - "integrity": "sha512-/t88Ty3d5JWQbWYgaOGCCYfXRwV1+be02WqYYlL6h0lEiUAMPM8o8qKGO01YIkOHzka2up08wvgYD0mDiI+q3Q==", + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", "license": "MIT", "dependencies": { "debug": "^4.4.0", @@ -161,44 +557,567 @@ "parseurl": "^1.3.3", "statuses": "^2.0.1" }, + "engines": { + "node": ">= 18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/fresh": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/iconv-lite": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", + "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/media-typer": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.1.tgz", + "integrity": "sha512-yz3xRaG20c6/BOzvYoDaGtPmGscs7YivItZEEqe6GbwNfHuxu9YNmvnEkMzKldAGY4/80pRcQRZSEnhquk9XuQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/merge-descriptors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/negotiator": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz", + "integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/qs": { + "version": "6.15.3", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.3.tgz", + "integrity": "sha512-O9gl3zCl5h5blw1KGUzQKhA5oUXSl8rwUIM5o0S3nCXMliSvy5Dzx7/DJcI+SwgICv+IneSZwhBh1oSyEHA71A==", + "license": "BSD-3-Clause", + "dependencies": { + "es-define-property": "^1.0.1", + "side-channel": "^1.1.1" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/send": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", + "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.1", + "mime-types": "^3.0.2", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/serve-static": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", + "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", + "license": "MIT", + "dependencies": { + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/type-is": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", + "integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==", + "license": "MIT", + "dependencies": { + "content-type": "^2.0.0", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/type-is/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/@verygoodplugins/mcp-automem": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/@verygoodplugins/mcp-automem/-/mcp-automem-0.15.0.tgz", + "integrity": "sha512-WsnFmalaVTMps9gaQ4Bhg688OPrFzNrFSv6iZpEQcp6FslTnUsXZnW7SxjzWfkxnSFXRERcLPB8NNAyXHsB5/w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/prompts": "^8.5.2", + "@modelcontextprotocol/sdk": "^1.29.0", + "dotenv": "^17.4.2", + "node-fetch": "^3.3.2", + "yaml": "^2.9.0" + }, + "bin": { + "mcp-automem": "dist/index.js" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@verygoodplugins/mcp-automem/node_modules/@modelcontextprotocol/sdk": { + "version": "1.30.0", + "resolved": "https://registry.npmjs.org/@modelcontextprotocol/sdk/-/sdk-1.30.0.tgz", + "integrity": "sha512-xKd8OIzlqNzcqcNumGAa6g+PW2kjD5vrpcKOnfldAUPP3j7lnqMPwlTXQm8gF+UwH72z0lqaRbjr9hqGz0eITA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@hono/node-server": "^1.19.9 || ^2.0.5", + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1", + "content-type": "^1.0.5", + "cors": "^2.8.5", + "cross-spawn": "^7.0.5", + "eventsource": "^3.0.2", + "eventsource-parser": "^3.0.0", + "express": "^5.2.1", + "express-rate-limit": "^8.2.1", + "hono": "^4.11.4", + "jose": "^6.1.3", + "json-schema-typed": "^8.0.2", + "pkce-challenge": "^5.0.0", + "raw-body": "^3.0.0", + "zod": "^3.25 || ^4.0", + "zod-to-json-schema": "^3.25.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@cfworker/json-schema": "^4.1.1", + "zod": "^3.25 || ^4.0" + }, + "peerDependenciesMeta": { + "@cfworker/json-schema": { + "optional": true + }, + "zod": { + "optional": false + } + } + }, + "node_modules/@verygoodplugins/mcp-automem/node_modules/accepts": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", + "dev": true, + "license": "MIT", + "dependencies": { + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/@verygoodplugins/mcp-automem/node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/@verygoodplugins/mcp-automem/node_modules/body-parser": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.3.0.tgz", + "integrity": "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==", + "dev": true, + "license": "MIT", + "dependencies": { + "bytes": "^3.1.2", + "content-type": "^2.0.0", + "debug": "^4.4.3", + "http-errors": "^2.0.1", + "iconv-lite": "^0.7.2", + "on-finished": "^2.4.1", + "qs": "^6.15.2", + "raw-body": "^3.0.2", + "type-is": "^2.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@verygoodplugins/mcp-automem/node_modules/body-parser/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@verygoodplugins/mcp-automem/node_modules/content-disposition": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz", + "integrity": "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@verygoodplugins/mcp-automem/node_modules/cookie-signature": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.6.0" + } + }, + "node_modules/@verygoodplugins/mcp-automem/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@verygoodplugins/mcp-automem/node_modules/express": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", + "dev": true, + "license": "MIT", + "dependencies": { + "accepts": "^2.0.0", + "body-parser": "^2.2.1", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "depd": "^2.0.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@verygoodplugins/mcp-automem/node_modules/express-rate-limit": { + "version": "8.6.2", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-8.6.2.tgz", + "integrity": "sha512-YH4ru+eOJxQABscKFfRCy9R7x9QFGdezclVMwwgFFndzS2Xnm0uo6B0ABZsLhcpeptGv2qvuJVWlQr9gQZoC3A==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "ip-address": "^10.2.0" + }, + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://github.com/sponsors/express-rate-limit" + }, + "peerDependencies": { + "express": ">= 4.11" + } + }, + "node_modules/@verygoodplugins/mcp-automem/node_modules/finalhandler": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" + }, + "engines": { + "node": ">= 18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@verygoodplugins/mcp-automem/node_modules/fresh": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", + "dev": true, + "license": "MIT", "engines": { "node": ">= 0.8" } }, - "node_modules/@modelcontextprotocol/sdk/node_modules/fresh": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", - "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", + "node_modules/@verygoodplugins/mcp-automem/node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "dev": true, "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, "engines": { "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, - "node_modules/@modelcontextprotocol/sdk/node_modules/iconv-lite": { - "version": "0.6.3", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", - "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "node_modules/@verygoodplugins/mcp-automem/node_modules/iconv-lite": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", + "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", + "dev": true, "license": "MIT", "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" }, "engines": { "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, - "node_modules/@modelcontextprotocol/sdk/node_modules/media-typer": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", - "integrity": "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==", + "node_modules/@verygoodplugins/mcp-automem/node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true, + "license": "MIT" + }, + "node_modules/@verygoodplugins/mcp-automem/node_modules/media-typer": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.1.tgz", + "integrity": "sha512-yz3xRaG20c6/BOzvYoDaGtPmGscs7YivItZEEqe6GbwNfHuxu9YNmvnEkMzKldAGY4/80pRcQRZSEnhquk9XuQ==", + "dev": true, "license": "MIT", "engines": { "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, - "node_modules/@modelcontextprotocol/sdk/node_modules/merge-descriptors": { + "node_modules/@verygoodplugins/mcp-automem/node_modules/merge-descriptors": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", + "dev": true, "license": "MIT", "engines": { "node": ">=18" @@ -207,49 +1126,59 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/@modelcontextprotocol/sdk/node_modules/mime-db": { + "node_modules/@verygoodplugins/mcp-automem/node_modules/mime-db": { "version": "1.54.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "dev": true, "license": "MIT", "engines": { "node": ">= 0.6" } }, - "node_modules/@modelcontextprotocol/sdk/node_modules/mime-types": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.1.tgz", - "integrity": "sha512-xRc4oEhT6eaBpU1XF7AjpOFD+xQmXNB5OVKwp4tqCuBpHLS/ZbBDrc07mYTDqVMg6PfxUjjNp85O6Cd2Z/5HWA==", + "node_modules/@verygoodplugins/mcp-automem/node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "dev": true, "license": "MIT", "dependencies": { "mime-db": "^1.54.0" }, "engines": { - "node": ">= 0.6" + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, - "node_modules/@modelcontextprotocol/sdk/node_modules/ms": { + "node_modules/@verygoodplugins/mcp-automem/node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, "license": "MIT" }, - "node_modules/@modelcontextprotocol/sdk/node_modules/negotiator": { + "node_modules/@verygoodplugins/mcp-automem/node_modules/negotiator": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz", "integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==", + "dev": true, "license": "MIT", "engines": { "node": ">= 0.6" } }, - "node_modules/@modelcontextprotocol/sdk/node_modules/qs": { - "version": "6.14.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.0.tgz", - "integrity": "sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w==", + "node_modules/@verygoodplugins/mcp-automem/node_modules/qs": { + "version": "6.15.3", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.3.tgz", + "integrity": "sha512-O9gl3zCl5h5blw1KGUzQKhA5oUXSl8rwUIM5o0S3nCXMliSvy5Dzx7/DJcI+SwgICv+IneSZwhBh1oSyEHA71A==", + "dev": true, "license": "BSD-3-Clause", "dependencies": { - "side-channel": "^1.1.0" + "es-define-property": "^1.0.1", + "side-channel": "^1.1.1" }, "engines": { "node": ">=0.6" @@ -258,32 +1187,38 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/@modelcontextprotocol/sdk/node_modules/send": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/send/-/send-1.2.0.tgz", - "integrity": "sha512-uaW0WwXKpL9blXE2o0bRhoL2EGXIrZxQ2ZQ4mgcfoBxdFmQold+qWsD2jLrfZ0trjKL6vOw0j//eAwcALFjKSw==", + "node_modules/@verygoodplugins/mcp-automem/node_modules/send": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", + "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", + "dev": true, "license": "MIT", "dependencies": { - "debug": "^4.3.5", + "debug": "^4.4.3", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "etag": "^1.8.1", "fresh": "^2.0.0", - "http-errors": "^2.0.0", - "mime-types": "^3.0.1", + "http-errors": "^2.0.1", + "mime-types": "^3.0.2", "ms": "^2.1.3", "on-finished": "^2.4.1", "range-parser": "^1.2.1", - "statuses": "^2.0.1" + "statuses": "^2.0.2" }, "engines": { "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, - "node_modules/@modelcontextprotocol/sdk/node_modules/serve-static": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.0.tgz", - "integrity": "sha512-61g9pCh0Vnh7IutZjtLGGpTA355+OPn2TyDv/6ivP2h/AdAVX9azsoxmg2/M6nZeQZNYBEwIcsne1mJd9oQItQ==", + "node_modules/@verygoodplugins/mcp-automem/node_modules/serve-static": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", + "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", + "dev": true, "license": "MIT", "dependencies": { "encodeurl": "^2.0.0", @@ -293,20 +1228,53 @@ }, "engines": { "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, - "node_modules/@modelcontextprotocol/sdk/node_modules/type-is": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.0.1.tgz", - "integrity": "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw==", + "node_modules/@verygoodplugins/mcp-automem/node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/@verygoodplugins/mcp-automem/node_modules/type-is": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", + "integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==", + "dev": true, "license": "MIT", "dependencies": { - "content-type": "^1.0.5", + "content-type": "^2.0.0", "media-typer": "^1.1.0", "mime-types": "^3.0.0" }, "engines": { - "node": ">= 0.6" + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@verygoodplugins/mcp-automem/node_modules/type-is/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/accepts": { @@ -323,9 +1291,9 @@ } }, "node_modules/ajv": { - "version": "6.12.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz", + "integrity": "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==", "license": "MIT", "dependencies": { "fast-deep-equal": "^3.1.1", @@ -338,6 +1306,48 @@ "url": "https://github.com/sponsors/epoberezkin" } }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/ajv-formats/node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats/node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true, + "license": "MIT" + }, "node_modules/array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", @@ -421,6 +1431,23 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/chardet": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/chardet/-/chardet-2.2.0.tgz", + "integrity": "sha512-rddelWYNPRrXq6PtNEN2S3f6t9ILzvqaN5pVgi4kqt9jHQaXIial9PznB5iSPVlQSLNaaH22ItWz3EJtQ10+OA==", + "dev": true, + "license": "MIT" + }, + "node_modules/cli-width": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-4.1.0.tgz", + "integrity": "sha512-ouuZd4/dm2Sw5Gmqy6bGyNNNe1qt9RpmxveLSO7KcgsTnU7RXfsw+/bukWGo1abgBiMAic068rclZsO4IWmmxQ==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">= 12" + } + }, "node_modules/content-disposition": { "version": "0.5.4", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", @@ -484,6 +1511,16 @@ "node": ">= 8" } }, + "node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, "node_modules/debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", @@ -512,6 +1549,19 @@ "npm": "1.2.8000 || >= 1.4.16" } }, + "node_modules/dotenv": { + "version": "17.4.2", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.4.2.tgz", + "integrity": "sha512-nI4U3TottKAcAD9LLud4Cb7b2QztQMUEfHbvhTH09bqXTxnSie8WnjPALV/WMCrJZ6UV/qHJ6L03OqO3LcdYZw==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, "node_modules/dunder-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", @@ -680,6 +1730,74 @@ "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", "license": "MIT" }, + "node_modules/fast-string-truncated-width": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/fast-string-truncated-width/-/fast-string-truncated-width-3.0.3.tgz", + "integrity": "sha512-0jjjIEL6+0jag3l2XWWizO64/aZVtpiGE3t0Zgqxv0DPuxiMjvB3M24fCyhZUO4KomJQPj3LTSUnDP3GpdwC0g==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-string-width": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/fast-string-width/-/fast-string-width-3.0.2.tgz", + "integrity": "sha512-gX8LrtNEI5hq8DVUfRQMbr5lpaS4nMIWV+7XEbXk2b8kiQIizgnlr12B4dA3ZEx3308ze0O4Q1R+cHts8kyUJg==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-string-truncated-width": "^3.0.2" + } + }, + "node_modules/fast-uri": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.5.tgz", + "integrity": "sha512-gHwA1O9LDIcKunMKhObS/HimwtehO1nPUECKAu5TpKgaO19fcWEl4bliWe1jWxVFvIXztJjjQ4L8XQ1EU9f7Jw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/fast-wrap-ansi": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/fast-wrap-ansi/-/fast-wrap-ansi-0.2.2.tgz", + "integrity": "sha512-7F2Fl+TjRSenLqlU3UjSH0iyqopqoZIu7eZVpEirP2g1GtWa2G/ecEmBdgz31+Mxr+ELclgg6sokpSFIQiZ02Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-string-width": "^3.0.2" + } + }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, "node_modules/finalhandler": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.1.tgz", @@ -698,6 +1816,19 @@ "node": ">= 0.8" } }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, "node_modules/forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -798,6 +1929,16 @@ "node": ">= 0.4" } }, + "node_modules/hono": { + "version": "4.13.2", + "resolved": "https://registry.npmjs.org/hono/-/hono-4.13.2.tgz", + "integrity": "sha512-JydRilDRkYBQMt9qR9U92mXxmbGqsqSn/IKOrh4e7/gEbn+0zSr8igTu0obwJoNGN4sez28DIql7FBHWydoJpA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=16.9.0" + } + }, "node_modules/http-errors": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", @@ -832,6 +1973,16 @@ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", "license": "ISC" }, + "node_modules/ip-address": { + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.5.0.tgz", + "integrity": "sha512-R5SnVLJmgYYvf2F2ZgwSBnelz5G4q5AxIC277GDfUaNbrZKNANcBC7RHqYYePlszf4kBolVkJauG0ZjHHFh55g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -853,12 +2004,29 @@ "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", "license": "ISC" }, + "node_modules/jose": { + "version": "6.2.8", + "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.8.tgz", + "integrity": "sha512-Bsdjwm3Qsd/P0jR+BHDe3LytDfY7WBq2HmCCLIwuVRHMuEC9ae7/R474GIUdF1NgCyZjzVo/A9DOiOBtXq8ZoQ==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, "node_modules/json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", "license": "MIT" }, + "node_modules/json-schema-typed": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/json-schema-typed/-/json-schema-typed-8.0.2.tgz", + "integrity": "sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA==", + "dev": true, + "license": "BSD-2-Clause" + }, "node_modules/math-intrinsics": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", @@ -934,6 +2102,16 @@ "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", "license": "MIT" }, + "node_modules/mute-stream": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-3.0.0.tgz", + "integrity": "sha512-dkEJPVvun4FryqBmZ5KhDo0K9iDXAwn08tMLDinNdRBNPcYEDiWYysLcc6k3mjTMlbP9KyylvRpd4wFtwrT9rw==", + "dev": true, + "license": "ISC", + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, "node_modules/negotiator": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", @@ -943,6 +2121,46 @@ "node": ">= 0.6" } }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "deprecated": "Use your platform's native DOMException instead", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "dev": true, + "license": "MIT", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, "node_modules/object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", @@ -1065,24 +2283,44 @@ } }, "node_modules/raw-body": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.1.tgz", - "integrity": "sha512-9G8cA+tuMS75+6G/TzW8OtLzmBDMo8p1JRxN5AZ+LAp8uxGA8V8GZm4GQ4/N5QNQEnLmg6SS7wyuSmbKepiKqA==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", "license": "MIT", "dependencies": { - "bytes": "3.1.2", - "http-errors": "2.0.0", - "iconv-lite": "0.7.0", - "unpipe": "1.0.0" + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.7.0", + "unpipe": "~1.0.0" }, "engines": { "node": ">= 0.10" } }, + "node_modules/raw-body/node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/raw-body/node_modules/iconv-lite": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.0.tgz", - "integrity": "sha512-cf6L2Ds3h57VVmkZe+Pn+5APsT7FpqJtEhhieDCvrE2MK5Qk9MyffgQyuxQTm6BChfeZNtcOLHp9IcWRVcIcBQ==", + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", + "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", "license": "MIT", "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" @@ -1095,6 +2333,25 @@ "url": "https://opencollective.com/express" } }, + "node_modules/raw-body/node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/router": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", @@ -1135,9 +2392,9 @@ "license": "MIT" }, "node_modules/router/node_modules/path-to-regexp": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.3.0.tgz", - "integrity": "sha512-7jdwVIRtsP8MYpdXSwOS0YdD0Du+qOoF/AEPIt88PcCFrZCzx41oxku1jD88hZBwbNUIEfpqvuhjFaMAqMTWnA==", + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", + "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==", "license": "MIT", "funding": { "type": "opencollective", @@ -1252,14 +2509,14 @@ } }, "node_modules/side-channel": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", - "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz", + "integrity": "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==", "license": "MIT", "dependencies": { "es-errors": "^1.3.0", - "object-inspect": "^1.13.3", - "side-channel-list": "^1.0.0", + "object-inspect": "^1.13.4", + "side-channel-list": "^1.0.1", "side-channel-map": "^1.0.1", "side-channel-weakmap": "^1.0.2" }, @@ -1271,13 +2528,13 @@ } }, "node_modules/side-channel-list": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.0.tgz", - "integrity": "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", "license": "MIT", "dependencies": { "es-errors": "^1.3.0", - "object-inspect": "^1.13.3" + "object-inspect": "^1.13.4" }, "engines": { "node": ">= 0.4" @@ -1323,6 +2580,19 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/statuses": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", @@ -1390,6 +2660,16 @@ "node": ">= 0.8" } }, + "node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -1411,22 +2691,38 @@ "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", "license": "ISC" }, + "node_modules/yaml": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz", + "integrity": "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==", + "dev": true, + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + }, + "funding": { + "url": "https://github.com/sponsors/eemeli" + } + }, "node_modules/zod": { - "version": "3.25.76", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", - "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz", + "integrity": "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==", "license": "MIT", "funding": { "url": "https://github.com/sponsors/colinhacks" } }, "node_modules/zod-to-json-schema": { - "version": "3.24.6", - "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.24.6.tgz", - "integrity": "sha512-h/z3PKvcTcTetyjl1fkj79MHNEjm+HpD6NXheWjzOekY7kV+lwDYnHw+ivHkijnCSMz1yJaWBD9vu/Fcmk+vEg==", + "version": "3.25.2", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.2.tgz", + "integrity": "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA==", "license": "ISC", "peerDependencies": { - "zod": "^3.24.1" + "zod": "^3.25.28 || ^4" } } } diff --git a/mcp-sse-server/package.json b/mcp-sse-server/package.json index d26992f..f69a0fe 100644 --- a/mcp-sse-server/package.json +++ b/mcp-sse-server/package.json @@ -8,7 +8,10 @@ "test": "node --test" }, "dependencies": { - "@modelcontextprotocol/sdk": "^1.20.0", + "@modelcontextprotocol/sdk": "1.20.0", "express": "^4.19.2" + }, + "devDependencies": { + "@verygoodplugins/mcp-automem": "^0.15.0" } } diff --git a/mcp-sse-server/parity/clients.js b/mcp-sse-server/parity/clients.js new file mode 100644 index 0000000..b5261ae --- /dev/null +++ b/mcp-sse-server/parity/clients.js @@ -0,0 +1,68 @@ +/** + * Connects one MCP client to each transport so the two can be diffed. + * + * Remote: the bridge is booted in-process on an ephemeral port (same pattern as + * test/server.test.js's withServer helper) and driven over streamable HTTP. + * stdio: the published @verygoodplugins/mcp-automem bin is spawned as a child + * process and driven over stdio. + * + * Both point at the same live AutoMem service, so any difference in the output + * is a difference between the transports and not between two datasets. + */ +import { createRequire } from 'node:module'; +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js'; +import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js'; +import { createApp } from '../server.js'; + +const require = createRequire(import.meta.url); + +export const API_URL = process.env.AUTOMEM_PARITY_API_URL || 'http://localhost:8001'; +export const API_TOKEN = process.env.AUTOMEM_PARITY_API_TOKEN || 'test-token'; + +export async function connectBothTransports() { + const closers = []; + + // --- remote --------------------------------------------------------------- + // server.js resolves the upstream inside its route handlers, so setting these + // before createApp() is enough. + process.env.AUTOMEM_API_URL = API_URL; + process.env.AUTOMEM_API_TOKEN = API_TOKEN; + + const app = createApp(); + const httpServer = await new Promise((resolve) => { + const s = app.listen(0, '127.0.0.1', () => resolve(s)); + }); + closers.push(() => new Promise((r) => httpServer.close(r))); + const { port } = httpServer.address(); + + const remote = new Client({ name: 'parity-harness', version: '1.0.0' }, {}); + const remoteTransport = new StreamableHTTPClientTransport( + new URL(`http://127.0.0.1:${port}/mcp`), + { requestInit: { headers: { Authorization: `Bearer ${API_TOKEN}` } } } + ); + await remote.connect(remoteTransport); + closers.push(() => remote.close()); + + // --- stdio ---------------------------------------------------------------- + const stdioEntry = require.resolve('@verygoodplugins/mcp-automem/dist/index.js'); + const stdio = new Client({ name: 'parity-harness', version: '1.0.0' }, {}); + const stdioTransport = new StdioClientTransport({ + command: process.execPath, + args: [stdioEntry], + env: { ...process.env, AUTOMEM_API_URL: API_URL, AUTOMEM_API_KEY: API_TOKEN }, + stderr: 'ignore', + }); + await stdio.connect(stdioTransport); + closers.push(() => stdio.close()); + + return { + remote, + stdio, + async close() { + for (const c of closers.reverse()) { + await Promise.resolve(c()).catch(() => {}); + } + }, + }; +} diff --git a/mcp-sse-server/parity/normalize.js b/mcp-sse-server/parity/normalize.js new file mode 100644 index 0000000..f796b95 --- /dev/null +++ b/mcp-sse-server/parity/normalize.js @@ -0,0 +1,46 @@ +/** + * Normalization helpers for the cross-transport parity harness. + * + * Lives outside test/ on purpose: `node --test` treats every file under a + * `test/` directory as a test file, so helper modules placed there would be + * executed as (empty) test suites. + */ + +/** + * Recursively sort object keys so deepStrictEqual is not sensitive to the order + * in which two independent implementations happened to build the same object. + */ +export function normalizeKeys(value) { + if (Array.isArray(value)) return value.map(normalizeKeys); + if (value && typeof value === 'object') { + return Object.fromEntries( + Object.keys(value) + .sort() + .map((k) => [k, normalizeKeys(value[k])]) + ); + } + return value; +} + +const UUID_RE = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi; +const ISO_RE = /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})/g; +const SCORE_RE = /(score=|Score: |"final_score":\s*|"score":\s*)[\d.]+/g; +const MS_RE = /("query_time_ms":\s*)[\d.]+/g; + +/** + * Replace values that legitimately differ run-to-run so two transports can be + * compared on the parts that must match. + * + * `scopeTag` is the per-transport uuid4 tag namespace; each transport writes + * under its own so neither sees the other's memories, which means the tag has + * to collapse to a constant before comparison. + */ +export function redact(text, scopeTag) { + let out = String(text); + if (scopeTag) out = out.split(scopeTag).join(''); + return out + .replace(UUID_RE, '') + .replace(ISO_RE, '') + .replace(SCORE_RE, '$1') + .replace(MS_RE, '$1'); +} diff --git a/mcp-sse-server/test/parity.test.js b/mcp-sse-server/test/parity.test.js new file mode 100644 index 0000000..ca91beb --- /dev/null +++ b/mcp-sse-server/test/parity.test.js @@ -0,0 +1,46 @@ +/** + * Cross-transport parity: the remote bridge and the stdio package must expose + * one identical MCP surface. See docs/MCP_TRANSPORT_PARITY.md for the contract + * and for the short list of differences that are allowed. + * + * Needs a live AutoMem at :8001, so it is gated. Run it with: + * make test-parity + */ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { connectBothTransports } from '../parity/clients.js'; +import { normalizeKeys } from '../parity/normalize.js'; + +const GATE = + process.env.AUTOMEM_RUN_PARITY_TESTS === '1' + ? false + : 'set AUTOMEM_RUN_PARITY_TESTS=1 with a live AutoMem at :8001 (make test-parity)'; + +test('tools/list is identical across transports', { skip: GATE }, async () => { + const { remote, stdio, close } = await connectBothTransports(); + try { + const a = normalizeKeys((await remote.listTools()).tools); + const b = normalizeKeys((await stdio.listTools()).tools); + assert.deepStrictEqual(a, b); + } finally { + await close(); + } +}); + +test('server capabilities and instructions match', { skip: GATE }, async () => { + const { remote, stdio, close } = await connectBothTransports(); + try { + assert.deepStrictEqual( + normalizeKeys(remote.getServerCapabilities()), + normalizeKeys(stdio.getServerCapabilities()) + ); + assert.equal(remote.getInstructions(), stdio.getInstructions()); + + // serverInfo.name is an allowlisted difference — clients must be able to + // tell the transports apart (docs/MCP_TRANSPORT_PARITY.md). + assert.equal(remote.getServerVersion().name, 'automem-mcp-sse'); + assert.equal(stdio.getServerVersion().name, 'mcp-automem'); + } finally { + await close(); + } +}); From 2260c0c46a0f2548c2686294d23a20b99b0ddd7e Mon Sep 17 00:00:00 2001 From: Jack Arturo Date: Fri, 14 Aug 2026 13:47:23 -0400 Subject: [PATCH 04/33] test(mcp): add tools/call parity scenario matrix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nineteen scenarios covering every mode on both transports — batch and supersede store, ranked/detailed/items/json/empty recall, id fetch, tag enumeration, relation-prop and batch association, update, single and bulk delete, health, and three error paths. Each transport writes under its own uuid4 tag namespace so neither sees the other's memories; UUIDs, timestamps, scores and query_time_ms are redacted before comparison. Cleanup deletes both namespaces by tag. Red as of this commit, first failure on `store single`: remote Memory stored: stdio Memory stored successfully!\n\nMemory ID: Co-Authored-By: Claude Opus 5 --- mcp-sse-server/parity/scenarios.js | 184 +++++++++++++++++++++++++++++ mcp-sse-server/test/parity.test.js | 63 +++++++++- 2 files changed, 246 insertions(+), 1 deletion(-) create mode 100644 mcp-sse-server/parity/scenarios.js diff --git a/mcp-sse-server/parity/scenarios.js b/mcp-sse-server/parity/scenarios.js new file mode 100644 index 0000000..e24c159 --- /dev/null +++ b/mcp-sse-server/parity/scenarios.js @@ -0,0 +1,184 @@ +/** + * The tools/call scenario matrix for the parity harness. + * + * Each scenario is a short sequence of calls run independently against each + * transport, under that transport's own `tag` namespace so neither sees the + * other's writes. `$PREV..memory_id` is substituted with the memory id + * produced by call of the same scenario. + */ +export function buildScenarios(tag) { + const base = { tags: [tag], importance: 0.7 }; + + return [ + { + name: 'store single', + calls: [ + { + tool: 'store_memory', + args: { ...base, content: 'Parity fixture alpha. Chose PostgreSQL for ACID.' }, + }, + ], + }, + { + // Distinct importance values are load-bearing. GET /memory/by-tag orders by + // `importance DESC, timestamp DESC, id ASC` (automem/api/memory.py:259). With + // equal importance and sub-millisecond writes, `id ASC` becomes the tiebreaker + // — and ids differ between the two tag namespaces by construction, so the + // `recall exhaustive` scenario would fail on ordering while actually in parity. + name: 'store batch', + calls: [ + { + tool: 'store_memory', + args: { + memories: [ + { content: 'Parity batch one.', tags: [tag], importance: 0.9 }, + { content: 'Parity batch two.', tags: [tag], importance: 0.7 }, + { content: 'Parity batch three.', tags: [tag], importance: 0.5 }, + ], + }, + }, + ], + }, + { + name: 'store supersede', + calls: [ + { tool: 'store_memory', args: { ...base, content: 'Parity supersede original.' } }, + { + tool: 'store_memory', + args: { + ...base, + content: 'Parity supersede replacement.', + supersedes_memory_id: '$PREV.0.memory_id', + supersede_reason: 'parity harness', + }, + }, + ], + }, + { + name: 'recall ranked text', + calls: [ + { tool: 'recall_memory', args: { query: 'Parity fixture', tags: [tag], limit: 5 } }, + ], + }, + { + name: 'recall detailed', + calls: [ + { + tool: 'recall_memory', + args: { query: 'Parity fixture', tags: [tag], format: 'detailed' }, + }, + ], + }, + { + name: 'recall items', + calls: [ + { tool: 'recall_memory', args: { query: 'Parity fixture', tags: [tag], format: 'items' } }, + ], + }, + { + name: 'recall json', + calls: [ + { tool: 'recall_memory', args: { query: 'Parity fixture', tags: [tag], format: 'json' } }, + ], + }, + { + name: 'recall empty', + calls: [ + { + tool: 'recall_memory', + args: { query: 'zzz-no-such-thing', tags: [`${tag}-absent`] }, + }, + ], + }, + { + name: 'recall id fetch', + calls: [ + { tool: 'store_memory', args: { ...base, content: 'Parity id-fetch target.' } }, + { tool: 'recall_memory', args: { memory_id: '$PREV.0.memory_id' } }, + ], + }, + { + name: 'recall exhaustive', + calls: [ + { tool: 'recall_memory', args: { tags: [tag], exhaustive: true, limit: 2, offset: 0 } }, + ], + }, + { + name: 'associate single', + calls: [ + { tool: 'store_memory', args: { ...base, content: 'Parity assoc source.' } }, + { tool: 'store_memory', args: { ...base, content: 'Parity assoc target.' } }, + { + tool: 'associate_memories', + args: { + memory1_id: '$PREV.0.memory_id', + memory2_id: '$PREV.1.memory_id', + type: 'EXEMPLIFIES', + strength: 0.8, + pattern_type: 'parity', + confidence: 0.9, + }, + }, + ], + }, + { + name: 'associate batch partial failure', + calls: [ + { tool: 'store_memory', args: { ...base, content: 'Parity batch assoc source.' } }, + { + tool: 'associate_memories', + args: { + associations: [ + { + memory1_id: '$PREV.0.memory_id', + memory2_id: '00000000-0000-4000-8000-000000000000', + type: 'RELATES_TO', + strength: 0.5, + }, + ], + }, + }, + ], + }, + { + name: 'update', + calls: [ + { tool: 'store_memory', args: { ...base, content: 'Parity update original.' } }, + { tool: 'update_memory', args: { memory_id: '$PREV.0.memory_id', importance: 0.95 } }, + ], + }, + { + name: 'delete single', + calls: [ + { tool: 'store_memory', args: { ...base, content: 'Parity delete target.' } }, + { tool: 'delete_memory', args: { memory_id: '$PREV.0.memory_id' } }, + ], + }, + { + name: 'delete by tag', + calls: [ + { + tool: 'store_memory', + args: { content: 'Parity bulk delete target.', tags: [`${tag}-bulk`], importance: 0.7 }, + }, + { tool: 'delete_memory', args: { tags: [`${tag}-bulk`] } }, + ], + }, + { + name: 'health', + calls: [{ tool: 'check_database_health', args: {} }], + }, + { + name: 'error: exhaustive without tags', + calls: [{ tool: 'recall_memory', args: { exhaustive: true } }], + }, + { + name: 'error: store content over hard limit', + calls: [{ tool: 'store_memory', args: { ...base, content: 'x'.repeat(2100) } }], + }, + { + name: 'error: unknown tool', + calls: [{ tool: 'no_such_tool', args: {} }], + }, + ]; +} diff --git a/mcp-sse-server/test/parity.test.js b/mcp-sse-server/test/parity.test.js index ca91beb..9d85870 100644 --- a/mcp-sse-server/test/parity.test.js +++ b/mcp-sse-server/test/parity.test.js @@ -8,8 +8,43 @@ */ import test from 'node:test'; import assert from 'node:assert/strict'; +import { randomUUID } from 'node:crypto'; import { connectBothTransports } from '../parity/clients.js'; -import { normalizeKeys } from '../parity/normalize.js'; +import { normalizeKeys, redact } from '../parity/normalize.js'; +import { buildScenarios } from '../parity/scenarios.js'; + +const UUID_RE = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i; + +/** Resolve "$PREV..memory_id" against ids already collected in this scenario. */ +function resolveArgs(args, prior) { + return JSON.parse(JSON.stringify(args), (_key, value) => + typeof value === 'string' && value.startsWith('$PREV.') + ? prior[Number(value.split('.')[1])] + : value + ); +} + +/** Run every scenario against one transport, under its own tag namespace. */ +async function runScenarios(client, tag) { + const out = []; + for (const scenario of buildScenarios(tag)) { + const ids = []; + const rendered = []; + for (const call of scenario.calls) { + const res = await client + .callTool({ name: call.tool, arguments: resolveArgs(call.args, ids) }) + .catch((e) => ({ + content: [{ type: 'text', text: `THREW: ${e.message}` }], + isError: true, + })); + const text = (res.content || []).map((c) => c.text ?? '').join('\n'); + ids.push(res.structuredContent?.memory_id ?? (text.match(UUID_RE) || [])[0]); + rendered.push({ isError: Boolean(res.isError), text: redact(text, tag) }); + } + out.push({ name: scenario.name, rendered }); + } + return out; +} const GATE = process.env.AUTOMEM_RUN_PARITY_TESTS === '1' @@ -44,3 +79,29 @@ test('server capabilities and instructions match', { skip: GATE }, async () => { await close(); } }); + +test('tools/call renders identically across transports', { skip: GATE }, async () => { + const { remote, stdio, close } = await connectBothTransports(); + const remoteTag = `parity-remote-${randomUUID()}`; + const stdioTag = `parity-stdio-${randomUUID()}`; + try { + const a = await runScenarios(remote, remoteTag); + const b = await runScenarios(stdio, stdioTag); + for (let i = 0; i < a.length; i++) { + assert.deepStrictEqual(a[i], b[i], `scenario mismatch: ${a[i].name}`); + } + } finally { + // Bulk delete by tag exists only on the stdio transport for now, so cleanup + // for both namespaces runs there. Swallowed so a cleanup failure can never + // mask an assertion failure. + await stdio + .callTool({ + name: 'delete_memory', + arguments: { + tags: [remoteTag, stdioTag, `${remoteTag}-bulk`, `${stdioTag}-bulk`], + }, + }) + .catch(() => {}); + await close(); + } +}); From b1822063458cc6a764172d252cfa4ac47c5b5f8d Mon Sep 17 00:00:00 2001 From: Jack Arturo Date: Fri, 14 Aug 2026 13:50:01 -0400 Subject: [PATCH 05/33] ci(mcp): run node suite in make test and add parity workflow make test now chains the MCP bridge's node suite, which was previously only reachable in CI and invisible locally. make test-parity brings the stack up and diffs the two transports in one command. The workflow runs on PRs touching the bridge or the HTTP API, plus weekly as a drift alarm against newly published mcp-automem versions. It is continue-on-error until the bridge moves onto the shared surface. Co-Authored-By: Claude Opus 5 --- .github/workflows/mcp-parity.yml | 63 ++++++++++++++++++++++++++++++++ Makefile | 21 ++++++++++- 2 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/mcp-parity.yml diff --git a/.github/workflows/mcp-parity.yml b/.github/workflows/mcp-parity.yml new file mode 100644 index 0000000..7d8d096 --- /dev/null +++ b/.github/workflows/mcp-parity.yml @@ -0,0 +1,63 @@ +name: MCP Transport Parity + +# Diffs the remote MCP bridge against the stdio package against one live +# AutoMem. See docs/MCP_TRANSPORT_PARITY.md for the contract. + +on: + pull_request: + paths: + - 'mcp-sse-server/**' + - 'automem/api/**' + - 'app.py' + schedule: + # Weekly drift alarm: catches a newly published mcp-automem the bridge has not adopted. + - cron: '0 6 * * 1' + workflow_dispatch: + +jobs: + parity: + runs-on: ubuntu-latest + # Phase 1 only. The harness is legitimately red until the bridge is moved + # onto the shared MCP surface; this is removed in the same change. + continue-on-error: true + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: '20' + cache: npm + cache-dependency-path: mcp-sse-server/package-lock.json + + - name: Start AutoMem stack + env: + AUTOMEM_API_TOKEN: test-token + ADMIN_API_TOKEN: test-admin-token + run: docker compose up -d + + - name: Wait for /health + run: | + for i in $(seq 1 60); do + if curl -fsS http://localhost:8001/health > /dev/null; then + echo "AutoMem is up after ${i}s" + exit 0 + fi + sleep 1 + done + echo "AutoMem did not become healthy within 60s" + docker compose logs flask-api + exit 1 + + - name: Run parity harness + working-directory: mcp-sse-server + env: + AUTOMEM_RUN_PARITY_TESTS: '1' + AUTOMEM_PARITY_API_URL: http://localhost:8001 + AUTOMEM_PARITY_API_TOKEN: test-token + run: | + npm ci + npm test + + - name: Dump service logs on failure + if: failure() + run: docker compose logs --tail=200 diff --git a/Makefile b/Makefile index 31aa92c..f82d23a 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Makefile - Development commands -.PHONY: help install dev stop test fmt lint test-integration test-live test-locomo test-locomo-live test-longmemeval test-longmemeval-live test-longmemeval-watch clean logs deploy deploy-check bench-current-state bench-health +.PHONY: help install dev stop test test-node test-parity fmt lint test-integration test-live test-locomo test-locomo-live test-longmemeval test-longmemeval-live test-longmemeval-watch clean logs deploy deploy-check bench-current-state bench-health VENV_DIR := $(if $(wildcard .venv/bin/python),.venv,venv) VENV_BIN := $(VENV_DIR)/bin @@ -67,6 +67,25 @@ test: $(MAKE) install; \ fi PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 $(VENV_BIN)/pytest -rs -m unit + @$(MAKE) test-node + +# Run the MCP bridge's node suite (parity tests self-skip without a live stack) +test-node: + @echo "🧪 Running MCP bridge node tests..." + @cd mcp-sse-server && npm ci --silent && npm test + +# Diff the remote and stdio MCP transports against one live AutoMem. +# See docs/MCP_TRANSPORT_PARITY.md. +test-parity: + @echo "🐳 Starting Docker services..." + @AUTOMEM_API_TOKEN=test-token ADMIN_API_TOKEN=test-admin-token docker compose up -d + @echo "⏳ Waiting for AutoMem to be ready..." + @for i in $$(seq 1 60); do \ + if curl -fsS http://localhost:8001/health > /dev/null 2>&1; then break; fi; \ + sleep 1; \ + done + @echo "🧪 Diffing MCP transports..." + @cd mcp-sse-server && npm ci --silent && AUTOMEM_RUN_PARITY_TESTS=1 npm test # Format code fmt: From bcef37638d1f44aa72298889a50cd5105ae30c75 Mon Sep 17 00:00:00 2001 From: Jack Arturo Date: Sat, 22 Aug 2026 22:08:22 -0400 Subject: [PATCH 06/33] docs: target feature PRs at develop (#228) ## Summary `develop` is the GitHub default branch again. Spell that in AGENTS.md so agents stop opening feature PRs against `main`. Release-please and GHCR `:stable` stay on `main`. ## Test plan - [x] Confirm `gh repo view --json defaultBranchRef` is `develop` - [x] Confirm this PR itself targets `develop` --- AGENTS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/AGENTS.md b/AGENTS.md index b1edbf5..1a7fdb1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -81,6 +81,7 @@ The benchmark system uses **snapshot-based evaluation**: ingest once, eval many ## Commit & Pull Requests +- Feature PRs target `develop` (repo default). Promote `develop` to `main` with a validated release merge; release-please and GHCR `:stable` then run on `main`. Do not open feature work onto `main`. - PR titles must use Conventional Commit format because squash merges use the PR title as the release commit title. Do not prefix titles with `[codex]`, `[claude]`, `[copilot]`, `[wip]`, or similar labels; put agent/status context in the PR body. - Use Conventional Commit types: `feat`, `fix`, `docs`, `refactor`, `test`, `ci`, `build`, `chore`, `perf`, `revert` (e.g., `feat(api): add /analyze endpoint`). - For public API changes, use `feat(api): ...` unless the change is strictly a bug fix with no new public surface. For docs-only changes, use `docs: ...`; for release automation, use `ci(release): ...` or `chore(release): ...`. From 7a92ebd6fb5e107553d6300f94c5170a36e9335e Mon Sep 17 00:00:00 2001 From: Jack Arturo Date: Sat, 22 Aug 2026 22:08:37 -0400 Subject: [PATCH 07/33] fix(stream): emit live memory operations on GET /stream (#227) ## Summary `GET /stream` was silent for store and recall. Those `emit_event` calls lived only in unused `runtime_*_routes` handlers after the app.py split; the live blueprints never published them. This wires the live path and fills in the rest of the write surface. ## Events | Type | Source | |---|---| | `memory.store` | `POST /memory`; one summary event for `POST /memory/batch` | | `memory.recall` | `GET /recall` | | `memory.update` | `PATCH /memory/{id}` | | `memory.delete` | `DELETE /memory/{id}` and bulk `DELETE /memory/by-tag` | | `memory.associate` | `POST /associate` (single) and one summary event for batch | Failed `4xx` requests do not emit. Batch operations send one summary (`count`) instead of one event per item. `GET /memory/{id}` and by-tag listing stay off the stream. `scripts/automem_watch.py` tracks the new types, skips garbage detection on batch summaries, and preserves explicit `count: 0`. `docs/API.md` documents the catalog. ## Test plan - [x] `pytest tests/test_stream_events.py` (13 passed) - [x] `pytest tests/test_app.py tests/contracts/test_routes_contract.py` - [x] flake8 on touched Python files - [ ] CI unit/lint jobs on this PR ## Risk Additive SSE payloads only. Existing enrichment and consolidation events are unchanged. Watcher still runs as `python scripts/automem_watch.py` without a package import. --- automem/api/memory.py | 101 +++++++++++- automem/api/recall.py | 15 +- automem/api/stream.py | 26 +++ docs/API.md | 17 ++ scripts/README.md | 2 +- scripts/automem_watch.py | 65 +++++++- tests/test_stream_events.py | 320 ++++++++++++++++++++++++++++++++++++ 7 files changed, 530 insertions(+), 16 deletions(-) create mode 100644 tests/test_stream_events.py diff --git a/automem/api/memory.py b/automem/api/memory.py index 95bc7c7..947df7b 100644 --- a/automem/api/memory.py +++ b/automem/api/memory.py @@ -8,6 +8,7 @@ from flask import Blueprint, abort, jsonify, request from flask.typing import ResponseReturnValue +from automem.api.stream import emit_event, preview_text from automem.config import ( CLASSIFICATION_MODEL, MEMORY_AUTO_SUMMARIZE, @@ -248,6 +249,15 @@ def _create_association_batch( _association_failure(row["index"], "One or both memories do not exist") ) + emit_event( + "memory.associate", + { + "count": len(succeeded), + "failed_count": len(failed), + "relation_types": sorted({item["relation_type"] for item in succeeded})[:5], + }, + utc_now_fn, + ) return _batch_association_response( succeeded=succeeded, failed=failed, @@ -666,9 +676,6 @@ def store() -> Any: logger.error("FalkorDB store returned no memory row for %s", memory_id) abort(500, description="Failed to store memory in FalkorDB") - # Queue enrichment - enqueue_enrichment(memory_id) - # Handle embeddings embedding_status = "skipped" qdrant_client = get_qdrant_client() @@ -751,6 +758,21 @@ def store() -> Any: "enrichment_queued": bool(state.enrichment_queue), }, ) + emit_event( + "memory.store", + { + "id": memory_id, + "content_preview": preview_text(content), + "type": memory_type, + "importance": importance, + "tags": tags[:5], + "size_bytes": len(content.encode("utf-8")), + "elapsed_ms": int(response["query_time_ms"]), + "count": 1, + }, + utc_now, + ) + enqueue_enrichment(memory_id) return jsonify(response), 201 @bp.route("/memory/", methods=["GET"]) @@ -792,6 +814,24 @@ def update(memory_id: str) -> Any: payload = request.get_json(silent=True) if not isinstance(payload, dict): abort(400, description="JSON body is required") + changed_fields = sorted( + key + for key in payload + if key + in { + "content", + "tags", + "importance", + "type", + "confidence", + "timestamp", + "metadata", + "t_valid", + "t_invalid", + "updated_at", + "last_accessed", + } + ) graph = get_memory_graph() if graph is None: @@ -937,6 +977,18 @@ def update(memory_id: str) -> Any: collection_name, ) + emit_event( + "memory.update", + { + "id": memory_id, + "content_preview": preview_text(new_content), + "type": memory_type, + "importance": importance, + "tags": tags[:5] if isinstance(tags, list) else [], + "fields": changed_fields, + }, + utc_now, + ) return jsonify({"status": "success", "memory_id": memory_id}) @bp.route("/memory/", methods=["DELETE"]) @@ -959,6 +1011,14 @@ def delete(memory_id: str) -> Any: logger=logger, ) + emit_event( + "memory.delete", + { + "id": memory_id, + "count": 1, + }, + utc_now, + ) return jsonify({"status": "success", "memory_id": memory_id}) @bp.route("/memory/by-tag", methods=["GET", "DELETE"]) @@ -1005,6 +1065,14 @@ def by_tag() -> Any: ) deleted_count += len(memory_ids) + emit_event( + "memory.delete", + { + "count": deleted_count, + "tags": tags[:5], + }, + utc_now, + ) return jsonify({"status": "success", "tags": tags, "deleted_count": deleted_count}) memories, has_more = _load_memories_by_tag_page( @@ -1118,6 +1186,17 @@ def associate() -> Any: for prop in relation_config.get("properties", []): if prop in relationship_props: response[prop] = relationship_props[prop] + emit_event( + "memory.associate", + { + "memory1_id": memory1_id, + "memory2_id": memory2_id, + "relation_type": relation_type, + "strength": strength, + "count": 1, + }, + utc_now, + ) return jsonify(response), 201 @bp.route("/memory/batch", methods=["POST"]) @@ -1355,10 +1434,6 @@ def store_batch() -> Any: for v in validated: enqueue_embedding(v["id"], v["content"]) qdrant_status = "queued" - # Queue enrichment for all - for v in validated: - enqueue_enrichment(v["id"]) - elapsed_ms = round((time.perf_counter() - query_start) * 1000, 2) logger.info( "batch_stored", @@ -1368,6 +1443,18 @@ def store_batch() -> Any: "qdrant_status": qdrant_status, }, ) + emit_event( + "memory.store", + { + "count": len(validated), + "ids": [item["id"] for item in validated][:5], + "content_preview": f"{len(validated)} memories stored", + "elapsed_ms": int(elapsed_ms), + }, + utc_now, + ) + for v in validated: + enqueue_enrichment(v["id"]) return ( jsonify( diff --git a/automem/api/recall.py b/automem/api/recall.py index c12eac2..9fddb41 100644 --- a/automem/api/recall.py +++ b/automem/api/recall.py @@ -9,6 +9,7 @@ from flask import Blueprint, abort, jsonify, request +from automem.api.stream import emit_event, preview_text from automem.config import ( COLLECTION_NAME, DEFAULT_EXPAND_RELATIONS, @@ -29,7 +30,7 @@ ) from automem.search.runtime_recall_helpers import _hydrate_vector_relations from automem.utils.graph import _serialize_node -from automem.utils.time import _parse_iso_datetime, query_has_temporal_intent +from automem.utils.time import _parse_iso_datetime, query_has_temporal_intent, utc_now DEFAULT_STYLE_PRIORITY_TAGS: Set[str] = { "coding-style", @@ -2597,6 +2598,18 @@ def _rank_local_results(results: List[Dict[str, Any]]) -> List[Dict[str, Any]]: ), }, ) + recall_preview = query_text or " | ".join(q for q in queries_to_run if q) + emit_event( + "memory.recall", + { + "query": preview_text(recall_preview, 50) if recall_preview else "(no query)", + "limit": limit, + "result_count": len(results), + "elapsed_ms": int(response["query_time_ms"]), + "tags": tag_filters[:3] if tag_filters else [], + }, + utc_now, + ) # Update last_accessed for direct matches (not expanded related memories) if on_access and seed_results: diff --git a/automem/api/stream.py b/automem/api/stream.py index 98f0cb1..14f2bac 100644 --- a/automem/api/stream.py +++ b/automem/api/stream.py @@ -3,6 +3,11 @@ Provides a /stream endpoint that emits events for memory operations, enrichment, and consolidation tasks. Uses an in-memory subscriber pattern with bounded queues per client. + +Event types: + memory.store / memory.recall / memory.update / memory.delete / memory.associate + enrichment.start / enrichment.complete / enrichment.failed + consolidation.run """ from __future__ import annotations @@ -19,6 +24,27 @@ _subscribers_lock = Lock() +def preview_text(value: Any, limit: int = 100) -> str: + """Truncate *value* for SSE payloads without leaking huge content.""" + text = "" if value is None else str(value) + return text[:limit] + "..." if len(text) > limit else text + + +def event_count(data: Dict[str, Any], default: int = 1) -> int: + """Read an SSE payload ``count``, preserving explicit zeros.""" + if not isinstance(data, dict) or "count" not in data: + return default + try: + return int(data["count"]) + except (TypeError, ValueError): + return default + + +def is_single_memory_store(data: Dict[str, Any]) -> bool: + """True when a ``memory.store`` payload describes one concrete memory.""" + return bool(isinstance(data, dict) and data.get("id") and event_count(data) == 1) + + def emit_event(event_type: str, data: Dict[str, Any], utc_now: Callable[[], str]) -> None: """Emit an event to all SSE subscribers. diff --git a/docs/API.md b/docs/API.md index 1126577..47ed739 100644 --- a/docs/API.md +++ b/docs/API.md @@ -160,6 +160,23 @@ Consolidation - GET `/consolidate/status` - Response: `{ "status": "success", "next_runs": {...}, "history": [...] }` +Stream + +- GET `/stream` + - Authenticated SSE feed of in-process memory operations. Each event is `data: {json}` with `{ "type", "timestamp", "data" }`. Keepalive comments (`: keepalive`) are sent every 30 seconds. + - Event types: + - `memory.store` — single store (`id`, `content_preview`, `type`, `importance`, `tags`, `size_bytes`, `elapsed_ms`, `count: 1`) or batch store (`count`, `ids`, `content_preview`, `elapsed_ms`) + - `memory.recall` — ranked recall (`query`, `limit`, `result_count`, `elapsed_ms`, `tags`) + - `memory.update` — patch (`id`, `content_preview`, `type`, `importance`, `tags`, `fields`) + - `memory.delete` — single delete (`id`, `count: 1`) or bulk-by-tag (`count`, `tags`) + - `memory.associate` — single edge (`memory1_id`, `memory2_id`, `relation_type`, `strength`, `count: 1`) or batch (`count`, `failed_count`, `relation_types`) + - `enrichment.start` / `enrichment.complete` / `enrichment.failed` + - `consolidation.run` + - Failed validation (`4xx`) does not emit. Watch with `python scripts/automem_watch.py --url … --token …`. + +- GET `/stream/status` + - Response: `{ "subscribers": N }` + Notes - Tag matching supports exact and prefix semantics; vector searches are filtered by tag conditions when provided. diff --git a/scripts/README.md b/scripts/README.md index 6db5fba..f06b678 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -36,7 +36,7 @@ Run these as part of normal upkeep. | [`backup_automem.py`](backup_automem.py) | `routine` | Timestamped FalkorDB + Qdrant backup; optional S3 upload and old-backup cleanup. Cron-friendly. `--s3-bucket`, `--cleanup --keep N`. | | [`restore_from_backup.py`](restore_from_backup.py) | `routine` · `recovery` | Restore FalkorDB + Qdrant from a backup (local snapshot or downloaded API tarball). `--backup-timestamp`, `--backup-dir snapshot.tar.gz`, `--dry-run`. | | [`health_monitor.py`](health_monitor.py) | `routine` | Background service: watches FalkorDB/Qdrant health, checks graph↔vector consistency, triggers recovery, alerts. `--interval 300`. Containerized via [`Dockerfile.health-monitor`](Dockerfile.health-monitor). See [docs/HEALTH_MONITORING.md](../docs/HEALTH_MONITORING.md). | -| [`automem_watch.py`](automem_watch.py) | `routine` | Real-time terminal UI over the SSE event stream; flags garbage-write patterns and consolidation timing. `--url`, `--token`. | +| [`automem_watch.py`](automem_watch.py) | `routine` | Real-time terminal UI over `GET /stream`; tracks store/recall/update/delete/associate plus enrichment and consolidation. `--url`, `--token`. | | [`audit_relevance.py`](audit_relevance.py) | `routine` | Audit the `relevance_score` distribution from a backup file (default) or a live instance (`--live`). | | [`reembed_embeddings.py`](reembed_embeddings.py) | `routine` | Re-embed all memories and upsert vectors into Qdrant using the configured provider. `--batch-size`, `--limit`. Used after provider/dimension changes — see [docs/MIGRATIONS.md](../docs/MIGRATIONS.md). | | [`reclassify_with_llm.py`](reclassify_with_llm.py) | `routine` | Reclassify fallback `type='Memory'` records via the configured classification LLM. `--provider`; env `CLASSIFICATION_MODEL` / `CLASSIFICATION_BASE_URL` / `CLASSIFICATION_API_KEY`. | diff --git a/scripts/automem_watch.py b/scripts/automem_watch.py index 42f8fa5..f6a6f41 100755 --- a/scripts/automem_watch.py +++ b/scripts/automem_watch.py @@ -36,6 +36,21 @@ sys.exit(1) +def event_count(data: Dict, default: int = 1) -> int: + """Read an SSE payload count, preserving explicit zeros.""" + if not isinstance(data, dict) or "count" not in data: + return default + try: + return int(data["count"]) + except (TypeError, ValueError): + return default + + +def is_single_memory_store(data: Dict) -> bool: + """True when a memory.store payload describes one concrete memory.""" + return bool(isinstance(data, dict) and data.get("id") and event_count(data) == 1) + + class GarbageDetector: """Detect suspicious patterns in memory stores.""" @@ -134,6 +149,9 @@ def __init__(self, url: str, token: str, min_content_length: int = 10): self.stats = { "stores": 0, "recalls": 0, + "updates": 0, + "deletes": 0, + "associates": 0, "enriched": 0, "consolidated": 0, "errors": 0, @@ -172,15 +190,22 @@ def _process_event(self, event: Dict) -> None: event_type = event.get("type", "") # Update stats + data = event.get("data", {}) or {} if event_type == "memory.store": - self.stats["stores"] += 1 - # Check for garbage - warning = self.garbage.analyze(event) - if warning: - ts = datetime.now().strftime("%H:%M:%S") - self.errors.appendleft(f"[{ts}] [GARBAGE] {warning}") + self.stats["stores"] += event_count(data) + if is_single_memory_store(data): + warning = self.garbage.analyze(event) + if warning: + ts = datetime.now().strftime("%H:%M:%S") + self.errors.appendleft(f"[{ts}] [GARBAGE] {warning}") elif event_type == "memory.recall": self.stats["recalls"] += 1 + elif event_type == "memory.update": + self.stats["updates"] += 1 + elif event_type == "memory.delete": + self.stats["deletes"] += event_count(data) + elif event_type == "memory.associate": + self.stats["associates"] += event_count(data) elif event_type == "enrichment.complete": self.stats["enriched"] += 1 elif event_type == "enrichment.failed": @@ -254,8 +279,11 @@ def render(self) -> Layout: # Format based on event type if event_type == "memory.store": + count = data.get("count", 1) preview = data.get("content_preview", "")[:40] - details = f"{preview}... ({data.get('type', '?')})" + details = f"{preview} ({data.get('type', '?')})" + if count and count > 1: + details = f"{count} stored" type_style = "green" elif event_type == "memory.recall": query = data.get("query", "")[:30] @@ -263,6 +291,26 @@ def render(self) -> Layout: f"'{query}' -> {data.get('result_count', 0)} ({data.get('elapsed_ms', 0)}ms)" ) type_style = "cyan" + elif event_type == "memory.update": + preview = data.get("content_preview", "")[:40] + fields = ",".join(data.get("fields") or []) + details = f"{preview} [{fields}]" if fields else preview + type_style = "yellow" + elif event_type == "memory.delete": + count = data.get("count", 1) + deleted_id = data.get("id", "")[:8] + tags = ",".join(data.get("tags") or []) + details = f"{count} deleted" + if deleted_id: + details = f"{deleted_id}... ({count})" + elif tags: + details = f"{details} tags={tags}" + type_style = "red" + elif event_type == "memory.associate": + count = data.get("count", 1) + relation = data.get("relation_type") or ",".join(data.get("relation_types") or []) + details = f"{relation or 'RELATES_TO'} x{count}" + type_style = "blue" elif event_type == "enrichment.start": details = f"{data.get('memory_id', '')[:8]}... attempt {data.get('attempt', 1)}" type_style = "yellow" @@ -307,6 +355,9 @@ def render(self) -> Layout: stats_text = Text() stats_text.append(f"Stores: {self.stats['stores']}\n", style="green") stats_text.append(f"Recalls: {self.stats['recalls']}\n", style="cyan") + stats_text.append(f"Updates: {self.stats['updates']}\n", style="yellow") + stats_text.append(f"Deletes: {self.stats['deletes']}\n", style="red") + stats_text.append(f"Associates: {self.stats['associates']}\n", style="blue") stats_text.append(f"Enriched: {self.stats['enriched']}\n", style="yellow") stats_text.append(f"Consolidated: {self.stats['consolidated']}\n", style="magenta") stats_text.append(f"Errors: {self.stats['errors']}\n", style="red") diff --git a/tests/test_stream_events.py b/tests/test_stream_events.py new file mode 100644 index 0000000..49c7fa4 --- /dev/null +++ b/tests/test_stream_events.py @@ -0,0 +1,320 @@ +import json +from queue import Empty, Queue + +import pytest + +import app +from automem.api import stream as stream_mod +from automem.api.stream import event_count, is_single_memory_store +from tests.support.fake_graph import FakeGraph + + +@pytest.fixture(autouse=True) +def reset_state(monkeypatch): + state = app.ServiceState() + graph = FakeGraph() + state.memory_graph = graph + monkeypatch.setattr(app, "state", state) + monkeypatch.setattr(app, "init_falkordb", lambda: None) + monkeypatch.setattr(app, "init_qdrant", lambda: None) + monkeypatch.setattr(app, "API_TOKEN", "test-token") + monkeypatch.setattr(app, "ADMIN_TOKEN", "test-admin-token") + yield graph + + +@pytest.fixture +def client(): + return app.app.test_client() + + +@pytest.fixture +def auth_headers(): + return {"Authorization": "Bearer test-token"} + + +@pytest.fixture +def sse_queue(): + queue: Queue = Queue(maxsize=100) + with stream_mod._subscribers_lock: + stream_mod._subscribers.append(queue) + try: + yield queue + finally: + with stream_mod._subscribers_lock: + if queue in stream_mod._subscribers: + stream_mod._subscribers.remove(queue) + + +def _drain(queue: Queue) -> list[dict]: + events: list[dict] = [] + while True: + try: + raw = queue.get_nowait() + except Empty: + break + assert raw.startswith("data: ") + events.append(json.loads(raw[len("data: ") :].strip())) + return events + + +def _store(client, auth_headers, content: str, **fields) -> str: + payload = {"content": content, **fields} + response = client.post( + "/memory", + data=json.dumps(payload), + content_type="application/json", + headers=auth_headers, + ) + assert response.status_code == 201, response.get_json() + return response.get_json()["memory_id"] + + +def test_emit_event_delivers_json_payload_to_subscribers(sse_queue): + stream_mod.emit_event("memory.store", {"id": "abc"}, lambda: "2026-08-23T00:00:00Z") + events = _drain(sse_queue) + assert events == [ + { + "type": "memory.store", + "timestamp": "2026-08-23T00:00:00Z", + "data": {"id": "abc"}, + } + ] + + +def test_store_emits_memory_store_event(client, auth_headers, sse_queue): + memory_id = _store( + client, + auth_headers, + "SSE store coverage probe", + tags=["sse", "automem"], + importance=0.8, + type="Insight", + ) + events = [event for event in _drain(sse_queue) if event["type"] == "memory.store"] + assert len(events) == 1 + data = events[0]["data"] + assert data["id"] == memory_id + assert data["content_preview"] == "SSE store coverage probe" + assert data["type"] == "Insight" + assert data["importance"] == 0.8 + assert data["tags"][:2] == ["sse", "automem"] + assert data["size_bytes"] == len("SSE store coverage probe".encode("utf-8")) + assert "elapsed_ms" in data + + +def test_store_event_size_bytes_uses_utf8_length(client, auth_headers, sse_queue): + memory_id = _store(client, auth_headers, "café") + events = [event for event in _drain(sse_queue) if event["type"] == "memory.store"] + assert events[0]["data"]["id"] == memory_id + assert events[0]["data"]["size_bytes"] == len("café".encode("utf-8")) + assert events[0]["data"]["size_bytes"] > len("café") + + +def test_failed_store_does_not_emit(client, auth_headers, sse_queue): + response = client.post( + "/memory", + data=json.dumps({}), + content_type="application/json", + headers=auth_headers, + ) + assert response.status_code == 400 + assert _drain(sse_queue) == [] + + +def test_recall_emits_memory_recall_event(client, auth_headers, sse_queue): + response = client.get("/recall?query=hello+world&limit=7&tags=sse", headers=auth_headers) + assert response.status_code == 200 + events = [event for event in _drain(sse_queue) if event["type"] == "memory.recall"] + assert len(events) == 1 + data = events[0]["data"] + assert data["query"] == "hello world" + assert data["limit"] == 7 + assert data["result_count"] == 0 + assert data["tags"] == ["sse"] + assert "elapsed_ms" in data + + +def test_recall_event_includes_multi_query_text(client, auth_headers, sse_queue): + response = client.get("/recall?queries=alpha&queries=beta", headers=auth_headers) + assert response.status_code == 200 + events = [event for event in _drain(sse_queue) if event["type"] == "memory.recall"] + assert len(events) == 1 + assert events[0]["data"]["query"] == "alpha | beta" + + +def test_associate_emits_memory_associate_event(client, auth_headers, sse_queue): + memory1_id = _store(client, auth_headers, "Source memory for SSE associate") + memory2_id = _store(client, auth_headers, "Target memory for SSE associate") + _drain(sse_queue) + + response = client.post( + "/associate", + data=json.dumps( + { + "memory1_id": memory1_id, + "memory2_id": memory2_id, + "type": "RELATES_TO", + "strength": 0.9, + } + ), + content_type="application/json", + headers=auth_headers, + ) + assert response.status_code == 201 + events = [event for event in _drain(sse_queue) if event["type"] == "memory.associate"] + assert len(events) == 1 + data = events[0]["data"] + assert data["memory1_id"] == memory1_id + assert data["memory2_id"] == memory2_id + assert data["relation_type"] == "RELATES_TO" + assert data["strength"] == 0.9 + assert data["count"] == 1 + + +def test_failed_associate_does_not_emit(client, auth_headers, sse_queue): + same_id = "a0000000-0000-0000-0000-000000000001" + response = client.post( + "/associate", + data=json.dumps({"memory1_id": same_id, "memory2_id": same_id}), + content_type="application/json", + headers=auth_headers, + ) + assert response.status_code == 400 + assert _drain(sse_queue) == [] + + +def test_batch_associate_emits_one_summary_event(client, auth_headers, sse_queue): + first_id = _store(client, auth_headers, "Batch associate source") + second_id = _store(client, auth_headers, "Batch associate target") + third_id = _store(client, auth_headers, "Batch associate extra") + _drain(sse_queue) + + response = client.post( + "/associate", + data=json.dumps( + { + "associations": [ + { + "memory1_id": first_id, + "memory2_id": second_id, + "type": "RELATES_TO", + "strength": 0.8, + }, + { + "memory1_id": second_id, + "memory2_id": third_id, + "type": "LEADS_TO", + "strength": 0.7, + }, + ] + } + ), + content_type="application/json", + headers=auth_headers, + ) + assert response.status_code == 201 + events = [event for event in _drain(sse_queue) if event["type"] == "memory.associate"] + assert len(events) == 1 + data = events[0]["data"] + assert data["count"] == 2 + assert data["failed_count"] == 0 + assert sorted(data["relation_types"]) == ["LEADS_TO", "RELATES_TO"] + + +def test_update_emits_memory_update_event(client, auth_headers, sse_queue): + memory_id = _store(client, auth_headers, "Original content", tags=["sse"]) + _drain(sse_queue) + + response = client.patch( + f"/memory/{memory_id}", + data=json.dumps({"content": "Updated content", "importance": 0.95}), + content_type="application/json", + headers=auth_headers, + ) + assert response.status_code == 200 + events = [event for event in _drain(sse_queue) if event["type"] == "memory.update"] + assert len(events) == 1 + data = events[0]["data"] + assert data["id"] == memory_id + assert data["content_preview"] == "Updated content" + assert "content" in data["fields"] + assert "importance" in data["fields"] + + +def test_update_event_includes_timestamp_fields(client, auth_headers, sse_queue): + memory_id = _store(client, auth_headers, "Timestamp patch probe") + _drain(sse_queue) + response = client.patch( + f"/memory/{memory_id}", + data=json.dumps( + { + "updated_at": "2026-08-22T12:00:00Z", + "last_accessed": "2026-08-22T12:00:01Z", + } + ), + content_type="application/json", + headers=auth_headers, + ) + assert response.status_code == 200 + events = [event for event in _drain(sse_queue) if event["type"] == "memory.update"] + assert events[0]["data"]["fields"] == ["last_accessed", "updated_at"] + + +def test_delete_emits_memory_delete_event(client, auth_headers, sse_queue): + memory_id = _store(client, auth_headers, "Delete me") + _drain(sse_queue) + + response = client.delete(f"/memory/{memory_id}", headers=auth_headers) + assert response.status_code == 200 + events = [event for event in _drain(sse_queue) if event["type"] == "memory.delete"] + assert len(events) == 1 + assert events[0]["data"]["id"] == memory_id + assert events[0]["data"]["count"] == 1 + + +def test_batch_store_emits_one_store_event(client, auth_headers, sse_queue): + response = client.post( + "/memory/batch", + json={ + "memories": [ + {"content": "Batch SSE one", "tags": ["sse"]}, + {"content": "Batch SSE two", "tags": ["sse"]}, + ] + }, + headers=auth_headers, + ) + assert response.status_code == 201, response.get_json() + events = [event for event in _drain(sse_queue) if event["type"] == "memory.store"] + assert len(events) == 1 + data = events[0]["data"] + assert data["count"] == 2 + assert data["content_preview"].startswith("2 memories stored") + + +def test_delete_by_tag_emits_memory_delete_event(client, auth_headers, sse_queue): + _store(client, auth_headers, "Tagged for bulk delete", tags=["sse-bulk"]) + _drain(sse_queue) + + response = client.delete("/memory/by-tag?tags=sse-bulk", headers=auth_headers) + assert response.status_code == 200 + body = response.get_json() + assert body["deleted_count"] == 1 + events = [event for event in _drain(sse_queue) if event["type"] == "memory.delete"] + assert len(events) == 1 + assert events[0]["data"]["count"] == 1 + assert events[0]["data"]["tags"] == ["sse-bulk"] + + +def test_event_count_preserves_explicit_zero(): + assert event_count({}) == 1 + assert event_count({"count": 2}) == 2 + assert event_count({"count": 0}) == 0 + assert is_single_memory_store({"id": "abc", "count": 1}) is True + assert is_single_memory_store({"count": 2, "ids": ["a", "b"]}) is False + assert is_single_memory_store({"content_preview": "2 memories stored"}) is False + + +def test_stream_status_reports_subscriber_count(client, auth_headers, sse_queue): + response = client.get("/stream/status", headers=auth_headers) + assert response.status_code == 200 + assert response.get_json()["subscribers"] >= 1 From 009d58e9a16326200c6aff976761136dcd9654a8 Mon Sep 17 00:00:00 2001 From: Zack Katz Date: Sat, 22 Aug 2026 23:02:28 -0400 Subject: [PATCH 08/33] fix(backup): page FalkorDB export by node id instead of deep SKIP (#221) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `export_falkordb_artifact` pages the graph with `SKIP LIMIT `. That loses data two ways on a graph of any size. ## 1. Deep SKIP exceeds the server timeout FalkorDB re-scans and discards every skipped row, so each batch costs more than the last. On my instance (20,682 nodes / 238,238 relationships) the tail batches reached 1120 ms of server execution and tripped the instance's `TIMEOUT` config (1000), killing the backup partway through: ``` INFO | Exported FalkorDB relationship batch: 10000 relationships (total: 210000) INFO | Exported FalkorDB relationship batch: 10000 relationships (total: 220000) ERROR | ❌ FalkorDB backup failed: Query timed out ERROR | ❌ Backup failed: Query timed out ``` Measured against that graph, same rows returned: | Pagination | Server execution | |---|---| | `SKIP 220000 LIMIT 10000` | 1120 ms (`Filter` over `All Node Scan`) | | `WHERE id(a) >= lo AND id(a) < hi` | 37–42 ms (`NodeByIdSeek`) | This has been failing roughly every other scheduled run for me since the graph crossed ~220k relationships. ## 2. `RESULTSET_SIZE` truncates silently FalkorDB caps a result set at `RESULTSET_SIZE` and returns the short set **with no error**. A query covering 65,900 relationships came back with exactly 10,000 rows and no warning. The loop breaks when a batch returns fewer rows than `batch_size`, so any `RESULTSET_SIZE` below `batch_size` ends the export after the first batch. Running the current exporter unmodified against a 5,000-node / 15,000-relationship graph with `RESULTSET_SIZE=1000`: ``` UPSTREAM exported: {'node_count': 1000, 'relationship_count': 1000} lost: 4000 nodes, 14000 relationships (no error raised) PATCHED exported: {'node_count': 5000, 'relationship_count': 15000} ``` 80% of nodes and 93% of relationships silently missing, and the backup reports success. At the 10000 default the batch size and the cap coincide, so nothing is lost today — but the failure mode is invisible when it does fire, and it fires on the *first* batch. ## The change Pages on internal node id ranges, which FalkorDB plans as `NodeByIdSeek`. Every relationship has exactly one source node, so ranges over `id(a)` partition the edge set with no overlap or gaps. - Batches stay under the result-set cap, read from `GRAPH.CONFIG GET RESULTSET_SIZE` and bounded by `batch_size`. - A batch that *reaches* the cap is ambiguous between complete and truncated, so its range is subdivided and retried rather than trusted. - A single node whose out-degree alone reaches the cap falls back to `SKIP` bounded by that one node's edges. - Queries carry an explicit timeout — a bulk export shouldn't inherit an interactive query's `TIMEOUT` budget. - Exported totals are verified against `count()` before the artifact is written, with a 1% tolerance for concurrent writes. A larger shortfall raises `BackupError` instead of writing a backup that looks complete. Relationship windows self-tune (256 node ids, doubling while batches stay clear of the cap), so batch count adapts to graph density rather than being fixed. **The backup file format is unchanged**, so existing restore tooling reads these backups as-is. ## Verification Against a live FalkorDB instance: - The current exporter reproduces the timeout at 220,000 relationships. - This one exports 20,715 nodes and 238,668 relationships, matching live `count()` exactly, zero duplicate ids, in 24 s. Test suite: 642 passed, 1 skipped (`pytest -m "not integration and not live"`), including the 24 existing `test_backup_endpoint.py` tests. `black`, `isort`, and `flake8` clean. `tests/support/fake_graph.py` gains id-range and `count()`/`max()` handling plus an optional `resultset_cap` that truncates silently the way the server does. **Six of the eight new tests fail against the current exporter**; the two that pass are the small-graph and empty-graph cases it already handled. ## One thing I did not change The export is not atomic — nodes are read, then relationships. A relationship created mid-export can point at a node newer than the node phase, so it lands in the backup with a target that isn't there. I saw 16 such edges in a run where the graph grew by 24 nodes during the export. `_restore_relationship_batches` already skips these with a warning, and the race predates this change (the old exporter had a wider window, being slower), so I left it alone. Happy to follow up if you'd like it addressed. --- automem/backup.py | 278 +++++++++++++++++++++------ tests/support/fake_graph.py | 110 ++++++++--- tests/test_falkordb_backup_export.py | 157 +++++++++++++++ 3 files changed, 455 insertions(+), 90 deletions(-) create mode 100644 tests/test_falkordb_backup_export.py diff --git a/automem/backup.py b/automem/backup.py index 095c56b..19e51ea 100644 --- a/automem/backup.py +++ b/automem/backup.py @@ -14,6 +14,47 @@ VALID_BACKUP_INCLUDES = ("falkordb", "qdrant") STREAM_QUEUE_MAX_CHUNKS = 8 +# FalkorDB caps every result set at RESULTSET_SIZE and returns the short set +# with no error, so a batch that reaches the cap may have lost rows silently. +DEFAULT_RESULTSET_CAP = 10000 +INITIAL_RELATIONSHIP_ID_CHUNK = 256 +MAX_RELATIONSHIP_ID_CHUNK = 4096 +# A bulk export should not inherit an interactive query's TIMEOUT budget. +BACKUP_QUERY_TIMEOUT_MS = 300000 +# Concurrent writes shift the totals mid-export; a wider gap means lost rows. +BACKUP_COUNT_TOLERANCE = 0.01 + +NODE_EXPORT_QUERY = """ + MATCH (n) + WHERE id(n) >= {lo} AND id(n) < {hi} + RETURN + id(n) as id, + labels(n) as labels, + properties(n) as props + """ + +RELATIONSHIP_EXPORT_QUERY = """ + MATCH (a)-[r]->(b) + WHERE id(a) >= {lo} AND id(a) < {hi} + RETURN + id(a) as source_id, + type(r) as rel_type, + id(b) as target_id, + properties(r) as props + """ + +SINGLE_NODE_RELATIONSHIP_EXPORT_QUERY = """ + MATCH (a)-[r]->(b) + WHERE id(a) = {node_id} + RETURN + id(a) as source_id, + type(r) as rel_type, + id(b) as target_id, + properties(r) as props + ORDER BY id(r) + SKIP {offset} LIMIT {limit} + """ + class BackupError(RuntimeError): """Raised when backup creation fails.""" @@ -70,89 +111,204 @@ def _query_rows(result: Any) -> list[Any]: return list(getattr(result, "result_set", []) or []) -def export_falkordb_artifact( - *, - graph: Any, - graph_name: str, - timestamp: str, - batch_size: int = 10000, - logger: Any = None, -) -> BackupArtifact: - """Export FalkorDB graph data as a compressed JSON backup artifact.""" - if graph is None: - raise BackupError("FalkorDB is unavailable") +def _backup_rows(graph: Any, query: str) -> list[Any]: + return _query_rows(graph.query(query, timeout=BACKUP_QUERY_TIMEOUT_MS)) - nodes: list[dict[str, Any]] = [] - offset = 0 - while True: - rows = _query_rows( - graph.query( - f""" - MATCH (n) - RETURN - id(n) as id, - labels(n) as labels, - properties(n) as props - SKIP {offset} LIMIT {batch_size} - """ - ) - ) - if not rows: - break +def _backup_scalar(graph: Any, query: str) -> Any: + rows = _backup_rows(graph, query) + if not rows or not rows[0]: + return None + return rows[0][0] + + +def _resultset_cap(graph: Any, batch_size: int, logger: Any = None) -> int: + """Largest batch that FalkorDB will return in full.""" + configured = DEFAULT_RESULTSET_CAP + try: + raw = graph.execute_command("GRAPH.CONFIG", "GET", "RESULTSET_SIZE") + reported = int(raw[1]) + # 0 or negative means unlimited. + if reported > 0: + configured = reported + except Exception as exc: # noqa: BLE001 - config read is best-effort + if logger: + logger.debug("Could not read RESULTSET_SIZE (%s); assuming %d", exc, configured) - for row in rows: - nodes.append({"id": row[0], "labels": row[1], "properties": row[2]}) + return max(1, min(batch_size, configured)) - if logger: + +def _export_falkordb_nodes( + graph: Any, max_node_id: int, cap: int, logger: Any = None +) -> list[dict[str, Any]]: + nodes: list[dict[str, Any]] = [] + # One row per node, so half the cap can never reach it. + chunk = max(1, cap // 2) + lo = 0 + + while lo <= max_node_id: + hi = min(lo + chunk, max_node_id + 1) + rows = _backup_rows(graph, NODE_EXPORT_QUERY.format(lo=lo, hi=hi)) + nodes.extend({"id": row[0], "labels": row[1], "properties": row[2]} for row in rows) + + if logger and rows: logger.info( "Exported FalkorDB node batch: %d nodes (total: %d)", len(rows), len(nodes), ) - if len(rows) < batch_size: - break - offset += batch_size + lo = hi + + return nodes + +def _export_node_relationships(graph: Any, node_id: int, cap: int) -> list[dict[str, Any]]: + """Page one node's out-edges, for a node whose degree alone reaches the cap.""" relationships: list[dict[str, Any]] = [] offset = 0 while True: - rows = _query_rows( - graph.query( - f""" - MATCH (a)-[r]->(b) - RETURN - id(a) as source_id, - type(r) as rel_type, - id(b) as target_id, - properties(r) as props - SKIP {offset} LIMIT {batch_size} - """ - ) + rows = _backup_rows( + graph, + SINGLE_NODE_RELATIONSHIP_EXPORT_QUERY.format(node_id=node_id, offset=offset, limit=cap), ) - if not rows: - break + relationships.extend( + { + "source_id": row[0], + "type": row[1], + "target_id": row[2], + "properties": row[3], + } + for row in rows + ) + if len(rows) < cap: + return relationships + offset += cap - for row in rows: - relationships.append( - { - "source_id": row[0], - "type": row[1], - "target_id": row[2], - "properties": row[3], - } - ) - if logger: +def _export_falkordb_relationships( + graph: Any, max_node_id: int, cap: int, logger: Any = None +) -> list[dict[str, Any]]: + relationships: list[dict[str, Any]] = [] + chunk = INITIAL_RELATIONSHIP_ID_CHUNK + lo = 0 + + while lo <= max_node_id: + hi = min(lo + chunk, max_node_id + 1) + rows = _backup_rows(graph, RELATIONSHIP_EXPORT_QUERY.format(lo=lo, hi=hi)) + + # At the cap the result set may have been truncated server-side, and a + # truncated batch is indistinguishable from a complete one. Narrow the + # range and retry rather than trust it. + reached_cap = len(rows) >= cap + if reached_cap and hi - lo > 1: + chunk = max(1, (hi - lo) // 4) + continue + if reached_cap: + relationships.extend(_export_node_relationships(graph, lo, cap)) + if logger: + logger.info( + "Exported FalkorDB relationships for high-degree node %d (total: %d)", + lo, + len(relationships), + ) + lo += 1 + chunk = INITIAL_RELATIONSHIP_ID_CHUNK + continue + + relationships.extend( + { + "source_id": row[0], + "type": row[1], + "target_id": row[2], + "properties": row[3], + } + for row in rows + ) + if logger and rows: logger.info( "Exported FalkorDB relationship batch: %d relationships (total: %d)", len(rows), len(relationships), ) - if len(rows) < batch_size: - break - offset += batch_size + + lo = hi + room_to_grow = len(rows) * 3 < cap and chunk < MAX_RELATIONSHIP_ID_CHUNK + if room_to_grow: + chunk = min(chunk * 2, MAX_RELATIONSHIP_ID_CHUNK) + + return relationships + + +def _verify_export_count( + *, kind: str, exported: int, before: int, after: int, logger: Any = None +) -> None: + """Fail a truncated export instead of writing a backup that looks complete.""" + expected = min(before, after) + if exported >= expected: + return + + shortfall = expected - exported + if exported < expected * (1 - BACKUP_COUNT_TOLERANCE): + raise BackupError( + f"FalkorDB {kind} export is short by {shortfall} " + f"(exported {exported}, graph held {expected}) - refusing to write a partial backup" + ) + if logger: + logger.warning( + "FalkorDB %s export is short by %d (exported %d, graph held %d); " + "within tolerance for concurrent writes", + kind, + shortfall, + exported, + expected, + ) + + +def export_falkordb_artifact( + *, + graph: Any, + graph_name: str, + timestamp: str, + batch_size: int = 10000, + logger: Any = None, +) -> BackupArtifact: + """Export FalkorDB graph data as a compressed JSON backup artifact. + + Pages on internal node id ranges, which FalkorDB plans as ``NodeByIdSeek``. + ``SKIP `` re-scans every skipped row, so its cost grows with depth + until batches exceed the server's ``TIMEOUT`` and the export dies partway + through a large graph. + """ + if graph is None: + raise BackupError("FalkorDB is unavailable") + + cap = _resultset_cap(graph, batch_size, logger) + max_node_id = _backup_scalar(graph, "MATCH (n) RETURN max(id(n))") + node_count_before = _backup_scalar(graph, "MATCH (n) RETURN count(n)") or 0 + relationship_count_before = _backup_scalar(graph, "MATCH ()-[r]->() RETURN count(r)") or 0 + + if max_node_id is None: + nodes: list[dict[str, Any]] = [] + relationships: list[dict[str, Any]] = [] + else: + nodes = _export_falkordb_nodes(graph, int(max_node_id), cap, logger) + relationships = _export_falkordb_relationships(graph, int(max_node_id), cap, logger) + + _verify_export_count( + kind="node", + exported=len(nodes), + before=node_count_before, + after=_backup_scalar(graph, "MATCH (n) RETURN count(n)") or 0, + logger=logger, + ) + _verify_export_count( + kind="relationship", + exported=len(relationships), + before=relationship_count_before, + after=_backup_scalar(graph, "MATCH ()-[r]->() RETURN count(r)") or 0, + logger=logger, + ) stats = { "node_count": len(nodes), diff --git a/tests/support/fake_graph.py b/tests/support/fake_graph.py index 9b912dc..450da75 100644 --- a/tests/support/fake_graph.py +++ b/tests/support/fake_graph.py @@ -32,6 +32,21 @@ def _skip_limit(query: str) -> tuple[int, int | None]: return int(match.group(1)), int(match.group(2)) +def _id_range(query: str, alias: str) -> tuple[int, int] | None: + """Parse the `id(x) >= lo AND id(x) < hi` window the backup exporter pages with.""" + match = re.search( + rf"id\({alias}\)\s*>=\s*(\d+)\s+AND\s+id\({alias}\)\s*<\s*(\d+)", " ".join(query.split()) + ) + if not match: + return None + return int(match.group(1)), int(match.group(2)) + + +def _single_id(query: str, alias: str) -> int | None: + match = re.search(rf"id\({alias}\)\s*=\s*(\d+)", " ".join(query.split())) + return int(match.group(1)) if match else None + + def _memory_type_allowed(memory: Dict[str, Any], excluded_types: List[str]) -> bool: if not excluded_types: return True @@ -48,6 +63,10 @@ class FakeGraph: def __init__(self, *, seed_enrichment_fixture: bool = False) -> None: self.queries: List[tuple[str, Dict[str, Any]]] = [] + self.query_kwargs: List[Dict[str, Any]] = [] + + # Set to mimic FalkorDB's RESULTSET_SIZE cap (silent truncation). + self.resultset_cap: int | None = None # Generic memory storage + associations self.memories: Dict[str, Dict[str, Any]] = {} @@ -94,8 +113,46 @@ def __init__(self, *, seed_enrichment_fixture: bool = False) -> None: ["mem-c", "Automation habit noted"], ] + def _backup_node_rows(self) -> List[List[Any]]: + return [ + [index, ["Memory"], dict(memory)] + for index, (_memory_id, memory) in enumerate(sorted(self.memories.items())) + ] + + def _backup_relationship_rows(self) -> List[List[Any]]: + backup_ids = {memory_id: index for index, memory_id in enumerate(sorted(self.memories))} + rows: List[List[Any]] = [] + for rel in self.relationships: + source_id = str(rel.get("id1") or "") + target_id = str(rel.get("id2") or "") + if source_id not in backup_ids or target_id not in backup_ids: + continue + props = {key: value for key, value in rel.items() if key not in {"id1", "id2", "type"}} + rows.append( + [ + backup_ids[source_id], + str(rel.get("type") or "RELATES_TO"), + backup_ids[target_id], + props, + ] + ) + return sorted(rows, key=lambda row: row[0]) + + def _apply_result_limits(self, rows: List[List[Any]], query: str) -> List[List[Any]]: + """Apply SKIP/LIMIT, then RESULTSET_SIZE truncation the way the server does: silently.""" + skip, limit = _skip_limit(query) + rows = rows[skip : None if limit is None else skip + limit] + if self.resultset_cap is not None: + rows = rows[: self.resultset_cap] + return rows + + def execute_command(self, *args: Any) -> Any: + if args[:3] == ("GRAPH.CONFIG", "GET", "RESULTSET_SIZE"): + return ["RESULTSET_SIZE", self.resultset_cap or 0] + raise NotImplementedError(f"FakeGraph.execute_command{args}") + def query(self, query: str, params: Dict[str, Any] | None = None, **kwargs: Any) -> FakeResult: - del kwargs + self.query_kwargs.append(dict(kwargs)) params = params or {} self.queries.append((query, params)) @@ -196,41 +253,36 @@ def query(self, query: str, params: Dict[str, Any] | None = None, **kwargs: Any) return FakeResult([]) # Full graph backup export + if "RETURN max(id(n))" in query: + node_rows = self._backup_node_rows() + return FakeResult([[node_rows[-1][0] if node_rows else None]]) + + if "RETURN count(n)" in query: + return FakeResult([[len(self._backup_node_rows())]]) + + if "RETURN count(r)" in query: + return FakeResult([[len(self._backup_relationship_rows())]]) + if "MATCH (n)" in query and "id(n) as id" in query and "properties(n) as props" in query: - memory_items = sorted(self.memories.items(), key=lambda item: item[0]) - rows = [ - [index, ["Memory"], dict(memory)] - for index, (_memory_id, memory) in enumerate(memory_items) - ] - skip, limit = _skip_limit(query) - return FakeResult(rows[skip : None if limit is None else skip + limit]) + rows = self._backup_node_rows() + id_range = _id_range(query, "n") + if id_range: + rows = [row for row in rows if id_range[0] <= row[0] < id_range[1]] + return FakeResult(self._apply_result_limits(rows, query)) if ( "MATCH (a)-[r]->(b)" in query and "id(a) as source_id" in query and "properties(r) as props" in query ): - memory_ids = [memory_id for memory_id, _memory in sorted(self.memories.items())] - backup_ids = {memory_id: index for index, memory_id in enumerate(memory_ids)} - rows = [] - for rel in self.relationships: - source_id = str(rel.get("id1") or "") - target_id = str(rel.get("id2") or "") - if source_id not in backup_ids or target_id not in backup_ids: - continue - props = { - key: value for key, value in rel.items() if key not in {"id1", "id2", "type"} - } - rows.append( - [ - backup_ids[source_id], - str(rel.get("type") or "RELATES_TO"), - backup_ids[target_id], - props, - ] - ) - skip, limit = _skip_limit(query) - return FakeResult(rows[skip : None if limit is None else skip + limit]) + rows = self._backup_relationship_rows() + id_range = _id_range(query, "a") + single_id = _single_id(query, "a") + if id_range: + rows = [row for row in rows if id_range[0] <= row[0] < id_range[1]] + elif single_id is not None: + rows = [row for row in rows if row[0] == single_id] + return FakeResult(self._apply_result_limits(rows, query)) # Batch memory create/upsert if "UNWIND $memories AS m" in query and "MERGE (node:Memory {id: m.id})" in query: diff --git a/tests/test_falkordb_backup_export.py b/tests/test_falkordb_backup_export.py new file mode 100644 index 0000000..11b27ec --- /dev/null +++ b/tests/test_falkordb_backup_export.py @@ -0,0 +1,157 @@ +"""Regression tests for the FalkorDB backup export pagination. + +Covers the two ways the previous ``SKIP LIMIT `` paging +lost data: batches that grew slower with depth until they exceeded the server's +``TIMEOUT``, and FalkorDB's ``RESULTSET_SIZE`` cap silently truncating a result +set that the loop then read as "the last page". +""" + +from __future__ import annotations + +import gzip +import io +import json +import re +from typing import Any + +import pytest + +from automem.backup import BackupError, export_falkordb_artifact +from tests.support.fake_graph import FakeGraph + + +def build_graph(memory_count: int, degrees: dict[int, int] | None = None) -> FakeGraph: + """Seed a graph with `memory_count` memories and per-node out-degrees.""" + graph = FakeGraph() + for index in range(memory_count): + memory_id = f"mem-{index:05d}" + graph.memories[memory_id] = {"id": memory_id, "content": f"memory {index}"} + graph.nodes.add(memory_id) + + for source, degree in (degrees or {}).items(): + for step in range(degree): + graph.relationships.append( + { + "id1": f"mem-{source:05d}", + "id2": f"mem-{(source + step + 1) % memory_count:05d}", + "type": "RELATES_TO", + "strength": 0.5, + } + ) + return graph + + +def export(graph: FakeGraph, **kwargs: Any) -> dict[str, Any]: + artifact = export_falkordb_artifact( + graph=graph, graph_name="memories", timestamp="20260101_000000", **kwargs + ) + with gzip.open(io.BytesIO(artifact.data), "rt", encoding="utf-8") as handle: + return json.load(handle) + + +def assert_exported_everything(graph: FakeGraph, payload: dict[str, Any]) -> None: + assert payload["stats"]["node_count"] == len(graph.memories) + assert payload["stats"]["relationship_count"] == len(graph.relationships) + assert len(payload["nodes"]) == len(graph.memories) + + exported_ids = [node["id"] for node in payload["nodes"]] + assert len(set(exported_ids)) == len(exported_ids), "a node was exported twice" + + exported_edges = sorted( + (rel["source_id"], rel["type"], rel["target_id"]) for rel in payload["relationships"] + ) + assert len(exported_edges) == len(graph.relationships) + node_ids = set(exported_ids) + assert all(edge[0] in node_ids and edge[2] in node_ids for edge in exported_edges) + + +def test_export_round_trips_a_small_graph() -> None: + graph = build_graph(50, {index: 2 for index in range(50)}) + assert_exported_everything(graph, export(graph)) + + +def test_export_is_complete_when_resultset_cap_is_below_batch_size() -> None: + """RESULTSET_SIZE under batch_size used to end the export after one batch.""" + graph = build_graph(2500, {index: 3 for index in range(2500)}) + graph.resultset_cap = 1000 + + payload = export(graph) + + assert_exported_everything(graph, payload) + assert payload["stats"] == {"node_count": 2500, "relationship_count": 7500} + + +def test_export_subdivides_a_dense_relationship_range() -> None: + """A node id window holding more relationships than the cap must be split.""" + degrees = {index: 1 for index in range(300)} + degrees.update({index: 60 for index in range(40, 45)}) + graph = build_graph(300, degrees) + graph.resultset_cap = 100 + + assert_exported_everything(graph, export(graph)) + + +def test_export_pages_a_single_high_degree_node() -> None: + """One node with more out-edges than the cap falls back to bounded SKIP.""" + degrees = {index: 2 for index in range(200)} + degrees[7] = 250 + graph = build_graph(200, degrees) + graph.resultset_cap = 100 + + payload = export(graph) + + assert_exported_everything(graph, payload) + hub_edges = [rel for rel in payload["relationships"] if rel["source_id"] == 7] + assert len(hub_edges) == 250 + + +def test_export_handles_out_degree_exactly_at_the_cap() -> None: + """Rows == cap is ambiguous between complete and truncated; treat it as truncated.""" + degrees = {index: 1 for index in range(50)} + degrees[3] = 100 + graph = build_graph(50, degrees) + graph.resultset_cap = 100 + + assert_exported_everything(graph, export(graph)) + + +def test_export_rejects_a_silently_truncated_result() -> None: + """A backup that lost rows must fail rather than look complete.""" + + class LossyGraph(FakeGraph): + def _apply_result_limits(self, rows: list[Any], query: str) -> list[Any]: + kept = super()._apply_result_limits(rows, query) + return [row for index, row in enumerate(kept) if index % 5] + + graph = LossyGraph() + for index in range(300): + memory_id = f"mem-{index:05d}" + graph.memories[memory_id] = {"id": memory_id} + + with pytest.raises(BackupError, match="short by"): + export(graph) + + +def test_export_handles_an_empty_graph() -> None: + payload = export(FakeGraph()) + + assert payload["nodes"] == [] + assert payload["relationships"] == [] + assert payload["stats"] == {"node_count": 0, "relationship_count": 0} + + +def test_export_avoids_deep_skip_and_always_sets_a_timeout() -> None: + """Deep SKIP is what exceeded the server TIMEOUT on a large graph.""" + graph = build_graph(400, {index: 3 for index in range(400)}) + + export(graph) + + assert graph.query_kwargs, "no queries were issued" + assert all(kwargs.get("timeout") for kwargs in graph.query_kwargs) + + offsets = [ + int(match.group(1)) + for query, _params in graph.queries + if (match := re.search(r"\bSKIP\s+(\d+)", query)) + ] + assert all(offset == 0 for offset in offsets), f"deep SKIP still in use: {offsets}" From daa906bda3d089eaba9f7e5d2543615902bfc712 Mon Sep 17 00:00:00 2001 From: Jack Arturo Date: Sun, 23 Aug 2026 05:06:20 +0200 Subject: [PATCH 09/33] style(tests): black-format qdrant payload index assertions --- tests/test_vector_size_safety.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/test_vector_size_safety.py b/tests/test_vector_size_safety.py index 4485c6b..1425bae 100644 --- a/tests/test_vector_size_safety.py +++ b/tests/test_vector_size_safety.py @@ -276,17 +276,13 @@ def test_indexes_every_field_recall_filters_on(self): """`type` backs RECALL_EXCLUDED_TYPES; without an index Qdrant 400s every search.""" qdrant = self._run_ensure(SimpleNamespace(KEYWORD="keyword")) - indexed = { - call.kwargs["field_name"] for call in qdrant.create_payload_index.call_args_list - } + indexed = {call.kwargs["field_name"] for call in qdrant.create_payload_index.call_args_list} assert indexed == {"tags", "tag_prefixes", "type"} def test_indexes_every_field_without_payload_schema_enum(self): qdrant = self._run_ensure(None) - indexed = { - call.kwargs["field_name"] for call in qdrant.create_payload_index.call_args_list - } + indexed = {call.kwargs["field_name"] for call in qdrant.create_payload_index.call_args_list} assert indexed == {"tags", "tag_prefixes", "type"} assert all( call.kwargs["field_schema"] == "keyword" From ea148e5e15e0b37ac98cf49a71470bc15b54f721 Mon Sep 17 00:00:00 2001 From: mikemikimike <13286568797@163.com> Date: Sun, 23 Aug 2026 12:14:29 +0800 Subject: [PATCH 10/33] fix(enrichment): circuit-break exhausted quota calls --- app.py | 1 + automem/api/enrichment.py | 1 + automem/api/memory.py | 2 + automem/classification/memory_classifier.py | 9 ++++ automem/config.py | 2 + automem/service_state.py | 55 ++++++++++++++++++++- automem/utils/text.py | 11 ++++- tests/test_enrichment_circuit.py | 34 +++++++++++++ 8 files changed, 112 insertions(+), 3 deletions(-) create mode 100644 tests/test_enrichment_circuit.py diff --git a/app.py b/app.py index b24e1b4..35e6a96 100644 --- a/app.py +++ b/app.py @@ -318,6 +318,7 @@ def require_api_token() -> None: classification_model=CLASSIFICATION_MODEL, logger=logger, stats=state.classification_stats, + circuit=state.enrichment_circuit, ) diff --git a/automem/api/enrichment.py b/automem/api/enrichment.py index ba5b2ae..c8841ed 100644 --- a/automem/api/enrichment.py +++ b/automem/api/enrichment.py @@ -30,6 +30,7 @@ def enrichment_status() -> Any: "max_attempts": max_attempts, "stats": state.enrichment_stats.to_dict(), "classification": state.classification_stats.to_dict(), + "circuit": state.enrichment_circuit.to_dict(), } return jsonify(response) diff --git a/automem/api/memory.py b/automem/api/memory.py index 947df7b..4f9fd33 100644 --- a/automem/api/memory.py +++ b/automem/api/memory.py @@ -514,6 +514,7 @@ def store() -> Any: openai_client, CLASSIFICATION_MODEL, MEMORY_SUMMARY_TARGET_LENGTH, + state.enrichment_circuit, ) if summary: original_content = content @@ -1248,6 +1249,7 @@ def store_batch() -> Any: openai_client, CLASSIFICATION_MODEL, MEMORY_SUMMARY_TARGET_LENGTH, + state.enrichment_circuit, ) if summary: logger.info( diff --git a/automem/classification/memory_classifier.py b/automem/classification/memory_classifier.py index 17f6509..4869f96 100644 --- a/automem/classification/memory_classifier.py +++ b/automem/classification/memory_classifier.py @@ -97,6 +97,7 @@ def __init__( classification_model: str, logger: Any, stats: Any = None, + circuit: Any = None, ) -> None: self._normalize_memory_type = normalize_memory_type self._ensure_openai_client = ensure_openai_client @@ -104,6 +105,7 @@ def __init__( self._classification_model = classification_model self._logger = logger self._stats = stats + self._circuit = circuit def classify(self, content: str, *, use_llm: bool = True) -> tuple[str, float]: """Classify memory type and return confidence score.""" @@ -134,12 +136,17 @@ def classify(self, content: str, *, use_llm: bool = True) -> tuple[str, float]: except Exception as exc: self._logger.exception("LLM classification failed, using fallback") llm_error = str(exc) + if self._circuit is not None: + self._circuit.record_failure(llm_error) if self._stats is not None: self._stats.record_fallback(llm_error) return "Memory", 0.3 def _classify_with_llm(self, content: str) -> Optional[tuple[str, float]]: + if self._circuit is not None and not self._circuit.allow_request(): + self._logger.info("Skipping LLM classification while enrichment circuit is open") + return None client = self._get_openai_client() if client is None: self._ensure_openai_client() @@ -166,6 +173,8 @@ def _classify_with_llm(self, content: str) -> Optional[tuple[str, float]]: response_format={"type": "json_object"}, **extra_params, ) + if self._circuit is not None: + self._circuit.record_success() raw_content = response.choices[0].message.content if not raw_content: diff --git a/automem/config.py b/automem/config.py index 5241c62..555e9a9 100644 --- a/automem/config.py +++ b/automem/config.py @@ -178,6 +178,8 @@ } # Target length for summarized content MEMORY_SUMMARY_TARGET_LENGTH = int(os.getenv("MEMORY_SUMMARY_TARGET_LENGTH", "300")) +# Cooldown after a definitive LLM quota exhaustion response. +ENRICHMENT_CIRCUIT_COOLDOWN_SECONDS = float(os.getenv("ENRICHMENT_CIRCUIT_COOLDOWN_SECONDS", "300")) # Memory types for classification MEMORY_TYPES = {"Decision", "Pattern", "Preference", "Style", "Habit", "Insight", "Context"} diff --git a/automem/service_state.py b/automem/service_state.py index e583182..af532bb 100644 --- a/automem/service_state.py +++ b/automem/service_state.py @@ -4,15 +4,65 @@ from queue import Queue from threading import Event, Lock, Thread from typing import Any, Dict, Optional, Set +import time from falkordb import FalkorDB from qdrant_client import QdrantClient -from automem.config import VECTOR_SIZE +from automem.config import ENRICHMENT_CIRCUIT_COOLDOWN_SECONDS, VECTOR_SIZE from automem.embedding.provider import EmbeddingProvider from automem.utils.time import utc_now +class EnrichmentCircuit: + """Fail-soft cooldown for definitive LLM quota failures.""" + + def __init__(self, cooldown_seconds: float = 300, clock: Any = time.monotonic) -> None: + self._cooldown_seconds = cooldown_seconds + self._clock = clock + self._opened_until = 0.0 + self._probe_pending = False + self._lock = Lock() + self.circuit_open_skips = 0 + self.recoveries = 0 + + def allow_request(self) -> bool: + with self._lock: + now = self._clock() + if now < self._opened_until: + self.circuit_open_skips += 1 + return False + if self._opened_until: + if self._probe_pending: + self.circuit_open_skips += 1 + return False + self._probe_pending = True + return True + + def record_failure(self, error: str) -> bool: + if "insufficient_quota" not in error.lower() and "quota" not in error.lower(): + return False + with self._lock: + self._opened_until = self._clock() + self._cooldown_seconds + self._probe_pending = False + return True + + def record_success(self) -> None: + with self._lock: + if self._opened_until: + self.recoveries += 1 + self._opened_until = 0.0 + self._probe_pending = False + + def to_dict(self) -> Dict[str, Any]: + with self._lock: + return { + "circuit_open_skips": self.circuit_open_skips, + "recoveries": self.recoveries, + "open": self._clock() < self._opened_until, + } + + @dataclass class EnrichmentStats: processed_total: int = 0 @@ -106,6 +156,9 @@ class ServiceState: enrichment_thread: Optional[Thread] = None enrichment_stats: EnrichmentStats = field(default_factory=EnrichmentStats) classification_stats: ClassificationStats = field(default_factory=ClassificationStats) + enrichment_circuit: EnrichmentCircuit = field( + default_factory=lambda: EnrichmentCircuit(ENRICHMENT_CIRCUIT_COOLDOWN_SECONDS) + ) enrichment_inflight: Set[str] = field(default_factory=set) enrichment_pending: Set[str] = field(default_factory=set) enrichment_lock: Lock = field(default_factory=Lock) diff --git a/automem/utils/text.py b/automem/utils/text.py index 2c4a4c6..ce15fdf 100644 --- a/automem/utils/text.py +++ b/automem/utils/text.py @@ -122,6 +122,7 @@ def summarize_content( openai_client: Any, model: str, target_length: int = 300, + circuit: Any = None, ) -> Optional[str]: """Summarize content using an LLM to fit within target length. @@ -137,9 +138,11 @@ def summarize_content( if openai_client is None: logger.warning("Cannot summarize: OpenAI client not available") return None - if not content or len(content) <= target_length: return content + if circuit is not None and not circuit.allow_request(): + logger.info("Skipping summarization while enrichment circuit is open") + return None try: system_prompt = SUMMARIZE_SYSTEM_PROMPT.format(target_length=target_length) @@ -163,6 +166,8 @@ def summarize_content( ], **extra_params, ) + if circuit is not None: + circuit.record_success() summary = response.choices[0].message.content.strip() @@ -183,8 +188,10 @@ def summarize_content( ) return None - except Exception: + except Exception as exc: logger.exception("Memory summarization failed") + if circuit is not None: + circuit.record_failure(str(exc)) return None diff --git a/tests/test_enrichment_circuit.py b/tests/test_enrichment_circuit.py new file mode 100644 index 0000000..618cc1c --- /dev/null +++ b/tests/test_enrichment_circuit.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +from datetime import timedelta + +from automem.service_state import EnrichmentCircuit + + +def test_quota_failure_opens_circuit_and_skips_requests(): + circuit = EnrichmentCircuit(cooldown_seconds=60, clock=lambda: 100.0) + + assert circuit.allow_request() is True + assert circuit.record_failure("429 insufficient_quota") is True + assert circuit.allow_request() is False + assert circuit.to_dict()["circuit_open_skips"] == 1 + + +def test_non_quota_failure_does_not_open_circuit(): + circuit = EnrichmentCircuit(cooldown_seconds=60, clock=lambda: 100.0) + + assert circuit.record_failure("connection reset") is False + assert circuit.allow_request() is True + + +def test_successful_probe_closes_circuit_and_records_recovery(): + now = [100.0] + circuit = EnrichmentCircuit(cooldown_seconds=60, clock=lambda: now[0]) + + circuit.record_failure("insufficient_quota") + now[0] += 60 + assert circuit.allow_request() is True + circuit.record_success() + + assert circuit.allow_request() is True + assert circuit.to_dict()["recoveries"] == 1 From 6566675564e4a65f4d9453ecd9f0f42bbdc96930 Mon Sep 17 00:00:00 2001 From: mikemikimike <13286568797@163.com> Date: Sun, 23 Aug 2026 12:30:29 +0800 Subject: [PATCH 11/33] test(enrichment): cover probe recovery and request suppression --- automem/service_state.py | 4 +++ tests/test_enrichment_circuit.py | 57 +++++++++++++++++++++++++++++++- 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/automem/service_state.py b/automem/service_state.py index af532bb..31eb63e 100644 --- a/automem/service_state.py +++ b/automem/service_state.py @@ -41,6 +41,10 @@ def allow_request(self) -> bool: def record_failure(self, error: str) -> bool: if "insufficient_quota" not in error.lower() and "quota" not in error.lower(): + with self._lock: + if self._probe_pending: + self._opened_until = 0.0 + self._probe_pending = False return False with self._lock: self._opened_until = self._clock() + self._cooldown_seconds diff --git a/tests/test_enrichment_circuit.py b/tests/test_enrichment_circuit.py index 618cc1c..fd6690d 100644 --- a/tests/test_enrichment_circuit.py +++ b/tests/test_enrichment_circuit.py @@ -1,8 +1,11 @@ from __future__ import annotations -from datetime import timedelta +from types import SimpleNamespace +import logging +from automem.classification.memory_classifier import MemoryClassifier from automem.service_state import EnrichmentCircuit +from automem.utils.text import summarize_content def test_quota_failure_opens_circuit_and_skips_requests(): @@ -32,3 +35,55 @@ def test_successful_probe_closes_circuit_and_records_recovery(): assert circuit.allow_request() is True assert circuit.to_dict()["recoveries"] == 1 + + +def test_failed_probe_does_not_permanently_block_future_requests(): + now = [100.0] + circuit = EnrichmentCircuit(cooldown_seconds=60, clock=lambda: now[0]) + + circuit.record_failure("insufficient_quota") + now[0] += 60 + assert circuit.allow_request() is True + + +def test_classifier_makes_no_second_request_while_circuit_is_open(): + calls = [] + + def create(*args, **kwargs): + calls.append(1) + raise RuntimeError("insufficient_quota") + + client = SimpleNamespace(chat=SimpleNamespace(completions=SimpleNamespace(create=create))) + circuit = EnrichmentCircuit(cooldown_seconds=60, clock=lambda: 100.0) + classifier = MemoryClassifier( + normalize_memory_type=lambda raw: (raw, False), + ensure_openai_client=lambda: None, + get_openai_client=lambda: client, + classification_model="gpt-4o-mini", + logger=logging.getLogger(__name__), + circuit=circuit, + ) + + classifier.classify("qwxz flibber jabberwock snorkelblatt") + classifier.classify("qwxz flibber jabberwock snorkelblatt") + + assert len(calls) == 1 + + +def test_summarizer_makes_no_second_request_while_circuit_is_open(): + calls = [] + + def create(*args, **kwargs): + calls.append(1) + raise RuntimeError("insufficient_quota") + + client = SimpleNamespace(chat=SimpleNamespace(completions=SimpleNamespace(create=create))) + circuit = EnrichmentCircuit(cooldown_seconds=60, clock=lambda: 100.0) + content = "x" * 600 + + summarize_content(content, client, "gpt-4o-mini", 300, circuit) + summarize_content(content, client, "gpt-4o-mini", 300, circuit) + + assert len(calls) == 1 + assert circuit.record_failure("connection reset") is False + assert circuit.allow_request() is True From 97d67fa1d1ad6a5a992410410850a21b6dc83b1b Mon Sep 17 00:00:00 2001 From: mikemikimike <13286568797@163.com> Date: Sun, 23 Aug 2026 19:01:17 +0800 Subject: [PATCH 12/33] test(enrichment): fix probe recovery assertions --- tests/test_enrichment_circuit.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_enrichment_circuit.py b/tests/test_enrichment_circuit.py index fd6690d..bf29993 100644 --- a/tests/test_enrichment_circuit.py +++ b/tests/test_enrichment_circuit.py @@ -31,6 +31,8 @@ def test_successful_probe_closes_circuit_and_records_recovery(): circuit.record_failure("insufficient_quota") now[0] += 60 assert circuit.allow_request() is True + assert circuit.record_failure("connection reset") is False + assert circuit.allow_request() is True circuit.record_success() assert circuit.allow_request() is True @@ -85,5 +87,3 @@ def create(*args, **kwargs): summarize_content(content, client, "gpt-4o-mini", 300, circuit) assert len(calls) == 1 - assert circuit.record_failure("connection reset") is False - assert circuit.allow_request() is True From 83df3525aac5df53121a95940786942a06424aa1 Mon Sep 17 00:00:00 2001 From: mikemikimike <13286568797@163.com> Date: Sun, 23 Aug 2026 20:29:12 +0800 Subject: [PATCH 13/33] test(enrichment): separate probe success and failure cases --- tests/test_enrichment_circuit.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_enrichment_circuit.py b/tests/test_enrichment_circuit.py index bf29993..04cfbbd 100644 --- a/tests/test_enrichment_circuit.py +++ b/tests/test_enrichment_circuit.py @@ -31,8 +31,6 @@ def test_successful_probe_closes_circuit_and_records_recovery(): circuit.record_failure("insufficient_quota") now[0] += 60 assert circuit.allow_request() is True - assert circuit.record_failure("connection reset") is False - assert circuit.allow_request() is True circuit.record_success() assert circuit.allow_request() is True @@ -46,6 +44,8 @@ def test_failed_probe_does_not_permanently_block_future_requests(): circuit.record_failure("insufficient_quota") now[0] += 60 assert circuit.allow_request() is True + assert circuit.record_failure("connection reset") is False + assert circuit.allow_request() is True def test_classifier_makes_no_second_request_while_circuit_is_open(): From 4cf9ef1f2352bdbe65d9eea0758711788bcfe37d Mon Sep 17 00:00:00 2001 From: Jack Arturo Date: Sun, 23 Aug 2026 21:00:32 +0200 Subject: [PATCH 14/33] fix(mcp): close false-failure and coverage gaps in the parity harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three defects found by local review before this ever reached GitHub: - The weekly drift alarm could never alarm. npm ci reinstalls exactly what package-lock.json pins, so the scheduled run re-tested the same version forever. It now resolves the latest published package on the schedule path only; PR runs stay reproducible. - The health scenario compared global memory/vector/enrichment counters. The two transports run their batches sequentially against one AutoMem, so the second batch always sees the first's fixtures — a guaranteed false failure no ordering could fix. Those counters are now redacted. - structuredContent was read for chaining and then discarded, so a mismatch in memory_ids, recall count, or health statistics would pass whenever the text matched. It is now normalized, redacted and compared. Also report every differing scenario in one run rather than throwing on the first, so a multi-scenario gap is not a fix-one-rerun loop at ~90s a cycle. Co-Authored-By: Claude Opus 5 --- .github/workflows/mcp-parity.yml | 8 ++++++++ mcp-sse-server/parity/normalize.js | 25 ++++++++++++++++++++++++- mcp-sse-server/test/parity.test.js | 29 +++++++++++++++++++++++++++-- 3 files changed, 59 insertions(+), 3 deletions(-) diff --git a/.github/workflows/mcp-parity.yml b/.github/workflows/mcp-parity.yml index 7d8d096..56a6ae5 100644 --- a/.github/workflows/mcp-parity.yml +++ b/.github/workflows/mcp-parity.yml @@ -56,6 +56,14 @@ jobs: AUTOMEM_PARITY_API_TOKEN: test-token run: | npm ci + # npm ci reinstalls exactly what package-lock.json pins, so on the + # weekly run it would re-test the same version forever and could + # never alarm. Resolve the latest published package on that path + # only; PR runs stay reproducible. + if [ "${{ github.event_name }}" = "schedule" ]; then + npm install --no-save @verygoodplugins/mcp-automem@latest + fi + node -p "'stdio package under test: ' + require('@verygoodplugins/mcp-automem/package.json').version" npm test - name: Dump service logs on failure diff --git a/mcp-sse-server/parity/normalize.js b/mcp-sse-server/parity/normalize.js index f796b95..7c7ab3a 100644 --- a/mcp-sse-server/parity/normalize.js +++ b/mcp-sse-server/parity/normalize.js @@ -27,6 +27,27 @@ const ISO_RE = /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2 const SCORE_RE = /(score=|Score: |"final_score":\s*|"score":\s*)[\d.]+/g; const MS_RE = /("query_time_ms":\s*)[\d.]+/g; +// Global service counters. The two transports run their scenario batches +// sequentially against one shared AutoMem, so by the time the second batch +// calls /health the first has already inserted its fixtures. These counters +// therefore always differ, and that difference is the harness's own writes — +// not a transport difference. Redact rather than reorder: no ordering makes +// a global counter agree across two sequential batches. +const VOLATILE_COUNT_KEYS = [ + 'memory_count', + 'vector_count', + 'queue_depth', + 'pending', + 'inflight', + 'processed', + 'failed', +]; +const JSON_COUNT_RE = new RegExp( + `("(?:${VOLATILE_COUNT_KEYS.join('|')})":\\s*)\\d+`, + 'g' +); +const LABEL_COUNT_RE = /((?:Memory|Vector) count: )\d+/g; + /** * Replace values that legitimately differ run-to-run so two transports can be * compared on the parts that must match. @@ -42,5 +63,7 @@ export function redact(text, scopeTag) { .replace(UUID_RE, '') .replace(ISO_RE, '') .replace(SCORE_RE, '$1') - .replace(MS_RE, '$1'); + .replace(MS_RE, '$1') + .replace(JSON_COUNT_RE, '$1') + .replace(LABEL_COUNT_RE, '$1'); } diff --git a/mcp-sse-server/test/parity.test.js b/mcp-sse-server/test/parity.test.js index 9d85870..720e7b8 100644 --- a/mcp-sse-server/test/parity.test.js +++ b/mcp-sse-server/test/parity.test.js @@ -39,7 +39,14 @@ async function runScenarios(client, tag) { })); const text = (res.content || []).map((c) => c.text ?? '').join('\n'); ids.push(res.structuredContent?.memory_id ?? (text.match(UUID_RE) || [])[0]); - rendered.push({ isError: Boolean(res.isError), text: redact(text, tag) }); + + // structuredContent is client-visible machine-readable output, so it is + // part of the contract too. Comparing only text would let a mismatch in + // memory_ids, recall count, or health statistics pass unnoticed. + const structured = res.structuredContent + ? JSON.parse(redact(JSON.stringify(normalizeKeys(res.structuredContent)), tag)) + : null; + rendered.push({ isError: Boolean(res.isError), text: redact(text, tag), structured }); } out.push({ name: scenario.name, rendered }); } @@ -87,8 +94,26 @@ test('tools/call renders identically across transports', { skip: GATE }, async ( try { const a = await runScenarios(remote, remoteTag); const b = await runScenarios(stdio, stdioTag); + + // Collect every mismatch before failing. A run costs ~90s, so reporting + // one scenario at a time turns a multi-scenario gap into a fix-one, + // rerun, fix-the-next loop. + const mismatched = []; for (let i = 0; i < a.length; i++) { - assert.deepStrictEqual(a[i], b[i], `scenario mismatch: ${a[i].name}`); + try { + assert.deepStrictEqual(a[i], b[i]); + } catch { + mismatched.push(i); + } + } + if (mismatched.length) { + const names = mismatched.map((i) => a[i].name).join(', '); + const first = mismatched[0]; + assert.deepStrictEqual( + a[first], + b[first], + `${mismatched.length}/${a.length} scenarios differ: ${names}\nFirst mismatch (${a[first].name}) diffed below.` + ); } } finally { // Bulk delete by tag exists only on the stdio transport for now, so cleanup From 9ac5b4eeb68dca4f29b8be71437b6702ba1aecd8 Mon Sep 17 00:00:00 2001 From: Jack Arturo Date: Sun, 23 Aug 2026 21:13:54 +0200 Subject: [PATCH 15/33] fix(mcp): align the parity harness with its own allowlist MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second local review pass, three more defects in the harness itself: - redact() stripped only the UUID inside the remote's " (request_id: ...)" error suffix, but the contract allowlists the whole suffix. The three error scenarios would have stayed red forever once everything else aligned. - The initialize check asserted both server names and neither version, so the remote's hardcoded 0.1.0 disagreeing with its own package.json — drift the audit explicitly records — could pass undetected. Each side now has to report its own package version; cross-transport equality is not required because the two packages version independently. - make test-parity fell through its readiness loop and ran the diff against a dead stack, reporting transport mismatches that were really a missing API. It now exits non-zero with service logs, matching the workflow. Co-Authored-By: Claude Opus 5 --- Makefile | 12 +++++++++--- docs/MCP_TRANSPORT_PARITY.md | 1 + mcp-sse-server/parity/normalize.js | 6 ++++++ mcp-sse-server/test/parity.test.js | 22 ++++++++++++++++++++-- 4 files changed, 36 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index f82d23a..f8e3c1c 100644 --- a/Makefile +++ b/Makefile @@ -80,10 +80,16 @@ test-parity: @echo "🐳 Starting Docker services..." @AUTOMEM_API_TOKEN=test-token ADMIN_API_TOKEN=test-admin-token docker compose up -d @echo "⏳ Waiting for AutoMem to be ready..." - @for i in $$(seq 1 60); do \ - if curl -fsS http://localhost:8001/health > /dev/null 2>&1; then break; fi; \ + @ready=0; \ + for i in $$(seq 1 60); do \ + if curl -fsS http://localhost:8001/health > /dev/null 2>&1; then ready=1; break; fi; \ sleep 1; \ - done + done; \ + if [ "$$ready" != "1" ]; then \ + echo "❌ AutoMem did not become healthy within 60s"; \ + docker compose logs --tail=50 flask-api; \ + exit 1; \ + fi @echo "🧪 Diffing MCP transports..." @cd mcp-sse-server && npm ci --silent && AUTOMEM_RUN_PARITY_TESTS=1 npm test diff --git a/docs/MCP_TRANSPORT_PARITY.md b/docs/MCP_TRANSPORT_PARITY.md index 77fe71e..26333e4 100644 --- a/docs/MCP_TRANSPORT_PARITY.md +++ b/docs/MCP_TRANSPORT_PARITY.md @@ -101,6 +101,7 @@ allowlists exactly these and nothing else. | Difference | remote | stdio | Why | | ---------------------- | ------------------------------------------------------------------------------------- | ---------------------------- | -------------------------------------------------------------------------------------------------------------------- | | `serverInfo.name` | `automem-mcp-sse` | `mcp-automem` | Clients must be able to tell the transports apart. | +| `serverInfo.version` | the bridge's `package.json` | the published package's | Two packages version independently. Each side must still report its **own** package version — the harness pins that self-consistency, which is how the remote's stale `0.1.0` is caught. | | Auth mechanisms | `Authorization: Bearer`, `X-API-Key` / `X-API-Token`, `?api_key=` / `?apiKey=` / `?api_token=` | `Authorization: Bearer` only | Browser and EventSource clients cannot set headers. | | Error text suffix | ` (request_id: )` | none | Observability on a hosted service. Same code path — the bridge supplies a `requestIdProvider`, stdio does not. | | Timeout / retry policy | `UPSTREAM_TIMEOUT_MS` (15 s), `UPSTREAM_MAX_RETRIES` (2) | 25 s, 3 | Tuned per deployment; injected via `AutoMemConfig`. | diff --git a/mcp-sse-server/parity/normalize.js b/mcp-sse-server/parity/normalize.js index 7c7ab3a..14ffcf2 100644 --- a/mcp-sse-server/parity/normalize.js +++ b/mcp-sse-server/parity/normalize.js @@ -22,6 +22,11 @@ export function normalizeKeys(value) { return value; } +// The remote transport appends " (request_id: )" to error text for +// observability on a hosted service. docs/MCP_TRANSPORT_PARITY.md allowlists +// the whole suffix, so strip it entirely — redacting only the UUID inside it +// would leave the three error scenarios permanently red. +const REQUEST_ID_SUFFIX_RE = /\s*\(request_id: [^)]*\)/g; const UUID_RE = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi; const ISO_RE = /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})/g; const SCORE_RE = /(score=|Score: |"final_score":\s*|"score":\s*)[\d.]+/g; @@ -60,6 +65,7 @@ export function redact(text, scopeTag) { let out = String(text); if (scopeTag) out = out.split(scopeTag).join(''); return out + .replace(REQUEST_ID_SUFFIX_RE, '') .replace(UUID_RE, '') .replace(ISO_RE, '') .replace(SCORE_RE, '$1') diff --git a/mcp-sse-server/test/parity.test.js b/mcp-sse-server/test/parity.test.js index 720e7b8..b5acf36 100644 --- a/mcp-sse-server/test/parity.test.js +++ b/mcp-sse-server/test/parity.test.js @@ -9,10 +9,15 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import { randomUUID } from 'node:crypto'; +import { createRequire } from 'node:module'; import { connectBothTransports } from '../parity/clients.js'; import { normalizeKeys, redact } from '../parity/normalize.js'; import { buildScenarios } from '../parity/scenarios.js'; +const require = createRequire(import.meta.url); +const bridgePkg = require('../package.json'); +const stdioPkg = require('@verygoodplugins/mcp-automem/package.json'); + const UUID_RE = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i; /** Resolve "$PREV..memory_id" against ids already collected in this scenario. */ @@ -78,10 +83,23 @@ test('server capabilities and instructions match', { skip: GATE }, async () => { ); assert.equal(remote.getInstructions(), stdio.getInstructions()); - // serverInfo.name is an allowlisted difference — clients must be able to - // tell the transports apart (docs/MCP_TRANSPORT_PARITY.md). + // serverInfo is transport-specific and allowlisted to differ, but each + // side must still report its OWN package version. The remote's hardcoded + // 0.1.0 disagreeing with its package.json is exactly the drift the audit + // records, so pin self-consistency rather than cross-transport equality. assert.equal(remote.getServerVersion().name, 'automem-mcp-sse'); assert.equal(stdio.getServerVersion().name, 'mcp-automem'); + + assert.equal( + remote.getServerVersion().version, + bridgePkg.version, + 'remote serverInfo.version must match mcp-sse-server/package.json' + ); + assert.equal( + stdio.getServerVersion().version, + stdioPkg.version, + 'stdio serverInfo.version must match the published package version' + ); } finally { await close(); } From 89346e9c87f319ad1814cf2a2061e306c005a36c Mon Sep 17 00:00:00 2001 From: Jack Arturo Date: Sun, 23 Aug 2026 21:24:11 +0200 Subject: [PATCH 16/33] fix(mcp): close partial connections when parity setup fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit connectBothTransports opens the HTTP listener before connecting the stdio child. If that child failed to start, the rejection escaped before close() was returned, leaving the listener and remote client alive — and node --test waits on open handles, so the scheduled job would hang until GitHub's timeout instead of reporting the startup failure. The weekly @latest install added in this branch is exactly what makes that reachable: an incompatible entry point resolves to a child that cannot start. Setup is now wrapped, and the accumulated closers run before rethrowing. Verified by removing the package and re-running: the error surfaces in 82ms and the process exits on its own instead of being killed by a watchdog. Co-Authored-By: Claude Opus 5 --- mcp-sse-server/parity/clients.js | 87 ++++++++++++++++++-------------- 1 file changed, 49 insertions(+), 38 deletions(-) diff --git a/mcp-sse-server/parity/clients.js b/mcp-sse-server/parity/clients.js index b5261ae..951bd06 100644 --- a/mcp-sse-server/parity/clients.js +++ b/mcp-sse-server/parity/clients.js @@ -22,47 +22,58 @@ export const API_TOKEN = process.env.AUTOMEM_PARITY_API_TOKEN || 'test-token'; export async function connectBothTransports() { const closers = []; + const closeAll = async () => { + for (const c of closers.reverse()) { + await Promise.resolve(c()).catch(() => {}); + } + closers.length = 0; + }; - // --- remote --------------------------------------------------------------- - // server.js resolves the upstream inside its route handlers, so setting these - // before createApp() is enough. - process.env.AUTOMEM_API_URL = API_URL; - process.env.AUTOMEM_API_TOKEN = API_TOKEN; + let remote; + let stdio; - const app = createApp(); - const httpServer = await new Promise((resolve) => { - const s = app.listen(0, '127.0.0.1', () => resolve(s)); - }); - closers.push(() => new Promise((r) => httpServer.close(r))); - const { port } = httpServer.address(); + // Setup opens an HTTP listener before the stdio child is connected. If that + // child fails to start — the weekly @latest install resolving an incompatible + // entry point is the realistic case — the listener would otherwise stay open, + // and `node --test` waits on open handles, so the job would hang until + // GitHub's timeout instead of reporting the startup failure. + try { + // --- remote ------------------------------------------------------------- + // server.js resolves the upstream inside its route handlers, so setting + // these before createApp() is enough. + process.env.AUTOMEM_API_URL = API_URL; + process.env.AUTOMEM_API_TOKEN = API_TOKEN; - const remote = new Client({ name: 'parity-harness', version: '1.0.0' }, {}); - const remoteTransport = new StreamableHTTPClientTransport( - new URL(`http://127.0.0.1:${port}/mcp`), - { requestInit: { headers: { Authorization: `Bearer ${API_TOKEN}` } } } - ); - await remote.connect(remoteTransport); - closers.push(() => remote.close()); + const app = createApp(); + const httpServer = await new Promise((resolve) => { + const s = app.listen(0, '127.0.0.1', () => resolve(s)); + }); + closers.push(() => new Promise((r) => httpServer.close(r))); + const { port } = httpServer.address(); - // --- stdio ---------------------------------------------------------------- - const stdioEntry = require.resolve('@verygoodplugins/mcp-automem/dist/index.js'); - const stdio = new Client({ name: 'parity-harness', version: '1.0.0' }, {}); - const stdioTransport = new StdioClientTransport({ - command: process.execPath, - args: [stdioEntry], - env: { ...process.env, AUTOMEM_API_URL: API_URL, AUTOMEM_API_KEY: API_TOKEN }, - stderr: 'ignore', - }); - await stdio.connect(stdioTransport); - closers.push(() => stdio.close()); + remote = new Client({ name: 'parity-harness', version: '1.0.0' }, {}); + const remoteTransport = new StreamableHTTPClientTransport( + new URL(`http://127.0.0.1:${port}/mcp`), + { requestInit: { headers: { Authorization: `Bearer ${API_TOKEN}` } } } + ); + await remote.connect(remoteTransport); + closers.push(() => remote.close()); - return { - remote, - stdio, - async close() { - for (const c of closers.reverse()) { - await Promise.resolve(c()).catch(() => {}); - } - }, - }; + // --- stdio -------------------------------------------------------------- + const stdioEntry = require.resolve('@verygoodplugins/mcp-automem/dist/index.js'); + stdio = new Client({ name: 'parity-harness', version: '1.0.0' }, {}); + const stdioTransport = new StdioClientTransport({ + command: process.execPath, + args: [stdioEntry], + env: { ...process.env, AUTOMEM_API_URL: API_URL, AUTOMEM_API_KEY: API_TOKEN }, + stderr: 'ignore', + }); + await stdio.connect(stdioTransport); + closers.push(() => stdio.close()); + } catch (error) { + await closeAll(); + throw error; + } + + return { remote, stdio, close: closeAll }; } From fa458b00044598a73556fdf10347541bdabfae26 Mon Sep 17 00:00:00 2001 From: Jack Arturo Date: Sun, 23 Aug 2026 21:59:51 +0200 Subject: [PATCH 17/33] fix(mcp): isolate parity fixtures per scenario and drop the false alarm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 2 of Codex review, three findings plus one caught locally. - Register transport/client closers *before* awaiting the handshake. The previous fix only covered a child that failed to spawn; if the process started but the MCP handshake rejected, the closer was never registered and the live child kept node --test open. - Give every scenario its own -sN namespace and seed its own fixtures. Sharing one namespace meant a write-capability gap corrupted every later comparison: the remote rejects `store batch` while stdio inserts three records, so subsequent recalls compared two different datasets and reported mismatches even where recall rendering was already in parity. That hides which fixes actually worked, precisely when landing them one at a time. Seeding uses single-store only, never a mode one transport lacks. - Remove the weekly cron and the @latest install. While the harness is intentionally red, continue-on-error leaves the workflow successful either way, so the scheduled run had the same external result before and after a regression — an alarm that could not alarm. It returns with the change that turns the harness green. Also, found while verifying: structuredContent was re-parsed after redaction, but redaction substitutes / placeholders for numbers, so the redacted form is deliberately not valid JSON and JSON.parse threw. It is now compared as a normalized, redacted string. test-parity now uses port 8011. A developer running a local AutoMem install holds 8001, and this harness writes fixtures and bulk-deletes by tag — it must never point at a real memory store. Co-Authored-By: Claude Opus 5 --- .github/workflows/mcp-parity.yml | 22 +- Makefile | 19 +- docs/MCP_TRANSPORT_PARITY.md | 15 +- mcp-sse-server/parity/clients.js | 13 +- mcp-sse-server/parity/scenarios.js | 408 +++++++++++++++++------------ mcp-sse-server/test/parity.test.js | 16 +- 6 files changed, 292 insertions(+), 201 deletions(-) diff --git a/.github/workflows/mcp-parity.yml b/.github/workflows/mcp-parity.yml index 56a6ae5..8d73031 100644 --- a/.github/workflows/mcp-parity.yml +++ b/.github/workflows/mcp-parity.yml @@ -2,6 +2,13 @@ name: MCP Transport Parity # Diffs the remote MCP bridge against the stdio package against one live # AutoMem. See docs/MCP_TRANSPORT_PARITY.md for the contract. +# +# No scheduled run yet, deliberately. While the harness is intentionally red +# against the known gaps, `continue-on-error` leaves the workflow successful +# either way, so a cron job could not distinguish "same known gaps" from "a +# newly published mcp-automem added drift" — it would be an alarm that cannot +# alarm. The weekly drift check lands with the change that turns the harness +# green, where a failure is a real signal. on: pull_request: @@ -9,9 +16,6 @@ on: - 'mcp-sse-server/**' - 'automem/api/**' - 'app.py' - schedule: - # Weekly drift alarm: catches a newly published mcp-automem the bridge has not adopted. - - cron: '0 6 * * 1' workflow_dispatch: jobs: @@ -31,6 +35,7 @@ jobs: - name: Start AutoMem stack env: + AUTOMEM_API_HOST_PORT: '8011' AUTOMEM_API_TOKEN: test-token ADMIN_API_TOKEN: test-admin-token run: docker compose up -d @@ -38,7 +43,7 @@ jobs: - name: Wait for /health run: | for i in $(seq 1 60); do - if curl -fsS http://localhost:8001/health > /dev/null; then + if curl -fsS http://localhost:8011/health > /dev/null; then echo "AutoMem is up after ${i}s" exit 0 fi @@ -52,17 +57,10 @@ jobs: working-directory: mcp-sse-server env: AUTOMEM_RUN_PARITY_TESTS: '1' - AUTOMEM_PARITY_API_URL: http://localhost:8001 + AUTOMEM_PARITY_API_URL: http://localhost:8011 AUTOMEM_PARITY_API_TOKEN: test-token run: | npm ci - # npm ci reinstalls exactly what package-lock.json pins, so on the - # weekly run it would re-test the same version forever and could - # never alarm. Resolve the latest published package on that path - # only; PR runs stay reproducible. - if [ "${{ github.event_name }}" = "schedule" ]; then - npm install --no-save @verygoodplugins/mcp-automem@latest - fi node -p "'stdio package under test: ' + require('@verygoodplugins/mcp-automem/package.json').version" npm test diff --git a/Makefile b/Makefile index f8e3c1c..472c4b5 100644 --- a/Makefile +++ b/Makefile @@ -76,22 +76,31 @@ test-node: # Diff the remote and stdio MCP transports against one live AutoMem. # See docs/MCP_TRANSPORT_PARITY.md. +# Deliberately not 8001. The harness writes fixtures and bulk-deletes by tag, +# and a developer running a local AutoMem install (~/.automem/server) already +# holds 8001 — pointing this at that instance would seed test data into a real +# memory store. Own port, own stack, no collision. +PARITY_PORT ?= 8011 + test-parity: - @echo "🐳 Starting Docker services..." - @AUTOMEM_API_TOKEN=test-token ADMIN_API_TOKEN=test-admin-token docker compose up -d + @echo "🐳 Starting Docker services on port $(PARITY_PORT)..." + @AUTOMEM_API_HOST_PORT=$(PARITY_PORT) AUTOMEM_API_TOKEN=test-token ADMIN_API_TOKEN=test-admin-token docker compose up -d @echo "⏳ Waiting for AutoMem to be ready..." @ready=0; \ for i in $$(seq 1 60); do \ - if curl -fsS http://localhost:8001/health > /dev/null 2>&1; then ready=1; break; fi; \ + if curl -fsS http://localhost:$(PARITY_PORT)/health > /dev/null 2>&1; then ready=1; break; fi; \ sleep 1; \ done; \ if [ "$$ready" != "1" ]; then \ - echo "❌ AutoMem did not become healthy within 60s"; \ + echo "❌ AutoMem did not become healthy on port $(PARITY_PORT) within 60s"; \ docker compose logs --tail=50 flask-api; \ exit 1; \ fi @echo "🧪 Diffing MCP transports..." - @cd mcp-sse-server && npm ci --silent && AUTOMEM_RUN_PARITY_TESTS=1 npm test + @cd mcp-sse-server && npm ci --silent && \ + AUTOMEM_RUN_PARITY_TESTS=1 \ + AUTOMEM_PARITY_API_URL=http://localhost:$(PARITY_PORT) \ + npm test # Format code fmt: diff --git a/docs/MCP_TRANSPORT_PARITY.md b/docs/MCP_TRANSPORT_PARITY.md index 26333e4..d56c12c 100644 --- a/docs/MCP_TRANSPORT_PARITY.md +++ b/docs/MCP_TRANSPORT_PARITY.md @@ -133,5 +133,16 @@ It lives in `mcp-sse-server/parity/` with its entry point at The harness is gated on `AUTOMEM_RUN_PARITY_TESTS=1` because it needs a live service. Without the gate it skips cleanly, which is what CI's `node-test` job sees. `.github/workflows/mcp-parity.yml` runs it with the stack up on PRs that -touch `mcp-sse-server/**`, `automem/api/**`, or `app.py`, plus weekly as a drift -alarm against newly published `mcp-automem` versions. +touch `mcp-sse-server/**`, `automem/api/**`, or `app.py`. + +It brings its own stack up on port **8011**, not the usual 8001. The harness +writes fixtures and bulk-deletes by tag, and a developer running a local +AutoMem install (`~/.automem/server`) already holds 8001 — pointing the harness +at that instance would seed test data into a real memory store. + +**There is no scheduled drift run yet, deliberately.** While the harness is +intentionally red against the known gaps, `continue-on-error` leaves the +workflow successful either way, so a cron job could not distinguish "same known +gaps" from "a newly published `mcp-automem` added drift". That would be an alarm +that cannot alarm. The weekly check lands with the change that turns the harness +green, where a failure is a real signal. diff --git a/mcp-sse-server/parity/clients.js b/mcp-sse-server/parity/clients.js index 951bd06..8fa6839 100644 --- a/mcp-sse-server/parity/clients.js +++ b/mcp-sse-server/parity/clients.js @@ -56,8 +56,13 @@ export async function connectBothTransports() { new URL(`http://127.0.0.1:${port}/mcp`), { requestInit: { headers: { Authorization: `Bearer ${API_TOKEN}` } } } ); - await remote.connect(remoteTransport); + // Registered before the await: if the handshake itself rejects or times + // out, a closer added afterwards would never exist and the resource would + // leak. Closers swallow their own errors, so closing something that never + // finished connecting is safe. + closers.push(() => remoteTransport.close()); closers.push(() => remote.close()); + await remote.connect(remoteTransport); // --- stdio -------------------------------------------------------------- const stdioEntry = require.resolve('@verygoodplugins/mcp-automem/dist/index.js'); @@ -68,8 +73,12 @@ export async function connectBothTransports() { env: { ...process.env, AUTOMEM_API_URL: API_URL, AUTOMEM_API_KEY: API_TOKEN }, stderr: 'ignore', }); - await stdio.connect(stdioTransport); + // Same ordering rule, and it matters more here: the child process is + // already spawned by the time the handshake runs, so a closer registered + // after the await would leave a live child holding node --test open. + closers.push(() => stdioTransport.close()); closers.push(() => stdio.close()); + await stdio.connect(stdioTransport); } catch (error) { await closeAll(); throw error; diff --git a/mcp-sse-server/parity/scenarios.js b/mcp-sse-server/parity/scenarios.js index e24c159..ab74112 100644 --- a/mcp-sse-server/parity/scenarios.js +++ b/mcp-sse-server/parity/scenarios.js @@ -1,184 +1,244 @@ /** * The tools/call scenario matrix for the parity harness. * - * Each scenario is a short sequence of calls run independently against each - * transport, under that transport's own `tag` namespace so neither sees the - * other's writes. `$PREV..memory_id` is substituted with the memory id - * produced by call of the same scenario. + * Each scenario runs independently against each transport, under that + * transport's own `tag` namespace so neither sees the other's writes. + * `$PREV..memory_id` is substituted with the memory id produced by call + * of the same scenario. + * + * Every scenario also owns a private `-sN` namespace and seeds its own + * fixtures. Sharing one namespace across scenarios looks tidier but is + * actively misleading: when a write capability differs — the remote rejects + * `store batch` while stdio inserts three records — every later recall in the + * shared tag compares two different datasets, so it reports a mismatch even + * when recall rendering is already in parity. That hides which fixes actually + * worked, exactly when you are landing them one at a time. + * + * Seeding therefore uses single-store only, never a mode one transport lacks. + * Every memory also carries the root `tag`, so cleanup stays a single + * bulk-delete per transport namespace. */ -export function buildScenarios(tag) { - const base = { tags: [tag], importance: 0.7 }; - - return [ - { - name: 'store single', - calls: [ - { - tool: 'store_memory', - args: { ...base, content: 'Parity fixture alpha. Chose PostgreSQL for ACID.' }, - }, - ], - }, - { - // Distinct importance values are load-bearing. GET /memory/by-tag orders by - // `importance DESC, timestamp DESC, id ASC` (automem/api/memory.py:259). With - // equal importance and sub-millisecond writes, `id ASC` becomes the tiebreaker - // — and ids differ between the two tag namespaces by construction, so the - // `recall exhaustive` scenario would fail on ordering while actually in parity. - name: 'store batch', - calls: [ - { - tool: 'store_memory', - args: { - memories: [ - { content: 'Parity batch one.', tags: [tag], importance: 0.9 }, - { content: 'Parity batch two.', tags: [tag], importance: 0.7 }, - { content: 'Parity batch three.', tags: [tag], importance: 0.5 }, - ], - }, - }, - ], - }, - { - name: 'store supersede', - calls: [ - { tool: 'store_memory', args: { ...base, content: 'Parity supersede original.' } }, - { - tool: 'store_memory', - args: { - ...base, - content: 'Parity supersede replacement.', - supersedes_memory_id: '$PREV.0.memory_id', - supersede_reason: 'parity harness', - }, - }, - ], - }, - { - name: 'recall ranked text', - calls: [ - { tool: 'recall_memory', args: { query: 'Parity fixture', tags: [tag], limit: 5 } }, - ], - }, - { - name: 'recall detailed', - calls: [ - { - tool: 'recall_memory', - args: { query: 'Parity fixture', tags: [tag], format: 'detailed' }, + +// Each entry is (sTag, tag) => scenario, so the private namespace is bound at +// build time and every scenario is self-contained. +const SCENARIOS = [ + (s, t) => ({ + name: 'store single', + calls: [ + { + tool: 'store_memory', + args: { + content: 'Parity fixture alpha. Chose PostgreSQL for ACID.', + tags: [t, s], + importance: 0.7, }, - ], - }, - { - name: 'recall items', - calls: [ - { tool: 'recall_memory', args: { query: 'Parity fixture', tags: [tag], format: 'items' } }, - ], - }, - { - name: 'recall json', - calls: [ - { tool: 'recall_memory', args: { query: 'Parity fixture', tags: [tag], format: 'json' } }, - ], - }, - { - name: 'recall empty', - calls: [ - { - tool: 'recall_memory', - args: { query: 'zzz-no-such-thing', tags: [`${tag}-absent`] }, + }, + ], + }), + + (s, t) => ({ + name: 'store batch', + calls: [ + { + tool: 'store_memory', + args: { + memories: [ + { content: 'Parity batch one.', tags: [t, s], importance: 0.9 }, + { content: 'Parity batch two.', tags: [t, s], importance: 0.7 }, + { content: 'Parity batch three.', tags: [t, s], importance: 0.5 }, + ], }, - ], - }, - { - name: 'recall id fetch', - calls: [ - { tool: 'store_memory', args: { ...base, content: 'Parity id-fetch target.' } }, - { tool: 'recall_memory', args: { memory_id: '$PREV.0.memory_id' } }, - ], - }, - { - name: 'recall exhaustive', - calls: [ - { tool: 'recall_memory', args: { tags: [tag], exhaustive: true, limit: 2, offset: 0 } }, - ], - }, - { - name: 'associate single', - calls: [ - { tool: 'store_memory', args: { ...base, content: 'Parity assoc source.' } }, - { tool: 'store_memory', args: { ...base, content: 'Parity assoc target.' } }, - { - tool: 'associate_memories', - args: { - memory1_id: '$PREV.0.memory_id', - memory2_id: '$PREV.1.memory_id', - type: 'EXEMPLIFIES', - strength: 0.8, - pattern_type: 'parity', - confidence: 0.9, - }, + }, + ], + }), + + (s, t) => ({ + name: 'store supersede', + calls: [ + { + tool: 'store_memory', + args: { content: 'Parity supersede original.', tags: [t, s], importance: 0.7 }, + }, + { + tool: 'store_memory', + args: { + content: 'Parity supersede replacement.', + tags: [t, s], + importance: 0.7, + supersedes_memory_id: '$PREV.0.memory_id', + supersede_reason: 'parity harness', }, - ], - }, - { - name: 'associate batch partial failure', - calls: [ - { tool: 'store_memory', args: { ...base, content: 'Parity batch assoc source.' } }, - { - tool: 'associate_memories', - args: { - associations: [ - { - memory1_id: '$PREV.0.memory_id', - memory2_id: '00000000-0000-4000-8000-000000000000', - type: 'RELATES_TO', - strength: 0.5, - }, - ], - }, + }, + ], + }), + + (s, t) => ({ + name: 'recall ranked text', + calls: [ + { + tool: 'store_memory', + args: { content: 'Parity fixture alpha. Chose PostgreSQL for ACID.', tags: [t, s], importance: 0.9 }, + }, + { + tool: 'store_memory', + args: { content: 'Parity fixture beta. Qdrant holds the vectors.', tags: [t, s], importance: 0.7 }, + }, + { tool: 'recall_memory', args: { query: 'Parity fixture', tags: [s], limit: 5 } }, + ], + }), + + (s, t) => ({ + name: 'recall detailed', + calls: [ + { + tool: 'store_memory', + args: { content: 'Parity detailed fixture.', tags: [t, s], importance: 0.7 }, + }, + { tool: 'recall_memory', args: { query: 'Parity detailed', tags: [s], format: 'detailed' } }, + ], + }), + + (s, t) => ({ + name: 'recall items', + calls: [ + { + tool: 'store_memory', + args: { content: 'Parity items fixture.', tags: [t, s], importance: 0.7 }, + }, + { tool: 'recall_memory', args: { query: 'Parity items', tags: [s], format: 'items' } }, + ], + }), + + (s, t) => ({ + name: 'recall json', + calls: [ + { + tool: 'store_memory', + args: { content: 'Parity json fixture.', tags: [t, s], importance: 0.7 }, + }, + { tool: 'recall_memory', args: { query: 'Parity json', tags: [s], format: 'json' } }, + ], + }), + + (s) => ({ + name: 'recall empty', + calls: [ + { tool: 'recall_memory', args: { query: 'zzz-no-such-thing', tags: [`${s}-absent`] } }, + ], + }), + + (s, t) => ({ + name: 'recall id fetch', + calls: [ + { + tool: 'store_memory', + args: { content: 'Parity id-fetch target.', tags: [t, s], importance: 0.7 }, + }, + { tool: 'recall_memory', args: { memory_id: '$PREV.0.memory_id' } }, + ], + }), + + (s, t) => ({ + // Distinct importance values are load-bearing. GET /memory/by-tag orders by + // `importance DESC, timestamp DESC, id ASC` (automem/api/memory.py:259). + // With equal importance and sub-millisecond writes, `id ASC` becomes the + // tiebreaker — and ids differ between the two tag namespaces by + // construction, so ordering would differ while actually in parity. + name: 'recall exhaustive', + calls: [ + { tool: 'store_memory', args: { content: 'Parity enum one.', tags: [t, s], importance: 0.9 } }, + { tool: 'store_memory', args: { content: 'Parity enum two.', tags: [t, s], importance: 0.7 } }, + { tool: 'store_memory', args: { content: 'Parity enum three.', tags: [t, s], importance: 0.5 } }, + { tool: 'recall_memory', args: { tags: [s], exhaustive: true, limit: 2, offset: 0 } }, + ], + }), + + (s, t) => ({ + name: 'associate single', + calls: [ + { tool: 'store_memory', args: { content: 'Parity assoc source.', tags: [t, s], importance: 0.7 } }, + { tool: 'store_memory', args: { content: 'Parity assoc target.', tags: [t, s], importance: 0.7 } }, + { + tool: 'associate_memories', + args: { + memory1_id: '$PREV.0.memory_id', + memory2_id: '$PREV.1.memory_id', + type: 'EXEMPLIFIES', + strength: 0.8, + pattern_type: 'parity', + confidence: 0.9, }, - ], - }, - { - name: 'update', - calls: [ - { tool: 'store_memory', args: { ...base, content: 'Parity update original.' } }, - { tool: 'update_memory', args: { memory_id: '$PREV.0.memory_id', importance: 0.95 } }, - ], - }, - { - name: 'delete single', - calls: [ - { tool: 'store_memory', args: { ...base, content: 'Parity delete target.' } }, - { tool: 'delete_memory', args: { memory_id: '$PREV.0.memory_id' } }, - ], - }, - { - name: 'delete by tag', - calls: [ - { - tool: 'store_memory', - args: { content: 'Parity bulk delete target.', tags: [`${tag}-bulk`], importance: 0.7 }, + }, + ], + }), + + (s, t) => ({ + name: 'associate batch partial failure', + calls: [ + { tool: 'store_memory', args: { content: 'Parity batch assoc source.', tags: [t, s], importance: 0.7 } }, + { + tool: 'associate_memories', + args: { + associations: [ + { + memory1_id: '$PREV.0.memory_id', + memory2_id: '00000000-0000-4000-8000-000000000000', + type: 'RELATES_TO', + strength: 0.5, + }, + ], }, - { tool: 'delete_memory', args: { tags: [`${tag}-bulk`] } }, - ], - }, - { - name: 'health', - calls: [{ tool: 'check_database_health', args: {} }], - }, - { - name: 'error: exhaustive without tags', - calls: [{ tool: 'recall_memory', args: { exhaustive: true } }], - }, - { - name: 'error: store content over hard limit', - calls: [{ tool: 'store_memory', args: { ...base, content: 'x'.repeat(2100) } }], - }, - { - name: 'error: unknown tool', - calls: [{ tool: 'no_such_tool', args: {} }], - }, - ]; + }, + ], + }), + + (s, t) => ({ + name: 'update', + calls: [ + { tool: 'store_memory', args: { content: 'Parity update original.', tags: [t, s], importance: 0.7 } }, + { tool: 'update_memory', args: { memory_id: '$PREV.0.memory_id', importance: 0.95 } }, + ], + }), + + (s, t) => ({ + name: 'delete single', + calls: [ + { tool: 'store_memory', args: { content: 'Parity delete target.', tags: [t, s], importance: 0.7 } }, + { tool: 'delete_memory', args: { memory_id: '$PREV.0.memory_id' } }, + ], + }), + + (s, t) => ({ + name: 'delete by tag', + calls: [ + { tool: 'store_memory', args: { content: 'Parity bulk delete target.', tags: [t, s], importance: 0.7 } }, + { tool: 'delete_memory', args: { tags: [s] } }, + ], + }), + + () => ({ + name: 'health', + calls: [{ tool: 'check_database_health', args: {} }], + }), + + () => ({ + name: 'error: exhaustive without tags', + calls: [{ tool: 'recall_memory', args: { exhaustive: true } }], + }), + + (s, t) => ({ + name: 'error: store content over hard limit', + calls: [ + { tool: 'store_memory', args: { content: 'x'.repeat(2100), tags: [t, s], importance: 0.7 } }, + ], + }), + + () => ({ + name: 'error: unknown tool', + calls: [{ tool: 'no_such_tool', args: {} }], + }), +]; + +export function buildScenarios(tag) { + return SCENARIOS.map((make, i) => make(`${tag}-s${i}`, tag)); } diff --git a/mcp-sse-server/test/parity.test.js b/mcp-sse-server/test/parity.test.js index b5acf36..d3c2bad 100644 --- a/mcp-sse-server/test/parity.test.js +++ b/mcp-sse-server/test/parity.test.js @@ -48,8 +48,12 @@ async function runScenarios(client, tag) { // structuredContent is client-visible machine-readable output, so it is // part of the contract too. Comparing only text would let a mismatch in // memory_ids, recall count, or health statistics pass unnoticed. + // Compared as a redacted string, not re-parsed: redaction substitutes + // placeholders like for numeric values, so the redacted form is + // deliberately not valid JSON. Key order is normalized first so the + // string comparison stays meaningful. const structured = res.structuredContent - ? JSON.parse(redact(JSON.stringify(normalizeKeys(res.structuredContent)), tag)) + ? redact(JSON.stringify(normalizeKeys(res.structuredContent)), tag) : null; rendered.push({ isError: Boolean(res.isError), text: redact(text, tag), structured }); } @@ -134,15 +138,15 @@ test('tools/call renders identically across transports', { skip: GATE }, async ( ); } } finally { - // Bulk delete by tag exists only on the stdio transport for now, so cleanup - // for both namespaces runs there. Swallowed so a cleanup failure can never + // Every fixture carries its transport's root tag regardless of which + // per-scenario namespace it also has, so one bulk delete per namespace is + // enough. Bulk delete by tag exists only on the stdio transport for now, + // so cleanup for both runs there. Swallowed so a cleanup failure can never // mask an assertion failure. await stdio .callTool({ name: 'delete_memory', - arguments: { - tags: [remoteTag, stdioTag, `${remoteTag}-bulk`, `${stdioTag}-bulk`], - }, + arguments: { tags: [remoteTag, stdioTag] }, }) .catch(() => {}); await close(); From 29f359de4657e7de4306a0a2891da451977171e6 Mon Sep 17 00:00:00 2001 From: Jack Arturo Date: Fri, 28 Aug 2026 02:55:04 +0200 Subject: [PATCH 18/33] fix(enrichment): preserve quota circuit during in-flight calls --- automem/classification/memory_classifier.py | 38 ++++++++++------- automem/service_state.py | 24 ++++++----- automem/utils/text.py | 13 +++--- tests/test_enrichment_circuit.py | 45 ++++++++++++++++----- 4 files changed, 79 insertions(+), 41 deletions(-) diff --git a/automem/classification/memory_classifier.py b/automem/classification/memory_classifier.py index 4869f96..9f9721f 100644 --- a/automem/classification/memory_classifier.py +++ b/automem/classification/memory_classifier.py @@ -136,22 +136,25 @@ def classify(self, content: str, *, use_llm: bool = True) -> tuple[str, float]: except Exception as exc: self._logger.exception("LLM classification failed, using fallback") llm_error = str(exc) - if self._circuit is not None: - self._circuit.record_failure(llm_error) if self._stats is not None: self._stats.record_fallback(llm_error) return "Memory", 0.3 def _classify_with_llm(self, content: str) -> Optional[tuple[str, float]]: - if self._circuit is not None and not self._circuit.allow_request(): - self._logger.info("Skipping LLM classification while enrichment circuit is open") - return None + was_probe = False + if self._circuit is not None: + allowed, was_probe = self._circuit.begin_request() + if not allowed: + self._logger.info("Skipping LLM classification while enrichment circuit is open") + return None client = self._get_openai_client() if client is None: self._ensure_openai_client() client = self._get_openai_client() if client is None: + if self._circuit is not None: + self._circuit.record_failure("OpenAI client unavailable", was_probe) return None extra_params: dict[str, Any] = {} @@ -164,17 +167,22 @@ def _classify_with_llm(self, content: str) -> Optional[tuple[str, float]]: extra_params["max_tokens"] = 50 extra_params["temperature"] = 0.3 - response = client.chat.completions.create( - model=self._classification_model, - messages=[ - {"role": "system", "content": self.SYSTEM_PROMPT}, - {"role": "user", "content": content[:1000]}, - ], - response_format={"type": "json_object"}, - **extra_params, - ) + try: + response = client.chat.completions.create( + model=self._classification_model, + messages=[ + {"role": "system", "content": self.SYSTEM_PROMPT}, + {"role": "user", "content": content[:1000]}, + ], + response_format={"type": "json_object"}, + **extra_params, + ) + except Exception as exc: + if self._circuit is not None: + self._circuit.record_failure(str(exc), was_probe) + raise if self._circuit is not None: - self._circuit.record_success() + self._circuit.record_success(was_probe) raw_content = response.choices[0].message.content if not raw_content: diff --git a/automem/service_state.py b/automem/service_state.py index 31eb63e..c84cbff 100644 --- a/automem/service_state.py +++ b/automem/service_state.py @@ -1,10 +1,10 @@ from __future__ import annotations +import time from dataclasses import dataclass, field from queue import Queue from threading import Event, Lock, Thread from typing import Any, Dict, Optional, Set -import time from falkordb import FalkorDB from qdrant_client import QdrantClient @@ -26,23 +26,25 @@ def __init__(self, cooldown_seconds: float = 300, clock: Any = time.monotonic) - self.circuit_open_skips = 0 self.recoveries = 0 - def allow_request(self) -> bool: + def begin_request(self) -> tuple[bool, bool]: + """Reserve an LLM request and report whether it is the recovery probe.""" with self._lock: now = self._clock() if now < self._opened_until: self.circuit_open_skips += 1 - return False + return False, False if self._opened_until: if self._probe_pending: self.circuit_open_skips += 1 - return False + return False, False self._probe_pending = True - return True + return True, True + return True, False - def record_failure(self, error: str) -> bool: + def record_failure(self, error: str, was_probe: bool = False) -> bool: if "insufficient_quota" not in error.lower() and "quota" not in error.lower(): with self._lock: - if self._probe_pending: + if was_probe and self._probe_pending: self._opened_until = 0.0 self._probe_pending = False return False @@ -51,12 +53,12 @@ def record_failure(self, error: str) -> bool: self._probe_pending = False return True - def record_success(self) -> None: + def record_success(self, was_probe: bool = False) -> None: with self._lock: - if self._opened_until: + if was_probe and self._probe_pending: self.recoveries += 1 - self._opened_until = 0.0 - self._probe_pending = False + self._opened_until = 0.0 + self._probe_pending = False def to_dict(self) -> Dict[str, Any]: with self._lock: diff --git a/automem/utils/text.py b/automem/utils/text.py index ce15fdf..e6ad54c 100644 --- a/automem/utils/text.py +++ b/automem/utils/text.py @@ -140,9 +140,12 @@ def summarize_content( return None if not content or len(content) <= target_length: return content - if circuit is not None and not circuit.allow_request(): - logger.info("Skipping summarization while enrichment circuit is open") - return None + was_probe = False + if circuit is not None: + allowed, was_probe = circuit.begin_request() + if not allowed: + logger.info("Skipping summarization while enrichment circuit is open") + return None try: system_prompt = SUMMARIZE_SYSTEM_PROMPT.format(target_length=target_length) @@ -167,7 +170,7 @@ def summarize_content( **extra_params, ) if circuit is not None: - circuit.record_success() + circuit.record_success(was_probe) summary = response.choices[0].message.content.strip() @@ -191,7 +194,7 @@ def summarize_content( except Exception as exc: logger.exception("Memory summarization failed") if circuit is not None: - circuit.record_failure(str(exc)) + circuit.record_failure(str(exc), was_probe) return None diff --git a/tests/test_enrichment_circuit.py b/tests/test_enrichment_circuit.py index 04cfbbd..44ed5d1 100644 --- a/tests/test_enrichment_circuit.py +++ b/tests/test_enrichment_circuit.py @@ -1,7 +1,7 @@ from __future__ import annotations -from types import SimpleNamespace import logging +from types import SimpleNamespace from automem.classification.memory_classifier import MemoryClassifier from automem.service_state import EnrichmentCircuit @@ -11,9 +11,11 @@ def test_quota_failure_opens_circuit_and_skips_requests(): circuit = EnrichmentCircuit(cooldown_seconds=60, clock=lambda: 100.0) - assert circuit.allow_request() is True + allowed, _ = circuit.begin_request() + assert allowed is True assert circuit.record_failure("429 insufficient_quota") is True - assert circuit.allow_request() is False + allowed, _ = circuit.begin_request() + assert allowed is False assert circuit.to_dict()["circuit_open_skips"] == 1 @@ -21,7 +23,8 @@ def test_non_quota_failure_does_not_open_circuit(): circuit = EnrichmentCircuit(cooldown_seconds=60, clock=lambda: 100.0) assert circuit.record_failure("connection reset") is False - assert circuit.allow_request() is True + allowed, _ = circuit.begin_request() + assert allowed is True def test_successful_probe_closes_circuit_and_records_recovery(): @@ -30,10 +33,13 @@ def test_successful_probe_closes_circuit_and_records_recovery(): circuit.record_failure("insufficient_quota") now[0] += 60 - assert circuit.allow_request() is True - circuit.record_success() + allowed, was_probe = circuit.begin_request() + assert allowed is True + assert was_probe is True + circuit.record_success(was_probe) - assert circuit.allow_request() is True + allowed, _ = circuit.begin_request() + assert allowed is True assert circuit.to_dict()["recoveries"] == 1 @@ -43,9 +49,28 @@ def test_failed_probe_does_not_permanently_block_future_requests(): circuit.record_failure("insufficient_quota") now[0] += 60 - assert circuit.allow_request() is True - assert circuit.record_failure("connection reset") is False - assert circuit.allow_request() is True + allowed, was_probe = circuit.begin_request() + assert allowed is True + assert was_probe is True + assert circuit.record_failure("connection reset", was_probe) is False + allowed, _ = circuit.begin_request() + assert allowed is True + + +def test_inflight_success_does_not_close_quota_circuit(): + now = [100.0] + circuit = EnrichmentCircuit(cooldown_seconds=60, clock=lambda: now[0]) + + first_allowed, _ = circuit.begin_request() + second_allowed, second_was_probe = circuit.begin_request() + assert first_allowed is True + assert second_allowed is True + assert circuit.record_failure("insufficient_quota") is True + + circuit.record_success(second_was_probe) + + allowed, _ = circuit.begin_request() + assert allowed is False def test_classifier_makes_no_second_request_while_circuit_is_open(): From d9787cb3ccd1b69a85b0a5afbbcc8086488f0749 Mon Sep 17 00:00:00 2001 From: Jack Arturo Date: Fri, 28 Aug 2026 03:16:11 +0200 Subject: [PATCH 19/33] docs(enrichment): document quota circuit --- docs/API.md | 2 +- docs/ENVIRONMENT_VARIABLES.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/API.md b/docs/API.md index 47ed739..eb4d125 100644 --- a/docs/API.md +++ b/docs/API.md @@ -131,7 +131,7 @@ GET /recall?query=project%20plan&state_mode=history Enrichment - GET `/enrichment/status` - - Response: queue size, inflight/pending, stats, plus a `classification` block with type-classification counters (`llm_attempts`, `llm_successes`, `fallbacks`, `pattern_classifications`, `last_error`, `last_error_at`) for monitoring LLM-classification fallback rate. + - Response: queue size, inflight/pending, stats, plus a `classification` block with type-classification counters (`llm_attempts`, `llm_successes`, `fallbacks`, `pattern_classifications`, `last_error`, `last_error_at`) for monitoring LLM-classification fallback rate, and a `circuit` block with `open`, `circuit_open_skips`, and `recoveries` for monitoring quota-circuit state. - POST `/enrichment/reprocess` - Body: `{ "ids": ["..."] }` or query `?ids=a,b,c` diff --git a/docs/ENVIRONMENT_VARIABLES.md b/docs/ENVIRONMENT_VARIABLES.md index 2b49b10..e84c199 100644 --- a/docs/ENVIRONMENT_VARIABLES.md +++ b/docs/ENVIRONMENT_VARIABLES.md @@ -299,6 +299,7 @@ Controls entity extraction and relationship linking. | `ENRICHMENT_IDLE_SLEEP_SECONDS` | Sleep when queue empty | `2` | | `ENRICHMENT_FAILURE_BACKOFF_SECONDS` | Backoff on failure | `5` | | `ENRICHMENT_ENABLE_SUMMARIES` | Enable summarization | `true` | +| `ENRICHMENT_CIRCUIT_COOLDOWN_SECONDS` | Cooldown after LLM quota exhaustion before one recovery probe is allowed | `300` | | `ENRICHMENT_SPACY_MODEL` | spaCy model name | `en_core_web_sm` | | `JIT_ENRICHMENT_ENABLED` | Inline enrichment during recall | `true` | From e177b60115927352b973c3590f5ca429d37f05fc Mon Sep 17 00:00:00 2001 From: Jack Arturo Date: Fri, 28 Aug 2026 03:24:36 +0200 Subject: [PATCH 20/33] docs: add Qdrant on-disk storage design --- ...026-08-28-qdrant-on-disk-storage-design.md | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 docs/superpowers/specs/2026-08-28-qdrant-on-disk-storage-design.md diff --git a/docs/superpowers/specs/2026-08-28-qdrant-on-disk-storage-design.md b/docs/superpowers/specs/2026-08-28-qdrant-on-disk-storage-design.md new file mode 100644 index 0000000..e621359 --- /dev/null +++ b/docs/superpowers/specs/2026-08-28-qdrant-on-disk-storage-design.md @@ -0,0 +1,81 @@ +# Qdrant On-Disk Storage Design + +## Goal + +Reduce AutoMem's Qdrant RAM residency by creating collections with on-disk +vectors, HNSW index, and payload storage, and provide a safe operator-run +rebuild path for the existing production collection. + +## Scope + +- New Qdrant collections use `on_disk=True` for vectors, `HnswConfigDiff(on_disk=True)`, + and `on_disk_payload=True`. +- The existing restore utility gains an optional HNSW on-disk setting so an + operator can rebuild a collection using the same profile. +- Documentation provides a production runbook for backup, Qdrant-only restore, + verification, rollback, and a subsequent manual Railway volume resize. + +## Non-goals + +- Do not mutate an existing collection at application startup; Qdrant creation + settings only apply to a newly created collection. +- Do not run a production migration, change Railway volume size, or alter + replicas as part of this change. +- Do not change embedding dimensions, distance metric, payload indexes, or + recall semantics. + +## Architecture + +`ensure_qdrant_collection()` remains responsible only for collection creation. +When it creates a collection, it passes the on-disk profile to Qdrant. Existing +collections keep their settings unchanged. + +`scripts/restore_from_backup.py` remains the only destructive recovery path. +It will accept `QDRANT_RESTORE_HNSW_ON_DISK` alongside its existing optional +vector and payload storage settings and applies all selected settings only when +it recreates a collection during a forced Qdrant-only restore. + +The runbook uses the existing authenticated `/backup` export and restore tool: +take a portable backup, rebuild Qdrant with explicitly set on-disk options, +validate collection point count and representative recall, and retain the +backup for rollback. Railway volume reduction is a manual post-verification +operation. + +## Data Flow + +1. A fresh AutoMem deployment starts with no Qdrant collection. +2. Runtime creates the collection with on-disk vector, HNSW, and payload + settings, then creates the existing payload indexes. +3. For an existing collection, an operator exports a backup and runs a + Qdrant-only forced restore with the three on-disk options enabled. +4. Restore deletes and recreates only Qdrant, upserts the backup's points, and + waits for the expected point count. +5. The operator verifies count and recall before manually resizing Railway + storage. A retained backup supports restoring the prior data if validation + fails. + +## Error Handling and Rollback + +- Collection creation errors keep the existing fail-soft behavior: Qdrant is + disabled and the service continues without vector storage. +- Restore remains opt-in and destructive only with its existing confirmation or + `--force`; no startup path invokes it. +- The runbook requires retaining the pre-migration backup until validation is + complete. Re-running restore from that backup recreates the collection if + rollback is needed. +- Railway volume resize is deliberately outside automation because it is an + infrastructure and capacity decision requiring observed post-migration usage. + +## Testing + +- Unit-test creation of a missing collection and assert all three on-disk + settings are passed to `create_collection`. +- Unit-test restore configuration with HNSW on-disk enabled and disabled. +- Run focused Qdrant/restore tests, full unit tests, formatting, import sorting, + lint, and compilation. + +## Documentation + +Update the environment-variable reference for the restore setting and the +backup/monitoring guide with the exact production migration, verification, and +rollback commands. From 56da2b75f82a0e3a52a8d06551e251923c19b306 Mon Sep 17 00:00:00 2001 From: Jack Arturo Date: Fri, 28 Aug 2026 03:29:29 +0200 Subject: [PATCH 21/33] docs: add Qdrant on-disk implementation plan --- .../2026-08-28-qdrant-on-disk-storage.md | 213 ++++++++++++++++++ 1 file changed, 213 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-28-qdrant-on-disk-storage.md diff --git a/docs/superpowers/plans/2026-08-28-qdrant-on-disk-storage.md b/docs/superpowers/plans/2026-08-28-qdrant-on-disk-storage.md new file mode 100644 index 0000000..b0470f5 --- /dev/null +++ b/docs/superpowers/plans/2026-08-28-qdrant-on-disk-storage.md @@ -0,0 +1,213 @@ +# Qdrant On-Disk Storage Implementation Plan + +> For agentic workers: REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task by task. + +**Goal:** Make every newly created AutoMem Qdrant collection disk-backed for vectors, HNSW, and payloads, and provide an explicit, documented Qdrant-only restore path for migrating existing collections. + +**Architecture:** Collection creation is the only runtime configuration mutation: it supplies Qdrant's \`on_disk\` options when the collection does not already exist. Existing collections are untouched. The backup restore tool accepts an explicit HNSW disk-storage environment variable so operators can rebuild a selected Qdrant collection from a portable backup; docs make that destructive operation, verification, and manual Railway volume resizing explicit. + +**Tech Stack:** Python 3.12, qdrant-client, Flask, pytest, Bash, Markdown. + +## Global Constraints + +- Preserve the configuration of any existing Qdrant collection; never update it at API startup. +- Keep production migration opt-in and operator-driven. Do not automate a Railway volume resize. +- Keep the restore script's existing explicit \`--force\` confirmation model and its \`--qdrant-only\` scope. +- Treat a Qdrant backup as sensitive corpus data; the runbook must use environment variables for credentials and avoid embedding secrets. + +--- + +### Task 1: Set safe on-disk defaults for newly created collections + +**Files:** +- Modify: \`automem/stores/runtime_clients.py\` +- Modify: \`automem/service_runtime_bindings.py\` +- Modify: \`app.py\` +- Modify: \`tests/test_vector_size_safety.py\` + +**Step 1: Add the failing collection-creation regression test.** + +In \`TestEnsureQdrantCollectionPayloadIndexes\`, add a test for a missing collection. Call \`ensure_qdrant_collection\` with the Qdrant model shims, assert one \`create_collection\` call, and verify that: + +- \`vectors_config\` uses the resolved embedding dimension and \`Distance.COSINE\`; +- \`vectors_config.on_disk is True\`; +- \`hnsw_config.on_disk is True\`; and +- \`on_disk_payload is True\`. + +Also assert the existing payload indexes are still created after the collection is created. Keep the pre-existing collection test proving that initialization does not recreate or reconfigure an existing collection. + +**Step 2: Run the focused test to confirm the new assertion fails.** + +Run: + +\`\`\`bash +PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 /Users/jgarturo/Projects/OpenAI/automem/.venv/bin/pytest -q tests/test_vector_size_safety.py -k on_disk +\`\`\` + +Expected: failure because the current collection factory does not provide all three on-disk settings. + +**Step 3: Pass \`HnswConfigDiff\` through the composition root.** + +Import \`HnswConfigDiff\` together with the other Qdrant models in \`app.py\`, including both optional-import fallback branches. Add an \`hnsw_config_diff_cls\` parameter to \`create_service_runtime\`, pass it to \`ensure_qdrant_collection\`, and wire \`HnswConfigDiff\` from the \`app.py\` service runtime construction. Preserve the application's optional-dependency startup behavior. + +**Step 4: Configure only newly created collections.** + +Extend \`ensure_qdrant_collection\` to accept the HNSW configuration class. In its existing missing-collection branch, create the collection with: + +\`\`\`python +vectors_config=VectorParams(size=effective_dim, distance=Distance.COSINE, on_disk=True) +hnsw_config=HnswConfigDiff(on_disk=True) +on_disk_payload=True +\`\`\` + +Do not alter the early return for an existing collection or the vector-size mismatch handling. + +**Step 5: Run focused tests.** + +Run: + +\`\`\`bash +PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 /Users/jgarturo/Projects/OpenAI/automem/.venv/bin/pytest -q tests/test_vector_size_safety.py +\`\`\` + +Expected: all vector-size safety tests pass, including the new missing-collection regression case. + +**Step 6: Commit the runtime change.** + +\`\`\`bash +git add app.py automem/stores/runtime_clients.py automem/service_runtime_bindings.py tests/test_vector_size_safety.py +git commit -m "perf(qdrant): store new collection data on disk" +\`\`\` + +--- + +### Task 2: Support explicit disk-backed HNSW during Qdrant restores + +**Files:** +- Modify: \`scripts/restore_from_backup.py\` +- Modify: \`scripts/lab/clone_production.sh\` +- Modify: \`tests/test_backup_endpoint.py\` + +**Step 1: Extend the restore configuration test first.** + +In \`test_restore_qdrant_uses_optional_collection_tuning\`, monkeypatch a new \`QDRANT_RESTORE_HNSW_ON_DISK\` setting to \`True\`. Assert the \`HnswConfigDiff\` used for \`create_collection\` contains both the existing \`m=0\` option and \`on_disk is True\`. Add a focused lab-clone-script assertion for a \`QDRANT_RESTORE_HNSW_ON_DISK\` default of \`true\`. + +**Step 2: Run the targeted test to confirm it fails.** + +Run: + +\`\`\`bash +PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 /Users/jgarturo/Projects/OpenAI/automem/.venv/bin/pytest -q tests/test_backup_endpoint.py -k "optional_collection_tuning or lab_clone_uses_paced_qdrant_restore_defaults" +\`\`\` + +Expected: failure because the restore module and local-clone wrapper do not yet expose the HNSW disk setting. + +**Step 3: Add the optional restore environment setting.** + +Define \`QDRANT_RESTORE_HNSW_ON_DISK = _optional_bool_env("QDRANT_RESTORE_HNSW_ON_DISK")\` next to the other restore collection tuning values. Refactor \`_qdrant_collection_kwargs\` to build an HNSW keyword dictionary, adding \`m\` when configured and \`on_disk\` when configured, and instantiate \`HnswConfigDiff(**hnsw_kwargs)\` only when at least one setting is supplied. This must preserve \`m=0\` as an intentional value. + +**Step 4: Set the local production-clone default.** + +Pass \`QDRANT_RESTORE_HNSW_ON_DISK="\${QDRANT_RESTORE_HNSW_ON_DISK:-true}"\` in \`scripts/lab/clone_production.sh\`, adjacent to the existing HNSW/vector/payload restore settings. This makes cloned production data match the new disk-backed default without affecting a running production collection. + +**Step 5: Run the focused restore tests.** + +Run: + +\`\`\`bash +PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 /Users/jgarturo/Projects/OpenAI/automem/.venv/bin/pytest -q tests/test_backup_endpoint.py +\`\`\` + +Expected: all backup/restore tests pass, including the optional HNSW configuration and local clone wrapper checks. + +**Step 6: Commit the migration-tool change.** + +\`\`\`bash +git add scripts/restore_from_backup.py scripts/lab/clone_production.sh tests/test_backup_endpoint.py +git commit -m "perf(qdrant): support disk-backed HNSW restores" +\`\`\` + +--- + +### Task 3: Document the explicit production migration and operational controls + +**Files:** +- Modify: \`docs/ENVIRONMENT_VARIABLES.md\` +- Modify: \`docs/MONITORING_AND_BACKUPS.md\` + +**Step 1: Document the restore variable.** + +Add \`QDRANT_RESTORE_HNSW_ON_DISK\` to the existing Qdrant restore tuning documentation in \`docs/ENVIRONMENT_VARIABLES.md\`. Describe it as an optional boolean that places the restored collection's HNSW index on disk; leave it unset by default so direct restore users must opt in. + +**Step 2: Add a Qdrant-only migration runbook.** + +Under the API backup export section in \`docs/MONITORING_AND_BACKUPS.md\`, add a concise “Migrate an existing Qdrant collection to on-disk storage” subsection that: + +1. takes a \`?include=qdrant\` portable backup with the admin token; +2. records the pre-migration point count and takes a recoverable Railway volume backup/snapshot; +3. runs \`restore_from_backup.py\` against the intended Qdrant endpoint with \`--qdrant-only --force\` plus \`QDRANT_RESTORE_VECTOR_ON_DISK=true\`, \`QDRANT_RESTORE_HNSW_ON_DISK=true\`, and \`QDRANT_RESTORE_ON_DISK_PAYLOAD=true\`; +4. verifies point count and representative recall before calling the change complete; and +5. directs the operator to resize the Railway volume manually only after validation and a stable observation period. + +State clearly that the restore deletes and recreates the selected Qdrant collection and that a tested backup is the rollback path. Do not document an automated Railway resize command. + +**Step 3: Review documentation for credential and safety clarity.** + +Run: + +\`\`\`bash +rg -n "QDRANT_RESTORE_HNSW_ON_DISK|on-disk storage|qdrant-only|Railway volume" docs/ENVIRONMENT_VARIABLES.md docs/MONITORING_AND_BACKUPS.md +\`\`\` + +Expected: the variable, destructive restore scope, validation requirements, and manual-resize boundary are each discoverable. + +**Step 4: Commit the docs change.** + +\`\`\`bash +git add docs/ENVIRONMENT_VARIABLES.md docs/MONITORING_AND_BACKUPS.md +git commit -m "docs(qdrant): add on-disk migration runbook" +\`\`\` + +--- + +### Task 4: Verify the integrated change set + +**Files:** +- Verify: all files changed by Tasks 1–3 + +**Step 1: Run formatting and lint checks.** + +Run: + +\`\`\`bash +make fmt +make lint +\`\`\` + +Expected: Black/Isort produce no unintended changes and Flake8 passes. + +**Step 2: Run the full unit suite.** + +Run: + +\`\`\`bash +PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 /Users/jgarturo/Projects/OpenAI/automem/.venv/bin/pytest -q -m unit +\`\`\` + +Expected: full unit suite passes. + +**Step 3: Inspect the final diff and status.** + +Run: + +\`\`\`bash +git diff develop...HEAD --check +git status --short +git log --oneline develop..HEAD +\`\`\` + +Expected: no whitespace errors, only intended files changed, and focused conventional commits are present. + +**Step 4: Report the handoff boundary.** + +State that the code provides defaults for new collections and an explicit migration mechanism for existing collections, but production migration and any Railway volume resize remain a manual operator action after backup and verification. From c97b5b1134d9aaf188f8a6c0ad47d5af34234d74 Mon Sep 17 00:00:00 2001 From: Jack Arturo Date: Fri, 28 Aug 2026 03:32:29 +0200 Subject: [PATCH 22/33] perf(qdrant): store new collection data on disk --- app.py | 12 +++++++-- automem/service_runtime_bindings.py | 2 ++ automem/stores/runtime_clients.py | 9 ++++++- tests/test_vector_size_safety.py | 39 +++++++++++++++++++++++++++++ 4 files changed, 59 insertions(+), 3 deletions(-) diff --git a/app.py b/app.py index b24e1b4..ebd5626 100644 --- a/app.py +++ b/app.py @@ -32,7 +32,13 @@ UnexpectedResponse = Exception # type: ignore[misc,assignment] try: # Allow tests to import without full qdrant client installed - from qdrant_client.models import Distance, PayloadSchemaType, PointStruct, VectorParams + from qdrant_client.models import ( + Distance, + HnswConfigDiff, + PayloadSchemaType, + PointStruct, + VectorParams, + ) except Exception: # pragma: no cover - degraded import path try: from qdrant_client.http import models as _qmodels @@ -40,9 +46,10 @@ Distance = getattr(_qmodels, "Distance", None) PointStruct = getattr(_qmodels, "PointStruct", None) VectorParams = getattr(_qmodels, "VectorParams", None) + HnswConfigDiff = getattr(_qmodels, "HnswConfigDiff", None) PayloadSchemaType = getattr(_qmodels, "PayloadSchemaType", None) except Exception: - Distance = PointStruct = VectorParams = None + Distance = PointStruct = VectorParams = HnswConfigDiff = None PayloadSchemaType = None # Provide a simple PointStruct shim for tests/environments lacking qdrant models @@ -296,6 +303,7 @@ def require_api_token() -> None: collection_name=COLLECTION_NAME, get_effective_vector_size_fn=get_effective_vector_size, vector_params_cls=VectorParams, + hnsw_config_diff_cls=HnswConfigDiff, distance_enum=Distance, payload_schema_type_enum=PayloadSchemaType, get_init_falkordb_fn=lambda: init_falkordb, diff --git a/automem/service_runtime_bindings.py b/automem/service_runtime_bindings.py index 331940b..550ecad 100644 --- a/automem/service_runtime_bindings.py +++ b/automem/service_runtime_bindings.py @@ -42,6 +42,7 @@ def create_service_runtime( collection_name: str, get_effective_vector_size_fn: Callable[[], int], vector_params_cls: Any, + hnsw_config_diff_cls: Any, distance_enum: Any, payload_schema_type_enum: Any, get_init_falkordb_fn: Callable[[], Callable[[], None]], @@ -80,6 +81,7 @@ def ensure_qdrant_collection() -> None: vector_size_config=vector_size_config_fn(), get_effective_vector_size_fn=get_effective_vector_size_fn, vector_params_cls=vector_params_cls, + hnsw_config_diff_cls=hnsw_config_diff_cls, distance_enum=distance_enum, payload_schema_type_enum=payload_schema_type_enum, ) diff --git a/automem/stores/runtime_clients.py b/automem/stores/runtime_clients.py index b7ae939..dbb1c3e 100644 --- a/automem/stores/runtime_clients.py +++ b/automem/stores/runtime_clients.py @@ -113,6 +113,7 @@ def ensure_qdrant_collection( vector_size_config: int, get_effective_vector_size_fn: Callable[[Any], tuple[int, str]], vector_params_cls: Any, + hnsw_config_diff_cls: Any, distance_enum: Any, payload_schema_type_enum: Any, ) -> None: @@ -143,7 +144,13 @@ def ensure_qdrant_collection( ) state.qdrant.create_collection( collection_name=collection_name, - vectors_config=vector_params_cls(size=effective_dim, distance=distance_enum.COSINE), + vectors_config=vector_params_cls( + size=effective_dim, + distance=distance_enum.COSINE, + on_disk=True, + ), + hnsw_config=hnsw_config_diff_cls(on_disk=True), + on_disk_payload=True, ) ensure_payload_indexes = os.getenv("QDRANT_ENSURE_PAYLOAD_INDEXES", "true").lower() in { diff --git a/tests/test_vector_size_safety.py b/tests/test_vector_size_safety.py index 02bd80c..57216a8 100644 --- a/tests/test_vector_size_safety.py +++ b/tests/test_vector_size_safety.py @@ -251,6 +251,43 @@ def bad_config(): class TestEnsureQdrantCollectionPayloadIndexes: + def test_new_collection_uses_on_disk_storage(self): + from automem.stores.runtime_clients import ensure_qdrant_collection + + class RecordingVectorParams: + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + + class RecordingHnswConfigDiff: + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + + qdrant = MagicMock() + qdrant.get_collections.return_value = SimpleNamespace(collections=[]) + state = SimpleNamespace(qdrant=qdrant, effective_vector_size=None) + + ensure_qdrant_collection( + state=state, + logger=MagicMock(), + collection_name="memories", + vector_size_config=1024, + get_effective_vector_size_fn=lambda _client: (768, "configured"), + vector_params_cls=RecordingVectorParams, + hnsw_config_diff_cls=RecordingHnswConfigDiff, + distance_enum=SimpleNamespace(COSINE="Cosine"), + payload_schema_type_enum=SimpleNamespace(KEYWORD="keyword"), + ) + + qdrant.create_collection.assert_called_once() + create_kwargs = qdrant.create_collection.call_args.kwargs + assert create_kwargs["collection_name"] == "memories" + assert create_kwargs["vectors_config"].size == 768 + assert create_kwargs["vectors_config"].distance == "Cosine" + assert create_kwargs["vectors_config"].on_disk is True + assert create_kwargs["hnsw_config"].on_disk is True + assert create_kwargs["on_disk_payload"] is True + assert qdrant.create_payload_index.call_count == 2 + @patch.dict("os.environ", {"QDRANT_ENSURE_PAYLOAD_INDEXES": "false"}, clear=False) def test_can_skip_payload_indexes_for_restore_tuned_lab_collection(self): from automem.stores.runtime_clients import ensure_qdrant_collection @@ -269,11 +306,13 @@ def test_can_skip_payload_indexes_for_restore_tuned_lab_collection(self): vector_size_config=1024, get_effective_vector_size_fn=lambda _client: (1024, "collection"), vector_params_cls=MagicMock(), + hnsw_config_diff_cls=MagicMock(), distance_enum=SimpleNamespace(COSINE="Cosine"), payload_schema_type_enum=SimpleNamespace(KEYWORD="keyword"), ) assert state.qdrant is qdrant + qdrant.create_collection.assert_not_called() qdrant.create_payload_index.assert_not_called() logger.info.assert_any_call( "Skipping Qdrant payload indexes for collection '%s'", "memories" From b04b9e1cd75b69e652b066e2faf68ff9c8ec198f Mon Sep 17 00:00:00 2001 From: Jack Arturo Date: Fri, 28 Aug 2026 03:32:53 +0200 Subject: [PATCH 23/33] perf(qdrant): support disk-backed HNSW restores --- scripts/lab/clone_production.sh | 1 + scripts/restore_from_backup.py | 8 +++++++- tests/test_backup_endpoint.py | 3 +++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/scripts/lab/clone_production.sh b/scripts/lab/clone_production.sh index 146ef0a..0bbd217 100755 --- a/scripts/lab/clone_production.sh +++ b/scripts/lab/clone_production.sh @@ -433,6 +433,7 @@ QDRANT_RESTORE_INDEXING_THRESHOLD="${QDRANT_RESTORE_INDEXING_THRESHOLD:-0}" \ QDRANT_RESTORE_DEFAULT_SEGMENT_NUMBER="${QDRANT_RESTORE_DEFAULT_SEGMENT_NUMBER:-1}" \ QDRANT_RESTORE_MEMMAP_THRESHOLD="${QDRANT_RESTORE_MEMMAP_THRESHOLD:-0}" \ QDRANT_RESTORE_HNSW_M="${QDRANT_RESTORE_HNSW_M:-0}" \ +QDRANT_RESTORE_HNSW_ON_DISK="${QDRANT_RESTORE_HNSW_ON_DISK:-true}" \ QDRANT_RESTORE_VECTOR_ON_DISK="${QDRANT_RESTORE_VECTOR_ON_DISK:-true}" \ QDRANT_RESTORE_ON_DISK_PAYLOAD="${QDRANT_RESTORE_ON_DISK_PAYLOAD:-false}" \ "$PYTHON_BIN" "$PROJECT_ROOT/scripts/restore_from_backup.py" \ diff --git a/scripts/restore_from_backup.py b/scripts/restore_from_backup.py index 03992bb..77b86fd 100755 --- a/scripts/restore_from_backup.py +++ b/scripts/restore_from_backup.py @@ -110,6 +110,7 @@ def _float_env(name: str, default: float) -> float: QDRANT_RESTORE_DEFAULT_SEGMENT_NUMBER = _optional_int_env("QDRANT_RESTORE_DEFAULT_SEGMENT_NUMBER") QDRANT_RESTORE_MEMMAP_THRESHOLD = _optional_int_env("QDRANT_RESTORE_MEMMAP_THRESHOLD") QDRANT_RESTORE_HNSW_M = _optional_int_env("QDRANT_RESTORE_HNSW_M") +QDRANT_RESTORE_HNSW_ON_DISK = _optional_bool_env("QDRANT_RESTORE_HNSW_ON_DISK") QDRANT_RESTORE_VECTOR_ON_DISK = _optional_bool_env("QDRANT_RESTORE_VECTOR_ON_DISK") QDRANT_RESTORE_ON_DISK_PAYLOAD = _optional_bool_env("QDRANT_RESTORE_ON_DISK_PAYLOAD") QDRANT_RESTORE_READY_TIMEOUT_SECONDS = float( @@ -448,8 +449,13 @@ def _qdrant_collection_kwargs(vector_size: int) -> dict[str, Any]: optimizer_kwargs["memmap_threshold"] = QDRANT_RESTORE_MEMMAP_THRESHOLD if optimizer_kwargs: kwargs["optimizers_config"] = OptimizersConfigDiff(**optimizer_kwargs) + hnsw_kwargs: dict[str, Any] = {} if QDRANT_RESTORE_HNSW_M is not None: - kwargs["hnsw_config"] = HnswConfigDiff(m=QDRANT_RESTORE_HNSW_M) + hnsw_kwargs["m"] = QDRANT_RESTORE_HNSW_M + if QDRANT_RESTORE_HNSW_ON_DISK is not None: + hnsw_kwargs["on_disk"] = QDRANT_RESTORE_HNSW_ON_DISK + if hnsw_kwargs: + kwargs["hnsw_config"] = HnswConfigDiff(**hnsw_kwargs) if QDRANT_RESTORE_ON_DISK_PAYLOAD is not None: kwargs["on_disk_payload"] = QDRANT_RESTORE_ON_DISK_PAYLOAD return kwargs diff --git a/tests/test_backup_endpoint.py b/tests/test_backup_endpoint.py index 55733b2..dc888ec 100644 --- a/tests/test_backup_endpoint.py +++ b/tests/test_backup_endpoint.py @@ -489,6 +489,7 @@ def test_lab_clone_uses_paced_qdrant_restore_defaults() -> None: ) assert 'QDRANT_RESTORE_MEMMAP_THRESHOLD="${QDRANT_RESTORE_MEMMAP_THRESHOLD:-0}"' in script assert 'QDRANT_RESTORE_HNSW_M="${QDRANT_RESTORE_HNSW_M:-0}"' in script + assert 'QDRANT_RESTORE_HNSW_ON_DISK="${QDRANT_RESTORE_HNSW_ON_DISK:-true}"' in script assert 'QDRANT_RESTORE_VECTOR_ON_DISK="${QDRANT_RESTORE_VECTOR_ON_DISK:-true}"' in script assert 'QDRANT_RESTORE_ON_DISK_PAYLOAD="${QDRANT_RESTORE_ON_DISK_PAYLOAD:-false}"' in script assert 'QDRANT_PREFER_GRPC="${QDRANT_PREFER_GRPC:-true}"' in script @@ -662,6 +663,7 @@ def upsert( monkeypatch.setattr(restore_module, "QDRANT_RESTORE_DEFAULT_SEGMENT_NUMBER", 1) monkeypatch.setattr(restore_module, "QDRANT_RESTORE_MEMMAP_THRESHOLD", 0) monkeypatch.setattr(restore_module, "QDRANT_RESTORE_HNSW_M", 0) + monkeypatch.setattr(restore_module, "QDRANT_RESTORE_HNSW_ON_DISK", True) monkeypatch.setattr(restore_module, "QDRANT_RESTORE_VECTOR_ON_DISK", True) monkeypatch.setattr(restore_module, "QDRANT_RESTORE_ON_DISK_PAYLOAD", False) @@ -675,4 +677,5 @@ def upsert( assert optimizers.default_segment_number == 1 assert optimizers.memmap_threshold == 0 assert RecordingQdrantClient.create_kwargs["hnsw_config"].m == 0 + assert RecordingQdrantClient.create_kwargs["hnsw_config"].on_disk is True assert RecordingQdrantClient.create_kwargs["vectors_config"].on_disk is True From d05a2d9cd4f10c12f27dc7bf142702ae2b0590d1 Mon Sep 17 00:00:00 2001 From: Jack Arturo Date: Fri, 28 Aug 2026 03:33:17 +0200 Subject: [PATCH 24/33] docs(qdrant): add on-disk migration runbook --- docs/ENVIRONMENT_VARIABLES.md | 10 +++++++++ docs/MONITORING_AND_BACKUPS.md | 38 ++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/docs/ENVIRONMENT_VARIABLES.md b/docs/ENVIRONMENT_VARIABLES.md index 2b49b10..e54b855 100644 --- a/docs/ENVIRONMENT_VARIABLES.md +++ b/docs/ENVIRONMENT_VARIABLES.md @@ -197,6 +197,16 @@ When running Qdrant as a Railway service (instead of Qdrant Cloud), set `QDRANT_ **Backward Compatibility**: `MCP_MEMORY_HTTP_ENDPOINT` is deprecated but still supported (falls back to this if `AUTOMEM_API_URL` not set). +### Qdrant Restore Tuning + +These options apply only to `scripts/restore_from_backup.py`; they do not reconfigure an existing collection during normal API startup. + +| Variable | Description | Default | +|----------|-------------|---------| +| `QDRANT_RESTORE_HNSW_ON_DISK` | Optional boolean that stores the HNSW index on disk when the restore creates the collection | unset | + +Use this with `QDRANT_RESTORE_VECTOR_ON_DISK=true` and `QDRANT_RESTORE_ON_DISK_PAYLOAD=true` for a full disk-backed Qdrant migration. The restore process deletes and recreates the selected collection unless run in import mode; see [Monitoring & Backups](MONITORING_AND_BACKUPS.md#api-backup-export) before using it against production. + ### Health Monitor | Variable | Description | Default | diff --git a/docs/MONITORING_AND_BACKUPS.md b/docs/MONITORING_AND_BACKUPS.md index f317d20..ae3a9c5 100644 --- a/docs/MONITORING_AND_BACKUPS.md +++ b/docs/MONITORING_AND_BACKUPS.md @@ -160,6 +160,44 @@ python scripts/restore_from_backup.py --backup-dir snapshot.tar.gz --force `GET /backup` requires the admin token because it exports the full corpus. Add `?include=falkordb` or `?include=qdrant` to export only one store. +#### Migrate an Existing Qdrant Collection to On-Disk Storage + +New AutoMem collections store vectors, the HNSW index, and payloads on disk by default. Existing collections retain their configuration, so migrate them deliberately during a maintenance window. + +> **Warning:** This operation deletes and recreates the configured Qdrant collection. Take and test a portable backup first; that backup is the rollback path. + +1. Record the current point count and create a recoverable Railway volume snapshot before touching the collection: + + ```bash + curl -sS -H "api-key: $QDRANT_API_KEY" \ + "$QDRANT_URL/collections/$QDRANT_COLLECTION" | jq '.result.points_count' + ``` + +2. Export a Qdrant-only portable backup from AutoMem: + + ```bash + curl -H "X-Admin-Token: $ADMIN_API_TOKEN" \ + "$AUTOMEM_API_URL/backup?include=qdrant" \ + -o qdrant-before-on-disk-migration.tar.gz + ``` + +3. Restore that backup to the intended Qdrant service with all on-disk settings enabled: + + ```bash + QDRANT_RESTORE_VECTOR_ON_DISK=true \ + QDRANT_RESTORE_HNSW_ON_DISK=true \ + QDRANT_RESTORE_ON_DISK_PAYLOAD=true \ + python scripts/restore_from_backup.py \ + --backup-dir qdrant-before-on-disk-migration.tar.gz \ + --qdrant-only --force + ``` + + Ensure `QDRANT_URL`, `QDRANT_API_KEY`, and `QDRANT_COLLECTION` identify the production collection before running the command. + +4. Compare the restored point count with the value recorded in step 1 and run representative recall requests before declaring the migration complete. If validation fails, stop and restore the tested backup. + +5. Leave the existing Railway volume size unchanged through a stable observation period. Resize it manually only after the restored collection is healthy and its disk usage is understood; AutoMem does not automate that infrastructure change. + #### Local Backups (Development) The `backup_automem.py` script exports both FalkorDB and Qdrant to compressed JSON files: From db71f07096d920be2f1648c8705af8b648dd0179 Mon Sep 17 00:00:00 2001 From: Jack Arturo Date: Fri, 28 Aug 2026 03:48:36 +0200 Subject: [PATCH 25/33] docs(qdrant): harden on-disk migration runbook --- docs/MONITORING_AND_BACKUPS.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/MONITORING_AND_BACKUPS.md b/docs/MONITORING_AND_BACKUPS.md index ae3a9c5..9e77e3e 100644 --- a/docs/MONITORING_AND_BACKUPS.md +++ b/docs/MONITORING_AND_BACKUPS.md @@ -166,14 +166,16 @@ New AutoMem collections store vectors, the HNSW index, and payloads on disk by d > **Warning:** This operation deletes and recreates the configured Qdrant collection. Take and test a portable backup first; that backup is the rollback path. -1. Record the current point count and create a recoverable Railway volume snapshot before touching the collection: +1. Quiesce writes before taking the backup. Put API clients, workers, and scheduled writers into maintenance/read-only mode, then keep writes stopped through the restore and validation. The API may remain available only to serve the backup export; point counts alone cannot detect concurrent updates or same-count changes. + +2. Record the current point count and create a recoverable Railway volume snapshot before touching the collection: ```bash curl -sS -H "api-key: $QDRANT_API_KEY" \ "$QDRANT_URL/collections/$QDRANT_COLLECTION" | jq '.result.points_count' ``` -2. Export a Qdrant-only portable backup from AutoMem: +3. Export a Qdrant-only portable backup from AutoMem: ```bash curl -H "X-Admin-Token: $ADMIN_API_TOKEN" \ @@ -181,7 +183,7 @@ New AutoMem collections store vectors, the HNSW index, and payloads on disk by d -o qdrant-before-on-disk-migration.tar.gz ``` -3. Restore that backup to the intended Qdrant service with all on-disk settings enabled: +4. Restore that backup to the intended Qdrant service with all on-disk settings enabled: ```bash QDRANT_RESTORE_VECTOR_ON_DISK=true \ @@ -194,9 +196,11 @@ New AutoMem collections store vectors, the HNSW index, and payloads on disk by d Ensure `QDRANT_URL`, `QDRANT_API_KEY`, and `QDRANT_COLLECTION` identify the production collection before running the command. -4. Compare the restored point count with the value recorded in step 1 and run representative recall requests before declaring the migration complete. If validation fails, stop and restore the tested backup. +5. Restart the AutoMem API while writes are still quiesced. With `QDRANT_ENSURE_PAYLOAD_INDEXES=true` (the default), startup recreates the `tags`, `tag_prefixes`, and `type` payload indexes that the collection restore does not preserve. + +6. Compare the restored point count with the value recorded in step 2 and run representative recall requests before declaring the migration complete. If validation fails, stop and restore the tested backup. Resume writers only after this validation succeeds. -5. Leave the existing Railway volume size unchanged through a stable observation period. Resize it manually only after the restored collection is healthy and its disk usage is understood; AutoMem does not automate that infrastructure change. +7. Leave the existing Railway volume size unchanged through a stable observation period. Resize it manually only after the restored collection is healthy and its disk usage is understood; AutoMem does not automate that infrastructure change. #### Local Backups (Development) From 45e77f49220a3d503263dd3d5fcb100d95606e6c Mon Sep 17 00:00:00 2001 From: Jack Arturo Date: Fri, 28 Aug 2026 03:55:12 +0200 Subject: [PATCH 26/33] docs(qdrant): verify migration export before restore --- docs/MONITORING_AND_BACKUPS.md | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/docs/MONITORING_AND_BACKUPS.md b/docs/MONITORING_AND_BACKUPS.md index 9e77e3e..40e3b02 100644 --- a/docs/MONITORING_AND_BACKUPS.md +++ b/docs/MONITORING_AND_BACKUPS.md @@ -183,7 +183,17 @@ New AutoMem collections store vectors, the HNSW index, and payloads on disk by d -o qdrant-before-on-disk-migration.tar.gz ``` -4. Restore that backup to the intended Qdrant service with all on-disk settings enabled: +4. Dry-run the portable artifact and confirm its reported point count matches the value from step 2 **before** deleting the collection: + + ```bash + python scripts/restore_from_backup.py \ + --backup-dir qdrant-before-on-disk-migration.tar.gz \ + --qdrant-only --dry-run --force + ``` + + If the counts differ, do not run the destructive restore. The portable export is incomplete; recover from the Railway volume snapshot instead of restoring the same artifact. + +5. Restore that verified backup to the intended Qdrant service with all on-disk settings enabled: ```bash QDRANT_RESTORE_VECTOR_ON_DISK=true \ @@ -196,11 +206,11 @@ New AutoMem collections store vectors, the HNSW index, and payloads on disk by d Ensure `QDRANT_URL`, `QDRANT_API_KEY`, and `QDRANT_COLLECTION` identify the production collection before running the command. -5. Restart the AutoMem API while writes are still quiesced. With `QDRANT_ENSURE_PAYLOAD_INDEXES=true` (the default), startup recreates the `tags`, `tag_prefixes`, and `type` payload indexes that the collection restore does not preserve. +6. Restart the AutoMem API while writes are still quiesced. With `QDRANT_ENSURE_PAYLOAD_INDEXES=true` (the default), startup recreates the `tags`, `tag_prefixes`, and `type` payload indexes that the collection restore does not preserve. -6. Compare the restored point count with the value recorded in step 2 and run representative recall requests before declaring the migration complete. If validation fails, stop and restore the tested backup. Resume writers only after this validation succeeds. +7. Compare the restored point count with the value recorded in step 2 and run representative recall requests before declaring the migration complete. If validation fails, stop and restore the tested backup. Resume writers only after this validation succeeds. -7. Leave the existing Railway volume size unchanged through a stable observation period. Resize it manually only after the restored collection is healthy and its disk usage is understood; AutoMem does not automate that infrastructure change. +8. Leave the existing Railway volume size unchanged through a stable observation period. Resize it manually only after the restored collection is healthy and its disk usage is understood; AutoMem does not automate that infrastructure change. #### Local Backups (Development) From 9b31cdea4b5d89d4da8150e64547b9086e178105 Mon Sep 17 00:00:00 2001 From: Jack Arturo Date: Fri, 28 Aug 2026 03:59:19 +0200 Subject: [PATCH 27/33] docs(qdrant): limit migration to default topology --- docs/MONITORING_AND_BACKUPS.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/MONITORING_AND_BACKUPS.md b/docs/MONITORING_AND_BACKUPS.md index 40e3b02..06c42be 100644 --- a/docs/MONITORING_AND_BACKUPS.md +++ b/docs/MONITORING_AND_BACKUPS.md @@ -175,6 +175,16 @@ New AutoMem collections store vectors, the HNSW index, and payloads on disk by d "$QDRANT_URL/collections/$QDRANT_COLLECTION" | jq '.result.points_count' ``` + This runbook is for collections using Qdrant's default topology. Capture and review the existing collection configuration before continuing: + + ```bash + curl -sS -H "api-key: $QDRANT_API_KEY" \ + "$QDRANT_URL/collections/$QDRANT_COLLECTION" \ + -o qdrant-before-on-disk-migration-config.json + ``` + + The restore preserves points and vector size, then applies the documented on-disk tuning; it does not preserve custom sharding, replication, write-consistency, quantization, or strict-mode settings. If this configuration has non-default settings, stop before the export and use an operator-specific migration that reapplies them during collection creation. + 3. Export a Qdrant-only portable backup from AutoMem: ```bash From 93afcffdf936a955066b2ad3d97c2c453b86891e Mon Sep 17 00:00:00 2001 From: Jack Arturo Date: Fri, 28 Aug 2026 05:21:06 +0200 Subject: [PATCH 28/33] docs: align provider and maintenance guidance --- .../automem-regression-drift-scout/SKILL.md | 109 --------- .env.example | 11 +- .gitignore | 8 +- AGENTS.md | 4 +- CLAUDE.md | 14 +- INSTALLATION.md | 24 +- README.md | 39 ++-- docker-compose.yml | 13 +- docs/API.md | 2 +- docs/COMPARISON.md | 6 +- docs/ENVIRONMENT_VARIABLES.md | 104 +++++++-- docs/HEALTH_MONITORING.md | 5 +- docs/MIGRATIONS.md | 113 ++++++---- docs/MONITORING_AND_BACKUPS.md | 34 +-- docs/QDRANT_SETUP.md | 26 ++- docs/RAILWAY_DEPLOYMENT.md | 32 ++- .../2026-08-28-qdrant-on-disk-storage.md | 213 ------------------ ...026-08-28-qdrant-on-disk-storage-design.md | 81 ------- .../queries_context_tags_20260704.json | 63 ------ scripts/README.md | 132 ++++++----- .../archive/summarize_pipeline_20260611.py | 131 ----------- .../compare_pr80_bm25_only_f10_judge_off.json | 4 +- 22 files changed, 352 insertions(+), 816 deletions(-) delete mode 100644 .agents/skills/automem-regression-drift-scout/SKILL.md delete mode 100644 docs/superpowers/plans/2026-08-28-qdrant-on-disk-storage.md delete mode 100644 docs/superpowers/specs/2026-08-28-qdrant-on-disk-storage-design.md delete mode 100644 lab/test_sets/queries_context_tags_20260704.json delete mode 100644 scripts/archive/summarize_pipeline_20260611.py diff --git a/.agents/skills/automem-regression-drift-scout/SKILL.md b/.agents/skills/automem-regression-drift-scout/SKILL.md deleted file mode 100644 index e2c1f99..0000000 --- a/.agents/skills/automem-regression-drift-scout/SKILL.md +++ /dev/null @@ -1,109 +0,0 @@ ---- -name: automem-regression-drift-scout -description: Inspect AutoMem CI, benchmark health, and deployment drift as a read-only recurring automation; use for scheduled Codex maintenance runs before proposing issues or PR plans. -license: MIT -tags: [automem, automation, regression, benchmarks, deployment] -category: maintenance -agents: [codex] -metadata: - version: "1.0.0" -capabilities: - network: true - filesystem: readonly - tools: [Bash] ---- - -# AutoMem Regression Drift Scout - -## When to use - -Use this skill for scheduled or manual Codex runs that assess whether AutoMem -has fresh regression, benchmark, CI, or deployment-drift signals that need -human attention. It is not a general code-review skill and it is not a release -operator. - -## Safety rules - -- Default to read-only. -- Do not edit files. -- Do not deploy. -- Do not create issues or pull requests. -- Do not run long or paid live benchmarks unless the automation prompt - explicitly authorizes that run. -- Do not report benchmark claims unless they are grounded in - `benchmarks/EXPERIMENT_LOG.md`, local benchmark command output, or CI output. -- Do not treat missing credentials as a failure by itself; report the skipped - surface and continue with available local evidence. - -## Inputs - -- Active worktree root: - `WORKTREE_ROOT=$(git rev-parse --show-toplevel 2>/dev/null)` -- Repository guidance: `AGENTS.md` -- Benchmark source of truth: `benchmarks/EXPERIMENT_LOG.md` -- Local command surface: `Makefile`, `pyproject.toml`, `.github/workflows` -- Optional read-only GitHub status via `gh run list` -- Optional Railway status via `railway status` - -## Workflow - -1. Load repository rules from `AGENTS.md`. -2. Inspect the current branch and dirty state with `git status --short --branch`. -3. Read the current benchmark baseline and latest experiment notes from - `benchmarks/EXPERIMENT_LOG.md`. -4. Run the fast local health probe when dependencies are available: - `make bench-health`. -5. Check deploy drift without changing remote state: - `make deploy-check`. -6. Check recent CI status if GitHub CLI is authenticated: - `gh run list --limit 10`. -7. Check Railway status if the Railway CLI is already authenticated: - `railway status`. -8. Compare the evidence against the last run memory, open issues/PRs if - visible, and recent `automem` memories. Dedupe before escalating. -9. Choose exactly one tier: - `healthy`, `needs_issue`, or `needs_pr_plan`. - -## Command result handling - -- Do not trust the final `RECALL HEALTH: HEALTHY` banner by itself. Parse the - full output for `ERROR:`, `SKIP:`, tracebacks, and connection failures. -- If `make bench-health` reports `Connection refused` for `localhost:8001`, - classify the local API-dependent checks as a skipped surface unless the - automation expected the local stack to be running. -- If `make deploy-check` reports `railway CLI not found`, classify Railway - deploy drift as a skipped surface, not as an AutoMem regression. -- If read-only GitHub or Railway status is unavailable because a CLI is missing - or unauthenticated, record the exact command output and continue with the - remaining evidence. - -## Tier guidance - -- `healthy`: checks are green or skipped for known missing credentials, and no - new actionable drift appears. -- `needs_issue`: evidence shows a real problem, but the fix is unclear, depends - on infrastructure state, or needs human prioritization. -- `needs_pr_plan`: evidence points to a bounded code/docs change with clear - files, commands, and acceptance criteria. The scout should draft the plan, not - create the PR unless a later prompt explicitly upgrades the run to PR-capable. - -## Output - -Return a concise triage deliverable with: - -- tier -- evidence bundle with command names, timestamps, and key outputs -- confidence score and one-line rationale -- dedupe notes against issues, PRs, or previous memories -- recommended next action -- what to measure next time - -Always end with the required inbox item for the automation runner. - -## Anti-patterns - -- Running heavyweight live benchmarks as a routine scout. -- Treating a stale benchmark table as a regression without fresh evidence. -- Opening GitHub issues or PRs from the read-only scout. -- Reporting deploy drift from memory alone without checking current read-only - status. diff --git a/.env.example b/.env.example index d879583..19b9d17 100644 --- a/.env.example +++ b/.env.example @@ -11,12 +11,21 @@ QDRANT_HOST= QDRANT_PORT=6333 QDRANT_URL= QDRANT_COLLECTION=memories -VECTOR_SIZE=1024 # Must match your embedding provider (1024=voyage-4, 3072=text-embedding-3-large, 768=text-embedding-3-small) +# Recommended embeddings: Voyage is the default when its API key is configured. +EMBEDDING_PROVIDER=auto +VOYAGE_API_KEY= +VOYAGE_MODEL=voyage-4 +VECTOR_SIZE=1024 # voyage-4 and Ollama bge-m3; must match your selected provider PORT=8001 +# OpenAI is the automatic API fallback when VOYAGE_API_KEY is unset. OPENAI_API_KEY= # For OpenAI-compatible providers (OpenRouter, LiteLLM, vLLM, etc.): # OPENAI_BASE_URL=https://openrouter.ai/api/v1 # EMBEDDING_MODEL=openai/text-embedding-3-small +# For local/self-hosted Ollama embeddings (for example, multilingual BGE-M3): +# OLLAMA_BASE_URL=http://localhost:11434 +# OLLAMA_MODEL=bge-m3 +# FastEmbed is local/self-hosted only; do not rely on it as a cloud default. # --- LLM classification (scripts/reclassify_with_llm.py) --- # Optional. Defaults to OpenAI gpt-4o-mini via OPENAI_API_KEY. # OPENROUTER_API_KEY= # used by --provider openrouter diff --git a/.gitignore b/.gitignore index d8b3dce..daa1a10 100644 --- a/.gitignore +++ b/.gitignore @@ -33,10 +33,8 @@ node_modules/ # Experiment results (promote notable runs to tests/benchmarks/results/) /tests/benchmarks/experiments/results_*/ -# Recall Quality Lab data (snapshots, test results) -/lab/snapshots/ -/lab/results/ -/lab/test_sets/ +# Recall Quality Lab data (generated snapshots, test sets, and results) +/lab/ # Benchmark overrides and snapshots .env.bench @@ -47,3 +45,5 @@ node_modules/ benchmarks/baselines/locomo_baseline.json data/bm25_index.db data/ +/.agents +/docs/superpowers/ diff --git a/AGENTS.md b/AGENTS.md index 1a7fdb1..e416ed9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,7 +8,7 @@ - `benchmarks/`: Snapshot-based benchmark system. See `EXPERIMENT_LOG.md` for current baselines and results. - `scripts/bench/`: Benchmark tooling (ingest, eval, compare, health check). - `docs/`: API, testing, deployment, monitoring, and env var references. -- `scripts/`: Maintenance and ops helpers (backup, reembed, health monitor). +- `scripts/`: Maintenance and ops helpers (backup, reembed, health monitor). See the canonical [scripts catalog](scripts/README.md) for lifecycle and usage. - `mcp-sse-server/`: Optional MCP bridge used in some deployments. ## Build, Test, and Development @@ -76,7 +76,7 @@ The benchmark system uses **snapshot-based evaluation**: ingest once, eval many - `benchmarks/baselines/` — baseline result JSONs (small files committed, large ones gitignored). - `benchmarks/snapshots/` — Qdrant/FalkorDB snapshot data (gitignored, regenerate with `make bench-ingest`). - `benchmarks/results/` — per-run result JSONs (gitignored). -- `scripts/bench/` — shell and Python scripts driving ingest, eval, compare, and health checks. +- `scripts/bench/` — shell and Python scripts driving ingest, eval, compare, and health checks; see [scripts/README.md](scripts/README.md). - `tests/benchmarks/` — legacy benchmark harnesses (LoCoMo, LongMemEval) and historical result markdown files. ## Commit & Pull Requests diff --git a/CLAUDE.md b/CLAUDE.md index bf02473..e937c60 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -142,11 +142,13 @@ AutoMem uses a provider pattern with multiple embedding backends: #### Provider Priority (Auto-Selection) 1. **Voyage AI** (`voyage:voyage-4`) - If `VOYAGE_API_KEY` is set + - Recommended provider for new deployments: multilingual quality with a generous free tier; see [Voyage pricing](https://docs.voyageai.com/docs/pricing) for current limits and rates - High-quality embeddings with flexible model family (voyage-4, voyage-4-large, voyage-4-lite) - Supports output dimensions: 256, 512, 1024, 2048 - Requires network and API key 2. **OpenAI / OpenAI-compatible** (`openai:text-embedding-3-small`) - If `OPENAI_API_KEY` is set + - API fallback when Voyage is not configured - Semantic embeddings via API, truncated to `VECTOR_SIZE` via Matryoshka (OpenAI native only) - If `VECTOR_SIZE` > 1536, auto-upgrades to `text-embedding-3-large` (set `EMBEDDING_MODEL=text-embedding-3-large` to silence) - Supports any OpenAI-compatible endpoint via `OPENAI_BASE_URL` (OpenRouter, LiteLLM, vLLM, Azure, etc.) for both embeddings and classification/enrichment LLM calls @@ -155,23 +157,25 @@ AutoMem uses a provider pattern with multiple embedding backends: 3. **Ollama** (`ollama:nomic-embed-text`) - If `OLLAMA_BASE_URL` or `OLLAMA_MODEL` is configured - Fully local, easy model swapping - Requires running Ollama server + - Use `ollama pull bge-m3`, then `OLLAMA_MODEL=bge-m3` with `VECTOR_SIZE=1024` for local/self-hosted multilingual BGE-M3 4. **FastEmbed** (`fastembed:BAAI/bge-base-en-v1.5`) - Local ONNX model - Good quality semantic embeddings - No API key or internet required (after first download) - Downloads ~210MB model to `~/.config/automem/models/` on first use - 768 dimensions (default), also supports 384 and 1024 dim models + - Local/self-hosted only: its 1024d cloud fallback can consume roughly 4 GB RSS and raise hosting cost substantially 5. **Placeholder** (`placeholder`) - Hash-based fallback - Deterministic vectors from content hash - No semantic meaning, last resort only -**Upgrade safety:** `VECTOR_SIZE_AUTODETECT=true` (default) automatically adopts your existing collection dimension on startup. No manual action needed when updating. To enforce strict matching, set `VECTOR_SIZE_AUTODETECT=false`. +**Upgrade safety:** `VECTOR_SIZE_AUTODETECT=true` (default) only adopts an existing collection dimension. It does not make vectors from different models compatible: any provider or model change requires recreating the Qdrant collection and a full re-embed, including a swap between two 1024d models. To enforce strict dimension matching, set `VECTOR_SIZE_AUTODETECT=false`. #### Provider Configuration Control via `EMBEDDING_PROVIDER` environment variable: -- `auto` (default): Try Voyage → OpenAI → Ollama → FastEmbed → Placeholder +- `auto` (default): Try Voyage → OpenAI → Ollama (only when configured) → FastEmbed → Placeholder - `voyage`: Use Voyage only (fail if unavailable) - `openai`: Use OpenAI only (fail if unavailable). Also works with OpenAI-compatible providers when `OPENAI_BASE_URL` is set. - `ollama`: Use Ollama only (fail if unavailable) @@ -226,7 +230,7 @@ QDRANT_HOST= # OR hostname for self-hosted Qdrant (e.g. "qdrant" QDRANT_PORT=6333 # Port for self-hosted Qdrant (used with QDRANT_HOST) QDRANT_API_KEY= # Qdrant Cloud API key (optional, not needed for self-hosted) QDRANT_COLLECTION=memories # Collection name -VECTOR_SIZE=1024 # Embedding dimensions (1024 for voyage-4, 768 for small, 3072 for large) +VECTOR_SIZE=1024 # Embedding dimensions (1024 for voyage-4 or Ollama bge-m3, 768 for small, 3072 for large) VECTOR_SIZE_AUTODETECT=true # Adopt existing collection dim on startup (false = fail on mismatch) # API configuration @@ -236,8 +240,12 @@ ADMIN_API_TOKEN= # For admin endpoints # Embedding configuration EMBEDDING_PROVIDER=auto # auto|voyage|openai|ollama|local|placeholder +VOYAGE_API_KEY= # Recommended for new cloud deployments +VOYAGE_MODEL=voyage-4 OPENAI_API_KEY= # For OpenAI or compatible provider (optional) OPENAI_BASE_URL= # Custom endpoint for OpenAI-compatible APIs used by embeddings and classification/enrichment (optional) +OLLAMA_BASE_URL= # Intentional local/self-hosted Ollama endpoint +OLLAMA_MODEL= # e.g. bge-m3 (requires `ollama pull bge-m3`) # Consolidation intervals (seconds) CONSOLIDATION_DECAY_INTERVAL_SECONDS=86400 # 1 day (default) diff --git a/INSTALLATION.md b/INSTALLATION.md index 3a6a003..cc97494 100644 --- a/INSTALLATION.md +++ b/INSTALLATION.md @@ -125,7 +125,7 @@ _Screenshot pending: Railway services dashboard image will be added at `docs/img 1. Sign in with GitHub (if not logged in) 2. Review environment variables -3. (Optional) Add `OPENAI_API_KEY` for real embeddings instead of mock embeddings +3. Add `VOYAGE_API_KEY` for the recommended `voyage-4` embeddings (OpenAI is supported as the API fallback) 4. Click **"Deploy"** 5. Wait ~60 seconds for deployment to complete ✅ @@ -209,7 +209,9 @@ Reference these in AutoMem config via `${{service..internalHost}}` | `ADMIN_API_TOKEN` | Token for admin/enrichment endpoints | ✅ Yes | | `FALKORDB_HOST` | Internal hostname of FalkorDB service | ✅ Yes | | `FALKORDB_PORT` | FalkorDB port (usually `6379`) | ✅ Yes | - | `OPENAI_API_KEY` | Enables real embeddings | Recommended | + | `VOYAGE_API_KEY` | Recommended API key for `voyage-4` embeddings | Recommended | + | `OPENAI_API_KEY` | OpenAI-compatible embedding fallback | Optional | + | `EMBEDDING_PROVIDER` | Provider selection (`auto` prefers Voyage) | `auto` | | `FALKORDB_PASSWORD` | Password if set on FalkorDB | If enabled | | `QDRANT_URL` | Qdrant Cloud endpoint | Optional | | `QDRANT_API_KEY` | Qdrant API key | If using Qdrant | @@ -357,7 +359,7 @@ curl -X POST https://your-automem.railway.app/memory \ 3. **Set up monitoring** (optional): See [Health Monitoring Guide](docs/HEALTH_MONITORING.md) 👉 **[Full Railway Guide](docs/RAILWAY_DEPLOYMENT.md)** - Advanced configuration, monitoring, troubleshooting -👉 **[Deployment Checklist](docs/DEPLOYMENT_CHECKLIST.md)** - Step-by-step verification +👉 **[Post-Deploy Checklist](docs/RAILWAY_DEPLOYMENT.md#post-deploy-checklist)** - Step-by-step verification --- @@ -411,7 +413,7 @@ export QDRANT_URL=http://localhost:6333 # Full URL (Qdrant Cloud or explicit) python app.py ``` -The API will use deterministic placeholder embeddings if no `OPENAI_API_KEY` or Qdrant is configured. +Without Qdrant, semantic vector storage is unavailable. With Qdrant and `EMBEDDING_PROVIDER=auto`, AutoMem tries configured Voyage, OpenAI, explicitly configured Ollama, and then local FastEmbed. For Railway, configure Voyage rather than depending on the FastEmbed fallback. --- @@ -466,12 +468,14 @@ Admin operations additionally require `X-Admin-Token: ` header. | `VOYAGE_MODEL` | Voyage model (Voyage provider) | `voyage-4` | | `OPENAI_API_KEY` | API key (OpenAI or compatible provider) | _unset_ | | `OPENAI_BASE_URL` | Custom endpoint for OpenAI-compatible providers | _unset_ | +| `OLLAMA_BASE_URL` | Ollama endpoint for intentional local/self-hosted use | `http://localhost:11434` | +| `OLLAMA_MODEL` | Pulled Ollama embedding model | `nomic-embed-text` | 👉 **New to Qdrant?** See the [Qdrant Setup Guide](docs/QDRANT_SETUP.md) for setup options (self-hosted on Railway or Qdrant Cloud). -> **Upgrade safety:** `VECTOR_SIZE_AUTODETECT=true` (default) automatically adopts your existing collection dimension on startup. No manual action needed when updating — existing 3072d or 768d collections continue to work. +> **Upgrade safety:** `VECTOR_SIZE_AUTODETECT=true` (default) adopts an existing collection dimension to avoid a startup mismatch. It does not migrate vectors: changing embedding provider or model requires recreating the collection and re-embedding, even when both models are 1024d. > -> The recommended setup is Voyage (`voyage-4`) at 1024d. If you only have an OpenAI key, `text-embedding-3-small` is used as fallback and truncated to `VECTOR_SIZE` via Matryoshka. +> The recommended setup is Voyage (`voyage-4`) at 1024d. If Voyage is unavailable, an OpenAI-compatible provider is the API fallback. Use FastEmbed only for intentional local/self-hosted deployments; its 1024d cloud fallback can add roughly 4 GB RSS and materially increase hosting cost. #### Enrichment Pipeline @@ -610,7 +614,7 @@ Store a new memory. - **Explicit `type` preferred**: Send `type` when you know the classification for immediate, accurate categorization - **Auto-classification fallback**: Omit `type` to let enrichment pipeline classify based on content -- **Embedding auto-generation**: Service generates real embeddings (OpenAI) or placeholder vectors if omitted +- **Embedding auto-generation**: Service generates vectors through the configured provider; `auto` prefers Voyage, then OpenAI, configured Ollama, FastEmbed, and finally placeholders - **Timestamp defaults**: All time fields default to current UTC time if not provided - **Background enrichment**: Entity extraction and relationship building queued automatically - **Type validation**: Invalid types return `400 Bad Request` with list of valid options @@ -894,6 +898,10 @@ Perfect for migrations or after updating embedding model. ## Migration +For the current migration-script catalog, safety classification, and CLI flags, +see [scripts/README.md](scripts/README.md). Follow the migration runbook below +for the required backup and validation sequence. + ### From MCP SQLite Memory Service Use the migration helper to transfer memories from legacy MCP SQLite: @@ -994,7 +1002,7 @@ AUTOMEM_RUN_INTEGRATION_TESTS=1 \ - `AUTOMEM_ALLOW_LIVE=1` - Required for non-localhost endpoints - `AUTOMEM_TEST_API_TOKEN` / `AUTOMEM_TEST_ADMIN_TOKEN` - Auth tokens -See **[TESTING.md](TESTING.md)** for complete testing documentation. +See **[TESTING.md](docs/TESTING.md)** for complete testing documentation. --- diff --git a/README.md b/README.md index 0744fcb..e40c43c 100644 --- a/README.md +++ b/README.md @@ -8,49 +8,38 @@ Discord X LoCoMo on the neutral Agent Memory Benchmark + LongMemEval full on AutoMem's internal harness: 87.0% BEAM 10M on the neutral Agent Memory Benchmark Deploy on Railway

- Long-term memory for AI assistants. Graph + vector. Runs on your hardware. + Long-term memory for AI assistants — fast, private, and yours.

- - # AutoMem -Your AI forgets between sessions. RAG dumps documents that look similar. Vector databases match keywords but miss meaning. None of them learn. +[AutoMem](https://automem.ai) gives your AI a memory that survives the chat. + +Save the decisions, preferences, notes, and context that matter. The next time you open Claude, Cursor, Codex, ChatGPT, or another connected assistant, it can bring back the right details instead of making you repeat yourself. -AutoMem stores typed relationships *and* embeddings. When you ask "why did we choose PostgreSQL?", recall returns not just the matching memory — but the alternatives you considered, the principle behind the choice, and the related decisions that came after. +Ask, “Why did we choose PostgreSQL?” and AutoMem can return the decision, the alternatives you considered, the principle behind it, and the work that followed — not just a pile of similarly worded snippets. -On its *own* internal harness, AutoMem scores **87.00%** on LongMemEval full (**97.00%** recall@5) and **84.74%** on LoCoMo full — but those are engineering baselines scored by a self-hosted judge (`gpt-5.4-mini`), and the same answers can swing ~12 points on judge model alone. For comparison against other systems, the numbers that matter come from the **neutral** third-party board below. See [`benchmarks/EXPERIMENT_LOG.md`](benchmarks/EXPERIMENT_LOG.md) for methodology, judge policy, category breakdowns, and historical runs. +## Why AutoMem feels different -### On the neutral Agent Memory Benchmark +**No LLM call in the middle of recall.** AutoMem looks up memories directly through its graph and vector index. That keeps normal retrieval fast and avoids an extra generative-LLM charge every time your assistant needs context. It still uses embeddings — from Voyage, OpenAI, or a local provider — and optional enrichment can add more structure over time. -AutoMem **0.16.0** was run through the neutral [Agent Memory Benchmark](https://automem.ai/benchmarks) (AMB, by vectorize-io) on a self-spinning FalkorDB + Qdrant stack with **FastEmbed-local `bge-base-en-v1.5` (768d)** — no embedding API keys. The honest summary: AutoMem's strength is **large-context scaling and efficiency**, not verbatim conversational recall. +**More than vector search.** A vector match finds something similar; AutoMem also records typed relationships between memories. It can follow the connections between a decision, its rationale, and its consequences, so your assistant has a better chance of returning the *why*, not only the words it recognizes. -- **BEAM is the apples-to-apples axis** (same benchmark, same Gemini answerer + judge). AutoMem scores above Honcho at every BEAM tier, and the gap widens with scale: **+4.5pp at 100k, +0.7pp at 500k, +0.7pp at 1M, +16.8pp at 10M**. AutoMem degrades gracefully — **67.5% → 57.4%** (−10pp) across a 100× haystack increase — while Honcho holds roughly flat through 1M, then drops to 40.6% at 10M. That places AutoMem **#2 on BEAM**, behind vectorize's own Hindsight (~73→64% across the curve). -- **At 10M tokens, AutoMem holds 57.4% ±5.5%** while Honcho falls to ~41%. At that scale, context-stuffing is physically impossible, so the score reflects retrieval architecture, not context window. -- **Efficiency is architectural:** AutoMem feeds the answerer **~2.6–4.8k context tokens** at every scale (mean), versus 17–27k for the board leader on BEAM. -- **The honest other half:** on conversational Core-3, AutoMem **trails** the AMB leader Hindsight — locomo 85.1% vs 92%, longmemeval 74.4% vs 94.6%, personamem 76.1% vs 86.6%. Pick AutoMem for large-context scaling and efficiency, not for top-of-board verbatim recall. +**One memory across your tools.** Use the local MCP bridge with Claude Desktop, Cursor, Claude Code, Codex, Copilot, and more. For cloud agents, Remote MCP connects the same service to ChatGPT Developer Mode, Claude.ai, and ElevenLabs over HTTPS. Your memory is not locked to one chat app. -Outputs are committed and public, and [`AUTOMEM_REPRODUCE.md`](https://automem.ai/benchmarks) gives one command per split so you can **run it yourself**. AutoMem is **submitted to the neutral board ([provider PR #24](https://github.com/vectorize-io/agent-memory-benchmark/pull/24), under review)** — not yet live on the public leaderboard. Full head-to-head numbers live at [automem.ai/benchmarks](https://automem.ai/benchmarks). +**Own the data and the setup.** Run AutoMem locally with Docker, on your own infrastructure, or as a small Railway service group. It exposes both MCP and a REST API, so it fits into the tools and workflows you already use. -## Should you use AutoMem? +## Proven where long context gets hard -| Use AutoMem if... | Look elsewhere if... | -|---|---| -| You want one memory across Claude / Cursor / ChatGPT / Codex | You need SOC2 / HIPAA audit logs and row-level ACLs | -| You're comfortable self-hosting (Docker or Railway) | You want a managed SaaS with a polished dashboard | -| You're a solo dev, prosumer, or small team | You're running a multi-agent swarm needing per-agent memory isolation | -| You want to own your memory data | You need an enterprise SLA and dedicated support | +On the independent [Agent Memory Benchmark](https://automem.ai/benchmarks)'s BEAM long-context tests, AutoMem scored **57.4% at 10 million source tokens** while giving the answerer an average of only **~2.6–4.8k retrieved tokens**. That is the kind of efficiency that lets memory stay useful as an agent's history grows. -If your row is on the right, AutoMem isn't it — yet. Try [Mem0](https://mem0.ai), [Letta](https://letta.com), or [Zep](https://www.getzep.com) instead. +The full picture — test setup, raw outputs, methodology, historical runs, and reproduction commands — is in [automem.ai/benchmarks](https://automem.ai/benchmarks) and [`benchmarks/EXPERIMENT_LOG.md`](benchmarks/EXPERIMENT_LOG.md). ## How it works diff --git a/docker-compose.yml b/docker-compose.yml index 046d74f..4528cc9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -58,10 +58,19 @@ services: QDRANT_ENSURE_PAYLOAD_INDEXES: ${QDRANT_ENSURE_PAYLOAD_INDEXES:-true} AUTOMEM_API_TOKEN: ${AUTOMEM_API_TOKEN:-test-token} ADMIN_API_TOKEN: ${ADMIN_API_TOKEN:-test-admin-token} - OPENAI_API_KEY: ${OPENAI_API_KEY:-} VOYAGE_API_KEY: ${VOYAGE_API_KEY:-} + VOYAGE_MODEL: ${VOYAGE_MODEL:-voyage-4} + OPENAI_API_KEY: ${OPENAI_API_KEY:-} + OPENAI_BASE_URL: ${OPENAI_BASE_URL:-} + EMBEDDING_MODEL: ${EMBEDDING_MODEL:-text-embedding-3-small} + OLLAMA_BASE_URL: ${OLLAMA_BASE_URL:-} + OLLAMA_MODEL: ${OLLAMA_MODEL:-} + OLLAMA_TIMEOUT: ${OLLAMA_TIMEOUT:-30} + OLLAMA_MAX_RETRIES: ${OLLAMA_MAX_RETRIES:-2} VECTOR_SIZE: ${VECTOR_SIZE:-1024} - EMBEDDING_PROVIDER: ${EMBEDDING_PROVIDER:-auto} # auto|voyage|openai|local|placeholder + VECTOR_SIZE_AUTODETECT: ${VECTOR_SIZE_AUTODETECT:-true} + # auto tries Voyage, OpenAI, configured Ollama, FastEmbed, then placeholders. + EMBEDDING_PROVIDER: ${EMBEDDING_PROVIDER:-auto} MEMORY_CONTENT_HARD_LIMIT: ${MEMORY_CONTENT_HARD_LIMIT:-2000} MEMORY_AUTO_SUMMARIZE: ${MEMORY_AUTO_SUMMARIZE:-true} AUTOMEM_MODELS_DIR: /root/.config/automem/models # Keep in sync with volume mount diff --git a/docs/API.md b/docs/API.md index eb4d125..6616bc8 100644 --- a/docs/API.md +++ b/docs/API.md @@ -172,7 +172,7 @@ Stream - `memory.associate` — single edge (`memory1_id`, `memory2_id`, `relation_type`, `strength`, `count: 1`) or batch (`count`, `failed_count`, `relation_types`) - `enrichment.start` / `enrichment.complete` / `enrichment.failed` - `consolidation.run` - - Failed validation (`4xx`) does not emit. Watch with `python scripts/automem_watch.py --url … --token …`. + - Failed validation (`4xx`) does not emit. Watch with `python scripts/automem_watch.py --url … --token …`; see the [scripts catalog](../scripts/README.md#operations-and-maintenance) for its usage and safety notes. - GET `/stream/status` - Response: `{ "subscribers": N }` diff --git a/docs/COMPARISON.md b/docs/COMPARISON.md index 92f422b..b1b355d 100644 --- a/docs/COMPARISON.md +++ b/docs/COMPARISON.md @@ -2,7 +2,7 @@ This doc covers how AutoMem differs from the three things people most often compare it against: traditional RAG, pure vector databases, and rolling your own memory layer. It also covers the recall scoring formula in enough detail to evaluate whether AutoMem will solve your specific retrieval problem. -For the short version, skip to the README's [Should you use AutoMem?](../README.md#should-you-use-automem) section. For research foundations, see [RESEARCH.md](RESEARCH.md). +For the short version, start with the README's [Why AutoMem feels different](../README.md#why-automem-feels-different) section. For research foundations, see [RESEARCH.md](RESEARCH.md). --- @@ -86,13 +86,13 @@ When building your own is the right answer: These defaults reflect the current canonical benchmark posture: LongMemEval full at 87.00% with 97.00% recall@5, and LoCoMo full at 84.74%. For a query like `GET /recall?query=database+migration&tags=decision&time_query=last+month`, the temporal-alignment and tag components dominate; for `GET /recall?query=why+postgres&expand_relations=true`, the relation component does. -The Recall Quality Lab (`scripts/lab/`) lets you sweep any weight and A/B-compare configs against snapshots of production data without touching the service. +The Recall Quality Lab (`scripts/lab/`) lets you sweep any weight and A/B-compare configs against snapshots of production data without touching the service. The [scripts catalog](../scripts/README.md) explains when to clone, generate queries, test, compare, or sweep. --- ## Where to go next - **Want to deploy?** [INSTALLATION.md](../INSTALLATION.md) -- **Want to tune recall?** [`scripts/lab/`](../scripts/lab/) and [`docs/ENVIRONMENT_VARIABLES.md`](ENVIRONMENT_VARIABLES.md) +- **Want to tune recall?** [Recall Quality Lab in the scripts catalog](../scripts/README.md) and [`docs/ENVIRONMENT_VARIABLES.md`](ENVIRONMENT_VARIABLES.md) - **Want to understand the science?** [RESEARCH.md](RESEARCH.md) - **Want to talk to humans?** [Discord](https://automem.ai/discord) diff --git a/docs/ENVIRONMENT_VARIABLES.md b/docs/ENVIRONMENT_VARIABLES.md index d73fb41..98585ce 100644 --- a/docs/ENVIRONMENT_VARIABLES.md +++ b/docs/ENVIRONMENT_VARIABLES.md @@ -67,11 +67,22 @@ AutoMem supports five embedding backends with automatic fallback. - `auto` (default): Try Voyage → OpenAI → Ollama (if configured) → FastEmbed local model → Placeholder - `voyage`: Use Voyage API only (requires `VOYAGE_API_KEY`) - `openai`: Use OpenAI API only (requires `OPENAI_API_KEY`) -- `local`: Use FastEmbed local model only (~210MB download on first use) +- `local`: Use FastEmbed local model only - `ollama`: Use Ollama only (requires `OLLAMA_BASE_URL` and a pulled model) - `placeholder`: Use hash-based embeddings (no semantic search) -**Local Model Details:** +For new cloud deployments, configure Voyage and leave automatic fallback enabled: + +```bash +EMBEDDING_PROVIDER=auto +VOYAGE_API_KEY=pa-... +VOYAGE_MODEL=voyage-4 +VECTOR_SIZE=1024 +``` + +With `auto`, Ollama is attempted only when `OLLAMA_BASE_URL` or `OLLAMA_MODEL` is explicitly configured. FastEmbed is the final usable embedding fallback; do not rely on it as a cloud default. + +**FastEmbed (local/self-hosted only):** - Models: `BAAI/bge-small-en-v1.5` (384d), `BAAI/bge-base-en-v1.5` (768d), `BAAI/bge-large-en-v1.5` (1024d) - Size: ~67MB / ~210MB / ~1.2GB (cached to `~/.config/automem/models/`) - No API key or internet required after first download @@ -79,6 +90,7 @@ AutoMem supports five embedding backends with automatic fallback. - Recommended: pin `onnxruntime<1.20` with `fastembed` 0.4.x to avoid runtime issues - Docker: persist model cache with a volume (see docker-compose.yml) - Ensure `VECTOR_SIZE` equals the selected model's dimension (default 768) +- The 1024d fallback can add roughly 4 GB RSS in a cloud container. On Railway or another managed host, that materially raises the hosting bill; use Voyage instead unless you are intentionally self-hosting FastEmbed. **Ollama Details:** - Requires a running Ollama server (default `http://localhost:11434`) @@ -86,11 +98,24 @@ AutoMem supports five embedding backends with automatic fallback. - Embedding dimensions vary by model; set `VECTOR_SIZE` to match the model output - For `auto` mode, Ollama is only attempted if `OLLAMA_BASE_URL` or `OLLAMA_MODEL` is set +For a local/self-hosted multilingual option, run [BGE-M3 on Ollama](https://ollama.com/library/bge-m3): + +```bash +ollama pull bge-m3 + +EMBEDDING_PROVIDER=ollama +OLLAMA_MODEL=bge-m3 +VECTOR_SIZE=1024 +``` + +Set `OLLAMA_BASE_URL` when the Ollama server is not at `http://localhost:11434`. BGE-M3 is documented through Ollama; it is not a FastEmbed option in this project. + **Voyage Details:** - Requires `VOYAGE_API_KEY` - Optional model override via `VOYAGE_MODEL` (default `voyage-4`) - Voyage 4 family supports output dimensions `256`, `512`, `1024`, or `2048` - Set `VECTOR_SIZE` to one of the supported Voyage dimensions +- Voyage's free tier covers typical AutoMem usage. See the maintained [Voyage pricing page](https://docs.voyageai.com/docs/pricing) for current limits and rates. **OpenAI-Compatible Providers (OpenRouter, LiteLLM, Azure, vLLM, etc.):** @@ -120,19 +145,21 @@ VECTOR_SIZE=768 # must match the model's output | Provider | Cost | Pros | Cons | Best for | |----------|------|------|------|----------| -| Voyage | API usage | Strong quality, flexible model family, simple API | Requires outbound network and matching `VECTOR_SIZE` | Teams using Voyage for quality/cost balance | -| OpenAI | ~$0.13/1M tokens (large), ~$0.02/1M tokens (small) | Highest semantic quality, zero infra | Recurring API cost, outbound network required | Production accuracy with minimal ops | -| FastEmbed (local) | Hardware-only (CPU/GPU) | Offline after first download, consistent latency | Model download size, quality tied to model size | Self-hosted + cost-sensitive environments | +| Voyage | Free tier for typical usage, then API usage | Recommended default, strong multilingual quality, no model hosting | Requires outbound network and matching `VECTOR_SIZE` | New cloud deployments | +| OpenAI | API usage | OpenAI-compatible API fallback, zero model hosting | Requires outbound network | When Voyage is unavailable or an OpenAI-compatible endpoint is required | +| FastEmbed (local) | Hardware and memory | Offline after first download, consistent latency | 1024d fallback can add roughly 4 GB RSS in cloud containers | Intentional local/self-hosted deployments only | | Ollama | Hardware-only (CPU/GPU) | Fully local, easy model swapping | Requires running Ollama service, model dims vary | Self-hosted deployments with Ollama already in stack | | Placeholder | Free | Always available, deterministic | No semantic search quality | Development/testing without vectors | -**Dimension matching tip:** If embeddings fail with a size mismatch, confirm your model's output length and set `VECTOR_SIZE` accordingly (or use `VECTOR_SIZE_AUTODETECT=true` if you accept adopting the existing collection size). +See [Voyage pricing](https://docs.voyageai.com/docs/pricing) for current free-tier limits and rates rather than relying on fixed pricing in this table. + +**Dimension matching tip:** If embeddings fail with a size mismatch, confirm your model's output length and set `VECTOR_SIZE` accordingly. `VECTOR_SIZE_AUTODETECT=true` can adopt an existing collection's dimension for compatibility, but it does not make vectors from different models compatible. ## Hosting Considerations (Railway vs Self-Hosted) -- **Railway / managed PaaS:** Voyage and OpenAI are the simplest choices (no local model downloads). FastEmbed works but increases image size and cold-start time; use a persistent volume for `AUTOMEM_MODELS_DIR` if supported. Ollama typically requires a **separate service** (Railway does not ship Ollama by default), so you'll need to deploy Ollama elsewhere and set `OLLAMA_BASE_URL` to that service. -- **Self-hosted Docker/VPS:** FastEmbed and Ollama are straightforward and avoid API costs. Ollama benefits from GPU acceleration if available; otherwise expect higher latency on CPU. Ensure the Ollama base URL is reachable from the AutoMem container (`OLLAMA_BASE_URL=http://ollama:11434` in docker compose setups). -- **Dimension consistency:** Regardless of host, make sure `VECTOR_SIZE` matches the embedding model output. Changing models requires re-embedding existing memories. +- **Railway / managed PaaS:** Configure Voyage (`VOYAGE_MODEL=voyage-4`, `VECTOR_SIZE=1024`) for new deployments. OpenAI is the API fallback. Do not use FastEmbed as the default fallback: its 1024d model can add roughly 4 GB RSS and materially increase hosting cost. Ollama requires a separately operated service; Railway does not ship it by default. +- **Self-hosted Docker/VPS:** FastEmbed and Ollama are deliberate local options that avoid embedding API usage. Ollama benefits from GPU acceleration if available; otherwise expect higher CPU latency. Ensure the Ollama base URL is reachable from the AutoMem container (`OLLAMA_BASE_URL=http://ollama:11434` in Docker Compose setups). +- **Model-space consistency:** Regardless of host, make sure `VECTOR_SIZE` matches the embedding model output. Changing a provider or model requires a full re-embed even when both models use the same dimension. ## Optional Variables @@ -146,7 +173,7 @@ VECTOR_SIZE=768 # must match the model's output | `QDRANT_API_KEY` | Qdrant API key | - | `your-qdrant-key` | | `QDRANT_COLLECTION` | Collection name | `memories` | `memories` | | `QDRANT_ENSURE_PAYLOAD_INDEXES` | Create payload indexes on startup for faster filtered search | `true` | set `false` for read-only Qdrant credentials | -| `VECTOR_SIZE` | Embedding dimension | `1024` | `1024` (voyage-4), `3072` (large), `768` (small) | +| `VECTOR_SIZE` | Embedding dimension | `1024` | `1024` (voyage-4 or Ollama bge-m3), `3072` (OpenAI large), `768` (truncated OpenAI small) | | `VECTOR_SIZE_AUTODETECT` | Adopt existing collection dimension instead of failing on mismatch | `true` | `false` to enforce strict matching | 👉 **New to Qdrant?** See the [Qdrant Setup Guide](QDRANT_SETUP.md) for step-by-step instructions on creating a collection with the right settings. @@ -155,9 +182,10 @@ VECTOR_SIZE=768 # must match the model's output **Notes**: - Without Qdrant, AutoMem uses deterministic placeholder embeddings (for testing only). -- **Existing deployments on 3072d or 768d**: `VECTOR_SIZE_AUTODETECT=true` (default) automatically adopts your existing collection dimension on startup. No manual action needed after updating. +- **Existing deployments on 3072d or 768d**: `VECTOR_SIZE_AUTODETECT=true` (default) automatically adopts the existing collection dimension on startup. This avoids a dimension-startup failure; it does **not** migrate vectors to a new model. - To enforce strict matching (fail on mismatch), set `VECTOR_SIZE_AUTODETECT=false`. The server will exit with a clear error message and fix instructions. - When creating a new collection, the configured `VECTOR_SIZE` (default 1024 for voyage-4) is used. +- Changing embedding provider or model always requires a clean collection and a full re-embed, even when the old and new outputs are both 1024d. #### Self-Hosted Qdrant on Railway @@ -191,6 +219,9 @@ When running Qdrant as a Railway service (instead of Qdrant Cloud), set `QDRANT_ ### Scripts Only +See the [scripts catalog](../scripts/README.md) for when and how to run the +operational tools that consume these values. + | Variable | Description | Default | Used By | |----------|-------------|---------|---------| | `AUTOMEM_API_URL` | AutoMem API endpoint | `http://localhost:8001` | `recover_from_qdrant.py`, `health_monitor.py` | @@ -256,8 +287,8 @@ Controls embedding provider and classification model settings. | Variable | Description | Default | Options | |----------|-------------|---------|---------| | `EMBEDDING_MODEL` | OpenAI embedding model (used when provider is `openai`) | `text-embedding-3-small` | `text-embedding-3-small`, `text-embedding-3-large` | -| `VECTOR_SIZE` | Embedding dimension | `1024` | Must match embedding provider (1024=voyage-4, 1536=text-embedding-3-small native; choose ≤1536 when truncating, 3072=text-embedding-3-large native) | -| `VECTOR_SIZE_AUTODETECT` | Adopt existing collection dimension instead of failing on mismatch | `true` | `false` to enforce strict matching | +| `VECTOR_SIZE` | Embedding dimension | `1024` | Must match embedding provider (1024=voyage-4 or Ollama bge-m3; 1536=text-embedding-3-small native; choose ≤1536 when truncating; 3072=text-embedding-3-large native) | +| `VECTOR_SIZE_AUTODETECT` | Adopt existing collection dimension instead of failing on mismatch | `true` | Dimension compatibility only; `false` enforces strict matching | | `CLASSIFICATION_MODEL` | LLM for memory type classification | `gpt-4o-mini` | `gpt-4o-mini`, `gpt-4.1`, `gpt-5.1` | | `CLASSIFICATION_BASE_URL` | Optional OpenAI-compatible endpoint for `scripts/reclassify_with_llm.py` | _(unset → OpenAI)_ | e.g. `https://openrouter.ai/api/v1` | | `CLASSIFICATION_API_KEY` | API key paired with `CLASSIFICATION_BASE_URL` (never falls back across providers) | _(unset)_ | provider key | @@ -267,14 +298,16 @@ Controls embedding provider and classification model settings. | Provider / Model | Dimensions | Cost/1M tokens | Quality | Use Case | |------------------|-----------|----------------|---------|----------| -| `voyage-4` | 1024 | ~$0.05 | Excellent | **Recommended default** | -| `text-embedding-3-small` | 1536 native (truncatable) | $0.02 | Good | OpenAI fallback (truncated to `VECTOR_SIZE` via Matryoshka) | -| `text-embedding-3-large` | 3072 native | $0.13 | Excellent | Maximum precision (legacy default) | +| `voyage-4` | 1024 by default | [Current Voyage pricing](https://docs.voyageai.com/docs/pricing) | Excellent, multilingual | **Recommended default** | +| `text-embedding-3-small` | 1536 native (truncatable) | OpenAI API usage | Good | OpenAI fallback (truncated to `VECTOR_SIZE` via Matryoshka) | +| `text-embedding-3-large` | 3072 native | OpenAI API usage | Excellent | Explicit OpenAI precision option | +| `bge-m3` via Ollama | 1024 | Local hardware | Multilingual | Intentional local/self-hosted deployment | Recommended setup (Voyage): ```bash EMBEDDING_PROVIDER=auto # or voyage VOYAGE_API_KEY=pa-... +VOYAGE_MODEL=voyage-4 VECTOR_SIZE=1024 ``` @@ -286,7 +319,16 @@ EMBEDDING_MODEL=text-embedding-3-small VECTOR_SIZE=768 ``` -**Upgrade safety**: Changing embedding dimensions requires a full re-embed. By default, `VECTOR_SIZE_AUTODETECT=true` adopts the existing Qdrant collection dimension on startup, so updating AutoMem won't break existing data even if the default `VECTOR_SIZE` changes between releases. To enforce strict matching, set `VECTOR_SIZE_AUTODETECT=false` — the server will exit with a clear error and fix instructions if there's a mismatch. +To use local/self-hosted BGE-M3 through Ollama: + +```bash +ollama pull bge-m3 +EMBEDDING_PROVIDER=ollama +OLLAMA_MODEL=bge-m3 +VECTOR_SIZE=1024 +``` + +**Upgrade safety**: Changing an embedding provider, model, or dimension requires a full re-embed. This is true even when both models output 1024 dimensions: the vectors occupy different model spaces. `VECTOR_SIZE_AUTODETECT=true` only adopts the current Qdrant collection dimension to avoid a startup mismatch; it is not a model migration. To enforce strict dimension matching, set `VECTOR_SIZE_AUTODETECT=false`. **Classification Model Pricing (Dec 2025):** | Model | Input | Output | Notes | @@ -414,6 +456,9 @@ These variables are only used by test suites. The `scripts/lab/` directory provides a data-driven framework for testing and optimizing recall scoring. It uses IR metrics (Recall@K, MRR, NDCG) and statistical comparison to evaluate config changes. +For prerequisites, safe clone workflow, and exact commands, see the [scripts +catalog](../scripts/README.md). + **Makefile targets:** | Target | Description | Example | @@ -516,7 +561,7 @@ FALKORDB_HOST = ( ## Re-embedding Memories -When changing `EMBEDDING_MODEL` or `VECTOR_SIZE`, you must re-embed all existing memories: +When changing the embedding provider, model, or `VECTOR_SIZE`, you must re-embed all existing memories. This includes swaps between two 1024d models, such as `voyage-4` and `bge-m3`: equal dimensions do not make their vector spaces compatible. ### 1. Backup First ```bash @@ -524,13 +569,20 @@ python scripts/backup_automem.py ``` ### 2. Set New Environment Variables + +Configure the provider that should create the replacement vectors. For the recommended Voyage configuration: + ```bash -export EMBEDDING_MODEL=text-embedding-3-large # or text-embedding-3-small -export VECTOR_SIZE=3072 # 3072 for large, 768 for small +export EMBEDDING_PROVIDER=voyage +export VOYAGE_API_KEY=pa-... +export VOYAGE_MODEL=voyage-4 +export VECTOR_SIZE=1024 ``` +For OpenAI, set `EMBEDDING_PROVIDER=openai`, `OPENAI_API_KEY`, `EMBEDDING_MODEL`, and its matching `VECTOR_SIZE`. For local/self-hosted multilingual BGE-M3, run `ollama pull bge-m3` and set `EMBEDDING_PROVIDER=ollama`, `OLLAMA_MODEL=bge-m3`, and `VECTOR_SIZE=1024`. + ### 3. Recreate Qdrant Collection -The collection must be recreated with the new dimension: +Pause memory writes, then recreate the collection before re-embedding. This prevents old and new model spaces from being mixed. The re-embed script upserts vectors; it does **not** delete or create the Qdrant collection for you. ```bash # Delete old collection @@ -541,22 +593,24 @@ curl -X DELETE "$QDRANT_URL/collections/memories" \ curl -X PUT "$QDRANT_URL/collections/memories" \ -H "api-key: $QDRANT_API_KEY" \ -H "Content-Type: application/json" \ - -d '{"vectors": {"size": 3072, "distance": "Cosine"}}' # Use 768 for small + -d '{"vectors": {"size": 1024, "distance": "Cosine"}}' ``` +Replace `memories` with your `QDRANT_COLLECTION` name and `1024` with the selected model's output dimension. + ### 4. Re-embed All Memories ```bash python scripts/reembed_embeddings.py --batch-size 32 ``` -The script reads from FalkorDB (which retains all memory data) and re-creates the vector embeddings in Qdrant. +The script reads from FalkorDB (which retains all memory data) and re-creates the vector embeddings in Qdrant. It requires `QDRANT_URL`; unlike the API service, it does not construct a URL from `QDRANT_HOST` and `QDRANT_PORT`. -**Cost estimate:** ~$0.15 for 6000 memories with large embeddings, ~$0.02 with small. +After the full run, verify the service health and run a representative recall. `VECTOR_SIZE_AUTODETECT` is not a substitute for this procedure: it only accepts an existing collection dimension and never converts old vectors to a new model space. --- ## See Also - [Railway Deployment Guide](./RAILWAY_DEPLOYMENT.md) -- [Deployment Checklist](./DEPLOYMENT_CHECKLIST.md) +- [Post-Deploy Checklist](./RAILWAY_DEPLOYMENT.md#post-deploy-checklist) - [Installation Guide](../INSTALLATION.md) diff --git a/docs/HEALTH_MONITORING.md b/docs/HEALTH_MONITORING.md index ec142a5..466ed21 100644 --- a/docs/HEALTH_MONITORING.md +++ b/docs/HEALTH_MONITORING.md @@ -2,6 +2,9 @@ AutoMem includes a built-in health monitoring system that watches for data inconsistencies and optionally triggers automatic recovery. +For the current command catalog and safety notes for `health_monitor.py` and +`recover_from_qdrant.py`, see [scripts/README.md](../scripts/README.md). + ## Quick Start ### Alert-Only Mode (Recommended) @@ -386,4 +389,4 @@ python scripts/health_monitor.py --once | grep drift_percent - [Recovery Script Documentation](../scripts/recover_from_qdrant.py) - [Environment Variables](./ENVIRONMENT_VARIABLES.md) - [Railway Deployment](./RAILWAY_DEPLOYMENT.md) -- [Deployment Checklist](./DEPLOYMENT_CHECKLIST.md) +- [Post-Deploy Checklist](./RAILWAY_DEPLOYMENT.md#post-deploy-checklist) diff --git a/docs/MIGRATIONS.md b/docs/MIGRATIONS.md index 199c1dd..23acd61 100644 --- a/docs/MIGRATIONS.md +++ b/docs/MIGRATIONS.md @@ -2,11 +2,11 @@ This document provides step-by-step instructions for migrating between different AutoMem configurations. -**Heads up for existing deployments:** The default embedding dimension is now **1024d** (voyage-4) for new installs. If your Qdrant collection uses a different dimension (e.g. 3072d from `text-embedding-3-large` or 768d from `text-embedding-3-small`), **no action is needed** — `VECTOR_SIZE_AUTODETECT=true` (the default) automatically adopts your existing collection dimension on startup. To explicitly pin your dimension, set `VECTOR_SIZE=` in your `.env`. To enforce strict matching (fail on mismatch), set `VECTOR_SIZE_AUTODETECT=false`. +**Heads up for existing deployments:** New installs default to **1024d** with Voyage (`voyage-4`). If you are only updating AutoMem and keeping the same embedding provider and model, `VECTOR_SIZE_AUTODETECT=true` (the default) can adopt an existing collection dimension to avoid a startup mismatch. It does **not** migrate embeddings. Any provider or model change requires backing up, recreating the Qdrant collection, and fully re-embedding every memory—even when both models output 1024 dimensions. To explicitly pin a dimension, set `VECTOR_SIZE=`; set `VECTOR_SIZE_AUTODETECT=false` to fail on mismatch. ## Table of Contents -**Embedding dimension migrations** +**Embedding provider and model migrations** - [Migrating to 1024d (voyage-4 default)](#migrating-to-1024d-voyage-4-default) - [Upgrading to 3072d Embeddings](#upgrading-to-3072d-embeddings) - [Downgrading to 768d Embeddings](#downgrading-to-768d-embeddings) @@ -23,6 +23,8 @@ This document provides step-by-step instructions for migrating between different > The embedding-dimension sections are about re-vectorizing; the data & schema > sections are one-time, idempotent FalkorDB/Qdrant migrations. +> **Re-embed prerequisite:** `scripts/reembed_embeddings.py` requires a full `QDRANT_URL` (for example, `http://localhost:6333`); unlike the API service, the script does not construct one from `QDRANT_HOST` and `QDRANT_PORT`. + --- ## Migrating to 1024d (voyage-4 default) @@ -36,45 +38,47 @@ This document provides step-by-step instructions for migrating between different ```bash EMBEDDING_PROVIDER=voyage # or auto (will prefer Voyage if VOYAGE_API_KEY is set) VOYAGE_API_KEY=pa-... + VOYAGE_MODEL=voyage-4 VECTOR_SIZE=1024 + QDRANT_URL=http://localhost:6333 # required by scripts/reembed_embeddings.py ``` -3. **Delete and recreate the Qdrant collection**: +3. **Pause writes, then delete and recreate the Qdrant collection**: ```bash curl -X DELETE http://localhost:6333/collections/memories + curl -X PUT http://localhost:6333/collections/memories \ + -H 'Content-Type: application/json' \ + -d '{"vectors": {"size": 1024, "distance": "Cosine"}}' ``` 4. **Re-embed all memories**: ```bash - python scripts/reembed_embeddings.py + python scripts/reembed_embeddings.py --batch-size 32 ``` 5. **Verify**: Check that `/health` shows `vector_size: 1024` and recall returns results. -> **Alternatively**, set `VECTOR_SIZE_AUTODETECT=true` (the default) and AutoMem will adopt your existing collection dimension without migration. Only migrate when you want to switch embedding providers. +> `VECTOR_SIZE_AUTODETECT=true` can preserve the old collection dimension only when you keep its provider and model. It never makes an existing OpenAI, Voyage, Ollama, or FastEmbed vector compatible with another model space. --- ## Upgrading to 3072d Embeddings -**When to upgrade:** If you need better semantic precision and have the storage budget for 4x larger embeddings. +**When to upgrade:** If an evaluation shows that you need the explicit OpenAI `text-embedding-3-large` option and can accept its extra storage and API usage. This is not the recommended cloud default; new deployments should use Voyage `voyage-4` at 1024d. ### Pros ✅ -- **Better semantic precision**: ~5-10% improvement on benchmarks -- **Improved multi-hop reasoning**: Better at connecting related concepts -- **Recommended for production**: If accuracy is critical and storage is not a constraint +- **Higher-dimensional OpenAI option**: Native 3072d output +- **Explicit control**: Useful when an OpenAI-specific evaluation requires it ### Cons ❌ - **4x storage cost**: 768 → 3072 dimensions (4x more disk space) -- **4x embedding cost**: OpenAI charges per dimension -- **~20% slower search**: More dimensions = more computation +- **More API usage cost**: Review current provider pricing before migration +- **More search work**: Larger vectors increase storage and computation - **Migration required**: Cannot reuse existing embeddings -### Cost Comparison +### Storage Impact | Metric | 768d (small) | 3072d (large) | Multiplier | |--------|--------------|---------------|------------| -| Storage per 1M memories | ~3GB | ~12GB | 4x | -| OpenAI cost per 1M tokens | $0.02 | $0.13 | 6.5x | -| Search latency | ~50ms | ~60ms | 1.2x | -| Benchmark accuracy | 88.2% | 90.5% | +2.3pp | +| Vector dimensions | 768 | 3072 | 4x | +| Vector storage (all else equal) | Baseline | Approximately 4x | 4x | ### Migration Steps @@ -90,30 +94,44 @@ This creates timestamped backups in `backups/`: #### 2. Update Configuration ```bash # Add to your .env file +echo "EMBEDDING_PROVIDER=openai" >> .env echo "VECTOR_SIZE=3072" >> .env echo "EMBEDDING_MODEL=text-embedding-3-large" >> .env +echo "QDRANT_URL=http://localhost:6333" >> .env ``` Or export temporarily: ```bash +export EMBEDDING_PROVIDER=openai export VECTOR_SIZE=3072 export EMBEDDING_MODEL=text-embedding-3-large +export QDRANT_URL=http://localhost:6333 ``` -#### 3. Re-embed All Memories +#### 3. Pause Writes and Recreate the Qdrant Collection + +`reembed_embeddings.py` upserts into an existing collection; it does not recreate one. After the backup, stop or pause writes and recreate the collection for the new model space: + ```bash -python scripts/reembed_embeddings.py +curl -X DELETE http://localhost:6333/collections/memories +curl -X PUT http://localhost:6333/collections/memories \ + -H 'Content-Type: application/json' \ + -d '{"vectors": {"size": 3072, "distance": "Cosine"}}' +``` + +#### 4. Re-embed All Memories +```bash +python scripts/reembed_embeddings.py --batch-size 32 ``` This will: - Fetch all memories from FalkorDB (source of truth) - Generate new 3072d embeddings using OpenAI API -- Recreate Qdrant collection with new dimensions - Upsert all embeddings in batches **Expected time:** ~5-10 minutes per 10k memories -#### 4. Verify Migration +#### 5. Verify Migration Check Qdrant collection info: ```bash curl http://localhost:6333/collections/memories | jq '.result.config.params.vectors' @@ -127,7 +145,7 @@ Should show: } ``` -#### 5. Test Recall +#### 6. Test Recall ```bash curl -X POST http://localhost:8001/recall \ -H "Authorization: Bearer $AUTOMEM_API_TOKEN" \ @@ -137,7 +155,7 @@ curl -X POST http://localhost:8001/recall \ Verify results are returned and scores look reasonable. -#### 6. Restart Application +#### 7. Restart Application ```bash # If using Docker docker compose up -d @@ -164,6 +182,7 @@ python scripts/restore_from_backup.py backups/qdrant/qdrant_snapshot_YYYYMMDD_HH python scripts/restore_from_backup.py backups/falkordb/memories_YYYYMMDD_HHMMSS.rdb # 3. Revert configuration +export EMBEDDING_PROVIDER=openai export VECTOR_SIZE=768 export EMBEDDING_MODEL=text-embedding-3-small @@ -181,11 +200,18 @@ docker compose up -d Follow the same migration steps above, but use: ```bash +export EMBEDDING_PROVIDER=openai export VECTOR_SIZE=768 export EMBEDDING_MODEL=text-embedding-3-small ``` -Then run `reembed_embeddings.py` to recreate the collection with 768d vectors. +After backing up and pausing writes, recreate the collection with a 768d vector schema, then run: + +```bash +python scripts/reembed_embeddings.py --batch-size 32 +``` + +The script upserts the replacement vectors; it does not recreate the collection. A full re-embed is required even if a future source model happens to share the same dimension. --- @@ -214,8 +240,8 @@ python scripts/migrate_entity_nodes.py # apply (idempotent) > Promote nodes when you opt into that feature. If your clone has noisy generated > entity tags, clean them first with > [`scripts/lab/repair_entity_tags.py`](../scripts/lab/repair_entity_tags.py) -> (`--mode audit` → review the plan → `--mode execute`, with `--mode rollback` -> available). +> (`--mode canonicalize-safe` → review the generated `plan.jsonl` → +> `--execute --plan `, with `--rollback ` available). ### Backfill tag prefixes @@ -285,9 +311,9 @@ FATAL: Vector dimension mismatch detected! ``` **Solution** (pick one): -1. Set `VECTOR_SIZE_AUTODETECT=true` (default) to automatically adopt the existing collection dimension -2. Set `VECTOR_SIZE=` in your `.env` to match your data -3. Migrate to the new dimension: follow the [1024d](#migrating-to-1024d-voyage-4-default), [3072d](#upgrading-to-3072d-embeddings), or [768d](#downgrading-to-768d-embeddings) migration steps above +1. If you are keeping the same provider and model, set `VECTOR_SIZE_AUTODETECT=true` (default) to adopt the existing collection dimension +2. If you are keeping the same provider and model, set `VECTOR_SIZE=` in `.env` +3. If you are changing the provider or model, follow the [1024d](#migrating-to-1024d-voyage-4-default), [3072d](#upgrading-to-3072d-embeddings), or [768d](#downgrading-to-768d-embeddings) procedure: back up, recreate the collection, and fully re-embed ### Error: "OpenAI API rate limit" @@ -297,10 +323,10 @@ Rate limit exceeded during re-embedding ``` **Solution:** -The `reembed_embeddings.py` script uses whatever embedding provider is configured (Voyage, OpenAI, local, etc.). For large datasets: +The `reembed_embeddings.py` script uses the configured provider (Voyage, OpenAI, Ollama, FastEmbed, and so on). For large datasets: 1. Run during off-peak hours 2. Increase your provider's rate limits if applicable -3. Split migration into batches using `--batch-size` flag +3. Split work into smaller embedding requests with `--batch-size` ### Error: "Qdrant collection already exists" @@ -310,29 +336,31 @@ Collection 'memories' already exists with different dimension ``` **Solution:** -Delete and recreate: +Back up, pause writes, then delete **and recreate** the collection with the selected model's dimension before re-embedding. The script only upserts vectors: ```bash curl -X DELETE http://localhost:6333/collections/memories -python scripts/reembed_embeddings.py +curl -X PUT http://localhost:6333/collections/memories \ + -H 'Content-Type: application/json' \ + -d '{"vectors": {"size": 1024, "distance": "Cosine"}}' +python scripts/reembed_embeddings.py --batch-size 32 ``` -**⚠️ Warning:** This deletes all embeddings. Make sure FalkorDB still has the memories (embeddings will be regenerated from there). +**⚠️ Warning:** Replace `1024` with the selected model's output size. This deletes all embeddings; make sure FalkorDB still has the memories so they can be regenerated. ### Migration is slow **Symptoms:** - Taking hours for thousands of memories -- High OpenAI API costs +- High embedding-provider costs **Solutions:** -1. **Check batch size**: Script uses batches of 100 by default -2. **Parallel processing**: Use `--workers` flag (if implemented) -3. **Spot check first**: Test on a subset before full migration -4. **Use cheaper model for testing**: +1. **Check batch size**: Script defaults to batches of 32; lower `--batch-size` if the provider or host is constrained +2. **Spot check first**: On a newly recreated collection, test a subset before the full run: ```bash - export EMBEDDING_MODEL=text-embedding-3-small - python scripts/reembed_embeddings.py --dry-run + python scripts/reembed_embeddings.py --limit 100 --batch-size 32 ``` + Then rerun without `--limit` to migrate every memory. +3. **Use a non-production provider configuration for experiments** and keep Voyage (`voyage-4`) as the cloud default for the final migration. See [Voyage pricing](https://docs.voyageai.com/docs/pricing) for current limits and rates. ### Backup failed @@ -359,8 +387,9 @@ python scripts/reembed_embeddings.py ### Before Any Migration 1. ✅ **Always backup first** - Don't skip this step 2. ✅ **Test in staging** - If you have a staging environment -3. ✅ **Monitor costs** - Check OpenAI usage dashboard during migration -4. ✅ **Document current state** - Note current VECTOR_SIZE and EMBEDDING_MODEL +3. ✅ **Pause writes and recreate the collection** - Required for every provider/model change, including same-dimension swaps +4. ✅ **Monitor costs** - Check the selected provider's usage dashboard during migration +5. ✅ **Document current state** - Note the current provider, model, and `VECTOR_SIZE` ### After Migration 1. ✅ **Run benchmark tests** - Verify accuracy hasn't degraded diff --git a/docs/MONITORING_AND_BACKUPS.md b/docs/MONITORING_AND_BACKUPS.md index 06c42be..cf1281b 100644 --- a/docs/MONITORING_AND_BACKUPS.md +++ b/docs/MONITORING_AND_BACKUPS.md @@ -2,6 +2,9 @@ Complete guide to setting up automated health monitoring and backups for AutoMem on Railway. +For the current script inventory, safety classification, and exact CLI usage, +see the [scripts catalog](../scripts/README.md). + ## Overview AutoMem includes three layers of data protection: @@ -60,7 +63,7 @@ Create a new Railway service for continuous monitoring: ```bash # In Railway dashboard 1. Create new service from GitHub repo -2. Set Dockerfile path: scripts/Dockerfile.health-monitor (we'll create this) +2. Set Dockerfile path: `scripts/Dockerfile.health-monitor` (included in this repository) 3. Configure environment variables (same as main service) 4. Deploy ``` @@ -353,29 +356,14 @@ redis-cli -h monorail.proxy.rlwy.net -p 12345 -a YOUR_PASSWORD ping --- -**Advanced: Railway Backup Service** - -For Railway Pro users who want backups running on Railway: - -⚠️ **Note:** Railway's UI makes Dockerfile configuration complex. This method is for advanced users. - -The `scripts/Dockerfile.backup` exists and runs backups every 6 hours in a loop. However, deploying it requires CLI: - -```bash -cd /path/to/automem -railway link -railway up --service backup-service -``` - -Then configure in Railway dashboard: - -- Set Builder to Dockerfile -- Dockerfile Path: `scripts/Dockerfile.backup` -- Add environment variables (same as the AutoMem API service) - -**Cost:** ~$1-2/month +**Railway-hosted backup schedules** -**Recommendation:** Use GitHub Actions instead unless you have specific requirements for Railway-hosted backups. +AutoMem does not ship a standalone backup container recipe; the old +documentation reference was stale. Use the included [GitHub Actions backup +workflow](../.github/workflows/backup.yml), or configure your platform's +scheduler to run the documented `backup_automem.py` command with the required +database credentials and off-site destination. The GitHub Actions workflow is +the supported repository example. --- diff --git a/docs/QDRANT_SETUP.md b/docs/QDRANT_SETUP.md index 1e8170f..9e04be6 100644 --- a/docs/QDRANT_SETUP.md +++ b/docs/QDRANT_SETUP.md @@ -123,9 +123,9 @@ AutoMem uses dense text embeddings (Voyage/OpenAI/etc.) for semantic search. Key *Click image to view full size* -> **Using a smaller model?** If you set `EMBEDDING_MODEL=text-embedding-3-small`, use `768` dimensions instead and set `VECTOR_SIZE=768` in AutoMem. +> **Using OpenAI small?** If you set `EMBEDDING_PROVIDER=openai` and `EMBEDDING_MODEL=text-embedding-3-small`, use `768` dimensions and set `VECTOR_SIZE=768` in AutoMem. > -> **Using FastEmbed or Ollama?** Set dimensions to match your local model output (e.g., 384/768/1024 for FastEmbed). Ollama model dimensions vary—verify with a test embedding and set `VECTOR_SIZE` accordingly. +> **Using local/self-hosted BGE-M3?** Run `ollama pull bge-m3`, set `EMBEDDING_PROVIDER=ollama`, `OLLAMA_MODEL=bge-m3`, and `VECTOR_SIZE=1024`. Other Ollama models and local FastEmbed models must use their own output dimension. FastEmbed is local/self-hosted only, not a Railway default. #### Payload Indexes (Recommended) @@ -159,6 +159,12 @@ Add these to your AutoMem environment variables: QDRANT_URL="https://xxxxx-xxxxx.aws.cloud.qdrant.io" QDRANT_API_KEY="your-api-key-here" QDRANT_COLLECTION="memories" # Only if using custom name + +# Recommended cloud embeddings +EMBEDDING_PROVIDER=auto +VOYAGE_API_KEY="pa-..." +VOYAGE_MODEL="voyage-4" +VECTOR_SIZE=1024 ``` **Railway**: Add these in `AutoMem` → Variables, then redeploy. @@ -198,17 +204,18 @@ If `qdrant` shows `"disconnected"` or `"not configured"`: | Provider / Model | Dimensions | Cost | Quality | |------------------|------------|------|---------| -| `voyage-4` (recommended) | 1024 | ~$0.05/1M tokens | Excellent for short text | -| `text-embedding-3-small` | 1536 native (truncatable) | $0.02/1M tokens | Good OpenAI fallback | -| `text-embedding-3-large` | 3072 native (truncatable) | $0.13/1M tokens | Maximum precision | +| `voyage-4` (recommended) | 1024 | [Current Voyage pricing](https://docs.voyageai.com/docs/pricing) | Excellent, multilingual | +| `text-embedding-3-small` | 1536 native (truncatable) | OpenAI API usage | Good OpenAI fallback | +| `text-embedding-3-large` | 3072 native (truncatable) | OpenAI API usage | Explicit precision option | +| `bge-m3` via Ollama | 1024 | Local hardware | Multilingual, self-hosted | **To switch providers**: 1. Set `EMBEDDING_PROVIDER` and any required API key 2. Set `VECTOR_SIZE` to match the provider's output dimension -3. Create a new Qdrant collection with matching dimensions (or use `VECTOR_SIZE_AUTODETECT=true`) -4. Redeploy AutoMem +3. Back up, pause writes, and recreate the Qdrant collection with matching dimensions +4. Re-embed all memories through the selected provider, then redeploy AutoMem -> ⚠️ **Warning**: Changing embedding models requires re-embedding all existing memories. See [MIGRATIONS.md](MIGRATIONS.md) for the reembed script. +> ⚠️ **Warning**: Changing embedding providers or models requires a clean collection and a full re-embed, even if both models output the same dimension. `VECTOR_SIZE_AUTODETECT` only accepts an existing dimension; it does not migrate model spaces. See [MIGRATIONS.md](MIGRATIONS.md) for the complete procedure. ### Custom Collection Names @@ -265,6 +272,7 @@ AutoMem expects dimensions to match `VECTOR_SIZE`: - `voyage-4` → `VECTOR_SIZE=1024` (default) - `text-embedding-3-small` → `VECTOR_SIZE` ≤ 1536 (default: 768; auto-upgrades to `text-embedding-3-large` if exceeded) - `text-embedding-3-large` → `VECTOR_SIZE` ≤ 3072 (truncatable via Matryoshka) +- `bge-m3` via Ollama → `VECTOR_SIZE=1024` If you created the collection with wrong dimensions: 1. Delete the collection in Qdrant dashboard @@ -273,7 +281,7 @@ If you created the collection with wrong dimensions: ### Memories not appearing in semantic search -1. **Check `OPENAI_API_KEY`**: Required for generating embeddings +1. **Check provider configuration**: Voyage is recommended for cloud (`VOYAGE_API_KEY`, `VOYAGE_MODEL=voyage-4`); OpenAI is the API fallback; Ollama must be explicitly configured and reachable 2. **Verify embeddings exist**: Check `/health` shows `qdrant: connected` 3. **Wait for enrichment**: New memories are embedded async (few seconds) diff --git a/docs/RAILWAY_DEPLOYMENT.md b/docs/RAILWAY_DEPLOYMENT.md index 3478086..f30cd99 100644 --- a/docs/RAILWAY_DEPLOYMENT.md +++ b/docs/RAILWAY_DEPLOYMENT.md @@ -1,5 +1,8 @@ # Railway Deployment Guide +For operational scripts used after deployment (health checks, backups, recovery, +and migrations), see the [scripts catalog](../scripts/README.md). + Complete guide to deploying AutoMem on Railway with persistent storage, backups, and zero data loss. ## Quick Start (One-Click Deploy) @@ -49,7 +52,8 @@ flowchart TB subgraph external [External Services] QdrantCloud[(Qdrant Cloud
Optional managed alternative)] - OpenAI[OpenAI API
Embeddings] + Voyage[Voyage API
Recommended embeddings] + OpenAI[OpenAI API
Fallback embeddings] end end @@ -65,7 +69,8 @@ flowchart TB FlaskAPI -->|Internal
falkordb.railway.internal:6379| FalkorDB FlaskAPI -->|Internal
qdrant.railway.internal:6333| Qdrant FlaskAPI -.->|Optional alternative| QdrantCloud - FlaskAPI --> OpenAI + FlaskAPI --> Voyage + FlaskAPI -.-> OpenAI Enrichment --> FalkorDB Enrichment -.-> QdrantCloud @@ -143,12 +148,13 @@ After deploying, complete these steps to fully configure AutoMem: | Variable | Required | How to Get | | ---------------- | ----------- | -------------------------------------------------------------------- | -| `OPENAI_API_KEY` | Yes\* | [platform.openai.com/api-keys](https://platform.openai.com/api-keys) | +| `VOYAGE_API_KEY` | Recommended | [Voyage API key](https://dash.voyageai.com/) for `voyage-4` | +| `OPENAI_API_KEY` | Optional fallback | [platform.openai.com/api-keys](https://platform.openai.com/api-keys) | | `QDRANT_URL` | Option A | Qdrant Cloud URL — see [Qdrant Setup Guide](QDRANT_SETUP.md) | | `QDRANT_API_KEY` | Option A | Qdrant Cloud API key | | `QDRANT_HOST` | Option B | `qdrant` (self-hosted on Railway — see [Step 1b](#step-1b-qdrant-vector-database)) | -\*Without `OPENAI_API_KEY`, semantic search won't work (embeddings skipped). +Set `EMBEDDING_PROVIDER=auto`, `VOYAGE_MODEL=voyage-4`, and `VECTOR_SIZE=1024` with the Voyage key. In auto mode, OpenAI is the API fallback. Do not rely on FastEmbed for Railway: its 1024d fallback can add roughly 4 GB RSS and materially increase the service cost. See [Voyage pricing](https://docs.voyageai.com/docs/pricing) for the current free-tier limits and rates. 👉 **Qdrant Cloud?** Follow the [Qdrant Setup Guide](QDRANT_SETUP.md) for step-by-step collection setup. 👉 **Self-hosted Qdrant?** See [Step 1b](#step-1b-qdrant-vector-database) — remember to set `QDRANT__SERVICE__HOST=::` on the Qdrant service. @@ -316,8 +322,13 @@ Run Qdrant inside your Railway project for lower latency (internal networking, n AUTOMEM_API_TOKEN=${{shared.AUTOMEM_API_TOKEN}} ADMIN_API_TOKEN=${{shared.ADMIN_API_TOKEN}} - # OpenAI for embeddings (required for semantic search) - OPENAI_API_KEY= + # Recommended cloud embeddings: Voyage first, OpenAI fallback + EMBEDDING_PROVIDER=auto + VOYAGE_API_KEY= + VOYAGE_MODEL=voyage-4 + VECTOR_SIZE=1024 + # Optional: OpenAI-compatible API fallback when Voyage is unavailable + # OPENAI_API_KEY= # Vector search — pick ONE: # Option A: Qdrant Cloud @@ -344,8 +355,13 @@ Run Qdrant inside your Railway project for lower latency (internal networking, n AUTOMEM_API_TOKEN= ADMIN_API_TOKEN= - # OpenAI for embeddings - OPENAI_API_KEY= + # Recommended cloud embeddings: Voyage first, OpenAI fallback + EMBEDDING_PROVIDER=auto + VOYAGE_API_KEY= + VOYAGE_MODEL=voyage-4 + VECTOR_SIZE=1024 + # Optional: OpenAI-compatible API fallback when Voyage is unavailable + # OPENAI_API_KEY= # Vector search — pick ONE: # Option A: Qdrant Cloud diff --git a/docs/superpowers/plans/2026-08-28-qdrant-on-disk-storage.md b/docs/superpowers/plans/2026-08-28-qdrant-on-disk-storage.md deleted file mode 100644 index b0470f5..0000000 --- a/docs/superpowers/plans/2026-08-28-qdrant-on-disk-storage.md +++ /dev/null @@ -1,213 +0,0 @@ -# Qdrant On-Disk Storage Implementation Plan - -> For agentic workers: REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task by task. - -**Goal:** Make every newly created AutoMem Qdrant collection disk-backed for vectors, HNSW, and payloads, and provide an explicit, documented Qdrant-only restore path for migrating existing collections. - -**Architecture:** Collection creation is the only runtime configuration mutation: it supplies Qdrant's \`on_disk\` options when the collection does not already exist. Existing collections are untouched. The backup restore tool accepts an explicit HNSW disk-storage environment variable so operators can rebuild a selected Qdrant collection from a portable backup; docs make that destructive operation, verification, and manual Railway volume resizing explicit. - -**Tech Stack:** Python 3.12, qdrant-client, Flask, pytest, Bash, Markdown. - -## Global Constraints - -- Preserve the configuration of any existing Qdrant collection; never update it at API startup. -- Keep production migration opt-in and operator-driven. Do not automate a Railway volume resize. -- Keep the restore script's existing explicit \`--force\` confirmation model and its \`--qdrant-only\` scope. -- Treat a Qdrant backup as sensitive corpus data; the runbook must use environment variables for credentials and avoid embedding secrets. - ---- - -### Task 1: Set safe on-disk defaults for newly created collections - -**Files:** -- Modify: \`automem/stores/runtime_clients.py\` -- Modify: \`automem/service_runtime_bindings.py\` -- Modify: \`app.py\` -- Modify: \`tests/test_vector_size_safety.py\` - -**Step 1: Add the failing collection-creation regression test.** - -In \`TestEnsureQdrantCollectionPayloadIndexes\`, add a test for a missing collection. Call \`ensure_qdrant_collection\` with the Qdrant model shims, assert one \`create_collection\` call, and verify that: - -- \`vectors_config\` uses the resolved embedding dimension and \`Distance.COSINE\`; -- \`vectors_config.on_disk is True\`; -- \`hnsw_config.on_disk is True\`; and -- \`on_disk_payload is True\`. - -Also assert the existing payload indexes are still created after the collection is created. Keep the pre-existing collection test proving that initialization does not recreate or reconfigure an existing collection. - -**Step 2: Run the focused test to confirm the new assertion fails.** - -Run: - -\`\`\`bash -PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 /Users/jgarturo/Projects/OpenAI/automem/.venv/bin/pytest -q tests/test_vector_size_safety.py -k on_disk -\`\`\` - -Expected: failure because the current collection factory does not provide all three on-disk settings. - -**Step 3: Pass \`HnswConfigDiff\` through the composition root.** - -Import \`HnswConfigDiff\` together with the other Qdrant models in \`app.py\`, including both optional-import fallback branches. Add an \`hnsw_config_diff_cls\` parameter to \`create_service_runtime\`, pass it to \`ensure_qdrant_collection\`, and wire \`HnswConfigDiff\` from the \`app.py\` service runtime construction. Preserve the application's optional-dependency startup behavior. - -**Step 4: Configure only newly created collections.** - -Extend \`ensure_qdrant_collection\` to accept the HNSW configuration class. In its existing missing-collection branch, create the collection with: - -\`\`\`python -vectors_config=VectorParams(size=effective_dim, distance=Distance.COSINE, on_disk=True) -hnsw_config=HnswConfigDiff(on_disk=True) -on_disk_payload=True -\`\`\` - -Do not alter the early return for an existing collection or the vector-size mismatch handling. - -**Step 5: Run focused tests.** - -Run: - -\`\`\`bash -PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 /Users/jgarturo/Projects/OpenAI/automem/.venv/bin/pytest -q tests/test_vector_size_safety.py -\`\`\` - -Expected: all vector-size safety tests pass, including the new missing-collection regression case. - -**Step 6: Commit the runtime change.** - -\`\`\`bash -git add app.py automem/stores/runtime_clients.py automem/service_runtime_bindings.py tests/test_vector_size_safety.py -git commit -m "perf(qdrant): store new collection data on disk" -\`\`\` - ---- - -### Task 2: Support explicit disk-backed HNSW during Qdrant restores - -**Files:** -- Modify: \`scripts/restore_from_backup.py\` -- Modify: \`scripts/lab/clone_production.sh\` -- Modify: \`tests/test_backup_endpoint.py\` - -**Step 1: Extend the restore configuration test first.** - -In \`test_restore_qdrant_uses_optional_collection_tuning\`, monkeypatch a new \`QDRANT_RESTORE_HNSW_ON_DISK\` setting to \`True\`. Assert the \`HnswConfigDiff\` used for \`create_collection\` contains both the existing \`m=0\` option and \`on_disk is True\`. Add a focused lab-clone-script assertion for a \`QDRANT_RESTORE_HNSW_ON_DISK\` default of \`true\`. - -**Step 2: Run the targeted test to confirm it fails.** - -Run: - -\`\`\`bash -PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 /Users/jgarturo/Projects/OpenAI/automem/.venv/bin/pytest -q tests/test_backup_endpoint.py -k "optional_collection_tuning or lab_clone_uses_paced_qdrant_restore_defaults" -\`\`\` - -Expected: failure because the restore module and local-clone wrapper do not yet expose the HNSW disk setting. - -**Step 3: Add the optional restore environment setting.** - -Define \`QDRANT_RESTORE_HNSW_ON_DISK = _optional_bool_env("QDRANT_RESTORE_HNSW_ON_DISK")\` next to the other restore collection tuning values. Refactor \`_qdrant_collection_kwargs\` to build an HNSW keyword dictionary, adding \`m\` when configured and \`on_disk\` when configured, and instantiate \`HnswConfigDiff(**hnsw_kwargs)\` only when at least one setting is supplied. This must preserve \`m=0\` as an intentional value. - -**Step 4: Set the local production-clone default.** - -Pass \`QDRANT_RESTORE_HNSW_ON_DISK="\${QDRANT_RESTORE_HNSW_ON_DISK:-true}"\` in \`scripts/lab/clone_production.sh\`, adjacent to the existing HNSW/vector/payload restore settings. This makes cloned production data match the new disk-backed default without affecting a running production collection. - -**Step 5: Run the focused restore tests.** - -Run: - -\`\`\`bash -PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 /Users/jgarturo/Projects/OpenAI/automem/.venv/bin/pytest -q tests/test_backup_endpoint.py -\`\`\` - -Expected: all backup/restore tests pass, including the optional HNSW configuration and local clone wrapper checks. - -**Step 6: Commit the migration-tool change.** - -\`\`\`bash -git add scripts/restore_from_backup.py scripts/lab/clone_production.sh tests/test_backup_endpoint.py -git commit -m "perf(qdrant): support disk-backed HNSW restores" -\`\`\` - ---- - -### Task 3: Document the explicit production migration and operational controls - -**Files:** -- Modify: \`docs/ENVIRONMENT_VARIABLES.md\` -- Modify: \`docs/MONITORING_AND_BACKUPS.md\` - -**Step 1: Document the restore variable.** - -Add \`QDRANT_RESTORE_HNSW_ON_DISK\` to the existing Qdrant restore tuning documentation in \`docs/ENVIRONMENT_VARIABLES.md\`. Describe it as an optional boolean that places the restored collection's HNSW index on disk; leave it unset by default so direct restore users must opt in. - -**Step 2: Add a Qdrant-only migration runbook.** - -Under the API backup export section in \`docs/MONITORING_AND_BACKUPS.md\`, add a concise “Migrate an existing Qdrant collection to on-disk storage” subsection that: - -1. takes a \`?include=qdrant\` portable backup with the admin token; -2. records the pre-migration point count and takes a recoverable Railway volume backup/snapshot; -3. runs \`restore_from_backup.py\` against the intended Qdrant endpoint with \`--qdrant-only --force\` plus \`QDRANT_RESTORE_VECTOR_ON_DISK=true\`, \`QDRANT_RESTORE_HNSW_ON_DISK=true\`, and \`QDRANT_RESTORE_ON_DISK_PAYLOAD=true\`; -4. verifies point count and representative recall before calling the change complete; and -5. directs the operator to resize the Railway volume manually only after validation and a stable observation period. - -State clearly that the restore deletes and recreates the selected Qdrant collection and that a tested backup is the rollback path. Do not document an automated Railway resize command. - -**Step 3: Review documentation for credential and safety clarity.** - -Run: - -\`\`\`bash -rg -n "QDRANT_RESTORE_HNSW_ON_DISK|on-disk storage|qdrant-only|Railway volume" docs/ENVIRONMENT_VARIABLES.md docs/MONITORING_AND_BACKUPS.md -\`\`\` - -Expected: the variable, destructive restore scope, validation requirements, and manual-resize boundary are each discoverable. - -**Step 4: Commit the docs change.** - -\`\`\`bash -git add docs/ENVIRONMENT_VARIABLES.md docs/MONITORING_AND_BACKUPS.md -git commit -m "docs(qdrant): add on-disk migration runbook" -\`\`\` - ---- - -### Task 4: Verify the integrated change set - -**Files:** -- Verify: all files changed by Tasks 1–3 - -**Step 1: Run formatting and lint checks.** - -Run: - -\`\`\`bash -make fmt -make lint -\`\`\` - -Expected: Black/Isort produce no unintended changes and Flake8 passes. - -**Step 2: Run the full unit suite.** - -Run: - -\`\`\`bash -PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 /Users/jgarturo/Projects/OpenAI/automem/.venv/bin/pytest -q -m unit -\`\`\` - -Expected: full unit suite passes. - -**Step 3: Inspect the final diff and status.** - -Run: - -\`\`\`bash -git diff develop...HEAD --check -git status --short -git log --oneline develop..HEAD -\`\`\` - -Expected: no whitespace errors, only intended files changed, and focused conventional commits are present. - -**Step 4: Report the handoff boundary.** - -State that the code provides defaults for new collections and an explicit migration mechanism for existing collections, but production migration and any Railway volume resize remain a manual operator action after backup and verification. diff --git a/docs/superpowers/specs/2026-08-28-qdrant-on-disk-storage-design.md b/docs/superpowers/specs/2026-08-28-qdrant-on-disk-storage-design.md deleted file mode 100644 index e621359..0000000 --- a/docs/superpowers/specs/2026-08-28-qdrant-on-disk-storage-design.md +++ /dev/null @@ -1,81 +0,0 @@ -# Qdrant On-Disk Storage Design - -## Goal - -Reduce AutoMem's Qdrant RAM residency by creating collections with on-disk -vectors, HNSW index, and payload storage, and provide a safe operator-run -rebuild path for the existing production collection. - -## Scope - -- New Qdrant collections use `on_disk=True` for vectors, `HnswConfigDiff(on_disk=True)`, - and `on_disk_payload=True`. -- The existing restore utility gains an optional HNSW on-disk setting so an - operator can rebuild a collection using the same profile. -- Documentation provides a production runbook for backup, Qdrant-only restore, - verification, rollback, and a subsequent manual Railway volume resize. - -## Non-goals - -- Do not mutate an existing collection at application startup; Qdrant creation - settings only apply to a newly created collection. -- Do not run a production migration, change Railway volume size, or alter - replicas as part of this change. -- Do not change embedding dimensions, distance metric, payload indexes, or - recall semantics. - -## Architecture - -`ensure_qdrant_collection()` remains responsible only for collection creation. -When it creates a collection, it passes the on-disk profile to Qdrant. Existing -collections keep their settings unchanged. - -`scripts/restore_from_backup.py` remains the only destructive recovery path. -It will accept `QDRANT_RESTORE_HNSW_ON_DISK` alongside its existing optional -vector and payload storage settings and applies all selected settings only when -it recreates a collection during a forced Qdrant-only restore. - -The runbook uses the existing authenticated `/backup` export and restore tool: -take a portable backup, rebuild Qdrant with explicitly set on-disk options, -validate collection point count and representative recall, and retain the -backup for rollback. Railway volume reduction is a manual post-verification -operation. - -## Data Flow - -1. A fresh AutoMem deployment starts with no Qdrant collection. -2. Runtime creates the collection with on-disk vector, HNSW, and payload - settings, then creates the existing payload indexes. -3. For an existing collection, an operator exports a backup and runs a - Qdrant-only forced restore with the three on-disk options enabled. -4. Restore deletes and recreates only Qdrant, upserts the backup's points, and - waits for the expected point count. -5. The operator verifies count and recall before manually resizing Railway - storage. A retained backup supports restoring the prior data if validation - fails. - -## Error Handling and Rollback - -- Collection creation errors keep the existing fail-soft behavior: Qdrant is - disabled and the service continues without vector storage. -- Restore remains opt-in and destructive only with its existing confirmation or - `--force`; no startup path invokes it. -- The runbook requires retaining the pre-migration backup until validation is - complete. Re-running restore from that backup recreates the collection if - rollback is needed. -- Railway volume resize is deliberately outside automation because it is an - infrastructure and capacity decision requiring observed post-migration usage. - -## Testing - -- Unit-test creation of a missing collection and assert all three on-disk - settings are passed to `create_collection`. -- Unit-test restore configuration with HNSW on-disk enabled and disabled. -- Run focused Qdrant/restore tests, full unit tests, formatting, import sorting, - lint, and compilation. - -## Documentation - -Update the environment-variable reference for the restore setting and the -backup/monitoring guide with the exact production migration, verification, and -rollback commands. diff --git a/lab/test_sets/queries_context_tags_20260704.json b/lab/test_sets/queries_context_tags_20260704.json deleted file mode 100644 index e0d0890..0000000 --- a/lab/test_sets/queries_context_tags_20260704.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "metadata": { - "created": "2026-07-04T00:00:00Z", - "source": "issue-201-context-tags-probes", - "query_count": 5, - "notes": "Probe rows exercise per-query context_tags in the recall lab harness." - }, - "queries": [ - { - "query": "What issues do React SPA editors face with WP Fusion integrations regarding data loss?", - "expected_ids": [ - "d4fc88f1-52d1-489b-b39c-72982e406910" - ], - "category": "context_tags", - "context_tags": [ - "wp-fusion" - ] - }, - { - "query": "What decisions were made regarding the AutoApp chat UI on April 21, 2026?", - "expected_ids": [ - "d7f35a92-50b6-46cb-9f00-e5f0d4f94819" - ], - "category": "context_tags", - "context_tags": [ - "autoapp" - ] - }, - { - "query": "What upgrades were made to the AutoHub voice latency settings?", - "expected_ids": [ - "de37ff1a-db7f-4310-98af-22c03b2622b8" - ], - "category": "context_tags", - "context_tags": [ - "autohub", - "voice" - ] - }, - { - "query": "What issues were identified with mcp-wp taxonomy tools and their interaction with slugs?", - "expected_ids": [ - "cd5115b8-e003-46c0-9178-b6b3c0d46117" - ], - "category": "context_tags", - "context_tags": [ - "mcp-wp", - "wordpress" - ] - }, - { - "query": "What design constraints did Jack mention for the AutoMem server in June 2026?", - "expected_ids": [ - "24997f3a-3bb4-406c-bed3-4172dfb1caa2" - ], - "category": "context_tags", - "context_tags": [ - "automem", - "mcp-automem" - ] - } - ] -} diff --git a/scripts/README.md b/scripts/README.md index f06b678..3ef6da5 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1,10 +1,19 @@ # AutoMem scripts Operational, migration, recovery, and evaluation tooling for an AutoMem -instance. This file is the catalog — the single place to find "what scripts -exist and when do I run each one." For the narrative runbooks (why and when), -see the [docs portal](https://automem.ai/docs/); for exact flags, run a script -with `--help` or read its docstring header. +instance. This is the canonical inventory: every active executable in +`scripts/`, when to use it, and the safest way to start. + +Run commands from the repository root with the project environment available: + +```bash +source .venv/bin/activate +``` + +Most CLI scripts support `--help`; `cleanup_memory_types.py` and +`recover_from_qdrant.py` intentionally have no flags, so use their documented +commands only. For narrative runbooks, follow the linked documents rather than +inventing a new sequence. ## How to read this @@ -19,30 +28,35 @@ before running it: | Tag | Meaning | |---|---| -| `routine` | Safe to run repeatedly as normal operations. | -| `one-time` | Run once per instance or per upgrade. Idempotent, but not part of day-to-day ops. See [docs/MIGRATIONS.md](../docs/MIGRATIONS.md). | -| `recovery` | Break-glass. Only after data loss or corruption. See [docs/MONITORING_AND_BACKUPS.md](../docs/MONITORING_AND_BACKUPS.md). | +| `read-only` | Inspects data or streams events without writing. | +| `maintenance` | Operational tool that may write or queue work; review the scope and back up first when it changes stored data. | +| `one-time` | Run once per instance or per upgrade. Idempotent where noted, but not part of day-to-day ops. See [docs/MIGRATIONS.md](../docs/MIGRATIONS.md). | +| `recovery` | Break-glass and potentially destructive. Only after data loss or corruption. See [docs/MONITORING_AND_BACKUPS.md](../docs/MONITORING_AND_BACKUPS.md). | | `dev` | Local development / deployment helpers. | | `bench` · `lab` | Contributor evaluation and recall-tuning harnesses. Not needed to run AutoMem. See [docs/TESTING.md](../docs/TESTING.md) and [docs/RECALL_QUALITY_LAB.md](../docs/RECALL_QUALITY_LAB.md). | +Before running a script that can change FalkorDB or Qdrant, confirm the target +from `.env`, take a backup, and start with `--dry-run` or a small `--limit` when +the script offers one. Do not run recovery, re-embedding, or migration scripts +against production while writers are active. + --- -## Routine operations +## Operations and maintenance -Run these as part of normal upkeep. +Use these for normal operations or deliberate small maintenance tasks. -| Script | Lifecycle | What it does | -|---|---|---| -| [`backup_automem.py`](backup_automem.py) | `routine` | Timestamped FalkorDB + Qdrant backup; optional S3 upload and old-backup cleanup. Cron-friendly. `--s3-bucket`, `--cleanup --keep N`. | -| [`restore_from_backup.py`](restore_from_backup.py) | `routine` · `recovery` | Restore FalkorDB + Qdrant from a backup (local snapshot or downloaded API tarball). `--backup-timestamp`, `--backup-dir snapshot.tar.gz`, `--dry-run`. | -| [`health_monitor.py`](health_monitor.py) | `routine` | Background service: watches FalkorDB/Qdrant health, checks graph↔vector consistency, triggers recovery, alerts. `--interval 300`. Containerized via [`Dockerfile.health-monitor`](Dockerfile.health-monitor). See [docs/HEALTH_MONITORING.md](../docs/HEALTH_MONITORING.md). | -| [`automem_watch.py`](automem_watch.py) | `routine` | Real-time terminal UI over `GET /stream`; tracks store/recall/update/delete/associate plus enrichment and consolidation. `--url`, `--token`. | -| [`audit_relevance.py`](audit_relevance.py) | `routine` | Audit the `relevance_score` distribution from a backup file (default) or a live instance (`--live`). | -| [`reembed_embeddings.py`](reembed_embeddings.py) | `routine` | Re-embed all memories and upsert vectors into Qdrant using the configured provider. `--batch-size`, `--limit`. Used after provider/dimension changes — see [docs/MIGRATIONS.md](../docs/MIGRATIONS.md). | -| [`reclassify_with_llm.py`](reclassify_with_llm.py) | `routine` | Reclassify fallback `type='Memory'` records via the configured classification LLM. `--provider`; env `CLASSIFICATION_MODEL` / `CLASSIFICATION_BASE_URL` / `CLASSIFICATION_API_KEY`. | -| [`reenrich_batch.py`](reenrich_batch.py) | `routine` | Re-run enrichment over a batch of memories with current classification logic. | +| Script | Lifecycle | When to use it | Start here | +|---|---|---|---| +| [`backup_automem.py`](backup_automem.py) | `maintenance` | Create a portable FalkorDB + Qdrant backup before a migration or on a schedule. | `python scripts/backup_automem.py`; add `--s3-bucket … --cleanup --keep 7` for off-site retention. | +| [`health_monitor.py`](health_monitor.py) | `maintenance` | Continuously check API, FalkorDB, Qdrant, and drift; use alert-only mode by default. | `python scripts/health_monitor.py --once` or `--interval 300`. `--auto-recover` is an explicit, high-risk opt-in. See [docs/HEALTH_MONITORING.md](../docs/HEALTH_MONITORING.md). | +| [`automem_watch.py`](automem_watch.py) | `read-only` | Observe live store/recall/update/delete, enrichment, and consolidation events. | `python scripts/automem_watch.py --url "$AUTOMEM_API_URL" --token "$AUTOMEM_API_TOKEN"` | +| [`audit_relevance.py`](audit_relevance.py) | `read-only` | Inspect `relevance_score` distribution before or after scoring changes. | `python scripts/audit_relevance.py` reads the newest backup; use `--live` only for the configured instance. | +| [`reembed_embeddings.py`](reembed_embeddings.py) | `maintenance` | Replace vectors after an embedding provider, model, or dimension migration. | After backing up and recreating the collection, run `python scripts/reembed_embeddings.py --batch-size 32`. Requires `QDRANT_URL`; use `--limit 100` only for a smoke check. See [docs/MIGRATIONS.md](../docs/MIGRATIONS.md). | +| [`reclassify_with_llm.py`](reclassify_with_llm.py) | `maintenance` | Reclassify fallback `type='Memory'` records after changing classification logic. | Start with `python scripts/reclassify_with_llm.py --dry-run --limit 25`; apply only after review (the script asks for confirmation unless `--yes` is supplied). | +| [`reenrich_batch.py`](reenrich_batch.py) | `maintenance` | Queue a small batch for re-enrichment after an enrichment logic change. | `python scripts/reenrich_batch.py --limit 10`; this calls the configured API and queues work. | -### `browse_memories.py` — read-only database browser `dev` +### `browse_memories.py` — read-only database browser Interactive CLI over the production FalkorDB graph + Qdrant vectors. Connects with `.env` credentials and **never modifies data**. Four subcommands: @@ -72,16 +86,16 @@ and every graph relationship. `diagnose` reports issues at `[CRITICAL]` / ## One-time migrations -Run once per instance or when upgrading. Idempotent and safe to re-run, but not -part of routine ops. Full runbook: [docs/MIGRATIONS.md](../docs/MIGRATIONS.md). +Run when adopting a specific upgrade or repair. Most are idempotent, but they +are not routine operations; back up first and follow the linked runbook. -| Script | Lifecycle | What it does | +| Script | When to use it | Start here | |---|---|---| -| [`migrate_mcp_sqlite.py`](migrate_mcp_sqlite.py) | `one-time` | Import the legacy MCP `sqlite_vec.db` memory store into AutoMem via the API, preserving timestamps/tags/importance. `--db`, `--automem-url`, `--api-token`, `--dry-run`. | -| [`migrate_entity_nodes.py`](migrate_entity_nodes.py) | `one-time` | Promote `entity:{category}:{slug}` tags on Memory nodes into first-class `Entity` nodes linked by `REFERENCED_IN`. `--dry-run`. (0.16.0) | -| [`backfill_tag_prefixes.py`](backfill_tag_prefixes.py) | `one-time` | Compute and backfill `tag_prefixes` in FalkorDB + Qdrant from existing tags (keeps prefix-match recall consistent). | -| [`rescore_relevance.py`](rescore_relevance.py) | `one-time` | Recompute every `relevance_score` with the corrected decay formula (undoes the old over-aggressive 0.1 rate). `--dry-run`, `--target`. | -| [`cleanup_memory_types.py`](cleanup_memory_types.py) | `one-time` | Reclassify invalid memory types (e.g. `session_start`, `interaction`) back to valid types. No flags; reads `.env`. | +| [`migrate_mcp_sqlite.py`](migrate_mcp_sqlite.py) | Moving from the legacy MCP `sqlite_vec.db` store into AutoMem. | Preview first: `python scripts/migrate_mcp_sqlite.py --dry-run`; then pass `--db`, `--automem-url`, and `--api-token`. | +| [`migrate_entity_nodes.py`](migrate_entity_nodes.py) | Adopting first-class `Entity` nodes for legacy `entity:{category}:{slug}` tags. | `python scripts/migrate_entity_nodes.py --dry-run`, then rerun without the flag. | +| [`backfill_tag_prefixes.py`](backfill_tag_prefixes.py) | Restoring or introducing `tag_prefixes` so prefix recall remains consistent. | `python scripts/backfill_tag_prefixes.py --dry-run`; apply after review. Use `--no-qdrant` only when vector payload sync is intentionally deferred. | +| [`rescore_relevance.py`](rescore_relevance.py) | Repairing relevance scores produced with the previous over-aggressive decay formula. | `python scripts/rescore_relevance.py --dry-run`, then choose the intended `--target` before applying. | +| [`cleanup_memory_types.py`](cleanup_memory_types.py) | Repairing legacy invalid types such as `session_start` or `interaction`. | Back up, verify `.env` targets the intended instance, then run `python scripts/cleanup_memory_types.py`. It has no preview mode. | > `scripts/lab/repair_entity_tags.py` (`lab`, below) is the companion repair tool > for entity-tag noise on a local clone before promoting entity nodes. @@ -93,20 +107,20 @@ part of routine ops. Full runbook: [docs/MIGRATIONS.md](../docs/MIGRATIONS.md). Only reach for these after data loss or corruption. See [docs/MONITORING_AND_BACKUPS.md](../docs/MONITORING_AND_BACKUPS.md). -| Script | Lifecycle | What it does | +| Script | When to use it | Start here | |---|---|---| -| [`recover_from_qdrant.py`](recover_from_qdrant.py) | `recovery` | Rebuild the FalkorDB graph from Qdrant: reads every vector's payload and re-inserts via the API, which regenerates relationships. No flags; reads `.env`. | -| [`deduplicate_qdrant.py`](deduplicate_qdrant.py) | `recovery` | Remove duplicate Qdrant points (e.g. after a recovery run double-inserted). `--dry-run`, `--auto-confirm`. | -| [`restore_from_backup.py`](restore_from_backup.py) | `recovery` · `routine` | See Routine operations above. | +| [`restore_from_backup.py`](restore_from_backup.py) | Restoring one or both stores from a tested local backup or API-exported tarball. | Always preview first: `python scripts/restore_from_backup.py --dry-run`. Then select `--backup-timestamp` or `--backup-dir`, optionally `--falkordb-only` / `--qdrant-only`, and use `--force` only after review. | +| [`recover_from_qdrant.py`](recover_from_qdrant.py) | FalkorDB is lost or corrupt while Qdrant is known-good and complete. | **Destructive:** it clears the configured FalkorDB graph, then rebuilds it from Qdrant. Back up and verify Qdrant before `python scripts/recover_from_qdrant.py`. | +| [`deduplicate_qdrant.py`](deduplicate_qdrant.py) | Qdrant contains duplicates, commonly after a failed recovery or manual import. | `python scripts/deduplicate_qdrant.py --dry-run`; review, then rerun with `--yes` to delete. | --- -## Developer & deployment +## Developer and deployment -| Script | Lifecycle | What it does | Make target | +| Script | When to use it | Start here | Make target | |---|---|---|---| -| [`bootstrap_dev.sh`](bootstrap_dev.sh) | `dev` | Create `.venv` (Python 3.12), refresh `venv -> .venv`, install dev deps + pre-commit hooks. | `make install` | -| [`deploy_check.sh`](deploy_check.sh) | `dev` | Compare the live Railway deployment commit against `origin/main` to catch a silently disconnected GitHub integration. `DEPLOY_CHECK_QUIET=1` for CI. | `make deploy-check` | +| [`bootstrap_dev.sh`](bootstrap_dev.sh) | Setting up or repairing a local contributor environment. | `make install` (or `./scripts/bootstrap_dev.sh`) creates `.venv`, refreshes `venv -> .venv`, installs development dependencies, and installs pre-commit hooks. | `make install` | +| [`deploy_check.sh`](deploy_check.sh) | Checking that Railway is deploying the expected Git commit. | `./scripts/deploy_check.sh automem`; use `DEPLOY_CHECK_QUIET=1` only for CI-style exit-code checks. Requires linked Railway and GitHub CLIs. | `make deploy-check` | --- @@ -114,16 +128,16 @@ Only reach for these after data loss or corruption. See Snapshot-based LoCoMo / LongMemEval evaluation. See [docs/TESTING.md](../docs/TESTING.md). -| Script | What it does | Make target | -|---|---|---| -| [`bench/ingest_and_snapshot.sh`](bench/ingest_and_snapshot.sh) | Ingest a benchmark dataset into Docker AutoMem and snapshot the volumes (run once). | `make bench-ingest BENCH=locomo` | -| [`bench/restore_and_eval.sh`](bench/restore_and_eval.sh) | Restore a snapshot and evaluate a config (no re-ingest). | `make bench-eval BENCH=locomo CONFIG=baseline` | -| [`bench/compare_configs.sh`](bench/compare_configs.sh) | A/B two scoring configs against the same snapshot. | `make bench-compare` | -| [`bench/compare_branch.sh`](bench/compare_branch.sh) | Compare a git branch against `main` on a snapshot. | `make bench-compare-branch BRANCH=…` | -| [`bench/compare_results.py`](bench/compare_results.py) | Side-by-side table of two result JSON files. `--baseline`, `--test`, `--output`. | — | -| [`bench/analyze_locomo_results.py`](bench/analyze_locomo_results.py) | Markdown failure report from a LoCoMo results JSON. `--output`. (0.16.0) | — | -| [`bench/health_check.py`](bench/health_check.py) | Post-restore diagnostics: score distribution, entity quality, latency, precision on curated queries. | `make bench-health` | -| [`run_longmemeval_watch.sh`](run_longmemeval_watch.sh) | LongMemEval with persistent logging + desktop completion/crash notifications. | `make test-longmemeval-watch` | +| Script | When to use it | Start here | Make target | +|---|---|---|---| +| [`bench/ingest_and_snapshot.sh`](bench/ingest_and_snapshot.sh) | Creating a fresh benchmark snapshot after a corpus or embedding change. | `make bench-ingest BENCH=locomo`; this starts Docker and writes reusable snapshots. | `make bench-ingest BENCH=locomo` | +| [`bench/restore_and_eval.sh`](bench/restore_and_eval.sh) | Evaluating one scoring configuration against an existing snapshot. | `make bench-eval BENCH=locomo CONFIG=baseline` | `make bench-eval` | +| [`bench/compare_configs.sh`](bench/compare_configs.sh) | A/B comparing two scoring configurations on the same snapshot. | `make bench-compare BENCH=locomo BASELINE=baseline CONFIG=` | `make bench-compare` | +| [`bench/compare_branch.sh`](bench/compare_branch.sh) | Comparing a branch against `main` using a snapshot. | `make bench-compare-branch BRANCH=`; the script temporarily checks out refs, so begin from a clean worktree. | `make bench-compare-branch` | +| [`bench/compare_results.py`](bench/compare_results.py) | Reading two existing result JSON files without rerunning a benchmark. | `python scripts/bench/compare_results.py --baseline --test ` | — | +| [`bench/analyze_locomo_results.py`](bench/analyze_locomo_results.py) | Producing a Markdown failure report from a LoCoMo result JSON. | `python scripts/bench/analyze_locomo_results.py --output report.md` | — | +| [`bench/health_check.py`](bench/health_check.py) | Checking post-restore score distributions, entity quality, latency, and curated-query precision. | `make bench-health` | `make bench-health` | +| [`run_longmemeval_watch.sh`](run_longmemeval_watch.sh) | Running LongMemEval with persistent logging and local completion/crash notifications. | `make test-longmemeval-watch`; for a smaller run use `./scripts/run_longmemeval_watch.sh --max-questions 50`. | `make test-longmemeval-watch` | --- @@ -132,15 +146,15 @@ Snapshot-based LoCoMo / LongMemEval evaluation. See [docs/TESTING.md](../docs/TE Data-driven recall scoring experiments against a clone of production. Full workflow: [docs/RECALL_QUALITY_LAB.md](../docs/RECALL_QUALITY_LAB.md). -| Script | What it does | Make target | -|---|---|---| -| [`lab/clone_production.sh`](lab/clone_production.sh) | Clone production data into an isolated local Docker stack (direct DB backup, or `--restore-only` from a saved API tarball; supports custom ports for parallel sweeps). | `make lab-clone` | -| [`lab/create_test_queries.py`](lab/create_test_queries.py) | Generate a natural-question test set from local memories (via GPT-4o-mini). `--count`, `--output`, `--api-url`. | `make lab-queries` | -| [`lab/run_recall_test.py`](lab/run_recall_test.py) | Run a test set under a config, compute Recall@K / MRR / NDCG, A/B compare, and sweep a parameter. `--config`, `--compare`, `--sweep`. | `make lab-test` · `make lab-compare` · `make lab-sweep` | -| [`lab/repair_entity_tags.py`](lab/repair_entity_tags.py) | Audit → plan → execute/rollback repair of noisy generated entity tags on a clone. `--mode audit\|execute\|rollback`. (0.16.0) | — | -| [`lab/lab_metrics.py`](lab/lab_metrics.py) | **Library module** (not a CLI): pure, deterministic IR scoring functions (Recall@K, MRR, NDCG, distractor rate). Imported by `run_recall_test.py`. (0.16.0) | — | -| [`lab/lab_corpus.py`](lab/lab_corpus.py) | **Library module** (not a CLI): recall/corpus HTTP helpers behind injectable clients for unit-testable lab logic. (0.16.0) | — | -| `lab/configs/` | JSON scoring-weight overrides for A/B testing (`baseline.json`, `issue78_*.json`). | — | +| Script | When to use it | Start here | Make target | +|---|---|---|---| +| [`lab/clone_production.sh`](lab/clone_production.sh) | Creating an isolated local copy of production data for recall experiments. | `make lab-clone`; use `--restore-only ` for repeat experiments so production is not contacted again. | `make lab-clone` | +| [`lab/create_test_queries.py`](lab/create_test_queries.py) | Generating a natural-language evaluation set from a local clone. | `make lab-queries` or `python scripts/lab/create_test_queries.py --count 100`. | `make lab-queries` | +| [`lab/run_recall_test.py`](lab/run_recall_test.py) | Measuring one configuration, an A/B comparison, or a parameter sweep. | `make lab-test CONFIG=baseline`; use `make lab-compare` or `make lab-sweep` for the other modes. | `make lab-test` · `make lab-compare` · `make lab-sweep` | +| [`lab/repair_entity_tags.py`](lab/repair_entity_tags.py) | Repairing noisy generated entity tags on a **local clone**. | Plan first: `python scripts/lab/repair_entity_tags.py --mode canonicalize-safe`; review `/plan.jsonl`, then apply with `--execute --plan /plan.jsonl` or undo with `--rollback /rollback.jsonl`. | — | +| [`lab/lab_metrics.py`](lab/lab_metrics.py) | **Library module**, not a CLI; implements deterministic Recall@K, MRR, NDCG, and distractor-rate metrics. | Import from `run_recall_test.py` or tests. | — | +| [`lab/lab_corpus.py`](lab/lab_corpus.py) | **Library module**, not a CLI; centralizes injectable recall/corpus HTTP helpers. | Import from `run_recall_test.py` or tests. | — | +| [`lab/configs/`](lab/configs/) | Creating named JSON scoring-weight overrides for A/B tests. | Copy `baseline.json`, edit weights, then pass the filename without `.json` to `CONFIG`. | — | --- @@ -148,14 +162,12 @@ workflow: [docs/RECALL_QUALITY_LAB.md](../docs/RECALL_QUALITY_LAB.md). | File | What it is | |---|---| -| [`lib/common.sh`](lib/common.sh) | Shell helpers (color codes, `wait_for_api`) sourced by the `bench/` scripts. | -| [`archive/`](archive/) | Retired one-off scripts kept for reference (e.g. dated release-sweep summarizers). Not maintained. | - ---- +| [`lib/common.sh`](lib/common.sh) | Support module, not a CLI. Provides color helpers and `wait_for_api` to benchmark shell scripts. | +| [`Dockerfile.health-monitor`](Dockerfile.health-monitor) | Container recipe for running `health_monitor.py` in alert-only mode. Use only when you intentionally operate monitoring as a separate service. | ## See also -- [docs/MIGRATIONS.md](../docs/MIGRATIONS.md) — embedding-dimension and one-time data migrations +- [docs/MIGRATIONS.md](../docs/MIGRATIONS.md) — embedding-provider, model, and one-time data migrations - [docs/MONITORING_AND_BACKUPS.md](../docs/MONITORING_AND_BACKUPS.md) — backup/restore/recovery runbook - [docs/HEALTH_MONITORING.md](../docs/HEALTH_MONITORING.md) — `health_monitor.py` deployment - [docs/RECALL_QUALITY_LAB.md](../docs/RECALL_QUALITY_LAB.md) — the `lab/` harness end to end diff --git a/scripts/archive/summarize_pipeline_20260611.py b/scripts/archive/summarize_pipeline_20260611.py deleted file mode 100644 index 1b0e86d..0000000 --- a/scripts/archive/summarize_pipeline_20260611.py +++ /dev/null @@ -1,131 +0,0 @@ -#!/usr/bin/env python3 -"""Tabulate the 2026-06-11 release-verification sweep results. - -Pools prod_parity runs as baseline; reports per-config metric deltas with a -paired difference test on per-query recall@5 (computed from retrieved_ids[:5]) -against parity run 1, plus per-category R@10 deltas. The p-value uses a normal -(z) approximation of the paired t-statistic — accurate for the n=200 query -sets here, but not a Student's t p-value for small n. -""" - -import glob -import json -import math -import os -import sys -from collections import defaultdict - -RESULTS_DIR = sys.argv[1] if len(sys.argv) > 1 else "lab/results" -RUN_DATE = "20260611" - -CONFIG_ORDER = [ - "prod_parity", - "cap2", - "cap3", - "cap4", - "gate005", - "gate010", - "gate015", - "gate020", - "gate025", - "recw90", - "recw365", - "recexp", - "recbias", -] - - -def load_runs(): - runs = defaultdict(list) - for path in sorted(glob.glob(os.path.join(RESULTS_DIR, f"*_{RUN_DATE}_*.json"))): - name = os.path.basename(path) - for cfg in CONFIG_ORDER: - if name.startswith(cfg + "_" + RUN_DATE): - with open(path) as fh: - runs[cfg].append(json.load(fh)) - break - return runs - - -def perq_recall5(run): - out = {} - for q in run.get("queries", []): - exp = q.get("expected_ids") or [] - top5 = set((q.get("retrieved_ids") or [])[:5]) - out[q["query"]] = (sum(1 for e in exp if e in top5) / len(exp)) if exp else None - return out - - -def paired_z(a, b): - # z-approximation: t-statistic against the normal CDF (n=200 here, so t ~= z) - diffs = [y - x for x, y in zip(a, b)] - n = len(diffs) - if n < 2: - return float("nan") - mean = sum(diffs) / n - var = sum((d - mean) ** 2 for d in diffs) / (n - 1) - if var == 0: - return 1.0 - t = mean / math.sqrt(var / n) - return 2 * (1 - 0.5 * (1 + math.erf(abs(t) / math.sqrt(2)))) - - -def main(): - runs = load_runs() - if not runs.get("prod_parity"): - sys.exit(f"no prod_parity runs found in {RESULTS_DIR}") - - base_runs = runs["prod_parity"] - base_pq = perq_recall5(base_runs[0]) - base_r5 = sum(r["summary"]["recall_5"] for r in base_runs) / len(base_runs) - - print( - f"{'config':>12} {'runs':>4} {'R@5':>7} {'R@10':>7} {'MRR':>7} {'NDCG':>7} {'dR@5':>7} {'p':>8}" - ) - print("-" * 66) - for cfg in CONFIG_ORDER: - rs = runs.get(cfg, []) - if not rs: - continue - s = [r["summary"] for r in rs] - r5 = sum(x["recall_5"] for x in s) / len(s) - r10 = sum(x["recall_10"] for x in s) / len(s) - mrr = sum(x["mrr"] for x in s) / len(s) - ndcg = sum(x["ndcg_10"] for x in s) / len(s) - if cfg == "prod_parity": - print( - f"{cfg:>12} {len(rs):>4} {r5:7.3f} {r10:7.3f} {mrr:7.3f} {ndcg:7.3f} {'—':>7} {'—':>8}" - ) - continue - cand_pq = perq_recall5(rs[0]) - common = [q for q, v in base_pq.items() if v is not None and cand_pq.get(q) is not None] - p = paired_z([base_pq[q] for q in common], [cand_pq[q] for q in common]) - print( - f"{cfg:>12} {len(rs):>4} {r5:7.3f} {r10:7.3f} {mrr:7.3f} {ndcg:7.3f} {r5 - base_r5:+7.3f} {p:8.4f}" - ) - - print("\nPer-category R@10 delta vs parity run 1 (n in parens):") - base_cat = base_runs[0].get("by_category", {}) - cats = sorted(base_cat) - print(f"{'config':>12} " + " ".join(f"{c[:9]:>9}" for c in cats)) - print( - f"{'(n)':>12} " - + " ".join(f"{'(' + str(base_cat[c].get('count', 0)) + ')':>9}" for c in cats) - ) - for cfg in CONFIG_ORDER: - rs = runs.get(cfg, []) - if not rs: - continue - cat = rs[0].get("by_category", {}) - if cfg == "prod_parity": - row = [f"{base_cat[c].get('recall_10', 0):.3f}" for c in cats] - else: - row = [ - f"{cat.get(c, {}).get('recall_10', 0) - base_cat[c].get('recall_10', 0):+.3f}" - for c in cats - ] - print(f"{cfg:>12} " + " ".join(f"{v:>9}" for v in row)) - - -if __name__ == "__main__": - main() diff --git a/tests/benchmarks/results/compare_pr80_bm25_only_f10_judge_off.json b/tests/benchmarks/results/compare_pr80_bm25_only_f10_judge_off.json index 27ca469..df517e5 100644 --- a/tests/benchmarks/results/compare_pr80_bm25_only_f10_judge_off.json +++ b/tests/benchmarks/results/compare_pr80_bm25_only_f10_judge_off.json @@ -9,6 +9,6 @@ "Open Domain": -0.03508771929824561, "Complex Reasoning": 0.0 }, - "baseline_file": "/Users/jgarturo/Projects/OpenAI/automem/benchmarks/results/locomo-mini_baseline_20260310_233631.json", - "test_file": "/Users/jgarturo/Projects/OpenAI/automem/benchmarks/results/locomo-mini_pr80_bm25_only_f10_20260311_025443.json" + "baseline_file": "benchmarks/results/locomo-mini_baseline_20260310_233631.json", + "test_file": "benchmarks/results/locomo-mini_pr80_bm25_only_f10_20260311_025443.json" } From a8be394bc9598bb8e30edabcec202c66c7256b18 Mon Sep 17 00:00:00 2001 From: Jack Arturo Date: Fri, 28 Aug 2026 05:26:07 +0200 Subject: [PATCH 29/33] fix(docs): correct re-embedding migration setup --- docs/MIGRATIONS.md | 15 ++--- tests/test_regression_drift_scout_contract.py | 59 ------------------- 2 files changed, 8 insertions(+), 66 deletions(-) delete mode 100644 tests/test_regression_drift_scout_contract.py diff --git a/docs/MIGRATIONS.md b/docs/MIGRATIONS.md index 23acd61..1d426ca 100644 --- a/docs/MIGRATIONS.md +++ b/docs/MIGRATIONS.md @@ -36,16 +36,17 @@ This document provides step-by-step instructions for migrating between different 1. **Backup your data**: `python scripts/backup_automem.py` 2. **Set environment variables**: ```bash - EMBEDDING_PROVIDER=voyage # or auto (will prefer Voyage if VOYAGE_API_KEY is set) - VOYAGE_API_KEY=pa-... - VOYAGE_MODEL=voyage-4 - VECTOR_SIZE=1024 - QDRANT_URL=http://localhost:6333 # required by scripts/reembed_embeddings.py + export EMBEDDING_PROVIDER=voyage # or auto (will prefer Voyage if VOYAGE_API_KEY is set) + export VOYAGE_API_KEY=pa-... + export VOYAGE_MODEL=voyage-4 + export VECTOR_SIZE=1024 + # The re-embed script needs a URL; derive one if your deployment uses host/port. + export QDRANT_URL="${QDRANT_URL:-http://${QDRANT_HOST:-localhost}:${QDRANT_PORT:-6333}}" ``` 3. **Pause writes, then delete and recreate the Qdrant collection**: ```bash - curl -X DELETE http://localhost:6333/collections/memories - curl -X PUT http://localhost:6333/collections/memories \ + curl -X DELETE "$QDRANT_URL/collections/memories" + curl -X PUT "$QDRANT_URL/collections/memories" \ -H 'Content-Type: application/json' \ -d '{"vectors": {"size": 1024, "distance": "Cosine"}}' ``` diff --git a/tests/test_regression_drift_scout_contract.py b/tests/test_regression_drift_scout_contract.py deleted file mode 100644 index 016d83f..0000000 --- a/tests/test_regression_drift_scout_contract.py +++ /dev/null @@ -1,59 +0,0 @@ -import re -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -SKILL_PATH = ROOT / ".agents" / "skills" / "automem-regression-drift-scout" / "SKILL.md" - - -def read_skill() -> str: - return SKILL_PATH.read_text(encoding="utf-8") - - -def test_automem_regression_drift_scout_skill_exists_with_required_contract(): - source = read_skill() - - assert "name: automem-regression-drift-scout" in source - assert "description:" in source - assert "agents: [codex]" in source - assert "capabilities:" in source - assert "filesystem: readonly" in source - - -def test_automem_regression_drift_scout_is_read_only_by_default(): - source = read_skill() - - assert "read-only" in source - assert "Do not deploy" in source - assert "Do not edit files" in source - assert "Do not create issues or pull requests" in source - assert not re.search(r"`make deploy`|\bmake deploy(?:\s|$)", source) - - -def test_automem_regression_drift_scout_uses_project_truth_sources(): - source = read_skill() - - assert "benchmarks/EXPERIMENT_LOG.md" in source - assert "make bench-health" in source - assert "make deploy-check" in source - assert "gh run list" in source - assert "railway status" in source - - -def test_automem_regression_drift_scout_output_contract_is_tiered(): - source = read_skill() - - assert "healthy" in source - assert "needs_issue" in source - assert "needs_pr_plan" in source - assert "evidence bundle" in source - assert "confidence score" in source - assert "what to measure next" in source - - -def test_automem_regression_drift_scout_handles_partial_command_results(): - source = read_skill() - - assert "Do not trust the final `RECALL HEALTH: HEALTHY` banner by itself" in source - assert "Connection refused" in source - assert "railway CLI not found" in source - assert "skipped surface" in source From 4dd13258e9bda316adaf121da6fe652ab2057b7e Mon Sep 17 00:00:00 2001 From: Jack Arturo Date: Fri, 28 Aug 2026 05:32:09 +0200 Subject: [PATCH 30/33] fix(recovery): honor configured graph and cloud migrations --- docs/MIGRATIONS.md | 6 +++- scripts/recover_from_qdrant.py | 26 +++++++++------ tests/test_recover_from_qdrant.py | 53 +++++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 10 deletions(-) create mode 100644 tests/test_recover_from_qdrant.py diff --git a/docs/MIGRATIONS.md b/docs/MIGRATIONS.md index 1d426ca..40083bc 100644 --- a/docs/MIGRATIONS.md +++ b/docs/MIGRATIONS.md @@ -42,12 +42,16 @@ This document provides step-by-step instructions for migrating between different export VECTOR_SIZE=1024 # The re-embed script needs a URL; derive one if your deployment uses host/port. export QDRANT_URL="${QDRANT_URL:-http://${QDRANT_HOST:-localhost}:${QDRANT_PORT:-6333}}" + # Set QDRANT_API_KEY when the selected Qdrant instance requires authentication. + export QDRANT_API_KEY="${QDRANT_API_KEY:-}" ``` 3. **Pause writes, then delete and recreate the Qdrant collection**: ```bash - curl -X DELETE "$QDRANT_URL/collections/memories" + curl -X DELETE "$QDRANT_URL/collections/memories" \ + -H "api-key: $QDRANT_API_KEY" curl -X PUT "$QDRANT_URL/collections/memories" \ -H 'Content-Type: application/json' \ + -H "api-key: $QDRANT_API_KEY" \ -d '{"vectors": {"size": 1024, "distance": "Cosine"}}' ``` 4. **Re-embed all memories**: diff --git a/scripts/recover_from_qdrant.py b/scripts/recover_from_qdrant.py index 58bdd10..780254e 100755 --- a/scripts/recover_from_qdrant.py +++ b/scripts/recover_from_qdrant.py @@ -26,6 +26,7 @@ FALKORDB_HOST = os.getenv("FALKORDB_HOST", "localhost") FALKORDB_PORT = int(os.getenv("FALKORDB_PORT", "6379")) FALKORDB_PASSWORD = os.getenv("FALKORDB_PASSWORD") +FALKORDB_GRAPH = os.getenv("FALKORDB_GRAPH", "memories") BATCH_SIZE = 50 @@ -79,7 +80,7 @@ def restore_memory_to_graph_only(memory: Dict[str, Any], client) -> bool: try: # Store directly to FalkorDB graph - g = client.select_graph("memories") + g = client.select_graph(FALKORDB_GRAPH) # Build metadata string (exclude reserved fields to prevent overwriting) RESERVED_FIELDS = {"type", "confidence", "content", "timestamp", "importance", "tags", "id"} @@ -124,6 +125,19 @@ def restore_memory_to_graph_only(memory: Dict[str, Any], client) -> bool: return False +def clear_graph(client: FalkorDB) -> bool: + """Remove all nodes from the configured FalkorDB graph before recovery.""" + print(f"🗑️ Clearing graph '{FALKORDB_GRAPH}'...") + try: + graph = client.select_graph(FALKORDB_GRAPH) + graph.query("MATCH (n) DETACH DELETE n") + print("✅ Graph cleared\n") + return True + except Exception as exc: + print(f"⚠️ Could not clear graph: {exc}\n") + return False + + def main(): """Main recovery process.""" print("=" * 60) @@ -145,14 +159,8 @@ def main(): print(f"❌ Failed to connect to FalkorDB: {e}") sys.exit(1) - # Clear existing graph - print("🗑️ Clearing existing graph data...") - try: - g = client.select_graph("memories") - g.query("MATCH (n) DETACH DELETE n") - print("✅ Graph cleared\n") - except Exception as e: - print(f"⚠️ Could not clear graph: {e}\n") + # Clear the configured graph before restoring from Qdrant. + clear_graph(client) # Fetch all memories from Qdrant memories = get_all_memories() diff --git a/tests/test_recover_from_qdrant.py b/tests/test_recover_from_qdrant.py new file mode 100644 index 0000000..48ac39f --- /dev/null +++ b/tests/test_recover_from_qdrant.py @@ -0,0 +1,53 @@ +"""Regression coverage for the Qdrant-to-FalkorDB recovery helper.""" + +import importlib.util +from pathlib import Path + + +SCRIPT_PATH = Path(__file__).resolve().parents[1] / "scripts" / "recover_from_qdrant.py" + + +def load_recovery_module(): + spec = importlib.util.spec_from_file_location("recover_from_qdrant", SCRIPT_PATH) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class FakeGraph: + def __init__(self): + self.queries = [] + + def query(self, statement, params=None): + self.queries.append((statement, params)) + + +class FakeFalkorDB: + def __init__(self): + self.graph_names = [] + self.graph = FakeGraph() + + def select_graph(self, name): + self.graph_names.append(name) + return self.graph + + +def test_recovery_uses_configured_graph_for_clear_and_restore(monkeypatch): + monkeypatch.setenv("FALKORDB_GRAPH", "custom_memories") + recovery = load_recovery_module() + client = FakeFalkorDB() + + assert recovery.clear_graph(client) + assert recovery.restore_memory_to_graph_only( + { + "id": "memory-1", + "payload": { + "content": "Recovered memory", + "tags": [], + }, + }, + client, + ) + + assert client.graph_names == ["custom_memories", "custom_memories"] From abfd0c84628bbe5c90f52410523d383fdde8be10 Mon Sep 17 00:00:00 2001 From: Jack Arturo Date: Fri, 28 Aug 2026 05:34:19 +0200 Subject: [PATCH 31/33] style(test): format recovery regression --- tests/test_recover_from_qdrant.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_recover_from_qdrant.py b/tests/test_recover_from_qdrant.py index 48ac39f..2df9ab1 100644 --- a/tests/test_recover_from_qdrant.py +++ b/tests/test_recover_from_qdrant.py @@ -3,7 +3,6 @@ import importlib.util from pathlib import Path - SCRIPT_PATH = Path(__file__).resolve().parents[1] / "scripts" / "recover_from_qdrant.py" From 34684431293ef9b0a186bb8b4ffb4fc5ca03b4cc Mon Sep 17 00:00:00 2001 From: Jack Arturo Date: Fri, 28 Aug 2026 05:38:11 +0200 Subject: [PATCH 32/33] fix(docs): complete embedding migration runbooks --- docs/MIGRATIONS.md | 58 ++++++++++++++++++++++++++++++---------------- 1 file changed, 38 insertions(+), 20 deletions(-) diff --git a/docs/MIGRATIONS.md b/docs/MIGRATIONS.md index 40083bc..86ac61f 100644 --- a/docs/MIGRATIONS.md +++ b/docs/MIGRATIONS.md @@ -42,14 +42,15 @@ This document provides step-by-step instructions for migrating between different export VECTOR_SIZE=1024 # The re-embed script needs a URL; derive one if your deployment uses host/port. export QDRANT_URL="${QDRANT_URL:-http://${QDRANT_HOST:-localhost}:${QDRANT_PORT:-6333}}" + export QDRANT_COLLECTION="${QDRANT_COLLECTION:-memories}" # Set QDRANT_API_KEY when the selected Qdrant instance requires authentication. export QDRANT_API_KEY="${QDRANT_API_KEY:-}" ``` 3. **Pause writes, then delete and recreate the Qdrant collection**: ```bash - curl -X DELETE "$QDRANT_URL/collections/memories" \ + curl -X DELETE "$QDRANT_URL/collections/$QDRANT_COLLECTION" \ -H "api-key: $QDRANT_API_KEY" - curl -X PUT "$QDRANT_URL/collections/memories" \ + curl -X PUT "$QDRANT_URL/collections/$QDRANT_COLLECTION" \ -H 'Content-Type: application/json' \ -H "api-key: $QDRANT_API_KEY" \ -d '{"vectors": {"size": 1024, "distance": "Cosine"}}' @@ -58,7 +59,8 @@ This document provides step-by-step instructions for migrating between different ```bash python scripts/reembed_embeddings.py --batch-size 32 ``` -5. **Verify**: Check that `/health` shows `vector_size: 1024` and recall returns results. +5. **Restart or redeploy AutoMem with the selected provider configuration** before testing recall. For example, run `docker compose up -d` for Docker Compose or `railway up` for Railway. +6. **Verify**: Check that `/health` shows `vector_size: 1024` and recall returns results. > `VECTOR_SIZE_AUTODETECT=true` can preserve the old collection dimension only when you keep its provider and model. It never makes an existing OpenAI, Voyage, Ollama, or FastEmbed vector compatible with another model space. @@ -102,15 +104,18 @@ This creates timestamped backups in `backups/`: echo "EMBEDDING_PROVIDER=openai" >> .env echo "VECTOR_SIZE=3072" >> .env echo "EMBEDDING_MODEL=text-embedding-3-large" >> .env -echo "QDRANT_URL=http://localhost:6333" >> .env +echo "QDRANT_URL=${QDRANT_URL:-http://${QDRANT_HOST:-localhost}:${QDRANT_PORT:-6333}}" >> .env +echo "QDRANT_COLLECTION=${QDRANT_COLLECTION:-memories}" >> .env ``` -Or export temporarily: +Export the same settings for the collection commands and re-embed script: ```bash export EMBEDDING_PROVIDER=openai export VECTOR_SIZE=3072 export EMBEDDING_MODEL=text-embedding-3-large -export QDRANT_URL=http://localhost:6333 +export QDRANT_URL="${QDRANT_URL:-http://${QDRANT_HOST:-localhost}:${QDRANT_PORT:-6333}}" +export QDRANT_COLLECTION="${QDRANT_COLLECTION:-memories}" +export QDRANT_API_KEY="${QDRANT_API_KEY:-}" ``` #### 3. Pause Writes and Recreate the Qdrant Collection @@ -118,9 +123,11 @@ export QDRANT_URL=http://localhost:6333 `reembed_embeddings.py` upserts into an existing collection; it does not recreate one. After the backup, stop or pause writes and recreate the collection for the new model space: ```bash -curl -X DELETE http://localhost:6333/collections/memories -curl -X PUT http://localhost:6333/collections/memories \ +curl -X DELETE "$QDRANT_URL/collections/$QDRANT_COLLECTION" \ + -H "api-key: $QDRANT_API_KEY" +curl -X PUT "$QDRANT_URL/collections/$QDRANT_COLLECTION" \ -H 'Content-Type: application/json' \ + -H "api-key: $QDRANT_API_KEY" \ -d '{"vectors": {"size": 3072, "distance": "Cosine"}}' ``` @@ -139,7 +146,9 @@ This will: #### 5. Verify Migration Check Qdrant collection info: ```bash -curl http://localhost:6333/collections/memories | jq '.result.config.params.vectors' +curl -H "api-key: $QDRANT_API_KEY" \ + "$QDRANT_URL/collections/$QDRANT_COLLECTION" \ + | jq '.result.config.params.vectors' ``` Should show: @@ -150,17 +159,11 @@ Should show: } ``` -#### 6. Test Recall -```bash -curl -X POST http://localhost:8001/recall \ - -H "Authorization: Bearer $AUTOMEM_API_TOKEN" \ - -H "Content-Type: application/json" \ - -d '{"query": "test recall", "limit": 5}' -``` +#### 6. Restart Application with the New Configuration -Verify results are returned and scores look reasonable. +Restart or redeploy before recall testing so the API generates query vectors in +the new model space: -#### 7. Restart Application ```bash # If using Docker docker compose up -d @@ -175,6 +178,16 @@ sudo systemctl restart automem railway up ``` +#### 7. Test Recall +```bash +curl -X POST http://localhost:8001/recall \ + -H "Authorization: Bearer $AUTOMEM_API_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"query": "test recall", "limit": 5}' +``` + +Verify results are returned and scores look reasonable. + ### Rollback Procedure If migration fails or results are poor: @@ -343,9 +356,14 @@ Collection 'memories' already exists with different dimension **Solution:** Back up, pause writes, then delete **and recreate** the collection with the selected model's dimension before re-embedding. The script only upserts vectors: ```bash -curl -X DELETE http://localhost:6333/collections/memories -curl -X PUT http://localhost:6333/collections/memories \ +export QDRANT_URL="${QDRANT_URL:-http://${QDRANT_HOST:-localhost}:${QDRANT_PORT:-6333}}" +export QDRANT_COLLECTION="${QDRANT_COLLECTION:-memories}" +export QDRANT_API_KEY="${QDRANT_API_KEY:-}" +curl -X DELETE "$QDRANT_URL/collections/$QDRANT_COLLECTION" \ + -H "api-key: $QDRANT_API_KEY" +curl -X PUT "$QDRANT_URL/collections/$QDRANT_COLLECTION" \ -H 'Content-Type: application/json' \ + -H "api-key: $QDRANT_API_KEY" \ -d '{"vectors": {"size": 1024, "distance": "Cosine"}}' python scripts/reembed_embeddings.py --batch-size 32 ``` From 0bc00969b1d9f3bb9c5875f82401ec4a5948cd65 Mon Sep 17 00:00:00 2001 From: Jack Arturo Date: Fri, 28 Aug 2026 05:43:31 +0200 Subject: [PATCH 33/33] fix(config): preserve Ollama defaults and migration checks --- docker-compose.yml | 7 +++++-- docs/MIGRATIONS.md | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4528cc9..63bb34f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -63,8 +63,11 @@ services: OPENAI_API_KEY: ${OPENAI_API_KEY:-} OPENAI_BASE_URL: ${OPENAI_BASE_URL:-} EMBEDDING_MODEL: ${EMBEDDING_MODEL:-text-embedding-3-small} - OLLAMA_BASE_URL: ${OLLAMA_BASE_URL:-} - OLLAMA_MODEL: ${OLLAMA_MODEL:-} + # Pass these through only when intentionally configured. This preserves + # Ollama's runtime defaults for EMBEDDING_PROVIDER=ollama and prevents + # auto-selection from treating an empty Compose value as configuration. + OLLAMA_BASE_URL: + OLLAMA_MODEL: OLLAMA_TIMEOUT: ${OLLAMA_TIMEOUT:-30} OLLAMA_MAX_RETRIES: ${OLLAMA_MAX_RETRIES:-2} VECTOR_SIZE: ${VECTOR_SIZE:-1024} diff --git a/docs/MIGRATIONS.md b/docs/MIGRATIONS.md index 86ac61f..e91bc19 100644 --- a/docs/MIGRATIONS.md +++ b/docs/MIGRATIONS.md @@ -60,7 +60,7 @@ This document provides step-by-step instructions for migrating between different python scripts/reembed_embeddings.py --batch-size 32 ``` 5. **Restart or redeploy AutoMem with the selected provider configuration** before testing recall. For example, run `docker compose up -d` for Docker Compose or `railway up` for Railway. -6. **Verify**: Check that `/health` shows `vector_size: 1024` and recall returns results. +6. **Verify**: Check that `/health` reports `vector_dimensions.collection: 1024` and recall returns results. > `VECTOR_SIZE_AUTODETECT=true` can preserve the old collection dimension only when you keep its provider and model. It never makes an existing OpenAI, Voyage, Ollama, or FastEmbed vector compatible with another model space.