diff --git a/.agents/skills/automem-regression-drift-scout/SKILL.md b/.agents/skills/automem-regression-drift-scout/SKILL.md deleted file mode 100644 index e2c1f999..00000000 --- a/.agents/skills/automem-regression-drift-scout/SKILL.md +++ /dev/null @@ -1,109 +0,0 @@ ---- -name: automem-regression-drift-scout -description: Inspect AutoMem CI, benchmark health, and deployment drift as a read-only recurring automation; use for scheduled Codex maintenance runs before proposing issues or PR plans. -license: MIT -tags: [automem, automation, regression, benchmarks, deployment] -category: maintenance -agents: [codex] -metadata: - version: "1.0.0" -capabilities: - network: true - filesystem: readonly - tools: [Bash] ---- - -# AutoMem Regression Drift Scout - -## When to use - -Use this skill for scheduled or manual Codex runs that assess whether AutoMem -has fresh regression, benchmark, CI, or deployment-drift signals that need -human attention. It is not a general code-review skill and it is not a release -operator. - -## Safety rules - -- Default to read-only. -- Do not edit files. -- Do not deploy. -- Do not create issues or pull requests. -- Do not run long or paid live benchmarks unless the automation prompt - explicitly authorizes that run. -- Do not report benchmark claims unless they are grounded in - `benchmarks/EXPERIMENT_LOG.md`, local benchmark command output, or CI output. -- Do not treat missing credentials as a failure by itself; report the skipped - surface and continue with available local evidence. - -## Inputs - -- Active worktree root: - `WORKTREE_ROOT=$(git rev-parse --show-toplevel 2>/dev/null)` -- Repository guidance: `AGENTS.md` -- Benchmark source of truth: `benchmarks/EXPERIMENT_LOG.md` -- Local command surface: `Makefile`, `pyproject.toml`, `.github/workflows` -- Optional read-only GitHub status via `gh run list` -- Optional Railway status via `railway status` - -## Workflow - -1. Load repository rules from `AGENTS.md`. -2. Inspect the current branch and dirty state with `git status --short --branch`. -3. Read the current benchmark baseline and latest experiment notes from - `benchmarks/EXPERIMENT_LOG.md`. -4. Run the fast local health probe when dependencies are available: - `make bench-health`. -5. Check deploy drift without changing remote state: - `make deploy-check`. -6. Check recent CI status if GitHub CLI is authenticated: - `gh run list --limit 10`. -7. Check Railway status if the Railway CLI is already authenticated: - `railway status`. -8. Compare the evidence against the last run memory, open issues/PRs if - visible, and recent `automem` memories. Dedupe before escalating. -9. Choose exactly one tier: - `healthy`, `needs_issue`, or `needs_pr_plan`. - -## Command result handling - -- Do not trust the final `RECALL HEALTH: HEALTHY` banner by itself. Parse the - full output for `ERROR:`, `SKIP:`, tracebacks, and connection failures. -- If `make bench-health` reports `Connection refused` for `localhost:8001`, - classify the local API-dependent checks as a skipped surface unless the - automation expected the local stack to be running. -- If `make deploy-check` reports `railway CLI not found`, classify Railway - deploy drift as a skipped surface, not as an AutoMem regression. -- If read-only GitHub or Railway status is unavailable because a CLI is missing - or unauthenticated, record the exact command output and continue with the - remaining evidence. - -## Tier guidance - -- `healthy`: checks are green or skipped for known missing credentials, and no - new actionable drift appears. -- `needs_issue`: evidence shows a real problem, but the fix is unclear, depends - on infrastructure state, or needs human prioritization. -- `needs_pr_plan`: evidence points to a bounded code/docs change with clear - files, commands, and acceptance criteria. The scout should draft the plan, not - create the PR unless a later prompt explicitly upgrades the run to PR-capable. - -## Output - -Return a concise triage deliverable with: - -- tier -- evidence bundle with command names, timestamps, and key outputs -- confidence score and one-line rationale -- dedupe notes against issues, PRs, or previous memories -- recommended next action -- what to measure next time - -Always end with the required inbox item for the automation runner. - -## Anti-patterns - -- Running heavyweight live benchmarks as a routine scout. -- Treating a stale benchmark table as a regression without fresh evidence. -- Opening GitHub issues or PRs from the read-only scout. -- Reporting deploy drift from memory alone without checking current read-only - status. diff --git a/.env.example b/.env.example index d8795835..19b9d17d 100644 --- a/.env.example +++ b/.env.example @@ -11,12 +11,21 @@ QDRANT_HOST= QDRANT_PORT=6333 QDRANT_URL= QDRANT_COLLECTION=memories -VECTOR_SIZE=1024 # Must match your embedding provider (1024=voyage-4, 3072=text-embedding-3-large, 768=text-embedding-3-small) +# Recommended embeddings: Voyage is the default when its API key is configured. +EMBEDDING_PROVIDER=auto +VOYAGE_API_KEY= +VOYAGE_MODEL=voyage-4 +VECTOR_SIZE=1024 # voyage-4 and Ollama bge-m3; must match your selected provider PORT=8001 +# OpenAI is the automatic API fallback when VOYAGE_API_KEY is unset. OPENAI_API_KEY= # For OpenAI-compatible providers (OpenRouter, LiteLLM, vLLM, etc.): # OPENAI_BASE_URL=https://openrouter.ai/api/v1 # EMBEDDING_MODEL=openai/text-embedding-3-small +# For local/self-hosted Ollama embeddings (for example, multilingual BGE-M3): +# OLLAMA_BASE_URL=http://localhost:11434 +# OLLAMA_MODEL=bge-m3 +# FastEmbed is local/self-hosted only; do not rely on it as a cloud default. # --- LLM classification (scripts/reclassify_with_llm.py) --- # Optional. Defaults to OpenAI gpt-4o-mini via OPENAI_API_KEY. # OPENROUTER_API_KEY= # used by --provider openrouter diff --git a/.github/workflows/mcp-parity.yml b/.github/workflows/mcp-parity.yml new file mode 100644 index 00000000..8d730310 --- /dev/null +++ b/.github/workflows/mcp-parity.yml @@ -0,0 +1,69 @@ +name: MCP Transport Parity + +# Diffs the remote MCP bridge against the stdio package against one live +# AutoMem. See docs/MCP_TRANSPORT_PARITY.md for the contract. +# +# No scheduled run yet, deliberately. While the harness is intentionally red +# against the known gaps, `continue-on-error` leaves the workflow successful +# either way, so a cron job could not distinguish "same known gaps" from "a +# newly published mcp-automem added drift" — it would be an alarm that cannot +# alarm. The weekly drift check lands with the change that turns the harness +# green, where a failure is a real signal. + +on: + pull_request: + paths: + - 'mcp-sse-server/**' + - 'automem/api/**' + - 'app.py' + workflow_dispatch: + +jobs: + parity: + runs-on: ubuntu-latest + # Phase 1 only. The harness is legitimately red until the bridge is moved + # onto the shared MCP surface; this is removed in the same change. + continue-on-error: true + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: '20' + cache: npm + cache-dependency-path: mcp-sse-server/package-lock.json + + - name: Start AutoMem stack + env: + AUTOMEM_API_HOST_PORT: '8011' + AUTOMEM_API_TOKEN: test-token + ADMIN_API_TOKEN: test-admin-token + run: docker compose up -d + + - name: Wait for /health + run: | + for i in $(seq 1 60); do + if curl -fsS http://localhost:8011/health > /dev/null; then + echo "AutoMem is up after ${i}s" + exit 0 + fi + sleep 1 + done + echo "AutoMem did not become healthy within 60s" + docker compose logs flask-api + exit 1 + + - name: Run parity harness + working-directory: mcp-sse-server + env: + AUTOMEM_RUN_PARITY_TESTS: '1' + AUTOMEM_PARITY_API_URL: http://localhost:8011 + AUTOMEM_PARITY_API_TOKEN: test-token + run: | + npm ci + node -p "'stdio package under test: ' + require('@verygoodplugins/mcp-automem/package.json').version" + npm test + + - name: Dump service logs on failure + if: failure() + run: docker compose logs --tail=200 diff --git a/.gitignore b/.gitignore index d8b3dce9..daa1a10f 100644 --- a/.gitignore +++ b/.gitignore @@ -33,10 +33,8 @@ node_modules/ # Experiment results (promote notable runs to tests/benchmarks/results/) /tests/benchmarks/experiments/results_*/ -# Recall Quality Lab data (snapshots, test results) -/lab/snapshots/ -/lab/results/ -/lab/test_sets/ +# Recall Quality Lab data (generated snapshots, test sets, and results) +/lab/ # Benchmark overrides and snapshots .env.bench @@ -47,3 +45,5 @@ node_modules/ benchmarks/baselines/locomo_baseline.json data/bm25_index.db data/ +/.agents +/docs/superpowers/ diff --git a/AGENTS.md b/AGENTS.md index b1edbf5d..e416ed95 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,7 +8,7 @@ - `benchmarks/`: Snapshot-based benchmark system. See `EXPERIMENT_LOG.md` for current baselines and results. - `scripts/bench/`: Benchmark tooling (ingest, eval, compare, health check). - `docs/`: API, testing, deployment, monitoring, and env var references. -- `scripts/`: Maintenance and ops helpers (backup, reembed, health monitor). +- `scripts/`: Maintenance and ops helpers (backup, reembed, health monitor). See the canonical [scripts catalog](scripts/README.md) for lifecycle and usage. - `mcp-sse-server/`: Optional MCP bridge used in some deployments. ## Build, Test, and Development @@ -76,11 +76,12 @@ The benchmark system uses **snapshot-based evaluation**: ingest once, eval many - `benchmarks/baselines/` — baseline result JSONs (small files committed, large ones gitignored). - `benchmarks/snapshots/` — Qdrant/FalkorDB snapshot data (gitignored, regenerate with `make bench-ingest`). - `benchmarks/results/` — per-run result JSONs (gitignored). -- `scripts/bench/` — shell and Python scripts driving ingest, eval, compare, and health checks. +- `scripts/bench/` — shell and Python scripts driving ingest, eval, compare, and health checks; see [scripts/README.md](scripts/README.md). - `tests/benchmarks/` — legacy benchmark harnesses (LoCoMo, LongMemEval) and historical result markdown files. ## Commit & Pull Requests +- Feature PRs target `develop` (repo default). Promote `develop` to `main` with a validated release merge; release-please and GHCR `:stable` then run on `main`. Do not open feature work onto `main`. - PR titles must use Conventional Commit format because squash merges use the PR title as the release commit title. Do not prefix titles with `[codex]`, `[claude]`, `[copilot]`, `[wip]`, or similar labels; put agent/status context in the PR body. - Use Conventional Commit types: `feat`, `fix`, `docs`, `refactor`, `test`, `ci`, `build`, `chore`, `perf`, `revert` (e.g., `feat(api): add /analyze endpoint`). - For public API changes, use `feat(api): ...` unless the change is strictly a bug fix with no new public surface. For docs-only changes, use `docs: ...`; for release automation, use `ci(release): ...` or `chore(release): ...`. diff --git a/CLAUDE.md b/CLAUDE.md index bf024738..e937c604 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -142,11 +142,13 @@ AutoMem uses a provider pattern with multiple embedding backends: #### Provider Priority (Auto-Selection) 1. **Voyage AI** (`voyage:voyage-4`) - If `VOYAGE_API_KEY` is set + - Recommended provider for new deployments: multilingual quality with a generous free tier; see [Voyage pricing](https://docs.voyageai.com/docs/pricing) for current limits and rates - High-quality embeddings with flexible model family (voyage-4, voyage-4-large, voyage-4-lite) - Supports output dimensions: 256, 512, 1024, 2048 - Requires network and API key 2. **OpenAI / OpenAI-compatible** (`openai:text-embedding-3-small`) - If `OPENAI_API_KEY` is set + - API fallback when Voyage is not configured - Semantic embeddings via API, truncated to `VECTOR_SIZE` via Matryoshka (OpenAI native only) - If `VECTOR_SIZE` > 1536, auto-upgrades to `text-embedding-3-large` (set `EMBEDDING_MODEL=text-embedding-3-large` to silence) - Supports any OpenAI-compatible endpoint via `OPENAI_BASE_URL` (OpenRouter, LiteLLM, vLLM, Azure, etc.) for both embeddings and classification/enrichment LLM calls @@ -155,23 +157,25 @@ AutoMem uses a provider pattern with multiple embedding backends: 3. **Ollama** (`ollama:nomic-embed-text`) - If `OLLAMA_BASE_URL` or `OLLAMA_MODEL` is configured - Fully local, easy model swapping - Requires running Ollama server + - Use `ollama pull bge-m3`, then `OLLAMA_MODEL=bge-m3` with `VECTOR_SIZE=1024` for local/self-hosted multilingual BGE-M3 4. **FastEmbed** (`fastembed:BAAI/bge-base-en-v1.5`) - Local ONNX model - Good quality semantic embeddings - No API key or internet required (after first download) - Downloads ~210MB model to `~/.config/automem/models/` on first use - 768 dimensions (default), also supports 384 and 1024 dim models + - Local/self-hosted only: its 1024d cloud fallback can consume roughly 4 GB RSS and raise hosting cost substantially 5. **Placeholder** (`placeholder`) - Hash-based fallback - Deterministic vectors from content hash - No semantic meaning, last resort only -**Upgrade safety:** `VECTOR_SIZE_AUTODETECT=true` (default) automatically adopts your existing collection dimension on startup. No manual action needed when updating. To enforce strict matching, set `VECTOR_SIZE_AUTODETECT=false`. +**Upgrade safety:** `VECTOR_SIZE_AUTODETECT=true` (default) only adopts an existing collection dimension. It does not make vectors from different models compatible: any provider or model change requires recreating the Qdrant collection and a full re-embed, including a swap between two 1024d models. To enforce strict dimension matching, set `VECTOR_SIZE_AUTODETECT=false`. #### Provider Configuration Control via `EMBEDDING_PROVIDER` environment variable: -- `auto` (default): Try Voyage → OpenAI → Ollama → FastEmbed → Placeholder +- `auto` (default): Try Voyage → OpenAI → Ollama (only when configured) → FastEmbed → Placeholder - `voyage`: Use Voyage only (fail if unavailable) - `openai`: Use OpenAI only (fail if unavailable). Also works with OpenAI-compatible providers when `OPENAI_BASE_URL` is set. - `ollama`: Use Ollama only (fail if unavailable) @@ -226,7 +230,7 @@ QDRANT_HOST= # OR hostname for self-hosted Qdrant (e.g. "qdrant" QDRANT_PORT=6333 # Port for self-hosted Qdrant (used with QDRANT_HOST) QDRANT_API_KEY= # Qdrant Cloud API key (optional, not needed for self-hosted) QDRANT_COLLECTION=memories # Collection name -VECTOR_SIZE=1024 # Embedding dimensions (1024 for voyage-4, 768 for small, 3072 for large) +VECTOR_SIZE=1024 # Embedding dimensions (1024 for voyage-4 or Ollama bge-m3, 768 for small, 3072 for large) VECTOR_SIZE_AUTODETECT=true # Adopt existing collection dim on startup (false = fail on mismatch) # API configuration @@ -236,8 +240,12 @@ ADMIN_API_TOKEN= # For admin endpoints # Embedding configuration EMBEDDING_PROVIDER=auto # auto|voyage|openai|ollama|local|placeholder +VOYAGE_API_KEY= # Recommended for new cloud deployments +VOYAGE_MODEL=voyage-4 OPENAI_API_KEY= # For OpenAI or compatible provider (optional) OPENAI_BASE_URL= # Custom endpoint for OpenAI-compatible APIs used by embeddings and classification/enrichment (optional) +OLLAMA_BASE_URL= # Intentional local/self-hosted Ollama endpoint +OLLAMA_MODEL= # e.g. bge-m3 (requires `ollama pull bge-m3`) # Consolidation intervals (seconds) CONSOLIDATION_DECAY_INTERVAL_SECONDS=86400 # 1 day (default) diff --git a/INSTALLATION.md b/INSTALLATION.md index 3a6a0031..cc974941 100644 --- a/INSTALLATION.md +++ b/INSTALLATION.md @@ -125,7 +125,7 @@ _Screenshot pending: Railway services dashboard image will be added at `docs/img 1. Sign in with GitHub (if not logged in) 2. Review environment variables -3. (Optional) Add `OPENAI_API_KEY` for real embeddings instead of mock embeddings +3. Add `VOYAGE_API_KEY` for the recommended `voyage-4` embeddings (OpenAI is supported as the API fallback) 4. Click **"Deploy"** 5. Wait ~60 seconds for deployment to complete ✅ @@ -209,7 +209,9 @@ Reference these in AutoMem config via `${{service..internalHost}}` | `ADMIN_API_TOKEN` | Token for admin/enrichment endpoints | ✅ Yes | | `FALKORDB_HOST` | Internal hostname of FalkorDB service | ✅ Yes | | `FALKORDB_PORT` | FalkorDB port (usually `6379`) | ✅ Yes | - | `OPENAI_API_KEY` | Enables real embeddings | Recommended | + | `VOYAGE_API_KEY` | Recommended API key for `voyage-4` embeddings | Recommended | + | `OPENAI_API_KEY` | OpenAI-compatible embedding fallback | Optional | + | `EMBEDDING_PROVIDER` | Provider selection (`auto` prefers Voyage) | `auto` | | `FALKORDB_PASSWORD` | Password if set on FalkorDB | If enabled | | `QDRANT_URL` | Qdrant Cloud endpoint | Optional | | `QDRANT_API_KEY` | Qdrant API key | If using Qdrant | @@ -357,7 +359,7 @@ curl -X POST https://your-automem.railway.app/memory \ 3. **Set up monitoring** (optional): See [Health Monitoring Guide](docs/HEALTH_MONITORING.md) 👉 **[Full Railway Guide](docs/RAILWAY_DEPLOYMENT.md)** - Advanced configuration, monitoring, troubleshooting -👉 **[Deployment Checklist](docs/DEPLOYMENT_CHECKLIST.md)** - Step-by-step verification +👉 **[Post-Deploy Checklist](docs/RAILWAY_DEPLOYMENT.md#post-deploy-checklist)** - Step-by-step verification --- @@ -411,7 +413,7 @@ export QDRANT_URL=http://localhost:6333 # Full URL (Qdrant Cloud or explicit) python app.py ``` -The API will use deterministic placeholder embeddings if no `OPENAI_API_KEY` or Qdrant is configured. +Without Qdrant, semantic vector storage is unavailable. With Qdrant and `EMBEDDING_PROVIDER=auto`, AutoMem tries configured Voyage, OpenAI, explicitly configured Ollama, and then local FastEmbed. For Railway, configure Voyage rather than depending on the FastEmbed fallback. --- @@ -466,12 +468,14 @@ Admin operations additionally require `X-Admin-Token: ` header. | `VOYAGE_MODEL` | Voyage model (Voyage provider) | `voyage-4` | | `OPENAI_API_KEY` | API key (OpenAI or compatible provider) | _unset_ | | `OPENAI_BASE_URL` | Custom endpoint for OpenAI-compatible providers | _unset_ | +| `OLLAMA_BASE_URL` | Ollama endpoint for intentional local/self-hosted use | `http://localhost:11434` | +| `OLLAMA_MODEL` | Pulled Ollama embedding model | `nomic-embed-text` | 👉 **New to Qdrant?** See the [Qdrant Setup Guide](docs/QDRANT_SETUP.md) for setup options (self-hosted on Railway or Qdrant Cloud). -> **Upgrade safety:** `VECTOR_SIZE_AUTODETECT=true` (default) automatically adopts your existing collection dimension on startup. No manual action needed when updating — existing 3072d or 768d collections continue to work. +> **Upgrade safety:** `VECTOR_SIZE_AUTODETECT=true` (default) adopts an existing collection dimension to avoid a startup mismatch. It does not migrate vectors: changing embedding provider or model requires recreating the collection and re-embedding, even when both models are 1024d. > -> The recommended setup is Voyage (`voyage-4`) at 1024d. If you only have an OpenAI key, `text-embedding-3-small` is used as fallback and truncated to `VECTOR_SIZE` via Matryoshka. +> The recommended setup is Voyage (`voyage-4`) at 1024d. If Voyage is unavailable, an OpenAI-compatible provider is the API fallback. Use FastEmbed only for intentional local/self-hosted deployments; its 1024d cloud fallback can add roughly 4 GB RSS and materially increase hosting cost. #### Enrichment Pipeline @@ -610,7 +614,7 @@ Store a new memory. - **Explicit `type` preferred**: Send `type` when you know the classification for immediate, accurate categorization - **Auto-classification fallback**: Omit `type` to let enrichment pipeline classify based on content -- **Embedding auto-generation**: Service generates real embeddings (OpenAI) or placeholder vectors if omitted +- **Embedding auto-generation**: Service generates vectors through the configured provider; `auto` prefers Voyage, then OpenAI, configured Ollama, FastEmbed, and finally placeholders - **Timestamp defaults**: All time fields default to current UTC time if not provided - **Background enrichment**: Entity extraction and relationship building queued automatically - **Type validation**: Invalid types return `400 Bad Request` with list of valid options @@ -894,6 +898,10 @@ Perfect for migrations or after updating embedding model. ## Migration +For the current migration-script catalog, safety classification, and CLI flags, +see [scripts/README.md](scripts/README.md). Follow the migration runbook below +for the required backup and validation sequence. + ### From MCP SQLite Memory Service Use the migration helper to transfer memories from legacy MCP SQLite: @@ -994,7 +1002,7 @@ AUTOMEM_RUN_INTEGRATION_TESTS=1 \ - `AUTOMEM_ALLOW_LIVE=1` - Required for non-localhost endpoints - `AUTOMEM_TEST_API_TOKEN` / `AUTOMEM_TEST_ADMIN_TOKEN` - Auth tokens -See **[TESTING.md](TESTING.md)** for complete testing documentation. +See **[TESTING.md](docs/TESTING.md)** for complete testing documentation. --- diff --git a/Makefile b/Makefile index 31aa92c5..472c4b57 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Makefile - Development commands -.PHONY: help install dev stop test fmt lint test-integration test-live test-locomo test-locomo-live test-longmemeval test-longmemeval-live test-longmemeval-watch clean logs deploy deploy-check bench-current-state bench-health +.PHONY: help install dev stop test test-node test-parity fmt lint test-integration test-live test-locomo test-locomo-live test-longmemeval test-longmemeval-live test-longmemeval-watch clean logs deploy deploy-check bench-current-state bench-health VENV_DIR := $(if $(wildcard .venv/bin/python),.venv,venv) VENV_BIN := $(VENV_DIR)/bin @@ -67,6 +67,40 @@ test: $(MAKE) install; \ fi PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 $(VENV_BIN)/pytest -rs -m unit + @$(MAKE) test-node + +# Run the MCP bridge's node suite (parity tests self-skip without a live stack) +test-node: + @echo "🧪 Running MCP bridge node tests..." + @cd mcp-sse-server && npm ci --silent && npm test + +# Diff the remote and stdio MCP transports against one live AutoMem. +# See docs/MCP_TRANSPORT_PARITY.md. +# Deliberately not 8001. The harness writes fixtures and bulk-deletes by tag, +# and a developer running a local AutoMem install (~/.automem/server) already +# holds 8001 — pointing this at that instance would seed test data into a real +# memory store. Own port, own stack, no collision. +PARITY_PORT ?= 8011 + +test-parity: + @echo "🐳 Starting Docker services on port $(PARITY_PORT)..." + @AUTOMEM_API_HOST_PORT=$(PARITY_PORT) AUTOMEM_API_TOKEN=test-token ADMIN_API_TOKEN=test-admin-token docker compose up -d + @echo "⏳ Waiting for AutoMem to be ready..." + @ready=0; \ + for i in $$(seq 1 60); do \ + if curl -fsS http://localhost:$(PARITY_PORT)/health > /dev/null 2>&1; then ready=1; break; fi; \ + sleep 1; \ + done; \ + if [ "$$ready" != "1" ]; then \ + echo "❌ AutoMem did not become healthy on port $(PARITY_PORT) within 60s"; \ + docker compose logs --tail=50 flask-api; \ + exit 1; \ + fi + @echo "🧪 Diffing MCP transports..." + @cd mcp-sse-server && npm ci --silent && \ + AUTOMEM_RUN_PARITY_TESTS=1 \ + AUTOMEM_PARITY_API_URL=http://localhost:$(PARITY_PORT) \ + npm test # Format code fmt: diff --git a/README.md b/README.md index 0744fcba..e40c43cb 100644 --- a/README.md +++ b/README.md @@ -8,49 +8,38 @@ Discord X LoCoMo on the neutral Agent Memory Benchmark + LongMemEval full on AutoMem's internal harness: 87.0% BEAM 10M on the neutral Agent Memory Benchmark Deploy on Railway

- Long-term memory for AI assistants. Graph + vector. Runs on your hardware. + Long-term memory for AI assistants — fast, private, and yours.

- - # AutoMem -Your AI forgets between sessions. RAG dumps documents that look similar. Vector databases match keywords but miss meaning. None of them learn. +[AutoMem](https://automem.ai) gives your AI a memory that survives the chat. + +Save the decisions, preferences, notes, and context that matter. The next time you open Claude, Cursor, Codex, ChatGPT, or another connected assistant, it can bring back the right details instead of making you repeat yourself. -AutoMem stores typed relationships *and* embeddings. When you ask "why did we choose PostgreSQL?", recall returns not just the matching memory — but the alternatives you considered, the principle behind the choice, and the related decisions that came after. +Ask, “Why did we choose PostgreSQL?” and AutoMem can return the decision, the alternatives you considered, the principle behind it, and the work that followed — not just a pile of similarly worded snippets. -On its *own* internal harness, AutoMem scores **87.00%** on LongMemEval full (**97.00%** recall@5) and **84.74%** on LoCoMo full — but those are engineering baselines scored by a self-hosted judge (`gpt-5.4-mini`), and the same answers can swing ~12 points on judge model alone. For comparison against other systems, the numbers that matter come from the **neutral** third-party board below. See [`benchmarks/EXPERIMENT_LOG.md`](benchmarks/EXPERIMENT_LOG.md) for methodology, judge policy, category breakdowns, and historical runs. +## Why AutoMem feels different -### On the neutral Agent Memory Benchmark +**No LLM call in the middle of recall.** AutoMem looks up memories directly through its graph and vector index. That keeps normal retrieval fast and avoids an extra generative-LLM charge every time your assistant needs context. It still uses embeddings — from Voyage, OpenAI, or a local provider — and optional enrichment can add more structure over time. -AutoMem **0.16.0** was run through the neutral [Agent Memory Benchmark](https://automem.ai/benchmarks) (AMB, by vectorize-io) on a self-spinning FalkorDB + Qdrant stack with **FastEmbed-local `bge-base-en-v1.5` (768d)** — no embedding API keys. The honest summary: AutoMem's strength is **large-context scaling and efficiency**, not verbatim conversational recall. +**More than vector search.** A vector match finds something similar; AutoMem also records typed relationships between memories. It can follow the connections between a decision, its rationale, and its consequences, so your assistant has a better chance of returning the *why*, not only the words it recognizes. -- **BEAM is the apples-to-apples axis** (same benchmark, same Gemini answerer + judge). AutoMem scores above Honcho at every BEAM tier, and the gap widens with scale: **+4.5pp at 100k, +0.7pp at 500k, +0.7pp at 1M, +16.8pp at 10M**. AutoMem degrades gracefully — **67.5% → 57.4%** (−10pp) across a 100× haystack increase — while Honcho holds roughly flat through 1M, then drops to 40.6% at 10M. That places AutoMem **#2 on BEAM**, behind vectorize's own Hindsight (~73→64% across the curve). -- **At 10M tokens, AutoMem holds 57.4% ±5.5%** while Honcho falls to ~41%. At that scale, context-stuffing is physically impossible, so the score reflects retrieval architecture, not context window. -- **Efficiency is architectural:** AutoMem feeds the answerer **~2.6–4.8k context tokens** at every scale (mean), versus 17–27k for the board leader on BEAM. -- **The honest other half:** on conversational Core-3, AutoMem **trails** the AMB leader Hindsight — locomo 85.1% vs 92%, longmemeval 74.4% vs 94.6%, personamem 76.1% vs 86.6%. Pick AutoMem for large-context scaling and efficiency, not for top-of-board verbatim recall. +**One memory across your tools.** Use the local MCP bridge with Claude Desktop, Cursor, Claude Code, Codex, Copilot, and more. For cloud agents, Remote MCP connects the same service to ChatGPT Developer Mode, Claude.ai, and ElevenLabs over HTTPS. Your memory is not locked to one chat app. -Outputs are committed and public, and [`AUTOMEM_REPRODUCE.md`](https://automem.ai/benchmarks) gives one command per split so you can **run it yourself**. AutoMem is **submitted to the neutral board ([provider PR #24](https://github.com/vectorize-io/agent-memory-benchmark/pull/24), under review)** — not yet live on the public leaderboard. Full head-to-head numbers live at [automem.ai/benchmarks](https://automem.ai/benchmarks). +**Own the data and the setup.** Run AutoMem locally with Docker, on your own infrastructure, or as a small Railway service group. It exposes both MCP and a REST API, so it fits into the tools and workflows you already use. -## Should you use AutoMem? +## Proven where long context gets hard -| Use AutoMem if... | Look elsewhere if... | -|---|---| -| You want one memory across Claude / Cursor / ChatGPT / Codex | You need SOC2 / HIPAA audit logs and row-level ACLs | -| You're comfortable self-hosting (Docker or Railway) | You want a managed SaaS with a polished dashboard | -| You're a solo dev, prosumer, or small team | You're running a multi-agent swarm needing per-agent memory isolation | -| You want to own your memory data | You need an enterprise SLA and dedicated support | +On the independent [Agent Memory Benchmark](https://automem.ai/benchmarks)'s BEAM long-context tests, AutoMem scored **57.4% at 10 million source tokens** while giving the answerer an average of only **~2.6–4.8k retrieved tokens**. That is the kind of efficiency that lets memory stay useful as an agent's history grows. -If your row is on the right, AutoMem isn't it — yet. Try [Mem0](https://mem0.ai), [Letta](https://letta.com), or [Zep](https://www.getzep.com) instead. +The full picture — test setup, raw outputs, methodology, historical runs, and reproduction commands — is in [automem.ai/benchmarks](https://automem.ai/benchmarks) and [`benchmarks/EXPERIMENT_LOG.md`](benchmarks/EXPERIMENT_LOG.md). ## How it works diff --git a/app.py b/app.py index b24e1b4e..89591a62 100644 --- a/app.py +++ b/app.py @@ -32,7 +32,13 @@ UnexpectedResponse = Exception # type: ignore[misc,assignment] try: # Allow tests to import without full qdrant client installed - from qdrant_client.models import Distance, PayloadSchemaType, PointStruct, VectorParams + from qdrant_client.models import ( + Distance, + HnswConfigDiff, + PayloadSchemaType, + PointStruct, + VectorParams, + ) except Exception: # pragma: no cover - degraded import path try: from qdrant_client.http import models as _qmodels @@ -40,9 +46,10 @@ Distance = getattr(_qmodels, "Distance", None) PointStruct = getattr(_qmodels, "PointStruct", None) VectorParams = getattr(_qmodels, "VectorParams", None) + HnswConfigDiff = getattr(_qmodels, "HnswConfigDiff", None) PayloadSchemaType = getattr(_qmodels, "PayloadSchemaType", None) except Exception: - Distance = PointStruct = VectorParams = None + Distance = PointStruct = VectorParams = HnswConfigDiff = None PayloadSchemaType = None # Provide a simple PointStruct shim for tests/environments lacking qdrant models @@ -296,6 +303,7 @@ def require_api_token() -> None: collection_name=COLLECTION_NAME, get_effective_vector_size_fn=get_effective_vector_size, vector_params_cls=VectorParams, + hnsw_config_diff_cls=HnswConfigDiff, distance_enum=Distance, payload_schema_type_enum=PayloadSchemaType, get_init_falkordb_fn=lambda: init_falkordb, @@ -318,6 +326,7 @@ def require_api_token() -> None: classification_model=CLASSIFICATION_MODEL, logger=logger, stats=state.classification_stats, + circuit=state.enrichment_circuit, ) diff --git a/automem/api/enrichment.py b/automem/api/enrichment.py index ba5b2ae9..c8841ed4 100644 --- a/automem/api/enrichment.py +++ b/automem/api/enrichment.py @@ -30,6 +30,7 @@ def enrichment_status() -> Any: "max_attempts": max_attempts, "stats": state.enrichment_stats.to_dict(), "classification": state.classification_stats.to_dict(), + "circuit": state.enrichment_circuit.to_dict(), } return jsonify(response) diff --git a/automem/api/memory.py b/automem/api/memory.py index 95bc7c76..4f9fd330 100644 --- a/automem/api/memory.py +++ b/automem/api/memory.py @@ -8,6 +8,7 @@ from flask import Blueprint, abort, jsonify, request from flask.typing import ResponseReturnValue +from automem.api.stream import emit_event, preview_text from automem.config import ( CLASSIFICATION_MODEL, MEMORY_AUTO_SUMMARIZE, @@ -248,6 +249,15 @@ def _create_association_batch( _association_failure(row["index"], "One or both memories do not exist") ) + emit_event( + "memory.associate", + { + "count": len(succeeded), + "failed_count": len(failed), + "relation_types": sorted({item["relation_type"] for item in succeeded})[:5], + }, + utc_now_fn, + ) return _batch_association_response( succeeded=succeeded, failed=failed, @@ -504,6 +514,7 @@ def store() -> Any: openai_client, CLASSIFICATION_MODEL, MEMORY_SUMMARY_TARGET_LENGTH, + state.enrichment_circuit, ) if summary: original_content = content @@ -666,9 +677,6 @@ def store() -> Any: logger.error("FalkorDB store returned no memory row for %s", memory_id) abort(500, description="Failed to store memory in FalkorDB") - # Queue enrichment - enqueue_enrichment(memory_id) - # Handle embeddings embedding_status = "skipped" qdrant_client = get_qdrant_client() @@ -751,6 +759,21 @@ def store() -> Any: "enrichment_queued": bool(state.enrichment_queue), }, ) + emit_event( + "memory.store", + { + "id": memory_id, + "content_preview": preview_text(content), + "type": memory_type, + "importance": importance, + "tags": tags[:5], + "size_bytes": len(content.encode("utf-8")), + "elapsed_ms": int(response["query_time_ms"]), + "count": 1, + }, + utc_now, + ) + enqueue_enrichment(memory_id) return jsonify(response), 201 @bp.route("/memory/", methods=["GET"]) @@ -792,6 +815,24 @@ def update(memory_id: str) -> Any: payload = request.get_json(silent=True) if not isinstance(payload, dict): abort(400, description="JSON body is required") + changed_fields = sorted( + key + for key in payload + if key + in { + "content", + "tags", + "importance", + "type", + "confidence", + "timestamp", + "metadata", + "t_valid", + "t_invalid", + "updated_at", + "last_accessed", + } + ) graph = get_memory_graph() if graph is None: @@ -937,6 +978,18 @@ def update(memory_id: str) -> Any: collection_name, ) + emit_event( + "memory.update", + { + "id": memory_id, + "content_preview": preview_text(new_content), + "type": memory_type, + "importance": importance, + "tags": tags[:5] if isinstance(tags, list) else [], + "fields": changed_fields, + }, + utc_now, + ) return jsonify({"status": "success", "memory_id": memory_id}) @bp.route("/memory/", methods=["DELETE"]) @@ -959,6 +1012,14 @@ def delete(memory_id: str) -> Any: logger=logger, ) + emit_event( + "memory.delete", + { + "id": memory_id, + "count": 1, + }, + utc_now, + ) return jsonify({"status": "success", "memory_id": memory_id}) @bp.route("/memory/by-tag", methods=["GET", "DELETE"]) @@ -1005,6 +1066,14 @@ def by_tag() -> Any: ) deleted_count += len(memory_ids) + emit_event( + "memory.delete", + { + "count": deleted_count, + "tags": tags[:5], + }, + utc_now, + ) return jsonify({"status": "success", "tags": tags, "deleted_count": deleted_count}) memories, has_more = _load_memories_by_tag_page( @@ -1118,6 +1187,17 @@ def associate() -> Any: for prop in relation_config.get("properties", []): if prop in relationship_props: response[prop] = relationship_props[prop] + emit_event( + "memory.associate", + { + "memory1_id": memory1_id, + "memory2_id": memory2_id, + "relation_type": relation_type, + "strength": strength, + "count": 1, + }, + utc_now, + ) return jsonify(response), 201 @bp.route("/memory/batch", methods=["POST"]) @@ -1169,6 +1249,7 @@ def store_batch() -> Any: openai_client, CLASSIFICATION_MODEL, MEMORY_SUMMARY_TARGET_LENGTH, + state.enrichment_circuit, ) if summary: logger.info( @@ -1355,10 +1436,6 @@ def store_batch() -> Any: for v in validated: enqueue_embedding(v["id"], v["content"]) qdrant_status = "queued" - # Queue enrichment for all - for v in validated: - enqueue_enrichment(v["id"]) - elapsed_ms = round((time.perf_counter() - query_start) * 1000, 2) logger.info( "batch_stored", @@ -1368,6 +1445,18 @@ def store_batch() -> Any: "qdrant_status": qdrant_status, }, ) + emit_event( + "memory.store", + { + "count": len(validated), + "ids": [item["id"] for item in validated][:5], + "content_preview": f"{len(validated)} memories stored", + "elapsed_ms": int(elapsed_ms), + }, + utc_now, + ) + for v in validated: + enqueue_enrichment(v["id"]) return ( jsonify( diff --git a/automem/api/recall.py b/automem/api/recall.py index c12eac28..9fddb41d 100644 --- a/automem/api/recall.py +++ b/automem/api/recall.py @@ -9,6 +9,7 @@ from flask import Blueprint, abort, jsonify, request +from automem.api.stream import emit_event, preview_text from automem.config import ( COLLECTION_NAME, DEFAULT_EXPAND_RELATIONS, @@ -29,7 +30,7 @@ ) from automem.search.runtime_recall_helpers import _hydrate_vector_relations from automem.utils.graph import _serialize_node -from automem.utils.time import _parse_iso_datetime, query_has_temporal_intent +from automem.utils.time import _parse_iso_datetime, query_has_temporal_intent, utc_now DEFAULT_STYLE_PRIORITY_TAGS: Set[str] = { "coding-style", @@ -2597,6 +2598,18 @@ def _rank_local_results(results: List[Dict[str, Any]]) -> List[Dict[str, Any]]: ), }, ) + recall_preview = query_text or " | ".join(q for q in queries_to_run if q) + emit_event( + "memory.recall", + { + "query": preview_text(recall_preview, 50) if recall_preview else "(no query)", + "limit": limit, + "result_count": len(results), + "elapsed_ms": int(response["query_time_ms"]), + "tags": tag_filters[:3] if tag_filters else [], + }, + utc_now, + ) # Update last_accessed for direct matches (not expanded related memories) if on_access and seed_results: diff --git a/automem/api/stream.py b/automem/api/stream.py index 98f0cb10..14f2bac8 100644 --- a/automem/api/stream.py +++ b/automem/api/stream.py @@ -3,6 +3,11 @@ Provides a /stream endpoint that emits events for memory operations, enrichment, and consolidation tasks. Uses an in-memory subscriber pattern with bounded queues per client. + +Event types: + memory.store / memory.recall / memory.update / memory.delete / memory.associate + enrichment.start / enrichment.complete / enrichment.failed + consolidation.run """ from __future__ import annotations @@ -19,6 +24,27 @@ _subscribers_lock = Lock() +def preview_text(value: Any, limit: int = 100) -> str: + """Truncate *value* for SSE payloads without leaking huge content.""" + text = "" if value is None else str(value) + return text[:limit] + "..." if len(text) > limit else text + + +def event_count(data: Dict[str, Any], default: int = 1) -> int: + """Read an SSE payload ``count``, preserving explicit zeros.""" + if not isinstance(data, dict) or "count" not in data: + return default + try: + return int(data["count"]) + except (TypeError, ValueError): + return default + + +def is_single_memory_store(data: Dict[str, Any]) -> bool: + """True when a ``memory.store`` payload describes one concrete memory.""" + return bool(isinstance(data, dict) and data.get("id") and event_count(data) == 1) + + def emit_event(event_type: str, data: Dict[str, Any], utc_now: Callable[[], str]) -> None: """Emit an event to all SSE subscribers. diff --git a/automem/backup.py b/automem/backup.py index 095c56b3..19e51ea4 100644 --- a/automem/backup.py +++ b/automem/backup.py @@ -14,6 +14,47 @@ VALID_BACKUP_INCLUDES = ("falkordb", "qdrant") STREAM_QUEUE_MAX_CHUNKS = 8 +# FalkorDB caps every result set at RESULTSET_SIZE and returns the short set +# with no error, so a batch that reaches the cap may have lost rows silently. +DEFAULT_RESULTSET_CAP = 10000 +INITIAL_RELATIONSHIP_ID_CHUNK = 256 +MAX_RELATIONSHIP_ID_CHUNK = 4096 +# A bulk export should not inherit an interactive query's TIMEOUT budget. +BACKUP_QUERY_TIMEOUT_MS = 300000 +# Concurrent writes shift the totals mid-export; a wider gap means lost rows. +BACKUP_COUNT_TOLERANCE = 0.01 + +NODE_EXPORT_QUERY = """ + MATCH (n) + WHERE id(n) >= {lo} AND id(n) < {hi} + RETURN + id(n) as id, + labels(n) as labels, + properties(n) as props + """ + +RELATIONSHIP_EXPORT_QUERY = """ + MATCH (a)-[r]->(b) + WHERE id(a) >= {lo} AND id(a) < {hi} + RETURN + id(a) as source_id, + type(r) as rel_type, + id(b) as target_id, + properties(r) as props + """ + +SINGLE_NODE_RELATIONSHIP_EXPORT_QUERY = """ + MATCH (a)-[r]->(b) + WHERE id(a) = {node_id} + RETURN + id(a) as source_id, + type(r) as rel_type, + id(b) as target_id, + properties(r) as props + ORDER BY id(r) + SKIP {offset} LIMIT {limit} + """ + class BackupError(RuntimeError): """Raised when backup creation fails.""" @@ -70,89 +111,204 @@ def _query_rows(result: Any) -> list[Any]: return list(getattr(result, "result_set", []) or []) -def export_falkordb_artifact( - *, - graph: Any, - graph_name: str, - timestamp: str, - batch_size: int = 10000, - logger: Any = None, -) -> BackupArtifact: - """Export FalkorDB graph data as a compressed JSON backup artifact.""" - if graph is None: - raise BackupError("FalkorDB is unavailable") +def _backup_rows(graph: Any, query: str) -> list[Any]: + return _query_rows(graph.query(query, timeout=BACKUP_QUERY_TIMEOUT_MS)) - nodes: list[dict[str, Any]] = [] - offset = 0 - while True: - rows = _query_rows( - graph.query( - f""" - MATCH (n) - RETURN - id(n) as id, - labels(n) as labels, - properties(n) as props - SKIP {offset} LIMIT {batch_size} - """ - ) - ) - if not rows: - break +def _backup_scalar(graph: Any, query: str) -> Any: + rows = _backup_rows(graph, query) + if not rows or not rows[0]: + return None + return rows[0][0] + + +def _resultset_cap(graph: Any, batch_size: int, logger: Any = None) -> int: + """Largest batch that FalkorDB will return in full.""" + configured = DEFAULT_RESULTSET_CAP + try: + raw = graph.execute_command("GRAPH.CONFIG", "GET", "RESULTSET_SIZE") + reported = int(raw[1]) + # 0 or negative means unlimited. + if reported > 0: + configured = reported + except Exception as exc: # noqa: BLE001 - config read is best-effort + if logger: + logger.debug("Could not read RESULTSET_SIZE (%s); assuming %d", exc, configured) - for row in rows: - nodes.append({"id": row[0], "labels": row[1], "properties": row[2]}) + return max(1, min(batch_size, configured)) - if logger: + +def _export_falkordb_nodes( + graph: Any, max_node_id: int, cap: int, logger: Any = None +) -> list[dict[str, Any]]: + nodes: list[dict[str, Any]] = [] + # One row per node, so half the cap can never reach it. + chunk = max(1, cap // 2) + lo = 0 + + while lo <= max_node_id: + hi = min(lo + chunk, max_node_id + 1) + rows = _backup_rows(graph, NODE_EXPORT_QUERY.format(lo=lo, hi=hi)) + nodes.extend({"id": row[0], "labels": row[1], "properties": row[2]} for row in rows) + + if logger and rows: logger.info( "Exported FalkorDB node batch: %d nodes (total: %d)", len(rows), len(nodes), ) - if len(rows) < batch_size: - break - offset += batch_size + lo = hi + + return nodes + +def _export_node_relationships(graph: Any, node_id: int, cap: int) -> list[dict[str, Any]]: + """Page one node's out-edges, for a node whose degree alone reaches the cap.""" relationships: list[dict[str, Any]] = [] offset = 0 while True: - rows = _query_rows( - graph.query( - f""" - MATCH (a)-[r]->(b) - RETURN - id(a) as source_id, - type(r) as rel_type, - id(b) as target_id, - properties(r) as props - SKIP {offset} LIMIT {batch_size} - """ - ) + rows = _backup_rows( + graph, + SINGLE_NODE_RELATIONSHIP_EXPORT_QUERY.format(node_id=node_id, offset=offset, limit=cap), ) - if not rows: - break + relationships.extend( + { + "source_id": row[0], + "type": row[1], + "target_id": row[2], + "properties": row[3], + } + for row in rows + ) + if len(rows) < cap: + return relationships + offset += cap - for row in rows: - relationships.append( - { - "source_id": row[0], - "type": row[1], - "target_id": row[2], - "properties": row[3], - } - ) - if logger: +def _export_falkordb_relationships( + graph: Any, max_node_id: int, cap: int, logger: Any = None +) -> list[dict[str, Any]]: + relationships: list[dict[str, Any]] = [] + chunk = INITIAL_RELATIONSHIP_ID_CHUNK + lo = 0 + + while lo <= max_node_id: + hi = min(lo + chunk, max_node_id + 1) + rows = _backup_rows(graph, RELATIONSHIP_EXPORT_QUERY.format(lo=lo, hi=hi)) + + # At the cap the result set may have been truncated server-side, and a + # truncated batch is indistinguishable from a complete one. Narrow the + # range and retry rather than trust it. + reached_cap = len(rows) >= cap + if reached_cap and hi - lo > 1: + chunk = max(1, (hi - lo) // 4) + continue + if reached_cap: + relationships.extend(_export_node_relationships(graph, lo, cap)) + if logger: + logger.info( + "Exported FalkorDB relationships for high-degree node %d (total: %d)", + lo, + len(relationships), + ) + lo += 1 + chunk = INITIAL_RELATIONSHIP_ID_CHUNK + continue + + relationships.extend( + { + "source_id": row[0], + "type": row[1], + "target_id": row[2], + "properties": row[3], + } + for row in rows + ) + if logger and rows: logger.info( "Exported FalkorDB relationship batch: %d relationships (total: %d)", len(rows), len(relationships), ) - if len(rows) < batch_size: - break - offset += batch_size + + lo = hi + room_to_grow = len(rows) * 3 < cap and chunk < MAX_RELATIONSHIP_ID_CHUNK + if room_to_grow: + chunk = min(chunk * 2, MAX_RELATIONSHIP_ID_CHUNK) + + return relationships + + +def _verify_export_count( + *, kind: str, exported: int, before: int, after: int, logger: Any = None +) -> None: + """Fail a truncated export instead of writing a backup that looks complete.""" + expected = min(before, after) + if exported >= expected: + return + + shortfall = expected - exported + if exported < expected * (1 - BACKUP_COUNT_TOLERANCE): + raise BackupError( + f"FalkorDB {kind} export is short by {shortfall} " + f"(exported {exported}, graph held {expected}) - refusing to write a partial backup" + ) + if logger: + logger.warning( + "FalkorDB %s export is short by %d (exported %d, graph held %d); " + "within tolerance for concurrent writes", + kind, + shortfall, + exported, + expected, + ) + + +def export_falkordb_artifact( + *, + graph: Any, + graph_name: str, + timestamp: str, + batch_size: int = 10000, + logger: Any = None, +) -> BackupArtifact: + """Export FalkorDB graph data as a compressed JSON backup artifact. + + Pages on internal node id ranges, which FalkorDB plans as ``NodeByIdSeek``. + ``SKIP `` re-scans every skipped row, so its cost grows with depth + until batches exceed the server's ``TIMEOUT`` and the export dies partway + through a large graph. + """ + if graph is None: + raise BackupError("FalkorDB is unavailable") + + cap = _resultset_cap(graph, batch_size, logger) + max_node_id = _backup_scalar(graph, "MATCH (n) RETURN max(id(n))") + node_count_before = _backup_scalar(graph, "MATCH (n) RETURN count(n)") or 0 + relationship_count_before = _backup_scalar(graph, "MATCH ()-[r]->() RETURN count(r)") or 0 + + if max_node_id is None: + nodes: list[dict[str, Any]] = [] + relationships: list[dict[str, Any]] = [] + else: + nodes = _export_falkordb_nodes(graph, int(max_node_id), cap, logger) + relationships = _export_falkordb_relationships(graph, int(max_node_id), cap, logger) + + _verify_export_count( + kind="node", + exported=len(nodes), + before=node_count_before, + after=_backup_scalar(graph, "MATCH (n) RETURN count(n)") or 0, + logger=logger, + ) + _verify_export_count( + kind="relationship", + exported=len(relationships), + before=relationship_count_before, + after=_backup_scalar(graph, "MATCH ()-[r]->() RETURN count(r)") or 0, + logger=logger, + ) stats = { "node_count": len(nodes), diff --git a/automem/classification/memory_classifier.py b/automem/classification/memory_classifier.py index 17f6509e..9f9721fd 100644 --- a/automem/classification/memory_classifier.py +++ b/automem/classification/memory_classifier.py @@ -97,6 +97,7 @@ def __init__( classification_model: str, logger: Any, stats: Any = None, + circuit: Any = None, ) -> None: self._normalize_memory_type = normalize_memory_type self._ensure_openai_client = ensure_openai_client @@ -104,6 +105,7 @@ def __init__( self._classification_model = classification_model self._logger = logger self._stats = stats + self._circuit = circuit def classify(self, content: str, *, use_llm: bool = True) -> tuple[str, float]: """Classify memory type and return confidence score.""" @@ -140,11 +142,19 @@ def classify(self, content: str, *, use_llm: bool = True) -> tuple[str, float]: return "Memory", 0.3 def _classify_with_llm(self, content: str) -> Optional[tuple[str, float]]: + was_probe = False + if self._circuit is not None: + allowed, was_probe = self._circuit.begin_request() + if not allowed: + self._logger.info("Skipping LLM classification while enrichment circuit is open") + return None client = self._get_openai_client() if client is None: self._ensure_openai_client() client = self._get_openai_client() if client is None: + if self._circuit is not None: + self._circuit.record_failure("OpenAI client unavailable", was_probe) return None extra_params: dict[str, Any] = {} @@ -157,15 +167,22 @@ def _classify_with_llm(self, content: str) -> Optional[tuple[str, float]]: extra_params["max_tokens"] = 50 extra_params["temperature"] = 0.3 - response = client.chat.completions.create( - model=self._classification_model, - messages=[ - {"role": "system", "content": self.SYSTEM_PROMPT}, - {"role": "user", "content": content[:1000]}, - ], - response_format={"type": "json_object"}, - **extra_params, - ) + try: + response = client.chat.completions.create( + model=self._classification_model, + messages=[ + {"role": "system", "content": self.SYSTEM_PROMPT}, + {"role": "user", "content": content[:1000]}, + ], + response_format={"type": "json_object"}, + **extra_params, + ) + except Exception as exc: + if self._circuit is not None: + self._circuit.record_failure(str(exc), was_probe) + raise + if self._circuit is not None: + self._circuit.record_success(was_probe) raw_content = response.choices[0].message.content if not raw_content: diff --git a/automem/config.py b/automem/config.py index 5241c621..555e9a9c 100644 --- a/automem/config.py +++ b/automem/config.py @@ -178,6 +178,8 @@ } # Target length for summarized content MEMORY_SUMMARY_TARGET_LENGTH = int(os.getenv("MEMORY_SUMMARY_TARGET_LENGTH", "300")) +# Cooldown after a definitive LLM quota exhaustion response. +ENRICHMENT_CIRCUIT_COOLDOWN_SECONDS = float(os.getenv("ENRICHMENT_CIRCUIT_COOLDOWN_SECONDS", "300")) # Memory types for classification MEMORY_TYPES = {"Decision", "Pattern", "Preference", "Style", "Habit", "Insight", "Context"} diff --git a/automem/service_runtime_bindings.py b/automem/service_runtime_bindings.py index 331940bd..550ecadb 100644 --- a/automem/service_runtime_bindings.py +++ b/automem/service_runtime_bindings.py @@ -42,6 +42,7 @@ def create_service_runtime( collection_name: str, get_effective_vector_size_fn: Callable[[], int], vector_params_cls: Any, + hnsw_config_diff_cls: Any, distance_enum: Any, payload_schema_type_enum: Any, get_init_falkordb_fn: Callable[[], Callable[[], None]], @@ -80,6 +81,7 @@ def ensure_qdrant_collection() -> None: vector_size_config=vector_size_config_fn(), get_effective_vector_size_fn=get_effective_vector_size_fn, vector_params_cls=vector_params_cls, + hnsw_config_diff_cls=hnsw_config_diff_cls, distance_enum=distance_enum, payload_schema_type_enum=payload_schema_type_enum, ) diff --git a/automem/service_state.py b/automem/service_state.py index e5831825..c84cbffa 100644 --- a/automem/service_state.py +++ b/automem/service_state.py @@ -1,5 +1,6 @@ from __future__ import annotations +import time from dataclasses import dataclass, field from queue import Queue from threading import Event, Lock, Thread @@ -8,11 +9,66 @@ from falkordb import FalkorDB from qdrant_client import QdrantClient -from automem.config import VECTOR_SIZE +from automem.config import ENRICHMENT_CIRCUIT_COOLDOWN_SECONDS, VECTOR_SIZE from automem.embedding.provider import EmbeddingProvider from automem.utils.time import utc_now +class EnrichmentCircuit: + """Fail-soft cooldown for definitive LLM quota failures.""" + + def __init__(self, cooldown_seconds: float = 300, clock: Any = time.monotonic) -> None: + self._cooldown_seconds = cooldown_seconds + self._clock = clock + self._opened_until = 0.0 + self._probe_pending = False + self._lock = Lock() + self.circuit_open_skips = 0 + self.recoveries = 0 + + def begin_request(self) -> tuple[bool, bool]: + """Reserve an LLM request and report whether it is the recovery probe.""" + with self._lock: + now = self._clock() + if now < self._opened_until: + self.circuit_open_skips += 1 + return False, False + if self._opened_until: + if self._probe_pending: + self.circuit_open_skips += 1 + return False, False + self._probe_pending = True + return True, True + return True, False + + def record_failure(self, error: str, was_probe: bool = False) -> bool: + if "insufficient_quota" not in error.lower() and "quota" not in error.lower(): + with self._lock: + if was_probe and self._probe_pending: + self._opened_until = 0.0 + self._probe_pending = False + return False + with self._lock: + self._opened_until = self._clock() + self._cooldown_seconds + self._probe_pending = False + return True + + def record_success(self, was_probe: bool = False) -> None: + with self._lock: + if was_probe and self._probe_pending: + self.recoveries += 1 + self._opened_until = 0.0 + self._probe_pending = False + + def to_dict(self) -> Dict[str, Any]: + with self._lock: + return { + "circuit_open_skips": self.circuit_open_skips, + "recoveries": self.recoveries, + "open": self._clock() < self._opened_until, + } + + @dataclass class EnrichmentStats: processed_total: int = 0 @@ -106,6 +162,9 @@ class ServiceState: enrichment_thread: Optional[Thread] = None enrichment_stats: EnrichmentStats = field(default_factory=EnrichmentStats) classification_stats: ClassificationStats = field(default_factory=ClassificationStats) + enrichment_circuit: EnrichmentCircuit = field( + default_factory=lambda: EnrichmentCircuit(ENRICHMENT_CIRCUIT_COOLDOWN_SECONDS) + ) enrichment_inflight: Set[str] = field(default_factory=set) enrichment_pending: Set[str] = field(default_factory=set) enrichment_lock: Lock = field(default_factory=Lock) diff --git a/automem/stores/runtime_clients.py b/automem/stores/runtime_clients.py index b7ae9390..df3bac47 100644 --- a/automem/stores/runtime_clients.py +++ b/automem/stores/runtime_clients.py @@ -3,6 +3,11 @@ import os from typing import Any, Callable +# Qdrant rejects an entire search with 400 when a filter references an unindexed payload +# field, so every field recall filters on must be indexed. `type` backs RECALL_EXCLUDED_TYPES, +# which applies to all vector searches since that setting defaults to a non-empty value. +QDRANT_FILTERED_PAYLOAD_FIELDS = ("tags", "tag_prefixes", "type") + def init_falkordb( *, @@ -113,6 +118,7 @@ def ensure_qdrant_collection( vector_size_config: int, get_effective_vector_size_fn: Callable[[Any], tuple[int, str]], vector_params_cls: Any, + hnsw_config_diff_cls: Any, distance_enum: Any, payload_schema_type_enum: Any, ) -> None: @@ -143,7 +149,13 @@ def ensure_qdrant_collection( ) state.qdrant.create_collection( collection_name=collection_name, - vectors_config=vector_params_cls(size=effective_dim, distance=distance_enum.COSINE), + vectors_config=vector_params_cls( + size=effective_dim, + distance=distance_enum.COSINE, + on_disk=True, + ), + hnsw_config=hnsw_config_diff_cls(on_disk=True), + on_disk_payload=True, ) ensure_payload_indexes = os.getenv("QDRANT_ENSURE_PAYLOAD_INDEXES", "true").lower() in { @@ -157,27 +169,12 @@ def ensure_qdrant_collection( return logger.info("Ensuring Qdrant payload indexes for collection '%s'", collection_name) - if payload_schema_type_enum: - state.qdrant.create_payload_index( - collection_name=collection_name, - field_name="tags", - field_schema=payload_schema_type_enum.KEYWORD, - ) - state.qdrant.create_payload_index( - collection_name=collection_name, - field_name="tag_prefixes", - field_schema=payload_schema_type_enum.KEYWORD, - ) - else: - state.qdrant.create_payload_index( - collection_name=collection_name, - field_name="tags", - field_schema="keyword", - ) + keyword_schema = payload_schema_type_enum.KEYWORD if payload_schema_type_enum else "keyword" + for field_name in QDRANT_FILTERED_PAYLOAD_FIELDS: state.qdrant.create_payload_index( collection_name=collection_name, - field_name="tag_prefixes", - field_schema="keyword", + field_name=field_name, + field_schema=keyword_schema, ) except ValueError: raise diff --git a/automem/utils/text.py b/automem/utils/text.py index 2c4a4c64..e6ad54c7 100644 --- a/automem/utils/text.py +++ b/automem/utils/text.py @@ -122,6 +122,7 @@ def summarize_content( openai_client: Any, model: str, target_length: int = 300, + circuit: Any = None, ) -> Optional[str]: """Summarize content using an LLM to fit within target length. @@ -137,9 +138,14 @@ def summarize_content( if openai_client is None: logger.warning("Cannot summarize: OpenAI client not available") return None - if not content or len(content) <= target_length: return content + was_probe = False + if circuit is not None: + allowed, was_probe = circuit.begin_request() + if not allowed: + logger.info("Skipping summarization while enrichment circuit is open") + return None try: system_prompt = SUMMARIZE_SYSTEM_PROMPT.format(target_length=target_length) @@ -163,6 +169,8 @@ def summarize_content( ], **extra_params, ) + if circuit is not None: + circuit.record_success(was_probe) summary = response.choices[0].message.content.strip() @@ -183,8 +191,10 @@ def summarize_content( ) return None - except Exception: + except Exception as exc: logger.exception("Memory summarization failed") + if circuit is not None: + circuit.record_failure(str(exc), was_probe) return None diff --git a/docker-compose.yml b/docker-compose.yml index 046d74fd..63bb34f7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -58,10 +58,22 @@ services: QDRANT_ENSURE_PAYLOAD_INDEXES: ${QDRANT_ENSURE_PAYLOAD_INDEXES:-true} AUTOMEM_API_TOKEN: ${AUTOMEM_API_TOKEN:-test-token} ADMIN_API_TOKEN: ${ADMIN_API_TOKEN:-test-admin-token} - OPENAI_API_KEY: ${OPENAI_API_KEY:-} VOYAGE_API_KEY: ${VOYAGE_API_KEY:-} + VOYAGE_MODEL: ${VOYAGE_MODEL:-voyage-4} + OPENAI_API_KEY: ${OPENAI_API_KEY:-} + OPENAI_BASE_URL: ${OPENAI_BASE_URL:-} + EMBEDDING_MODEL: ${EMBEDDING_MODEL:-text-embedding-3-small} + # Pass these through only when intentionally configured. This preserves + # Ollama's runtime defaults for EMBEDDING_PROVIDER=ollama and prevents + # auto-selection from treating an empty Compose value as configuration. + OLLAMA_BASE_URL: + OLLAMA_MODEL: + OLLAMA_TIMEOUT: ${OLLAMA_TIMEOUT:-30} + OLLAMA_MAX_RETRIES: ${OLLAMA_MAX_RETRIES:-2} VECTOR_SIZE: ${VECTOR_SIZE:-1024} - EMBEDDING_PROVIDER: ${EMBEDDING_PROVIDER:-auto} # auto|voyage|openai|local|placeholder + VECTOR_SIZE_AUTODETECT: ${VECTOR_SIZE_AUTODETECT:-true} + # auto tries Voyage, OpenAI, configured Ollama, FastEmbed, then placeholders. + EMBEDDING_PROVIDER: ${EMBEDDING_PROVIDER:-auto} MEMORY_CONTENT_HARD_LIMIT: ${MEMORY_CONTENT_HARD_LIMIT:-2000} MEMORY_AUTO_SUMMARIZE: ${MEMORY_AUTO_SUMMARIZE:-true} AUTOMEM_MODELS_DIR: /root/.config/automem/models # Keep in sync with volume mount diff --git a/docs/API.md b/docs/API.md index 1126577f..6616bc85 100644 --- a/docs/API.md +++ b/docs/API.md @@ -131,7 +131,7 @@ GET /recall?query=project%20plan&state_mode=history Enrichment - GET `/enrichment/status` - - Response: queue size, inflight/pending, stats, plus a `classification` block with type-classification counters (`llm_attempts`, `llm_successes`, `fallbacks`, `pattern_classifications`, `last_error`, `last_error_at`) for monitoring LLM-classification fallback rate. + - Response: queue size, inflight/pending, stats, plus a `classification` block with type-classification counters (`llm_attempts`, `llm_successes`, `fallbacks`, `pattern_classifications`, `last_error`, `last_error_at`) for monitoring LLM-classification fallback rate, and a `circuit` block with `open`, `circuit_open_skips`, and `recoveries` for monitoring quota-circuit state. - POST `/enrichment/reprocess` - Body: `{ "ids": ["..."] }` or query `?ids=a,b,c` @@ -160,6 +160,23 @@ Consolidation - GET `/consolidate/status` - Response: `{ "status": "success", "next_runs": {...}, "history": [...] }` +Stream + +- GET `/stream` + - Authenticated SSE feed of in-process memory operations. Each event is `data: {json}` with `{ "type", "timestamp", "data" }`. Keepalive comments (`: keepalive`) are sent every 30 seconds. + - Event types: + - `memory.store` — single store (`id`, `content_preview`, `type`, `importance`, `tags`, `size_bytes`, `elapsed_ms`, `count: 1`) or batch store (`count`, `ids`, `content_preview`, `elapsed_ms`) + - `memory.recall` — ranked recall (`query`, `limit`, `result_count`, `elapsed_ms`, `tags`) + - `memory.update` — patch (`id`, `content_preview`, `type`, `importance`, `tags`, `fields`) + - `memory.delete` — single delete (`id`, `count: 1`) or bulk-by-tag (`count`, `tags`) + - `memory.associate` — single edge (`memory1_id`, `memory2_id`, `relation_type`, `strength`, `count: 1`) or batch (`count`, `failed_count`, `relation_types`) + - `enrichment.start` / `enrichment.complete` / `enrichment.failed` + - `consolidation.run` + - Failed validation (`4xx`) does not emit. Watch with `python scripts/automem_watch.py --url … --token …`; see the [scripts catalog](../scripts/README.md#operations-and-maintenance) for its usage and safety notes. + +- GET `/stream/status` + - Response: `{ "subscribers": N }` + Notes - Tag matching supports exact and prefix semantics; vector searches are filtered by tag conditions when provided. diff --git a/docs/COMPARISON.md b/docs/COMPARISON.md index 92f422b0..b1b355dc 100644 --- a/docs/COMPARISON.md +++ b/docs/COMPARISON.md @@ -2,7 +2,7 @@ This doc covers how AutoMem differs from the three things people most often compare it against: traditional RAG, pure vector databases, and rolling your own memory layer. It also covers the recall scoring formula in enough detail to evaluate whether AutoMem will solve your specific retrieval problem. -For the short version, skip to the README's [Should you use AutoMem?](../README.md#should-you-use-automem) section. For research foundations, see [RESEARCH.md](RESEARCH.md). +For the short version, start with the README's [Why AutoMem feels different](../README.md#why-automem-feels-different) section. For research foundations, see [RESEARCH.md](RESEARCH.md). --- @@ -86,13 +86,13 @@ When building your own is the right answer: These defaults reflect the current canonical benchmark posture: LongMemEval full at 87.00% with 97.00% recall@5, and LoCoMo full at 84.74%. For a query like `GET /recall?query=database+migration&tags=decision&time_query=last+month`, the temporal-alignment and tag components dominate; for `GET /recall?query=why+postgres&expand_relations=true`, the relation component does. -The Recall Quality Lab (`scripts/lab/`) lets you sweep any weight and A/B-compare configs against snapshots of production data without touching the service. +The Recall Quality Lab (`scripts/lab/`) lets you sweep any weight and A/B-compare configs against snapshots of production data without touching the service. The [scripts catalog](../scripts/README.md) explains when to clone, generate queries, test, compare, or sweep. --- ## Where to go next - **Want to deploy?** [INSTALLATION.md](../INSTALLATION.md) -- **Want to tune recall?** [`scripts/lab/`](../scripts/lab/) and [`docs/ENVIRONMENT_VARIABLES.md`](ENVIRONMENT_VARIABLES.md) +- **Want to tune recall?** [Recall Quality Lab in the scripts catalog](../scripts/README.md) and [`docs/ENVIRONMENT_VARIABLES.md`](ENVIRONMENT_VARIABLES.md) - **Want to understand the science?** [RESEARCH.md](RESEARCH.md) - **Want to talk to humans?** [Discord](https://automem.ai/discord) diff --git a/docs/ENVIRONMENT_VARIABLES.md b/docs/ENVIRONMENT_VARIABLES.md index 2b49b108..98585ce7 100644 --- a/docs/ENVIRONMENT_VARIABLES.md +++ b/docs/ENVIRONMENT_VARIABLES.md @@ -67,11 +67,22 @@ AutoMem supports five embedding backends with automatic fallback. - `auto` (default): Try Voyage → OpenAI → Ollama (if configured) → FastEmbed local model → Placeholder - `voyage`: Use Voyage API only (requires `VOYAGE_API_KEY`) - `openai`: Use OpenAI API only (requires `OPENAI_API_KEY`) -- `local`: Use FastEmbed local model only (~210MB download on first use) +- `local`: Use FastEmbed local model only - `ollama`: Use Ollama only (requires `OLLAMA_BASE_URL` and a pulled model) - `placeholder`: Use hash-based embeddings (no semantic search) -**Local Model Details:** +For new cloud deployments, configure Voyage and leave automatic fallback enabled: + +```bash +EMBEDDING_PROVIDER=auto +VOYAGE_API_KEY=pa-... +VOYAGE_MODEL=voyage-4 +VECTOR_SIZE=1024 +``` + +With `auto`, Ollama is attempted only when `OLLAMA_BASE_URL` or `OLLAMA_MODEL` is explicitly configured. FastEmbed is the final usable embedding fallback; do not rely on it as a cloud default. + +**FastEmbed (local/self-hosted only):** - Models: `BAAI/bge-small-en-v1.5` (384d), `BAAI/bge-base-en-v1.5` (768d), `BAAI/bge-large-en-v1.5` (1024d) - Size: ~67MB / ~210MB / ~1.2GB (cached to `~/.config/automem/models/`) - No API key or internet required after first download @@ -79,6 +90,7 @@ AutoMem supports five embedding backends with automatic fallback. - Recommended: pin `onnxruntime<1.20` with `fastembed` 0.4.x to avoid runtime issues - Docker: persist model cache with a volume (see docker-compose.yml) - Ensure `VECTOR_SIZE` equals the selected model's dimension (default 768) +- The 1024d fallback can add roughly 4 GB RSS in a cloud container. On Railway or another managed host, that materially raises the hosting bill; use Voyage instead unless you are intentionally self-hosting FastEmbed. **Ollama Details:** - Requires a running Ollama server (default `http://localhost:11434`) @@ -86,11 +98,24 @@ AutoMem supports five embedding backends with automatic fallback. - Embedding dimensions vary by model; set `VECTOR_SIZE` to match the model output - For `auto` mode, Ollama is only attempted if `OLLAMA_BASE_URL` or `OLLAMA_MODEL` is set +For a local/self-hosted multilingual option, run [BGE-M3 on Ollama](https://ollama.com/library/bge-m3): + +```bash +ollama pull bge-m3 + +EMBEDDING_PROVIDER=ollama +OLLAMA_MODEL=bge-m3 +VECTOR_SIZE=1024 +``` + +Set `OLLAMA_BASE_URL` when the Ollama server is not at `http://localhost:11434`. BGE-M3 is documented through Ollama; it is not a FastEmbed option in this project. + **Voyage Details:** - Requires `VOYAGE_API_KEY` - Optional model override via `VOYAGE_MODEL` (default `voyage-4`) - Voyage 4 family supports output dimensions `256`, `512`, `1024`, or `2048` - Set `VECTOR_SIZE` to one of the supported Voyage dimensions +- Voyage's free tier covers typical AutoMem usage. See the maintained [Voyage pricing page](https://docs.voyageai.com/docs/pricing) for current limits and rates. **OpenAI-Compatible Providers (OpenRouter, LiteLLM, Azure, vLLM, etc.):** @@ -120,19 +145,21 @@ VECTOR_SIZE=768 # must match the model's output | Provider | Cost | Pros | Cons | Best for | |----------|------|------|------|----------| -| Voyage | API usage | Strong quality, flexible model family, simple API | Requires outbound network and matching `VECTOR_SIZE` | Teams using Voyage for quality/cost balance | -| OpenAI | ~$0.13/1M tokens (large), ~$0.02/1M tokens (small) | Highest semantic quality, zero infra | Recurring API cost, outbound network required | Production accuracy with minimal ops | -| FastEmbed (local) | Hardware-only (CPU/GPU) | Offline after first download, consistent latency | Model download size, quality tied to model size | Self-hosted + cost-sensitive environments | +| Voyage | Free tier for typical usage, then API usage | Recommended default, strong multilingual quality, no model hosting | Requires outbound network and matching `VECTOR_SIZE` | New cloud deployments | +| OpenAI | API usage | OpenAI-compatible API fallback, zero model hosting | Requires outbound network | When Voyage is unavailable or an OpenAI-compatible endpoint is required | +| FastEmbed (local) | Hardware and memory | Offline after first download, consistent latency | 1024d fallback can add roughly 4 GB RSS in cloud containers | Intentional local/self-hosted deployments only | | Ollama | Hardware-only (CPU/GPU) | Fully local, easy model swapping | Requires running Ollama service, model dims vary | Self-hosted deployments with Ollama already in stack | | Placeholder | Free | Always available, deterministic | No semantic search quality | Development/testing without vectors | -**Dimension matching tip:** If embeddings fail with a size mismatch, confirm your model's output length and set `VECTOR_SIZE` accordingly (or use `VECTOR_SIZE_AUTODETECT=true` if you accept adopting the existing collection size). +See [Voyage pricing](https://docs.voyageai.com/docs/pricing) for current free-tier limits and rates rather than relying on fixed pricing in this table. + +**Dimension matching tip:** If embeddings fail with a size mismatch, confirm your model's output length and set `VECTOR_SIZE` accordingly. `VECTOR_SIZE_AUTODETECT=true` can adopt an existing collection's dimension for compatibility, but it does not make vectors from different models compatible. ## Hosting Considerations (Railway vs Self-Hosted) -- **Railway / managed PaaS:** Voyage and OpenAI are the simplest choices (no local model downloads). FastEmbed works but increases image size and cold-start time; use a persistent volume for `AUTOMEM_MODELS_DIR` if supported. Ollama typically requires a **separate service** (Railway does not ship Ollama by default), so you'll need to deploy Ollama elsewhere and set `OLLAMA_BASE_URL` to that service. -- **Self-hosted Docker/VPS:** FastEmbed and Ollama are straightforward and avoid API costs. Ollama benefits from GPU acceleration if available; otherwise expect higher latency on CPU. Ensure the Ollama base URL is reachable from the AutoMem container (`OLLAMA_BASE_URL=http://ollama:11434` in docker compose setups). -- **Dimension consistency:** Regardless of host, make sure `VECTOR_SIZE` matches the embedding model output. Changing models requires re-embedding existing memories. +- **Railway / managed PaaS:** Configure Voyage (`VOYAGE_MODEL=voyage-4`, `VECTOR_SIZE=1024`) for new deployments. OpenAI is the API fallback. Do not use FastEmbed as the default fallback: its 1024d model can add roughly 4 GB RSS and materially increase hosting cost. Ollama requires a separately operated service; Railway does not ship it by default. +- **Self-hosted Docker/VPS:** FastEmbed and Ollama are deliberate local options that avoid embedding API usage. Ollama benefits from GPU acceleration if available; otherwise expect higher CPU latency. Ensure the Ollama base URL is reachable from the AutoMem container (`OLLAMA_BASE_URL=http://ollama:11434` in Docker Compose setups). +- **Model-space consistency:** Regardless of host, make sure `VECTOR_SIZE` matches the embedding model output. Changing a provider or model requires a full re-embed even when both models use the same dimension. ## Optional Variables @@ -146,7 +173,7 @@ VECTOR_SIZE=768 # must match the model's output | `QDRANT_API_KEY` | Qdrant API key | - | `your-qdrant-key` | | `QDRANT_COLLECTION` | Collection name | `memories` | `memories` | | `QDRANT_ENSURE_PAYLOAD_INDEXES` | Create payload indexes on startup for faster filtered search | `true` | set `false` for read-only Qdrant credentials | -| `VECTOR_SIZE` | Embedding dimension | `1024` | `1024` (voyage-4), `3072` (large), `768` (small) | +| `VECTOR_SIZE` | Embedding dimension | `1024` | `1024` (voyage-4 or Ollama bge-m3), `3072` (OpenAI large), `768` (truncated OpenAI small) | | `VECTOR_SIZE_AUTODETECT` | Adopt existing collection dimension instead of failing on mismatch | `true` | `false` to enforce strict matching | 👉 **New to Qdrant?** See the [Qdrant Setup Guide](QDRANT_SETUP.md) for step-by-step instructions on creating a collection with the right settings. @@ -155,9 +182,10 @@ VECTOR_SIZE=768 # must match the model's output **Notes**: - Without Qdrant, AutoMem uses deterministic placeholder embeddings (for testing only). -- **Existing deployments on 3072d or 768d**: `VECTOR_SIZE_AUTODETECT=true` (default) automatically adopts your existing collection dimension on startup. No manual action needed after updating. +- **Existing deployments on 3072d or 768d**: `VECTOR_SIZE_AUTODETECT=true` (default) automatically adopts the existing collection dimension on startup. This avoids a dimension-startup failure; it does **not** migrate vectors to a new model. - To enforce strict matching (fail on mismatch), set `VECTOR_SIZE_AUTODETECT=false`. The server will exit with a clear error message and fix instructions. - When creating a new collection, the configured `VECTOR_SIZE` (default 1024 for voyage-4) is used. +- Changing embedding provider or model always requires a clean collection and a full re-embed, even when the old and new outputs are both 1024d. #### Self-Hosted Qdrant on Railway @@ -191,12 +219,25 @@ When running Qdrant as a Railway service (instead of Qdrant Cloud), set `QDRANT_ ### Scripts Only +See the [scripts catalog](../scripts/README.md) for when and how to run the +operational tools that consume these values. + | Variable | Description | Default | Used By | |----------|-------------|---------|---------| | `AUTOMEM_API_URL` | AutoMem API endpoint | `http://localhost:8001` | `recover_from_qdrant.py`, `health_monitor.py` | **Backward Compatibility**: `MCP_MEMORY_HTTP_ENDPOINT` is deprecated but still supported (falls back to this if `AUTOMEM_API_URL` not set). +### Qdrant Restore Tuning + +These options apply only to `scripts/restore_from_backup.py`; they do not reconfigure an existing collection during normal API startup. + +| Variable | Description | Default | +|----------|-------------|---------| +| `QDRANT_RESTORE_HNSW_ON_DISK` | Optional boolean that stores the HNSW index on disk when the restore creates the collection | unset | + +Use this with `QDRANT_RESTORE_VECTOR_ON_DISK=true` and `QDRANT_RESTORE_ON_DISK_PAYLOAD=true` for a full disk-backed Qdrant migration. The restore process deletes and recreates the selected collection unless run in import mode; see [Monitoring & Backups](MONITORING_AND_BACKUPS.md#api-backup-export) before using it against production. + ### Health Monitor | Variable | Description | Default | @@ -246,8 +287,8 @@ Controls embedding provider and classification model settings. | Variable | Description | Default | Options | |----------|-------------|---------|---------| | `EMBEDDING_MODEL` | OpenAI embedding model (used when provider is `openai`) | `text-embedding-3-small` | `text-embedding-3-small`, `text-embedding-3-large` | -| `VECTOR_SIZE` | Embedding dimension | `1024` | Must match embedding provider (1024=voyage-4, 1536=text-embedding-3-small native; choose ≤1536 when truncating, 3072=text-embedding-3-large native) | -| `VECTOR_SIZE_AUTODETECT` | Adopt existing collection dimension instead of failing on mismatch | `true` | `false` to enforce strict matching | +| `VECTOR_SIZE` | Embedding dimension | `1024` | Must match embedding provider (1024=voyage-4 or Ollama bge-m3; 1536=text-embedding-3-small native; choose ≤1536 when truncating; 3072=text-embedding-3-large native) | +| `VECTOR_SIZE_AUTODETECT` | Adopt existing collection dimension instead of failing on mismatch | `true` | Dimension compatibility only; `false` enforces strict matching | | `CLASSIFICATION_MODEL` | LLM for memory type classification | `gpt-4o-mini` | `gpt-4o-mini`, `gpt-4.1`, `gpt-5.1` | | `CLASSIFICATION_BASE_URL` | Optional OpenAI-compatible endpoint for `scripts/reclassify_with_llm.py` | _(unset → OpenAI)_ | e.g. `https://openrouter.ai/api/v1` | | `CLASSIFICATION_API_KEY` | API key paired with `CLASSIFICATION_BASE_URL` (never falls back across providers) | _(unset)_ | provider key | @@ -257,14 +298,16 @@ Controls embedding provider and classification model settings. | Provider / Model | Dimensions | Cost/1M tokens | Quality | Use Case | |------------------|-----------|----------------|---------|----------| -| `voyage-4` | 1024 | ~$0.05 | Excellent | **Recommended default** | -| `text-embedding-3-small` | 1536 native (truncatable) | $0.02 | Good | OpenAI fallback (truncated to `VECTOR_SIZE` via Matryoshka) | -| `text-embedding-3-large` | 3072 native | $0.13 | Excellent | Maximum precision (legacy default) | +| `voyage-4` | 1024 by default | [Current Voyage pricing](https://docs.voyageai.com/docs/pricing) | Excellent, multilingual | **Recommended default** | +| `text-embedding-3-small` | 1536 native (truncatable) | OpenAI API usage | Good | OpenAI fallback (truncated to `VECTOR_SIZE` via Matryoshka) | +| `text-embedding-3-large` | 3072 native | OpenAI API usage | Excellent | Explicit OpenAI precision option | +| `bge-m3` via Ollama | 1024 | Local hardware | Multilingual | Intentional local/self-hosted deployment | Recommended setup (Voyage): ```bash EMBEDDING_PROVIDER=auto # or voyage VOYAGE_API_KEY=pa-... +VOYAGE_MODEL=voyage-4 VECTOR_SIZE=1024 ``` @@ -276,7 +319,16 @@ EMBEDDING_MODEL=text-embedding-3-small VECTOR_SIZE=768 ``` -**Upgrade safety**: Changing embedding dimensions requires a full re-embed. By default, `VECTOR_SIZE_AUTODETECT=true` adopts the existing Qdrant collection dimension on startup, so updating AutoMem won't break existing data even if the default `VECTOR_SIZE` changes between releases. To enforce strict matching, set `VECTOR_SIZE_AUTODETECT=false` — the server will exit with a clear error and fix instructions if there's a mismatch. +To use local/self-hosted BGE-M3 through Ollama: + +```bash +ollama pull bge-m3 +EMBEDDING_PROVIDER=ollama +OLLAMA_MODEL=bge-m3 +VECTOR_SIZE=1024 +``` + +**Upgrade safety**: Changing an embedding provider, model, or dimension requires a full re-embed. This is true even when both models output 1024 dimensions: the vectors occupy different model spaces. `VECTOR_SIZE_AUTODETECT=true` only adopts the current Qdrant collection dimension to avoid a startup mismatch; it is not a model migration. To enforce strict dimension matching, set `VECTOR_SIZE_AUTODETECT=false`. **Classification Model Pricing (Dec 2025):** | Model | Input | Output | Notes | @@ -299,6 +351,7 @@ Controls entity extraction and relationship linking. | `ENRICHMENT_IDLE_SLEEP_SECONDS` | Sleep when queue empty | `2` | | `ENRICHMENT_FAILURE_BACKOFF_SECONDS` | Backoff on failure | `5` | | `ENRICHMENT_ENABLE_SUMMARIES` | Enable summarization | `true` | +| `ENRICHMENT_CIRCUIT_COOLDOWN_SECONDS` | Cooldown after LLM quota exhaustion before one recovery probe is allowed | `300` | | `ENRICHMENT_SPACY_MODEL` | spaCy model name | `en_core_web_sm` | | `JIT_ENRICHMENT_ENABLED` | Inline enrichment during recall | `true` | @@ -403,6 +456,9 @@ These variables are only used by test suites. The `scripts/lab/` directory provides a data-driven framework for testing and optimizing recall scoring. It uses IR metrics (Recall@K, MRR, NDCG) and statistical comparison to evaluate config changes. +For prerequisites, safe clone workflow, and exact commands, see the [scripts +catalog](../scripts/README.md). + **Makefile targets:** | Target | Description | Example | @@ -505,7 +561,7 @@ FALKORDB_HOST = ( ## Re-embedding Memories -When changing `EMBEDDING_MODEL` or `VECTOR_SIZE`, you must re-embed all existing memories: +When changing the embedding provider, model, or `VECTOR_SIZE`, you must re-embed all existing memories. This includes swaps between two 1024d models, such as `voyage-4` and `bge-m3`: equal dimensions do not make their vector spaces compatible. ### 1. Backup First ```bash @@ -513,13 +569,20 @@ python scripts/backup_automem.py ``` ### 2. Set New Environment Variables + +Configure the provider that should create the replacement vectors. For the recommended Voyage configuration: + ```bash -export EMBEDDING_MODEL=text-embedding-3-large # or text-embedding-3-small -export VECTOR_SIZE=3072 # 3072 for large, 768 for small +export EMBEDDING_PROVIDER=voyage +export VOYAGE_API_KEY=pa-... +export VOYAGE_MODEL=voyage-4 +export VECTOR_SIZE=1024 ``` +For OpenAI, set `EMBEDDING_PROVIDER=openai`, `OPENAI_API_KEY`, `EMBEDDING_MODEL`, and its matching `VECTOR_SIZE`. For local/self-hosted multilingual BGE-M3, run `ollama pull bge-m3` and set `EMBEDDING_PROVIDER=ollama`, `OLLAMA_MODEL=bge-m3`, and `VECTOR_SIZE=1024`. + ### 3. Recreate Qdrant Collection -The collection must be recreated with the new dimension: +Pause memory writes, then recreate the collection before re-embedding. This prevents old and new model spaces from being mixed. The re-embed script upserts vectors; it does **not** delete or create the Qdrant collection for you. ```bash # Delete old collection @@ -530,22 +593,24 @@ curl -X DELETE "$QDRANT_URL/collections/memories" \ curl -X PUT "$QDRANT_URL/collections/memories" \ -H "api-key: $QDRANT_API_KEY" \ -H "Content-Type: application/json" \ - -d '{"vectors": {"size": 3072, "distance": "Cosine"}}' # Use 768 for small + -d '{"vectors": {"size": 1024, "distance": "Cosine"}}' ``` +Replace `memories` with your `QDRANT_COLLECTION` name and `1024` with the selected model's output dimension. + ### 4. Re-embed All Memories ```bash python scripts/reembed_embeddings.py --batch-size 32 ``` -The script reads from FalkorDB (which retains all memory data) and re-creates the vector embeddings in Qdrant. +The script reads from FalkorDB (which retains all memory data) and re-creates the vector embeddings in Qdrant. It requires `QDRANT_URL`; unlike the API service, it does not construct a URL from `QDRANT_HOST` and `QDRANT_PORT`. -**Cost estimate:** ~$0.15 for 6000 memories with large embeddings, ~$0.02 with small. +After the full run, verify the service health and run a representative recall. `VECTOR_SIZE_AUTODETECT` is not a substitute for this procedure: it only accepts an existing collection dimension and never converts old vectors to a new model space. --- ## See Also - [Railway Deployment Guide](./RAILWAY_DEPLOYMENT.md) -- [Deployment Checklist](./DEPLOYMENT_CHECKLIST.md) +- [Post-Deploy Checklist](./RAILWAY_DEPLOYMENT.md#post-deploy-checklist) - [Installation Guide](../INSTALLATION.md) diff --git a/docs/HEALTH_MONITORING.md b/docs/HEALTH_MONITORING.md index ec142a57..466ed21e 100644 --- a/docs/HEALTH_MONITORING.md +++ b/docs/HEALTH_MONITORING.md @@ -2,6 +2,9 @@ AutoMem includes a built-in health monitoring system that watches for data inconsistencies and optionally triggers automatic recovery. +For the current command catalog and safety notes for `health_monitor.py` and +`recover_from_qdrant.py`, see [scripts/README.md](../scripts/README.md). + ## Quick Start ### Alert-Only Mode (Recommended) @@ -386,4 +389,4 @@ python scripts/health_monitor.py --once | grep drift_percent - [Recovery Script Documentation](../scripts/recover_from_qdrant.py) - [Environment Variables](./ENVIRONMENT_VARIABLES.md) - [Railway Deployment](./RAILWAY_DEPLOYMENT.md) -- [Deployment Checklist](./DEPLOYMENT_CHECKLIST.md) +- [Post-Deploy Checklist](./RAILWAY_DEPLOYMENT.md#post-deploy-checklist) diff --git a/docs/MCP_TRANSPORT_PARITY.md b/docs/MCP_TRANSPORT_PARITY.md new file mode 100644 index 00000000..d56c12c5 --- /dev/null +++ b/docs/MCP_TRANSPORT_PARITY.md @@ -0,0 +1,148 @@ +# MCP Transport Parity + +AutoMem ships its MCP surface over two transports: + +| Transport | Implementation | Package / path | +| ---------- | ------------------------------------------- | --------------------------------------------- | +| **stdio** | `@verygoodplugins/mcp-automem` (TypeScript) | [verygoodplugins/mcp-automem](https://github.com/verygoodplugins/mcp-automem) | +| **remote** | Streamable HTTP + SSE bridge (ESM JS) | `mcp-sse-server/server.js` (this repo) | + +`mcp-automem`'s `server.json` publishes stdio, streamable-HTTP, and SSE as **one +server with one shared 6-tool array**. That is the contract: a client that picks +"AutoMem" out of the registry must get the same tools, the same request mapping, +and the same rendered output regardless of which transport it connected over. + +This document records the audit that established where the two had drifted, and +the short list of differences that remain intentional. + +## Why this exists + +[PR #224](https://github.com/verygoodplugins/automem/pull/224) fixed one symptom: +the remote bridge's compact recall block dropped the memory's stored date. An +agent replaying that text read a two-week-old itinerary as today's plan, because +nothing in the output said how old the memory was. AutoHub's parser already +looked for a `Created:` line — the remote transport simply never emitted one, +while the stdio package always had. + +That single missing line turned out to be one instance of a much wider split. A +manual schema sync had already been attempted once +(`d99b86d fix(mcp-sse): sync tool schemas for SSE/MCP parity (#104)`) and had +since drifted again, which is the case against maintaining two hand-synced +copies. + +## Audit: where the transports had diverged + +Snapshot taken against `mcp-automem` 0.15.0 and `mcp-sse-server` at +`4b5eaaf`. Both transports registered the same six tool names in the same order — +`store_memory`, `recall_memory`, `associate_memories`, `update_memory`, +`delete_memory`, `check_database_health`. Everything below that differed. + +### Tool definitions + +| Axis | stdio | remote | +| --------------------------------- | -------------------------------------------------------------------- | ----------------------------------------------------------- | +| `description` | Multi-paragraph: modes, "When to use", "Examples" | One line each | +| `title` | Present (and duplicated in `annotations.title`) | Absent | +| `outputSchema` | On all 6 | On none | +| `annotations` | `readOnlyHint`, `destructiveHint`, `idempotentHint`, `openWorldHint` | `readOnlyHint`, `destructiveHint` only | +| `_meta['anthropic/alwaysLoad']` | On `store_memory`, `recall_memory`, `associate_memories` | Absent | +| server `instructions` | Set (~450 chars) | Not set | +| `serverInfo` | `mcp-automem` / package version | `automem-mcp-sse` / `0.1.0`, disagreeing with its own `0.2.0` | + +### Capability missing on the remote transport + +- **`recall_memory`** — 38 params vs 27. Absent remotely: `memory_id` (ID-fetch + mode), `exhaustive` + `offset` (tag-enumeration mode), `exclude_tags`, + `current_only`, `state_mode`, `state_debug`, `recency_bias`, `min_score`, + `adaptive_floor`, `expand_respect_tags`. `limit` capped at 50 remotely vs 200 + on stdio. stdio declared defaults (`expansion_limit: 25`, `relation_limit: 5`, + `current_only: true`); remote declared none. +- **`store_memory`** — no `memories[]` batch mode, and no supersede mode + (`supersedes_memory_id` / `supersede_relation` / `supersede_reason`, a + four-call sequence with a compensating delete on partial failure). +- **`delete_memory`** — no bulk-delete-by-tag. +- **`associate_memories`** — none of the nine relation-specific properties + (`context`, `reason`, `pattern_type`, `confidence`, `resolution`, + `observations`, `timestamp`, `transformation`, `role`). `additionalProperties: + true` let them through unvalidated, but an agent reading the schema never + learned to send them. +- **`update_memory`** — remote advertised `embedding` and forwarded it; + `PATCH /memory/` (`automem/api/memory.py:789`) never reads it. Dead field. + +### Response rendering + +| Call | stdio | remote | +| ----------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------- | +| recall `text` | `1. [tags] (importance: raw) score=0.123 [match] relations=N`, ` ID:`, ` Created: … Updated: …`; summary-first, 400-char preview, 18k-token budget, trailer | `1. [tags] score=0.123`, ` ID:` — no date, no budget, no summary | +| recall `detailed` | 2-space indent, no numbering, `Created:` | no indent, numbered, `Timestamp:` | +| recall `items` | `[] ` blocks, no header | `Found N memories:` header block plus compact blocks | +| recall `json` | own structured envelope | raw upstream response | +| recall empty | `No memories found matching your query.` | `No memories found.` | +| `store_memory` | `Memory stored successfully!\n\nMemory ID: ` | `Memory stored: ` | +| `update_memory` | `Memory updated successfully!` | `Updated ` | +| `delete_memory` | `Memory deleted successfully!` | `Deleted ` | +| `check_database_health` | formatted block with a status emoji | `JSON.stringify(r)` | +| error | `Error: ` | `AutoMem error: (request_id: …)` | +| `structuredContent` | on all six successes | never | + +### Not a parity gap: `store_memory.id` + +Both transports advertised an `id` parameter, and +`automem/api/memory.py:475` mints a server-side UUID unconditionally ("Always +generate server-side UUID to prevent collision/overwrite attacks"). Neither +transport could honor it. Rather than copy the lie into parity, `id` was dropped +from the shared schema. + +## Accepted transport-level differences + +These are intentional and are **not** parity violations. The differential harness +allowlists exactly these and nothing else. + +| Difference | remote | stdio | Why | +| ---------------------- | ------------------------------------------------------------------------------------- | ---------------------------- | -------------------------------------------------------------------------------------------------------------------- | +| `serverInfo.name` | `automem-mcp-sse` | `mcp-automem` | Clients must be able to tell the transports apart. | +| `serverInfo.version` | the bridge's `package.json` | the published package's | Two packages version independently. Each side must still report its **own** package version — the harness pins that self-consistency, which is how the remote's stale `0.1.0` is caught. | +| Auth mechanisms | `Authorization: Bearer`, `X-API-Key` / `X-API-Token`, `?api_key=` / `?apiKey=` / `?api_token=` | `Authorization: Bearer` only | Browser and EventSource clients cannot set headers. | +| Error text suffix | ` (request_id: )` | none | Observability on a hosted service. Same code path — the bridge supplies a `requestIdProvider`, stdio does not. | +| Timeout / retry policy | `UPSTREAM_TIMEOUT_MS` (15 s), `UPSTREAM_MAX_RETRIES` (2) | 25 s, 3 | Tuned per deployment; injected via `AutoMemConfig`. | +| Transport surface | `/mcp`, `/mcp/sse`, `/mcp/messages`, `/health`, `/ready`, `/alexa` | stdio only | Not part of the MCP tool surface. | + +Everything else — tool names, order, `title`, `description`, `inputSchema`, +`outputSchema`, `annotations`, `_meta`, server `instructions`, and every byte of +rendered `tools/call` text — must be identical. + +## How parity is enforced + +A differential harness drives both transports against a single +docker-compose AutoMem and diffs them: + +```bash +make test-parity +``` + +It lives in `mcp-sse-server/parity/` with its entry point at +`mcp-sse-server/test/parity.test.js`, and it asserts three things: + +1. `tools/list` is deep-equal across transports after key-order normalization. +2. Server capabilities and `instructions` match; `serverInfo.name` is allowlisted + to differ. +3. A 19-scenario `tools/call` matrix renders identical text on both, after + redacting values that legitimately vary per run (UUIDs, timestamps, scores, + `query_time_ms`, and the per-transport tag namespace). + +The harness is gated on `AUTOMEM_RUN_PARITY_TESTS=1` because it needs a live +service. Without the gate it skips cleanly, which is what CI's `node-test` job +sees. `.github/workflows/mcp-parity.yml` runs it with the stack up on PRs that +touch `mcp-sse-server/**`, `automem/api/**`, or `app.py`. + +It brings its own stack up on port **8011**, not the usual 8001. The harness +writes fixtures and bulk-deletes by tag, and a developer running a local +AutoMem install (`~/.automem/server`) already holds 8001 — pointing the harness +at that instance would seed test data into a real memory store. + +**There is no scheduled drift run yet, deliberately.** While the harness is +intentionally red against the known gaps, `continue-on-error` leaves the +workflow successful either way, so a cron job could not distinguish "same known +gaps" from "a newly published `mcp-automem` added drift". That would be an alarm +that cannot alarm. The weekly check lands with the change that turns the harness +green, where a failure is a real signal. diff --git a/docs/MIGRATIONS.md b/docs/MIGRATIONS.md index 199c1ddf..e91bc195 100644 --- a/docs/MIGRATIONS.md +++ b/docs/MIGRATIONS.md @@ -2,11 +2,11 @@ This document provides step-by-step instructions for migrating between different AutoMem configurations. -**Heads up for existing deployments:** The default embedding dimension is now **1024d** (voyage-4) for new installs. If your Qdrant collection uses a different dimension (e.g. 3072d from `text-embedding-3-large` or 768d from `text-embedding-3-small`), **no action is needed** — `VECTOR_SIZE_AUTODETECT=true` (the default) automatically adopts your existing collection dimension on startup. To explicitly pin your dimension, set `VECTOR_SIZE=` in your `.env`. To enforce strict matching (fail on mismatch), set `VECTOR_SIZE_AUTODETECT=false`. +**Heads up for existing deployments:** New installs default to **1024d** with Voyage (`voyage-4`). If you are only updating AutoMem and keeping the same embedding provider and model, `VECTOR_SIZE_AUTODETECT=true` (the default) can adopt an existing collection dimension to avoid a startup mismatch. It does **not** migrate embeddings. Any provider or model change requires backing up, recreating the Qdrant collection, and fully re-embedding every memory—even when both models output 1024 dimensions. To explicitly pin a dimension, set `VECTOR_SIZE=`; set `VECTOR_SIZE_AUTODETECT=false` to fail on mismatch. ## Table of Contents -**Embedding dimension migrations** +**Embedding provider and model migrations** - [Migrating to 1024d (voyage-4 default)](#migrating-to-1024d-voyage-4-default) - [Upgrading to 3072d Embeddings](#upgrading-to-3072d-embeddings) - [Downgrading to 768d Embeddings](#downgrading-to-768d-embeddings) @@ -23,6 +23,8 @@ This document provides step-by-step instructions for migrating between different > The embedding-dimension sections are about re-vectorizing; the data & schema > sections are one-time, idempotent FalkorDB/Qdrant migrations. +> **Re-embed prerequisite:** `scripts/reembed_embeddings.py` requires a full `QDRANT_URL` (for example, `http://localhost:6333`); unlike the API service, the script does not construct one from `QDRANT_HOST` and `QDRANT_PORT`. + --- ## Migrating to 1024d (voyage-4 default) @@ -34,47 +36,56 @@ This document provides step-by-step instructions for migrating between different 1. **Backup your data**: `python scripts/backup_automem.py` 2. **Set environment variables**: ```bash - EMBEDDING_PROVIDER=voyage # or auto (will prefer Voyage if VOYAGE_API_KEY is set) - VOYAGE_API_KEY=pa-... - VECTOR_SIZE=1024 + export EMBEDDING_PROVIDER=voyage # or auto (will prefer Voyage if VOYAGE_API_KEY is set) + export VOYAGE_API_KEY=pa-... + export VOYAGE_MODEL=voyage-4 + export VECTOR_SIZE=1024 + # The re-embed script needs a URL; derive one if your deployment uses host/port. + export QDRANT_URL="${QDRANT_URL:-http://${QDRANT_HOST:-localhost}:${QDRANT_PORT:-6333}}" + export QDRANT_COLLECTION="${QDRANT_COLLECTION:-memories}" + # Set QDRANT_API_KEY when the selected Qdrant instance requires authentication. + export QDRANT_API_KEY="${QDRANT_API_KEY:-}" ``` -3. **Delete and recreate the Qdrant collection**: +3. **Pause writes, then delete and recreate the Qdrant collection**: ```bash - curl -X DELETE http://localhost:6333/collections/memories + curl -X DELETE "$QDRANT_URL/collections/$QDRANT_COLLECTION" \ + -H "api-key: $QDRANT_API_KEY" + curl -X PUT "$QDRANT_URL/collections/$QDRANT_COLLECTION" \ + -H 'Content-Type: application/json' \ + -H "api-key: $QDRANT_API_KEY" \ + -d '{"vectors": {"size": 1024, "distance": "Cosine"}}' ``` 4. **Re-embed all memories**: ```bash - python scripts/reembed_embeddings.py + python scripts/reembed_embeddings.py --batch-size 32 ``` -5. **Verify**: Check that `/health` shows `vector_size: 1024` and recall returns results. +5. **Restart or redeploy AutoMem with the selected provider configuration** before testing recall. For example, run `docker compose up -d` for Docker Compose or `railway up` for Railway. +6. **Verify**: Check that `/health` reports `vector_dimensions.collection: 1024` and recall returns results. -> **Alternatively**, set `VECTOR_SIZE_AUTODETECT=true` (the default) and AutoMem will adopt your existing collection dimension without migration. Only migrate when you want to switch embedding providers. +> `VECTOR_SIZE_AUTODETECT=true` can preserve the old collection dimension only when you keep its provider and model. It never makes an existing OpenAI, Voyage, Ollama, or FastEmbed vector compatible with another model space. --- ## Upgrading to 3072d Embeddings -**When to upgrade:** If you need better semantic precision and have the storage budget for 4x larger embeddings. +**When to upgrade:** If an evaluation shows that you need the explicit OpenAI `text-embedding-3-large` option and can accept its extra storage and API usage. This is not the recommended cloud default; new deployments should use Voyage `voyage-4` at 1024d. ### Pros ✅ -- **Better semantic precision**: ~5-10% improvement on benchmarks -- **Improved multi-hop reasoning**: Better at connecting related concepts -- **Recommended for production**: If accuracy is critical and storage is not a constraint +- **Higher-dimensional OpenAI option**: Native 3072d output +- **Explicit control**: Useful when an OpenAI-specific evaluation requires it ### Cons ❌ - **4x storage cost**: 768 → 3072 dimensions (4x more disk space) -- **4x embedding cost**: OpenAI charges per dimension -- **~20% slower search**: More dimensions = more computation +- **More API usage cost**: Review current provider pricing before migration +- **More search work**: Larger vectors increase storage and computation - **Migration required**: Cannot reuse existing embeddings -### Cost Comparison +### Storage Impact | Metric | 768d (small) | 3072d (large) | Multiplier | |--------|--------------|---------------|------------| -| Storage per 1M memories | ~3GB | ~12GB | 4x | -| OpenAI cost per 1M tokens | $0.02 | $0.13 | 6.5x | -| Search latency | ~50ms | ~60ms | 1.2x | -| Benchmark accuracy | 88.2% | 90.5% | +2.3pp | +| Vector dimensions | 768 | 3072 | 4x | +| Vector storage (all else equal) | Baseline | Approximately 4x | 4x | ### Migration Steps @@ -90,33 +101,54 @@ This creates timestamped backups in `backups/`: #### 2. Update Configuration ```bash # Add to your .env file +echo "EMBEDDING_PROVIDER=openai" >> .env echo "VECTOR_SIZE=3072" >> .env echo "EMBEDDING_MODEL=text-embedding-3-large" >> .env +echo "QDRANT_URL=${QDRANT_URL:-http://${QDRANT_HOST:-localhost}:${QDRANT_PORT:-6333}}" >> .env +echo "QDRANT_COLLECTION=${QDRANT_COLLECTION:-memories}" >> .env ``` -Or export temporarily: +Export the same settings for the collection commands and re-embed script: ```bash +export EMBEDDING_PROVIDER=openai export VECTOR_SIZE=3072 export EMBEDDING_MODEL=text-embedding-3-large +export QDRANT_URL="${QDRANT_URL:-http://${QDRANT_HOST:-localhost}:${QDRANT_PORT:-6333}}" +export QDRANT_COLLECTION="${QDRANT_COLLECTION:-memories}" +export QDRANT_API_KEY="${QDRANT_API_KEY:-}" ``` -#### 3. Re-embed All Memories +#### 3. Pause Writes and Recreate the Qdrant Collection + +`reembed_embeddings.py` upserts into an existing collection; it does not recreate one. After the backup, stop or pause writes and recreate the collection for the new model space: + ```bash -python scripts/reembed_embeddings.py +curl -X DELETE "$QDRANT_URL/collections/$QDRANT_COLLECTION" \ + -H "api-key: $QDRANT_API_KEY" +curl -X PUT "$QDRANT_URL/collections/$QDRANT_COLLECTION" \ + -H 'Content-Type: application/json' \ + -H "api-key: $QDRANT_API_KEY" \ + -d '{"vectors": {"size": 3072, "distance": "Cosine"}}' +``` + +#### 4. Re-embed All Memories +```bash +python scripts/reembed_embeddings.py --batch-size 32 ``` This will: - Fetch all memories from FalkorDB (source of truth) - Generate new 3072d embeddings using OpenAI API -- Recreate Qdrant collection with new dimensions - Upsert all embeddings in batches **Expected time:** ~5-10 minutes per 10k memories -#### 4. Verify Migration +#### 5. Verify Migration Check Qdrant collection info: ```bash -curl http://localhost:6333/collections/memories | jq '.result.config.params.vectors' +curl -H "api-key: $QDRANT_API_KEY" \ + "$QDRANT_URL/collections/$QDRANT_COLLECTION" \ + | jq '.result.config.params.vectors' ``` Should show: @@ -127,17 +159,11 @@ Should show: } ``` -#### 5. Test Recall -```bash -curl -X POST http://localhost:8001/recall \ - -H "Authorization: Bearer $AUTOMEM_API_TOKEN" \ - -H "Content-Type: application/json" \ - -d '{"query": "test recall", "limit": 5}' -``` +#### 6. Restart Application with the New Configuration -Verify results are returned and scores look reasonable. +Restart or redeploy before recall testing so the API generates query vectors in +the new model space: -#### 6. Restart Application ```bash # If using Docker docker compose up -d @@ -152,6 +178,16 @@ sudo systemctl restart automem railway up ``` +#### 7. Test Recall +```bash +curl -X POST http://localhost:8001/recall \ + -H "Authorization: Bearer $AUTOMEM_API_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"query": "test recall", "limit": 5}' +``` + +Verify results are returned and scores look reasonable. + ### Rollback Procedure If migration fails or results are poor: @@ -164,6 +200,7 @@ python scripts/restore_from_backup.py backups/qdrant/qdrant_snapshot_YYYYMMDD_HH python scripts/restore_from_backup.py backups/falkordb/memories_YYYYMMDD_HHMMSS.rdb # 3. Revert configuration +export EMBEDDING_PROVIDER=openai export VECTOR_SIZE=768 export EMBEDDING_MODEL=text-embedding-3-small @@ -181,11 +218,18 @@ docker compose up -d Follow the same migration steps above, but use: ```bash +export EMBEDDING_PROVIDER=openai export VECTOR_SIZE=768 export EMBEDDING_MODEL=text-embedding-3-small ``` -Then run `reembed_embeddings.py` to recreate the collection with 768d vectors. +After backing up and pausing writes, recreate the collection with a 768d vector schema, then run: + +```bash +python scripts/reembed_embeddings.py --batch-size 32 +``` + +The script upserts the replacement vectors; it does not recreate the collection. A full re-embed is required even if a future source model happens to share the same dimension. --- @@ -214,8 +258,8 @@ python scripts/migrate_entity_nodes.py # apply (idempotent) > Promote nodes when you opt into that feature. If your clone has noisy generated > entity tags, clean them first with > [`scripts/lab/repair_entity_tags.py`](../scripts/lab/repair_entity_tags.py) -> (`--mode audit` → review the plan → `--mode execute`, with `--mode rollback` -> available). +> (`--mode canonicalize-safe` → review the generated `plan.jsonl` → +> `--execute --plan `, with `--rollback ` available). ### Backfill tag prefixes @@ -285,9 +329,9 @@ FATAL: Vector dimension mismatch detected! ``` **Solution** (pick one): -1. Set `VECTOR_SIZE_AUTODETECT=true` (default) to automatically adopt the existing collection dimension -2. Set `VECTOR_SIZE=` in your `.env` to match your data -3. Migrate to the new dimension: follow the [1024d](#migrating-to-1024d-voyage-4-default), [3072d](#upgrading-to-3072d-embeddings), or [768d](#downgrading-to-768d-embeddings) migration steps above +1. If you are keeping the same provider and model, set `VECTOR_SIZE_AUTODETECT=true` (default) to adopt the existing collection dimension +2. If you are keeping the same provider and model, set `VECTOR_SIZE=` in `.env` +3. If you are changing the provider or model, follow the [1024d](#migrating-to-1024d-voyage-4-default), [3072d](#upgrading-to-3072d-embeddings), or [768d](#downgrading-to-768d-embeddings) procedure: back up, recreate the collection, and fully re-embed ### Error: "OpenAI API rate limit" @@ -297,10 +341,10 @@ Rate limit exceeded during re-embedding ``` **Solution:** -The `reembed_embeddings.py` script uses whatever embedding provider is configured (Voyage, OpenAI, local, etc.). For large datasets: +The `reembed_embeddings.py` script uses the configured provider (Voyage, OpenAI, Ollama, FastEmbed, and so on). For large datasets: 1. Run during off-peak hours 2. Increase your provider's rate limits if applicable -3. Split migration into batches using `--batch-size` flag +3. Split work into smaller embedding requests with `--batch-size` ### Error: "Qdrant collection already exists" @@ -310,29 +354,36 @@ Collection 'memories' already exists with different dimension ``` **Solution:** -Delete and recreate: +Back up, pause writes, then delete **and recreate** the collection with the selected model's dimension before re-embedding. The script only upserts vectors: ```bash -curl -X DELETE http://localhost:6333/collections/memories -python scripts/reembed_embeddings.py +export QDRANT_URL="${QDRANT_URL:-http://${QDRANT_HOST:-localhost}:${QDRANT_PORT:-6333}}" +export QDRANT_COLLECTION="${QDRANT_COLLECTION:-memories}" +export QDRANT_API_KEY="${QDRANT_API_KEY:-}" +curl -X DELETE "$QDRANT_URL/collections/$QDRANT_COLLECTION" \ + -H "api-key: $QDRANT_API_KEY" +curl -X PUT "$QDRANT_URL/collections/$QDRANT_COLLECTION" \ + -H 'Content-Type: application/json' \ + -H "api-key: $QDRANT_API_KEY" \ + -d '{"vectors": {"size": 1024, "distance": "Cosine"}}' +python scripts/reembed_embeddings.py --batch-size 32 ``` -**⚠️ Warning:** This deletes all embeddings. Make sure FalkorDB still has the memories (embeddings will be regenerated from there). +**⚠️ Warning:** Replace `1024` with the selected model's output size. This deletes all embeddings; make sure FalkorDB still has the memories so they can be regenerated. ### Migration is slow **Symptoms:** - Taking hours for thousands of memories -- High OpenAI API costs +- High embedding-provider costs **Solutions:** -1. **Check batch size**: Script uses batches of 100 by default -2. **Parallel processing**: Use `--workers` flag (if implemented) -3. **Spot check first**: Test on a subset before full migration -4. **Use cheaper model for testing**: +1. **Check batch size**: Script defaults to batches of 32; lower `--batch-size` if the provider or host is constrained +2. **Spot check first**: On a newly recreated collection, test a subset before the full run: ```bash - export EMBEDDING_MODEL=text-embedding-3-small - python scripts/reembed_embeddings.py --dry-run + python scripts/reembed_embeddings.py --limit 100 --batch-size 32 ``` + Then rerun without `--limit` to migrate every memory. +3. **Use a non-production provider configuration for experiments** and keep Voyage (`voyage-4`) as the cloud default for the final migration. See [Voyage pricing](https://docs.voyageai.com/docs/pricing) for current limits and rates. ### Backup failed @@ -359,8 +410,9 @@ python scripts/reembed_embeddings.py ### Before Any Migration 1. ✅ **Always backup first** - Don't skip this step 2. ✅ **Test in staging** - If you have a staging environment -3. ✅ **Monitor costs** - Check OpenAI usage dashboard during migration -4. ✅ **Document current state** - Note current VECTOR_SIZE and EMBEDDING_MODEL +3. ✅ **Pause writes and recreate the collection** - Required for every provider/model change, including same-dimension swaps +4. ✅ **Monitor costs** - Check the selected provider's usage dashboard during migration +5. ✅ **Document current state** - Note the current provider, model, and `VECTOR_SIZE` ### After Migration 1. ✅ **Run benchmark tests** - Verify accuracy hasn't degraded diff --git a/docs/MONITORING_AND_BACKUPS.md b/docs/MONITORING_AND_BACKUPS.md index f317d20c..cf1281b2 100644 --- a/docs/MONITORING_AND_BACKUPS.md +++ b/docs/MONITORING_AND_BACKUPS.md @@ -2,6 +2,9 @@ Complete guide to setting up automated health monitoring and backups for AutoMem on Railway. +For the current script inventory, safety classification, and exact CLI usage, +see the [scripts catalog](../scripts/README.md). + ## Overview AutoMem includes three layers of data protection: @@ -60,7 +63,7 @@ Create a new Railway service for continuous monitoring: ```bash # In Railway dashboard 1. Create new service from GitHub repo -2. Set Dockerfile path: scripts/Dockerfile.health-monitor (we'll create this) +2. Set Dockerfile path: `scripts/Dockerfile.health-monitor` (included in this repository) 3. Configure environment variables (same as main service) 4. Deploy ``` @@ -160,6 +163,68 @@ python scripts/restore_from_backup.py --backup-dir snapshot.tar.gz --force `GET /backup` requires the admin token because it exports the full corpus. Add `?include=falkordb` or `?include=qdrant` to export only one store. +#### Migrate an Existing Qdrant Collection to On-Disk Storage + +New AutoMem collections store vectors, the HNSW index, and payloads on disk by default. Existing collections retain their configuration, so migrate them deliberately during a maintenance window. + +> **Warning:** This operation deletes and recreates the configured Qdrant collection. Take and test a portable backup first; that backup is the rollback path. + +1. Quiesce writes before taking the backup. Put API clients, workers, and scheduled writers into maintenance/read-only mode, then keep writes stopped through the restore and validation. The API may remain available only to serve the backup export; point counts alone cannot detect concurrent updates or same-count changes. + +2. Record the current point count and create a recoverable Railway volume snapshot before touching the collection: + + ```bash + curl -sS -H "api-key: $QDRANT_API_KEY" \ + "$QDRANT_URL/collections/$QDRANT_COLLECTION" | jq '.result.points_count' + ``` + + This runbook is for collections using Qdrant's default topology. Capture and review the existing collection configuration before continuing: + + ```bash + curl -sS -H "api-key: $QDRANT_API_KEY" \ + "$QDRANT_URL/collections/$QDRANT_COLLECTION" \ + -o qdrant-before-on-disk-migration-config.json + ``` + + The restore preserves points and vector size, then applies the documented on-disk tuning; it does not preserve custom sharding, replication, write-consistency, quantization, or strict-mode settings. If this configuration has non-default settings, stop before the export and use an operator-specific migration that reapplies them during collection creation. + +3. Export a Qdrant-only portable backup from AutoMem: + + ```bash + curl -H "X-Admin-Token: $ADMIN_API_TOKEN" \ + "$AUTOMEM_API_URL/backup?include=qdrant" \ + -o qdrant-before-on-disk-migration.tar.gz + ``` + +4. Dry-run the portable artifact and confirm its reported point count matches the value from step 2 **before** deleting the collection: + + ```bash + python scripts/restore_from_backup.py \ + --backup-dir qdrant-before-on-disk-migration.tar.gz \ + --qdrant-only --dry-run --force + ``` + + If the counts differ, do not run the destructive restore. The portable export is incomplete; recover from the Railway volume snapshot instead of restoring the same artifact. + +5. Restore that verified backup to the intended Qdrant service with all on-disk settings enabled: + + ```bash + QDRANT_RESTORE_VECTOR_ON_DISK=true \ + QDRANT_RESTORE_HNSW_ON_DISK=true \ + QDRANT_RESTORE_ON_DISK_PAYLOAD=true \ + python scripts/restore_from_backup.py \ + --backup-dir qdrant-before-on-disk-migration.tar.gz \ + --qdrant-only --force + ``` + + Ensure `QDRANT_URL`, `QDRANT_API_KEY`, and `QDRANT_COLLECTION` identify the production collection before running the command. + +6. Restart the AutoMem API while writes are still quiesced. With `QDRANT_ENSURE_PAYLOAD_INDEXES=true` (the default), startup recreates the `tags`, `tag_prefixes`, and `type` payload indexes that the collection restore does not preserve. + +7. Compare the restored point count with the value recorded in step 2 and run representative recall requests before declaring the migration complete. If validation fails, stop and restore the tested backup. Resume writers only after this validation succeeds. + +8. Leave the existing Railway volume size unchanged through a stable observation period. Resize it manually only after the restored collection is healthy and its disk usage is understood; AutoMem does not automate that infrastructure change. + #### Local Backups (Development) The `backup_automem.py` script exports both FalkorDB and Qdrant to compressed JSON files: @@ -291,29 +356,14 @@ redis-cli -h monorail.proxy.rlwy.net -p 12345 -a YOUR_PASSWORD ping --- -**Advanced: Railway Backup Service** - -For Railway Pro users who want backups running on Railway: - -⚠️ **Note:** Railway's UI makes Dockerfile configuration complex. This method is for advanced users. - -The `scripts/Dockerfile.backup` exists and runs backups every 6 hours in a loop. However, deploying it requires CLI: - -```bash -cd /path/to/automem -railway link -railway up --service backup-service -``` - -Then configure in Railway dashboard: - -- Set Builder to Dockerfile -- Dockerfile Path: `scripts/Dockerfile.backup` -- Add environment variables (same as the AutoMem API service) - -**Cost:** ~$1-2/month +**Railway-hosted backup schedules** -**Recommendation:** Use GitHub Actions instead unless you have specific requirements for Railway-hosted backups. +AutoMem does not ship a standalone backup container recipe; the old +documentation reference was stale. Use the included [GitHub Actions backup +workflow](../.github/workflows/backup.yml), or configure your platform's +scheduler to run the documented `backup_automem.py` command with the required +database credentials and off-site destination. The GitHub Actions workflow is +the supported repository example. --- diff --git a/docs/QDRANT_SETUP.md b/docs/QDRANT_SETUP.md index 1e8170f3..9e04be64 100644 --- a/docs/QDRANT_SETUP.md +++ b/docs/QDRANT_SETUP.md @@ -123,9 +123,9 @@ AutoMem uses dense text embeddings (Voyage/OpenAI/etc.) for semantic search. Key *Click image to view full size* -> **Using a smaller model?** If you set `EMBEDDING_MODEL=text-embedding-3-small`, use `768` dimensions instead and set `VECTOR_SIZE=768` in AutoMem. +> **Using OpenAI small?** If you set `EMBEDDING_PROVIDER=openai` and `EMBEDDING_MODEL=text-embedding-3-small`, use `768` dimensions and set `VECTOR_SIZE=768` in AutoMem. > -> **Using FastEmbed or Ollama?** Set dimensions to match your local model output (e.g., 384/768/1024 for FastEmbed). Ollama model dimensions vary—verify with a test embedding and set `VECTOR_SIZE` accordingly. +> **Using local/self-hosted BGE-M3?** Run `ollama pull bge-m3`, set `EMBEDDING_PROVIDER=ollama`, `OLLAMA_MODEL=bge-m3`, and `VECTOR_SIZE=1024`. Other Ollama models and local FastEmbed models must use their own output dimension. FastEmbed is local/self-hosted only, not a Railway default. #### Payload Indexes (Recommended) @@ -159,6 +159,12 @@ Add these to your AutoMem environment variables: QDRANT_URL="https://xxxxx-xxxxx.aws.cloud.qdrant.io" QDRANT_API_KEY="your-api-key-here" QDRANT_COLLECTION="memories" # Only if using custom name + +# Recommended cloud embeddings +EMBEDDING_PROVIDER=auto +VOYAGE_API_KEY="pa-..." +VOYAGE_MODEL="voyage-4" +VECTOR_SIZE=1024 ``` **Railway**: Add these in `AutoMem` → Variables, then redeploy. @@ -198,17 +204,18 @@ If `qdrant` shows `"disconnected"` or `"not configured"`: | Provider / Model | Dimensions | Cost | Quality | |------------------|------------|------|---------| -| `voyage-4` (recommended) | 1024 | ~$0.05/1M tokens | Excellent for short text | -| `text-embedding-3-small` | 1536 native (truncatable) | $0.02/1M tokens | Good OpenAI fallback | -| `text-embedding-3-large` | 3072 native (truncatable) | $0.13/1M tokens | Maximum precision | +| `voyage-4` (recommended) | 1024 | [Current Voyage pricing](https://docs.voyageai.com/docs/pricing) | Excellent, multilingual | +| `text-embedding-3-small` | 1536 native (truncatable) | OpenAI API usage | Good OpenAI fallback | +| `text-embedding-3-large` | 3072 native (truncatable) | OpenAI API usage | Explicit precision option | +| `bge-m3` via Ollama | 1024 | Local hardware | Multilingual, self-hosted | **To switch providers**: 1. Set `EMBEDDING_PROVIDER` and any required API key 2. Set `VECTOR_SIZE` to match the provider's output dimension -3. Create a new Qdrant collection with matching dimensions (or use `VECTOR_SIZE_AUTODETECT=true`) -4. Redeploy AutoMem +3. Back up, pause writes, and recreate the Qdrant collection with matching dimensions +4. Re-embed all memories through the selected provider, then redeploy AutoMem -> ⚠️ **Warning**: Changing embedding models requires re-embedding all existing memories. See [MIGRATIONS.md](MIGRATIONS.md) for the reembed script. +> ⚠️ **Warning**: Changing embedding providers or models requires a clean collection and a full re-embed, even if both models output the same dimension. `VECTOR_SIZE_AUTODETECT` only accepts an existing dimension; it does not migrate model spaces. See [MIGRATIONS.md](MIGRATIONS.md) for the complete procedure. ### Custom Collection Names @@ -265,6 +272,7 @@ AutoMem expects dimensions to match `VECTOR_SIZE`: - `voyage-4` → `VECTOR_SIZE=1024` (default) - `text-embedding-3-small` → `VECTOR_SIZE` ≤ 1536 (default: 768; auto-upgrades to `text-embedding-3-large` if exceeded) - `text-embedding-3-large` → `VECTOR_SIZE` ≤ 3072 (truncatable via Matryoshka) +- `bge-m3` via Ollama → `VECTOR_SIZE=1024` If you created the collection with wrong dimensions: 1. Delete the collection in Qdrant dashboard @@ -273,7 +281,7 @@ If you created the collection with wrong dimensions: ### Memories not appearing in semantic search -1. **Check `OPENAI_API_KEY`**: Required for generating embeddings +1. **Check provider configuration**: Voyage is recommended for cloud (`VOYAGE_API_KEY`, `VOYAGE_MODEL=voyage-4`); OpenAI is the API fallback; Ollama must be explicitly configured and reachable 2. **Verify embeddings exist**: Check `/health` shows `qdrant: connected` 3. **Wait for enrichment**: New memories are embedded async (few seconds) diff --git a/docs/RAILWAY_DEPLOYMENT.md b/docs/RAILWAY_DEPLOYMENT.md index 34780862..f30cd993 100644 --- a/docs/RAILWAY_DEPLOYMENT.md +++ b/docs/RAILWAY_DEPLOYMENT.md @@ -1,5 +1,8 @@ # Railway Deployment Guide +For operational scripts used after deployment (health checks, backups, recovery, +and migrations), see the [scripts catalog](../scripts/README.md). + Complete guide to deploying AutoMem on Railway with persistent storage, backups, and zero data loss. ## Quick Start (One-Click Deploy) @@ -49,7 +52,8 @@ flowchart TB subgraph external [External Services] QdrantCloud[(Qdrant Cloud
Optional managed alternative)] - OpenAI[OpenAI API
Embeddings] + Voyage[Voyage API
Recommended embeddings] + OpenAI[OpenAI API
Fallback embeddings] end end @@ -65,7 +69,8 @@ flowchart TB FlaskAPI -->|Internal
falkordb.railway.internal:6379| FalkorDB FlaskAPI -->|Internal
qdrant.railway.internal:6333| Qdrant FlaskAPI -.->|Optional alternative| QdrantCloud - FlaskAPI --> OpenAI + FlaskAPI --> Voyage + FlaskAPI -.-> OpenAI Enrichment --> FalkorDB Enrichment -.-> QdrantCloud @@ -143,12 +148,13 @@ After deploying, complete these steps to fully configure AutoMem: | Variable | Required | How to Get | | ---------------- | ----------- | -------------------------------------------------------------------- | -| `OPENAI_API_KEY` | Yes\* | [platform.openai.com/api-keys](https://platform.openai.com/api-keys) | +| `VOYAGE_API_KEY` | Recommended | [Voyage API key](https://dash.voyageai.com/) for `voyage-4` | +| `OPENAI_API_KEY` | Optional fallback | [platform.openai.com/api-keys](https://platform.openai.com/api-keys) | | `QDRANT_URL` | Option A | Qdrant Cloud URL — see [Qdrant Setup Guide](QDRANT_SETUP.md) | | `QDRANT_API_KEY` | Option A | Qdrant Cloud API key | | `QDRANT_HOST` | Option B | `qdrant` (self-hosted on Railway — see [Step 1b](#step-1b-qdrant-vector-database)) | -\*Without `OPENAI_API_KEY`, semantic search won't work (embeddings skipped). +Set `EMBEDDING_PROVIDER=auto`, `VOYAGE_MODEL=voyage-4`, and `VECTOR_SIZE=1024` with the Voyage key. In auto mode, OpenAI is the API fallback. Do not rely on FastEmbed for Railway: its 1024d fallback can add roughly 4 GB RSS and materially increase the service cost. See [Voyage pricing](https://docs.voyageai.com/docs/pricing) for the current free-tier limits and rates. 👉 **Qdrant Cloud?** Follow the [Qdrant Setup Guide](QDRANT_SETUP.md) for step-by-step collection setup. 👉 **Self-hosted Qdrant?** See [Step 1b](#step-1b-qdrant-vector-database) — remember to set `QDRANT__SERVICE__HOST=::` on the Qdrant service. @@ -316,8 +322,13 @@ Run Qdrant inside your Railway project for lower latency (internal networking, n AUTOMEM_API_TOKEN=${{shared.AUTOMEM_API_TOKEN}} ADMIN_API_TOKEN=${{shared.ADMIN_API_TOKEN}} - # OpenAI for embeddings (required for semantic search) - OPENAI_API_KEY= + # Recommended cloud embeddings: Voyage first, OpenAI fallback + EMBEDDING_PROVIDER=auto + VOYAGE_API_KEY= + VOYAGE_MODEL=voyage-4 + VECTOR_SIZE=1024 + # Optional: OpenAI-compatible API fallback when Voyage is unavailable + # OPENAI_API_KEY= # Vector search — pick ONE: # Option A: Qdrant Cloud @@ -344,8 +355,13 @@ Run Qdrant inside your Railway project for lower latency (internal networking, n AUTOMEM_API_TOKEN= ADMIN_API_TOKEN= - # OpenAI for embeddings - OPENAI_API_KEY= + # Recommended cloud embeddings: Voyage first, OpenAI fallback + EMBEDDING_PROVIDER=auto + VOYAGE_API_KEY= + VOYAGE_MODEL=voyage-4 + VECTOR_SIZE=1024 + # Optional: OpenAI-compatible API fallback when Voyage is unavailable + # OPENAI_API_KEY= # Vector search — pick ONE: # Option A: Qdrant Cloud diff --git a/lab/test_sets/queries_context_tags_20260704.json b/lab/test_sets/queries_context_tags_20260704.json deleted file mode 100644 index e0d0890c..00000000 --- a/lab/test_sets/queries_context_tags_20260704.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "metadata": { - "created": "2026-07-04T00:00:00Z", - "source": "issue-201-context-tags-probes", - "query_count": 5, - "notes": "Probe rows exercise per-query context_tags in the recall lab harness." - }, - "queries": [ - { - "query": "What issues do React SPA editors face with WP Fusion integrations regarding data loss?", - "expected_ids": [ - "d4fc88f1-52d1-489b-b39c-72982e406910" - ], - "category": "context_tags", - "context_tags": [ - "wp-fusion" - ] - }, - { - "query": "What decisions were made regarding the AutoApp chat UI on April 21, 2026?", - "expected_ids": [ - "d7f35a92-50b6-46cb-9f00-e5f0d4f94819" - ], - "category": "context_tags", - "context_tags": [ - "autoapp" - ] - }, - { - "query": "What upgrades were made to the AutoHub voice latency settings?", - "expected_ids": [ - "de37ff1a-db7f-4310-98af-22c03b2622b8" - ], - "category": "context_tags", - "context_tags": [ - "autohub", - "voice" - ] - }, - { - "query": "What issues were identified with mcp-wp taxonomy tools and their interaction with slugs?", - "expected_ids": [ - "cd5115b8-e003-46c0-9178-b6b3c0d46117" - ], - "category": "context_tags", - "context_tags": [ - "mcp-wp", - "wordpress" - ] - }, - { - "query": "What design constraints did Jack mention for the AutoMem server in June 2026?", - "expected_ids": [ - "24997f3a-3bb4-406c-bed3-4172dfb1caa2" - ], - "category": "context_tags", - "context_tags": [ - "automem", - "mcp-automem" - ] - } - ] -} diff --git a/mcp-sse-server/package-lock.json b/mcp-sse-server/package-lock.json index ac2750b2..ddbac670 100644 --- a/mcp-sse-server/package-lock.json +++ b/mcp-sse-server/package-lock.json @@ -1,15 +1,392 @@ { "name": "automem-mcp-sse-server", - "version": "0.1.0", + "version": "0.2.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "automem-mcp-sse-server", - "version": "0.1.0", + "version": "0.2.0", "dependencies": { - "@modelcontextprotocol/sdk": "^1.20.0", + "@modelcontextprotocol/sdk": "1.20.0", "express": "^4.19.2" + }, + "devDependencies": { + "@verygoodplugins/mcp-automem": "^0.15.0" + } + }, + "node_modules/@hono/node-server": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-2.1.1.tgz", + "integrity": "sha512-ELuehkj5VCBdgEw9zs+ivkKwyzzUCSQuE96YmiPvn1ECBoZCczbFXJLeEGMTYjphP6gydh4pHMqEYPVMYUVgQg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "hono": "^4" + } + }, + "node_modules/@inquirer/ansi": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/@inquirer/ansi/-/ansi-2.0.7.tgz", + "integrity": "sha512-3eTuUO1vH2cZm2ZKHeQxnOqlTi9EfZDGgIe3BL3I4u+rJHocr9Fz86M4fjYABPvFnQG/gGK551HqDiIcETwU6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + } + }, + "node_modules/@inquirer/checkbox": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/@inquirer/checkbox/-/checkbox-5.2.1.tgz", + "integrity": "sha512-b6xmA/VlTe0ZgDQHDui+Nav470u7u49nRd8/iuhOcQPO9Ch7lGuogydhi2VOmNlZ+zXcM8IcPuNSwQcdJaF/kw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/ansi": "^2.0.7", + "@inquirer/core": "^11.2.1", + "@inquirer/figures": "^2.0.7", + "@inquirer/type": "^4.0.7" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/confirm": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/@inquirer/confirm/-/confirm-6.1.1.tgz", + "integrity": "sha512-eb8DBZcz/2qHWQda4rk2JiQk5h9QV/cVHi1yjt0f69WFZMRFn0sJTye3EAP8icut8UDMjQPsaH5KbcOogefrFQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/core": "^11.2.1", + "@inquirer/type": "^4.0.7" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/core": { + "version": "11.2.1", + "resolved": "https://registry.npmjs.org/@inquirer/core/-/core-11.2.1.tgz", + "integrity": "sha512-Qd6GJT1yVyrZZCfN8W2qKF5ApmqryXRhRKCuip8h01x2w/esJQ2XIYc6f9abMIHgKQdBfFTSOdbHRLAhuM09UA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/ansi": "^2.0.7", + "@inquirer/figures": "^2.0.7", + "@inquirer/type": "^4.0.7", + "cli-width": "^4.1.0", + "fast-wrap-ansi": "^0.2.0", + "mute-stream": "^3.0.0", + "signal-exit": "^4.1.0" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/editor": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/@inquirer/editor/-/editor-5.2.2.tgz", + "integrity": "sha512-ZRVd/oD+sYsUd5zVm0NflqEzlqfYCyHNsqkHl2oWXEUHs12tCbcSFi+wVFEvD8+LGRaMUsVrE7qeo6lSG/S1Vg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/core": "^11.2.1", + "@inquirer/external-editor": "^3.0.3", + "@inquirer/type": "^4.0.7" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/expand": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/@inquirer/expand/-/expand-5.1.1.tgz", + "integrity": "sha512-YmQpenjbFSHAK3sOd44puHh3V1KXXr+JiNpUztoSQ4drLh2rTVzTap/YtlAVu/5xavifIlBfNEzJ/neZJ1a/1g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/core": "^11.2.1", + "@inquirer/type": "^4.0.7" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/external-editor": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@inquirer/external-editor/-/external-editor-3.0.3.tgz", + "integrity": "sha512-6thf5I8q7lZwzGLAxPaaGEREEkZ3nyePPDQ1oyobblxmEE8mqTLguScP7pDjUTAibiyb4hfXl+qjUEJ+di/aNA==", + "dev": true, + "license": "MIT", + "dependencies": { + "chardet": "^2.1.1", + "iconv-lite": "^0.7.2" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/external-editor/node_modules/iconv-lite": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", + "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@inquirer/figures": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/@inquirer/figures/-/figures-2.0.7.tgz", + "integrity": "sha512-aJ8TBPOGB6f/2qziPfElISTCEd5XOYTFckA2SGjhNmiKzfK/u4ot3v0DUzGVdUnKjN10EqnnEPck36BkyfLnJw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + } + }, + "node_modules/@inquirer/input": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/@inquirer/input/-/input-5.1.2.tgz", + "integrity": "sha512-9K/DDBSQpOyZSkt6sOVP9Vo0TR7atX2kuILsUu0x3wVcVbe97lJwIJKMLdMw25tDYuXl/qp6erT0Xs1rfmcfZg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/core": "^11.2.1", + "@inquirer/type": "^4.0.7" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/number": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/@inquirer/number/-/number-4.1.1.tgz", + "integrity": "sha512-XF4IXAbPnGPgw0wsbC/i2tPcyfdZgDpUlhsqU0SfT4IRIGWha6Xm9VRgN5yYxJq+jnyXlfXI/nQ3ulfk0iEICA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/core": "^11.2.1", + "@inquirer/type": "^4.0.7" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/password": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/@inquirer/password/-/password-5.1.1.tgz", + "integrity": "sha512-3XBfF7DAsp5qeDsvN5Rd1HmbNokVvEQoUM0QLrRcybC9nX96w3Pbmu7qUsb3IT3J3jBvs2+mTXaKHOUsgHMLzg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/ansi": "^2.0.7", + "@inquirer/core": "^11.2.1", + "@inquirer/type": "^4.0.7" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/prompts": { + "version": "8.5.2", + "resolved": "https://registry.npmjs.org/@inquirer/prompts/-/prompts-8.5.2.tgz", + "integrity": "sha512-IYR/3C/paEVVQYQvdDlFZVjRCJVYHHON0XXMH91KO9GSxs0TdKYWlUdvfQl2EfAHDxUaN3IBffkE/BDTh5nJ6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/checkbox": "^5.2.1", + "@inquirer/confirm": "^6.1.1", + "@inquirer/editor": "^5.2.2", + "@inquirer/expand": "^5.1.1", + "@inquirer/input": "^5.1.2", + "@inquirer/number": "^4.1.1", + "@inquirer/password": "^5.1.1", + "@inquirer/rawlist": "^5.3.1", + "@inquirer/search": "^4.2.1", + "@inquirer/select": "^5.2.1" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/rawlist": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/@inquirer/rawlist/-/rawlist-5.3.1.tgz", + "integrity": "sha512-QqdTqQddL3qPX/PPrjobpsO25NZ4dWXgTLenrR445L2ptLEYE6Z+PD5c5CNDJNx4ugRgELAIpSIJxZaO2jJ2Og==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/core": "^11.2.1", + "@inquirer/type": "^4.0.7" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/search": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/@inquirer/search/-/search-4.2.1.tgz", + "integrity": "sha512-xJj8QWKRSrfKoBIITLZK61dD3zwo0Rz11fgDImku30/Oe81zMdIdGgrLY2h6RkJ+KZ/GhNYIRMKnH/62qBTA5g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/core": "^11.2.1", + "@inquirer/figures": "^2.0.7", + "@inquirer/type": "^4.0.7" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/select": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/@inquirer/select/-/select-5.2.1.tgz", + "integrity": "sha512-FlDndEUww8m7BfukO2nJa25vhD+H5jxxCv4oGioKqzyWz3nPHhhw4LKdYRSlXuAx7DsdWia7iyaBPKKS95Evfw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/ansi": "^2.0.7", + "@inquirer/core": "^11.2.1", + "@inquirer/figures": "^2.0.7", + "@inquirer/type": "^4.0.7" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/type": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/@inquirer/type/-/type-4.0.7.tgz", + "integrity": "sha512-t28inv14nMQ1PhKpsJPY+kEs/c00qzeCOS2gTNRyTjG5d6qsVA2fItxW4hkvGZ5lvanGLdtCzVIx5dwdRpN1+g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } } }, "node_modules/@modelcontextprotocol/sdk": { @@ -49,35 +426,53 @@ } }, "node_modules/@modelcontextprotocol/sdk/node_modules/body-parser": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.2.0.tgz", - "integrity": "sha512-02qvAaxv8tp7fBa/mw1ga98OGm+eCbqzJOKoRt70sLmfEEi+jyBYVTDGfCL/k06/4EMk/z01gCe7HoCH/f2LTg==", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.3.0.tgz", + "integrity": "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==", "license": "MIT", "dependencies": { "bytes": "^3.1.2", - "content-type": "^1.0.5", - "debug": "^4.4.0", - "http-errors": "^2.0.0", - "iconv-lite": "^0.6.3", + "content-type": "^2.0.0", + "debug": "^4.4.3", + "http-errors": "^2.0.1", + "iconv-lite": "^0.7.2", "on-finished": "^2.4.1", - "qs": "^6.14.0", - "raw-body": "^3.0.0", - "type-is": "^2.0.0" + "qs": "^6.15.2", + "raw-body": "^3.0.2", + "type-is": "^2.1.0" + }, + "engines": { + "node": ">=18" }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/body-parser/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "license": "MIT", "engines": { "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/@modelcontextprotocol/sdk/node_modules/content-disposition": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.0.0.tgz", - "integrity": "sha512-Au9nRL8VNUut/XSzbQA38+M78dzP4D+eqg3gfJHMIHHYa3bg067xj1KxMUWj+VULbiZMowKngFFbKczUrNJ1mg==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz", + "integrity": "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==", "license": "MIT", - "dependencies": { - "safe-buffer": "5.2.1" - }, "engines": { - "node": ">= 0.6" + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/@modelcontextprotocol/sdk/node_modules/cookie-signature": { @@ -107,18 +502,19 @@ } }, "node_modules/@modelcontextprotocol/sdk/node_modules/express": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/express/-/express-5.1.0.tgz", - "integrity": "sha512-DT9ck5YIRU+8GYzzU5kT3eHGA5iL+1Zd0EutOmTE9Dtk+Tvuzd23VBU+ec7HPNSTxXYO55gPV/hq4pSBJDjFpA==", + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", "license": "MIT", "dependencies": { "accepts": "^2.0.0", - "body-parser": "^2.2.0", + "body-parser": "^2.2.1", "content-disposition": "^1.0.0", "content-type": "^1.0.5", "cookie": "^0.7.1", "cookie-signature": "^1.2.1", "debug": "^4.4.0", + "depd": "^2.0.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "etag": "^1.8.1", @@ -149,9 +545,9 @@ } }, "node_modules/@modelcontextprotocol/sdk/node_modules/finalhandler": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.0.tgz", - "integrity": "sha512-/t88Ty3d5JWQbWYgaOGCCYfXRwV1+be02WqYYlL6h0lEiUAMPM8o8qKGO01YIkOHzka2up08wvgYD0mDiI+q3Q==", + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", "license": "MIT", "dependencies": { "debug": "^4.4.0", @@ -161,44 +557,567 @@ "parseurl": "^1.3.3", "statuses": "^2.0.1" }, + "engines": { + "node": ">= 18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/fresh": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/iconv-lite": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", + "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/media-typer": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.1.tgz", + "integrity": "sha512-yz3xRaG20c6/BOzvYoDaGtPmGscs7YivItZEEqe6GbwNfHuxu9YNmvnEkMzKldAGY4/80pRcQRZSEnhquk9XuQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/merge-descriptors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/negotiator": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz", + "integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/qs": { + "version": "6.15.3", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.3.tgz", + "integrity": "sha512-O9gl3zCl5h5blw1KGUzQKhA5oUXSl8rwUIM5o0S3nCXMliSvy5Dzx7/DJcI+SwgICv+IneSZwhBh1oSyEHA71A==", + "license": "BSD-3-Clause", + "dependencies": { + "es-define-property": "^1.0.1", + "side-channel": "^1.1.1" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/send": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", + "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.1", + "mime-types": "^3.0.2", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/serve-static": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", + "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", + "license": "MIT", + "dependencies": { + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/type-is": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", + "integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==", + "license": "MIT", + "dependencies": { + "content-type": "^2.0.0", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/type-is/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/@verygoodplugins/mcp-automem": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/@verygoodplugins/mcp-automem/-/mcp-automem-0.15.0.tgz", + "integrity": "sha512-WsnFmalaVTMps9gaQ4Bhg688OPrFzNrFSv6iZpEQcp6FslTnUsXZnW7SxjzWfkxnSFXRERcLPB8NNAyXHsB5/w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/prompts": "^8.5.2", + "@modelcontextprotocol/sdk": "^1.29.0", + "dotenv": "^17.4.2", + "node-fetch": "^3.3.2", + "yaml": "^2.9.0" + }, + "bin": { + "mcp-automem": "dist/index.js" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@verygoodplugins/mcp-automem/node_modules/@modelcontextprotocol/sdk": { + "version": "1.30.0", + "resolved": "https://registry.npmjs.org/@modelcontextprotocol/sdk/-/sdk-1.30.0.tgz", + "integrity": "sha512-xKd8OIzlqNzcqcNumGAa6g+PW2kjD5vrpcKOnfldAUPP3j7lnqMPwlTXQm8gF+UwH72z0lqaRbjr9hqGz0eITA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@hono/node-server": "^1.19.9 || ^2.0.5", + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1", + "content-type": "^1.0.5", + "cors": "^2.8.5", + "cross-spawn": "^7.0.5", + "eventsource": "^3.0.2", + "eventsource-parser": "^3.0.0", + "express": "^5.2.1", + "express-rate-limit": "^8.2.1", + "hono": "^4.11.4", + "jose": "^6.1.3", + "json-schema-typed": "^8.0.2", + "pkce-challenge": "^5.0.0", + "raw-body": "^3.0.0", + "zod": "^3.25 || ^4.0", + "zod-to-json-schema": "^3.25.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@cfworker/json-schema": "^4.1.1", + "zod": "^3.25 || ^4.0" + }, + "peerDependenciesMeta": { + "@cfworker/json-schema": { + "optional": true + }, + "zod": { + "optional": false + } + } + }, + "node_modules/@verygoodplugins/mcp-automem/node_modules/accepts": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", + "dev": true, + "license": "MIT", + "dependencies": { + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/@verygoodplugins/mcp-automem/node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/@verygoodplugins/mcp-automem/node_modules/body-parser": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.3.0.tgz", + "integrity": "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==", + "dev": true, + "license": "MIT", + "dependencies": { + "bytes": "^3.1.2", + "content-type": "^2.0.0", + "debug": "^4.4.3", + "http-errors": "^2.0.1", + "iconv-lite": "^0.7.2", + "on-finished": "^2.4.1", + "qs": "^6.15.2", + "raw-body": "^3.0.2", + "type-is": "^2.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@verygoodplugins/mcp-automem/node_modules/body-parser/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@verygoodplugins/mcp-automem/node_modules/content-disposition": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz", + "integrity": "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@verygoodplugins/mcp-automem/node_modules/cookie-signature": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.6.0" + } + }, + "node_modules/@verygoodplugins/mcp-automem/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@verygoodplugins/mcp-automem/node_modules/express": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", + "dev": true, + "license": "MIT", + "dependencies": { + "accepts": "^2.0.0", + "body-parser": "^2.2.1", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "depd": "^2.0.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@verygoodplugins/mcp-automem/node_modules/express-rate-limit": { + "version": "8.6.2", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-8.6.2.tgz", + "integrity": "sha512-YH4ru+eOJxQABscKFfRCy9R7x9QFGdezclVMwwgFFndzS2Xnm0uo6B0ABZsLhcpeptGv2qvuJVWlQr9gQZoC3A==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "ip-address": "^10.2.0" + }, + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://github.com/sponsors/express-rate-limit" + }, + "peerDependencies": { + "express": ">= 4.11" + } + }, + "node_modules/@verygoodplugins/mcp-automem/node_modules/finalhandler": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" + }, + "engines": { + "node": ">= 18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@verygoodplugins/mcp-automem/node_modules/fresh": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", + "dev": true, + "license": "MIT", "engines": { "node": ">= 0.8" } }, - "node_modules/@modelcontextprotocol/sdk/node_modules/fresh": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", - "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", + "node_modules/@verygoodplugins/mcp-automem/node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "dev": true, "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, "engines": { "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, - "node_modules/@modelcontextprotocol/sdk/node_modules/iconv-lite": { - "version": "0.6.3", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", - "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "node_modules/@verygoodplugins/mcp-automem/node_modules/iconv-lite": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", + "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", + "dev": true, "license": "MIT", "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" }, "engines": { "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, - "node_modules/@modelcontextprotocol/sdk/node_modules/media-typer": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", - "integrity": "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==", + "node_modules/@verygoodplugins/mcp-automem/node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true, + "license": "MIT" + }, + "node_modules/@verygoodplugins/mcp-automem/node_modules/media-typer": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.1.tgz", + "integrity": "sha512-yz3xRaG20c6/BOzvYoDaGtPmGscs7YivItZEEqe6GbwNfHuxu9YNmvnEkMzKldAGY4/80pRcQRZSEnhquk9XuQ==", + "dev": true, "license": "MIT", "engines": { "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, - "node_modules/@modelcontextprotocol/sdk/node_modules/merge-descriptors": { + "node_modules/@verygoodplugins/mcp-automem/node_modules/merge-descriptors": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", + "dev": true, "license": "MIT", "engines": { "node": ">=18" @@ -207,49 +1126,59 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/@modelcontextprotocol/sdk/node_modules/mime-db": { + "node_modules/@verygoodplugins/mcp-automem/node_modules/mime-db": { "version": "1.54.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "dev": true, "license": "MIT", "engines": { "node": ">= 0.6" } }, - "node_modules/@modelcontextprotocol/sdk/node_modules/mime-types": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.1.tgz", - "integrity": "sha512-xRc4oEhT6eaBpU1XF7AjpOFD+xQmXNB5OVKwp4tqCuBpHLS/ZbBDrc07mYTDqVMg6PfxUjjNp85O6Cd2Z/5HWA==", + "node_modules/@verygoodplugins/mcp-automem/node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "dev": true, "license": "MIT", "dependencies": { "mime-db": "^1.54.0" }, "engines": { - "node": ">= 0.6" + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, - "node_modules/@modelcontextprotocol/sdk/node_modules/ms": { + "node_modules/@verygoodplugins/mcp-automem/node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, "license": "MIT" }, - "node_modules/@modelcontextprotocol/sdk/node_modules/negotiator": { + "node_modules/@verygoodplugins/mcp-automem/node_modules/negotiator": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz", "integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==", + "dev": true, "license": "MIT", "engines": { "node": ">= 0.6" } }, - "node_modules/@modelcontextprotocol/sdk/node_modules/qs": { - "version": "6.14.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.0.tgz", - "integrity": "sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w==", + "node_modules/@verygoodplugins/mcp-automem/node_modules/qs": { + "version": "6.15.3", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.3.tgz", + "integrity": "sha512-O9gl3zCl5h5blw1KGUzQKhA5oUXSl8rwUIM5o0S3nCXMliSvy5Dzx7/DJcI+SwgICv+IneSZwhBh1oSyEHA71A==", + "dev": true, "license": "BSD-3-Clause", "dependencies": { - "side-channel": "^1.1.0" + "es-define-property": "^1.0.1", + "side-channel": "^1.1.1" }, "engines": { "node": ">=0.6" @@ -258,32 +1187,38 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/@modelcontextprotocol/sdk/node_modules/send": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/send/-/send-1.2.0.tgz", - "integrity": "sha512-uaW0WwXKpL9blXE2o0bRhoL2EGXIrZxQ2ZQ4mgcfoBxdFmQold+qWsD2jLrfZ0trjKL6vOw0j//eAwcALFjKSw==", + "node_modules/@verygoodplugins/mcp-automem/node_modules/send": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", + "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", + "dev": true, "license": "MIT", "dependencies": { - "debug": "^4.3.5", + "debug": "^4.4.3", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "etag": "^1.8.1", "fresh": "^2.0.0", - "http-errors": "^2.0.0", - "mime-types": "^3.0.1", + "http-errors": "^2.0.1", + "mime-types": "^3.0.2", "ms": "^2.1.3", "on-finished": "^2.4.1", "range-parser": "^1.2.1", - "statuses": "^2.0.1" + "statuses": "^2.0.2" }, "engines": { "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, - "node_modules/@modelcontextprotocol/sdk/node_modules/serve-static": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.0.tgz", - "integrity": "sha512-61g9pCh0Vnh7IutZjtLGGpTA355+OPn2TyDv/6ivP2h/AdAVX9azsoxmg2/M6nZeQZNYBEwIcsne1mJd9oQItQ==", + "node_modules/@verygoodplugins/mcp-automem/node_modules/serve-static": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", + "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", + "dev": true, "license": "MIT", "dependencies": { "encodeurl": "^2.0.0", @@ -293,20 +1228,53 @@ }, "engines": { "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, - "node_modules/@modelcontextprotocol/sdk/node_modules/type-is": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.0.1.tgz", - "integrity": "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw==", + "node_modules/@verygoodplugins/mcp-automem/node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/@verygoodplugins/mcp-automem/node_modules/type-is": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", + "integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==", + "dev": true, "license": "MIT", "dependencies": { - "content-type": "^1.0.5", + "content-type": "^2.0.0", "media-typer": "^1.1.0", "mime-types": "^3.0.0" }, "engines": { - "node": ">= 0.6" + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@verygoodplugins/mcp-automem/node_modules/type-is/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/accepts": { @@ -323,9 +1291,9 @@ } }, "node_modules/ajv": { - "version": "6.12.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz", + "integrity": "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==", "license": "MIT", "dependencies": { "fast-deep-equal": "^3.1.1", @@ -338,6 +1306,48 @@ "url": "https://github.com/sponsors/epoberezkin" } }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/ajv-formats/node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats/node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true, + "license": "MIT" + }, "node_modules/array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", @@ -421,6 +1431,23 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/chardet": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/chardet/-/chardet-2.2.0.tgz", + "integrity": "sha512-rddelWYNPRrXq6PtNEN2S3f6t9ILzvqaN5pVgi4kqt9jHQaXIial9PznB5iSPVlQSLNaaH22ItWz3EJtQ10+OA==", + "dev": true, + "license": "MIT" + }, + "node_modules/cli-width": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-4.1.0.tgz", + "integrity": "sha512-ouuZd4/dm2Sw5Gmqy6bGyNNNe1qt9RpmxveLSO7KcgsTnU7RXfsw+/bukWGo1abgBiMAic068rclZsO4IWmmxQ==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">= 12" + } + }, "node_modules/content-disposition": { "version": "0.5.4", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", @@ -484,6 +1511,16 @@ "node": ">= 8" } }, + "node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, "node_modules/debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", @@ -512,6 +1549,19 @@ "npm": "1.2.8000 || >= 1.4.16" } }, + "node_modules/dotenv": { + "version": "17.4.2", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.4.2.tgz", + "integrity": "sha512-nI4U3TottKAcAD9LLud4Cb7b2QztQMUEfHbvhTH09bqXTxnSie8WnjPALV/WMCrJZ6UV/qHJ6L03OqO3LcdYZw==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, "node_modules/dunder-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", @@ -680,6 +1730,74 @@ "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", "license": "MIT" }, + "node_modules/fast-string-truncated-width": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/fast-string-truncated-width/-/fast-string-truncated-width-3.0.3.tgz", + "integrity": "sha512-0jjjIEL6+0jag3l2XWWizO64/aZVtpiGE3t0Zgqxv0DPuxiMjvB3M24fCyhZUO4KomJQPj3LTSUnDP3GpdwC0g==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-string-width": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/fast-string-width/-/fast-string-width-3.0.2.tgz", + "integrity": "sha512-gX8LrtNEI5hq8DVUfRQMbr5lpaS4nMIWV+7XEbXk2b8kiQIizgnlr12B4dA3ZEx3308ze0O4Q1R+cHts8kyUJg==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-string-truncated-width": "^3.0.2" + } + }, + "node_modules/fast-uri": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.5.tgz", + "integrity": "sha512-gHwA1O9LDIcKunMKhObS/HimwtehO1nPUECKAu5TpKgaO19fcWEl4bliWe1jWxVFvIXztJjjQ4L8XQ1EU9f7Jw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/fast-wrap-ansi": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/fast-wrap-ansi/-/fast-wrap-ansi-0.2.2.tgz", + "integrity": "sha512-7F2Fl+TjRSenLqlU3UjSH0iyqopqoZIu7eZVpEirP2g1GtWa2G/ecEmBdgz31+Mxr+ELclgg6sokpSFIQiZ02Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-string-width": "^3.0.2" + } + }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, "node_modules/finalhandler": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.1.tgz", @@ -698,6 +1816,19 @@ "node": ">= 0.8" } }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, "node_modules/forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -798,6 +1929,16 @@ "node": ">= 0.4" } }, + "node_modules/hono": { + "version": "4.13.2", + "resolved": "https://registry.npmjs.org/hono/-/hono-4.13.2.tgz", + "integrity": "sha512-JydRilDRkYBQMt9qR9U92mXxmbGqsqSn/IKOrh4e7/gEbn+0zSr8igTu0obwJoNGN4sez28DIql7FBHWydoJpA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=16.9.0" + } + }, "node_modules/http-errors": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", @@ -832,6 +1973,16 @@ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", "license": "ISC" }, + "node_modules/ip-address": { + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.5.0.tgz", + "integrity": "sha512-R5SnVLJmgYYvf2F2ZgwSBnelz5G4q5AxIC277GDfUaNbrZKNANcBC7RHqYYePlszf4kBolVkJauG0ZjHHFh55g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -853,12 +2004,29 @@ "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", "license": "ISC" }, + "node_modules/jose": { + "version": "6.2.8", + "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.8.tgz", + "integrity": "sha512-Bsdjwm3Qsd/P0jR+BHDe3LytDfY7WBq2HmCCLIwuVRHMuEC9ae7/R474GIUdF1NgCyZjzVo/A9DOiOBtXq8ZoQ==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, "node_modules/json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", "license": "MIT" }, + "node_modules/json-schema-typed": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/json-schema-typed/-/json-schema-typed-8.0.2.tgz", + "integrity": "sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA==", + "dev": true, + "license": "BSD-2-Clause" + }, "node_modules/math-intrinsics": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", @@ -934,6 +2102,16 @@ "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", "license": "MIT" }, + "node_modules/mute-stream": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-3.0.0.tgz", + "integrity": "sha512-dkEJPVvun4FryqBmZ5KhDo0K9iDXAwn08tMLDinNdRBNPcYEDiWYysLcc6k3mjTMlbP9KyylvRpd4wFtwrT9rw==", + "dev": true, + "license": "ISC", + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, "node_modules/negotiator": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", @@ -943,6 +2121,46 @@ "node": ">= 0.6" } }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "deprecated": "Use your platform's native DOMException instead", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "dev": true, + "license": "MIT", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, "node_modules/object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", @@ -1065,24 +2283,44 @@ } }, "node_modules/raw-body": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.1.tgz", - "integrity": "sha512-9G8cA+tuMS75+6G/TzW8OtLzmBDMo8p1JRxN5AZ+LAp8uxGA8V8GZm4GQ4/N5QNQEnLmg6SS7wyuSmbKepiKqA==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", "license": "MIT", "dependencies": { - "bytes": "3.1.2", - "http-errors": "2.0.0", - "iconv-lite": "0.7.0", - "unpipe": "1.0.0" + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.7.0", + "unpipe": "~1.0.0" }, "engines": { "node": ">= 0.10" } }, + "node_modules/raw-body/node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/raw-body/node_modules/iconv-lite": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.0.tgz", - "integrity": "sha512-cf6L2Ds3h57VVmkZe+Pn+5APsT7FpqJtEhhieDCvrE2MK5Qk9MyffgQyuxQTm6BChfeZNtcOLHp9IcWRVcIcBQ==", + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", + "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", "license": "MIT", "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" @@ -1095,6 +2333,25 @@ "url": "https://opencollective.com/express" } }, + "node_modules/raw-body/node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/router": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", @@ -1135,9 +2392,9 @@ "license": "MIT" }, "node_modules/router/node_modules/path-to-regexp": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.3.0.tgz", - "integrity": "sha512-7jdwVIRtsP8MYpdXSwOS0YdD0Du+qOoF/AEPIt88PcCFrZCzx41oxku1jD88hZBwbNUIEfpqvuhjFaMAqMTWnA==", + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", + "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==", "license": "MIT", "funding": { "type": "opencollective", @@ -1252,14 +2509,14 @@ } }, "node_modules/side-channel": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", - "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz", + "integrity": "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==", "license": "MIT", "dependencies": { "es-errors": "^1.3.0", - "object-inspect": "^1.13.3", - "side-channel-list": "^1.0.0", + "object-inspect": "^1.13.4", + "side-channel-list": "^1.0.1", "side-channel-map": "^1.0.1", "side-channel-weakmap": "^1.0.2" }, @@ -1271,13 +2528,13 @@ } }, "node_modules/side-channel-list": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.0.tgz", - "integrity": "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", "license": "MIT", "dependencies": { "es-errors": "^1.3.0", - "object-inspect": "^1.13.3" + "object-inspect": "^1.13.4" }, "engines": { "node": ">= 0.4" @@ -1323,6 +2580,19 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/statuses": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", @@ -1390,6 +2660,16 @@ "node": ">= 0.8" } }, + "node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -1411,22 +2691,38 @@ "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", "license": "ISC" }, + "node_modules/yaml": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz", + "integrity": "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==", + "dev": true, + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + }, + "funding": { + "url": "https://github.com/sponsors/eemeli" + } + }, "node_modules/zod": { - "version": "3.25.76", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", - "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz", + "integrity": "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==", "license": "MIT", "funding": { "url": "https://github.com/sponsors/colinhacks" } }, "node_modules/zod-to-json-schema": { - "version": "3.24.6", - "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.24.6.tgz", - "integrity": "sha512-h/z3PKvcTcTetyjl1fkj79MHNEjm+HpD6NXheWjzOekY7kV+lwDYnHw+ivHkijnCSMz1yJaWBD9vu/Fcmk+vEg==", + "version": "3.25.2", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.2.tgz", + "integrity": "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA==", "license": "ISC", "peerDependencies": { - "zod": "^3.24.1" + "zod": "^3.25.28 || ^4" } } } diff --git a/mcp-sse-server/package.json b/mcp-sse-server/package.json index d26992f2..f69a0fe1 100644 --- a/mcp-sse-server/package.json +++ b/mcp-sse-server/package.json @@ -8,7 +8,10 @@ "test": "node --test" }, "dependencies": { - "@modelcontextprotocol/sdk": "^1.20.0", + "@modelcontextprotocol/sdk": "1.20.0", "express": "^4.19.2" + }, + "devDependencies": { + "@verygoodplugins/mcp-automem": "^0.15.0" } } diff --git a/mcp-sse-server/parity/clients.js b/mcp-sse-server/parity/clients.js new file mode 100644 index 00000000..8fa6839a --- /dev/null +++ b/mcp-sse-server/parity/clients.js @@ -0,0 +1,88 @@ +/** + * Connects one MCP client to each transport so the two can be diffed. + * + * Remote: the bridge is booted in-process on an ephemeral port (same pattern as + * test/server.test.js's withServer helper) and driven over streamable HTTP. + * stdio: the published @verygoodplugins/mcp-automem bin is spawned as a child + * process and driven over stdio. + * + * Both point at the same live AutoMem service, so any difference in the output + * is a difference between the transports and not between two datasets. + */ +import { createRequire } from 'node:module'; +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js'; +import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js'; +import { createApp } from '../server.js'; + +const require = createRequire(import.meta.url); + +export const API_URL = process.env.AUTOMEM_PARITY_API_URL || 'http://localhost:8001'; +export const API_TOKEN = process.env.AUTOMEM_PARITY_API_TOKEN || 'test-token'; + +export async function connectBothTransports() { + const closers = []; + const closeAll = async () => { + for (const c of closers.reverse()) { + await Promise.resolve(c()).catch(() => {}); + } + closers.length = 0; + }; + + let remote; + let stdio; + + // Setup opens an HTTP listener before the stdio child is connected. If that + // child fails to start — the weekly @latest install resolving an incompatible + // entry point is the realistic case — the listener would otherwise stay open, + // and `node --test` waits on open handles, so the job would hang until + // GitHub's timeout instead of reporting the startup failure. + try { + // --- remote ------------------------------------------------------------- + // server.js resolves the upstream inside its route handlers, so setting + // these before createApp() is enough. + process.env.AUTOMEM_API_URL = API_URL; + process.env.AUTOMEM_API_TOKEN = API_TOKEN; + + const app = createApp(); + const httpServer = await new Promise((resolve) => { + const s = app.listen(0, '127.0.0.1', () => resolve(s)); + }); + closers.push(() => new Promise((r) => httpServer.close(r))); + const { port } = httpServer.address(); + + remote = new Client({ name: 'parity-harness', version: '1.0.0' }, {}); + const remoteTransport = new StreamableHTTPClientTransport( + new URL(`http://127.0.0.1:${port}/mcp`), + { requestInit: { headers: { Authorization: `Bearer ${API_TOKEN}` } } } + ); + // Registered before the await: if the handshake itself rejects or times + // out, a closer added afterwards would never exist and the resource would + // leak. Closers swallow their own errors, so closing something that never + // finished connecting is safe. + closers.push(() => remoteTransport.close()); + closers.push(() => remote.close()); + await remote.connect(remoteTransport); + + // --- stdio -------------------------------------------------------------- + const stdioEntry = require.resolve('@verygoodplugins/mcp-automem/dist/index.js'); + stdio = new Client({ name: 'parity-harness', version: '1.0.0' }, {}); + const stdioTransport = new StdioClientTransport({ + command: process.execPath, + args: [stdioEntry], + env: { ...process.env, AUTOMEM_API_URL: API_URL, AUTOMEM_API_KEY: API_TOKEN }, + stderr: 'ignore', + }); + // Same ordering rule, and it matters more here: the child process is + // already spawned by the time the handshake runs, so a closer registered + // after the await would leave a live child holding node --test open. + closers.push(() => stdioTransport.close()); + closers.push(() => stdio.close()); + await stdio.connect(stdioTransport); + } catch (error) { + await closeAll(); + throw error; + } + + return { remote, stdio, close: closeAll }; +} diff --git a/mcp-sse-server/parity/normalize.js b/mcp-sse-server/parity/normalize.js new file mode 100644 index 00000000..14ffcf24 --- /dev/null +++ b/mcp-sse-server/parity/normalize.js @@ -0,0 +1,75 @@ +/** + * Normalization helpers for the cross-transport parity harness. + * + * Lives outside test/ on purpose: `node --test` treats every file under a + * `test/` directory as a test file, so helper modules placed there would be + * executed as (empty) test suites. + */ + +/** + * Recursively sort object keys so deepStrictEqual is not sensitive to the order + * in which two independent implementations happened to build the same object. + */ +export function normalizeKeys(value) { + if (Array.isArray(value)) return value.map(normalizeKeys); + if (value && typeof value === 'object') { + return Object.fromEntries( + Object.keys(value) + .sort() + .map((k) => [k, normalizeKeys(value[k])]) + ); + } + return value; +} + +// The remote transport appends " (request_id: )" to error text for +// observability on a hosted service. docs/MCP_TRANSPORT_PARITY.md allowlists +// the whole suffix, so strip it entirely — redacting only the UUID inside it +// would leave the three error scenarios permanently red. +const REQUEST_ID_SUFFIX_RE = /\s*\(request_id: [^)]*\)/g; +const UUID_RE = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi; +const ISO_RE = /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})/g; +const SCORE_RE = /(score=|Score: |"final_score":\s*|"score":\s*)[\d.]+/g; +const MS_RE = /("query_time_ms":\s*)[\d.]+/g; + +// Global service counters. The two transports run their scenario batches +// sequentially against one shared AutoMem, so by the time the second batch +// calls /health the first has already inserted its fixtures. These counters +// therefore always differ, and that difference is the harness's own writes — +// not a transport difference. Redact rather than reorder: no ordering makes +// a global counter agree across two sequential batches. +const VOLATILE_COUNT_KEYS = [ + 'memory_count', + 'vector_count', + 'queue_depth', + 'pending', + 'inflight', + 'processed', + 'failed', +]; +const JSON_COUNT_RE = new RegExp( + `("(?:${VOLATILE_COUNT_KEYS.join('|')})":\\s*)\\d+`, + 'g' +); +const LABEL_COUNT_RE = /((?:Memory|Vector) count: )\d+/g; + +/** + * Replace values that legitimately differ run-to-run so two transports can be + * compared on the parts that must match. + * + * `scopeTag` is the per-transport uuid4 tag namespace; each transport writes + * under its own so neither sees the other's memories, which means the tag has + * to collapse to a constant before comparison. + */ +export function redact(text, scopeTag) { + let out = String(text); + if (scopeTag) out = out.split(scopeTag).join(''); + return out + .replace(REQUEST_ID_SUFFIX_RE, '') + .replace(UUID_RE, '') + .replace(ISO_RE, '') + .replace(SCORE_RE, '$1') + .replace(MS_RE, '$1') + .replace(JSON_COUNT_RE, '$1') + .replace(LABEL_COUNT_RE, '$1'); +} diff --git a/mcp-sse-server/parity/scenarios.js b/mcp-sse-server/parity/scenarios.js new file mode 100644 index 00000000..ab74112b --- /dev/null +++ b/mcp-sse-server/parity/scenarios.js @@ -0,0 +1,244 @@ +/** + * The tools/call scenario matrix for the parity harness. + * + * Each scenario runs independently against each transport, under that + * transport's own `tag` namespace so neither sees the other's writes. + * `$PREV..memory_id` is substituted with the memory id produced by call + * of the same scenario. + * + * Every scenario also owns a private `-sN` namespace and seeds its own + * fixtures. Sharing one namespace across scenarios looks tidier but is + * actively misleading: when a write capability differs — the remote rejects + * `store batch` while stdio inserts three records — every later recall in the + * shared tag compares two different datasets, so it reports a mismatch even + * when recall rendering is already in parity. That hides which fixes actually + * worked, exactly when you are landing them one at a time. + * + * Seeding therefore uses single-store only, never a mode one transport lacks. + * Every memory also carries the root `tag`, so cleanup stays a single + * bulk-delete per transport namespace. + */ + +// Each entry is (sTag, tag) => scenario, so the private namespace is bound at +// build time and every scenario is self-contained. +const SCENARIOS = [ + (s, t) => ({ + name: 'store single', + calls: [ + { + tool: 'store_memory', + args: { + content: 'Parity fixture alpha. Chose PostgreSQL for ACID.', + tags: [t, s], + importance: 0.7, + }, + }, + ], + }), + + (s, t) => ({ + name: 'store batch', + calls: [ + { + tool: 'store_memory', + args: { + memories: [ + { content: 'Parity batch one.', tags: [t, s], importance: 0.9 }, + { content: 'Parity batch two.', tags: [t, s], importance: 0.7 }, + { content: 'Parity batch three.', tags: [t, s], importance: 0.5 }, + ], + }, + }, + ], + }), + + (s, t) => ({ + name: 'store supersede', + calls: [ + { + tool: 'store_memory', + args: { content: 'Parity supersede original.', tags: [t, s], importance: 0.7 }, + }, + { + tool: 'store_memory', + args: { + content: 'Parity supersede replacement.', + tags: [t, s], + importance: 0.7, + supersedes_memory_id: '$PREV.0.memory_id', + supersede_reason: 'parity harness', + }, + }, + ], + }), + + (s, t) => ({ + name: 'recall ranked text', + calls: [ + { + tool: 'store_memory', + args: { content: 'Parity fixture alpha. Chose PostgreSQL for ACID.', tags: [t, s], importance: 0.9 }, + }, + { + tool: 'store_memory', + args: { content: 'Parity fixture beta. Qdrant holds the vectors.', tags: [t, s], importance: 0.7 }, + }, + { tool: 'recall_memory', args: { query: 'Parity fixture', tags: [s], limit: 5 } }, + ], + }), + + (s, t) => ({ + name: 'recall detailed', + calls: [ + { + tool: 'store_memory', + args: { content: 'Parity detailed fixture.', tags: [t, s], importance: 0.7 }, + }, + { tool: 'recall_memory', args: { query: 'Parity detailed', tags: [s], format: 'detailed' } }, + ], + }), + + (s, t) => ({ + name: 'recall items', + calls: [ + { + tool: 'store_memory', + args: { content: 'Parity items fixture.', tags: [t, s], importance: 0.7 }, + }, + { tool: 'recall_memory', args: { query: 'Parity items', tags: [s], format: 'items' } }, + ], + }), + + (s, t) => ({ + name: 'recall json', + calls: [ + { + tool: 'store_memory', + args: { content: 'Parity json fixture.', tags: [t, s], importance: 0.7 }, + }, + { tool: 'recall_memory', args: { query: 'Parity json', tags: [s], format: 'json' } }, + ], + }), + + (s) => ({ + name: 'recall empty', + calls: [ + { tool: 'recall_memory', args: { query: 'zzz-no-such-thing', tags: [`${s}-absent`] } }, + ], + }), + + (s, t) => ({ + name: 'recall id fetch', + calls: [ + { + tool: 'store_memory', + args: { content: 'Parity id-fetch target.', tags: [t, s], importance: 0.7 }, + }, + { tool: 'recall_memory', args: { memory_id: '$PREV.0.memory_id' } }, + ], + }), + + (s, t) => ({ + // Distinct importance values are load-bearing. GET /memory/by-tag orders by + // `importance DESC, timestamp DESC, id ASC` (automem/api/memory.py:259). + // With equal importance and sub-millisecond writes, `id ASC` becomes the + // tiebreaker — and ids differ between the two tag namespaces by + // construction, so ordering would differ while actually in parity. + name: 'recall exhaustive', + calls: [ + { tool: 'store_memory', args: { content: 'Parity enum one.', tags: [t, s], importance: 0.9 } }, + { tool: 'store_memory', args: { content: 'Parity enum two.', tags: [t, s], importance: 0.7 } }, + { tool: 'store_memory', args: { content: 'Parity enum three.', tags: [t, s], importance: 0.5 } }, + { tool: 'recall_memory', args: { tags: [s], exhaustive: true, limit: 2, offset: 0 } }, + ], + }), + + (s, t) => ({ + name: 'associate single', + calls: [ + { tool: 'store_memory', args: { content: 'Parity assoc source.', tags: [t, s], importance: 0.7 } }, + { tool: 'store_memory', args: { content: 'Parity assoc target.', tags: [t, s], importance: 0.7 } }, + { + tool: 'associate_memories', + args: { + memory1_id: '$PREV.0.memory_id', + memory2_id: '$PREV.1.memory_id', + type: 'EXEMPLIFIES', + strength: 0.8, + pattern_type: 'parity', + confidence: 0.9, + }, + }, + ], + }), + + (s, t) => ({ + name: 'associate batch partial failure', + calls: [ + { tool: 'store_memory', args: { content: 'Parity batch assoc source.', tags: [t, s], importance: 0.7 } }, + { + tool: 'associate_memories', + args: { + associations: [ + { + memory1_id: '$PREV.0.memory_id', + memory2_id: '00000000-0000-4000-8000-000000000000', + type: 'RELATES_TO', + strength: 0.5, + }, + ], + }, + }, + ], + }), + + (s, t) => ({ + name: 'update', + calls: [ + { tool: 'store_memory', args: { content: 'Parity update original.', tags: [t, s], importance: 0.7 } }, + { tool: 'update_memory', args: { memory_id: '$PREV.0.memory_id', importance: 0.95 } }, + ], + }), + + (s, t) => ({ + name: 'delete single', + calls: [ + { tool: 'store_memory', args: { content: 'Parity delete target.', tags: [t, s], importance: 0.7 } }, + { tool: 'delete_memory', args: { memory_id: '$PREV.0.memory_id' } }, + ], + }), + + (s, t) => ({ + name: 'delete by tag', + calls: [ + { tool: 'store_memory', args: { content: 'Parity bulk delete target.', tags: [t, s], importance: 0.7 } }, + { tool: 'delete_memory', args: { tags: [s] } }, + ], + }), + + () => ({ + name: 'health', + calls: [{ tool: 'check_database_health', args: {} }], + }), + + () => ({ + name: 'error: exhaustive without tags', + calls: [{ tool: 'recall_memory', args: { exhaustive: true } }], + }), + + (s, t) => ({ + name: 'error: store content over hard limit', + calls: [ + { tool: 'store_memory', args: { content: 'x'.repeat(2100), tags: [t, s], importance: 0.7 } }, + ], + }), + + () => ({ + name: 'error: unknown tool', + calls: [{ tool: 'no_such_tool', args: {} }], + }), +]; + +export function buildScenarios(tag) { + return SCENARIOS.map((make, i) => make(`${tag}-s${i}`, tag)); +} diff --git a/mcp-sse-server/test/parity.test.js b/mcp-sse-server/test/parity.test.js new file mode 100644 index 00000000..d3c2bad6 --- /dev/null +++ b/mcp-sse-server/test/parity.test.js @@ -0,0 +1,154 @@ +/** + * Cross-transport parity: the remote bridge and the stdio package must expose + * one identical MCP surface. See docs/MCP_TRANSPORT_PARITY.md for the contract + * and for the short list of differences that are allowed. + * + * Needs a live AutoMem at :8001, so it is gated. Run it with: + * make test-parity + */ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { randomUUID } from 'node:crypto'; +import { createRequire } from 'node:module'; +import { connectBothTransports } from '../parity/clients.js'; +import { normalizeKeys, redact } from '../parity/normalize.js'; +import { buildScenarios } from '../parity/scenarios.js'; + +const require = createRequire(import.meta.url); +const bridgePkg = require('../package.json'); +const stdioPkg = require('@verygoodplugins/mcp-automem/package.json'); + +const UUID_RE = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i; + +/** Resolve "$PREV..memory_id" against ids already collected in this scenario. */ +function resolveArgs(args, prior) { + return JSON.parse(JSON.stringify(args), (_key, value) => + typeof value === 'string' && value.startsWith('$PREV.') + ? prior[Number(value.split('.')[1])] + : value + ); +} + +/** Run every scenario against one transport, under its own tag namespace. */ +async function runScenarios(client, tag) { + const out = []; + for (const scenario of buildScenarios(tag)) { + const ids = []; + const rendered = []; + for (const call of scenario.calls) { + const res = await client + .callTool({ name: call.tool, arguments: resolveArgs(call.args, ids) }) + .catch((e) => ({ + content: [{ type: 'text', text: `THREW: ${e.message}` }], + isError: true, + })); + const text = (res.content || []).map((c) => c.text ?? '').join('\n'); + ids.push(res.structuredContent?.memory_id ?? (text.match(UUID_RE) || [])[0]); + + // structuredContent is client-visible machine-readable output, so it is + // part of the contract too. Comparing only text would let a mismatch in + // memory_ids, recall count, or health statistics pass unnoticed. + // Compared as a redacted string, not re-parsed: redaction substitutes + // placeholders like for numeric values, so the redacted form is + // deliberately not valid JSON. Key order is normalized first so the + // string comparison stays meaningful. + const structured = res.structuredContent + ? redact(JSON.stringify(normalizeKeys(res.structuredContent)), tag) + : null; + rendered.push({ isError: Boolean(res.isError), text: redact(text, tag), structured }); + } + out.push({ name: scenario.name, rendered }); + } + return out; +} + +const GATE = + process.env.AUTOMEM_RUN_PARITY_TESTS === '1' + ? false + : 'set AUTOMEM_RUN_PARITY_TESTS=1 with a live AutoMem at :8001 (make test-parity)'; + +test('tools/list is identical across transports', { skip: GATE }, async () => { + const { remote, stdio, close } = await connectBothTransports(); + try { + const a = normalizeKeys((await remote.listTools()).tools); + const b = normalizeKeys((await stdio.listTools()).tools); + assert.deepStrictEqual(a, b); + } finally { + await close(); + } +}); + +test('server capabilities and instructions match', { skip: GATE }, async () => { + const { remote, stdio, close } = await connectBothTransports(); + try { + assert.deepStrictEqual( + normalizeKeys(remote.getServerCapabilities()), + normalizeKeys(stdio.getServerCapabilities()) + ); + assert.equal(remote.getInstructions(), stdio.getInstructions()); + + // serverInfo is transport-specific and allowlisted to differ, but each + // side must still report its OWN package version. The remote's hardcoded + // 0.1.0 disagreeing with its package.json is exactly the drift the audit + // records, so pin self-consistency rather than cross-transport equality. + assert.equal(remote.getServerVersion().name, 'automem-mcp-sse'); + assert.equal(stdio.getServerVersion().name, 'mcp-automem'); + + assert.equal( + remote.getServerVersion().version, + bridgePkg.version, + 'remote serverInfo.version must match mcp-sse-server/package.json' + ); + assert.equal( + stdio.getServerVersion().version, + stdioPkg.version, + 'stdio serverInfo.version must match the published package version' + ); + } finally { + await close(); + } +}); + +test('tools/call renders identically across transports', { skip: GATE }, async () => { + const { remote, stdio, close } = await connectBothTransports(); + const remoteTag = `parity-remote-${randomUUID()}`; + const stdioTag = `parity-stdio-${randomUUID()}`; + try { + const a = await runScenarios(remote, remoteTag); + const b = await runScenarios(stdio, stdioTag); + + // Collect every mismatch before failing. A run costs ~90s, so reporting + // one scenario at a time turns a multi-scenario gap into a fix-one, + // rerun, fix-the-next loop. + const mismatched = []; + for (let i = 0; i < a.length; i++) { + try { + assert.deepStrictEqual(a[i], b[i]); + } catch { + mismatched.push(i); + } + } + if (mismatched.length) { + const names = mismatched.map((i) => a[i].name).join(', '); + const first = mismatched[0]; + assert.deepStrictEqual( + a[first], + b[first], + `${mismatched.length}/${a.length} scenarios differ: ${names}\nFirst mismatch (${a[first].name}) diffed below.` + ); + } + } finally { + // Every fixture carries its transport's root tag regardless of which + // per-scenario namespace it also has, so one bulk delete per namespace is + // enough. Bulk delete by tag exists only on the stdio transport for now, + // so cleanup for both runs there. Swallowed so a cleanup failure can never + // mask an assertion failure. + await stdio + .callTool({ + name: 'delete_memory', + arguments: { tags: [remoteTag, stdioTag] }, + }) + .catch(() => {}); + await close(); + } +}); diff --git a/scripts/README.md b/scripts/README.md index 6db5fba6..3ef6da5e 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1,10 +1,19 @@ # AutoMem scripts Operational, migration, recovery, and evaluation tooling for an AutoMem -instance. This file is the catalog — the single place to find "what scripts -exist and when do I run each one." For the narrative runbooks (why and when), -see the [docs portal](https://automem.ai/docs/); for exact flags, run a script -with `--help` or read its docstring header. +instance. This is the canonical inventory: every active executable in +`scripts/`, when to use it, and the safest way to start. + +Run commands from the repository root with the project environment available: + +```bash +source .venv/bin/activate +``` + +Most CLI scripts support `--help`; `cleanup_memory_types.py` and +`recover_from_qdrant.py` intentionally have no flags, so use their documented +commands only. For narrative runbooks, follow the linked documents rather than +inventing a new sequence. ## How to read this @@ -19,30 +28,35 @@ before running it: | Tag | Meaning | |---|---| -| `routine` | Safe to run repeatedly as normal operations. | -| `one-time` | Run once per instance or per upgrade. Idempotent, but not part of day-to-day ops. See [docs/MIGRATIONS.md](../docs/MIGRATIONS.md). | -| `recovery` | Break-glass. Only after data loss or corruption. See [docs/MONITORING_AND_BACKUPS.md](../docs/MONITORING_AND_BACKUPS.md). | +| `read-only` | Inspects data or streams events without writing. | +| `maintenance` | Operational tool that may write or queue work; review the scope and back up first when it changes stored data. | +| `one-time` | Run once per instance or per upgrade. Idempotent where noted, but not part of day-to-day ops. See [docs/MIGRATIONS.md](../docs/MIGRATIONS.md). | +| `recovery` | Break-glass and potentially destructive. Only after data loss or corruption. See [docs/MONITORING_AND_BACKUPS.md](../docs/MONITORING_AND_BACKUPS.md). | | `dev` | Local development / deployment helpers. | | `bench` · `lab` | Contributor evaluation and recall-tuning harnesses. Not needed to run AutoMem. See [docs/TESTING.md](../docs/TESTING.md) and [docs/RECALL_QUALITY_LAB.md](../docs/RECALL_QUALITY_LAB.md). | +Before running a script that can change FalkorDB or Qdrant, confirm the target +from `.env`, take a backup, and start with `--dry-run` or a small `--limit` when +the script offers one. Do not run recovery, re-embedding, or migration scripts +against production while writers are active. + --- -## Routine operations +## Operations and maintenance -Run these as part of normal upkeep. +Use these for normal operations or deliberate small maintenance tasks. -| Script | Lifecycle | What it does | -|---|---|---| -| [`backup_automem.py`](backup_automem.py) | `routine` | Timestamped FalkorDB + Qdrant backup; optional S3 upload and old-backup cleanup. Cron-friendly. `--s3-bucket`, `--cleanup --keep N`. | -| [`restore_from_backup.py`](restore_from_backup.py) | `routine` · `recovery` | Restore FalkorDB + Qdrant from a backup (local snapshot or downloaded API tarball). `--backup-timestamp`, `--backup-dir snapshot.tar.gz`, `--dry-run`. | -| [`health_monitor.py`](health_monitor.py) | `routine` | Background service: watches FalkorDB/Qdrant health, checks graph↔vector consistency, triggers recovery, alerts. `--interval 300`. Containerized via [`Dockerfile.health-monitor`](Dockerfile.health-monitor). See [docs/HEALTH_MONITORING.md](../docs/HEALTH_MONITORING.md). | -| [`automem_watch.py`](automem_watch.py) | `routine` | Real-time terminal UI over the SSE event stream; flags garbage-write patterns and consolidation timing. `--url`, `--token`. | -| [`audit_relevance.py`](audit_relevance.py) | `routine` | Audit the `relevance_score` distribution from a backup file (default) or a live instance (`--live`). | -| [`reembed_embeddings.py`](reembed_embeddings.py) | `routine` | Re-embed all memories and upsert vectors into Qdrant using the configured provider. `--batch-size`, `--limit`. Used after provider/dimension changes — see [docs/MIGRATIONS.md](../docs/MIGRATIONS.md). | -| [`reclassify_with_llm.py`](reclassify_with_llm.py) | `routine` | Reclassify fallback `type='Memory'` records via the configured classification LLM. `--provider`; env `CLASSIFICATION_MODEL` / `CLASSIFICATION_BASE_URL` / `CLASSIFICATION_API_KEY`. | -| [`reenrich_batch.py`](reenrich_batch.py) | `routine` | Re-run enrichment over a batch of memories with current classification logic. | +| Script | Lifecycle | When to use it | Start here | +|---|---|---|---| +| [`backup_automem.py`](backup_automem.py) | `maintenance` | Create a portable FalkorDB + Qdrant backup before a migration or on a schedule. | `python scripts/backup_automem.py`; add `--s3-bucket … --cleanup --keep 7` for off-site retention. | +| [`health_monitor.py`](health_monitor.py) | `maintenance` | Continuously check API, FalkorDB, Qdrant, and drift; use alert-only mode by default. | `python scripts/health_monitor.py --once` or `--interval 300`. `--auto-recover` is an explicit, high-risk opt-in. See [docs/HEALTH_MONITORING.md](../docs/HEALTH_MONITORING.md). | +| [`automem_watch.py`](automem_watch.py) | `read-only` | Observe live store/recall/update/delete, enrichment, and consolidation events. | `python scripts/automem_watch.py --url "$AUTOMEM_API_URL" --token "$AUTOMEM_API_TOKEN"` | +| [`audit_relevance.py`](audit_relevance.py) | `read-only` | Inspect `relevance_score` distribution before or after scoring changes. | `python scripts/audit_relevance.py` reads the newest backup; use `--live` only for the configured instance. | +| [`reembed_embeddings.py`](reembed_embeddings.py) | `maintenance` | Replace vectors after an embedding provider, model, or dimension migration. | After backing up and recreating the collection, run `python scripts/reembed_embeddings.py --batch-size 32`. Requires `QDRANT_URL`; use `--limit 100` only for a smoke check. See [docs/MIGRATIONS.md](../docs/MIGRATIONS.md). | +| [`reclassify_with_llm.py`](reclassify_with_llm.py) | `maintenance` | Reclassify fallback `type='Memory'` records after changing classification logic. | Start with `python scripts/reclassify_with_llm.py --dry-run --limit 25`; apply only after review (the script asks for confirmation unless `--yes` is supplied). | +| [`reenrich_batch.py`](reenrich_batch.py) | `maintenance` | Queue a small batch for re-enrichment after an enrichment logic change. | `python scripts/reenrich_batch.py --limit 10`; this calls the configured API and queues work. | -### `browse_memories.py` — read-only database browser `dev` +### `browse_memories.py` — read-only database browser Interactive CLI over the production FalkorDB graph + Qdrant vectors. Connects with `.env` credentials and **never modifies data**. Four subcommands: @@ -72,16 +86,16 @@ and every graph relationship. `diagnose` reports issues at `[CRITICAL]` / ## One-time migrations -Run once per instance or when upgrading. Idempotent and safe to re-run, but not -part of routine ops. Full runbook: [docs/MIGRATIONS.md](../docs/MIGRATIONS.md). +Run when adopting a specific upgrade or repair. Most are idempotent, but they +are not routine operations; back up first and follow the linked runbook. -| Script | Lifecycle | What it does | +| Script | When to use it | Start here | |---|---|---| -| [`migrate_mcp_sqlite.py`](migrate_mcp_sqlite.py) | `one-time` | Import the legacy MCP `sqlite_vec.db` memory store into AutoMem via the API, preserving timestamps/tags/importance. `--db`, `--automem-url`, `--api-token`, `--dry-run`. | -| [`migrate_entity_nodes.py`](migrate_entity_nodes.py) | `one-time` | Promote `entity:{category}:{slug}` tags on Memory nodes into first-class `Entity` nodes linked by `REFERENCED_IN`. `--dry-run`. (0.16.0) | -| [`backfill_tag_prefixes.py`](backfill_tag_prefixes.py) | `one-time` | Compute and backfill `tag_prefixes` in FalkorDB + Qdrant from existing tags (keeps prefix-match recall consistent). | -| [`rescore_relevance.py`](rescore_relevance.py) | `one-time` | Recompute every `relevance_score` with the corrected decay formula (undoes the old over-aggressive 0.1 rate). `--dry-run`, `--target`. | -| [`cleanup_memory_types.py`](cleanup_memory_types.py) | `one-time` | Reclassify invalid memory types (e.g. `session_start`, `interaction`) back to valid types. No flags; reads `.env`. | +| [`migrate_mcp_sqlite.py`](migrate_mcp_sqlite.py) | Moving from the legacy MCP `sqlite_vec.db` store into AutoMem. | Preview first: `python scripts/migrate_mcp_sqlite.py --dry-run`; then pass `--db`, `--automem-url`, and `--api-token`. | +| [`migrate_entity_nodes.py`](migrate_entity_nodes.py) | Adopting first-class `Entity` nodes for legacy `entity:{category}:{slug}` tags. | `python scripts/migrate_entity_nodes.py --dry-run`, then rerun without the flag. | +| [`backfill_tag_prefixes.py`](backfill_tag_prefixes.py) | Restoring or introducing `tag_prefixes` so prefix recall remains consistent. | `python scripts/backfill_tag_prefixes.py --dry-run`; apply after review. Use `--no-qdrant` only when vector payload sync is intentionally deferred. | +| [`rescore_relevance.py`](rescore_relevance.py) | Repairing relevance scores produced with the previous over-aggressive decay formula. | `python scripts/rescore_relevance.py --dry-run`, then choose the intended `--target` before applying. | +| [`cleanup_memory_types.py`](cleanup_memory_types.py) | Repairing legacy invalid types such as `session_start` or `interaction`. | Back up, verify `.env` targets the intended instance, then run `python scripts/cleanup_memory_types.py`. It has no preview mode. | > `scripts/lab/repair_entity_tags.py` (`lab`, below) is the companion repair tool > for entity-tag noise on a local clone before promoting entity nodes. @@ -93,20 +107,20 @@ part of routine ops. Full runbook: [docs/MIGRATIONS.md](../docs/MIGRATIONS.md). Only reach for these after data loss or corruption. See [docs/MONITORING_AND_BACKUPS.md](../docs/MONITORING_AND_BACKUPS.md). -| Script | Lifecycle | What it does | +| Script | When to use it | Start here | |---|---|---| -| [`recover_from_qdrant.py`](recover_from_qdrant.py) | `recovery` | Rebuild the FalkorDB graph from Qdrant: reads every vector's payload and re-inserts via the API, which regenerates relationships. No flags; reads `.env`. | -| [`deduplicate_qdrant.py`](deduplicate_qdrant.py) | `recovery` | Remove duplicate Qdrant points (e.g. after a recovery run double-inserted). `--dry-run`, `--auto-confirm`. | -| [`restore_from_backup.py`](restore_from_backup.py) | `recovery` · `routine` | See Routine operations above. | +| [`restore_from_backup.py`](restore_from_backup.py) | Restoring one or both stores from a tested local backup or API-exported tarball. | Always preview first: `python scripts/restore_from_backup.py --dry-run`. Then select `--backup-timestamp` or `--backup-dir`, optionally `--falkordb-only` / `--qdrant-only`, and use `--force` only after review. | +| [`recover_from_qdrant.py`](recover_from_qdrant.py) | FalkorDB is lost or corrupt while Qdrant is known-good and complete. | **Destructive:** it clears the configured FalkorDB graph, then rebuilds it from Qdrant. Back up and verify Qdrant before `python scripts/recover_from_qdrant.py`. | +| [`deduplicate_qdrant.py`](deduplicate_qdrant.py) | Qdrant contains duplicates, commonly after a failed recovery or manual import. | `python scripts/deduplicate_qdrant.py --dry-run`; review, then rerun with `--yes` to delete. | --- -## Developer & deployment +## Developer and deployment -| Script | Lifecycle | What it does | Make target | +| Script | When to use it | Start here | Make target | |---|---|---|---| -| [`bootstrap_dev.sh`](bootstrap_dev.sh) | `dev` | Create `.venv` (Python 3.12), refresh `venv -> .venv`, install dev deps + pre-commit hooks. | `make install` | -| [`deploy_check.sh`](deploy_check.sh) | `dev` | Compare the live Railway deployment commit against `origin/main` to catch a silently disconnected GitHub integration. `DEPLOY_CHECK_QUIET=1` for CI. | `make deploy-check` | +| [`bootstrap_dev.sh`](bootstrap_dev.sh) | Setting up or repairing a local contributor environment. | `make install` (or `./scripts/bootstrap_dev.sh`) creates `.venv`, refreshes `venv -> .venv`, installs development dependencies, and installs pre-commit hooks. | `make install` | +| [`deploy_check.sh`](deploy_check.sh) | Checking that Railway is deploying the expected Git commit. | `./scripts/deploy_check.sh automem`; use `DEPLOY_CHECK_QUIET=1` only for CI-style exit-code checks. Requires linked Railway and GitHub CLIs. | `make deploy-check` | --- @@ -114,16 +128,16 @@ Only reach for these after data loss or corruption. See Snapshot-based LoCoMo / LongMemEval evaluation. See [docs/TESTING.md](../docs/TESTING.md). -| Script | What it does | Make target | -|---|---|---| -| [`bench/ingest_and_snapshot.sh`](bench/ingest_and_snapshot.sh) | Ingest a benchmark dataset into Docker AutoMem and snapshot the volumes (run once). | `make bench-ingest BENCH=locomo` | -| [`bench/restore_and_eval.sh`](bench/restore_and_eval.sh) | Restore a snapshot and evaluate a config (no re-ingest). | `make bench-eval BENCH=locomo CONFIG=baseline` | -| [`bench/compare_configs.sh`](bench/compare_configs.sh) | A/B two scoring configs against the same snapshot. | `make bench-compare` | -| [`bench/compare_branch.sh`](bench/compare_branch.sh) | Compare a git branch against `main` on a snapshot. | `make bench-compare-branch BRANCH=…` | -| [`bench/compare_results.py`](bench/compare_results.py) | Side-by-side table of two result JSON files. `--baseline`, `--test`, `--output`. | — | -| [`bench/analyze_locomo_results.py`](bench/analyze_locomo_results.py) | Markdown failure report from a LoCoMo results JSON. `--output`. (0.16.0) | — | -| [`bench/health_check.py`](bench/health_check.py) | Post-restore diagnostics: score distribution, entity quality, latency, precision on curated queries. | `make bench-health` | -| [`run_longmemeval_watch.sh`](run_longmemeval_watch.sh) | LongMemEval with persistent logging + desktop completion/crash notifications. | `make test-longmemeval-watch` | +| Script | When to use it | Start here | Make target | +|---|---|---|---| +| [`bench/ingest_and_snapshot.sh`](bench/ingest_and_snapshot.sh) | Creating a fresh benchmark snapshot after a corpus or embedding change. | `make bench-ingest BENCH=locomo`; this starts Docker and writes reusable snapshots. | `make bench-ingest BENCH=locomo` | +| [`bench/restore_and_eval.sh`](bench/restore_and_eval.sh) | Evaluating one scoring configuration against an existing snapshot. | `make bench-eval BENCH=locomo CONFIG=baseline` | `make bench-eval` | +| [`bench/compare_configs.sh`](bench/compare_configs.sh) | A/B comparing two scoring configurations on the same snapshot. | `make bench-compare BENCH=locomo BASELINE=baseline CONFIG=` | `make bench-compare` | +| [`bench/compare_branch.sh`](bench/compare_branch.sh) | Comparing a branch against `main` using a snapshot. | `make bench-compare-branch BRANCH=`; the script temporarily checks out refs, so begin from a clean worktree. | `make bench-compare-branch` | +| [`bench/compare_results.py`](bench/compare_results.py) | Reading two existing result JSON files without rerunning a benchmark. | `python scripts/bench/compare_results.py --baseline --test ` | — | +| [`bench/analyze_locomo_results.py`](bench/analyze_locomo_results.py) | Producing a Markdown failure report from a LoCoMo result JSON. | `python scripts/bench/analyze_locomo_results.py --output report.md` | — | +| [`bench/health_check.py`](bench/health_check.py) | Checking post-restore score distributions, entity quality, latency, and curated-query precision. | `make bench-health` | `make bench-health` | +| [`run_longmemeval_watch.sh`](run_longmemeval_watch.sh) | Running LongMemEval with persistent logging and local completion/crash notifications. | `make test-longmemeval-watch`; for a smaller run use `./scripts/run_longmemeval_watch.sh --max-questions 50`. | `make test-longmemeval-watch` | --- @@ -132,15 +146,15 @@ Snapshot-based LoCoMo / LongMemEval evaluation. See [docs/TESTING.md](../docs/TE Data-driven recall scoring experiments against a clone of production. Full workflow: [docs/RECALL_QUALITY_LAB.md](../docs/RECALL_QUALITY_LAB.md). -| Script | What it does | Make target | -|---|---|---| -| [`lab/clone_production.sh`](lab/clone_production.sh) | Clone production data into an isolated local Docker stack (direct DB backup, or `--restore-only` from a saved API tarball; supports custom ports for parallel sweeps). | `make lab-clone` | -| [`lab/create_test_queries.py`](lab/create_test_queries.py) | Generate a natural-question test set from local memories (via GPT-4o-mini). `--count`, `--output`, `--api-url`. | `make lab-queries` | -| [`lab/run_recall_test.py`](lab/run_recall_test.py) | Run a test set under a config, compute Recall@K / MRR / NDCG, A/B compare, and sweep a parameter. `--config`, `--compare`, `--sweep`. | `make lab-test` · `make lab-compare` · `make lab-sweep` | -| [`lab/repair_entity_tags.py`](lab/repair_entity_tags.py) | Audit → plan → execute/rollback repair of noisy generated entity tags on a clone. `--mode audit\|execute\|rollback`. (0.16.0) | — | -| [`lab/lab_metrics.py`](lab/lab_metrics.py) | **Library module** (not a CLI): pure, deterministic IR scoring functions (Recall@K, MRR, NDCG, distractor rate). Imported by `run_recall_test.py`. (0.16.0) | — | -| [`lab/lab_corpus.py`](lab/lab_corpus.py) | **Library module** (not a CLI): recall/corpus HTTP helpers behind injectable clients for unit-testable lab logic. (0.16.0) | — | -| `lab/configs/` | JSON scoring-weight overrides for A/B testing (`baseline.json`, `issue78_*.json`). | — | +| Script | When to use it | Start here | Make target | +|---|---|---|---| +| [`lab/clone_production.sh`](lab/clone_production.sh) | Creating an isolated local copy of production data for recall experiments. | `make lab-clone`; use `--restore-only ` for repeat experiments so production is not contacted again. | `make lab-clone` | +| [`lab/create_test_queries.py`](lab/create_test_queries.py) | Generating a natural-language evaluation set from a local clone. | `make lab-queries` or `python scripts/lab/create_test_queries.py --count 100`. | `make lab-queries` | +| [`lab/run_recall_test.py`](lab/run_recall_test.py) | Measuring one configuration, an A/B comparison, or a parameter sweep. | `make lab-test CONFIG=baseline`; use `make lab-compare` or `make lab-sweep` for the other modes. | `make lab-test` · `make lab-compare` · `make lab-sweep` | +| [`lab/repair_entity_tags.py`](lab/repair_entity_tags.py) | Repairing noisy generated entity tags on a **local clone**. | Plan first: `python scripts/lab/repair_entity_tags.py --mode canonicalize-safe`; review `/plan.jsonl`, then apply with `--execute --plan /plan.jsonl` or undo with `--rollback /rollback.jsonl`. | — | +| [`lab/lab_metrics.py`](lab/lab_metrics.py) | **Library module**, not a CLI; implements deterministic Recall@K, MRR, NDCG, and distractor-rate metrics. | Import from `run_recall_test.py` or tests. | — | +| [`lab/lab_corpus.py`](lab/lab_corpus.py) | **Library module**, not a CLI; centralizes injectable recall/corpus HTTP helpers. | Import from `run_recall_test.py` or tests. | — | +| [`lab/configs/`](lab/configs/) | Creating named JSON scoring-weight overrides for A/B tests. | Copy `baseline.json`, edit weights, then pass the filename without `.json` to `CONFIG`. | — | --- @@ -148,14 +162,12 @@ workflow: [docs/RECALL_QUALITY_LAB.md](../docs/RECALL_QUALITY_LAB.md). | File | What it is | |---|---| -| [`lib/common.sh`](lib/common.sh) | Shell helpers (color codes, `wait_for_api`) sourced by the `bench/` scripts. | -| [`archive/`](archive/) | Retired one-off scripts kept for reference (e.g. dated release-sweep summarizers). Not maintained. | - ---- +| [`lib/common.sh`](lib/common.sh) | Support module, not a CLI. Provides color helpers and `wait_for_api` to benchmark shell scripts. | +| [`Dockerfile.health-monitor`](Dockerfile.health-monitor) | Container recipe for running `health_monitor.py` in alert-only mode. Use only when you intentionally operate monitoring as a separate service. | ## See also -- [docs/MIGRATIONS.md](../docs/MIGRATIONS.md) — embedding-dimension and one-time data migrations +- [docs/MIGRATIONS.md](../docs/MIGRATIONS.md) — embedding-provider, model, and one-time data migrations - [docs/MONITORING_AND_BACKUPS.md](../docs/MONITORING_AND_BACKUPS.md) — backup/restore/recovery runbook - [docs/HEALTH_MONITORING.md](../docs/HEALTH_MONITORING.md) — `health_monitor.py` deployment - [docs/RECALL_QUALITY_LAB.md](../docs/RECALL_QUALITY_LAB.md) — the `lab/` harness end to end diff --git a/scripts/archive/summarize_pipeline_20260611.py b/scripts/archive/summarize_pipeline_20260611.py deleted file mode 100644 index 1b0e86d7..00000000 --- a/scripts/archive/summarize_pipeline_20260611.py +++ /dev/null @@ -1,131 +0,0 @@ -#!/usr/bin/env python3 -"""Tabulate the 2026-06-11 release-verification sweep results. - -Pools prod_parity runs as baseline; reports per-config metric deltas with a -paired difference test on per-query recall@5 (computed from retrieved_ids[:5]) -against parity run 1, plus per-category R@10 deltas. The p-value uses a normal -(z) approximation of the paired t-statistic — accurate for the n=200 query -sets here, but not a Student's t p-value for small n. -""" - -import glob -import json -import math -import os -import sys -from collections import defaultdict - -RESULTS_DIR = sys.argv[1] if len(sys.argv) > 1 else "lab/results" -RUN_DATE = "20260611" - -CONFIG_ORDER = [ - "prod_parity", - "cap2", - "cap3", - "cap4", - "gate005", - "gate010", - "gate015", - "gate020", - "gate025", - "recw90", - "recw365", - "recexp", - "recbias", -] - - -def load_runs(): - runs = defaultdict(list) - for path in sorted(glob.glob(os.path.join(RESULTS_DIR, f"*_{RUN_DATE}_*.json"))): - name = os.path.basename(path) - for cfg in CONFIG_ORDER: - if name.startswith(cfg + "_" + RUN_DATE): - with open(path) as fh: - runs[cfg].append(json.load(fh)) - break - return runs - - -def perq_recall5(run): - out = {} - for q in run.get("queries", []): - exp = q.get("expected_ids") or [] - top5 = set((q.get("retrieved_ids") or [])[:5]) - out[q["query"]] = (sum(1 for e in exp if e in top5) / len(exp)) if exp else None - return out - - -def paired_z(a, b): - # z-approximation: t-statistic against the normal CDF (n=200 here, so t ~= z) - diffs = [y - x for x, y in zip(a, b)] - n = len(diffs) - if n < 2: - return float("nan") - mean = sum(diffs) / n - var = sum((d - mean) ** 2 for d in diffs) / (n - 1) - if var == 0: - return 1.0 - t = mean / math.sqrt(var / n) - return 2 * (1 - 0.5 * (1 + math.erf(abs(t) / math.sqrt(2)))) - - -def main(): - runs = load_runs() - if not runs.get("prod_parity"): - sys.exit(f"no prod_parity runs found in {RESULTS_DIR}") - - base_runs = runs["prod_parity"] - base_pq = perq_recall5(base_runs[0]) - base_r5 = sum(r["summary"]["recall_5"] for r in base_runs) / len(base_runs) - - print( - f"{'config':>12} {'runs':>4} {'R@5':>7} {'R@10':>7} {'MRR':>7} {'NDCG':>7} {'dR@5':>7} {'p':>8}" - ) - print("-" * 66) - for cfg in CONFIG_ORDER: - rs = runs.get(cfg, []) - if not rs: - continue - s = [r["summary"] for r in rs] - r5 = sum(x["recall_5"] for x in s) / len(s) - r10 = sum(x["recall_10"] for x in s) / len(s) - mrr = sum(x["mrr"] for x in s) / len(s) - ndcg = sum(x["ndcg_10"] for x in s) / len(s) - if cfg == "prod_parity": - print( - f"{cfg:>12} {len(rs):>4} {r5:7.3f} {r10:7.3f} {mrr:7.3f} {ndcg:7.3f} {'—':>7} {'—':>8}" - ) - continue - cand_pq = perq_recall5(rs[0]) - common = [q for q, v in base_pq.items() if v is not None and cand_pq.get(q) is not None] - p = paired_z([base_pq[q] for q in common], [cand_pq[q] for q in common]) - print( - f"{cfg:>12} {len(rs):>4} {r5:7.3f} {r10:7.3f} {mrr:7.3f} {ndcg:7.3f} {r5 - base_r5:+7.3f} {p:8.4f}" - ) - - print("\nPer-category R@10 delta vs parity run 1 (n in parens):") - base_cat = base_runs[0].get("by_category", {}) - cats = sorted(base_cat) - print(f"{'config':>12} " + " ".join(f"{c[:9]:>9}" for c in cats)) - print( - f"{'(n)':>12} " - + " ".join(f"{'(' + str(base_cat[c].get('count', 0)) + ')':>9}" for c in cats) - ) - for cfg in CONFIG_ORDER: - rs = runs.get(cfg, []) - if not rs: - continue - cat = rs[0].get("by_category", {}) - if cfg == "prod_parity": - row = [f"{base_cat[c].get('recall_10', 0):.3f}" for c in cats] - else: - row = [ - f"{cat.get(c, {}).get('recall_10', 0) - base_cat[c].get('recall_10', 0):+.3f}" - for c in cats - ] - print(f"{cfg:>12} " + " ".join(f"{v:>9}" for v in row)) - - -if __name__ == "__main__": - main() diff --git a/scripts/automem_watch.py b/scripts/automem_watch.py index 42f8fa59..f6a6f412 100755 --- a/scripts/automem_watch.py +++ b/scripts/automem_watch.py @@ -36,6 +36,21 @@ sys.exit(1) +def event_count(data: Dict, default: int = 1) -> int: + """Read an SSE payload count, preserving explicit zeros.""" + if not isinstance(data, dict) or "count" not in data: + return default + try: + return int(data["count"]) + except (TypeError, ValueError): + return default + + +def is_single_memory_store(data: Dict) -> bool: + """True when a memory.store payload describes one concrete memory.""" + return bool(isinstance(data, dict) and data.get("id") and event_count(data) == 1) + + class GarbageDetector: """Detect suspicious patterns in memory stores.""" @@ -134,6 +149,9 @@ def __init__(self, url: str, token: str, min_content_length: int = 10): self.stats = { "stores": 0, "recalls": 0, + "updates": 0, + "deletes": 0, + "associates": 0, "enriched": 0, "consolidated": 0, "errors": 0, @@ -172,15 +190,22 @@ def _process_event(self, event: Dict) -> None: event_type = event.get("type", "") # Update stats + data = event.get("data", {}) or {} if event_type == "memory.store": - self.stats["stores"] += 1 - # Check for garbage - warning = self.garbage.analyze(event) - if warning: - ts = datetime.now().strftime("%H:%M:%S") - self.errors.appendleft(f"[{ts}] [GARBAGE] {warning}") + self.stats["stores"] += event_count(data) + if is_single_memory_store(data): + warning = self.garbage.analyze(event) + if warning: + ts = datetime.now().strftime("%H:%M:%S") + self.errors.appendleft(f"[{ts}] [GARBAGE] {warning}") elif event_type == "memory.recall": self.stats["recalls"] += 1 + elif event_type == "memory.update": + self.stats["updates"] += 1 + elif event_type == "memory.delete": + self.stats["deletes"] += event_count(data) + elif event_type == "memory.associate": + self.stats["associates"] += event_count(data) elif event_type == "enrichment.complete": self.stats["enriched"] += 1 elif event_type == "enrichment.failed": @@ -254,8 +279,11 @@ def render(self) -> Layout: # Format based on event type if event_type == "memory.store": + count = data.get("count", 1) preview = data.get("content_preview", "")[:40] - details = f"{preview}... ({data.get('type', '?')})" + details = f"{preview} ({data.get('type', '?')})" + if count and count > 1: + details = f"{count} stored" type_style = "green" elif event_type == "memory.recall": query = data.get("query", "")[:30] @@ -263,6 +291,26 @@ def render(self) -> Layout: f"'{query}' -> {data.get('result_count', 0)} ({data.get('elapsed_ms', 0)}ms)" ) type_style = "cyan" + elif event_type == "memory.update": + preview = data.get("content_preview", "")[:40] + fields = ",".join(data.get("fields") or []) + details = f"{preview} [{fields}]" if fields else preview + type_style = "yellow" + elif event_type == "memory.delete": + count = data.get("count", 1) + deleted_id = data.get("id", "")[:8] + tags = ",".join(data.get("tags") or []) + details = f"{count} deleted" + if deleted_id: + details = f"{deleted_id}... ({count})" + elif tags: + details = f"{details} tags={tags}" + type_style = "red" + elif event_type == "memory.associate": + count = data.get("count", 1) + relation = data.get("relation_type") or ",".join(data.get("relation_types") or []) + details = f"{relation or 'RELATES_TO'} x{count}" + type_style = "blue" elif event_type == "enrichment.start": details = f"{data.get('memory_id', '')[:8]}... attempt {data.get('attempt', 1)}" type_style = "yellow" @@ -307,6 +355,9 @@ def render(self) -> Layout: stats_text = Text() stats_text.append(f"Stores: {self.stats['stores']}\n", style="green") stats_text.append(f"Recalls: {self.stats['recalls']}\n", style="cyan") + stats_text.append(f"Updates: {self.stats['updates']}\n", style="yellow") + stats_text.append(f"Deletes: {self.stats['deletes']}\n", style="red") + stats_text.append(f"Associates: {self.stats['associates']}\n", style="blue") stats_text.append(f"Enriched: {self.stats['enriched']}\n", style="yellow") stats_text.append(f"Consolidated: {self.stats['consolidated']}\n", style="magenta") stats_text.append(f"Errors: {self.stats['errors']}\n", style="red") diff --git a/scripts/lab/clone_production.sh b/scripts/lab/clone_production.sh index 146ef0af..0bbd2178 100755 --- a/scripts/lab/clone_production.sh +++ b/scripts/lab/clone_production.sh @@ -433,6 +433,7 @@ QDRANT_RESTORE_INDEXING_THRESHOLD="${QDRANT_RESTORE_INDEXING_THRESHOLD:-0}" \ QDRANT_RESTORE_DEFAULT_SEGMENT_NUMBER="${QDRANT_RESTORE_DEFAULT_SEGMENT_NUMBER:-1}" \ QDRANT_RESTORE_MEMMAP_THRESHOLD="${QDRANT_RESTORE_MEMMAP_THRESHOLD:-0}" \ QDRANT_RESTORE_HNSW_M="${QDRANT_RESTORE_HNSW_M:-0}" \ +QDRANT_RESTORE_HNSW_ON_DISK="${QDRANT_RESTORE_HNSW_ON_DISK:-true}" \ QDRANT_RESTORE_VECTOR_ON_DISK="${QDRANT_RESTORE_VECTOR_ON_DISK:-true}" \ QDRANT_RESTORE_ON_DISK_PAYLOAD="${QDRANT_RESTORE_ON_DISK_PAYLOAD:-false}" \ "$PYTHON_BIN" "$PROJECT_ROOT/scripts/restore_from_backup.py" \ diff --git a/scripts/recover_from_qdrant.py b/scripts/recover_from_qdrant.py index 58bdd102..780254ef 100755 --- a/scripts/recover_from_qdrant.py +++ b/scripts/recover_from_qdrant.py @@ -26,6 +26,7 @@ FALKORDB_HOST = os.getenv("FALKORDB_HOST", "localhost") FALKORDB_PORT = int(os.getenv("FALKORDB_PORT", "6379")) FALKORDB_PASSWORD = os.getenv("FALKORDB_PASSWORD") +FALKORDB_GRAPH = os.getenv("FALKORDB_GRAPH", "memories") BATCH_SIZE = 50 @@ -79,7 +80,7 @@ def restore_memory_to_graph_only(memory: Dict[str, Any], client) -> bool: try: # Store directly to FalkorDB graph - g = client.select_graph("memories") + g = client.select_graph(FALKORDB_GRAPH) # Build metadata string (exclude reserved fields to prevent overwriting) RESERVED_FIELDS = {"type", "confidence", "content", "timestamp", "importance", "tags", "id"} @@ -124,6 +125,19 @@ def restore_memory_to_graph_only(memory: Dict[str, Any], client) -> bool: return False +def clear_graph(client: FalkorDB) -> bool: + """Remove all nodes from the configured FalkorDB graph before recovery.""" + print(f"🗑️ Clearing graph '{FALKORDB_GRAPH}'...") + try: + graph = client.select_graph(FALKORDB_GRAPH) + graph.query("MATCH (n) DETACH DELETE n") + print("✅ Graph cleared\n") + return True + except Exception as exc: + print(f"⚠️ Could not clear graph: {exc}\n") + return False + + def main(): """Main recovery process.""" print("=" * 60) @@ -145,14 +159,8 @@ def main(): print(f"❌ Failed to connect to FalkorDB: {e}") sys.exit(1) - # Clear existing graph - print("🗑️ Clearing existing graph data...") - try: - g = client.select_graph("memories") - g.query("MATCH (n) DETACH DELETE n") - print("✅ Graph cleared\n") - except Exception as e: - print(f"⚠️ Could not clear graph: {e}\n") + # Clear the configured graph before restoring from Qdrant. + clear_graph(client) # Fetch all memories from Qdrant memories = get_all_memories() diff --git a/scripts/restore_from_backup.py b/scripts/restore_from_backup.py index 03992bbf..77b86fd2 100755 --- a/scripts/restore_from_backup.py +++ b/scripts/restore_from_backup.py @@ -110,6 +110,7 @@ def _float_env(name: str, default: float) -> float: QDRANT_RESTORE_DEFAULT_SEGMENT_NUMBER = _optional_int_env("QDRANT_RESTORE_DEFAULT_SEGMENT_NUMBER") QDRANT_RESTORE_MEMMAP_THRESHOLD = _optional_int_env("QDRANT_RESTORE_MEMMAP_THRESHOLD") QDRANT_RESTORE_HNSW_M = _optional_int_env("QDRANT_RESTORE_HNSW_M") +QDRANT_RESTORE_HNSW_ON_DISK = _optional_bool_env("QDRANT_RESTORE_HNSW_ON_DISK") QDRANT_RESTORE_VECTOR_ON_DISK = _optional_bool_env("QDRANT_RESTORE_VECTOR_ON_DISK") QDRANT_RESTORE_ON_DISK_PAYLOAD = _optional_bool_env("QDRANT_RESTORE_ON_DISK_PAYLOAD") QDRANT_RESTORE_READY_TIMEOUT_SECONDS = float( @@ -448,8 +449,13 @@ def _qdrant_collection_kwargs(vector_size: int) -> dict[str, Any]: optimizer_kwargs["memmap_threshold"] = QDRANT_RESTORE_MEMMAP_THRESHOLD if optimizer_kwargs: kwargs["optimizers_config"] = OptimizersConfigDiff(**optimizer_kwargs) + hnsw_kwargs: dict[str, Any] = {} if QDRANT_RESTORE_HNSW_M is not None: - kwargs["hnsw_config"] = HnswConfigDiff(m=QDRANT_RESTORE_HNSW_M) + hnsw_kwargs["m"] = QDRANT_RESTORE_HNSW_M + if QDRANT_RESTORE_HNSW_ON_DISK is not None: + hnsw_kwargs["on_disk"] = QDRANT_RESTORE_HNSW_ON_DISK + if hnsw_kwargs: + kwargs["hnsw_config"] = HnswConfigDiff(**hnsw_kwargs) if QDRANT_RESTORE_ON_DISK_PAYLOAD is not None: kwargs["on_disk_payload"] = QDRANT_RESTORE_ON_DISK_PAYLOAD return kwargs diff --git a/tests/benchmarks/results/compare_pr80_bm25_only_f10_judge_off.json b/tests/benchmarks/results/compare_pr80_bm25_only_f10_judge_off.json index 27ca4691..df517e5d 100644 --- a/tests/benchmarks/results/compare_pr80_bm25_only_f10_judge_off.json +++ b/tests/benchmarks/results/compare_pr80_bm25_only_f10_judge_off.json @@ -9,6 +9,6 @@ "Open Domain": -0.03508771929824561, "Complex Reasoning": 0.0 }, - "baseline_file": "/Users/jgarturo/Projects/OpenAI/automem/benchmarks/results/locomo-mini_baseline_20260310_233631.json", - "test_file": "/Users/jgarturo/Projects/OpenAI/automem/benchmarks/results/locomo-mini_pr80_bm25_only_f10_20260311_025443.json" + "baseline_file": "benchmarks/results/locomo-mini_baseline_20260310_233631.json", + "test_file": "benchmarks/results/locomo-mini_pr80_bm25_only_f10_20260311_025443.json" } diff --git a/tests/support/fake_graph.py b/tests/support/fake_graph.py index 9b912dc8..450da750 100644 --- a/tests/support/fake_graph.py +++ b/tests/support/fake_graph.py @@ -32,6 +32,21 @@ def _skip_limit(query: str) -> tuple[int, int | None]: return int(match.group(1)), int(match.group(2)) +def _id_range(query: str, alias: str) -> tuple[int, int] | None: + """Parse the `id(x) >= lo AND id(x) < hi` window the backup exporter pages with.""" + match = re.search( + rf"id\({alias}\)\s*>=\s*(\d+)\s+AND\s+id\({alias}\)\s*<\s*(\d+)", " ".join(query.split()) + ) + if not match: + return None + return int(match.group(1)), int(match.group(2)) + + +def _single_id(query: str, alias: str) -> int | None: + match = re.search(rf"id\({alias}\)\s*=\s*(\d+)", " ".join(query.split())) + return int(match.group(1)) if match else None + + def _memory_type_allowed(memory: Dict[str, Any], excluded_types: List[str]) -> bool: if not excluded_types: return True @@ -48,6 +63,10 @@ class FakeGraph: def __init__(self, *, seed_enrichment_fixture: bool = False) -> None: self.queries: List[tuple[str, Dict[str, Any]]] = [] + self.query_kwargs: List[Dict[str, Any]] = [] + + # Set to mimic FalkorDB's RESULTSET_SIZE cap (silent truncation). + self.resultset_cap: int | None = None # Generic memory storage + associations self.memories: Dict[str, Dict[str, Any]] = {} @@ -94,8 +113,46 @@ def __init__(self, *, seed_enrichment_fixture: bool = False) -> None: ["mem-c", "Automation habit noted"], ] + def _backup_node_rows(self) -> List[List[Any]]: + return [ + [index, ["Memory"], dict(memory)] + for index, (_memory_id, memory) in enumerate(sorted(self.memories.items())) + ] + + def _backup_relationship_rows(self) -> List[List[Any]]: + backup_ids = {memory_id: index for index, memory_id in enumerate(sorted(self.memories))} + rows: List[List[Any]] = [] + for rel in self.relationships: + source_id = str(rel.get("id1") or "") + target_id = str(rel.get("id2") or "") + if source_id not in backup_ids or target_id not in backup_ids: + continue + props = {key: value for key, value in rel.items() if key not in {"id1", "id2", "type"}} + rows.append( + [ + backup_ids[source_id], + str(rel.get("type") or "RELATES_TO"), + backup_ids[target_id], + props, + ] + ) + return sorted(rows, key=lambda row: row[0]) + + def _apply_result_limits(self, rows: List[List[Any]], query: str) -> List[List[Any]]: + """Apply SKIP/LIMIT, then RESULTSET_SIZE truncation the way the server does: silently.""" + skip, limit = _skip_limit(query) + rows = rows[skip : None if limit is None else skip + limit] + if self.resultset_cap is not None: + rows = rows[: self.resultset_cap] + return rows + + def execute_command(self, *args: Any) -> Any: + if args[:3] == ("GRAPH.CONFIG", "GET", "RESULTSET_SIZE"): + return ["RESULTSET_SIZE", self.resultset_cap or 0] + raise NotImplementedError(f"FakeGraph.execute_command{args}") + def query(self, query: str, params: Dict[str, Any] | None = None, **kwargs: Any) -> FakeResult: - del kwargs + self.query_kwargs.append(dict(kwargs)) params = params or {} self.queries.append((query, params)) @@ -196,41 +253,36 @@ def query(self, query: str, params: Dict[str, Any] | None = None, **kwargs: Any) return FakeResult([]) # Full graph backup export + if "RETURN max(id(n))" in query: + node_rows = self._backup_node_rows() + return FakeResult([[node_rows[-1][0] if node_rows else None]]) + + if "RETURN count(n)" in query: + return FakeResult([[len(self._backup_node_rows())]]) + + if "RETURN count(r)" in query: + return FakeResult([[len(self._backup_relationship_rows())]]) + if "MATCH (n)" in query and "id(n) as id" in query and "properties(n) as props" in query: - memory_items = sorted(self.memories.items(), key=lambda item: item[0]) - rows = [ - [index, ["Memory"], dict(memory)] - for index, (_memory_id, memory) in enumerate(memory_items) - ] - skip, limit = _skip_limit(query) - return FakeResult(rows[skip : None if limit is None else skip + limit]) + rows = self._backup_node_rows() + id_range = _id_range(query, "n") + if id_range: + rows = [row for row in rows if id_range[0] <= row[0] < id_range[1]] + return FakeResult(self._apply_result_limits(rows, query)) if ( "MATCH (a)-[r]->(b)" in query and "id(a) as source_id" in query and "properties(r) as props" in query ): - memory_ids = [memory_id for memory_id, _memory in sorted(self.memories.items())] - backup_ids = {memory_id: index for index, memory_id in enumerate(memory_ids)} - rows = [] - for rel in self.relationships: - source_id = str(rel.get("id1") or "") - target_id = str(rel.get("id2") or "") - if source_id not in backup_ids or target_id not in backup_ids: - continue - props = { - key: value for key, value in rel.items() if key not in {"id1", "id2", "type"} - } - rows.append( - [ - backup_ids[source_id], - str(rel.get("type") or "RELATES_TO"), - backup_ids[target_id], - props, - ] - ) - skip, limit = _skip_limit(query) - return FakeResult(rows[skip : None if limit is None else skip + limit]) + rows = self._backup_relationship_rows() + id_range = _id_range(query, "a") + single_id = _single_id(query, "a") + if id_range: + rows = [row for row in rows if id_range[0] <= row[0] < id_range[1]] + elif single_id is not None: + rows = [row for row in rows if row[0] == single_id] + return FakeResult(self._apply_result_limits(rows, query)) # Batch memory create/upsert if "UNWIND $memories AS m" in query and "MERGE (node:Memory {id: m.id})" in query: diff --git a/tests/test_backup_endpoint.py b/tests/test_backup_endpoint.py index 55733b2b..dc888ec7 100644 --- a/tests/test_backup_endpoint.py +++ b/tests/test_backup_endpoint.py @@ -489,6 +489,7 @@ def test_lab_clone_uses_paced_qdrant_restore_defaults() -> None: ) assert 'QDRANT_RESTORE_MEMMAP_THRESHOLD="${QDRANT_RESTORE_MEMMAP_THRESHOLD:-0}"' in script assert 'QDRANT_RESTORE_HNSW_M="${QDRANT_RESTORE_HNSW_M:-0}"' in script + assert 'QDRANT_RESTORE_HNSW_ON_DISK="${QDRANT_RESTORE_HNSW_ON_DISK:-true}"' in script assert 'QDRANT_RESTORE_VECTOR_ON_DISK="${QDRANT_RESTORE_VECTOR_ON_DISK:-true}"' in script assert 'QDRANT_RESTORE_ON_DISK_PAYLOAD="${QDRANT_RESTORE_ON_DISK_PAYLOAD:-false}"' in script assert 'QDRANT_PREFER_GRPC="${QDRANT_PREFER_GRPC:-true}"' in script @@ -662,6 +663,7 @@ def upsert( monkeypatch.setattr(restore_module, "QDRANT_RESTORE_DEFAULT_SEGMENT_NUMBER", 1) monkeypatch.setattr(restore_module, "QDRANT_RESTORE_MEMMAP_THRESHOLD", 0) monkeypatch.setattr(restore_module, "QDRANT_RESTORE_HNSW_M", 0) + monkeypatch.setattr(restore_module, "QDRANT_RESTORE_HNSW_ON_DISK", True) monkeypatch.setattr(restore_module, "QDRANT_RESTORE_VECTOR_ON_DISK", True) monkeypatch.setattr(restore_module, "QDRANT_RESTORE_ON_DISK_PAYLOAD", False) @@ -675,4 +677,5 @@ def upsert( assert optimizers.default_segment_number == 1 assert optimizers.memmap_threshold == 0 assert RecordingQdrantClient.create_kwargs["hnsw_config"].m == 0 + assert RecordingQdrantClient.create_kwargs["hnsw_config"].on_disk is True assert RecordingQdrantClient.create_kwargs["vectors_config"].on_disk is True diff --git a/tests/test_enrichment_circuit.py b/tests/test_enrichment_circuit.py new file mode 100644 index 00000000..44ed5d1b --- /dev/null +++ b/tests/test_enrichment_circuit.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +import logging +from types import SimpleNamespace + +from automem.classification.memory_classifier import MemoryClassifier +from automem.service_state import EnrichmentCircuit +from automem.utils.text import summarize_content + + +def test_quota_failure_opens_circuit_and_skips_requests(): + circuit = EnrichmentCircuit(cooldown_seconds=60, clock=lambda: 100.0) + + allowed, _ = circuit.begin_request() + assert allowed is True + assert circuit.record_failure("429 insufficient_quota") is True + allowed, _ = circuit.begin_request() + assert allowed is False + assert circuit.to_dict()["circuit_open_skips"] == 1 + + +def test_non_quota_failure_does_not_open_circuit(): + circuit = EnrichmentCircuit(cooldown_seconds=60, clock=lambda: 100.0) + + assert circuit.record_failure("connection reset") is False + allowed, _ = circuit.begin_request() + assert allowed is True + + +def test_successful_probe_closes_circuit_and_records_recovery(): + now = [100.0] + circuit = EnrichmentCircuit(cooldown_seconds=60, clock=lambda: now[0]) + + circuit.record_failure("insufficient_quota") + now[0] += 60 + allowed, was_probe = circuit.begin_request() + assert allowed is True + assert was_probe is True + circuit.record_success(was_probe) + + allowed, _ = circuit.begin_request() + assert allowed is True + assert circuit.to_dict()["recoveries"] == 1 + + +def test_failed_probe_does_not_permanently_block_future_requests(): + now = [100.0] + circuit = EnrichmentCircuit(cooldown_seconds=60, clock=lambda: now[0]) + + circuit.record_failure("insufficient_quota") + now[0] += 60 + allowed, was_probe = circuit.begin_request() + assert allowed is True + assert was_probe is True + assert circuit.record_failure("connection reset", was_probe) is False + allowed, _ = circuit.begin_request() + assert allowed is True + + +def test_inflight_success_does_not_close_quota_circuit(): + now = [100.0] + circuit = EnrichmentCircuit(cooldown_seconds=60, clock=lambda: now[0]) + + first_allowed, _ = circuit.begin_request() + second_allowed, second_was_probe = circuit.begin_request() + assert first_allowed is True + assert second_allowed is True + assert circuit.record_failure("insufficient_quota") is True + + circuit.record_success(second_was_probe) + + allowed, _ = circuit.begin_request() + assert allowed is False + + +def test_classifier_makes_no_second_request_while_circuit_is_open(): + calls = [] + + def create(*args, **kwargs): + calls.append(1) + raise RuntimeError("insufficient_quota") + + client = SimpleNamespace(chat=SimpleNamespace(completions=SimpleNamespace(create=create))) + circuit = EnrichmentCircuit(cooldown_seconds=60, clock=lambda: 100.0) + classifier = MemoryClassifier( + normalize_memory_type=lambda raw: (raw, False), + ensure_openai_client=lambda: None, + get_openai_client=lambda: client, + classification_model="gpt-4o-mini", + logger=logging.getLogger(__name__), + circuit=circuit, + ) + + classifier.classify("qwxz flibber jabberwock snorkelblatt") + classifier.classify("qwxz flibber jabberwock snorkelblatt") + + assert len(calls) == 1 + + +def test_summarizer_makes_no_second_request_while_circuit_is_open(): + calls = [] + + def create(*args, **kwargs): + calls.append(1) + raise RuntimeError("insufficient_quota") + + client = SimpleNamespace(chat=SimpleNamespace(completions=SimpleNamespace(create=create))) + circuit = EnrichmentCircuit(cooldown_seconds=60, clock=lambda: 100.0) + content = "x" * 600 + + summarize_content(content, client, "gpt-4o-mini", 300, circuit) + summarize_content(content, client, "gpt-4o-mini", 300, circuit) + + assert len(calls) == 1 diff --git a/tests/test_falkordb_backup_export.py b/tests/test_falkordb_backup_export.py new file mode 100644 index 00000000..11b27ec4 --- /dev/null +++ b/tests/test_falkordb_backup_export.py @@ -0,0 +1,157 @@ +"""Regression tests for the FalkorDB backup export pagination. + +Covers the two ways the previous ``SKIP LIMIT `` paging +lost data: batches that grew slower with depth until they exceeded the server's +``TIMEOUT``, and FalkorDB's ``RESULTSET_SIZE`` cap silently truncating a result +set that the loop then read as "the last page". +""" + +from __future__ import annotations + +import gzip +import io +import json +import re +from typing import Any + +import pytest + +from automem.backup import BackupError, export_falkordb_artifact +from tests.support.fake_graph import FakeGraph + + +def build_graph(memory_count: int, degrees: dict[int, int] | None = None) -> FakeGraph: + """Seed a graph with `memory_count` memories and per-node out-degrees.""" + graph = FakeGraph() + for index in range(memory_count): + memory_id = f"mem-{index:05d}" + graph.memories[memory_id] = {"id": memory_id, "content": f"memory {index}"} + graph.nodes.add(memory_id) + + for source, degree in (degrees or {}).items(): + for step in range(degree): + graph.relationships.append( + { + "id1": f"mem-{source:05d}", + "id2": f"mem-{(source + step + 1) % memory_count:05d}", + "type": "RELATES_TO", + "strength": 0.5, + } + ) + return graph + + +def export(graph: FakeGraph, **kwargs: Any) -> dict[str, Any]: + artifact = export_falkordb_artifact( + graph=graph, graph_name="memories", timestamp="20260101_000000", **kwargs + ) + with gzip.open(io.BytesIO(artifact.data), "rt", encoding="utf-8") as handle: + return json.load(handle) + + +def assert_exported_everything(graph: FakeGraph, payload: dict[str, Any]) -> None: + assert payload["stats"]["node_count"] == len(graph.memories) + assert payload["stats"]["relationship_count"] == len(graph.relationships) + assert len(payload["nodes"]) == len(graph.memories) + + exported_ids = [node["id"] for node in payload["nodes"]] + assert len(set(exported_ids)) == len(exported_ids), "a node was exported twice" + + exported_edges = sorted( + (rel["source_id"], rel["type"], rel["target_id"]) for rel in payload["relationships"] + ) + assert len(exported_edges) == len(graph.relationships) + node_ids = set(exported_ids) + assert all(edge[0] in node_ids and edge[2] in node_ids for edge in exported_edges) + + +def test_export_round_trips_a_small_graph() -> None: + graph = build_graph(50, {index: 2 for index in range(50)}) + assert_exported_everything(graph, export(graph)) + + +def test_export_is_complete_when_resultset_cap_is_below_batch_size() -> None: + """RESULTSET_SIZE under batch_size used to end the export after one batch.""" + graph = build_graph(2500, {index: 3 for index in range(2500)}) + graph.resultset_cap = 1000 + + payload = export(graph) + + assert_exported_everything(graph, payload) + assert payload["stats"] == {"node_count": 2500, "relationship_count": 7500} + + +def test_export_subdivides_a_dense_relationship_range() -> None: + """A node id window holding more relationships than the cap must be split.""" + degrees = {index: 1 for index in range(300)} + degrees.update({index: 60 for index in range(40, 45)}) + graph = build_graph(300, degrees) + graph.resultset_cap = 100 + + assert_exported_everything(graph, export(graph)) + + +def test_export_pages_a_single_high_degree_node() -> None: + """One node with more out-edges than the cap falls back to bounded SKIP.""" + degrees = {index: 2 for index in range(200)} + degrees[7] = 250 + graph = build_graph(200, degrees) + graph.resultset_cap = 100 + + payload = export(graph) + + assert_exported_everything(graph, payload) + hub_edges = [rel for rel in payload["relationships"] if rel["source_id"] == 7] + assert len(hub_edges) == 250 + + +def test_export_handles_out_degree_exactly_at_the_cap() -> None: + """Rows == cap is ambiguous between complete and truncated; treat it as truncated.""" + degrees = {index: 1 for index in range(50)} + degrees[3] = 100 + graph = build_graph(50, degrees) + graph.resultset_cap = 100 + + assert_exported_everything(graph, export(graph)) + + +def test_export_rejects_a_silently_truncated_result() -> None: + """A backup that lost rows must fail rather than look complete.""" + + class LossyGraph(FakeGraph): + def _apply_result_limits(self, rows: list[Any], query: str) -> list[Any]: + kept = super()._apply_result_limits(rows, query) + return [row for index, row in enumerate(kept) if index % 5] + + graph = LossyGraph() + for index in range(300): + memory_id = f"mem-{index:05d}" + graph.memories[memory_id] = {"id": memory_id} + + with pytest.raises(BackupError, match="short by"): + export(graph) + + +def test_export_handles_an_empty_graph() -> None: + payload = export(FakeGraph()) + + assert payload["nodes"] == [] + assert payload["relationships"] == [] + assert payload["stats"] == {"node_count": 0, "relationship_count": 0} + + +def test_export_avoids_deep_skip_and_always_sets_a_timeout() -> None: + """Deep SKIP is what exceeded the server TIMEOUT on a large graph.""" + graph = build_graph(400, {index: 3 for index in range(400)}) + + export(graph) + + assert graph.query_kwargs, "no queries were issued" + assert all(kwargs.get("timeout") for kwargs in graph.query_kwargs) + + offsets = [ + int(match.group(1)) + for query, _params in graph.queries + if (match := re.search(r"\bSKIP\s+(\d+)", query)) + ] + assert all(offset == 0 for offset in offsets), f"deep SKIP still in use: {offsets}" diff --git a/tests/test_recover_from_qdrant.py b/tests/test_recover_from_qdrant.py new file mode 100644 index 00000000..2df9ab13 --- /dev/null +++ b/tests/test_recover_from_qdrant.py @@ -0,0 +1,52 @@ +"""Regression coverage for the Qdrant-to-FalkorDB recovery helper.""" + +import importlib.util +from pathlib import Path + +SCRIPT_PATH = Path(__file__).resolve().parents[1] / "scripts" / "recover_from_qdrant.py" + + +def load_recovery_module(): + spec = importlib.util.spec_from_file_location("recover_from_qdrant", SCRIPT_PATH) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class FakeGraph: + def __init__(self): + self.queries = [] + + def query(self, statement, params=None): + self.queries.append((statement, params)) + + +class FakeFalkorDB: + def __init__(self): + self.graph_names = [] + self.graph = FakeGraph() + + def select_graph(self, name): + self.graph_names.append(name) + return self.graph + + +def test_recovery_uses_configured_graph_for_clear_and_restore(monkeypatch): + monkeypatch.setenv("FALKORDB_GRAPH", "custom_memories") + recovery = load_recovery_module() + client = FakeFalkorDB() + + assert recovery.clear_graph(client) + assert recovery.restore_memory_to_graph_only( + { + "id": "memory-1", + "payload": { + "content": "Recovered memory", + "tags": [], + }, + }, + client, + ) + + assert client.graph_names == ["custom_memories", "custom_memories"] diff --git a/tests/test_regression_drift_scout_contract.py b/tests/test_regression_drift_scout_contract.py deleted file mode 100644 index 016d83f9..00000000 --- a/tests/test_regression_drift_scout_contract.py +++ /dev/null @@ -1,59 +0,0 @@ -import re -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -SKILL_PATH = ROOT / ".agents" / "skills" / "automem-regression-drift-scout" / "SKILL.md" - - -def read_skill() -> str: - return SKILL_PATH.read_text(encoding="utf-8") - - -def test_automem_regression_drift_scout_skill_exists_with_required_contract(): - source = read_skill() - - assert "name: automem-regression-drift-scout" in source - assert "description:" in source - assert "agents: [codex]" in source - assert "capabilities:" in source - assert "filesystem: readonly" in source - - -def test_automem_regression_drift_scout_is_read_only_by_default(): - source = read_skill() - - assert "read-only" in source - assert "Do not deploy" in source - assert "Do not edit files" in source - assert "Do not create issues or pull requests" in source - assert not re.search(r"`make deploy`|\bmake deploy(?:\s|$)", source) - - -def test_automem_regression_drift_scout_uses_project_truth_sources(): - source = read_skill() - - assert "benchmarks/EXPERIMENT_LOG.md" in source - assert "make bench-health" in source - assert "make deploy-check" in source - assert "gh run list" in source - assert "railway status" in source - - -def test_automem_regression_drift_scout_output_contract_is_tiered(): - source = read_skill() - - assert "healthy" in source - assert "needs_issue" in source - assert "needs_pr_plan" in source - assert "evidence bundle" in source - assert "confidence score" in source - assert "what to measure next" in source - - -def test_automem_regression_drift_scout_handles_partial_command_results(): - source = read_skill() - - assert "Do not trust the final `RECALL HEALTH: HEALTHY` banner by itself" in source - assert "Connection refused" in source - assert "railway CLI not found" in source - assert "skipped surface" in source diff --git a/tests/test_stream_events.py b/tests/test_stream_events.py new file mode 100644 index 00000000..49c7fa4c --- /dev/null +++ b/tests/test_stream_events.py @@ -0,0 +1,320 @@ +import json +from queue import Empty, Queue + +import pytest + +import app +from automem.api import stream as stream_mod +from automem.api.stream import event_count, is_single_memory_store +from tests.support.fake_graph import FakeGraph + + +@pytest.fixture(autouse=True) +def reset_state(monkeypatch): + state = app.ServiceState() + graph = FakeGraph() + state.memory_graph = graph + monkeypatch.setattr(app, "state", state) + monkeypatch.setattr(app, "init_falkordb", lambda: None) + monkeypatch.setattr(app, "init_qdrant", lambda: None) + monkeypatch.setattr(app, "API_TOKEN", "test-token") + monkeypatch.setattr(app, "ADMIN_TOKEN", "test-admin-token") + yield graph + + +@pytest.fixture +def client(): + return app.app.test_client() + + +@pytest.fixture +def auth_headers(): + return {"Authorization": "Bearer test-token"} + + +@pytest.fixture +def sse_queue(): + queue: Queue = Queue(maxsize=100) + with stream_mod._subscribers_lock: + stream_mod._subscribers.append(queue) + try: + yield queue + finally: + with stream_mod._subscribers_lock: + if queue in stream_mod._subscribers: + stream_mod._subscribers.remove(queue) + + +def _drain(queue: Queue) -> list[dict]: + events: list[dict] = [] + while True: + try: + raw = queue.get_nowait() + except Empty: + break + assert raw.startswith("data: ") + events.append(json.loads(raw[len("data: ") :].strip())) + return events + + +def _store(client, auth_headers, content: str, **fields) -> str: + payload = {"content": content, **fields} + response = client.post( + "/memory", + data=json.dumps(payload), + content_type="application/json", + headers=auth_headers, + ) + assert response.status_code == 201, response.get_json() + return response.get_json()["memory_id"] + + +def test_emit_event_delivers_json_payload_to_subscribers(sse_queue): + stream_mod.emit_event("memory.store", {"id": "abc"}, lambda: "2026-08-23T00:00:00Z") + events = _drain(sse_queue) + assert events == [ + { + "type": "memory.store", + "timestamp": "2026-08-23T00:00:00Z", + "data": {"id": "abc"}, + } + ] + + +def test_store_emits_memory_store_event(client, auth_headers, sse_queue): + memory_id = _store( + client, + auth_headers, + "SSE store coverage probe", + tags=["sse", "automem"], + importance=0.8, + type="Insight", + ) + events = [event for event in _drain(sse_queue) if event["type"] == "memory.store"] + assert len(events) == 1 + data = events[0]["data"] + assert data["id"] == memory_id + assert data["content_preview"] == "SSE store coverage probe" + assert data["type"] == "Insight" + assert data["importance"] == 0.8 + assert data["tags"][:2] == ["sse", "automem"] + assert data["size_bytes"] == len("SSE store coverage probe".encode("utf-8")) + assert "elapsed_ms" in data + + +def test_store_event_size_bytes_uses_utf8_length(client, auth_headers, sse_queue): + memory_id = _store(client, auth_headers, "café") + events = [event for event in _drain(sse_queue) if event["type"] == "memory.store"] + assert events[0]["data"]["id"] == memory_id + assert events[0]["data"]["size_bytes"] == len("café".encode("utf-8")) + assert events[0]["data"]["size_bytes"] > len("café") + + +def test_failed_store_does_not_emit(client, auth_headers, sse_queue): + response = client.post( + "/memory", + data=json.dumps({}), + content_type="application/json", + headers=auth_headers, + ) + assert response.status_code == 400 + assert _drain(sse_queue) == [] + + +def test_recall_emits_memory_recall_event(client, auth_headers, sse_queue): + response = client.get("/recall?query=hello+world&limit=7&tags=sse", headers=auth_headers) + assert response.status_code == 200 + events = [event for event in _drain(sse_queue) if event["type"] == "memory.recall"] + assert len(events) == 1 + data = events[0]["data"] + assert data["query"] == "hello world" + assert data["limit"] == 7 + assert data["result_count"] == 0 + assert data["tags"] == ["sse"] + assert "elapsed_ms" in data + + +def test_recall_event_includes_multi_query_text(client, auth_headers, sse_queue): + response = client.get("/recall?queries=alpha&queries=beta", headers=auth_headers) + assert response.status_code == 200 + events = [event for event in _drain(sse_queue) if event["type"] == "memory.recall"] + assert len(events) == 1 + assert events[0]["data"]["query"] == "alpha | beta" + + +def test_associate_emits_memory_associate_event(client, auth_headers, sse_queue): + memory1_id = _store(client, auth_headers, "Source memory for SSE associate") + memory2_id = _store(client, auth_headers, "Target memory for SSE associate") + _drain(sse_queue) + + response = client.post( + "/associate", + data=json.dumps( + { + "memory1_id": memory1_id, + "memory2_id": memory2_id, + "type": "RELATES_TO", + "strength": 0.9, + } + ), + content_type="application/json", + headers=auth_headers, + ) + assert response.status_code == 201 + events = [event for event in _drain(sse_queue) if event["type"] == "memory.associate"] + assert len(events) == 1 + data = events[0]["data"] + assert data["memory1_id"] == memory1_id + assert data["memory2_id"] == memory2_id + assert data["relation_type"] == "RELATES_TO" + assert data["strength"] == 0.9 + assert data["count"] == 1 + + +def test_failed_associate_does_not_emit(client, auth_headers, sse_queue): + same_id = "a0000000-0000-0000-0000-000000000001" + response = client.post( + "/associate", + data=json.dumps({"memory1_id": same_id, "memory2_id": same_id}), + content_type="application/json", + headers=auth_headers, + ) + assert response.status_code == 400 + assert _drain(sse_queue) == [] + + +def test_batch_associate_emits_one_summary_event(client, auth_headers, sse_queue): + first_id = _store(client, auth_headers, "Batch associate source") + second_id = _store(client, auth_headers, "Batch associate target") + third_id = _store(client, auth_headers, "Batch associate extra") + _drain(sse_queue) + + response = client.post( + "/associate", + data=json.dumps( + { + "associations": [ + { + "memory1_id": first_id, + "memory2_id": second_id, + "type": "RELATES_TO", + "strength": 0.8, + }, + { + "memory1_id": second_id, + "memory2_id": third_id, + "type": "LEADS_TO", + "strength": 0.7, + }, + ] + } + ), + content_type="application/json", + headers=auth_headers, + ) + assert response.status_code == 201 + events = [event for event in _drain(sse_queue) if event["type"] == "memory.associate"] + assert len(events) == 1 + data = events[0]["data"] + assert data["count"] == 2 + assert data["failed_count"] == 0 + assert sorted(data["relation_types"]) == ["LEADS_TO", "RELATES_TO"] + + +def test_update_emits_memory_update_event(client, auth_headers, sse_queue): + memory_id = _store(client, auth_headers, "Original content", tags=["sse"]) + _drain(sse_queue) + + response = client.patch( + f"/memory/{memory_id}", + data=json.dumps({"content": "Updated content", "importance": 0.95}), + content_type="application/json", + headers=auth_headers, + ) + assert response.status_code == 200 + events = [event for event in _drain(sse_queue) if event["type"] == "memory.update"] + assert len(events) == 1 + data = events[0]["data"] + assert data["id"] == memory_id + assert data["content_preview"] == "Updated content" + assert "content" in data["fields"] + assert "importance" in data["fields"] + + +def test_update_event_includes_timestamp_fields(client, auth_headers, sse_queue): + memory_id = _store(client, auth_headers, "Timestamp patch probe") + _drain(sse_queue) + response = client.patch( + f"/memory/{memory_id}", + data=json.dumps( + { + "updated_at": "2026-08-22T12:00:00Z", + "last_accessed": "2026-08-22T12:00:01Z", + } + ), + content_type="application/json", + headers=auth_headers, + ) + assert response.status_code == 200 + events = [event for event in _drain(sse_queue) if event["type"] == "memory.update"] + assert events[0]["data"]["fields"] == ["last_accessed", "updated_at"] + + +def test_delete_emits_memory_delete_event(client, auth_headers, sse_queue): + memory_id = _store(client, auth_headers, "Delete me") + _drain(sse_queue) + + response = client.delete(f"/memory/{memory_id}", headers=auth_headers) + assert response.status_code == 200 + events = [event for event in _drain(sse_queue) if event["type"] == "memory.delete"] + assert len(events) == 1 + assert events[0]["data"]["id"] == memory_id + assert events[0]["data"]["count"] == 1 + + +def test_batch_store_emits_one_store_event(client, auth_headers, sse_queue): + response = client.post( + "/memory/batch", + json={ + "memories": [ + {"content": "Batch SSE one", "tags": ["sse"]}, + {"content": "Batch SSE two", "tags": ["sse"]}, + ] + }, + headers=auth_headers, + ) + assert response.status_code == 201, response.get_json() + events = [event for event in _drain(sse_queue) if event["type"] == "memory.store"] + assert len(events) == 1 + data = events[0]["data"] + assert data["count"] == 2 + assert data["content_preview"].startswith("2 memories stored") + + +def test_delete_by_tag_emits_memory_delete_event(client, auth_headers, sse_queue): + _store(client, auth_headers, "Tagged for bulk delete", tags=["sse-bulk"]) + _drain(sse_queue) + + response = client.delete("/memory/by-tag?tags=sse-bulk", headers=auth_headers) + assert response.status_code == 200 + body = response.get_json() + assert body["deleted_count"] == 1 + events = [event for event in _drain(sse_queue) if event["type"] == "memory.delete"] + assert len(events) == 1 + assert events[0]["data"]["count"] == 1 + assert events[0]["data"]["tags"] == ["sse-bulk"] + + +def test_event_count_preserves_explicit_zero(): + assert event_count({}) == 1 + assert event_count({"count": 2}) == 2 + assert event_count({"count": 0}) == 0 + assert is_single_memory_store({"id": "abc", "count": 1}) is True + assert is_single_memory_store({"count": 2, "ids": ["a", "b"]}) is False + assert is_single_memory_store({"content_preview": "2 memories stored"}) is False + + +def test_stream_status_reports_subscriber_count(client, auth_headers, sse_queue): + response = client.get("/stream/status", headers=auth_headers) + assert response.status_code == 200 + assert response.get_json()["subscribers"] >= 1 diff --git a/tests/test_vector_size_safety.py b/tests/test_vector_size_safety.py index 02bd80c0..65cde0bb 100644 --- a/tests/test_vector_size_safety.py +++ b/tests/test_vector_size_safety.py @@ -251,6 +251,83 @@ def bad_config(): class TestEnsureQdrantCollectionPayloadIndexes: + def test_new_collection_uses_on_disk_storage(self): + from automem.stores.runtime_clients import ensure_qdrant_collection + + class RecordingVectorParams: + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + + class RecordingHnswConfigDiff: + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + + qdrant = MagicMock() + qdrant.get_collections.return_value = SimpleNamespace(collections=[]) + + state = SimpleNamespace(qdrant=qdrant, effective_vector_size=None) + + ensure_qdrant_collection( + state=state, + logger=MagicMock(), + collection_name="memories", + vector_size_config=1024, + get_effective_vector_size_fn=lambda _client: (768, "configured"), + vector_params_cls=RecordingVectorParams, + hnsw_config_diff_cls=RecordingHnswConfigDiff, + distance_enum=SimpleNamespace(COSINE="Cosine"), + payload_schema_type_enum=SimpleNamespace(KEYWORD="keyword"), + ) + + qdrant.create_collection.assert_called_once() + create_kwargs = qdrant.create_collection.call_args.kwargs + assert create_kwargs["collection_name"] == "memories" + assert create_kwargs["vectors_config"].size == 768 + assert create_kwargs["vectors_config"].distance == "Cosine" + assert create_kwargs["vectors_config"].on_disk is True + assert create_kwargs["hnsw_config"].on_disk is True + assert create_kwargs["on_disk_payload"] is True + assert qdrant.create_payload_index.call_count == 3 + + def _run_ensure(self, payload_schema_type_enum): + from automem.stores.runtime_clients import ensure_qdrant_collection + + qdrant = MagicMock() + qdrant.get_collections.return_value = SimpleNamespace( + collections=[SimpleNamespace(name="memories")] + ) + state = SimpleNamespace(qdrant=qdrant, effective_vector_size=None) + + ensure_qdrant_collection( + state=state, + logger=MagicMock(), + collection_name="memories", + vector_size_config=1024, + get_effective_vector_size_fn=lambda _client: (1024, "collection"), + vector_params_cls=MagicMock(), + hnsw_config_diff_cls=MagicMock(), + distance_enum=SimpleNamespace(COSINE="Cosine"), + payload_schema_type_enum=payload_schema_type_enum, + ) + return qdrant + + def test_indexes_every_field_recall_filters_on(self): + """`type` backs RECALL_EXCLUDED_TYPES; without an index Qdrant 400s every search.""" + qdrant = self._run_ensure(SimpleNamespace(KEYWORD="keyword")) + + indexed = {call.kwargs["field_name"] for call in qdrant.create_payload_index.call_args_list} + assert indexed == {"tags", "tag_prefixes", "type"} + + def test_indexes_every_field_without_payload_schema_enum(self): + qdrant = self._run_ensure(None) + + indexed = {call.kwargs["field_name"] for call in qdrant.create_payload_index.call_args_list} + assert indexed == {"tags", "tag_prefixes", "type"} + assert all( + call.kwargs["field_schema"] == "keyword" + for call in qdrant.create_payload_index.call_args_list + ) + @patch.dict("os.environ", {"QDRANT_ENSURE_PAYLOAD_INDEXES": "false"}, clear=False) def test_can_skip_payload_indexes_for_restore_tuned_lab_collection(self): from automem.stores.runtime_clients import ensure_qdrant_collection @@ -269,11 +346,13 @@ def test_can_skip_payload_indexes_for_restore_tuned_lab_collection(self): vector_size_config=1024, get_effective_vector_size_fn=lambda _client: (1024, "collection"), vector_params_cls=MagicMock(), + hnsw_config_diff_cls=MagicMock(), distance_enum=SimpleNamespace(COSINE="Cosine"), payload_schema_type_enum=SimpleNamespace(KEYWORD="keyword"), ) assert state.qdrant is qdrant + qdrant.create_collection.assert_not_called() qdrant.create_payload_index.assert_not_called() logger.info.assert_any_call( "Skipping Qdrant payload indexes for collection '%s'", "memories"