diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..cb35c22 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,47 @@ +name: Release + +on: + push: + tags: + - "v*" + +permissions: {} + +jobs: + build: + name: Build distribution + runs-on: ubuntu-24.04 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.11" + - name: Install uv + run: python -m pip install "uv==0.13.1" + - name: Build package distributions + run: | + uv build + uvx twine check dist/* + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: dist + path: dist/ + + publish: + name: Publish to PyPI + needs: build + runs-on: ubuntu-24.04 + environment: pypi + permissions: + id-token: write + steps: + - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7 + with: + name: dist + path: dist/ + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 diff --git a/CHANGELOG.md b/CHANGELOG.md index bff0e14..a41753e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,18 @@ are historical; consult the [current guides](docs/README.md) for present behavio The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Changed + +- Set the runtime vector RRF weight to 8.0 from the frozen Tier M dev benchmark + (passage MRR 0.3953 → 0.5663 with reranking off); other lane weights remain 1.0. + The full weight/reranker matrix and model environment are recorded in the + [retrieval guide](docs/retrieval-and-graph.md#round-14-dev-benchmark). +- New document revisions use lossless Turkish article boundaries in automatic + mode. Optional BGE reranking remains off by default, and query embedding calls + run before acquiring the shared search connection. + ## [0.9.0] - 2026-10-28 ### Added diff --git a/README.md b/README.md index d2a934c..6d0e4cb 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Package version: **0.9.0**. V4 is the current API. Production qualification rema ## Installation ```bash -pip install "mesa-memory[adapters]" +pip install mesa-memory ``` Supported on Linux and macOS natively. On Windows, run via Docker or WSL. diff --git a/docs/benchmarks/round14_dev.json b/docs/benchmarks/round14_dev.json new file mode 100644 index 0000000..231c760 --- /dev/null +++ b/docs/benchmarks/round14_dev.json @@ -0,0 +1,235 @@ +{ + "split": "dev", + "measured_source_commit": "1e3500d7f9802b977b5729832f45ab903ccd9f08", + "environment": { + "packages": { + "torch": "2.11.0+cu130", + "sentence-transformers": "5.7.0", + "transformers": "5.21.0", + "numpy": "2.5.4", + "lancedb": "0.41.0", + "kuzu": "0.11.3" + }, + "model_revisions": { + "BAAI/bge-m3": "5617a9f61b028005a4858fdac845db406aefb181", + "BAAI/bge-reranker-v2-m3": "953dc6f6f85a1b2dbfca4c34a2796e7dde08d41e" + }, + "source_sha256": { + "mesa_storage/dao.py": "15c138ed26e251a08687ba62e29d9d7541808ab52cecd1c42bed56259ae0240f", + "mesa_memory/config.py": "a175327bee61ad07496b46a5ce79f046fa5e23468302daecfd06f12a37f1fe4c", + "mesa_memory/reranking.py": "1f9d4d8ec01044e53cef0606ed8a2a34f0862774a5f97994dd6af7cff7a8c3ef", + "mesa_memory/api/server.py": "a459097575c00dab7e6ed25760ee9a9ddaf3b0c9e37fa8bd28661d1e2cbe2521" + }, + "cpu_affinity_count": 12, + "cpu_model": "Intel(R) Xeon(R) CPU @ 2.20GHz" + }, + "document_count": 141, + "chunk_count": 5041, + "question_count": 101, + "answerable_count": 85, + "limit": 50, + "rerank_depth": 50, + "embedding_model": "BAAI/bge-m3", + "rerank_model": "BAAI/bge-reranker-v2-m3", + "other_lane_weights": { + "bm25": 1, + "assertion": 1, + "graph": 1 + }, + "selected_default_vector_weight": 8, + "best_disabled_mrr_gain": 0.17100820871377637, + "selection_rule": "Best disabled MRR; keep 1 unless its unrounded MRR improves by at least 0.02; ties use the smaller weight.", + "runtime": { + "python": "3.13.16", + "torch": "2.11.0+cu130", + "gpu": "NVIDIA A100-SXM4-40GB", + "cpu_threads": 8, + "embedding": "BAAI/bge-m3", + "reranker": "BAAI/bge-reranker-v2-m3", + "max_length": 1024, + "depth": 50, + "limit": 50, + "graph": true, + "extraction": false, + "transport": "HTTP route via ASGI (no TCP)", + "split": "dev" + }, + "runs": [ + { + "vector_weight": 1, + "rerank_enabled": false, + "metrics": { + "count_total": 101, + "count_answerable": 85, + "count_no_answer": 16, + "passage_hit@1": 0.2941, + "passage_hit@5": 0.5176, + "passage_hit@10": 0.6, + "passage_mrr": 0.3953, + "document_hit@1": 0.4941, + "document_hit@5": 0.7059, + "latency_p50_sec": 1.2693, + "latency_p95_sec": 2.0035, + "suppressed_vector_count": 267, + "no_answer_mean_top_fused_score": 0.061, + "no_answer_mean_top_vector_sim": 0.0 + }, + "passage_mrr_unrounded": 0.39533925255237007, + "warmup_seconds": 5.303557358000035 + }, + { + "vector_weight": 1, + "rerank_enabled": true, + "metrics": { + "count_total": 101, + "count_answerable": 85, + "count_no_answer": 16, + "passage_hit@1": 0.6353, + "passage_hit@5": 0.7412, + "passage_hit@10": 0.7647, + "passage_mrr": 0.6816, + "document_hit@1": 0.6941, + "document_hit@5": 0.8118, + "latency_p50_sec": 1.9548, + "latency_p95_sec": 2.5736, + "suppressed_vector_count": 200, + "no_answer_mean_top_fused_score": 0.0558, + "no_answer_mean_top_vector_sim": 0.0 + }, + "passage_mrr_unrounded": 0.6815644636232872, + "warmup_seconds": 7.091411409000102 + }, + { + "vector_weight": 2, + "rerank_enabled": false, + "metrics": { + "count_total": 101, + "count_answerable": 85, + "count_no_answer": 16, + "passage_hit@1": 0.3647, + "passage_hit@5": 0.5765, + "passage_hit@10": 0.6824, + "passage_mrr": 0.4565, + "document_hit@1": 0.5529, + "document_hit@5": 0.7294, + "latency_p50_sec": 1.2708, + "latency_p95_sec": 1.964, + "suppressed_vector_count": 304, + "no_answer_mean_top_fused_score": 0.0763, + "no_answer_mean_top_vector_sim": 0.0 + }, + "passage_mrr_unrounded": 0.45648975693603044, + "warmup_seconds": 2.664795117000722 + }, + { + "vector_weight": 2, + "rerank_enabled": true, + "metrics": { + "count_total": 101, + "count_answerable": 85, + "count_no_answer": 16, + "passage_hit@1": 0.6588, + "passage_hit@5": 0.7529, + "passage_hit@10": 0.7882, + "passage_mrr": 0.7055, + "document_hit@1": 0.7176, + "document_hit@5": 0.8235, + "latency_p50_sec": 1.9569, + "latency_p95_sec": 2.6364, + "suppressed_vector_count": 290, + "no_answer_mean_top_fused_score": 0.0713, + "no_answer_mean_top_vector_sim": 0.0 + }, + "passage_mrr_unrounded": 0.7055448473095532, + "warmup_seconds": 3.410192898999412 + }, + { + "vector_weight": 4, + "rerank_enabled": false, + "metrics": { + "count_total": 101, + "count_answerable": 85, + "count_no_answer": 16, + "passage_hit@1": 0.4118, + "passage_hit@5": 0.6588, + "passage_hit@10": 0.7529, + "passage_mrr": 0.5152, + "document_hit@1": 0.5765, + "document_hit@5": 0.7765, + "latency_p50_sec": 1.2836, + "latency_p95_sec": 1.9469, + "suppressed_vector_count": 215, + "no_answer_mean_top_fused_score": 0.1086, + "no_answer_mean_top_vector_sim": 0.0 + }, + "passage_mrr_unrounded": 0.5152291602446402, + "warmup_seconds": 3.09878310699969 + }, + { + "vector_weight": 4, + "rerank_enabled": true, + "metrics": { + "count_total": 101, + "count_answerable": 85, + "count_no_answer": 16, + "passage_hit@1": 0.6588, + "passage_hit@5": 0.7765, + "passage_hit@10": 0.8, + "passage_mrr": 0.7069, + "document_hit@1": 0.7294, + "document_hit@5": 0.8471, + "latency_p50_sec": 1.9279, + "latency_p95_sec": 2.61, + "suppressed_vector_count": 291, + "no_answer_mean_top_fused_score": 0.1023, + "no_answer_mean_top_vector_sim": 0.0 + }, + "passage_mrr_unrounded": 0.7069098548510313, + "warmup_seconds": 3.857730206999804 + }, + { + "vector_weight": 8, + "rerank_enabled": false, + "metrics": { + "count_total": 101, + "count_answerable": 85, + "count_no_answer": 16, + "passage_hit@1": 0.4706, + "passage_hit@5": 0.6706, + "passage_hit@10": 0.7529, + "passage_mrr": 0.5663, + "document_hit@1": 0.6353, + "document_hit@5": 0.7647, + "latency_p50_sec": 1.2849, + "latency_p95_sec": 1.9115, + "suppressed_vector_count": 147, + "no_answer_mean_top_fused_score": 0.1732, + "no_answer_mean_top_vector_sim": 0.0 + }, + "passage_mrr_unrounded": 0.5663474612661464, + "warmup_seconds": 3.122546444999898 + }, + { + "vector_weight": 8, + "rerank_enabled": true, + "metrics": { + "count_total": 101, + "count_answerable": 85, + "count_no_answer": 16, + "passage_hit@1": 0.6471, + "passage_hit@5": 0.7647, + "passage_hit@10": 0.7882, + "passage_mrr": 0.6973, + "document_hit@1": 0.7176, + "document_hit@5": 0.8353, + "latency_p50_sec": 1.9161, + "latency_p95_sec": 2.536, + "suppressed_vector_count": 287, + "no_answer_mean_top_fused_score": 0.1642, + "no_answer_mean_top_vector_sim": 0.0 + }, + "passage_mrr_unrounded": 0.6973020117137764, + "warmup_seconds": 3.4035570070000176 + } + ] +} diff --git a/docs/configuration.md b/docs/configuration.md index e48f413..48702b2 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -30,6 +30,26 @@ and `MESA_LOG_FORMAT` to json. Secrets must be injected privately. ## Providers and capacity +`MESA_CHUNKING=auto` (default) splits new document revisions at Turkish article +headings when at least three occur at line starts. The preamble stays separate; +long articles use the existing paragraph/sentence splitting with a 512-token +limit. All slices preserve the exact source text. Other documents keep the same +plain chunks; `MESA_CHUNKING=plain` selects that plan for every document. Retrying +a pending upload preserves the plan matching its already admitted chunk texts, +even after restarting or changing this setting. Active revisions are unchanged. +If an admitted prefix fits two different plans, the retry returns 409 without +appending chunks, because the stored text cannot identify the original plan. + +`MESA_RERANK_ENABLED=false` keeps the existing fused ranking and does not import +ML packages. Enabling it requires `pip install "mesa-memory[ml]"`; startup checks +that the package is installed, and the model loads lazily on the first search. +`MESA_RERANK_MODEL` defaults to `BAAI/bge-reranker-v2-m3` (Apache-2.0), with +`MESA_RERANK_DEPTH=50` (range 1–50). The scorer uses authorized full chunk texts +after releasing the search connection, in a worker thread, then returns the +requested number of results. Tied scores retain fused order. Results keep their +fusion fields, including `rrf_score` and `final_score`, and add `rerank_score`. +The first search includes model-load latency; subsequent searches reuse it. + Model and external-provider flags default false. Validation defaults to mode 0; mode 1 or 2 must be set explicitly via `MESA_TIER3_MODE`. Mode 2 needs distinct A/B validator identities. Extraction and validation are separate. @@ -130,7 +150,7 @@ An endpoint is considered **local** when its scheme is `http` or `https` and its If an optional provider or model extra is selected but not installed, server startup halts with a clear one-line installation command: - Ollama LLM, Ollama embedding, or OpenAI-compatible embedding: - `pip install "mesa-memory[adapters]"` + `pip install mesa-memory` - Local transformer embedding (`sentence-transformers`): `pip install "mesa-memory[ml]"` @@ -206,10 +226,15 @@ Field-declaration reference; compare it with `MesaConfig` when configuration cha | `metrics_admission_threshold` | `METRICS_ADMISSION_THRESHOLD` | `0.8` | | `metrics_divergence_threshold` | `METRICS_DIVERGENCE_THRESHOLD` | `0.5` | | `rrf_k` | `MESA_RRF_K` | `60` | -| `rrf_vector_weight` | `MESA_RRF_VECTOR_WEIGHT` | `1.0` | +| `rrf_vector_weight` | `MESA_RRF_VECTOR_WEIGHT` | `8.0` | | `rrf_bm25_weight` | `MESA_RRF_BM25_WEIGHT` | `1.0` | | `rrf_assertion_weight` | `MESA_RRF_ASSERTION_WEIGHT` | `1.0` | | `rrf_graph_weight` | `MESA_RRF_GRAPH_WEIGHT` | `1.0` | +| `search_scope_cache` | `MESA_SEARCH_SCOPE_CACHE` | `True` | +| `chunking` | `MESA_CHUNKING` | `'auto'` | +| `rerank_enabled` | `MESA_RERANK_ENABLED` | `False` | +| `rerank_model` | `MESA_RERANK_MODEL` | `'BAAI/bge-reranker-v2-m3'` | +| `rerank_depth` | `MESA_RERANK_DEPTH` | `50` | | `t_route` | `MESA_T_ROUTE` | `0.85` | | `v4_rebuild_enabled` | `MESA_V4_REBUILD_ENABLED` | `False` | | `v4_graph_timeout_seconds` | `MESA_V4_GRAPH_TIMEOUT_SECONDS` | `15.0` | diff --git a/docs/getting-started.md b/docs/getting-started.md index 1acc104..e79564a 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -6,15 +6,15 @@ This guide covers getting started with MESA as a new user, from installation to ## Installation -Install MESA with the adapters extra: +Install MESA: ```bash -pip install "mesa-memory[adapters]" +pip install mesa-memory ``` Supported natively on Linux and macOS. On Windows, use Docker or WSL. -The development version can be installed directly from git: `pip install "mesa-memory[adapters] @ git+https://github.com/Yasou13/MESA"`. +The development version can be installed directly from git: `pip install "mesa-memory @ git+https://github.com/Yasou13/MESA"`. Or install from source with [uv](https://docs.astral.sh/uv/getting-started/installation/): @@ -45,7 +45,7 @@ mesa search "your query" mesa status ``` -By default, `mesa add` waits for all chunks to commit and the revision to become active, printing `searchable` when complete (pass `--no-wait` to return immediately). +By default, `mesa add` waits for all chunks to commit and the revision to become active, printing `searchable` when complete (pass `--no-wait` to return immediately). To delete an ingested document, run `mesa delete ` or `mesa delete --source `. ### 3. Python SDK @@ -84,7 +84,7 @@ MESA_EMBEDDING_MODEL=ollama:bge-m3 Under decision D31, local `openai_compatible` servers (e.g. vLLM started with `--port 8001`, at `http://127.0.0.1:8001/v1`, since MESA itself uses port 8000) are also permitted with `MESA_EXTERNAL_PROVIDER_ENABLED=false` on a local endpoint and require no API key. For external cloud APIs (OpenAI, Anthropic), set `MESA_EXTERNAL_PROVIDER_ENABLED=true` and supply the required API keys. Missing extras raise clear, actionable installation messages: -- Ollama LLM, Ollama embedding, or OpenAI-compatible embedding → `pip install "mesa-memory[adapters]"` +- Anthropic LLM requires the adapters extra: `pip install "mesa-memory[adapters]"` - Local embedding (`sentence-transformers`) → `pip install "mesa-memory[ml]"` ### 6. Graph toggle diff --git a/docs/release.md b/docs/release.md index c09f0bc..d60edd7 100644 --- a/docs/release.md +++ b/docs/release.md @@ -113,17 +113,31 @@ yapılmaz; eski storage ve v3 compatibility servisi korunur. ## Yayınlama (yalnızca depo sahibi) -PyPI yayını ve sürüm etiketleme adımları: +PyPI yayını GitHub Actions üzerinden OIDC ile token/secret kullanılmadan trusted publishing ile yapılır (D42). + +### Bir kerelik kurulum (depo sahibi) + +1. **PyPI Trusted Publisher**: + - PyPI hesabında `mesa-memory` projesi (veya "Add a pending publisher" sekmesi) altında GitHub publisher eklenir: + - Owner: `Yasou13` + - Repository: `MESA` + - Workflow name: `release.yml` + - Environment name: `pypi` +2. **GitHub Environment**: + - `Yasou13/MESA` repository ayarlarında (`Settings > Environments`) `pypi` isimli bir environment oluşturulur. + +### Yayınlama adımı + +Sürüm kapıları geçtikten sonra etiketi oluşturup göndermek yeterlidir: ```bash -uv build -uvx twine check dist/* -uv publish -git tag v0.9.0 -git push origin v0.9.0 +git tag vX.Y.Z +git push origin vX.Y.Z ``` -`README.md` dosyasındaki kurulum satırı bu release commit'inde `pip install "mesa-memory[adapters]"` olarak tamamlanmıştır. +Tag push edildiğinde `.github/workflows/release.yml` otomatik tetiklenir, `uv build` ile paketleri üretir, `uvx twine check` ile doğrular ve `pypa/gh-action-pypi-publish` ile PyPI'a yayınlar. + +`README.md` dosyasındaki kurulum satırı bu release commit'inde `pip install mesa-memory` olarak tamamlanmıştır. ## Sürüme özel yayın istisnaları diff --git a/docs/retrieval-and-graph.md b/docs/retrieval-and-graph.md index 8f5eae5..a17eb0e 100644 --- a/docs/retrieval-and-graph.md +++ b/docs/retrieval-and-graph.md @@ -7,11 +7,11 @@ agent, committed-state, jurisdiction and temporal eligibility are checked before candidate rank contribution. Physical vector IDs and graph assertions must agree with canonical ownership/evidence. -Scope predicates limit eligible rows; they are not evidence of statistically -independent rankings. The lexical query calls `bm25()` on the shared -`v4_assertions_fts` table. Do not infer complete cross-scope ranking isolation -from a WHERE filter or from the `scope_audit` label alone; candidate/rank and -observable diagnostic behavior require separate negative tests. +BM25 uses a temporary FTS5 index containing only authorized assertions, so +out-of-scope rows do not enter its corpus statistics. The scope cache reuses +that index only while SQLite's data version matches the pinned read snapshot. +The `scope_audit` describes eligibility; negative tests also check rank and +observable diagnostic isolation. | Lane | Candidate signal | |---|---| @@ -21,9 +21,11 @@ observable diagnostic behavior require separate negative tests. | Graph | Bounded paths from legal, vector, lexical and entity-name seeds | For candidate `c`, `RRF(c) = Σ w_lane / (k + rank_lane(c))`, with ranks starting -at 1, default `k=60`, and all four default weights `1.0`. Duplicate backend hits -cast one vote. Ties use candidate identity. The source is -`mesa_storage/retrieval_scope.py`; runtime overrides are in [configuration](configuration.md). +at 1 and default `k=60`. Runtime defaults are vector weight `8.0` and the other +three weights `1.0`, selected by the [dev benchmark](#round-14-dev-benchmark). +Duplicate backend hits cast one vote. Ties use candidate identity. The RRF +helper in `mesa_storage/retrieval_scope.py` uses unit weights when none are +passed; the server passes its [configuration](configuration.md) to the DAO. Lexical exact citation +200 and same-statute +40 change the BM25/lexical lane ordering. Assertion exact citation +150 and same-statute +40 change assertion @@ -34,6 +36,11 @@ factor 1.0. HTTP/SDK `final_score` is final fused relevance; higher is better. `rrf_score`, `legal_factor` and lane `raw_score` are separate diagnostics; MCP response adapters may expose a normalized `score` field. +Optional [reranking](configuration.md#providers-and-capacity) orders fused +candidates by a cross-encoder score after the search connection is released. +It adds `rerank_score`; `rrf_score` and `final_score` retain their fusion values. +Equal reranker scores retain fused order. Reranking is off by default. + Typed ingestion identity uses a bounded alias ontology and immutable provenance. It improves scoped pre-rank candidate discovery, including article chunks with little title context. Reserved caller metadata cannot opt into trusted identity. @@ -91,3 +98,64 @@ needed, jurisdiction. These are request patterns, not measured benchmark results Core context packing is distinct from the independently sealed Profile B context and scorer in [E2E Certification](architecture-v4.md#ecosystem). + +## Round 14 dev benchmark + +Measured on 2026-11-01 using the frozen Tier M corpus: 141 documents and 5,041 +chunks ingested through this branch's authenticated document endpoint, with +article chunking enabled, extraction off, embeddings and graph on. Only the +101-question dev split was evaluated (85 answerable, 16 no-answer); the test +split was not accessed. The unmodified `mesa_eval.py` computes passage hit@5 +and MRR on answerable questions; latency covers all 101 questions. Retrieval +limit and reranker depth are both 50, with max_length 1024 for the reranker. + +Embeddings use `BAAI/bge-m3` (1024 dimensions, normalized), and reranking uses +`BAAI/bge-reranker-v2-m3`. Each row below uses the same complete store, GPU +models and other lane weights of 1. Model revisions, package versions, source +hashes, unrounded MRR and the selection rule are in +[the measurement data](benchmarks/round14_dev.json). + +| Vector weight | Reranker | Passage hit@5 | Passage MRR | Search p50 (s) | Search p95 (s) | +|---:|---|---:|---:|---:|---:| +| 1 | Off | 0.5176 | 0.3953 | 1.2693 | 2.0035 | +| 1 | On | 0.7412 | 0.6816 | 1.9548 | 2.5736 | +| 2 | Off | 0.5765 | 0.4565 | 1.2708 | 1.9640 | +| 2 | On | 0.7529 | 0.7055 | 1.9569 | 2.6364 | +| 4 | Off | 0.6588 | 0.5152 | 1.2836 | 1.9469 | +| 4 | On | 0.7765 | 0.7069 | 1.9279 | 2.6100 | +| 8 | Off | 0.6706 | 0.5663 | 1.2849 | 1.9115 | +| 8 | On | 0.7647 | 0.6973 | 1.9161 | 2.5360 | + +The default is selected from **reranker-off** MRR because reranking is off by +default. Weight 8 has unrounded MRR 0.566347, versus 0.395339 at weight 1: a +gain of 0.171008, exceeding the required 0.02. The runtime default +is therefore **8.0**. Without that gain the default would stay 1; ties use the +smaller weight. Weight 4 has the best reranker-on MRR in this sweep. Overrides +remain available through `MESA_RRF_VECTOR_WEIGHT`. These measurements describe +this Turkish legal corpus and these models; other corpora/models need their own +measurements. + +A separate device pass at vector weight 1 measured the optional reranker: + +| Inference device / reranker | Passage hit@5 | Passage MRR | Search p50 (s) | Search p95 (s) | +|---|---:|---:|---:|---:| +| GPU / Off | 0.5176 | 0.3953 | 1.2531 | 1.9953 | +| GPU / On | 0.7412 | 0.6816 | 1.9627 | 2.6029 | +| CPU / On | 0.7412 | 0.6816 | 26.9158 | 33.8756 | + +Hardware: Colab A100-SXM4 40GB and Intel Xeon 2.20GHz CPU, with eight CPU +threads. Python 3.13.16, Torch 2.11.0+cu130, sentence-transformers 5.7.0, +transformers 5.21.0, NumPy 2.5.4, LanceDB 0.41.0 and Kùzu 0.11.3. These are +compatible installed Colab packages, rather than the lockfile's exact ML pins; +the local gate uses the locked dev environment. Both models were explicitly +acquired before runtime startup. The device pass moves both models to CPU +without quantization for its CPU row. + +Requests use the actual authenticated HTTP route through ASGI, without TCP. +Passes run sequentially, reusing the store and caches; one first-query warmup +is excluded from each pass (warmup values are in the data). These are descriptive +latencies, including embedding, scoped retrieval and optional reranking, rather +than a controlled hardware speedup claim. The benchmark store was ingested with +the measured source commit and then reused after the search-lock change, which +kept all 101 dev top-50 result dictionaries identical. The original frozen +corpus and question files were read only throughout. diff --git a/examples/colab_quickstart.ipynb b/examples/colab_quickstart.ipynb index bf37d31..b1cee8e 100644 --- a/examples/colab_quickstart.ipynb +++ b/examples/colab_quickstart.ipynb @@ -6,8 +6,8 @@ "metadata": {}, "outputs": [], "source": [ - "# Step 1: Install MESA with adapters extra (D34)\n", - "!pip install \"mesa-memory[adapters]\"" + "# Step 1: Install MESA (D34)\n", + "!pip install mesa-memory" ] }, { diff --git a/mesa_api/v4_router.py b/mesa_api/v4_router.py index 33ffe70..9fba898 100644 --- a/mesa_api/v4_router.py +++ b/mesa_api/v4_router.py @@ -23,7 +23,7 @@ from mesa_memory.config import config, configured_embedding_identity from mesa_memory.consolidation.policy import ValidationPolicy from mesa_memory.context_builder import ContextBuilder -from mesa_memory.document_chunking import chunk_document +from mesa_memory.document_chunking import chunk_document, chunk_legal_document from mesa_memory.embedding.service import ( EmbeddingGenerationError, EmbeddingIdentityMismatchError, @@ -1305,7 +1305,44 @@ async def ingest_document( "mutation_ids": [admitted[i] for i in sorted(admitted)], "duplicate": True, } - chunks = chunk_document(payload.content) + chunks = ( + chunk_legal_document(payload.content) + if config.chunking == "auto" + else chunk_document(payload.content) + ) + if admitted: + # Admissions are already scoped above. Their durable payloads, + # including chunks committed before a restart, pin the plan. + texts = {} + for ordinal, mutation_id in admitted.items(): + mutation = await dao.get_projection_mutation(mutation_id) + if mutation is None: + raise HTTPException(409, "admitted chunk is unavailable") + texts[ordinal] = mutation["content_payload"] + + def matches(plan: list[str]) -> bool: + return all( + ordinal < len(plan) and plan[ordinal] == text + for ordinal, text in texts.items() + ) + + alternative = ( + chunk_document(payload.content) + if config.chunking == "auto" + else chunk_legal_document(payload.content) + ) + preferred_matches = matches(chunks) + alternative_matches = matches(alternative) + if preferred_matches and alternative_matches and chunks != alternative: + raise HTTPException( + 409, "admitted chunking plan is ambiguous; no chunks appended" + ) + if not preferred_matches: + if not alternative_matches: + raise HTTPException( + 409, "admitted chunks do not match a supported chunking plan" + ) + chunks = alternative duplicate = len(admitted) == len(chunks) for ordinal, chunk in enumerate(chunks): if ordinal in admitted: diff --git a/mesa_client/client.py b/mesa_client/client.py index 470efec..c59bb01 100644 --- a/mesa_client/client.py +++ b/mesa_client/client.py @@ -67,7 +67,7 @@ class MesaNetworkError(MesaClientError): pass -class MesaValidationError(MesaClientError): +class MesaValidationError(MesaClientError, ValueError): """Exception raised when Pydantic validation fails for request or response payloads.""" pass @@ -380,6 +380,18 @@ async def _op() -> httpx.Response: {"COMMITTED", "REJECTED", "DEAD_LETTER", "ROLLED_BACK", "BLOCKED"} ) +_TERMINAL_FAILURE_MUTATION_STATES = frozenset( + { + "REJECTED", + "DEAD_LETTER", + "BLOCKED", + "CANCELLED", + "ROLLED_BACK", + "PURGED", + "FAILED", + } +) + def _v4_control_value(value: str, *, label: str) -> str: if ( @@ -790,11 +802,14 @@ def wait_until_revision_active( dataset_id: str, document_id: str, revision_id: str, + mutation_ids: list[str] | None = None, timeout_seconds: float = 600.0, poll_interval_seconds: float = 1.0, ) -> dict[str, Any]: """Poll revision status once per second until ACTIVE or terminal failure.""" deadline = time.monotonic() + timeout_seconds + mut_list = [str(m) for m in mutation_ids] if mutation_ids else [] + mut_cursor = 0 while True: resp = self.list_revisions( tenant_id=tenant_id, @@ -827,6 +842,26 @@ def wait_until_revision_active( raise MesaClientError( f"Revision {revision_id} entered unexpected state: {status}" ) + if mut_cursor < len(mut_list): + mut_id = mut_list[mut_cursor] + mut_resp = self.status(mut_id) + mut_state = str(mut_resp.get("state", "")) + if mut_state == "COMMITTED": + mut_cursor += 1 + elif mut_state in _TERMINAL_FAILURE_MUTATION_STATES: + err_class = ( + mut_resp.get("failure_class") + or mut_resp.get("rejection_reason") + ) + if not err_class: + for proj in mut_resp.get("projections", []): + if proj.get("last_error_class"): + err_class = proj["last_error_class"] + break + err_class = err_class or "unknown" + raise MesaClientError( + f"Mutation {mut_id} entered terminal state {mut_state} with error class {err_class}" + ) if time.monotonic() >= deadline: raise MesaNetworkError( f"Timed out waiting for revision {revision_id} to become ACTIVE" @@ -1218,11 +1253,14 @@ async def wait_until_revision_active( dataset_id: str, document_id: str, revision_id: str, + mutation_ids: list[str] | None = None, timeout_seconds: float = 600.0, poll_interval_seconds: float = 1.0, ) -> dict[str, Any]: """Poll revision status once per second until ACTIVE or terminal failure.""" deadline = time.monotonic() + timeout_seconds + mut_list = [str(m) for m in mutation_ids] if mutation_ids else [] + mut_cursor = 0 while True: resp = await self.list_revisions( tenant_id=tenant_id, @@ -1255,6 +1293,26 @@ async def wait_until_revision_active( raise MesaClientError( f"Revision {revision_id} entered unexpected state: {status}" ) + if mut_cursor < len(mut_list): + mut_id = mut_list[mut_cursor] + mut_resp = await self.status(mut_id) + mut_state = str(mut_resp.get("state", "")) + if mut_state == "COMMITTED": + mut_cursor += 1 + elif mut_state in _TERMINAL_FAILURE_MUTATION_STATES: + err_class = ( + mut_resp.get("failure_class") + or mut_resp.get("rejection_reason") + ) + if not err_class: + for proj in mut_resp.get("projections", []): + if proj.get("last_error_class"): + err_class = proj["last_error_class"] + break + err_class = err_class or "unknown" + raise MesaClientError( + f"Mutation {mut_id} entered terminal state {mut_state} with error class {err_class}" + ) if time.monotonic() >= deadline: raise MesaNetworkError( f"Timed out waiting for revision {revision_id} to become ACTIVE" @@ -1262,6 +1320,10 @@ async def wait_until_revision_active( await asyncio.sleep(poll_interval_seconds) +def _document_id_from_source(source: str) -> str: + return f"doc-{hashlib.sha256(source.encode('utf-8')).hexdigest()[:32]}" + + class Mesa: """Convenience SDK entrypoint for fast local use and scripting (D30). @@ -1363,7 +1425,7 @@ def add( deadline: float | None = None, ) -> dict[str, Any]: """Ingest a document with server-side chunking (D29).""" - doc_id = document_id or f"doc-{hashlib.sha256(source.encode('utf-8')).hexdigest()[:32]}" + doc_id = document_id or _document_id_from_source(source) doc_title = title if title is not None else source session_id = self._ensure_session() total_timeout = deadline if deadline is not None else timeout @@ -1378,12 +1440,19 @@ def add( ) if wait: rev_id = str(res["revision_id"]) + raw_mutation_ids = res.get("mutation_ids") + mutation_ids = ( + [str(m) for m in raw_mutation_ids] + if isinstance(raw_mutation_ids, list) + else None + ) self._client.wait_until_revision_active( tenant_id=self.tenant_id, workspace_id=self.workspace_id, dataset_id=self.dataset_id, document_id=doc_id, revision_id=rev_id, + mutation_ids=mutation_ids, timeout_seconds=total_timeout, poll_interval_seconds=1.0, ) @@ -1407,13 +1476,29 @@ def context(self, query: str, max_tokens: int = 2000) -> str: ) return str(resp.get("context", "")) - def delete(self, document_id: str) -> dict[str, Any]: + def delete( + self, + document_id: str | None = None, + *, + source: str | None = None, + ) -> dict[str, Any]: """Purge a document from the catalog.""" + if (document_id is None and source is None) or ( + document_id is not None and source is not None + ): + raise MesaValidationError( + "Exactly one of document_id or source is required" + ) + doc_id = ( + document_id + if document_id is not None + else _document_id_from_source(source) # type: ignore[arg-type] + ) return self._client.purge_document( tenant_id=self.tenant_id, workspace_id=self.workspace_id, dataset_id=self.dataset_id, - document_id=document_id, + document_id=doc_id, ) def close(self) -> None: diff --git a/mesa_mcp/local_cli.py b/mesa_mcp/local_cli.py index 84e22be..e937868 100644 --- a/mesa_mcp/local_cli.py +++ b/mesa_mcp/local_cli.py @@ -203,16 +203,25 @@ def search_command(args: argparse.Namespace) -> None: def delete_command(args: argparse.Namespace) -> None: from mesa_client import Mesa + doc_arg = getattr(args, "document_id", None) + source = getattr(args, "source", None) + if (doc_arg is None and source is None) or ( + doc_arg is not None and source is not None + ): + sys.stderr.write("Error: Exactly one of document_id or --source is required\n") + sys.exit(1) + mesa = Mesa() try: - res = mesa.delete(args.document_id) + res = mesa.delete(document_id=doc_arg, source=source) finally: mesa.close() if getattr(args, "json", False): print(json.dumps(res, indent=2)) else: - print(f"Purged document {args.document_id}") + doc_id = res.get("document_id") or doc_arg or source + print(f"Purged document {doc_id}") def status_command(args: argparse.Namespace) -> None: @@ -343,7 +352,17 @@ def register_local_commands(subparsers: argparse._SubParsersAction) -> None: delete_parser = subparsers.add_parser( "delete", help="purge a document from MESA" ) - delete_parser.add_argument("document_id", help="document ID to purge") + delete_parser.add_argument( + "document_id", + nargs="?", + default=None, + help="document ID to purge", + ) + delete_parser.add_argument( + "--source", + default=None, + help="source reference to purge", + ) delete_parser.add_argument("--json", action="store_true", help="output JSON") status_parser = subparsers.add_parser( diff --git a/mesa_memory/api/server.py b/mesa_memory/api/server.py index eaba6b3..d69cfad 100644 --- a/mesa_memory/api/server.py +++ b/mesa_memory/api/server.py @@ -295,11 +295,16 @@ async def _run_combined_durable_consumer( async def _runtime_lifespan(app: FastAPI, runtime: RuntimeProfileConfig): refresh_config_from_environment() state.is_ready = False - state.obs_layer = ObservabilityLayer() state.background_tasks = set() state.worker_supervisor = WorkerSupervisor(max_restarts=3) + reranker = None + if config.rerank_enabled: + from mesa_memory.reranking import CrossEncoderReranker + + reranker = CrossEncoderReranker(config.rerank_model) + # Initialize asynchronous storage engines state.sqlite_engine = AsyncEngine(db_path=str(_SQLITE_PATH)) await state.sqlite_engine.initialize() @@ -372,6 +377,9 @@ async def _runtime_lifespan(app: FastAPI, runtime: RuntimeProfileConfig): "assertion": config.rrf_assertion_weight, "graph": config.rrf_graph_weight, }, + search_scope_cache_enabled=config.search_scope_cache, + reranker=reranker, + rerank_depth=config.rerank_depth, ) await state.dao.initialize() if runtime.profile is RuntimeProfile.COMBINED: diff --git a/mesa_memory/config.py b/mesa_memory/config.py index 3bd32ae..7f2994c 100644 --- a/mesa_memory/config.py +++ b/mesa_memory/config.py @@ -4,7 +4,7 @@ from dataclasses import dataclass from enum import Enum from pathlib import Path -from typing import Any, Mapping, Optional +from typing import Any, Literal, Mapping, Optional import psutil from dotenv import load_dotenv @@ -613,6 +613,9 @@ def vector_worker_limit(self) -> int: # Observability (Module 4) histogram_max_size: int = 10000 + chunking: Literal["auto", "plain"] = Field( + "auto", validation_alias="MESA_CHUNKING" + ) metrics_admission_threshold: float = 0.8 metrics_divergence_threshold: float = 0.5 @@ -621,12 +624,20 @@ def vector_worker_limit(self) -> int: # score-space weights here because scores from those lanes are not # comparable. rrf_k: int = Field(60, validation_alias="MESA_RRF_K") - rrf_vector_weight: float = Field(1.0, validation_alias="MESA_RRF_VECTOR_WEIGHT") + rrf_vector_weight: float = Field(8.0, validation_alias="MESA_RRF_VECTOR_WEIGHT") rrf_bm25_weight: float = Field(1.0, validation_alias="MESA_RRF_BM25_WEIGHT") rrf_assertion_weight: float = Field( 1.0, validation_alias="MESA_RRF_ASSERTION_WEIGHT" ) rrf_graph_weight: float = Field(1.0, validation_alias="MESA_RRF_GRAPH_WEIGHT") + search_scope_cache: bool = Field( + True, validation_alias="MESA_SEARCH_SCOPE_CACHE" + ) + rerank_enabled: bool = Field(False, validation_alias="MESA_RERANK_ENABLED") + rerank_model: str = Field( + "BAAI/bge-reranker-v2-m3", min_length=1, validation_alias="MESA_RERANK_MODEL" + ) + rerank_depth: int = Field(50, ge=1, le=50, validation_alias="MESA_RERANK_DEPTH") t_route: float = Field(0.85, validation_alias="MESA_T_ROUTE") # V4 projection rebuild remains an explicit operator opt-in. Enabling this diff --git a/mesa_memory/document_chunking.py b/mesa_memory/document_chunking.py index 275893c..8fde8cb 100644 --- a/mesa_memory/document_chunking.py +++ b/mesa_memory/document_chunking.py @@ -30,6 +30,14 @@ "şti", }) _TRAILING_WORD = re.compile(r"([^\W_]+)$") +_ARTICLE_HEADING = re.compile( + r"^[ \t]*(?:(?:GEÇİCİ|EK)[ \t]+)?MADDE[ \t]+\d+" + r"(?:/[A-ZÇĞİÖŞÜ]+)?(?:[ \t]+ila[ \t]+\d+)?[ \t]*[-–—−:]", + re.MULTILINE | re.IGNORECASE, +) +_CASE_LAW_PREAMBLE = re.compile( + r'^[ \t]*["“]?İçtihat Metni["”]?[ \t]*\r?$', re.MULTILINE | re.IGNORECASE +) def _is_sentence_end(prefix: str, m: re.Match) -> bool: @@ -97,3 +105,19 @@ def chunk_document(content: str) -> list[str]: chunks.append(content[start:end]) start = end return chunks + + +def chunk_legal_document(content: str) -> list[str]: + """Keep Turkish article headings at slice starts without changing the text.""" + starts = [match.start() for match in _ARTICLE_HEADING.finditer(content)] + if len(starts) < 3 or _CASE_LAW_PREAMBLE.search(content[:starts[0]]): + return chunk_document(content) + boundaries = sorted({0, *starts, len(content)}) + chunks = [] + for start, end in zip(boundaries, boundaries[1:]): + piece = content[start:end] + if len(piece) <= 32768 and count_tokens(piece, "openai") <= 512: + chunks.append(piece) + else: + chunks.extend(chunk_document(piece)) + return chunks diff --git a/mesa_memory/reranking.py b/mesa_memory/reranking.py new file mode 100644 index 0000000..0c81bc9 --- /dev/null +++ b/mesa_memory/reranking.py @@ -0,0 +1,27 @@ +"""Optional, lazily loaded cross-encoder for already authorized passages.""" + +import importlib.util +import threading +from typing import Any + + +class CrossEncoderReranker: + def __init__(self, model: str) -> None: + if importlib.util.find_spec("sentence_transformers") is None: + raise RuntimeError('Reranking requires: pip install "mesa-memory[ml]"') + self._model_name = model + self._model: Any = None + self._load_lock = threading.Lock() + + def score(self, query: str, texts: list[str]) -> list[float]: + """Called in a worker thread, including the one-time model load.""" + with self._load_lock: + if self._model is None: + from sentence_transformers import CrossEncoder + + self._model = CrossEncoder(self._model_name, max_length=1024) + return [ + float(value) for value in self._model.predict( + [(query, text) for text in texts], show_progress_bar=False + ) + ] diff --git a/mesa_storage/dao.py b/mesa_storage/dao.py index 49d0bf9..c7364f3 100644 --- a/mesa_storage/dao.py +++ b/mesa_storage/dao.py @@ -16,10 +16,11 @@ from __future__ import annotations +import asyncio import hashlib import json import logging -import os +import math import re import sqlite3 import unicodedata @@ -63,7 +64,11 @@ stable_graph_path_id, ) from mesa_storage.sqlite_engine import AsyncEngine -from mesa_storage.vector_engine import SemanticRuntimeDisabledError, VectorEngine +from mesa_storage.vector_engine import ( + SemanticRuntimeDisabledError, + VectorEngine, + VectorSearchError, +) logger = logging.getLogger("MESA_DAO") @@ -539,6 +544,8 @@ class MemoryDAO: "_search_scope_cache", "_search_cache_db", "_search_data_version", + "_reranker", + "_rerank_depth", ) def __init__( @@ -553,6 +560,9 @@ def __init__( graph_enabled: bool = True, rrf_k: int = V4_RRF_DEFAULT_K, rrf_weights: Mapping[str, float] | None = None, + search_scope_cache_enabled: bool = True, + reranker: Any = None, + rerank_depth: int = 50, ) -> None: self._sql = sqlite_engine self._vec = vector_engine @@ -567,14 +577,14 @@ def __init__( self._graph_enabled = graph_enabled self._rrf_k = rrf_k self._rrf_weights = dict(rrf_weights or V4_RRF_LANE_WEIGHTS) - self._search_scope_cache_enabled = os.environ.get( - "MESA_SEARCH_SCOPE_CACHE", "true" - ).strip().lower() not in {"0", "false", "no", "off"} + self._search_scope_cache_enabled = search_scope_cache_enabled self._search_scope_cache: OrderedDict[tuple[Any, ...], dict[str, Any]] = ( OrderedDict() ) self._search_cache_db: aiosqlite.Connection | None = None self._search_data_version: int | None = None + self._reranker = reranker + self._rerank_depth = rerank_depth @property def catalog(self) -> CatalogRepositoryPort: @@ -5793,12 +5803,24 @@ async def search_v4_memory( certification_metadata: dict[str, Any] | None = None, ) -> list[dict[str, Any]]: """Reuse only version-valid authorized scope data, never query results.""" + _assert_valid_agent_id(agent_id) + if not tenant_id or not dataset_ids or not query.strip() or not 1 <= limit <= 50: + raise ValueError("invalid v4 search scope") + query_vector: list[float] | None = None + query_error: Exception | None = None + if self._embedding_enabled and self._vec is not None: + try: + query_vector = await self._vec.compute_query_embedding(query) + except Exception as exc: + # Preserve degradation/error semantics: a provider error matters + # only if the authorized scope actually has compatible vectors. + query_error = exc kwargs: dict[str, Any] = dict( tenant_id=tenant_id, agent_id=agent_id, dataset_ids=dataset_ids, query=query, - limit=limit, + limit=max(limit, self._rerank_depth) if self._reranker is not None else limit, jurisdiction=jurisdiction, valid_at=valid_at, valid_from=valid_from, @@ -5808,9 +5830,14 @@ async def search_v4_memory( graph_enabled=graph_enabled, request_principal_id=request_principal_id, certification_metadata=certification_metadata, + _query_vector=query_vector, + _query_error=query_error, ) if not self._search_scope_cache_enabled: - return await self._search_v4_memory_uncached(**kwargs) + if self._reranker is None: + return await self._search_v4_memory_uncached(**kwargs) + result = await self._search_v4_memory_uncached(**kwargs) + return await self._rerank_results(query, result, limit) async with self._sql.search_connection() as db: if self._search_cache_db is not db: self._search_scope_cache.clear() @@ -5859,7 +5886,6 @@ async def search_v4_memory( **kwargs, _search_db=db, _scope=scope ) await db.commit() # Only TEMP can be written on this connection. - return result except BaseException: await db.rollback() # Rollback can restore tables evicted earlier in this search. @@ -5868,6 +5894,23 @@ async def search_v4_memory( await self._clear_search_scope_cache(db) self._search_data_version = None raise + if self._reranker is not None: + return await self._rerank_results(query, result, limit) + return result + + async def _rerank_results( + self, query: str, results: list[dict[str, Any]], limit: int + ) -> list[dict[str, Any]]: + if not results: + return [] + texts = [row.pop("_rerank_text") for row in results] + scores = await asyncio.to_thread(self._reranker.score, query, texts) + if len(scores) != len(results) or any(not math.isfinite(s) for s in scores): + raise RuntimeError("reranker returned invalid scores") + for row, score in zip(results, scores): + row["rerank_score"] = float(score) + # Python's stable sort retains fused order for tied scores. + return sorted(results, key=lambda row: -row["rerank_score"])[:limit] async def _clear_search_scope_cache(self, db: aiosqlite.Connection) -> None: for scope in self._search_scope_cache.values(): @@ -5903,6 +5946,8 @@ async def _search_v4_memory_uncached( certification_metadata: dict[str, Any] | None = None, _search_db: aiosqlite.Connection | None = None, _scope: dict[str, Any] | None = None, + _query_vector: list[float] | None = None, + _query_error: Exception | None = None, ) -> list[dict[str, Any]]: """Dataset-filter every retrieval lane, then combine ranks with RRF.""" _assert_valid_agent_id(agent_id) @@ -6353,9 +6398,12 @@ async def _search_v4_memory_uncached( if not searchable_vector_ids: vector_rows = [] else: - query_vector = await self._vec.compute_query_embedding(query) + if _query_error is not None: + raise _query_error + if _query_vector is None: + raise VectorSearchError("query embedding is unavailable") vector_rows = await self._vec.search( - query_vector, + _query_vector, agent_id=agent_id, allowed_node_ids=searchable_vector_ids, limit=min(500, max(limit * 10, 50)), @@ -7367,15 +7415,20 @@ async def _search_v4_memory_uncached( } ) metadata_by_mutation: dict[str, dict[str, Any]] = {} + rerank_texts: dict[str, str] = {} async with self._search_read_connection(_search_db) as db: if mutation_ids: mutation_placeholders = ",".join("?" for _ in mutation_ids) async with db.execute( - "SELECT mutation_id, metadata_json FROM memory_mutations " + "SELECT mutation_id, metadata_json" + + (", content_payload" if self._reranker is not None else "") + + " FROM memory_mutations " f"WHERE mutation_id IN ({mutation_placeholders})", mutation_ids, ) as cursor: for row in await cursor.fetchall(): + if self._reranker is not None: + rerank_texts[str(row["mutation_id"])] = str(row["content_payload"]) try: metadata = json.loads(str(row["metadata_json"] or "{}")) except (TypeError, json.JSONDecodeError): @@ -7498,6 +7551,8 @@ async def _search_v4_memory_uncached( ], "retrieval_provenance": retrieval_provenance, } + if self._reranker is not None: + result["_rerank_text"] = rerank_texts[str(cand_prov[0]["mutation_id"])] if certification_metadata is not None: first_provenance = cand_prov[0] result["scope_identity"] = { diff --git a/pyproject.toml b/pyproject.toml index b86e3a2..4c81e62 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,8 @@ dependencies = [ "slowapi>=0.1.9", "prometheus_client>=0.20.0", # LLM Provider SDKs (Core) - # Note: Specific SDKs (openai, anthropic, etc.) are moved to optional [adapters] + "openai>=2.36.0", + "ollama>=0.6.1", # Pydantic / Settings "pydantic>=2.13.0", "pydantic-settings>=2.14.0", @@ -58,8 +59,6 @@ dependencies = [ [project.optional-dependencies] adapters = [ "anthropic>=0.100.0", - "openai>=2.36.0", - "ollama>=0.6.1", ] ml = [ "torch>=2.11.0", diff --git a/tests/test_deployment_assets.py b/tests/test_deployment_assets.py index 496658a..5b93419 100644 --- a/tests/test_deployment_assets.py +++ b/tests/test_deployment_assets.py @@ -196,20 +196,34 @@ def test_documented_install_lines_use_published_pypi_package() -> None: quickstart = (ROOT / "docs/getting-started.md").read_text(encoding="utf-8") colab = (ROOT / "examples/colab_quickstart.ipynb").read_text(encoding="utf-8") - assert 'pip install "mesa-memory[adapters]"' in readme + assert "pip install mesa-memory" in readme assert "git+https://github.com/Yasou13/MESA" not in readme - assert 'pip install "mesa-memory[adapters]"' in quickstart + assert "pip install mesa-memory" in quickstart assert ( - 'The development version can be installed directly from git: ' - '`pip install "mesa-memory[adapters] @ git+https://github.com/Yasou13/MESA"`.' + "The development version can be installed directly from git: " + '`pip install "mesa-memory @ git+https://github.com/Yasou13/MESA"`.' in quickstart ) - assert '!pip install \\"mesa-memory[adapters]\\"' in colab + assert "!pip install mesa-memory" in colab assert "git+https://github.com/Yasou13/MESA" not in colab +def test_openai_and_ollama_in_core_dependencies() -> None: + import tomllib + data = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8")) + core_deps = data["project"]["dependencies"] + optional_adapters = data["project"]["optional-dependencies"]["adapters"] + + assert any(dep.startswith("openai>=") for dep in core_deps) + assert any(dep.startswith("ollama>=") for dep in core_deps) + assert not any(dep.startswith("openai") for dep in optional_adapters) + assert not any(dep.startswith("ollama") for dep in optional_adapters) + assert any(dep.startswith("anthropic>=") for dep in optional_adapters) + + + def test_documented_public_api_examples_match_authenticated_route_contract() -> None: quickstart = (ROOT / "docs/getting-started.md").read_text(encoding="utf-8") v4_example = (ROOT / "examples/v4_quickstart.py").read_text(encoding="utf-8") diff --git a/tests/test_p402_mesa_sdk.py b/tests/test_p402_mesa_sdk.py index bdc46c5..fc72d3a 100644 --- a/tests/test_p402_mesa_sdk.py +++ b/tests/test_p402_mesa_sdk.py @@ -20,6 +20,7 @@ class FakeMesaServerHandler(BaseHTTPRequestHandler): requests_log: list[dict[str, Any]] = [] documents_503_count: int = 0 revisions_status_sequence: list[str] = [] + mutations: dict[str, dict[str, Any]] = {} def log_message(self, format: str, *args: Any) -> None: pass # Quiet logs during test runs @@ -123,6 +124,23 @@ def do_GET(self) -> None: ] } self.wfile.write(json.dumps(resp).encode("utf-8")) + elif "/v4/mutations/" in self.path: + mut_id = self.path.split("/v4/mutations/")[-1].split("?")[0] + mut_info = FakeMesaServerHandler.mutations.get( + mut_id, {"state": "COMMITTED", "mutation_id": mut_id} + ) + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + resp = { + "mutation_id": mut_id, + "candidate_id": f"cand-{mut_id}", + "state": mut_info.get("state", "COMMITTED"), + "failure_class": mut_info.get("failure_class"), + "rejection_reason": mut_info.get("rejection_reason"), + "projections": mut_info.get("projections", []), + } + self.wfile.write(json.dumps(resp).encode("utf-8")) else: self.send_response(404) self.end_headers() @@ -148,6 +166,7 @@ def fake_server(): FakeMesaServerHandler.requests_log = [] FakeMesaServerHandler.documents_503_count = 0 FakeMesaServerHandler.revisions_status_sequence = [] + FakeMesaServerHandler.mutations = {} server = HTTPServer(("127.0.0.1", 0), FakeMesaServerHandler) host, port = server.server_address url = f"http://{host}:{port}" @@ -392,3 +411,93 @@ def test_mesa_add_wait_raises_on_failed_revision(fake_server): mesa.add("Kira metni", source="file://kira.txt", wait=True) assert "FAILED" in str(exc_info.value) + +def test_mesa_add_wait_raises_immediately_on_chunk_failure(fake_server): + url, logs = fake_server + FakeMesaServerHandler.revisions_status_sequence = ["PENDING", "PENDING"] + FakeMesaServerHandler.mutations = { + "mut-1": {"state": "DEAD_LETTER", "failure_class": "Tier3Unavailable"} + } + with Mesa(base_url=url, api_key="test-key", dataset_id="legal") as mesa: + with pytest.raises(MesaClientError) as exc_info: + mesa.add("Kira metni", source="file://kira.txt", wait=True, timeout=5.0) + msg = str(exc_info.value) + assert "mut-1" in msg + assert "DEAD_LETTER" in msg + assert "Tier3Unavailable" in msg + + mut_logs = [log for log in logs if "/v4/mutations/mut-1" in log["path"]] + assert len(mut_logs) == 1 + + +def test_mesa_add_wait_advances_through_mutations(fake_server): + url, logs = fake_server + FakeMesaServerHandler.revisions_status_sequence = ["PENDING", "PENDING", "ACTIVE"] + FakeMesaServerHandler.mutations = { + "mut-1": {"state": "COMMITTED"}, + "mut-2": {"state": "COMMITTED"}, + } + with Mesa(base_url=url, api_key="test-key", dataset_id="legal") as mesa: + res = mesa.add("Kira metni", source="file://kira.txt", wait=True, timeout=5.0) + assert res["document_id"].startswith("doc-") + + mut1_logs = [log for log in logs if "/v4/mutations/mut-1" in log["path"]] + mut2_logs = [log for log in logs if "/v4/mutations/mut-2" in log["path"]] + assert len(mut1_logs) == 1 + assert len(mut2_logs) == 1 + + +@pytest.mark.asyncio +async def test_async_wait_raises_immediately_on_chunk_failure(fake_server): + url, logs = fake_server + FakeMesaServerHandler.revisions_status_sequence = ["PENDING", "PENDING"] + FakeMesaServerHandler.mutations = { + "mut-1": {"state": "DEAD_LETTER", "failure_class": "Tier3Unavailable"} + } + client = AsyncMesaV4Client(base_url=url, api_key="test-key") + try: + with pytest.raises(MesaClientError) as exc_info: + await client.wait_until_revision_active( + tenant_id="local", + workspace_id="default", + dataset_id="default", + document_id="doc-test", + revision_id="rev-test1234", + mutation_ids=["mut-1", "mut-2"], + timeout_seconds=5.0, + poll_interval_seconds=0.01, + ) + msg = str(exc_info.value) + assert "mut-1" in msg + assert "DEAD_LETTER" in msg + assert "Tier3Unavailable" in msg + finally: + await client.aclose() + + +def test_mesa_sdk_delete_by_source(fake_server): + url, logs = fake_server + source = "file://sozlesme.txt" + expected_doc_id = f"doc-{hashlib.sha256(source.encode('utf-8')).hexdigest()[:32]}" + with Mesa(base_url=url, api_key="test-key", dataset_id="legal") as mesa: + res = mesa.delete(source=source) + assert res["status"] == "purged" + assert any( + log["method"] == "DELETE" and expected_doc_id in log["path"] + for log in logs + ) + + +def test_mesa_sdk_delete_validation(fake_server): + url, logs = fake_server + with Mesa(base_url=url, api_key="test-key", dataset_id="legal") as mesa: + # Neither document_id nor source + with pytest.raises((MesaClientError, ValueError)): + mesa.delete() + + # Both document_id and source + with pytest.raises((MesaClientError, ValueError)): + mesa.delete(document_id="doc-123", source="file://sozlesme.txt") + + + diff --git a/tests/test_p403_local_cli.py b/tests/test_p403_local_cli.py index 31deb4f..a4a260e 100644 --- a/tests/test_p403_local_cli.py +++ b/tests/test_p403_local_cli.py @@ -287,6 +287,11 @@ def test_codex_cli_parser_routes_to_local_commands(): assert args_delete.group == "delete" assert args_delete.document_id == "doc-123" + args_delete_source = parser.parse_args(["delete", "--source", "file://doc.txt"]) + assert args_delete_source.group == "delete" + assert args_delete_source.document_id is None + assert args_delete_source.source == "file://doc.txt" + args_status = parser.parse_args(["status", "--json"]) assert args_status.group == "status" assert args_status.json is True @@ -385,3 +390,77 @@ def test_cli_add_prints_server_conflict_detail_without_traceback(tmp_path, capsy assert exit_info.value.code == 1 _, err = capsys.readouterr() assert err == f"Error 409: {detail}\n" + + +def test_cli_add_prints_mutation_failure_without_traceback(tmp_path, capsys): + from mesa_client.client import MesaClientError + from mesa_mcp.local_cli import handle_local_command + + msg = "Mutation mut-1 entered terminal state DEAD_LETTER with error class Tier3Unavailable" + document = tmp_path / "a.txt" + document.write_text("A") + mock_mesa = MagicMock() + mock_mesa.add.side_effect = MesaClientError(msg) + with patch("mesa_client.Mesa", return_value=mock_mesa): + with pytest.raises(SystemExit) as exit_info: + handle_local_command( + argparse.Namespace( + group="add", + file=str(document), + source="file://a.txt", + title=None, + json=False, + ) + ) + assert exit_info.value.code == 1 + _, err = capsys.readouterr() + assert err == f"Error: {msg}\n" + + +def test_cli_delete_by_source(capsys): + from mesa_mcp import codex_cli + from mesa_mcp.local_cli import delete_command + + mock_mesa = MagicMock() + mock_mesa.delete.return_value = {"status": "purged", "document_id": "doc-derived-123"} + with patch("mesa_client.Mesa", return_value=mock_mesa): + parser = codex_cli._parser() + args = parser.parse_args(["delete", "--source", "file://doc.txt"]) + delete_command(args) + out, _ = capsys.readouterr() + mock_mesa.delete.assert_called_once_with(document_id=None, source="file://doc.txt") + assert "Purged document doc-derived-123" in out + + +def test_cli_delete_requires_exactly_one_of_id_or_source(capsys): + from mesa_mcp.local_cli import handle_local_command + + # 1. Neither + with pytest.raises(SystemExit) as exc_info: + handle_local_command( + argparse.Namespace( + group="delete", + document_id=None, + source=None, + json=False, + ) + ) + assert exc_info.value.code == 1 + _, err = capsys.readouterr() + assert "Exactly one" in err + + # 2. Both + with pytest.raises(SystemExit) as exc_info: + handle_local_command( + argparse.Namespace( + group="delete", + document_id="doc-1", + source="file://doc.txt", + json=False, + ) + ) + assert exc_info.value.code == 1 + _, err = capsys.readouterr() + assert "Exactly one" in err + + diff --git a/tests/test_p702_search_scope_cache.py b/tests/test_p702_search_scope_cache.py index c83fe36..dfe39bf 100644 --- a/tests/test_p702_search_scope_cache.py +++ b/tests/test_p702_search_scope_cache.py @@ -78,9 +78,7 @@ async def scope_env(tmp_path, monkeypatch): sql, vector, graph, dao = await _environment(tmp_path) await _seed(dao, 1) await _seed(dao, 2) - monkeypatch.setenv("MESA_SEARCH_SCOPE_CACHE", "false") - uncached = MemoryDAO(sql, vector, graph) - monkeypatch.setenv("MESA_SEARCH_SCOPE_CACHE", "true") + uncached = MemoryDAO(sql, vector, graph, search_scope_cache_enabled=False) try: yield sql, dao, uncached finally: @@ -499,3 +497,11 @@ async def test_active_vector_graph_and_query_options_remain_identical(scope_env) assert any( "graph" in row["retrieval_provenance"]["origins"] for row in result ) + + +def test_dao_search_scope_cache_enabled_parameter(): + dao_enabled = MemoryDAO(None, None, None, search_scope_cache_enabled=True) # type: ignore[arg-type] + assert dao_enabled._search_scope_cache_enabled is True + dao_disabled = MemoryDAO(None, None, None, search_scope_cache_enabled=False) # type: ignore[arg-type] + assert dao_disabled._search_scope_cache_enabled is False + diff --git a/tests/test_p801_article_chunking.py b/tests/test_p801_article_chunking.py new file mode 100644 index 0000000..48e718d --- /dev/null +++ b/tests/test_p801_article_chunking.py @@ -0,0 +1,206 @@ +"""Article boundaries must preserve text and durable upload plans.""" + +import json +import os +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from mesa_memory.adapter.tokenizer import count_tokens +from mesa_memory.config import MesaConfig +from mesa_memory.document_chunking import chunk_document +from mesa_storage.dao import QueueUnavailableError +from tests.test_document_ingestion import _add, _payload, _rows +from tests.test_model_optional_runtime import _consume +from tests.test_model_optional_runtime import model_runtime as model_runtime + + +def _legal_text(): + return "Önsöz.\n" + "".join( + f"MADDE {i}- " + "Kiracı bedeli zamanında öder. " * 130 + "\n" + for i in range(1, 5) + ) + + +@pytest.mark.parametrize("separator", ["-", "–", "—", ":"]) +def test_article_heading_variants_and_long_articles(separator): + from mesa_memory.document_chunking import chunk_legal_document + + text = ( + "Önsöz.\r\n" + f"MADDE 12{separator} Kural.\r\n" + f" Madde 13 {separator} " + "Uzun hüküm. " * 400 + "\r\n" + f"GEÇİCİ MADDE 3{separator} Geçiş.\r\n" + f"Ek Madde 1{separator} Ek hüküm.\r\n" + ) + chunks = chunk_legal_document(text) + assert "".join(chunks) == text + assert all(chunk and count_tokens(chunk, "openai") <= 512 for chunk in chunks) + assert chunks[0] == "Önsöz.\r\n" + assert any(c.startswith(" Madde 13") for c in chunks) + assert any(c.startswith("GEÇİCİ MADDE 3") for c in chunks) + assert chunks[-1].startswith("Ek Madde 1") + + +@pytest.mark.parametrize("text", [ + "", "Karar. " * 800, + "MADDE 1- Kural.\nMADDE 2- Kural.\n" + "Karar. " * 800, + "madde kapsamına giren karar.\n" * 500, + "Aktarılan MADDE 1- Kural. MADDE 2- Kural. MADDE 3- Kural. " * 80, +]) +def test_non_legislation_and_fewer_than_three_headings_keep_plain_chunks(text): + from mesa_memory.document_chunking import chunk_legal_document + + assert chunk_legal_document(text) == chunk_document(text) + + +def test_article_at_start_has_no_empty_preamble_chunk(): + from mesa_memory.document_chunking import chunk_legal_document + + pieces = [f"Madde {i} – Kural.\n" for i in range(1, 4)] + assert chunk_legal_document("".join(pieces)) == pieces + + +def test_decision_quoting_many_articles_keeps_plain_chunks(): + from mesa_memory.document_chunking import chunk_legal_document + + text = 'Hukuk Genel Kurulu 2023/858 E., 2023/1399 K.\n\n"İçtihat Metni"\n' + _legal_text() + assert chunk_legal_document(text) == chunk_document(text) + + +def test_chunking_config(monkeypatch): + monkeypatch.delenv("MESA_CHUNKING", raising=False) + assert MesaConfig(_env_file=None).chunking == "auto" + monkeypatch.setenv("MESA_CHUNKING", "plain") + assert MesaConfig(_env_file=None).chunking == "plain" + monkeypatch.setenv("MESA_CHUNKING", "unknown") + with pytest.raises(ValidationError): + MesaConfig(_env_file=None) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("initial,resumed", [("plain", "auto"), ("auto", "plain")]) +async def test_pending_revision_keeps_admitted_plan_after_restart( + model_runtime, monkeypatch, initial, resumed +): + from mesa_memory.document_chunking import chunk_legal_document + + text = _legal_text() + expected = chunk_document(text) if initial == "plain" else chunk_legal_document(text) + assert chunk_document(text) != chunk_legal_document(text) + async with model_runtime(MESA_CHUNKING=initial) as env: + original = type(env.dao).admit_v4_memory + calls = 0 + + async def interrupt(self, **kwargs): + nonlocal calls + calls += 1 + if calls == 2: + raise QueueUnavailableError("injected interruption") + return await original(self, **kwargs) + + monkeypatch.setattr(type(env.dao), "admit_v4_memory", interrupt) + failed = await env.client.post("/v4/documents", json=_payload(env, text)) + assert failed.status_code == 503, failed.text + first = await _rows(env, "memory_mutations") + assert len(first) == 1 + assert first[0]["content_payload"] == expected[0] + await _consume(env, 3) + monkeypatch.setattr(type(env.dao), "admit_v4_memory", original) + await env.restart(MESA_CHUNKING=resumed) + result = await _add(env, text) + stored = sorted(await _rows(env, "source_chunks"), key=lambda r: r["ordinal"]) + assert [c["content_payload"] for c in stored] == expected + assert result["chunk_count"] == len(expected) + assert result["mutation_ids"][0] == first[0]["mutation_id"] + await _consume(env, len(expected) + 2) + assert (await _rows(env, "document_revisions"))[0]["status"] == "ACTIVE" + + +@pytest.mark.asyncio +async def test_unknown_admitted_plan_fails_without_appending(model_runtime, monkeypatch): + async with model_runtime() as env: + text = _legal_text() + original = type(env.dao).admit_v4_memory + calls = 0 + + async def interrupt(self, **kwargs): + nonlocal calls + calls += 1 + if calls == 2: + raise QueueUnavailableError("injected interruption") + return await original(self, **kwargs) + + monkeypatch.setattr(type(env.dao), "admit_v4_memory", interrupt) + failed = await env.client.post("/v4/documents", json=_payload(env, text)) + assert failed.status_code == 503 + async with env.dao._sql.transaction() as db: + await db.execute("UPDATE memory_mutations SET content_payload = 'unknown plan'") + await db.commit() + before = await _rows(env, "memory_mutations") + monkeypatch.setattr(type(env.dao), "admit_v4_memory", original) + response = await env.client.post("/v4/documents", json=_payload(env, text)) + assert response.status_code == 409, response.text + assert await _rows(env, "memory_mutations") == before + + +@pytest.mark.asyncio +async def test_ambiguous_admitted_prefix_never_silently_changes_plan(model_runtime, monkeypatch): + from mesa_memory.document_chunking import chunk_legal_document + + # A blank-line preamble can be identical even when the later plans differ. + text = _legal_text().replace("Önsöz.\n", "Önsöz.\n\n", 1) + assert chunk_document(text)[0] == chunk_legal_document(text)[0] + assert chunk_document(text) != chunk_legal_document(text) + async with model_runtime(MESA_CHUNKING="plain") as env: + original = type(env.dao).admit_v4_memory + calls = 0 + + async def interrupt(self, **kwargs): + nonlocal calls + calls += 1 + if calls == 2: + raise QueueUnavailableError("injected interruption") + return await original(self, **kwargs) + + monkeypatch.setattr(type(env.dao), "admit_v4_memory", interrupt) + response = await env.client.post("/v4/documents", json=_payload(env, text)) + assert response.status_code == 503 + before = await _rows(env, "memory_mutations") + monkeypatch.setattr(type(env.dao), "admit_v4_memory", original) + await env.restart(MESA_CHUNKING="auto") + response = await env.client.post("/v4/documents", json=_payload(env, text)) + assert response.status_code == 409 + assert "ambiguous" in response.json()["detail"] + assert await _rows(env, "memory_mutations") == before + + +@pytest.mark.skipif(not os.environ.get("MESA_BENCH_ROOT"), reason="frozen corpus is external") +def test_frozen_tier_m_article_boundaries_and_lossless_slices(): + from mesa_memory.document_chunking import chunk_legal_document + + root = Path(os.environ["MESA_BENCH_ROOT"]) + documents = [ + d for line in (root / "documents.jsonl").read_text().splitlines() + if "M" in (d := json.loads(line))["tiers"] + ] + starts = {} + for document in documents: + text = (root / "corpus" / document["file"]).read_text() + chunks = chunk_legal_document(text) + assert "".join(chunks) == text, document["file"] + assert all(c and count_tokens(c, "openai") <= 512 for c in chunks) + if document["type"] not in {"legislation", "regulation", "communique"}: + assert chunks == chunk_document(text), document["file"] + offset = 0 + starts[document["file"]] = set() + for chunk in chunks: + starts[document["file"]].add(offset) + offset += len(chunk) + articles = [ + a for line in (root / "articles.jsonl").read_text().splitlines() + if (a := json.loads(line))["file"] in starts + ] + matched = sum(a["char_start"] in starts[a["file"]] for a in articles) + assert matched / len(articles) >= 0.95 diff --git a/tests/test_p802_reranker.py b/tests/test_p802_reranker.py new file mode 100644 index 0000000..6a94e31 --- /dev/null +++ b/tests/test_p802_reranker.py @@ -0,0 +1,144 @@ +"""Reranking runs on authorized chunk text after releasing SQLite.""" + +import asyncio +import importlib.util +import sys +import threading +from types import SimpleNamespace + +import pytest +from pydantic import ValidationError + +from mesa_memory.config import MesaConfig +from mesa_storage.dao import MemoryDAO +from tests.test_model_optional_runtime import model_runtime as model_runtime +from tests.test_p702_search_scope_cache import _search, _seed +from tests.test_v4_bm25_scope_isolation import _environment + + +def test_rerank_config_defaults_and_validation(monkeypatch): + for name in ("MESA_RERANK_ENABLED", "MESA_RERANK_MODEL", "MESA_RERANK_DEPTH"): + monkeypatch.delenv(name, raising=False) + cfg = MesaConfig(_env_file=None) + assert cfg.rerank_enabled is False + assert cfg.rerank_model == "BAAI/bge-reranker-v2-m3" + assert cfg.rerank_depth == 50 + monkeypatch.setenv("MESA_RERANK_DEPTH", "0") + with pytest.raises(ValidationError): + MesaConfig(_env_file=None) + + +def test_model_is_lazy_loaded_once_with_fake_cross_encoder(monkeypatch): + from mesa_memory.reranking import CrossEncoderReranker + + loaded = [] + + class FakeModel: + def __init__(self, name, **kwargs): + loaded.append((name, kwargs)) + + def predict(self, pairs, **kwargs): + assert pairs == [("query", "chunk")] + return [0.75] + + monkeypatch.setattr(importlib.util, "find_spec", lambda name: object()) + monkeypatch.setitem(sys.modules, "sentence_transformers", SimpleNamespace(CrossEncoder=FakeModel)) + scorer = CrossEncoderReranker("fake") + assert not loaded + assert scorer.score("query", ["chunk"]) == [0.75] + assert scorer.score("query", ["chunk"]) == [0.75] + assert loaded == [("fake", {"max_length": 1024})] + + +def test_missing_ml_extra_fails_before_search(monkeypatch): + from mesa_memory.reranking import CrossEncoderReranker + + monkeypatch.setattr(importlib.util, "find_spec", lambda name: None) + with pytest.raises(RuntimeError, match=r'pip install "mesa-memory\[ml\]"'): + CrossEncoderReranker("fake") + + +@pytest.mark.asyncio +async def test_missing_ml_extra_stops_runtime_startup(model_runtime, monkeypatch): + from mesa_memory.api import server + + # A previous lifespan may leave completed tasks tied to its closed loop. + closed_loop = asyncio.new_event_loop() + stale_task = closed_loop.create_future() + stale_task.cancel() + closed_loop.close() + monkeypatch.setattr(server.state, "background_tasks", {stale_task}, raising=False) + original = importlib.util.find_spec + monkeypatch.setattr(importlib.util, "find_spec", lambda name: ( + None if name == "sentence_transformers" else original(name) + )) + with pytest.raises(RuntimeError, match=r'pip install "mesa-memory\[ml\]"'): + async with model_runtime(MESA_RERANK_ENABLED=True): + pytest.fail("runtime must not start without the selected optional package") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("scores", [[float("nan")], [], [float("inf")]]) +async def test_invalid_model_scores_fail_instead_of_returning_partial_ranking(scores): + dao = MemoryDAO(None, None, reranker=SimpleNamespace(score=lambda *_: scores)) + with pytest.raises(RuntimeError, match="invalid scores"): + await dao._rerank_results("query", [{"_rerank_text": "passage"}], 1) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("cache", [False, True]) +async def test_rerank_depth_full_chunk_text_stable_ties_and_released_connection(tmp_path, cache): + sql, vector, graph, plain = await _environment(tmp_path) + event_thread = threading.get_ident() + try: + for number in range(1, 5): + await _seed(plain, number) + # Foreign evidence must never be sent to the model. + await _seed(plain, 5, tenant="other") + fused = await _search(plain, limit=4) + assert len(fused) == 4 + seen = [] + + def score(query, texts): + assert threading.get_ident() != event_thread + assert not sql._search_lock.locked() + assert not sql._search_db or not sql._search_db.in_transaction + assert query == "kira bedeli" + assert len(texts) == 4 + assert all("Article 12 privacy compliance." in text for text in texts) + assert not any(text.endswith(" 5") for text in texts) + seen.extend(texts) + return [0.1, 0.9, 0.9, 0.2] + + reranked = MemoryDAO(sql, vector, graph, search_scope_cache_enabled=cache, + reranker=SimpleNamespace(score=score), rerank_depth=4) + result = await _search(reranked, limit=2) + assert [r["assertion_id"] for r in result] == [r["assertion_id"] for r in fused[1:3]] + for result_row, fused_row in zip(result, fused[1:3]): + assert result_row == {**fused_row, "rerank_score": 0.9} + assert seen + # A score call in a worker does not stall the event loop. + await asyncio.sleep(0) + finally: + await sql.close() + + +@pytest.mark.asyncio +async def test_disabled_reranker_keeps_complete_results_and_never_imports_ml(tmp_path, monkeypatch): + import builtins + + sql, vector, graph, dao = await _environment(tmp_path) + original = builtins.__import__ + + def guard(name, *args, **kwargs): + assert not name.startswith("sentence_transformers") + return original(name, *args, **kwargs) + + try: + await _seed(dao, 1) + expected = await _search(dao) + monkeypatch.setattr(builtins, "__import__", guard) + disabled = MemoryDAO(sql, vector, graph, reranker=None) + assert await _search(disabled) == expected + finally: + await sql.close() diff --git a/tests/test_p803_search_embedding_concurrency.py b/tests/test_p803_search_embedding_concurrency.py new file mode 100644 index 0000000..d2d334c --- /dev/null +++ b/tests/test_p803_search_embedding_concurrency.py @@ -0,0 +1,74 @@ +"""Query providers must overlap without holding the shared search connection.""" + +import asyncio +import time + +import pytest + +from mesa_storage.dao import MemoryDAO +from tests.test_p702_search_scope_cache import _search, _seed +from tests.test_v4_bm25_scope_isolation import _environment +from tests.test_v4_retrieval_hardening_phase1 import TEST_EMBEDDING_IDENTITY + + +@pytest.mark.asyncio +@pytest.mark.parametrize("cache", [True, False]) +async def test_eight_slow_query_embeddings_overlap_and_preserve_results(tmp_path, cache): + sql, vector, graph, _dao = await _environment(tmp_path) + vector.embedding_identity = TEST_EMBEDDING_IDENTITY + dao = MemoryDAO(sql, vector, graph, search_scope_cache_enabled=cache) + held = [] + + async def slow_embedding(query): + held.append(sql._search_lock.locked()) + await asyncio.sleep(0.3) + return [1.0, 0.0] + + async def vector_search(query_vector, *, allowed_node_ids, **kwargs): + assert query_vector == [1.0, 0.0] + return [ + {"node_id": node, "_distance": i / 10} + for i, node in enumerate(sorted(allowed_node_ids)) + ] + + vector.compute_query_embedding = slow_embedding + vector.search = vector_search + try: + await _seed(dao, 1) + await _seed(dao, 2) + await _seed(dao, 3, tenant="other") + queries = [(f"kira bedeli {i}", "t" if i % 2 == 0 else "other") for i in range(8)] + + async def search(query, tenant): + return await _search(dao, query, tenant_id=tenant, graph_enabled=False) + + expected = [await search(*query) for query in queries] + assert all(expected) + held.clear() + start = time.perf_counter() + results = await asyncio.gather(*(search(*query) for query in queries)) + elapsed = time.perf_counter() - start + print(f"eight searches cache={cache}: {elapsed:.6f}s; provider lock states={held}") + assert results == expected + assert len(held) == 8 + assert elapsed < 1.2 + assert not any(held) + finally: + await sql.close() + + +@pytest.mark.asyncio +async def test_incompatible_vectors_keep_bm25_when_query_provider_is_unavailable(tmp_path): + sql, vector, graph, dao = await _environment(tmp_path) + try: + # This fixture has vector artifacts but no compatible runtime identity. + await _seed(dao, 1) + expected = await _search(dao) + + async def unavailable(query): + raise RuntimeError("provider unavailable") + + vector.compute_query_embedding = unavailable + assert await _search(dao) == expected + finally: + await sql.close() diff --git a/tests/test_p804_dev_defaults.py b/tests/test_p804_dev_defaults.py new file mode 100644 index 0000000..7b39b71 --- /dev/null +++ b/tests/test_p804_dev_defaults.py @@ -0,0 +1,34 @@ +"""The runtime default must follow the recorded DEV-only selection rule.""" + +import json +from pathlib import Path + +from mesa_memory.config import MesaConfig + + +def test_recorded_dev_matrix_covers_all_requested_weights_and_modes(): + data = json.loads((Path(__file__).parents[1] / "docs/benchmarks/round14_dev.json").read_text()) + assert data["split"] == "dev" + assert data["other_lane_weights"] == {"bm25": 1, "assertion": 1, "graph": 1} + assert (data["document_count"], data["chunk_count"]) == (141, 5041) + assert {(r["vector_weight"], r["rerank_enabled"]) for r in data["runs"]} == { + (weight, mode) for weight in (1, 2, 4, 8) for mode in (False, True) + } + assert len(data["runs"]) == 8 + for run in data["runs"]: + assert run["metrics"]["count_total"] == 101 + assert run["metrics"]["count_answerable"] == 85 + assert 0 <= run["passage_mrr_unrounded"] <= 1 + assert round(run["passage_mrr_unrounded"], 4) == run["metrics"]["passage_mrr"] + + +def test_default_uses_best_disabled_dev_mrr_only_with_required_gain(monkeypatch): + data = json.loads((Path(__file__).parents[1] / "docs/benchmarks/round14_dev.json").read_text()) + disabled = [r for r in data["runs"] if not r["rerank_enabled"]] + baseline = next(r for r in disabled if r["vector_weight"] == 1) + best = max(disabled, key=lambda r: (r["passage_mrr_unrounded"], -r["vector_weight"])) + gain = best["passage_mrr_unrounded"] - baseline["passage_mrr_unrounded"] + expected = best["vector_weight"] if gain >= 0.02 else 1 + monkeypatch.delenv("MESA_RRF_VECTOR_WEIGHT", raising=False) + assert MesaConfig(_env_file=None).rrf_vector_weight == expected + assert data["selected_default_vector_weight"] == expected diff --git a/tests/test_retrieval_scope_contract.py b/tests/test_retrieval_scope_contract.py index d334e5f..cb0679a 100644 --- a/tests/test_retrieval_scope_contract.py +++ b/tests/test_retrieval_scope_contract.py @@ -230,3 +230,27 @@ def test_mesaconfig_accepts_zero_rrf_lane_weight() -> None: assert [c[0] for c in fused_zero_vector] == [c[0] for c in fused_only_bm25] +def test_mesaconfig_search_scope_cache_defaults_and_env(monkeypatch) -> None: + from mesa_memory.config import MesaConfig + + # 1. Default is True + monkeypatch.delenv("MESA_SEARCH_SCOPE_CACHE", raising=False) + cfg = MesaConfig() + assert cfg.search_scope_cache is True + + # 2. Env variable 'false' + monkeypatch.setenv("MESA_SEARCH_SCOPE_CACHE", "false") + cfg_off = MesaConfig() + assert cfg_off.search_scope_cache is False + + # 3. Env variable '0' + monkeypatch.setenv("MESA_SEARCH_SCOPE_CACHE", "0") + cfg_zero = MesaConfig() + assert cfg_zero.search_scope_cache is False + + # 4. Explicit kwarg + cfg_explicit = MesaConfig(search_scope_cache=False) + assert cfg_explicit.search_scope_cache is False + + + diff --git a/uv.lock b/uv.lock index cb35912..3981378 100644 --- a/uv.lock +++ b/uv.lock @@ -2176,6 +2176,8 @@ dependencies = [ { name = "lancedb" }, { name = "mcp" }, { name = "numpy" }, + { name = "ollama" }, + { name = "openai" }, { name = "prometheus-client" }, { name = "psutil" }, { name = "pyarrow" }, @@ -2195,8 +2197,6 @@ dependencies = [ [package.optional-dependencies] adapters = [ { name = "anthropic" }, - { name = "ollama" }, - { name = "openai" }, ] dev = [ { name = "black" }, @@ -2243,8 +2243,8 @@ requires-dist = [ { name = "mcp", specifier = ">=1.30.0,<2.0.0" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.10.0" }, { name = "numpy", specifier = ">=2.2.0" }, - { name = "ollama", marker = "extra == 'adapters'", specifier = ">=0.6.1" }, - { name = "openai", marker = "extra == 'adapters'", specifier = ">=2.36.0" }, + { name = "ollama", specifier = ">=0.6.1" }, + { name = "openai", specifier = ">=2.36.0" }, { name = "prometheus-client", specifier = ">=0.20.0" }, { name = "psutil", specifier = ">=7.0.0" }, { name = "pyarrow", specifier = ">=24.0.0" },