diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0f8c71b..8b60332 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,14 +34,16 @@ jobs: if: runner.os == 'Linux' run: >- uvx --python 3.13 --from pytest==9.1.1 - --with pyyaml==6.0.3 --with sqlite-vec==0.1.9 pytest -q + --with pyyaml==6.0.3 --with sqlite-vec==0.1.9 + --with fastembed==0.8.0 pytest -q - name: Run macOS portable tests if: runner.os == 'macOS' run: >- uvx --python 3.13 --from pytest==9.1.1 --with pyyaml==6.0.3 --with sqlite-vec==0.1.9 - pytest -q -k "not test_index_rebuild_installs_and_loads_sqlite_vec_when_available" + --with fastembed==0.8.0 + pytest -q -k "not test_production_index_rebuild_keeps_proposed_notes_in_vector_discovery" - name: Validate skill packages run: bin/work-bundle-skill validate diff --git a/README.md b/README.md index 2903dd7..475d8bd 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,12 @@ bin/install-work-bundle-skills --dry-run Run the deterministic repository gate with isolated dependencies: ```bash -uvx --python 3.13 --from pytest==9.1.1 --with pyyaml==6.0.3 --with sqlite-vec==0.1.9 pytest -q +uvx --python 3.13 --from pytest==9.1.1 --with pyyaml==6.0.3 --with sqlite-vec==0.1.9 --with fastembed==0.8.0 pytest -q bin/work-bundle-skill validate ``` + +Run the keep-summarizing CLI through its pinned uv-managed environment: + +```bash +uv run scripts/ks.py --help +``` diff --git a/scripts/keep-summarizing/indexes.py b/scripts/keep-summarizing/indexes.py index 7b46309..9f03686 100644 --- a/scripts/keep-summarizing/indexes.py +++ b/scripts/keep-summarizing/indexes.py @@ -1,41 +1,24 @@ from core import * -VECTOR_DIMENSIONS = 64 +from functools import lru_cache +from importlib.metadata import PackageNotFoundError, version + +VECTOR_DIMENSIONS = 384 SQLITE_VEC_PACKAGE = "sqlite-vec" SQLITE_VEC_IMPORT = "sqlite_vec" +EMBEDDING_PACKAGE = "fastembed" +EMBEDDING_PACKAGE_VERSION = "0.8.0" +EMBEDDING_MODEL = "BAAI/bge-small-en-v1.5" +EMBEDDING_MODEL_VERSION = "v1.5" +VECTOR_INDEX_SCHEMA = "knowledge-chunk-vec-v2" +VECTOR_CHUNKING = "document-body-v1" def install_sqlite_vec() -> tuple[object | None, str | None]: try: return __import__(SQLITE_VEC_IMPORT), None - except ImportError: - pass - - commands = [ - [sys.executable, "-m", "pip", "install", "--disable-pip-version-check", SQLITE_VEC_PACKAGE], - [ - sys.executable, - "-m", - "pip", - "install", - "--disable-pip-version-check", - "--index-url", - "https://pypi.org/simple", - SQLITE_VEC_PACKAGE, - ], - ] - errors: list[str] = [] - for command in commands: - result = subprocess.run(command, capture_output=True, text=True, check=False) - if result.returncode == 0: - try: - return __import__(SQLITE_VEC_IMPORT), None - except ImportError as exc: - errors.append(f"{command!r}: installed but import failed: {exc}") - else: - detail = (result.stderr or result.stdout).strip() - errors.append(f"{command!r}: {detail}") - return None, "sqlite-vec install failed: " + " | ".join(errors) + except ImportError as exc: + return None, f"sqlite-vec unavailable in the uv-managed environment: {exc}" def load_sqlite_vec(conn: sqlite3.Connection) -> tuple[object | None, str | None]: @@ -55,18 +38,49 @@ def load_sqlite_vec(conn: sqlite3.Connection) -> tuple[object | None, str | None pass -def local_text_vector(text: str) -> list[float]: - vector = [0.0] * VECTOR_DIMENSIONS - terms = [term for term in re.split(r"\W+", text.lower()) if term] - for term in terms: - digest = hashlib.sha256(term.encode("utf-8")).digest() - index = int.from_bytes(digest[:2], "big") % VECTOR_DIMENSIONS - sign = 1.0 if digest[2] % 2 == 0 else -1.0 - vector[index] += sign - magnitude = sum(value * value for value in vector) ** 0.5 - if not magnitude: - return vector - return [value / magnitude for value in vector] +@lru_cache(maxsize=1) +def embedding_model() -> object: + from fastembed import TextEmbedding + + return TextEmbedding(model_name=EMBEDDING_MODEL) + + +def embedding_backend_status() -> tuple[object | None, str | None]: + try: + installed_version = version(EMBEDDING_PACKAGE) + if installed_version != EMBEDDING_PACKAGE_VERSION: + return None, ( + f"FastEmbed version mismatch: expected {EMBEDDING_PACKAGE_VERSION}, " + f"got {installed_version}" + ) + return embedding_model(), None + except (ImportError, OSError, RuntimeError, ValueError, PackageNotFoundError) as exc: + return None, f"FastEmbed model unavailable in the uv-managed environment: {exc}" + + +def local_text_vector(text: str, *, query: bool = False) -> list[float]: + model, error = embedding_backend_status() + if model is None: + raise RuntimeError(error or "FastEmbed model unavailable") + embed = model.query_embed if query else model.passage_embed + vector = list(next(iter(embed([text])))) + if len(vector) != VECTOR_DIMENSIONS: + raise RuntimeError( + f"FastEmbed model dimension mismatch: expected {VECTOR_DIMENSIONS}, got {len(vector)}" + ) + return [float(value) for value in vector] + + +def expected_vector_metadata() -> dict[str, object]: + return { + "embedding_model": EMBEDDING_MODEL, + "embedding_model_version": EMBEDDING_MODEL_VERSION, + "embedding_package": EMBEDDING_PACKAGE, + "embedding_package_version": EMBEDDING_PACKAGE_VERSION, + "dimensions": VECTOR_DIMENSIONS, + "chunking": VECTOR_CHUNKING, + "index_schema": VECTOR_INDEX_SCHEMA, + } def markdown_files(root: Path) -> list[Path]: candidates = list((root / "notes").glob("**/*.md")) + list((root / "context-packs").glob("*.md")) @@ -206,7 +220,7 @@ def build_vector_index_status(root: Path, chunks: list[dict[str, object]], proje "project": project, "artifact": VECTOR_INDEX_ARTIFACT_FILE, "chunks_considered": len(chunks), - "backend": "sqlite-local-vector", + "backend": "sqlite-vec", } if not install_missing: @@ -234,6 +248,20 @@ def build_vector_index_status(root: Path, chunks: list[dict[str, object]], proje } artifact_path.write_text("", encoding="utf-8") else: + model, model_error = embedding_backend_status() + if model is None: + status = { + **base_status, + "status": "unavailable", + "chunks_indexed": 0, + "reason": model_error or "FastEmbed model unavailable", + "fallback": "sqlite_fts", + } + artifact_path.write_text("", encoding="utf-8") + (indexes / VECTOR_INDEX_STATUS_FILE).write_text( + json.dumps(status, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" + ) + return status conn.execute("DROP TABLE IF EXISTS knowledge_chunk_vec") conn.execute( f""" @@ -269,6 +297,7 @@ def build_vector_index_status(root: Path, chunks: list[dict[str, object]], proje "path": chunk["path"], "backend": "sqlite-vec", "dimensions": VECTOR_DIMENSIONS, + "embedding_model": EMBEDDING_MODEL, } ) conn.commit() @@ -283,8 +312,8 @@ def build_vector_index_status(root: Path, chunks: list[dict[str, object]], proje "chunks_indexed": len(rows), "extension": "sqlite-vec", "extension_version": version, - "dimensions": VECTOR_DIMENSIONS, "table": "knowledge_chunk_vec", + **expected_vector_metadata(), } finally: conn.close() @@ -312,7 +341,9 @@ def cmd_index(args: argparse.Namespace) -> None: continue status = str(fm.get("status", "draft")) sensitivity = str(fm.get("sensitivity", "normal")) - include = status not in config["exclude_status"] and sensitivity not in config["exclude_sensitivity"] + # Discovery is authority-neutral: lifecycle status is evaluated only after + # retrieval, while sensitivity remains a legitimate indexing boundary. + include = sensitivity not in config["exclude_sensitivity"] doc = { "id": fm.get("id", rel), "path": rel, diff --git a/scripts/keep-summarizing/query.py b/scripts/keep-summarizing/query.py index ffc591b..af861ca 100644 --- a/scripts/keep-summarizing/query.py +++ b/scripts/keep-summarizing/query.py @@ -1,5 +1,14 @@ from core import * -from indexes import cmd_index +from indexes import ( + cmd_index, + expected_vector_metadata, + load_sqlite_vec, + local_text_vector, +) + + +RRF_K = 60 +MAX_VECTOR_DISTANCE = 1.0 def fts_literal_query(query: str) -> str: @@ -30,7 +39,40 @@ def vector_index_status(root: Path) -> dict[str, object]: return {"status": "failed", "reason": "invalid vector index status", "fallback": "sqlite_fts"} -def candidate_record(row: sqlite3.Row, anchors: list[str], policy_hint: str | None, fusion_rank: int) -> dict[str, object]: +def vector_compatibility(status: dict[str, object]) -> tuple[str, str | None]: + state = str(status.get("status", "unavailable")) + if state != "rebuilt": + return state if state in {"unavailable", "failed"} else "unavailable", str( + status.get("reason") or "vector index is not rebuilt" + ) + expected = expected_vector_metadata() + mismatches = [ + key + for key in ( + "embedding_model", + "embedding_model_version", + "embedding_package", + "embedding_package_version", + "dimensions", + "chunking", + "index_schema", + ) + if status.get(key) != expected[key] + ] + if mismatches: + return "failed", f"vector index requires rebuild: incompatible {', '.join(mismatches)}" + return "rebuilt", None + + +def candidate_record( + row: sqlite3.Row | dict[str, object], + anchors: list[str], + policy_hint: str | None, + fusion_rank: int, + *, + from_fts: bool, + from_vector: bool, +) -> dict[str, object]: result = dict(row) tags = result.get("tags", "[]") if isinstance(tags, str): @@ -50,13 +92,13 @@ def candidate_record(row: sqlite3.Row, anchors: list[str], policy_hint: str | No "summary": result.get("summary", ""), "tags": tags if isinstance(tags, list) else [], "mechanical_sources": { - "fts": True, - "vector": False, + "fts": from_fts, + "vector": from_vector, "bfs": False, }, "mechanical_scores": { - "fts_rank": result.get("rank"), - "vector_distance": None, + "fts_rank": result.get("rank") if from_fts else None, + "vector_distance": result.get("vector_distance") if from_vector else None, "fusion_rank": fusion_rank, "bfs_depth": None, }, @@ -69,6 +111,36 @@ def candidate_record(row: sqlite3.Row, anchors: list[str], policy_hint: str | No } +def reciprocal_rank_fusion( + fts_rows: list[sqlite3.Row | dict[str, object]], + vector_rows: list[sqlite3.Row | dict[str, object]], + limit: int, +) -> list[tuple[dict[str, object], bool, bool]]: + merged: dict[str, dict[str, object]] = {} + scores: dict[str, float] = {} + best_rank: dict[str, int] = {} + sources: dict[str, set[str]] = {} + for source, rows in (("fts", fts_rows), ("vector", vector_rows)): + for rank, row in enumerate(rows, start=1): + record = dict(row) + candidate_id = str(record.get("id") or record.get("document_id") or "") + if not candidate_id: + continue + merged.setdefault(candidate_id, record) + if source == "vector": + merged[candidate_id]["vector_distance"] = record.get("vector_distance") + elif merged[candidate_id].get("rank") is None: + merged[candidate_id]["rank"] = record.get("rank") + scores[candidate_id] = scores.get(candidate_id, 0.0) + 1.0 / (RRF_K + rank) + best_rank[candidate_id] = min(best_rank.get(candidate_id, rank), rank) + sources.setdefault(candidate_id, set()).add(source) + ordered = sorted(merged, key=lambda item: (-scores[item], best_rank[item], item))[:limit] + return [ + (merged[item], "fts" in sources[item], "vector" in sources[item]) + for item in ordered + ] + + def cmd_query(args: argparse.Namespace) -> None: root = project_dir(args.project, args) db_path = root / "indexes" / "knowledge.sqlite" @@ -90,23 +162,71 @@ def cmd_query(args: argparse.Namespace) -> None: conn.row_factory = sqlite3.Row try: vector_status = vector_index_status(root) + vector_state, vector_reason = vector_compatibility(vector_status) + vector_rows: list[sqlite3.Row | dict[str, object]] = [] + if vector_state == "rebuilt": + sqlite_vec, load_error = load_sqlite_vec(conn) + if sqlite_vec is None: + vector_state = "unavailable" + vector_reason = load_error or "sqlite-vec unavailable" + else: + try: + query_vector = local_text_vector(args.query, query=True) + vector_sql = """ + SELECT n.*, v.distance AS vector_distance + FROM knowledge_chunk_vec v + JOIN knowledge_note n ON n.id = v.document_id + WHERE v.embedding MATCH ? AND k = ? + ORDER BY v.distance + """ + vector_rows = [ + row + for row in conn.execute( + vector_sql, + [sqlite_vec.serialize_float32(query_vector), max(args.limit * 4, 20)], + ) + if float(dict(row).get("vector_distance", float("inf"))) <= MAX_VECTOR_DISTANCE + ] + vector_state = "queried" + except (ImportError, OSError, RuntimeError, ValueError, sqlite3.Error) as exc: + vector_state = "failed" + vector_reason = f"vector query failed: {exc}" + vector_rows = [] + fts_rows = list(conn.execute(sql, [fts_literal_query(args.query), max(args.limit * 4, 20)])) + trace = { + "policy_hint": policy_hint, + "query_anchors": anchors, + "sources": { + "fts": "queried", + "vector": vector_state, + "bfs": "not_configured", + }, + } + if vector_reason: + trace["source_details"] = {"vector": {"reason": vector_reason}} print( json.dumps( { - "query_trace": { - "policy_hint": policy_hint, - "query_anchors": anchors, - "sources": { - "fts": "queried", - "vector": vector_status.get("status", "unavailable"), - "bfs": "not_configured", - }, - } + "query_trace": trace }, ensure_ascii=False, ) ) - for fusion_rank, row in enumerate(conn.execute(sql, [fts_literal_query(args.query), args.limit]), start=1): - print(json.dumps(candidate_record(row, anchors, policy_hint, fusion_rank), ensure_ascii=False)) + for fusion_rank, (row, from_fts, from_vector) in enumerate( + reciprocal_rank_fusion(fts_rows, vector_rows, args.limit), start=1 + ): + print( + json.dumps( + candidate_record( + row, + anchors, + policy_hint, + fusion_rank, + from_fts=from_fts, + from_vector=from_vector, + ), + ensure_ascii=False, + ) + ) finally: conn.close() diff --git a/scripts/ks.py b/scripts/ks.py index 7e4f0ac..381772a 100755 --- a/scripts/ks.py +++ b/scripts/ks.py @@ -1,4 +1,12 @@ #!/usr/bin/env python3 +# /// script +# requires-python = ">=3.13" +# dependencies = [ +# "pyyaml==6.0.3", +# "sqlite-vec==0.1.9", +# "fastembed==0.8.0", +# ] +# /// """Compatibility entrypoint for keep-summarizing helpers.""" from __future__ import annotations diff --git a/tests/test_keep_summarizing_query.py b/tests/test_keep_summarizing_query.py index c407d91..d7d1c92 100644 --- a/tests/test_keep_summarizing_query.py +++ b/tests/test_keep_summarizing_query.py @@ -54,11 +54,6 @@ def load_keep_summarizing_modules() -> tuple[object, object]: "should_block", "retrieval_role", } -MISSING_HYBRID_RETRIEVAL = pytest.mark.xfail( - strict=True, - reason="WOR-58 production hybrid retrieval is intentionally deferred to a separate slice", -) - HYBRID_CONTRACT_NOTES = [ { "id": "note-exact-api", @@ -212,10 +207,7 @@ def candidate_by_id(candidates: list[dict[str, object]], candidate_id: str) -> d def write_vector_status(root: Path, **overrides: object) -> Path: status = { "status": "rebuilt", - "embedding_model": "fixture-model", - "embedding_model_version": "fixture-v1", - "dimensions": indexes.VECTOR_DIMENSIONS, - "index_schema": "fixture-v1", + **indexes.expected_vector_metadata(), **overrides, } status_path = root / "indexes" / indexes.VECTOR_INDEX_STATUS_FILE @@ -254,6 +246,7 @@ def test_neutral_candidate_discovery_spans_every_lifecycle_without_stage_gate( "policy_hint": None, "query_anchors": ["shared", "discovery", "fixture"], "sources": {"fts": "queried", "vector": "unavailable", "bfs": "not_configured"}, + "source_details": {"vector": {"reason": "missing vector index status"}}, } assert {candidate["lifecycle_stage"] for candidate in candidates} == { lifecycle for lifecycle, _ in LIFECYCLE_FIXTURES @@ -331,7 +324,7 @@ def test_query_trace_reports_vector_unavailable_status( assert trace["sources"]["vector"] == "unavailable" -def test_index_rebuild_installs_and_loads_sqlite_vec_when_available( +def test_production_index_rebuild_keeps_proposed_notes_in_vector_discovery( tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: root = tmp_path / ".work-bundle" / "knowledge" @@ -354,6 +347,49 @@ def test_index_rebuild_installs_and_loads_sqlite_vec_when_available( # Vector Index Vector index fixture body. +""", + encoding="utf-8", + ) + proposed = root / "notes" / "development-design" / "retrieval" / "conceptual-match.md" + proposed.parent.mkdir(parents=True) + proposed.write_text( + """--- +id: note-proposed-paraphrase +title: Conceptual Match Without Shared Wording +lifecycle_stage: development_design +perspective: development-design/retrieval +status: proposed +source_type: source_note +summary: Meaning based recall across vocabulary mismatch. +tags: + - semantic-recall +updated_at: 2026-08-28 +--- + +# Conceptual Match Without Shared Wording + +A reader asks how to locate advice that means the same thing even when none of the original wording is repeated. +""", + encoding="utf-8", + ) + confidential = root / "notes" / "implementation" / "confidential.md" + confidential.parent.mkdir(parents=True) + confidential.write_text( + """--- +id: note-confidential +title: Confidential Fixture +lifecycle_stage: implementation +perspective: implementation/security +status: current +source_type: source_note +sensitivity: confidential +summary: This note must not enter vector discovery. +updated_at: 2026-08-28 +--- + +# Confidential Fixture + +Confidential discovery material. """, encoding="utf-8", ) @@ -372,8 +408,27 @@ def test_index_rebuild_installs_and_loads_sqlite_vec_when_available( vector_status = payload["vector_status"] assert vector_status["status"] == "rebuilt" assert vector_status["extension"] == "sqlite-vec" - assert vector_status["chunks_indexed"] == 1 - assert (root / "indexes" / "vector-index.jsonl").read_text(encoding="utf-8") + assert vector_status["chunks_indexed"] == 2 + assert vector_status["embedding_model"] == indexes.EMBEDDING_MODEL + assert vector_status["embedding_model_version"] == indexes.EMBEDDING_MODEL_VERSION + assert vector_status["embedding_package_version"] == indexes.EMBEDDING_PACKAGE_VERSION + assert vector_status["dimensions"] == indexes.VECTOR_DIMENSIONS + assert vector_status["chunking"] == indexes.VECTOR_CHUNKING + assert vector_status["index_schema"] == indexes.VECTOR_INDEX_SCHEMA + vector_artifact = (root / "indexes" / "vector-index.jsonl").read_text(encoding="utf-8") + assert vector_artifact + assert "note-confidential" not in vector_artifact + + trace, candidates = run_query( + root, + capsys, + query="retrieve semantically similar knowledge using different terms", + limit=4, + ) + proposed_candidate = candidate_by_id(candidates, "note-proposed-paraphrase") + assert vector_trace_status(trace) == "queried" + assert proposed_candidate["status"] == "proposed" + assert proposed_candidate["mechanical_sources"] == {"fts": False, "vector": True, "bfs": False} def test_query_output_omits_forbidden_semantic_fields( @@ -395,7 +450,6 @@ def test_hybrid_retrieval_contract_exact_identifier_keeps_lexical_win( assert candidates[0]["mechanical_sources"]["fts"] is True -@MISSING_HYBRID_RETRIEVAL def test_hybrid_retrieval_contract_paraphrase_has_vector_provenance_without_lexical_overlap( hybrid_vector_root: Path, capsys: pytest.CaptureFixture[str] ) -> None: @@ -412,7 +466,6 @@ def test_hybrid_retrieval_contract_paraphrase_has_vector_provenance_without_lexi assert isinstance(paraphrase["mechanical_scores"]["vector_distance"], float) -@MISSING_HYBRID_RETRIEVAL def test_hybrid_retrieval_contract_deduplicates_both_sources_and_is_deterministic( hybrid_vector_root: Path, capsys: pytest.CaptureFixture[str] ) -> None: @@ -430,7 +483,6 @@ def test_hybrid_retrieval_contract_deduplicates_both_sources_and_is_deterministi assert "note-noise" not in {candidate["id"] for candidate in first} -@MISSING_HYBRID_RETRIEVAL def test_hybrid_retrieval_contract_uses_reciprocal_rank_fusion_not_source_append( hybrid_retrieval_root: Path, capsys: pytest.CaptureFixture[str], @@ -466,8 +518,15 @@ def execute(self, sql: str, _parameters: object) -> list[dict[str, object]]: def close(self) -> None: return None + class FakeSqliteVec: + @staticmethod + def serialize_float32(_values: object) -> bytes: + return b"fixture-vector" + write_vector_status(hybrid_retrieval_root) monkeypatch.setattr(query.sqlite3, "connect", lambda _path: FakeHybridConnection()) + monkeypatch.setattr(query, "load_sqlite_vec", lambda _connection: (FakeSqliteVec(), None)) + monkeypatch.setattr(query, "local_text_vector", lambda _text, query: [0.0] * indexes.VECTOR_DIMENSIONS) trace, candidates = run_query(hybrid_retrieval_root, capsys, query="fixture fusion", limit=3) @@ -480,7 +539,6 @@ def close(self) -> None: } -@MISSING_HYBRID_RETRIEVAL def test_hybrid_retrieval_contract_fallback_reports_reason_and_keeps_fts( hybrid_retrieval_root: Path, capsys: pytest.CaptureFixture[str], @@ -500,7 +558,6 @@ def test_hybrid_retrieval_contract_fallback_reports_reason_and_keeps_fts( assert all(candidate["mechanical_sources"]["vector"] is False for candidate in candidates) -@MISSING_HYBRID_RETRIEVAL def test_hybrid_retrieval_contract_runtime_has_no_package_manager_shellout() -> None: source = "\n".join( path.read_text(encoding="utf-8") @@ -514,6 +571,15 @@ def test_hybrid_retrieval_contract_runtime_has_no_package_manager_shellout() -> assert '"-m", "uv"' not in source +def test_hybrid_retrieval_contract_runtime_declares_pinned_uv_dependencies() -> None: + source = (REPO_ROOT / "scripts" / "ks.py").read_text(encoding="utf-8") + + assert '# /// script' in source + assert '"pyyaml==6.0.3"' in source + assert '"sqlite-vec==0.1.9"' in source + assert '"fastembed==0.8.0"' in source + + @pytest.mark.parametrize( ("field", "value"), [ @@ -525,7 +591,6 @@ def test_hybrid_retrieval_contract_runtime_has_no_package_manager_shellout() -> ("index_schema", None), ], ) -@MISSING_HYBRID_RETRIEVAL def test_hybrid_retrieval_contract_incompatible_rebuilt_index_requires_rebuild( hybrid_retrieval_root: Path, capsys: pytest.CaptureFixture[str],