From 05a33d2e563c61c08683d5e80e0dbbbcd03690c0 Mon Sep 17 00:00:00 2001 From: "Robert D. Schoening" Date: Mon, 20 Jul 2026 20:46:48 -0400 Subject: [PATCH] feat(search): optional BM25 relevance ranking (--rank / rank=True) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Slice 1 (engine + CLI) of the ranking feature. Opt-in; changes only the ORDER of matches, never which match, and preserves exact substring semantics. Design discovery mid-build (see ops/RELEVANCE_RANKING_DESIGN.md correction): the "one ORDER BY bm25" plan doesn't work for the default path — literal searches go through _direct_scan_search (Python substring scan, so "bob" matches "bobcat") and never touch FTS5, so the index's bm25() is unreachable there. Routing them through FTS5 would change WHICH results match (token vs substring), breaking the "only order changes" promise. Resolution (Option B): a Python BM25 scorer applied to the matched set after matching — uniform on both the substring and whole-word paths, exact-substring-preserving, deterministic. - indexer._rank_matches: BM25 over the matched set (term rarity/IDF, saturated TF, length normalization); ties keep original order. - indexer.search_with_index: applies it when config["rank"] and not use_context (context regroups by file, which ranking would scramble). - api.search: new rank=False param, threaded via search_config; docstring. - cli: --rank flag + help text; prints a note if --rank is used without an index (index-gated — ranking lives on the indexed path). - tests: unit scorer (ordering, determinism, empty/no-terms) + integration through search() (same matches reordered; no-index no-op is safe). Scope: CLI + engine. MCP wiring and GUI checkbox are follow-up slices. No version bump (Unreleased). Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Robert D. Schoening --- CHANGELOG.md | 13 ++++++++++ docs/USER_GUIDE.md | 1 + peekdocs/api.py | 8 ++++++ peekdocs/cli.py | 12 +++++++++ peekdocs/indexer.py | 53 ++++++++++++++++++++++++++++++++++++++++ tests/test_api.py | 59 +++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 146 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2bc372a8..0be90619 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,19 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ## [Unreleased] ### Added +- **Optional relevance ranking (`--rank`) — order matches by BM25 instead of + file order.** Opt-in; **changes only the order, never which matches are + returned**, and preserves peekdocs's exact substring matching. Ranks by term + rarity, term frequency (with saturation, so repetition can't dominate), and + match density — so a short on-point paragraph outranks a file that merely + repeats a common word. Deterministic (same query → same order). Available in + the CLI (`peekdocs --rank `) and the Python API (`search(..., rank=True)`); + requires the search index (a note is printed if it's missing) and applies to + non-context searches. Regex/fuzzy searches are unaffected. *(Implemented as a + Python scorer over the matched set — the literal-search path scans substrings + rather than FTS5 tokens, so the index's own `bm25()` isn't reachable there; + the Python scorer gives uniform ranking on both paths while keeping substring + semantics.)* - **MCP `search_documents` gains a `detail` output mode — `full` (default) or `locations`.** `locations` returns only each match's file and line, dropping the (often paragraph-sized) matched text — far more token-efficient, so a diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index d37efc98..387180df 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -768,6 +768,7 @@ peekdocs has twenty-nine flags that can be mixed and matched: | `--index-status` (index-status) | Show index info — file count, line count, database size, creation date, and settings | | `--inverse` (inverse) | Inverse search — list files that do NOT contain the search terms. See [Inverse Search](#inverse-search) | | `--open FMT` (open) | Automatically open the report when the search finishes. Specify the format: `docx`, `txt`, `csv`, `json`, `pdf`, or `html`. For csv/json/pdf/html, the output format is auto-generated if not already enabled — no need to also specify `-o`. Opens in whatever app you've set as your OS default for that file type | +| `--rank` (rank) | Order matches by **relevance (BM25)** instead of the default file-then-line order — a short, on-point passage floats above a file that merely repeats a common word. Opt-in; it changes only the *order*, never *which* matches are returned, and it's deterministic (same query → same order). Requires the [search index](#search-index-optional) (prints a note if none exists) and applies to non-context searches; regex and fuzzy searches are unaffected. | | `--output-dir PATH` (output-dir) | Write all output files (reports, error log, CSV, JSON, PDF) to the specified directory instead of the search folder | | `-A N` (after) | Show N lines after each match. What counts as a "line" matches the unit peekdocs indexes per format: a literal line for plain text and source code, a paragraph for Word (.docx) and PDF, a row for Excel. On paragraph-heavy formats, `-A 3` can include several sentences or even pages of surrounding text. | | `-B N` (before) | Show N lines before each match. Same per-format meaning as `-A N` — see above. | diff --git a/peekdocs/api.py b/peekdocs/api.py index e1495bbf..549cf902 100644 --- a/peekdocs/api.py +++ b/peekdocs/api.py @@ -85,6 +85,7 @@ def search( expression: str | None = None, range_filters: list[str] | None = None, max_file_size_mb: int = 100, + rank: bool = False, ) -> SearchResult: """Search documents and return structured results. @@ -135,6 +136,12 @@ def search( range_filters : list[str], optional Range filter specs (e.g. ["amount:1000..5000", "date:2024-01-01..2024-12-31"]). Content ranges filter matched lines; metadata ranges filter files. + rank : bool, optional + Order matches by BM25 relevance (most relevant first) instead of the + default file-then-line order. Opt-in; changes only the order, never which + matches are returned. Requires the FTS5 index (no effect on direct-scan, + regex, or fuzzy searches). Applies to non-context searches; when context + lines are requested, matches regroup by file and rank order is not kept. Returns ------- @@ -231,6 +238,7 @@ def search( "metadata_ranges": metadata_ranges, "filename_ranges": filename_ranges, "max_file_size_mb": max_file_size_mb, + "rank": rank, } # ── Determine search path ─────────────────────────────────── diff --git a/peekdocs/cli.py b/peekdocs/cli.py index c7fc9a7e..c49e7d8d 100644 --- a/peekdocs/cli.py +++ b/peekdocs/cli.py @@ -123,6 +123,8 @@ ' --index-status Show index file count, size, last updated\n' ' --index-clear Delete the search index\n' ' --no-index Skip the index for this search (direct scan)\n' + ' --rank Order matches by relevance (BM25) instead of file order;\n' + ' needs the search index\n' '\n' '── Settings & Info ──────────────────────────────────────────────\n' ' --suite NAME Run a search suite (group of saved searches) by name.\n' @@ -1948,6 +1950,7 @@ def _index_progress(done, total_count, filename): _set_paths_rc(**_rc_path_args) return 0 if total_matches > 0 else 1 + want_rank = "--rank" in args no_index = "--no-index" in args if no_index: args.remove("--no-index") @@ -2070,6 +2073,14 @@ def _index_progress(done, total_count, filename): # Determine index mode for display _will_use_index = index_exists(cwd) and not no_index + # --rank needs the index (BM25-style relevance is applied on the indexed + # path). If ranking was requested but the index won't be used, say so rather + # than silently returning file-order results. + if want_rank and not _will_use_index and not (stdout_json or minimal or quiet): + reason = "no index in this folder — run `peekdocs --index`" if not no_index \ + else "--no-index was set" + print(f"Note: --rank needs the search index ({reason}); results are in file order.", + file=sys.stderr) display_label = expression if expression else ' '.join(search_terms) if not display_label and range_specs_raw: display_label = " ".join(range_specs_raw) @@ -2193,6 +2204,7 @@ def _cli_progress(done, total_count, filename): expression=expression, range_filters=range_specs_raw or None, max_file_size_mb=parsed.get("max_file_size_mb", 100), + rank=want_rank, ) except KeyboardInterrupt: spinner_stop.set() diff --git a/peekdocs/indexer.py b/peekdocs/indexer.py index 47933299..98df93c0 100644 --- a/peekdocs/indexer.py +++ b/peekdocs/indexer.py @@ -1,6 +1,7 @@ """SQLite FTS5 index for peekdocs.""" from __future__ import annotations +import math import os import re import sqlite3 @@ -621,10 +622,62 @@ def _norm(p): matches = [(fd, fn, ln, tx) for fd, fn, ln, tx in matches if file_matches_filename_ranges(fn, filename_ranges)] + # Optional relevance ranking (opt-in via config["rank"]). Applied last, to + # the fully-filtered flat match list, so it reorders exactly what's returned. + # Skipped when context lines are requested — those results are grouped by + # file (context windows), which ranking would scramble. + if config.get("rank") and not config.get("use_context"): + matches = _rank_matches(matches, config.get("search_terms", [])) + conn.close() return matches, skipped, all_indexed_files +def _rank_matches( + matches: list[tuple[str, str, int, str]], search_terms: list[str] +) -> list[tuple[str, str, int, str]]: + """Reorder already-matched rows by BM25 relevance (most relevant first). + + A Python scorer applied *after* matching, so it changes only the order — + never which matches are returned — and preserves peekdocs's exact substring + semantics (the literal-search path scans substrings, not FTS5 tokens, so the + index's own ``bm25()`` isn't reachable there; this gives uniform ranking on + both the substring and whole-word paths). + + Classic BM25 per matched paragraph: term frequency with saturation (``k1``), + rarity/IDF computed over the matched set, and length normalization (``b``). + Deterministic — same inputs give the same order (ties keep original order). + """ + if not matches or not search_terms: + return matches + terms = [t.lower() for t in search_terms if t] + if not terms: + return matches + + texts = [m[3].lower() for m in matches] + lengths = [max(1, len(t.split())) for t in texts] + avgdl = sum(lengths) / len(lengths) + n = len(matches) + df = {t: sum(1 for txt in texts if t in txt) for t in terms} + + k1, b = 1.5, 0.75 + scored = [] + for i, m in enumerate(matches): + txt, dl = texts[i], lengths[i] + score = 0.0 + for t in terms: + n_t = df[t] + f = txt.count(t) + if n_t == 0 or f == 0: + continue + idf = math.log(1 + (n - n_t + 0.5) / (n_t + 0.5)) + score += idf * (f * (k1 + 1)) / (f + k1 * (1 - b + b * dl / avgdl)) + scored.append((score, i, m)) + + scored.sort(key=lambda s: (-s[0], s[1])) # score desc; stable by orig index + return [m for _, _, m in scored] + + def _can_use_direct_scan(config: dict[str, Any]) -> bool: """Determine if the search can use a direct paragraph scan. diff --git a/tests/test_api.py b/tests/test_api.py index fba891e7..9ef818ba 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -460,3 +460,62 @@ def test_fn_date_no_match(self, tmp_path, monkeypatch): result = search(["budget"], directory=str(tmp_path), range_filters=["fn:date:2024-01-01..2024-12-31"]) assert len(result.matches) == 0 + + +# ── Relevance ranking (BM25) ───────────────────────────────────── + +class TestRanking: + def test_rank_matches_unit_orders_by_relevance(self): + # dense on-point paragraph beats repetitive-common-word beats buried. + from peekdocs.indexer import _rank_matches + matches = [ + ("/d", "buried.txt", 1, "the roof over the whole building is grey and large " * 4), + ("/d", "dense.txt", 1, "the roof warranty period is ten years"), + ("/d", "repetitive.txt", 1, "warranty warranty warranty warranty boilerplate terms"), + ] + ranked = _rank_matches(matches, ["roof", "warranty"]) + names = [m[1] for m in ranked] + assert names[0] == "dense.txt" # short + both terms = most relevant + assert names[-1] == "buried.txt" # term buried in long text = least + # only reordered, never dropped + assert sorted(m[1] for m in ranked) == sorted(m[1] for m in matches) + + def test_rank_matches_is_deterministic(self): + from peekdocs.indexer import _rank_matches + matches = [("/d", f"f{i}.txt", 1, "budget line and revenue detail") for i in range(5)] + a = _rank_matches(list(matches), ["budget"]) + b = _rank_matches(list(matches), ["budget"]) + assert [m[1] for m in a] == [m[1] for m in b] + + def test_rank_matches_empty_and_no_terms(self): + from peekdocs.indexer import _rank_matches + assert _rank_matches([], ["x"]) == [] + one = [("/d", "f.txt", 1, "text")] + assert _rank_matches(one, []) == one # no terms → unchanged + + def test_search_rank_reorders_same_matches(self, tmp_path, monkeypatch): + from peekdocs.indexer import build_index + (tmp_path / "buried.txt").write_text( + "the roof over the whole building is grey and large and old " * 4 + "\n") + (tmp_path / "dense.txt").write_text("the roof warranty period is ten years\n") + (tmp_path / "repetitive.txt").write_text( + "warranty warranty warranty warranty boilerplate terms apply\n") + monkeypatch.chdir(tmp_path) + build_index(str(tmp_path), recursive=True) + + base = search(["roof", "warranty"], directory=str(tmp_path), use_index=True) + ranked = search(["roof", "warranty"], directory=str(tmp_path), use_index=True, rank=True) + + base_names = [m.filename for m in base.matches] + ranked_names = [m.filename for m in ranked.matches] + # same matches, only the order changes + assert sorted(base_names) == sorted(ranked_names) + assert ranked_names[0] == "dense.txt" + assert ranked_names != base_names # ranking actually did something here + + def test_rank_without_index_is_safe_noop(self, tmp_path, monkeypatch): + _make_docx(tmp_path / "doc.docx", ["budget report", "budget summary"]) + monkeypatch.chdir(tmp_path) + # No index built + rank=True must not error; results just come back unranked. + result = search(["budget"], directory=str(tmp_path), use_index=False, rank=True) + assert len(result.matches) == 2