feat(search): optional BM25 relevance ranking (--rank)

CHANGELOG.md

@@ -13,6 +13,19 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 ## [Unreleased]
 
 ### Added
+- **Optional relevance ranking (`--rank`) — order matches by BM25 instead of
+  file order.** Opt-in; **changes only the order, never which matches are
+  returned**, and preserves peekdocs's exact substring matching. Ranks by term
+  rarity, term frequency (with saturation, so repetition can't dominate), and
+  match density — so a short on-point paragraph outranks a file that merely
+  repeats a common word. Deterministic (same query → same order). Available in
+  the CLI (`peekdocs --rank <terms>`) and the Python API (`search(..., rank=True)`);
+  requires the search index (a note is printed if it's missing) and applies to
+  non-context searches. Regex/fuzzy searches are unaffected. *(Implemented as a
+  Python scorer over the matched set — the literal-search path scans substrings
+  rather than FTS5 tokens, so the index's own `bm25()` isn't reachable there;
+  the Python scorer gives uniform ranking on both paths while keeping substring
+  semantics.)*
 - **MCP `search_documents` gains a `detail` output mode — `full` (default) or
   `locations`.** `locations` returns only each match's file and line, dropping
   the (often paragraph-sized) matched text — far more token-efficient, so a

docs/USER_GUIDE.md

@@ -768,6 +768,7 @@ peekdocs has twenty-nine flags that can be mixed and matched:
 | `--index-status` (index-status) | Show index info — file count, line count, database size, creation date, and settings |
 | `--inverse` (inverse) | Inverse search — list files that do NOT contain the search terms. See [Inverse Search](#inverse-search) |
 | `--open FMT` (open) | Automatically open the report when the search finishes. Specify the format: `docx`, `txt`, `csv`, `json`, `pdf`, or `html`. For csv/json/pdf/html, the output format is auto-generated if not already enabled — no need to also specify `-o`. Opens in whatever app you've set as your OS default for that file type |
+| `--rank` (rank) | Order matches by **relevance (BM25)** instead of the default file-then-line order — a short, on-point passage floats above a file that merely repeats a common word. Opt-in; it changes only the *order*, never *which* matches are returned, and it's deterministic (same query → same order). Requires the [search index](#search-index-optional) (prints a note if none exists) and applies to non-context searches; regex and fuzzy searches are unaffected. |
 | `--output-dir PATH` (output-dir) | Write all output files (reports, error log, CSV, JSON, PDF) to the specified directory instead of the search folder |
 | `-A N` (after) | Show N lines after each match. What counts as a "line" matches the unit peekdocs indexes per format: a literal line for plain text and source code, a paragraph for Word (.docx) and PDF, a row for Excel. On paragraph-heavy formats, `-A 3` can include several sentences or even pages of surrounding text. |
 | `-B N` (before) | Show N lines before each match. Same per-format meaning as `-A N` — see above. |

peekdocs/api.py

@@ -85,6 +85,7 @@ def search(
     expression: str | None = None,
     range_filters: list[str] | None = None,
     max_file_size_mb: int = 100,
+    rank: bool = False,
 ) -> SearchResult:
     """Search documents and return structured results.
 
@@ -135,6 +136,12 @@ def search(
     range_filters : list[str], optional
         Range filter specs (e.g. ["amount:1000..5000", "date:2024-01-01..2024-12-31"]).
         Content ranges filter matched lines; metadata ranges filter files.
+    rank : bool, optional
+        Order matches by BM25 relevance (most relevant first) instead of the
+        default file-then-line order. Opt-in; changes only the order, never which
+        matches are returned. Requires the FTS5 index (no effect on direct-scan,
+        regex, or fuzzy searches). Applies to non-context searches; when context
+        lines are requested, matches regroup by file and rank order is not kept.
 
     Returns
     -------
@@ -231,6 +238,7 @@ def search(
         "metadata_ranges": metadata_ranges,
         "filename_ranges": filename_ranges,
         "max_file_size_mb": max_file_size_mb,
+        "rank": rank,
     }
 
     # ── Determine search path ───────────────────────────────────

peekdocs/cli.py

@@ -123,6 +123,8 @@
     '  --index-status     Show index file count, size, last updated\n'
     '  --index-clear      Delete the search index\n'
     '  --no-index         Skip the index for this search (direct scan)\n'
+    '  --rank             Order matches by relevance (BM25) instead of file order;\n'
+    '                     needs the search index\n'
     '\n'
     '── Settings & Info ──────────────────────────────────────────────\n'
     '  --suite NAME       Run a search suite (group of saved searches) by name.\n'
@@ -1948,6 +1950,7 @@ def _index_progress(done, total_count, filename):
             _set_paths_rc(**_rc_path_args)
         return 0 if total_matches > 0 else 1
 
+    want_rank = "--rank" in args
     no_index = "--no-index" in args
     if no_index:
         args.remove("--no-index")
@@ -2070,6 +2073,14 @@ def _index_progress(done, total_count, filename):
 
     # Determine index mode for display
     _will_use_index = index_exists(cwd) and not no_index
+    # --rank needs the index (BM25-style relevance is applied on the indexed
+    # path). If ranking was requested but the index won't be used, say so rather
+    # than silently returning file-order results.
+    if want_rank and not _will_use_index and not (stdout_json or minimal or quiet):
+        reason = "no index in this folder — run `peekdocs --index`" if not no_index \
+            else "--no-index was set"
+        print(f"Note: --rank needs the search index ({reason}); results are in file order.",
+              file=sys.stderr)
     display_label = expression if expression else ' '.join(search_terms)
     if not display_label and range_specs_raw:
         display_label = " ".join(range_specs_raw)
@@ -2193,6 +2204,7 @@ def _cli_progress(done, total_count, filename):
             expression=expression,
             range_filters=range_specs_raw or None,
             max_file_size_mb=parsed.get("max_file_size_mb", 100),
+            rank=want_rank,
         )
     except KeyboardInterrupt:
         spinner_stop.set()

peekdocs/indexer.py

@@ -1,6 +1,7 @@
 """SQLite FTS5 index for peekdocs."""
 from __future__ import annotations
 
+import math
 import os
 import re
 import sqlite3
@@ -621,10 +622,62 @@ def _norm(p):
         matches = [(fd, fn, ln, tx) for fd, fn, ln, tx in matches
                    if file_matches_filename_ranges(fn, filename_ranges)]
 
+    # Optional relevance ranking (opt-in via config["rank"]). Applied last, to
+    # the fully-filtered flat match list, so it reorders exactly what's returned.
+    # Skipped when context lines are requested — those results are grouped by
+    # file (context windows), which ranking would scramble.
+    if config.get("rank") and not config.get("use_context"):
+        matches = _rank_matches(matches, config.get("search_terms", []))
+
     conn.close()
     return matches, skipped, all_indexed_files
 
 
+def _rank_matches(
+    matches: list[tuple[str, str, int, str]], search_terms: list[str]
+) -> list[tuple[str, str, int, str]]:
+    """Reorder already-matched rows by BM25 relevance (most relevant first).
+
+    A Python scorer applied *after* matching, so it changes only the order —
+    never which matches are returned — and preserves peekdocs's exact substring
+    semantics (the literal-search path scans substrings, not FTS5 tokens, so the
+    index's own ``bm25()`` isn't reachable there; this gives uniform ranking on
+    both the substring and whole-word paths).
+
+    Classic BM25 per matched paragraph: term frequency with saturation (``k1``),
+    rarity/IDF computed over the matched set, and length normalization (``b``).
+    Deterministic — same inputs give the same order (ties keep original order).
+    """
+    if not matches or not search_terms:
+        return matches
+    terms = [t.lower() for t in search_terms if t]
+    if not terms:
+        return matches
+
+    texts = [m[3].lower() for m in matches]
+    lengths = [max(1, len(t.split())) for t in texts]
+    avgdl = sum(lengths) / len(lengths)
+    n = len(matches)
+    df = {t: sum(1 for txt in texts if t in txt) for t in terms}
+
+    k1, b = 1.5, 0.75
+    scored = []
+    for i, m in enumerate(matches):
+        txt, dl = texts[i], lengths[i]
+        score = 0.0
+        for t in terms:
+            n_t = df[t]
+            f = txt.count(t)
+            if n_t == 0 or f == 0:
+                continue
+            idf = math.log(1 + (n - n_t + 0.5) / (n_t + 0.5))
+            score += idf * (f * (k1 + 1)) / (f + k1 * (1 - b + b * dl / avgdl))
+        scored.append((score, i, m))
+
+    scored.sort(key=lambda s: (-s[0], s[1]))  # score desc; stable by orig index
+    return [m for _, _, m in scored]
+
+
 def _can_use_direct_scan(config: dict[str, Any]) -> bool:
     """Determine if the search can use a direct paragraph scan.
 

tests/test_api.py

@@ -460,3 +460,62 @@ def test_fn_date_no_match(self, tmp_path, monkeypatch):
         result = search(["budget"], directory=str(tmp_path),
                          range_filters=["fn:date:2024-01-01..2024-12-31"])
         assert len(result.matches) == 0
+
+
+# ── Relevance ranking (BM25) ─────────────────────────────────────
+
+class TestRanking:
+    def test_rank_matches_unit_orders_by_relevance(self):
+        # dense on-point paragraph beats repetitive-common-word beats buried.
+        from peekdocs.indexer import _rank_matches
+        matches = [
+            ("/d", "buried.txt", 1, "the roof over the whole building is grey and large " * 4),
+            ("/d", "dense.txt", 1, "the roof warranty period is ten years"),
+            ("/d", "repetitive.txt", 1, "warranty warranty warranty warranty boilerplate terms"),
+        ]
+        ranked = _rank_matches(matches, ["roof", "warranty"])
+        names = [m[1] for m in ranked]
+        assert names[0] == "dense.txt"      # short + both terms = most relevant
+        assert names[-1] == "buried.txt"    # term buried in long text = least
+        # only reordered, never dropped
+        assert sorted(m[1] for m in ranked) == sorted(m[1] for m in matches)
+
+    def test_rank_matches_is_deterministic(self):
+        from peekdocs.indexer import _rank_matches
+        matches = [("/d", f"f{i}.txt", 1, "budget line and revenue detail") for i in range(5)]
+        a = _rank_matches(list(matches), ["budget"])
+        b = _rank_matches(list(matches), ["budget"])
+        assert [m[1] for m in a] == [m[1] for m in b]
+
+    def test_rank_matches_empty_and_no_terms(self):
+        from peekdocs.indexer import _rank_matches
+        assert _rank_matches([], ["x"]) == []
+        one = [("/d", "f.txt", 1, "text")]
+        assert _rank_matches(one, []) == one  # no terms → unchanged
+
+    def test_search_rank_reorders_same_matches(self, tmp_path, monkeypatch):
+        from peekdocs.indexer import build_index
+        (tmp_path / "buried.txt").write_text(
+            "the roof over the whole building is grey and large and old " * 4 + "\n")
+        (tmp_path / "dense.txt").write_text("the roof warranty period is ten years\n")
+        (tmp_path / "repetitive.txt").write_text(
+            "warranty warranty warranty warranty boilerplate terms apply\n")
+        monkeypatch.chdir(tmp_path)
+        build_index(str(tmp_path), recursive=True)
+
+        base = search(["roof", "warranty"], directory=str(tmp_path), use_index=True)
+        ranked = search(["roof", "warranty"], directory=str(tmp_path), use_index=True, rank=True)
+
+        base_names = [m.filename for m in base.matches]
+        ranked_names = [m.filename for m in ranked.matches]
+        # same matches, only the order changes
+        assert sorted(base_names) == sorted(ranked_names)
+        assert ranked_names[0] == "dense.txt"
+        assert ranked_names != base_names  # ranking actually did something here
+
+    def test_rank_without_index_is_safe_noop(self, tmp_path, monkeypatch):
+        _make_docx(tmp_path / "doc.docx", ["budget report", "budget summary"])
+        monkeypatch.chdir(tmp_path)
+        # No index built + rank=True must not error; results just come back unranked.
+        result = search(["budget"], directory=str(tmp_path), use_index=False, rank=True)
+        assert len(result.matches) == 2