diff --git a/CHANGELOG.md b/CHANGELOG.md index 2bc372a..0be9061 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,19 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ## [Unreleased] ### Added +- **Optional relevance ranking (`--rank`) — order matches by BM25 instead of + file order.** Opt-in; **changes only the order, never which matches are + returned**, and preserves peekdocs's exact substring matching. Ranks by term + rarity, term frequency (with saturation, so repetition can't dominate), and + match density — so a short on-point paragraph outranks a file that merely + repeats a common word. Deterministic (same query → same order). Available in + the CLI (`peekdocs --rank `) and the Python API (`search(..., rank=True)`); + requires the search index (a note is printed if it's missing) and applies to + non-context searches. Regex/fuzzy searches are unaffected. *(Implemented as a + Python scorer over the matched set — the literal-search path scans substrings + rather than FTS5 tokens, so the index's own `bm25()` isn't reachable there; + the Python scorer gives uniform ranking on both paths while keeping substring + semantics.)* - **MCP `search_documents` gains a `detail` output mode — `full` (default) or `locations`.** `locations` returns only each match's file and line, dropping the (often paragraph-sized) matched text — far more token-efficient, so a diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index d37efc9..387180d 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -768,6 +768,7 @@ peekdocs has twenty-nine flags that can be mixed and matched: | `--index-status` (index-status) | Show index info — file count, line count, database size, creation date, and settings | | `--inverse` (inverse) | Inverse search — list files that do NOT contain the search terms. See [Inverse Search](#inverse-search) | | `--open FMT` (open) | Automatically open the report when the search finishes. Specify the format: `docx`, `txt`, `csv`, `json`, `pdf`, or `html`. For csv/json/pdf/html, the output format is auto-generated if not already enabled — no need to also specify `-o`. Opens in whatever app you've set as your OS default for that file type | +| `--rank` (rank) | Order matches by **relevance (BM25)** instead of the default file-then-line order — a short, on-point passage floats above a file that merely repeats a common word. Opt-in; it changes only the *order*, never *which* matches are returned, and it's deterministic (same query → same order). Requires the [search index](#search-index-optional) (prints a note if none exists) and applies to non-context searches; regex and fuzzy searches are unaffected. | | `--output-dir PATH` (output-dir) | Write all output files (reports, error log, CSV, JSON, PDF) to the specified directory instead of the search folder | | `-A N` (after) | Show N lines after each match. What counts as a "line" matches the unit peekdocs indexes per format: a literal line for plain text and source code, a paragraph for Word (.docx) and PDF, a row for Excel. On paragraph-heavy formats, `-A 3` can include several sentences or even pages of surrounding text. | | `-B N` (before) | Show N lines before each match. Same per-format meaning as `-A N` — see above. | diff --git a/peekdocs/api.py b/peekdocs/api.py index e1495bb..549cf90 100644 --- a/peekdocs/api.py +++ b/peekdocs/api.py @@ -85,6 +85,7 @@ def search( expression: str | None = None, range_filters: list[str] | None = None, max_file_size_mb: int = 100, + rank: bool = False, ) -> SearchResult: """Search documents and return structured results. @@ -135,6 +136,12 @@ def search( range_filters : list[str], optional Range filter specs (e.g. ["amount:1000..5000", "date:2024-01-01..2024-12-31"]). Content ranges filter matched lines; metadata ranges filter files. + rank : bool, optional + Order matches by BM25 relevance (most relevant first) instead of the + default file-then-line order. Opt-in; changes only the order, never which + matches are returned. Requires the FTS5 index (no effect on direct-scan, + regex, or fuzzy searches). Applies to non-context searches; when context + lines are requested, matches regroup by file and rank order is not kept. Returns ------- @@ -231,6 +238,7 @@ def search( "metadata_ranges": metadata_ranges, "filename_ranges": filename_ranges, "max_file_size_mb": max_file_size_mb, + "rank": rank, } # ── Determine search path ─────────────────────────────────── diff --git a/peekdocs/cli.py b/peekdocs/cli.py index c7fc9a7..c49e7d8 100644 --- a/peekdocs/cli.py +++ b/peekdocs/cli.py @@ -123,6 +123,8 @@ ' --index-status Show index file count, size, last updated\n' ' --index-clear Delete the search index\n' ' --no-index Skip the index for this search (direct scan)\n' + ' --rank Order matches by relevance (BM25) instead of file order;\n' + ' needs the search index\n' '\n' '── Settings & Info ──────────────────────────────────────────────\n' ' --suite NAME Run a search suite (group of saved searches) by name.\n' @@ -1948,6 +1950,7 @@ def _index_progress(done, total_count, filename): _set_paths_rc(**_rc_path_args) return 0 if total_matches > 0 else 1 + want_rank = "--rank" in args no_index = "--no-index" in args if no_index: args.remove("--no-index") @@ -2070,6 +2073,14 @@ def _index_progress(done, total_count, filename): # Determine index mode for display _will_use_index = index_exists(cwd) and not no_index + # --rank needs the index (BM25-style relevance is applied on the indexed + # path). If ranking was requested but the index won't be used, say so rather + # than silently returning file-order results. + if want_rank and not _will_use_index and not (stdout_json or minimal or quiet): + reason = "no index in this folder — run `peekdocs --index`" if not no_index \ + else "--no-index was set" + print(f"Note: --rank needs the search index ({reason}); results are in file order.", + file=sys.stderr) display_label = expression if expression else ' '.join(search_terms) if not display_label and range_specs_raw: display_label = " ".join(range_specs_raw) @@ -2193,6 +2204,7 @@ def _cli_progress(done, total_count, filename): expression=expression, range_filters=range_specs_raw or None, max_file_size_mb=parsed.get("max_file_size_mb", 100), + rank=want_rank, ) except KeyboardInterrupt: spinner_stop.set() diff --git a/peekdocs/indexer.py b/peekdocs/indexer.py index 4793329..98df93c 100644 --- a/peekdocs/indexer.py +++ b/peekdocs/indexer.py @@ -1,6 +1,7 @@ """SQLite FTS5 index for peekdocs.""" from __future__ import annotations +import math import os import re import sqlite3 @@ -621,10 +622,62 @@ def _norm(p): matches = [(fd, fn, ln, tx) for fd, fn, ln, tx in matches if file_matches_filename_ranges(fn, filename_ranges)] + # Optional relevance ranking (opt-in via config["rank"]). Applied last, to + # the fully-filtered flat match list, so it reorders exactly what's returned. + # Skipped when context lines are requested — those results are grouped by + # file (context windows), which ranking would scramble. + if config.get("rank") and not config.get("use_context"): + matches = _rank_matches(matches, config.get("search_terms", [])) + conn.close() return matches, skipped, all_indexed_files +def _rank_matches( + matches: list[tuple[str, str, int, str]], search_terms: list[str] +) -> list[tuple[str, str, int, str]]: + """Reorder already-matched rows by BM25 relevance (most relevant first). + + A Python scorer applied *after* matching, so it changes only the order — + never which matches are returned — and preserves peekdocs's exact substring + semantics (the literal-search path scans substrings, not FTS5 tokens, so the + index's own ``bm25()`` isn't reachable there; this gives uniform ranking on + both the substring and whole-word paths). + + Classic BM25 per matched paragraph: term frequency with saturation (``k1``), + rarity/IDF computed over the matched set, and length normalization (``b``). + Deterministic — same inputs give the same order (ties keep original order). + """ + if not matches or not search_terms: + return matches + terms = [t.lower() for t in search_terms if t] + if not terms: + return matches + + texts = [m[3].lower() for m in matches] + lengths = [max(1, len(t.split())) for t in texts] + avgdl = sum(lengths) / len(lengths) + n = len(matches) + df = {t: sum(1 for txt in texts if t in txt) for t in terms} + + k1, b = 1.5, 0.75 + scored = [] + for i, m in enumerate(matches): + txt, dl = texts[i], lengths[i] + score = 0.0 + for t in terms: + n_t = df[t] + f = txt.count(t) + if n_t == 0 or f == 0: + continue + idf = math.log(1 + (n - n_t + 0.5) / (n_t + 0.5)) + score += idf * (f * (k1 + 1)) / (f + k1 * (1 - b + b * dl / avgdl)) + scored.append((score, i, m)) + + scored.sort(key=lambda s: (-s[0], s[1])) # score desc; stable by orig index + return [m for _, _, m in scored] + + def _can_use_direct_scan(config: dict[str, Any]) -> bool: """Determine if the search can use a direct paragraph scan. diff --git a/tests/test_api.py b/tests/test_api.py index fba891e..9ef818b 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -460,3 +460,62 @@ def test_fn_date_no_match(self, tmp_path, monkeypatch): result = search(["budget"], directory=str(tmp_path), range_filters=["fn:date:2024-01-01..2024-12-31"]) assert len(result.matches) == 0 + + +# ── Relevance ranking (BM25) ───────────────────────────────────── + +class TestRanking: + def test_rank_matches_unit_orders_by_relevance(self): + # dense on-point paragraph beats repetitive-common-word beats buried. + from peekdocs.indexer import _rank_matches + matches = [ + ("/d", "buried.txt", 1, "the roof over the whole building is grey and large " * 4), + ("/d", "dense.txt", 1, "the roof warranty period is ten years"), + ("/d", "repetitive.txt", 1, "warranty warranty warranty warranty boilerplate terms"), + ] + ranked = _rank_matches(matches, ["roof", "warranty"]) + names = [m[1] for m in ranked] + assert names[0] == "dense.txt" # short + both terms = most relevant + assert names[-1] == "buried.txt" # term buried in long text = least + # only reordered, never dropped + assert sorted(m[1] for m in ranked) == sorted(m[1] for m in matches) + + def test_rank_matches_is_deterministic(self): + from peekdocs.indexer import _rank_matches + matches = [("/d", f"f{i}.txt", 1, "budget line and revenue detail") for i in range(5)] + a = _rank_matches(list(matches), ["budget"]) + b = _rank_matches(list(matches), ["budget"]) + assert [m[1] for m in a] == [m[1] for m in b] + + def test_rank_matches_empty_and_no_terms(self): + from peekdocs.indexer import _rank_matches + assert _rank_matches([], ["x"]) == [] + one = [("/d", "f.txt", 1, "text")] + assert _rank_matches(one, []) == one # no terms → unchanged + + def test_search_rank_reorders_same_matches(self, tmp_path, monkeypatch): + from peekdocs.indexer import build_index + (tmp_path / "buried.txt").write_text( + "the roof over the whole building is grey and large and old " * 4 + "\n") + (tmp_path / "dense.txt").write_text("the roof warranty period is ten years\n") + (tmp_path / "repetitive.txt").write_text( + "warranty warranty warranty warranty boilerplate terms apply\n") + monkeypatch.chdir(tmp_path) + build_index(str(tmp_path), recursive=True) + + base = search(["roof", "warranty"], directory=str(tmp_path), use_index=True) + ranked = search(["roof", "warranty"], directory=str(tmp_path), use_index=True, rank=True) + + base_names = [m.filename for m in base.matches] + ranked_names = [m.filename for m in ranked.matches] + # same matches, only the order changes + assert sorted(base_names) == sorted(ranked_names) + assert ranked_names[0] == "dense.txt" + assert ranked_names != base_names # ranking actually did something here + + def test_rank_without_index_is_safe_noop(self, tmp_path, monkeypatch): + _make_docx(tmp_path / "doc.docx", ["budget report", "budget summary"]) + monkeypatch.chdir(tmp_path) + # No index built + rank=True must not error; results just come back unranked. + result = search(["budget"], directory=str(tmp_path), use_index=False, rank=True) + assert len(result.matches) == 2