From 9d9713c23ee52411fae5c8ffd286bbcb7574dba9 Mon Sep 17 00:00:00 2001 From: "Robert D. Schoening" Date: Mon, 20 Jul 2026 17:01:29 -0400 Subject: [PATCH] feat(mcp): add `detail` output mode to search_documents (full | locations) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lean output slice 1: `locations` returns only file + line per match, dropping the (often paragraph-sized) matched text. Far more token-efficient — a small local model fits many more results in its context window, and it's a natural "which files mention X?" first pass before reading specific ones with get_document_context. Attacks the context-overflow problem from peekdocs's side rather than only via LM Studio settings. Design (per discussion): - Default is `full` (unchanged behavior); the model only sees less when a request explicitly asks for `locations`, and the tool description tells it to say so when it does — avoids "answered confidently from a partial picture". - `_match_dicts(matches, detail)` is the single chokepoint; unknown values fall back to full. - Truncation `note` now points at `detail=locations` as a way to fit more — gated by `_cap(..., detail_hint=True)` so it appears ONLY on search_documents (the tool that accepts the param), never on get_document_context / suite / collection notes (which would be an invent-a-param trap). Scope: search_documents only (MVP). `snippet` mode and suite/collection fan-out are deliberate follow-up slices. Tests: full-includes-text, locations-omits-text, unknown-falls-back, detail-hint-gating. Docs: USER_GUIDE Tools row + "want to see more" third lever; LOCAL_AI_SETUP overflow troubleshooting (c); CHANGELOG. No version bump. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Robert D. Schoening --- CHANGELOG.md | 11 ++++++++ docs/LOCAL_AI_SETUP.md | 4 ++- docs/USER_GUIDE.md | 4 ++- peekdocs/mcp_server.py | 61 +++++++++++++++++++++++++++++++--------- tests/test_mcp_server.py | 38 +++++++++++++++++++++++++ 5 files changed, 103 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c718f8..2bc372a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,17 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ## [Unreleased] ### Added +- **MCP `search_documents` gains a `detail` output mode — `full` (default) or + `locations`.** `locations` returns only each match's file and line, dropping + the (often paragraph-sized) matched text — far more token-efficient, so a + small local model can fit many more results in its context window, and a + natural first pass for "which files mention X?" before reading specific ones + with `get_document_context`. `full` is unchanged and remains the default, so + the model only sees less when a request explicitly asks for it (and the tool + description tells it to say so when it does). When a `full` search truncates, + the response `note` now also points at `detail=locations` as a way to fit + more — surfaced only on the tool that accepts the parameter, never on the + others. - **MCP setup helper — generate and (optionally) write the LM Studio `mcp.json` so an AI assistant can drive peekdocs.** Available three ways, all sharing one core module (`peekdocs/mcp_setup.py`): (a) `peekdocs-mcp` diff --git a/docs/LOCAL_AI_SETUP.md b/docs/LOCAL_AI_SETUP.md index a49a926..91816ad 100644 --- a/docs/LOCAL_AI_SETUP.md +++ b/docs/LOCAL_AI_SETUP.md @@ -376,7 +376,9 @@ reads the file at startup). reload the model (see [Step 5](#step-5--turn-it-on)); or **(b)** return fewer/shorter matches — lower `--max-results`, scope the search to a subfolder or single file, or start a **fresh chat** to clear history. If you're already at a low `--max-results` and still overflowing, **(a) is the real - fix** — raise the window rather than starving the answer. + fix** — raise the window rather than starving the answer. **(c)** if your question only needs to + know *which* files match, ask the assistant to **return locations only** (no matched text) — that + fits far more results in the window, and it can read the ones that matter afterward. - **The model tells you to *type a command*** (something like `peekdocs -q "…" --context-before 5`). **Don't run it — it's invented.** When peekdocs is connected through MCP, the model **calls a tool** (you'll see a tool-call appear before the answer); it does **not** drive peekdocs with diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index ce62e09..9557743 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -1691,7 +1691,7 @@ These files are often in hidden folders; the [beginner guide](LOCAL_AI_SETUP.md# | Tool | What it does | |---|---| -| `search_documents` | Search for terms, regex, fuzzy, whole-word, or a boolean expression across a folder; returns matching lines with file and line number. Supports context lines, type filters, and exclusions. | +| `search_documents` | Search for terms, regex, fuzzy, whole-word, or a boolean expression across a folder; returns matching lines with file and line number. Supports context lines, type filters, and exclusions. A `detail` mode chooses how much comes back per match: `full` (default) includes the matched text; `locations` returns just file + line (no text) — far more token-efficient for a broad "which files?" pass or when results are being truncated, then read the ones you want with `get_document_context`. | | `get_document_context` | Return the lines surrounding matches of a query within one named file. | | `inventory_folder` | List the searchable files in a folder (path, size, modified time, type) without reading their contents. | | `list_supported_file_types` | List the extensions peekdocs can search (optionally including OCR image types). | @@ -1852,6 +1852,8 @@ A handy property this surfaces: within the server's `--root` fence, any single s They're a **pair**: raising `--max-results` without enough context risks the *"exceeds the available context size"* error, and a bigger context does nothing if the cap still stops results at 25. And neither helps a **census** question (an exact count or an exhaustive list) — for those, ask peekdocs directly rather than chasing ever-higher limits (see [What to ask](#what-to-ask--and-what-to-send-straight-to-peekdocs)). +A third lever attacks the problem from peekdocs's side: ask for **less per match**. If your question only needs to know *which* files match — not their exact wording — tell the assistant to return **locations only** (it passes `detail=locations` to `search_documents`, dropping the matched text). That fits far more matches in the same window; the assistant can then read the few that matter with `get_document_context`. When a search is truncated, the response note points this out too. + **Or run the model in the cloud (e.g. Claude) instead.** Two of the constraints above came from the *local* model: 8192 tokens of memory forced the low cap, and a 7B model is a modest reasoner. A **cloud** assistant such as Claude Desktop or Claude Code sidesteps both — same peekdocs server, same tools, just a more capable model behind the host — at the cost of the matched snippets leaving your machine. In this example the trade looks like: | | Local model (as above) | Cloud assistant (e.g. Claude) | diff --git a/peekdocs/mcp_server.py b/peekdocs/mcp_server.py index b9462d9..8c1386e 100644 --- a/peekdocs/mcp_server.py +++ b/peekdocs/mcp_server.py @@ -81,39 +81,66 @@ def _resolve_dir(directory: Optional[str]) -> str: return _resolve_within_roots(directory) -def _cap(items: list) -> tuple[list, dict[str, Any]]: +def _cap(items: list, *, detail_hint: bool = False) -> tuple[list, dict[str, Any]]: """Truncate ``items`` to ``max_results`` and describe what was dropped. On truncation the envelope carries a plain-language ``note`` alongside the machine fields, stating the real reason (the ``max_results`` cap). Assistants reliably relay a ready-made sentence but tend to invent a cause when handed only the numbers — the note keeps their narration honest. + + ``detail_hint`` adds a pointer to the ``detail=locations`` output mode — set + it only for tools that actually accept ``detail`` (currently + ``search_documents``), so the note never suggests a parameter the calling + tool doesn't support. """ total = len(items) cap = _CONFIG.max_results if total > cap: + note = ( + f"Showing {cap} of {total} results — capped by max_results " + f"({cap}). To see more, ask to narrow the request (e.g. a more " + "specific term or folder) or raise the max_results limit." + ) + if detail_hint: + note += ( + " Or, if you only need to know which files match, re-run with " + "detail=locations to fit many more." + ) return items[:cap], { "truncated": True, "total": total, "returned": cap, - "note": ( - f"Showing {cap} of {total} results — capped by max_results " - f"({cap}). To see more, ask to narrow the request (e.g. a more " - "specific term or folder) or raise the max_results limit." - ), + "note": note, } return items, {"truncated": False, "total": total, "returned": total} -def _match_dicts(matches) -> list[dict[str, Any]]: - return [ - { +#: Valid values for the ``detail`` output mode. +_DETAIL_MODES = ("full", "locations") + + +def _match_dicts(matches, detail: str = "full") -> list[dict[str, Any]]: + """Serialize matches to dicts at the requested ``detail`` level. + + ``full`` (default) includes each match's text; ``locations`` returns only + ``file`` + ``line`` (no text), which is far more token-efficient — useful + when a small local model's context window can't hold many full matches, or + for a broad "which files mention X?" pass before drilling in with + :func:`get_document_context`. Unknown values fall back to ``full``. + """ + if detail not in _DETAIL_MODES: + detail = "full" + out: list[dict[str, Any]] = [] + for m in matches: + d: dict[str, Any] = { "file": os.path.join(m.file_dir, m.filename), "line": m.line_num, - "text": m.text, } - for m in matches - ] + if detail == "full": + d["text"] = m.text + out.append(d) + return out # ── Tool implementations (plain functions — no MCP dependency) ────── @@ -134,6 +161,7 @@ def search_documents( range_filters: Optional[list[str]] = None, use_ocr: Optional[bool] = None, allow_index_write: Optional[bool] = None, + detail: str = "full", ) -> dict[str, Any]: """Search local documents for text and return matching lines. @@ -155,6 +183,13 @@ def search_documents( scanned PDFs and images (requires Tesseract). allow_index_write: opt in to using/refreshing the on-disk search index (writes .peekdocs.db); off by default to keep the search purely read-only. + detail: how much to return per match. "full" (default) includes each + match's text. "locations" returns only file + line, no text — far more + token-efficient; use it for a broad "which files mention X?" pass, or when + results are being truncated and you want to fit more, then call + get_document_context on the files you care about to read their text. Only + request "locations" when the user's question doesn't need the matched text + itself; if you use it, say so — you have not seen the surrounding wording. """ recursive = _CONFIG.recursive_default if recursive is None else recursive use_ocr = _CONFIG.ocr_default if use_ocr is None else use_ocr @@ -179,7 +214,7 @@ def search_documents( use_ocr=use_ocr, use_index=None if allow_index_write else False, ) - matches, envelope = _cap(_match_dicts(result.matches)) + matches, envelope = _cap(_match_dicts(result.matches, detail), detail_hint=True) return { "searched_directory": d, "matches": matches, diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index af924c1..c1c6da2 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -90,6 +90,15 @@ def test_truncation_note_states_real_reason(self): assert "max_results" in env["note"] assert "2 of 5" in env["note"] + def test_detail_hint_only_when_requested(self): + # The detail=locations pointer must appear ONLY for tools that accept + # the detail param — never on the shared default note. + m._CONFIG.max_results = 2 + _, plain = m._cap([1, 2, 3, 4, 5]) + _, hinted = m._cap([1, 2, 3, 4, 5], detail_hint=True) + assert "detail=locations" not in plain["note"] + assert "detail=locations" in hinted["note"] + def test_under_cap_untouched(self): m._CONFIG.max_results = 10 rows, env = m._cap([1, 2]) @@ -137,6 +146,35 @@ def test_truncation_envelope(self, tmp_path): assert len(out["matches"]) == 1 +class TestDetailModes: + def test_full_includes_text_by_default(self, tmp_path): + _seed(tmp_path) + out = m.search_documents(["fox"], directory=str(tmp_path), recursive=True) + assert all("text" in x for x in out["matches"]) + + def test_locations_omits_text(self, tmp_path): + _seed(tmp_path) + out = m.search_documents( + ["fox"], directory=str(tmp_path), recursive=True, detail="locations" + ) + assert out["matches"] # still returns the matches + assert all("text" not in x for x in out["matches"]) + assert all({"file", "line"} <= x.keys() for x in out["matches"]) + + def test_unknown_detail_falls_back_to_full(self, tmp_path): + _seed(tmp_path) + out = m.search_documents( + ["fox"], directory=str(tmp_path), recursive=True, detail="bogus" + ) + assert all("text" in x for x in out["matches"]) + + def test_locations_search_still_truncation_hints_detail(self, tmp_path): + _seed(tmp_path) + m._CONFIG.max_results = 1 + out = m.search_documents(["fox"], directory=str(tmp_path), recursive=True) + assert "detail=locations" in out["note"] + + class TestOtherTools: def test_get_document_context(self, tmp_path): _seed(tmp_path)