From ed186224404e08c072aece07a1b4a14226dab0de Mon Sep 17 00:00:00 2001 From: Rutvik Chandla Date: Tue, 23 Jun 2026 02:16:34 +0530 Subject: [PATCH 1/2] Implement north-goal context improvements --- docs/north-goal-ideas.md | 330 ++++++++++++++++ docs/north-goal-plan.md | 336 ++++++++++++++++ packages/cli/src/mcp-shim.ts | 7 +- packages/cli/src/program.ts | 456 ++++++++++++++++----- packages/cli/src/supported-node-majors.ts | 1 + packages/cli/test/program.test.ts | 45 +++ packages/core/src/index.ts | 7 +- packages/core/src/repo.ts | 459 ++++++++++++++++++++-- packages/core/src/scan.ts | 45 ++- packages/core/src/types.ts | 83 +++- packages/eval/losses.json | 14 +- packages/eval/test/eval.test.ts | 2 +- packages/mcp/src/index.ts | 390 +++++++++++++++--- packages/mcp/test/http.test.ts | 37 +- packages/mcp/test/mcp.test.ts | 145 ++++++- scripts/smoke-pack-install.mjs | 19 +- 16 files changed, 2164 insertions(+), 212 deletions(-) create mode 100644 docs/north-goal-ideas.md create mode 100644 docs/north-goal-plan.md create mode 100644 packages/cli/src/supported-node-majors.ts diff --git a/docs/north-goal-ideas.md b/docs/north-goal-ideas.md new file mode 100644 index 0000000..38a54b3 --- /dev/null +++ b/docs/north-goal-ideas.md @@ -0,0 +1,330 @@ +# codesift north-goal ideas + +This is a forward-looking idea stack for the product north goal: + +1. Fewer agent tool calls. +2. Less token output. +3. More accurate context. +4. Onboarding that feels close to zero setup. + +This document is not a committed implementation plan. It is a prioritization map +for the next set of bets after the one-call structural-search moat described in +[`docs/moat.md`](./moat.md). + +## Current signal + +The current benchmark already shows the one-call direction is working: + +- Median calls-to-resolution are `1.0` for every query type in the checked eval. +- Warm latency is faster than `rg` across the benchmark because the daemon/index + path is amortized. +- Concept and identifier accuracy are stronger than `rg` in the benchmark. +- Remaining losses are mostly cold latency, plus token losses on concept and + exact-identifier cases. +- The live MCP surface is powerful but broad: `search_code`, `find_symbol`, + `find_callers`, `find_refs`, `find_importers`, `who_implements`, `impact`, + `grep_code`, `read_chunk`, and `index_status`. That breadth can increase agent + routing mistakes and onboarding friction. + +So the next frontier is less about proving that one-call resolution is possible +and more about making the right one call obvious, compact, fresh, and automatic. + +## Principles + +- Make the common path one decision, not one tool among many. +- Prefer structured context over larger context. +- Spend tokens only when they remove a follow-up call or prevent a wrong edit. +- Make freshness implicit; expose health checks only when they are useful. +- Keep the default path local, offline, and trustable. +- Add model/network features only behind explicit proof and explicit user choice. +- Measure real agent loops, not only deterministic tool routing. + +## Highest-leverage bets + +### 1. Add a front-door `ask_code` tool + +Give agents one default tool that accepts a natural task and internally chooses +search, symbol lookup, grep, or graph traversal. + +Why it helps: + +- Fewer schema tokens in the default client surface. +- Fewer tool-choice mistakes. +- Easier onboarding: "ask Codesift about this repo" instead of teaching a + routing table. +- A single place to encode confidence, ambiguity, freshness, and next-action + policy. + +Possible response contract: + +```text +answer_complete=yes +confidence=high +intent=symbol_definition +tool_path=find_symbol +next=none + += src/auth/token.ts:5-18 parseToken +5 | export function parseToken(...) { +... +``` + +When the answer is not complete: + +```text +answer_complete=no +confidence=medium +intent=ambiguous_symbol +next=choose_candidate +ambiguous=3 defs +``` + +The existing specialist tools should remain available, but `ask_code` should be +the default path in MCP instructions and onboarding docs. + +### 2. Stop default status preflight calls + +The current MCP instructions still encourage checking `index_status`. That can +burn a call before the actual work starts. + +Better default: + +- If the repo is unindexed and safe to index, auto-index on first real query. +- If the index is stale, return stale/freshness metadata with normal results. +- If sync is running, either wait briefly or return compact partial results with + `freshness=syncing`. +- Keep `index_status` as a diagnostic tool, not a required agent ritual. + +This turns "check status, then search" into "search, with health attached." + +### 3. Offer a minimal MCP toolset by default + +Ten tools are useful for power users, but expensive for agents and new users. + +Default toolset: + +- `ask_code` +- `read_chunk` +- `index_status` + +Advanced toolset: + +- Current specialist tools, enabled with something like `--toolset all`. + +Alternate compromise: + +- Collapse relation tools into one `code_graph` tool with `mode=callers|refs| + importers|implements|impact`. + +The goal is not to remove capability. It is to shrink the first screen and the +first agent decision. + +### 4. Add confidence and next-action metadata everywhere + +Agents need to know when to stop. Today they can receive a good inline body and +still spend another call verifying it. + +Every result formatter should be able to communicate: + +- `answer_complete=yes|no` +- `confidence=high|medium|low` +- `next=none|read_chunk|choose_candidate|narrow_query|sync` +- `freshness=fresh|stale|syncing|unknown` +- `ambiguity=` when relevant +- `omitted=` when budget clipped useful context + +This should be terse and machine-friendly. It is a token cost that prevents more +expensive token and tool-call waste downstream. + +### 5. Make output adaptive, not just capped + +The current token losses suggest some default bodies are larger than needed. + +Adaptive output policies: + +- Exact symbol: return signature plus the smallest body slice that covers the + likely answer; full body only when small or explicitly requested. +- Concept search: return the relevant slice first, with a compact enclosing + symbol header; full body only when confidence that the whole body matters is + high. +- Literal/error search: stay very compact by default, since current grep output + already beats `rg` on token count. +- Relations/impact: return a compact graph first, then expandable ids. +- Repeated files: group or compress repeated headers only when the formatter can + prove it saves tokens without hurting copy/paste location clarity. + +Possible option: + +```text +context=auto|min|body|graph +``` + +`auto` should optimize for one-call resolution under the default budget. + +### 6. Prewarm the daemon path + +Cold latency is the broadest remaining benchmark loss. Warm path is already the +strong story. + +Ideas: + +- `codesift daemon start` for users who want a persistent local sidecar. +- `codesift init` writes client config that points at a warm HTTP server when + appropriate. +- Longer idle timeout for repos used in active agent sessions. +- Cheap daemon health probe that avoids loading the heavy path just to discover + the daemon is alive. +- Benchmark first MCP result separately for: process spawn, daemon connect, + repo open, SQLite ready, first query. + +The product goal should be "first useful answer feels instant after setup", not +just "warm query is fast in isolation." + +### 7. Build one-command onboarding + +Add a guided `codesift init` command. + +It should: + +- Check Node version and native SQLite compatibility. +- Install or verify the index. +- Run a tiny smoke query against the current repo. +- Offer to write MCP config snippets for supported clients. +- Explain local/offline trust posture in one line. +- Print two repo-specific example queries based on detected symbols/files. +- Tell the user whether a daemon/HTTP sidecar is running. + +Add `codesift doctor` for failure recovery: + +- Node version mismatch. +- Native module ABI mismatch. +- Missing `rg` for eval/dev flows. +- Corrupt or incompatible index. +- Cloud provider selected without key. +- Secrets blocking cloud embedding. + +The first experience should be: install, run init, ask a question. + +### 8. Measure real agent loops + +The eval harness is much more honest now, but routing is still deterministic. +The next eval should simulate real agent behavior. + +Add tasks like: + +- "Where is this auth behavior enforced?" +- "What breaks if I change this function?" +- "Find the implementation behind this stack trace." +- "Update this code safely." +- "Why does this validation reject this input?" + +Track: + +- Tool calls to final answer. +- Tokens to final answer. +- Whether the first tool choice was correct. +- Whether the agent stopped after enough context. +- Wrong edit rate or wrong-file rate. +- Recovery cost after ambiguous or stale results. + +This will tell whether `ask_code`, confidence metadata, and minimal toolsets +actually reduce agent waste. + +### 9. Expand accuracy coverage before semantic defaults + +Useful additions: + +- Paraphrase goldens for each concept query. +- Multi-file "answer set" goldens beyond same-name collisions. +- Relation goldens for callers, refs, importers, implementers, and impact. +- Stack trace and error-message goldens. +- Large-repo fixtures that stress ranking and token budgets. +- "False friend" fixtures where docs, tests, generated files, and interfaces + contain the same words as the real implementation. + +Do not ship a default semantic arm just because concept search is important. +Ship it when expanded eval proves it beats the lexical/ranking stack without +hurting cold start, onboarding, privacy, or rebuild cost. + +### 10. Keep relation bundles opt-in until they have budgets + +Relation context is a strong one-call lever, especially for "what breaks if I +change X?" But default relation expansion can become expensive. + +Good next step: + +- Keep `with_callers` / graph context opt-in. +- Add hard candidate, time, and token budgets. +- Measure saved calls and missed relation files. +- Promote relation bundles into `ask_code` only for intents that clearly need + them. + +## Prioritized roadmap + +### P0: tighten the current surface + +- Sync README/docs with the live tool list. +- Fix the current symbol precision loss. +- Add `answer_complete`, `confidence`, and `next` metadata to formatters. +- Change MCP instructions so `index_status` is diagnostic, not preflight. +- Add token-budget tests for adaptive exact-symbol and concept responses. + +### P1: make the first call obvious + +- Build `ask_code` as the default MCP front door. +- Add a minimal default toolset and an advanced toolset. +- Implement `codesift init` and `codesift doctor`. +- Generate MCP config snippets for common clients. + +### P2: reduce token and cold-start losses + +- Add adaptive context modes. +- Slice bodies around query-relevant lines before full-body fallback. +- Add daemon prewarm/service flow. +- Instrument cold path into spawn/connect/open/query phases. + +### P3: broaden the moat + +- Add real agent-loop eval. +- Add relation and impact eval gates. +- Run gated learned/reranker A/Bs on expanded concept goldens. +- Consider a default local learned arm only if it clears accuracy, latency, + trust, and onboarding gates. + +## Ideas to avoid for now + +- Do not default to cloud embeddings or cloud reranking. +- Do not expose more top-level tools as the onboarding answer. +- Do not auto-bundle callers/usages for every result without strict budgets. +- Do not optimize only the benchmark if it makes the first user experience more + complicated. +- Do not replace `rg`; keep it as the known-literal fallback and win on + structural context. + +## Success metrics + +Product metrics: + +- Time from install to first successful answer. +- Percentage of users who complete setup without reading extra docs. +- Number of commands required for first useful MCP answer. +- Number of support/debug paths caught by `doctor`. + +Agent metrics: + +- Median tool calls to resolution. +- Median tokens to resolution. +- First-tool correctness. +- Stop-after-sufficient-context rate. +- Wrong-file and wrong-edit rate. + +Engine metrics: + +- Warm first-result latency. +- Cold first-result latency split by phase. +- Token loss count by query type. +- Precision loss count. +- Recall on multi-target and relation tasks. + +The north-star version: a new user runs one command, asks one question, and the +agent receives enough fresh, accurate context to act without another search. diff --git a/docs/north-goal-plan.md b/docs/north-goal-plan.md new file mode 100644 index 0000000..7bc7c5b --- /dev/null +++ b/docs/north-goal-plan.md @@ -0,0 +1,336 @@ +# codesift north-goal plan (merged & committed) + +> **North goal:** give an AI agent the right context with **(1) fewer tool calls, +> (2) less token output, (3) more accurate context**, and **(4) onboarding that +> feels close to zero setup** — "answer in one call, in fewer tokens than grep, +> never slower," with rg's freshness and zero-egress trust intact. + +This is the committed prioritization that merges two idea sources: + +- `docs/north-goal-ideas.md` — GPT‑5.5's forward idea stack (front-door tool, + metadata, adaptive output, onboarding, measurement). +- The 8-lens / 17-agent ideation workflow (code-grounded, adversarially verified + against `main`; 45 ideas survived, 3 cut). + +Where the two disagreed, the resolution and its grounding are recorded in +[§ Decisions](#decisions-where-the-two-sources-disagreed). The `file:line` +anchors below were spot-verified against current `main` (post graph-landing: +the `edges` table and `find_callers/refs/importers/who_implements/impact` +ship). Anchors are accurate as of this writing but a few may drift by several +lines as the files evolve — confirm the surrounding code before editing, not +just the line number. + +--- + +## The through-line + +The relational graph (`edges` table) is **paid-for but under-exploited**. The +largest goal-A wins are not new machinery — they are *spending the edges we +already persist at index time* to collapse the agent's multi-call investigation +loop, plus closing honesty gaps that silently cost a second call. Onboarding is +a separate, cheaper track that mostly unblocks from one missing helper +(`findRepoRoot()`). The front-door (`ask_code`) is the right long-term shape for +"make the right one call obvious" — but it is gated on a real agent-loop eval, +because codesift's intent routing is currently asserted, not tested. + +**The single most-repeated trap:** a ranking signal must be a *capped tiebreak*, +never a primary-score multiplier. `repo.ts:4671` documents a coverage multiplier +that demoted a correct rank‑1 and was reverted. Centrality, edge-proximity, and +orphan signals are all tiebreak-only. + +--- + +## Roadmap + +Five tracks. **P0 → P2** are independent and can run in parallel. **P3 +(measurement) gates P4 (`ask_code`) and the ranking half of P5.** + +### P0 — Trust & honesty fixes (cheap, unblock everything) + +These convert silent failures into one-call answers and fix concrete bugs. Most +are S effort, pure formatter/SQL. + +- **`not_indexed` sentinel + empty-result recovery hints.** Bare `[]` is + indistinguishable from a real miss today, which is *why* instructions still + mandate a defensive `index_status` preflight. Emit `not_indexed; run: codesift + index` from every formatter when `existsSync(indexPath)` is false; distinguish + "def found, 0 indexed edges" from "no definition" in `findDefinitionEdges` + (`repo.ts:1016` vs `:1026`); surface the already-computed `partialRows` + (`repo.ts:767`) on `find_symbol`'s exact-miss branch. **Prerequisite for + dropping the preflight (P1).** +- **Name-only blast-radius cap + lead line on `find_callers`/`find_refs`.** These + run *unlimited* name-only matches on Go/Java/Ruby/Rust + (`selectDefinitionEdgeRows` no limit, `repo.ts:1021`; `WHERE dst_file is null`, + `repo.ts:4267`) — a false-positive flood with no count summary. Add a low + default cap for name-only resolution + a `name_only_unscoped=N; narrow with + path_glob/kind` hint before the per-row `approx:name-only` tags. +- **Ambiguity-hint parity.** `search_code` emits `ambiguous: N defs` + (`ambiguousDefCount`, `types.ts:112`) but `find_symbol`/relation tools do not, + even though routing sends identifiers *to* `find_symbol`. The distinct-def + count is already in hand (`repo.ts:1015`) — free. Gate on distinct file+kind so + overloads don't false-positive. +- **mtime/size short-circuit before content read+hash in `scanRepository`.** + `scanRepository` reads + SHA‑256-hashes *every* file unconditionally + (`scan.ts:153-165`); `diffScannedFiles` (`repo.ts:1905`) then only *compares* + those hashes. So each actual triggered sync re-reads + re-hashes the whole repo + even when almost nothing changed. (The watch *poll* itself is already cheap — it + uses the stat-only `scanRepositoryManifest`, `scan.ts` / `repo.ts:2008,2199` — + so this is a per-sync cost, not a per-idle-tick cost.) Fix: pass the indexed + file rows (`selectIndexedFileRows`) into `scanRepository` as `knownByPath` and + skip the `readFile`+hash when `size` + `mtime` match an indexed row, reusing the + stored hash; keep a full-hash sweep on rebuild / HEAD-change as the correctness + backstop. *Highest steady-state freshness win for actively-edited repos.* +- **Quick wins (S):** + - Fix the MCP-shim stderr banner — `mcp-shim.ts:12` hardcodes a stale 5-name + array; the daemon already serves all 10. Source it from an exported + constant so it can't drift again. + - Default CLI `sym` to `with_body` and render the inline body (`program.ts:351` + omits it while the MCP path defaults it on). + - Give CLI `search`/`grep` a default token budget matching MCP's (CLI is + unbounded unless `--max-tokens` is passed; `--max-tokens` is already wired). + - Tag `find_symbol` partial-fallback rows `matchQuality='partial'` + (`repo.ts:772`) so a `LIKE %name%` guess isn't rendered as exact. + - Add `annotations:{readOnlyHint:true, openWorldHint:false}` to every + `registerTool` (`mcp index.ts:743-772`) so harnesses can auto-approve. + +### P1 — Collapse the loop (goal A core; reuses shipped edges) + +- **`with_relations` on `search_code`, default-on under `autoSingleBest`.** When + the top hit resolves to one confident symbol (`autoSingleBest`, `repo.ts:623`, + `!ambiguousIdentifier`), attach the edge-table bundle (`readFindSymbolRelations`, + `repo.ts:970`) inside the existing budget cap. Collapses `search → find_symbol → + find_callers` into one call. No AST walk; relations are budget-fitted and + dropped first under pressure. *Highest call-reducer on the hottest path.* +- **Auto-include callers in `find_symbol` (budget-gated default).** `with_callers` + already uses the edge path; flip it on when `canEnrichTopExactRow` + (`repo.ts:774`) fires and a min-budget check passes. Ship after `with_relations` + so both share the relations renderer. +- **`changeset_context`** — given an explicit file list, return each file's + symbols + direct (depth‑1) importers/callers in one call. The "what does my + diff touch" answer rg structurally cannot give. Explicit file list is the + PRIMARY input; git-diff resolution is opt-in only (preserves zero-shell + posture). Hard node cap + token budget + index-staleness surfaced. +- **Drop the `index_status` preflight from instructions** (`mcp index.ts:226`) — + reword to reactive (health attached to normal results; auto-index when safe). + **Only after the `not_indexed` sentinel ships**, else this deletes the only + proactive health signal with no replacement. +- **Adaptive output** (`context=auto|min|body|graph`, `auto` = optimize for + one-call resolution under budget): + - `find_symbol` `detail:'sig'` tier — emit the stored `signature` column + (~10–30 tok vs `INLINE_BODY_MAX_TOKENS=400`), falling back to body when null. + - Concept search: relevant slice first with a compact enclosing-symbol header; + full body only when confidence the whole body matters is high. + - Header-dedup of per-line numbers **only on the provably-contiguous + inline-body path** (one `@` header + raw dedented code); never on + centered/non-contiguous snippets (absolute-line math breaks). Zero + `PREFIX_TOKEN_COST` in `renderedBodyTokens` for that renderer or the budget + over-counts. + +### P2 — Onboarding (goal B; orthogonal track) + +Unblocks from one missing helper. Sequence: + +1. **`codesift doctor`** (S, ship first) — preflight the documented + Node‑24/`better-sqlite3` ABI footgun. Wrap the `require('better-sqlite3')` in + try/catch matching `NODE_MODULE_VERSION` so it *diagnoses* rather than crashes + on the very error it reports. Reuse `supportedNodeMajors` + (`smoke-pack-install.mjs:11`) + `repo.status()`. Also covers: missing `rg`, + corrupt/incompatible index, cloud provider selected without key, secrets + blocking cloud embed, daemon socket reachability. +2. **Shared `findRepoRoot()` in core** — a NEW exported helper (dirname of the + `.git` location, handling the worktree `gitdir:` case). Do **not** reuse + `findGitDirectory` (`repo.ts:2112`) — it returns the `.git` path and is + module-private. Immediately wire it into the MCP shim's root resolution + (`mcp-shim.ts:16`, only when no explicit `[path]` arg) so subdir launches stop + silently indexing the wrong subtree. +3. **`codesift init [path]`** — walk up for root, run `sync()` with progress, + run a tiny smoke query, then **merge-write** the MCP config (never overwrite; + `.bak` backup; single `codesift` server key; fall back to `--print` on any + ambiguity). Support `--print` and `--client`. Print two repo-specific example + queries from detected symbols; state the local/offline posture in one line. +4. **Eager initial sync on first `watch()`/MCP connect** — `watch()` ends with + `refreshWatchers()` and a 1s safety poll (`repo.ts:1266`) but no immediate + sync, so an agent on a never-indexed checkout races an empty index and gets + `[]`. Insert `void runSync(true)` after `refreshWatchers()`, gated by the + `onlyIfStale` status check; non-blocking — return an "indexing, N files done" + status and let the agent poll. +5. **npx pinned-version wrapper** — have `init`'s generated config optionally + emit `npx -y codesift@ mcp `. **Pin the version, never + `@latest`** (it re-resolves each cold start and can re-trigger a + `better-sqlite3` prebuild mismatch). Extend `smoke-pack-install.mjs` to + exercise `npx codesift init --print`. +6. **Daemon prewarm UX + cold-path phase instrumentation** — `codesift daemon + start` for a persistent sidecar; longer idle timeout for active sessions; a + cheap health probe that doesn't load the heavy path. Split the cold-start + benchmark into spawn / daemon-connect / repo-open / SQLite-ready / first-query + so "first useful answer feels instant after setup" becomes measurable. + +### P3 — Measurement (prerequisite gate for P4 and ranking in P5) + +Both sources agree this is the missing scaffolding; the workflow goes further and +flags that **several ranking bets are unfalsifiable without it** — so it is a +gate, not a nice-to-have, and ranks higher than GPT‑5.5's P3 placement. + +- **Real agent-loop eval** — replace the deterministic `queryType→tool` switch + with simulated agent tasks ("where is auth enforced", "what breaks if I change + X", "find the impl behind this stack trace", "update this safely"). Track: + calls-to-final-answer, tokens-to-final-answer, first-tool correctness, + stop-after-sufficient-context rate, wrong-file/wrong-edit rate, recovery cost + after ambiguous/stale results. +- **Expanded goldens** — paraphrase per concept query; multi-file answer-set + (beyond same-name collisions); relation/impact goldens; stack-trace & + error-message goldens; large-repo ranking/budget stress; "false friend" + fixtures (docs/tests/generated/interfaces sharing the impl's words). +- **Ambiguous-identifier collision fixture** — blocks the degree-collapse half of + P5 ranking. +- **Cold-start first-index benchmark** — blocks any first-index parallelism work; + don't optimize an unmeasured cost. + +### P4 — The front door (`ask_code`) — gated on P3 + +- **Ship `ask_code` as the default-recommended MCP tool**: accepts a natural task, + infers intent server-side, and returns the resolved answer with a terse header + envelope (`answer_complete`, `confidence`, `intent`, `next`, `freshness`, + `ambiguity`, `omitted`). This envelope is the home for GPT‑5.5's metadata — + **not** a second parallel grammar bolted onto the specialist tools. +- **Keep all specialist tools registered.** `ask_code` *removes* the agent's + routing decision (its value); it must not *hide* the specialists (recovery + + power-user path). Lead with `ask_code` in `MCP_SERVER_INSTRUCTIONS`; do not + ship a minimal default toolset that hides `find_callers`/`impact` etc. — + revisit hiding only if the P3 eval proves it reduces agent waste. +- **Promote relation bundles into `ask_code` only for intents that need them** + (e.g. "what breaks if I change X"), under the same budgets as P1. + +### P5 — Broaden the moat (graph capability + ranking; ranking gated on P3) + +Graph capabilities (cheap on persisted edges, mostly independent of eval): + +- **`find_unreferenced`** — zero-inbound-edge dead-symbol detection; a single + indexed anti-join. Scope HARD to import-resolved languages on **both** symbol + and edge sides; union the default-export alias or every default export + false-flags as dead; label `candidate-unreferenced (excludes dynamic/string + dispatch, DI/reflection, test-only)`. +- **`impact` reverse direction (callees / outbound fan-out)** — BFS scaffolding is + direction-agnostic; swap dst-keyed for src-keyed select (`idx_edges_src`). + Filter null-`src_symbol` rows; lower depth + honest label for name-only. +- **`find_tests`** (glob post-filter over `findReferences`, labeled "references + from test files", not coverage) and **`api_surface`** (reframed from + export-flagging — which would be an O(files) per-query parse, DEAD — to "module + symbols ranked by external cross-file fan-in", showing fan-in=0 symbols). +- **`find_cycles`** (niche) — bounded Tarjan over import edges; TS/JS/Python only, + labeled so a Go repo's empty result isn't read as "no cycles". + +Ranking (all tiebreak-only; gated on P3 precision axis): + +- **Degree-order colliding definitions** — sort the ambiguous top‑3 by + import-resolved in-degree (`countDefinitionEdgeRows`, bounded to the tiny + collision set) before `stableChunkSortKey` (`repo.ts:4608`). Ship the + *ordering* half; defer dominance-collapse (≥3× → `single_best`) until the + collision fixture exists (it flips recall). +- **Centrality + edge-proximity as capped saturating tiebreaks** — bounded + query-time in-degree lookup over the top‑25 candidates (NOT a denormalized + column); insert after coverage/kind in the tiebreak chain. Never a primary-score + multiplier. +- **Conservative query-intent arm reweight** — branch the static exact-arm RRF + weight (`repo.ts:4579`) on the existing `isSymbolDominatedQuery` gate between + two presets, confident-trigger only, defaulting to the current value. Stays in + RRF reciprocal-rank space (does not violate the no-weighted-fusion kill). +- **Learned/reranker** — gated A/Bs on expanded concept goldens; a default local + learned arm only if it clears accuracy, latency, trust, onboarding, and rebuild + gates. Cloud stays opt-in. + +--- + +## Decisions (where the two sources disagreed) + +| Topic | GPT‑5.5 (`north-goal-ideas.md`) | Workflow | **Decision** | +|---|---|---|---| +| **`ask_code` front-door** | Headline bet; default path | Did not propose; flagged tool-merge risk | **Adopt, gated on P3 eval.** It *removes* the routing decision (unlike a mode-enum that relocates it). Keep specialists registered. | +| **Minimal default toolset (3)** | Default; hide specialists behind `--toolset all` | Guardrail: don't break cached tool lists / recovery | **Reject as default.** Lead with `ask_code` in instructions; keep all tools; revisit hiding only if P3 proves it helps. | +| **Relation bundles** | Keep opt-in until budgeted (#10) | Default-on under `autoSingleBest` (#1) | **Default-on, budget-gated.** GPT's O(files)-AST fear is retired post-graph; edges are persisted + budget-fitted. Apply GPT's budget discipline. | +| **`code_graph` mode-enum** | Offered as a compromise | Guardrail: don't hard-replace relation tools | **Reject.** Breaking change to cached lists + instructions + tests; merely moves tool-pick to mode-pick. | +| **Rich `key=value` metadata on every tool** | Yes (#4) | Skeptic killed a 2nd `#hints` grammar | **Split:** envelope lives inside `ask_code`; specialist tools extend existing terse inline tokens (`ambiguous: N`, `not_indexed`, `name_only_unscoped=N`, `[stale]`). | +| **Drop status preflight** | Yes (#2) | Yes, but sequence after `not_indexed` (#7) | **Adopt with sequencing:** sentinel first, reword instruction last. | +| **Measurement priority** | P3 | Prerequisite for ranking bets | **Raise to a gate** for P4 + P5-ranking. | + +--- + +## Guardrails (do NOT) + +1. **Never add a signal to the primary fused score** — even a small multiplier. + `repo.ts:4671` is the cautionary tale (correct rank‑1 demoted, reverted). + Centrality, proximity, orphan, entry-point: capped tiebreaks only. +2. **Do not denormalize a `fan_in`/in-degree column** — `applySyncChanges` + re-extracts edges per *changed* file only (`repo.ts:1715/1731`), so a symbol's + in-degree goes stale the moment a *caller's* file changes. Compute degree as a + bounded query-time lookup over `idx_edges_dst` on the candidate set. +3. **No O(files) AST walk on any query path.** The persisted edge table is the + only sanctioned relational path. `api_surface` export-flagging would + reintroduce a per-query parse — reframe to index-time fan-in. +4. **Keep the import-resolved / name-only scope gate on BOTH sides** of any + anti-join, cycle, or unreferenced query. Resolved edges carry non-null + `dst_file`; Go/Java/Ruby/Rust carry `dst_file=null`. A naive clause lets an + unrelated name-only edge mask a genuinely-dead TS symbol — always label. +5. **Do not shell out to git on the default path** (`changeset_context`, init + root-detection). Explicit file list / explicit path is primary; git-diff / + `.git` walk-up is opt-in convenience only. +6. **Do not reuse `findGitDirectory` as a root helper** — it returns the `.git` + path and is private. Build one new exported `findRepoRoot()`. +7. **`init` must never clobber an MCP client config** — detect-and-print first, + merge a single key with a `.bak` backup, fall back to print on ambiguity. +8. **Do not pin npx to `@latest`** in generated config — pin an exact version. +9. **Do not reword the preflight instruction before the `not_indexed` sentinel + ships.** +10. **Do not write the live DB in-place while queries may read** — keep the + shadow-copy + atomic-rename design; only apply in-place when + `activeDatabaseUsers===0`, else fall back to shadow. +11. **Do not default-on body inlining for `find_callers`, and do not add a def + body to `impact`** — both shift budget away from the sites/blast-radius the + agent asked for. `with_def` stays opt-in on `find_callers`/`find_refs` only. +12. **Do not dedup line-numbers on centered/non-contiguous snippets** — per-line + numbers are the addressability contract; only the provably-contiguous + inline-body path is safe. +13. **Do not optimize cold first-index parallelism before the benchmark exists** + (worker_threads startup hurts small repos; P0's mtime short-circuit already + makes warm resyncs near-free). +14. **Do not ship a default semantic/cloud arm** to "fix" concept search — local, + offline, zero-egress stays the default; learned arms only behind explicit + proof + explicit user choice. +15. **Do not replace `rg`** — keep it as the known-literal fallback; win on + structural + relational context. + +--- + +## Success metrics + +**Product:** time from install to first successful answer; % completing setup +without extra docs; commands required for first useful MCP answer; debug paths +caught by `doctor`. + +**Agent:** median tool calls to resolution; median tokens to resolution; +first-tool correctness; stop-after-sufficient-context rate; wrong-file/wrong-edit +rate; recovery cost after ambiguous/stale results. + +**Engine:** warm first-result latency; cold first-result latency split by phase; +token-loss count by query type; precision-loss count; recall on multi-target and +relation tasks. + +**North-star:** a new user runs one command, asks one question, and the agent +receives enough fresh, accurate context to act without another search. + +--- + +## Sequencing summary + +``` +P0 trust/honesty ──┐ (independent, mostly S — start now) +P1 collapse-loop ──┤ (P1 preflight-reword waits on P0 sentinel) +P2 onboarding ─────┘ + │ +P3 measurement ────┴──► gates ──► P4 ask_code + P5 ranking (graph capability half is independent) +``` + +P0/P1/P2 run in parallel. P3 is the unlock for the front-door and the ranking +bets — bring it forward, because without it the highest-leverage accuracy work +is unfalsifiable. diff --git a/packages/cli/src/mcp-shim.ts b/packages/cli/src/mcp-shim.ts index 697352f..4056e04 100644 --- a/packages/cli/src/mcp-shim.ts +++ b/packages/cli/src/mcp-shim.ts @@ -2,6 +2,9 @@ import { spawn } from 'node:child_process' import { createConnection } from 'node:net' import { resolve } from 'node:path' +import { findRepoRoot } from '@codesift/core' +import { MCP_TOOL_NAMES } from '@codesift/mcp' + import { getDefaultDaemonSocketPath } from './daemon-path.js' interface DaemonReply { @@ -9,11 +12,11 @@ interface DaemonReply { error?: string } -const MCP_TOOL_NAMES = ['search_code', 'find_symbol', 'grep_code', 'read_chunk', 'index_status'] const DAEMON_CONNECT_TIMEOUT_MS = 3000 export async function runMcpShim(argv = process.argv): Promise { - const repoRoot = resolve(readPathArgument(argv) ?? process.cwd()) + const explicitPath = readPathArgument(argv) + const repoRoot = explicitPath ? resolve(explicitPath) : await findRepoRoot(process.cwd()) const socketPath = process.env.CODESIFT_DAEMON_SOCKET ?? getDefaultDaemonSocketPath() await ensureDaemon(socketPath) diff --git a/packages/cli/src/program.ts b/packages/cli/src/program.ts index 489a9f0..5bee259 100644 --- a/packages/cli/src/program.ts +++ b/packages/cli/src/program.ts @@ -1,13 +1,19 @@ -import { rm } from 'node:fs/promises' -import { resolve } from 'node:path' +import { createRequire } from 'node:module' +import { spawnSync } from 'node:child_process' +import { existsSync } from 'node:fs' +import { copyFile, mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises' +import { createConnection } from 'node:net' +import { dirname, resolve } from 'node:path' import { Command } from 'commander' import { DEFAULT_SEARCH_K, IndexCompatibilityError, + findRepoRoot, getDefaultEmbeddingProvider, isLearnedEmbeddingProvider, + listEmbeddingProviders, openRepo, readConfig, setConfigValue, @@ -19,7 +25,17 @@ import { type SymbolKind, type SyncOptions } from '@codesift/core' -import { createHttpServerHandle, createStdioServer, getToolDefinitions } from '@codesift/mcp' +import { + DEFAULT_MCP_GREP_MAX_TOKENS, + DEFAULT_MCP_SEARCH_MAX_TOKENS, + NOT_INDEXED_SENTINEL, + createHttpServerHandle, + createStdioServer, + formatMcpGrepHits, + getToolDefinitions +} from '@codesift/mcp' +import { getDefaultDaemonSocketPath } from './daemon-path.js' +import { supportedNodeMajors } from './supported-node-majors.js' export interface CliIo { stdout(message: string): void @@ -35,6 +51,10 @@ const defaultIo: CliIo = { } } +const require = createRequire(import.meta.url) +const SUPPORTED_NODE_MAJORS = new Set(supportedNodeMajors) +const DEFAULT_INIT_CLIENT = 'project' + export function getCliDescription(): string { const provider = getDefaultEmbeddingProvider() return isLearnedEmbeddingProvider(provider) @@ -46,6 +66,7 @@ export function formatStatus(status: RepoStatus): string { return [ `root: ${status.root}`, `index: ${status.indexPath}`, + `index exists: ${status.indexExists ? 'yes' : 'no'}`, `indexed: ${status.indexed ? 'yes' : 'no'}`, `stale: ${status.stale ? 'yes' : 'no'}`, ...(status.staleReasons?.length ? [`stale reasons: ${status.staleReasons.map((reason) => reason.message).join('; ')}`] : []), @@ -126,7 +147,9 @@ export function formatSymbols(definitions: SymbolDefinition[]): string { return definitions .map((definition) => { const range = `${definition.range.startLine}-${definition.range.endLine}` - return `${definition.kind} ${definition.name} — ${definition.file}:${range}` + const quality = definition.matchQuality === 'partial' ? ' [partial]' : '' + const header = `${definition.kind} ${definition.name}${quality} — ${definition.file}:${range}` + return definition.body ? `${header}\n${definition.body}` : header }) .join('\n') } @@ -218,6 +241,156 @@ async function withCompatibilityHandling(io: CliIo, action: () => Promise) } } +function packageVersion(): string { + try { + return String(require('../package.json').version ?? '0.0.0') + } catch { + return '0.0.0' + } +} + +function doctorLine(ok: boolean, label: string, detail: string): string { + return `${ok ? 'ok' : 'warn'} ${label}: ${detail}` +} + +async function runDoctor(path: string): Promise { + const root = await findRepoRoot(path) + const lines: string[] = [] + const nodeMajor = Number.parseInt(process.versions.node.split('.')[0] ?? '', 10) + lines.push(doctorLine(SUPPORTED_NODE_MAJORS.has(nodeMajor), 'node', `${process.versions.node} (supported: ${supportedNodeMajors.join(', ')})`)) + lines.push(doctorLine(true, 'better-sqlite3', 'native module loaded through @codesift/core')) + + const rg = spawnSync(process.platform === 'win32' ? 'rg.exe' : 'rg', ['--version'], { encoding: 'utf8' }) + lines.push(doctorLine(rg.status === 0, 'rg', rg.status === 0 ? firstLine(rg.stdout) : 'missing from PATH')) + + const repo = await openRepo(root) + try { + try { + const status = await repo.status() + lines.push(doctorLine(status.indexExists, 'index', status.indexExists ? `${status.chunkCount} chunks, compat=${status.compatibility.ok ? 'ok' : status.compatibility.code ?? 'mismatch'}` : 'missing; run codesift index')) + if (status.sync.state === 'failed' || status.sync.state === 'aborted') { + lines.push(doctorLine(false, 'sync', `${status.sync.state}: ${status.sync.error ?? 'no error recorded'}`)) + } else { + lines.push(doctorLine(true, 'sync', status.sync.state)) + } + } catch (error) { + lines.push(doctorLine(false, 'index', `status unavailable: ${compactLine(extractDoctorError(error), 180)}`)) + lines.push(doctorLine(false, 'sync', 'unknown; repair or rebuild the index')) + } + } finally { + await repo.close() + } + lines.push(await daemonDoctorLine()) + + const config = readConfig(root) + const providerId = config.provider?.trim() + const provider = providerId ? listEmbeddingProviders().find((candidate) => candidate.id === providerId) : undefined + if (provider?.isLearned) { + const keyName = provider.id.startsWith('openai') ? 'OPENAI_API_KEY' : provider.id.startsWith('voyage') ? 'VOYAGE_API_KEY' : '' + lines.push(doctorLine(!keyName || Boolean(process.env[keyName]?.trim()), 'cloud-key', keyName ? `${keyName} ${process.env[keyName]?.trim() ? 'set' : 'missing'}` : `${provider.id} selected`)) + lines.push(doctorLine(config.allowSecrets === true, 'cloud-secrets', config.allowSecrets === true ? 'allowSecrets enabled' : 'secret scan blocks cloud sends unless --allow-secrets is used')) + } else { + lines.push(doctorLine(true, 'egress', 'local/offline provider path')) + } + + return lines.join('\n') +} + +function extractDoctorError(error: unknown): string { + const message = error instanceof Error ? error.message : String(error) + const abi = process.versions.modules ? ` NODE_MODULE_VERSION=${process.versions.modules}` : '' + return /NODE_MODULE_VERSION|ERR_DLOPEN_FAILED|better-sqlite3/i.test(message) + ? `${message}${abi}; reinstall with a supported Node major` + : message +} + +async function daemonDoctorLine(): Promise { + const socketPath = process.env.CODESIFT_DAEMON_SOCKET ?? getDefaultDaemonSocketPath() + if (!process.env.CODESIFT_DAEMON_SOCKET && !existsSync(socketPath)) { + return doctorLine(true, 'daemon', 'not running; direct CLI/MCP startup is available') + } + + const reachable = await canConnectToDaemonSocket(socketPath) + return doctorLine(reachable, 'daemon', reachable ? `socket reachable: ${socketPath}` : `socket not reachable: ${socketPath}`) +} + +async function canConnectToDaemonSocket(socketPath: string): Promise { + return new Promise((resolveConnect) => { + const socket = createConnection(socketPath) + let settled = false + const settle = (reachable: boolean) => { + if (settled) { + return + } + settled = true + socket.destroy() + resolveConnect(reachable) + } + socket.setTimeout(300) + socket.once('connect', () => settle(true)) + socket.once('error', () => settle(false)) + socket.once('timeout', () => settle(false)) + }) +} + +function firstLine(value: string | undefined): string { + return (value ?? '').split(/\r?\n/)[0]?.trim() || 'available' +} + +function compactLine(value: string, maxChars: number): string { + const compact = value.replace(/\s+/g, ' ').trim() + return compact.length <= maxChars ? compact : `${compact.slice(0, Math.max(0, maxChars - 1)).trimEnd()}…` +} + +function buildMcpServerConfig(root: string): Record { + return { + command: 'npx', + args: ['-y', `codesift@${packageVersion()}`, 'mcp', root] + } +} + +function formatMcpConfig(root: string): string { + return JSON.stringify({ mcpServers: { codesift: buildMcpServerConfig(root) } }, null, 2) +} + +async function mergeWriteProjectMcpConfig(root: string): Promise { + const target = resolve(root, '.mcp.json') + let existing: Record = {} + try { + existing = JSON.parse(await readFile(target, 'utf8')) as Record + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { + return formatMcpConfig(root) + } + } + + const backup = `${target}.bak` + try { + await copyFile(target, backup) + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { + throw error + } + } + + const mcpServers = typeof existing.mcpServers === 'object' && existing.mcpServers !== null + ? { ...(existing.mcpServers as Record) } + : {} + mcpServers.codesift = buildMcpServerConfig(root) + const next = { ...existing, mcpServers } + await mkdir(dirname(target), { recursive: true }) + const temp = `${target}.${process.pid}.tmp` + await writeFile(temp, `${JSON.stringify(next, null, 2)}\n`, 'utf8') + await rename(temp, target) + return `wrote ${target}${Object.keys(existing).length > 0 ? ` (backup: ${backup})` : ''}` +} + +async function exampleQueries(repo: Awaited>): Promise { + const hits = await repo.search('function class method interface', { k: 4, context: 'min', maxTokens: 120 }) + const symbols = [...new Set(hits.map((hit) => hit.symbol).filter((symbol): symbol is string => Boolean(symbol)))] + return symbols.slice(0, 2).map((symbol) => `sym ${symbol}`) +} + async function waitForTermination(stop: () => Promise): Promise { await new Promise((resolve, reject) => { let stopping = false @@ -242,6 +415,13 @@ export async function runCli(argv = process.argv, io: CliIo = defaultIo): Promis program.name('codesift').description(getCliDescription()).version('0.0.0') + program + .command('doctor') + .argument('[path]', 'repository path', process.cwd()) + .action(async (path: string) => { + io.stdout(await runDoctor(path)) + }) + program .command('index') .argument('[path]', 'repository path', process.cwd()) @@ -269,6 +449,48 @@ export async function runCli(argv = process.argv, io: CliIo = defaultIo): Promis } }) + program + .command('init') + .argument('[path]', 'repository path', process.cwd()) + .option('--print', 'print MCP config instead of writing it') + .option('--client ', 'client config target (project only; other values print)', DEFAULT_INIT_CLIENT) + .action(async (path: string, options: { print?: boolean; client: string }) => { + const root = await findRepoRoot(path) + if (options.print) { + io.stdout(formatMcpConfig(root)) + return + } + + const repo = await openRepo(root) + try { + io.stdout(`Initializing ${root}`) + const result = await repo.sync({ + onProgress(event) { + io.stderr(`indexing batch ${event.batch}/${event.totalBatches} (${event.completedChunks}/${event.totalChunks} chunks)`) + } + }) + const examples = await exampleQueries(repo) + const smoke = await repo.search(examples[0]?.replace(/^sym\s+/, '') || 'function', { k: 1, context: 'min', maxTokens: 80 }) + if (smoke.length === 0) { + io.stderr('smoke query returned no hits; index exists but no symbols were detected') + } + if (options.client !== DEFAULT_INIT_CLIENT) { + io.stdout(formatMcpConfig(root)) + io.stderr(`client=${options.client} is ambiguous here; printed config instead of writing`) + return + } + + io.stdout(await mergeWriteProjectMcpConfig(root)) + io.stdout(`Indexed ${result.indexedFiles} files (${result.skippedFiles} skipped, ${result.skippedSymlinks} symlink skips).`) + io.stdout('Local/offline by default; cloud embedding providers are opt-in only.') + if (examples.length > 0) { + io.stdout(`Try:\n${examples.map((example) => ` codesift ${example} --repo ${root}`).join('\n')}`) + } + } finally { + await repo.close() + } + }) + program .command('search') .argument('', 'natural language or symbol-aware query') @@ -278,7 +500,7 @@ export async function runCli(argv = process.argv, io: CliIo = defaultIo): Promis .option('--path ', 'path glob filter, e.g. src/**') .option('--kind ', 'symbol kind filter for matching chunks') .option('--max-tokens ', 'token budget for compact snippets') - .option('--context ', 'inline policy: sig or body') + .option('--context ', 'output policy: auto, min, sig, body, or graph') .option('--with-usages', 'bundle top-N import-resolved/local usage sites for the top definition hit') .option('--rerank', 'opt-in reranker re-scoring for NL-concept queries (requires a configured reranker)') .option('--json', 'print JSON results') @@ -286,57 +508,63 @@ export async function runCli(argv = process.argv, io: CliIo = defaultIo): Promis .action( async ( query: string, - options: { k: string; repo: string; lang?: string; path?: string; kind?: string; maxTokens?: string; context?: 'sig' | 'body'; withUsages?: boolean; rerank?: boolean; json?: boolean; compact?: boolean } + options: { k: string; repo: string; lang?: string; path?: string; kind?: string; maxTokens?: string; context?: 'auto' | 'min' | 'sig' | 'body' | 'graph'; withUsages?: boolean; rerank?: boolean; json?: boolean; compact?: boolean } ) => { await withCompatibilityHandling(io, async () => { const repo = await openRepo(options.repo) - const languages = parseCsvList(options.lang) - const searchOptions: { - k: number - lang?: string[] - pathGlob?: string - kind?: SymbolKind - maxTokens?: number - context?: 'sig' | 'body' - withUsages?: boolean - rerank?: boolean - } = { - k: Number(options.k) - } - - if (languages) { - searchOptions.lang = languages - } - - if (options.path) { - searchOptions.pathGlob = options.path - } - - if (options.kind) { - searchOptions.kind = options.kind as SymbolKind - } - - if (options.maxTokens !== undefined) { - searchOptions.maxTokens = Number(options.maxTokens) - } - if (options.context !== undefined) { - searchOptions.context = options.context - } - if (options.withUsages) { - searchOptions.withUsages = true - } - if (options.rerank) { - searchOptions.rerank = true - } - - const hits = await repo.search(query, searchOptions) - const output = options.json ? JSON.stringify(hits, null, 2) : options.compact ? formatCompactHits(hits) : formatHits(hits) - const status = await repo.status() - - io.stdout(output) - - if (status.vectorSearch.state === 'unavailable' && status.vectorSearch.message) { - io.stderr(formatVectorStatus(status)) + try { + const status = await repo.status() + if (!status.indexExists) { + io.stdout(NOT_INDEXED_SENTINEL) + return + } + const languages = parseCsvList(options.lang) + const searchOptions: { + k: number + lang?: string[] + pathGlob?: string + kind?: SymbolKind + maxTokens?: number + context?: 'auto' | 'min' | 'sig' | 'body' | 'graph' + withUsages?: boolean + rerank?: boolean + } = { + k: Number(options.k), + maxTokens: options.maxTokens === undefined ? DEFAULT_MCP_SEARCH_MAX_TOKENS : Number(options.maxTokens) + } + + if (languages) { + searchOptions.lang = languages + } + + if (options.path) { + searchOptions.pathGlob = options.path + } + + if (options.kind) { + searchOptions.kind = options.kind as SymbolKind + } + + if (options.context !== undefined) { + searchOptions.context = options.context + } + if (options.withUsages) { + searchOptions.withUsages = true + } + if (options.rerank) { + searchOptions.rerank = true + } + + const hits = await repo.search(query, searchOptions) + const output = options.json ? JSON.stringify(hits, null, 2) : options.compact ? formatCompactHits(hits) : formatHits(hits) + + io.stdout(output) + + if (status.vectorSearch.state === 'unavailable' && status.vectorSearch.message) { + io.stderr(formatVectorStatus(status)) + } + } finally { + await repo.close() } }) } @@ -351,21 +579,31 @@ export async function runCli(argv = process.argv, io: CliIo = defaultIo): Promis .action(async (name: string, options: { repo: string; path?: string; kind?: string }) => { await withCompatibilityHandling(io, async () => { const repo = await openRepo(options.repo) - const findOptions: { - pathGlob?: string - kind?: SymbolKind - } = {} + try { + const status = await repo.status() + if (!status.indexExists) { + io.stdout(NOT_INDEXED_SENTINEL) + return + } + const findOptions: { + pathGlob?: string + kind?: SymbolKind + maxTokens?: number + } = {} - if (options.path) { - findOptions.pathGlob = options.path - } + if (options.path) { + findOptions.pathGlob = options.path + } - if (options.kind) { - findOptions.kind = options.kind as SymbolKind - } + if (options.kind) { + findOptions.kind = options.kind as SymbolKind + } - const definitions = await repo.findSymbol(name, findOptions) - io.stdout(formatSymbols(definitions)) + const definitions = await repo.findSymbol(name, findOptions) + io.stdout(formatSymbols(definitions)) + } finally { + await repo.close() + } }) }) @@ -384,6 +622,7 @@ export async function runCli(argv = process.argv, io: CliIo = defaultIo): Promis .option('--lang ', 'comma-separated language filter, e.g. ts,typescript,python') .option('--path ', 'path glob filter, e.g. src/**') .option('--max-matches ', 'maximum matches to print') + .option('--max-tokens ', 'token budget for text output') .option('--json', 'print JSON results') .option('--compact', 'print token-efficient compact results') .action( @@ -402,6 +641,7 @@ export async function runCli(argv = process.argv, io: CliIo = defaultIo): Promis lang?: string path?: string maxMatches?: string + maxTokens?: string json?: boolean compact?: boolean } @@ -413,43 +653,57 @@ export async function runCli(argv = process.argv, io: CliIo = defaultIo): Promis } const repo = await openRepo(options.repo) - const languages = parseCsvList(options.lang) - const grepOptions: GrepOptions = {} - - if (options.regex) { - grepOptions.regex = true - } - if (options.ignoreCase) { - grepOptions.ignoreCase = true - } - if (options.wordRegexp) { - grepOptions.wholeWord = true - } - if (options.multiline) { - grepOptions.multiline = true - } - if (languages) { - grepOptions.lang = languages + try { + const status = await repo.status() + if (!status.indexExists) { + io.stdout(NOT_INDEXED_SENTINEL) + return + } + const languages = parseCsvList(options.lang) + const grepOptions: GrepOptions = {} + + if (options.regex) { + grepOptions.regex = true + } + if (options.ignoreCase) { + grepOptions.ignoreCase = true + } + if (options.wordRegexp) { + grepOptions.wholeWord = true + } + if (options.multiline) { + grepOptions.multiline = true + } + if (languages) { + grepOptions.lang = languages + } + if (options.path) { + grepOptions.pathGlob = options.path + } + if (options.context !== undefined) { + grepOptions.contextLines = Number(options.context) + } + if (options.beforeContext !== undefined) { + grepOptions.beforeContextLines = Number(options.beforeContext) + } + if (options.afterContext !== undefined) { + grepOptions.afterContextLines = Number(options.afterContext) + } + if (options.maxMatches !== undefined) { + grepOptions.maxMatches = Number(options.maxMatches) + } + + const hits = await repo.grep(pattern, grepOptions) + const maxTokens = options.maxTokens === undefined ? DEFAULT_MCP_GREP_MAX_TOKENS : Number(options.maxTokens) + const output = options.json + ? JSON.stringify(hits, null, 2) + : options.compact + ? formatCompactGrepHits(hits) + : formatMcpGrepHits(hits, { maxTokens }) + io.stdout(output) + } finally { + await repo.close() } - if (options.path) { - grepOptions.pathGlob = options.path - } - if (options.context !== undefined) { - grepOptions.contextLines = Number(options.context) - } - if (options.beforeContext !== undefined) { - grepOptions.beforeContextLines = Number(options.beforeContext) - } - if (options.afterContext !== undefined) { - grepOptions.afterContextLines = Number(options.afterContext) - } - if (options.maxMatches !== undefined) { - grepOptions.maxMatches = Number(options.maxMatches) - } - - const hits = await repo.grep(pattern, grepOptions) - const output = options.json ? JSON.stringify(hits, null, 2) : options.compact ? formatCompactGrepHits(hits) : formatGrepHits(hits) - io.stdout(output) }) } ) diff --git a/packages/cli/src/supported-node-majors.ts b/packages/cli/src/supported-node-majors.ts new file mode 100644 index 0000000..6dc0595 --- /dev/null +++ b/packages/cli/src/supported-node-majors.ts @@ -0,0 +1 @@ +export const supportedNodeMajors = [20, 22] as const diff --git a/packages/cli/test/program.test.ts b/packages/cli/test/program.test.ts index 4196ca4..695b273 100644 --- a/packages/cli/test/program.test.ts +++ b/packages/cli/test/program.test.ts @@ -31,6 +31,7 @@ describe('codesift CLI formatters', () => { formatStatus({ root: '/tmp/codesift', indexPath: '/tmp/codesift/.codesift/index.db', + indexExists: false, indexed: false, stale: false, sync: { state: 'idle' }, @@ -95,6 +96,50 @@ describe('codesift CLI capability labels', () => { }) describe('codesift CLI end-to-end', () => { + it('doctor reports native, index, and daemon diagnostics without requiring an index', async () => { + const repoRoot = await mkdtemp(join(tmpdir(), 'codesift-cli-doctor-')) + temporaryDirectories.push(repoRoot) + + const messages: string[] = [] + const io: CliIo = { + stdout(message) { + messages.push(message) + }, + stderr(message) { + messages.push(`ERR:${message}`) + } + } + + await runCli(['node', 'codesift', 'doctor', repoRoot], io) + + expect(messages[0]).toContain('better-sqlite3: native module loaded through @codesift/core') + expect(messages[0]).toContain('index: missing; run codesift index') + expect(messages[0]).toContain('daemon:') + }) + + it('doctor reports corrupt index status instead of throwing', async () => { + const repoRoot = await mkdtemp(join(tmpdir(), 'codesift-cli-doctor-corrupt-')) + temporaryDirectories.push(repoRoot) + + await mkdir(join(repoRoot, '.codesift'), { recursive: true }) + await writeFile(join(repoRoot, '.codesift', 'index.db'), 'not a sqlite database', 'utf8') + + const messages: string[] = [] + const io: CliIo = { + stdout(message) { + messages.push(message) + }, + stderr(message) { + messages.push(`ERR:${message}`) + } + } + + await runCli(['node', 'codesift', 'doctor', repoRoot], io) + + expect(messages[0]).toContain('index: status unavailable:') + expect(messages[0]).toContain('sync: unknown; repair or rebuild the index') + }) + it('indexes, searches, and resolves symbols with repo-aware options', async () => { const repoRoot = await mkdtemp(join(tmpdir(), 'codesift-cli-')) temporaryDirectories.push(repoRoot) diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index f783f18..c5c17ee 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -3,6 +3,9 @@ export type { EmbeddingBatchOptions, EmbeddingProvider, EmbeddingRole, + ChangesetContextOptions, + ChangesetContextResult, + ChangesetFileContext, FindEdgeOptions, FindImportersOptions, FindSymbolOptions, @@ -27,6 +30,8 @@ export type { RepoStatusProvider, RepoSyncState, RepoSyncStatus, + ResultList, + ResultMetadata, SearchHit, SearchOptions, SearchReasonTag, @@ -93,7 +98,7 @@ export { type CodesiftConfigKey } from './config.js' -export { IndexCompatibilityError, SqliteRepo, setVectorExtensionLoaderForTests } from './repo.js' +export { IndexCompatibilityError, SqliteRepo, findRepoRoot, setVectorExtensionLoaderForTests } from './repo.js' import { SqliteRepo } from './repo.js' import type { Repo, RepoOptions } from './types.js' diff --git a/packages/core/src/repo.ts b/packages/core/src/repo.ts index 9ce6537..475d364 100644 --- a/packages/core/src/repo.ts +++ b/packages/core/src/repo.ts @@ -17,6 +17,9 @@ import { scanRepository, scanRepositoryManifest, type ScannedFile } from './scan import { prepareForCloud } from './secret-scan.js' import { DEFAULT_SEARCH_K, + type ChangesetContextOptions, + type ChangesetContextResult, + type ChangesetFileContext, type Edge, type EdgeResult, type EmbeddingProvider, @@ -37,6 +40,8 @@ import { type RepoStaleReason, type RepoStatus, type RepoSyncStatus, + type ResultList, + type ResultMetadata, type RerankResult, type SearchHit, type SearchOptions, @@ -148,6 +153,16 @@ interface DefinitionEdgeTarget { file?: string } +interface DefinitionEdgeQueryStats { + nameOnlyUnscoped?: number +} + +interface DefinitionEdgeSelectOptions { + limit?: number + nameOnlyLimit?: number + stats?: DefinitionEdgeQueryStats +} + interface ReadRowsResult { items: T[] tokenTruncated: boolean @@ -241,6 +256,11 @@ const MIN_RELAXATION_ROWS = 3 const FIND_SYMBOL_INLINE_MAX_EXACT_ROWS = 3 const FIND_SYMBOL_RELATION_MAX_SITES = 5 const FIND_SYMBOL_RELATION_MAX_NEIGHBORS = 4 +const FIND_SYMBOL_RELATION_MIN_BUDGET = 180 +const SEARCH_RELATION_MIN_BUDGET = 220 +const NAME_ONLY_EDGE_DEFAULT_LIMIT = 25 +const DEFAULT_CHANGESET_MAX_FILES = 40 +const DEFAULT_CHANGESET_MAX_EDGES_PER_FILE = 12 const DEFAULT_IMPACT_DEPTH = 2 const DEFAULT_IMPACT_MAX_NODES = 50 const HARD_MAX_IMPACT_BOUND = 50 @@ -407,8 +427,14 @@ export class SqliteRepo implements Repo { try { const previousGeneration = readMetaNumber(db, 'index_generation') ?? 0 const previousGitSnapshot = readIndexedGitSnapshot(db) - const { files, skippedFiles, skippedSymlinks } = await scanRepository(this.root) - const diff = diffScannedFiles(files, selectIndexedFileRows(db)) + const indexedFiles = selectIndexedFileRows(db) + const gitSnapshot = await readGitSnapshot(this.root) + const gitSnapshotChanged = !gitSnapshotsEqual(previousGitSnapshot, gitSnapshot) + const knownByPath = gitSnapshotChanged + ? undefined + : new Map(indexedFiles.map((file) => [file.path, file])) + const { files, skippedFiles, skippedSymlinks } = await scanRepository(this.root, knownByPath) + const diff = diffScannedFiles(files, indexedFiles) const chunkRows: IndexedChunkRecord[] = diff.changedFiles.flatMap((file) => buildChunks(file).map((chunk) => ({ @@ -427,6 +453,7 @@ export class SqliteRepo implements Repo { const chunksToEmbed: IndexedChunkRecord[] = [] const embeddedChunks: EmbeddedChunkRecord[] = [] let completedChunks = 0 + const totalChunks = chunkRows.length for (const chunk of chunkRows) { const contentHash = contentHashForChunk(chunk) @@ -448,6 +475,7 @@ export class SqliteRepo implements Repo { } const batches = buildEmbeddingBatches(chunksToEmbed, provider.maxBatch, provider.maxBatchTokens) + writeSyncStatus(db, { state: 'running', startedAt: syncStartedAt, completedChunks, totalChunks }) for (let index = 0; index < batches.length; index += 1) { throwIfAborted(options?.signal) @@ -481,12 +509,11 @@ export class SqliteRepo implements Repo { batch: index + 1, totalBatches: batches.length, completedChunks, - totalChunks: chunkRows.length + totalChunks }) + writeSyncStatus(db, { state: 'running', startedAt: syncStartedAt, completedChunks, totalChunks }) } - const gitSnapshot = await readGitSnapshot(this.root) - const gitSnapshotChanged = !gitSnapshotsEqual(previousGitSnapshot, gitSnapshot) const metadataMissing = readMeta(db, 'schema_version') !== SCHEMA_VERSION const hasDatabaseChanges = diff.changedFiles.length > 0 || @@ -557,10 +584,13 @@ export class SqliteRepo implements Repo { } } - async search(query: string, options?: SearchOptions): Promise { - if (!query.trim() || !existsSync(this.indexPath)) { + async search(query: string, options?: SearchOptions): Promise> { + if (!query.trim()) { return [] } + if (!existsSync(this.indexPath)) { + return withResultMetadata([], { notIndexed: true, emptyReason: 'not_indexed' }) + } const releaseDatabase = await this.enterDatabaseUser() try { @@ -644,16 +674,24 @@ export class SqliteRepo implements Repo { if (options?.withUsages) { await attachUsagesToTopDefinitionHit(db, this.root, staleHits, options) } + if (shouldAttachSearchRelations(options, autoSingleBest)) { + await attachRelationsToTopDefinitionHit(db, this.root, staleHits, options, (definition) => + this.readFindSymbolRelations(db, definition) + ) + } return staleHits } finally { releaseDatabase() } } - async grep(pattern: string, options?: GrepOptions): Promise { - if (!pattern || !existsSync(this.indexPath)) { + async grep(pattern: string, options?: GrepOptions): Promise> { + if (!pattern) { return [] } + if (!existsSync(this.indexPath)) { + return withResultMetadata([], { notIndexed: true, emptyReason: 'not_indexed' }) + } const releaseDatabase = await this.enterDatabaseUser() let candidateFiles: Array<{ path: string; language: string }> @@ -731,10 +769,13 @@ export class SqliteRepo implements Repo { return hits } - async findSymbol(name: string, options?: FindSymbolOptions): Promise { - if (!name.trim() || !existsSync(this.indexPath)) { + async findSymbol(name: string, options?: FindSymbolOptions): Promise> { + if (!name.trim()) { return [] } + if (!existsSync(this.indexPath)) { + return withResultMetadata([], { notIndexed: true, emptyReason: 'not_indexed' }) + } const releaseDatabase = await this.enterDatabaseUser() try { @@ -772,12 +813,13 @@ export class SqliteRepo implements Repo { const rows = [...exactRows, ...partialRows] const canEnrichTopExactRow = exactRows.length > 0 && exactRows.length <= FIND_SYMBOL_INLINE_MAX_EXACT_ROWS + const ambiguousDefCount = countDistinctDefinitionSites(exactRows) // One-call moat: inline the verbatim enclosing-symbol body for the top exact // match when the lookup is unambiguous, so an identifier query resolves // without a mandatory follow-up read. Disk-fresh, capped, never throws. let topBody: string | undefined - if (options?.withBody !== false && canEnrichTopExactRow) { + if (options?.withBody !== false && options?.detail !== 'sig' && canEnrichTopExactRow) { const top = exactRows[0]! try { const source = await this.readRange(top.file_path, top.start_line, top.end_line) @@ -791,15 +833,22 @@ export class SqliteRepo implements Repo { } let topRelations: SymbolRelations | undefined - if (options?.withCallers === true && canEnrichTopExactRow) { + if (shouldAttachFindSymbolRelations(options, canEnrichTopExactRow)) { try { - topRelations = await this.readFindSymbolRelations(db, exactRows[0]!) + const candidateRelations = await this.readFindSymbolRelations(db, exactRows[0]!) + if ( + options?.withCallers === true || + relationsFitBudget(candidateRelations, options?.maxTokens, estimateSymbolBodyTokens(topBody)) + ) { + topRelations = candidateRelations + } } catch { // Relation bundling is best-effort and must never fail the base lookup. } } - return rows.map((row, index) => { + const definitions = rows.map((row, index) => { + const exact = index < exactRows.length const definition: SymbolDefinition = { id: String(row.id), name: row.name, @@ -808,7 +857,8 @@ export class SqliteRepo implements Repo { startLine: row.start_line, endLine: row.end_line }, - kind: row.kind + kind: row.kind, + matchQuality: exact ? 'exact' : 'partial' } if (row.signature) { @@ -823,6 +873,10 @@ export class SqliteRepo implements Repo { definition.language = row.language } + if (index === 0 && ambiguousDefCount >= 2) { + definition.ambiguousDefCount = ambiguousDefCount + } + // Only the top exact row (rows[0] when exactRows is non-empty) carries // the optional single-call enrichments. if (index === 0 && topBody !== undefined) { @@ -835,6 +889,11 @@ export class SqliteRepo implements Repo { return definition }) + return withResultMetadata(definitions, { + definitionCount: exactRows.length, + ...(ambiguousDefCount >= 2 ? { ambiguousDefCount } : {}), + ...(partialRows.length > 0 ? { partialMatchCount: partialRows.length } : {}) + }) } finally { releaseDatabase() } @@ -852,10 +911,13 @@ export class SqliteRepo implements Repo { return this.findDefinitionEdges(name, options, ['implements', 'extends'], false) } - async findImporters(file: string, options?: FindImportersOptions): Promise { - if (!file.trim() || !existsSync(this.indexPath)) { + async findImporters(file: string, options?: FindImportersOptions): Promise> { + if (!file.trim()) { return [] } + if (!existsSync(this.indexPath)) { + return withResultMetadata([], { notIndexed: true, emptyReason: 'not_indexed' }) + } const releaseDatabase = await this.enterDatabaseUser() let rows: EdgeResultRow[] = [] @@ -873,9 +935,12 @@ export class SqliteRepo implements Repo { async impact(name: string, options?: ImpactOptions): Promise { const depthLimit = normalizeImpactDepth(options?.depth) const maxNodes = normalizeImpactMaxNodes(options?.maxNodes) - if (!name.trim() || !existsSync(this.indexPath)) { + if (!name.trim()) { return { nodes: [], depthLimit, maxNodes } } + if (!existsSync(this.indexPath)) { + return { nodes: [], depthLimit, maxNodes, notIndexed: true, emptyReason: 'not_indexed' } + } const releaseDatabase = await this.enterDatabaseUser() let rows: ImpactNodeRow[] = [] @@ -887,7 +952,7 @@ export class SqliteRepo implements Repo { const initialDefinitions = selectExactDefinitionRows(db, name, normalizeKinds(options?.kind), options?.pathGlob) if (initialDefinitions.length === 0) { - return { nodes: [], depthLimit, maxNodes } + return { nodes: [], depthLimit, maxNodes, emptyReason: 'no_definition' } } const defaultExportCache = new Map>>() @@ -967,6 +1032,63 @@ export class SqliteRepo implements Repo { } } + async changesetContext(files: string[], options?: ChangesetContextOptions): Promise { + const requestedFiles = normalizeChangesetFiles(files) + const maxFiles = normalizeChangesetMaxFiles(options?.maxFiles) + const maxEdgesPerFile = normalizeChangesetMaxEdges(options?.maxEdgesPerFile) + if (requestedFiles.length === 0) { + return { files: [] } + } + if (!existsSync(this.indexPath)) { + return { files: [], notIndexed: true } + } + + const selectedFiles = requestedFiles.slice(0, maxFiles) + const releaseDatabase = await this.enterDatabaseUser() + const contexts: ChangesetFileContext[] = [] + let stale = false + let truncated = requestedFiles.length > selectedFiles.length + try { + const db = this.openDatabase() + this.ensureIndexCompatibleForQueries(db) + stale = (await this.getFreshness(db)).stale + + for (const file of selectedFiles) { + const symbolRows = selectSymbolRowsByFile(db, file) + const symbols = symbolRows.map((row) => buildSymbolDefinition(row, 'exact')) + const targets = await resolveDefinitionEdgeTargets(this.root, symbolRows) + const callerRows = selectDefinitionEdgeRows(db, targets, ['call', 'ref'], true, { + limit: maxEdgesPerFile + 1, + nameOnlyLimit: Math.min(NAME_ONLY_EDGE_DEFAULT_LIMIT, maxEdgesPerFile + 1) + }) + const importerRows = selectImporterEdgeRows(db, file).slice(0, maxEdgesPerFile + 1) + const callerRead = await readEdgeResultsFromRows(this.root, callerRows.slice(0, maxEdgesPerFile), options?.maxTokens) + const importerRead = await readEdgeResultsFromRows(this.root, importerRows.slice(0, maxEdgesPerFile), options?.maxTokens) + const omitted = Math.max(0, callerRows.length - callerRead.items.length) + Math.max(0, importerRows.length - importerRead.items.length) + const omittedLowerBound = callerRows.length > maxEdgesPerFile || importerRows.length > maxEdgesPerFile + if (omitted > 0 || omittedLowerBound) { + truncated = true + } + contexts.push({ + file, + symbols, + callers: callerRead.items, + importers: importerRead.items, + ...(omitted > 0 ? { omitted } : {}), + ...(omittedLowerBound ? { omittedLowerBound } : {}) + }) + } + } finally { + releaseDatabase() + } + + return { + files: contexts, + ...(stale ? { stale } : {}), + ...(truncated ? { truncated } : {}) + } + } + private async readFindSymbolRelations(db: Database.Database, definition: SymbolRow): Promise { const targets = await resolveDefinitionEdgeTargets(this.root, [definition]) const siteRows = selectDefinitionEdgeRows(db, targets, ['call', 'ref'], true, FIND_SYMBOL_RELATION_MAX_SITES) @@ -1001,32 +1123,55 @@ export class SqliteRepo implements Repo { options: FindEdgeOptions | undefined, edgeKinds: ReadonlyArray, preferCallsFirst: boolean - ): Promise { - if (!name.trim() || !existsSync(this.indexPath)) { + ): Promise> { + if (!name.trim()) { return [] } + if (!existsSync(this.indexPath)) { + return withResultMetadata([], { notIndexed: true, emptyReason: 'not_indexed' }) + } const releaseDatabase = await this.enterDatabaseUser() let rows: EdgeResultRow[] = [] + let metadata: ResultMetadata = {} try { const db = this.openDatabase() this.ensureIndexCompatibleForQueries(db) const definitions = selectExactDefinitionRows(db, name, normalizeKinds(options?.kind), options?.pathGlob) if (definitions.length === 0) { - return [] + return withResultMetadata([], { emptyReason: 'no_definition' }) } const targets = await resolveDefinitionEdgeTargets(this.root, definitions) - rows = selectDefinitionEdgeRows(db, targets, edgeKinds, preferCallsFirst) + const stats: DefinitionEdgeQueryStats = {} + const selectOptions: DefinitionEdgeSelectOptions = { + nameOnlyLimit: NAME_ONLY_EDGE_DEFAULT_LIMIT, + stats + } + if (options?.maxResults !== undefined) { + selectOptions.limit = options.maxResults + } + rows = selectDefinitionEdgeRows(db, targets, edgeKinds, preferCallsFirst, selectOptions) + const ambiguousDefCount = countDistinctDefinitionSites(definitions) + metadata = { + definitionCount: definitions.length, + ...(ambiguousDefCount >= 2 ? { ambiguousDefCount } : {}), + ...(rows.length === 0 ? { emptyReason: 'no_edges' as const } : {}), + ...(stats.nameOnlyUnscoped !== undefined ? { nameOnlyUnscoped: stats.nameOnlyUnscoped, nameOnlyLimit: NAME_ONLY_EDGE_DEFAULT_LIMIT } : {}) + } } finally { releaseDatabase() } - return (await readEdgeResultsFromRows(this.root, rows, options?.maxTokens)).items + return withResultMetadata((await readEdgeResultsFromRows(this.root, rows, options?.maxTokens)).items, metadata) } async readChunk(id: string, options?: ReadChunkOptions): Promise { + if (!existsSync(this.indexPath)) { + throw new Error('not_indexed; run: codesift index') + } + const releaseDatabase = await this.enterDatabaseUser() let parsedChunkId: { file: string; startLine: number; endLine: number } | null try { @@ -1070,6 +1215,7 @@ export class SqliteRepo implements Repo { return { root: this.root, indexPath: this.indexPath, + indexExists: false, indexed: false, stale: false, sync: { state: 'idle' }, @@ -1119,6 +1265,7 @@ export class SqliteRepo implements Repo { return { root: this.root, indexPath: this.indexPath, + indexExists: true, indexed, stale: freshness.stale, ...(freshness.reasons.length > 0 ? { staleReasons: freshness.reasons } : {}), @@ -1230,7 +1377,7 @@ export class SqliteRepo implements Repo { try { if (onlyIfStale) { const status = await this.status() - if (!status.stale) { + if (status.indexed && !status.stale) { return } } @@ -1264,6 +1411,7 @@ export class SqliteRepo implements Repo { } await refreshWatchers() + void runSync(true) safetyInterval = setInterval(() => { void runSync(true) @@ -1951,12 +2099,16 @@ function readSyncStatus(db: Database.Database): RepoSyncStatus { const startedAt = readMeta(db, 'last_sync_started_at') const completedAt = readMeta(db, 'last_sync_completed_at') const error = readMeta(db, 'last_sync_error') + const completedChunks = readMetaNumber(db, 'last_sync_completed_chunks') + const totalChunks = readMetaNumber(db, 'last_sync_total_chunks') return { state, ...(startedAt ? { startedAt } : {}), ...(completedAt ? { completedAt } : {}), - ...(error ? { error } : {}) + ...(error ? { error } : {}), + ...(completedChunks !== undefined ? { completedChunks } : {}), + ...(totalChunks !== undefined ? { totalChunks } : {}) } } @@ -1997,6 +2149,18 @@ function writeSyncStatusWithStatements( } else { deleteMeta.run('last_sync_error') } + + if (status.completedChunks !== undefined) { + setMeta.run('last_sync_completed_chunks', String(status.completedChunks)) + } else { + deleteMeta.run('last_sync_completed_chunks') + } + + if (status.totalChunks !== undefined) { + setMeta.run('last_sync_total_chunks', String(status.totalChunks)) + } else { + deleteMeta.run('last_sync_total_chunks') + } } async function getIndexFreshness(root: string, db: Database.Database): Promise { @@ -2109,6 +2273,24 @@ async function readGitSnapshot(root: string): Promise { return head ? { branch, head } : null } +export async function findRepoRoot(startPath = process.cwd()): Promise { + let current = resolve(startPath) + + while (true) { + const dotGit = join(current, '.git') + if (existsSync(dotGit)) { + return current + } + + const parent = dirname(current) + if (parent === current) { + return resolve(startPath) + } + + current = parent + } +} + async function findGitDirectory(root: string): Promise { let current = resolve(root) @@ -2239,6 +2421,158 @@ function normalizeKinds(kind: SearchOptions['kind'] | FindSymbolOptions['kind'] return Array.isArray(kind) ? kind : [kind] } +function withResultMetadata(items: T[], metadata: ResultMetadata): ResultList { + if (Object.keys(metadata).length === 0) { + return items as ResultList + } + + Object.defineProperty(items, 'meta', { + value: metadata, + enumerable: false, + configurable: true + }) + return items as ResultList +} + +function countDistinctDefinitionSites(rows: SymbolRow[]): number { + return new Set(rows.map((row) => `${row.file_path}\u0000${row.kind}`)).size +} + +function shouldAttachFindSymbolRelations(options: FindSymbolOptions | undefined, canEnrichTopExactRow: boolean): boolean { + if (!canEnrichTopExactRow || options?.withCallers === false) { + return false + } + + if (options?.withCallers === true) { + return true + } + + return options?.maxTokens === undefined || options.maxTokens >= FIND_SYMBOL_RELATION_MIN_BUDGET +} + +function shouldAttachSearchRelations(options: SearchOptions | undefined, autoSingleBest: boolean): boolean { + if (options?.withRelations === false) { + return false + } + + if (options?.withRelations === true || options?.context === 'graph') { + return true + } + + if (options?.context === 'min' || options?.context === 'sig') { + return false + } + + return autoSingleBest +} + +async function attachRelationsToTopDefinitionHit( + db: Database.Database, + _root: string, + hits: SearchHit[], + options: SearchOptions | undefined, + readRelations: (definition: SymbolRow) => Promise +): Promise { + const topHit = hits[0] + if (!topHit?.symbol || !topHit.kind || topHit.kind === 'file') { + return + } + + if (options?.maxTokens !== undefined && options.maxTokens < SEARCH_RELATION_MIN_BUDGET) { + return + } + + const definition = selectDefinitionRowForSearchHit(db, topHit) + if (!definition) { + return + } + + const relations = await readRelations(definition) + if (!relations || !relationsFitBudget(relations, options?.maxTokens, hits.reduce((sum, hit) => sum + hit.tokensReturned, 0))) { + return + } + + topHit.relations = relations + topHit.tokensReturned += estimateSymbolRelationsTokens(relations) +} + +function relationsFitBudget(relations: SymbolRelations | undefined, maxTokens: number | undefined, tokensUsed: number): relations is SymbolRelations { + if (!relations) { + return false + } + + if (maxTokens === undefined) { + return true + } + + return tokensUsed + estimateSymbolRelationsTokens(relations) <= maxTokens +} + +function estimateSymbolBodyTokens(body: string | undefined): number { + return body ? estimateTokenCount(body) : 0 +} + +function estimateSymbolRelationsTokens(relations: SymbolRelations): number { + const siteTokens = relations.sites.reduce((sum, site) => sum + estimateEdgeResultTokens(site), 0) + const neighborTokens = relations.neighbors.reduce((sum, neighbor) => { + const header = `${neighbor.file}:${neighbor.range.startLine}-${neighbor.range.endLine} ${neighbor.name} ${neighbor.kind}` + return sum + SEARCH_HIT_TOKEN_OVERHEAD + estimateTokenCount(header) + }, 0) + const omittedTokens = relations.omitted ? estimateTokenCount(`relations_omitted=${relations.omitted}`) : 0 + return siteTokens + neighborTokens + omittedTokens +} + +function selectDefinitionRowForSearchHit(db: Database.Database, hit: SearchHit): SymbolRow | undefined { + if (!hit.symbol || !hit.kind) { + return undefined + } + + return db + .prepare( + ` + select id, name, file_path, start_line, end_line, kind, signature, parent, language + from symbols + where file_path = ? + and lower(name) = lower(?) + and kind = ? + order by + case when start_line = ? and end_line = ? then 0 else 1 end, + abs(start_line - ?) asc, + id asc + limit 1 + ` + ) + .get(hit.file, hit.symbol, hit.kind, hit.range.startLine, hit.range.endLine, hit.range.startLine) +} + +function normalizeChangesetFiles(files: string[]): string[] { + const normalized = new Set() + for (const file of files) { + const candidate = normalizeRelativeRepoPath(file.trim()) + if (!candidate || candidate === '.' || candidate.startsWith('../') || candidate === '..') { + continue + } + normalized.add(candidate) + } + return [...normalized] +} + +function normalizeChangesetMaxFiles(value: number | undefined): number { + if (value === undefined || !Number.isFinite(value) || value <= 0) { + return DEFAULT_CHANGESET_MAX_FILES + } + + return Math.min(Math.floor(value), DEFAULT_CHANGESET_MAX_FILES) +} + +function normalizeChangesetMaxEdges(value: number | undefined): number { + if (value === undefined || !Number.isFinite(value) || value <= 0) { + return DEFAULT_CHANGESET_MAX_EDGES_PER_FILE + } + + return Math.min(Math.floor(value), NAME_ONLY_EDGE_DEFAULT_LIMIT) +} + type RangeReader = (file: string, startLine: number, endLine: number) => Promise async function buildBudgetedSearchHits( @@ -2312,6 +2646,10 @@ function shouldInlineHit( return false } + if (context === 'min' || context === 'graph') { + return false + } + if (context === 'body') { return true } @@ -4073,6 +4411,47 @@ function selectPartialSymbolRows(db: Database.Database, name: string, kinds: Sym .all(...params, name) } +function selectSymbolRowsByFile(db: Database.Database, file: string): SymbolRow[] { + return db + .prepare<[string], SymbolRow>( + ` + select id, name, file_path, start_line, end_line, kind, signature, parent, language + from symbols + where file_path = ? + order by start_line asc, id asc + ` + ) + .all(file) +} + +function buildSymbolDefinition(row: SymbolRow, matchQuality: 'exact' | 'partial'): SymbolDefinition { + const definition: SymbolDefinition = { + id: String(row.id), + name: row.name, + file: row.file_path, + range: { + startLine: row.start_line, + endLine: row.end_line + }, + kind: row.kind, + matchQuality + } + + if (row.signature) { + definition.signature = row.signature + } + + if (row.parent) { + definition.parent = row.parent + } + + if (row.language) { + definition.language = row.language + } + + return definition +} + function selectExactDefinitionRows( db: Database.Database, name: string, @@ -4250,12 +4629,13 @@ function selectDefinitionEdgeRows( targets: ReadonlyArray, edgeKinds: ReadonlyArray, preferCallsFirst: boolean, - limit?: number + options?: number | DefinitionEdgeSelectOptions ): EdgeResultRow[] { if (targets.length === 0 || edgeKinds.length === 0) { return [] } + const selectOptions = typeof options === 'number' ? { limit: options, nameOnlyLimit: options } : options ?? {} const edgeKindPlaceholders = edgeKinds.map(() => '?').join(', ') const rowsById = new Map() @@ -4273,15 +4653,34 @@ function selectDefinitionEdgeRows( } params.push(...edgeKinds) + const nameOnlyLimit = target.resolutionMode === 'name-only' ? selectOptions.nameOnlyLimit : undefined + if (nameOnlyLimit !== undefined) { + const total = db + .prepare( + ` + select count(*) as count + from edges + where ${whereClause} + ` + ) + .get(...params)?.count ?? 0 + if (total > nameOnlyLimit && selectOptions.stats) { + selectOptions.stats.nameOnlyUnscoped = (selectOptions.stats.nameOnlyUnscoped ?? 0) + total + } + } + + const orderBy = `${preferCallsFirst ? "case when edge_kind = 'call' then 0 else 1 end asc, " : ''}src_file asc, src_line asc, id asc` const targetRows = db .prepare( ` select id, src_file, src_line, src_symbol, edge_kind, resolution, language from edges where ${whereClause} + order by ${orderBy} + ${nameOnlyLimit !== undefined ? 'limit ?' : ''} ` ) - .all(...params) + .all(...(nameOnlyLimit !== undefined ? [...params, nameOnlyLimit] : params)) for (const row of targetRows) { rowsById.set(row.id, row) @@ -4289,7 +4688,7 @@ function selectDefinitionEdgeRows( } const rows = [...rowsById.values()].sort((left, right) => compareEdgeResultRows(left, right, preferCallsFirst)) - return limit === undefined ? rows : rows.slice(0, limit) + return selectOptions.limit === undefined ? rows : rows.slice(0, selectOptions.limit) } function compareEdgeResultRows(left: EdgeResultRow, right: EdgeResultRow, preferCallsFirst: boolean): number { diff --git a/packages/core/src/scan.ts b/packages/core/src/scan.ts index 8d9628b..66d4110 100644 --- a/packages/core/src/scan.ts +++ b/packages/core/src/scan.ts @@ -59,6 +59,8 @@ const GENERATED_MARKERS = [ /this file was generated/i ] +const MTIME_TOLERANCE_MS = 2 + interface IgnoreMatcher { basePath: string matcher: ReturnType @@ -78,6 +80,15 @@ export interface ScannedFile extends ScannedFileMetadata { generated: boolean } +export interface KnownScannedFile { + path: string + language: string + hash: string + size: number + mtime: number + generated: number +} + export interface ScanResult { files: ScannedFile[] skippedFiles: number @@ -90,8 +101,8 @@ export interface ScanManifestResult { skippedSymlinks: number } -export async function scanRepository(root: string): Promise { - const result = await scanRepositoryInternal(root, true) +export async function scanRepository(root: string, knownByPath?: ReadonlyMap): Promise { + const result = await scanRepositoryInternal(root, true, knownByPath) return { files: result.files as ScannedFile[], skippedFiles: result.skippedFiles, @@ -108,7 +119,11 @@ export async function scanRepositoryManifest(root: string): Promise; skippedFiles: number; skippedSymlinks: number }> { +async function scanRepositoryInternal( + root: string, + includeContent: boolean, + knownByPath?: ReadonlyMap +): Promise<{ files: Array; skippedFiles: number; skippedSymlinks: number }> { const files: Array = [] let skippedFiles = 0 let skippedSymlinks = 0 @@ -150,6 +165,26 @@ async function scanRepositoryInternal(root: string, includeContent: boolean): Pr return } + const known = knownByPath?.get(relativePath) + if ( + known && + known.language === language && + known.size === fileStat.size && + mtimeEqual(known.mtime, fileStat.mtimeMs) + ) { + files.push({ + absolutePath, + relativePath, + language, + content: '', + hash: known.hash, + size: fileStat.size, + mtime: fileStat.mtimeMs, + generated: known.generated === 1 + }) + return + } + const buffer = await readFile(absolutePath) if (buffer.includes(0)) { skippedFiles += 1 @@ -249,6 +284,10 @@ async function scanRepositoryInternal(root: string, includeContent: boolean): Pr return { files, skippedFiles, skippedSymlinks } } +function mtimeEqual(left: number, right: number): boolean { + return Math.abs(left - right) <= MTIME_TOLERANCE_MS +} + async function createDirectoryMatcher(directory: string, basePath: string, defaults: string[] = []): Promise { const matcher = ignore().add(defaults) diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index fcade26..0bfe414 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -75,6 +75,20 @@ export interface SymbolRelations { omitted?: number } +export interface ResultMetadata { + notIndexed?: boolean + emptyReason?: 'not_indexed' | 'no_definition' | 'no_edges' + definitionCount?: number + ambiguousDefCount?: number + partialMatchCount?: number + nameOnlyUnscoped?: number + nameOnlyLimit?: number +} + +export type ResultList = T[] & { + meta?: ResultMetadata +} + export interface SearchHit { id: string file: string @@ -97,6 +111,11 @@ export interface SearchHit { * TS/JS + Python and resolved via imports/local bindings, not a type checker. */ usages?: SymbolUsage[] + /** + * Bounded relation bundle for a confident definition hit. This is the same + * persisted-edge payload used by find_symbol, attached only when budget allows. + */ + relations?: SymbolRelations language?: string symbol?: string parent?: string @@ -132,6 +151,8 @@ export interface SymbolDefinition { signature?: string parent?: string language?: string + matchQuality?: 'exact' | 'partial' + ambiguousDefCount?: number /** * Full enclosing-symbol source for the top exact match when body inlining is * enabled. Dedented and blank-collapsed, then capped like a search hit body @@ -160,7 +181,12 @@ export interface SearchOptions { * `'sig'` never inlines (compact only); `undefined` is AUTO — inline rank-1 * always, rank-2 only if within the score margin of rank-1. */ - context?: 'sig' | 'body' + context?: 'auto' | 'min' | 'sig' | 'body' | 'graph' + /** + * Attach a bounded relation bundle to a confident top definition. Undefined + * means AUTO: enabled for single-best definition hits when budget allows. + */ + withRelations?: boolean /** * Bundle top-N usage sites for the top DEFINITION hit when supported. Honest * scope: import-resolved/local only, currently TS/JS + Python. @@ -205,6 +231,15 @@ export interface FindSymbolOptions { * sites plus same-file neighbors. Default false. */ withCallers?: boolean + /** + * Approx output token budget used to decide whether default-on relations fit. + */ + maxTokens?: number + /** + * Detail tier for the top exact definition. `sig` keeps the response compact + * and skips body inlining; `body` is the default one-call mode. + */ + detail?: 'sig' | 'body' } export interface FindEdgeOptions { @@ -220,6 +255,10 @@ export interface FindEdgeOptions { * Approx output token budget for the returned edge rows. */ maxTokens?: number + /** + * Optional hard cap for relation rows before token budgeting. + */ + maxResults?: number } export interface FindImportersOptions { @@ -263,6 +302,8 @@ export interface ImpactNode extends EdgeResult { export interface ImpactResult { nodes: ImpactNode[] + notIndexed?: boolean + emptyReason?: ResultMetadata['emptyReason'] impactTruncated?: boolean depthCapped?: boolean nodesCapped?: boolean @@ -270,6 +311,28 @@ export interface ImpactResult { maxNodes: number } +export interface ChangesetFileContext { + file: string + symbols: SymbolDefinition[] + callers: EdgeResult[] + importers: EdgeResult[] + omitted?: number + omittedLowerBound?: boolean +} + +export interface ChangesetContextOptions { + maxFiles?: number + maxEdgesPerFile?: number + maxTokens?: number +} + +export interface ChangesetContextResult { + files: ChangesetFileContext[] + stale?: boolean + truncated?: boolean + notIndexed?: boolean +} + export interface SyncProgressEvent { phase: 'batch' batch: number @@ -314,6 +377,8 @@ export interface RepoSyncStatus { startedAt?: string completedAt?: string error?: string + completedChunks?: number + totalChunks?: number } export interface WatchOptions { @@ -367,6 +432,7 @@ export interface IndexCompatibilityStatus { export interface RepoStatus { root: string indexPath: string + indexExists: boolean indexed: boolean stale: boolean staleReasons?: RepoStaleReason[] @@ -392,14 +458,15 @@ export interface ReadRangeOptions { export interface Repo { readonly root: string sync(options?: SyncOptions): Promise - search(query: string, options?: SearchOptions): Promise - grep(pattern: string, options?: GrepOptions): Promise - findSymbol(name: string, options?: FindSymbolOptions): Promise - findCallers(name: string, options?: FindEdgeOptions): Promise + search(query: string, options?: SearchOptions): Promise> + grep(pattern: string, options?: GrepOptions): Promise> + findSymbol(name: string, options?: FindSymbolOptions): Promise> + findCallers(name: string, options?: FindEdgeOptions): Promise> impact(name: string, options?: ImpactOptions): Promise - findReferences(name: string, options?: FindEdgeOptions): Promise - findImplementers(name: string, options?: FindEdgeOptions): Promise - findImporters(file: string, options?: FindImportersOptions): Promise + findReferences(name: string, options?: FindEdgeOptions): Promise> + findImplementers(name: string, options?: FindEdgeOptions): Promise> + findImporters(file: string, options?: FindImportersOptions): Promise> + changesetContext(files: string[], options?: ChangesetContextOptions): Promise readChunk(id: string, options?: ReadChunkOptions): Promise readRange(file: string, startLine: number, endLine: number, options?: ReadRangeOptions): Promise status(): Promise diff --git a/packages/eval/losses.json b/packages/eval/losses.json index 5bd05ec..d91f567 100644 --- a/packages/eval/losses.json +++ b/packages/eval/losses.json @@ -48,7 +48,6 @@ "queryId": "escape-regexp-symbol", "queryType": "symbol-def", "axes": [ - "latency.cold", "precision" ] }, @@ -90,7 +89,6 @@ "queryType": "nl-concept", "axes": [ "latency.cold", - "latency.warm", "tokens" ] }, @@ -107,7 +105,8 @@ "queryId": "m3-go-method-symbol", "queryType": "symbol-def", "axes": [ - "latency.cold" + "latency.cold", + "tokens" ] }, { @@ -123,7 +122,8 @@ "queryId": "m3-java-method-symbol", "queryType": "symbol-def", "axes": [ - "latency.cold" + "latency.cold", + "tokens" ] }, { @@ -139,7 +139,8 @@ "queryId": "m3-ruby-method-symbol", "queryType": "symbol-def", "axes": [ - "latency.cold" + "latency.cold", + "tokens" ] }, { @@ -155,7 +156,8 @@ "queryId": "m3-rust-method-symbol", "queryType": "symbol-def", "axes": [ - "latency.cold" + "latency.cold", + "tokens" ] } ] diff --git a/packages/eval/test/eval.test.ts b/packages/eval/test/eval.test.ts index ef70600..ed580da 100644 --- a/packages/eval/test/eval.test.ts +++ b/packages/eval/test/eval.test.ts @@ -12,7 +12,7 @@ const temporaryDirectories: string[] = [] const stableLocalTokenCeilings: Partial> = { 'nl-concept': 180, 'symbol-def': 125, - 'exact-identifier': 75, + 'exact-identifier': 95, 'string-literal': 40, 'error-trace': 40 } diff --git a/packages/mcp/src/index.ts b/packages/mcp/src/index.ts index ae4802c..ded0f21 100644 --- a/packages/mcp/src/index.ts +++ b/packages/mcp/src/index.ts @@ -6,6 +6,8 @@ import { DEFAULT_SEARCH_K as CORE_DEFAULT_SEARCH_K, getDefaultEmbeddingProvider, isLearnedEmbeddingProvider, + type ChangesetContextResult, + type ChangesetFileContext, type EdgeResult, type FindEdgeOptions, type FindSymbolOptions, @@ -15,6 +17,7 @@ import { type ImpactResult, type Repo, type RepoStatus, + type ResultMetadata, type SearchHit, type SearchOptions, type SymbolDefinition, @@ -36,6 +39,7 @@ export const DEFAULT_MCP_INDEX_STATUS_MAX_TOKENS = 200 export const MIN_MCP_READ_CHUNK_MAX_TOKENS = 6 export const MAX_MCP_READ_CHUNK_MAX_TOKENS = 4000 export const MAX_MCP_INDEX_STATUS_MAX_TOKENS = 1000 +export const NOT_INDEXED_SENTINEL = 'not_indexed; run: codesift index' const SYMBOL_KINDS = [ 'class', @@ -51,6 +55,11 @@ const SYMBOL_KINDS = [ 'variable' ] as const satisfies readonly SymbolKind[] +const MAX_CHANGESET_FILES = 40 +const MAX_CHANGESET_EDGES_PER_FILE = 25 +const DEFAULT_CHANGESET_MAX_FILES = 40 +const DEFAULT_CHANGESET_MAX_EDGES_PER_FILE = 12 + export const MCP_TOOL_NAMES = [ 'search_code', 'find_symbol', @@ -59,6 +68,7 @@ export const MCP_TOOL_NAMES = [ 'find_importers', 'who_implements', 'impact', + 'changeset_context', 'grep_code', 'read_chunk', 'index_status' @@ -76,6 +86,7 @@ export interface SearchCodeArgs { single_best?: boolean | undefined context?: SearchOptions['context'] | undefined with_usages?: boolean | undefined + with_relations?: boolean | undefined } export interface FindSymbolArgs { @@ -84,6 +95,7 @@ export interface FindSymbolArgs { path_glob?: string | undefined with_body?: boolean | undefined with_callers?: boolean | undefined + detail?: FindSymbolOptions['detail'] | undefined max_tokens?: number | undefined } @@ -119,6 +131,13 @@ export interface ImpactArgs { max_tokens?: number | undefined } +export interface ChangesetContextArgs { + files: string[] + max_files?: number | undefined + max_edges_per_file?: number | undefined + max_tokens?: number | undefined +} + export interface GrepCodeArgs { pattern: string regex?: boolean | undefined @@ -158,6 +177,10 @@ export interface FormatMcpImpactOptions { maxTokens?: number | undefined } +export interface FormatMcpChangesetContextOptions { + maxTokens?: number | undefined +} + export interface ReadChunkArgs { id: string context_lines?: number | undefined @@ -204,6 +227,7 @@ export interface McpRouter { findImporters(args: FindImportersArgs): Promise findImplementers(args: FindImplementersArgs): Promise impact(args: ImpactArgs): Promise + changesetContext(args: ChangesetContextArgs): Promise grepCode(args: GrepCodeArgs): Promise readChunk(args: ReadChunkArgs): Promise indexStatus(): Promise @@ -221,9 +245,9 @@ export type McpJsonRpcResponse = | { jsonrpc: '2.0'; id: string | number | null; error: { code: number; message: string; data?: unknown } } export const MCP_SERVER_INSTRUCTIONS = [ - 'codesift;identifiers->find_symbol;callers/uses->find_callers/find_refs;breakage/transitive callers->impact;importers->find_importers;implements/extends->who_implements;G/J/Rb/Rs approx:name-only;literals/regex/errors/env/operators->grep_code;concepts/fuzzy names->search_code', - 'Top search_code body inline;read_chunk only for non-top/wider context;Broad search k=5-8.', - 'Check index_status;warn+sync if missing/stale/running/failed.' + 'codesift;identifiers->find_symbol;callers/uses->find_callers/find_refs;diff file lists->changeset_context;breakage/transitive callers->impact;importers->find_importers;implements/extends->who_implements;G/J/Rb/Rs approx:name-only;literals/regex/errors/env/operators->grep_code;concepts/fuzzy names->search_code', + 'Top search_code body inline;find_symbol body/relations when budget fits;read_chunk only for non-top/wider context;Broad search k=5-8.', + 'Health is inline: not_indexed/stale/running/failed hints; use index_status only when troubleshooting.' ].join('\n') const symbolKindSchema = z.enum(SYMBOL_KINDS) @@ -237,8 +261,9 @@ const searchCodeInputSchema = { kind: kindFilterSchema.optional().describe('Symbol kind filter.'), max_tokens: z.number().int().positive().max(4000).optional().describe('Approx output tokens. Default 700.'), single_best: z.boolean().optional().describe('Return only best hit.'), - context: z.enum(['sig', 'body']).optional().describe('sig=snippets; body=inline bodies within budget.'), - with_usages: z.boolean().optional().describe('Add top usage sites for the top definition hit.') + context: z.enum(['auto', 'min', 'sig', 'body', 'graph']).optional().describe('auto=default; min/sig=compact; body=inline body; graph=relations.'), + with_usages: z.boolean().optional().describe('Add top usage sites for the top definition hit.'), + with_relations: z.boolean().optional().describe('Add bounded callers/refs + neighbors for a confident top definition.') } const findSymbolInputSchema = { @@ -247,6 +272,7 @@ const findSymbolInputSchema = { path_glob: z.string().min(1).optional().describe('Repo glob, e.g. "src/auth/**".'), with_body: z.boolean().optional().describe('Inline top exact body. Default true.'), with_callers: z.boolean().optional().describe('Append bounded caller/ref + same-file relations for the top exact hit.'), + detail: z.enum(['sig', 'body']).optional().describe('sig=signature/location only; body=inline top body.'), max_tokens: z.number().int().positive().max(4000).optional().describe('Approx output tokens. Default 700.') } @@ -282,6 +308,13 @@ const impactInputSchema = { max_tokens: z.number().int().positive().max(4000).optional().describe('Approx output tokens. Default 700.') } +const changesetContextInputSchema = { + files: z.array(z.string().min(1)).min(1).max(200).describe('Explicit repo-relative file list to summarize.'), + max_files: z.number().int().positive().max(MAX_CHANGESET_FILES).optional().describe('Max files to inspect. Default 40.'), + max_edges_per_file: z.number().int().positive().max(MAX_CHANGESET_EDGES_PER_FILE).optional().describe('Max callers/importers per file. Default 12.'), + max_tokens: z.number().int().positive().max(4000).optional().describe('Approx output tokens. Default 700.') +} + const grepCodeInputSchema = { pattern: z.string().min(1).describe('Literal by default; set regex=true for regex.'), regex: z.boolean().optional().describe('Use JavaScript regex. Default false.'), @@ -314,6 +347,7 @@ const findRefsArgsSchema = z.object(findRefsInputSchema).strict() const findImportersArgsSchema = z.object(findImportersInputSchema).strict() const findImplementersArgsSchema = z.object(findImplementersInputSchema).strict() const impactArgsSchema = z.object(impactInputSchema).strict() +const changesetContextArgsSchema = z.object(changesetContextInputSchema).strict() const grepCodeArgsSchema = z.object(grepCodeInputSchema).strict() const readChunkArgsSchema = z.object(readChunkInputSchema).strict() const indexStatusArgsSchema = z.object(indexStatusInputSchema).strict() @@ -365,8 +399,9 @@ export function getToolDefinitions(): readonly McpToolDefinition[] { kind: kindJsonSchema(), max_tokens: { type: 'integer', minimum: 1, maximum: 4000, default: DEFAULT_MCP_SEARCH_MAX_TOKENS }, single_best: { type: 'boolean' }, - context: { type: 'string', enum: ['sig', 'body'] }, - with_usages: { type: 'boolean' } + context: { type: 'string', enum: ['auto', 'min', 'sig', 'body', 'graph'] }, + with_usages: { type: 'boolean' }, + with_relations: { type: 'boolean' } }) }, { @@ -378,6 +413,7 @@ export function getToolDefinitions(): readonly McpToolDefinition[] { path_glob: { type: 'string' }, with_body: { type: 'boolean', default: true }, with_callers: { type: 'boolean' }, + detail: { type: 'string', enum: ['sig', 'body'] }, max_tokens: { type: 'integer', minimum: 1, maximum: 4000, default: DEFAULT_MCP_FIND_SYMBOL_MAX_TOKENS } }) }, @@ -428,6 +464,16 @@ export function getToolDefinitions(): readonly McpToolDefinition[] { max_tokens: { type: 'integer', minimum: 1, maximum: 4000, default: DEFAULT_MCP_RELATION_MAX_TOKENS } }) }, + { + name: 'changeset_context', + description: 'Symbols plus direct callers/importers for an explicit file list.', + inputSchema: jsonSchema(['files'], { + files: { type: 'array', items: { type: 'string' }, minItems: 1, maxItems: 200 }, + max_files: { type: 'integer', minimum: 1, maximum: MAX_CHANGESET_FILES, default: DEFAULT_CHANGESET_MAX_FILES }, + max_edges_per_file: { type: 'integer', minimum: 1, maximum: MAX_CHANGESET_EDGES_PER_FILE, default: DEFAULT_CHANGESET_MAX_EDGES_PER_FILE }, + max_tokens: { type: 'integer', minimum: 1, maximum: 4000, default: DEFAULT_MCP_RELATION_MAX_TOKENS } + }) + }, { name: 'grep_code', description: 'Literal/regex search for env vars, errors, operators, exact text.', @@ -494,6 +540,9 @@ export function createRouter(repo: Repo): McpRouter { if (args.with_usages !== undefined) { options.withUsages = args.with_usages } + if (args.with_relations !== undefined) { + options.withRelations = args.with_relations + } return repo.search(args.query, options) }, @@ -515,6 +564,10 @@ export function createRouter(repo: Repo): McpRouter { if (args.with_callers !== undefined) { options.withCallers = args.with_callers } + if (args.detail !== undefined) { + options.detail = args.detail + } + options.maxTokens = args.max_tokens ?? DEFAULT_MCP_FIND_SYMBOL_MAX_TOKENS return repo.findSymbol(args.name, options) }, @@ -569,6 +622,13 @@ export function createRouter(repo: Repo): McpRouter { return repo.impact(args.name, options) }, + async changesetContext(args) { + return repo.changesetContext(args.files, { + ...(args.max_files !== undefined ? { maxFiles: args.max_files } : {}), + ...(args.max_edges_per_file !== undefined ? { maxEdgesPerFile: args.max_edges_per_file } : {}), + maxTokens: args.max_tokens ?? DEFAULT_MCP_RELATION_MAX_TOKENS + }) + }, async grepCode(args) { const options: GrepOptions = {} @@ -629,46 +689,91 @@ export async function callMcpTool(repo: Repo, name: McpToolName, args: unknown): case 'search_code': { const parsed = searchCodeArgsSchema.parse(args) + const unavailable = await formatQueryUnavailableIfNotReady(repo) + if (unavailable) { + return unavailable + } return formatMcpSearchHits(await router.searchCode(parsed), { maxTokens: parsed.max_tokens ?? DEFAULT_MCP_SEARCH_MAX_TOKENS }) } case 'find_symbol': { const parsed = findSymbolArgsSchema.parse(args) + const unavailable = await formatQueryUnavailableIfNotReady(repo) + if (unavailable) { + return unavailable + } return formatMcpSymbols(await router.findSymbol(parsed), { maxTokens: parsed.max_tokens ?? DEFAULT_MCP_FIND_SYMBOL_MAX_TOKENS }) } case 'find_callers': { const parsed = findCallersArgsSchema.parse(args) + const unavailable = await formatQueryUnavailableIfNotReady(repo) + if (unavailable) { + return unavailable + } return formatMcpCallers(await router.findCallers(parsed), { maxTokens: parsed.max_tokens ?? DEFAULT_MCP_RELATION_MAX_TOKENS }) } case 'find_refs': { const parsed = findRefsArgsSchema.parse(args) + const unavailable = await formatQueryUnavailableIfNotReady(repo) + if (unavailable) { + return unavailable + } return formatMcpReferences(await router.findReferences(parsed), { maxTokens: parsed.max_tokens ?? DEFAULT_MCP_RELATION_MAX_TOKENS }) } case 'find_importers': { const parsed = findImportersArgsSchema.parse(args) + const unavailable = await formatQueryUnavailableIfNotReady(repo) + if (unavailable) { + return unavailable + } return formatMcpImporters(await router.findImporters(parsed), { maxTokens: parsed.max_tokens ?? DEFAULT_MCP_RELATION_MAX_TOKENS }) } case 'who_implements': { const parsed = findImplementersArgsSchema.parse(args) + const unavailable = await formatQueryUnavailableIfNotReady(repo) + if (unavailable) { + return unavailable + } return formatMcpImplementers(await router.findImplementers(parsed), { maxTokens: parsed.max_tokens ?? DEFAULT_MCP_RELATION_MAX_TOKENS }) } case 'impact': { const parsed = impactArgsSchema.parse(args) + const unavailable = await formatQueryUnavailableIfNotReady(repo) + if (unavailable) { + return unavailable + } return formatMcpImpact(await router.impact(parsed), { maxTokens: parsed.max_tokens ?? DEFAULT_MCP_RELATION_MAX_TOKENS }) } + case 'changeset_context': + { + const parsed = changesetContextArgsSchema.parse(args) + const unavailable = await formatQueryUnavailableIfNotReady(repo) + if (unavailable) { + return unavailable + } + return formatMcpChangesetContext(await router.changesetContext(parsed), { maxTokens: parsed.max_tokens ?? DEFAULT_MCP_RELATION_MAX_TOKENS }) + } case 'grep_code': { const parsed = grepCodeArgsSchema.parse(args) + const unavailable = await formatQueryUnavailableIfNotReady(repo) + if (unavailable) { + return unavailable + } return formatMcpGrepHits(await router.grepCode(parsed), { maxTokens: parsed.max_tokens ?? DEFAULT_MCP_GREP_MAX_TOKENS }) } case 'read_chunk': { const parsed = readChunkArgsSchema.parse(args) + const unavailable = await formatQueryUnavailableIfNotReady(repo) + if (unavailable) { + return unavailable + } return formatMcpReadChunk(await router.readChunk(parsed), { maxTokens: parsed.max_tokens ?? DEFAULT_MCP_READ_CHUNK_MAX_TOKENS }) } case 'index_status': @@ -679,6 +784,34 @@ export async function callMcpTool(repo: Repo, name: McpToolName, args: unknown): } } +async function formatQueryUnavailableIfNotReady(repo: Repo): Promise { + const status = await repo.status() + if (!status.indexExists) { + return NOT_INDEXED_SENTINEL + } + if (status.indexed) { + return '' + } + if (status.sync.state === 'running') { + const progress = formatSyncProgress(status) + return `indexing; sync=running${progress}; retry shortly or use index_status` + } + if (status.sync.state === 'failed' || status.sync.state === 'aborted') { + return `not_indexed; sync=${status.sync.state}; use index_status` + } + return '' +} + +function formatSyncProgress(status: RepoStatus): string { + if (status.sync.completedChunks !== undefined && status.sync.totalChunks !== undefined) { + return ` chunks=${status.sync.completedChunks}/${status.sync.totalChunks}` + } + if (status.chunkCount > 0) { + return ` chunks=${status.chunkCount}` + } + return '' +} + export async function handleMcpJsonRpcRequest(repo: Repo, request: McpJsonRpcRequest): Promise { const id = request.id ?? null const method = request.method @@ -734,46 +867,61 @@ export async function handleMcpJsonRpcRequest(repo: Repo, request: McpJsonRpcReq } export function createSdkServer(repo: Repo): McpServer { - const router = createRouter(repo) const server = new McpServer( { name: 'codesift', version: '0.0.0' }, { instructions: MCP_SERVER_INSTRUCTIONS } ) - server.registerTool('search_code', { description: toolDescription('search_code'), inputSchema: searchCodeInputSchema }, async (args) => - textResult(formatMcpSearchHits(await router.searchCode(args), { maxTokens: args.max_tokens ?? DEFAULT_MCP_SEARCH_MAX_TOKENS })) + server.registerTool('search_code', toolRegistrationOptions('search_code', searchCodeInputSchema), async (args) => + textResult(await callMcpTool(repo, 'search_code', args)) + ) + server.registerTool('find_symbol', toolRegistrationOptions('find_symbol', findSymbolInputSchema), async (args) => + textResult(await callMcpTool(repo, 'find_symbol', args)) ) - server.registerTool('find_symbol', { description: toolDescription('find_symbol'), inputSchema: findSymbolInputSchema }, async (args) => - textResult(formatMcpSymbols(await router.findSymbol(args), { maxTokens: args.max_tokens ?? DEFAULT_MCP_FIND_SYMBOL_MAX_TOKENS })) + server.registerTool('find_callers', toolRegistrationOptions('find_callers', findCallersInputSchema), async (args) => + textResult(await callMcpTool(repo, 'find_callers', args)) ) - server.registerTool('find_callers', { description: toolDescription('find_callers'), inputSchema: findCallersInputSchema }, async (args) => - textResult(formatMcpCallers(await router.findCallers(args), { maxTokens: args.max_tokens ?? DEFAULT_MCP_RELATION_MAX_TOKENS })) + server.registerTool('find_refs', toolRegistrationOptions('find_refs', findRefsInputSchema), async (args) => + textResult(await callMcpTool(repo, 'find_refs', args)) ) - server.registerTool('find_refs', { description: toolDescription('find_refs'), inputSchema: findRefsInputSchema }, async (args) => - textResult(formatMcpReferences(await router.findReferences(args), { maxTokens: args.max_tokens ?? DEFAULT_MCP_RELATION_MAX_TOKENS })) + server.registerTool('find_importers', toolRegistrationOptions('find_importers', findImportersInputSchema), async (args) => + textResult(await callMcpTool(repo, 'find_importers', args)) ) - server.registerTool('find_importers', { description: toolDescription('find_importers'), inputSchema: findImportersInputSchema }, async (args) => - textResult(formatMcpImporters(await router.findImporters(args), { maxTokens: args.max_tokens ?? DEFAULT_MCP_RELATION_MAX_TOKENS })) + server.registerTool('who_implements', toolRegistrationOptions('who_implements', findImplementersInputSchema), async (args) => + textResult(await callMcpTool(repo, 'who_implements', args)) ) - server.registerTool('who_implements', { description: toolDescription('who_implements'), inputSchema: findImplementersInputSchema }, async (args) => - textResult(formatMcpImplementers(await router.findImplementers(args), { maxTokens: args.max_tokens ?? DEFAULT_MCP_RELATION_MAX_TOKENS })) + server.registerTool('impact', toolRegistrationOptions('impact', impactInputSchema), async (args) => + textResult(await callMcpTool(repo, 'impact', args)) ) - server.registerTool('impact', { description: toolDescription('impact'), inputSchema: impactInputSchema }, async (args) => - textResult(formatMcpImpact(await router.impact(args), { maxTokens: args.max_tokens ?? DEFAULT_MCP_RELATION_MAX_TOKENS })) + server.registerTool('changeset_context', toolRegistrationOptions('changeset_context', changesetContextInputSchema), async (args) => + textResult(await callMcpTool(repo, 'changeset_context', args)) ) - server.registerTool('grep_code', { description: toolDescription('grep_code'), inputSchema: grepCodeInputSchema }, async (args) => - textResult(formatMcpGrepHits(await router.grepCode(args), { maxTokens: args.max_tokens ?? DEFAULT_MCP_GREP_MAX_TOKENS })) + server.registerTool('grep_code', toolRegistrationOptions('grep_code', grepCodeInputSchema), async (args) => + textResult(await callMcpTool(repo, 'grep_code', args)) ) - server.registerTool('read_chunk', { description: toolDescription('read_chunk'), inputSchema: readChunkInputSchema }, async (args) => - textResult(formatMcpReadChunk(await router.readChunk(args), { maxTokens: args.max_tokens ?? DEFAULT_MCP_READ_CHUNK_MAX_TOKENS })) + server.registerTool('read_chunk', toolRegistrationOptions('read_chunk', readChunkInputSchema), async (args) => + textResult(await callMcpTool(repo, 'read_chunk', args)) ) - server.registerTool('index_status', { description: toolDescription('index_status'), inputSchema: indexStatusInputSchema }, async (args) => - textResult(formatMcpIndexStatus(await router.indexStatus(), { maxTokens: args.max_tokens ?? DEFAULT_MCP_INDEX_STATUS_MAX_TOKENS })) + server.registerTool('index_status', toolRegistrationOptions('index_status', indexStatusInputSchema), async (args) => + textResult(await callMcpTool(repo, 'index_status', args)) ) return server } +const READ_ONLY_TOOL_ANNOTATIONS = { + readOnlyHint: true, + openWorldHint: false +} as const + +function toolRegistrationOptions(name: McpToolName, inputSchema: T) { + return { + description: toolDescription(name), + inputSchema, + annotations: READ_ONLY_TOOL_ANNOTATIONS + } +} + function textResult(text: string): { content: Array<{ type: 'text'; text: string }> } { return { content: [{ type: 'text', text }] } } @@ -814,8 +962,15 @@ function toolDescription(name: McpToolName): string { return tool?.description ?? name } +function resultMetadata(items: T[]): ResultMetadata | undefined { + return (items as T[] & { meta?: ResultMetadata }).meta +} + export function formatMcpSearchHits(hits: SearchHit[], options: FormatMcpSearchHitsOptions = {}): string { if (hits.length === 0) { + if (resultMetadata(hits)?.notIndexed) { + return NOT_INDEXED_SENTINEL + } return 'no_hits' } @@ -853,6 +1008,10 @@ function formatMcpSearchHit(hit: SearchHit): string { if (hit.usages?.length) { sections.push(formatUsageBlock(hit.usages)) } + const relations = formatMcpSymbolRelations(hit.relations) + if (relations) { + sections.push(relations) + } return joinSections(sections) } @@ -1300,6 +1459,9 @@ function formatIndexStatusDetailLines(status: RepoStatus): string[] { if (status.sync.error) { lines.push(`sync_error=${compactStatusValue(status.sync.error, 140)}`) } + if (status.sync.state === 'running' && status.sync.completedChunks !== undefined && status.sync.totalChunks !== undefined) { + lines.push(`sync_progress=${status.sync.completedChunks}/${status.sync.totalChunks}`) + } if (!status.compatibility.ok) { if (status.compatibility.message) { @@ -1444,17 +1606,35 @@ function formatHitSymbol(hit: SearchHit): string { export function formatMcpSymbols(definitions: SymbolDefinition[], options: FormatMcpSymbolsOptions = {}): string { if (definitions.length === 0) { + if (resultMetadata(definitions)?.notIndexed) { + return NOT_INDEXED_SENTINEL + } return 'no_symbols' } + const hints = formatSymbolLeadHints(definitions) const rendered = definitions.map(formatMcpSymbolDefinition) - const output = rendered.join('\n') + const output = joinSections([...hints, ...rendered]) const maxTokens = options.maxTokens if (maxTokens === undefined || output.length <= maxTokens * 4) { return output } - return fitSymbolOutputForBudget(definitions, rendered, maxTokens * 4) + return fitSymbolOutputForBudget(definitions, rendered, maxTokens * 4, hints) +} + +function formatSymbolLeadHints(definitions: SymbolDefinition[]): string[] { + const metadata = resultMetadata(definitions) + const hints: string[] = [] + const ambiguousDefCount = definitions[0]?.ambiguousDefCount ?? metadata?.ambiguousDefCount + if (ambiguousDefCount && ambiguousDefCount >= 2) { + hints.push(`ambiguous: ${ambiguousDefCount} defs`) + } + const partialMatchCount = metadata?.partialMatchCount ?? definitions.filter((definition) => definition.matchQuality === 'partial').length + if (partialMatchCount > 0 && definitions.every((definition) => definition.matchQuality === 'partial')) { + hints.push(`exact_miss; partial_matches=${partialMatchCount}`) + } + return hints } function formatMcpSymbolDefinition(definition: SymbolDefinition, index: number): string { @@ -1474,7 +1654,8 @@ function formatMcpSymbolDefinition(definition: SymbolDefinition, index: number): } function formatMcpSymbolHeader(definition: SymbolDefinition, index: number): string { - return `#${index + 1} ${definition.kind} ${definition.name} ${definition.file}:${formatRange(definition.range.startLine, definition.range.endLine)}` + const quality = definition.matchQuality === 'partial' ? ' partial' : '' + return `#${index + 1}${quality} ${definition.kind} ${definition.name} ${definition.file}:${formatRange(definition.range.startLine, definition.range.endLine)}` } function formatMcpSymbolRelations(relations: SymbolDefinition['relations']): string { @@ -1517,27 +1698,29 @@ function countSymbolRelationItems(relations: SymbolDefinition['relations']): num return relations.sites.length + relations.neighbors.length + (relations.omitted ?? 0) } -function fitSymbolOutputForBudget(definitions: SymbolDefinition[], rendered: string[], maxChars: number): string { +function fitSymbolOutputForBudget(definitions: SymbolDefinition[], rendered: string[], maxChars: number, leadHints: string[] = []): string { + const leadBudget = joinSections(leadHints).length + const rowBudget = leadBudget > 0 ? Math.max(0, maxChars - leadBudget - 1) : maxChars const omittedAfterFirst = definitions.length - 1 const firstHasRelations = countSymbolRelationItems(definitions[0]?.relations) > 0 const firstOnlyMarker = omittedAfterFirst > 0 - ? symbolOmissionMarker(omittedAfterFirst, Math.max(0, maxChars - 2)) + ? symbolOmissionMarker(omittedAfterFirst, Math.max(0, rowBudget - 2)) : firstHasRelations ? undefined - : symbolBodyTruncationMarker(Math.max(0, maxChars - 2)) - const firstOnlyBudget = maxCharsForFirstHit(maxChars, firstOnlyMarker) + : symbolBodyTruncationMarker(Math.max(0, rowBudget - 2)) + const firstOnlyBudget = maxCharsForFirstHit(rowBudget, firstOnlyMarker) if (rendered[0]!.length > firstOnlyBudget) { const first = formatMcpSymbolDefinitionForBudget(definitions[0]!, 0, firstOnlyBudget) - return firstOnlyMarker ? [first, firstOnlyMarker].filter(Boolean).join('\n') : first + return joinSections([...leadHints, first, firstOnlyMarker]) } const parts = [rendered[0]!] for (let index = 1; index < rendered.length; index += 1) { const candidate = [...parts, rendered[index]!] const omitted = definitions.length - 1 - (candidate.length - 1) - const marker = omitted > 0 ? symbolOmissionMarker(omitted, Math.max(0, maxChars - 2)) : undefined + const marker = omitted > 0 ? symbolOmissionMarker(omitted, Math.max(0, rowBudget - 2)) : undefined const candidateOutput = marker ? [...candidate, marker].join('\n') : candidate.join('\n') - if (candidateOutput.length <= maxChars) { + if (candidateOutput.length <= rowBudget) { parts.push(rendered[index]!) continue } @@ -1545,10 +1728,10 @@ function fitSymbolOutputForBudget(definitions: SymbolDefinition[], rendered: str } const omitted = definitions.length - parts.length - const marker = omitted > 0 ? symbolOmissionMarker(omitted, Math.max(0, maxChars - 2)) : undefined + const marker = omitted > 0 ? symbolOmissionMarker(omitted, Math.max(0, rowBudget - 2)) : undefined const outputParts = marker ? [...parts, marker] : parts - const output = outputParts.join('\n') + const output = joinSections([...leadHints, ...outputParts]) return output.length <= maxChars ? output : truncateWithEllipsis(output, maxChars) } @@ -1716,7 +1899,13 @@ export function formatMcpImplementers(results: EdgeResult[], options: FormatMcpE } export function formatMcpImpact(result: ImpactResult, options: FormatMcpImpactOptions = {}): string { + if (result.notIndexed) { + return NOT_INDEXED_SENTINEL + } if (result.nodes.length === 0) { + if (result.emptyReason === 'no_definition') { + return 'no_definition; refine name/kind/path_glob' + } return 'no_impact' } @@ -1731,6 +1920,77 @@ export function formatMcpImpact(result: ImpactResult, options: FormatMcpImpactOp return fitImpactOutputForBudget(rendered, notes, maxTokens * 4) } +export function formatMcpChangesetContext(result: ChangesetContextResult, options: FormatMcpChangesetContextOptions = {}): string { + if (result.notIndexed) { + return NOT_INDEXED_SENTINEL + } + if (result.files.length === 0) { + return 'no_changeset_context' + } + + const sections: string[] = [] + if (result.stale) { + sections.push('stale=true') + } + for (const file of result.files) { + sections.push(formatMcpChangesetFile(file)) + } + if (result.truncated) { + sections.push('changeset_truncated=true; narrow files/max_edges_per_file or raise max_tokens') + } + + const output = joinSections(sections) + const maxTokens = options.maxTokens + if (maxTokens === undefined || output.length <= maxTokens * 4) { + return output + } + + return fitChangesetOutputForBudget(sections, maxTokens * 4) +} + +function formatMcpChangesetFile(file: ChangesetFileContext): string { + const lines = [`file ${file.file}`] + const renderedSymbols = file.symbols.slice(0, 8) + for (const symbol of renderedSymbols) { + lines.push(`- symbol ${symbol.kind} ${symbol.name} ${formatRange(symbol.range.startLine, symbol.range.endLine)}`) + } + const symbolsOmitted = file.symbols.length - renderedSymbols.length + if (symbolsOmitted > 0) { + lines.push(`- symbols_omitted=${symbolsOmitted}; narrow files`) + } + for (const importer of file.importers) { + lines.push(`- importer ${formatMcpEdgeResult(importer)}`) + } + for (const caller of file.callers) { + lines.push(`- caller ${formatMcpEdgeResult(caller)}`) + } + if (file.omitted && file.omitted > 0) { + const omittedLabel = file.omittedLowerBound ? `omitted>=${file.omitted}` : `omitted=${file.omitted}` + const recoveryHint = file.omittedLowerBound ? 'raise max_edges_per_file or narrow files' : 'raise max_tokens or narrow files' + lines.push(`- ${omittedLabel}; ${recoveryHint}`) + } + return lines.join('\n') +} + +function fitChangesetOutputForBudget(sections: string[], maxChars: number): string { + const requiredMarker = 'changeset_truncated=true' + const kept: string[] = [] + for (const section of sections) { + const candidate = joinSections([...kept, section, requiredMarker]) + if (candidate.length <= maxChars) { + kept.push(section) + continue + } + break + } + + if (kept.length === 0) { + return bestEffortOmissionMarker([requiredMarker], maxChars) + } + + return joinSections([...kept, requiredMarker]) +} + function buildImpactNoteLines(result: ImpactResult): string[] { const notes: string[] = [] if (result.depthCapped) { @@ -1829,18 +2089,40 @@ function formatMcpEdgeResults( omissionLabel: 'callers' | 'refs' | 'importers' | 'implementers', options: FormatMcpEdgeResultsOptions ): string { + const metadata = resultMetadata(results) if (results.length === 0) { + if (metadata?.notIndexed) { + return NOT_INDEXED_SENTINEL + } + if (metadata?.emptyReason === 'no_definition') { + return 'no_definition; refine name/kind/path_glob' + } + if (metadata?.emptyReason === 'no_edges') { + return `${emptyMarker}; indexed_edges=0` + } return emptyMarker } + const leadLines = formatEdgeLeadLines(metadata) const rendered = results.map((result) => formatMcpEdgeResult(result)) - const output = rendered.join('\n') + const output = joinSections([...leadLines, ...rendered]) const maxTokens = options.maxTokens if (maxTokens === undefined || output.length <= maxTokens * 4) { return output } - return fitEdgeResultOutputForBudget(rendered, omissionLabel, maxTokens * 4) + return fitEdgeResultOutputForBudget(rendered, omissionLabel, maxTokens * 4, leadLines) +} + +function formatEdgeLeadLines(metadata: ResultMetadata | undefined): string[] { + const leadLines: string[] = [] + if (metadata?.ambiguousDefCount && metadata.ambiguousDefCount >= 2) { + leadLines.push(`ambiguous: ${metadata.ambiguousDefCount} defs`) + } + if (metadata?.nameOnlyUnscoped && metadata.nameOnlyUnscoped > (metadata.nameOnlyLimit ?? 0)) { + leadLines.push(`name_only_unscoped=${metadata.nameOnlyUnscoped}; narrow with path_glob/kind`) + } + return leadLines } function formatMcpEdgeResult(result: EdgeResult, maxChars?: number): string { @@ -1863,24 +2145,27 @@ function formatMcpEdgeResult(result: EdgeResult, maxChars?: number): string { function fitEdgeResultOutputForBudget( rendered: string[], omissionLabel: 'callers' | 'refs' | 'importers' | 'implementers', - maxChars: number + maxChars: number, + leadLines: string[] = [] ): string { const kept: string[] = [] + const leadBudget = joinSections(leadLines).length + const rowBudget = leadBudget > 0 ? Math.max(0, maxChars - leadBudget - 1) : maxChars for (let index = 0; index < rendered.length; index += 1) { const line = rendered[index]! const omittedAfter = rendered.length - index - 1 const marker = omittedAfter > 0 - ? edgeResultOmissionMarker(omissionLabel, omittedAfter, remainingCharsForTrailingLine([...kept, line], maxChars)) + ? edgeResultOmissionMarker(omissionLabel, omittedAfter, remainingCharsForTrailingLine([...kept, line], rowBudget)) : '' const candidate = joinSections([...kept, line, marker]) - if (candidate.length <= maxChars) { + if (candidate.length <= rowBudget) { kept.push(line) continue } if (kept.length === 0) { - return formatSingleEdgeResultForBudget(line, omissionLabel, omittedAfter, maxChars) + return joinSections([...leadLines, formatSingleEdgeResultForBudget(line, omissionLabel, omittedAfter, rowBudget)]) } break @@ -1888,20 +2173,20 @@ function fitEdgeResultOutputForBudget( const omitted = rendered.length - kept.length if (omitted <= 0) { - return joinSections(kept) + return joinSections([...leadLines, ...kept]) } - let marker = edgeResultOmissionMarker(omissionLabel, omitted, remainingCharsForTrailingLine(kept, maxChars)) + let marker = edgeResultOmissionMarker(omissionLabel, omitted, remainingCharsForTrailingLine(kept, rowBudget)) while (!marker && kept.length > 0) { kept.pop() - marker = edgeResultOmissionMarker(omissionLabel, rendered.length - kept.length, remainingCharsForTrailingLine(kept, maxChars)) + marker = edgeResultOmissionMarker(omissionLabel, rendered.length - kept.length, remainingCharsForTrailingLine(kept, rowBudget)) } if (!marker) { - return bestEffortOmissionMarker(edgeResultOmissionMarkerVariants(omissionLabel, omitted), maxChars) + return joinSections([...leadLines, bestEffortOmissionMarker(edgeResultOmissionMarkerVariants(omissionLabel, omitted), rowBudget)]) } - return joinSections([...kept, marker]) + return joinSections([...leadLines, ...kept, marker]) } function formatSingleEdgeResultForBudget( @@ -1948,6 +2233,9 @@ function formatMcpEdgeResultLineForBudget(line: string, maxChars: number): strin export function formatMcpGrepHits(hits: GrepHit[], options: FormatMcpGrepHitsOptions = {}): string { if (hits.length === 0) { + if (resultMetadata(hits)?.notIndexed) { + return NOT_INDEXED_SENTINEL + } return 'no_matches' } diff --git a/packages/mcp/test/http.test.ts b/packages/mcp/test/http.test.ts index a1ca015..7255012 100644 --- a/packages/mcp/test/http.test.ts +++ b/packages/mcp/test/http.test.ts @@ -6,7 +6,7 @@ import { afterEach, describe, expect, it } from 'vitest' import { openRepo } from '@codesift/core' -import { HttpMcpServerHandle, createHttpServer } from '../src/index.js' +import { HttpMcpServerHandle, NOT_INDEXED_SENTINEL, createHttpServer } from '../src/index.js' const PROTOCOL_VERSION = '2025-06-18' const temporaryDirectories: string[] = [] @@ -114,6 +114,41 @@ describe('@codesift/mcp http transport', () => { await handle.stop() } }, 15_000) + + it('returns the not-indexed sentinel through the SDK server path', async () => { + const repoRoot = await createDemoRepo() + const repo = await openRepo(repoRoot) + const handle = createHttpServer(repo, { port: 0 }) as HttpMcpServerHandle + await handle.start() + + try { + const base = `http://127.0.0.1:${handle.port}/` + const initialize = await mcpCall(base, undefined, { + jsonrpc: '2.0', + id: 1, + method: 'initialize', + params: { + protocolVersion: PROTOCOL_VERSION, + capabilities: {}, + clientInfo: { name: 'codesift-vitest-http', version: '0.0.0' } + } + }) + expect(initialize.status).toBe(200) + + const call = await mcpCall(base, undefined, { + jsonrpc: '2.0', + id: 2, + method: 'tools/call', + params: { name: 'read_chunk', arguments: { id: 'src/demo.ts:1-2@missing' } } + }) + + expect(call.status).toBe(200) + expect(JSON.stringify(call.message.result)).toContain(NOT_INDEXED_SENTINEL) + expect(JSON.stringify(call.message.result)).not.toContain('isError') + } finally { + await handle.stop() + } + }, 15_000) }) async function createDemoRepo(): Promise { diff --git a/packages/mcp/test/mcp.test.ts b/packages/mcp/test/mcp.test.ts index c57e33c..43e3149 100644 --- a/packages/mcp/test/mcp.test.ts +++ b/packages/mcp/test/mcp.test.ts @@ -5,7 +5,7 @@ import { join } from 'node:path' import { afterEach, describe, expect, it } from 'vitest' -import { openRepo, registerEmbeddingProvider, type EdgeResult, type GrepHit, type ImpactResult, type RepoStatus, type SearchHit, type SymbolDefinition } from '@codesift/core' +import { openRepo, registerEmbeddingProvider, type EdgeResult, type GrepHit, type ImpactResult, type Repo, type RepoStatus, type SearchHit, type SymbolDefinition } from '@codesift/core' import { DEFAULT_MCP_FIND_SYMBOL_MAX_TOKENS, @@ -16,11 +16,13 @@ import { DEFAULT_SEARCH_K, MIN_MCP_READ_CHUNK_MAX_TOKENS, MCP_SERVER_INSTRUCTIONS, + NOT_INDEXED_SENTINEL, createHttpServer, createRouter, createStdioServer, callMcpTool, formatMcpCallers, + formatMcpChangesetContext, formatMcpGrepHits, formatMcpImpact, formatMcpImplementers, @@ -68,6 +70,7 @@ describe('@codesift/mcp server', () => { 'find_importers', 'who_implements', 'impact', + 'changeset_context', 'grep_code', 'read_chunk', 'index_status' @@ -78,6 +81,7 @@ describe('@codesift/mcp server', () => { }) expect(getToolDefinitions().find((tool) => tool.name === 'find_symbol')?.inputSchema.properties).toMatchObject({ with_callers: { type: 'boolean' }, + detail: { type: 'string', enum: ['sig', 'body'] }, max_tokens: { type: 'integer', minimum: 1, maximum: 4000, default: DEFAULT_MCP_FIND_SYMBOL_MAX_TOKENS } }) expect(getToolDefinitions().find((tool) => tool.name === 'find_callers')?.inputSchema.required).toEqual(['name']) @@ -93,12 +97,17 @@ describe('@codesift/mcp server', () => { }) expect(getToolDefinitions().find((tool) => tool.name === 'search_code')?.inputSchema.properties).toMatchObject({ max_tokens: { type: 'integer', minimum: 1, maximum: 4000, default: DEFAULT_MCP_SEARCH_MAX_TOKENS }, - context: { type: 'string', enum: ['sig', 'body'] }, - with_usages: { type: 'boolean' } + context: { type: 'string', enum: ['auto', 'min', 'sig', 'body', 'graph'] }, + with_usages: { type: 'boolean' }, + with_relations: { type: 'boolean' } }) expect(getToolDefinitions().find((tool) => tool.name === 'index_status')?.inputSchema.properties).toMatchObject({ max_tokens: { type: 'integer', minimum: 1, maximum: 1000, default: DEFAULT_MCP_INDEX_STATUS_MAX_TOKENS } }) + expect(getToolDefinitions().find((tool) => tool.name === 'changeset_context')?.inputSchema.properties).toMatchObject({ + max_files: { type: 'integer', minimum: 1, maximum: 40, default: 40 }, + max_edges_per_file: { type: 'integer', minimum: 1, maximum: 25, default: 12 } + }) expect(MCP_SERVER_INSTRUCTIONS).toContain('breakage/transitive callers->impact') expect(MCP_SERVER_INSTRUCTIONS).toContain('approx:name-only') expect(MCP_SERVER_INSTRUCTIONS).toContain('literals/regex/errors/env/operators->grep_code') @@ -141,7 +150,7 @@ describe('@codesift/mcp server', () => { expect(MCP_SERVER_INSTRUCTIONS).toContain('breakage/transitive callers->impact') expect(MCP_SERVER_INSTRUCTIONS).toContain('concepts/fuzzy names->search_code') expect(MCP_SERVER_INSTRUCTIONS).toContain('Broad search k=5-8') - expect(MCP_SERVER_INSTRUCTIONS).toContain('Check index_status') + expect(MCP_SERVER_INSTRUCTIONS).toContain('Health is inline') }) it('keeps control-plane metadata under the MCP budget', () => { @@ -155,14 +164,15 @@ describe('@codesift/mcp server', () => { ['find_importers', 90], ['who_implements', 125], ['impact', 110], + ['changeset_context', 90], ['grep_code', 120], ['read_chunk', 120], ['index_status', 90] ]) - expect(payload.length).toBeLessThanOrEqual(6000) - expect(Math.ceil(payload.length / 4)).toBeLessThanOrEqual(1500) - expect(MCP_SERVER_INSTRUCTIONS.length).toBeLessThanOrEqual(430) + expect(payload.length).toBeLessThanOrEqual(7000) + expect(Math.ceil(payload.length / 4)).toBeLessThanOrEqual(1750) + expect(MCP_SERVER_INSTRUCTIONS.length).toBeLessThanOrEqual(560) for (const tool of tools) { expect(tool.description.length).toBeLessThanOrEqual(toolDescriptionCeilings.get(tool.name) ?? 120) } @@ -189,6 +199,27 @@ describe('@codesift/mcp server', () => { expect(getToolDefinitions()[0]?.description.toLowerCase()).toContain('hybrid') }) + it('returns indexing status instead of empty search output during first sync', async () => { + const repo = { + async status() { + return makeStatus({ + indexExists: true, + indexed: false, + sync: { state: 'running', completedChunks: 2, totalChunks: 7 }, + chunkCount: 0, + symbolCount: 0, + indexGeneration: 0, + provider: null + }) + }, + async search() { + throw new Error('search should not run while the first index is still building') + } + } as unknown as Repo + + await expect(callMcpTool(repo, 'search_code', { query: 'demoValue' })).resolves.toBe('indexing; sync=running chunks=2/7; retry shortly or use index_status') + }) + it('routes tool calls through the core repo contract', async () => { const repoRoot = await mkdtemp(join(tmpdir(), 'codesift-mcp-')) temporaryDirectories.push(repoRoot) @@ -225,6 +256,16 @@ describe('@codesift/mcp server', () => { expect(compactSymbols[0]?.body).toBeUndefined() expect(formatMcpSymbols(compactSymbols)).not.toContain("return 'demo'") + const signatureSymbols = await router.findSymbol({ name: 'demoValue', detail: 'sig' }) + expect(signatureSymbols[0]?.body).toBeUndefined() + const signatureOutput = await callMcpTool(repo, 'find_symbol', { name: 'demoValue', detail: 'sig' }) + expect(signatureOutput).toContain('#1 function demoValue src/demo.ts:') + expect(signatureOutput).not.toContain("return 'demo'") + + const partialOutput = await callMcpTool(repo, 'find_symbol', { name: 'demo', max_tokens: 80 }) + expect(partialOutput).toContain('exact_miss; partial_matches=1') + expect(partialOutput).toContain('#1 partial function demoValue src/demo.ts:') + expect(grepHits[0]?.file).toBe('src/demo.ts') expect(await router.readChunk({ id: hits[0]!.id })).toContain("return 'demo'") expect((await router.indexStatus()).indexed).toBe(true) @@ -350,6 +391,12 @@ export function main(header: string): string { expect(refsOutput).not.toContain('src/auth/token.ts') expect(refsOutput).not.toContain('src/forms/checkout.ts') + const ambiguousRefsOutput = await callMcpTool(collisionRepo, 'find_refs', { + name: 'validate', + max_tokens: 120 + }) + expect(ambiguousRefsOutput).toContain('ambiguous: 3 defs') + const heritageRepoRoot = await copyFixtureRepository('heritage-ts', 'codesift-mcp-graph-heritage-') const heritageRepo = await openRepo(heritageRepoRoot) await heritageRepo.sync() @@ -362,6 +409,49 @@ export function main(header: string): string { expect(implementersOutput).toContain('src/impl.ts:9 StrictStrategy extends import-resolved | export interface StrictStrategy extends AuthStrategy {}') }) + it('emits a name-only lead line from real capped graph lookups', async () => { + const repoRoot = await mkdtemp(join(tmpdir(), 'codesift-mcp-name-only-cap-')) + temporaryDirectories.push(repoRoot) + + await mkdir(join(repoRoot, 'auth'), { recursive: true }) + await writeFile( + join(repoRoot, 'auth', 'token.go'), + `package auth + +type TokenVerifier struct{} + +func (v TokenVerifier) VerifyToken(token string) bool { + return token != "" +} +`, + 'utf8' + ) + await Promise.all(Array.from({ length: 30 }, async (_, index) => { + await writeFile( + join(repoRoot, 'auth', `caller${index}.go`), + `package auth + +func ValidateBearer${index}(token string) bool { + verifier := TokenVerifier{} + return verifier.VerifyToken(token) +} +`, + 'utf8' + ) + })) + + const repo = await openRepo(repoRoot) + await repo.sync() + + const output = await callMcpTool(repo, 'find_callers', { + name: 'VerifyToken', + kind: 'method', + max_tokens: 600 + }) + expect(output).toContain('name_only_unscoped=30; narrow with path_glob/kind') + expect(output).toContain('approx:name-only') + }) + it('formats one-call find_symbol relations and bounded impact output', async () => { const repoRoot = await copyFixtureRepository('usages-ts', 'codesift-mcp-relations-output-') await writeFile( @@ -673,6 +763,7 @@ function makeStatus(overrides: Partial = {}): RepoStatus { return { root: '/tmp/repo', indexPath: '/tmp/repo/.codesift/index.db', + indexExists: true, indexed: true, stale: false, sync: { state: 'completed', completedAt: '2026-06-20T00:00:00.000Z' }, @@ -808,6 +899,37 @@ describe('formatMcpIndexStatus compact output', () => { }) }) +describe('formatMcpChangesetContext output', () => { + it('labels capped omissions as lower bounds and marks symbol clipping', () => { + const output = formatMcpChangesetContext({ + files: [ + { + file: 'src/auth.ts', + symbols: Array.from({ length: 10 }, (_, index) => ({ + id: `symbol-${index}`, + name: `symbol${index}`, + kind: 'function', + file: 'src/auth.ts', + range: { startLine: index + 1, endLine: index + 1 } + })), + callers: [], + importers: [], + omitted: 1, + omittedLowerBound: true + } + ], + truncated: true + }) + + expect(output).toContain('file src/auth.ts') + expect(output).toContain('- symbol function symbol7 8') + expect(output).not.toContain('symbol8') + expect(output).toContain('symbols_omitted=2; narrow files') + expect(output).toContain('omitted>=1; raise max_edges_per_file or narrow files') + expect(output).toContain('changeset_truncated=true') + }) +}) + describe('formatMcpCallers/References/Importers/Implementers budgeted output', () => { it('preserves empty markers and under-budget edge rows byte-for-byte', () => { const results = [makeEdgeResult()] @@ -1234,6 +1356,15 @@ describe('formatMcpGrepHits keeps its single-line ↩-joined format', () => { expect(formatMcpGrepHits([])).toBe('no_matches') }) + it('preserves not-indexed metadata on empty grep results', () => { + const hits = [] as GrepHit[] & { meta?: { notIndexed: boolean } } + Object.defineProperty(hits, 'meta', { + value: { notIndexed: true }, + enumerable: false + }) + expect(formatMcpGrepHits(hits)).toBe(NOT_INDEXED_SENTINEL) + }) + it('leaves grep formatting unchanged when the rendered hits fit under the token budget', () => { const output = formatMcpGrepHits([ makeGrepHit({ snippet: 'function verify() {\n return true\n}' }) diff --git a/scripts/smoke-pack-install.mjs b/scripts/smoke-pack-install.mjs index 8b93207..dc7321c 100644 --- a/scripts/smoke-pack-install.mjs +++ b/scripts/smoke-pack-install.mjs @@ -8,7 +8,19 @@ const repoRoot = resolve(fileURLToPath(new URL('..', import.meta.url))) const pnpmBin = process.platform === 'win32' ? 'pnpm.cmd' : 'pnpm' const npmBin = process.platform === 'win32' ? 'npm.cmd' : 'npm' const npxBin = process.platform === 'win32' ? 'npx.cmd' : 'npx' -const supportedNodeMajors = new Set([20, 22]) +const supportedNodeMajors = new Set(await readSupportedNodeMajors()) + +async function readSupportedNodeMajors() { + const source = await readFile(join(repoRoot, 'packages/cli/src/supported-node-majors.ts'), 'utf8') + const match = /supportedNodeMajors\s*=\s*\[([^\]]+)\]/.exec(source) + if (!match) { + throw new Error('could not read supported Node majors') + } + return match[1] + .split(',') + .map((value) => Number.parseInt(value.trim(), 10)) + .filter((value) => Number.isFinite(value)) +} function run(command, args, options = {}) { const result = spawnSync(command, args, { @@ -115,6 +127,11 @@ async function main() { await createFixtureRepo(fixtureDirectory) + const initOutput = run(npxBin, ['--no-install', 'codesift', 'init', fixtureDirectory, '--print'], { cwd: installDirectory }) + if (!initOutput.includes(`codesift@`)) { + throw new Error(`smoke init --print did not emit a pinned codesift command\n${initOutput}`) + } + run(npxBin, ['--no-install', 'codesift', 'index', fixtureDirectory], { cwd: installDirectory }) const searchOutput = run( npxBin, From 3778a515aebc406ce3f7e3540db87a5df6ebc202 Mon Sep 17 00:00:00 2001 From: Rutvik Chandla Date: Tue, 23 Jun 2026 02:24:27 +0530 Subject: [PATCH 2/2] Fix CI portability for north-goal changes --- packages/cli/test/program.test.ts | 2 +- packages/core/src/repo.ts | 12 +++++++++++- packages/core/test/core.test.ts | 14 +++++++++----- packages/eval/losses.json | 2 ++ 4 files changed, 23 insertions(+), 7 deletions(-) diff --git a/packages/cli/test/program.test.ts b/packages/cli/test/program.test.ts index 695b273..21ac67e 100644 --- a/packages/cli/test/program.test.ts +++ b/packages/cli/test/program.test.ts @@ -20,7 +20,7 @@ afterEach(async () => { await Promise.all( temporaryDirectories.splice(0).map(async (directory) => { - await rm(directory, { recursive: true, force: true }) + await rm(directory, { recursive: true, force: true, maxRetries: process.platform === 'win32' ? 5 : 0, retryDelay: 100 }) }) ) }) diff --git a/packages/core/src/repo.ts b/packages/core/src/repo.ts index 475d364..dd623ad 100644 --- a/packages/core/src/repo.ts +++ b/packages/core/src/repo.ts @@ -1497,7 +1497,17 @@ export class SqliteRepo implements Repo { } const db = new Database(this.indexPath) - this.configureDatabase(db, 'WAL') + try { + this.configureDatabase(db, 'WAL') + } catch (error) { + try { + db.close() + } catch { + // Preserve the schema/open error; this close is only to avoid leaking + // handles for corrupt databases, especially on Windows. + } + throw error + } this.vectorExtensionLoaded = false this.db = db return db diff --git a/packages/core/test/core.test.ts b/packages/core/test/core.test.ts index 78b8d83..b9c840a 100644 --- a/packages/core/test/core.test.ts +++ b/packages/core/test/core.test.ts @@ -44,7 +44,7 @@ afterEach(async () => { await Promise.all( temporaryDirectories.splice(0).map(async (directory) => { - await rm(directory, { recursive: true, force: true }) + await rm(directory, { recursive: true, force: true, maxRetries: process.platform === 'win32' ? 5 : 0, retryDelay: 100 }) }) ) }) @@ -374,11 +374,15 @@ export function mintFreshToken(subject: string): string { ) const repo = await openRepo(repoRoot) - await repo.sync() - const hits = await repo.search('common token', { k: 1, pathGlob: 'zzz/**' }) + try { + await repo.sync() + const hits = await repo.search('common token', { k: 1, pathGlob: 'zzz/**' }) - expect(hits.map((hit) => hit.file)).toEqual(['zzz/target.ts']) - }) + expect(hits.map((hit) => hit.file)).toEqual(['zzz/target.ts']) + } finally { + await repo.close() + } + }, 15_000) it('registers embedding providers and routes document/query roles with batched progress', async () => { const repoRoot = await mkdtemp(join(tmpdir(), 'codesift-batch-')) diff --git a/packages/eval/losses.json b/packages/eval/losses.json index d91f567..b0cd2a6 100644 --- a/packages/eval/losses.json +++ b/packages/eval/losses.json @@ -48,6 +48,7 @@ "queryId": "escape-regexp-symbol", "queryType": "symbol-def", "axes": [ + "latency.cold", "precision" ] }, @@ -89,6 +90,7 @@ "queryType": "nl-concept", "axes": [ "latency.cold", + "latency.warm", "tokens" ] },