diff --git a/.agents/plugins/marketplace.json b/.agents/plugins/marketplace.json new file mode 100644 index 0000000..d2591e3 --- /dev/null +++ b/.agents/plugins/marketplace.json @@ -0,0 +1,20 @@ +{ + "name": "Mixedbread-Grep", + "interface": { + "displayName": "Mixedbread" + }, + "plugins": [ + { + "name": "mgrep", + "source": { + "source": "local", + "path": "./plugins/mgrep" + }, + "policy": { + "installation": "AVAILABLE", + "authentication": "ON_INSTALL" + }, + "category": "Developer Tools" + } + ] +} diff --git a/README.md b/README.md index 6adffe6..e2d4ea6 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,44 @@ These commands sign you in (if needed) and add Mixedbread `mgrep` support to the agent. After that you only have to start the agent in your project folder, thats it. +### Codex + +With the Codex CLI installed, run: + +```bash +mgrep install-codex +``` + +This authenticates with Mixedbread, adds the `Mixedbread-Grep` marketplace, and +installs `mgrep@Mixedbread-Grep`. The plugin includes an MCP server with a +read-only `search` tool and a short skill. The MCP server syncs the session's +project directory in the background while Codex runs. + +A SessionStart hook adds search guidance on startup and resume, helping Codex +choose semantic search for conceptual or cross-file questions and rg or grep for +exact matches. On first start, Codex asks you to review the hook ("Hooks need +review"); you can also review it with `/hooks`. This one-time review allows the +hook to run. The hook only adds context; the MCP server owns background sync. + +Use MCP `search` for conceptual questions; `path` is optional and defaults to the +session's project directory. Use rg or grep for exact strings, symbol names, and +regexes. The MCP server runs `mgrep mcp --expose-tools`. Searches within the +project wait for the watcher's initial sync. If the watcher cannot start, or the +search path is outside the project, the first search syncs that root once. + +The plugin forwards `MXBAI_API_KEY` and `MXBAI_STORE` from Codex's environment. +You can also authenticate with `mgrep login`. No API key is written into plugin +configuration. To install from a local checkout or another Git URL, set +`MGREP_CODEX_MARKETPLACE` before running `mgrep install-codex`. + +```bash +mgrep uninstall-codex +``` + +Uninstall removes the plugin, its marketplace, and the legacy MCP registration. +Both commands remove the old mgrep block from `$CODEX_HOME/AGENTS.md` and +`~/.codex/AGENTS.md`, preserving surrounding instructions. + ### More Agents Coming Soon More agents (Cursor, Windsurf, etc.) are on the way—this section will grow as soon as each integration lands. diff --git a/package.json b/package.json index 81e653b..fa9e5de 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ "build": "tsc", "postbuild": "chmod +x dist/index.js && cp -r plugins dist/", "dev": "npx tsc && node dist/index.js", - "test": "bats test/test.bats --filter-tags !long-running", + "test": "bats test/ --filter-tags !long-running", "format": "biome check --write .", "format:check": "biome check .", "lint": "biome lint --check .", diff --git a/plugins/mgrep/.codex-plugin/plugin.json b/plugins/mgrep/.codex-plugin/plugin.json new file mode 100644 index 0000000..b43b255 --- /dev/null +++ b/plugins/mgrep/.codex-plugin/plugin.json @@ -0,0 +1,17 @@ +{ + "name": "mgrep", + "version": "0.1.13", + "description": "Search your local files using Mixedbread", + "author": { + "name": "Joel Dierkes" + }, + "skills": "./codex/skills/", + "mcpServers": "./codex/mcp.json", + "hooks": "./codex/hooks.json", + "interface": { + "displayName": "mgrep", + "shortDescription": "Semantic search for your project files", + "developerName": "Mixedbread", + "category": "Developer Tools" + } +} diff --git a/plugins/mgrep/codex/hooks.json b/plugins/mgrep/codex/hooks.json new file mode 100644 index 0000000..b386a64 --- /dev/null +++ b/plugins/mgrep/codex/hooks.json @@ -0,0 +1,16 @@ +{ + "hooks": { + "SessionStart": [ + { + "matcher": "startup|resume", + "hooks": [ + { + "type": "command", + "command": "python3 \"${CLAUDE_PLUGIN_ROOT:-$PLUGIN_ROOT}/codex/session_start.py\"", + "timeout": 5 + } + ] + } + ] + } +} diff --git a/plugins/mgrep/codex/mcp.json b/plugins/mgrep/codex/mcp.json new file mode 100644 index 0000000..563b998 --- /dev/null +++ b/plugins/mgrep/codex/mcp.json @@ -0,0 +1,9 @@ +{ + "mcpServers": { + "mgrep": { + "command": "mgrep", + "args": ["mcp", "--expose-tools"], + "env_vars": ["MXBAI_API_KEY", "MXBAI_STORE"] + } + } +} diff --git a/plugins/mgrep/codex/session_start.py b/plugins/mgrep/codex/session_start.py new file mode 100644 index 0000000..ebe5ff0 --- /dev/null +++ b/plugins/mgrep/codex/session_start.py @@ -0,0 +1,46 @@ +import json +import os +import sys +from pathlib import Path + + +def read_update_notice(): + try: + cache_path = Path.home() / ".mgrep" / "update-check.json" + with open(cache_path, "r", encoding="utf-8") as handle: + cache = json.load(handle) + latest = cache.get("latestVersion") + if not isinstance(latest, str): + return "" + return f" An mgrep update is available (latest: {latest}). Tell the user to run `mgrep update` to upgrade." + except Exception: + return "" + + +def main(): + try: + payload = json.load(sys.stdin) + except (ValueError, OSError): + payload = {} + cwd = payload.get("cwd") if isinstance(payload, dict) else None + if not isinstance(cwd, str) or not cwd or "\0" in cwd: + cwd = os.getcwd() + cwd = os.path.abspath(cwd) + context = ( + "mgrep semantic search is available through the `search` tool of the mgrep MCP server. " + "Use it for conceptual or cross-file questions where you don't know the exact identifier, " + "such as 'where is authentication handled'. " + f"It searches {cwd} by default. " + "Use rg or grep for exact strings, symbol names and regexes. " + "Both are fine; pick whichever fits the question." + ) + print(json.dumps({ + "hookSpecificOutput": { + "hookEventName": "SessionStart", + "additionalContext": context + read_update_notice(), + } + })) + + +if __name__ == "__main__": + main() diff --git a/plugins/mgrep/codex/skills/mgrep/SKILL.md b/plugins/mgrep/codex/skills/mgrep/SKILL.md new file mode 100644 index 0000000..1c58ca2 --- /dev/null +++ b/plugins/mgrep/codex/skills/mgrep/SKILL.md @@ -0,0 +1,28 @@ +--- +name: mgrep +description: Semantic search for conceptual questions about project files, alongside rg and grep for exact matches. +--- + +Use the mgrep MCP server's `search` tool for conceptual or cross-file questions +when you do not know the exact identifier, such as "where is authentication +handled". Provide a natural-language `query`; `path` is optional and defaults +to the session's project directory. Results contain file paths and line ranges; +read only those ranges. The MCP server keeps the session's project directory synced. + +Use rg or grep for exact strings, symbol names, and regexes. Both approaches are +fine; choose whichever fits the question. + +Outside MCP, use the CLI from your project directory: + +```bash +mgrep search "how are failed uploads retried" +mgrep search --sync "where is authentication handled" ./src +mgrep search --web --answer "how does this API work" +``` + +Prefer the MCP tool inside Codex to avoid shell network approval prompts. The CLI +works in a terminal or when MCP is unavailable. Set `web` and `answer` to true +for an MCP web lookup. Use `maxCount` (CLI `-m`) to limit results. + +`content: true` and CLI `-c` include full matching content and can produce large +output. Start with paths and line ranges, then read the relevant ranges. diff --git a/src/commands/search.ts b/src/commands/search.ts index 880379e..a9621ca 100644 --- a/src/commands/search.ts +++ b/src/commands/search.ts @@ -1,4 +1,3 @@ -import { join, normalize } from "node:path"; import type { Command } from "commander"; import { Command as CommanderCommand, InvalidArgumentError } from "commander"; import { @@ -8,13 +7,8 @@ import { } from "../lib/config.js"; import { createFileSystem, createStore } from "../lib/context.js"; import { DEFAULT_IGNORE_PATTERNS } from "../lib/file.js"; -import type { - AskResponse, - ChunkType, - FileMetadata, - SearchResponse, - Store, -} from "../lib/store.js"; +import { resolveSearchPath, runSearch } from "../lib/search.js"; +import type { Store } from "../lib/store.js"; import { createIndexingSpinner, formatDryRunSummary, @@ -26,107 +20,6 @@ import { QuotaExceededError, } from "../lib/utils.js"; -function extractSources(response: AskResponse): { [key: number]: ChunkType } { - const sources: { [key: number]: ChunkType } = {}; - const answer = response.answer; - - // Match ALL cite tags and capture the i="..." - const citeTags = answer.match(/= 0 && - end >= start && - end < response.sources.length - ) { - for (let i = start; i <= end; i++) { - sources[i] = response.sources[i]; - } - } - } - - return sources; -} - -function formatAskResponse(response: AskResponse, show_content: boolean) { - const sources = extractSources(response); - const sourceEntries = Object.entries(sources).map( - ([index, chunk]) => `${index}: ${formatChunk(chunk, show_content)}`, - ); - return `${response.answer}\n\n${sourceEntries.join("\n")}`; -} - -function formatSearchResponse(response: SearchResponse, show_content: boolean) { - return response.data - .map((chunk) => formatChunk(chunk, show_content)) - .join("\n"); -} - -function isWebResult(chunk: ChunkType): boolean { - return ( - "filename" in chunk && - typeof chunk.filename === "string" && - chunk.filename.startsWith("http") - ); -} - -function formatChunk(chunk: ChunkType, show_content: boolean) { - const pwd = process.cwd(); - - if (isWebResult(chunk) && chunk.type === "text") { - const url = "filename" in chunk ? chunk.filename : "Unknown URL"; - const content = show_content ? chunk.text : ""; - return `${url} (${(chunk.score * 100).toFixed(2)}% match)${content ? `\n${content}` : ""}`; - } - - const path = - (chunk.metadata as FileMetadata)?.path?.replace(pwd, "") ?? "Unknown path"; - let line_range = ""; - let content = ""; - switch (chunk.type) { - case "text": { - const start_line = (chunk.generated_metadata?.start_line as number) + 1; - const end_line = - start_line + (chunk.generated_metadata?.num_lines as number); - line_range = `:${start_line}-${end_line}`; - content = show_content ? chunk.text : ""; - break; - } - case "image_url": - line_range = - chunk.generated_metadata?.type === "pdf" - ? `, page ${chunk.chunk_index + 1}` - : ""; - break; - case "audio_url": - line_range = ""; - break; - case "video_url": - line_range = ""; - break; - } - - return `.${path}${line_range} (${(chunk.score * 100).toFixed(2)}% match)${content ? `\n${content}` : ""}`; -} - function parseBooleanEnv( envVar: string | undefined, defaultValue: boolean, @@ -138,9 +31,14 @@ function parseBooleanEnv( /** * Syncs local files to the store with progress indication. + * @param store The authenticated store. + * @param storeName The store to sync. + * @param root The absolute directory to sync. + * @param dryRun Whether to report changes without applying them. + * @param config The project indexing limits and settings. * @returns true if the caller should return early (dry-run mode), false otherwise */ -async function syncFiles( +export async function syncFiles( store: Store, storeName: string, root: string, @@ -285,9 +183,7 @@ export const search: Command = new CommanderCommand("search") }; const config = loadConfig(root, cliOptions); - const search_path = exec_path?.startsWith("/") - ? exec_path - : normalize(join(root, exec_path ?? "")); + const search_path = resolveSearchPath(root, exec_path); if (options.sync && isAtOrAboveHomeDirectory(search_path)) { console.error( @@ -316,45 +212,21 @@ export const search: Command = new CommanderCommand("search") } } - const storeIds = options.web - ? [options.store, "mixedbread/web"] - : [options.store]; - - const filters = { - all: [ - { - key: "path", - operator: "starts_with" as const, - value: search_path, - }, - ], - }; - - const searchOptions = { - rerank: options.rerank, - ...(options.agentic && { agentic: true }), - }; - - let response: string; - if (!options.answer) { - const results = await store.search( - storeIds, - pattern, - parseInt(options.maxCount, 10), - searchOptions, - filters, - ); - response = formatSearchResponse(results, options.content); - } else { - const results = await store.ask( - storeIds, - pattern, - parseInt(options.maxCount, 10), - searchOptions, - filters, - ); - response = formatAskResponse(results, options.content); - } + const response = await runSearch( + store, + { + store: options.store, + query: pattern, + path: search_path, + maxCount: parseInt(options.maxCount, 10), + content: options.content, + answer: options.answer, + web: options.web, + rerank: options.rerank, + agentic: options.agentic, + }, + root, + ); console.log(response); } catch (error) { diff --git a/src/commands/watch.ts b/src/commands/watch.ts index 50e9d3d..cd39db0 100644 --- a/src/commands/watch.ts +++ b/src/commands/watch.ts @@ -4,6 +4,7 @@ import { Command, InvalidArgumentError } from "commander"; import { type CliConfigOptions, loadConfig } from "../lib/config.js"; import { createFileSystem, createStore } from "../lib/context.js"; import { DEFAULT_IGNORE_PATTERNS } from "../lib/file.js"; +import type { Store } from "../lib/store.js"; import { createIndexingSpinner, formatDryRunSummary, @@ -22,13 +23,20 @@ export interface WatchOptions { dryRun: boolean; maxFileSize?: number; maxFileCount?: number; + createStore?: () => Promise; } -export async function startWatch(options: WatchOptions): Promise { +/** + * Starts in-process synchronization and file watchers for the current directory. + * @param options Store, sync limits, and optional non-interactive store factory. + * @returns Whether initial synchronization completed and file watchers were set up, + * regardless of per-file upload errors. + */ +export async function startWatch(options: WatchOptions): Promise { let refreshInterval: NodeJS.Timeout | undefined; try { - const store = await createStore(); + const store = await (options.createStore ?? createStore)(); // Refresh JWT token every 5 minutes (before 15-minute expiration) if (!options.dryRun) { @@ -60,7 +68,7 @@ export async function startWatch(options: WatchOptions): Promise { "Please run this command from within a specific project subdirectory.", ); process.exitCode = 1; - return; + return false; } const cliOptions: CliConfigOptions = { @@ -112,7 +120,7 @@ export async function startWatch(options: WatchOptions): Promise { includeTotal: true, }), ); - return; + return false; } } catch (e) { if (e instanceof QuotaExceededError) { @@ -123,7 +131,9 @@ export async function startWatch(options: WatchOptions): Promise { console.error( " Upgrade your plan at https://platform.mixedbread.com to continue syncing.\n", ); - process.exit(1); + clearInterval(refreshInterval); + process.exitCode = 1; + return false; } if (e instanceof MaxFileCountExceededError) { spinner.fail("File count exceeded"); @@ -131,7 +141,9 @@ export async function startWatch(options: WatchOptions): Promise { console.error( " Increase the limit with --max-file-count or MGREP_MAX_FILE_COUNT environment variable.\n", ); - process.exit(1); + clearInterval(refreshInterval); + process.exitCode = 1; + return false; } spinner.fail("Initial upload failed"); throw e; @@ -146,6 +158,7 @@ export async function startWatch(options: WatchOptions): Promise { // the callback filter runs, which exhausts the kernel watcher limit on // large repos. const watchers = new Map(); + let watchSetupSucceeded = true; function isMissingPathError(error: unknown): boolean { return ( @@ -163,6 +176,7 @@ export async function startWatch(options: WatchOptions): Promise { return null; } + watchSetupSucceeded = false; console.warn(`Warning: failed to inspect path ${filePath}:`, error); return undefined; } @@ -270,8 +284,14 @@ export async function startWatch(options: WatchOptions): Promise { handleFileEvent(eventType, dirPath, name); }); + watcher.on("error", (error) => { + watchSetupSucceeded = false; + console.error(`Failed to watch ${dirPath}:`, error); + closeWatcher(dirPath); + }); watchers.set(dirPath, watcher); } catch (error) { + watchSetupSucceeded = false; console.warn(`Warning: failed to watch ${dirPath}:`, error); return; } @@ -285,11 +305,15 @@ export async function startWatch(options: WatchOptions): Promise { watchDirectory(path.join(dirPath, entry.name)); } } catch (error) { + watchSetupSucceeded = false; console.warn(`Warning: failed to read directory ${dirPath}:`, error); } } watchDirectory(watchRoot); + // fs.watch can report startup failures asynchronously. + await new Promise((resolve) => setImmediate(resolve)); + return watchSetupSucceeded && watchers.has(watchRoot); } catch (error) { if (refreshInterval) { clearInterval(refreshInterval); @@ -297,6 +321,7 @@ export async function startWatch(options: WatchOptions): Promise { const message = error instanceof Error ? error.message : "Unknown error"; console.error("Failed to start watcher:", message); process.exitCode = 1; + return false; } } diff --git a/src/commands/watch_mcp.ts b/src/commands/watch_mcp.ts index fe992e6..7758adc 100644 --- a/src/commands/watch_mcp.ts +++ b/src/commands/watch_mcp.ts @@ -1,94 +1,276 @@ +import { realpathSync } from "node:fs"; +import { resolve, sep } from "node:path"; +import { fileURLToPath } from "node:url"; import { Server } from "@modelcontextprotocol/sdk/server/index.js"; import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"; import { CallToolRequestSchema, ListToolsRequestSchema, + ResultSchema, } from "@modelcontextprotocol/sdk/types.js"; import { Command } from "commander"; +import { z } from "zod"; +import { loadConfig } from "../lib/config.js"; +import { createStore } from "../lib/context.js"; +import { runSearch } from "../lib/search.js"; +import { isAtOrAboveHomeDirectory } from "../lib/utils.js"; +import { getPackageVersion } from "../lib/version.js"; +import { syncFiles } from "./search.js"; import { startWatch } from "./watch.js"; +// The SDK's default roots schema rejects non-file URIs before we can skip them. +const rootsResult = z.object({ + roots: z.array(z.object({ uri: z.string() })), +}); + +const searchArguments = z.strictObject({ + query: z.string().min(1), + path: z.string().optional(), + maxCount: z.number().int().positive().default(10), + content: z.boolean().default(false), + answer: z.boolean().default(false), + web: z.boolean().default(false), +}); + export const watchMcp = new Command("mcp") .description("Start MCP server for mgrep") + .option("--expose-tools", "Expose search tools via MCP", false) + .option("--no-watch", "Skip background file sync") .action(async (_options, cmd) => { - process.on("SIGINT", () => { - console.error("Received SIGINT, shutting down gracefully..."); - process.exit(0); - }); - - process.on("SIGTERM", () => { - console.error("Received SIGTERM, shutting down gracefully..."); - process.exit(0); - }); - - // Prevent unhandled promise rejections from crashing the MCP server - process.on("unhandledRejection", (reason, promise) => { - console.error( - "[ERROR] Unhandled Rejection at:", - promise, - "reason:", - reason, - ); - }); - - // The MCP server is writing to stdout, so all logs are written to stderr + // The MCP server is writing to stdout, so all logs are written to stderr. console.log = (...args: unknown[]) => { process.stderr.write(`[LOG] ${args.join(" ")}\n`); }; - console.error = (...args: unknown[]) => { process.stderr.write(`[ERROR] ${args.join(" ")}\n`); }; - console.debug = (...args: unknown[]) => { process.stderr.write(`[DEBUG] ${args.join(" ")}\n`); }; + console.info = console.log; - const options: { - store: string; - } = cmd.optsWithGlobals(); + process.on("unhandledRejection", (reason) => { + console.error("Unhandled rejection:", reason); + }); + const options: { store: string; exposeTools: boolean; watch: boolean } = + cmd.optsWithGlobals(); + const configuredSyncWaitMs = Number( + process.env.MGREP_MCP_SYNC_WAIT_MS?.trim() || "20000", + ); + const syncWaitMs = + Number.isInteger(configuredSyncWaitMs) && + configuredSyncWaitMs >= 0 && + configuredSyncWaitMs <= 2147483647 + ? configuredSyncWaitMs + : 20000; + const root = process.cwd(); const transport = new StdioServerTransport(); const server = new Server( + { name: "mgrep", version: getPackageVersion() }, { - name: "mgrep", - version: "0.1.3", - }, - { - capabilities: { - tools: {}, - }, + capabilities: { tools: {} }, + instructions: `mgrep semantic search is available through this server's \`search\` tool. Use it for conceptual or cross-file questions where you don't know the exact identifier, such as 'where is authentication handled'. It searches ${root} by default. Use rg or grep for exact strings, symbol names and regexes. Both are fine; pick whichever fits the question.`, }, ); - server.setRequestHandler(ListToolsRequestSchema, async () => { - return { - tools: [], - }; + let syncTimer: NodeJS.Timeout | undefined; + let closing = false; + const firstSyncs = new Map>(); + const serverRoot = realpathSync(root); + const serverRootPrefix = serverRoot.endsWith(sep) + ? serverRoot + : `${serverRoot}${sep}`; + let completeWatch: (watching: boolean) => void = () => {}; + // Requests can arrive before the delayed watcher startup completes. + const watcherReady = new Promise((resolve) => { + completeWatch = resolve; }); + if (!options.watch) completeWatch(false); - server.setRequestHandler(CallToolRequestSchema, async (_request) => { - return { - result: "Not implemented", - }; - }); + const resolveRoot = async (searchPath?: string): Promise => { + if (searchPath !== undefined) + return realpathSync(resolve(root, searchPath)); + if (server.getClientCapabilities()?.roots) { + const { roots } = rootsResult.parse( + await server.request( + { method: "roots/list" }, + ResultSchema.passthrough(), + ), + ); + const fileRoot = roots.find((entry) => entry.uri.startsWith("file://")); + if (fileRoot) return realpathSync(fileURLToPath(fileRoot.uri)); + } + if (options.watch) return root; + throw new Error( + "An absolute project path is required; pass the absolute project directory as path.", + ); + }; - await server.connect(transport); + const shutdown = async (): Promise => { + if (closing) return; + closing = true; + clearTimeout(syncTimer); + completeWatch(false); + try { + await server.close(); + } finally { + // In-process fs.watch handles must not outlive the MCP client. + process.exit(0); + } + }; - const startBackgroundSync = async () => { - console.log("[SYNC] Scheduling initial sync in 5 seconds..."); + server.onclose = () => void shutdown(); + process.stdin.once("end", () => void shutdown()); + process.stdin.once("close", () => void shutdown()); + for (const signal of ["SIGINT", "SIGTERM"] as const) { + process.once(signal, () => { + console.error(`Received ${signal}, shutting down gracefully...`); + void shutdown(); + }); + } - setTimeout(async () => { - console.log("[SYNC] Starting file sync..."); - try { - await startWatch({ store: options.store, dryRun: false }); - } catch (error) { - const errorMessage = - error instanceof Error ? error.message : String(error); - console.error("[SYNC] Sync failed:", errorMessage); - } - }, 1000); - }; + server.setRequestHandler(ListToolsRequestSchema, async () => ({ + tools: options.exposeTools + ? [ + { + name: "search", + description: + "Semantic search over the project's files, indexed by mgrep. Use it for conceptual or cross-file questions where you don't know the exact identifier, such as 'where is authentication handled' or 'how are failed uploads retried'. For exact strings, symbol names or regexes, use rg or grep instead. path is optional and defaults to the session's project directory. Results are file paths with line ranges; read only those ranges. Set web and answer to true for a web lookup.", + inputSchema: { + type: "object", + properties: { + query: { + type: "string", + minLength: 1, + description: "A full natural-language question.", + }, + path: { + type: "string", + description: + "A path relative to the server cwd or an absolute path.", + }, + maxCount: { + type: "integer", + minimum: 1, + multipleOf: 1, + default: 10, + }, + content: { type: "boolean", default: false }, + answer: { type: "boolean", default: false }, + web: { type: "boolean", default: false }, + }, + required: options.watch ? ["query"] : ["query", "path"], + additionalProperties: false, + }, + annotations: { + title: "mgrep semantic search", + readOnlyHint: true, + destructiveHint: false, + idempotentHint: true, + openWorldHint: false, + }, + }, + ] + : [], + })); - startBackgroundSync().catch((error) => { - console.error("[SYNC] Background sync setup failed:", error); + server.setRequestHandler(CallToolRequestSchema, async (request) => { + try { + if (!options.exposeTools || request.params.name !== "search") { + throw new Error(`Unknown tool: ${request.params.name}`); + } + const args = searchArguments.parse(request.params.arguments ?? {}); + const searchRoot = await resolveRoot(args.path); + let watcherStatus: boolean | undefined = false; + if ( + options.watch && + (searchRoot === serverRoot || searchRoot.startsWith(serverRootPrefix)) + ) { + let waitTimer: NodeJS.Timeout | undefined; + try { + watcherStatus = await Promise.race([ + watcherReady, + new Promise((resolve) => { + waitTimer = setTimeout(() => resolve(undefined), syncWaitMs); + }), + ]); + } finally { + clearTimeout(waitTimer); + } + } + const store = await createStore({ interactive: false }); + if (watcherStatus === false && !isAtOrAboveHomeDirectory(searchRoot)) { + let firstSync = firstSyncs.get(searchRoot); + if (!firstSync) { + firstSync = syncFiles( + store, + options.store, + searchRoot, + false, + loadConfig(searchRoot), + ).then(() => undefined); + firstSyncs.set(searchRoot, firstSync); + } + try { + await firstSync; + } catch (error) { + firstSyncs.delete(searchRoot); + throw error; + } + } + let text = await runSearch( + store, + { + ...args, + store: options.store, + path: searchRoot, + rerank: true, + agentic: false, + }, + searchRoot, + ); + if (watcherStatus === undefined) { + text = [ + text, + "Note: the initial mgrep sync is still running; results may be incomplete.", + ] + .filter(Boolean) + .join("\n"); + } + return { content: [{ type: "text", text }] }; + } catch (error) { + const text = error instanceof Error ? error.message : String(error); + return { content: [{ type: "text", text }], isError: true }; + } }); + + await server.connect(transport); + if (closing || !options.watch) return; + + console.log("[SYNC] Scheduling initial sync in 1 second..."); + syncTimer = setTimeout(async () => { + if (isAtOrAboveHomeDirectory(root)) { + console.error( + "[SYNC] Background sync skipped: cwd is at or above the home directory.", + ); + completeWatch(false); + return; + } + console.log("[SYNC] Starting file sync..."); + try { + completeWatch( + await startWatch({ + store: options.store, + dryRun: false, + createStore: () => createStore({ interactive: false }), + }), + ); + } catch (error) { + console.error("[SYNC] Failed to start watcher:", error); + completeWatch(false); + } finally { + // Watch's CLI error status must not affect a healthy MCP session. + process.exitCode = 0; + } + }, 1000); }); diff --git a/src/index.ts b/src/index.ts index af89474..a2ce394 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,7 +1,4 @@ #!/usr/bin/env node -import * as fs from "node:fs"; -import * as path from "node:path"; -import { fileURLToPath } from "node:url"; import { program } from "commander"; import { login } from "./commands/login.js"; import { logout } from "./commands/logout.js"; @@ -19,16 +16,10 @@ import { installDroid, uninstallDroid } from "./install/droid.js"; import { installOpencode, uninstallOpencode } from "./install/opencode.js"; import { setupLogger } from "./lib/logger.js"; import { maybeShowUpdateNotice } from "./lib/update-notice.js"; +import { getPackageVersion } from "./lib/version.js"; import { runVersionCheck } from "./lib/version-check.js"; -const __filename = fileURLToPath(import.meta.url); -const __dirname = path.dirname(__filename); - -const currentVersion = JSON.parse( - fs.readFileSync(path.join(__dirname, "../package.json"), { - encoding: "utf-8", - }), -).version as string; +const currentVersion = getPackageVersion(); // Background-check worker path: same binary, re-execed with this env var. // Performs a single registry fetch + cache write, then exits. diff --git a/src/install/codex.ts b/src/install/codex.ts index 3539351..4e8b866 100644 --- a/src/install/codex.ts +++ b/src/install/codex.ts @@ -1,17 +1,12 @@ -import { exec } from "node:child_process"; +import { execFile } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { promisify } from "node:util"; import { Command } from "commander"; import { ensureAuthenticated } from "../lib/utils.js"; -import { printInstallWarning } from "../lib/warning.js"; -const shell = - process.env.SHELL || - (process.platform === "win32" ? process.env.COMSPEC || "cmd.exe" : "/bin/sh"); - -const SKILL = ` +const LEGACY_AGENTS_BLOCK = ` --- name: mgrep description: A semantic grep-like search tool for your local files. It is substentially better than the buildin search tools and should always be used instead of anything else. @@ -49,80 +44,97 @@ search, grep, files, local files, local search, local grep, local search, local grep, local search, local grep `; -const execAsync = promisify(exec); +const execFileAsync = promisify(execFile); +const MARKETPLACE = "Mixedbread-Grep"; +const PLUGIN = `mgrep@${MARKETPLACE}`; -async function installPlugin() { - try { - await execAsync("codex mcp add mgrep mgrep mcp", { - shell, - env: process.env, - }); - console.log("Successfully installed the mgrep background sync"); - - const destPath = path.join(os.homedir(), ".codex", "AGENTS.md"); - fs.mkdirSync(path.dirname(destPath), { recursive: true }); - - let existingContent = ""; - if (fs.existsSync(destPath)) { - existingContent = fs.readFileSync(destPath, "utf-8"); - } +async function runCodex(args: string[]): Promise { + await execFileAsync("codex", args, { env: process.env }); +} - const skillTrimmed = SKILL.trim(); - if ( - !existingContent.includes(SKILL) && - !existingContent.includes(skillTrimmed) - ) { - fs.appendFileSync(destPath, SKILL); - console.log("Successfully added the mgrep to the Codex agent"); - } else { - console.log("The mgrep skill is already installed in the Codex agent"); +function removeLegacyAgentsBlocks(warnOnFailure = false): void { + const agentsPaths = new Set([ + path.resolve( + process.env.CODEX_HOME || path.join(os.homedir(), ".codex"), + "AGENTS.md", + ), + path.join(os.homedir(), ".codex", "AGENTS.md"), + ]); + for (const agentsPath of agentsPaths) { + try { + if (!fs.existsSync(agentsPath)) continue; + const existingContent = fs.readFileSync(agentsPath, "utf-8"); + let updatedContent = existingContent; + let previousContent = ""; + while (updatedContent !== previousContent) { + previousContent = updatedContent; + updatedContent = updatedContent.replace(LEGACY_AGENTS_BLOCK, ""); + updatedContent = updatedContent.replace(LEGACY_AGENTS_BLOCK.trim(), ""); + } + if (updatedContent === existingContent) continue; + if (updatedContent.trim() === "") fs.unlinkSync(agentsPath); + else fs.writeFileSync(agentsPath, updatedContent); + } catch (error) { + if (!warnOnFailure) throw error; + console.warn( + `Warning: Could not clean legacy instructions in ${agentsPath}: ${error}`, + ); } - - printInstallWarning("Codex", "mgrep uninstall-codex"); - } catch (error) { - console.error(`Error installing plugin: ${error}`); - process.exit(1); } } -async function uninstallPlugin() { +async function installPlugin(): Promise { try { - await execAsync("codex mcp remove mgrep", { shell, env: process.env }); + await runCodex(["mcp", "remove", "mgrep"]).catch(() => undefined); + removeLegacyAgentsBlocks(); + const source = + process.env.MGREP_CODEX_MARKETPLACE || + "https://github.com/mixedbread-ai/mgrep.git"; + try { + await runCodex(["plugin", "marketplace", "add", source]); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (!/already (?:added|exists|registered|configured)/i.test(message)) + throw error; + } + await runCodex(["plugin", "add", PLUGIN]); + console.log("Successfully installed the mgrep plugin for Codex"); + console.log( + "On first start, Codex asks you to review the plugin's SessionStart hook (or use /hooks); it only adds search guidance. The plugin's MCP server syncs the session's project directory in the background while Codex runs.", + ); } catch (error) { - console.error(`Error uninstalling plugin: ${error}`); - process.exit(1); + console.error(`Error installing plugin: ${error}`); + process.exitCode = 1; } +} - const destPath = path.join(os.homedir(), ".codex", "AGENTS.md"); - if (fs.existsSync(destPath)) { - const existingContent = fs.readFileSync(destPath, "utf-8"); - let updatedContent = existingContent; - let previousContent = ""; - - while (updatedContent !== previousContent) { - previousContent = updatedContent; - updatedContent = updatedContent.replace(SKILL, ""); - updatedContent = updatedContent.replace(SKILL.trim(), ""); - } - - if (updatedContent.trim() === "") { - fs.unlinkSync(destPath); - } else { - fs.writeFileSync(destPath, updatedContent); +async function uninstallPlugin(): Promise { + for (const args of [ + ["plugin", "remove", PLUGIN], + ["plugin", "marketplace", "remove", MARKETPLACE], + ["mcp", "remove", "mgrep"], + ]) { + try { + await runCodex(args); + } catch (error) { + console.warn(`Warning: Could not run codex ${args.join(" ")}: ${error}`); } } - console.log("Successfully removed the mgrep from the Codex agent"); + removeLegacyAgentsBlocks(true); + console.log("Finished removing the mgrep Codex integration"); } +/** Installs the native mgrep Codex plugin and removes legacy agent instructions. */ export const installCodex = new Command("install-codex") - .description("Install the Codex agent") - .action(async () => { + .description("Install the mgrep plugin for Codex") + .action(async (): Promise => { await ensureAuthenticated(); await installPlugin(); }); +/** Removes the native mgrep Codex plugin and legacy agent instructions. */ export const uninstallCodex = new Command("uninstall-codex") - .description("Uninstall the Codex agent") - .action(async () => { + .description("Uninstall the mgrep plugin for Codex") + .action(async (): Promise => { await uninstallPlugin(); }); diff --git a/src/lib/context.ts b/src/lib/context.ts index 52db3ef..220a4ea 100644 --- a/src/lib/context.ts +++ b/src/lib/context.ts @@ -13,16 +13,29 @@ const BASE_URL = isDevelopment() ? "http://localhost:8000" : "https://api.mixedbread.com"; +/** Options controlling store authentication. */ +export interface CreateStoreOptions { + interactive?: boolean; +} + /** * Creates an authenticated Store instance * Supports authentication via MXBAI_API_KEY env var or OAuth token + * @param options Authentication options; interactive defaults to true. + * Set interactive to false to skip the login prompt and let getJWTToken throw + * when credentials are missing. Test mode always returns a TestStore first. + * @returns The authenticated store, or a TestStore in test mode. */ -export async function createStore(): Promise { +export async function createStore( + options: CreateStoreOptions = {}, +): Promise { if (isTest) { return new TestStore(); } - await ensureAuthenticated(); + if (options.interactive ?? true) { + await ensureAuthenticated(); + } async function createClient() { const jwtToken = await getJWTToken(); diff --git a/src/lib/search.ts b/src/lib/search.ts new file mode 100644 index 0000000..ad0de55 --- /dev/null +++ b/src/lib/search.ts @@ -0,0 +1,189 @@ +import { isAbsolute, join, normalize, relative } from "node:path"; +import type { + AskResponse, + ChunkType, + FileMetadata, + SearchResponse, + Store, +} from "./store.js"; + +export interface RunSearchOptions { + store: string; + query: string; + path: string; + maxCount: number; + content: boolean; + answer: boolean; + web: boolean; + rerank: boolean; + agentic: boolean; +} + +/** + * Resolves the search filter path without changing absolute paths. + * @param root The absolute search root. + * @param execPath An optional relative or absolute search path. + * @returns The absolute path used by the starts_with filter. + */ +export function resolveSearchPath(root: string, execPath?: string): string { + return execPath !== undefined && isAbsolute(execPath) + ? execPath + : normalize(join(root, execPath ?? "")); +} + +/** + * Searches indexed files or generates an answer without syncing files. + * @param store The authenticated store implementation. + * @param options Search, filter, and output options. + * @param root The absolute root used to make result paths relative. + * @returns The formatted text shared by the CLI and MCP tool. + */ +export async function runSearch( + store: Store, + options: RunSearchOptions, + root: string, +): Promise { + const storeIds = options.web + ? [options.store, "mixedbread/web"] + : [options.store]; + const filters = { + all: [ + { key: "path", operator: "starts_with" as const, value: options.path }, + ], + }; + const searchOptions = { + rerank: options.rerank, + ...(options.agentic && { agentic: true }), + }; + + if (options.answer) { + const results = await store.ask( + storeIds, + options.query, + options.maxCount, + searchOptions, + filters, + ); + return formatAskResponse(results, options.content, root); + } + + const results = await store.search( + storeIds, + options.query, + options.maxCount, + searchOptions, + filters, + ); + return formatSearchResponse(results, options.content, root); +} + +function extractSources(response: AskResponse): { [key: number]: ChunkType } { + const sources: { [key: number]: ChunkType } = {}; + const answer = response.answer; + + // Match ALL cite tags and capture the i="..." + const citeTags = answer.match(/= 0 && + end >= start && + end < response.sources.length + ) { + for (let i = start; i <= end; i++) { + sources[i] = response.sources[i]; + } + } + } + + return sources; +} + +function formatAskResponse( + response: AskResponse, + show_content: boolean, + root: string, +): string { + const sources = extractSources(response); + const sourceEntries = Object.entries(sources).map( + ([index, chunk]) => `${index}: ${formatChunk(chunk, show_content, root)}`, + ); + return `${response.answer}\n\n${sourceEntries.join("\n")}`; +} + +function formatSearchResponse( + response: SearchResponse, + show_content: boolean, + root: string, +): string { + return response.data + .map((chunk) => formatChunk(chunk, show_content, root)) + .join("\n"); +} + +function isWebResult(chunk: ChunkType): boolean { + return ( + "filename" in chunk && + typeof chunk.filename === "string" && + chunk.filename.startsWith("http") + ); +} + +function formatChunk( + chunk: ChunkType, + show_content: boolean, + root: string, +): string { + if (isWebResult(chunk) && chunk.type === "text") { + const url = "filename" in chunk ? chunk.filename : "Unknown URL"; + const content = show_content ? chunk.text : ""; + return `${url} (${(chunk.score * 100).toFixed(2)}% match)${content ? `\n${content}` : ""}`; + } + + const filePath = (chunk.metadata as FileMetadata)?.path; + const path = filePath ? `./${relative(root, filePath)}` : ".Unknown path"; + let line_range = ""; + let content = ""; + switch (chunk.type) { + case "text": { + const start_line = (chunk.generated_metadata?.start_line as number) + 1; + const end_line = + start_line + (chunk.generated_metadata?.num_lines as number); + line_range = `:${start_line}-${end_line}`; + content = show_content ? chunk.text : ""; + break; + } + case "image_url": + line_range = + chunk.generated_metadata?.type === "pdf" + ? `, page ${chunk.chunk_index + 1}` + : ""; + break; + case "audio_url": + line_range = ""; + break; + case "video_url": + line_range = ""; + break; + } + + return `${path}${line_range} (${(chunk.score * 100).toFixed(2)}% match)${content ? `\n${content}` : ""}`; +} diff --git a/src/lib/update-notice.ts b/src/lib/update-notice.ts index 589b52e..04e9476 100644 --- a/src/lib/update-notice.ts +++ b/src/lib/update-notice.ts @@ -17,7 +17,7 @@ import { const SUPPRESSED_COMMANDS = new Set([ "watch", - "watch-mcp", + "mcp", "update", "login", "logout", diff --git a/src/lib/version.ts b/src/lib/version.ts new file mode 100644 index 0000000..fe6f6e2 --- /dev/null +++ b/src/lib/version.ts @@ -0,0 +1,11 @@ +import { readFileSync } from "node:fs"; + +/** + * Reads the version from the installed package relative to this module. + * @returns The package version used by the CLI and MCP server. + */ +export function getPackageVersion(): string { + return JSON.parse( + readFileSync(new URL("../../package.json", import.meta.url), "utf-8"), + ).version as string; +} diff --git a/test/codex.bats b/test/codex.bats new file mode 100644 index 0000000..3c994af --- /dev/null +++ b/test/codex.bats @@ -0,0 +1,347 @@ +#!/bin/bash + +setup_file() { + pnpm build +} + +setup() { + bats_require_minimum_version 1.5.0 + load '../node_modules/bats-support/load' + load '../node_modules/bats-assert/load' + DIR="$( cd "$( dirname "$BATS_TEST_FILENAME" )" >/dev/null 2>&1 && pwd )" + export HOME="$BATS_TEST_TMPDIR/home" + export CODEX_HOME="$BATS_TEST_TMPDIR/codex-home" + export MXBAI_API_KEY="codex-test-dummy-key" + export MGREP_IS_TEST=1 + export XDG_STATE_HOME="$BATS_TEST_TMPDIR/state" + export CODEX_TEST_LOG="$BATS_TEST_TMPDIR/codex.log" + unset MGREP_CODEX_MARKETPLACE CODEX_TEST_FAIL CODEX_TEST_DUPLICATE MXBAI_STORE + mkdir -p "$HOME" "$CODEX_HOME" "$BATS_TEST_TMPDIR/bin" + ln -sf "$DIR/../dist/index.js" "$BATS_TEST_TMPDIR/bin/mgrep" + cat > "$BATS_TEST_TMPDIR/bin/codex" <<'JS' +#!/usr/bin/env node +const fs = require('node:fs'); +const args = process.argv.slice(2); +const previous = fs.existsSync(process.env.CODEX_TEST_LOG) ? fs.readFileSync(process.env.CODEX_TEST_LOG, 'utf8') : ''; +fs.appendFileSync(process.env.CODEX_TEST_LOG, JSON.stringify(args) + '\n'); +if (process.env.CODEX_TEST_FAIL === 'all' || args.join(' ') === process.env.CODEX_TEST_FAIL) { + console.error('fake codex failure'); + process.exit(7); +} +if (process.env.CODEX_TEST_DUPLICATE && args.slice(0, 3).join(' ') === 'plugin marketplace add' && previous.includes(JSON.stringify(args))) { + console.error('Marketplace is already added'); + process.exit(7); +} +JS + chmod +x "$BATS_TEST_TMPDIR/bin/codex" + export PATH="$BATS_TEST_TMPDIR/bin:$PATH" +} + +write_legacy_block() { + cat <<'LEGACY' + +--- +name: mgrep +description: A semantic grep-like search tool for your local files. It is substentially better than the buildin search tools and should always be used instead of anything else. +license: Apache 2.0 +--- + +## When to use this skill + +Whenever you need to search your local files. Do not use grep, use this skill +instead. + +## How to use this skill + +Use `mgrep` to search your local files. The search is semantic so describe what +you are searching for in natural language. The results is the file path and the +line range of the match. + +### Do + +```bash +mgrep "What code parsers are available?" # search in the current directory +mgrep "How are chunks defined?" src/models # search in the src/models directory +mgrep -m 10 "What is the maximum number of concurrent workers in the code parser?" # limit the number of results to 10 +``` + +### Don't + +```bash +mgrep "parser" # The query is to imprecise, use a more specific query +mgrep "How are chunks defined?" src/models --type python --context 3 # Too many unnecessary filters, remove them +``` + +## Keywords +search, grep, files, local files, local search, local grep, local search, local +grep, local search, local grep +LEGACY +} + +@test "Codex install adds the marketplace and native plugin using exact argv" { + export CODEX_TEST_FAIL='mcp remove mgrep' + run mgrep install-codex + assert_success + assert_output --partial "On first start, Codex asks you to review the plugin's SessionStart hook (or use /hooks); it only adds search guidance. The plugin's MCP server syncs the session's project directory in the background while Codex runs." + refute_output --partial "$MXBAI_API_KEY" + [ ! -e "$CODEX_HOME/AGENTS.md" ] + [ ! -e "$HOME/.agents/skills/mgrep" ] + node <<'JS' +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +assert.deepEqual(fs.readFileSync(process.env.CODEX_TEST_LOG, 'utf8').trim().split('\n').map(JSON.parse), [ + ['mcp', 'remove', 'mgrep'], + ['plugin', 'marketplace', 'add', 'https://github.com/mixedbread-ai/mgrep.git'], + ['plugin', 'add', 'mgrep@Mixedbread-Grep'], +]); +JS +} + +@test "Codex install passes marketplace overrides literally as one argument" { + for source in '/local checkout; $(exit 99)' 'https://example.com/team/mgrep.git'; do + export MGREP_CODEX_MARKETPLACE="$source" + run mgrep install-codex + assert_success + node <<'JS' +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const calls = fs.readFileSync(process.env.CODEX_TEST_LOG, 'utf8').trim().split('\n').map(JSON.parse); +assert.deepEqual(calls.at(-2), ['plugin', 'marketplace', 'add', process.env.MGREP_CODEX_MARKETPLACE]); +JS + done +} + +@test "Codex install removes raw and trimmed legacy blocks in both homes preserving surrounding content" { + write_legacy_block > "$BATS_TEST_TMPDIR/legacy.md" + mkdir -p "$HOME/.codex" + node <<'JS' +const fs = require('node:fs'); +const path = require('node:path'); +const legacy = fs.readFileSync(path.join(process.env.BATS_TEST_TMPDIR, 'legacy.md'), 'utf8'); +for (const root of [process.env.CODEX_HOME, path.join(process.env.HOME, '.codex')]) { + fs.writeFileSync(path.join(root, 'AGENTS.md'), 'Before\n' + legacy + legacy.trim() + '\nBetween\n' + legacy + legacy.trim() + '\nAfter\n'); +} +JS + run mgrep install-codex + assert_success + printf 'Before\n\nBetween\n\nAfter\n' > "$BATS_TEST_TMPDIR/expected.md" + cmp "$CODEX_HOME/AGENTS.md" "$BATS_TEST_TMPDIR/expected.md" + cmp "$HOME/.codex/AGENTS.md" "$BATS_TEST_TMPDIR/expected.md" +} + +@test "Codex install deletes only legacy-only AGENTS and preserves untouched whitespace" { + write_legacy_block > "$CODEX_HOME/AGENTS.md" + mkdir -p "$HOME/.codex" + printf ' \n\t\n' > "$HOME/.codex/AGENTS.md" + cp "$HOME/.codex/AGENTS.md" "$BATS_TEST_TMPDIR/expected.md" + run mgrep install-codex + assert_success + [ ! -e "$CODEX_HOME/AGENTS.md" ] + cmp "$HOME/.codex/AGENTS.md" "$BATS_TEST_TMPDIR/expected.md" +} + +@test "Codex install is idempotent when an already-added marketplace reports failure" { + export CODEX_TEST_DUPLICATE=1 + write_legacy_block > "$CODEX_HOME/AGENTS.md" + run mgrep install-codex + assert_success + run mgrep install-codex + assert_success + [ ! -e "$CODEX_HOME/AGENTS.md" ] + node <<'JS' +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const calls = fs.readFileSync(process.env.CODEX_TEST_LOG, 'utf8').trim().split('\n').map(JSON.parse); +assert.equal(calls.length, 6); +assert.deepEqual(calls.slice(0, 3), calls.slice(3)); +JS +} + +@test "Codex install reports genuine marketplace and plugin failures" { + for step in 'plugin marketplace add https://github.com/mixedbread-ai/mgrep.git' 'plugin add mgrep@Mixedbread-Grep'; do + export CODEX_TEST_FAIL="$step" + run mgrep install-codex + assert_failure 1 + assert_output --partial 'fake codex failure' + done +} + +@test "Codex uninstall invokes every step and cleans both homes after failures" { + export CODEX_TEST_FAIL=all + mkdir -p "$HOME/.codex" + write_legacy_block > "$CODEX_HOME/AGENTS.md" + { printf 'Keep this\n'; write_legacy_block; } > "$HOME/.codex/AGENTS.md" + run mgrep uninstall-codex + assert_success + assert_output --partial 'Warning: Could not run codex plugin remove' + assert_output --partial 'Warning: Could not run codex plugin marketplace remove' + assert_output --partial 'Warning: Could not run codex mcp remove' + [ ! -e "$CODEX_HOME/AGENTS.md" ] + printf 'Keep this\n' > "$BATS_TEST_TMPDIR/expected.md" + cmp "$HOME/.codex/AGENTS.md" "$BATS_TEST_TMPDIR/expected.md" + node <<'JS' +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +assert.deepEqual(fs.readFileSync(process.env.CODEX_TEST_LOG, 'utf8').trim().split('\n').map(JSON.parse), [ + ['plugin', 'remove', 'mgrep@Mixedbread-Grep'], + ['plugin', 'marketplace', 'remove', 'Mixedbread-Grep'], + ['mcp', 'remove', 'mgrep'], +]); +JS +} + +@test "Codex uninstall warns for one failed cleanup file and continues with the other" { + mkdir -p "$CODEX_HOME/AGENTS.md" "$HOME/.codex" + write_legacy_block > "$HOME/.codex/AGENTS.md" + run mgrep uninstall-codex + assert_success + assert_output --partial "Warning: Could not clean legacy instructions in $CODEX_HOME/AGENTS.md" + [ ! -e "$HOME/.codex/AGENTS.md" ] +} + +@test "Codex plugin manifest version and metadata match the package and Claude manifest" { + node - "$DIR/.." <<'JS' +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const read = (name) => JSON.parse(fs.readFileSync(path.join(process.argv[2], name), 'utf8')); +const plugin = read('plugins/mgrep/.codex-plugin/plugin.json'); +const claude = read('plugins/mgrep/.claude-plugin/plugin.json'); +assert.equal(plugin.version, read('package.json').version); +assert.equal(plugin.description, claude.description); +assert.deepEqual(plugin.author, claude.author); +assert.equal(plugin.skills, './codex/skills/'); +assert.equal(plugin.mcpServers, './codex/mcp.json'); +assert.equal(plugin.hooks, './codex/hooks.json'); +assert.deepEqual(read('plugins/mgrep/codex/hooks.json'), { + hooks: { + SessionStart: [{ + matcher: 'startup|resume', + hooks: [{ + type: 'command', + command: 'python3 "${CLAUDE_PLUGIN_ROOT:-$PLUGIN_ROOT}/codex/session_start.py"', + timeout: 5, + }], + }], + }, +}); +assert.equal(fs.existsSync(path.join(process.argv[2], 'plugins/mgrep/hooks/hooks.json')), false); +assert.deepEqual(read('plugins/mgrep/codex/mcp.json').mcpServers.mgrep, { + command: 'mgrep', args: ['mcp', '--expose-tools'], + env_vars: ['MXBAI_API_KEY', 'MXBAI_STORE'], +}); + +JS +} + +assert_session_start_context() { + python3 - "$output" "$1" "${2:-}" <<'PY' +import json +import sys + +raw, cwd, suffix = sys.argv[1:] +assert len(raw.splitlines()) == 1 +assert json.loads(raw) == { + 'hookSpecificOutput': { + 'hookEventName': 'SessionStart', + 'additionalContext': ( + "mgrep semantic search is available through the `search` tool of the mgrep MCP server. " + "Use it for conceptual or cross-file questions where you don't know the exact identifier, " + "such as 'where is authentication handled'. " + f"It searches {cwd} by default. " + "Use rg or grep for exact strings, symbol names and regexes. " + "Both are fine; pick whichever fits the question." + ) + suffix, + }, +} +PY + [ -z "$stderr" ] +} + +@test "Codex SessionStart hook prints one JSON object with the payload cwd" { + local project="$BATS_TEST_TMPDIR/project with spaces" + mkdir -p "$project" + run --separate-stderr python3 "$DIR/../plugins/mgrep/codex/session_start.py" <<< "{\"cwd\":\"$project\"}" + assert_success + assert_session_start_context "$project" +} + +@test "Codex SessionStart hook falls back to the process cwd on empty stdin" { + local project="$BATS_TEST_TMPDIR/project" + mkdir -p "$project" + cd "$project" + run --separate-stderr python3 "$DIR/../plugins/mgrep/codex/session_start.py" < /dev/null + assert_success + assert_session_start_context "$(pwd -P)" +} + +@test "Codex SessionStart hook falls back on invalid payloads and unusable cwd values" { + local project="$BATS_TEST_TMPDIR/project" + mkdir -p "$project" + cd "$project" + for payload in 'not JSON' '{' 'null' '[]' '42' '"text"' '{}' '{"cwd":null}' '{"cwd":42}' '{"cwd":""}' '{"cwd":"\u0000"}'; do + run --separate-stderr python3 "$DIR/../plugins/mgrep/codex/session_start.py" <<< "$payload" + assert_success + assert_session_start_context "$(pwd -P)" + done +} + +@test "Codex SessionStart hook makes a relative payload cwd absolute" { + local project="$BATS_TEST_TMPDIR/project" + mkdir -p "$project" + cd "$project" + run --separate-stderr python3 "$DIR/../plugins/mgrep/codex/session_start.py" <<< '{"cwd":"nested/../source"}' + assert_success + assert_session_start_context "$(pwd -P)/source" +} + +@test "Codex SessionStart hook includes the cached update notice" { + mkdir -p "$HOME/.mgrep" + printf '%s\n' '{"latestVersion":"9.9.9"}' > "$HOME/.mgrep/update-check.json" + run --separate-stderr python3 "$DIR/../plugins/mgrep/codex/session_start.py" < /dev/null + assert_success + assert_session_start_context "$(pwd -P)" ' An mgrep update is available (latest: 9.9.9). Tell the user to run `mgrep update` to upgrade.' +} + +@test "Codex SessionStart hook ignores malformed update notices" { + mkdir -p "$HOME/.mgrep" + for cache in 'not JSON' 'null' '[]' '{}' '{"latestVersion":42}'; do + printf '%s\n' "$cache" > "$HOME/.mgrep/update-check.json" + run --separate-stderr python3 "$DIR/../plugins/mgrep/codex/session_start.py" < /dev/null + assert_success + assert_session_start_context "$(pwd -P)" + done +} + +@test "Codex SessionStart hook creates no files or changes to existing files" { + python3 - "$DIR/../plugins/mgrep/codex/session_start.py" <<'PY' +import json +import os +from pathlib import Path +import subprocess +import sys + +hook = Path(sys.argv[1]).resolve() +root = Path(os.environ['BATS_TEST_TMPDIR']) +project = root / 'project' +project.mkdir() +cache = Path.home() / '.mgrep' / 'update-check.json' +cache.parent.mkdir() +cache.write_text(json.dumps({'latestVersion': '9.9.9'})) + +def snapshot(): + return { + str(path): path.read_bytes() if path.is_file() else None + for tree in [root, hook.parent] + for path in tree.rglob('*') + } + +before = snapshot() +for payload in ['', 'not JSON', json.dumps({'cwd': str(project), 'session_id': 'steering-only'})]: + result = subprocess.run([sys.executable, str(hook)], input=payload, cwd=project, text=True, capture_output=True) + assert result.returncode == 0, result + assert result.stderr == '', result.stderr + assert json.loads(result.stdout)['hookSpecificOutput']['hookEventName'] == 'SessionStart' + assert snapshot() == before +PY +} diff --git a/test/helpers/mcp-client.mjs b/test/helpers/mcp-client.mjs new file mode 100644 index 0000000..32922ee --- /dev/null +++ b/test/helpers/mcp-client.mjs @@ -0,0 +1,109 @@ +import assert from "node:assert/strict"; +import { writeFileSync } from "node:fs"; +import { setTimeout as delay } from "node:timers/promises"; +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js"; +import { ListRootsRequestSchema } from "@modelcontextprotocol/sdk/types.js"; + +const args = process.argv.slice(2); +const separator = args.indexOf("--"); +if (separator < 0) { + throw new Error( + "Usage: mcp-client.mjs initialize|list|call |calls |close|eof -- ", + ); +} +const [action, name, json] = args.slice(0, separator); +const transport = new StdioClientTransport({ + command: "mgrep", + args: args.slice(separator + 1), + env: process.env, + stderr: "pipe", +}); +const roots = process.env.MGREP_MCP_ROOTS; +const client = new Client( + { name: "mgrep-test", version: "1.0.0" }, + { + capabilities: roots !== undefined ? { roots: {} } : {}, + }, +); +if (roots !== undefined) { + client.setRequestHandler(ListRootsRequestSchema, async () => { + if (process.env.MGREP_MCP_FORBID_ROOTS) + throw new Error("Unexpected roots/list"); + return { roots: JSON.parse(roots) }; + }); +} +let stderr = ""; +transport.stderr.on("data", (chunk) => { + stderr += chunk.toString(); +}); +const timeout = setTimeout(() => { + console.error(`MCP client timed out\n${stderr}`); + void client.close().finally(() => process.exit(1)); +}, 15000); + +try { + await client.connect(transport); + const waitFor = process.env.MGREP_MCP_WAIT_FOR; + while (waitFor && !stderr.includes(waitFor)) { + if (transport.pid === null) throw new Error("MCP server exited early"); + await delay(25); + } + + let result; + switch (action) { + case "initialize": + result = { instructions: client.getInstructions() }; + break; + case "list": + result = await client.listTools(); + break; + case "call-with-logs": + case "call": + result = await client.callTool({ name, arguments: JSON.parse(json) }); + await client.ping(); + break; + case "calls-with-logs": + case "calls": + result = []; + for (const step of JSON.parse(name)) { + if (step.write) writeFileSync(step.write.path, step.write.text); + else + result.push( + await client.callTool({ name: "search", arguments: step }), + ); + } + await client.ping(); + break; + case "idle": + await delay(1500); + await client.ping(); + result = { stderr }; + break; + case "close": + case "eof": { + // SDK 1.22 close() sends SIGTERM; use its child stdin to test EOF separately. + const child = transport._process; + const exited = new Promise((resolve) => { + child.once("exit", (code, signal) => resolve([code, signal])); + }); + if (action === "eof") child.stdin.end(); + else await client.close(); + const [code, signal] = await exited; + assert.equal(code, 0); + assert.equal(signal, null); + result = { exited: true, code, signal }; + break; + } + default: + throw new Error(`Unknown action: ${action}`); + } + if (action.endsWith("-with-logs")) result = { result, stderr }; + console.log(JSON.stringify(result)); +} catch (error) { + console.error(stderr); + throw error; +} finally { + clearTimeout(timeout); + await client.close(); +} diff --git a/test/mcp.bats b/test/mcp.bats new file mode 100644 index 0000000..e431543 --- /dev/null +++ b/test/mcp.bats @@ -0,0 +1,535 @@ +#!/bin/bash + +setup_file() { + pnpm build +} + +setup() { + load '../node_modules/bats-support/load' + load '../node_modules/bats-assert/load' + + DIR="$( cd "$( dirname "$BATS_TEST_FILENAME" )" >/dev/null 2>&1 && pwd )" + mkdir -p "$BATS_TEST_TMPDIR/bin" + ln -sf "$DIR/../dist/index.js" "$BATS_TEST_TMPDIR/bin/mgrep" + export PATH="$BATS_TEST_TMPDIR/bin:$PATH" + export MGREP_IS_TEST=1 + export MGREP_TEST_STORE_PATH="$BATS_TEST_TMPDIR/mgrep-test-store.json" + export XDG_STATE_HOME="$BATS_TEST_TMPDIR/state" + unset MGREP_MCP_WAIT_FOR MGREP_MCP_ROOTS MGREP_MCP_FORBID_ROOTS MGREP_MCP_SYNC_WAIT_MS + + mkdir -p "$BATS_TEST_TMPDIR/test-store" + printf 'Hello, world!\nThis is a test file.\n' > "$BATS_TEST_TMPDIR/test-store/test.txt" + printf 'Hello, world!\nA second one.\n' > "$BATS_TEST_TMPDIR/test-store/test-2.txt" + printf 'Hello, world!\nA third one.\n' > "$BATS_TEST_TMPDIR/test-store/test-3.txt" + cd "$BATS_TEST_TMPDIR/test-store" + mgrep search --sync test +} + +mcp_client() { + node "$DIR/helpers/mcp-client.mjs" "$@" +} + +@test "MCP lists no tools without opt-in" { + run mcp_client list -- mcp + assert_success + assert_equal "$output" '{"tools":[]}' +} + +@test "MCP exposes one read-only search tool with the specified schema" { + run mcp_client list -- mcp --expose-tools + assert_success + node - "$output" <<'JS' +const assert = require('node:assert/strict'); +const { tools } = JSON.parse(process.argv[2]); +assert.equal(tools.length, 1); +assert.equal(tools[0].name, 'search'); +assert.deepEqual(tools[0].annotations, { + title: 'mgrep semantic search', readOnlyHint: true, destructiveHint: false, + idempotentHint: true, openWorldHint: false, +}); +assert.deepEqual(tools[0].inputSchema.required, ['query']); +const properties = tools[0].inputSchema.properties; +assert.equal(Object.hasOwn(properties.path, "default"), false); +assert.equal(properties.maxCount.type, "integer"); +assert.equal(tools[0].description, "Semantic search over the project's files, indexed by mgrep. Use it for conceptual or cross-file questions where you don't know the exact identifier, such as 'where is authentication handled' or 'how are failed uploads retried'. For exact strings, symbol names or regexes, use rg or grep instead. path is optional and defaults to the session's project directory. Results are file paths with line ranges; read only those ranges. Set web and answer to true for a web lookup."); +assert.equal(properties.maxCount.default, 10); +for (const key of ['content', 'answer', 'web']) assert.equal(properties[key].default, false); +JS +} + +@test "MCP search returns fixture text and matches CLI output" { + local expected + expected="$(mgrep search --content test)" + run mcp_client call search '{"query":"test","content":true}' -- mcp --expose-tools + assert_success + assert_output --partial 'test.txt' + node - "$output" "$expected" <<'JS' +const assert = require('node:assert/strict'); +assert.deepEqual(JSON.parse(process.argv[2]), { content: [{ type: 'text', text: process.argv[3] }] }); +JS +} + +@test "MCP search supports answer, web, relative and absolute paths and maxCount" { + local expected + expected="$(mgrep search --answer --content --web test .)" + run mcp_client call search '{"query":"test","path":".","answer":true,"content":true,"web":true}' -- --store custom-store mcp --expose-tools + assert_success + node - "$output" "$expected" <<'JS' +const assert = require('node:assert/strict'); +assert.equal(JSON.parse(process.argv[2]).content[0].text, process.argv[3]); +JS + run mcp_client call search "{\"query\":\"Hello\",\"path\":\"$PWD\",\"maxCount\":1}" -- mcp --expose-tools + assert_success + node - "$output" <<'JS' +const assert = require('node:assert/strict'); +assert.equal(JSON.parse(process.argv[2]).content[0].text.split('\n').length, 1); +JS +} + +@test "MCP unknown and disabled tools return errors without crashing" { + run mcp_client call unknown '{}' -- mcp --expose-tools + assert_success + assert_output --partial '"isError":true' + assert_output --partial 'Unknown tool: unknown' + run mcp_client call search '{"query":"test"}' -- mcp + assert_success + assert_output --partial '"isError":true' + assert_output --partial 'Unknown tool: search' +} + +@test "MCP invalid arguments return errors without crashing" { + local arguments + for arguments in '{}' '{"query":4}' '{"query":""}' '{"query":"test","path":4}' '{"query":"test","maxCount":0}' '{"query":"test","maxCount":1.5}' '{"query":"test","content":"true"}' '{"query":"test","sync":true}'; do + run mcp_client call search "$arguments" -- mcp --expose-tools + assert_success + assert_output --partial '"isError":true' + done +} + +@test "MCP first search syncs files without a watcher" { + printf 'unsynced-needle\n' > new.txt + run mcp_client call search "{\"query\":\"unsynced-needle\",\"path\":\"$PWD\"}" -- mcp --expose-tools --no-watch + assert_success + assert_output --partial './new.txt' +} + +@test "MCP survives watcher file count failure" { + printf 'new file\n' > new.txt + printf 'another new file\n' > another.txt + export MGREP_MAX_FILE_COUNT=1 + run mgrep watch + assert_failure 1 + assert_output --partial '❌ Files to sync (2) exceeds the maximum allowed (1). No files were synced.' + refute_output --partial 'Failed to start watcher:' + + export MGREP_MCP_WAIT_FOR='❌ Files to sync' + run mcp_client call search '{"query":"test"}' -- mcp --expose-tools + assert_success + assert_output --partial '"isError":true' + assert_output --partial 'exceeds the maximum allowed' +} + +@test "MCP skips background sync at and above home while serving search" { + export HOME="$(pwd -P)" + export MGREP_MCP_WAIT_FOR='Background sync skipped:' + run mcp_client call search '{"query":"test"}' -- mcp --expose-tools + assert_success + assert_output --partial 'test.txt' + export HOME="$(pwd -P)/child-home" + run mcp_client call search '{"query":"test"}' -- mcp --expose-tools + assert_success + assert_output --partial 'test.txt' +} + +@test "MCP authentication failure returns an error and watcher failure keeps serving" { + export MGREP_IS_TEST=0 + export HOME="$BATS_TEST_TMPDIR/empty-home" + mkdir -p "$HOME" + unset MXBAI_API_KEY + export MGREP_MCP_WAIT_FOR='Failed to start watcher:' + run mcp_client call search '{"query":"test"}' -- mcp --expose-tools + assert_success + assert_output --partial '"isError":true' + assert_output --partial 'No authentication token found' +} + +@test "MCP exits cleanly after client.close with active watchers" { + export MGREP_MCP_WAIT_FOR='[LOG] Watching for file changes in' + run mcp_client close -- mcp --expose-tools + assert_success + assert_equal "$output" '{"exited":true,"code":0,"signal":null}' +} + +@test "MCP exits cleanly on stdin EOF with active watchers" { + export MGREP_MCP_WAIT_FOR='[LOG] Watching for file changes in' + run mcp_client eof -- mcp --expose-tools + assert_success + assert_equal "$output" '{"exited":true,"code":0,"signal":null}' +} + +@test "MCP exits cleanly on stdin EOF before the watcher starts" { + run mcp_client eof -- mcp + assert_success + assert_equal "$output" '{"exited":true,"code":0,"signal":null}' +} + +@test "MCP survives watcher quota failure and returns search quota errors" { + local preload + preload="$(node --input-type=module - "$DIR/../dist" <<'JS' +import { pathToFileURL } from 'node:url'; +const base = pathToFileURL(`${process.argv[2]}/`).href; +const code = ` +import { TestStore } from ${JSON.stringify(`${base}lib/store.js`)}; +import ${JSON.stringify(`${base}lib/auth.js`)}; +import { QuotaExceededError } from ${JSON.stringify(`${base}lib/utils.js`)}; +TestStore.prototype.uploadFile = async () => { throw new QuotaExceededError('quota test'); }; +const search = TestStore.prototype.search; +TestStore.prototype.search = async function (...args) { + if (args[1] === 'quota') throw new QuotaExceededError('search quota test'); + return search.apply(this, args); +};`; +console.log(`data:text/javascript;base64,${Buffer.from(code).toString('base64')}`); +JS +)" + printf 'new file\n' > new.txt + export NODE_OPTIONS="--import=$preload" + run mgrep watch + assert_failure 1 + assert_output --partial "❌ Free tier quota exceeded. You've reached the monthly limit of 2,000,000 store tokens." + refute_output --partial 'Failed to start watcher:' + + export MGREP_MCP_WAIT_FOR='❌ Free tier quota exceeded.' + run mcp_client call search '{"query":"test"}' -- mcp --expose-tools + assert_success + assert_output --partial '"isError":true' + assert_output --partial 'quota test' + unset MGREP_MCP_WAIT_FOR + export HOME="$(pwd -P)" + run mcp_client call search '{"query":"quota"}' -- mcp --expose-tools + assert_success + assert_equal "$output" '{"content":[{"type":"text","text":"search quota test"}],"isError":true}' +} + +@test "Shared search forwards stores, filters, options and preserves formatting" { + node --input-type=module - "$DIR/../dist/lib/search.js" <<'JS' +import assert from 'node:assert/strict'; +import { pathToFileURL } from 'node:url'; +const { resolveSearchPath, runSearch } = await import(pathToFileURL(process.argv[2])); +const root = process.cwd(); +assert.equal(resolveSearchPath(root), root); +assert.equal(resolveSearchPath(root, 'a/../b'), `${root}/b`); +assert.equal(resolveSearchPath(root, '/absolute/../unchanged'), '/absolute/../unchanged'); +const chunk = { + type: 'text', text: 'fixture', score: 0.75, metadata: { path: `${root}/test.txt` }, + generated_metadata: { start_line: 0, num_lines: 2 }, +}; +const web = { type: 'text', text: 'web fixture', score: 1, filename: 'https://example.com' }; +let received; +const store = { + search: async (...args) => { received = args; return { data: [chunk, web] }; }, + ask: async (...args) => { received = args; return { answer: '', sources: [chunk, web] }; }, +}; +const options = { + store: 'custom', query: 'question', path: root, maxCount: 3, content: true, + answer: false, web: true, rerank: true, agentic: false, +}; +assert.equal(await runSearch(store, options, root), './test.txt:1-3 (75.00% match)\nfixture\nhttps://example.com (100.00% match)\nweb fixture'); +assert.deepEqual(received, [['custom', 'mixedbread/web'], 'question', 3, { rerank: true }, + { all: [{ key: 'path', operator: 'starts_with', value: root }] }]); +assert.equal(await runSearch(store, { ...options, answer: true, web: false, rerank: false, agentic: true }, root), + '\n\n0: ./test.txt:1-3 (75.00% match)\nfixture\n1: https://example.com (100.00% match)\nweb fixture'); +assert.deepEqual(received.slice(0, 4), [['custom'], 'question', 3, { rerank: false, agentic: true }]); +JS +} + +@test "MCP no-watch requires path in its schema and errors without a root" { + run mcp_client list -- mcp --expose-tools --no-watch + assert_success + node - "$output" <<'JS' +const assert = require('node:assert/strict'); +const schema = JSON.parse(process.argv[2]).tools[0].inputSchema; +assert.deepEqual(schema.required, ['query', 'path']); +assert.equal(Object.hasOwn(schema.properties.path, 'default'), false); +JS + run mcp_client call search '{"query":"test"}' -- mcp --expose-tools --no-watch + assert_success + assert_output --partial '"isError":true' + assert_output --partial 'An absolute project path is required' +} + +@test "MCP no-watch never schedules or starts background sync" { + run mcp_client idle -- mcp --expose-tools --no-watch + assert_success + assert_equal "$output" '{"stderr":""}' +} + +@test "MCP uses the first file root and makes results relative to it" { + mkdir -p "$BATS_TEST_TMPDIR/project/src" "$BATS_TEST_TMPDIR/other" + printf 'root-needle\n' > "$BATS_TEST_TMPDIR/project/src/x.ts" + printf 'root-needle\n' > "$BATS_TEST_TMPDIR/other/wrong.txt" + export MGREP_MCP_ROOTS="[{\"uri\":\"https://example.com\"},{\"uri\":\"file://$BATS_TEST_TMPDIR/project\"},{\"uri\":\"file://$BATS_TEST_TMPDIR/other\"}]" + run mcp_client call search '{"query":"root-needle"}' -- mcp --expose-tools --no-watch + assert_success + assert_output --partial './src/x.ts:1-2' + refute_output --partial 'wrong.txt' + refute_output --partial "$BATS_TEST_TMPDIR" +} + +@test "MCP explicit absolute and relative paths win over client roots" { + mkdir -p alternate/src + printf 'explicit-needle\n' > alternate/src/x.ts + export MGREP_MCP_ROOTS="[{\"uri\":\"file://$BATS_TEST_TMPDIR/missing\"}]" + export MGREP_MCP_FORBID_ROOTS=1 + for root in './alternate' "$PWD/alternate"; do + run mcp_client call search "{\"query\":\"explicit-needle\",\"path\":\"$root\"}" -- mcp --expose-tools --no-watch + assert_success + assert_output --partial './src/x.ts:1-2' + refute_output --partial '"isError":true' + done +} + +@test "MCP roots with no file URI fall back to cwd only with watch enabled" { + export MGREP_MCP_ROOTS='[{"uri":"https://example.com"}]' + run mcp_client call search '{"query":"test"}' -- mcp --expose-tools + assert_success + assert_output --partial './test.txt' + run mcp_client call search '{"query":"test"}' -- mcp --expose-tools --no-watch + assert_success + assert_output --partial 'An absolute project path is required' +} + +@test "MCP first-search sync runs once per resolved root during a server lifetime" { + mkdir -p alternate + run mcp_client calls "[{\"query\":\"test\",\"path\":\"$PWD\"},{\"write\":{\"path\":\"$PWD/later.txt\",\"text\":\"later-needle\"}},{\"query\":\"later-needle\",\"path\":\".\"},{\"write\":{\"path\":\"$PWD/alternate/second.txt\",\"text\":\"second-root-needle\"}},{\"query\":\"second-root-needle\",\"path\":\"alternate\"}]" -- mcp --expose-tools --no-watch + assert_success + node - "$output" <<'JS' +const assert = require('node:assert/strict'); +const results = JSON.parse(process.argv[2]); +assert.match(results[0].content[0].text, /test.txt/); +assert.equal(results[1].content[0].text, ''); +assert.match(results[2].content[0].text, /^\.\/second.txt:1-2/); +JS +} + +@test "MCP first-search sync skips roots at and above home with no watcher" { + printf 'unsynced-home-needle\n' > new.txt + for root in "$PWD" "$PWD/.."; do + export HOME="$(pwd -P)" + run mcp_client call search "{\"query\":\"unsynced-home-needle\",\"path\":\"$root\"}" -- mcp --expose-tools --no-watch + assert_success + assert_equal "$output" '{"content":[{"type":"text","text":""}]}' + done +} + +@test "MCP initialize instructions describe semantic search and include the server cwd" { + run mcp_client initialize -- mcp --expose-tools + assert_success + node - "$output" "$(pwd -P)" <<'JS' +const assert = require('node:assert/strict'); +assert.equal(JSON.parse(process.argv[2]).instructions, `mgrep semantic search is available through this server's \`search\` tool. Use it for conceptual or cross-file questions where you don't know the exact identifier, such as 'where is authentication handled'. It searches ${process.argv[3]} by default. Use rg or grep for exact strings, symbol names and regexes. Both are fine; pick whichever fits the question.`); +JS +} + +@test "MCP optional path waits for the initial watcher sync before searching cwd" { + printf 'watcher-needle\n' > new.txt + run mcp_client call-with-logs search '{"query":"watcher-needle"}' -- mcp --expose-tools + assert_success + node - "$output" <<'JS' +const assert = require('node:assert/strict'); +const { result, stderr } = JSON.parse(process.argv[2]); +assert.match(result.content[0].text, /\.\/new.txt/); +assert.equal(result.isError, undefined); +assert.equal((stderr.match(/Initial sync complete/g) || []).length, 1); +assert.doesNotMatch(stderr, /Indexing complete/); +JS +} + +@test "MCP watcher per-file upload errors do not trigger a second sync" { + local preload + preload="$(node --input-type=module - "$DIR/../dist" <<'JS' +import { pathToFileURL } from 'node:url'; +const module = pathToFileURL(`${process.argv[2]}/lib/store.js`).href; +const code = `import { TestStore } from ${JSON.stringify(module)}; +const listFiles = TestStore.prototype.listFiles; +TestStore.prototype.listFiles = async function* (...args) { + console.error('test sync pass'); + yield* listFiles.apply(this, args); +}; +TestStore.prototype.uploadFile = async (_storeId, file) => { + for await (const chunk of file) {} + throw new Error('per-file upload test'); +};`; +console.log(`data:text/javascript;base64,${Buffer.from(code).toString('base64')}`); +JS +)" + printf 'failed upload\n' > new.txt + export NODE_OPTIONS="--import=$preload" + run mcp_client call-with-logs search '{"query":"test"}' -- mcp --expose-tools + assert_success + node - "$output" <<'JS' +const assert = require('node:assert/strict'); +const { result, stderr } = JSON.parse(process.argv[2]); +assert.match(result.content[0].text, /\.\/test.txt/); +assert.equal(result.isError, undefined); +assert.match(stderr, /Initial sync complete.*errors 1/); +assert.match(stderr, /1 file\(s\) failed to upload/); +assert.match(stderr, /\[LOG\] Watching for file changes in/); +assert.equal((stderr.match(/test sync pass/g) || []).length, 1); +assert.doesNotMatch(stderr, /Indexing complete/); +assert.doesNotMatch(result.content[0].text, /Note:/); +JS +} + +@test "MCP search times out waiting for initial sync and searches the existing index without syncing again" { + local preload + preload="$(node --input-type=module - "$DIR/../dist" <<'JS' +import { pathToFileURL } from 'node:url'; +const module = pathToFileURL(`${process.argv[2]}/lib/store.js`).href; +const code = `import { TestStore } from ${JSON.stringify(module)}; +const listFiles = TestStore.prototype.listFiles; +TestStore.prototype.listFiles = async function* (...args) { + console.error('test sync pass'); + yield* listFiles.apply(this, args); +}; +TestStore.prototype.uploadFile = async (_storeId, file) => { + for await (const chunk of file) {} + console.error('test upload blocked'); + await new Promise(() => {}); +};`; +console.log(`data:text/javascript;base64,${Buffer.from(code).toString('base64')}`); +JS +)" + printf 'not-indexed-needle\n' > new.txt + export NODE_OPTIONS="--import=$preload" + export MGREP_MCP_WAIT_FOR='test upload blocked' + export MGREP_MCP_SYNC_WAIT_MS=50 + run mcp_client calls-with-logs '[{"query":"test"},{"query":"not-indexed-needle","path":"."}]' -- mcp --expose-tools + assert_success + node - "$output" <<'JS' +const assert = require('node:assert/strict'); +const { result, stderr } = JSON.parse(process.argv[2]); +const note = 'Note: the initial mgrep sync is still running; results may be incomplete.'; +assert.match(result[0].content[0].text, /\.\/test.txt/); +for (const entry of result) { + assert.equal(entry.isError, undefined); + assert.equal(entry.content[0].text.split('\n').at(-1), note); +} +assert.equal(result[1].content[0].text, note); +assert.equal((stderr.match(/test sync pass/g) || []).length, 1); +assert.equal((stderr.match(/test upload blocked/g) || []).length, 1); +assert.doesNotMatch(stderr, /Initial sync complete|Indexing complete/); +JS +} + +@test "MCP searches inside cwd do not sync again once the watcher is active" { + mkdir -p nested + printf 'nested-needle\n' > nested/child.txt + export MGREP_MCP_WAIT_FOR='[LOG] Watching for file changes in' + run mcp_client calls-with-logs '[{"query":"nested-needle","path":"nested"},{"query":"nested-needle"},{"query":"nested-needle","path":"."}]' -- mcp --expose-tools + assert_success + node - "$output" <<'JS' +const assert = require('node:assert/strict'); +const { result, stderr } = JSON.parse(process.argv[2]); +assert.match(result[0].content[0].text, /^\.\/child.txt/); +for (const entry of result.slice(1)) assert.match(entry.content[0].text, /^\.\/nested\/child.txt/); +assert.equal((stderr.match(/Initial sync complete/g) || []).length, 1); +assert.doesNotMatch(stderr, /Indexing complete/); +JS +} + +@test "MCP searches outside cwd keep first-search sync per root with a watcher" { + mkdir -p ../test-store-sibling + printf 'outside-needle\n' > ../test-store-sibling/outside.txt + export MGREP_MCP_WAIT_FOR='[LOG] Watching for file changes in' + run mcp_client calls-with-logs '[{"query":"outside-needle","path":"../test-store-sibling"},{"query":"outside-needle","path":"../test-store-sibling"}]' -- mcp --expose-tools + assert_success + node - "$output" <<'JS' +const assert = require('node:assert/strict'); +const { result, stderr } = JSON.parse(process.argv[2]); +for (const entry of result) assert.match(entry.content[0].text, /^\.\/outside.txt/); +assert.equal((stderr.match(/Initial sync complete/g) || []).length, 1); +assert.equal((stderr.match(/Indexing complete/g) || []).length, 1); +JS +} + +@test "MCP failed watcher falls back to first-search sync once per root" { + local preload + preload="$(node --input-type=module - "$DIR/../dist" <<'JS' +import { pathToFileURL } from 'node:url'; +const module = pathToFileURL(`${process.argv[2]}/lib/store.js`).href; +const code = `import { TestStore } from ${JSON.stringify(module)}; +TestStore.prototype.retrieve = async () => { throw new Error('watcher startup test'); }; +TestStore.prototype.create = async () => { throw new Error('watcher startup test'); };`; +console.log(`data:text/javascript;base64,${Buffer.from(code).toString('base64')}`); +JS +)" + printf 'fallback-needle\n' > new.txt + export NODE_OPTIONS="--import=$preload" + run mcp_client calls-with-logs '[{"query":"fallback-needle"},{"query":"fallback-needle","path":"."}]' -- mcp --expose-tools + assert_success + node - "$output" <<'JS' +const assert = require('node:assert/strict'); +const { result, stderr } = JSON.parse(process.argv[2]); +for (const entry of result) { + assert.match(entry.content[0].text, /^\.\/new.txt/); + assert.equal(entry.isError, undefined); +} +assert.match(stderr, /Failed to start watcher: watcher startup test/); +assert.equal((stderr.match(/Indexing complete/g) || []).length, 1); +JS +} + +@test "startWatch reports readiness after complete sync and returns false for startup failures or dry run" { + local scenario + for scenario in ready dry-run home quota file-count store-error watch-error async-watch-error partial-sync; do + run node --input-type=module - "$DIR/../dist" "$scenario" <<'JS' +import assert from 'node:assert/strict'; +import { EventEmitter } from 'node:events'; +import fs from 'node:fs'; +import { syncBuiltinESMExports } from 'node:module'; +import { pathToFileURL } from 'node:url'; +const base = pathToFileURL(`${process.argv[2]}/`).href; +const { startWatch } = await import(`${base}commands/watch.js`); +const { TestStore } = await import(`${base}lib/store.js`); +const { QuotaExceededError } = await import(`${base}lib/utils.js`); +const scenario = process.argv[3]; +const store = new TestStore(); +const watchers = []; +fs.watch = () => { + if (scenario === 'watch-error') throw new Error('watch setup test'); + const watcher = new EventEmitter(); + watcher.close = () => {}; + watchers.push(watcher); + if (scenario === 'async-watch-error') process.nextTick(() => watcher.emit('error', new Error('async watch setup test'))); + return watcher; +}; +syncBuiltinESMExports(); +if (scenario === 'home') process.env.HOME = process.cwd(); +if (['quota', 'file-count', 'partial-sync'].includes(scenario)) { + fs.writeFileSync('contract-a.txt', 'contract fixture'); + fs.writeFileSync('contract-b.txt', 'contract fixture'); +} +if (['quota', 'partial-sync'].includes(scenario)) store.uploadFile = async (_storeId, file) => { + for await (const chunk of file) {} + if (scenario === 'quota') throw new QuotaExceededError('contract quota'); + throw new Error('contract upload'); +}; +const result = await startWatch({ + store: 'test-store', dryRun: scenario === 'dry-run', + maxFileCount: scenario === 'file-count' ? 1 : undefined, + createStore: async () => { + if (scenario === 'store-error') throw new Error('contract store'); + return store; + }, +}); +const watching = ['ready', 'partial-sync'].includes(scenario); +assert.equal(result, watching, scenario); +if (watching) assert.ok(watchers.length > 0); +assert.equal(process.exitCode || 0, ['home', 'quota', 'file-count', 'store-error'].includes(scenario) ? 1 : 0, scenario); +process.exitCode = 0; +fs.rmSync('contract-a.txt', { force: true }); +fs.rmSync('contract-b.txt', { force: true }); +JS + assert_success + done +} diff --git a/test/test.bats b/test/test.bats index d3cf846..c8d4df8 100755 --- a/test/test.bats +++ b/test/test.bats @@ -46,7 +46,7 @@ teardown() { run mgrep --version assert_success - assert_output --regexp '^[0-9]+\.[0-9]+\.[0-9]+$' + assert_equal "$output" "$(node -p "JSON.parse(require('node:fs').readFileSync('$DIR/../package.json', 'utf8')).version")" } @test "Search" {