From fab66ea607ab08f84823c95be1e7f97aa4e69f98 Mon Sep 17 00:00:00 2001 From: yashi Date: Tue, 4 Aug 2026 22:02:21 -0700 Subject: [PATCH] feat: add full-text search over personal and code wikis Agents often explore wiki memory with multi-hop ls/grep/read. Add a dedicated searchWiki helper, openwiki search CLI, and openwiki_search_wiki agent tool so retrieval is a first-class path for Personal and Code Brain. Co-authored-by: Cursor --- .changeset/wiki-search-tool.md | 5 + src/agent/index.ts | 9 +- src/agent/prompts/code.ts | 6 +- src/agent/prompts/personal.ts | 6 +- src/cli.tsx | 37 +++++ src/commands.ts | 165 ++++++++++++++++++++++ src/search/index.ts | 10 ++ src/search/resolve-wiki-root.ts | 46 +++++++ src/search/search-wiki.ts | 233 ++++++++++++++++++++++++++++++++ src/search/tools.ts | 87 ++++++++++++ src/search/types.ts | 18 +++ test/search-wiki.test.ts | 215 +++++++++++++++++++++++++++++ 12 files changed, 834 insertions(+), 3 deletions(-) create mode 100644 .changeset/wiki-search-tool.md create mode 100644 src/search/index.ts create mode 100644 src/search/resolve-wiki-root.ts create mode 100644 src/search/search-wiki.ts create mode 100644 src/search/tools.ts create mode 100644 src/search/types.ts create mode 100644 test/search-wiki.test.ts diff --git a/.changeset/wiki-search-tool.md b/.changeset/wiki-search-tool.md new file mode 100644 index 00000000..0252e70f --- /dev/null +++ b/.changeset/wiki-search-tool.md @@ -0,0 +1,5 @@ +--- +"openwiki": minor +--- + +Add `openwiki search` / `openwiki personal search` and an `openwiki_search_wiki` agent tool for full-text wiki retrieval. diff --git a/src/agent/index.ts b/src/agent/index.ts index dd028088..405800cb 100644 --- a/src/agent/index.ts +++ b/src/agent/index.ts @@ -19,6 +19,7 @@ import { type GlobResult, } from "deepagents"; import { createOpenWikiConnectorTools } from "../connectors/tools.js"; +import { createWikiSearchTool } from "../search/index.js"; import { DEBUG_ENV_KEYS, loadOpenWikiEnv, @@ -366,7 +367,13 @@ function createOpenWikiAgentGraph( return createDeepAgent({ model: options.model, - tools: createOpenWikiConnectorTools(), + tools: [ + ...createOpenWikiConnectorTools(), + createWikiSearchTool({ + cwd: options.cwd, + outputMode: options.outputMode, + }), + ], checkpointer: options.checkpointer, backend, middleware: diff --git a/src/agent/prompts/code.ts b/src/agent/prompts/code.ts index eadf6771..90a2a0e3 100644 --- a/src/agent/prompts/code.ts +++ b/src/agent/prompts/code.ts @@ -21,7 +21,8 @@ Run discipline: - Inspect the repository tree, workspace and package manifests, existing docs, entrypoints, routing and schema files, public surfaces, and representative implementation and tests.{OPENWIKIIGNORE_INSTRUCTIONS} Wiki-first question answering: -- For ordinary chat questions, inspect the generated wiki under /openwiki first. Use quickstart/index pages, section pages, and targeted grep/glob over the wiki before looking at source files. +- For ordinary chat questions, call openwiki_search_wiki first to locate relevant pages under /openwiki, then read only the best matching virtualPath hits. Prefer search over broad ls/grep crawls. +- If search is unavailable or returns nothing useful, fall back to quickstart/index pages and targeted grep/glob over the wiki before looking at source files. - If the user asks you to "look at the wiki", answer "based on the wiki", report "what the wiki says", or otherwise frames the request around the wiki, use only /openwiki pages unless the wiki cannot support the answer. - Assume the generated wiki contains the answer most of the time. Do not exhaustively read source files just because they exist. @@ -43,6 +44,9 @@ OpenWiki CLI reference: - \`openwiki --update [message]\` updates repository documentation under openwiki/ (code mode). - \`openwiki personal --init [message]\` initializes the local personal brain wiki under ~/.openwiki/wiki. - \`openwiki code --init [message]\` initializes repository documentation under openwiki/. +- \`openwiki personal search \` full-text searches the local personal brain without starting an agent. +- \`openwiki code search \` full-text searches repository documentation under openwiki/. +- \`openwiki search [--mode personal|code] [--limit ] \` full-text searches a wiki (defaults to personal). - \`openwiki --mode code --init [message]\` initializes repository documentation under openwiki/. - \`openwiki --mode personal --init [message]\` initializes the local personal brain wiki under ~/.openwiki/wiki. - \`openwiki -p "message"\` or \`openwiki --print "message"\` runs once, prints the final assistant output, and exits. diff --git a/src/agent/prompts/personal.ts b/src/agent/prompts/personal.ts index ef139181..0bc93f7a 100644 --- a/src/agent/prompts/personal.ts +++ b/src/agent/prompts/personal.ts @@ -41,7 +41,8 @@ Connector ingestion discipline: Wiki-first question answering: -- For ordinary chat questions, inspect the generated wiki under the virtual root / first. Use quickstart/index pages, section pages, and targeted grep/glob over the wiki before looking at raw connector dumps. +- For ordinary chat questions, call openwiki_search_wiki first to locate relevant wiki pages, then read only the best matching virtualPath hits. Prefer search over broad ls/grep crawls across the wiki. +- If search is unavailable or returns nothing useful, fall back to quickstart/index pages and targeted grep/glob over the wiki before looking at raw connector dumps. - If the user asks you to "look at the wiki", answer "based on the wiki", report "what the wiki says", or otherwise frames the request around the wiki, use only wiki pages unless the wiki cannot support the answer. - Assume the synthesized wiki contains the answer most of the time. Do not inspect raw connector data just because it exists. - Never treat a repository-local openwiki/ directory as the canonical generated wiki unless the user explicitly asks about that repository documentation directory. @@ -75,6 +76,9 @@ OpenWiki CLI reference: - \`openwiki --update [message]\` updates repository documentation under openwiki/ (code mode). - \`openwiki personal --init [message]\` initializes the local personal brain wiki under ~/.openwiki/wiki. - \`openwiki code --init [message]\` initializes repository documentation under openwiki/. +- \`openwiki personal search \` full-text searches the local personal brain without starting an agent. +- \`openwiki code search \` full-text searches repository documentation under openwiki/. +- \`openwiki search [--mode personal|code] [--limit ] \` full-text searches a wiki (defaults to personal). - \`openwiki --mode code --init [message]\` initializes repository documentation under openwiki/. - \`openwiki --mode personal --init [message]\` initializes the local personal brain wiki under ~/.openwiki/wiki. - \`openwiki -p "message"\` or \`openwiki --print "message"\` runs once, prints the final assistant output, and exits. diff --git a/src/cli.tsx b/src/cli.tsx index 5d93634e..c360d06c 100644 --- a/src/cli.tsx +++ b/src/cli.tsx @@ -59,6 +59,11 @@ import { saveOpenWikiOnboardingConfig, } from "./onboarding.js"; import { openWikiLocalWikiDir } from "./openwiki-home.js"; +import { + resolveWikiRoot, + searchWiki, + virtualRootForRunMode, +} from "./search/index.js"; import { deleteConnectorSchedules, getSavedPowerScheduleStatus, @@ -3795,6 +3800,8 @@ if (command.kind === "auth") { await runCronCommand(command); } else if (command.kind === "ingest") { await runIngestCommand(command); +} else if (command.kind === "search") { + await runSearchCommand(command); } else if (command.kind === "visualize") { await runVisualizeCommand(command); } else if (shouldPrintStartupError(argv, parsedCommand, command)) { @@ -3853,6 +3860,36 @@ async function runVisualizeCommand( } } +async function runSearchCommand( + command: Extract, +): Promise { + try { + const rootDir = resolveWikiRoot(command.mode, process.cwd()); + const hits = await searchWiki({ + rootDir, + query: command.query, + virtualRoot: virtualRootForRunMode(command.mode), + maxResults: command.limit ?? undefined, + }); + + if (hits.length === 0) { + process.stdout.write( + `No matches for ${JSON.stringify(command.query)} in ${rootDir}\n`, + ); + process.exitCode = 0; + return; + } + + for (const hit of hits) { + process.stdout.write(`${hit.path}:${hit.line}: ${hit.snippet}\n`); + } + process.exitCode = 0; + } catch (error) { + process.stderr.write(`${getErrorMessage(error)}\n`); + process.exitCode = 1; + } +} + async function runCronCommand( command: Extract, ): Promise { diff --git a/src/commands.ts b/src/commands.ts index 3ddb03ee..eb1b1f15 100644 --- a/src/commands.ts +++ b/src/commands.ts @@ -60,6 +60,13 @@ export type CliCommand = exitCode: 0; target: CronTarget | null; } + | { + kind: "search"; + exitCode: 0; + limit: number | null; + mode: OpenWikiRunMode; + query: string; + } | { kind: "help"; exitCode: 0 } | { kind: "run"; @@ -393,6 +400,14 @@ export function parseCommand(argv: string[]): CliCommand { } } + if (argv[0] === "search") { + return parseSearchCommand(argv.slice(1), null); + } + + if (isOpenWikiRunMode(argv[0]) && argv[1] === "search") { + return parseSearchCommand(argv.slice(2), argv[0]); + } + if (isOpenWikiRunMode(argv[0])) { return parseRunCommand(argv.slice(1), argv[0], "positional"); } @@ -400,6 +415,132 @@ export function parseCommand(argv: string[]): CliCommand { return parseRunCommand(argv, "code", "default"); } +function parseSearchCommand( + argv: string[], + initialMode: OpenWikiRunMode | null, +): CliCommand { + let mode = initialMode; + let limit: number | null = null; + const queryParts: string[] = []; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + + if (arg === "--help" || arg === "-h") { + return { kind: "help", exitCode: 0 }; + } + + if (arg === "--mode") { + if (initialMode !== null) { + return { + kind: "error", + exitCode: 1, + message: + "Do not pass --mode when using openwiki personal search or openwiki code search.", + }; + } + + const nextArg = argv[index + 1]; + if (!nextArg || nextArg.startsWith("-")) { + return { + kind: "error", + exitCode: 1, + message: "--mode requires personal or code.", + }; + } + if (!isOpenWikiRunMode(nextArg)) { + return { + kind: "error", + exitCode: 1, + message: `Invalid mode: ${nextArg}. Expected personal or code.`, + }; + } + mode = nextArg; + index += 1; + continue; + } + + if (arg.startsWith("--mode=")) { + if (initialMode !== null) { + return { + kind: "error", + exitCode: 1, + message: + "Do not pass --mode when using openwiki personal search or openwiki code search.", + }; + } + const nextArg = arg.slice("--mode=".length); + if (!isOpenWikiRunMode(nextArg)) { + return { + kind: "error", + exitCode: 1, + message: `Invalid mode: ${nextArg}. Expected personal or code.`, + }; + } + mode = nextArg; + continue; + } + + if (arg === "--limit") { + const nextArg = argv[index + 1]; + if (!nextArg || nextArg.startsWith("-")) { + return { + kind: "error", + exitCode: 1, + message: "--limit requires a number.", + }; + } + limit = Number(nextArg); + index += 1; + continue; + } + + if (arg.startsWith("--limit=")) { + limit = Number(arg.slice("--limit=".length)); + continue; + } + + if (arg.startsWith("-")) { + return { + kind: "error", + exitCode: 1, + message: `Unknown option for search: ${arg}`, + }; + } + + queryParts.push(arg); + } + + if ( + limit !== null && + (!Number.isInteger(limit) || limit < 1 || limit > 100) + ) { + return { + kind: "error", + exitCode: 1, + message: "--limit must be an integer between 1 and 100.", + }; + } + + const query = queryParts.join(" ").trim(); + if (!query) { + return { + kind: "error", + exitCode: 1, + message: + "Usage: openwiki personal search | openwiki search [--mode personal|code] [--limit ] ", + }; + } + + return { + kind: "search", + exitCode: 0, + limit, + mode: mode ?? "personal", + query, + }; +} + function parseRunCommand( argv: string[], initialMode: OpenWikiRunMode, @@ -747,6 +888,9 @@ export const helpContent: HelpContent = { "openwiki cron pause all", "openwiki cron resume all", "openwiki cron delete all", + "openwiki personal search ", + "openwiki code search ", + "openwiki search [--mode personal|code] [--limit ] ", "openwiki ngrok start [url] [--port ]", "openwiki visualize [path] [--port ] [--no-open]", ], @@ -761,6 +905,21 @@ export const helpContent: HelpContent = { description: "Run OpenWiki as your local personal brain over configured sources, writing to ~/.openwiki/wiki.", }, + { + label: "openwiki personal search ", + description: + "Full-text search the local personal brain wiki under ~/.openwiki/wiki without starting an agent.", + }, + { + label: "openwiki code search ", + description: + "Full-text search repository documentation under ./openwiki without starting an agent.", + }, + { + label: "openwiki search ", + description: + "Full-text search a wiki (defaults to personal mode; pass --mode code for repository docs).", + }, { label: "openwiki", description: @@ -859,6 +1018,10 @@ export const helpContent: HelpContent = { description: "Write the exact anonymous telemetry payload to a local JSON file.", }, + { + label: "--limit ", + description: "For search: maximum number of hits to print (1-100).", + }, { label: "--port ", description: @@ -895,6 +1058,8 @@ export const helpContent: HelpContent = { "openwiki cron pause all", "openwiki cron resume all", "openwiki cron delete all", + 'openwiki personal search "middleware"', + 'openwiki search --mode personal --limit 10 "filesystem backends"', "openwiki auth slack", "openwiki auth gmail", "openwiki auth notion", diff --git a/src/search/index.ts b/src/search/index.ts new file mode 100644 index 00000000..a07f83fb --- /dev/null +++ b/src/search/index.ts @@ -0,0 +1,10 @@ +export type { WikiSearchHit, WikiSearchOptions } from "./types.js"; +export { searchWiki, tokenizeQuery } from "./search-wiki.js"; +export { + resolveWikiRoot, + resolveWikiRootFromOutputMode, + runModeToOutputMode, + virtualRootForOutputMode, + virtualRootForRunMode, +} from "./resolve-wiki-root.js"; +export { createWikiSearchTool } from "./tools.js"; diff --git a/src/search/resolve-wiki-root.ts b/src/search/resolve-wiki-root.ts new file mode 100644 index 00000000..1347644f --- /dev/null +++ b/src/search/resolve-wiki-root.ts @@ -0,0 +1,46 @@ +import path from "node:path"; +import { openWikiLocalWikiDir } from "../openwiki-home.js"; +import type { OpenWikiOutputMode } from "../agent/types.js"; + +const OPEN_WIKI_DIR = "openwiki"; + +export type WikiSearchRunMode = "personal" | "code"; + +export function runModeToOutputMode( + mode: WikiSearchRunMode, +): OpenWikiOutputMode { + return mode === "code" ? "repository" : "local-wiki"; +} + +export function resolveWikiRoot( + mode: WikiSearchRunMode, + cwd: string = process.cwd(), +): string { + if (mode === "personal") { + return openWikiLocalWikiDir; + } + + return path.join(cwd, OPEN_WIKI_DIR); +} + +export function resolveWikiRootFromOutputMode( + outputMode: OpenWikiOutputMode, + cwd: string, +): string { + if (outputMode === "local-wiki") { + // Personal-brain agent runs use the wiki directory as cwd. + return cwd; + } + + return path.join(cwd, OPEN_WIKI_DIR); +} + +export function virtualRootForOutputMode( + outputMode: OpenWikiOutputMode, +): string { + return outputMode === "local-wiki" ? "/" : "/openwiki/"; +} + +export function virtualRootForRunMode(mode: WikiSearchRunMode): string { + return mode === "personal" ? "/" : "/openwiki/"; +} diff --git a/src/search/search-wiki.ts b/src/search/search-wiki.ts new file mode 100644 index 00000000..decbd04d --- /dev/null +++ b/src/search/search-wiki.ts @@ -0,0 +1,233 @@ +import { lstat, readdir, readFile } from "node:fs/promises"; +import path from "node:path"; +import type { WikiSearchHit, WikiSearchOptions } from "./types.js"; + +const DEFAULT_MAX_RESULTS = 20; +const DEFAULT_MAX_FILE_BYTES = 500_000; +const MAX_SNIPPET_LENGTH = 160; + +export async function searchWiki( + options: WikiSearchOptions, +): Promise { + const query = options.query.trim(); + if (!query) { + return []; + } + + const terms = tokenizeQuery(query); + if (terms.length === 0) { + return []; + } + + const maxResults = clamp(options.maxResults ?? DEFAULT_MAX_RESULTS, 1, 100); + const maxFileBytes = clamp( + options.maxFileBytes ?? DEFAULT_MAX_FILE_BYTES, + 1, + 2_000_000, + ); + const virtualRoot = normalizeVirtualRoot(options.virtualRoot ?? "/"); + + const files = await listMarkdownFiles(options.rootDir, options.rootDir); + const hits: WikiSearchHit[] = []; + + for (const absolutePath of files) { + const relativePath = toPosixRelative(options.rootDir, absolutePath); + let content: string; + try { + const raw = await readFile(absolutePath); + if (raw.byteLength > maxFileBytes) { + continue; + } + content = raw.toString("utf8"); + } catch { + continue; + } + + const lines = content.split(/\r?\n/u); + const pathBoost = scorePathMatch(relativePath, terms); + let fileTermHits = 0; + + for (let index = 0; index < lines.length; index += 1) { + const line = lines[index] ?? ""; + const lower = line.toLowerCase(); + const matchedTerms = terms.filter((term) => lower.includes(term)); + if (matchedTerms.length === 0) { + continue; + } + + // Prefer lines that cover more query terms (AND-ish ranking). + const coverage = matchedTerms.length / terms.length; + if (coverage < 1 && terms.length > 1 && matchedTerms.length === 1) { + // Still keep single-term hits, but rank them lower than multi-term. + } + + fileTermHits += matchedTerms.length; + const score = + matchedTerms.length * 10 + + coverage * 5 + + pathBoost + + titleLineBoost(line); + + hits.push({ + path: relativePath, + virtualPath: toVirtualPath(relativePath, virtualRoot), + line: index + 1, + snippet: truncateSnippet(line.trim()), + score, + }); + } + + // If the path matched but no line did, still emit a path-level hit so + // filename matches (e.g. deepagents-backends.md for "backends") surface. + if (pathBoost > 0 && fileTermHits === 0) { + hits.push({ + path: relativePath, + virtualPath: toVirtualPath(relativePath, virtualRoot), + line: 1, + snippet: `(filename match) ${relativePath}`, + score: pathBoost, + }); + } + } + + hits.sort((left, right) => { + if (right.score !== left.score) { + return right.score - left.score; + } + if (left.path !== right.path) { + return left.path.localeCompare(right.path); + } + return left.line - right.line; + }); + + return dedupeHits(hits).slice(0, maxResults); +} + +export function tokenizeQuery(query: string): string[] { + return query + .toLowerCase() + .split(/[^a-z0-9_./-]+/u) + .map((term) => term.trim()) + .filter((term) => term.length > 0); +} + +async function listMarkdownFiles( + rootDir: string, + currentDir: string, +): Promise { + let entries; + try { + await assertNotSymlink(currentDir); + entries = await readdir(currentDir, { withFileTypes: true }); + } catch (error) { + if (isFileNotFoundError(error)) { + return []; + } + throw error; + } + + const files: string[] = []; + for (const entry of entries) { + if (entry.name === ".git" || entry.name === "node_modules") { + continue; + } + + const entryPath = path.join(currentDir, entry.name); + try { + await assertNotSymlink(entryPath); + } catch { + continue; + } + + if (entry.isDirectory()) { + files.push(...(await listMarkdownFiles(rootDir, entryPath))); + } else if (entry.isFile() && entry.name.toLowerCase().endsWith(".md")) { + files.push(entryPath); + } + } + + return files; +} + +async function assertNotSymlink(filePath: string): Promise { + const entryStat = await lstat(filePath); + if (entryStat.isSymbolicLink()) { + throw new Error("Symbolic links are not followed during wiki search."); + } +} + +function scorePathMatch(relativePath: string, terms: string[]): number { + const haystack = relativePath.toLowerCase(); + let score = 0; + for (const term of terms) { + if (haystack.includes(term)) { + score += 8; + } + } + return score; +} + +function titleLineBoost(line: string): number { + const trimmed = line.trim(); + if (trimmed.startsWith("#")) { + return 3; + } + if (trimmed.startsWith("title:")) { + return 2; + } + return 0; +} + +function dedupeHits(hits: WikiSearchHit[]): WikiSearchHit[] { + const seen = new Set(); + const result: WikiSearchHit[] = []; + for (const hit of hits) { + const key = `${hit.path}:${hit.line}`; + if (seen.has(key)) { + continue; + } + seen.add(key); + result.push(hit); + } + return result; +} + +function toPosixRelative(rootDir: string, absolutePath: string): string { + return path + .relative(rootDir, absolutePath) + .split(path.sep) + .join(path.posix.sep); +} + +function normalizeVirtualRoot(virtualRoot: string): string { + if (virtualRoot === "/") { + return "/"; + } + return virtualRoot.endsWith("/") ? virtualRoot : `${virtualRoot}/`; +} + +function toVirtualPath(relativePath: string, virtualRoot: string): string { + if (virtualRoot === "/") { + return `/${relativePath}`; + } + return `${virtualRoot}${relativePath}`; +} + +function truncateSnippet(snippet: string): string { + if (snippet.length <= MAX_SNIPPET_LENGTH) { + return snippet; + } + return `${snippet.slice(0, MAX_SNIPPET_LENGTH - 1)}…`; +} + +function clamp(value: number, min: number, max: number): number { + return Math.max(min, Math.min(max, value)); +} + +function isFileNotFoundError(error: unknown): boolean { + return ( + error instanceof Error && + "code" in error && + (error as NodeJS.ErrnoException).code === "ENOENT" + ); +} diff --git a/src/search/tools.ts b/src/search/tools.ts new file mode 100644 index 00000000..d2917b34 --- /dev/null +++ b/src/search/tools.ts @@ -0,0 +1,87 @@ +import { + DynamicStructuredTool, + type StructuredToolInterface, +} from "@langchain/core/tools"; +import type { OpenWikiOutputMode } from "../agent/types.js"; +import { + resolveWikiRootFromOutputMode, + virtualRootForOutputMode, +} from "./resolve-wiki-root.js"; +import { searchWiki } from "./search-wiki.js"; + +export function createWikiSearchTool(options: { + cwd: string; + outputMode: OpenWikiOutputMode; +}): StructuredToolInterface { + const rootDir = resolveWikiRootFromOutputMode( + options.outputMode, + options.cwd, + ); + const virtualRoot = virtualRootForOutputMode(options.outputMode); + + return new DynamicStructuredTool({ + name: "openwiki_search_wiki", + description: + "Full-text search over the OpenWiki markdown wiki. Use this before broad ls/grep/read_file crawls when looking for existing wiki knowledge. Returns ranked hits with virtualPath, line, and snippet. Then read only the best matching pages.", + schema: { + type: "object", + properties: { + query: { + type: "string", + description: "Search query (keywords or a short phrase).", + }, + maxResults: { + type: "number", + description: "Maximum hits to return (default 20, max 100).", + }, + }, + required: ["query"], + additionalProperties: false, + } as const, + func: async (input) => { + const query = getStringInput(input, "query"); + const maxResults = getNumberInput(input, "maxResults") ?? undefined; + const hits = await searchWiki({ + rootDir, + query, + virtualRoot, + maxResults, + }); + + return JSON.stringify( + { + query, + rootDirNote: + options.outputMode === "local-wiki" + ? "Searched the personal brain wiki (virtual root /)." + : "Searched the repository openwiki/ docs (virtual root /openwiki/).", + hitCount: hits.length, + hits, + }, + null, + 2, + ); + }, + }); +} + +function getStringInput(input: unknown, key: string): string { + if (!isRecord(input) || typeof input[key] !== "string") { + throw new Error(`Missing string input: ${key}`); + } + return input[key]; +} + +function getNumberInput(input: unknown, key: string): number | null { + if (!isRecord(input) || input[key] === undefined) { + return null; + } + if (typeof input[key] !== "number") { + throw new Error(`Expected number input: ${key}`); + } + return input[key]; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} diff --git a/src/search/types.ts b/src/search/types.ts new file mode 100644 index 00000000..3b823272 --- /dev/null +++ b/src/search/types.ts @@ -0,0 +1,18 @@ +export type WikiSearchHit = { + /** Wiki-relative POSIX path, e.g. topics/deepagents-harness.md */ + path: string; + /** Virtual filesystem path for OpenWiki agent tools, e.g. /topics/... */ + virtualPath: string; + line: number; + snippet: string; + score: number; +}; + +export type WikiSearchOptions = { + rootDir: string; + query: string; + /** Prefix for agent-facing virtual paths. "/" for personal, "/openwiki/" for code. */ + virtualRoot?: string; + maxResults?: number; + maxFileBytes?: number; +}; diff --git a/test/search-wiki.test.ts b/test/search-wiki.test.ts new file mode 100644 index 00000000..a42409b1 --- /dev/null +++ b/test/search-wiki.test.ts @@ -0,0 +1,215 @@ +import { mkdtemp, mkdir, symlink, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, test } from "vitest"; +import { getHelpText, parseCommand } from "../src/commands.ts"; +import { searchWiki, tokenizeQuery } from "../src/search/search-wiki.ts"; +import { + resolveWikiRoot, + virtualRootForRunMode, +} from "../src/search/resolve-wiki-root.ts"; + +const tempDirs: string[] = []; + +afterEach(async () => { + // Best-effort cleanup; tests should not depend on leftover fixtures. + for (const dir of tempDirs.splice(0)) { + try { + const { rm } = await import("node:fs/promises"); + await rm(dir, { recursive: true, force: true }); + } catch { + // ignore + } + } +}); + +async function createFixtureWiki(): Promise { + const root = await mkdtemp(path.join(os.tmpdir(), "openwiki-search-")); + tempDirs.push(root); + await mkdir(path.join(root, "topics"), { recursive: true }); + await writeFile( + path.join(root, "topics", "deepagents-harness.md"), + [ + "# Deep Agents harness", + "", + "FilesystemMiddleware exposes ls and read_file tools.", + "SummarizationMiddleware compacts history.", + ].join("\n"), + "utf8", + ); + await writeFile( + path.join(root, "topics", "deepagents-backends.md"), + [ + "# Deep Agents backends", + "", + "FilesystemBackend stores files under root_dir.", + "StateBackend is ephemeral.", + ].join("\n"), + "utf8", + ); + await writeFile( + path.join(root, "quickstart.md"), + "# Quickstart\n\nStart with harness architecture.\n", + "utf8", + ); + return root; +} + +describe("tokenizeQuery", () => { + test("splits on punctuation and lowercases", () => { + expect(tokenizeQuery("Filesystem Backends!")).toEqual([ + "filesystem", + "backends", + ]); + }); +}); + +describe("searchWiki", () => { + test("returns ranked hits with path line and snippet", async () => { + const root = await createFixtureWiki(); + const hits = await searchWiki({ + rootDir: root, + query: "FilesystemMiddleware", + virtualRoot: "/", + }); + + expect(hits.length).toBeGreaterThan(0); + expect(hits[0]?.path).toBe("topics/deepagents-harness.md"); + expect(hits[0]?.virtualPath).toBe("/topics/deepagents-harness.md"); + expect(hits[0]?.snippet.toLowerCase()).toContain("filesystemmiddleware"); + expect(hits[0]?.line).toBeGreaterThan(0); + }); + + test("multi-term queries prefer lines covering more terms", async () => { + const root = await createFixtureWiki(); + const hits = await searchWiki({ + rootDir: root, + query: "filesystem backends", + virtualRoot: "/", + maxResults: 10, + }); + + expect(hits.some((hit) => hit.path.includes("backends"))).toBe(true); + expect(hits[0]?.score).toBeGreaterThan(0); + }); + + test("respects maxResults", async () => { + const root = await createFixtureWiki(); + const hits = await searchWiki({ + rootDir: root, + query: "Deep", + maxResults: 1, + }); + expect(hits).toHaveLength(1); + }); + + test("skips symlink files", async () => { + const root = await createFixtureWiki(); + const target = path.join(root, "topics", "deepagents-harness.md"); + const linkPath = path.join(root, "topics", "linked.md"); + await symlink(target, linkPath); + + const hits = await searchWiki({ + rootDir: root, + query: "FilesystemMiddleware", + }); + + expect(hits.every((hit) => hit.path !== "topics/linked.md")).toBe(true); + }); + + test("uses /openwiki virtual root for code mode", async () => { + const root = await createFixtureWiki(); + const hits = await searchWiki({ + rootDir: root, + query: "Quickstart", + virtualRoot: "/openwiki/", + }); + + expect(hits[0]?.virtualPath).toBe("/openwiki/quickstart.md"); + }); + + test("returns empty for blank query", async () => { + const root = await createFixtureWiki(); + expect(await searchWiki({ rootDir: root, query: " " })).toEqual([]); + }); +}); + +describe("resolveWikiRoot", () => { + test("personal mode uses ~/.openwiki/wiki", () => { + const root = resolveWikiRoot("personal", "/tmp/repo"); + expect(root.endsWith(`${path.sep}.openwiki${path.sep}wiki`)).toBe(true); + }); + + test("code mode uses ./openwiki under cwd", () => { + expect(resolveWikiRoot("code", "/tmp/repo")).toBe( + path.join("/tmp/repo", "openwiki"), + ); + }); + + test("virtual roots match mode", () => { + expect(virtualRootForRunMode("personal")).toBe("/"); + expect(virtualRootForRunMode("code")).toBe("/openwiki/"); + }); +}); + +describe("parseCommand — search", () => { + test("personal search parses query and mode", () => { + expect(parseCommand(["personal", "search", "middleware"])).toMatchObject({ + kind: "search", + mode: "personal", + query: "middleware", + limit: null, + }); + }); + + test("search with --mode and --limit", () => { + expect( + parseCommand([ + "search", + "--mode", + "code", + "--limit", + "5", + "filesystem", + "backends", + ]), + ).toMatchObject({ + kind: "search", + mode: "code", + query: "filesystem backends", + limit: 5, + }); + }); + + test("defaults bare search to personal mode", () => { + expect(parseCommand(["search", "middleware"])).toMatchObject({ + kind: "search", + mode: "personal", + query: "middleware", + }); + }); + + test("errors when query missing", () => { + expect(parseCommand(["personal", "search"])).toMatchObject({ + kind: "error", + exitCode: 1, + }); + }); + + test("errors on invalid limit", () => { + expect( + parseCommand(["search", "--limit", "0", "middleware"]), + ).toMatchObject({ + kind: "error", + exitCode: 1, + }); + }); + + test("help mentions personal search", () => { + const helpText = getHelpText(); + expect(helpText).toContain("openwiki personal search "); + expect(helpText).toContain( + "openwiki search [--mode personal|code] [--limit ] ", + ); + }); +});