diff --git a/AGENTS.md b/AGENTS.md index 959eadd5..5b345770 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,7 +14,13 @@ When working in this repository, read the OpenWiki quickstart first, then follow ## OpenWiki -This repository uses OpenWiki for recurring code documentation. Start with `openwiki/quickstart.md`, then follow its links to architecture, workflows, domain concepts, operations, integrations, testing guidance, and source maps. +This repository has a generated `openwiki/` evidence index. It is optional just-in-time context, not required startup reading. + +- If implementation ownership, behavioral invariants, analogous tests, or shipped surfaces are unclear, call `openwiki_retrieval.change_surface` once with the task before broad exploration. Inspect its cited source and tests directly; do not reread the returned wiki pages. +- Use `openwiki_retrieval.search` only for a concrete unresolved evidence gap. Reconsult when source contradicts the brief, work enters an uncited subsystem, or an unfamiliar failure reveals a missing contract. +- Treat source code and tests as authoritative. A brief's unknowns and review items are verification gaps, not automatic requirements. +- Before finishing a public or cross-package change, call `change_surface` with the task and the repository-relative changed paths. Verify relevant flagged exports, registration, generated surfaces, consumer paths, and focused tests. +- Prefer the narrowest quiet validation that proves the changed behavior. Preserve complete failure output. The scheduled OpenWiki GitHub Actions workflow refreshes the repository wiki. Do not hand-edit generated OpenWiki pages unless explicitly asked; prefer updating source code/docs and letting OpenWiki regenerate. diff --git a/CLAUDE.md b/CLAUDE.md index 959eadd5..5b345770 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -14,7 +14,13 @@ When working in this repository, read the OpenWiki quickstart first, then follow ## OpenWiki -This repository uses OpenWiki for recurring code documentation. Start with `openwiki/quickstart.md`, then follow its links to architecture, workflows, domain concepts, operations, integrations, testing guidance, and source maps. +This repository has a generated `openwiki/` evidence index. It is optional just-in-time context, not required startup reading. + +- If implementation ownership, behavioral invariants, analogous tests, or shipped surfaces are unclear, call `openwiki_retrieval.change_surface` once with the task before broad exploration. Inspect its cited source and tests directly; do not reread the returned wiki pages. +- Use `openwiki_retrieval.search` only for a concrete unresolved evidence gap. Reconsult when source contradicts the brief, work enters an uncited subsystem, or an unfamiliar failure reveals a missing contract. +- Treat source code and tests as authoritative. A brief's unknowns and review items are verification gaps, not automatic requirements. +- Before finishing a public or cross-package change, call `change_surface` with the task and the repository-relative changed paths. Verify relevant flagged exports, registration, generated surfaces, consumer paths, and focused tests. +- Prefer the narrowest quiet validation that proves the changed behavior. Preserve complete failure output. The scheduled OpenWiki GitHub Actions workflow refreshes the repository wiki. Do not hand-edit generated OpenWiki pages unless explicitly asked; prefer updating source code/docs and letting OpenWiki regenerate. diff --git a/README.md b/README.md index 0e42593e..bcfe15d4 100644 --- a/README.md +++ b/README.md @@ -171,7 +171,7 @@ Bare `openwiki` runs in code mode for the current repository. It creates initial Bare `openwiki --init` and `openwiki --update` default to code mode and operate on repository documentation. Use the `personal` positional mode or `--mode personal` to initialize or update the local personal brain wiki. -On each `code` run, `openwiki` maintains both an `AGENTS.md` and a `CLAUDE.md` at the repository root, adding prompting that instructs your coding agent to reference the wiki when searching for context. Each file is created if it does not already exist. If a file is present, OpenWiki only rewrites its own `…` block and leaves the rest of your content untouched (appending the block the first time). The scheduled GitHub Actions workflow includes these files, along with the workflow itself, in the documentation pull request. +On each `code` run, `openwiki` maintains both an `AGENTS.md` and a `CLAUDE.md` at the repository root, adding a just-in-time navigation workflow: start from the quickstart, search only relevant wiki pages, consult source maps before broad repository searches, and return to the wiki at subsystem or debugging boundaries. Source code and tests remain authoritative. Each file is created if it does not already exist. If a file is present, OpenWiki only rewrites its own `…` block and leaves the rest of your content untouched (appending the block the first time). The scheduled GitHub Actions workflow includes these files, along with the workflow itself, in the documentation pull request. Repository-specific wiki instructions are stored separately in `openwiki/INSTRUCTIONS.md`. This file is a shared, user-authored brief for the diff --git a/package.json b/package.json index dfd338ae..d5ec2b44 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,8 @@ "node": ">=22" }, "bin": { - "openwiki": "./dist/cli.js" + "openwiki": "./dist/cli.js", + "openwiki-retrieval-mcp": "./dist/retrieval/mcp-server.js" }, "repository": { "type": "git", diff --git a/src/agent/prompt.ts b/src/agent/prompt.ts index b58bc10a..599ced2b 100644 --- a/src/agent/prompt.ts +++ b/src/agent/prompt.ts @@ -58,11 +58,11 @@ Run discipline: - ${output.filesystemRootInstruction} - Never pass host absolute paths like /Users/... to filesystem tools; that creates nested paths inside the repo instead of touching the intended file. - Shell execute commands run on the host. If you use execute, run commands from the current runtime root unless a source-specific instruction explicitly tells you to inspect a connector raw file or configured local repository path. -- Do not exhaustively read every file. For a local knowledge wiki, inspect the existing wiki structure and only the relevant connector evidence or configured local repository paths. For an explicit repository source, inspect the repository tree, package/config files, README-style files, entrypoints, routing files, database/schema files, and representative files for each major domain. +- For a local knowledge wiki, do not exhaustively read every file; inspect the existing wiki structure and only the relevant connector evidence or configured local repository paths. ${discoveryHint} - Prefer grep/glob and short targeted reads over full-file reads when files are large. -- Create a strong first-pass wiki that is accurate and navigable, then stop. The wiki can be refined in later update runs. -- Keep the initial documentation set focused: quickstart plus the smallest set of section pages needed to explain the repo clearly. +- For an explicit repository source, inspect the repository tree, package and workspace manifests, README-style files, entrypoints, routing files, database/schema files, and representative implementation and test files for every important domain. +- Prioritize the most important, durable information. Keep individual pages concise and avoid redundant or low-signal detail, but do not use concision as a reason to omit important domains, independent components, or relationships. - ${output.searchBoundaryInstruction} ${createOpenWikiIgnoreInstructions(openWikiIgnore)} @@ -94,16 +94,19 @@ ${output.wikiFirstAnsweringInstruction} - When you do inspect raw data, keep reads narrow: list latest raw items for the relevant connector, open only the specific files needed, and summarize only the minimum evidence required to answer or update the wiki. Subagent discipline: -- You may use the task tool to parallelize read-only research during init and update runs when the repository has multiple substantial domains. -- Default to 1-2 subagents for large or unfamiliar repositories. Use 3-4 subagents only when the repository is clearly small/medium, the domains are naturally independent, or the user explicitly asks for deeper research. -- Subagents must only inspect and summarize. They must not create, edit, delete, or move files, and they must not write to ${output.docsLocation}. -- Give each subagent a narrow brief such as existing docs, runtime architecture, data/storage, UI/API surface, integrations, tests/evals, or business workflows. -- Ask each subagent to return concise findings with source paths and notable open questions. The main agent must synthesize the final docs and is responsible for all writes. -- Treat subagent reports as internal discovery notes. Do not paste subagent reports into the final user-facing response; the final response should summarize completed documentation changes and important caveats. +- Use the task tool when independent repository areas or cross-cutting concerns can be investigated or documented in parallel. Choose the number and sequence of subagents from the repository's discovered complexity rather than a preset limit. +- In a monorepo, consider assigning a scoped subagent to each substantial service, package, application, or workspace. Closely related small units may share a subagent when that produces a clearer domain boundary. +- Delegation is iterative, not one-and-done. After the first reports or drafts return, reassess coverage and spawn additional subagents for newly discovered components, cross-package workflows, shared contracts, contradictions, or evidence gaps. +- Give each subagent a narrow brief such as one service/package/workspace, existing docs, runtime architecture, data/storage, UI/API surface, integrations, tests/evals, or a cross-component business workflow. +- Subagents may inspect and summarize, or may draft/write explicitly assigned documentation pages when that improves throughput. Any delegated writes must stay inside ${output.docsLocation}, use non-overlapping page ownership, and follow the same source-grounding and security rules as the main agent. Never have parallel subagents edit the same file. +- Ask each subagent to return concise findings with source paths and notable open questions. The main agent is responsible for the final synthesized documentation state, including delegated writes. +- The main agent must review delegated pages, reconcile terminology and duplicated content, add cross-component context, and verify navigation and relationship links before finishing. +- Treat subagent reports as internal discovery notes. Do not paste reports into the final user-facing response; summarize completed documentation changes and important caveats. Planning discipline: -- After discovery and before writing final documentation, create a temporary ${output.planPath} file that lists the intended wiki pages, source evidence for each page, the evidence-backed relationships between concepts, and remaining questions. +- After discovery and before writing final documentation, create a temporary ${output.planPath} file that inventories the important domains and independent components, lists the intended wiki pages and source evidence for each page, records whether each area is documented, covered by another page, or deferred, and captures remaining questions. - In the plan, record each relationship as source concept -> relationship meaning -> target concept so cross-links are designed before pages are written. +- Revisit the plan after initial subagent findings. Expand or reorganize it when discovery reveals additional services, packages, workspaces, workflows, or cross-component relationships. - Use ${output.planPath} when writing this temporary plan with filesystem tools. - The temporary ${output.planPath} is removed automatically after the run, so you do not need to delete it. Do not treat it as a wiki concept or link to it from other pages. @@ -150,13 +153,16 @@ Documentation goals: - Prefer clear Markdown with stable links between pages. - Organize the docs like human documentation, not a raw file inventory. - Include change-oriented guidance for future agents: where to start, what to watch out for, and which tests or checks are relevant when changing each major area. -- Keep the docs concise enough to maintain. Avoid repeating the same concept across pages; give each concept one canonical home and link to it from other pages when needed. +- Keep each page concise, specific, and centered on important information. Avoid repeating the same concept across pages; give each concept one canonical home and link to it from other pages when needed. Concision should reduce redundancy and verbosity, not repository coverage. - Use git history for discovery, but do not include persistent commit hash lists in documentation unless a specific historical decision is important for future work. +${createCodingAgentUtilityRequirements(outputMode, output)} + OKF relationship modeling: - Treat every non-reserved Markdown document as a concept node. Standard Markdown links between concept documents are directed relationship edges; tags, resource fields, directory placement, source-code references, and index.md links do not replace concept-to-concept links. - Model meaningful runtime, dependency, ownership, data-flow, security, lifecycle, and user-flow relationships, not only navigation from ${output.quickstartPath}. - Put a concept link in the sentence that explains the relationship. Use the surrounding prose to state its meaning, such as \`dispatches to\`, \`depends on\`, \`shares infrastructure with\`, \`is configured through\`, \`is surfaced by\`, or \`is secured by\`. +- When separate pages document services, packages, or workspaces that interact, link them at the point where the runtime call, dependency, shared data, ownership boundary, lifecycle, or contract is explained. Add links from both pages when the relationship is important to understanding each side. - Do not add links solely to increase graph density, and do not automatically add reciprocal links. Add an inverse link only when it helps explain the target concept and is supported by evidence. - ${output.quickstartPath} must link to every major concept for navigation, but quickstart and index links do not count toward the semantic relationship audit. - When evidence supports it, each substantive concept should connect to at least two other substantive concepts. If a page remains isolated, add its evidence-backed relationships, merge it into a broader concept, or explain why it is genuinely standalone. @@ -186,6 +192,21 @@ timestamp: - Produce valid YAML. Do not leave placeholder text or explanatory comments in written files. - Preserve all existing producer-defined front matter fields when updating a concept. Unknown extension fields are valid OKF and must survive round trips. Change metadata only when the underlying fact or meaningful content changes. - The description field is especially useful for retrieval tools. When present, make it clear, detailed, and optimized for search. +- In repository mode, use the optional namespaced \`openwiki\` producer extension when source evidence supports it. Keep values concise and omit empty keys: + + +openwiki: + roles: [architecture, domain] # One or more of architecture, delivery, domain, integration, operations, repository, testing, workflow + change_kinds: [lifecycle, public-api] # Short kebab-case routing facets + source_paths: [path/to/canonical-source.ts] + symbols: [PublicSymbol, owningInternalSymbol] + test_paths: [path/to/focused.test.ts] + invariants: [A concise externally observable contract.] + validation_commands: [the narrowest non-destructive check] + + +- Use \`type\` as a free-form human concept kind. Use \`openwiki.roles\` for stable retrieval roles and \`tags\` for specific domain facets; do not use generic shared tags as a substitute for explicit concept links. +- Treat \`source_paths\`, \`test_paths\`, invariants, and validation commands as evidence-backed routing metadata, not exhaustive requirements. Never place secrets, credentials, or commands that expose them in metadata. - When updating an existing Markdown concept, preserve accurate body content and correct its opening front matter only when needed for compliance or accuracy. - OpenWiki repairs front matter deterministically after every run, so a page is never rejected for missing or invalid front matter. If a page's front matter contains \`openwiki_generated: true\`, that metadata was code-derived as a fallback: replace it with an accurate \`type\`, \`title\`, and \`description\` grounded in the page body, then remove the \`openwiki_generated\` field. - If a page's front matter contains an \`openwiki_translation_pending\` field, ignore it: it is a translation-system marker that OpenWiki manages automatically. Do not add, edit, remove, or act on it. @@ -193,19 +214,21 @@ timestamp: Section quality rules: - Do not create a directory unless it represents a real documentation area. - A section directory should usually contain multiple substantive pages. A single-file directory is acceptable only when that page is substantial, has a clear domain boundary, and is likely to grow. -- Avoid thin pages. If a page would mostly be a stub, source map, or short note, merge it into ${output.quickstartPath} or a broader section page instead. -- Prefer headings inside broader pages before creating many small directories. - Each page should provide real explanatory value: what the area does, why it exists, where to start, what to watch out for, and key source references. -- Before finishing an init or update run, review the ${output.docsLocation} tree. Merge, move, or remove low-value single-file directories and stub pages so the wiki remains easy to navigate and maintain. -- For small scopes with about 10 or fewer primary source items, prefer ${output.quickstartPath} plus at most 1-2 supporting pages. Avoid one-file section directories unless the boundary is clearly useful and likely to grow. -- Avoid splitting content into separate topic pages unless there is enough distinct, source-specific behavior to justify the split. +- Before finishing an init or update run, review the ${output.docsLocation} tree. Remove low-value stubs and redundant content while preserving useful coverage of independent components and important relationships. + +Repository decomposition and coverage: +- For repository sources, identify independent services, applications, packages, libraries, and workspaces from manifests, build configuration, entrypoints, and directory boundaries before choosing the documentation structure. +- Give each substantial independent component its own page or clearly identifiable section when it has distinct responsibilities, runtime behavior, APIs, data ownership, dependencies, operational guidance, or tests. Closely coupled or very small components may share a page when their relationship is explained clearly. +- In a monorepo, organize service/package/workspace documentation so readers can navigate both by component and by cross-component workflow. Wiki breadth should reflect meaningful repository boundaries and complexity; do not force repositories of different sizes into a predetermined page count. +- Document the important responsibilities, interfaces, dependencies, data flows, operational constraints, extension points, and change-safety guidance for each component. Do not turn the wiki into a file-by-file inventory. Required documentation structure: - ${output.quickstartPath} must be the entrypoint. - ${output.quickstartPath} must include a high-level overview and links to every major section. - When writing required documentation with filesystem tools or narrow shell execute, use ${output.writePathExample}. - ${output.sectionDirectoryInstruction} -- Each section directory should contain focused Markdown pages; if a directory would contain only one short page, prefer a broader page or a heading in ${output.quickstartPath}. +- Each section directory should contain focused Markdown pages whose boundaries follow the repository's actual components and domains. - Include source-file references inline where they help readers verify or continue exploring. - Source Map sections are optional. Add one only when it materially improves navigation for that page. Prefer inline source references for short pages. - Track the last successful documentation update in ${output.metadataPath}. @@ -269,6 +292,31 @@ function createOpenWikiIgnoreInstructions( ${patterns}`; } +function createCodingAgentUtilityRequirements( + outputMode: OpenWikiOutputMode, + output: OutputPromptConfig, +): string { + if (outputMode !== "repository") { + return ""; + } + + return `Coding-agent utility requirements: +- Optimize the repository wiki to reduce exploratory source searches during future code changes. It must help an agent identify where to start, which invariants matter, and how to validate narrowly; it must not attempt to anticipate or encode a specific future task. +- ${output.quickstartPath} must contain a compact task-routing table with columns for change area or user intent, relevant wiki page, exact source entry points, important symbols or types, focused tests, and the minimal validation command. Route broad change categories supported by repository evidence, not hypothetical one-off features. +- Every substantive architecture, domain, runtime, workflow, integration, or operations page must make change navigation explicit when applicable: when to consult the page; runtime invariants and lifecycle ordering; extension points; exact source files and important symbols; focused tests; minimal validation commands; and scope boundaries such as generated files or broader checks that are normally unnecessary. +- Prefer symbol-level mappings such as Concept -> Public API -> Implementation -> Tests. Do not merely list directories. Explain why each path or symbol matters and what behavior it owns. Avoid stale line-number references; prefer stable paths and symbol names. +- Document evidence-backed change recipes for recurring extension seams discovered in source or recent history, such as adding a query/modifier, extending a domain abstraction, changing lifecycle behavior, adding persistence/serialization, or updating a public export. Each recipe should identify implementation seams, affected caches or lifecycle hooks, focused tests, likely non-goals, and escalation conditions. +- For every public or cross-package extension seam, document the complete change surface: implementation symbols; internal barrel exports; package or public entrypoints; generated, bundled, or publish mirrors; initialization, registration, or factory wiring; the consumer import path; focused internal tests; and consumer/package tests. Omit a layer only when repository evidence shows it does not exist. +- Make the distinction between internal correctness and shipped-surface correctness explicit. A new API is not complete merely because its defining module typechecks or its unit tests pass; future agents must be able to verify that the API resolves from the import path real consumers use and that required registration or generated artifacts are present. +- Separate ordinary focused checks from expensive integration, root-test, release, package-build, generated-artifact, and performance checks. Label expensive checks as conditional and state the source-backed condition that makes each one necessary. Do not encourage broad validation by default. +- When a change crosses a public, package, generated-artifact, or runtime-registration boundary, identify the narrowest consumer-facing smoke test or package validation command that exercises that boundary. Record any source-backed synchronization command and the canonical source of generated files so agents do not validate only an internal package or hand-edit derived output. +- For stateful or lifecycle extension seams, document a source-backed behavioral test matrix when applicable: initial state; false-to-true and true-to-false transitions; unchanged updates; missing prerequisites; isolation between independent instances and tracker identity; reset, reuse, and observation-window boundaries; deferred or re-entrant mutation including net/coalesced effects; and composition between static and temporal constraints. Record constructor or composition invariants when they are externally observable. Link each invariant to the narrowest existing test or test location so future agents can turn every acceptance criterion into a focused check. +- Make analogous tests retrievable by describing the behavior and invariant they exercise, not just the implementation symbol. When large test files cover multiple lifecycle phases, identify the relevant suite or stable test names so a future \`search\` call scoped to \`tests\` can reach the right section without reading from the top. +- Keep validation commands narrow and quiet by default. Identify flags or focused commands that suppress successful output while preserving complete failure diagnostics; do not make agents consume verbose build logs merely to confirm success. +- Keep navigation stable and concise: use one canonical home per concept, link to it instead of duplicating prose, and keep operational/release guidance out of runtime reading paths unless it is genuinely required. +- Before finishing, simulate navigation for representative adjacent changes grounded in the repository's actual components and history. Verify that a future agent can reach the first implementation files, important symbols/invariants, focused tests, and minimal validation command from the quickstart without a repository-wide search. Repair navigation gaps found by this audit.`; +} + export function createModeInstructions( command: OpenWikiCommand, outputMode: OpenWikiOutputMode = "local-wiki", @@ -294,8 +342,7 @@ export function createModeInstructions( - ${output.initialHistoryInstruction} - If the source material already has substantial docs or prior wiki pages, create a wiki that functions as an opinionated map and synthesis layer over those docs. - Create ${output.quickstartPath} first, then the linked section pages. -- Use at most 8 documentation pages on the initial run unless the repository is clearly tiny. -- Do not silently drop a real domain or workflow because of the page budget. If it is not fully documented, record it in the \`## Backlog\` section of ${output.quickstartPath} with its area name, source anchor, and a one-line reason. +- Do not silently drop a real domain, independent component, or workflow. Document it at the appropriate level or record it in the \`## Backlog\` section of ${output.quickstartPath} with its area name, source anchor, and a one-line reason. - Do not try to document every source file. Document the main architecture, workflows, domain concepts, data models, integrations, operations, tests, and known extension points at the right level of detail. - The CLI will record successful run metadata in ${output.metadataPath} after you finish. `.trim(); @@ -309,16 +356,16 @@ export function createModeInstructions( - If source-specific connector raw data paths are supplied, inspect those files and update the wiki from that local evidence. Do not run all connector ingestions from inside the agent. - ${output.updateEvidenceInstruction} - Before editing, build a docs impact plan from the changed source files: source change -> docs affected -> edit needed -> why. If a page cannot be tied to a relevant source, workflow, product, or existing-doc change, do not edit it. -- Update runs must be surgical. Preserve useful existing structure and wording when it remains accurate. Prefer replacing one stale sentence over adding new paragraphs. -- Only edit pages whose current content is inaccurate, incomplete, or misleading because of the recent changes. Do not refresh every page. +- Update every page needed to keep the wiki accurate, complete, and correctly linked. There is no preset limit on the number of pages or sections an update may change or add. +- Preserve useful existing structure and wording when it remains accurate, and avoid unrelated formatting or prose churn. +- Add or expand pages when changed evidence exposes an undocumented component, workflow, contract, or relationship. An update may improve incomplete coverage discovered during the run even when that work spans multiple pages. - Keep each concept in one canonical page. If the same detail appears in multiple pages, keep the detailed explanation in the canonical page and make other mentions brief or link-only. - Do not make formatting-only edits. Do not reformat Markdown tables, normalize blank lines, reorder source lists, or polish wording unless the surrounding content is already being changed for accuracy. - When updating a page that documents a runtime flow, lifecycle, or data model but has no diagram, adding one is a valuable improvement, not a formatting-only change. Add it opportunistically when you are already editing that area or have spare diff budget, following the diagram discipline above. - Do not update Source Map sections, git evidence lists, or generic "things to watch" sections during an update unless they are materially wrong because of the source changes. - Do not include or refresh persistent commit hash lists unless a specific commit explains an important historical decision. -- Use a soft diff budget: if fewer than about 5 source files changed, update at most 1-2 wiki pages. Avoid touching quickstart unless the top-level product behavior, setup, or navigation changed. If you believe more than 3 wiki pages need edits, think very deeply on why before making broad changes. - Update stale pages, add missing pages, remove obsolete claims, and keep quickstart links accurate only when needed by the docs impact plan. -- Promote a backlog entry when recent changes touch that area or the update has spare documentation budget, then document the area and remove the entry from the backlog. +- Promote backlog entries whenever the available evidence is sufficient to document them accurately, then remove the completed entries from the backlog. - Do not let the backlog grow silently: every identified area must remain either documented or represented by a concise backlog entry with a source anchor and reason. - Updates may be a no-op. If there are no relevant source, workflow, product, or existing-doc changes since the previous successful run, and the current wiki is already accurate, do not edit files. Say that the wiki is already current. - The CLI will record successful run metadata in ${output.metadataPath} after you finish. @@ -360,7 +407,7 @@ ${context.gitSummary} ` Update the existing OpenWiki documentation for ${output.subjectLabel}. -Inspect ${output.docsLocation}, identify recent source changes or newly ingested connector evidence, and refresh only the documentation pages directly affected by those changes. Use the git evidence below when available. Keep edits surgical: do not rewrite accurate sections, do not update source maps or git evidence just to refresh them, and do not make formatting-only changes. If the wiki is already current, do not edit files. The CLI will update ${output.metadataPath} only when OpenWiki content changes. +Inspect ${output.docsLocation}, identify recent source changes or newly ingested connector evidence, and update every documentation page needed to keep the wiki accurate, complete, and correctly linked. Use the git evidence below when available. Preserve unrelated accurate content and avoid formatting-only changes. If the wiki is already current, do not edit files. The CLI will update ${output.metadataPath} only when OpenWiki content changes. Last update metadata: ${formatLastUpdate(context.lastUpdate)} diff --git a/src/code-mode.ts b/src/code-mode.ts index e8350e7d..9a1b80d8 100644 --- a/src/code-mode.ts +++ b/src/code-mode.ts @@ -268,7 +268,13 @@ function createCodeModeAgentsSnippet(): string { ## OpenWiki -This repository uses OpenWiki for recurring code documentation. Start with \`openwiki/quickstart.md\`, then follow its links to architecture, workflows, domain concepts, operations, integrations, testing guidance, and source maps. +This repository has a generated \`openwiki/\` evidence index. It is optional just-in-time context, not required startup reading. + +- If implementation ownership, behavioral invariants, analogous tests, or shipped surfaces are unclear, call \`openwiki_retrieval.change_surface\` once with the task before broad exploration. Inspect its cited source and tests directly; do not reread the returned wiki pages. +- Use \`openwiki_retrieval.search\` only for a concrete unresolved evidence gap. Reconsult when source contradicts the brief, work enters an uncited subsystem, or an unfamiliar failure reveals a missing contract. +- Treat source code and tests as authoritative. A brief's unknowns and review items are verification gaps, not automatic requirements. +- Before finishing a public or cross-package change, call \`change_surface\` with the task and the repository-relative changed paths. Verify relevant flagged exports, registration, generated surfaces, consumer paths, and focused tests. +- Prefer the narrowest quiet validation that proves the changed behavior. Preserve complete failure output. The scheduled OpenWiki GitHub Actions workflow refreshes the repository wiki. Do not hand-edit generated OpenWiki pages unless explicitly asked; prefer updating source code/docs and letting OpenWiki regenerate. diff --git a/src/retrieval/mcp-server.ts b/src/retrieval/mcp-server.ts new file mode 100644 index 00000000..2e504dca --- /dev/null +++ b/src/retrieval/mcp-server.ts @@ -0,0 +1,182 @@ +#!/usr/bin/env node + +import { createInterface } from "node:readline"; +import { OPENWIKI_VERSION } from "../constants.js"; +import { RETRIEVAL_TOOL_DEFINITIONS } from "./mcp-tools.js"; +import { RetrievalService } from "./search-service.js"; +import type { EmbeddingProvider } from "./semantic.js"; +import type { SearchScope } from "./types.js"; + +interface JsonRpcRequest { + id?: number | string; + jsonrpc?: string; + method?: string; + params?: Record; +} + +const options = parseOptions(process.argv.slice(2)); +const service = new RetrievalService(options); +const input = createInterface({ input: process.stdin, crlfDelay: Infinity }); + +input.on("line", (line) => { + void handleLine(line); +}); + +async function handleLine(line: string): Promise { + let request: JsonRpcRequest; + try { + request = JSON.parse(line) as JsonRpcRequest; + } catch { + writeError(null, -32700, "Invalid JSON-RPC message."); + return; + } + if (request.jsonrpc !== "2.0" || typeof request.method !== "string") { + writeError(request.id ?? null, -32600, "Invalid JSON-RPC request."); + return; + } + if (request.id === undefined) return; + try { + switch (request.method) { + case "initialize": + writeResult(request.id, { + capabilities: { tools: { listChanged: false } }, + instructions: + "Use change_surface when ownership, invariants, analogous tests, or shipped surfaces are unclear. Inspect its citations directly instead of rereading wiki pages. Reuse it with changed_paths for a final cross-surface review. Use search only for a concrete unresolved evidence gap.", + protocolVersion: "2025-06-18", + serverInfo: { name: "openwiki-retrieval", version: OPENWIKI_VERSION }, + }); + return; + case "ping": + writeResult(request.id, {}); + return; + case "tools/list": + writeResult(request.id, { tools: RETRIEVAL_TOOL_DEFINITIONS }); + return; + case "tools/call": + await callTool(request.id, request.params ?? {}); + return; + default: + writeError(request.id, -32601, "Method not found."); + } + } catch (error) { + const message = + error instanceof Error ? error.message : "Retrieval failed."; + writeResult(request.id, { + content: [{ text: message.slice(0, 500), type: "text" }], + isError: true, + }); + } +} + +async function callTool( + id: number | string, + params: Record, +): Promise { + const name = typeof params.name === "string" ? params.name : ""; + const args = isRecord(params.arguments) ? params.arguments : {}; + const limit = optionalInteger(args.limit, 8); + let result: unknown; + switch (name) { + case "search": + result = await service.search( + requiredString(args.query, "query"), + optionalScope(args.scope), + limit, + ); + break; + case "change_surface": + result = await service.changeSurface( + requiredString(args.query, "query"), + optionalInteger(args.limit, 6), + optionalStringArray(args.changed_paths, "changed_paths"), + ); + break; + default: + throw new Error(`Unknown retrieval tool: ${name || "(missing)"}.`); + } + writeResult(id, { + content: [{ text: JSON.stringify(result), type: "text" }], + }); +} + +function parseOptions(args: string[]): { + embeddingProvider: EmbeddingProvider; + repoRoot: string; + wikiRoot: string; +} { + const values = new Map(); + for (let index = 0; index < args.length; index += 2) { + const flag = args[index]; + const value = args[index + 1]; + if (!flag?.startsWith("--") || !value || value.startsWith("--")) { + throw new Error( + "Expected --repo-root, --wiki-root, and optional --embedding-provider values.", + ); + } + values.set(flag, value); + } + const provider = values.get("--embedding-provider") ?? "local"; + if (provider !== "local" && provider !== "openai") { + throw new Error("embedding provider must be local or openai."); + } + return { + embeddingProvider: provider, + repoRoot: values.get("--repo-root") ?? process.cwd(), + wikiRoot: values.get("--wiki-root") ?? `${process.cwd()}/openwiki`, + }; +} + +function requiredString(value: unknown, name: string): string { + if (typeof value !== "string" || !value.trim()) { + throw new Error(`${name} is required.`); + } + return value; +} + +function optionalInteger(value: unknown, fallback: number): number { + return typeof value === "number" && Number.isInteger(value) + ? value + : fallback; +} + +function optionalScope(value: unknown): SearchScope { + return value === "source_code" || + value === "tests" || + value === "wiki" || + value === "all" + ? value + : "all"; +} + +function optionalStringArray(value: unknown, name: string): string[] { + if (value === undefined) return []; + if (!Array.isArray(value)) { + throw new Error(`${name} must be an array of strings.`); + } + const result: string[] = []; + for (const item of value as unknown[]) { + if (typeof item !== "string") { + throw new Error(`${name} must be an array of strings.`); + } + result.push(item); + } + return result; +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function writeResult(id: number | string, result: unknown): void { + process.stdout.write(`${JSON.stringify({ id, jsonrpc: "2.0", result })}\n`); +} + +function writeError( + id: number | string | null, + code: number, + message: string, +): void { + process.stdout.write( + `${JSON.stringify({ error: { code, message }, id, jsonrpc: "2.0" })}\n`, + ); +} diff --git a/src/retrieval/mcp-tools.ts b/src/retrieval/mcp-tools.ts new file mode 100644 index 00000000..641dfa39 --- /dev/null +++ b/src/retrieval/mcp-tools.ts @@ -0,0 +1,72 @@ +interface ToolDefinition { + annotations: { destructiveHint: false; readOnlyHint: true }; + description: string; + inputSchema: object; + name: string; +} + +export const SEARCH_SCOPES = ["all", "wiki", "source_code", "tests"] as const; + +function integerSchema( + minimum: number, + maximum: number, + defaultValue: number, +): object { + return { default: defaultValue, maximum, minimum, type: "integer" }; +} + +function querySchema(properties: Record): object { + return { + additionalProperties: false, + properties: { + query: { maxLength: 500, minLength: 1, type: "string" }, + ...properties, + }, + required: ["query"], + type: "object", + }; +} + +function tool( + name: string, + description: string, + inputSchema: object, +): ToolDefinition { + return { + annotations: { destructiveHint: false, readOnlyHint: true }, + description, + inputSchema, + name, + }; +} + +export const RETRIEVAL_TOOL_DEFINITIONS = [ + tool( + "search", + "Retrieve focused wiki guidance, implementation evidence, or analogous tests with automatic lexical, semantic, and OKF ranking. Use wiki for contracts and invariants; verify every citation in source.", + querySchema({ + limit: integerSchema(1, 10, 5), + scope: { + default: "all", + description: + "Search all indexed content, only generated wiki pages, implementation source excluding tests, or only test/spec files.", + enum: SEARCH_SCOPES, + type: "string", + }, + }), + ), + tool( + "change_surface", + "Build a compact, evidence-backed task brief before broad exploration: likely owners, invariants, analogous tests, conditional delivery surfaces, and narrow validation. Pass changed_paths later to review documented adjacent surfaces.", + querySchema({ + changed_paths: { + description: + "Optional repository-relative paths already changed. When present, the response flags documented adjacent surfaces to verify; flags are evidence gaps, not automatic requirements.", + items: { maxLength: 300, minLength: 1, type: "string" }, + maxItems: 50, + type: "array", + }, + limit: integerSchema(1, 8, 6), + }), + ), +] as const satisfies readonly ToolDefinition[]; diff --git a/src/retrieval/ranking.ts b/src/retrieval/ranking.ts new file mode 100644 index 00000000..edf7e175 --- /dev/null +++ b/src/retrieval/ranking.ts @@ -0,0 +1,293 @@ +import type { IndexedChunk, RankedHit } from "./types.js"; + +const STOP_WORDS = new Set([ + "a", + "an", + "and", + "are", + "as", + "at", + "be", + "by", + "for", + "from", + "how", + "in", + "is", + "it", + "of", + "on", + "or", + "that", + "the", + "this", + "to", + "with", +]); + +const SYNONYM_GROUPS = [ + [ + "api", + "consumer", + "entrypoint", + "export", + "package", + "public", + "publish", + "surface", + ], + ["build", "bundle", "copy", "dist", "generated", "mirror", "release", "sync"], + ["factory", "initialize", "install", "register", "registry", "setup", "wire"], + ["assert", "check", "spec", "test", "validate", "verify"], + ["defer", "buffer", "batch", "command", "flush", "queue"], + ["predicate", "filter", "query", "select", "where"], + ["relation", "edge", "link", "pair", "target"], + ["aspect", "composite", "trait", "mixin", "schema"], + ["diff", "restore", "rollback", "snapshot", "state"], + ["initial", "baseline", "empty", "first", "setup"], + ["add", "added", "enter", "gain", "insert", "true"], + ["remove", "removed", "exit", "lose", "delete", "false"], + ["change", "changed", "mutate", "transition", "update"], + ["unchanged", "noop", "idempotent", "stable"], + ["independent", "isolation", "instance", "tracker"], + ["reset", "reuse", "window", "observation", "generation"], + ["defer", "reentrant", "coalesce", "net", "flush"], + ["compose", "composition", "combine", "mixed"], +] as const; + +const SYNONYMS = buildSynonyms(); + +export function tokenize(value: string): string[] { + const separated = value + .replace(/[_-]+/gu, " ") + .replace(/([a-z0-9])([A-Z])/gu, "$1 $2") + .replace(/([A-Z]+)([A-Z][a-z])/gu, "$1 $2") + .toLowerCase(); + const terms = separated.match(/[a-z0-9]+/gu) ?? []; + return terms + .map((term) => stem(term)) + .filter((term) => term.length > 1 && !STOP_WORDS.has(term)); +} + +export function expandQueryTerms(query: string): string[] { + const base = tokenize(query); + const expanded = new Set(base); + for (const term of base) { + for (const synonym of SYNONYMS.get(term) ?? []) expanded.add(synonym); + } + return [...expanded]; +} + +export function rankKeyword( + chunks: IndexedChunk[], + query: string, +): RankedHit[] { + const phrase = query.trim().toLowerCase(); + const queryTerms = expandQueryTerms(query); + return chunks + .map((chunk) => { + const path = chunk.path.toLowerCase(); + const title = `${chunk.title ?? ""} ${chunk.heading ?? ""}`.toLowerCase(); + const metadata = chunk.fields.toLowerCase(); + const text = chunk.text.toLowerCase(); + const pathTerms = new Set(tokenize(path)); + const titleTerms = new Set(tokenize(title)); + const metadataTerms = new Set(tokenize(metadata)); + const textTerms = new Set(tokenize(text)); + let score = 0; + if (phrase) { + if (path.includes(phrase)) score += 10; + if (title.includes(phrase)) score += 9; + if (metadata.includes(phrase)) score += 7; + if (text.includes(phrase)) score += 5; + } + for (const term of queryTerms) { + if (path.includes(term) || pathTerms.has(term)) score += 3.5; + if (title.includes(term) || titleTerms.has(term)) score += 3; + if (metadata.includes(term) || metadataTerms.has(term)) score += 2; + if (text.includes(term) || textTerms.has(term)) score += 1; + } + return { chunk, score }; + }) + .filter((hit) => hit.score > 0) + .sort(compareHits); +} + +export function rankBm25(chunks: IndexedChunk[], query: string): RankedHit[] { + const queryTerms = expandQueryTerms(query); + if (queryTerms.length === 0 || chunks.length === 0) return []; + const documents = chunks.map((chunk) => tokenize(searchableText(chunk))); + const averageLength = + documents.reduce((sum, terms) => sum + terms.length, 0) / + documents.length || 1; + const documentFrequency = new Map(); + for (const terms of documents) { + for (const term of new Set(terms)) { + documentFrequency.set(term, (documentFrequency.get(term) ?? 0) + 1); + } + } + const k1 = 1.5; + const b = 0.75; + return chunks + .map((chunk, index) => { + const terms = documents[index] ?? []; + const frequencies = frequenciesOf(terms); + let score = 0; + for (const term of queryTerms) { + const frequency = frequencies.get(term) ?? 0; + if (frequency === 0) continue; + const df = documentFrequency.get(term) ?? 0; + const idf = Math.log(1 + (chunks.length - df + 0.5) / (df + 0.5)); + const denominator = + frequency + k1 * (1 - b + b * (terms.length / averageLength)); + score += idf * ((frequency * (k1 + 1)) / denominator); + } + if ( + chunk.title && + queryTerms.some((term) => tokenize(chunk.title ?? "").includes(term)) + ) { + score *= 1.35; + } + if (queryTerms.some((term) => tokenize(chunk.path).includes(term))) + score *= 1.2; + return { chunk, score }; + }) + .filter((hit) => hit.score > 0) + .sort(compareHits); +} + +export function rankLocalVectors( + chunks: IndexedChunk[], + query: string, +): RankedHit[] { + const queryVector = vectorize(query); + return chunks + .map((chunk) => ({ + chunk, + score: cosine(queryVector, vectorize(searchableText(chunk))), + })) + .filter((hit) => hit.score > 0) + .sort(compareHits); +} + +export function reciprocalRankFusion( + rankedLists: { hits: RankedHit[]; name: string; weight: number }[], + k = 60, +): RankedHit[] { + const fused = new Map(); + for (const list of rankedLists) { + list.hits.forEach((hit, index) => { + const contribution = list.weight / (k + index + 1); + const existing = fused.get(hit.chunk.id) ?? { + chunk: hit.chunk, + score: 0, + signals: {}, + }; + existing.score += contribution; + existing.signals = { + ...(existing.signals ?? {}), + [list.name]: hit.score, + }; + fused.set(hit.chunk.id, existing); + }); + } + return [...fused.values()].sort(compareHits); +} + +export function searchableText(chunk: IndexedChunk): string { + return [ + chunk.path, + chunk.title, + chunk.heading, + chunk.description, + chunk.type, + chunk.roles.join(" "), + chunk.tags.join(" "), + chunk.resource, + chunk.fields, + chunk.text, + ] + .filter(Boolean) + .join("\n"); +} + +function vectorize(value: string, dimensions = 768): Float64Array { + const vector = new Float64Array(dimensions); + const terms = expandTermsForVector(value); + for (const term of terms) { + const index = fnv1a(term) % dimensions; + const sign = (fnv1a(`sign:${term}`) & 1) === 0 ? 1 : -1; + vector[index] += sign; + } + return vector; +} + +function expandTermsForVector(value: string): string[] { + const terms = tokenize(value); + const expanded = [...terms]; + for (const term of terms) { + for (const synonym of SYNONYMS.get(term) ?? []) expanded.push(synonym); + } + for (let index = 0; index + 1 < terms.length; index += 1) { + expanded.push(`${terms[index]}:${terms[index + 1]}`); + } + return expanded; +} + +function cosine(left: Float64Array, right: Float64Array): number { + let dot = 0; + let leftNorm = 0; + let rightNorm = 0; + for (let index = 0; index < left.length; index += 1) { + const l = left[index] ?? 0; + const r = right[index] ?? 0; + dot += l * r; + leftNorm += l * l; + rightNorm += r * r; + } + return leftNorm && rightNorm ? dot / Math.sqrt(leftNorm * rightNorm) : 0; +} + +function frequenciesOf(terms: string[]): Map { + const frequencies = new Map(); + for (const term of terms) + frequencies.set(term, (frequencies.get(term) ?? 0) + 1); + return frequencies; +} + +function compareHits(left: RankedHit, right: RankedHit): number { + return ( + right.score - left.score || left.chunk.path.localeCompare(right.chunk.path) + ); +} + +function stem(term: string): string { + if (term.length > 5 && term.endsWith("ing")) return term.slice(0, -3); + if (term.length > 4 && term.endsWith("ed")) return term.slice(0, -2); + if (term.length > 4 && term.endsWith("es")) return term.slice(0, -2); + if (term.length > 3 && term.endsWith("s")) return term.slice(0, -1); + return term; +} + +function buildSynonyms(): Map { + const result = new Map(); + for (const group of SYNONYM_GROUPS) { + const normalized = group.map((term) => stem(term)); + for (const term of normalized) { + result.set( + term, + normalized.filter((candidate) => candidate !== term), + ); + } + } + return result; +} + +function fnv1a(value: string): number { + let hash = 0x811c9dc5; + for (let index = 0; index < value.length; index += 1) { + hash ^= value.charCodeAt(index); + hash = Math.imul(hash, 0x01000193); + } + return hash >>> 0; +} diff --git a/src/retrieval/repository-index.ts b/src/retrieval/repository-index.ts new file mode 100644 index 00000000..a688c1e2 --- /dev/null +++ b/src/retrieval/repository-index.ts @@ -0,0 +1,510 @@ +import { lstat, readFile, readdir, realpath, stat } from "node:fs/promises"; +import path from "node:path"; +import { + parseFrontmatterFields, + splitFrontmatter, +} from "../okf/frontmatter.js"; +import type { + DocumentRole, + IndexedChunk, + OkfConcept, + OkfRelationship, + OpenWikiMetadata, + RepositoryCorpus, +} from "./types.js"; + +const MAX_FILE_BYTES = 256_000; +const MAX_FILES = 5_000; +const SOURCE_LINES_PER_CHUNK = 80; +const SOURCE_LINE_OVERLAP = 16; +const SOURCE_EXTENSIONS = new Set([ + ".c", + ".cc", + ".cpp", + ".css", + ".go", + ".h", + ".hpp", + ".html", + ".java", + ".js", + ".json", + ".jsx", + ".md", + ".mjs", + ".py", + ".rb", + ".rs", + ".sh", + ".sql", + ".toml", + ".ts", + ".tsx", + ".yaml", + ".yml", +]); +const EXCLUDED_DIRECTORIES = new Set([ + ".cache", + ".git", + ".hg", + ".next", + ".svn", + ".turbo", + ".venv", + "build", + "coverage", + "dist", + "node_modules", + "target", + "vendor", +]); +const SECRET_FILE = + /^(?:\.env(?:\..*)?|.*\.(?:crt|jks|key|keystore|p12|pem|pfx)|credentials\.json|token(?:\.json)?|cookies?(?:\.(?:db|sqlite|txt))?|\.git-credentials|hosts\.yml)$/iu; +const MARKDOWN_LINK = /\[([^\]]+)\]\(([^)]+)\)/gu; +const TEST_NAME = + /\b(?:describe|it|test)(?:\.(?:each|only|skip|todo))?\s*\(\s*(["'`])([^\n]{1,160}?)\1/gu; +const DOCUMENT_ROLES = new Set([ + "architecture", + "delivery", + "domain", + "integration", + "operations", + "reference", + "repository", + "testing", + "workflow", +]); + +export interface RepositoryIndexOptions { + repoRoot: string; + wikiRoot: string; +} + +export async function buildRepositoryCorpus( + options: RepositoryIndexOptions, +): Promise { + const repoRoot = await resolveDirectory(options.repoRoot, "repository root"); + const wikiRoot = await resolveDirectory(options.wikiRoot, "wiki root"); + const wikiPages = await readWikiPages(wikiRoot); + const concepts = buildConcepts(wikiPages); + connectIncomingRelationships(concepts); + return { + chunks: [ + ...wikiPages.flatMap((page) => page.chunks), + ...(await readSourceChunks(repoRoot, wikiRoot)), + ], + concepts, + }; +} + +interface WikiPage { + chunks: IndexedChunk[]; + concept: OkfConcept; +} + +async function readWikiPages(wikiRoot: string): Promise { + const files = await walkFiles(wikiRoot, (file) => file.endsWith(".md")); + return Promise.all( + files.map(async (file) => { + const content = await readBoundedTextFile(wikiRoot, file); + const relative = toPosix(path.relative(wikiRoot, file)); + const conceptPath = `openwiki/${relative}`; + const fields = parseFrontmatterFields(content) ?? {}; + const { body } = splitFrontmatter(content); + const title = stringField(fields.title) ?? firstHeading(body) ?? relative; + const description = stringField(fields.description); + const type = stringField(fields.type) ?? "Reference"; + const resource = stringField(fields.resource); + const tags = stringArray(fields.tags); + const metadata = parseOpenWikiMetadata(fields.openwiki); + const roles = inferDocumentRoles(type, tags, relative, metadata.roles); + return { + chunks: chunkWikiPage({ + body, + conceptPath, + description, + fields, + relative, + resource, + roles, + tags, + title, + type, + }), + concept: { + ...(description ? { description } : {}), + incoming: new Set(), + metadata: { ...metadata, roles }, + path: conceptPath, + relationships: extractRelationships(body, relative), + ...(resource ? { resource } : {}), + roles, + tags, + title, + type, + }, + }; + }), + ); +} + +async function readSourceChunks( + repoRoot: string, + wikiRoot: string, +): Promise { + const files = await walkFiles(repoRoot, (file) => { + const extension = path.extname(file).toLowerCase(); + return SOURCE_EXTENSIONS.has(extension); + }); + const chunks: IndexedChunk[] = []; + for (const file of files) { + if (isContained(wikiRoot, file)) continue; + const content = await readBoundedTextFile(repoRoot, file); + const relative = toPosix(path.relative(repoRoot, file)); + const lines = content.split(/\r?\n/u); + for ( + let start = 0; + start < lines.length; + start += SOURCE_LINES_PER_CHUNK - SOURCE_LINE_OVERLAP + ) { + const selected = lines.slice(start, start + SOURCE_LINES_PER_CHUNK); + if (selected.every((line) => !line.trim())) continue; + const lineStart = start + 1; + const lineEnd = start + selected.length; + const text = selected.join("\n"); + const testNames = extractTestNames(text); + chunks.push({ + fields: [relative, ...testNames].join("\n"), + id: `source:${relative}:${lineStart}`, + kind: "source", + lineEnd, + lineStart, + path: relative, + roles: [], + scope: "source_code", + tags: pathTags(relative), + ...(testNames.length > 0 ? { testNames } : {}), + text, + title: path.basename(relative), + }); + } + } + return chunks; +} + +function chunkWikiPage(input: { + body: string; + conceptPath: string; + description?: string; + fields: Record; + relative: string; + resource?: string; + roles: DocumentRole[]; + tags: string[]; + title: string; + type: string; +}): IndexedChunk[] { + const lines = input.body.split(/\r?\n/u); + const headingIndexes = lines + .map((line, index) => (/^#{1,3}\s+\S/u.test(line) ? index : -1)) + .filter((index) => index >= 0); + if (headingIndexes.length === 0) headingIndexes.push(0); + return headingIndexes.map((start, index) => { + const end = headingIndexes[index + 1] ?? lines.length; + const selected = lines.slice(start, end); + const heading = selected[0]?.replace(/^#{1,3}\s+/u, "").trim(); + return { + conceptPath: input.conceptPath, + ...(input.description ? { description: input.description } : {}), + fields: JSON.stringify(input.fields), + ...(heading ? { heading } : {}), + id: `wiki:${input.relative}:${start + 1}`, + kind: "wiki-section", + lineEnd: Math.max(start + 1, end), + lineStart: start + 1, + path: input.conceptPath, + ...(input.resource ? { resource: input.resource } : {}), + roles: input.roles, + scope: "wiki", + tags: input.tags, + text: [input.description, selected.join("\n")].filter(Boolean).join("\n"), + title: input.title, + type: input.type, + }; + }); +} + +function buildConcepts(pages: WikiPage[]): Map { + return new Map(pages.map((page) => [page.concept.path, page.concept])); +} + +function connectIncomingRelationships(concepts: Map): void { + for (const concept of concepts.values()) { + concept.relationships = concept.relationships.filter((relationship) => { + const target = concepts.get(relationship.target); + if (!target) return false; + target.incoming.add(concept.path); + return true; + }); + } +} + +function extractRelationships( + body: string, + sourceRelative: string, +): OkfRelationship[] { + const relationships: OkfRelationship[] = []; + for (const match of body.matchAll(MARKDOWN_LINK)) { + const rawTarget = (match[2] ?? "").trim().split("#", 1)[0] ?? ""; + if (!rawTarget || /^[A-Za-z][A-Za-z0-9+.-]*:/u.test(rawTarget)) continue; + const sourceDirectory = path.posix.dirname(toPosix(sourceRelative)); + const resolved = path.posix.normalize( + path.posix.join(sourceDirectory, rawTarget), + ); + if (resolved.startsWith("../") || path.posix.isAbsolute(resolved)) continue; + const target = `openwiki/${resolved.endsWith(".md") ? resolved : `${resolved}.md`}`; + const offset = match.index ?? 0; + relationships.push({ + context: relationshipContext(body, offset), + kind: relationshipKind(relationshipContext(body, offset), sourceRelative), + target, + }); + } + return relationships; +} + +async function walkFiles( + root: string, + include: (file: string) => boolean, +): Promise { + const files: string[] = []; + const pending = [root]; + while (pending.length > 0 && files.length < MAX_FILES) { + const directory = pending.pop(); + if (!directory) break; + const entries = await readdir(directory, { withFileTypes: true }); + for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) { + if (isSecretName(entry.name)) continue; + const candidate = path.join(directory, entry.name); + if (!isContained(root, candidate)) continue; + if (entry.isSymbolicLink()) continue; + if (entry.isDirectory()) { + if (!EXCLUDED_DIRECTORIES.has(entry.name)) pending.push(candidate); + } else if (entry.isFile() && include(candidate)) { + files.push(candidate); + if (files.length >= MAX_FILES) break; + } + } + } + return files; +} + +async function readBoundedTextFile( + root: string, + file: string, +): Promise { + const resolved = await realpath(file); + if (!isContained(root, resolved) || isSecretPath(resolved)) { + throw new Error( + "Refusing to read a path outside the indexed root or a secret-like file.", + ); + } + const info = await stat(resolved); + if (info.size > MAX_FILE_BYTES) return ""; + const content = await readFile(resolved, "utf8"); + return content.includes("\0") ? "" : content; +} + +async function resolveDirectory(value: string, label: string): Promise { + const resolved = await realpath(path.resolve(value)); + const info = await lstat(resolved); + if (!info.isDirectory() || info.isSymbolicLink()) { + throw new Error(`${label} must be a real directory.`); + } + return resolved; +} + +function isContained(root: string, candidate: string): boolean { + const relative = path.relative(root, path.resolve(candidate)); + return ( + relative === "" || + (!relative.startsWith(`..${path.sep}`) && + relative !== ".." && + !path.isAbsolute(relative)) + ); +} + +function isSecretPath(file: string): boolean { + return file.split(path.sep).some(isSecretName); +} + +function isSecretName(name: string): boolean { + return ( + SECRET_FILE.test(name) || + /(?:credential|private[_-]?key|secret)/iu.test(name) + ); +} + +function relationshipContext(body: string, offset: number): string { + const start = Math.max(0, body.lastIndexOf("\n", offset - 160)); + const endCandidate = body.indexOf("\n", offset + 160); + const end = endCandidate === -1 ? body.length : endCandidate; + return body.slice(start, end).replace(/\s+/gu, " ").trim().slice(0, 320); +} + +function pathTags(relative: string): string[] { + return toPosix(relative) + .split("/") + .slice(0, -1) + .filter((part) => part.length > 1); +} + +function firstHeading(body: string): string | undefined { + return /^#\s+(.+?)\s*$/mu.exec(body)?.[1]?.trim(); +} + +function extractTestNames(value: string): string[] { + return [ + ...new Set( + [...value.matchAll(TEST_NAME)] + .map((match) => match[2]?.trim()) + .filter((name): name is string => Boolean(name)), + ), + ]; +} + +function stringField(value: unknown): string | undefined { + return typeof value === "string" && value.trim() ? value.trim() : undefined; +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function stringArray(value: unknown): string[] { + return Array.isArray(value) + ? value.filter( + (item): item is string => + typeof item === "string" && item.trim().length > 0, + ) + : []; +} + +function parseOpenWikiMetadata(value: unknown): OpenWikiMetadata { + const record = isRecord(value) ? value : {}; + return { + changeKinds: slugArray(record.change_kinds, 16), + invariants: boundedStringArray(record.invariants, 16, 400), + roles: boundedStringArray(record.roles, 9, 40).filter( + (role): role is DocumentRole => DOCUMENT_ROLES.has(role as DocumentRole), + ), + sourcePaths: pathArray(record.source_paths, 32), + symbols: boundedStringArray(record.symbols, 48, 120).filter((symbol) => + /^[A-Za-z_$][A-Za-z0-9_$.:-]*$/u.test(symbol), + ), + testPaths: pathArray(record.test_paths, 32), + validationCommands: boundedStringArray(record.validation_commands, 12, 300), + }; +} + +function inferDocumentRoles( + type: string, + tags: string[], + relative: string, + declared: DocumentRole[], +): DocumentRole[] { + const value = `${type} ${tags.join(" ")} ${relative}`.toLowerCase(); + const roles = new Set(declared); + const add = (role: DocumentRole, pattern: RegExp): void => { + if (pattern.test(value)) roles.add(role); + }; + add( + "architecture", + /\b(?:architecture|engine|interface|memory|runtime|storage|system)\b/u, + ); + add("delivery", /\b(?:artifact|build|delivery|package|publish|release)\b/u); + add("domain", /\b(?:concept|data|domain|model|query|schema)\b/u); + add( + "integration", + /\b(?:ecosystem|integration|platform|plugin|provider|react)\b/u, + ); + add( + "operations", + /\b(?:contribution|development|operations|practice|tooling)\b/u, + ); + add("repository", /\b(?:project|quickstart|repository)\b/u); + add("testing", /\b(?:quality|test|testing|validation|verification)\b/u); + add("workflow", /\b(?:automation|ingestion|lifecycle|playbook|workflow)\b/u); + if (roles.size === 0) roles.add("reference"); + return [...roles]; +} + +function relationshipKind( + context: string, + sourceRelative: string, +): OkfRelationship["kind"] { + if (/^(?:quickstart|index)\.md$/u.test(path.posix.basename(sourceRelative))) { + return "navigation"; + } + if ( + /\b(?:export|package|publish|release|ship|surface|bundle|deliver)\w*\b/iu.test( + context, + ) + ) { + return "delivery"; + } + if ( + /\b(?:lifecycle|transition|reset|reuse|before|after|enter|exit)\w*\b/iu.test( + context, + ) + ) { + return "lifecycle"; + } + if ( + /\b(?:call|depend|dispatch|own|share|configure|secure|adapt|consume)\w*\b/iu.test( + context, + ) + ) { + return "dependency"; + } + if (/\b(?:start|navigate|read|see|guide|overview)\w*\b/iu.test(context)) { + return "navigation"; + } + return "related"; +} + +function boundedStringArray( + value: unknown, + maximumItems: number, + maximumLength: number, +): string[] { + if (!Array.isArray(value)) return []; + return [ + ...new Set( + value + .filter((item): item is string => typeof item === "string") + .map((item) => item.replace(/\s+/gu, " ").trim()) + .filter((item) => item.length > 0 && item.length <= maximumLength), + ), + ].slice(0, maximumItems); +} + +function slugArray(value: unknown, maximumItems: number): string[] { + return boundedStringArray(value, maximumItems, 60) + .map((item) => item.toLowerCase()) + .filter((item) => /^[a-z0-9]+(?:-[a-z0-9]+)*$/u.test(item)); +} + +function pathArray(value: unknown, maximumItems: number): string[] { + return boundedStringArray(value, maximumItems, 300).filter( + (item) => + !path.posix.isAbsolute(item) && + !item.includes("\\") && + !item.split("/").some((part) => part === "" || part === "..") && + !item.split("/").some(isSecretName), + ); +} + +function toPosix(value: string): string { + return value.split(path.sep).join("/"); +} diff --git a/src/retrieval/search-service.ts b/src/retrieval/search-service.ts new file mode 100644 index 00000000..6887f6ce --- /dev/null +++ b/src/retrieval/search-service.ts @@ -0,0 +1,991 @@ +import { buildRepositoryCorpus } from "./repository-index.js"; +import { + rankBm25, + rankKeyword, + reciprocalRankFusion, + tokenize, +} from "./ranking.js"; +import { SemanticRanker, type EmbeddingProvider } from "./semantic.js"; +import type { + BriefInvariant, + ChangeSurfaceResponse, + CoverageReviewItem, + DocumentRole, + EvidenceReference, + IndexedChunk, + OkfConcept, + OpenWikiMetadata, + RankedHit, + RepositoryCorpus, + SearchResponse, + SearchResultItem, + SearchScope, + SourceSurfaceCategory, + ValidationReference, +} from "./types.js"; + +const DEFAULT_LIMIT = 6; +const MAX_SEARCH_LIMIT = 10; +const MAX_SURFACE_LIMIT = 8; +const MAX_QUERY_LENGTH = 500; +const MAX_SNIPPET_LENGTH = 220; +const MAX_CHANGED_PATHS = 50; +const MAX_CONCEPTS = 3; +const MAX_INVARIANTS = 4; +const MAX_VALIDATION_COMMANDS = 3; + +export interface RetrievalServiceOptions { + embeddingProvider: EmbeddingProvider; + repoRoot: string; + wikiRoot: string; +} + +export class RetrievalService { + private corpusPromise: Promise | undefined; + private readonly semantic: SemanticRanker; + + constructor(private readonly options: RetrievalServiceOptions) { + this.semantic = new SemanticRanker(options.embeddingProvider); + } + + async search( + query: string, + scope: SearchScope = "all", + limit = DEFAULT_LIMIT, + ): Promise { + const corpus = await this.corpus(); + const validQuery = validateQuery(query); + const validScope = validateScope(scope); + const chunks = scopedChunks(corpus.chunks, validScope); + const semantic = await this.semantic.rank(chunks, validQuery); + const lists = [ + { hits: rankKeyword(chunks, validQuery), name: "keyword", weight: 0.75 }, + { hits: rankBm25(chunks, validQuery), name: "bm25", weight: 1 }, + { hits: semantic.hits, name: "semantic", weight: 0.9 }, + ]; + if (validScope === "all" || validScope === "wiki") { + lists.push({ + hits: rankOkfGraph(corpus, validQuery, 1), + name: "okf_graph", + weight: 0.8, + }); + } + const ranked = reciprocalRankFusion(lists); + return response( + validQuery, + validScope === "tests" ? deduplicateTestMirrors(ranked) : ranked, + normalizeLimit(limit, MAX_SEARCH_LIMIT, DEFAULT_LIMIT), + validScope, + ); + } + + async changeSurface( + query: string, + limit = 6, + changedPaths: string[] = [], + ): Promise { + const corpus = await this.corpus(); + const validQuery = validateQuery(query); + const validLimit = normalizeLimit(limit, MAX_SURFACE_LIMIT, 6); + const validChangedPaths = validateChangedPaths(changedPaths); + const metadataRoles = inferQueryRoles(validQuery); + const conceptChunks = selectConceptChunks( + corpus, + validQuery, + metadataRoles, + MAX_CONCEPTS, + ); + const selectedConcepts = selectedOkfConcepts(corpus, conceptChunks); + const metadata = mergeMetadata(selectedConcepts); + const referencedPaths = new Set([ + ...extractPaths(conceptChunks.map((chunk) => chunk.text).join("\n")), + ...metadata.sourcePaths, + ...metadata.testPaths, + ]); + const symbols = [...new Set(metadata.symbols)]; + const sourceQuery = [validQuery, ...symbols.slice(0, 10)].join(" "); + const sourceCorpus = sourceChunks(corpus.chunks).filter( + (chunk) => !isRepositoryGuidance(chunk), + ); + const source = boostReferencedPaths( + reciprocalRankFusion([ + { + hits: rankBm25(sourceCorpus, sourceQuery), + name: "bm25", + weight: 1, + }, + { + hits: rankKeyword(sourceCorpus, sourceQuery), + name: "keyword", + weight: 0.9, + }, + ]), + referencedPaths, + ).slice(0, 120); + const invariants = collectInvariants( + selectedConcepts, + conceptChunks, + validQuery, + ); + const ownershipHits = uniquePathHits( + source.filter( + (hit) => + !isTestChunk(hit.chunk) && + categorize(hit.chunk).includes("implementation"), + ), + ).slice(0, Math.min(3, validLimit)); + const deliveryRequested = requiresDeliveryReview( + validQuery, + metadataRoles, + validChangedPaths, + ); + const deliveryHits = deliveryRequested + ? uniquePathHits( + source.filter((hit) => + categorize(hit.chunk).some((category) => + [ + "consumer", + "exports", + "initialization", + "publish_generated", + ].includes(category), + ), + ), + ).slice(0, 2) + : []; + const testRankers = [ + { + hits: rankBm25(sourceCorpus.filter(isTestChunk), validQuery), + name: "test_bm25", + weight: 1.2, + }, + { + hits: rankKeyword(sourceCorpus.filter(isTestChunk), validQuery), + name: "test_keyword", + weight: 1, + }, + ]; + if (invariants.length > 0) { + testRankers.push({ + hits: rankBm25( + sourceCorpus.filter(isTestChunk), + invariants.map((invariant) => invariant.text).join(" "), + ), + name: "invariant_bm25", + weight: 0.35, + }); + } + const testHits = uniquePathHits( + deduplicateTestMirrors( + boostReferencedPaths( + reciprocalRankFusion(testRankers), + new Set(metadata.testPaths), + 2.5, + ), + ), + ).slice(0, Math.min(3, validLimit)); + const ownership = ownershipHits.map((hit) => + toEvidenceReference(hit, ownershipReason(hit.chunk, referencedPaths)), + ); + const tests = testHits.map((hit) => + toEvidenceReference(hit, "Analogous behavior or regression coverage."), + ); + const delivery = deliveryHits.map((hit) => + toEvidenceReference(hit, deliveryReason(hit.chunk)), + ); + const validation = collectValidation(selectedConcepts, conceptChunks); + const unknowns = collectUnknowns({ + delivery, + deliveryRequested, + invariants, + ownership, + tests, + }); + const review = buildCoverageReview( + validChangedPaths, + [...ownership, ...delivery], + referencedPaths, + ); + return { + brief: { + delivery, + invariants, + ownership, + tests, + unknowns, + validation, + }, + provenance: { + changedPaths: validChangedPaths, + metadataRoles, + wikiConceptPaths: [ + ...new Set(conceptChunks.map((chunk) => chunk.path)), + ], + wikiReferencedSourcePaths: [...referencedPaths], + }, + query: validQuery, + ...(review.length > 0 ? { review } : {}), + }; + } + + private corpus(): Promise { + this.corpusPromise ??= buildRepositoryCorpus(this.options); + return this.corpusPromise; + } +} + +function rankOkfGraph( + corpus: RepositoryCorpus, + query: string, + hops: number, +): RankedHit[] { + const wikiChunks = corpus.chunks.filter((chunk) => chunk.scope === "wiki"); + const seeds = rankBm25(wikiChunks, query).slice(0, 20); + const scores = new Map(); + const seedConcepts = new Set(); + for (const [index, hit] of seeds.entries()) { + if (!hit.chunk.conceptPath) continue; + const score = 1 / (index + 1); + scores.set( + hit.chunk.conceptPath, + (scores.get(hit.chunk.conceptPath) ?? 0) + score, + ); + seedConcepts.add(hit.chunk.conceptPath); + } + let frontier = seedConcepts; + for (let hop = 0; hop < hops; hop += 1) { + const next = new Set(); + for (const conceptPath of frontier) { + const concept = corpus.concepts.get(conceptPath); + if (!concept) continue; + const base = scores.get(conceptPath) ?? 0; + for (const neighbor of graphNeighbors(concept, query)) { + scores.set(neighbor, (scores.get(neighbor) ?? 0) + base * 0.35); + next.add(neighbor); + } + } + frontier = next; + } + return [...scores.entries()] + .map(([conceptPath, score]) => { + const chunk = bestConceptChunk(wikiChunks, conceptPath, query); + return chunk ? { chunk, score } : null; + }) + .filter((hit): hit is RankedHit => hit !== null) + .sort((left, right) => right.score - left.score); +} + +function graphNeighbors(concept: OkfConcept, query: string): Set { + const queryTerms = new Set(tokenize(query)); + const desiredKinds = new Set([ + "dependency", + "lifecycle", + "related", + ]); + if (/\b(?:export|package|public|publish|release|ship)\w*\b/iu.test(query)) { + desiredKinds.add("delivery"); + } + return new Set( + concept.relationships + .filter( + (relationship) => + desiredKinds.has(relationship.kind) && + (relationship.kind !== "related" || + tokenize(relationship.context).some((term) => + queryTerms.has(term), + )), + ) + .map((relationship) => relationship.target), + ); +} + +function bestConceptChunk( + chunks: IndexedChunk[], + conceptPath: string, + query: string, +): IndexedChunk | undefined { + return ( + rankBm25( + chunks.filter((chunk) => chunk.conceptPath === conceptPath), + query, + )[0]?.chunk ?? chunks.find((chunk) => chunk.conceptPath === conceptPath) + ); +} + +function selectConceptChunks( + corpus: RepositoryCorpus, + query: string, + desiredRoles: DocumentRole[], + limit: number, +): IndexedChunk[] { + const wikiChunks = corpus.chunks.filter((chunk) => chunk.scope === "wiki"); + const representatives = [...corpus.concepts.values()] + .map((concept) => { + const base = wikiChunks.find( + (chunk) => chunk.conceptPath === concept.path, + ); + if (!base) return undefined; + return { + ...base, + fields: [ + concept.title, + concept.type, + concept.description, + concept.roles.join(" "), + concept.tags.join(" "), + concept.resource, + concept.metadata.changeKinds.join(" "), + concept.metadata.sourcePaths.join(" "), + concept.metadata.symbols.join(" "), + concept.metadata.testPaths.join(" "), + ] + .filter(Boolean) + .join("\n"), + text: concept.description ?? "", + } satisfies IndexedChunk; + }) + .filter((chunk): chunk is IndexedChunk => chunk !== undefined); + const relevantRepresentatives = representatives.filter((chunk) => + hasDistinctiveMetadataMatch(chunk, query), + ); + if (relevantRepresentatives.length === 0) return []; + const neighborPaths = new Set(); + for (const seed of rankBm25(relevantRepresentatives, query).slice(0, 4)) { + const concept = seed.chunk.conceptPath + ? corpus.concepts.get(seed.chunk.conceptPath) + : undefined; + if (!concept) continue; + for (const neighbor of graphNeighbors(concept, query)) { + neighborPaths.add(neighbor); + } + } + const ranked = reciprocalRankFusion([ + { + hits: rankBm25(relevantRepresentatives, query), + name: "metadata_bm25", + weight: 1, + }, + { + hits: rankKeyword(relevantRepresentatives, query), + name: "metadata_keyword", + weight: 0.9, + }, + ]) + .map((hit) => { + const overlap = hit.chunk.roles.filter((role) => + desiredRoles.includes(role), + ).length; + const repositoryOnly = + hit.chunk.roles.includes("repository") && + hit.chunk.roles.every((role) => + ["repository", "reference"].includes(role), + ); + return { + ...hit, + score: + hit.score * + (1 + overlap * 0.18) * + (neighborPaths.has(hit.chunk.conceptPath ?? "") ? 1.15 : 1) * + (repositoryOnly ? 0.65 : 1), + }; + }) + .sort((left, right) => right.score - left.score); + const selected: RankedHit[] = []; + const coveredRoles = new Set(); + for (const hit of ranked) { + if (selected.length >= limit) break; + if ( + selected.some( + (candidate) => candidate.chunk.conceptPath === hit.chunk.conceptPath, + ) + ) { + continue; + } + const addsRole = hit.chunk.roles.some( + (role) => desiredRoles.includes(role) && !coveredRoles.has(role), + ); + if (selected.length < 2 || addsRole || selected.length + 1 === limit) { + selected.push(hit); + hit.chunk.roles.forEach((role) => coveredRoles.add(role)); + } + } + return selected + .map((hit) => + hit.chunk.conceptPath + ? bestConceptChunk(wikiChunks, hit.chunk.conceptPath, query) + : undefined, + ) + .filter((chunk): chunk is IndexedChunk => chunk !== undefined); +} + +const GENERIC_ROUTING_TERMS = new Set([ + "add", + "agent", + "change", + "cod", + "code", + "implement", + "improve", + "repository", + "task", + "update", +]); + +function hasDistinctiveMetadataMatch( + chunk: IndexedChunk, + query: string, +): boolean { + const queryTerms = new Set( + tokenize(query).filter((term) => !GENERIC_ROUTING_TERMS.has(term)), + ); + if (queryTerms.size === 0) return true; + const metadataTerms = new Set( + tokenize( + [ + chunk.path, + chunk.title, + chunk.description, + chunk.type, + chunk.tags.join(" "), + chunk.roles.join(" "), + chunk.fields, + ] + .filter(Boolean) + .join(" "), + ), + ); + return [...queryTerms].some((term) => metadataTerms.has(term)); +} + +function selectedOkfConcepts( + corpus: RepositoryCorpus, + chunks: IndexedChunk[], +): OkfConcept[] { + return [ + ...new Map( + chunks + .map((chunk) => + chunk.conceptPath + ? corpus.concepts.get(chunk.conceptPath) + : undefined, + ) + .filter((concept): concept is OkfConcept => concept !== undefined) + .map((concept) => [concept.path, concept]), + ).values(), + ]; +} + +function mergeMetadata(concepts: OkfConcept[]): OpenWikiMetadata { + const merge = ( + select: (metadata: OpenWikiMetadata) => string[], + ): string[] => [ + ...new Set(concepts.flatMap((concept) => select(concept.metadata))), + ]; + return { + changeKinds: merge((metadata) => metadata.changeKinds), + invariants: merge((metadata) => metadata.invariants), + roles: [...new Set(concepts.flatMap((concept) => concept.metadata.roles))], + sourcePaths: merge((metadata) => metadata.sourcePaths), + symbols: merge((metadata) => metadata.symbols), + testPaths: merge((metadata) => metadata.testPaths), + validationCommands: merge((metadata) => metadata.validationCommands), + }; +} + +function inferQueryRoles(query: string): DocumentRole[] { + const roles = new Set(["architecture", "domain"]); + const add = (role: DocumentRole, pattern: RegExp): void => { + if (pattern.test(query)) roles.add(role); + }; + add( + "delivery", + /\b(?:api|artifact|build|consumer|export|package|public|publish|release|ship)\w*\b/iu, + ); + add( + "integration", + /\b(?:adapter|integration|middleware|plugin|provider|react|router)\w*\b/iu, + ); + add( + "operations", + /\b(?:ci|cli|configure|deploy|development|install|operations|tooling)\w*\b/iu, + ); + add( + "testing", + /\b(?:behavior|compatibility|invariant|regression|test|validate|verify)\w*\b/iu, + ); + add( + "workflow", + /\b(?:defer|event|lifecycle|reset|rollback|state|transition|workflow)\w*\b/iu, + ); + return [...roles]; +} + +function collectInvariants( + concepts: OkfConcept[], + chunks: IndexedChunk[], + query: string, +): BriefInvariant[] { + const candidates: (BriefInvariant & { score: number })[] = []; + for (const concept of concepts) { + for (const invariant of concept.metadata.invariants) { + candidates.push({ + lineEnd: 1, + lineStart: 1, + path: concept.path, + score: invariantScore(invariant, query) + 8, + text: invariant, + }); + } + } + for (const chunk of chunks) { + const lines = chunk.text + .split(/\r?\n/gu) + .map((line) => line.replace(/^\s*(?:[-*]|\d+\.)\s+/u, "").trim()) + .filter((line) => line.length >= 24 && line.length <= 500); + for (const line of lines) { + if (!isInvariantText(line)) continue; + if (!hasTermOverlap(line, query)) continue; + candidates.push({ + lineEnd: chunk.lineEnd, + lineStart: chunk.lineStart, + path: chunk.path, + score: invariantScore(line, query), + text: compactText(line, 240), + }); + } + } + const seen = new Set(); + return candidates + .sort((left, right) => right.score - left.score) + .filter((candidate) => candidate.score > 0) + .filter((candidate) => { + const key = candidate.text.toLowerCase(); + if (seen.has(key)) return false; + seen.add(key); + return true; + }) + .slice(0, MAX_INVARIANTS) + .map((candidate) => ({ + lineEnd: candidate.lineEnd, + lineStart: candidate.lineStart, + path: candidate.path, + text: candidate.text, + })); +} + +function isInvariantText(value: string): boolean { + return /\b(?:must|should not|do not|don't|never|preserve|remain|only|before|after|unchanged|idempotent|reset|reuse|invariant|required|incomplete)\b/iu.test( + value, + ); +} + +function invariantScore(value: string, query: string): number { + const queryTerms = new Set(tokenize(query)); + const overlap = tokenize(value).filter((term) => queryTerms.has(term)).length; + const force = /\b(?:must|do not|don't|never|required|invariant)\b/iu.test( + value, + ) + ? 4 + : 0; + return overlap * 2 + force; +} + +function hasTermOverlap(value: string, query: string): boolean { + const queryTerms = new Set( + tokenize(query).filter((term) => !GENERIC_ROUTING_TERMS.has(term)), + ); + return tokenize(value).some((term) => queryTerms.has(term)); +} + +function collectValidation( + concepts: OkfConcept[], + chunks: IndexedChunk[], +): ValidationReference[] { + const candidates: ValidationReference[] = []; + for (const concept of concepts) { + for (const command of concept.metadata.validationCommands) { + if (isSafeDisplayedCommand(command)) { + candidates.push({ command, path: concept.path }); + } + } + } + for (const chunk of chunks) { + for (const match of chunk.text.matchAll(/`([^`\n]{3,300})`/gu)) { + const command = match[1]?.trim(); + if (command && looksLikeValidationCommand(command)) { + candidates.push({ command, path: chunk.path }); + } + } + } + const seen = new Set(); + return candidates + .filter((candidate) => { + if (seen.has(candidate.command)) return false; + seen.add(candidate.command); + return true; + }) + .slice(0, MAX_VALIDATION_COMMANDS); +} + +function looksLikeValidationCommand(value: string): boolean { + return ( + isSafeDisplayedCommand(value) && + /^(?:bun|cargo|go|make|npm|npx|pnpm|pytest|python\s+-m\s+pytest|ruff|uv\s+run|yarn)\b/iu.test( + value, + ) && + /\b(?:build|check|lint|test|typecheck|verify|vitest)\b/iu.test(value) + ); +} + +function isSafeDisplayedCommand(value: string): boolean { + return ( + value.length <= 300 && + !/[\r\n\0]/u.test(value) && + !/(?:\.env|credential|private[_-]?key|secret|token)/iu.test(value) + ); +} + +function uniquePathHits(hits: RankedHit[]): RankedHit[] { + const seen = new Set(); + return hits.filter((hit) => { + if (seen.has(hit.chunk.path)) return false; + seen.add(hit.chunk.path); + return true; + }); +} + +function toEvidenceReference( + hit: RankedHit, + reason: string, +): EvidenceReference { + return { + lineEnd: hit.chunk.lineEnd, + lineStart: hit.chunk.lineStart, + path: hit.chunk.path, + reason, + ...(hit.chunk.testNames && hit.chunk.testNames.length > 0 + ? { testNames: hit.chunk.testNames.slice(0, 6) } + : {}), + ...(hit.chunk.title ? { title: hit.chunk.title } : {}), + }; +} + +function ownershipReason(chunk: IndexedChunk, references: Set): string { + return pathMatchesReference(chunk.path, references) + ? "Named by the selected OpenWiki concept as an implementation anchor." + : "Highest-ranked implementation ownership candidate; verify in source."; +} + +function deliveryReason(chunk: IndexedChunk): string { + const categories = categorize(chunk); + if (categories.includes("exports")) + return "Public or package export surface."; + if (categories.includes("publish_generated")) { + return "Generated, packaged, or publish-facing surface."; + } + if (categories.includes("consumer")) return "Consumer-facing usage surface."; + return "Initialization or registration surface."; +} + +function requiresDeliveryReview( + query: string, + roles: DocumentRole[], + changedPaths: string[], +): boolean { + return ( + roles.includes("delivery") || + /\b(?:api|consumer|export|package|public|publish|release|ship)\w*\b/iu.test( + query, + ) || + changedPaths.some((candidate) => + /(?:^|\/)(?:index\.[cm]?[jt]sx?|package\.json|dist|publish)(?:$|\/)/u.test( + candidate, + ), + ) + ); +} + +function collectUnknowns(input: { + delivery: EvidenceReference[]; + deliveryRequested: boolean; + invariants: BriefInvariant[]; + ownership: EvidenceReference[]; + tests: EvidenceReference[]; +}): string[] { + const unknowns: string[] = []; + if (input.ownership.length === 0) { + unknowns.push( + "No implementation owner was established; locate it in source.", + ); + } + if (input.invariants.length === 0) { + unknowns.push("No explicit behavioral invariant was found in the wiki."); + } + if (input.tests.length === 0) { + unknowns.push( + "No analogous focused test was found; add task-specific coverage.", + ); + } + if (input.deliveryRequested && input.delivery.length === 0) { + unknowns.push( + "No shipped-surface evidence was found; verify exports manually.", + ); + } + return unknowns; +} + +function buildCoverageReview( + changedPaths: string[], + evidence: EvidenceReference[], + referencedPaths: Set, +): CoverageReviewItem[] { + if (changedPaths.length === 0) return []; + const normalizedChanges = new Set(changedPaths); + const candidates = [...evidence.map((item) => item.path), ...referencedPaths]; + const seen = new Set(); + return candidates + .filter((candidate) => { + if (seen.has(candidate) || normalizedChanges.has(candidate)) return false; + seen.add(candidate); + return true; + }) + .slice(0, 4) + .map((candidate) => ({ + path: candidate, + reason: + "Documented adjacent surface is absent from changed_paths; verify that it is intentionally unaffected.", + })); +} + +function boostReferencedPaths( + hits: RankedHit[], + paths: Set, + multiplier = 1.6, +): RankedHit[] { + return hits + .map((hit) => ({ + ...hit, + score: + hit.score * + (pathMatchesReference(hit.chunk.path, paths) ? multiplier : 1), + })) + .sort((left, right) => right.score - left.score); +} + +function pathMatchesReference(path: string, references: Set): boolean { + return [...references].some( + (candidate) => path === candidate || path.endsWith(candidate), + ); +} + +function categorize(chunk: IndexedChunk): SourceSurfaceCategory[] { + const value = `${chunk.path}\n${chunk.text}`; + const categories = new Set(); + if ( + /\b(?:exports|entrypoint|public api)\b/iu.test(value) || + /\bexport\s+(?:\*|\{[^}]+\})\s+from\b/iu.test(chunk.text) || + /(?:^|\/)index\.[cm]?[jt]sx?$/u.test(chunk.path) + ) { + categories.add("exports"); + } + if ( + /\b(?:publish|generated|bundle|build artifact|package\.json|dist)\b/iu.test( + value, + ) + ) { + categories.add("publish_generated"); + } + if ( + /\b(?:initialize|register|registry|factory|createStore|createWorld|setup)\b/u.test( + value, + ) + ) { + categories.add("initialization"); + } + if (isTestChunk(chunk)) { + categories.add("tests"); + } + if ( + /\bimport\s+.+\s+from\s+['"][^./]/u.test(chunk.text) || + /(?:^|\/)(?:examples?|apps?|publish\/tests)(?:\/|$)/iu.test(chunk.path) + ) { + categories.add("consumer"); + } + if (categories.size === 0 || /(?:^|\/)src(?:\/|$)/u.test(chunk.path)) { + categories.add("implementation"); + } + return [...categories]; +} + +function isTestChunk(chunk: IndexedChunk): boolean { + return ( + /(?:^|\/)(?:test|tests|spec|specs)(?:\/|$)/iu.test(chunk.path) || + /(?:^|[._-])(?:test|tests|spec|specs)(?:[._-]|$)/iu.test(chunk.path) + ); +} + +function isRepositoryGuidance(chunk: IndexedChunk): boolean { + return /(?:^|\/)(?:AGENTS|CLAUDE)\.md$/iu.test(chunk.path); +} + +function deduplicateTestMirrors(hits: RankedHit[]): RankedHit[] { + const deduplicated = new Map(); + for (const hit of hits) { + const key = canonicalTestKey(hit.chunk); + const current = deduplicated.get(key); + if ( + !current || + (isGeneratedTestPath(current.chunk.path) && + !isGeneratedTestPath(hit.chunk.path)) + ) { + deduplicated.set(key, hit); + } + } + return [...deduplicated.values()]; +} + +function canonicalTestKey(chunk: IndexedChunk): string { + const normalizedPath = chunk.path + .replace( + /(?:^|\/)packages\/publish\/tests\/(?:core\/)?/u, + "packages/core/tests/", + ) + .replace(/(?:^|\/)(?:generated|publish)\/tests\//u, "tests/"); + return `${normalizedPath}:${chunk.lineStart}:${(chunk.testNames ?? []).join("|")}`; +} + +function isGeneratedTestPath(value: string): boolean { + return /(?:^|\/)(?:generated|publish)(?:\/|$)/u.test(value); +} + +function extractPaths(value: string): Set { + const paths = value.match( + /(?:^|[\s`("'])([A-Za-z0-9_.-]+\/(?:[A-Za-z0-9_.-]+\/)*[A-Za-z0-9_.-]+\.[A-Za-z0-9]+)/gmu, + ); + return new Set( + (paths ?? []).map((item) => item.trim().replace(/^[`("']/u, "")), + ); +} + +function response( + query: string, + hits: RankedHit[], + limit: number, + scope: SearchScope, +): SearchResponse { + return { + query, + results: hits.slice(0, limit).map(toResultItem), + scope, + }; +} + +function toResultItem(hit: RankedHit): SearchResultItem { + return { + ...(hit.chunk.heading ? { heading: hit.chunk.heading } : {}), + lineEnd: hit.chunk.lineEnd, + lineStart: hit.chunk.lineStart, + path: hit.chunk.path, + snippet: compactSnippet(hit.chunk.text), + ...(hit.chunk.tags.length > 0 ? { tags: hit.chunk.tags } : {}), + ...(hit.chunk.testNames && hit.chunk.testNames.length > 0 + ? { testNames: hit.chunk.testNames } + : {}), + ...(hit.chunk.title ? { title: hit.chunk.title } : {}), + ...(hit.chunk.type ? { type: hit.chunk.type } : {}), + }; +} + +function compactSnippet(value: string): string { + return compactText(value, MAX_SNIPPET_LENGTH); +} + +function compactText(value: string, limit: number): string { + return value.replace(/\s+/gu, " ").trim().slice(0, limit); +} + +function scopedChunks( + chunks: IndexedChunk[], + scope: SearchScope, +): IndexedChunk[] { + const valid = validateScope(scope); + if (valid === "all") return chunks; + if (valid === "wiki") { + return chunks.filter((chunk) => chunk.scope === "wiki"); + } + if (valid === "tests") { + return sourceChunks(chunks).filter(isTestChunk); + } + return sourceChunks(chunks).filter((chunk) => !isTestChunk(chunk)); +} + +function validateScope(scope: SearchScope): SearchScope { + if ( + scope !== "all" && + scope !== "source_code" && + scope !== "tests" && + scope !== "wiki" + ) { + throw new Error("scope must be all, source_code, tests, or wiki."); + } + return scope; +} + +function sourceChunks(chunks: IndexedChunk[]): IndexedChunk[] { + return chunks.filter((chunk) => chunk.scope === "source_code"); +} + +function validateQuery(query: string): string { + if ( + typeof query !== "string" || + !query.trim() || + query.length > MAX_QUERY_LENGTH + ) { + throw new Error(`query must be 1-${MAX_QUERY_LENGTH} characters.`); + } + return query.trim(); +} + +function validateChangedPaths(paths: string[]): string[] { + if (!Array.isArray(paths)) { + throw new Error( + "changed_paths must be an array of repository-relative paths.", + ); + } + if (paths.length > MAX_CHANGED_PATHS) { + throw new Error( + `changed_paths must contain at most ${MAX_CHANGED_PATHS} paths.`, + ); + } + return [ + ...new Set( + paths.map((candidate) => { + if ( + typeof candidate !== "string" || + !candidate.trim() || + candidate.length > 300 || + candidate.startsWith("/") || + candidate.includes("\\") || + candidate.split("/").some((part) => part === "" || part === "..") || + /(?:^|\/)(?:\.env(?:\..*)?|credentials\.json|secrets?|tokens?)(?:\/|$)/iu.test( + candidate, + ) + ) { + throw new Error( + "changed_paths must contain safe repository-relative paths.", + ); + } + return candidate.trim(); + }), + ), + ]; +} + +function normalizeLimit( + limit: number, + maximum: number, + fallback: number, +): number { + if (!Number.isInteger(limit)) return fallback; + return Math.max(1, Math.min(maximum, limit)); +} diff --git a/src/retrieval/semantic.ts b/src/retrieval/semantic.ts new file mode 100644 index 00000000..ca430e15 --- /dev/null +++ b/src/retrieval/semantic.ts @@ -0,0 +1,99 @@ +import { rankBm25, rankLocalVectors, searchableText } from "./ranking.js"; +import type { IndexedChunk, RankedHit } from "./types.js"; + +export type EmbeddingProvider = "local" | "openai"; + +const OPENAI_CANDIDATE_LIMIT = 120; + +export class SemanticRanker { + private readonly vectorCache = new Map(); + + constructor(private readonly provider: EmbeddingProvider) {} + + async rank( + chunks: IndexedChunk[], + query: string, + ): Promise<{ engine: string; hits: RankedHit[] }> { + if (this.provider !== "openai" || !process.env.OPENAI_API_KEY) { + return { + engine: "local-hashed-vector", + hits: rankLocalVectors(chunks, query), + }; + } + try { + const candidates = selectOpenAiCandidates(chunks, query); + const embeddings = await this.openAiEmbeddings(); + const queryVector = await embeddings.embedQuery(query); + const missing = candidates.filter( + (chunk) => !this.vectorCache.has(chunk.id), + ); + if (missing.length > 0) { + const vectors = await embeddings.embedDocuments( + missing.map(searchableText), + ); + missing.forEach((chunk, index) => { + const vector = vectors[index]; + if (vector) this.vectorCache.set(chunk.id, vector); + }); + } + const hits = candidates + .map((chunk) => ({ + chunk, + score: cosine(queryVector, this.vectorCache.get(chunk.id) ?? []), + })) + .filter((hit) => hit.score > 0) + .sort((left, right) => right.score - left.score); + return { engine: "openai:text-embedding-3-small", hits }; + } catch { + return { + engine: "local-hashed-vector:fallback", + hits: rankLocalVectors(chunks, query), + }; + } + } + + private async openAiEmbeddings(): Promise<{ + embedDocuments(texts: string[]): Promise; + embedQuery(text: string): Promise; + }> { + const { OpenAIEmbeddings } = await import("@langchain/openai"); + return new OpenAIEmbeddings({ + apiKey: process.env.OPENAI_API_KEY, + batchSize: 64, + configuration: process.env.OPENAI_BASE_URL + ? { baseURL: process.env.OPENAI_BASE_URL } + : undefined, + model: "text-embedding-3-small", + }); + } +} + +function selectOpenAiCandidates( + chunks: IndexedChunk[], + query: string, +): IndexedChunk[] { + const wiki = chunks.filter((chunk) => chunk.scope === "wiki"); + const lexical = rankBm25(chunks, query) + .slice(0, OPENAI_CANDIDATE_LIMIT) + .map((hit) => hit.chunk); + return [ + ...new Map( + [...wiki, ...lexical].map((chunk) => [chunk.id, chunk]), + ).values(), + ].slice(0, OPENAI_CANDIDATE_LIMIT); +} + +function cosine(left: number[], right: number[]): number { + if (left.length === 0 || left.length !== right.length) return 0; + let dot = 0; + let leftNorm = 0; + let rightNorm = 0; + for (let index = 0; index < left.length; index += 1) { + const l = left[index] ?? 0; + const r = right[index] ?? 0; + dot += l * r; + leftNorm += l * l; + rightNorm += r * r; + } + return leftNorm && rightNorm ? dot / Math.sqrt(leftNorm * rightNorm) : 0; +} diff --git a/src/retrieval/types.ts b/src/retrieval/types.ts new file mode 100644 index 00000000..4d82b360 --- /dev/null +++ b/src/retrieval/types.ts @@ -0,0 +1,154 @@ +export type SearchScope = "all" | "source_code" | "tests" | "wiki"; + +export type IndexedScope = "source_code" | "wiki"; + +export type ChunkKind = "source" | "wiki-section"; + +export type DocumentRole = + | "architecture" + | "delivery" + | "domain" + | "integration" + | "operations" + | "reference" + | "repository" + | "testing" + | "workflow"; + +export interface OpenWikiMetadata { + changeKinds: string[]; + invariants: string[]; + roles: DocumentRole[]; + sourcePaths: string[]; + symbols: string[]; + testPaths: string[]; + validationCommands: string[]; +} + +export interface IndexedChunk { + conceptPath?: string; + description?: string; + fields: string; + heading?: string; + id: string; + kind: ChunkKind; + lineEnd: number; + lineStart: number; + path: string; + resource?: string; + roles: DocumentRole[]; + scope: IndexedScope; + tags: string[]; + testNames?: string[]; + text: string; + title?: string; + type?: string; +} + +export interface OkfRelationship { + context: string; + kind: "dependency" | "delivery" | "lifecycle" | "navigation" | "related"; + target: string; +} + +export interface OkfConcept { + description?: string; + incoming: Set; + metadata: OpenWikiMetadata; + path: string; + relationships: OkfRelationship[]; + resource?: string; + roles: DocumentRole[]; + tags: string[]; + title: string; + type: string; +} + +export interface RepositoryCorpus { + chunks: IndexedChunk[]; + concepts: Map; +} + +export interface RankedHit { + chunk: IndexedChunk; + score: number; + signals?: Record; +} + +export interface SearchResultItem { + heading?: string; + lineEnd: number; + lineStart: number; + path: string; + snippet: string; + tags?: string[]; + testNames?: string[]; + title?: string; + type?: string; +} + +export interface SearchResponse { + query: string; + results: SearchResultItem[]; + scope: SearchScope; +} + +export type SourceSurfaceCategory = + | "consumer" + | "exports" + | "implementation" + | "initialization" + | "publish_generated" + | "tests"; + +export type ChangeSurfaceCategory = SourceSurfaceCategory | "state_transitions"; + +export interface EvidenceReference { + lineEnd: number; + lineStart: number; + path: string; + reason: string; + symbols?: string[]; + testNames?: string[]; + title?: string; +} + +export interface ChangeSurfaceProvenance { + changedPaths: string[]; + metadataRoles: DocumentRole[]; + wikiConceptPaths: string[]; + wikiReferencedSourcePaths: string[]; +} + +export interface BriefInvariant { + lineEnd: number; + lineStart: number; + path: string; + text: string; +} + +export interface ValidationReference { + command: string; + path: string; +} + +export interface CoverageReviewItem { + path: string; + reason: string; +} + +export interface ChangeSurfaceBrief { + delivery: EvidenceReference[]; + invariants: BriefInvariant[]; + ownership: EvidenceReference[]; + tests: EvidenceReference[]; + unknowns: string[]; + validation: ValidationReference[]; +} + +export interface ChangeSurfaceResponse { + brief: ChangeSurfaceBrief; + provenance: ChangeSurfaceProvenance; + query: string; + review?: CoverageReviewItem[]; +} diff --git a/test/agent-navigation-guidance.test.ts b/test/agent-navigation-guidance.test.ts new file mode 100644 index 00000000..58b2435f --- /dev/null +++ b/test/agent-navigation-guidance.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, test } from "vitest"; +import { createSystemPrompt } from "../src/agent/prompt.ts"; + +describe("repository coding-agent documentation guidance", () => { + test("requires change-oriented navigation and validation guidance", () => { + const prompt = createSystemPrompt("init", "repository"); + + expect(prompt).toContain("Coding-agent utility requirements"); + expect(prompt).toContain("compact task-routing table"); + expect(prompt).toContain("exact source entry points"); + expect(prompt).toContain("important symbols or types"); + expect(prompt).toContain("runtime invariants and lifecycle ordering"); + expect(prompt).toContain("evidence-backed change recipes"); + expect(prompt).toContain("complete change surface"); + expect(prompt).toContain("shipped-surface correctness"); + expect(prompt).toContain("consumer-facing smoke test"); + expect(prompt).toContain("behavioral test matrix"); + expect(prompt).toContain("isolation between independent instances"); + expect(prompt).toContain("scoped to `tests`"); + expect(prompt).toContain("observation-window boundaries"); + expect(prompt).toContain("net/coalesced effects"); + expect(prompt).toContain("narrow and quiet"); + expect(prompt).toContain("Label expensive checks as conditional"); + expect(prompt).toContain( + "simulate navigation for representative adjacent changes", + ); + }); + + test("does not apply repository coding guidance to the personal wiki", () => { + const prompt = createSystemPrompt("init", "local-wiki"); + + expect(prompt).not.toContain("Coding-agent utility requirements"); + expect(prompt).not.toContain("compact task-routing table"); + }); +}); diff --git a/test/code-mode.test.ts b/test/code-mode.test.ts index c841fb29..66cbdcfe 100644 --- a/test/code-mode.test.ts +++ b/test/code-mode.test.ts @@ -43,6 +43,16 @@ describe("ensureCodeModeRepoSetup agent files", () => { expect(content).toContain(SNIPPET_START); expect(content).toContain(SNIPPET_END); expect(content).toContain("## OpenWiki"); + expect(content).toContain("optional just-in-time context"); + expect(content).toContain("not required startup reading"); + expect(content).toContain("openwiki_retrieval.change_surface"); + expect(content).toContain("do not reread the returned wiki pages"); + expect(content).toContain("concrete unresolved evidence gap"); + expect(content).toContain("repository-relative changed paths"); + expect(content).toContain("verification gaps"); + expect(content).not.toContain("trace_symbols"); + expect(content).toContain("quiet validation"); + expect(content.length).toBeLessThan(2_500); } }); diff --git a/test/retrieval.test.ts b/test/retrieval.test.ts new file mode 100644 index 00000000..96071952 --- /dev/null +++ b/test/retrieval.test.ts @@ -0,0 +1,343 @@ +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, test } from "vitest"; +import { + RETRIEVAL_TOOL_DEFINITIONS, + SEARCH_SCOPES, +} from "../src/retrieval/mcp-tools.ts"; +import { tokenize } from "../src/retrieval/ranking.ts"; +import { RetrievalService } from "../src/retrieval/search-service.ts"; + +let root = ""; +let repoRoot = ""; +let wikiRoot = ""; + +beforeEach(async () => { + root = await mkdtemp(path.join(tmpdir(), "openwiki-retrieval-")); + repoRoot = path.join(root, "repo"); + wikiRoot = path.join(root, "wiki"); + await Promise.all([ + mkdir(path.join(repoRoot, "packages/core/src/query"), { recursive: true }), + mkdir(path.join(repoRoot, "packages/core/src/relation"), { + recursive: true, + }), + mkdir(path.join(repoRoot, "packages/core/tests"), { recursive: true }), + mkdir(path.join(repoRoot, "packages/publish/src"), { recursive: true }), + mkdir(path.join(repoRoot, "packages/publish/tests"), { recursive: true }), + mkdir(path.join(repoRoot, "secrets"), { recursive: true }), + mkdir(path.join(wikiRoot, "architecture"), { recursive: true }), + ]); + await Promise.all([ + writeFile( + path.join(wikiRoot, "quickstart.md"), + `--- +type: Quickstart +title: Koota quickstart +description: Routes query changes into runtime and package validation. +tags: [query, navigation] +--- + +# Quickstart + +For query changes, follow the [runtime contract](architecture/runtime.md). +`, + ), + writeFile( + path.join(wikiRoot, "architecture/runtime.md"), + `--- +type: Architecture +title: Query runtime and package contract +description: Connects predicate implementation to public exports and consumer tests. +tags: [query, package, runtime] +openwiki: + roles: [architecture, testing, delivery] + change_kinds: [public-api, lifecycle] + source_paths: + - packages/core/src/query/predicate.ts + - packages/core/src/index.ts + - packages/publish/src/index.ts + symbols: [createPredicate] + test_paths: [packages/core/tests/predicate.test.ts] + invariants: + - Predicate transitions must remain independent between instances. + validation_commands: [pnpm -F core test predicate.test.ts] +--- + +# Query runtime + +Implement predicates in \`packages/core/src/query/predicate.ts\`, export them from +\`packages/core/src/index.ts\`, mirror the public surface through +\`packages/publish/src/index.ts\`, and validate consumer imports in +\`packages/publish/tests/predicate.test.ts\`. + +Unchanged predicate inputs must not emit a transition. Public exports must remain +available from the consumer package. + +The [quickstart](../quickstart.md) routes adjacent changes here. +`, + ), + writeFile( + path.join(repoRoot, "packages/core/src/query/predicate.ts"), + "export const PUBLIC_PREDICATE_FACTORY = true;\nexport function createPredicate() { return true; }\n", + ), + writeFile( + path.join(repoRoot, "packages/core/src/relation/relation-events.ts"), + "export function removeRelationPair() { emitRelationEvent('remove'); }\nfunction emitRelationEvent(type: string) { return type; }\n", + ), + writeFile( + path.join(repoRoot, "packages/core/src/index.ts"), + "export { createPredicate } from './query/predicate';\n", + ), + writeFile( + path.join(repoRoot, "packages/publish/src/index.ts"), + "export { createPredicate } from '@koota/core';\n", + ), + writeFile( + path.join(repoRoot, "packages/core/tests/predicate.test.ts"), + "import { createPredicate } from '../src';\ndescribe('predicate lifecycle', () => {\n test('tracks false-to-true transitions independently', () => createPredicate());\n});\n", + ), + writeFile( + path.join(repoRoot, "packages/publish/tests/predicate.test.ts"), + "import { createPredicate } from 'koota';\ndescribe('predicate lifecycle', () => {\n test('tracks false-to-true transitions independently', () => createPredicate());\n});\n", + ), + writeFile( + path.join(repoRoot, "AGENTS.md"), + "Public API exports initialize register factory predicate relation removal tests.\n", + ), + writeFile( + path.join(repoRoot, "packages/core/tests/unrelated.test.ts"), + "test('generic public package initialization', () => true);\n", + ), + writeFile( + path.join(repoRoot, ".env"), + "SECRET_PREDICATE_SURFACE=never-index-this\n", + ), + writeFile( + path.join(repoRoot, "secrets/credentials.json"), + '{"note":"predicate consumer package"}\n', + ), + ]); +}); + +afterEach(async () => { + await rm(root, { force: true, recursive: true }); +}); + +function service(): RetrievalService { + return new RetrievalService({ + embeddingProvider: "local", + repoRoot, + wikiRoot, + }); +} + +describe("OKF-aware repository retrieval", () => { + test("exposes two concise workflow-oriented MCP tools", () => { + expect(RETRIEVAL_TOOL_DEFINITIONS.map((tool) => tool.name)).toEqual([ + "search", + "change_surface", + ]); + expect( + RETRIEVAL_TOOL_DEFINITIONS.every( + (tool) => + tool.description.length >= 100 && tool.description.length < 300, + ), + ).toBe(true); + expect(SEARCH_SCOPES).toEqual(["all", "wiki", "source_code", "tests"]); + }); + + test("automatically combines lexical, semantic, and OKF ranking", async () => { + const retrieval = service(); + const exact = await retrieval.search("createPredicate", "source_code", 5); + const concept = await retrieval.search("query navigation", "wiki", 5); + const consumer = await retrieval.search( + "consumer-facing package surface", + "all", + 5, + ); + + expect(exact.results[0]?.path).toMatch(/predicate|index/u); + expect(concept.results.map((hit) => hit.path)).toContain( + "openwiki/architecture/runtime.md", + ); + expect( + consumer.results.some( + (hit) => + hit.path.includes("runtime.md") || hit.path.includes("publish"), + ), + ).toBe(true); + }); + + test("matches snake and kebab compounds to camel-case source symbols", async () => { + expect(tokenize("name_mapping HttpApi http-api")).toEqual([ + "name", + "mapp", + "http", + "api", + "http", + "api", + ]); + await writeFile( + path.join(repoRoot, "packages/core/src/query/NameMapping.ts"), + "export class NameMapping {}\n", + ); + + const result = await service().search("name_mapping", "source_code", 5); + + expect(result.results[0]?.path).toContain("NameMapping.ts"); + }); + + test("supports distinct wiki, source_code, and tests scopes", async () => { + const retrieval = service(); + const source = await retrieval.search("createPredicate", "source_code", 10); + const tests = await retrieval.search( + "false-to-true independent predicate transition", + "tests", + 10, + ); + + expect(source.results.every((hit) => !/test|spec/iu.test(hit.path))).toBe( + true, + ); + expect(tests.scope).toBe("tests"); + expect(tests.results.length).toBeGreaterThan(0); + expect(tests.results.every((hit) => /test|spec/iu.test(hit.path))).toBe( + true, + ); + expect(tests.results.flatMap((hit) => hit.testNames ?? [])).toContain( + "tracks false-to-true transitions independently", + ); + expect( + tests.results.filter((hit) => hit.path.endsWith("predicate.test.ts")), + ).toHaveLength(1); + expect(tests.results[0]).not.toHaveProperty("signals"); + expect(tests.results[0]).not.toHaveProperty("score"); + }); + + test("clamps broad result requests to the public maximum", async () => { + const result = await service().search( + "predicate query public API", + "all", + 50, + ); + + expect(result.results.length).toBeLessThanOrEqual(10); + }); + + test("change_surface returns a bounded metadata-routed task brief", async () => { + const surface = await service().changeSurface( + "add createPredicate query API and track relation removal events", + 7, + ); + + expect(surface.brief.ownership[0]?.path).toContain("predicate.ts"); + expect(surface.brief.ownership[0]?.reason).toContain("OpenWiki"); + expect(surface.brief.invariants.map((item) => item.text).join(" ")).toMatch( + /independent|unchanged/iu, + ); + expect(surface.brief.tests[0]?.path).toContain("predicate.test.ts"); + expect(surface.brief.tests[0]?.testNames).toContain( + "tracks false-to-true transitions independently", + ); + expect(surface.brief.delivery.length).toBeGreaterThan(0); + expect(surface.brief.validation[0]?.command).toContain("pnpm -F core test"); + const evidence = [ + ...surface.brief.ownership, + ...surface.brief.tests, + ...surface.brief.delivery, + ]; + expect(evidence.every((result) => result.path !== "AGENTS.md")).toBe(true); + expect(surface.provenance.wikiConceptPaths).toContain( + "openwiki/architecture/runtime.md", + ); + expect(surface.provenance.metadataRoles).toContain("delivery"); + expect(surface.provenance.wikiReferencedSourcePaths).toContain( + "packages/core/src/query/predicate.ts", + ); + expect(JSON.stringify(surface).length).toBeLessThan(5_000); + }); + + test("change_surface reviews changed paths without treating gaps as requirements", async () => { + const surface = await service().changeSurface( + "add a public createPredicate API", + 6, + ["packages/core/src/query/predicate.ts"], + ); + + expect(surface.provenance.changedPaths).toEqual([ + "packages/core/src/query/predicate.ts", + ]); + expect(surface.review?.some((item) => item.path.includes("index.ts"))).toBe( + true, + ); + expect( + surface.review?.every((item) => item.reason.includes("verify")), + ).toBe(true); + await expect( + service().changeSurface("public API", 6, ["../.env"]), + ).rejects.toThrow("safe repository-relative paths"); + }); + + test("weak wiki matches do not invent invariants or symbol ownership", async () => { + await Promise.all([ + writeFile( + path.join(wikiRoot, "architecture/noisy-routing.md"), + `--- +type: Architecture +title: Metadata routing notes +description: Notes about OKF metadata routing and retrieval briefs. +tags: [okf, metadata, retrieval] +--- + +# Metadata routing notes + +Personal reminders should not be mixed into work commitments. The unrelated +example helper is \`wrongOwner\`. +`, + ), + writeFile( + path.join(repoRoot, "packages/core/src/query/metadata-router.ts"), + "export function routeOkfMetadataRetrievalBrief() { return true; }\n", + ), + writeFile( + path.join(repoRoot, "packages/core/src/relation/wrong-owner.ts"), + "export function wrongOwner() { return wrongOwner; }\n", + ), + ]); + + const surface = await service().changeSurface( + "improve OKF metadata routing retrieval briefs", + ); + + expect(surface.brief.ownership[0]?.path).toContain("metadata-router.ts"); + expect(surface.brief.invariants).toEqual([]); + expect(surface.brief.unknowns).toContain( + "No explicit behavioral invariant was found in the wiki.", + ); + }); + + test("never indexes secret-like files", async () => { + const result = await service().search( + "never-index-this credentials", + "all", + 50, + ); + + expect( + result.results.every( + (hit) => !/\.env|credentials\.json|secrets\//u.test(hit.path), + ), + ).toBe(true); + expect(result.results.map((hit) => hit.snippet).join("\n")).not.toContain( + "never-index-this", + ); + }); + + test("bounds query length", async () => { + const retrieval = service(); + await expect(retrieval.search("x".repeat(501), "all", 5)).rejects.toThrow( + "query must be", + ); + }); +});