From 28f2faad8d4f12ccd68f1df1563e69e62ef80a22 Mon Sep 17 00:00:00 2001 From: bracesproul Date: Mon, 27 Jul 2026 10:39:27 -0700 Subject: [PATCH 1/3] feat: Add DeepSWE evals and start hill climbing --- AGENTS.md | 14 +- CLAUDE.md | 14 +- README.md | 2 +- evals/deepswe/.gitignore | 6 + evals/deepswe/OPENWIKI_TRACE_FINDINGS.md | 77 +++ evals/deepswe/README.md | 188 +++++++ evals/deepswe/deepswe_langsmith.py | 69 +++ evals/deepswe/openwiki_codex.py | 365 ++++++++++++++ evals/deepswe/run.py | 595 +++++++++++++++++++++++ evals/deepswe/test_run.py | 295 +++++++++++ package.json | 3 +- src/agent/prompt.ts | 78 ++- src/code-mode.ts | 14 +- src/retrieval/mcp-server.ts | 283 +++++++++++ src/retrieval/ranking.ts | 276 +++++++++++ src/retrieval/repository-index.ts | 350 +++++++++++++ src/retrieval/search-service.ts | 522 ++++++++++++++++++++ src/retrieval/semantic.ts | 103 ++++ src/retrieval/types.ts | 85 ++++ test/agent-navigation-guidance.test.ts | 32 ++ test/code-mode.test.ts | 9 + test/prompt.test.ts | 65 --- test/retrieval.test.ts | 219 +++++++++ 23 files changed, 3570 insertions(+), 94 deletions(-) create mode 100644 evals/deepswe/.gitignore create mode 100644 evals/deepswe/OPENWIKI_TRACE_FINDINGS.md create mode 100644 evals/deepswe/README.md create mode 100644 evals/deepswe/deepswe_langsmith.py create mode 100644 evals/deepswe/openwiki_codex.py create mode 100644 evals/deepswe/run.py create mode 100644 evals/deepswe/test_run.py create mode 100644 src/retrieval/mcp-server.ts create mode 100644 src/retrieval/ranking.ts create mode 100644 src/retrieval/repository-index.ts create mode 100644 src/retrieval/search-service.ts create mode 100644 src/retrieval/semantic.ts create mode 100644 src/retrieval/types.ts create mode 100644 test/agent-navigation-guidance.test.ts delete mode 100644 test/prompt.test.ts create mode 100644 test/retrieval.test.ts diff --git a/AGENTS.md b/AGENTS.md index 959eadd5..d343f293 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,7 +14,19 @@ When working in this repository, read the OpenWiki quickstart first, then follow ## OpenWiki -This repository uses OpenWiki for recurring code documentation. Start with `openwiki/quickstart.md`, then follow its links to architecture, workflows, domain concepts, operations, integrations, testing guidance, and source maps. +This repository uses OpenWiki for recurring code documentation. Use `openwiki/` as a just-in-time repository index: + +- At task start, read `openwiki/quickstart.md`, then search the wiki for the task's concepts and read only the relevant linked pages. +- Before a repository-wide `rg`, `find`, or exploratory directory scan, check the wiki's source maps. When they name relevant files, symbols, or tests, inspect those paths directly. +- Re-consult the wiki when entering a different subsystem, when source evidence contradicts the current understanding, or when blocked by an unfamiliar test or build failure. +- Treat source code and tests as authoritative. Verify wiki claims in source before editing. +- Before finishing a public API or cross-package change, trace it from implementation through barrel/package exports, generated or publish mirrors, initialization or registration, and the import path consumers actually use. Consult the relevant wiki integration or delivery guidance and run the narrowest consumer-facing check; internal unit tests alone do not prove the shipped surface works. +- If an `openwiki_retrieval` MCP server is available, use `change_surface` before editing and `symbol_trace` after adding or changing each public symbol. Treat missing groups as verification gaps to investigate against repository architecture, not automatic requirements. +- For stateful or lifecycle changes, translate each externally observable acceptance criterion into a focused test checklist. Cover relevant initial state, both transition directions, unchanged updates, missing dependencies, independent instances, reset or reuse, deferred or re-entrant mutation, and composition; map every criterion to a passing test before finishing. +- If the retrieval server provides `test_search`, use it with that behavior matrix to find analogous focused tests, then inspect the cited tests directly before implementing lifecycle semantics. +- When the repository generates or copies package artifacts, identify the canonical source and repository-supported synchronization command. Do not hand-edit derived output unless the documented workflow explicitly requires it. +- Do not read operations, release, or integration pages unless the task affects those areas. If a requested feature does not exist yet, use the wiki to locate extension points, then inspect source rather than searching the wiki for the implementation. +- Do not reread pages already consulted unless new evidence requires it. The scheduled OpenWiki GitHub Actions workflow refreshes the repository wiki. Do not hand-edit generated OpenWiki pages unless explicitly asked; prefer updating source code/docs and letting OpenWiki regenerate. diff --git a/CLAUDE.md b/CLAUDE.md index 959eadd5..d343f293 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -14,7 +14,19 @@ When working in this repository, read the OpenWiki quickstart first, then follow ## OpenWiki -This repository uses OpenWiki for recurring code documentation. Start with `openwiki/quickstart.md`, then follow its links to architecture, workflows, domain concepts, operations, integrations, testing guidance, and source maps. +This repository uses OpenWiki for recurring code documentation. Use `openwiki/` as a just-in-time repository index: + +- At task start, read `openwiki/quickstart.md`, then search the wiki for the task's concepts and read only the relevant linked pages. +- Before a repository-wide `rg`, `find`, or exploratory directory scan, check the wiki's source maps. When they name relevant files, symbols, or tests, inspect those paths directly. +- Re-consult the wiki when entering a different subsystem, when source evidence contradicts the current understanding, or when blocked by an unfamiliar test or build failure. +- Treat source code and tests as authoritative. Verify wiki claims in source before editing. +- Before finishing a public API or cross-package change, trace it from implementation through barrel/package exports, generated or publish mirrors, initialization or registration, and the import path consumers actually use. Consult the relevant wiki integration or delivery guidance and run the narrowest consumer-facing check; internal unit tests alone do not prove the shipped surface works. +- If an `openwiki_retrieval` MCP server is available, use `change_surface` before editing and `symbol_trace` after adding or changing each public symbol. Treat missing groups as verification gaps to investigate against repository architecture, not automatic requirements. +- For stateful or lifecycle changes, translate each externally observable acceptance criterion into a focused test checklist. Cover relevant initial state, both transition directions, unchanged updates, missing dependencies, independent instances, reset or reuse, deferred or re-entrant mutation, and composition; map every criterion to a passing test before finishing. +- If the retrieval server provides `test_search`, use it with that behavior matrix to find analogous focused tests, then inspect the cited tests directly before implementing lifecycle semantics. +- When the repository generates or copies package artifacts, identify the canonical source and repository-supported synchronization command. Do not hand-edit derived output unless the documented workflow explicitly requires it. +- Do not read operations, release, or integration pages unless the task affects those areas. If a requested feature does not exist yet, use the wiki to locate extension points, then inspect source rather than searching the wiki for the implementation. +- Do not reread pages already consulted unless new evidence requires it. The scheduled OpenWiki GitHub Actions workflow refreshes the repository wiki. Do not hand-edit generated OpenWiki pages unless explicitly asked; prefer updating source code/docs and letting OpenWiki regenerate. diff --git a/README.md b/README.md index 1ccc96d0..4e53b9b6 100644 --- a/README.md +++ b/README.md @@ -171,7 +171,7 @@ Bare `openwiki` runs in code mode for the current repository. It creates initial Bare `openwiki --init` and `openwiki --update` default to code mode and operate on repository documentation. Use the `personal` positional mode or `--mode personal` to initialize or update the local personal brain wiki. -On each `code` run, `openwiki` maintains both an `AGENTS.md` and a `CLAUDE.md` at the repository root, adding prompting that instructs your coding agent to reference the wiki when searching for context. Each file is created if it does not already exist. If a file is present, OpenWiki only rewrites its own `…` block and leaves the rest of your content untouched (appending the block the first time). The scheduled GitHub Actions workflow includes these files, along with the workflow itself, in the documentation pull request. +On each `code` run, `openwiki` maintains both an `AGENTS.md` and a `CLAUDE.md` at the repository root, adding a just-in-time navigation workflow: start from the quickstart, search only relevant wiki pages, consult source maps before broad repository searches, and return to the wiki at subsystem or debugging boundaries. Source code and tests remain authoritative. Each file is created if it does not already exist. If a file is present, OpenWiki only rewrites its own `…` block and leaves the rest of your content untouched (appending the block the first time). The scheduled GitHub Actions workflow includes these files, along with the workflow itself, in the documentation pull request. Repository-specific wiki instructions are stored separately in `openwiki/INSTRUCTIONS.md`. This file is a shared, user-authored brief for the diff --git a/evals/deepswe/.gitignore b/evals/deepswe/.gitignore new file mode 100644 index 00000000..bec45047 --- /dev/null +++ b/evals/deepswe/.gitignore @@ -0,0 +1,6 @@ +.cache/ +artifacts/ +results/ +summaries/ +__pycache__/ +*.pyc diff --git a/evals/deepswe/OPENWIKI_TRACE_FINDINGS.md b/evals/deepswe/OPENWIKI_TRACE_FINDINGS.md new file mode 100644 index 00000000..d52f7c4e --- /dev/null +++ b/evals/deepswe/OPENWIKI_TRACE_FINDINGS.md @@ -0,0 +1,77 @@ +# OpenWiki trace findings: Koota DeepSWE + +## Scope + +This analysis compares 15 baseline and 15 OpenWiki trials: three attempts on each of five `pmndrs/koota` tasks. Wiki-generation tokens are excluded from the coding-agent token comparison. Wall-clock results are directionally useful but are confounded by concurrency, retries, and infrastructure variance. + +## Executive diagnosis + +OpenWiki improved full solves from **3/15 to 6/15** and reduced file-edit actions by **31%**. Its clearest benefit is better change-surface coverage: agents search less broadly, modify fewer files repeatedly, and more consistently validate package exports and consumer paths. It is most useful on changes that cross lifecycle, relation, and publish boundaries. + +That gain currently costs context. OpenWiki increased uncached coding-agent input by **36.5%**, cumulative input by **24.4%**, and total tool calls from **52.3 to 57.9** per trial. Output tokens were essentially flat. The excess comes primarily from large retrieval responses, direct full-page wiki reads that duplicate retrieval, and verbose build/test output—not from longer reasoning or more editing. + +The reported **35.5% lower mean end-to-end time should not yet be treated as a product claim**. The difficult pair, query, and composite tasks were much faster, but deferred mutation and entity snapshot were slower; run scheduling and infrastructure varied. A controlled sequential rerun is needed to isolate the effect. + +## Where OpenWiki helped + +| Signal | Baseline | OpenWiki | Interpretation | +| --- | ---: | ---: | --- | +| Full solves | 3/15 | 6/15 | Promising quality improvement; sample is still small | +| Mean partial score | 0.9892 | 0.9928 | Failures became narrower | +| File-edit actions/trial | 23.2 | 16.0 | Less rework and patch churn | +| `rg` commands/trial | 4.93 | 2.40 | Retrieval replaced broad text search | +| Tool calls/trial | 52.3 | 57.9 | Retrieval added more calls than it eliminated | +| Uncached input | baseline | +36.5% | Retrieval remains too expensive | + +The strongest task-level result was pair-relation tracking. Baseline trials failed across cancellation, exclusive replacement, destruction, wildcard removal, coexistence, and transition cases. OpenWiki narrowed this to one repeated mixed-requirement edge case and achieved one full solve. The traces show agents explicitly converting requirements into lifecycle checks, following the public package surface, and finding a bundler-only issue through consumer validation. + +Entity snapshots improved from 2/3 to 3/3 solves. Deferred mutation improved from 1/3 to 2/3, although its remaining failure exposed unmodeled net/coalesced effects such as add→remove and remove→add. Composite-aspect failures narrowed to unchanged-update, constructor-arity, and one `Not(aspect)` transition edge. + +## Where it did not help enough + +Query predicates remained 0/3. All OpenWiki trials missed the held-out `Changed(predicate)` and `Removed(predicate)` truth-transition semantics; some also missed `Added(predicate)` and tracker independence. Agents found the right subsystem and wrote plausible tests, but their tests did not reproduce the verifier's observation-window behavior. This is a semantic-modeling and test-design failure, not a navigation failure. + +The wiki should describe these runtime contracts explicitly: + +- Observation-window boundaries: when added, removed, and changed state becomes visible and when it resets. +- Tracker identity and isolation across predicate/query instances. +- Truth-transition state machines, including false→true, true→false, and unchanged updates. +- The interaction between static query constraints and temporal tracking constraints. +- Net/coalesced effects of deferred and re-entrant mutations. +- No-op update semantics and constructor invariants for composed aspects. + +These should be expressed as compact behavior matrices with links to authoritative source and focused tests. More architectural prose or more file snippets will not address the observed failures. + +## Experiment verdicts + +- **H1 surface gate and H2 OKF retrieval:** quality scores are invalid because these ran before patch transport was fixed. H2 still demonstrated a clear payload failure: `change_surface` returned 177k characters initially and 148k at final verification. +- **H3b compact retrieval plus `symbol_trace`:** compaction worked. `change_surface` fell to roughly 8–10k characters and the query task scored 39/43, but the run still used 126k uncached input tokens and 14 retrieval calls. +- **H4 behavior matrix:** the clearest win. It retained the same 39/43 score with 107k uncached input tokens and only five retrieval calls. Keep this policy. +- **H5 mandatory `test_search`:** no quality gain; the score remained 39/43 while uncached input rose to 130.5k and retrieval calls to nine. Keep test search optional until its precision improves. + +## Retrieval-tool decisions + +Across OpenWiki trials, retrieval was called 135 times, averaging nine calls and about 70k returned characters per trial. Fourteen calls (10.4%) were invalid because requested limits exceeded 20 or `symbol_trace` rejected dotted/multiple symbols. Fixing these retries is the first priority. + +| Tool | Calls | Decision | +| --- | ---: | --- | +| `symbol_trace` | 54 | Keep, but batch symbols, accept dotted names, cap output, and replace per-symbol prompting with one final surface audit | +| `change_surface` | 32 | Keep; make the initial result pointer-first and run final verification only for public/export/generated/registration changes | +| `test_search` | 16 | Keep optional; deduplicate canonical/generated mirrors and return exact test names plus short behavioral snippets | +| `hybrid_search` | 13 | Keep as the default broad discovery tool; it already incorporates semantic ranking | +| Keyword/BM25/OKF graph | 20 total | Preserve as retrieval engines, but consider exposing them as modes or fallbacks behind hybrid search rather than separate default tools | +| Standalone semantic search | 0 | Hide from the default surface unless it gains a distinct workflow; do not remove semantic ranking from hybrid search | + +`symbol_trace` is overused: 29 of its 54 calls came from the entity-snapshot task. `change_surface` is commonly called twice and sometimes three times with overlapping results. The tool surface should guide agents toward four workflows—change mapping, broad discovery, focused test discovery, and batched public-surface verification—rather than exposing every ranking implementation as a separate choice. + +## Recommended next experiments + +1. **Fix tool ergonomics:** clamp limits, accept dotted symbols, add multi-symbol tracing, and eliminate identical retry calls. +2. **Run the H4 policy with batched tracing:** compare current `symbol_trace` against one final batch audit. +3. **Make retrieval pointer-first:** return paths, symbols, test names, and small snippets by default; expand only on request. Target under 20k retrieval characters per trial. +4. **Improve `test_search`:** rank by requested transition behavior and observation phase, deduplicate generated mirrors, then compare optional use against H4 alone. +5. **Add quiet validation guidance:** capture failures in full but suppress successful build/test logs. OpenWiki trials produced substantially more validation output. +6. **Use query predicates as the discriminator:** test whether new observation-window and tracker-state wiki content converts the repeated 39/43 result into a full solve. +7. **Repeat timing under controlled scheduling:** same task order, concurrency, warmup, and infrastructure; report medians and successful-trial timing separately. + +The near-term objective should be to preserve OpenWiki's solve-rate and rework gains while removing duplicated context. The best current direction is **behavior-matrix prompting plus compact, workflow-oriented retrieval**, not mandatory use of more tools. diff --git a/evals/deepswe/README.md b/evals/deepswe/README.md new file mode 100644 index 00000000..12054012 --- /dev/null +++ b/evals/deepswe/README.md @@ -0,0 +1,188 @@ +# DeepSWE OpenWiki evaluation + +This harness runs a paired DeepSWE experiment with the same tasks, seed, model, +reasoning effort, attempts, and Harbor environment in both conditions: + +- `baseline`: Codex receives only the DeepSWE task and repository. +- `openwiki`: OpenWiki first documents an isolated clone of the agent-visible + repository, then the same Codex adapter is instructed to read the generated + quickstart and use OpenWiki's read-only OKF-aware retrieval MCP server before + solving the task. + +The harness pins: + +- DeepSWE commit `6db64a40f3318d8659238ff34a8cc4b491c49205` +- `harbor[langsmith]==0.20.0` +- `litellm==1.83.14` (Harbor's supported lower bound, pinned to avoid a newer + release's local Rust build requirement) +- Codex CLI `0.118.0` +- the current OpenWiki checkout, packed locally for each treatment run + +## Safety and isolation + +DeepSWE uses a separate verifier environment. Its held-out `tests/` and +`solution/` directories are not present in the agent container. The treatment +adapter additionally runs OpenWiki against `/tmp/openwiki-source`, a local clone +of `/app`; OpenWiki never runs from the benchmark task directory and cannot see +the verifier or reference solution. + +Generated wiki files remain outside `/app`, so DeepSWE's patch extraction cannot +include them. Codex is explicitly told that `/app` is the source of truth and +that all code changes belong there. + +DeepSWE v1.1 normally requires Pier 0.3's `pre_artifacts.sh` lifecycle to copy +committed work into its separate verifier. This harness retains Harbor 0.20 for +the official LangSmith integration, so the shared Codex adapter performs the +same validated base-to-final-HEAD diff capture into +`/logs/artifacts/model.patch`. It also configures a fixed repository-local eval +author so task-required commits succeed. Both baseline and OpenWiki conditions +use this identical compatibility path. + +The treatment installs the packed OpenWiki artifact with dependency lifecycle +scripts disabled, then explicitly rebuilds and verifies only the existing pinned +`better-sqlite3` native dependency required by OpenWiki's checkpointer. + +Credentials are injected at runtime by Harbor. They are never written into an +image, command argument, generated wiki, or result summary. Do not enable Harbor's +debug mode for credentialed runs. + +DeepSWE disables general container internet access. The harness narrowly allows +the Debian, NodeSource, npm, GitHub release-asset, and OpenAI API hosts needed to +install and run Codex and OpenWiki's pinned SQLite binding, plus the LangSmith +API and trace-ingest hosts required by every traced run. The adapter uses the +task image's existing Node runtime and installs the pinned Codex CLI directly, +avoiding Harbor's NVM bootstrap. +If `OPENAI_BASE_URL` uses another gateway, pass its hostname (not a URL) with +`--allow-host gateway.example.com`. The separate verifier environment remains +offline. + +## Requirements + +- Python 3.12 (Harbor's supported runtime; selected explicitly through `uvx`) +- `uv`/`uvx` +- `pnpm` +- Docker for local runs, or a configured Modal account +- `OPENAI_API_KEY` available in the process environment or an env file passed + by path with `--env-file` +- `LANGSMITH_API_KEY` available the same way + +The project does not add Harbor as a package dependency. `uvx` downloads the +pinned Apache-2.0 runner and its official LangSmith extra into its tool cache +when a run starts. + +## LangSmith datasets, experiments, and traces + +Every evaluation command enables Harbor's official `langsmith` plugin. The +plugin creates or updates one stable dataset named +`deepswe-openwiki-6db64a40f331` by default. Baseline and OpenWiki jobs use that +same dataset and create separate, uniquely named experiments whose names begin +with the corresponding Harbor job name (for example, +`pilot-01-baseline-seed-0` and `pilot-01-openwiki-seed-0`). +Ambient Harbor experiment-name/ID overrides are cleared to prevent the two +conditions from being merged accidentally. See the official +[LangSmith Harbor integration](https://docs.langchain.com/langsmith/harbor-integrations) +for the resulting run and feedback schema. + +The harness subclasses the official plugin only to omit DeepSWE's count metrics +(such as `f2p_total`) from LangSmith feedback. Harbor 0.20 otherwise sends all +numeric rewards as bounded scores, which LangSmith rejects when a count is +greater than one. Normalized metrics are rounded to LangSmith's supported four +decimal places, and the primary reward remains a feedback score. All counts +remain available in trial outputs and local results. + +Each experiment contains one root run per trial, environment/agent/verifier +phase runs, verifier reward feedback, Harbor error feedback, and reported token +and cost usage. The OpenWiki condition also enables OpenWiki's LangChain v2 +tracing and routes those generation traces to the OpenWiki experiment. Codex +CLI itself does not emit native LangSmith LLM/tool spans; Harbor still records +its agent phase, ATIF trajectory-derived totals, tokens, cost, result, and +feedback. + +Use `--langsmith-dataset NAME` to override the shared dataset. Self-hosted or +multi-workspace LangSmith installations can also use `--langsmith-endpoint URL` +and `--langsmith-workspace-id ID`. Dataset sync and fail-fast behavior are +always enabled so a run cannot silently omit its LangSmith evaluation record. + +## Commands + +Inspect both commands without downloading tasks, building images, or calling a +model: + +```bash +python3 evals/deepswe/run.py paired --n-tasks 2 --dry-run +``` + +Prepare the pinned DeepSWE checkout and pack the current OpenWiki source: + +```bash +python3 evals/deepswe/run.py prepare +``` + +Run only the baseline: + +```bash +source ~/.zshrc && python3 evals/deepswe/run.py baseline \ + --n-tasks 10 \ + --seed 0 \ + --model openai/gpt-5.6-terra \ + --reasoning-effort high +``` + +Run only the OpenWiki condition: + +```bash +source ~/.zshrc && python3 evals/deepswe/run.py openwiki \ + --n-tasks 10 \ + --seed 0 \ + --model openai/gpt-5.6-terra \ + --openwiki-model gpt-5.6-terra \ + --reasoning-effort high +``` + +Run both paired conditions and summarize them: + +```bash +source ~/.zshrc && python3 evals/deepswe/run.py paired \ + --run-name pilot-01 \ + --n-tasks 10 \ + --seed 0 \ + --model openai/gpt-5.6-terra \ + --openwiki-model gpt-5.6-terra \ + --reasoning-effort high +``` + +Use `--task ''` one or more times to select named tasks. The harness uses +`--seed` to sample one exact task list and passes that same list to both arms. +Use `--attempts 3` for repeated trials and `--environment modal` for Harbor's +hosted parallel environment. + +Treatment runs register `openwiki-retrieval-mcp` inside Codex's isolated home. +It provides keyword, BM25, semantic-vector, OKF graph, hybrid, and change-surface +tools over `/app` plus the generated wiki. Local deterministic vectors are the +default. Pass `--retrieval-embedding-provider openai` to opt into bounded +`text-embedding-3-small` reranking; provider failures fall back to local vectors. + +If runs already exist, summarize them without invoking Harbor: + +```bash +python3 evals/deepswe/run.py summarize --run-name pilot-01 --seed 0 +``` + +## Outputs and interpretation + +Harbor writes raw jobs to `evals/deepswe/results/`. The harness writes aggregate +JSON and trial-level CSV files to `evals/deepswe/summaries/`, including: + +- binary reward and exception type +- input, cached, and output tokens used by Codex +- Codex cost and agent steps +- agent and total wall-clock time +- OpenWiki generation wall-clock time + +Efficiency should be compared among successful trials as well as across all +trials. A faster failure is not an efficiency improvement. + +OpenWiki's current CLI does not expose generation token usage to Harbor's local +summary, so treatment summaries include its wall-clock time but not its tokens +or provider cost. Its LangSmith generation traces in the same experiment provide +generation-token details. diff --git a/evals/deepswe/deepswe_langsmith.py b/evals/deepswe/deepswe_langsmith.py new file mode 100644 index 00000000..2131f2ba --- /dev/null +++ b/evals/deepswe/deepswe_langsmith.py @@ -0,0 +1,69 @@ +"""DeepSWE-compatible feedback handling for Harbor's LangSmith plugin.""" + +from __future__ import annotations + +import math +import warnings +from typing import Any + +from harbor_langsmith import LangSmithPlugin +from requests import RequestException + + +class DeepSWELangSmithPlugin(LangSmithPlugin): + """Preserve count rewards as values instead of invalid LangSmith scores.""" + + def _post_feedback(self, payload: dict[str, Any]) -> None: + """Publish feedback without allowing telemetry failures to abort a trial.""" + try: + self._request( + "POST", "/feedback", json=payload, ok_statuses={200, 201, 409} + ) + except RequestException as exc: + response = exc.response + status = response.status_code if response is not None else "unavailable" + request_id = "unavailable" + if response is not None: + request_id = response.headers.get( + "x-request-id", response.headers.get("x-langsmith-trace", "unavailable") + ) + warnings.warn( + "LangSmith feedback publish failed and was skipped: " + f"error={type(exc).__name__}, status={status}, " + f"request_id={request_id}", + RuntimeWarning, + stacklevel=2, + ) + + def _create_feedback(self, run_id: str, result: Any) -> None: + if result.verifier_result is not None: + for key, value in result.verifier_result.rewards.items(): + payload: dict[str, Any] = { + "id": self._stable_uuid(run_id, "feedback", key), + "run_id": run_id, + "key": key, + "feedback_source_type": "api", + } + is_number = isinstance(value, (int, float)) and not isinstance( + value, bool + ) + if is_number and math.isfinite(float(value)) and -1 <= value <= 1: + # LangSmith rejects feedback scores with more than four decimal + # places. DeepSWE's partial reward uses full float precision. + payload["score"] = round(float(value), 4) + else: + continue + self._post_feedback(payload) + + if result.exception_info is not None: + self._post_feedback( + { + "id": self._stable_uuid(run_id, "feedback", "harbor_error"), + "run_id": run_id, + "key": "harbor_error", + "score": 1, + "value": result.exception_info.exception_type, + "comment": result.exception_info.exception_message, + "feedback_source_type": "api", + } + ) diff --git a/evals/deepswe/openwiki_codex.py b/evals/deepswe/openwiki_codex.py new file mode 100644 index 00000000..a1a2cf8d --- /dev/null +++ b/evals/deepswe/openwiki_codex.py @@ -0,0 +1,365 @@ +"""Harbor Codex adapters for paired DeepSWE/OpenWiki evaluations.""" + +from __future__ import annotations + +import json +import re +import shlex +import time +from pathlib import Path, PurePosixPath +from typing import Any + +from harbor.agents.installed.codex import Codex +from harbor.environments.base import BaseEnvironment +from harbor.models.agent.context import AgentContext +from harbor_langsmith import parent_env + + +_MODEL_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/@+-]*$") +_GIT_COMMIT_RE = re.compile(r"^[0-9a-f]{40}$") +_APP_DIR = PurePosixPath("/app") +_OPENWIKI_SOURCE_DIR = PurePosixPath("/tmp/openwiki-source") +_OPENWIKI_HOME_DIR = PurePosixPath("/tmp/openwiki-home") +_REMOTE_PACKAGE_PATH = PurePosixPath("/tmp/openwiki-eval.tgz") +_OPENWIKI_LOG_PATH = PurePosixPath("/logs/agent/openwiki.log") + + +class BaselineCodex(Codex): + """Codex with credential-safe Harbor command logging for the control arm.""" + + @staticmethod + def name() -> str: + return "codex-baseline" + + async def install(self, environment: BaseEnvironment) -> None: + """Install pinned Codex without Harbor's unnecessary NVM bootstrap.""" + + if await self._installed_codex_satisfies_version(environment): + return + if self._version is None or not re.fullmatch(r"\d+\.\d+\.\d+", self._version): + raise ValueError("a pinned semantic Codex version is required") + await self.exec_as_root( + environment, + command=( + "if command -v rg >/dev/null 2>&1; then :; " + "elif command -v apk >/dev/null 2>&1; then " + "apk add --no-cache ripgrep; " + "elif command -v apt-get >/dev/null 2>&1; then " + "apt-get update && apt-get install -y ripgrep; " + "elif command -v yum >/dev/null 2>&1; then " + "yum install -y ripgrep; " + "else echo 'No supported package manager for ripgrep' >&2; exit 1; fi" + ), + env={"DEBIAN_FRONTEND": "noninteractive"}, + ) + package = shlex.quote(f"@openai/codex@{self._version}") + await self.exec_as_agent( + environment, + command=( + f"npm install -g {package} --ignore-scripts --no-audit --no-fund && " + "codex --version" + ), + ) + + async def run( + self, + instruction: str, + environment: BaseEnvironment, + context: AgentContext, + ) -> None: + """Run Codex and capture committed work for DeepSWE's verifier. + + DeepSWE v1.1 normally relies on Pier's ``pre_artifacts.sh`` lifecycle. + Harbor 0.20 does not execute that hook, so the adapter performs the same + base-to-final-HEAD patch capture after the agent exits. + """ + + head_result = await self.exec_as_agent( + environment, + command="git rev-parse HEAD", + cwd=_APP_DIR.as_posix(), + ) + start_head = (head_result.stdout or "").strip() + if not _GIT_COMMIT_RE.fullmatch(start_head): + raise RuntimeError("Task repository returned an invalid starting commit") + await self.exec_as_agent( + environment, + command=( + "git config user.name 'DeepSWE Eval' && " + "git config user.email 'deepswe-eval@local.invalid'" + ), + cwd=_APP_DIR.as_posix(), + ) + try: + await super().run(instruction, environment, context) + finally: + patch_path = PurePosixPath("/logs/artifacts/model.patch") + await self.exec_as_agent( + environment, + command=( + "umask 077; mkdir -p /logs/artifacts && " + f"git diff --binary {shlex.quote(start_head)} HEAD > " + f"{shlex.quote(patch_path.as_posix())} && " + f"chmod 0600 {shlex.quote(patch_path.as_posix())}" + ), + cwd=_APP_DIR.as_posix(), + ) + + async def _exec( + self, + environment: BaseEnvironment, + command: str, + user: str | int | None = None, + env: dict[str, str] | None = None, + cwd: str | None = None, + timeout_sec: int | None = None, + ) -> Any: + """Execute without logging environment values or command output. + + Harbor's upstream helper includes the per-command environment in debug log + metadata. Eval jobs necessarily pass provider credentials, so this + adapter deliberately keeps environment values out of logs. + """ + + result = await environment.exec( + command=f"set -o pipefail; {command}", + user=user, + env=env, + cwd=cwd, + timeout_sec=timeout_sec, + ) + if result.return_code != 0: + raise RuntimeError( + f"Sandbox command failed with exit code {result.return_code}; " + "inspect the trial logs for non-sensitive diagnostics." + ) + return result + + +class OpenWikiCodex(BaselineCodex): + """Generate OpenWiki in an isolated clone before running the same Codex agent.""" + + def __init__( + self, + *args: Any, + openwiki_package: str, + openwiki_model: str, + openwiki_timeout_sec: int = 5400, + retrieval_embedding_provider: str = "local", + **kwargs: Any, + ) -> None: + package_path = Path(openwiki_package).expanduser().resolve() + if package_path.suffix != ".tgz" or not package_path.is_file(): + raise ValueError("openwiki_package must be an existing .tgz file") + if not _MODEL_ID_RE.fullmatch(openwiki_model): + raise ValueError("openwiki_model contains unsupported characters") + if openwiki_timeout_sec <= 0 or openwiki_timeout_sec > 14_400: + raise ValueError("openwiki_timeout_sec must be between 1 and 14400") + if retrieval_embedding_provider not in {"local", "openai"}: + raise ValueError("retrieval_embedding_provider must be local or openai") + + self._openwiki_package = package_path + self._openwiki_model = openwiki_model + self._openwiki_timeout_sec = openwiki_timeout_sec + self._retrieval_embedding_provider = retrieval_embedding_provider + super().__init__(*args, **kwargs) + + @staticmethod + def name() -> str: + return "codex-openwiki" + + async def setup(self, environment: BaseEnvironment) -> None: + await super().setup(environment) + await environment.upload_file( + self._openwiki_package, _REMOTE_PACKAGE_PATH.as_posix() + ) + await self.exec_as_root( + environment, + command=f"chmod 0644 {shlex.quote(_REMOTE_PACKAGE_PATH.as_posix())}", + ) + await self.exec_as_agent( + environment, + command=( + "if [ -s ~/.nvm/nvm.sh ]; then . ~/.nvm/nvm.sh; fi; " + f"npm install -g {shlex.quote(_REMOTE_PACKAGE_PATH.as_posix())} " + "--ignore-scripts --no-audit --no-fund && " + 'cd "$(npm root -g)/openwiki" && ' + "npm rebuild better-sqlite3 --foreground-scripts " + "--no-audit --no-fund && " + "node -e \"require('better-sqlite3')\" && " + "command -v openwiki >/dev/null" + ), + ) + openwiki_bin_result = await self.exec_as_agent( + environment, + command=( + "if [ -s ~/.nvm/nvm.sh ]; then . ~/.nvm/nvm.sh; fi; command -v openwiki" + ), + ) + openwiki_bin_lines = [ + line.strip() + for line in (openwiki_bin_result.stdout or "").splitlines() + if line.strip() + ] + openwiki_bin = openwiki_bin_lines[-1] if openwiki_bin_lines else "" + if not re.fullmatch(r"/[A-Za-z0-9._/@+-]+", openwiki_bin): + raise RuntimeError("OpenWiki installation returned an invalid binary path") + await self.exec_as_root( + environment, + command=(f"ln -sf {shlex.quote(openwiki_bin)} /usr/local/bin/openwiki"), + ) + retrieval_bin_result = await self.exec_as_agent( + environment, + command=( + "if [ -s ~/.nvm/nvm.sh ]; then . ~/.nvm/nvm.sh; fi; " + "command -v openwiki-retrieval-mcp" + ), + ) + retrieval_bin_lines = [ + line.strip() + for line in (retrieval_bin_result.stdout or "").splitlines() + if line.strip() + ] + retrieval_bin = retrieval_bin_lines[-1] if retrieval_bin_lines else "" + if not re.fullmatch(r"/[A-Za-z0-9._/@+-]+", retrieval_bin): + raise RuntimeError("OpenWiki retrieval installation returned an invalid path") + await self.exec_as_root( + environment, + command=( + f"ln -sf {shlex.quote(retrieval_bin)} " + "/usr/local/bin/openwiki-retrieval-mcp" + ), + ) + + def _build_register_mcp_servers_command(self) -> str: + """Register the fixed read-only OpenWiki retrieval server for Codex.""" + + provider = shlex.quote(self._retrieval_embedding_provider) + return ( + "codex mcp add openwiki_retrieval -- " + "/usr/local/bin/openwiki-retrieval-mcp " + f"--repo-root {_APP_DIR.as_posix()} " + f"--wiki-root {(_OPENWIKI_SOURCE_DIR / 'openwiki').as_posix()} " + f"--embedding-provider {provider}" + ) + + async def run( + self, + instruction: str, + environment: BaseEnvironment, + context: AgentContext, + ) -> None: + started = time.monotonic() + status = "failure" + try: + await self.exec_as_agent( + environment, + command=( + f"mkdir -p {shlex.quote(_OPENWIKI_HOME_DIR.as_posix())} && " + f"git clone --quiet --no-hardlinks " + f"{shlex.quote(_APP_DIR.as_posix())} " + f"{shlex.quote(_OPENWIKI_SOURCE_DIR.as_posix())}" + ), + timeout_sec=600, + ) + + trace_env = parent_env(self.context_id) + wiki_env = { + "HOME": _OPENWIKI_HOME_DIR.as_posix(), + "OPENAI_API_KEY": self._get_env("OPENAI_API_KEY") or "", + "LANGSMITH_API_KEY": self._get_env("LANGSMITH_API_KEY") or "", + "LANGCHAIN_TRACING_V2": "true", + "OPENWIKI_PROVIDER": "openai", + "OPENWIKI_MODEL_ID": self._openwiki_model, + "OPENWIKI_TELEMETRY_DISABLED": "1", + "DO_NOT_TRACK": "1", + **trace_env, + } + if project := trace_env.get("LANGSMITH_PROJECT"): + # OpenWiki currently uses the LangChain v2 tracing variable. + wiki_env["LANGCHAIN_PROJECT"] = project + for key in ("LANGSMITH_ENDPOINT", "LANGSMITH_WORKSPACE_ID"): + if value := self._get_env(key): + wiki_env[key] = value + if openai_base_url := self._get_env("OPENAI_BASE_URL"): + wiki_env["OPENAI_BASE_URL"] = openai_base_url + await self.exec_as_agent( + environment, + command=( + "if [ -s ~/.nvm/nvm.sh ]; then . ~/.nvm/nvm.sh; fi; " + "openwiki code --init --print " + f"> {shlex.quote(_OPENWIKI_LOG_PATH.as_posix())} 2>&1" + ), + env=wiki_env, + cwd=_OPENWIKI_SOURCE_DIR.as_posix(), + timeout_sec=self._openwiki_timeout_sec, + ) + status = "success" + finally: + elapsed = round(time.monotonic() - started, 3) + openwiki_metadata = { + "status": status, + "duration_seconds": elapsed, + "model": self._openwiki_model, + "quickstart": ( + _OPENWIKI_SOURCE_DIR / "openwiki" / "quickstart.md" + ).as_posix(), + } + context.metadata = { + **(context.metadata or {}), + "openwiki": openwiki_metadata, + } + (self.logs_dir / "openwiki.json").write_text( + json.dumps(openwiki_metadata, indent=2) + "\n", + encoding="utf-8", + ) + + quickstart_path = ( + _OPENWIKI_SOURCE_DIR / "openwiki" / "quickstart.md" + ).as_posix() + treatment_instruction = ( + "OpenWiki treatment condition: use the generated wiki and the read-only " + "openwiki_retrieval MCP tools as a just-in-time repository index. At task " + "start, call change_surface with the requested change, then " + f"read {quickstart_path}, search the wiki for the task concepts, and read " + "only the relevant linked pages. Before a repository-wide rg, find, or " + "exploratory directory scan, check the wiki source maps and inspect named " + "files, symbols, and tests directly. Re-consult the wiki when entering a " + "different subsystem, when source contradicts the current understanding, " + "or when blocked by an unfamiliar test or build failure. Do not read " + "operations, release, or integration pages unless the task affects them, " + "and do not reread pages without new evidence. Before finishing a public " + "API or cross-package change, trace the change from its implementation " + "through internal and package exports, generated or publish mirrors, " + "initialization or registration, and the import path real consumers use. " + "Consult the wiki's relevant integration or delivery guidance and run the " + "narrowest consumer-facing check; passing only internal unit tests does not " + "prove the shipped surface works. If the repository generates or copies " + "package artifacts, follow its documented synchronization workflow rather " + "than assuming the defining source module is sufficient. " + "For stateful or lifecycle behavior, turn every externally observable " + "acceptance criterion into a test checklist before editing. Where relevant, " + "cover initial state, false-to-true and true-to-false transitions, unchanged " + "updates, missing dependencies, independent instances, reset or reuse, " + "deferred or re-entrant mutation, and composition with adjacent features. " + "When behavior is unfamiliar, the relevant tests are large, or no analogous " + "focused check is known, use test_search with that behavior matrix and " + "inspect the cited tests directly. " + "Before committing, map each criterion to a passing focused test; one happy " + "path does not establish transition or isolation correctness. " + "Use hybrid_search for broad ranked discovery, okf_graph_search to follow " + "related concepts and cross-package relationships, semantic_search when " + "the repository uses unfamiliar vocabulary, BM25 for precise concepts, " + "test_search when analogous behavioral checks are needed, and keyword_search " + "for exact symbols. Verify all retrieval excerpts in " + "source before editing. After adding or changing a public symbol, call " + "symbol_trace for that exact identifier; investigate missing export, " + "publish, consumer, initialization, or test groups when the repository's " + "architecture requires them. Call change_surface again before finalizing " + "if the patch added a public API, generated artifact, or registration path. " + "The wiki is a navigation aid generated from the same base checkout. " + "Treat /app as the source of truth, make all code changes only in /app, " + "and do not edit /tmp/openwiki-source.\n\n" + f"{instruction}" + ) + await super().run(treatment_instruction, environment, context) diff --git a/evals/deepswe/run.py b/evals/deepswe/run.py new file mode 100644 index 00000000..40b714e0 --- /dev/null +++ b/evals/deepswe/run.py @@ -0,0 +1,595 @@ +#!/usr/bin/env python3 +"""Prepare, run, and summarize paired DeepSWE/OpenWiki evaluations.""" + +from __future__ import annotations + +import argparse +import csv +import fnmatch +import json +import os +import random +import re +import shlex +import subprocess +import sys +import tomllib +from collections import defaultdict +from datetime import datetime +from pathlib import Path +from typing import Any, Iterable, Sequence +from urllib.parse import urlsplit + + +DEEPSWE_REPOSITORY = "https://github.com/datacurve-ai/deep-swe.git" +DEEPSWE_COMMIT = "6db64a40f3318d8659238ff34a8cc4b491c49205" +HARBOR_PACKAGE = "harbor[langsmith]==0.20.0" +LITELLM_PACKAGE = "litellm==1.83.14" +CODEX_VERSION = "0.144.6" +DEFAULT_MODEL = "openai/gpt-5.6-terra" +DEFAULT_OPENWIKI_MODEL = "gpt-5.6-terra" +DEFAULT_LANGSMITH_DATASET = f"deepswe-openwiki-{DEEPSWE_COMMIT[:12]}" +DEFAULT_ALLOWED_HOSTS = ( + "deb.debian.org", + "deb.nodesource.com", + "registry.npmjs.org", + "github.com", + "release-assets.githubusercontent.com", + "api.openai.com", + "gateway.smith.langchain.com", + "api.smith.langchain.com", +) +SAFE_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/@+-]*$") +TASK_FILTER_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/@+*?\[\]-]*$") +DNS_LABEL_RE = re.compile(r"^[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?$") + +EVAL_DIR = Path(__file__).resolve().parent +PROJECT_ROOT = EVAL_DIR.parents[1] +DEFAULT_DEEPSWE_DIR = EVAL_DIR / ".cache" / "deep-swe" +DEFAULT_ARTIFACTS_DIR = EVAL_DIR / "artifacts" +DEFAULT_JOBS_DIR = EVAL_DIR / "results" +DEFAULT_SUMMARY_DIR = EVAL_DIR / "summaries" + +SENSITIVE_FLAGS = {"--agent-env", "--ae"} +LANGSMITH_ENV_UNSET = { + "HARBOR_LANGSMITH_EXPERIMENT", + "HARBOR_LANGSMITH_EXPERIMENT_ID", +} +SUMMARY_FIELDS = [ + "condition", + "task_name", + "trial_name", + "reward", + "exception_type", + "input_tokens", + "cache_tokens", + "output_tokens", + "cost_usd", + "agent_steps", + "agent_duration_seconds", + "total_duration_seconds", + "openwiki_duration_seconds", +] + + +def validate_id(value: str, label: str) -> str: + if not SAFE_ID_RE.fullmatch(value): + raise ValueError(f"{label} contains unsupported characters: {value!r}") + return value + + +def validate_task_filter(value: str) -> str: + if not TASK_FILTER_RE.fullmatch(value): + raise ValueError(f"task filter contains unsupported characters: {value!r}") + return value + + +def validate_endpoint(value: str) -> str: + parsed = urlsplit(value) + if ( + parsed.scheme not in {"http", "https"} + or not parsed.netloc + or parsed.username is not None + or parsed.password is not None + or parsed.query + or parsed.fragment + ): + raise ValueError( + "LangSmith endpoint must be an http(s) URL without credentials, " + "a query, or a fragment" + ) + return value + + +def validate_host(value: str) -> str: + labels = value.split(".") + if len(value) > 253 or any(not DNS_LABEL_RE.fullmatch(label) for label in labels): + raise ValueError(f"allowed host must be a plain DNS hostname: {value!r}") + return value.lower() + + +def display_command(argv: Sequence[str]) -> str: + """Render an argv list while redacting values of known sensitive flags.""" + + rendered: list[str] = [] + redact_next = False + for arg in argv: + if redact_next: + rendered.append("") + redact_next = False + continue + if arg in SENSITIVE_FLAGS: + rendered.append(arg) + redact_next = True + continue + rendered.append(arg) + return shlex.join(rendered) + + +def run_checked( + argv: Sequence[str], + *, + cwd: Path = PROJECT_ROOT, + dry_run: bool = False, + env_overrides: dict[str, str] | None = None, + env_unset: Iterable[str] = (), +) -> None: + print(f"+ {display_command(argv)}") + if dry_run: + return + env = os.environ.copy() + for key in env_unset: + env.pop(key, None) + env.update(env_overrides or {}) + subprocess.run(list(argv), cwd=cwd, env=env, check=True, shell=False) + + +def prepare_deepswe(destination: Path, *, dry_run: bool) -> None: + if not destination.exists(): + destination.parent.mkdir(parents=True, exist_ok=True) + run_checked( + ["git", "clone", "--depth", "1", DEEPSWE_REPOSITORY, str(destination)], + dry_run=dry_run, + ) + if dry_run and not destination.exists(): + print( + f"+ git -C {shlex.quote(str(destination))} " + f"checkout --detach {DEEPSWE_COMMIT}" + ) + return + + current = subprocess.run( + ["git", "-C", str(destination), "rev-parse", "HEAD"], + check=True, + capture_output=True, + text=True, + shell=False, + ).stdout.strip() + if current != DEEPSWE_COMMIT: + run_checked( + ["git", "-C", str(destination), "fetch", "origin", DEEPSWE_COMMIT], + dry_run=dry_run, + ) + run_checked( + ["git", "-C", str(destination), "checkout", "--detach", DEEPSWE_COMMIT], + dry_run=dry_run, + ) + + +def pack_openwiki(artifacts_dir: Path, *, dry_run: bool) -> Path: + package_path = artifacts_dir / "openwiki-eval.tgz" + if dry_run: + print( + "+ pnpm pack --pack-destination " + f"{shlex.quote(str(artifacts_dir))} && rename package to " + f"{shlex.quote(str(package_path))}" + ) + return package_path + + artifacts_dir.mkdir(parents=True, exist_ok=True) + for stale_package in artifacts_dir.glob("openwiki-*.tgz"): + stale_package.unlink() + run_checked( + ["pnpm", "pack", "--pack-destination", str(artifacts_dir)], + cwd=PROJECT_ROOT, + ) + created = sorted(artifacts_dir.glob("openwiki-*.tgz")) + if len(created) != 1: + candidates = sorted(artifacts_dir.glob("openwiki-*.tgz")) + if len(candidates) != 1: + raise RuntimeError("Could not identify the package produced by pnpm pack") + created = candidates + if package_path.exists(): + package_path.unlink() + created[0].replace(package_path) + return package_path + + +def harbor_args( + args: argparse.Namespace, + *, + condition: str, + package_path: Path | None = None, + selected_tasks: Sequence[str] | None = None, +) -> list[str]: + validate_id(args.model, "model") + validate_id(args.reasoning_effort, "reasoning effort") + validate_id(args.run_name, "run name") + for task in args.task: + validate_task_filter(task) + + job_name = f"{args.run_name}-{condition}-seed-{args.seed}" + command = [ + "uvx", + "--python", + "3.12", + "--from", + HARBOR_PACKAGE, + "--with", + LITELLM_PACKAGE, + "harbor", + "run", + "--path", + str(args.deepswe_dir / "tasks"), + "--jobs-dir", + str(args.jobs_dir), + "--job-name", + job_name, + "--agent", + ( + "openwiki_codex:BaselineCodex" + if condition == "baseline" + else "openwiki_codex:OpenWikiCodex" + ), + "--model", + args.model, + "--agent-kwarg", + f"reasoning_effort={args.reasoning_effort}", + "--agent-kwarg", + f"version={CODEX_VERSION}", + "--env", + args.environment, + "--n-attempts", + str(args.attempts), + "--n-concurrent", + str(args.concurrency), + "--n-tasks", + str(len(selected_tasks) if selected_tasks is not None else args.n_tasks), + "--plugin", + "deepswe_langsmith:DeepSWELangSmithPlugin", + "--yes", + ] + for task in selected_tasks if selected_tasks is not None else args.task: + command.extend(["--include-task-name", task]) + allowed_hosts = dict.fromkeys( + validate_host(host) for host in (*DEFAULT_ALLOWED_HOSTS, *args.allow_host) + ) + for host in allowed_hosts: + command.extend(["--allow-environment-host", host]) + if args.env_file is not None: + command.extend(["--env-file", str(args.env_file)]) + if condition == "openwiki": + if package_path is None: + raise ValueError("package_path is required for the OpenWiki condition") + validate_id(args.openwiki_model, "OpenWiki model") + command.extend( + [ + "--agent-kwarg", + f"openwiki_package={package_path.resolve()}", + "--agent-kwarg", + f"openwiki_model={args.openwiki_model}", + "--agent-kwarg", + f"openwiki_timeout_sec={args.openwiki_timeout}", + "--agent-kwarg", + f"retrieval_embedding_provider={args.retrieval_embedding_provider}", + ] + ) + return command + + +def ensure_credentials(args: argparse.Namespace) -> None: + if args.dry_run or args.env_file is not None: + return + if not os.environ.get("OPENAI_API_KEY"): + raise RuntimeError( + "OPENAI_API_KEY is not set. Source your shell configuration or pass " + "--env-file; the harness never reads or prints the key value." + ) + if not os.environ.get("LANGSMITH_API_KEY"): + raise RuntimeError( + "LANGSMITH_API_KEY is not set. Source your shell configuration or pass " + "--env-file; the harness never reads or prints the key value." + ) + + +def langsmith_env(args: argparse.Namespace) -> dict[str, str]: + """Return non-secret Harbor/LangSmith configuration for one job.""" + + validate_id(args.langsmith_dataset, "LangSmith dataset") + overrides = { + "PYTHONPATH": str(EVAL_DIR), + "HARBOR_LANGSMITH_DATASET": args.langsmith_dataset, + "HARBOR_LANGSMITH_SYNC_DATASET": "true", + "HARBOR_LANGSMITH_FAIL_FAST": "true", + } + if args.langsmith_endpoint: + overrides["LANGSMITH_ENDPOINT"] = validate_endpoint(args.langsmith_endpoint) + if args.langsmith_workspace_id: + validate_id(args.langsmith_workspace_id, "LangSmith workspace ID") + overrides["LANGSMITH_WORKSPACE_ID"] = args.langsmith_workspace_id + return overrides + + +def prepare(args: argparse.Namespace) -> Path: + prepare_deepswe(args.deepswe_dir, dry_run=args.dry_run) + return pack_openwiki(args.artifacts_dir, dry_run=args.dry_run) + + +def select_tasks(args: argparse.Namespace) -> list[str] | None: + """Select an exact, reproducible Harbor task set from the pinned checkout.""" + + tasks_dir = args.deepswe_dir / "tasks" + if not tasks_dir.is_dir(): + return None + candidates: list[tuple[str, str]] = [] + for config_path in sorted(tasks_dir.glob("*/task.toml")): + config = tomllib.loads(config_path.read_text(encoding="utf-8")) + configured_name = config.get("task", {}).get("name") + local_id = config_path.parent.name + validate_task_filter(local_id) + if isinstance(configured_name, str): + candidates.append((local_id, configured_name)) + if args.task: + candidates = [ + candidate + for candidate in candidates + if any( + fnmatch.fnmatchcase(candidate[0], pattern) + or fnmatch.fnmatchcase(candidate[1], pattern) + or fnmatch.fnmatchcase(candidate[1].split("/")[-1], pattern) + for pattern in args.task + ) + ] + if not candidates: + raise ValueError("No DeepSWE tasks matched the requested filters") + rng = random.Random(args.seed) + rng.shuffle(candidates) + return [local_id for local_id, _ in candidates[: args.n_tasks]] + + +def run_condition( + args: argparse.Namespace, + condition: str, + *, + package_path: Path | None = None, +) -> None: + ensure_credentials(args) + if not args.dry_run: + args.jobs_dir.mkdir(parents=True, exist_ok=True) + command = harbor_args( + args, + condition=condition, + package_path=package_path, + selected_tasks=select_tasks(args), + ) + run_checked( + command, + dry_run=args.dry_run, + env_overrides=langsmith_env(args), + env_unset=LANGSMITH_ENV_UNSET, + ) + + +def seconds_between(timing: dict[str, Any] | None) -> float | None: + if not timing or not timing.get("started_at") or not timing.get("finished_at"): + return None + started = datetime.fromisoformat(timing["started_at"]) + finished = datetime.fromisoformat(timing["finished_at"]) + return round((finished - started).total_seconds(), 3) + + +def load_trial_rows(job_dir: Path, condition: str) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for result_path in sorted(job_dir.glob("*/result.json")): + data = json.loads(result_path.read_text(encoding="utf-8")) + if "task_name" not in data: + continue + agent_result = data.get("agent_result") or {} + verifier_result = data.get("verifier_result") or {} + rewards = verifier_result.get("rewards") or {} + metadata = agent_result.get("metadata") or {} + openwiki = metadata.get("openwiki") or {} + exception = data.get("exception_info") or {} + rows.append( + { + "condition": condition, + "task_name": data["task_name"], + "trial_name": data.get("trial_name"), + "reward": rewards.get("reward"), + "exception_type": exception.get("exception_type"), + "input_tokens": agent_result.get("n_input_tokens"), + "cache_tokens": agent_result.get("n_cache_tokens"), + "output_tokens": agent_result.get("n_output_tokens"), + "cost_usd": agent_result.get("cost_usd"), + "agent_steps": ( + data.get("n_agent_steps") or agent_result.get("n_agent_steps") + ), + "agent_duration_seconds": seconds_between(data.get("agent_execution")), + "total_duration_seconds": seconds_between( + { + "started_at": data.get("started_at"), + "finished_at": data.get("finished_at"), + } + ), + "openwiki_duration_seconds": openwiki.get("duration_seconds"), + } + ) + return rows + + +def mean(values: Iterable[float | int | None]) -> float | None: + present = [float(value) for value in values if value is not None] + return round(sum(present) / len(present), 6) if present else None + + +def aggregate(rows: list[dict[str, Any]]) -> dict[str, Any]: + rewards = [row["reward"] for row in rows if row["reward"] in (0, 1)] + successful = [row for row in rows if row["reward"] == 1] + return { + "trials": len(rows), + "errors": sum(1 for row in rows if row["exception_type"]), + "invalid_rewards": sum( + 1 + for row in rows + if row["reward"] is not None and row["reward"] not in (0, 1) + ), + "solve_rate": mean(rewards), + "mean_input_tokens": mean(row["input_tokens"] for row in rows), + "mean_cache_tokens": mean(row["cache_tokens"] for row in rows), + "mean_output_tokens": mean(row["output_tokens"] for row in rows), + "mean_cost_usd": mean(row["cost_usd"] for row in rows), + "mean_agent_steps": mean(row["agent_steps"] for row in rows), + "mean_agent_duration_seconds": mean( + row["agent_duration_seconds"] for row in rows + ), + "mean_total_duration_seconds": mean( + row["total_duration_seconds"] for row in rows + ), + "mean_openwiki_duration_seconds": mean( + row["openwiki_duration_seconds"] for row in rows + ), + "successful_trials": len(successful), + "successful_mean_input_tokens": mean(row["input_tokens"] for row in successful), + "successful_mean_output_tokens": mean( + row["output_tokens"] for row in successful + ), + "successful_mean_agent_duration_seconds": mean( + row["agent_duration_seconds"] for row in successful + ), + } + + +def summarize(args: argparse.Namespace) -> tuple[Path, Path]: + baseline_dir = args.jobs_dir / f"{args.run_name}-baseline-seed-{args.seed}" + openwiki_dir = args.jobs_dir / f"{args.run_name}-openwiki-seed-{args.seed}" + rows = load_trial_rows(baseline_dir, "baseline") + load_trial_rows( + openwiki_dir, "openwiki" + ) + if not rows: + raise RuntimeError("No Harbor trial result.json files were found") + + by_condition: dict[str, list[dict[str, Any]]] = defaultdict(list) + for row in rows: + by_condition[row["condition"]].append(row) + + tasks_by_condition = { + condition: sorted(row["task_name"] for row in condition_rows) + for condition, condition_rows in by_condition.items() + } + paired_task_sets_match = tasks_by_condition.get( + "baseline" + ) == tasks_by_condition.get("openwiki") + summary = { + "run_name": args.run_name, + "seed": args.seed, + "paired_task_sets_match": paired_task_sets_match, + "conditions": { + condition: aggregate(condition_rows) + for condition, condition_rows in sorted(by_condition.items()) + }, + } + + args.summary_dir.mkdir(parents=True, exist_ok=True) + json_path = args.summary_dir / f"{args.run_name}-seed-{args.seed}.json" + csv_path = args.summary_dir / f"{args.run_name}-seed-{args.seed}-trials.csv" + json_path.write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8") + with csv_path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=SUMMARY_FIELDS) + writer.writeheader() + writer.writerows(rows) + print(json.dumps(summary, indent=2)) + print(f"Wrote {json_path}") + print(f"Wrote {csv_path}") + return json_path, csv_path + + +def add_common_options(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--deepswe-dir", type=Path, default=DEFAULT_DEEPSWE_DIR) + parser.add_argument("--artifacts-dir", type=Path, default=DEFAULT_ARTIFACTS_DIR) + parser.add_argument("--jobs-dir", type=Path, default=DEFAULT_JOBS_DIR) + parser.add_argument("--summary-dir", type=Path, default=DEFAULT_SUMMARY_DIR) + parser.add_argument("--run-name", default="deepswe-openwiki") + parser.add_argument("--model", default=DEFAULT_MODEL) + parser.add_argument("--openwiki-model", default=DEFAULT_OPENWIKI_MODEL) + parser.add_argument( + "--reasoning-effort", + choices=("low", "medium", "high", "xhigh", "max"), + default="high", + ) + parser.add_argument("--environment", choices=("docker", "modal"), default="docker") + parser.add_argument("--n-tasks", type=int, default=10) + parser.add_argument("--task", action="append", default=[]) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--attempts", type=int, default=1) + parser.add_argument("--concurrency", type=int, default=1) + parser.add_argument("--openwiki-timeout", type=int, default=5400) + parser.add_argument( + "--retrieval-embedding-provider", + choices=("local", "openai"), + default="local", + help="Vector engine used by the OpenWiki retrieval MCP server", + ) + parser.add_argument("--env-file", type=Path) + parser.add_argument( + "--allow-host", + action="append", + default=[], + help="Additional API gateway hostname allowed in the agent container", + ) + parser.add_argument("--langsmith-dataset", default=DEFAULT_LANGSMITH_DATASET) + parser.add_argument("--langsmith-endpoint") + parser.add_argument("--langsmith-workspace-id") + parser.add_argument("--dry-run", action="store_true") + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + for name in ("prepare", "baseline", "openwiki", "paired", "summarize"): + subparser = subparsers.add_parser(name) + add_common_options(subparser) + args = parser.parse_args(argv) + if args.n_tasks <= 0 or args.attempts <= 0 or args.concurrency <= 0: + parser.error("--n-tasks, --attempts, and --concurrency must be positive") + return args + + +def main(argv: Sequence[str] | None = None) -> int: + args = parse_args(argv) + try: + if args.command == "prepare": + prepare(args) + elif args.command == "baseline": + prepare_deepswe(args.deepswe_dir, dry_run=args.dry_run) + run_condition(args, "baseline") + elif args.command == "openwiki": + package_path = prepare(args) + run_condition(args, "openwiki", package_path=package_path) + elif args.command == "paired": + package_path = prepare(args) + run_condition(args, "baseline") + run_condition(args, "openwiki", package_path=package_path) + if not args.dry_run: + summarize(args) + elif args.command == "summarize": + summarize(args) + else: + raise AssertionError(f"Unhandled command: {args.command}") + except (OSError, ValueError, RuntimeError, subprocess.CalledProcessError) as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/deepswe/test_run.py b/evals/deepswe/test_run.py new file mode 100644 index 00000000..a51aa9f4 --- /dev/null +++ b/evals/deepswe/test_run.py @@ -0,0 +1,295 @@ +from __future__ import annotations + +import json +import os +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock, patch + +from requests import HTTPError, Response + +import deepswe_langsmith +import run as deepswe_run + + +class DeepSWEHarnessTests(unittest.TestCase): + def test_langsmith_feedback_rounds_scores_and_skips_invalid_metrics(self) -> None: + plugin = deepswe_langsmith.DeepSWELangSmithPlugin.__new__( + deepswe_langsmith.DeepSWELangSmithPlugin + ) + plugin._post_feedback = Mock() + result = SimpleNamespace( + verifier_result=SimpleNamespace( + rewards={ + "reward": 0, + "partial": 0.35877862595419846, + "count": 47, + "not_a_number": float("nan"), + "boolean": True, + } + ), + exception_info=None, + ) + + plugin._create_feedback("00000000-0000-0000-0000-000000000001", result) + + payloads = [call.args[0] for call in plugin._post_feedback.call_args_list] + self.assertEqual(["reward", "partial"], [payload["key"] for payload in payloads]) + self.assertEqual([0.0, 0.3588], [payload["score"] for payload in payloads]) + + def test_langsmith_feedback_http_error_does_not_abort_trial(self) -> None: + plugin = deepswe_langsmith.DeepSWELangSmithPlugin.__new__( + deepswe_langsmith.DeepSWELangSmithPlugin + ) + response = Response() + response.status_code = 422 + response.headers["x-request-id"] = "safe-request-id" + plugin._request = Mock(side_effect=HTTPError(response=response)) + + with self.assertWarnsRegex( + RuntimeWarning, "status=422, request_id=safe-request-id" + ): + plugin._post_feedback({"score": 0}) + + def test_langsmith_feedback_programming_error_still_propagates(self) -> None: + plugin = deepswe_langsmith.DeepSWELangSmithPlugin.__new__( + deepswe_langsmith.DeepSWELangSmithPlugin + ) + plugin._request = Mock(side_effect=TypeError("invalid payload")) + + with self.assertRaisesRegex(TypeError, "invalid payload"): + plugin._post_feedback({"score": 0}) + + def test_openwiki_install_rebuilds_only_native_sqlite_dependency(self) -> None: + adapter = (deepswe_run.EVAL_DIR / "openwiki_codex.py").read_text( + encoding="utf-8" + ) + self.assertIn("npm install -g", adapter) + self.assertIn("--ignore-scripts", adapter) + self.assertIn("npm rebuild better-sqlite3", adapter) + self.assertIn("require('better-sqlite3')", adapter) + + def test_adapter_captures_committed_patch_for_separate_verifier(self) -> None: + adapter = (deepswe_run.EVAL_DIR / "openwiki_codex.py").read_text( + encoding="utf-8" + ) + self.assertIn("_GIT_COMMIT_RE.fullmatch(start_head)", adapter) + self.assertIn("git config user.name 'DeepSWE Eval'", adapter) + self.assertIn("git diff --binary", adapter) + self.assertIn("/logs/artifacts/model.patch", adapter) + self.assertIn("chmod 0600", adapter) + + def test_openwiki_treatment_uses_just_in_time_navigation(self) -> None: + adapter = (deepswe_run.EVAL_DIR / "openwiki_codex.py").read_text( + encoding="utf-8" + ) + self.assertIn("just-in-time", adapter) + self.assertIn("Before a repository-wide rg", adapter) + self.assertIn("only the relevant linked pages", adapter) + self.assertIn("Re-consult the wiki", adapter) + self.assertIn("import path real consumers use", adapter) + self.assertIn("passing only internal unit tests", adapter) + self.assertIn("codex mcp add openwiki_retrieval", adapter) + self.assertIn("call change_surface", adapter) + self.assertIn("symbol_trace", adapter) + self.assertIn("okf_graph_search", adapter) + self.assertIn("every externally observable", adapter) + self.assertIn("independent instances", adapter) + self.assertIn("use test_search", adapter) + + def test_eval_defaults_use_terra_without_changing_openwiki_defaults(self) -> None: + args = deepswe_run.parse_args(["paired"]) + self.assertEqual("openai/gpt-5.6-terra", args.model) + self.assertEqual("gpt-5.6-terra", args.openwiki_model) + + def test_paired_commands_share_selection_and_agent_settings(self) -> None: + args = deepswe_run.parse_args( + [ + "paired", + "--n-tasks", + "7", + "--seed", + "42", + "--task", + "happy-dom-*", + "--dry-run", + ] + ) + package = args.artifacts_dir / "openwiki-eval.tgz" + baseline = deepswe_run.harbor_args(args, condition="baseline") + treatment = deepswe_run.harbor_args( + args, condition="openwiki", package_path=package + ) + + for flag in ( + "--path", + "--model", + "--env", + "--n-attempts", + "--n-concurrent", + "--n-tasks", + "--include-task-name", + ): + self.assertEqual( + baseline[baseline.index(flag) + 1], treatment[treatment.index(flag) + 1] + ) + self.assertIn("openwiki_codex:BaselineCodex", baseline) + self.assertIn("openwiki_codex:OpenWikiCodex", treatment) + self.assertEqual(2, baseline.count("--agent-kwarg")) + self.assertEqual(6, treatment.count("--agent-kwarg")) + self.assertIn("retrieval_embedding_provider=local", treatment) + self.assertIn(f"version={deepswe_run.CODEX_VERSION}", baseline) + self.assertIn("gateway.smith.langchain.com", baseline) + self.assertIn("api.smith.langchain.com", baseline) + self.assertEqual(1, baseline.count("--plugin")) + self.assertEqual( + "deepswe_langsmith:DeepSWELangSmithPlugin", + baseline[baseline.index("--plugin") + 1], + ) + self.assertNotEqual( + baseline[baseline.index("--job-name") + 1], + treatment[treatment.index("--job-name") + 1], + ) + + baseline_env = deepswe_run.langsmith_env(args) + treatment_env = deepswe_run.langsmith_env(args) + self.assertEqual( + baseline_env["HARBOR_LANGSMITH_DATASET"], + treatment_env["HARBOR_LANGSMITH_DATASET"], + ) + self.assertEqual("true", baseline_env["HARBOR_LANGSMITH_SYNC_DATASET"]) + self.assertEqual("true", baseline_env["HARBOR_LANGSMITH_FAIL_FAST"]) + + for host in deepswe_run.DEFAULT_ALLOWED_HOSTS: + self.assertIn(host, baseline) + self.assertIn(host, treatment) + + def test_custom_allowed_host_is_validated_and_included(self) -> None: + args = deepswe_run.parse_args( + ["baseline", "--allow-host", "Gateway.Example.com"] + ) + command = deepswe_run.harbor_args(args, condition="baseline") + self.assertIn("gateway.example.com", command) + + invalid = deepswe_run.parse_args( + ["baseline", "--allow-host", "https://gateway.example.com/v1"] + ) + with self.assertRaisesRegex(ValueError, "plain DNS hostname"): + deepswe_run.harbor_args(invalid, condition="baseline") + + def test_display_command_redacts_agent_env(self) -> None: + rendered = deepswe_run.display_command( + ["harbor", "run", "--agent-env", "API_TOKEN=example-sensitive-value"] + ) + self.assertNotIn("example-sensitive-value", rendered) + self.assertIn("", rendered) + + def test_credentials_require_openai_and_langsmith(self) -> None: + args = deepswe_run.parse_args(["baseline"]) + with patch.dict(os.environ, {}, clear=True): + with self.assertRaisesRegex(RuntimeError, "OPENAI_API_KEY"): + deepswe_run.ensure_credentials(args) + with patch.dict(os.environ, {"OPENAI_API_KEY": "present"}, clear=True): + with self.assertRaisesRegex(RuntimeError, "LANGSMITH_API_KEY"): + deepswe_run.ensure_credentials(args) + with patch.dict( + os.environ, + {"OPENAI_API_KEY": "present", "LANGSMITH_API_KEY": "present"}, + clear=True, + ): + deepswe_run.ensure_credentials(args) + + def test_langsmith_endpoint_rejects_embedded_credentials(self) -> None: + args = deepswe_run.parse_args( + ["baseline", "--langsmith-endpoint", "https://user:secret@example.com"] + ) + with self.assertRaisesRegex(ValueError, "without credentials"): + deepswe_run.langsmith_env(args) + + def test_run_clears_ambient_experiment_overrides(self) -> None: + with ( + patch.dict( + os.environ, + { + "HARBOR_LANGSMITH_EXPERIMENT": "ambient", + "HARBOR_LANGSMITH_EXPERIMENT_ID": "ambient-id", + }, + ), + patch.object(deepswe_run.subprocess, "run") as run, + ): + deepswe_run.run_checked( + ["harbor", "run"], + env_unset=deepswe_run.LANGSMITH_ENV_UNSET, + ) + child_env = run.call_args.kwargs["env"] + self.assertNotIn("HARBOR_LANGSMITH_EXPERIMENT", child_env) + self.assertNotIn("HARBOR_LANGSMITH_EXPERIMENT_ID", child_env) + + def test_seeded_task_selection_is_reproducible(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + for name in ("alpha", "beta", "gamma"): + task_dir = root / "tasks" / name + task_dir.mkdir(parents=True) + (task_dir / "task.toml").write_text( + f'[task]\nname = "datacurve/{name}"\n', encoding="utf-8" + ) + args = deepswe_run.parse_args( + [ + "paired", + "--deepswe-dir", + str(root), + "--n-tasks", + "2", + "--seed", + "42", + ] + ) + first = deepswe_run.select_tasks(args) + second = deepswe_run.select_tasks(args) + self.assertEqual(first, second) + self.assertEqual(2, len(first or [])) + self.assertTrue(all("/" not in task_id for task_id in first or [])) + + def test_load_and_aggregate_trial_rows(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + job_dir = Path(temp_dir) + trial_dir = job_dir / "trial-1" + trial_dir.mkdir() + (trial_dir / "result.json").write_text( + json.dumps( + { + "task_name": "example-task", + "trial_name": "example-task__attempt-1", + "started_at": "2026-07-21T10:00:00+00:00", + "finished_at": "2026-07-21T10:02:00+00:00", + "agent_execution": { + "started_at": "2026-07-21T10:00:30+00:00", + "finished_at": "2026-07-21T10:01:30+00:00", + }, + "n_agent_steps": 12, + "agent_result": { + "n_input_tokens": 100, + "n_cache_tokens": 40, + "n_output_tokens": 25, + "cost_usd": 0.5, + "metadata": {"openwiki": {"duration_seconds": 20.0}}, + }, + "verifier_result": {"rewards": {"reward": 1}}, + } + ), + encoding="utf-8", + ) + + rows = deepswe_run.load_trial_rows(job_dir, "openwiki") + self.assertEqual(1, len(rows)) + self.assertEqual(120.0, rows[0]["total_duration_seconds"]) + self.assertEqual(60.0, rows[0]["agent_duration_seconds"]) + self.assertEqual(20.0, rows[0]["openwiki_duration_seconds"]) + self.assertEqual(1.0, deepswe_run.aggregate(rows)["solve_rate"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/package.json b/package.json index e1ddfe47..1150a356 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,8 @@ "node": ">=22" }, "bin": { - "openwiki": "./dist/cli.js" + "openwiki": "./dist/cli.js", + "openwiki-retrieval-mcp": "./dist/retrieval/mcp-server.js" }, "files": [ "dist", diff --git a/src/agent/prompt.ts b/src/agent/prompt.ts index 560ded1d..4eb06fc2 100644 --- a/src/agent/prompt.ts +++ b/src/agent/prompt.ts @@ -32,11 +32,10 @@ Run discipline: - ${output.filesystemRootInstruction} - Never pass host absolute paths like /Users/... to filesystem tools; that creates nested paths inside the repo instead of touching the intended file. - Shell execute commands run on the host. If you use execute, run commands from the current runtime root unless a source-specific instruction explicitly tells you to inspect a connector raw file or configured local repository path. -- Do not exhaustively read every file. For a local knowledge wiki, inspect the existing wiki structure and only the relevant connector evidence or configured local repository paths. For an explicit repository source, inspect the repository tree, package/config files, README-style files, entrypoints, routing files, database/schema files, and representative files for each major domain. - Do not call glob with **/* from the root. Use targeted discovery by directory and extension. Prefer shell commands like rg --files with excludes for .git, node_modules, dist, build, cache directories, and existing generated wiki output. - Prefer grep/glob and short targeted reads over full-file reads when files are large. -- Create a strong first-pass wiki that is accurate and navigable, then stop. The wiki can be refined in later update runs. -- Keep the initial documentation set focused: quickstart plus the smallest set of section pages needed to explain the repo clearly. +- For an explicit repository source, inspect the repository tree, package and workspace manifests, README-style files, entrypoints, routing files, database/schema files, and representative implementation and test files for every important domain. +- Prioritize the most important, durable information. Keep individual pages concise and avoid redundant or low-signal detail, but do not use concision as a reason to omit important domains, independent components, or relationships. - ${output.searchBoundaryInstruction} Connector ingestion discipline: @@ -67,16 +66,19 @@ ${output.wikiFirstAnsweringInstruction} - When you do inspect raw data, keep reads narrow: list latest raw items for the relevant connector, open only the specific files needed, and summarize only the minimum evidence required to answer or update the wiki. Subagent discipline: -- You may use the task tool to parallelize read-only research during init and update runs when the repository has multiple substantial domains. -- Default to 1-2 subagents for large or unfamiliar repositories. Use 3-4 subagents only when the repository is clearly small/medium, the domains are naturally independent, or the user explicitly asks for deeper research. -- Subagents must only inspect and summarize. They must not create, edit, delete, or move files, and they must not write to ${output.docsLocation}. -- Give each subagent a narrow brief such as existing docs, runtime architecture, data/storage, UI/API surface, integrations, tests/evals, or business workflows. -- Ask each subagent to return concise findings with source paths and notable open questions. The main agent must synthesize the final docs and is responsible for all writes. -- Treat subagent reports as internal discovery notes. Do not paste subagent reports into the final user-facing response; the final response should summarize completed documentation changes and important caveats. +- Use the task tool when independent repository areas or cross-cutting concerns can be investigated or documented in parallel. Choose the number and sequence of subagents from the repository's discovered complexity rather than a preset limit. +- In a monorepo, consider assigning a scoped subagent to each substantial service, package, application, or workspace. Closely related small units may share a subagent when that produces a clearer domain boundary. +- Delegation is iterative, not one-and-done. After the first reports or drafts return, reassess coverage and spawn additional subagents for newly discovered components, cross-package workflows, shared contracts, contradictions, or evidence gaps. +- Give each subagent a narrow brief such as one service/package/workspace, existing docs, runtime architecture, data/storage, UI/API surface, integrations, tests/evals, or a cross-component business workflow. +- Subagents may inspect and summarize, or may draft/write explicitly assigned documentation pages when that improves throughput. Any delegated writes must stay inside ${output.docsLocation}, use non-overlapping page ownership, and follow the same source-grounding and security rules as the main agent. Never have parallel subagents edit the same file. +- Ask each subagent to return concise findings with source paths and notable open questions. The main agent is responsible for the final synthesized documentation state, including delegated writes. +- The main agent must review delegated pages, reconcile terminology and duplicated content, add cross-component context, and verify navigation and relationship links before finishing. +- Treat subagent reports as internal discovery notes. Do not paste reports into the final user-facing response; summarize completed documentation changes and important caveats. Planning discipline: -- After discovery and before writing final documentation, create a temporary ${output.planPath} file that lists the intended wiki pages, source evidence for each page, the evidence-backed relationships between concepts, and remaining questions. +- After discovery and before writing final documentation, create a temporary ${output.planPath} file that inventories the important domains and independent components, lists the intended wiki pages and source evidence for each page, records whether each area is documented, covered by another page, or deferred, and captures remaining questions. - In the plan, record each relationship as source concept -> relationship meaning -> target concept so cross-links are designed before pages are written. +- Revisit the plan after initial subagent findings. Expand or reorganize it when discovery reveals additional services, packages, workspaces, workflows, or cross-component relationships. - Use ${output.planPath} when writing this temporary plan with filesystem tools. - Before completing the run, delete ${output.planPath}. If there is no filesystem delete tool, use shell execute from the runtime root, for example ${output.removePlanCommand}. - Do not leave ${output.planPath} in the final wiki. @@ -129,13 +131,16 @@ Documentation goals: - Prefer clear Markdown with stable links between pages. - Organize the docs like human documentation, not a raw file inventory. - Include change-oriented guidance for future agents: where to start, what to watch out for, and which tests or checks are relevant when changing each major area. -- Keep the docs concise enough to maintain. Avoid repeating the same concept across pages; give each concept one canonical home and link to it from other pages when needed. +- Keep each page concise, specific, and centered on important information. Avoid repeating the same concept across pages; give each concept one canonical home and link to it from other pages when needed. Concision should reduce redundancy and verbosity, not repository coverage. - Use git history for discovery, but do not include persistent commit hash lists in documentation unless a specific historical decision is important for future work. +${createCodingAgentUtilityRequirements(outputMode, output)} + OKF relationship modeling: - Treat every non-reserved Markdown document as a concept node. Standard Markdown links between concept documents are directed relationship edges; tags, resource fields, directory placement, source-code references, and index.md links do not replace concept-to-concept links. - Model meaningful runtime, dependency, ownership, data-flow, security, lifecycle, and user-flow relationships, not only navigation from ${output.quickstartPath}. - Put a concept link in the sentence that explains the relationship. Use the surrounding prose to state its meaning, such as \`dispatches to\`, \`depends on\`, \`shares infrastructure with\`, \`is configured through\`, \`is surfaced by\`, or \`is secured by\`. +- When separate pages document services, packages, or workspaces that interact, link them at the point where the runtime call, dependency, shared data, ownership boundary, lifecycle, or contract is explained. Add links from both pages when the relationship is important to understanding each side. - Do not add links solely to increase graph density, and do not automatically add reciprocal links. Add an inverse link only when it helps explain the target concept and is supported by evidence. - ${output.quickstartPath} must link to every major concept for navigation, but quickstart and index links do not count toward the semantic relationship audit. - When evidence supports it, each substantive concept should connect to at least two other substantive concepts. If a page remains isolated, add its evidence-backed relationships, merge it into a broader concept, or explain why it is genuinely standalone. @@ -171,19 +176,21 @@ timestamp: Section quality rules: - Do not create a directory unless it represents a real documentation area. - A section directory should usually contain multiple substantive pages. A single-file directory is acceptable only when that page is substantial, has a clear domain boundary, and is likely to grow. -- Avoid thin pages. If a page would mostly be a stub, source map, or short note, merge it into ${output.quickstartPath} or a broader section page instead. -- Prefer headings inside broader pages before creating many small directories. - Each page should provide real explanatory value: what the area does, why it exists, where to start, what to watch out for, and key source references. -- Before finishing an init or update run, review the ${output.docsLocation} tree. Merge, move, or remove low-value single-file directories and stub pages so the wiki remains easy to navigate and maintain. -- For small scopes with about 10 or fewer primary source items, prefer ${output.quickstartPath} plus at most 1-2 supporting pages. Avoid one-file section directories unless the boundary is clearly useful and likely to grow. -- Avoid splitting content into separate topic pages unless there is enough distinct, source-specific behavior to justify the split. +- Before finishing an init or update run, review the ${output.docsLocation} tree. Remove low-value stubs and redundant content while preserving useful coverage of independent components and important relationships. + +Repository decomposition and coverage: +- For repository sources, identify independent services, applications, packages, libraries, and workspaces from manifests, build configuration, entrypoints, and directory boundaries before choosing the documentation structure. +- Give each substantial independent component its own page or clearly identifiable section when it has distinct responsibilities, runtime behavior, APIs, data ownership, dependencies, operational guidance, or tests. Closely coupled or very small components may share a page when their relationship is explained clearly. +- In a monorepo, organize service/package/workspace documentation so readers can navigate both by component and by cross-component workflow. Wiki breadth should reflect meaningful repository boundaries and complexity; do not force repositories of different sizes into a predetermined page count. +- Document the important responsibilities, interfaces, dependencies, data flows, operational constraints, extension points, and change-safety guidance for each component. Do not turn the wiki into a file-by-file inventory. Required documentation structure: - ${output.quickstartPath} must be the entrypoint. - ${output.quickstartPath} must include a high-level overview and links to every major section. - When writing required documentation with filesystem tools or narrow shell execute, use ${output.writePathExample}. - ${output.sectionDirectoryInstruction} -- Each section directory should contain focused Markdown pages; if a directory would contain only one short page, prefer a broader page or a heading in ${output.quickstartPath}. +- Each section directory should contain focused Markdown pages whose boundaries follow the repository's actual components and domains. - Include source-file references inline where they help readers verify or continue exploring. - Source Map sections are optional. Add one only when it materially improves navigation for that page. Prefer inline source references for short pages. - Track the last successful documentation update in ${output.metadataPath}. @@ -200,6 +207,30 @@ ${createModeInstructions(command, outputMode)} `.trim(); } +function createCodingAgentUtilityRequirements( + outputMode: OpenWikiOutputMode, + output: OutputPromptConfig, +): string { + if (outputMode !== "repository") { + return ""; + } + + return `Coding-agent utility requirements: +- Optimize the repository wiki to reduce exploratory source searches during future code changes. It must help an agent identify where to start, which invariants matter, and how to validate narrowly; it must not attempt to anticipate or encode a specific future task. +- ${output.quickstartPath} must contain a compact task-routing table with columns for change area or user intent, relevant wiki page, exact source entry points, important symbols or types, focused tests, and the minimal validation command. Route broad change categories supported by repository evidence, not hypothetical one-off features. +- Every substantive architecture, domain, runtime, workflow, integration, or operations page must make change navigation explicit when applicable: when to consult the page; runtime invariants and lifecycle ordering; extension points; exact source files and important symbols; focused tests; minimal validation commands; and scope boundaries such as generated files or broader checks that are normally unnecessary. +- Prefer symbol-level mappings such as Concept -> Public API -> Implementation -> Tests. Do not merely list directories. Explain why each path or symbol matters and what behavior it owns. Avoid stale line-number references; prefer stable paths and symbol names. +- Document evidence-backed change recipes for recurring extension seams discovered in source or recent history, such as adding a query/modifier, extending a domain abstraction, changing lifecycle behavior, adding persistence/serialization, or updating a public export. Each recipe should identify implementation seams, affected caches or lifecycle hooks, focused tests, likely non-goals, and escalation conditions. +- For every public or cross-package extension seam, document the complete change surface: implementation symbols; internal barrel exports; package or public entrypoints; generated, bundled, or publish mirrors; initialization, registration, or factory wiring; the consumer import path; focused internal tests; and consumer/package tests. Omit a layer only when repository evidence shows it does not exist. +- Make the distinction between internal correctness and shipped-surface correctness explicit. A new API is not complete merely because its defining module typechecks or its unit tests pass; future agents must be able to verify that the API resolves from the import path real consumers use and that required registration or generated artifacts are present. +- Separate ordinary focused checks from expensive integration, root-test, release, package-build, generated-artifact, and performance checks. Label expensive checks as conditional and state the source-backed condition that makes each one necessary. Do not encourage broad validation by default. +- When a change crosses a public, package, generated-artifact, or runtime-registration boundary, identify the narrowest consumer-facing smoke test or package validation command that exercises that boundary. Record any source-backed synchronization command and the canonical source of generated files so agents do not validate only an internal package or hand-edit derived output. +- For stateful or lifecycle extension seams, document a source-backed behavioral test matrix when applicable: initial state; both transition directions; unchanged updates; missing prerequisites; isolation between independent instances; reset or reuse; deferred or re-entrant mutation; and composition with adjacent features. Link each invariant to the narrowest existing test or test location so future agents can turn every externally observable acceptance criterion into a focused check. +- Make analogous tests retrievable by describing the behavior and invariant they exercise, not just the implementation symbol. When large test files cover multiple lifecycle phases, identify the relevant suite or stable test names so a future \`test_search\` can reach the right section without reading from the top. +- Keep navigation stable and concise: use one canonical home per concept, link to it instead of duplicating prose, and keep operational/release guidance out of runtime reading paths unless it is genuinely required. +- Before finishing, simulate navigation for representative adjacent changes grounded in the repository's actual components and history. Verify that a future agent can reach the first implementation files, important symbols/invariants, focused tests, and minimal validation command from the quickstart without a repository-wide search. Repair navigation gaps found by this audit.`; +} + export function createModeInstructions( command: OpenWikiCommand, outputMode: OpenWikiOutputMode = "local-wiki", @@ -225,8 +256,7 @@ export function createModeInstructions( - ${output.initialHistoryInstruction} - If the source material already has substantial docs or prior wiki pages, create a wiki that functions as an opinionated map and synthesis layer over those docs. - Create ${output.quickstartPath} first, then the linked section pages. -- Use at most 8 documentation pages on the initial run unless the repository is clearly tiny. -- Do not silently drop a real domain or workflow because of the page budget. If it is not fully documented, record it in the \`## Backlog\` section of ${output.quickstartPath} with its area name, source anchor, and a one-line reason. +- Do not silently drop a real domain, independent component, or workflow. Document it at the appropriate level or record it in the \`## Backlog\` section of ${output.quickstartPath} with its area name, source anchor, and a one-line reason. - Do not try to document every source file. Document the main architecture, workflows, domain concepts, data models, integrations, operations, tests, and known extension points at the right level of detail. - The CLI will record successful run metadata in ${output.metadataPath} after you finish. `.trim(); @@ -240,15 +270,15 @@ export function createModeInstructions( - If source-specific connector raw data paths are supplied, inspect those files and update the wiki from that local evidence. Do not run all connector ingestions from inside the agent. - ${output.updateEvidenceInstruction} - Before editing, build a docs impact plan from the changed source files: source change -> docs affected -> edit needed -> why. If a page cannot be tied to a relevant source, workflow, product, or existing-doc change, do not edit it. -- Update runs must be surgical. Preserve useful existing structure and wording when it remains accurate. Prefer replacing one stale sentence over adding new paragraphs. -- Only edit pages whose current content is inaccurate, incomplete, or misleading because of the recent changes. Do not refresh every page. +- Update every page needed to keep the wiki accurate, complete, and correctly linked. There is no preset limit on the number of pages or sections an update may change or add. +- Preserve useful existing structure and wording when it remains accurate, and avoid unrelated formatting or prose churn. +- Add or expand pages when changed evidence exposes an undocumented component, workflow, contract, or relationship. An update may improve incomplete coverage discovered during the run even when that work spans multiple pages. - Keep each concept in one canonical page. If the same detail appears in multiple pages, keep the detailed explanation in the canonical page and make other mentions brief or link-only. - Do not make formatting-only edits. Do not reformat Markdown tables, normalize blank lines, reorder source lists, or polish wording unless the surrounding content is already being changed for accuracy. - Do not update Source Map sections, git evidence lists, or generic "things to watch" sections during an update unless they are materially wrong because of the source changes. - Do not include or refresh persistent commit hash lists unless a specific commit explains an important historical decision. -- Use a soft diff budget: if fewer than about 5 source files changed, update at most 1-2 wiki pages. Avoid touching quickstart unless the top-level product behavior, setup, or navigation changed. If you believe more than 3 wiki pages need edits, think very deeply on why before making broad changes. - Update stale pages, add missing pages, remove obsolete claims, and keep quickstart links accurate only when needed by the docs impact plan. -- Promote a backlog entry when recent changes touch that area or the update has spare documentation budget, then document the area and remove the entry from the backlog. +- Promote backlog entries whenever the available evidence is sufficient to document them accurately, then remove the completed entries from the backlog. - Do not let the backlog grow silently: every identified area must remain either documented or represented by a concise backlog entry with a source anchor and reason. - Updates may be a no-op. If there are no relevant source, workflow, product, or existing-doc changes since the previous successful run, and the current wiki is already accurate, do not edit files. Say that the wiki is already current. - The CLI will record successful run metadata in ${output.metadataPath} after you finish. @@ -290,7 +320,7 @@ ${context.gitSummary} ` Update the existing OpenWiki documentation for ${output.subjectLabel}. -Inspect ${output.docsLocation}, identify recent source changes or newly ingested connector evidence, and refresh only the documentation pages directly affected by those changes. Use the git evidence below when available. Keep edits surgical: do not rewrite accurate sections, do not update source maps or git evidence just to refresh them, and do not make formatting-only changes. If the wiki is already current, do not edit files. The CLI will update ${output.metadataPath} only when OpenWiki content changes. +Inspect ${output.docsLocation}, identify recent source changes or newly ingested connector evidence, and update every documentation page needed to keep the wiki accurate, complete, and correctly linked. Use the git evidence below when available. Preserve unrelated accurate content and avoid formatting-only changes. If the wiki is already current, do not edit files. The CLI will update ${output.metadataPath} only when OpenWiki content changes. Last update metadata: ${formatLastUpdate(context.lastUpdate)} diff --git a/src/code-mode.ts b/src/code-mode.ts index f3447c5e..4e4e1aa1 100644 --- a/src/code-mode.ts +++ b/src/code-mode.ts @@ -126,7 +126,19 @@ function createCodeModeAgentsSnippet(): string { ## OpenWiki -This repository uses OpenWiki for recurring code documentation. Start with \`openwiki/quickstart.md\`, then follow its links to architecture, workflows, domain concepts, operations, integrations, testing guidance, and source maps. +This repository uses OpenWiki for recurring code documentation. Use \`openwiki/\` as a just-in-time repository index: + +- At task start, read \`openwiki/quickstart.md\`, then search the wiki for the task's concepts and read only the relevant linked pages. +- Before a repository-wide \`rg\`, \`find\`, or exploratory directory scan, check the wiki's source maps. When they name relevant files, symbols, or tests, inspect those paths directly. +- Re-consult the wiki when entering a different subsystem, when source evidence contradicts the current understanding, or when blocked by an unfamiliar test or build failure. +- Treat source code and tests as authoritative. Verify wiki claims in source before editing. +- Before finishing a public API or cross-package change, trace it from implementation through barrel/package exports, generated or publish mirrors, initialization or registration, and the import path consumers actually use. Consult the relevant wiki integration or delivery guidance and run the narrowest consumer-facing check; internal unit tests alone do not prove the shipped surface works. +- If an \`openwiki_retrieval\` MCP server is available, use \`change_surface\` before editing and \`symbol_trace\` after adding or changing each public symbol. Treat missing groups as verification gaps to investigate against repository architecture, not automatic requirements. +- For stateful or lifecycle changes, translate each externally observable acceptance criterion into a focused test checklist. Cover relevant initial state, both transition directions, unchanged updates, missing dependencies, independent instances, reset or reuse, deferred or re-entrant mutation, and composition; map every criterion to a passing test before finishing. +- If the retrieval server provides \`test_search\`, use it with that behavior matrix to find analogous focused tests, then inspect the cited tests directly before implementing lifecycle semantics. +- When the repository generates or copies package artifacts, identify the canonical source and repository-supported synchronization command. Do not hand-edit derived output unless the documented workflow explicitly requires it. +- Do not read operations, release, or integration pages unless the task affects those areas. If a requested feature does not exist yet, use the wiki to locate extension points, then inspect source rather than searching the wiki for the implementation. +- Do not reread pages already consulted unless new evidence requires it. The scheduled OpenWiki GitHub Actions workflow refreshes the repository wiki. Do not hand-edit generated OpenWiki pages unless explicitly asked; prefer updating source code/docs and letting OpenWiki regenerate. diff --git a/src/retrieval/mcp-server.ts b/src/retrieval/mcp-server.ts new file mode 100644 index 00000000..ee5de035 --- /dev/null +++ b/src/retrieval/mcp-server.ts @@ -0,0 +1,283 @@ +#!/usr/bin/env node + +import { createInterface } from "node:readline"; +import { OPENWIKI_VERSION } from "../constants.js"; +import { RetrievalService } from "./search-service.js"; +import type { EmbeddingProvider } from "./semantic.js"; +import type { SearchScope } from "./types.js"; + +interface JsonRpcRequest { + id?: number | string; + jsonrpc?: string; + method?: string; + params?: Record; +} + +const TOOL_DEFINITIONS = [ + tool( + "symbol_trace", + "After editing, re-index source and trace one exact public symbol through implementation, exports, publish/generated mirrors, initialization, consumer imports, and tests. Missing groups are verification gaps, not proof that a layer is required.", + querySchema({ limit: integerSchema(1, 12, 6) }), + ), + tool( + "change_surface", + "Find the complete change surface for a feature: relevant OKF concepts, implementation, exports, publish/generated mirrors, initialization, consumer imports, and tests. Use this first for public or cross-package changes.", + querySchema({ limit: integerSchema(1, 12, 6) }), + ), + tool( + "test_search", + "Find analogous focused tests using hybrid keyword, BM25, and semantic ranking restricted to test/spec source chunks. Use this to derive lifecycle, transition, isolation, reset, and composition checks before implementing stateful behavior.", + searchSchema(), + ), + tool( + "hybrid_search", + "Hybrid reciprocal-rank search across BM25, semantic vectors, weighted keywords, and the OKF concept graph.", + searchSchema(), + ), + tool( + "okf_graph_search", + "Search OKF concept metadata, then expand across semantic Markdown relationships, incoming links, and shared tags.", + querySchema({ + hops: integerSchema(0, 2, 1), + limit: integerSchema(1, 20, 8), + }), + ), + tool( + "semantic_search", + "Vector semantic search over bounded wiki/source candidates. The response reports whether OpenAI embeddings or the deterministic local vector fallback was used.", + searchSchema(), + ), + tool( + "bm25_search", + "BM25 lexical search over wiki sections and source-code chunks.", + searchSchema(), + ), + tool( + "keyword_search", + "Fast field-weighted exact and token search over OKF metadata, headings, paths, and content.", + searchSchema(), + ), +] as const; + +const options = parseOptions(process.argv.slice(2)); +const service = new RetrievalService(options); +const input = createInterface({ input: process.stdin, crlfDelay: Infinity }); + +input.on("line", (line) => { + void handleLine(line); +}); + +async function handleLine(line: string): Promise { + let request: JsonRpcRequest; + try { + request = JSON.parse(line) as JsonRpcRequest; + } catch { + writeError(null, -32700, "Invalid JSON-RPC message."); + return; + } + if (request.jsonrpc !== "2.0" || typeof request.method !== "string") { + writeError(request.id ?? null, -32600, "Invalid JSON-RPC request."); + return; + } + if (request.id === undefined) return; + try { + switch (request.method) { + case "initialize": + writeResult(request.id, { + capabilities: { tools: { listChanged: false } }, + instructions: + "Use change_surface first for public, cross-package, generated-artifact, or runtime-registration changes. Verify returned citations in source before editing. Use hybrid_search for broad discovery, okf_graph_search for related concepts, semantic_search for vocabulary mismatch, BM25 for precise terms, and keyword_search for exact symbols. All tools are read-only and return bounded excerpts.", + protocolVersion: "2025-06-18", + serverInfo: { name: "openwiki-retrieval", version: OPENWIKI_VERSION }, + }); + return; + case "ping": + writeResult(request.id, {}); + return; + case "tools/list": + writeResult(request.id, { tools: TOOL_DEFINITIONS }); + return; + case "tools/call": + await callTool(request.id, request.params ?? {}); + return; + default: + writeError(request.id, -32601, "Method not found."); + } + } catch (error) { + const message = + error instanceof Error ? error.message : "Retrieval failed."; + writeResult(request.id, { + content: [{ text: message.slice(0, 500), type: "text" }], + isError: true, + }); + } +} + +async function callTool( + id: number | string, + params: Record, +): Promise { + const name = typeof params.name === "string" ? params.name : ""; + const args = isRecord(params.arguments) ? params.arguments : {}; + const query = requiredString(args.query, "query"); + const limit = optionalInteger(args.limit, 8); + let result: unknown; + switch (name) { + case "symbol_trace": + result = await service.symbolTrace(query, optionalInteger(args.limit, 6)); + break; + case "change_surface": + result = await service.changeSurface( + query, + optionalInteger(args.limit, 6), + ); + break; + case "test_search": + result = await service.testSearch(query, optionalInteger(args.limit, 5)); + break; + case "hybrid_search": + result = await service.hybridSearch( + query, + optionalScope(args.scope), + limit, + ); + break; + case "okf_graph_search": + result = await service.okfGraphSearch( + query, + limit, + optionalInteger(args.hops, 1), + ); + break; + case "semantic_search": + result = await service.semanticSearch( + query, + optionalScope(args.scope), + limit, + ); + break; + case "bm25_search": + result = await service.bm25Search( + query, + optionalScope(args.scope), + limit, + ); + break; + case "keyword_search": + result = await service.keywordSearch( + query, + optionalScope(args.scope), + limit, + ); + break; + default: + throw new Error(`Unknown retrieval tool: ${name || "(missing)"}.`); + } + writeResult(id, { + content: [{ text: JSON.stringify(result), type: "text" }], + }); +} + +function parseOptions(args: string[]): { + embeddingProvider: EmbeddingProvider; + repoRoot: string; + wikiRoot: string; +} { + const values = new Map(); + for (let index = 0; index < args.length; index += 2) { + const flag = args[index]; + const value = args[index + 1]; + if (!flag?.startsWith("--") || !value || value.startsWith("--")) { + throw new Error( + "Expected --repo-root, --wiki-root, and optional --embedding-provider values.", + ); + } + values.set(flag, value); + } + const provider = values.get("--embedding-provider") ?? "local"; + if (provider !== "local" && provider !== "openai") { + throw new Error("embedding provider must be local or openai."); + } + return { + embeddingProvider: provider, + repoRoot: values.get("--repo-root") ?? process.cwd(), + wikiRoot: values.get("--wiki-root") ?? `${process.cwd()}/openwiki`, + }; +} + +function tool(name: string, description: string, inputSchema: object): object { + return { + annotations: { destructiveHint: false, readOnlyHint: true }, + description, + inputSchema, + name, + }; +} + +function searchSchema(): object { + return querySchema({ + limit: integerSchema(1, 10, 5), + scope: { + default: "all", + enum: ["all", "wiki", "source"], + type: "string", + }, + }); +} + +function querySchema(properties: Record): object { + return { + additionalProperties: false, + properties: { + query: { maxLength: 500, minLength: 1, type: "string" }, + ...properties, + }, + required: ["query"], + type: "object", + }; +} + +function integerSchema( + minimum: number, + maximum: number, + defaultValue: number, +): object { + return { default: defaultValue, maximum, minimum, type: "integer" }; +} + +function requiredString(value: unknown, name: string): string { + if (typeof value !== "string" || !value.trim()) { + throw new Error(`${name} is required.`); + } + return value; +} + +function optionalInteger(value: unknown, fallback: number): number { + return typeof value === "number" && Number.isInteger(value) + ? value + : fallback; +} + +function optionalScope(value: unknown): SearchScope { + return value === "source" || value === "wiki" || value === "all" + ? value + : "all"; +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function writeResult(id: number | string, result: unknown): void { + process.stdout.write(`${JSON.stringify({ id, jsonrpc: "2.0", result })}\n`); +} + +function writeError( + id: number | string | null, + code: number, + message: string, +): void { + process.stdout.write( + `${JSON.stringify({ error: { code, message }, id, jsonrpc: "2.0" })}\n`, + ); +} diff --git a/src/retrieval/ranking.ts b/src/retrieval/ranking.ts new file mode 100644 index 00000000..acf7fa87 --- /dev/null +++ b/src/retrieval/ranking.ts @@ -0,0 +1,276 @@ +import type { IndexedChunk, RankedHit } from "./types.js"; + +const STOP_WORDS = new Set([ + "a", + "an", + "and", + "are", + "as", + "at", + "be", + "by", + "for", + "from", + "how", + "in", + "is", + "it", + "of", + "on", + "or", + "that", + "the", + "this", + "to", + "with", +]); + +const SYNONYM_GROUPS = [ + [ + "api", + "consumer", + "entrypoint", + "export", + "package", + "public", + "publish", + "surface", + ], + ["build", "bundle", "copy", "dist", "generated", "mirror", "release", "sync"], + ["factory", "initialize", "install", "register", "registry", "setup", "wire"], + ["assert", "check", "spec", "test", "validate", "verify"], + ["defer", "buffer", "batch", "command", "flush", "queue"], + ["predicate", "filter", "query", "select", "where"], + ["relation", "edge", "link", "pair", "target"], + ["aspect", "composite", "trait", "mixin", "schema"], + ["diff", "restore", "rollback", "snapshot", "state"], +] as const; + +const SYNONYMS = buildSynonyms(); + +export function tokenize(value: string): string[] { + const separated = value + .replace(/([a-z0-9])([A-Z])/gu, "$1 $2") + .replace(/([A-Z]+)([A-Z][a-z])/gu, "$1 $2") + .toLowerCase(); + const terms = separated.match(/[a-z0-9][a-z0-9_-]*/gu) ?? []; + return terms + .map((term) => stem(term.replace(/[_-]+/gu, ""))) + .filter((term) => term.length > 1 && !STOP_WORDS.has(term)); +} + +export function expandQueryTerms(query: string): string[] { + const base = tokenize(query); + const expanded = new Set(base); + for (const term of base) { + for (const synonym of SYNONYMS.get(term) ?? []) expanded.add(synonym); + } + return [...expanded]; +} + +export function rankKeyword( + chunks: IndexedChunk[], + query: string, +): RankedHit[] { + const phrase = query.trim().toLowerCase(); + const queryTerms = expandQueryTerms(query); + return chunks + .map((chunk) => { + const path = chunk.path.toLowerCase(); + const title = `${chunk.title ?? ""} ${chunk.heading ?? ""}`.toLowerCase(); + const metadata = chunk.fields.toLowerCase(); + const text = chunk.text.toLowerCase(); + let score = 0; + if (phrase) { + if (path.includes(phrase)) score += 10; + if (title.includes(phrase)) score += 9; + if (metadata.includes(phrase)) score += 7; + if (text.includes(phrase)) score += 5; + } + for (const term of queryTerms) { + if (path.includes(term)) score += 3.5; + if (title.includes(term)) score += 3; + if (metadata.includes(term)) score += 2; + if (text.includes(term)) score += 1; + } + return { chunk, score }; + }) + .filter((hit) => hit.score > 0) + .sort(compareHits); +} + +export function rankBm25(chunks: IndexedChunk[], query: string): RankedHit[] { + const queryTerms = expandQueryTerms(query); + if (queryTerms.length === 0 || chunks.length === 0) return []; + const documents = chunks.map((chunk) => tokenize(searchableText(chunk))); + const averageLength = + documents.reduce((sum, terms) => sum + terms.length, 0) / + documents.length || 1; + const documentFrequency = new Map(); + for (const terms of documents) { + for (const term of new Set(terms)) { + documentFrequency.set(term, (documentFrequency.get(term) ?? 0) + 1); + } + } + const k1 = 1.5; + const b = 0.75; + return chunks + .map((chunk, index) => { + const terms = documents[index] ?? []; + const frequencies = frequenciesOf(terms); + let score = 0; + for (const term of queryTerms) { + const frequency = frequencies.get(term) ?? 0; + if (frequency === 0) continue; + const df = documentFrequency.get(term) ?? 0; + const idf = Math.log(1 + (chunks.length - df + 0.5) / (df + 0.5)); + const denominator = + frequency + k1 * (1 - b + b * (terms.length / averageLength)); + score += idf * ((frequency * (k1 + 1)) / denominator); + } + if ( + chunk.title && + queryTerms.some((term) => tokenize(chunk.title ?? "").includes(term)) + ) { + score *= 1.35; + } + if (queryTerms.some((term) => tokenize(chunk.path).includes(term))) + score *= 1.2; + return { chunk, score }; + }) + .filter((hit) => hit.score > 0) + .sort(compareHits); +} + +export function rankLocalVectors( + chunks: IndexedChunk[], + query: string, +): RankedHit[] { + const queryVector = vectorize(query); + return chunks + .map((chunk) => ({ + chunk, + score: cosine(queryVector, vectorize(searchableText(chunk))), + })) + .filter((hit) => hit.score > 0) + .sort(compareHits); +} + +export function reciprocalRankFusion( + rankedLists: { hits: RankedHit[]; name: string; weight: number }[], + k = 60, +): RankedHit[] { + const fused = new Map(); + for (const list of rankedLists) { + list.hits.forEach((hit, index) => { + const contribution = list.weight / (k + index + 1); + const existing = fused.get(hit.chunk.id) ?? { + chunk: hit.chunk, + score: 0, + signals: {}, + }; + existing.score += contribution; + existing.signals = { + ...(existing.signals ?? {}), + [list.name]: hit.score, + }; + fused.set(hit.chunk.id, existing); + }); + } + return [...fused.values()].sort(compareHits); +} + +export function searchableText(chunk: IndexedChunk): string { + return [ + chunk.path, + chunk.title, + chunk.heading, + chunk.type, + chunk.tags.join(" "), + chunk.fields, + chunk.text, + ] + .filter(Boolean) + .join("\n"); +} + +function vectorize(value: string, dimensions = 768): Float64Array { + const vector = new Float64Array(dimensions); + const terms = expandTermsForVector(value); + for (const term of terms) { + const index = fnv1a(term) % dimensions; + const sign = (fnv1a(`sign:${term}`) & 1) === 0 ? 1 : -1; + vector[index] += sign; + } + return vector; +} + +function expandTermsForVector(value: string): string[] { + const terms = tokenize(value); + const expanded = [...terms]; + for (const term of terms) { + for (const synonym of SYNONYMS.get(term) ?? []) expanded.push(synonym); + } + for (let index = 0; index + 1 < terms.length; index += 1) { + expanded.push(`${terms[index]}:${terms[index + 1]}`); + } + return expanded; +} + +function cosine(left: Float64Array, right: Float64Array): number { + let dot = 0; + let leftNorm = 0; + let rightNorm = 0; + for (let index = 0; index < left.length; index += 1) { + const l = left[index] ?? 0; + const r = right[index] ?? 0; + dot += l * r; + leftNorm += l * l; + rightNorm += r * r; + } + return leftNorm && rightNorm ? dot / Math.sqrt(leftNorm * rightNorm) : 0; +} + +function frequenciesOf(terms: string[]): Map { + const frequencies = new Map(); + for (const term of terms) + frequencies.set(term, (frequencies.get(term) ?? 0) + 1); + return frequencies; +} + +function compareHits(left: RankedHit, right: RankedHit): number { + return ( + right.score - left.score || left.chunk.path.localeCompare(right.chunk.path) + ); +} + +function stem(term: string): string { + if (term.length > 5 && term.endsWith("ing")) return term.slice(0, -3); + if (term.length > 4 && term.endsWith("ed")) return term.slice(0, -2); + if (term.length > 4 && term.endsWith("es")) return term.slice(0, -2); + if (term.length > 3 && term.endsWith("s")) return term.slice(0, -1); + return term; +} + +function buildSynonyms(): Map { + const result = new Map(); + for (const group of SYNONYM_GROUPS) { + const normalized = group.map((term) => stem(term)); + for (const term of normalized) { + result.set( + term, + normalized.filter((candidate) => candidate !== term), + ); + } + } + return result; +} + +function fnv1a(value: string): number { + let hash = 0x811c9dc5; + for (let index = 0; index < value.length; index += 1) { + hash ^= value.charCodeAt(index); + hash = Math.imul(hash, 0x01000193); + } + return hash >>> 0; +} diff --git a/src/retrieval/repository-index.ts b/src/retrieval/repository-index.ts new file mode 100644 index 00000000..c6280331 --- /dev/null +++ b/src/retrieval/repository-index.ts @@ -0,0 +1,350 @@ +import { lstat, readFile, readdir, realpath, stat } from "node:fs/promises"; +import path from "node:path"; +import { + parseFrontmatterFields, + splitFrontmatter, +} from "../okf/frontmatter.js"; +import type { + IndexedChunk, + OkfConcept, + OkfRelationship, + RepositoryCorpus, +} from "./types.js"; + +const MAX_FILE_BYTES = 256_000; +const MAX_FILES = 5_000; +const SOURCE_LINES_PER_CHUNK = 80; +const SOURCE_LINE_OVERLAP = 16; +const SOURCE_EXTENSIONS = new Set([ + ".c", + ".cc", + ".cpp", + ".css", + ".go", + ".h", + ".hpp", + ".html", + ".java", + ".js", + ".json", + ".jsx", + ".md", + ".mjs", + ".py", + ".rb", + ".rs", + ".sh", + ".sql", + ".toml", + ".ts", + ".tsx", + ".yaml", + ".yml", +]); +const EXCLUDED_DIRECTORIES = new Set([ + ".cache", + ".git", + ".hg", + ".next", + ".svn", + ".turbo", + ".venv", + "build", + "coverage", + "dist", + "node_modules", + "target", + "vendor", +]); +const SECRET_FILE = + /^(?:\.env(?:\..*)?|.*\.(?:crt|jks|key|keystore|p12|pem|pfx)|credentials\.json|token(?:\.json)?|cookies?(?:\.(?:db|sqlite|txt))?|\.git-credentials|hosts\.yml)$/iu; +const MARKDOWN_LINK = /\[([^\]]+)\]\(([^)]+)\)/gu; + +export interface RepositoryIndexOptions { + repoRoot: string; + wikiRoot: string; +} + +export async function buildRepositoryCorpus( + options: RepositoryIndexOptions, +): Promise { + const repoRoot = await resolveDirectory(options.repoRoot, "repository root"); + const wikiRoot = await resolveDirectory(options.wikiRoot, "wiki root"); + const wikiPages = await readWikiPages(wikiRoot); + const concepts = buildConcepts(wikiPages); + connectIncomingRelationships(concepts); + return { + chunks: [ + ...wikiPages.flatMap((page) => page.chunks), + ...(await readSourceChunks(repoRoot, wikiRoot)), + ], + concepts, + }; +} + +interface WikiPage { + chunks: IndexedChunk[]; + concept: OkfConcept; +} + +async function readWikiPages(wikiRoot: string): Promise { + const files = await walkFiles(wikiRoot, (file) => file.endsWith(".md")); + return Promise.all( + files.map(async (file) => { + const content = await readBoundedTextFile(wikiRoot, file); + const relative = toPosix(path.relative(wikiRoot, file)); + const conceptPath = `openwiki/${relative}`; + const fields = parseFrontmatterFields(content) ?? {}; + const { body } = splitFrontmatter(content); + const title = stringField(fields.title) ?? firstHeading(body) ?? relative; + const description = stringField(fields.description); + const type = stringField(fields.type) ?? "Reference"; + const resource = stringField(fields.resource); + const tags = stringArray(fields.tags); + return { + chunks: chunkWikiPage({ + body, + conceptPath, + description, + fields, + relative, + tags, + title, + type, + }), + concept: { + ...(description ? { description } : {}), + incoming: new Set(), + path: conceptPath, + relationships: extractRelationships(body, relative), + ...(resource ? { resource } : {}), + tags, + title, + type, + }, + }; + }), + ); +} + +async function readSourceChunks( + repoRoot: string, + wikiRoot: string, +): Promise { + const files = await walkFiles(repoRoot, (file) => { + const extension = path.extname(file).toLowerCase(); + return SOURCE_EXTENSIONS.has(extension); + }); + const chunks: IndexedChunk[] = []; + for (const file of files) { + if (isContained(wikiRoot, file)) continue; + const content = await readBoundedTextFile(repoRoot, file); + const relative = toPosix(path.relative(repoRoot, file)); + const lines = content.split(/\r?\n/u); + for ( + let start = 0; + start < lines.length; + start += SOURCE_LINES_PER_CHUNK - SOURCE_LINE_OVERLAP + ) { + const selected = lines.slice(start, start + SOURCE_LINES_PER_CHUNK); + if (selected.every((line) => !line.trim())) continue; + const lineStart = start + 1; + const lineEnd = start + selected.length; + chunks.push({ + fields: relative, + id: `source:${relative}:${lineStart}`, + kind: "source", + lineEnd, + lineStart, + path: relative, + scope: "source", + tags: pathTags(relative), + text: selected.join("\n"), + title: path.basename(relative), + }); + } + } + return chunks; +} + +function chunkWikiPage(input: { + body: string; + conceptPath: string; + description?: string; + fields: Record; + relative: string; + tags: string[]; + title: string; + type: string; +}): IndexedChunk[] { + const lines = input.body.split(/\r?\n/u); + const headingIndexes = lines + .map((line, index) => (/^#{1,3}\s+\S/u.test(line) ? index : -1)) + .filter((index) => index >= 0); + if (headingIndexes.length === 0) headingIndexes.push(0); + return headingIndexes.map((start, index) => { + const end = headingIndexes[index + 1] ?? lines.length; + const selected = lines.slice(start, end); + const heading = selected[0]?.replace(/^#{1,3}\s+/u, "").trim(); + return { + conceptPath: input.conceptPath, + fields: JSON.stringify(input.fields), + ...(heading ? { heading } : {}), + id: `wiki:${input.relative}:${start + 1}`, + kind: "wiki-section", + lineEnd: Math.max(start + 1, end), + lineStart: start + 1, + path: input.conceptPath, + scope: "wiki", + tags: input.tags, + text: [input.description, selected.join("\n")].filter(Boolean).join("\n"), + title: input.title, + type: input.type, + }; + }); +} + +function buildConcepts(pages: WikiPage[]): Map { + return new Map(pages.map((page) => [page.concept.path, page.concept])); +} + +function connectIncomingRelationships(concepts: Map): void { + for (const concept of concepts.values()) { + concept.relationships = concept.relationships.filter((relationship) => { + const target = concepts.get(relationship.target); + if (!target) return false; + target.incoming.add(concept.path); + return true; + }); + } +} + +function extractRelationships( + body: string, + sourceRelative: string, +): OkfRelationship[] { + const relationships: OkfRelationship[] = []; + for (const match of body.matchAll(MARKDOWN_LINK)) { + const rawTarget = (match[2] ?? "").trim().split("#", 1)[0] ?? ""; + if (!rawTarget || /^[A-Za-z][A-Za-z0-9+.-]*:/u.test(rawTarget)) continue; + const sourceDirectory = path.posix.dirname(toPosix(sourceRelative)); + const resolved = path.posix.normalize( + path.posix.join(sourceDirectory, rawTarget), + ); + if (resolved.startsWith("../") || path.posix.isAbsolute(resolved)) continue; + const target = `openwiki/${resolved.endsWith(".md") ? resolved : `${resolved}.md`}`; + const offset = match.index ?? 0; + relationships.push({ + context: relationshipContext(body, offset), + target, + }); + } + return relationships; +} + +async function walkFiles( + root: string, + include: (file: string) => boolean, +): Promise { + const files: string[] = []; + const pending = [root]; + while (pending.length > 0 && files.length < MAX_FILES) { + const directory = pending.pop(); + if (!directory) break; + const entries = await readdir(directory, { withFileTypes: true }); + for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) { + if (isSecretName(entry.name)) continue; + const candidate = path.join(directory, entry.name); + if (!isContained(root, candidate)) continue; + if (entry.isSymbolicLink()) continue; + if (entry.isDirectory()) { + if (!EXCLUDED_DIRECTORIES.has(entry.name)) pending.push(candidate); + } else if (entry.isFile() && include(candidate)) { + files.push(candidate); + if (files.length >= MAX_FILES) break; + } + } + } + return files; +} + +async function readBoundedTextFile( + root: string, + file: string, +): Promise { + const resolved = await realpath(file); + if (!isContained(root, resolved) || isSecretPath(resolved)) { + throw new Error( + "Refusing to read a path outside the indexed root or a secret-like file.", + ); + } + const info = await stat(resolved); + if (info.size > MAX_FILE_BYTES) return ""; + const content = await readFile(resolved, "utf8"); + return content.includes("\0") ? "" : content; +} + +async function resolveDirectory(value: string, label: string): Promise { + const resolved = await realpath(path.resolve(value)); + const info = await lstat(resolved); + if (!info.isDirectory() || info.isSymbolicLink()) { + throw new Error(`${label} must be a real directory.`); + } + return resolved; +} + +function isContained(root: string, candidate: string): boolean { + const relative = path.relative(root, path.resolve(candidate)); + return ( + relative === "" || + (!relative.startsWith(`..${path.sep}`) && + relative !== ".." && + !path.isAbsolute(relative)) + ); +} + +function isSecretPath(file: string): boolean { + return file.split(path.sep).some(isSecretName); +} + +function isSecretName(name: string): boolean { + return ( + SECRET_FILE.test(name) || + /(?:credential|private[_-]?key|secret)/iu.test(name) + ); +} + +function relationshipContext(body: string, offset: number): string { + const start = Math.max(0, body.lastIndexOf("\n", offset - 160)); + const endCandidate = body.indexOf("\n", offset + 160); + const end = endCandidate === -1 ? body.length : endCandidate; + return body.slice(start, end).replace(/\s+/gu, " ").trim().slice(0, 320); +} + +function pathTags(relative: string): string[] { + return toPosix(relative) + .split("/") + .slice(0, -1) + .filter((part) => part.length > 1); +} + +function firstHeading(body: string): string | undefined { + return /^#\s+(.+?)\s*$/mu.exec(body)?.[1]?.trim(); +} + +function stringField(value: unknown): string | undefined { + return typeof value === "string" && value.trim() ? value.trim() : undefined; +} + +function stringArray(value: unknown): string[] { + return Array.isArray(value) + ? value.filter( + (item): item is string => + typeof item === "string" && item.trim().length > 0, + ) + : []; +} + +function toPosix(value: string): string { + return value.split(path.sep).join("/"); +} diff --git a/src/retrieval/search-service.ts b/src/retrieval/search-service.ts new file mode 100644 index 00000000..4acde44d --- /dev/null +++ b/src/retrieval/search-service.ts @@ -0,0 +1,522 @@ +import { buildRepositoryCorpus } from "./repository-index.js"; +import { + rankBm25, + rankKeyword, + reciprocalRankFusion, + tokenize, +} from "./ranking.js"; +import { SemanticRanker, type EmbeddingProvider } from "./semantic.js"; +import type { + ChangeSurfaceCategory, + ChangeSurfaceResponse, + IndexedChunk, + OkfConcept, + RankedHit, + RepositoryCorpus, + SearchResponse, + SearchResultItem, + SearchScope, + SymbolTraceResponse, +} from "./types.js"; + +const DEFAULT_LIMIT = 8; +const MAX_LIMIT = 20; +const MAX_QUERY_LENGTH = 500; +const MAX_RELATED_CONCEPTS = 3; +const MAX_SNIPPET_LENGTH = 320; +const IDENTIFIER = /^[A-Za-z_$][A-Za-z0-9_$]{0,99}$/u; +const SURFACE_CATEGORY_ORDER: ChangeSurfaceCategory[] = [ + "consumer", + "tests", + "exports", + "publish_generated", + "initialization", + "implementation", +]; + +export interface RetrievalServiceOptions { + embeddingProvider: EmbeddingProvider; + repoRoot: string; + wikiRoot: string; +} + +export class RetrievalService { + private corpusPromise: Promise | undefined; + private readonly semantic: SemanticRanker; + + constructor(private readonly options: RetrievalServiceOptions) { + this.semantic = new SemanticRanker(options.embeddingProvider); + } + + async keywordSearch( + query: string, + scope: SearchScope = "all", + limit = DEFAULT_LIMIT, + ): Promise { + const { chunks } = await this.corpus(); + return response( + "field-weighted-keyword", + query, + rankKeyword(scopedChunks(chunks, scope), validateQuery(query)), + validateLimit(limit), + ); + } + + async bm25Search( + query: string, + scope: SearchScope = "all", + limit = DEFAULT_LIMIT, + ): Promise { + const { chunks } = await this.corpus(); + return response( + "bm25", + query, + rankBm25(scopedChunks(chunks, scope), validateQuery(query)), + validateLimit(limit), + ); + } + + async semanticSearch( + query: string, + scope: SearchScope = "all", + limit = DEFAULT_LIMIT, + ): Promise { + const { chunks } = await this.corpus(); + const ranked = await this.semantic.rank( + chunks, + validateQuery(query), + validateScope(scope), + ); + return response(ranked.engine, query, ranked.hits, validateLimit(limit)); + } + + async okfGraphSearch( + query: string, + limit = DEFAULT_LIMIT, + hops = 1, + ): Promise { + const corpus = await this.corpus(); + const validQuery = validateQuery(query); + const validHops = + Number.isInteger(hops) && hops >= 0 && hops <= 2 ? hops : 1; + const hits = rankOkfGraph(corpus, validQuery, validHops); + return response("okf-graph", validQuery, hits, validateLimit(limit)); + } + + async hybridSearch( + query: string, + scope: SearchScope = "all", + limit = DEFAULT_LIMIT, + ): Promise { + const corpus = await this.corpus(); + const validQuery = validateQuery(query); + const validScope = validateScope(scope); + const chunks = scopedChunks(corpus.chunks, validScope); + const semantic = await this.semantic.rank( + corpus.chunks, + validQuery, + validScope, + ); + const lists = [ + { hits: rankKeyword(chunks, validQuery), name: "keyword", weight: 0.55 }, + { hits: rankBm25(chunks, validQuery), name: "bm25", weight: 1 }, + { hits: semantic.hits, name: "semantic", weight: 0.9 }, + ]; + if (validScope !== "source") { + lists.push({ + hits: rankOkfGraph(corpus, validQuery, 1), + name: "okf_graph", + weight: 0.8, + }); + } + return response( + `hybrid-rrf:${semantic.engine}`, + validQuery, + reciprocalRankFusion(lists), + validateLimit(limit), + ); + } + + async testSearch(query: string, limit = 5): Promise { + const validQuery = validateQuery(query); + const validLimit = validateLimit(limit); + const testChunks = (await this.corpus()).chunks.filter( + (chunk) => chunk.scope === "source" && isTestChunk(chunk), + ); + const semantic = await this.semantic.rank(testChunks, validQuery, "source"); + return response( + `test-hybrid-rrf:${semantic.engine}`, + validQuery, + reciprocalRankFusion([ + { + hits: rankKeyword(testChunks, validQuery), + name: "keyword", + weight: 0.6, + }, + { hits: rankBm25(testChunks, validQuery), name: "bm25", weight: 1 }, + { hits: semantic.hits, name: "semantic", weight: 0.9 }, + ]), + validLimit, + ); + } + + async changeSurface( + query: string, + limit = 6, + ): Promise { + const corpus = await this.corpus(); + const validQuery = validateQuery(query); + const validLimit = validateLimit(limit); + const concepts = await this.hybridSearch(validQuery, "wiki", validLimit); + const conceptChunks = conceptHits(corpus.chunks, concepts.results); + const referencedPaths = extractPaths( + conceptChunks.map((chunk) => chunk.text).join("\n"), + ); + const symbols = extractSymbols( + `${validQuery}\n${conceptChunks.map((chunk) => chunk.text).join("\n")}`, + ); + const expandedQuery = [validQuery, ...symbols.slice(0, 24)].join(" "); + const source = reciprocalRankFusion([ + { + hits: rankBm25(scopedChunks(corpus.chunks, "source"), expandedQuery), + name: "bm25", + weight: 1, + }, + { + hits: boostReferencedPaths( + rankKeyword(scopedChunks(corpus.chunks, "source"), expandedQuery), + referencedPaths, + ), + name: "wiki_paths", + weight: 1.1, + }, + ]).slice(0, 160); + const groups = emptySurfaceGroups(); + const candidates = emptySurfaceGroups(); + for (const hit of source) { + for (const category of categorize(hit.chunk)) { + candidates[category].push(toResultItem(hit)); + } + } + fillSurfaceGroups(groups, candidates, validLimit); + return { + groups, + query: validQuery, + relatedConcepts: concepts.results.slice(0, MAX_RELATED_CONCEPTS), + }; + } + + async symbolTrace(query: string, limit = 6): Promise { + const symbol = validateIdentifier(query); + const validLimit = validateLimit(limit); + this.corpusPromise = undefined; + const sourceChunks = scopedChunks((await this.corpus()).chunks, "source"); + const exactIdentifier = new RegExp( + `(?:^|[^A-Za-z0-9_$])${escapeRegExp(symbol)}(?:$|[^A-Za-z0-9_$])`, + "u", + ); + const candidates = emptySurfaceGroups(); + for (const hit of rankKeyword(sourceChunks, symbol)) { + if (!exactIdentifier.test(hit.chunk.text)) continue; + for (const category of categorize(hit.chunk)) { + candidates[category].push(toResultItem(hit)); + } + } + const groups = emptySurfaceGroups(); + fillSurfaceGroups(groups, candidates, validLimit); + return { + groups, + missing: SURFACE_CATEGORY_ORDER.filter( + (category) => groups[category].length === 0, + ), + symbol, + }; + } + + private corpus(): Promise { + this.corpusPromise ??= buildRepositoryCorpus(this.options); + return this.corpusPromise; + } +} + +function rankOkfGraph( + corpus: RepositoryCorpus, + query: string, + hops: number, +): RankedHit[] { + const wikiChunks = corpus.chunks.filter((chunk) => chunk.scope === "wiki"); + const seeds = rankBm25(wikiChunks, query).slice(0, 20); + const scores = new Map(); + const seedConcepts = new Set(); + for (const [index, hit] of seeds.entries()) { + if (!hit.chunk.conceptPath) continue; + const score = 1 / (index + 1); + scores.set( + hit.chunk.conceptPath, + (scores.get(hit.chunk.conceptPath) ?? 0) + score, + ); + seedConcepts.add(hit.chunk.conceptPath); + } + let frontier = seedConcepts; + for (let hop = 0; hop < hops; hop += 1) { + const next = new Set(); + for (const conceptPath of frontier) { + const concept = corpus.concepts.get(conceptPath); + if (!concept) continue; + const base = scores.get(conceptPath) ?? 0; + for (const neighbor of graphNeighbors(concept, corpus.concepts)) { + scores.set(neighbor, (scores.get(neighbor) ?? 0) + base * 0.35); + next.add(neighbor); + } + } + frontier = next; + } + return [...scores.entries()] + .map(([conceptPath, score]) => { + const chunk = bestConceptChunk(wikiChunks, conceptPath, query); + return chunk ? { chunk, score } : null; + }) + .filter((hit): hit is RankedHit => hit !== null) + .sort((left, right) => right.score - left.score); +} + +function graphNeighbors( + concept: OkfConcept, + concepts: Map, +): Set { + const neighbors = new Set([ + ...concept.relationships.map((relationship) => relationship.target), + ...concept.incoming, + ]); + if (concept.tags.length > 0) { + for (const candidate of concepts.values()) { + if ( + candidate.path !== concept.path && + candidate.tags.some((tag) => concept.tags.includes(tag)) + ) { + neighbors.add(candidate.path); + } + } + } + return neighbors; +} + +function bestConceptChunk( + chunks: IndexedChunk[], + conceptPath: string, + query: string, +): IndexedChunk | undefined { + return ( + rankBm25( + chunks.filter((chunk) => chunk.conceptPath === conceptPath), + query, + )[0]?.chunk ?? chunks.find((chunk) => chunk.conceptPath === conceptPath) + ); +} + +function conceptHits( + chunks: IndexedChunk[], + results: SearchResultItem[], +): IndexedChunk[] { + const keys = new Set(results.map((item) => `${item.path}:${item.lineStart}`)); + return chunks.filter((chunk) => keys.has(`${chunk.path}:${chunk.lineStart}`)); +} + +function boostReferencedPaths( + hits: RankedHit[], + paths: Set, +): RankedHit[] { + return hits + .map((hit) => ({ + ...hit, + score: + hit.score * + ([...paths].some( + (candidate) => + hit.chunk.path === candidate || hit.chunk.path.endsWith(candidate), + ) + ? 2.5 + : 1), + })) + .sort((left, right) => right.score - left.score); +} + +function categorize(chunk: IndexedChunk): ChangeSurfaceCategory[] { + const value = `${chunk.path}\n${chunk.text}`; + const categories = new Set(); + if ( + /\b(?:exports|entrypoint|public api)\b/iu.test(value) || + /\bexport\s+(?:\*|\{[^}]+\})\s+from\b/iu.test(chunk.text) || + /(?:^|\/)index\.[cm]?[jt]sx?$/u.test(chunk.path) + ) { + categories.add("exports"); + } + if ( + /\b(?:publish|generated|bundle|build artifact|package\.json|dist)\b/iu.test( + value, + ) + ) { + categories.add("publish_generated"); + } + if ( + /\b(?:initialize|register|registry|factory|createStore|createWorld|setup)\b/u.test( + value, + ) + ) { + categories.add("initialization"); + } + if (isTestChunk(chunk)) { + categories.add("tests"); + } + if ( + /\bimport\s+.+\s+from\s+['"][^./]/u.test(chunk.text) || + /(?:^|\/)(?:examples?|apps?|publish\/tests)(?:\/|$)/iu.test(chunk.path) + ) { + categories.add("consumer"); + } + if (categories.size === 0 || /(?:^|\/)src(?:\/|$)/u.test(chunk.path)) { + categories.add("implementation"); + } + return [...categories]; +} + +function isTestChunk(chunk: IndexedChunk): boolean { + return ( + /(?:^|\/)(?:test|tests|spec|specs)(?:\/|\.)/iu.test(chunk.path) || + /\b(?:describe|it|test)\s*\(/u.test(chunk.text) + ); +} + +function extractPaths(value: string): Set { + const paths = value.match( + /(?:^|[\s`("'])([A-Za-z0-9_.-]+\/(?:[A-Za-z0-9_.-]+\/)*[A-Za-z0-9_.-]+\.[A-Za-z0-9]+)/gmu, + ); + return new Set( + (paths ?? []).map((item) => item.trim().replace(/^[`("']/u, "")), + ); +} + +function extractSymbols(value: string): string[] { + const symbols = new Set(); + for (const match of value.matchAll(/`([A-Za-z_$][A-Za-z0-9_$]{2,})`/gu)) { + if (match[1]) symbols.add(match[1]); + } + for (const term of tokenize(value)) { + if (term.length >= 4) symbols.add(term); + } + return [...symbols]; +} + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); +} + +function emptySurfaceGroups(): Record< + ChangeSurfaceCategory, + SearchResultItem[] +> { + return { + consumer: [], + exports: [], + implementation: [], + initialization: [], + publish_generated: [], + tests: [], + }; +} + +function fillSurfaceGroups( + groups: Record, + candidates: Record, + totalLimit: number, +): void { + let remaining = totalLimit; + let index = 0; + while (remaining > 0) { + let added = false; + for (const category of SURFACE_CATEGORY_ORDER) { + const candidate = candidates[category][index]; + if (!candidate || remaining === 0) continue; + groups[category].push(candidate); + remaining -= 1; + added = true; + } + if (!added) return; + index += 1; + } +} + +function response( + engine: string, + query: string, + hits: RankedHit[], + limit: number, +): SearchResponse { + return { + engine, + query, + results: hits.slice(0, limit).map(toResultItem), + }; +} + +function toResultItem(hit: RankedHit): SearchResultItem { + return { + ...(hit.chunk.heading ? { heading: hit.chunk.heading } : {}), + lineEnd: hit.chunk.lineEnd, + lineStart: hit.chunk.lineStart, + path: hit.chunk.path, + score: Number(hit.score.toFixed(6)), + ...(hit.signals ? { signals: hit.signals } : {}), + snippet: compactSnippet(hit.chunk.text), + ...(hit.chunk.tags.length > 0 ? { tags: hit.chunk.tags } : {}), + ...(hit.chunk.title ? { title: hit.chunk.title } : {}), + ...(hit.chunk.type ? { type: hit.chunk.type } : {}), + }; +} + +function compactSnippet(value: string): string { + return value.replace(/\s+/gu, " ").trim().slice(0, MAX_SNIPPET_LENGTH); +} + +function scopedChunks( + chunks: IndexedChunk[], + scope: SearchScope, +): IndexedChunk[] { + const valid = validateScope(scope); + return valid === "all" + ? chunks + : chunks.filter((chunk) => chunk.scope === valid); +} + +function validateScope(scope: SearchScope): SearchScope { + if (scope !== "all" && scope !== "source" && scope !== "wiki") { + throw new Error("scope must be all, source, or wiki."); + } + return scope; +} + +function validateQuery(query: string): string { + if ( + typeof query !== "string" || + !query.trim() || + query.length > MAX_QUERY_LENGTH + ) { + throw new Error(`query must be 1-${MAX_QUERY_LENGTH} characters.`); + } + return query.trim(); +} + +function validateIdentifier(query: string): string { + const identifier = validateQuery(query); + if (!IDENTIFIER.test(identifier)) { + throw new Error("symbol must be a single 1-100 character identifier."); + } + return identifier; +} + +function validateLimit(limit: number): number { + if (!Number.isInteger(limit) || limit < 1 || limit > MAX_LIMIT) { + throw new Error(`limit must be an integer between 1 and ${MAX_LIMIT}.`); + } + return limit; +} diff --git a/src/retrieval/semantic.ts b/src/retrieval/semantic.ts new file mode 100644 index 00000000..65ad9403 --- /dev/null +++ b/src/retrieval/semantic.ts @@ -0,0 +1,103 @@ +import { rankBm25, rankLocalVectors, searchableText } from "./ranking.js"; +import type { IndexedChunk, RankedHit, SearchScope } from "./types.js"; + +export type EmbeddingProvider = "local" | "openai"; + +const OPENAI_CANDIDATE_LIMIT = 120; + +export class SemanticRanker { + private readonly vectorCache = new Map(); + + constructor(private readonly provider: EmbeddingProvider) {} + + async rank( + chunks: IndexedChunk[], + query: string, + scope: SearchScope, + ): Promise<{ engine: string; hits: RankedHit[] }> { + const scoped = chunks.filter( + (chunk) => scope === "all" || chunk.scope === scope, + ); + if (this.provider !== "openai" || !process.env.OPENAI_API_KEY) { + return { + engine: "local-hashed-vector", + hits: rankLocalVectors(scoped, query), + }; + } + try { + const candidates = selectOpenAiCandidates(scoped, query); + const embeddings = await this.openAiEmbeddings(); + const queryVector = await embeddings.embedQuery(query); + const missing = candidates.filter( + (chunk) => !this.vectorCache.has(chunk.id), + ); + if (missing.length > 0) { + const vectors = await embeddings.embedDocuments( + missing.map(searchableText), + ); + missing.forEach((chunk, index) => { + const vector = vectors[index]; + if (vector) this.vectorCache.set(chunk.id, vector); + }); + } + const hits = candidates + .map((chunk) => ({ + chunk, + score: cosine(queryVector, this.vectorCache.get(chunk.id) ?? []), + })) + .filter((hit) => hit.score > 0) + .sort((left, right) => right.score - left.score); + return { engine: "openai:text-embedding-3-small", hits }; + } catch { + return { + engine: "local-hashed-vector:fallback", + hits: rankLocalVectors(scoped, query), + }; + } + } + + private async openAiEmbeddings(): Promise<{ + embedDocuments(texts: string[]): Promise; + embedQuery(text: string): Promise; + }> { + const { OpenAIEmbeddings } = await import("@langchain/openai"); + return new OpenAIEmbeddings({ + apiKey: process.env.OPENAI_API_KEY, + batchSize: 64, + configuration: process.env.OPENAI_BASE_URL + ? { baseURL: process.env.OPENAI_BASE_URL } + : undefined, + model: "text-embedding-3-small", + }); + } +} + +function selectOpenAiCandidates( + chunks: IndexedChunk[], + query: string, +): IndexedChunk[] { + const wiki = chunks.filter((chunk) => chunk.scope === "wiki"); + const lexical = rankBm25(chunks, query) + .slice(0, OPENAI_CANDIDATE_LIMIT) + .map((hit) => hit.chunk); + return [ + ...new Map( + [...wiki, ...lexical].map((chunk) => [chunk.id, chunk]), + ).values(), + ].slice(0, OPENAI_CANDIDATE_LIMIT); +} + +function cosine(left: number[], right: number[]): number { + if (left.length === 0 || left.length !== right.length) return 0; + let dot = 0; + let leftNorm = 0; + let rightNorm = 0; + for (let index = 0; index < left.length; index += 1) { + const l = left[index] ?? 0; + const r = right[index] ?? 0; + dot += l * r; + leftNorm += l * l; + rightNorm += r * r; + } + return leftNorm && rightNorm ? dot / Math.sqrt(leftNorm * rightNorm) : 0; +} diff --git a/src/retrieval/types.ts b/src/retrieval/types.ts new file mode 100644 index 00000000..9f917880 --- /dev/null +++ b/src/retrieval/types.ts @@ -0,0 +1,85 @@ +export type SearchScope = "all" | "source" | "wiki"; + +export type ChunkKind = "source" | "wiki-section"; + +export interface IndexedChunk { + conceptPath?: string; + fields: string; + heading?: string; + id: string; + kind: ChunkKind; + lineEnd: number; + lineStart: number; + path: string; + scope: Exclude; + tags: string[]; + text: string; + title?: string; + type?: string; +} + +export interface OkfRelationship { + context: string; + target: string; +} + +export interface OkfConcept { + description?: string; + incoming: Set; + path: string; + relationships: OkfRelationship[]; + resource?: string; + tags: string[]; + title: string; + type: string; +} + +export interface RepositoryCorpus { + chunks: IndexedChunk[]; + concepts: Map; +} + +export interface RankedHit { + chunk: IndexedChunk; + score: number; + signals?: Record; +} + +export interface SearchResultItem { + heading?: string; + lineEnd: number; + lineStart: number; + path: string; + score: number; + signals?: Record; + snippet: string; + tags?: string[]; + title?: string; + type?: string; +} + +export interface SearchResponse { + engine: string; + query: string; + results: SearchResultItem[]; +} + +export type ChangeSurfaceCategory = + | "consumer" + | "exports" + | "implementation" + | "initialization" + | "publish_generated" + | "tests"; + +export interface ChangeSurfaceResponse { + groups: Record; + query: string; + relatedConcepts: SearchResultItem[]; +} + +export interface SymbolTraceResponse { + groups: Record; + missing: ChangeSurfaceCategory[]; + symbol: string; +} diff --git a/test/agent-navigation-guidance.test.ts b/test/agent-navigation-guidance.test.ts new file mode 100644 index 00000000..b36b5f15 --- /dev/null +++ b/test/agent-navigation-guidance.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, test } from "vitest"; +import { createSystemPrompt } from "../src/agent/prompt.ts"; + +describe("repository coding-agent documentation guidance", () => { + test("requires change-oriented navigation and validation guidance", () => { + const prompt = createSystemPrompt("init", "repository"); + + expect(prompt).toContain("Coding-agent utility requirements"); + expect(prompt).toContain("compact task-routing table"); + expect(prompt).toContain("exact source entry points"); + expect(prompt).toContain("important symbols or types"); + expect(prompt).toContain("runtime invariants and lifecycle ordering"); + expect(prompt).toContain("evidence-backed change recipes"); + expect(prompt).toContain("complete change surface"); + expect(prompt).toContain("shipped-surface correctness"); + expect(prompt).toContain("consumer-facing smoke test"); + expect(prompt).toContain("behavioral test matrix"); + expect(prompt).toContain("isolation between independent instances"); + expect(prompt).toContain("test_search"); + expect(prompt).toContain("Label expensive checks as conditional"); + expect(prompt).toContain( + "simulate navigation for representative adjacent changes", + ); + }); + + test("does not apply repository coding guidance to the personal wiki", () => { + const prompt = createSystemPrompt("init", "local-wiki"); + + expect(prompt).not.toContain("Coding-agent utility requirements"); + expect(prompt).not.toContain("compact task-routing table"); + }); +}); diff --git a/test/code-mode.test.ts b/test/code-mode.test.ts index 11df0650..2b1247da 100644 --- a/test/code-mode.test.ts +++ b/test/code-mode.test.ts @@ -43,6 +43,15 @@ describe("ensureCodeModeRepoSetup agent files", () => { expect(content).toContain(SNIPPET_START); expect(content).toContain(SNIPPET_END); expect(content).toContain("## OpenWiki"); + expect(content).toContain("just-in-time repository index"); + expect(content).toContain("Before a repository-wide"); + expect(content).toContain("Re-consult the wiki"); + expect(content).toContain("import path consumers actually use"); + expect(content).toContain("internal unit tests alone"); + expect(content).toContain("symbol_trace"); + expect(content).toContain("independent instances"); + expect(content).toContain("deferred or re-entrant mutation"); + expect(content).toContain("test_search"); } }); diff --git a/test/prompt.test.ts b/test/prompt.test.ts deleted file mode 100644 index 45050f03..00000000 --- a/test/prompt.test.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { describe, expect, test } from "vitest"; -import { createSystemPrompt } from "../src/agent/prompt.ts"; - -/** - * Guards against the 0.2 regression where the shared "Canonical wiki location" - * and "Wiki-first question answering" blocks hardcoded ~/.openwiki/wiki and - * leaked into repository (code) mode. In code mode the filesystem virtual root - * maps to the repo, so instructing the model to use ~/.openwiki/wiki made it - * type non-absolute host paths into filesystem tools and crash the run. - */ -describe("createSystemPrompt filesystem path guidance", () => { - const commands = ["init", "update", "chat"] as const; - - describe("repository mode", () => { - for (const command of commands) { - test(`${command}: does not point the wiki at ~/.openwiki/wiki`, () => { - const prompt = createSystemPrompt(command, "repository"); - - // The canonical location must be the repo-local /openwiki, never the - // personal-brain home dir. - expect(prompt).not.toMatch(/lives in ~\/\.openwiki\/wiki/); - expect(prompt).not.toMatch(/inspect ~\/\.openwiki\/wiki first/); - expect(prompt).toContain("/openwiki"); - }); - } - }); - - describe("local-wiki mode", () => { - for (const command of commands) { - test(`${command}: roots the wiki at ~/.openwiki/wiki via virtual /`, () => { - const prompt = createSystemPrompt(command, "local-wiki"); - - expect(prompt).toContain("~/.openwiki/wiki"); - expect(prompt).toContain("/quickstart.md"); - }); - } - }); - - test("both modes forbid typing host/tilde paths into filesystem tools", () => { - for (const outputMode of ["repository", "local-wiki"] as const) { - const prompt = createSystemPrompt("update", outputMode); - expect(prompt).toMatch( - /Never type ~, ~\/\.openwiki\/wiki, or host paths/, - ); - } - }); -}); - -/** - * The deterministic post-run pass repairs missing or invalid front matter and - * tags the page `openwiki_generated`. The prompt must tell the agent that code - * owns conformance and that it should enrich those flagged pages, so quality - * fills in over later runs instead of code guessing forever. - */ -describe("createSystemPrompt openwiki_generated enrichment guidance", () => { - for (const outputMode of ["repository", "local-wiki"] as const) { - test(`${outputMode} mode: instructs the agent to enrich and clear the mark`, () => { - const prompt = createSystemPrompt("update", outputMode); - - expect(prompt).toContain("openwiki_generated: true"); - expect(prompt).toMatch(/repairs front matter deterministically/); - expect(prompt).toMatch(/remove the `openwiki_generated` field/); - }); - } -}); diff --git a/test/retrieval.test.ts b/test/retrieval.test.ts new file mode 100644 index 00000000..e7b51a69 --- /dev/null +++ b/test/retrieval.test.ts @@ -0,0 +1,219 @@ +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, test } from "vitest"; +import { RetrievalService } from "../src/retrieval/search-service.ts"; + +let root = ""; +let repoRoot = ""; +let wikiRoot = ""; + +beforeEach(async () => { + root = await mkdtemp(path.join(tmpdir(), "openwiki-retrieval-")); + repoRoot = path.join(root, "repo"); + wikiRoot = path.join(root, "wiki"); + await Promise.all([ + mkdir(path.join(repoRoot, "packages/core/src/query"), { recursive: true }), + mkdir(path.join(repoRoot, "packages/publish/src"), { recursive: true }), + mkdir(path.join(repoRoot, "packages/publish/tests"), { recursive: true }), + mkdir(path.join(repoRoot, "secrets"), { recursive: true }), + mkdir(path.join(wikiRoot, "architecture"), { recursive: true }), + ]); + await Promise.all([ + writeFile( + path.join(wikiRoot, "quickstart.md"), + `--- +type: Quickstart +title: Koota quickstart +description: Routes query changes into runtime and package validation. +tags: [query, navigation] +--- + +# Quickstart + +For query changes, follow the [runtime contract](architecture/runtime.md). +`, + ), + writeFile( + path.join(wikiRoot, "architecture/runtime.md"), + `--- +type: Architecture +title: Query runtime and package contract +description: Connects predicate implementation to public exports and consumer tests. +tags: [query, package, runtime] +--- + +# Query runtime + +Implement predicates in \`packages/core/src/query/predicate.ts\`, export them from +\`packages/core/src/index.ts\`, mirror the public surface through +\`packages/publish/src/index.ts\`, and validate consumer imports in +\`packages/publish/tests/predicate.test.ts\`. + +The [quickstart](../quickstart.md) routes adjacent changes here. +`, + ), + writeFile( + path.join(repoRoot, "packages/core/src/query/predicate.ts"), + "export function createPredicate() { return true; }\n", + ), + writeFile( + path.join(repoRoot, "packages/core/src/index.ts"), + "export { createPredicate } from './query/predicate';\n", + ), + writeFile( + path.join(repoRoot, "packages/publish/src/index.ts"), + "export { createPredicate } from '@koota/core';\n", + ), + writeFile( + path.join(repoRoot, "packages/publish/tests/predicate.test.ts"), + "import { createPredicate } from 'koota';\ntest('public import', () => createPredicate());\n", + ), + writeFile( + path.join(repoRoot, ".env"), + "SECRET_PREDICATE_SURFACE=never-index-this\n", + ), + writeFile( + path.join(repoRoot, "secrets/credentials.json"), + '{"note":"predicate consumer package"}\n', + ), + ]); +}); + +afterEach(async () => { + await rm(root, { force: true, recursive: true }); +}); + +function service(): RetrievalService { + return new RetrievalService({ + embeddingProvider: "local", + repoRoot, + wikiRoot, + }); +} + +describe("OKF-aware repository retrieval", () => { + test("supports keyword, BM25, and local vector ranking", async () => { + const retrieval = service(); + const keyword = await retrieval.keywordSearch("createPredicate", "all", 5); + const bm25 = await retrieval.bm25Search( + "predicate consumer import", + "all", + 5, + ); + const semantic = await retrieval.semanticSearch( + "consumer-facing package surface", + "all", + 5, + ); + + expect(keyword.results[0]?.path).toMatch(/predicate|index/u); + expect(bm25.results.some((hit) => hit.path.includes("publish/tests"))).toBe( + true, + ); + expect(semantic.engine).toBe("local-hashed-vector"); + expect( + semantic.results.some( + (hit) => + hit.path.includes("runtime.md") || hit.path.includes("publish"), + ), + ).toBe(true); + }); + + test("expands retrieval through OKF links and shared tags", async () => { + const graph = await service().okfGraphSearch("query navigation", 5, 2); + + expect(graph.results.map((hit) => hit.path)).toContain( + "openwiki/architecture/runtime.md", + ); + expect(graph.results.map((hit) => hit.path)).toContain( + "openwiki/quickstart.md", + ); + }); + + test("hybrid search reports component scores", async () => { + const hybrid = await service().hybridSearch( + "add predicate query public API", + "all", + 5, + ); + + expect(hybrid.engine).toContain("hybrid-rrf"); + expect(hybrid.results[0]?.signals).toBeDefined(); + }); + + test("test_search returns only bounded test citations", async () => { + const result = await service().testSearch( + "public predicate import lifecycle transition", + 3, + ); + + expect(result.engine).toContain("test-hybrid-rrf"); + expect(result.results.length).toBeLessThanOrEqual(3); + expect(result.results.length).toBeGreaterThan(0); + expect( + result.results.every((hit) => /(?:test|spec)/iu.test(hit.path)), + ).toBe(true); + }); + + test("change_surface groups cross-package evidence", async () => { + const surface = await service().changeSurface( + "add createPredicate query API", + 6, + ); + + expect(surface.relatedConcepts[0]?.path).toContain("openwiki/"); + expect(surface.groups.implementation.length).toBeGreaterThan(0); + expect(surface.groups.exports.length).toBeGreaterThan(0); + expect(surface.groups.publish_generated.length).toBeGreaterThan(0); + expect(surface.groups.consumer.length).toBeGreaterThan(0); + expect(surface.groups.tests.length).toBeGreaterThan(0); + const results = Object.values(surface.groups).flat(); + expect(results).toHaveLength(6); + expect(surface.relatedConcepts.length).toBeLessThanOrEqual(3); + expect( + Math.max(...results.map((result) => result.snippet.length)), + ).toBeLessThanOrEqual(320); + expect(JSON.stringify(surface).length).toBeLessThan(6_000); + }); + + test("symbol_trace refreshes post-edit source and reports missing layers", async () => { + const retrieval = service(); + await retrieval.keywordSearch("createMatcher", "source", 5); + await writeFile( + path.join(repoRoot, "packages/core/src/query/matcher.ts"), + "export function createMatcher() { return true; }\n", + ); + + const trace = await retrieval.symbolTrace("createMatcher", 6); + + expect(trace.groups.implementation[0]?.path).toContain("matcher.ts"); + expect(trace.missing).toContain("consumer"); + expect(trace.missing).toContain("tests"); + expect(trace.missing).toContain("exports"); + expect(Object.values(trace.groups).flat()).toHaveLength(1); + await expect( + retrieval.symbolTrace("createMatcher(); rm -rf /", 6), + ).rejects.toThrow("single 1-100 character identifier"); + }); + + test("never indexes secret-like files", async () => { + const result = await service().keywordSearch( + "never-index-this credentials", + "all", + 20, + ); + + expect(result.results).toEqual([]); + }); + + test("bounds query length and result limits", async () => { + const retrieval = service(); + await expect(retrieval.keywordSearch("x", "all", 21)).rejects.toThrow( + "limit must be", + ); + await expect( + retrieval.keywordSearch("x".repeat(501), "all", 5), + ).rejects.toThrow("query must be"); + }); +}); From 27eda79d37d64639882c132e2ed9193b46e755f1 Mon Sep 17 00:00:00 2001 From: bracesproul Date: Mon, 27 Jul 2026 15:14:23 -0700 Subject: [PATCH 2/3] more tasks and better results --- AGENTS.md | 12 +- CLAUDE.md | 12 +- evals/deepswe/OPENWIKI_OPTIMIZATION_LOG.md | 222 +++++++++++++ evals/deepswe/OPENWIKI_TRACE_FINDINGS.md | 59 ++-- evals/deepswe/README.md | 58 +++- evals/deepswe/openwiki_codex.py | 48 +-- evals/deepswe/run.py | 258 ++++++++++++++- evals/deepswe/test_run.py | 221 ++++++++++++- src/agent/prompt.ts | 5 +- src/code-mode.ts | 12 +- src/retrieval/mcp-server.ts | 161 ++------- src/retrieval/mcp-tools.ts | 81 +++++ src/retrieval/ranking.ts | 21 +- src/retrieval/repository-index.ts | 21 +- src/retrieval/search-service.ts | 359 ++++++++++++--------- src/retrieval/semantic.ts | 12 +- src/retrieval/types.ts | 28 +- test/agent-navigation-guidance.test.ts | 5 +- test/code-mode.test.ts | 10 +- test/retrieval.test.ts | 165 ++++++---- 20 files changed, 1289 insertions(+), 481 deletions(-) create mode 100644 evals/deepswe/OPENWIKI_OPTIMIZATION_LOG.md create mode 100644 src/retrieval/mcp-tools.ts diff --git a/AGENTS.md b/AGENTS.md index d343f293..b05ff113 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,17 +16,17 @@ When working in this repository, read the OpenWiki quickstart first, then follow This repository uses OpenWiki for recurring code documentation. Use `openwiki/` as a just-in-time repository index: -- At task start, read `openwiki/quickstart.md`, then search the wiki for the task's concepts and read only the relevant linked pages. +- At task start, read `openwiki/quickstart.md`, then search the wiki for the task's concepts and read only the relevant linked sections. - Before a repository-wide `rg`, `find`, or exploratory directory scan, check the wiki's source maps. When they name relevant files, symbols, or tests, inspect those paths directly. -- Re-consult the wiki when entering a different subsystem, when source evidence contradicts the current understanding, or when blocked by an unfamiliar test or build failure. +- Re-consult the wiki when entering a different subsystem, when source evidence contradicts the current understanding, or when blocked by an unfamiliar test or build failure. Do not reread content already returned by retrieval unless surrounding context is needed. - Treat source code and tests as authoritative. Verify wiki claims in source before editing. - Before finishing a public API or cross-package change, trace it from implementation through barrel/package exports, generated or publish mirrors, initialization or registration, and the import path consumers actually use. Consult the relevant wiki integration or delivery guidance and run the narrowest consumer-facing check; internal unit tests alone do not prove the shipped surface works. -- If an `openwiki_retrieval` MCP server is available, use `change_surface` before editing and `symbol_trace` after adding or changing each public symbol. Treat missing groups as verification gaps to investigate against repository architecture, not automatic requirements. -- For stateful or lifecycle changes, translate each externally observable acceptance criterion into a focused test checklist. Cover relevant initial state, both transition directions, unchanged updates, missing dependencies, independent instances, reset or reuse, deferred or re-entrant mutation, and composition; map every criterion to a passing test before finishing. -- If the retrieval server provides `test_search`, use it with that behavior matrix to find analogous focused tests, then inspect the cited tests directly before implementing lifecycle semantics. +- If an `openwiki_retrieval` MCP server is available, use `search` for focused retrieval. Use `change_surface` before public, cross-package, generated-artifact, or runtime-registration edits. After changing public symbols, call `trace_symbols` once with all of them and treat missing groups as verification gaps, not automatic requirements. +- For stateful or lifecycle changes, translate each externally observable acceptance criterion into a focused test checklist. Cover relevant initial state, false-to-true and true-to-false transitions, unchanged updates, missing dependencies, independent tracker instances, reset/reuse and observation windows, deferred or re-entrant net effects, and composition between static and temporal constraints; map every criterion to a passing test before finishing. +- When behavior is unfamiliar, relevant tests are large, or no analogous focused check is known, call `search` with the `tests` scope using that behavior matrix, then inspect the cited tests directly. Skip this search when the exact focused test is already known. - When the repository generates or copies package artifacts, identify the canonical source and repository-supported synchronization command. Do not hand-edit derived output unless the documented workflow explicitly requires it. - Do not read operations, release, or integration pages unless the task affects those areas. If a requested feature does not exist yet, use the wiki to locate extension points, then inspect source rather than searching the wiki for the implementation. -- Do not reread pages already consulted unless new evidence requires it. +- Prefer the narrowest quiet validation command available. Suppress successful build/test noise when possible, but preserve complete failure output. The scheduled OpenWiki GitHub Actions workflow refreshes the repository wiki. Do not hand-edit generated OpenWiki pages unless explicitly asked; prefer updating source code/docs and letting OpenWiki regenerate. diff --git a/CLAUDE.md b/CLAUDE.md index d343f293..b05ff113 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -16,17 +16,17 @@ When working in this repository, read the OpenWiki quickstart first, then follow This repository uses OpenWiki for recurring code documentation. Use `openwiki/` as a just-in-time repository index: -- At task start, read `openwiki/quickstart.md`, then search the wiki for the task's concepts and read only the relevant linked pages. +- At task start, read `openwiki/quickstart.md`, then search the wiki for the task's concepts and read only the relevant linked sections. - Before a repository-wide `rg`, `find`, or exploratory directory scan, check the wiki's source maps. When they name relevant files, symbols, or tests, inspect those paths directly. -- Re-consult the wiki when entering a different subsystem, when source evidence contradicts the current understanding, or when blocked by an unfamiliar test or build failure. +- Re-consult the wiki when entering a different subsystem, when source evidence contradicts the current understanding, or when blocked by an unfamiliar test or build failure. Do not reread content already returned by retrieval unless surrounding context is needed. - Treat source code and tests as authoritative. Verify wiki claims in source before editing. - Before finishing a public API or cross-package change, trace it from implementation through barrel/package exports, generated or publish mirrors, initialization or registration, and the import path consumers actually use. Consult the relevant wiki integration or delivery guidance and run the narrowest consumer-facing check; internal unit tests alone do not prove the shipped surface works. -- If an `openwiki_retrieval` MCP server is available, use `change_surface` before editing and `symbol_trace` after adding or changing each public symbol. Treat missing groups as verification gaps to investigate against repository architecture, not automatic requirements. -- For stateful or lifecycle changes, translate each externally observable acceptance criterion into a focused test checklist. Cover relevant initial state, both transition directions, unchanged updates, missing dependencies, independent instances, reset or reuse, deferred or re-entrant mutation, and composition; map every criterion to a passing test before finishing. -- If the retrieval server provides `test_search`, use it with that behavior matrix to find analogous focused tests, then inspect the cited tests directly before implementing lifecycle semantics. +- If an `openwiki_retrieval` MCP server is available, use `search` for focused retrieval. Use `change_surface` before public, cross-package, generated-artifact, or runtime-registration edits. After changing public symbols, call `trace_symbols` once with all of them and treat missing groups as verification gaps, not automatic requirements. +- For stateful or lifecycle changes, translate each externally observable acceptance criterion into a focused test checklist. Cover relevant initial state, false-to-true and true-to-false transitions, unchanged updates, missing dependencies, independent tracker instances, reset/reuse and observation windows, deferred or re-entrant net effects, and composition between static and temporal constraints; map every criterion to a passing test before finishing. +- When behavior is unfamiliar, relevant tests are large, or no analogous focused check is known, call `search` with the `tests` scope using that behavior matrix, then inspect the cited tests directly. Skip this search when the exact focused test is already known. - When the repository generates or copies package artifacts, identify the canonical source and repository-supported synchronization command. Do not hand-edit derived output unless the documented workflow explicitly requires it. - Do not read operations, release, or integration pages unless the task affects those areas. If a requested feature does not exist yet, use the wiki to locate extension points, then inspect source rather than searching the wiki for the implementation. -- Do not reread pages already consulted unless new evidence requires it. +- Prefer the narrowest quiet validation command available. Suppress successful build/test noise when possible, but preserve complete failure output. The scheduled OpenWiki GitHub Actions workflow refreshes the repository wiki. Do not hand-edit generated OpenWiki pages unless explicitly asked; prefer updating source code/docs and letting OpenWiki regenerate. diff --git a/evals/deepswe/OPENWIKI_OPTIMIZATION_LOG.md b/evals/deepswe/OPENWIKI_OPTIMIZATION_LOG.md new file mode 100644 index 00000000..1a7f55ca --- /dev/null +++ b/evals/deepswe/OPENWIKI_OPTIMIZATION_LOG.md @@ -0,0 +1,222 @@ +# OpenWiki optimization loop: Koota DeepSWE + +## Objective and protocol + +Primary objectives, in order: preserve or improve task score, then reduce coding-agent tokens and total tool calls. Wiki-generation tokens are excluded. Retrieval calls, returned characters, edit actions, and duration are diagnostic metrics. + +Each hypothesis is cumulative unless its result causes an explicit rollback. The discriminator is `koota-pair-relation-tracking`, run three times concurrently with Codex `gpt-5.6-terra`, high reasoning, and OpenAI semantic reranking. This task was selected because it separates the old and current OpenWiki cohorts and exposes semantic-modeling failures rather than basic navigation failures. After five iterations, the best configuration is run on all five Koota tasks with three attempts each. + +Runs stop for rate limits only after logs confirm a provider or HTTP 429. Infrastructure failures are diagnosed separately. + +## Reference cohorts + +| Cohort | Full solves | Mean partial | Uncached input/trial | Cumulative input/trial | Output/trial | Tool calls/trial | Retrieval calls/trial | Retrieval chars/trial | Edit actions/trial | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| Baseline, 15 valid | 3/15 | 0.989192 | 100,184 | 4,175,334 | 30,345 | 51.13 | 0.00 | 0 | 23.2 | +| Old OpenWiki, 15 valid | 6/15 | 0.992832 | 136,772 | 5,192,270 | 30,478 | 57.93 | 9.00 | 70,413 | 16.0 | +| Current OpenWiki, 15 valid | 4/15 | 0.984533 | 125,617 | 5,838,432 | 32,721 | 60.73 | 5.73 | 30,650 | 21.0 | + +The current retrieval surface eliminated invalid calls, cut retrieval calls 36%, retrieval payload 57%, broad `rg` 83%, and validation commands 26%. Those savings were outweighed by 75 additional edit actions across the cohort. Pair tracking was the dominant regression: old OpenWiki solved 1/3 with 0.9968 mean partial; current OpenWiki solved 0/3 with 0.9508 mean partial. + +## Hypothesis 1: explicit state model, one canonical ledger, less prompt duplication + +### Proposed change + +- Before editing stateful behavior, require a compact model of all state identity axes, transitions, observation/reset windows, and the canonical owner or event ledger. +- Require explicit input/event/expected-result oracle rows mapped to focused tests. +- Remove the eval adapter's duplicated workflow essay and defer to the generated `AGENTS.md`, keeping only quickstart, read-only retrieval, and filesystem-isolation instructions. + +### Expected benefit + +The old full-solving pair trace centralized events by tracker/factory, entity, relation, and target. Current failures distributed state across modifier, trait, and query call sites, causing coexistence, cancellation, composition, and observation-window bugs. Making the design checkpoint salient should improve correctness and reduce edit churn without adding retrieval calls or payload. + +### Outcome + +H1 recovered quality and sharply reduced cost relative to the current pair cohort: + +| Metric | Current pair | H1 | Change | +| --- | ---: | ---: | ---: | +| Full solves | 0/3 | 1/3 | +1 solve | +| Mean partial | 0.950794 | 0.990476 | +0.039683 | +| Uncached input/trial | 153,744 | 129,476 | -15.8% | +| Cumulative input/trial | 8,042,262 | 5,746,183 | -28.6% | +| Output/trial | 37,320 | 37,573 | +0.7% | +| Exec/apply calls/trial | 71.67 | 61.33 | -14.4% | +| Retrieval calls/trial | 6.00 | 3.00 | -50.0% | +| Retrieval chars/trial | 30,603 | 21,697 | -29.1% | +| Edit actions/trial | 24.00 | 23.67 | -1.4% | +| Tool calls/trial | 77.67 | 64.33 | -17.2% | + +The full solve made only `change_surface` and `trace_symbols` retrieval calls and created a centralized pair-tracking module. The other attempts missed one coexistence case and five removal/lifecycle cases respectively. + +The intended state-model guidance was not actually tested: all traces read upstream `/app/AGENTS.md`, while OpenWiki had written its managed block to the isolated `/tmp/openwiki-source/AGENTS.md`. The shorter treatment prompt therefore acted as a low-guidance/retrieval-restraint ablation. Its efficiency result is useful, but its quality gain cannot be attributed to the new managed prompt. + +## Hypothesis 2: make the generated state-model guidance visible + +### Proposed change + +Point the concise treatment instruction directly to `/tmp/openwiki-source/AGENTS.md` after the quickstart. Keep the long workflow out of the task prompt and retain the `/app` source-of-truth boundary. + +### Expected benefit + +One small file read should expose the identity-axis, canonical-ledger, observation-window, and explicit-oracle checkpoint. This should turn more attempts into the centralized architecture seen in both full solves and close the coexistence/removal gaps, with much less context cost than restoring the duplicated treatment essay. + +### Outcome + +All three traces read the generated managed block, so the hypothesis was tested. It was a loss: + +| Metric | H1 | H2 | Change | +| --- | ---: | ---: | ---: | +| Full solves | 1/3 | 0/3 | -1 solve | +| Mean partial | 0.990476 | 0.973016 | -0.017460 | +| Uncached input/trial | 129,476 | 153,067 | +18.2% | +| Cumulative input/trial | 5,746,183 | 6,980,095 | +21.5% | +| Output/trial | 37,573 | 38,454 | +2.3% | +| Exec/apply calls/trial | 61.33 | 66.33 | +8.2% | +| Retrieval calls/trial | 3.00 | 4.33 | +44.4% | +| Retrieval chars/trial | 21,697 | 25,776 | +18.8% | +| Edit actions/trial | 23.67 | 23.00 | -2.8% | +| Tool calls/trial | 64.33 | 70.67 | +9.8% | + +The visible block increased retrieval and validation but did not improve the semantic design. All three attempts created centralized pair-tracking utilities, yet all missed specific/non-last/wildcard removal, exclusive replacement, and destruction. One also missed trait-plus-pair coexistence. The abstract state-model instruction did not force agents to enumerate every mutation producer, and the long workflow diluted the key decision. + +## Hypothesis 3: compact, producer-aware managed guidance + +### Proposed change + +Reduce the managed block from eleven workflow bullets to five decision rules. Make retrieval evidence-driven rather than routine. For stateful work, require tracing every mutation/event producer and the state consumer before selecting one state owner, followed by explicit behavior rows and focused quiet validation. + +### Expected benefit + +The shorter block should recover H1's lower tokens and calls while retaining a concrete design guardrail. Producer tracing directly targets H2's repeated removal/destruction/replacement failures, which came from updating state consumers without covering all relation mutation paths. + +### Outcome + +Aborted before any coding-agent model calls after the user clarified that regressions must be reverted before continuing. This proposal is not counted as one of the five evaluated hypotheses. H2's explicit generated-`AGENTS.md` read and this untested compact-prompt edit were both rolled back to the H1 winner. + +## Hypothesis 3: transition-producer evidence in `change_surface` + +### Proposed change + +Starting from H1, keep the same three-tool surface and enhance the already-used `change_surface` response with one compact `state_transitions` group. It prioritizes authoritative add/remove/update/destroy/reset/defer/replacement producer code and excludes query-modifier consumers. The final `trace_symbols` schema is unchanged. + +### Expected benefit + +Every H1/H2 attempt already calls `change_surface`, so this should expose the relation removal, replacement, and destruction paths that repeated failures missed without adding a tool call. The payload increase is bounded to one short citation, avoiding H2's prompt and search overhead. + +### Outcome + +H3 is retained as the new winner: + +| Metric | H1 | H3 | Change | +| --- | ---: | ---: | ---: | +| Full solves | 1/3 | 2/3 | +1 solve | +| Mean partial | 0.990476 | 0.990476 | unchanged | +| Uncached input/trial | 129,476 | 122,029 | -5.8% | +| Cumulative input/trial | 5,746,183 | 5,318,199 | -7.4% | +| Output/trial | 37,573 | 33,751 | -10.2% | +| Exec/apply calls/trial | 61.33 | 61.33 | unchanged | +| Retrieval calls/trial | 3.00 | 3.33 | +11.1% | +| Retrieval chars/trial | 21,697 | 18,742 | -13.6% | +| Edit actions/trial | 23.67 | 24.33 | +2.8% | +| Tool calls/trial | 64.33 | 64.67 | +0.5% | + +One full solve received `packages/core/src/trait/trait.ts` as transition evidence and inspected trait/relation producers before editing. The other full solve made no retrieval calls, so its success is run variance rather than a retrieval win. The failed trial received the same producer citation but still missed removal/destruction/coexistence, showing that the extra evidence is helpful for some trajectories but not sufficient. H3 is retained because solve count improved while all token metrics fell materially. + +## Hypothesis 4: evidence-gap descriptions and smaller search payloads + +### Proposed change + +- Describe `change_surface` as a once-per-change evidence bundle that should be inspected before separate searches. +- Describe `search` as a tool for a specific unresolved gap, queried by symbol or observable behavior in the narrowest scope. +- Reduce search's default/maximum results from 5/10 to 4/6. + +### Expected benefit + +The two H3 retrieval users each made three searches returning about 14k characters after `change_surface`. Better descriptions should reduce redundant searches; the lower bound caps remaining payload while preserving top-ranked evidence. Score should remain unchanged. + +### Outcome + +H4 regressed and was rolled back before H5: + +| Metric | H3 | H4 | Change | +| --- | ---: | ---: | ---: | +| Full solves | 2/3 | 0/3 | -2 solves | +| Mean partial | 0.990476 | 0.977778 | -0.012698 | +| Uncached input/trial | 122,029 | 164,467 | +34.8% | +| Cumulative input/trial | 5,318,199 | 6,622,466 | +24.5% | +| Output/trial | 33,751 | 39,949 | +18.4% | +| Exec/apply calls/trial | 61.33 | 68.67 | +12.0% | +| Retrieval calls/trial | 3.33 | 4.00 | +20.0% | +| Retrieval chars/trial | 18,742 | 22,752 | +21.4% | +| Edit actions/trial | 24.33 | 25.67 | +5.5% | +| Tool calls/trial | 64.67 | 72.67 | +12.4% | + +The six-result cap reduced each search response, and one attempt used only one search, but the cohort as a whole made more retrieval and command calls and spent substantially more tokens. Tool descriptions did not reliably prevent redundant search. Both H4 descriptions and limits were reverted to H3 values. + +## Hypothesis 5: compact post-edit symbol traces + +### Proposed change + +Keep H3's pre-edit retrieval unchanged. Make `trace_symbols` return deduplicated path/line citations rather than repeated snippets, and reduce its per-group default/maximum from 4/6 to 2/3. + +### Expected benefit + +Trace output was the largest single retrieval response at 9-11k characters in H3. It occurs after implementation, and agents need group presence, paths, and missing groups—not duplicate source excerpts. Compaction should cut retrieval and input tokens without affecting solve quality or tool-call count. + +### Outcome + +H5 reduced trace payload but regressed the primary outcome, so it was rolled back: + +| Metric | H3 | H5 | Change | +| --- | ---: | ---: | ---: | +| Full solves | 2/3 | 1/3 | -1 solve | +| Mean partial | 0.990476 | 0.987302 | -0.003175 | +| Uncached input/trial | 122,029 | 119,204 | -2.3% | +| Cumulative input/trial | 5,318,199 | 5,825,158 | +9.5% | +| Output/trial | 33,751 | 33,222 | -1.6% | +| Exec/apply calls/trial | 61.33 | 64.67 | +5.4% | +| Retrieval calls/trial | 3.33 | 3.67 | +10.0% | +| Retrieval chars/trial | 18,742 | 13,226 | -29.4% | +| Edit actions/trial | 24.33 | 25.33 | +4.1% | +| Tool calls/trial | 64.67 | 68.33 | +5.7% | + +Per-call trace payload fell from 9-11k to 1.7-2.1k characters, proving the compaction mechanism worked. That local saving did not reduce cumulative context or calls, and solve quality fell. The compact citation type, lower trace limits, and description were reverted. H3 remains the winner. + +## Winner selected for the full suite + +H3 is the retained configuration: H1's concise eval treatment and three-tool workflow, plus one `state_transitions` producer citation in `change_surface`. H2, H4, and H5 were rolled back; the aborted compact-prompt run is excluded. + +### Full five-task, three-attempt result + +The winner run completed 15/15 valid trials with no infrastructure or rate-limit failures. + +| Cohort | Full solves | Mean partial | Uncached input/trial | Cumulative input/trial | Output/trial | Tool calls/trial | Retrieval calls/trial | Retrieval chars/trial | Edit actions/trial | Agent duration/trial | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| Baseline | 3/15 | 0.989192 | 100,184 | 4,175,334 | 30,345 | 51.13 | 0.00 | 0 | 23.20 | 471.1s | +| Old OpenWiki | 6/15 | 0.992832 | 136,772 | 5,192,270 | 30,478 | 57.93 | 9.00 | 70,413 | 16.00 | 929.6s | +| Current OpenWiki | 4/15 | 0.984533 | 125,617 | 5,838,432 | 32,721 | 60.73 | 5.73 | 30,650 | 21.00 | 841.0s | +| H3 winner | **7/15** | **0.994350** | 132,469 | 6,198,174 | 34,006 | 61.07 | **3.20** | **18,291** | 19.07 | **599.7s** | + +H3 has the best quality: +4 full solves over baseline, +1 over old OpenWiki, and +3 over current OpenWiki. Against current OpenWiki, it holds total tool calls nearly flat (+0.6%), cuts retrieval calls 44%, retrieval payload 40%, edit actions 9%, and agent duration 29%. The tradeoff is +5.5% uncached input, +6.2% cumulative input, and +3.9% output tokens. Against baseline, quality improves substantially but costs 32% more uncached input, 48% more cumulative input, and 19% more tool calls. + +Tool-call accounting treats every Codex exec/apply invocation and every MCP retrieval as one call. Edit actions are already included in exec/apply calls and are reported separately as a churn diagnostic; they are not double-counted in total tool calls. + +| Task | Full solves | Mean partial | Uncached input | Cumulative input | Output | Tool calls | Retrieval calls | Retrieval chars | Edit actions | Agent duration | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| Composite aspects | 1/3 | 0.995516 | 157,513 | 8,609,858 | 45,190 | 80.67 | 3.67 | 18,540 | 30.33 | 764.7s | +| Deferred mutation | 3/3 | 1.000000 | 116,375 | 4,779,118 | 27,944 | 56.00 | 5.33 | 24,894 | 13.00 | 511.4s | +| Entity snapshots | 2/3 | 0.994911 | 87,182 | 2,307,435 | 24,183 | 35.00 | 1.33 | 11,825 | 8.33 | 374.4s | +| Pair tracking | 1/3 | 0.996825 | 165,004 | 9,510,027 | 39,297 | 80.00 | 3.00 | 20,210 | 25.67 | 720.6s | +| Query predicates | 0/3 | 0.984496 | 136,271 | 5,784,433 | 33,415 | 53.67 | 2.67 | 15,988 | 18.00 | 627.7s | + +Compared with old/current OpenWiki task solves, H3 improved composite aspects to 1/3 and deferred mutation to 3/3, retained 1/3 pair solves, and remained 0/3 on query predicates. Entity snapshots fell from 3/3 to 2/3; its one miss was limited to tag-relation omission and roundtrip world-diff identity. + +The remaining failures are concentrated and consistent: + +- Composite: constructor arity and one removed-constituent transition. +- Pair: trait-plus-pair or static-plus-temporal conjunction semantics. +- Query predicates: `Added`/`Removed`/`Changed(predicate)` observation windows and independent predicate trackers. +- Entity: tag-relation snapshot omission and exact roundtrip diff identity. + +The strongest retained product change is transition-producer evidence inside the existing `change_surface` tool. The clearest negative finding is that more or more-forceful prompting did not help: making the long managed block visible increased tokens/calls and worsened score. Search-result caps and compact final tracing both reduced their local payloads but did not improve end-to-end efficiency or quality, so both were reverted. diff --git a/evals/deepswe/OPENWIKI_TRACE_FINDINGS.md b/evals/deepswe/OPENWIKI_TRACE_FINDINGS.md index d52f7c4e..0669c08a 100644 --- a/evals/deepswe/OPENWIKI_TRACE_FINDINGS.md +++ b/evals/deepswe/OPENWIKI_TRACE_FINDINGS.md @@ -12,16 +12,28 @@ That gain currently costs context. OpenWiki increased uncached coding-agent inpu The reported **35.5% lower mean end-to-end time should not yet be treated as a product claim**. The difficult pair, query, and composite tasks were much faster, but deferred mutation and entity snapshot were slower; run scheduling and infrastructure varied. A controlled sequential rerun is needed to isolate the effect. +## Changes applied from these findings + +The retrieval and prompting changes are implemented but have not yet been re-evaluated on DeepSWE: + +- The MCP surface is reduced from eight tools to `search`, `change_surface`, and batched `trace_symbols`. +- `search` automatically combines exact, BM25, semantic, and OKF ranking behind `all`, `wiki`, `source_code`, and `tests` scopes. Ranking implementation is no longer an agent choice. +- `tests` results include stable test names, deduplicate canonical/generated publish mirrors, and receive lifecycle/transition vocabulary expansion. +- `trace_symbols` accepts up to 12 plain or dotted identifiers, re-indexes once, deduplicates symbols, and returns one grouped audit. +- Limits are clamped to documented bounds instead of failing and forcing retries. Search payloads omit ranking diagnostics, snippets are shorter, and `change_surface` returns at most two related wiki concepts. +- Generated agent guidance now makes test-scoped search conditional, calls `change_surface` only for cross-boundary changes, performs one final symbol batch, avoids duplicate wiki/retrieval reads, and requests quiet focused validation. +- Wiki-generation guidance now asks for explicit observation windows, tracker identity, truth transitions, static-plus-temporal composition, net/coalesced deferred effects, unchanged-update behavior, constructor invariants, exact test names, and quiet validation commands. + ## Where OpenWiki helped -| Signal | Baseline | OpenWiki | Interpretation | -| --- | ---: | ---: | --- | -| Full solves | 3/15 | 6/15 | Promising quality improvement; sample is still small | -| Mean partial score | 0.9892 | 0.9928 | Failures became narrower | -| File-edit actions/trial | 23.2 | 16.0 | Less rework and patch churn | -| `rg` commands/trial | 4.93 | 2.40 | Retrieval replaced broad text search | -| Tool calls/trial | 52.3 | 57.9 | Retrieval added more calls than it eliminated | -| Uncached input | baseline | +36.5% | Retrieval remains too expensive | +| Signal | Baseline | OpenWiki | Interpretation | +| ----------------------- | -------: | -------: | ---------------------------------------------------- | +| Full solves | 3/15 | 6/15 | Promising quality improvement; sample is still small | +| Mean partial score | 0.9892 | 0.9928 | Failures became narrower | +| File-edit actions/trial | 23.2 | 16.0 | Less rework and patch churn | +| `rg` commands/trial | 4.93 | 2.40 | Retrieval replaced broad text search | +| Tool calls/trial | 52.3 | 57.9 | Retrieval added more calls than it eliminated | +| Uncached input | baseline | +36.5% | Retrieval remains too expensive | The strongest task-level result was pair-relation tracking. Baseline trials failed across cancellation, exclusive replacement, destruction, wildcard removal, coexistence, and transition cases. OpenWiki narrowed this to one repeated mixed-requirement edge case and achieved one full solve. The traces show agents explicitly converting requirements into lifecycle checks, following the public package surface, and finding a bundler-only issue through consumer validation. @@ -53,25 +65,24 @@ These should be expressed as compact behavior matrices with links to authoritati Across OpenWiki trials, retrieval was called 135 times, averaging nine calls and about 70k returned characters per trial. Fourteen calls (10.4%) were invalid because requested limits exceeded 20 or `symbol_trace` rejected dotted/multiple symbols. Fixing these retries is the first priority. -| Tool | Calls | Decision | -| --- | ---: | --- | -| `symbol_trace` | 54 | Keep, but batch symbols, accept dotted names, cap output, and replace per-symbol prompting with one final surface audit | -| `change_surface` | 32 | Keep; make the initial result pointer-first and run final verification only for public/export/generated/registration changes | -| `test_search` | 16 | Keep optional; deduplicate canonical/generated mirrors and return exact test names plus short behavioral snippets | -| `hybrid_search` | 13 | Keep as the default broad discovery tool; it already incorporates semantic ranking | -| Keyword/BM25/OKF graph | 20 total | Preserve as retrieval engines, but consider exposing them as modes or fallbacks behind hybrid search rather than separate default tools | -| Standalone semantic search | 0 | Hide from the default surface unless it gains a distinct workflow; do not remove semantic ranking from hybrid search | +| Previous tool | Calls | Applied decision | +| -------------------------- | -------: | -------------------------------------------------------------------------------------------------------------- | +| `symbol_trace` | 54 | Replaced by batched `trace_symbols`, including dotted names and one shared re-index | +| `change_surface` | 32 | Retained with compact results; prompt use is limited to public/cross-package/generated/registration changes | +| `test_search` | 16 | Folded into `search(scope: "tests")`; use is conditional and results expose test names and deduplicate mirrors | +| `hybrid_search` | 13 | Replaced by `search`; hybrid ranking remains the automatic default | +| Keyword/BM25/OKF graph | 20 total | Removed from the agent tool surface and retained as internal ranking engines | +| Standalone semantic search | 0 | Removed from the tool surface; semantic ranking remains inside `search` | -`symbol_trace` is overused: 29 of its 54 calls came from the entity-snapshot task. `change_surface` is commonly called twice and sometimes three times with overlapping results. The tool surface should guide agents toward four workflows—change mapping, broad discovery, focused test discovery, and batched public-surface verification—rather than exposing every ranking implementation as a separate choice. +`symbol_trace` was overused: 29 of its 54 calls came from the entity-snapshot task. `change_surface` was commonly called twice and sometimes three times with overlapping results. The new surface represents three agent decisions—search, pre-edit change mapping, and post-edit batched verification—while scope selects the corpus without exposing ranking internals. ## Recommended next experiments -1. **Fix tool ergonomics:** clamp limits, accept dotted symbols, add multi-symbol tracing, and eliminate identical retry calls. -2. **Run the H4 policy with batched tracing:** compare current `symbol_trace` against one final batch audit. -3. **Make retrieval pointer-first:** return paths, symbols, test names, and small snippets by default; expand only on request. Target under 20k retrieval characters per trial. -4. **Improve `test_search`:** rank by requested transition behavior and observation phase, deduplicate generated mirrors, then compare optional use against H4 alone. -5. **Add quiet validation guidance:** capture failures in full but suppress successful build/test logs. OpenWiki trials produced substantially more validation output. -6. **Use query predicates as the discriminator:** test whether new observation-window and tracker-state wiki content converts the repeated 39/43 result into a full solve. -7. **Repeat timing under controlled scheduling:** same task order, concurrency, warmup, and infrastructure; report medians and successful-trial timing separately. +1. **Re-run the H4 policy with the new surface:** measure retrieval calls, invalid calls, payload characters, coding-agent tokens, edits, and score. Invalid retrieval calls should fall to zero. +2. **Validate compact retrieval:** target under 20k retrieval characters per trial without reducing solve rate or partial score. +3. **Ablate test scope:** compare conditional `search(scope: "tests")` against the same policy with test retrieval disabled. +4. **Use query predicates as the discriminator:** test whether new observation-window and tracker-state wiki content converts the repeated 39/43 result into a full solve. +5. **Measure batched tracing:** compare symbols per call, trace payload, and public-surface misses against the old per-symbol behavior. +6. **Repeat timing under controlled scheduling:** same task order, concurrency, warmup, and infrastructure; report medians and successful-trial timing separately. The near-term objective should be to preserve OpenWiki's solve-rate and rework gains while removing duplicated context. The best current direction is **behavior-matrix prompting plus compact, workflow-oriented retrieval**, not mandatory use of more tools. diff --git a/evals/deepswe/README.md b/evals/deepswe/README.md index 12054012..1c085a6f 100644 --- a/evals/deepswe/README.md +++ b/evals/deepswe/README.md @@ -52,6 +52,12 @@ install and run Codex and OpenWiki's pinned SQLite binding, plus the LangSmith API and trace-ingest hosts required by every traced run. The adapter uses the task image's existing Node runtime and installs the pinned Codex CLI directly, avoiding Harbor's NVM bootstrap. + +For Docker runs, the harness removes inactive per-trial networks after each job +and checks completed prior jobs for stale networks before launching. Cleanup is +restricted to networks derived from Harbor result directories, verifies exact +Docker Compose ownership labels, and skips every network with an attached +container; it never performs a global Docker network prune. If `OPENAI_BASE_URL` uses another gateway, pass its hostname (not a URL) with `--allow-host gateway.example.com`. The separate verifier environment remains offline. @@ -156,11 +162,55 @@ Use `--task ''` one or more times to select named tasks. The harness uses Use `--attempts 3` for repeated trials and `--environment modal` for Harbor's hosted parallel environment. +### Named OpenWiki task suites + +Use `--task-suite` for the two exact, reproducible OpenWiki cohorts. A suite +selects all of its members regardless of `--n-tasks` and cannot be combined +with `--task`: + +```bash +# Existing fast iteration set: the five Koota tasks +python3 evals/deepswe/run.py paired --task-suite koota-5 + +# Broader set: the five Koota tasks plus 15 independent repositories +python3 evals/deepswe/run.py paired --task-suite openwiki-20 +``` + +The 15 tasks added to `openwiki-20` are not exposed as a separate runnable +suite. They were selected from the user-provided `gpt-5.6-terra [medium]` +leaderboard export. Across their 42 listed trials they had an 81% failure rate, +40.8 mean steps, and $0.88 mean reported cost. The local harness uses high +reasoning, so these figures are selection signals rather than expected results. +The export did not include token counts; reported cost and steps are only +proxies for token intensity. + +| Task | Repository / language | Terra-medium signal | What it stresses | +| ------------------------------------------ | ----------------------------- | ---------------------- | ------------------------------------------------------------------ | +| `adaptix-name-mapping-aliases` | Adaptix / Python | 1/4 failed, 47.0 steps | High-cost positive control; mapping and serialization seams | +| `dynamodb-toolbox-lazy-recursive-schemas` | DynamoDB Toolbox / TypeScript | 4/4 failed, 40.8 steps | Recursive types, DTO round trips, JSON Schema, update expressions | +| `pebble-durability-wait-apis` | Pebble / Go | 2/2 failed, 45.5 steps | Concurrency, durability callbacks, waits, metrics, reset behavior | +| `scriggo-method-declarations` | Scriggo / Go | 2/2 failed, 44.0 steps | Compiler/runtime method sets and interface dispatch | +| `helm-unified-manifest-stream` | Helm / Go | 1/4 failed, 42.8 steps | Large-repo positive control across multiple command paths | +| `fastapi-implicit-head-options` | FastAPI / Python | 2/3 failed, 38.7 steps | Routing inheritance, configuration, HEAD/OPTIONS semantics | +| `boa-hierarchical-evaluation-cancellation` | Boa / Rust | 3/3 failed, 38.0 steps | Nested cancellation and async lifecycle propagation | +| `bandit-structured-nosec-directives` | Bandit / Python | 2/2 failed, 39.0 steps | Parser state, scoped directives, selector semantics | +| `effect-sse-httpapi-streaming` | Effect / TypeScript | 3/3 failed, 42.3 steps | Large monorepo; server/client streaming and public API propagation | +| `katex-multicolumn-array-spans` | KaTeX / JavaScript | 2/2 failed, 40.5 steps | Parser-to-layout invariants and error handling | +| `prometheus-transactional-reload-status` | Prometheus / Go | 1/2 failed, 36.5 steps | Large repo; transactions, rollback, persistence, HTTP status | +| `opa-template-string-reconstruction` | OPA / Go | 3/3 failed, 39.7 steps | Compiler AST reconstruction and syntax preservation | +| `oxvg-structural-selector-preservation` | OXVG / Rust | 3/3 failed, 39.7 steps | Optimizer correctness under structural CSS selectors | +| `kgateway-consistent-hash-policy` | kgateway / Go | 2/2 failed, 38.5 steps | Kubernetes API-to-runtime translation and merge behavior | +| `python-statemachine-state-data-scoping` | python-statemachine / Python | 3/3 failed, 36.3 steps | Hierarchical state ownership, history, isolation, lifecycle resets | + Treatment runs register `openwiki-retrieval-mcp` inside Codex's isolated home. -It provides keyword, BM25, semantic-vector, OKF graph, hybrid, and change-surface -tools over `/app` plus the generated wiki. Local deterministic vectors are the -default. Pass `--retrieval-embedding-provider openai` to opt into bounded -`text-embedding-3-small` reranking; provider failures fall back to local vectors. +It exposes three read-only workflows over `/app` and the generated wiki: +`search` with `all`, `wiki`, `source_code`, and `tests` scopes; +`change_surface` for pre-edit cross-boundary mapping; and batched +`trace_symbols` for post-edit public-surface verification. Search automatically +combines exact, BM25, semantic-vector, and OKF graph ranking. Local deterministic +vectors are the default. Pass `--retrieval-embedding-provider openai` to opt into +bounded `text-embedding-3-small` reranking; provider failures fall back to local +vectors. If runs already exist, summarize them without invoking Harbor: diff --git a/evals/deepswe/openwiki_codex.py b/evals/deepswe/openwiki_codex.py index a1a2cf8d..c7955d93 100644 --- a/evals/deepswe/openwiki_codex.py +++ b/evals/deepswe/openwiki_codex.py @@ -318,48 +318,12 @@ async def run( _OPENWIKI_SOURCE_DIR / "openwiki" / "quickstart.md" ).as_posix() treatment_instruction = ( - "OpenWiki treatment condition: use the generated wiki and the read-only " - "openwiki_retrieval MCP tools as a just-in-time repository index. At task " - "start, call change_surface with the requested change, then " - f"read {quickstart_path}, search the wiki for the task concepts, and read " - "only the relevant linked pages. Before a repository-wide rg, find, or " - "exploratory directory scan, check the wiki source maps and inspect named " - "files, symbols, and tests directly. Re-consult the wiki when entering a " - "different subsystem, when source contradicts the current understanding, " - "or when blocked by an unfamiliar test or build failure. Do not read " - "operations, release, or integration pages unless the task affects them, " - "and do not reread pages without new evidence. Before finishing a public " - "API or cross-package change, trace the change from its implementation " - "through internal and package exports, generated or publish mirrors, " - "initialization or registration, and the import path real consumers use. " - "Consult the wiki's relevant integration or delivery guidance and run the " - "narrowest consumer-facing check; passing only internal unit tests does not " - "prove the shipped surface works. If the repository generates or copies " - "package artifacts, follow its documented synchronization workflow rather " - "than assuming the defining source module is sufficient. " - "For stateful or lifecycle behavior, turn every externally observable " - "acceptance criterion into a test checklist before editing. Where relevant, " - "cover initial state, false-to-true and true-to-false transitions, unchanged " - "updates, missing dependencies, independent instances, reset or reuse, " - "deferred or re-entrant mutation, and composition with adjacent features. " - "When behavior is unfamiliar, the relevant tests are large, or no analogous " - "focused check is known, use test_search with that behavior matrix and " - "inspect the cited tests directly. " - "Before committing, map each criterion to a passing focused test; one happy " - "path does not establish transition or isolation correctness. " - "Use hybrid_search for broad ranked discovery, okf_graph_search to follow " - "related concepts and cross-package relationships, semantic_search when " - "the repository uses unfamiliar vocabulary, BM25 for precise concepts, " - "test_search when analogous behavioral checks are needed, and keyword_search " - "for exact symbols. Verify all retrieval excerpts in " - "source before editing. After adding or changing a public symbol, call " - "symbol_trace for that exact identifier; investigate missing export, " - "publish, consumer, initialization, or test groups when the repository's " - "architecture requires them. Call change_surface again before finalizing " - "if the patch added a public API, generated artifact, or registration path. " - "The wiki is a navigation aid generated from the same base checkout. " - "Treat /app as the source of truth, make all code changes only in /app, " - "and do not edit /tmp/openwiki-source.\n\n" + "OpenWiki treatment condition: follow the OpenWiki instructions installed " + "in the repository's AGENTS.md. Start with the generated quickstart at " + f"{quickstart_path} and use the read-only openwiki_retrieval MCP tools as " + "needed. The wiki was generated from the same base checkout: treat /app as " + "the source of truth, make all code changes only in /app, and do not edit " + "/tmp/openwiki-source.\n\n" f"{instruction}" ) await super().run(treatment_instruction, environment, context) diff --git a/evals/deepswe/run.py b/evals/deepswe/run.py index 40b714e0..4bae42d9 100644 --- a/evals/deepswe/run.py +++ b/evals/deepswe/run.py @@ -14,6 +14,7 @@ import subprocess import sys import tomllib +import warnings from collections import defaultdict from datetime import datetime from pathlib import Path @@ -42,6 +43,7 @@ SAFE_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/@+-]*$") TASK_FILTER_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/@+*?\[\]-]*$") DNS_LABEL_RE = re.compile(r"^[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?$") +DOCKER_NETWORK_RE = re.compile(r"^[a-z0-9][a-z0-9_-]*$") EVAL_DIR = Path(__file__).resolve().parent PROJECT_ROOT = EVAL_DIR.parents[1] @@ -50,6 +52,35 @@ DEFAULT_JOBS_DIR = EVAL_DIR / "results" DEFAULT_SUMMARY_DIR = EVAL_DIR / "summaries" +KOOTA_5_TASKS = ( + "koota-composite-trait-aspects", + "koota-deferred-mutation-buffer", + "koota-entity-snapshot-rollback", + "koota-pair-relation-tracking", + "koota-query-predicates", +) +WIKI_STRESS_15_TASKS = ( + "adaptix-name-mapping-aliases", + "dynamodb-toolbox-lazy-recursive-schemas", + "pebble-durability-wait-apis", + "scriggo-method-declarations", + "helm-unified-manifest-stream", + "fastapi-implicit-head-options", + "boa-hierarchical-evaluation-cancellation", + "bandit-structured-nosec-directives", + "effect-sse-httpapi-streaming", + "katex-multicolumn-array-spans", + "prometheus-transactional-reload-status", + "opa-template-string-reconstruction", + "oxvg-structural-selector-preservation", + "kgateway-consistent-hash-policy", + "python-statemachine-state-data-scoping", +) +TASK_SUITES = { + "koota-5": KOOTA_5_TASKS, + "openwiki-20": (*KOOTA_5_TASKS, *WIKI_STRESS_15_TASKS), +} + SENSITIVE_FLAGS = {"--agent-env", "--ae"} LANGSMITH_ENV_UNSET = { "HARBOR_LANGSMITH_EXPERIMENT", @@ -144,6 +175,174 @@ def run_checked( subprocess.run(list(argv), cwd=cwd, env=env, check=True, shell=False) +def harbor_job_name(args: argparse.Namespace, condition: str) -> str: + return f"{args.run_name}-{condition}-seed-{args.seed}" + + +def _trial_network_identity( + trial_dir: Path, *, job_dir: Path +) -> tuple[str, str] | None: + """Return the expected Compose project and network for one direct trial child.""" + + try: + resolved_job = job_dir.resolve(strict=True) + resolved_trial = trial_dir.resolve(strict=True) + except OSError: + return None + if not resolved_trial.is_dir() or resolved_trial.parent != resolved_job: + return None + + project_name = f"{resolved_trial.name.lower()}__env" + network_name = f"{project_name}_default" + if not DOCKER_NETWORK_RE.fullmatch(network_name): + return None + return project_name, network_name + + +def _cleanup_docker_networks( + trials: Iterable[tuple[Path, Path]], +) -> None: + """Best-effort removal of inactive, label-verified Harbor trial networks.""" + + candidates: dict[str, str] = {} + for job_dir, trial_dir in trials: + identity = _trial_network_identity(trial_dir, job_dir=job_dir) + if identity is not None: + project_name, network_name = identity + candidates[network_name] = project_name + if not candidates: + return + + try: + listing = subprocess.run( + ["docker", "network", "ls", "--format", "{{.Name}}"], + check=False, + capture_output=True, + text=True, + shell=False, + ) + except OSError: + warnings.warn( + "Could not list Docker networks for Harbor cleanup; continuing", + RuntimeWarning, + stacklevel=2, + ) + return + if listing.returncode != 0: + warnings.warn( + "Could not list Docker networks for Harbor cleanup; continuing", + RuntimeWarning, + stacklevel=2, + ) + return + + available = set(listing.stdout.splitlines()) + for network_name, project_name in sorted(candidates.items()): + if network_name not in available: + continue + try: + inspected = subprocess.run( + ["docker", "network", "inspect", network_name], + check=False, + capture_output=True, + text=True, + shell=False, + ) + if inspected.returncode != 0: + continue + payload = json.loads(inspected.stdout) + if not isinstance(payload, list) or len(payload) != 1: + continue + network = payload[0] + if not isinstance(network, dict): + continue + labels = network.get("Labels") + containers = network.get("Containers") + if ( + network.get("Name") != network_name + or not isinstance(labels, dict) + or labels.get("com.docker.compose.project") != project_name + or labels.get("com.docker.compose.network") != "default" + or not isinstance(containers, dict) + or containers + ): + continue + removed = subprocess.run( + ["docker", "network", "rm", network_name], + check=False, + capture_output=True, + text=True, + shell=False, + ) + if removed.returncode != 0: + warnings.warn( + f"Could not remove inactive Harbor network {network_name!r}; " + "continuing", + RuntimeWarning, + stacklevel=2, + ) + except (OSError, json.JSONDecodeError, TypeError): + warnings.warn( + f"Could not verify Harbor network {network_name!r}; continuing", + RuntimeWarning, + stacklevel=2, + ) + + +def cleanup_stale_docker_networks(jobs_dir: Path) -> None: + """Clean inactive networks from completed Harbor trials under jobs_dir.""" + + trials: list[tuple[Path, Path]] = [] + try: + if not jobs_dir.is_dir(): + return + resolved_jobs = jobs_dir.resolve(strict=True) + for candidate in jobs_dir.iterdir(): + job_dir = candidate.resolve(strict=True) + if job_dir.parent != resolved_jobs: + continue + if not job_dir.is_dir() or not (job_dir / "config.json").is_file(): + continue + for trial_dir in job_dir.iterdir(): + if trial_dir.is_dir() and (trial_dir / "result.json").is_file(): + trials.append((job_dir, trial_dir)) + except OSError: + warnings.warn( + "Could not inspect Harbor results for Docker cleanup; continuing", + RuntimeWarning, + stacklevel=2, + ) + return + _cleanup_docker_networks(trials) + + +def cleanup_job_docker_networks(jobs_dir: Path, job_name: str) -> None: + """Clean inactive networks belonging to the exact current Harbor job.""" + + validate_id(job_name, "job name") + try: + resolved_jobs = jobs_dir.resolve(strict=True) + job_dir = (jobs_dir / job_name).resolve(strict=True) + except OSError: + return + if not job_dir.is_dir() or job_dir.parent != resolved_jobs: + return + try: + trials = [ + (job_dir, trial_dir) + for trial_dir in job_dir.iterdir() + if trial_dir.is_dir() + ] + except OSError: + warnings.warn( + "Could not inspect the current Harbor job for Docker cleanup; continuing", + RuntimeWarning, + stacklevel=2, + ) + return + _cleanup_docker_networks(trials) + + def prepare_deepswe(destination: Path, *, dry_run: bool) -> None: if not destination.exists(): destination.parent.mkdir(parents=True, exist_ok=True) @@ -218,7 +417,7 @@ def harbor_args( for task in args.task: validate_task_filter(task) - job_name = f"{args.run_name}-{condition}-seed-{args.seed}" + job_name = harbor_job_name(args, condition) command = [ "uvx", "--python", @@ -329,8 +528,13 @@ def select_tasks(args: argparse.Namespace) -> list[str] | None: """Select an exact, reproducible Harbor task set from the pinned checkout.""" tasks_dir = args.deepswe_dir / "tasks" + suite_tasks: list[str] | None = None + if args.task_suite: + suite_tasks = list(TASK_SUITES[args.task_suite]) + for task_id in suite_tasks: + validate_task_filter(task_id) if not tasks_dir.is_dir(): - return None + return suite_tasks candidates: list[tuple[str, str]] = [] for config_path in sorted(tasks_dir.glob("*/task.toml")): config = tomllib.loads(config_path.read_text(encoding="utf-8")) @@ -339,6 +543,15 @@ def select_tasks(args: argparse.Namespace) -> list[str] | None: validate_task_filter(local_id) if isinstance(configured_name, str): candidates.append((local_id, configured_name)) + if suite_tasks is not None: + available = {local_id for local_id, _ in candidates} + missing = [task_id for task_id in suite_tasks if task_id not in available] + if missing: + raise ValueError( + f"DeepSWE task suite {args.task_suite!r} is missing pinned tasks: " + f"{', '.join(missing)}" + ) + return suite_tasks if args.task: candidates = [ candidate @@ -372,12 +585,34 @@ def run_condition( package_path=package_path, selected_tasks=select_tasks(args), ) - run_checked( - command, - dry_run=args.dry_run, - env_overrides=langsmith_env(args), - env_unset=LANGSMITH_ENV_UNSET, - ) + clean_docker = args.environment == "docker" and not args.dry_run + job_name = harbor_job_name(args, condition) + if clean_docker: + try: + cleanup_stale_docker_networks(args.jobs_dir) + except Exception: + warnings.warn( + "Docker network preflight cleanup failed; continuing", + RuntimeWarning, + stacklevel=2, + ) + try: + run_checked( + command, + dry_run=args.dry_run, + env_overrides=langsmith_env(args), + env_unset=LANGSMITH_ENV_UNSET, + ) + finally: + if clean_docker: + try: + cleanup_job_docker_networks(args.jobs_dir, job_name) + except Exception: + warnings.warn( + "Docker network final cleanup failed; continuing", + RuntimeWarning, + stacklevel=2, + ) def seconds_between(timing: dict[str, Any] | None) -> float | None: @@ -529,6 +764,11 @@ def add_common_options(parser: argparse.ArgumentParser) -> None: parser.add_argument("--environment", choices=("docker", "modal"), default="docker") parser.add_argument("--n-tasks", type=int, default=10) parser.add_argument("--task", action="append", default=[]) + parser.add_argument( + "--task-suite", + choices=tuple(TASK_SUITES), + help="Exact named task set; selects all members regardless of --n-tasks", + ) parser.add_argument("--seed", type=int, default=0) parser.add_argument("--attempts", type=int, default=1) parser.add_argument("--concurrency", type=int, default=1) @@ -561,6 +801,8 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: args = parser.parse_args(argv) if args.n_tasks <= 0 or args.attempts <= 0 or args.concurrency <= 0: parser.error("--n-tasks, --attempts, and --concurrency must be positive") + if args.task_suite and args.task: + parser.error("--task-suite cannot be combined with --task") return args diff --git a/evals/deepswe/test_run.py b/evals/deepswe/test_run.py index a51aa9f4..2ff854d2 100644 --- a/evals/deepswe/test_run.py +++ b/evals/deepswe/test_run.py @@ -85,19 +85,14 @@ def test_openwiki_treatment_uses_just_in_time_navigation(self) -> None: adapter = (deepswe_run.EVAL_DIR / "openwiki_codex.py").read_text( encoding="utf-8" ) - self.assertIn("just-in-time", adapter) - self.assertIn("Before a repository-wide rg", adapter) - self.assertIn("only the relevant linked pages", adapter) - self.assertIn("Re-consult the wiki", adapter) - self.assertIn("import path real consumers use", adapter) - self.assertIn("passing only internal unit tests", adapter) + self.assertIn("generated quickstart", adapter) + self.assertIn("follow the OpenWiki instructions", adapter) self.assertIn("codex mcp add openwiki_retrieval", adapter) - self.assertIn("call change_surface", adapter) - self.assertIn("symbol_trace", adapter) - self.assertIn("okf_graph_search", adapter) - self.assertIn("every externally observable", adapter) - self.assertIn("independent instances", adapter) - self.assertIn("use test_search", adapter) + self.assertIn("read-only openwiki_retrieval MCP tools", adapter) + self.assertIn("treat /app as", adapter) + self.assertIn("the source of truth", adapter) + self.assertIn("do not edit", adapter) + self.assertIn("/tmp/openwiki-source", adapter) def test_eval_defaults_use_terra_without_changing_openwiki_defaults(self) -> None: args = deepswe_run.parse_args(["paired"]) @@ -227,6 +222,139 @@ def test_run_clears_ambient_experiment_overrides(self) -> None: self.assertNotIn("HARBOR_LANGSMITH_EXPERIMENT", child_env) self.assertNotIn("HARBOR_LANGSMITH_EXPERIMENT_ID", child_env) + def test_cleanup_removes_only_inactive_owned_trial_network(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + jobs_dir = Path(temp_dir) + job_dir = jobs_dir / "example-job" + trial_dir = job_dir / "koota-query__AbC123" + trial_dir.mkdir(parents=True) + (job_dir / "config.json").write_text("{}\n", encoding="utf-8") + (trial_dir / "result.json").write_text("{}\n", encoding="utf-8") + network_name = "koota-query__abc123__env_default" + project_name = "koota-query__abc123__env" + + def docker_run(command: list[str], **kwargs: object) -> SimpleNamespace: + self.assertFalse(kwargs["shell"]) + if command[2] == "ls": + return SimpleNamespace(returncode=0, stdout=f"{network_name}\n") + if command[2] == "inspect": + return SimpleNamespace( + returncode=0, + stdout=json.dumps( + [ + { + "Name": network_name, + "Labels": { + "com.docker.compose.project": project_name, + "com.docker.compose.network": "default", + }, + "Containers": {}, + } + ] + ), + ) + self.assertEqual(["docker", "network", "rm", network_name], command) + return SimpleNamespace(returncode=0, stdout=network_name) + + with patch.object( + deepswe_run.subprocess, "run", side_effect=docker_run + ) as run: + deepswe_run.cleanup_stale_docker_networks(jobs_dir) + + commands = [call.args[0] for call in run.call_args_list] + self.assertIn(["docker", "network", "inspect", network_name], commands) + self.assertIn(["docker", "network", "rm", network_name], commands) + + def test_cleanup_skips_active_or_foreign_trial_networks(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + job_dir = Path(temp_dir) / "example-job" + active_trial = job_dir / "active__AbC" + foreign_trial = job_dir / "foreign__DeF" + active_trial.mkdir(parents=True) + foreign_trial.mkdir() + network_names = { + "active__abc__env_default": { + "com.docker.compose.project": "active__abc__env", + "com.docker.compose.network": "default", + }, + "foreign__def__env_default": { + "com.docker.compose.project": "another-project", + "com.docker.compose.network": "default", + }, + } + + def docker_run(command: list[str], **kwargs: object) -> SimpleNamespace: + self.assertFalse(kwargs["shell"]) + if command[2] == "ls": + return SimpleNamespace( + returncode=0, stdout="\n".join(network_names) + "\n" + ) + network_name = command[3] + containers = ( + {"attached": {}} if network_name.startswith("active") else {} + ) + return SimpleNamespace( + returncode=0, + stdout=json.dumps( + [ + { + "Name": network_name, + "Labels": network_names[network_name], + "Containers": containers, + } + ] + ), + ) + + with patch.object( + deepswe_run.subprocess, "run", side_effect=docker_run + ) as run: + deepswe_run._cleanup_docker_networks( + [(job_dir, active_trial), (job_dir, foreign_trial)] + ) + + self.assertFalse( + any(call.args[0][2] == "rm" for call in run.call_args_list) + ) + + def test_run_condition_cleans_docker_networks_even_when_harbor_fails(self) -> None: + args = deepswe_run.parse_args( + ["baseline", "--env-file", "credentials.env"] + ) + with ( + patch.object(deepswe_run, "cleanup_stale_docker_networks") as preflight, + patch.object(deepswe_run, "cleanup_job_docker_networks") as final_cleanup, + patch.object( + deepswe_run, "run_checked", side_effect=RuntimeError("harbor failed") + ), + ): + with self.assertRaisesRegex(RuntimeError, "harbor failed"): + deepswe_run.run_condition(args, "baseline") + + preflight.assert_called_once_with(args.jobs_dir) + final_cleanup.assert_called_once_with( + args.jobs_dir, deepswe_run.harbor_job_name(args, "baseline") + ) + + def test_cleanup_failure_does_not_mask_harbor_failure(self) -> None: + args = deepswe_run.parse_args( + ["baseline", "--env-file", "credentials.env"] + ) + with ( + patch.object(deepswe_run, "cleanup_stale_docker_networks"), + patch.object( + deepswe_run, + "cleanup_job_docker_networks", + side_effect=RuntimeError("cleanup failed"), + ), + patch.object( + deepswe_run, "run_checked", side_effect=RuntimeError("harbor failed") + ), + self.assertWarnsRegex(RuntimeWarning, "final cleanup failed"), + ): + with self.assertRaisesRegex(RuntimeError, "harbor failed"): + deepswe_run.run_condition(args, "baseline") + def test_seeded_task_selection_is_reproducible(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: root = Path(temp_dir) @@ -253,6 +381,75 @@ def test_seeded_task_selection_is_reproducible(self) -> None: self.assertEqual(2, len(first or [])) self.assertTrue(all("/" not in task_id for task_id in first or [])) + def test_named_task_suites_are_independent_and_composable(self) -> None: + koota = deepswe_run.TASK_SUITES["koota-5"] + stress = deepswe_run.WIKI_STRESS_15_TASKS + combined = deepswe_run.TASK_SUITES["openwiki-20"] + + self.assertEqual({"koota-5", "openwiki-20"}, set(deepswe_run.TASK_SUITES)) + self.assertEqual(5, len(koota)) + self.assertEqual(15, len(stress)) + self.assertEqual(20, len(combined)) + self.assertTrue(set(koota).isdisjoint(stress)) + self.assertEqual((*koota, *stress), combined) + + def test_named_task_suite_selects_every_exact_member(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + expected = deepswe_run.TASK_SUITES["openwiki-20"] + for name in expected: + task_dir = root / "tasks" / name + task_dir.mkdir(parents=True) + (task_dir / "task.toml").write_text( + f'[task]\nname = "datacurve/{name}"\n', encoding="utf-8" + ) + args = deepswe_run.parse_args( + [ + "baseline", + "--deepswe-dir", + str(root), + "--task-suite", + "openwiki-20", + "--n-tasks", + "1", + ] + ) + + self.assertEqual(list(expected), deepswe_run.select_tasks(args)) + + def test_named_task_suite_reports_missing_pinned_tasks(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + task_dir = root / "tasks" / deepswe_run.KOOTA_5_TASKS[0] + task_dir.mkdir(parents=True) + (task_dir / "task.toml").write_text( + f'[task]\nname = "datacurve/{task_dir.name}"\n', encoding="utf-8" + ) + args = deepswe_run.parse_args( + [ + "baseline", + "--deepswe-dir", + str(root), + "--task-suite", + "koota-5", + ] + ) + + with self.assertRaisesRegex(ValueError, "missing pinned tasks"): + deepswe_run.select_tasks(args) + + def test_named_task_suite_cannot_be_combined_with_task_filter(self) -> None: + with self.assertRaises(SystemExit): + deepswe_run.parse_args( + [ + "baseline", + "--task-suite", + "koota-5", + "--task", + "koota-*", + ] + ) + def test_load_and_aggregate_trial_rows(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: job_dir = Path(temp_dir) diff --git a/src/agent/prompt.ts b/src/agent/prompt.ts index 4eb06fc2..382f413a 100644 --- a/src/agent/prompt.ts +++ b/src/agent/prompt.ts @@ -225,8 +225,9 @@ function createCodingAgentUtilityRequirements( - Make the distinction between internal correctness and shipped-surface correctness explicit. A new API is not complete merely because its defining module typechecks or its unit tests pass; future agents must be able to verify that the API resolves from the import path real consumers use and that required registration or generated artifacts are present. - Separate ordinary focused checks from expensive integration, root-test, release, package-build, generated-artifact, and performance checks. Label expensive checks as conditional and state the source-backed condition that makes each one necessary. Do not encourage broad validation by default. - When a change crosses a public, package, generated-artifact, or runtime-registration boundary, identify the narrowest consumer-facing smoke test or package validation command that exercises that boundary. Record any source-backed synchronization command and the canonical source of generated files so agents do not validate only an internal package or hand-edit derived output. -- For stateful or lifecycle extension seams, document a source-backed behavioral test matrix when applicable: initial state; both transition directions; unchanged updates; missing prerequisites; isolation between independent instances; reset or reuse; deferred or re-entrant mutation; and composition with adjacent features. Link each invariant to the narrowest existing test or test location so future agents can turn every externally observable acceptance criterion into a focused check. -- Make analogous tests retrievable by describing the behavior and invariant they exercise, not just the implementation symbol. When large test files cover multiple lifecycle phases, identify the relevant suite or stable test names so a future \`test_search\` can reach the right section without reading from the top. +- For stateful or lifecycle extension seams, document a source-backed behavioral test matrix when applicable: initial state; false-to-true and true-to-false transitions; unchanged updates; missing prerequisites; isolation between independent instances and tracker identity; reset, reuse, and observation-window boundaries; deferred or re-entrant mutation including net/coalesced effects; and composition between static and temporal constraints. Record constructor or composition invariants when they are externally observable. Link each invariant to the narrowest existing test or test location so future agents can turn every acceptance criterion into a focused check. +- Make analogous tests retrievable by describing the behavior and invariant they exercise, not just the implementation symbol. When large test files cover multiple lifecycle phases, identify the relevant suite or stable test names so a future \`search\` call scoped to \`tests\` can reach the right section without reading from the top. +- Keep validation commands narrow and quiet by default. Identify flags or focused commands that suppress successful output while preserving complete failure diagnostics; do not make agents consume verbose build logs merely to confirm success. - Keep navigation stable and concise: use one canonical home per concept, link to it instead of duplicating prose, and keep operational/release guidance out of runtime reading paths unless it is genuinely required. - Before finishing, simulate navigation for representative adjacent changes grounded in the repository's actual components and history. Verify that a future agent can reach the first implementation files, important symbols/invariants, focused tests, and minimal validation command from the quickstart without a repository-wide search. Repair navigation gaps found by this audit.`; } diff --git a/src/code-mode.ts b/src/code-mode.ts index 4e4e1aa1..79f60425 100644 --- a/src/code-mode.ts +++ b/src/code-mode.ts @@ -128,17 +128,17 @@ function createCodeModeAgentsSnippet(): string { This repository uses OpenWiki for recurring code documentation. Use \`openwiki/\` as a just-in-time repository index: -- At task start, read \`openwiki/quickstart.md\`, then search the wiki for the task's concepts and read only the relevant linked pages. +- At task start, read \`openwiki/quickstart.md\`, then search the wiki for the task's concepts and read only the relevant linked sections. - Before a repository-wide \`rg\`, \`find\`, or exploratory directory scan, check the wiki's source maps. When they name relevant files, symbols, or tests, inspect those paths directly. -- Re-consult the wiki when entering a different subsystem, when source evidence contradicts the current understanding, or when blocked by an unfamiliar test or build failure. +- Re-consult the wiki when entering a different subsystem, when source evidence contradicts the current understanding, or when blocked by an unfamiliar test or build failure. Do not reread content already returned by retrieval unless surrounding context is needed. - Treat source code and tests as authoritative. Verify wiki claims in source before editing. - Before finishing a public API or cross-package change, trace it from implementation through barrel/package exports, generated or publish mirrors, initialization or registration, and the import path consumers actually use. Consult the relevant wiki integration or delivery guidance and run the narrowest consumer-facing check; internal unit tests alone do not prove the shipped surface works. -- If an \`openwiki_retrieval\` MCP server is available, use \`change_surface\` before editing and \`symbol_trace\` after adding or changing each public symbol. Treat missing groups as verification gaps to investigate against repository architecture, not automatic requirements. -- For stateful or lifecycle changes, translate each externally observable acceptance criterion into a focused test checklist. Cover relevant initial state, both transition directions, unchanged updates, missing dependencies, independent instances, reset or reuse, deferred or re-entrant mutation, and composition; map every criterion to a passing test before finishing. -- If the retrieval server provides \`test_search\`, use it with that behavior matrix to find analogous focused tests, then inspect the cited tests directly before implementing lifecycle semantics. +- If an \`openwiki_retrieval\` MCP server is available, use \`search\` for focused retrieval. Use \`change_surface\` before public, cross-package, generated-artifact, or runtime-registration edits. After changing public symbols, call \`trace_symbols\` once with all of them and treat missing groups as verification gaps, not automatic requirements. +- Before editing stateful or lifecycle behavior, write a compact state model: list every identity axis that partitions state (such as tracker/factory, entity, relation or predicate, and target), define the relevant transitions and observation/reset window, and choose one canonical owner or ledger for those transitions. Turn each externally observable criterion into an input/event/expected-result oracle row. Cover relevant initial state, false-to-true and true-to-false transitions, unchanged updates, missing dependencies, independent instances, reset/reuse, deferred or re-entrant net effects, and static-plus-temporal composition; map every row to a passing focused test before finishing. +- When behavior is unfamiliar, relevant tests are large, or no analogous focused check is known, call \`search\` with the \`tests\` scope using that behavior matrix, then inspect the cited tests directly. Skip this search when the exact focused test is already known. - When the repository generates or copies package artifacts, identify the canonical source and repository-supported synchronization command. Do not hand-edit derived output unless the documented workflow explicitly requires it. - Do not read operations, release, or integration pages unless the task affects those areas. If a requested feature does not exist yet, use the wiki to locate extension points, then inspect source rather than searching the wiki for the implementation. -- Do not reread pages already consulted unless new evidence requires it. +- Prefer the narrowest quiet validation command available. Suppress successful build/test noise when possible, but preserve complete failure output. The scheduled OpenWiki GitHub Actions workflow refreshes the repository wiki. Do not hand-edit generated OpenWiki pages unless explicitly asked; prefer updating source code/docs and letting OpenWiki regenerate. diff --git a/src/retrieval/mcp-server.ts b/src/retrieval/mcp-server.ts index ee5de035..ed546df6 100644 --- a/src/retrieval/mcp-server.ts +++ b/src/retrieval/mcp-server.ts @@ -2,6 +2,7 @@ import { createInterface } from "node:readline"; import { OPENWIKI_VERSION } from "../constants.js"; +import { RETRIEVAL_TOOL_DEFINITIONS } from "./mcp-tools.js"; import { RetrievalService } from "./search-service.js"; import type { EmbeddingProvider } from "./semantic.js"; import type { SearchScope } from "./types.js"; @@ -13,52 +14,6 @@ interface JsonRpcRequest { params?: Record; } -const TOOL_DEFINITIONS = [ - tool( - "symbol_trace", - "After editing, re-index source and trace one exact public symbol through implementation, exports, publish/generated mirrors, initialization, consumer imports, and tests. Missing groups are verification gaps, not proof that a layer is required.", - querySchema({ limit: integerSchema(1, 12, 6) }), - ), - tool( - "change_surface", - "Find the complete change surface for a feature: relevant OKF concepts, implementation, exports, publish/generated mirrors, initialization, consumer imports, and tests. Use this first for public or cross-package changes.", - querySchema({ limit: integerSchema(1, 12, 6) }), - ), - tool( - "test_search", - "Find analogous focused tests using hybrid keyword, BM25, and semantic ranking restricted to test/spec source chunks. Use this to derive lifecycle, transition, isolation, reset, and composition checks before implementing stateful behavior.", - searchSchema(), - ), - tool( - "hybrid_search", - "Hybrid reciprocal-rank search across BM25, semantic vectors, weighted keywords, and the OKF concept graph.", - searchSchema(), - ), - tool( - "okf_graph_search", - "Search OKF concept metadata, then expand across semantic Markdown relationships, incoming links, and shared tags.", - querySchema({ - hops: integerSchema(0, 2, 1), - limit: integerSchema(1, 20, 8), - }), - ), - tool( - "semantic_search", - "Vector semantic search over bounded wiki/source candidates. The response reports whether OpenAI embeddings or the deterministic local vector fallback was used.", - searchSchema(), - ), - tool( - "bm25_search", - "BM25 lexical search over wiki sections and source-code chunks.", - searchSchema(), - ), - tool( - "keyword_search", - "Fast field-weighted exact and token search over OKF metadata, headings, paths, and content.", - searchSchema(), - ), -] as const; - const options = parseOptions(process.argv.slice(2)); const service = new RetrievalService(options); const input = createInterface({ input: process.stdin, crlfDelay: Infinity }); @@ -86,7 +41,7 @@ async function handleLine(line: string): Promise { writeResult(request.id, { capabilities: { tools: { listChanged: false } }, instructions: - "Use change_surface first for public, cross-package, generated-artifact, or runtime-registration changes. Verify returned citations in source before editing. Use hybrid_search for broad discovery, okf_graph_search for related concepts, semantic_search for vocabulary mismatch, BM25 for precise terms, and keyword_search for exact symbols. All tools are read-only and return bounded excerpts.", + "Use search for focused wiki, source_code, or tests retrieval. Use change_surface before public or cross-package edits, and trace_symbols once after changing public symbols. Verify citations in source. All tools are read-only and return bounded excerpts.", protocolVersion: "2025-06-18", serverInfo: { name: "openwiki-retrieval", version: OPENWIKI_VERSION }, }); @@ -95,7 +50,7 @@ async function handleLine(line: string): Promise { writeResult(request.id, {}); return; case "tools/list": - writeResult(request.id, { tools: TOOL_DEFINITIONS }); + writeResult(request.id, { tools: RETRIEVAL_TOOL_DEFINITIONS }); return; case "tools/call": await callTool(request.id, request.params ?? {}); @@ -119,55 +74,26 @@ async function callTool( ): Promise { const name = typeof params.name === "string" ? params.name : ""; const args = isRecord(params.arguments) ? params.arguments : {}; - const query = requiredString(args.query, "query"); const limit = optionalInteger(args.limit, 8); let result: unknown; switch (name) { - case "symbol_trace": - result = await service.symbolTrace(query, optionalInteger(args.limit, 6)); - break; - case "change_surface": - result = await service.changeSurface( - query, - optionalInteger(args.limit, 6), - ); - break; - case "test_search": - result = await service.testSearch(query, optionalInteger(args.limit, 5)); - break; - case "hybrid_search": - result = await service.hybridSearch( - query, - optionalScope(args.scope), - limit, - ); - break; - case "okf_graph_search": - result = await service.okfGraphSearch( - query, - limit, - optionalInteger(args.hops, 1), - ); - break; - case "semantic_search": - result = await service.semanticSearch( - query, + case "search": + result = await service.search( + requiredString(args.query, "query"), optionalScope(args.scope), limit, ); break; - case "bm25_search": - result = await service.bm25Search( - query, - optionalScope(args.scope), - limit, + case "change_surface": + result = await service.changeSurface( + requiredString(args.query, "query"), + optionalInteger(args.limit, 6), ); break; - case "keyword_search": - result = await service.keywordSearch( - query, - optionalScope(args.scope), - limit, + case "trace_symbols": + result = await service.traceSymbols( + requiredStrings(args.symbols, "symbols"), + optionalInteger(args.limit, 4), ); break; default: @@ -205,46 +131,6 @@ function parseOptions(args: string[]): { }; } -function tool(name: string, description: string, inputSchema: object): object { - return { - annotations: { destructiveHint: false, readOnlyHint: true }, - description, - inputSchema, - name, - }; -} - -function searchSchema(): object { - return querySchema({ - limit: integerSchema(1, 10, 5), - scope: { - default: "all", - enum: ["all", "wiki", "source"], - type: "string", - }, - }); -} - -function querySchema(properties: Record): object { - return { - additionalProperties: false, - properties: { - query: { maxLength: 500, minLength: 1, type: "string" }, - ...properties, - }, - required: ["query"], - type: "object", - }; -} - -function integerSchema( - minimum: number, - maximum: number, - defaultValue: number, -): object { - return { default: defaultValue, maximum, minimum, type: "integer" }; -} - function requiredString(value: unknown, name: string): string { if (typeof value !== "string" || !value.trim()) { throw new Error(`${name} is required.`); @@ -252,6 +138,20 @@ function requiredString(value: unknown, name: string): string { return value; } +function requiredStrings(value: unknown, name: string): string[] { + if (!Array.isArray(value) || value.length === 0) { + throw new Error(`${name} must be a non-empty string array.`); + } + const strings: string[] = []; + for (const item of value) { + if (typeof item !== "string" || !item.trim()) { + throw new Error(`${name} must be a non-empty string array.`); + } + strings.push(item); + } + return strings; +} + function optionalInteger(value: unknown, fallback: number): number { return typeof value === "number" && Number.isInteger(value) ? value @@ -259,7 +159,10 @@ function optionalInteger(value: unknown, fallback: number): number { } function optionalScope(value: unknown): SearchScope { - return value === "source" || value === "wiki" || value === "all" + return value === "source_code" || + value === "tests" || + value === "wiki" || + value === "all" ? value : "all"; } diff --git a/src/retrieval/mcp-tools.ts b/src/retrieval/mcp-tools.ts new file mode 100644 index 00000000..590f4f82 --- /dev/null +++ b/src/retrieval/mcp-tools.ts @@ -0,0 +1,81 @@ +interface ToolDefinition { + annotations: { destructiveHint: false; readOnlyHint: true }; + description: string; + inputSchema: object; + name: string; +} + +export const SEARCH_SCOPES = ["all", "wiki", "source_code", "tests"] as const; + +function integerSchema( + minimum: number, + maximum: number, + defaultValue: number, +): object { + return { default: defaultValue, maximum, minimum, type: "integer" }; +} + +function querySchema(properties: Record): object { + return { + additionalProperties: false, + properties: { + query: { maxLength: 500, minLength: 1, type: "string" }, + ...properties, + }, + required: ["query"], + type: "object", + }; +} + +function tool( + name: string, + description: string, + inputSchema: object, +): ToolDefinition { + return { + annotations: { destructiveHint: false, readOnlyHint: true }, + description, + inputSchema, + name, + }; +} + +export const RETRIEVAL_TOOL_DEFINITIONS = [ + tool( + "search", + "Search the wiki, implementation code, or tests with automatic exact, lexical, semantic, and OKF ranking. Use the tests scope only when analogous behavior is needed, and inspect cited source before relying on it.", + querySchema({ + limit: integerSchema(1, 10, 5), + scope: { + default: "all", + description: + "Search all indexed content, only generated wiki pages, implementation source excluding tests, or only test/spec files.", + enum: SEARCH_SCOPES, + type: "string", + }, + }), + ), + tool( + "change_surface", + "Map a public, stateful, or cross-package change before editing. Returns compact citations for implementation, state-transition producers, exports, publish mirrors, initialization, consumers, and tests.", + querySchema({ limit: integerSchema(1, 12, 7) }), + ), + tool( + "trace_symbols", + "After editing public symbols, re-index once and verify them together across implementation, exports, generated/publish mirrors, initialization, consumers, and tests. Missing groups are verification gaps, not automatic requirements.", + { + additionalProperties: false, + properties: { + limit: integerSchema(1, 6, 4), + symbols: { + items: { maxLength: 200, minLength: 1, type: "string" }, + maxItems: 12, + minItems: 1, + type: "array", + }, + }, + required: ["symbols"], + type: "object", + }, + ), +] as const satisfies readonly ToolDefinition[]; diff --git a/src/retrieval/ranking.ts b/src/retrieval/ranking.ts index acf7fa87..2c5648af 100644 --- a/src/retrieval/ranking.ts +++ b/src/retrieval/ranking.ts @@ -44,6 +44,15 @@ const SYNONYM_GROUPS = [ ["relation", "edge", "link", "pair", "target"], ["aspect", "composite", "trait", "mixin", "schema"], ["diff", "restore", "rollback", "snapshot", "state"], + ["initial", "baseline", "empty", "first", "setup"], + ["add", "added", "enter", "gain", "insert", "true"], + ["remove", "removed", "exit", "lose", "delete", "false"], + ["change", "changed", "mutate", "transition", "update"], + ["unchanged", "noop", "idempotent", "stable"], + ["independent", "isolation", "instance", "tracker"], + ["reset", "reuse", "window", "observation", "generation"], + ["defer", "reentrant", "coalesce", "net", "flush"], + ["compose", "composition", "combine", "mixed"], ] as const; const SYNONYMS = buildSynonyms(); @@ -80,6 +89,10 @@ export function rankKeyword( const title = `${chunk.title ?? ""} ${chunk.heading ?? ""}`.toLowerCase(); const metadata = chunk.fields.toLowerCase(); const text = chunk.text.toLowerCase(); + const pathTerms = new Set(tokenize(path)); + const titleTerms = new Set(tokenize(title)); + const metadataTerms = new Set(tokenize(metadata)); + const textTerms = new Set(tokenize(text)); let score = 0; if (phrase) { if (path.includes(phrase)) score += 10; @@ -88,10 +101,10 @@ export function rankKeyword( if (text.includes(phrase)) score += 5; } for (const term of queryTerms) { - if (path.includes(term)) score += 3.5; - if (title.includes(term)) score += 3; - if (metadata.includes(term)) score += 2; - if (text.includes(term)) score += 1; + if (path.includes(term) || pathTerms.has(term)) score += 3.5; + if (title.includes(term) || titleTerms.has(term)) score += 3; + if (metadata.includes(term) || metadataTerms.has(term)) score += 2; + if (text.includes(term) || textTerms.has(term)) score += 1; } return { chunk, score }; }) diff --git a/src/retrieval/repository-index.ts b/src/retrieval/repository-index.ts index c6280331..f740454b 100644 --- a/src/retrieval/repository-index.ts +++ b/src/retrieval/repository-index.ts @@ -59,6 +59,8 @@ const EXCLUDED_DIRECTORIES = new Set([ const SECRET_FILE = /^(?:\.env(?:\..*)?|.*\.(?:crt|jks|key|keystore|p12|pem|pfx)|credentials\.json|token(?:\.json)?|cookies?(?:\.(?:db|sqlite|txt))?|\.git-credentials|hosts\.yml)$/iu; const MARKDOWN_LINK = /\[([^\]]+)\]\(([^)]+)\)/gu; +const TEST_NAME = + /\b(?:describe|it|test)(?:\.(?:each|only|skip|todo))?\s*\(\s*(["'`])([^\n]{1,160}?)\1/gu; export interface RepositoryIndexOptions { repoRoot: string; @@ -150,16 +152,19 @@ async function readSourceChunks( if (selected.every((line) => !line.trim())) continue; const lineStart = start + 1; const lineEnd = start + selected.length; + const text = selected.join("\n"); + const testNames = extractTestNames(text); chunks.push({ - fields: relative, + fields: [relative, ...testNames].join("\n"), id: `source:${relative}:${lineStart}`, kind: "source", lineEnd, lineStart, path: relative, - scope: "source", + scope: "source_code", tags: pathTags(relative), - text: selected.join("\n"), + ...(testNames.length > 0 ? { testNames } : {}), + text, title: path.basename(relative), }); } @@ -332,6 +337,16 @@ function firstHeading(body: string): string | undefined { return /^#\s+(.+?)\s*$/mu.exec(body)?.[1]?.trim(); } +function extractTestNames(value: string): string[] { + return [ + ...new Set( + [...value.matchAll(TEST_NAME)] + .map((match) => match[2]?.trim()) + .filter((name): name is string => Boolean(name)), + ), + ]; +} + function stringField(value: unknown): string | undefined { return typeof value === "string" && value.trim() ? value.trim() : undefined; } diff --git a/src/retrieval/search-service.ts b/src/retrieval/search-service.ts index 4acde44d..bb3bbc41 100644 --- a/src/retrieval/search-service.ts +++ b/src/retrieval/search-service.ts @@ -16,22 +16,37 @@ import type { SearchResponse, SearchResultItem, SearchScope, + SymbolTraceCategory, + SymbolTraceResult, SymbolTraceResponse, } from "./types.js"; -const DEFAULT_LIMIT = 8; -const MAX_LIMIT = 20; +const DEFAULT_LIMIT = 6; +const MAX_SEARCH_LIMIT = 10; +const MAX_SURFACE_LIMIT = 12; +const MAX_TRACE_LIMIT = 6; +const MAX_SYMBOLS = 12; const MAX_QUERY_LENGTH = 500; -const MAX_RELATED_CONCEPTS = 3; -const MAX_SNIPPET_LENGTH = 320; -const IDENTIFIER = /^[A-Za-z_$][A-Za-z0-9_$]{0,99}$/u; -const SURFACE_CATEGORY_ORDER: ChangeSurfaceCategory[] = [ +const MAX_RELATED_CONCEPTS = 2; +const MAX_SNIPPET_LENGTH = 220; +const DOTTED_IDENTIFIER = + /^[A-Za-z_$][A-Za-z0-9_$]{0,99}(?:\.[A-Za-z_$][A-Za-z0-9_$]{0,99}){0,5}$/u; +const TRACE_CATEGORY_ORDER: SymbolTraceCategory[] = [ + "implementation", + "exports", + "publish_generated", + "initialization", "consumer", "tests", +]; +const CHANGE_SURFACE_CATEGORY_ORDER: ChangeSurfaceCategory[] = [ + "implementation", + "state_transitions", "exports", "publish_generated", "initialization", - "implementation", + "consumer", + "tests", ]; export interface RetrievalServiceOptions { @@ -48,62 +63,7 @@ export class RetrievalService { this.semantic = new SemanticRanker(options.embeddingProvider); } - async keywordSearch( - query: string, - scope: SearchScope = "all", - limit = DEFAULT_LIMIT, - ): Promise { - const { chunks } = await this.corpus(); - return response( - "field-weighted-keyword", - query, - rankKeyword(scopedChunks(chunks, scope), validateQuery(query)), - validateLimit(limit), - ); - } - - async bm25Search( - query: string, - scope: SearchScope = "all", - limit = DEFAULT_LIMIT, - ): Promise { - const { chunks } = await this.corpus(); - return response( - "bm25", - query, - rankBm25(scopedChunks(chunks, scope), validateQuery(query)), - validateLimit(limit), - ); - } - - async semanticSearch( - query: string, - scope: SearchScope = "all", - limit = DEFAULT_LIMIT, - ): Promise { - const { chunks } = await this.corpus(); - const ranked = await this.semantic.rank( - chunks, - validateQuery(query), - validateScope(scope), - ); - return response(ranked.engine, query, ranked.hits, validateLimit(limit)); - } - - async okfGraphSearch( - query: string, - limit = DEFAULT_LIMIT, - hops = 1, - ): Promise { - const corpus = await this.corpus(); - const validQuery = validateQuery(query); - const validHops = - Number.isInteger(hops) && hops >= 0 && hops <= 2 ? hops : 1; - const hits = rankOkfGraph(corpus, validQuery, validHops); - return response("okf-graph", validQuery, hits, validateLimit(limit)); - } - - async hybridSearch( + async search( query: string, scope: SearchScope = "all", limit = DEFAULT_LIMIT, @@ -112,62 +72,36 @@ export class RetrievalService { const validQuery = validateQuery(query); const validScope = validateScope(scope); const chunks = scopedChunks(corpus.chunks, validScope); - const semantic = await this.semantic.rank( - corpus.chunks, - validQuery, - validScope, - ); + const semantic = await this.semantic.rank(chunks, validQuery); const lists = [ - { hits: rankKeyword(chunks, validQuery), name: "keyword", weight: 0.55 }, + { hits: rankKeyword(chunks, validQuery), name: "keyword", weight: 0.75 }, { hits: rankBm25(chunks, validQuery), name: "bm25", weight: 1 }, { hits: semantic.hits, name: "semantic", weight: 0.9 }, ]; - if (validScope !== "source") { + if (validScope === "all" || validScope === "wiki") { lists.push({ hits: rankOkfGraph(corpus, validQuery, 1), name: "okf_graph", weight: 0.8, }); } + const ranked = reciprocalRankFusion(lists); return response( - `hybrid-rrf:${semantic.engine}`, - validQuery, - reciprocalRankFusion(lists), - validateLimit(limit), - ); - } - - async testSearch(query: string, limit = 5): Promise { - const validQuery = validateQuery(query); - const validLimit = validateLimit(limit); - const testChunks = (await this.corpus()).chunks.filter( - (chunk) => chunk.scope === "source" && isTestChunk(chunk), - ); - const semantic = await this.semantic.rank(testChunks, validQuery, "source"); - return response( - `test-hybrid-rrf:${semantic.engine}`, validQuery, - reciprocalRankFusion([ - { - hits: rankKeyword(testChunks, validQuery), - name: "keyword", - weight: 0.6, - }, - { hits: rankBm25(testChunks, validQuery), name: "bm25", weight: 1 }, - { hits: semantic.hits, name: "semantic", weight: 0.9 }, - ]), - validLimit, + validScope === "tests" ? deduplicateTestMirrors(ranked) : ranked, + normalizeLimit(limit, MAX_SEARCH_LIMIT, DEFAULT_LIMIT), + validScope, ); } async changeSurface( query: string, - limit = 6, + limit = 7, ): Promise { const corpus = await this.corpus(); const validQuery = validateQuery(query); - const validLimit = validateLimit(limit); - const concepts = await this.hybridSearch(validQuery, "wiki", validLimit); + const validLimit = normalizeLimit(limit, MAX_SURFACE_LIMIT, 6); + const concepts = await this.search(validQuery, "wiki", validLimit); const conceptChunks = conceptHits(corpus.chunks, concepts.results); const referencedPaths = extractPaths( conceptChunks.map((chunk) => chunk.text).join("\n"), @@ -178,27 +112,35 @@ export class RetrievalService { const expandedQuery = [validQuery, ...symbols.slice(0, 24)].join(" "); const source = reciprocalRankFusion([ { - hits: rankBm25(scopedChunks(corpus.chunks, "source"), expandedQuery), + hits: rankBm25(sourceChunks(corpus.chunks), expandedQuery), name: "bm25", weight: 1, }, { hits: boostReferencedPaths( - rankKeyword(scopedChunks(corpus.chunks, "source"), expandedQuery), + rankKeyword(sourceChunks(corpus.chunks), expandedQuery), referencedPaths, ), name: "wiki_paths", weight: 1.1, }, ]).slice(0, 160); - const groups = emptySurfaceGroups(); - const candidates = emptySurfaceGroups(); + const groups = emptyChangeSurfaceGroups(); + const candidates = emptyChangeSurfaceGroups(); for (const hit of source) { for (const category of categorize(hit.chunk)) { candidates[category].push(toResultItem(hit)); } + if (isStateTransitionProducer(hit.chunk)) { + candidates.state_transitions.push(toResultItem(hit)); + } } - fillSurfaceGroups(groups, candidates, validLimit); + fillGroups( + groups, + candidates, + validLimit, + CHANGE_SURFACE_CATEGORY_ORDER, + ); return { groups, query: validQuery, @@ -206,30 +148,18 @@ export class RetrievalService { }; } - async symbolTrace(query: string, limit = 6): Promise { - const symbol = validateIdentifier(query); - const validLimit = validateLimit(limit); + async traceSymbols( + symbols: string[], + limit = 4, + ): Promise { + const validSymbols = validateSymbols(symbols); + const validLimit = normalizeLimit(limit, MAX_TRACE_LIMIT, 4); this.corpusPromise = undefined; - const sourceChunks = scopedChunks((await this.corpus()).chunks, "source"); - const exactIdentifier = new RegExp( - `(?:^|[^A-Za-z0-9_$])${escapeRegExp(symbol)}(?:$|[^A-Za-z0-9_$])`, - "u", - ); - const candidates = emptySurfaceGroups(); - for (const hit of rankKeyword(sourceChunks, symbol)) { - if (!exactIdentifier.test(hit.chunk.text)) continue; - for (const category of categorize(hit.chunk)) { - candidates[category].push(toResultItem(hit)); - } - } - const groups = emptySurfaceGroups(); - fillSurfaceGroups(groups, candidates, validLimit); + const chunks = sourceChunks((await this.corpus()).chunks); return { - groups, - missing: SURFACE_CATEGORY_ORDER.filter( - (category) => groups[category].length === 0, + traces: validSymbols.map((symbol) => + traceSymbol(chunks, symbol, validLimit), ), - symbol, }; } @@ -341,9 +271,9 @@ function boostReferencedPaths( .sort((left, right) => right.score - left.score); } -function categorize(chunk: IndexedChunk): ChangeSurfaceCategory[] { +function categorize(chunk: IndexedChunk): SymbolTraceCategory[] { const value = `${chunk.path}\n${chunk.text}`; - const categories = new Set(); + const categories = new Set(); if ( /\b(?:exports|entrypoint|public api)\b/iu.test(value) || /\bexport\s+(?:\*|\{[^}]+\})\s+from\b/iu.test(chunk.text) || @@ -382,9 +312,92 @@ function categorize(chunk: IndexedChunk): ChangeSurfaceCategory[] { function isTestChunk(chunk: IndexedChunk): boolean { return ( - /(?:^|\/)(?:test|tests|spec|specs)(?:\/|\.)/iu.test(chunk.path) || - /\b(?:describe|it|test)\s*\(/u.test(chunk.text) + /(?:^|\/)(?:test|tests|spec|specs)(?:\/|$)/iu.test(chunk.path) || + /(?:^|[._-])(?:test|tests|spec|specs)(?:[._-]|$)/iu.test(chunk.path) + ); +} + +function isStateTransitionProducer(chunk: IndexedChunk): boolean { + if ( + isTestChunk(chunk) || + /(?:^|\/)query\/(?:modifier|modifiers)(?:\/|$)/u.test(chunk.path) + ) { + return false; + } + const producerPath = + /(?:^|\/)(?:actions?|entity|mutation|relation|store|trait|world)(?:\/|[._-])/u.test( + chunk.path, + ); + const transitionText = + /\b(?:add|change|defer|destroy|emit|flush|remove|replace|reset|trigger|update)(?:d|s|ing)?\b/iu.test( + chunk.text, + ); + return producerPath && transitionText; +} + +function traceSymbol( + chunks: IndexedChunk[], + symbol: string, + limit: number, +): SymbolTraceResult { + const leaf = symbol.split(".").at(-1) ?? symbol; + const fullPattern = symbol.split(".").map(escapeRegExp).join("\\s*\\.\\s*"); + const exactSymbol = new RegExp( + `(?:^|[^A-Za-z0-9_$])${fullPattern}(?:$|[^A-Za-z0-9_$])`, + "u", ); + const exactLeaf = new RegExp( + `(?:^|[^A-Za-z0-9_$])${escapeRegExp(leaf)}(?:$|[^A-Za-z0-9_$])`, + "u", + ); + const candidates = emptyTraceGroups(); + for (const hit of rankKeyword(chunks, `${symbol} ${leaf}`)) { + if (!exactSymbol.test(hit.chunk.text) && !exactLeaf.test(hit.chunk.text)) { + continue; + } + for (const category of categorize(hit.chunk)) { + candidates[category].push(toResultItem(hit)); + } + } + const groups = emptyTraceGroups(); + fillGroups(groups, candidates, limit, TRACE_CATEGORY_ORDER); + return { + groups, + missing: TRACE_CATEGORY_ORDER.filter( + (category) => candidates[category].length === 0, + ), + symbol, + }; +} + +function deduplicateTestMirrors(hits: RankedHit[]): RankedHit[] { + const deduplicated = new Map(); + for (const hit of hits) { + const key = canonicalTestKey(hit.chunk); + const current = deduplicated.get(key); + if ( + !current || + (isGeneratedTestPath(current.chunk.path) && + !isGeneratedTestPath(hit.chunk.path)) + ) { + deduplicated.set(key, hit); + } + } + return [...deduplicated.values()]; +} + +function canonicalTestKey(chunk: IndexedChunk): string { + const normalizedPath = chunk.path + .replace( + /(?:^|\/)packages\/publish\/tests\/(?:core\/)?/u, + "packages/core/tests/", + ) + .replace(/(?:^|\/)(?:generated|publish)\/tests\//u, "tests/"); + return `${normalizedPath}:${chunk.lineStart}:${(chunk.testNames ?? []).join("|")}`; +} + +function isGeneratedTestPath(value: string): boolean { + return /(?:^|\/)(?:generated|publish)(?:\/|$)/u.test(value); } function extractPaths(value: string): Set { @@ -411,10 +424,22 @@ function escapeRegExp(value: string): string { return value.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); } -function emptySurfaceGroups(): Record< +function emptyChangeSurfaceGroups(): Record< ChangeSurfaceCategory, SearchResultItem[] > { + return { + consumer: [], + exports: [], + implementation: [], + initialization: [], + publish_generated: [], + state_transitions: [], + tests: [], + }; +} + +function emptyTraceGroups(): Record { return { consumer: [], exports: [], @@ -425,16 +450,17 @@ function emptySurfaceGroups(): Record< }; } -function fillSurfaceGroups( - groups: Record, - candidates: Record, +function fillGroups( + groups: Record, + candidates: Record, totalLimit: number, + categoryOrder: readonly Category[], ): void { let remaining = totalLimit; let index = 0; while (remaining > 0) { let added = false; - for (const category of SURFACE_CATEGORY_ORDER) { + for (const category of categoryOrder) { const candidate = candidates[category][index]; if (!candidate || remaining === 0) continue; groups[category].push(candidate); @@ -447,15 +473,15 @@ function fillSurfaceGroups( } function response( - engine: string, query: string, hits: RankedHit[], limit: number, + scope: SearchScope, ): SearchResponse { return { - engine, query, results: hits.slice(0, limit).map(toResultItem), + scope, }; } @@ -465,10 +491,11 @@ function toResultItem(hit: RankedHit): SearchResultItem { lineEnd: hit.chunk.lineEnd, lineStart: hit.chunk.lineStart, path: hit.chunk.path, - score: Number(hit.score.toFixed(6)), - ...(hit.signals ? { signals: hit.signals } : {}), snippet: compactSnippet(hit.chunk.text), ...(hit.chunk.tags.length > 0 ? { tags: hit.chunk.tags } : {}), + ...(hit.chunk.testNames && hit.chunk.testNames.length > 0 + ? { testNames: hit.chunk.testNames } + : {}), ...(hit.chunk.title ? { title: hit.chunk.title } : {}), ...(hit.chunk.type ? { type: hit.chunk.type } : {}), }; @@ -483,18 +510,32 @@ function scopedChunks( scope: SearchScope, ): IndexedChunk[] { const valid = validateScope(scope); - return valid === "all" - ? chunks - : chunks.filter((chunk) => chunk.scope === valid); + if (valid === "all") return chunks; + if (valid === "wiki") { + return chunks.filter((chunk) => chunk.scope === "wiki"); + } + if (valid === "tests") { + return sourceChunks(chunks).filter(isTestChunk); + } + return sourceChunks(chunks).filter((chunk) => !isTestChunk(chunk)); } function validateScope(scope: SearchScope): SearchScope { - if (scope !== "all" && scope !== "source" && scope !== "wiki") { - throw new Error("scope must be all, source, or wiki."); + if ( + scope !== "all" && + scope !== "source_code" && + scope !== "tests" && + scope !== "wiki" + ) { + throw new Error("scope must be all, source_code, tests, or wiki."); } return scope; } +function sourceChunks(chunks: IndexedChunk[]): IndexedChunk[] { + return chunks.filter((chunk) => chunk.scope === "source_code"); +} + function validateQuery(query: string): string { if ( typeof query !== "string" || @@ -506,17 +547,29 @@ function validateQuery(query: string): string { return query.trim(); } -function validateIdentifier(query: string): string { - const identifier = validateQuery(query); - if (!IDENTIFIER.test(identifier)) { - throw new Error("symbol must be a single 1-100 character identifier."); +function validateSymbols(symbols: string[]): string[] { + if (!Array.isArray(symbols) || symbols.length === 0) { + throw new Error("symbols must contain at least one identifier."); } - return identifier; + const unique = [...new Set(symbols.map((symbol) => symbol.trim()))]; + if (unique.length > MAX_SYMBOLS) { + throw new Error(`symbols must contain at most ${MAX_SYMBOLS} identifiers.`); + } + for (const symbol of unique) { + if (symbol.length > 200 || !DOTTED_IDENTIFIER.test(symbol)) { + throw new Error( + "each symbol must be a plain or dotted identifier up to 200 characters.", + ); + } + } + return unique; } -function validateLimit(limit: number): number { - if (!Number.isInteger(limit) || limit < 1 || limit > MAX_LIMIT) { - throw new Error(`limit must be an integer between 1 and ${MAX_LIMIT}.`); - } - return limit; +function normalizeLimit( + limit: number, + maximum: number, + fallback: number, +): number { + if (!Number.isInteger(limit)) return fallback; + return Math.max(1, Math.min(maximum, limit)); } diff --git a/src/retrieval/semantic.ts b/src/retrieval/semantic.ts index 65ad9403..ca430e15 100644 --- a/src/retrieval/semantic.ts +++ b/src/retrieval/semantic.ts @@ -1,5 +1,5 @@ import { rankBm25, rankLocalVectors, searchableText } from "./ranking.js"; -import type { IndexedChunk, RankedHit, SearchScope } from "./types.js"; +import type { IndexedChunk, RankedHit } from "./types.js"; export type EmbeddingProvider = "local" | "openai"; @@ -13,19 +13,15 @@ export class SemanticRanker { async rank( chunks: IndexedChunk[], query: string, - scope: SearchScope, ): Promise<{ engine: string; hits: RankedHit[] }> { - const scoped = chunks.filter( - (chunk) => scope === "all" || chunk.scope === scope, - ); if (this.provider !== "openai" || !process.env.OPENAI_API_KEY) { return { engine: "local-hashed-vector", - hits: rankLocalVectors(scoped, query), + hits: rankLocalVectors(chunks, query), }; } try { - const candidates = selectOpenAiCandidates(scoped, query); + const candidates = selectOpenAiCandidates(chunks, query); const embeddings = await this.openAiEmbeddings(); const queryVector = await embeddings.embedQuery(query); const missing = candidates.filter( @@ -51,7 +47,7 @@ export class SemanticRanker { } catch { return { engine: "local-hashed-vector:fallback", - hits: rankLocalVectors(scoped, query), + hits: rankLocalVectors(chunks, query), }; } } diff --git a/src/retrieval/types.ts b/src/retrieval/types.ts index 9f917880..8c1916a1 100644 --- a/src/retrieval/types.ts +++ b/src/retrieval/types.ts @@ -1,4 +1,6 @@ -export type SearchScope = "all" | "source" | "wiki"; +export type SearchScope = "all" | "source_code" | "tests" | "wiki"; + +export type IndexedScope = "source_code" | "wiki"; export type ChunkKind = "source" | "wiki-section"; @@ -11,8 +13,9 @@ export interface IndexedChunk { lineEnd: number; lineStart: number; path: string; - scope: Exclude; + scope: IndexedScope; tags: string[]; + testNames?: string[]; text: string; title?: string; type?: string; @@ -50,21 +53,20 @@ export interface SearchResultItem { lineEnd: number; lineStart: number; path: string; - score: number; - signals?: Record; snippet: string; tags?: string[]; + testNames?: string[]; title?: string; type?: string; } export interface SearchResponse { - engine: string; query: string; results: SearchResultItem[]; + scope: SearchScope; } -export type ChangeSurfaceCategory = +export type SymbolTraceCategory = | "consumer" | "exports" | "implementation" @@ -72,14 +74,22 @@ export type ChangeSurfaceCategory = | "publish_generated" | "tests"; +export type ChangeSurfaceCategory = + | SymbolTraceCategory + | "state_transitions"; + export interface ChangeSurfaceResponse { groups: Record; query: string; relatedConcepts: SearchResultItem[]; } -export interface SymbolTraceResponse { - groups: Record; - missing: ChangeSurfaceCategory[]; +export interface SymbolTraceResult { + groups: Record; + missing: SymbolTraceCategory[]; symbol: string; } + +export interface SymbolTraceResponse { + traces: SymbolTraceResult[]; +} diff --git a/test/agent-navigation-guidance.test.ts b/test/agent-navigation-guidance.test.ts index b36b5f15..58b2435f 100644 --- a/test/agent-navigation-guidance.test.ts +++ b/test/agent-navigation-guidance.test.ts @@ -16,7 +16,10 @@ describe("repository coding-agent documentation guidance", () => { expect(prompt).toContain("consumer-facing smoke test"); expect(prompt).toContain("behavioral test matrix"); expect(prompt).toContain("isolation between independent instances"); - expect(prompt).toContain("test_search"); + expect(prompt).toContain("scoped to `tests`"); + expect(prompt).toContain("observation-window boundaries"); + expect(prompt).toContain("net/coalesced effects"); + expect(prompt).toContain("narrow and quiet"); expect(prompt).toContain("Label expensive checks as conditional"); expect(prompt).toContain( "simulate navigation for representative adjacent changes", diff --git a/test/code-mode.test.ts b/test/code-mode.test.ts index 2b1247da..1dc1df6d 100644 --- a/test/code-mode.test.ts +++ b/test/code-mode.test.ts @@ -48,10 +48,14 @@ describe("ensureCodeModeRepoSetup agent files", () => { expect(content).toContain("Re-consult the wiki"); expect(content).toContain("import path consumers actually use"); expect(content).toContain("internal unit tests alone"); - expect(content).toContain("symbol_trace"); + expect(content).toContain("trace_symbols"); + expect(content).toContain("every identity axis"); + expect(content).toContain("canonical owner or ledger"); + expect(content).toContain("input/event/expected-result oracle row"); expect(content).toContain("independent instances"); - expect(content).toContain("deferred or re-entrant mutation"); - expect(content).toContain("test_search"); + expect(content).toContain("deferred or re-entrant net effects"); + expect(content).toContain("\`tests\` scope"); + expect(content).toContain("quiet validation"); } }); diff --git a/test/retrieval.test.ts b/test/retrieval.test.ts index e7b51a69..436d805c 100644 --- a/test/retrieval.test.ts +++ b/test/retrieval.test.ts @@ -2,6 +2,10 @@ import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, test } from "vitest"; +import { + RETRIEVAL_TOOL_DEFINITIONS, + SEARCH_SCOPES, +} from "../src/retrieval/mcp-tools.ts"; import { RetrievalService } from "../src/retrieval/search-service.ts"; let root = ""; @@ -14,6 +18,8 @@ beforeEach(async () => { wikiRoot = path.join(root, "wiki"); await Promise.all([ mkdir(path.join(repoRoot, "packages/core/src/query"), { recursive: true }), + mkdir(path.join(repoRoot, "packages/core/src/relation"), { recursive: true }), + mkdir(path.join(repoRoot, "packages/core/tests"), { recursive: true }), mkdir(path.join(repoRoot, "packages/publish/src"), { recursive: true }), mkdir(path.join(repoRoot, "packages/publish/tests"), { recursive: true }), mkdir(path.join(repoRoot, "secrets"), { recursive: true }), @@ -55,7 +61,11 @@ The [quickstart](../quickstart.md) routes adjacent changes here. ), writeFile( path.join(repoRoot, "packages/core/src/query/predicate.ts"), - "export function createPredicate() { return true; }\n", + "export const PUBLIC_PREDICATE_FACTORY = true;\nexport function createPredicate() { return true; }\n", + ), + writeFile( + path.join(repoRoot, "packages/core/src/relation/relation-events.ts"), + "export function removeRelationPair() { emitRelationEvent('remove'); }\nfunction emitRelationEvent(type: string) { return type; }\n", ), writeFile( path.join(repoRoot, "packages/core/src/index.ts"), @@ -65,9 +75,13 @@ The [quickstart](../quickstart.md) routes adjacent changes here. path.join(repoRoot, "packages/publish/src/index.ts"), "export { createPredicate } from '@koota/core';\n", ), + writeFile( + path.join(repoRoot, "packages/core/tests/predicate.test.ts"), + "import { createPredicate } from '../src';\ndescribe('predicate lifecycle', () => {\n test('tracks false-to-true transitions independently', () => createPredicate());\n});\n", + ), writeFile( path.join(repoRoot, "packages/publish/tests/predicate.test.ts"), - "import { createPredicate } from 'koota';\ntest('public import', () => createPredicate());\n", + "import { createPredicate } from 'koota';\ndescribe('predicate lifecycle', () => {\n test('tracks false-to-true transitions independently', () => createPredicate());\n});\n", ), writeFile( path.join(repoRoot, ".env"), @@ -93,127 +107,156 @@ function service(): RetrievalService { } describe("OKF-aware repository retrieval", () => { - test("supports keyword, BM25, and local vector ranking", async () => { + test("exposes three concise workflow-oriented MCP tools", () => { + expect(RETRIEVAL_TOOL_DEFINITIONS.map((tool) => tool.name)).toEqual([ + "search", + "change_surface", + "trace_symbols", + ]); + expect( + RETRIEVAL_TOOL_DEFINITIONS.every( + (tool) => + tool.description.length >= 100 && tool.description.length < 300, + ), + ).toBe(true); + expect(SEARCH_SCOPES).toEqual(["all", "wiki", "source_code", "tests"]); + }); + + test("automatically combines lexical, semantic, and OKF ranking", async () => { const retrieval = service(); - const keyword = await retrieval.keywordSearch("createPredicate", "all", 5); - const bm25 = await retrieval.bm25Search( - "predicate consumer import", - "all", - 5, - ); - const semantic = await retrieval.semanticSearch( + const exact = await retrieval.search("createPredicate", "source_code", 5); + const concept = await retrieval.search("query navigation", "wiki", 5); + const consumer = await retrieval.search( "consumer-facing package surface", "all", 5, ); - expect(keyword.results[0]?.path).toMatch(/predicate|index/u); - expect(bm25.results.some((hit) => hit.path.includes("publish/tests"))).toBe( - true, + expect(exact.results[0]?.path).toMatch(/predicate|index/u); + expect(concept.results.map((hit) => hit.path)).toContain( + "openwiki/architecture/runtime.md", ); - expect(semantic.engine).toBe("local-hashed-vector"); expect( - semantic.results.some( + consumer.results.some( (hit) => hit.path.includes("runtime.md") || hit.path.includes("publish"), ), ).toBe(true); }); - test("expands retrieval through OKF links and shared tags", async () => { - const graph = await service().okfGraphSearch("query navigation", 5, 2); + test("supports distinct wiki, source_code, and tests scopes", async () => { + const retrieval = service(); + const source = await retrieval.search("createPredicate", "source_code", 10); + const tests = await retrieval.search( + "false-to-true independent predicate transition", + "tests", + 10, + ); - expect(graph.results.map((hit) => hit.path)).toContain( - "openwiki/architecture/runtime.md", + expect(source.results.every((hit) => !/test|spec/iu.test(hit.path))).toBe( + true, ); - expect(graph.results.map((hit) => hit.path)).toContain( - "openwiki/quickstart.md", + expect(tests.scope).toBe("tests"); + expect(tests.results.length).toBeGreaterThan(0); + expect(tests.results.every((hit) => /test|spec/iu.test(hit.path))).toBe( + true, ); - }); - - test("hybrid search reports component scores", async () => { - const hybrid = await service().hybridSearch( - "add predicate query public API", - "all", - 5, + expect(tests.results.flatMap((hit) => hit.testNames ?? [])).toContain( + "tracks false-to-true transitions independently", ); - - expect(hybrid.engine).toContain("hybrid-rrf"); - expect(hybrid.results[0]?.signals).toBeDefined(); + expect( + tests.results.filter((hit) => hit.path.endsWith("predicate.test.ts")), + ).toHaveLength(1); + expect(tests.results[0]).not.toHaveProperty("signals"); + expect(tests.results[0]).not.toHaveProperty("score"); }); - test("test_search returns only bounded test citations", async () => { - const result = await service().testSearch( - "public predicate import lifecycle transition", - 3, + test("clamps broad result requests to the public maximum", async () => { + const result = await service().search( + "predicate query public API", + "all", + 50, ); - expect(result.engine).toContain("test-hybrid-rrf"); - expect(result.results.length).toBeLessThanOrEqual(3); - expect(result.results.length).toBeGreaterThan(0); - expect( - result.results.every((hit) => /(?:test|spec)/iu.test(hit.path)), - ).toBe(true); + expect(result.results.length).toBeLessThanOrEqual(10); }); test("change_surface groups cross-package evidence", async () => { const surface = await service().changeSurface( - "add createPredicate query API", - 6, + "add createPredicate query API and track relation removal events", + 7, ); expect(surface.relatedConcepts[0]?.path).toContain("openwiki/"); expect(surface.groups.implementation.length).toBeGreaterThan(0); + expect(surface.groups.state_transitions[0]?.path).toContain("relation"); expect(surface.groups.exports.length).toBeGreaterThan(0); expect(surface.groups.publish_generated.length).toBeGreaterThan(0); expect(surface.groups.consumer.length).toBeGreaterThan(0); expect(surface.groups.tests.length).toBeGreaterThan(0); const results = Object.values(surface.groups).flat(); - expect(results).toHaveLength(6); - expect(surface.relatedConcepts.length).toBeLessThanOrEqual(3); + expect(results).toHaveLength(7); + expect(surface.relatedConcepts.length).toBeLessThanOrEqual(2); expect( Math.max(...results.map((result) => result.snippet.length)), - ).toBeLessThanOrEqual(320); - expect(JSON.stringify(surface).length).toBeLessThan(6_000); + ).toBeLessThanOrEqual(220); + expect(JSON.stringify(surface).length).toBeLessThan(5_000); }); - test("symbol_trace refreshes post-edit source and reports missing layers", async () => { + test("trace_symbols reindexes once and accepts batched dotted symbols", async () => { const retrieval = service(); - await retrieval.keywordSearch("createMatcher", "source", 5); + await retrieval.search("createMatcher", "source_code", 5); await writeFile( path.join(repoRoot, "packages/core/src/query/matcher.ts"), - "export function createMatcher() { return true; }\n", + "export function createMatcher() { return true; }\nexport const Entity = { changed() { return true; } };\n", ); - const trace = await retrieval.symbolTrace("createMatcher", 6); + const response = await retrieval.traceSymbols( + [ + "createMatcher", + "Entity.changed", + "PUBLIC_PREDICATE_FACTORY", + "createMatcher", + ], + 50, + ); + const trace = response.traces[0]; expect(trace.groups.implementation[0]?.path).toContain("matcher.ts"); expect(trace.missing).toContain("consumer"); expect(trace.missing).toContain("tests"); expect(trace.missing).toContain("exports"); expect(Object.values(trace.groups).flat()).toHaveLength(1); + expect(response.traces.map((item) => item.symbol)).toEqual([ + "createMatcher", + "Entity.changed", + "PUBLIC_PREDICATE_FACTORY", + ]); + expect(response.traces[1]?.groups.implementation[0]?.path).toContain( + "matcher.ts", + ); + expect(response.traces[2]?.groups.implementation[0]?.path).toContain( + "predicate.ts", + ); await expect( - retrieval.symbolTrace("createMatcher(); rm -rf /", 6), - ).rejects.toThrow("single 1-100 character identifier"); + retrieval.traceSymbols(["createMatcher(); rm -rf /"], 6), + ).rejects.toThrow("plain or dotted identifier"); }); test("never indexes secret-like files", async () => { - const result = await service().keywordSearch( + const result = await service().search( "never-index-this credentials", "all", - 20, + 50, ); expect(result.results).toEqual([]); }); - test("bounds query length and result limits", async () => { + test("bounds query length", async () => { const retrieval = service(); - await expect(retrieval.keywordSearch("x", "all", 21)).rejects.toThrow( - "limit must be", + await expect(retrieval.search("x".repeat(501), "all", 5)).rejects.toThrow( + "query must be", ); - await expect( - retrieval.keywordSearch("x".repeat(501), "all", 5), - ).rejects.toThrow("query must be"); }); }); From a3cbc8e37a6b2f0824ab434da288ade5dcc91c34 Mon Sep 17 00:00:00 2001 From: bracesproul Date: Wed, 29 Jul 2026 08:50:49 -0700 Subject: [PATCH 3/3] cr --- AGENTS.md | 20 +- CLAUDE.md | 20 +- evals/deepswe/OPENWIKI_OPTIMIZATION_LOG.md | 499 +++++++++-- evals/deepswe/README.md | 87 +- evals/deepswe/analyze_openwiki_usage.py | 440 +++++++++ evals/deepswe/openwiki_codex.py | 356 +++++++- evals/deepswe/run.py | 61 +- evals/deepswe/test_analyze_openwiki_usage.py | 243 +++++ evals/deepswe/test_run.py | 164 +++- src/agent/prompt.ts | 15 + src/code-mode.ts | 20 +- src/retrieval/mcp-server.ts | 38 +- src/retrieval/mcp-tools.ts | 31 +- src/retrieval/ranking.ts | 8 +- src/retrieval/repository-index.ts | 145 +++ src/retrieval/search-service.ts | 890 ++++++++++++++----- src/retrieval/types.ts | 87 +- test/code-mode.test.ts | 21 +- test/retrieval.test.ts | 181 ++-- 19 files changed, 2785 insertions(+), 541 deletions(-) create mode 100644 evals/deepswe/analyze_openwiki_usage.py create mode 100644 evals/deepswe/test_analyze_openwiki_usage.py diff --git a/AGENTS.md b/AGENTS.md index b05ff113..5b345770 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,19 +14,13 @@ When working in this repository, read the OpenWiki quickstart first, then follow ## OpenWiki -This repository uses OpenWiki for recurring code documentation. Use `openwiki/` as a just-in-time repository index: - -- At task start, read `openwiki/quickstart.md`, then search the wiki for the task's concepts and read only the relevant linked sections. -- Before a repository-wide `rg`, `find`, or exploratory directory scan, check the wiki's source maps. When they name relevant files, symbols, or tests, inspect those paths directly. -- Re-consult the wiki when entering a different subsystem, when source evidence contradicts the current understanding, or when blocked by an unfamiliar test or build failure. Do not reread content already returned by retrieval unless surrounding context is needed. -- Treat source code and tests as authoritative. Verify wiki claims in source before editing. -- Before finishing a public API or cross-package change, trace it from implementation through barrel/package exports, generated or publish mirrors, initialization or registration, and the import path consumers actually use. Consult the relevant wiki integration or delivery guidance and run the narrowest consumer-facing check; internal unit tests alone do not prove the shipped surface works. -- If an `openwiki_retrieval` MCP server is available, use `search` for focused retrieval. Use `change_surface` before public, cross-package, generated-artifact, or runtime-registration edits. After changing public symbols, call `trace_symbols` once with all of them and treat missing groups as verification gaps, not automatic requirements. -- For stateful or lifecycle changes, translate each externally observable acceptance criterion into a focused test checklist. Cover relevant initial state, false-to-true and true-to-false transitions, unchanged updates, missing dependencies, independent tracker instances, reset/reuse and observation windows, deferred or re-entrant net effects, and composition between static and temporal constraints; map every criterion to a passing test before finishing. -- When behavior is unfamiliar, relevant tests are large, or no analogous focused check is known, call `search` with the `tests` scope using that behavior matrix, then inspect the cited tests directly. Skip this search when the exact focused test is already known. -- When the repository generates or copies package artifacts, identify the canonical source and repository-supported synchronization command. Do not hand-edit derived output unless the documented workflow explicitly requires it. -- Do not read operations, release, or integration pages unless the task affects those areas. If a requested feature does not exist yet, use the wiki to locate extension points, then inspect source rather than searching the wiki for the implementation. -- Prefer the narrowest quiet validation command available. Suppress successful build/test noise when possible, but preserve complete failure output. +This repository has a generated `openwiki/` evidence index. It is optional just-in-time context, not required startup reading. + +- If implementation ownership, behavioral invariants, analogous tests, or shipped surfaces are unclear, call `openwiki_retrieval.change_surface` once with the task before broad exploration. Inspect its cited source and tests directly; do not reread the returned wiki pages. +- Use `openwiki_retrieval.search` only for a concrete unresolved evidence gap. Reconsult when source contradicts the brief, work enters an uncited subsystem, or an unfamiliar failure reveals a missing contract. +- Treat source code and tests as authoritative. A brief's unknowns and review items are verification gaps, not automatic requirements. +- Before finishing a public or cross-package change, call `change_surface` with the task and the repository-relative changed paths. Verify relevant flagged exports, registration, generated surfaces, consumer paths, and focused tests. +- Prefer the narrowest quiet validation that proves the changed behavior. Preserve complete failure output. The scheduled OpenWiki GitHub Actions workflow refreshes the repository wiki. Do not hand-edit generated OpenWiki pages unless explicitly asked; prefer updating source code/docs and letting OpenWiki regenerate. diff --git a/CLAUDE.md b/CLAUDE.md index b05ff113..5b345770 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -14,19 +14,13 @@ When working in this repository, read the OpenWiki quickstart first, then follow ## OpenWiki -This repository uses OpenWiki for recurring code documentation. Use `openwiki/` as a just-in-time repository index: - -- At task start, read `openwiki/quickstart.md`, then search the wiki for the task's concepts and read only the relevant linked sections. -- Before a repository-wide `rg`, `find`, or exploratory directory scan, check the wiki's source maps. When they name relevant files, symbols, or tests, inspect those paths directly. -- Re-consult the wiki when entering a different subsystem, when source evidence contradicts the current understanding, or when blocked by an unfamiliar test or build failure. Do not reread content already returned by retrieval unless surrounding context is needed. -- Treat source code and tests as authoritative. Verify wiki claims in source before editing. -- Before finishing a public API or cross-package change, trace it from implementation through barrel/package exports, generated or publish mirrors, initialization or registration, and the import path consumers actually use. Consult the relevant wiki integration or delivery guidance and run the narrowest consumer-facing check; internal unit tests alone do not prove the shipped surface works. -- If an `openwiki_retrieval` MCP server is available, use `search` for focused retrieval. Use `change_surface` before public, cross-package, generated-artifact, or runtime-registration edits. After changing public symbols, call `trace_symbols` once with all of them and treat missing groups as verification gaps, not automatic requirements. -- For stateful or lifecycle changes, translate each externally observable acceptance criterion into a focused test checklist. Cover relevant initial state, false-to-true and true-to-false transitions, unchanged updates, missing dependencies, independent tracker instances, reset/reuse and observation windows, deferred or re-entrant net effects, and composition between static and temporal constraints; map every criterion to a passing test before finishing. -- When behavior is unfamiliar, relevant tests are large, or no analogous focused check is known, call `search` with the `tests` scope using that behavior matrix, then inspect the cited tests directly. Skip this search when the exact focused test is already known. -- When the repository generates or copies package artifacts, identify the canonical source and repository-supported synchronization command. Do not hand-edit derived output unless the documented workflow explicitly requires it. -- Do not read operations, release, or integration pages unless the task affects those areas. If a requested feature does not exist yet, use the wiki to locate extension points, then inspect source rather than searching the wiki for the implementation. -- Prefer the narrowest quiet validation command available. Suppress successful build/test noise when possible, but preserve complete failure output. +This repository has a generated `openwiki/` evidence index. It is optional just-in-time context, not required startup reading. + +- If implementation ownership, behavioral invariants, analogous tests, or shipped surfaces are unclear, call `openwiki_retrieval.change_surface` once with the task before broad exploration. Inspect its cited source and tests directly; do not reread the returned wiki pages. +- Use `openwiki_retrieval.search` only for a concrete unresolved evidence gap. Reconsult when source contradicts the brief, work enters an uncited subsystem, or an unfamiliar failure reveals a missing contract. +- Treat source code and tests as authoritative. A brief's unknowns and review items are verification gaps, not automatic requirements. +- Before finishing a public or cross-package change, call `change_surface` with the task and the repository-relative changed paths. Verify relevant flagged exports, registration, generated surfaces, consumer paths, and focused tests. +- Prefer the narrowest quiet validation that proves the changed behavior. Preserve complete failure output. The scheduled OpenWiki GitHub Actions workflow refreshes the repository wiki. Do not hand-edit generated OpenWiki pages unless explicitly asked; prefer updating source code/docs and letting OpenWiki regenerate. diff --git a/evals/deepswe/OPENWIKI_OPTIMIZATION_LOG.md b/evals/deepswe/OPENWIKI_OPTIMIZATION_LOG.md index 1a7f55ca..19cfab4e 100644 --- a/evals/deepswe/OPENWIKI_OPTIMIZATION_LOG.md +++ b/evals/deepswe/OPENWIKI_OPTIMIZATION_LOG.md @@ -10,11 +10,11 @@ Runs stop for rate limits only after logs confirm a provider or HTTP 429. Infras ## Reference cohorts -| Cohort | Full solves | Mean partial | Uncached input/trial | Cumulative input/trial | Output/trial | Tool calls/trial | Retrieval calls/trial | Retrieval chars/trial | Edit actions/trial | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | -| Baseline, 15 valid | 3/15 | 0.989192 | 100,184 | 4,175,334 | 30,345 | 51.13 | 0.00 | 0 | 23.2 | -| Old OpenWiki, 15 valid | 6/15 | 0.992832 | 136,772 | 5,192,270 | 30,478 | 57.93 | 9.00 | 70,413 | 16.0 | -| Current OpenWiki, 15 valid | 4/15 | 0.984533 | 125,617 | 5,838,432 | 32,721 | 60.73 | 5.73 | 30,650 | 21.0 | +| Cohort | Full solves | Mean partial | Uncached input/trial | Cumulative input/trial | Output/trial | Tool calls/trial | Retrieval calls/trial | Retrieval chars/trial | Edit actions/trial | +| -------------------------- | ----------: | -----------: | -------------------: | ---------------------: | -----------: | ---------------: | --------------------: | --------------------: | -----------------: | +| Baseline, 15 valid | 3/15 | 0.989192 | 100,184 | 4,175,334 | 30,345 | 51.13 | 0.00 | 0 | 23.2 | +| Old OpenWiki, 15 valid | 6/15 | 0.992832 | 136,772 | 5,192,270 | 30,478 | 57.93 | 9.00 | 70,413 | 16.0 | +| Current OpenWiki, 15 valid | 4/15 | 0.984533 | 125,617 | 5,838,432 | 32,721 | 60.73 | 5.73 | 30,650 | 21.0 | The current retrieval surface eliminated invalid calls, cut retrieval calls 36%, retrieval payload 57%, broad `rg` 83%, and validation commands 26%. Those savings were outweighed by 75 additional edit actions across the cohort. Pair tracking was the dominant regression: old OpenWiki solved 1/3 with 0.9968 mean partial; current OpenWiki solved 0/3 with 0.9508 mean partial. @@ -34,18 +34,18 @@ The old full-solving pair trace centralized events by tracker/factory, entity, r H1 recovered quality and sharply reduced cost relative to the current pair cohort: -| Metric | Current pair | H1 | Change | -| --- | ---: | ---: | ---: | -| Full solves | 0/3 | 1/3 | +1 solve | -| Mean partial | 0.950794 | 0.990476 | +0.039683 | -| Uncached input/trial | 153,744 | 129,476 | -15.8% | -| Cumulative input/trial | 8,042,262 | 5,746,183 | -28.6% | -| Output/trial | 37,320 | 37,573 | +0.7% | -| Exec/apply calls/trial | 71.67 | 61.33 | -14.4% | -| Retrieval calls/trial | 6.00 | 3.00 | -50.0% | -| Retrieval chars/trial | 30,603 | 21,697 | -29.1% | -| Edit actions/trial | 24.00 | 23.67 | -1.4% | -| Tool calls/trial | 77.67 | 64.33 | -17.2% | +| Metric | Current pair | H1 | Change | +| ---------------------- | -----------: | --------: | --------: | +| Full solves | 0/3 | 1/3 | +1 solve | +| Mean partial | 0.950794 | 0.990476 | +0.039683 | +| Uncached input/trial | 153,744 | 129,476 | -15.8% | +| Cumulative input/trial | 8,042,262 | 5,746,183 | -28.6% | +| Output/trial | 37,320 | 37,573 | +0.7% | +| Exec/apply calls/trial | 71.67 | 61.33 | -14.4% | +| Retrieval calls/trial | 6.00 | 3.00 | -50.0% | +| Retrieval chars/trial | 30,603 | 21,697 | -29.1% | +| Edit actions/trial | 24.00 | 23.67 | -1.4% | +| Tool calls/trial | 77.67 | 64.33 | -17.2% | The full solve made only `change_surface` and `trace_symbols` retrieval calls and created a centralized pair-tracking module. The other attempts missed one coexistence case and five removal/lifecycle cases respectively. @@ -65,18 +65,18 @@ One small file read should expose the identity-axis, canonical-ledger, observati All three traces read the generated managed block, so the hypothesis was tested. It was a loss: -| Metric | H1 | H2 | Change | -| --- | ---: | ---: | ---: | -| Full solves | 1/3 | 0/3 | -1 solve | -| Mean partial | 0.990476 | 0.973016 | -0.017460 | -| Uncached input/trial | 129,476 | 153,067 | +18.2% | -| Cumulative input/trial | 5,746,183 | 6,980,095 | +21.5% | -| Output/trial | 37,573 | 38,454 | +2.3% | -| Exec/apply calls/trial | 61.33 | 66.33 | +8.2% | -| Retrieval calls/trial | 3.00 | 4.33 | +44.4% | -| Retrieval chars/trial | 21,697 | 25,776 | +18.8% | -| Edit actions/trial | 23.67 | 23.00 | -2.8% | -| Tool calls/trial | 64.33 | 70.67 | +9.8% | +| Metric | H1 | H2 | Change | +| ---------------------- | --------: | --------: | --------: | +| Full solves | 1/3 | 0/3 | -1 solve | +| Mean partial | 0.990476 | 0.973016 | -0.017460 | +| Uncached input/trial | 129,476 | 153,067 | +18.2% | +| Cumulative input/trial | 5,746,183 | 6,980,095 | +21.5% | +| Output/trial | 37,573 | 38,454 | +2.3% | +| Exec/apply calls/trial | 61.33 | 66.33 | +8.2% | +| Retrieval calls/trial | 3.00 | 4.33 | +44.4% | +| Retrieval chars/trial | 21,697 | 25,776 | +18.8% | +| Edit actions/trial | 23.67 | 23.00 | -2.8% | +| Tool calls/trial | 64.33 | 70.67 | +9.8% | The visible block increased retrieval and validation but did not improve the semantic design. All three attempts created centralized pair-tracking utilities, yet all missed specific/non-last/wildcard removal, exclusive replacement, and destruction. One also missed trait-plus-pair coexistence. The abstract state-model instruction did not force agents to enumerate every mutation producer, and the long workflow diluted the key decision. @@ -108,18 +108,18 @@ Every H1/H2 attempt already calls `change_surface`, so this should expose the re H3 is retained as the new winner: -| Metric | H1 | H3 | Change | -| --- | ---: | ---: | ---: | -| Full solves | 1/3 | 2/3 | +1 solve | -| Mean partial | 0.990476 | 0.990476 | unchanged | -| Uncached input/trial | 129,476 | 122,029 | -5.8% | -| Cumulative input/trial | 5,746,183 | 5,318,199 | -7.4% | -| Output/trial | 37,573 | 33,751 | -10.2% | -| Exec/apply calls/trial | 61.33 | 61.33 | unchanged | -| Retrieval calls/trial | 3.00 | 3.33 | +11.1% | -| Retrieval chars/trial | 21,697 | 18,742 | -13.6% | -| Edit actions/trial | 23.67 | 24.33 | +2.8% | -| Tool calls/trial | 64.33 | 64.67 | +0.5% | +| Metric | H1 | H3 | Change | +| ---------------------- | --------: | --------: | --------: | +| Full solves | 1/3 | 2/3 | +1 solve | +| Mean partial | 0.990476 | 0.990476 | unchanged | +| Uncached input/trial | 129,476 | 122,029 | -5.8% | +| Cumulative input/trial | 5,746,183 | 5,318,199 | -7.4% | +| Output/trial | 37,573 | 33,751 | -10.2% | +| Exec/apply calls/trial | 61.33 | 61.33 | unchanged | +| Retrieval calls/trial | 3.00 | 3.33 | +11.1% | +| Retrieval chars/trial | 21,697 | 18,742 | -13.6% | +| Edit actions/trial | 23.67 | 24.33 | +2.8% | +| Tool calls/trial | 64.33 | 64.67 | +0.5% | One full solve received `packages/core/src/trait/trait.ts` as transition evidence and inspected trait/relation producers before editing. The other full solve made no retrieval calls, so its success is run variance rather than a retrieval win. The failed trial received the same producer citation but still missed removal/destruction/coexistence, showing that the extra evidence is helpful for some trajectories but not sufficient. H3 is retained because solve count improved while all token metrics fell materially. @@ -139,18 +139,18 @@ The two H3 retrieval users each made three searches returning about 14k characte H4 regressed and was rolled back before H5: -| Metric | H3 | H4 | Change | -| --- | ---: | ---: | ---: | -| Full solves | 2/3 | 0/3 | -2 solves | -| Mean partial | 0.990476 | 0.977778 | -0.012698 | -| Uncached input/trial | 122,029 | 164,467 | +34.8% | -| Cumulative input/trial | 5,318,199 | 6,622,466 | +24.5% | -| Output/trial | 33,751 | 39,949 | +18.4% | -| Exec/apply calls/trial | 61.33 | 68.67 | +12.0% | -| Retrieval calls/trial | 3.33 | 4.00 | +20.0% | -| Retrieval chars/trial | 18,742 | 22,752 | +21.4% | -| Edit actions/trial | 24.33 | 25.67 | +5.5% | -| Tool calls/trial | 64.67 | 72.67 | +12.4% | +| Metric | H3 | H4 | Change | +| ---------------------- | --------: | --------: | --------: | +| Full solves | 2/3 | 0/3 | -2 solves | +| Mean partial | 0.990476 | 0.977778 | -0.012698 | +| Uncached input/trial | 122,029 | 164,467 | +34.8% | +| Cumulative input/trial | 5,318,199 | 6,622,466 | +24.5% | +| Output/trial | 33,751 | 39,949 | +18.4% | +| Exec/apply calls/trial | 61.33 | 68.67 | +12.0% | +| Retrieval calls/trial | 3.33 | 4.00 | +20.0% | +| Retrieval chars/trial | 18,742 | 22,752 | +21.4% | +| Edit actions/trial | 24.33 | 25.67 | +5.5% | +| Tool calls/trial | 64.67 | 72.67 | +12.4% | The six-result cap reduced each search response, and one attempt used only one search, but the cohort as a whole made more retrieval and command calls and spent substantially more tokens. Tool descriptions did not reliably prevent redundant search. Both H4 descriptions and limits were reverted to H3 values. @@ -168,18 +168,18 @@ Trace output was the largest single retrieval response at 9-11k characters in H3 H5 reduced trace payload but regressed the primary outcome, so it was rolled back: -| Metric | H3 | H5 | Change | -| --- | ---: | ---: | ---: | -| Full solves | 2/3 | 1/3 | -1 solve | -| Mean partial | 0.990476 | 0.987302 | -0.003175 | -| Uncached input/trial | 122,029 | 119,204 | -2.3% | -| Cumulative input/trial | 5,318,199 | 5,825,158 | +9.5% | -| Output/trial | 33,751 | 33,222 | -1.6% | -| Exec/apply calls/trial | 61.33 | 64.67 | +5.4% | -| Retrieval calls/trial | 3.33 | 3.67 | +10.0% | -| Retrieval chars/trial | 18,742 | 13,226 | -29.4% | -| Edit actions/trial | 24.33 | 25.33 | +4.1% | -| Tool calls/trial | 64.67 | 68.33 | +5.7% | +| Metric | H3 | H5 | Change | +| ---------------------- | --------: | --------: | --------: | +| Full solves | 2/3 | 1/3 | -1 solve | +| Mean partial | 0.990476 | 0.987302 | -0.003175 | +| Uncached input/trial | 122,029 | 119,204 | -2.3% | +| Cumulative input/trial | 5,318,199 | 5,825,158 | +9.5% | +| Output/trial | 33,751 | 33,222 | -1.6% | +| Exec/apply calls/trial | 61.33 | 64.67 | +5.4% | +| Retrieval calls/trial | 3.33 | 3.67 | +10.0% | +| Retrieval chars/trial | 18,742 | 13,226 | -29.4% | +| Edit actions/trial | 24.33 | 25.33 | +4.1% | +| Tool calls/trial | 64.67 | 68.33 | +5.7% | Per-call trace payload fell from 9-11k to 1.7-2.1k characters, proving the compaction mechanism worked. That local saving did not reduce cumulative context or calls, and solve quality fell. The compact citation type, lower trace limits, and description were reverted. H3 remains the winner. @@ -191,24 +191,24 @@ H3 is the retained configuration: H1's concise eval treatment and three-tool wor The winner run completed 15/15 valid trials with no infrastructure or rate-limit failures. -| Cohort | Full solves | Mean partial | Uncached input/trial | Cumulative input/trial | Output/trial | Tool calls/trial | Retrieval calls/trial | Retrieval chars/trial | Edit actions/trial | Agent duration/trial | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | -| Baseline | 3/15 | 0.989192 | 100,184 | 4,175,334 | 30,345 | 51.13 | 0.00 | 0 | 23.20 | 471.1s | -| Old OpenWiki | 6/15 | 0.992832 | 136,772 | 5,192,270 | 30,478 | 57.93 | 9.00 | 70,413 | 16.00 | 929.6s | -| Current OpenWiki | 4/15 | 0.984533 | 125,617 | 5,838,432 | 32,721 | 60.73 | 5.73 | 30,650 | 21.00 | 841.0s | -| H3 winner | **7/15** | **0.994350** | 132,469 | 6,198,174 | 34,006 | 61.07 | **3.20** | **18,291** | 19.07 | **599.7s** | +| Cohort | Full solves | Mean partial | Uncached input/trial | Cumulative input/trial | Output/trial | Tool calls/trial | Retrieval calls/trial | Retrieval chars/trial | Edit actions/trial | Agent duration/trial | +| ---------------- | ----------: | -----------: | -------------------: | ---------------------: | -----------: | ---------------: | --------------------: | --------------------: | -----------------: | -------------------: | +| Baseline | 3/15 | 0.989192 | 100,184 | 4,175,334 | 30,345 | 51.13 | 0.00 | 0 | 23.20 | 471.1s | +| Old OpenWiki | 6/15 | 0.992832 | 136,772 | 5,192,270 | 30,478 | 57.93 | 9.00 | 70,413 | 16.00 | 929.6s | +| Current OpenWiki | 4/15 | 0.984533 | 125,617 | 5,838,432 | 32,721 | 60.73 | 5.73 | 30,650 | 21.00 | 841.0s | +| H3 winner | **7/15** | **0.994350** | 132,469 | 6,198,174 | 34,006 | 61.07 | **3.20** | **18,291** | 19.07 | **599.7s** | H3 has the best quality: +4 full solves over baseline, +1 over old OpenWiki, and +3 over current OpenWiki. Against current OpenWiki, it holds total tool calls nearly flat (+0.6%), cuts retrieval calls 44%, retrieval payload 40%, edit actions 9%, and agent duration 29%. The tradeoff is +5.5% uncached input, +6.2% cumulative input, and +3.9% output tokens. Against baseline, quality improves substantially but costs 32% more uncached input, 48% more cumulative input, and 19% more tool calls. Tool-call accounting treats every Codex exec/apply invocation and every MCP retrieval as one call. Edit actions are already included in exec/apply calls and are reported separately as a churn diagnostic; they are not double-counted in total tool calls. -| Task | Full solves | Mean partial | Uncached input | Cumulative input | Output | Tool calls | Retrieval calls | Retrieval chars | Edit actions | Agent duration | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | -| Composite aspects | 1/3 | 0.995516 | 157,513 | 8,609,858 | 45,190 | 80.67 | 3.67 | 18,540 | 30.33 | 764.7s | -| Deferred mutation | 3/3 | 1.000000 | 116,375 | 4,779,118 | 27,944 | 56.00 | 5.33 | 24,894 | 13.00 | 511.4s | -| Entity snapshots | 2/3 | 0.994911 | 87,182 | 2,307,435 | 24,183 | 35.00 | 1.33 | 11,825 | 8.33 | 374.4s | -| Pair tracking | 1/3 | 0.996825 | 165,004 | 9,510,027 | 39,297 | 80.00 | 3.00 | 20,210 | 25.67 | 720.6s | -| Query predicates | 0/3 | 0.984496 | 136,271 | 5,784,433 | 33,415 | 53.67 | 2.67 | 15,988 | 18.00 | 627.7s | +| Task | Full solves | Mean partial | Uncached input | Cumulative input | Output | Tool calls | Retrieval calls | Retrieval chars | Edit actions | Agent duration | +| ----------------- | ----------: | -----------: | -------------: | ---------------: | -----: | ---------: | --------------: | --------------: | -----------: | -------------: | +| Composite aspects | 1/3 | 0.995516 | 157,513 | 8,609,858 | 45,190 | 80.67 | 3.67 | 18,540 | 30.33 | 764.7s | +| Deferred mutation | 3/3 | 1.000000 | 116,375 | 4,779,118 | 27,944 | 56.00 | 5.33 | 24,894 | 13.00 | 511.4s | +| Entity snapshots | 2/3 | 0.994911 | 87,182 | 2,307,435 | 24,183 | 35.00 | 1.33 | 11,825 | 8.33 | 374.4s | +| Pair tracking | 1/3 | 0.996825 | 165,004 | 9,510,027 | 39,297 | 80.00 | 3.00 | 20,210 | 25.67 | 720.6s | +| Query predicates | 0/3 | 0.984496 | 136,271 | 5,784,433 | 33,415 | 53.67 | 2.67 | 15,988 | 18.00 | 627.7s | Compared with old/current OpenWiki task solves, H3 improved composite aspects to 1/3 and deferred mutation to 3/3, retained 1/3 pair solves, and remained 0/3 on query predicates. Entity snapshots fell from 3/3 to 2/3; its one miss was limited to tag-relation omission and roundtrip world-diff identity. @@ -220,3 +220,346 @@ The remaining failures are concentrated and consistent: - Entity: tag-relation snapshot omission and exact roundtrip diff identity. The strongest retained product change is transition-producer evidence inside the existing `change_surface` tool. The clearest negative finding is that more or more-forceful prompting did not help: making the long managed block visible increased tokens/calls and worsened score. Search-result caps and compact final tracing both reduced their local payloads but did not improve end-to-end efficiency or quality, so both were reverted. + +## Hypothesis 6: deduplicated traces with preserved evidence + +### Proposed change + +Starting from H3, keep `trace_symbols`' original 4-default/6-maximum result limits, ranking, input schema, and tool description. Deduplicate exact path/line citations across batched symbols into a shared citation table. Preserve every symbol-to-category mapping with compact citation IDs, retain every missing-category signal, and include a short snippet on each citation that provides the first evidence for at least one non-empty category. Additional citations remain path/line-only. + +### Expected benefit + +H3 repeatedly returned the same export, initialization, and test citation—including the same 220-character snippet and metadata—for each symbol. H5 proved that trace compaction can reduce a 9–11k response to 1.7–2.1k characters, but it also lowered result limits and removed all evidence excerpts. H6 isolates the safer payload change: it should materially reduce trace characters while retaining H3's evidence breadth and enough source context to interpret every category. Because `trace_symbols` runs after implementation, score and pre-trace behavior should remain unchanged; total calls and cumulative tokens should not increase. + +### Outcome + +H6 regressed both quality and end-to-end efficiency, so the citation-table format was rolled back to H3: + +| Metric | H3 | H6 | Change | +| ------------------------------ | --------: | --------: | --------- | +| Full solves | 2/3 | 0/3 | -2 solves | +| Mean partial | 0.990476 | 0.969841 | -0.020635 | +| Uncached input/trial | 122,029 | 194,493 | +59.4% | +| Cumulative input/trial | 5,318,199 | 7,652,008 | +43.9% | +| Output/trial | 33,751 | 41,154 | +21.9% | +| Successful command/edit events | 57.67 | 63.67 | +10.4% | +| Retrieval calls/trial | 3.33 | 3.67 | +10.0% | +| Observed tool calls/trial | 61.00 | 67.33 | +10.4% | +| Retrieval chars/trial | 18,742 | 17,926 | -4.4% | +| Edit actions/trial | 24.33 | 24.67 | +1.4% | +| Agent duration/trial | 684.2s | 858.7s | +25.5% | + +The observed-call rows use directly comparable completed Codex command, file-change, and MCP events. The earlier tables' exec/apply totals additionally include failed or no-op apply attempts that do not produce file-change events; H6 had 15 visible failed apply attempts versus nine in H3, reinforcing the churn regression. + +The local trace saving was real. H3's two trace responses were 9.2k and 10.9k characters (10,069 mean); all three H6 responses were 6.2–7.4k (6,716 mean), a 33.3% reduction. That was materially smaller than H5's 1.7–2.1k responses because 16–20 of H6's 16–21 unique citations still needed snippets: each was the first evidence for at least one symbol/category. The shared table mainly removed repeated citations and metadata while preserving H3's breadth. + +That local saving did not translate into lower total context. All H6 agents continued working after tracing, making 2–3 more edits and 9–14 more commands; H3's trace users made 2–3 edits and 4–8 commands afterward. Two H6 trials missed specific/non-last/wildcard removal, exclusive replacement, and destruction. The third also missed specific/wildcard Added, `Or` and trait composition, cache isolation, and per-target result data. The extra tokens therefore came primarily from longer implementation, failed patch, validation, documentation, and publish-surface loops—not from consuming the trace response itself. + +Because tracing occurred late and the cohorts used different trajectories, this run does not prove that citation IDs caused the failures. It does prove that the partially compact representation is not a safe end-to-end win under the project's rollback criterion. H6 was reverted; H3 remains the retained configuration. + +## OpenWiki-20 optimization loop + +The broader loop uses the completed 20-task baseline and 40-trial OpenWiki cohort. Each discriminator is run twice. A hypothesis is retained only when it does not regress quality and improves coding-agent tokens, calls, or time; otherwise its product changes are reverted before the next hypothesis. Wiki-generation tokens remain excluded. + +### Hypothesis 1: task-first change-surface ranking + +#### Proposed change + +- Rank implementation evidence from the original task language, using generated-wiki paths only as a prior. +- Stop expanding the source query with every generic token from related wiki prose; retain only explicitly formatted symbols. +- Rank the tests group independently against the task instead of accepting whichever test happened to rank in the global source list. +- Exclude repository instruction files from change-surface evidence. + +#### Expected benefit + +The Helm failure received `AGENTS.md`, `pkg/cmd/root.go`, and an unrelated downloader test from `change_surface`, then widened a CLI presentation change into persisted release and Kubernetes lifecycle semantics. More precise task and test citations should reduce wrong-boundary edits and duplicate exploration. The main discriminator is Helm; Scriggo checks whether negative/parser tests become easier to locate. + +#### Success gate + +Across n=2 discriminator runs, preserve or improve mean partial score and full solves while reducing mean uncached or cumulative input and total tool calls. Inspect the returned citations directly. Revert if quality regresses or end-to-end efficiency does not improve. + +#### Outcome + +H1 achieved perfect discriminator quality but missed the efficiency gate, so it is retained only as the quality base for subsequent hypotheses. Across four valid trials (the infrastructure-failed wiki setup was replaced), it solved 4/4 with 1.000000 mean partial. It averaged 182,820 uncached input tokens, 16,368,626 cumulative input tokens, 46,168 output tokens, 109.25 observed tool calls, 3.25 retrieval calls, and 27.5 edit actions. Relative to the prior OpenWiki pair, it improved 2/4 full solves to 4/4 but increased uncached input about 6.7% and calls about 7.6%. Trace inspection showed agents recovered from still-poor initial citations through extra targeted search and validation; H1 alone does not beat the baseline on efficiency. + +### Hypothesis 2: unique change-surface evidence budget + +#### Proposed change + +Keep H1's task-first ranking, independent test ranking, and instruction-file exclusion, but deduplicate citations across `change_surface` groups. Preserve category coverage and the seven-result budget; when one chunk qualifies for multiple groups, keep its first category and advance the later group to its next distinct citation. Leave `trace_symbols` unchanged because earlier trace compaction regressed. + +#### Expected benefit + +The H1 Helm retry returned seven category slots but only four unique files: `pkg/cmd/root.go` and `pkg/cmd/install.go` each appeared twice. That crowded out task-named surfaces such as `template.go`, `upgrade.go`, and `get_manifest.go`, after which the agent performed manual searches. Seven distinct citations should expose more of the change boundary in the already-required pre-edit call, reducing follow-up searches and commands without increasing retrieval payload or changing prompt behavior. + +#### Success gate + +Across n=2 discriminator runs, preserve H1's score while reducing mean retrieval calls, total tool calls, or uncached/cumulative input. Inspect the actual citations and verify that `trace_symbols` output is unchanged. Revert if quality regresses or the unique evidence does not reduce end-to-end work. + +#### Outcome + +H2 failed the quality gate and was rolled back before H3. It solved only 1/4 with 0.853724 mean partial versus H1's 4/4 and 1.000000. It did reduce mean uncached input from 182,820 to 155,579 (-14.9%) and observed calls from 109.25 to 86.0 (-21.3%), proving that a broader set of distinct citations can shorten trajectories. The savings are unusable because both Scriggo trials and one Helm trial lost full solves. Deduplication worked mechanically, but the distinct evidence was still poorly ranked, including CLI help/generator files and unrelated checker tests. The agents then performed deep compiler/runtime discovery and debugging themselves; no H2 product code is retained. + +### Hypothesis 3: literal and path-aware change-surface ranking + +#### Proposed change + +Retain H1 but use literal-token BM25 and keyword rankers for `change_surface` source and test evidence. Keep synonym expansion for general `search`, semantic fallback, OKF concept discovery, and symbol tracing. Preserve the generated wiki's explicit referenced paths as a strong prior, so documentation still connects concepts to code without expanding task words into unrelated source vocabulary. + +#### Expected benefit + +H1 and H2 returned irrelevant Scriggo CLI help/generator files and unrelated tests because source ranking expanded generic query terms through broad synonym groups such as API/public/publish and setup/register. Literal task terms plus wiki path priors should favor task-named compiler, runtime, command, and test paths; that should reduce compensating searches and deep exploratory reads while preserving the quality gains of H1. + +#### Success gate + +Run Helm and Scriggo n=2 with all four trials concurrent. Preserve H1's full-solve count and improve mean uncached/cumulative input or observed tool calls. Revert H3 if quality regresses or ranking precision fails to improve end-to-end efficiency. + +#### Outcome + +H3 failed the quality gate and was rolled back. Its first completed Scriggo verifier scored only 0.624430 partial versus H1's 1.000000 in both trials. Literal ranking also failed its intended mechanism check: Helm still over-ranked `pkg/cmd/root.go`, while Scriggo returned renderer, CLI help, and unrelated multi-file template evidence rather than the method checker/emitter/runtime boundary. Frozen H3 trials continue only for diagnostic completeness; no H3 product code is retained. + +### Hypothesis 4: bounded retrieval-first workflow + +#### Proposed change + +Keep H1's task-first `change_surface` ranking, but replace the eval adapter's mandatory quickstart read with one bounded pre-edit `change_surface` brief. Ask the agent to inspect the cited source and tests, avoid separately reading the quickstart or linked wiki pages unless a specific evidence gap remains, stop discovery after locating the implementation, affected public/generated surface, and focused tests, and avoid unrelated broad validation after focused checks pass. + +#### Expected benefit + +The expensive Effect, Adaptix, and Koota traces read the quickstart, multiple full wiki pages, generated instructions, and overlapping retrieval before repeating broad source discovery. The efficient Helm and KGateway traces stopped much earlier once the correct surface was known. A single documentation-backed evidence bundle plus an explicit stop rule should retain OpenWiki's cross-boundary quality advantage while removing duplicate context, search, and validation calls. + +#### Success gate + +Run Effect, Adaptix, Prometheus, and KGateway twice each with all eight trials concurrent. Preserve or improve the baseline's aggregate quality on this mixed discriminator while bringing mean uncached input and observed tool calls down relative to H1; inspect whether full-page wiki reads and overlapping retrieval disappear. Revert the prompt if quality regresses. Only promote it to generated product guidance if it passes. + +#### Outcome + +H4 is retained as the new quality-and-call base, but it does not yet pass the token gate: + +| Metric | H4 result | +| -------------------------- | --------: | +| Full solves | 4/8 | +| Mean partial | 0.992970 | +| Median partial | 0.999641 | +| Uncached input/trial | 138,969 | +| Cumulative input/trial | 8,834,009 | +| Output/trial | 28,231 | +| Observed tool calls/trial | 70.75 | +| Retrieval calls/trial | 2.125 | +| Direct wiki reads/trial | 0.00 | +| Command-output chars/trial | 957,332 | + +Against the matched one-attempt baseline for Adaptix, Effect, Prometheus, and KGateway, H4 improved full solves from 1/4 to 4/8 and mean partial from 0.980085 to 0.992970. It reduced observed calls from 74.25 to 70.75, but uncached input remained 16.0% above the matched 119,866-token baseline and 25.1% above the overall 111,080-token baseline. Relative to H1 on these tasks, H4 removed every direct wiki-page read, cut retrieval to about two calls, substantially shortened each trajectory, recovered Effect from 0.598291 to 0.974359/1.000000, and reduced tokens; the remaining cost is no longer duplicate wiki consumption. + +The dominant residual signal is command payload. H4 averaged 957k command-output characters per trial. Agents still dumped large source windows and consumed verbose test, generation, type-check, and build output. Effect averaged 194,814 uncached tokens despite one full solve; one KGateway trial spent extra calls and context recovering from a broad generation command. H4's workflow constraint worked, but it did not bound evidence size within each command. + +### Hypothesis 5: bounded source and validation output + +#### Proposed change + +Retain H4 and require symbol-first, line-bounded source reads: inspect the cited range or a symbol-sized window of at most about 200 lines per command rather than dumping whole large files. Prefer repository-supported quiet validation flags. When unavailable, capture only test/build output in a short-lived task-local log, emit one concise success line or the relevant failure tail, delete the log immediately, and never redirect credential, environment, or configuration output. + +#### Expected benefit + +H4 already brings calls below the matched baseline, so further mandatory retrieval or fewer tools would target the wrong variable. Bounding per-call evidence should reduce uncached and cumulative context without removing source authority, tests, or failure diagnostics. It directly targets Effect's large multi-file reads and noisy package checks and KGateway's generation/test output while preserving H4's quality gains. + +#### Success gate + +Run the same four-task discriminator twice with all eight trials concurrent. Preserve H4's quality advantage over the matched baseline and keep observed calls at or below 75.2 per task. Reduce mean uncached input materially toward or below 111.1k, with lower command-output characters. Revert H5's output-budget instructions if quality regresses. + +#### Outcome + +H5 failed the quality and call gates and was rolled back. Across eight valid +trials it solved 2/8 with 0.975765 mean partial, versus H4's 4/8 and 0.992970. +It reduced uncached input to 120,576 tokens per trial, but observed calls rose +to roughly 77 per trial. Prometheus fell to 0.938144 in both attempts and +Effect remained partial in both attempts. Trace inspection showed that agents +ignored several source/output limits, launched overlapping checks, polled +background jobs, and collided on temporary logs. Smaller command responses did +not compensate for the added validation and recovery work, so none of H5's +source-window or temporary-log instructions is retained. + +### Hypothesis 7: serialized quiet final validation + +#### Proposed change + +Build on H4 and full compound-aware retrieval, but isolate the safe part of +H5's output idea. After focused checks pass, run repository-required final +test, lint, typecheck, build, and documentation commands once and serially. +Capture each command's output separately and expose only a one-line success or +the relevant failure tail. Explicitly avoid overlapping validation and +`sleep`/`ps` polling. Do not cap source reads, reuse one shared temporary log, +or require log deletion. + +#### Expected benefit + +Effect's own `AGENTS.md` mandates root lint, check, build, and docgen, so those +commands cannot simply be skipped. H4/H6 traces repeatedly launched them in +parallel, consumed megabytes of successful build output, hit memory pressure, +and spent additional calls polling background processes. Serial quiet checks +should preserve required validation and solve quality while reducing uncached +context, tool calls, and retry churn. Removing H5's source cap isolates output +control from the Prometheus quality regression. + +#### Success gate + +Run Effect and Prometheus twice each. Preserve at least H4's two full solves and +0.9884 matched mean partial score while reducing uncached tokens and keeping +observed calls at or below 75.2 per task. Inspect traces for overlapping checks, +polling commands, and validation-output characters. Revert the prompt if +quality regresses or agents do not follow it. + +#### Outcome + +H7 failed the quality gate and its prompt-only change was rolled back. One +Effect trial failed during agent setup before any model call and was excluded. +The three valid trials produced no full solves: Prometheus scored 0.989691 in +both attempts and Effect scored 0.974359, for 0.984580 mean partial. A +replacement Effect trial was not run because even a perfect result could reach +only 1/4 full solves, below H4's required 2/4. + +The mechanism worked locally: traces contained no `sleep`/`ps` polling or +overlapping background validation. Relative to the four matched H4 Effect and +Prometheus trials, the three valid H7 trials reduced mean uncached input from +153,507 to 113,415, cumulative input from 9,067,200 to 7,427,668, and command +output from about 364k to 271k extracted characters. Observed calls also fell +from about 75 to 70 per trial. Those aggregate savings were concentrated in +Prometheus. Effect still made 111 observed calls, repeated focused lint/tests +while fixing real failures, and took about 32 minutes of coding time. Quiet +serial validation therefore reduced successful-command noise but did not +prevent implementation/debugging loops, and the quality regression makes the +savings unusable. + +### Hypothesis 6a: snake-only lexical boundaries + +#### Proposed change + +Partially roll back Hypothesis 6 after its first valid Effect trial regressed. +Keep underscore boundary splitting, which directly improved Adaptix's +`name_mapping` to `NameMapping` evidence, but restore the prior joined-token +behavior for hyphenated terms. This prevents H6 from changing ranking for +Effect task phrases such as `text/event-stream`, `no-cache`, and `keep-alive`. +All other H4 behavior remains unchanged. + +#### Expected benefit + +The full compound change produced 2/2 Adaptix solves while reducing Adaptix +mean uncached input from 119.4k to 116.4k and observed calls from 63.5 to 50. +Its first valid Effect result fell to 0.9487, below H4's 0.9744 and 1.0. +Snake-only normalization should preserve the measured Adaptix improvement while +removing the only lexical change relevant to the Effect task. + +#### Success gate + +Run Adaptix and Effect twice each. Preserve H4's aggregate quality and retain +the Adaptix token/call improvement. Revert underscore splitting as well if the +quality regression persists. + +#### Outcome + +H6a was a clear regression and was rolled back. It solved 1/4 with 0.886662 +mean partial, 129,513 uncached input tokens, and 79.5 observed calls per trial. +Adaptix retained one full solve plus a 0.999641 partial, but Effect scored +0.948718 and 0.598291. Snake-only splitting did not isolate the Effect +regression and also lost one Adaptix full solve, so full underscore-and-hyphen +compound normalization was restored. + +### Hypothesis 6: compound-aware lexical retrieval + +#### Proposed change + +Retain H4's bounded retrieval-first workflow and task-first `change_surface` +ranking, but normalize snake_case and kebab-case boundaries the same way as +camelCase and PascalCase boundaries. For example, `name_mapping`, +`name-mapping`, and `NameMapping` should all contribute the terms `name` and +`mapping` instead of producing incompatible `namemapping` versus +`name`/`mapping` tokens. Leave synonym expansion, result limits, semantic +fallback, and tool descriptions unchanged. + +#### Expected benefit + +Generated wiki prose and benchmark tasks frequently use snake_case or +hyphenated API names while source symbols use camel case. The current mismatch +silently weakens both BM25 and keyword ranking, forcing agents to compensate +with manual source searches. Fixing the lexical boundary should improve the +already-required `change_surface` evidence without another tool call or a +larger response. Adaptix is the primary discriminator because `name_mapping` +must resolve to `NameMapping`; Effect provides a second compound-heavy check. + +#### Success gate + +Run Adaptix and Effect twice each, with all four trials concurrent. Preserve +H4's aggregate quality while reducing uncached input, cumulative input, or +observed calls. Inspect `change_surface` citations to confirm the compound +match changed the returned evidence. Revert if quality regresses or retrieval +precision does not improve end-to-end efficiency. + +#### Outcome + +H6 is retained. After replacing one infrastructure-failed Effect setup, the +four valid trials solved 3/4 with 0.987179 mean partial. Adaptix improved to 2/2 +full solves while averaging about 116.4k uncached tokens and 50 observed calls. +Effect produced one full solve and one 0.948718 partial. Across both tasks H6 +averaged about 136.0k uncached tokens and 67.75 calls, reducing each by roughly +13-14% from the matched H4 trials. H6a's failed ablation strengthened the +evidence that normalizing both snake_case and kebab-case boundaries is the +better retained retrieval behavior, despite remaining Effect variance. + +### Hypothesis 8: OKF metadata-routed task briefs + +#### Proposed change + +Replace broad wiki prose retrieval with one compact `change_surface` brief: +implementation ownership, explicit invariants, analogous tests, conditional +delivery surfaces, validation commands, unknowns, and a changed-path coverage +review. Treat OKF descriptions and inferred document roles as routing signals, +tags as weighted facets rather than graph edges, and explicit Markdown links as +relationships. Add a type-checked `openwiki` frontmatter extension for future +generated wikis (`roles`, `change_kinds`, `source_paths`, `symbols`, +`test_paths`, `invariants`, and `validation_commands`). Shorten the managed +AGENTS guidance so agents use the brief only when it can replace exploration, +inspect cited source/tests directly, and avoid rereading returned wiki pages. + +#### Expected benefit + +The previous workflow added a parallel wiki-reading phase. A small structured +brief should substitute for broad source discovery, make uncertainty explicit, +and use the existing OKF metadata as a cheap retrieval control plane. It should +reduce direct OpenWiki payload, redundant filesystem wiki reads, total calls, +and context while preserving or improving quality. + +#### Success gate + +Run the same 14-task fast subset at n=2 with cached wiki Markdown and all trials +parallel. Preserve or improve the prior 9/28 full solves and 0.965895 mean +partial score while reducing tokens and calls. Require compatible caches so the +experiment changes retrieval/runtime behavior without paying for or changing +wiki generation. If preliminary valid trials show no quality improvement, do +not spend tokens replacing infrastructure failures. + +#### Outcome + +The quality gate failed, so six Docker-subnet setup failures were not rerun. +The 22 valid trials solved 7/22 (31.8%) with 0.934685 mean and 0.994733 median +partial score. The prior OpenWiki cohort solved 9/28 (32.1%) with 0.965895 mean +partial, and baseline solved 10/28 (35.7%) with 0.969366 mean partial. One +deferred-mutation trial scored 0.030151 while its replicate fully solved, +accounting for most of the mean regression, but the aggregate solve rate still +showed no improvement. + +The 22 valid trials averaged 117,522 uncached input tokens, 6.264M cumulative +input, 30,355 output tokens, and 61.32 observed calls. Relative to prior +OpenWiki task means reweighted to the same 22-task composition, those are +improvements of 6.4%, 4.5%, 2.4%, and 0.6%, respectively. OpenWiki use fell +from 3.54 to 2.36 calls per trial; direct estimated overhead fell from 7,799 to +4,738 tokens. Removing direct OpenWiki overhead leaves 6.290M total tokens and +58.95 calls per trial, still above baseline's same-task-weighted 4.689M total +tokens and 56.70 calls. The brief therefore became smaller and more +consistently consulted, but did not yet replace enough downstream exploration +or improve implementation quality. + +The cached wikis predate the new namespaced `openwiki` extension, so this run +evaluated inferred roles, descriptions, tags, explicit links, retrieval output, +and AGENTS behavior—not producer-authored structured invariants, source paths, +test paths, or validation commands. A future clean generation experiment is +required to evaluate that half of the hypothesis. diff --git a/evals/deepswe/README.md b/evals/deepswe/README.md index 1c085a6f..e8ea4837 100644 --- a/evals/deepswe/README.md +++ b/evals/deepswe/README.md @@ -4,10 +4,11 @@ This harness runs a paired DeepSWE experiment with the same tasks, seed, model, reasoning effort, attempts, and Harbor environment in both conditions: - `baseline`: Codex receives only the DeepSWE task and repository. -- `openwiki`: OpenWiki first documents an isolated clone of the agent-visible - repository, then the same Codex adapter is instructed to read the generated - quickstart and use OpenWiki's read-only OKF-aware retrieval MCP server before - solving the task. +- `openwiki`: the adapter restores or generates OpenWiki in an isolated clone, + merges OpenWiki's managed instructions into the repository's root + `AGENTS.md`, and copies both `AGENTS.md` and `openwiki/` into `/app` before the + same Codex adapter solves the unchanged DeepSWE task. Codex automatically + loads root `AGENTS.md`; the harness adds no treatment-only task prompt. The harness pins: @@ -15,7 +16,7 @@ The harness pins: - `harbor[langsmith]==0.20.0` - `litellm==1.83.14` (Harbor's supported lower bound, pinned to avoid a newer release's local Rust build requirement) -- Codex CLI `0.118.0` +- Codex CLI `0.144.6` - the current OpenWiki checkout, packed locally for each treatment run ## Safety and isolation @@ -26,9 +27,10 @@ adapter additionally runs OpenWiki against `/tmp/openwiki-source`, a local clone of `/app`; OpenWiki never runs from the benchmark task directory and cannot see the verifier or reference solution. -Generated wiki files remain outside `/app`, so DeepSWE's patch extraction cannot -include them. Codex is explicitly told that `/app` is the source of truth and -that all code changes belong there. +The generated wiki and merged `AGENTS.md` are copied into `/app` so Codex sees +the same layout as a normal local OpenWiki user. They are hidden from Git status, +and the treatment adapter explicitly excludes `AGENTS.md` and `openwiki/**` from +the verifier patch. DeepSWE v1.1 normally requires Pier 0.3's `pre_artifacts.sh` lifecycle to copy committed work into its separate verifier. This harness retains Harbor 0.20 for @@ -52,6 +54,13 @@ install and run Codex and OpenWiki's pinned SQLite binding, plus the LangSmith API and trace-ingest hosts required by every traced run. The adapter uses the task image's existing Node runtime and installs the pinned Codex CLI directly, avoiding Harbor's NVM bootstrap. +When a task image lacks ripgrep, the adapter installs Debian's package after +disabling only a preconfigured `*nodesource*` apt source inside the disposable +container; stale NodeSource repositories must not prevent agent setup. +Parallel runs give agent setup three times Harbor's default deadline so +concurrent Codex package downloads do not fail before model execution. Override +this with `--agent-setup-timeout-multiplier` when local registry throughput +requires a different bound. For Docker runs, the harness removes inactive per-trial networks after each job and checks completed prior jobs for stale networks before launching. Cleanup is @@ -145,6 +154,18 @@ source ~/.zshrc && python3 evals/deepswe/run.py openwiki \ --reasoning-effort high ``` +Generated task wikis are cached on the host in +`evals/deepswe/.cache/openwiki-wikis`. The key includes the task repository's +base commit, the normalized OpenWiki package contents, and the OpenWiki model, +so unchanged reruns restore the same wiki instead of regenerating it. Use +`--openwiki-cache-dir PATH` to select another persistent cache location. The +first cache-aware run for a commit still generates and populates the cache. +By default, a package update may also reuse an older cache whose validated +`openwiki/.last-update.json` records the exact same task commit and model. Pass +`--no-reuse-compatible-wiki-cache` to disable that lookup. Pass +`--require-openwiki-cache` to fail before any wiki-generation model call on a +cache miss; use this for controlled reruns where wiki Markdown must stay fixed. + Run both paired conditions and summarize them: ```bash @@ -164,7 +185,7 @@ hosted parallel environment. ### Named OpenWiki task suites -Use `--task-suite` for the two exact, reproducible OpenWiki cohorts. A suite +Use `--task-suite` for the exact, reproducible OpenWiki cohorts. A suite selects all of its members regardless of `--n-tasks` and cannot be combined with `--task`: @@ -174,8 +195,28 @@ python3 evals/deepswe/run.py paired --task-suite koota-5 # Broader set: the five Koota tasks plus 15 independent repositories python3 evals/deepswe/run.py paired --task-suite openwiki-20 + +# Documentation-leverage set: ten cross-surface tasks from independent cohorts +python3 evals/deepswe/run.py paired --task-suite openwiki-doc-leverage-10 ``` +The `openwiki-doc-leverage-10` suite targets changes where repository +documentation should have high leverage: ownership and behavior are spread +across multiple runtime, serialization, integration, CLI, SDK, or delivery +surfaces. Its members are disjoint from `openwiki-20` so it can provide a fresh +test of the OpenWiki hypothesis: + +- `aiomonitor-task-snapshots-diff` +- `bandit-incremental-cache-control` +- `dynamodb-toolbox-conditional-attribute-requirements` +- `fastapi-deprecation-response-headers` +- `go-genai-streamed-function-args` +- `goreleaser-retry-publish-auditing` +- `gql-incremental-graphql-delivery` +- `igel-persist-feature-schema` +- `onedump-dump-encryption-pipeline` +- `testem-bail-on-test-failure` + The 15 tasks added to `openwiki-20` are not exposed as a separate runnable suite. They were selected from the user-provided `gpt-5.6-terra [medium]` leaderboard export. Across their 42 listed trials they had an 81% failure rate, @@ -203,14 +244,13 @@ proxies for token intensity. | `python-statemachine-state-data-scoping` | python-statemachine / Python | 3/3 failed, 36.3 steps | Hierarchical state ownership, history, isolation, lifecycle resets | Treatment runs register `openwiki-retrieval-mcp` inside Codex's isolated home. -It exposes three read-only workflows over `/app` and the generated wiki: -`search` with `all`, `wiki`, `source_code`, and `tests` scopes; -`change_surface` for pre-edit cross-boundary mapping; and batched -`trace_symbols` for post-edit public-surface verification. Search automatically -combines exact, BM25, semantic-vector, and OKF graph ranking. Local deterministic -vectors are the default. Pass `--retrieval-embedding-provider openai` to opt into -bounded `text-embedding-3-small` reranking; provider failures fall back to local -vectors. +It exposes two read-only workflows over `/app` and `/app/openwiki`: `search` +with `all`, `wiki`, `source_code`, and `tests` scopes, and `change_surface` for +bounded wiki guidance, cross-boundary source/test mapping, evidence gaps, and +wiki provenance. Search automatically combines exact, BM25, semantic-vector, +and OKF graph ranking. Local deterministic vectors are the default. Pass +`--retrieval-embedding-provider openai` to opt into bounded +`text-embedding-3-small` reranking; provider failures fall back to local vectors. If runs already exist, summarize them without invoking Harbor: @@ -236,3 +276,16 @@ OpenWiki's current CLI does not expose generation token usage to Harbor's local summary, so treatment summaries include its wall-clock time but not its tokens or provider cost. Its LangSmith generation traces in the same experiment provide generation-token details. + +To measure direct treatment overhead after a run, use: + +```bash +python3 evals/deepswe/analyze_openwiki_usage.py \ + --job-dir evals/deepswe/results/ +``` + +The analyzer separately reports OpenWiki MCP calls, shell reads under +`openwiki/`, serialized tool-call and result characters at four characters per +token, and one automatic inclusion of the managed OpenWiki `AGENTS.md` block. +It also reports token/tool totals after subtracting that estimated direct +overhead. It does not estimate repeated cached-context amplification. diff --git a/evals/deepswe/analyze_openwiki_usage.py b/evals/deepswe/analyze_openwiki_usage.py new file mode 100644 index 00000000..3585dabc --- /dev/null +++ b/evals/deepswe/analyze_openwiki_usage.py @@ -0,0 +1,440 @@ +#!/usr/bin/env python3 +"""Measure direct OpenWiki retrieval overhead in Codex DeepSWE traces. + +The estimate intentionally follows a simple, reproducible rule: recorded +result characters and canonical tool-call JSON characters each cost one token +per four characters. It does not attempt to model repeated context-window +charges or tokenizer-specific behavior. +""" + +from __future__ import annotations + +import argparse +import json +import re +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any, Iterable + + +CHARS_PER_TOKEN = 4 +OPENWIKI_SERVER = "openwiki_retrieval" +OPENWIKI_AGENTS_START = "" +OPENWIKI_AGENTS_END = "" +MAX_AGENTS_BYTES = 1_048_576 +TOOL_ITEM_TYPES = { + "command_execution", + "file_change", + "mcp_tool_call", + "todo_list", + "web_search", +} +OPENWIKI_PATH_RE = re.compile( + r"(? int: + return self.input_tokens - self.cached_input_tokens + + @property + def total_tokens(self) -> int: + return self.input_tokens + self.output_tokens + + +@dataclass(frozen=True) +class TrialMetrics: + task_name: str + trial_dir: str + total_tool_calls: int + openwiki_mcp_calls: int + openwiki_filesystem_calls: int + call_json_chars: int + result_chars: int + agents_prompt_chars: int + usage: Usage + + @property + def openwiki_calls(self) -> int: + return self.openwiki_mcp_calls + self.openwiki_filesystem_calls + + +def _dict(value: Any) -> dict[str, Any] | None: + return value if isinstance(value, dict) else None + + +def _string(value: Any) -> str | None: + return value if isinstance(value, str) else None + + +def _nonnegative_int(value: Any) -> int | None: + return value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else None + + +def canonical_call_json(name: str, args: dict[str, Any]) -> str: + """Return the stable representation used for tool-call token estimates.""" + + return json.dumps( + {"name": name, "args": args}, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + + +def is_openwiki_filesystem_read(command: str) -> bool: + """Whether a logged shell call reads from a generated OpenWiki directory.""" + + return bool(OPENWIKI_PATH_RE.search(command) and READ_COMMAND_RE.search(command)) + + +def _mcp_result_text(item: dict[str, Any]) -> str: + result = item.get("result") + result_dict = _dict(result) + if result_dict is not None: + content = result_dict.get("content") + if isinstance(content, list): + text_blocks = [] + for block in content: + block_dict = _dict(block) + if block_dict is not None: + text_value = _string(block_dict.get("text")) + if text_value is not None: + text_blocks.append(text_value) + if text_blocks: + return "".join(text_blocks) + if isinstance(result, str): + return result + error = _string(item.get("error")) + return error or "" + + +def _load_json_object(path: Path) -> dict[str, Any] | None: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError): + return None + return _dict(value) + + +def _managed_agents_chars(path: Path) -> int: + try: + if path.stat().st_size > MAX_AGENTS_BYTES: + return 0 + content = path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + return 0 + start = content.find(OPENWIKI_AGENTS_START) + end = content.find(OPENWIKI_AGENTS_END, start + len(OPENWIKI_AGENTS_START)) + if start < 0 or end < 0: + return 0 + return len(content[start : end + len(OPENWIKI_AGENTS_END)]) + + +def _iter_jsonl_objects(path: Path) -> Iterable[dict[str, Any]]: + try: + with path.open(encoding="utf-8") as trace: + for line in trace: + try: + value = json.loads(line) + except json.JSONDecodeError: + continue + value_dict = _dict(value) + if value_dict is not None: + yield value_dict + except (OSError, UnicodeDecodeError): + return + + +def _usage_from_event(event: dict[str, Any]) -> Usage | None: + if event.get("type") != "turn.completed": + return None + usage = _dict(event.get("usage")) + if usage is None: + return None + input_tokens = _nonnegative_int(usage.get("input_tokens")) + cached_input_tokens = _nonnegative_int(usage.get("cached_input_tokens")) + output_tokens = _nonnegative_int(usage.get("output_tokens")) + if input_tokens is None or cached_input_tokens is None or output_tokens is None: + return None + if cached_input_tokens > input_tokens: + return None + return Usage(input_tokens, cached_input_tokens, output_tokens) + + +def analyze_trace(trace_path: Path, task_name: str) -> TrialMetrics | None: + """Analyze one Codex JSONL trace, ignoring malformed or incomplete events.""" + + pending: dict[str, list[dict[str, Any]]] = {} + usage: Usage | None = None + total_tool_calls = 0 + mcp_calls = 0 + filesystem_calls = 0 + call_json_chars = 0 + result_chars = 0 + + for event in _iter_jsonl_objects(trace_path): + parsed_usage = _usage_from_event(event) + if parsed_usage is not None: + usage = parsed_usage + + event_type = event.get("type") + item = _dict(event.get("item")) + if item is None: + continue + item_id = _string(item.get("id")) + item_type = _string(item.get("type")) + if item_id is None or item_type is None: + continue + if event_type == "item.started": + pending.setdefault(item_id, []).append(item) + if item_type in TOOL_ITEM_TYPES: + total_tool_calls += 1 + elif event_type == "item.completed": + candidates = pending.get(item_id) + if not candidates: + continue + started_item = candidates.pop(0) + if not candidates: + pending.pop(item_id, None) + + started_type = started_item.get("type") + if ( + started_type == "mcp_tool_call" + and started_item.get("server") == OPENWIKI_SERVER + ): + tool = _string(started_item.get("tool")) + arguments = _dict(started_item.get("arguments")) + if tool is None or arguments is None: + continue + mcp_calls += 1 + call_json_chars += len( + canonical_call_json(f"{OPENWIKI_SERVER}.{tool}", arguments) + ) + result_chars += len(_mcp_result_text(item)) + elif started_type == "command_execution": + command = _string(started_item.get("command")) + if command is None or not is_openwiki_filesystem_read(command): + continue + filesystem_calls += 1 + call_json_chars += len( + canonical_call_json("command_execution", {"command": command}) + ) + output = _string(item.get("aggregated_output")) + if output is not None: + result_chars += len(output) + + if usage is None: + return None + + return TrialMetrics( + task_name=task_name, + trial_dir=str(trace_path.parent.parent), + total_tool_calls=total_tool_calls, + openwiki_mcp_calls=mcp_calls, + openwiki_filesystem_calls=filesystem_calls, + call_json_chars=call_json_chars, + result_chars=result_chars, + agents_prompt_chars=_managed_agents_chars( + trace_path.parent / "openwiki-agents.md" + ), + usage=usage, + ) + + +def _valid_trial(job_dir: Path, result_path: Path) -> tuple[str, Path] | None: + try: + resolved_result = result_path.resolve(strict=True) + resolved_result.relative_to(job_dir) + except (OSError, ValueError): + return None + + result = _load_json_object(resolved_result) + if result is None: + return None + task_name = _string(result.get("task_name")) + if task_name is None: + return None + trace_path = resolved_result.parent / "agent" / "codex.txt" + if not trace_path.is_file(): + return None + return task_name, trace_path + + +def collect_trials(job_dirs: Iterable[Path]) -> list[TrialMetrics]: + trials = [] + seen_trial_dirs: set[Path] = set() + for supplied_dir in job_dirs: + try: + job_dir = supplied_dir.resolve(strict=True) + except OSError as exc: + raise ValueError(f"job directory does not exist: {supplied_dir}") from exc + if not job_dir.is_dir(): + raise ValueError(f"job path is not a directory: {supplied_dir}") + for result_path in job_dir.rglob("result.json"): + valid = _valid_trial(job_dir, result_path) + if valid is None: + continue + task_name, trace_path = valid + trial_dir = trace_path.parent.parent.resolve() + if trial_dir in seen_trial_dirs: + continue + metrics = analyze_trace(trace_path, task_name) + if metrics is not None: + trials.append(metrics) + seen_trial_dirs.add(trial_dir) + return sorted(trials, key=lambda trial: (trial.task_name, trial.trial_dir)) + + +def _sum_usage(trials: Iterable[TrialMetrics]) -> dict[str, int]: + trial_list = list(trials) + return { + "input_tokens": sum(trial.usage.input_tokens for trial in trial_list), + "cached_input_tokens": sum( + trial.usage.cached_input_tokens for trial in trial_list + ), + "uncached_input_tokens": sum( + trial.usage.uncached_input_tokens for trial in trial_list + ), + "output_tokens": sum(trial.usage.output_tokens for trial in trial_list), + "total_tokens": sum(trial.usage.total_tokens for trial in trial_list), + } + + +def summarize(trials: list[TrialMetrics]) -> dict[str, Any]: + if not trials: + raise ValueError("no valid completed trials with Codex usage were found") + + count = len(trials) + total_tool_calls = sum(trial.total_tool_calls for trial in trials) + mcp_calls = sum(trial.openwiki_mcp_calls for trial in trials) + filesystem_calls = sum(trial.openwiki_filesystem_calls for trial in trials) + openwiki_calls = mcp_calls + filesystem_calls + call_chars = sum(trial.call_json_chars for trial in trials) + result_chars = sum(trial.result_chars for trial in trials) + agents_chars = sum(trial.agents_prompt_chars for trial in trials) + call_tokens = call_chars / CHARS_PER_TOKEN + result_tokens = result_chars / CHARS_PER_TOKEN + agents_tokens = agents_chars / CHARS_PER_TOKEN + direct_tool_tokens = call_tokens + result_tokens + overhead_tokens = direct_tool_tokens + agents_tokens + raw = _sum_usage(trials) + + adjusted = { + "input_tokens": raw["input_tokens"] - result_tokens - agents_tokens, + "uncached_input_tokens": ( + raw["uncached_input_tokens"] - result_tokens - agents_tokens + ), + "output_tokens": raw["output_tokens"] - call_tokens, + "total_tokens": raw["total_tokens"] - overhead_tokens, + "tool_calls": total_tool_calls - openwiki_calls, + } + per_trial = { + "raw_uncached_input_tokens": raw["uncached_input_tokens"] / count, + "raw_input_tokens": raw["input_tokens"] / count, + "raw_output_tokens": raw["output_tokens"] / count, + "raw_total_tokens": raw["total_tokens"] / count, + "raw_tool_calls": total_tool_calls / count, + "openwiki_calls": openwiki_calls / count, + "openwiki_estimated_tokens": overhead_tokens / count, + "adjusted_input_tokens": adjusted["input_tokens"] / count, + "adjusted_output_tokens": adjusted["output_tokens"] / count, + "adjusted_total_tokens": adjusted["total_tokens"] / count, + "adjusted_tool_calls": adjusted["tool_calls"] / count, + } + + return { + "estimation": { + "chars_per_token": CHARS_PER_TOKEN, + "scope": ( + "direct tool-call JSON, tool-result text, and one inclusion of " + "the managed OpenWiki AGENTS.md block" + ), + "caveat": ( + "AGENTS.md is automatically loaded, not a tool call. The estimate " + "does not model repeated cached-context amplification or " + "tokenizer-specific counts." + ), + }, + "trials": count, + "tasks": len({trial.task_name for trial in trials}), + "tool_calls": { + "all": total_tool_calls, + "openwiki": openwiki_calls, + "openwiki_mcp": mcp_calls, + "openwiki_filesystem": filesystem_calls, + "adjusted_without_openwiki": adjusted["tool_calls"], + }, + "openwiki_token_overhead": { + "call_json_chars": call_chars, + "call_json_tokens": call_tokens, + "result_chars": result_chars, + "result_tokens": result_tokens, + "agents_prompt_chars": agents_chars, + "agents_prompt_tokens": agents_tokens, + "direct_tool_tokens": direct_tool_tokens, + "total_tokens": overhead_tokens, + }, + "raw_usage": raw, + "adjusted_usage": adjusted, + "per_trial": per_trial, + "trial_details": [ + { + **{key: value for key, value in asdict(trial).items() if key != "usage"}, + "openwiki_calls": trial.openwiki_calls, + "estimated_openwiki_tokens": ( + trial.call_json_chars + + trial.result_chars + + trial.agents_prompt_chars + ) + / CHARS_PER_TOKEN, + "usage": asdict(trial.usage), + } + for trial in trials + ], + } + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--job-dir", + action="append", + required=True, + type=Path, + help="Harbor job directory; repeat for retry directories and replicates.", + ) + parser.add_argument( + "--json-output", + type=Path, + help="Also write the complete JSON summary to this path.", + ) + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + try: + summary = summarize(collect_trials(args.job_dir)) + except ValueError as exc: + raise SystemExit(str(exc)) from exc + rendered = json.dumps(summary, indent=2, sort_keys=True) + "\n" + print(rendered, end="") + if args.json_output is not None: + args.json_output.write_text(rendered, encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/deepswe/openwiki_codex.py b/evals/deepswe/openwiki_codex.py index c7955d93..421ecce7 100644 --- a/evals/deepswe/openwiki_codex.py +++ b/evals/deepswe/openwiki_codex.py @@ -3,8 +3,12 @@ from __future__ import annotations import json +import hashlib +import os import re import shlex +import tarfile +import tempfile import time from pathlib import Path, PurePosixPath from typing import Any @@ -21,7 +25,163 @@ _OPENWIKI_SOURCE_DIR = PurePosixPath("/tmp/openwiki-source") _OPENWIKI_HOME_DIR = PurePosixPath("/tmp/openwiki-home") _REMOTE_PACKAGE_PATH = PurePosixPath("/tmp/openwiki-eval.tgz") +_REMOTE_WIKI_CACHE_PATH = PurePosixPath("/tmp/openwiki-wiki-cache.tgz") _OPENWIKI_LOG_PATH = PurePosixPath("/logs/agent/openwiki.log") +_WIKI_CACHE_SCHEMA = "openwiki-eval-wiki-v1" +_MAX_CACHE_MEMBERS = 20_000 +_MAX_CACHE_UNCOMPRESSED_BYTES = 1_073_741_824 +_MAX_CACHE_METADATA_BYTES = 65_536 + + +def _normalized_package_digest(package_path: Path) -> str: + """Hash package contents without unstable tar ownership or timestamps.""" + + digest = hashlib.sha256() + seen: set[str] = set() + with tarfile.open(package_path, mode="r:*") as archive: + members = sorted(archive.getmembers(), key=lambda member: member.name) + if len(members) > _MAX_CACHE_MEMBERS: + raise ValueError("OpenWiki package contains too many archive members") + total_size = 0 + for member in members: + path = PurePosixPath(member.name) + if path.is_absolute() or ".." in path.parts or member.name in seen: + raise ValueError("OpenWiki package contains an unsafe archive path") + seen.add(member.name) + total_size += member.size + if total_size > _MAX_CACHE_UNCOMPRESSED_BYTES: + raise ValueError("OpenWiki package is too large to hash safely") + digest.update(member.name.encode("utf-8", errors="surrogateescape")) + digest.update(b"\0") + digest.update(member.type) + digest.update(b"\0") + digest.update(str(member.mode & 0o777).encode("ascii")) + digest.update(b"\0") + if member.isfile(): + extracted = archive.extractfile(member) + if extracted is None: + raise ValueError("OpenWiki package member could not be read") + for chunk in iter(lambda: extracted.read(1024 * 1024), b""): + digest.update(chunk) + elif member.issym() or member.islnk(): + digest.update(member.linkname.encode("utf-8", errors="surrogateescape")) + elif not member.isdir(): + raise ValueError("OpenWiki package contains an unsupported member type") + digest.update(b"\0") + return digest.hexdigest() + + +def _wiki_cache_key(base_commit: str, package_digest: str, model: str) -> str: + if not _GIT_COMMIT_RE.fullmatch(base_commit): + raise ValueError("invalid base commit for OpenWiki cache") + payload = json.dumps( + { + "schema": _WIKI_CACHE_SCHEMA, + "base_commit": base_commit, + "package_digest": package_digest, + "model": model, + }, + sort_keys=True, + separators=(",", ":"), + ) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def _validate_wiki_cache_archive(archive_path: Path) -> None: + """Reject archives that could write anywhere except openwiki/.""" + + seen: set[str] = set() + quickstart_found = False + total_size = 0 + with tarfile.open(archive_path, mode="r:*") as archive: + members = archive.getmembers() + if not members or len(members) > _MAX_CACHE_MEMBERS: + raise ValueError("OpenWiki cache has an invalid member count") + for member in members: + path = PurePosixPath(member.name) + normalized = path.as_posix().rstrip("/") + if ( + path.is_absolute() + or not path.parts + or path.parts[0] != "openwiki" + or ".." in path.parts + or normalized in {"", "."} + or normalized in seen + ): + raise ValueError("OpenWiki cache contains an unsafe archive path") + seen.add(normalized) + if not (member.isfile() or member.isdir()): + raise ValueError("OpenWiki cache may contain only files and directories") + total_size += member.size + if total_size > _MAX_CACHE_UNCOMPRESSED_BYTES: + raise ValueError("OpenWiki cache is too large to restore safely") + if normalized == "openwiki/quickstart.md" and member.isfile(): + quickstart_found = True + if not quickstart_found: + raise ValueError("OpenWiki cache is missing openwiki/quickstart.md") + + +def _wiki_cache_metadata(archive_path: Path) -> dict[str, str]: + """Read schema-checked update metadata from an already validated cache.""" + + _validate_wiki_cache_archive(archive_path) + with tarfile.open(archive_path, mode="r:*") as archive: + try: + member = archive.getmember("openwiki/.last-update.json") + except KeyError as exc: + raise ValueError("OpenWiki cache is missing update metadata") from exc + if not member.isfile() or member.size > _MAX_CACHE_METADATA_BYTES: + raise ValueError("OpenWiki cache has invalid update metadata") + extracted = archive.extractfile(member) + if extracted is None: + raise ValueError("OpenWiki cache update metadata could not be read") + try: + value = json.loads(extracted.read(_MAX_CACHE_METADATA_BYTES + 1)) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError("OpenWiki cache update metadata is invalid JSON") from exc + if not isinstance(value, dict): + raise ValueError("OpenWiki cache update metadata must be an object") + git_head = value.get("gitHead") + model = value.get("model") + if not isinstance(git_head, str) or not _GIT_COMMIT_RE.fullmatch(git_head): + raise ValueError("OpenWiki cache update metadata has an invalid gitHead") + if not isinstance(model, str) or not _MODEL_ID_RE.fullmatch(model): + raise ValueError("OpenWiki cache update metadata has an invalid model") + return {"gitHead": git_head, "model": model} + + +def _find_wiki_cache( + cache_dir: Path, + exact_path: Path, + *, + base_commit: str, + model: str, + reuse_compatible: bool, +) -> tuple[Path | None, str | None]: + """Find an exact-key cache, or a metadata-compatible older package cache.""" + + candidates = [exact_path] + if reuse_compatible: + candidates.extend( + path for path in sorted(cache_dir.glob("*.tgz")) if path != exact_path + ) + for index, candidate in enumerate(candidates): + if not candidate.is_file(): + continue + metadata = _wiki_cache_metadata(candidate) + if metadata == {"gitHead": base_commit, "model": model}: + return candidate, "exact" if index == 0 else "compatible" + return None, None + + +def _bool_option(value: bool | str, name: str) -> bool: + if isinstance(value, bool): + return value + if value == "true": + return True + if value == "false": + return False + raise ValueError(f"{name} must be true or false") class BaselineCodex(Codex): @@ -45,6 +205,8 @@ async def install(self, environment: BaseEnvironment) -> None: "elif command -v apk >/dev/null 2>&1; then " "apk add --no-cache ripgrep; " "elif command -v apt-get >/dev/null 2>&1; then " + "for source in /etc/apt/sources.list.d/*nodesource*; do " + "[ ! -f \"$source\" ] || mv \"$source\" \"$source.disabled\"; done; " "apt-get update && apt-get install -y ripgrep; " "elif command -v yum >/dev/null 2>&1; then " "yum install -y ripgrep; " @@ -98,13 +260,17 @@ async def run( environment, command=( "umask 077; mkdir -p /logs/artifacts && " - f"git diff --binary {shlex.quote(start_head)} HEAD > " + f"git diff --binary {shlex.quote(start_head)} HEAD" + f"{self._patch_pathspec()} > " f"{shlex.quote(patch_path.as_posix())} && " f"chmod 0600 {shlex.quote(patch_path.as_posix())}" ), cwd=_APP_DIR.as_posix(), ) + def _patch_pathspec(self) -> str: + return "" + async def _exec( self, environment: BaseEnvironment, @@ -144,8 +310,11 @@ def __init__( *args: Any, openwiki_package: str, openwiki_model: str, + openwiki_cache_dir: str, openwiki_timeout_sec: int = 5400, retrieval_embedding_provider: str = "local", + reuse_compatible_wiki_cache: bool | str = True, + require_openwiki_cache: bool | str = False, **kwargs: Any, ) -> None: package_path = Path(openwiki_package).expanduser().resolve() @@ -157,11 +326,23 @@ def __init__( raise ValueError("openwiki_timeout_sec must be between 1 and 14400") if retrieval_embedding_provider not in {"local", "openai"}: raise ValueError("retrieval_embedding_provider must be local or openai") + cache_dir = Path(openwiki_cache_dir).expanduser().resolve() + cache_dir.mkdir(mode=0o700, parents=True, exist_ok=True) + if not cache_dir.is_dir(): + raise ValueError("openwiki_cache_dir must be a directory") self._openwiki_package = package_path + self._openwiki_package_digest = _normalized_package_digest(package_path) self._openwiki_model = openwiki_model + self._openwiki_cache_dir = cache_dir self._openwiki_timeout_sec = openwiki_timeout_sec self._retrieval_embedding_provider = retrieval_embedding_provider + self._reuse_compatible_wiki_cache = _bool_option( + reuse_compatible_wiki_cache, "reuse_compatible_wiki_cache" + ) + self._require_openwiki_cache = _bool_option( + require_openwiki_cache, "require_openwiki_cache" + ) super().__init__(*args, **kwargs) @staticmethod @@ -239,10 +420,13 @@ def _build_register_mcp_servers_command(self) -> str: "codex mcp add openwiki_retrieval -- " "/usr/local/bin/openwiki-retrieval-mcp " f"--repo-root {_APP_DIR.as_posix()} " - f"--wiki-root {(_OPENWIKI_SOURCE_DIR / 'openwiki').as_posix()} " + f"--wiki-root {(_APP_DIR / 'openwiki').as_posix()} " f"--embedding-provider {provider}" ) + def _patch_pathspec(self) -> str: + return " -- . ':(exclude)AGENTS.md' ':(exclude)openwiki/**'" + async def run( self, instruction: str, @@ -251,6 +435,9 @@ async def run( ) -> None: started = time.monotonic() status = "failure" + cache_hit = False + cache_match: str | None = None + cache_key: str | None = None try: await self.exec_as_agent( environment, @@ -263,36 +450,134 @@ async def run( timeout_sec=600, ) - trace_env = parent_env(self.context_id) - wiki_env = { - "HOME": _OPENWIKI_HOME_DIR.as_posix(), - "OPENAI_API_KEY": self._get_env("OPENAI_API_KEY") or "", - "LANGSMITH_API_KEY": self._get_env("LANGSMITH_API_KEY") or "", - "LANGCHAIN_TRACING_V2": "true", - "OPENWIKI_PROVIDER": "openai", - "OPENWIKI_MODEL_ID": self._openwiki_model, - "OPENWIKI_TELEMETRY_DISABLED": "1", - "DO_NOT_TRACK": "1", - **trace_env, - } - if project := trace_env.get("LANGSMITH_PROJECT"): - # OpenWiki currently uses the LangChain v2 tracing variable. - wiki_env["LANGCHAIN_PROJECT"] = project - for key in ("LANGSMITH_ENDPOINT", "LANGSMITH_WORKSPACE_ID"): - if value := self._get_env(key): - wiki_env[key] = value - if openai_base_url := self._get_env("OPENAI_BASE_URL"): - wiki_env["OPENAI_BASE_URL"] = openai_base_url + head_result = await self.exec_as_agent( + environment, + command="git rev-parse HEAD", + cwd=_OPENWIKI_SOURCE_DIR.as_posix(), + ) + base_commit = (head_result.stdout or "").strip() + cache_key = _wiki_cache_key( + base_commit, self._openwiki_package_digest, self._openwiki_model + ) + cache_path = (self._openwiki_cache_dir / f"{cache_key}.tgz").resolve() + if cache_path.parent != self._openwiki_cache_dir: + raise RuntimeError("OpenWiki cache path escaped its configured directory") + + selected_cache, cache_match = _find_wiki_cache( + self._openwiki_cache_dir, + cache_path, + base_commit=base_commit, + model=self._openwiki_model, + reuse_compatible=self._reuse_compatible_wiki_cache, + ) + if selected_cache is not None: + await environment.upload_file( + selected_cache, _REMOTE_WIKI_CACHE_PATH.as_posix() + ) + await self.exec_as_agent( + environment, + command=( + f"tar -xzf {shlex.quote(_REMOTE_WIKI_CACHE_PATH.as_posix())} " + f"-C {shlex.quote(_OPENWIKI_SOURCE_DIR.as_posix())} && " + f"test -f {shlex.quote((_OPENWIKI_SOURCE_DIR / 'openwiki' / 'quickstart.md').as_posix())} && " + f"test ! -L {shlex.quote((_OPENWIKI_SOURCE_DIR / 'openwiki' / 'quickstart.md').as_posix())}" + ), + timeout_sec=600, + ) + cache_hit = True + else: + if self._require_openwiki_cache: + raise RuntimeError( + "No compatible OpenWiki cache exists for the task commit " + "and model; generation is disabled" + ) + trace_env = parent_env(self.context_id) + wiki_env = { + "HOME": _OPENWIKI_HOME_DIR.as_posix(), + "OPENAI_API_KEY": self._get_env("OPENAI_API_KEY") or "", + "LANGSMITH_API_KEY": self._get_env("LANGSMITH_API_KEY") or "", + "LANGCHAIN_TRACING_V2": "true", + "OPENWIKI_PROVIDER": "openai", + "OPENWIKI_MODEL_ID": self._openwiki_model, + "OPENWIKI_TELEMETRY_DISABLED": "1", + "DO_NOT_TRACK": "1", + **trace_env, + } + if project := trace_env.get("LANGSMITH_PROJECT"): + wiki_env["LANGCHAIN_PROJECT"] = project + for key in ("LANGSMITH_ENDPOINT", "LANGSMITH_WORKSPACE_ID"): + if value := self._get_env(key): + wiki_env[key] = value + if openai_base_url := self._get_env("OPENAI_BASE_URL"): + wiki_env["OPENAI_BASE_URL"] = openai_base_url + await self.exec_as_agent( + environment, + command=( + "if [ -s ~/.nvm/nvm.sh ]; then . ~/.nvm/nvm.sh; fi; " + "openwiki code --init --print " + f"> {shlex.quote(_OPENWIKI_LOG_PATH.as_posix())} 2>&1" + ), + env=wiki_env, + cwd=_OPENWIKI_SOURCE_DIR.as_posix(), + timeout_sec=self._openwiki_timeout_sec, + ) + await self.exec_as_agent( + environment, + command=( + f"test -f {shlex.quote((_OPENWIKI_SOURCE_DIR / 'openwiki' / 'quickstart.md').as_posix())} && " + f"tar -czf {shlex.quote(_REMOTE_WIKI_CACHE_PATH.as_posix())} " + f"-C {shlex.quote(_OPENWIKI_SOURCE_DIR.as_posix())} openwiki" + ), + timeout_sec=600, + ) + temp_handle = tempfile.NamedTemporaryFile( + prefix=f".{cache_key}.", + suffix=".tmp", + dir=self._openwiki_cache_dir, + delete=False, + ) + temp_path = Path(temp_handle.name) + temp_handle.close() + try: + await environment.download_file( + _REMOTE_WIKI_CACHE_PATH.as_posix(), temp_path + ) + _validate_wiki_cache_archive(temp_path) + os.chmod(temp_path, 0o600) + os.replace(temp_path, cache_path) + finally: + temp_path.unlink(missing_ok=True) + await self.exec_as_agent( environment, command=( "if [ -s ~/.nvm/nvm.sh ]; then . ~/.nvm/nvm.sh; fi; " - "openwiki code --init --print " - f"> {shlex.quote(_OPENWIKI_LOG_PATH.as_posix())} 2>&1" + "node --input-type=module -e \"const root=process.argv[1]; " + "const repo=process.argv[2]; const module=await " + "import('file://' + root + '/openwiki/dist/code-mode.js'); " + "await module.ensureCodeModeRepoSetup(repo)\" " + '"$(npm root -g)" ' + f"{shlex.quote(_OPENWIKI_SOURCE_DIR.as_posix())}" ), - env=wiki_env, - cwd=_OPENWIKI_SOURCE_DIR.as_posix(), - timeout_sec=self._openwiki_timeout_sec, + timeout_sec=600, + ) + await self.exec_as_agent( + environment, + command=( + f"rm -rf {shlex.quote((_APP_DIR / 'openwiki').as_posix())} && " + f"cp -R {shlex.quote((_OPENWIKI_SOURCE_DIR / 'openwiki').as_posix())} " + f"{shlex.quote((_APP_DIR / 'openwiki').as_posix())} && " + f"cp {shlex.quote((_OPENWIKI_SOURCE_DIR / 'AGENTS.md').as_posix())} " + f"{shlex.quote((_APP_DIR / 'AGENTS.md').as_posix())} && " + "mkdir -p /logs/agent && " + f"cp {shlex.quote((_APP_DIR / 'AGENTS.md').as_posix())} " + "/logs/agent/openwiki-agents.md && " + "printf '\\n/AGENTS.md\\n/openwiki/\\n' >> .git/info/exclude && " + "git ls-files -z -- AGENTS.md openwiki | " + "git update-index --skip-worktree -z --stdin" + ), + cwd=_APP_DIR.as_posix(), + timeout_sec=600, ) status = "success" finally: @@ -301,6 +586,9 @@ async def run( "status": status, "duration_seconds": elapsed, "model": self._openwiki_model, + "cache_hit": cache_hit, + "cache_match": cache_match, + "cache_key": cache_key, "quickstart": ( _OPENWIKI_SOURCE_DIR / "openwiki" / "quickstart.md" ).as_posix(), @@ -314,16 +602,4 @@ async def run( encoding="utf-8", ) - quickstart_path = ( - _OPENWIKI_SOURCE_DIR / "openwiki" / "quickstart.md" - ).as_posix() - treatment_instruction = ( - "OpenWiki treatment condition: follow the OpenWiki instructions installed " - "in the repository's AGENTS.md. Start with the generated quickstart at " - f"{quickstart_path} and use the read-only openwiki_retrieval MCP tools as " - "needed. The wiki was generated from the same base checkout: treat /app as " - "the source of truth, make all code changes only in /app, and do not edit " - "/tmp/openwiki-source.\n\n" - f"{instruction}" - ) - await super().run(treatment_instruction, environment, context) + await super().run(instruction, environment, context) diff --git a/evals/deepswe/run.py b/evals/deepswe/run.py index 4bae42d9..0fd38dfb 100644 --- a/evals/deepswe/run.py +++ b/evals/deepswe/run.py @@ -51,6 +51,7 @@ DEFAULT_ARTIFACTS_DIR = EVAL_DIR / "artifacts" DEFAULT_JOBS_DIR = EVAL_DIR / "results" DEFAULT_SUMMARY_DIR = EVAL_DIR / "summaries" +DEFAULT_OPENWIKI_CACHE_DIR = EVAL_DIR / ".cache" / "openwiki-wikis" KOOTA_5_TASKS = ( "koota-composite-trait-aspects", @@ -76,8 +77,21 @@ "kgateway-consistent-hash-policy", "python-statemachine-state-data-scoping", ) +DOC_LEVERAGE_10_TASKS = ( + "aiomonitor-task-snapshots-diff", + "bandit-incremental-cache-control", + "dynamodb-toolbox-conditional-attribute-requirements", + "fastapi-deprecation-response-headers", + "go-genai-streamed-function-args", + "goreleaser-retry-publish-auditing", + "gql-incremental-graphql-delivery", + "igel-persist-feature-schema", + "onedump-dump-encryption-pipeline", + "testem-bail-on-test-failure", +) TASK_SUITES = { "koota-5": KOOTA_5_TASKS, + "openwiki-doc-leverage-10": DOC_LEVERAGE_10_TASKS, "openwiki-20": (*KOOTA_5_TASKS, *WIKI_STRESS_15_TASKS), } @@ -452,6 +466,8 @@ def harbor_args( str(args.attempts), "--n-concurrent", str(args.concurrency), + "--agent-setup-timeout-multiplier", + str(args.agent_setup_timeout_multiplier), "--n-tasks", str(len(selected_tasks) if selected_tasks is not None else args.n_tasks), "--plugin", @@ -476,11 +492,18 @@ def harbor_args( "--agent-kwarg", f"openwiki_package={package_path.resolve()}", "--agent-kwarg", + f"openwiki_cache_dir={args.openwiki_cache_dir.resolve()}", + "--agent-kwarg", f"openwiki_model={args.openwiki_model}", "--agent-kwarg", f"openwiki_timeout_sec={args.openwiki_timeout}", "--agent-kwarg", f"retrieval_embedding_provider={args.retrieval_embedding_provider}", + "--agent-kwarg", + "reuse_compatible_wiki_cache=" + f"{str(args.reuse_compatible_wiki_cache).lower()}", + "--agent-kwarg", + f"require_openwiki_cache={str(args.require_openwiki_cache).lower()}", ] ) return command @@ -772,7 +795,33 @@ def add_common_options(parser: argparse.ArgumentParser) -> None: parser.add_argument("--seed", type=int, default=0) parser.add_argument("--attempts", type=int, default=1) parser.add_argument("--concurrency", type=int, default=1) + parser.add_argument( + "--agent-setup-timeout-multiplier", + type=float, + default=3.0, + help="Multiplier for Harbor's agent setup deadline (default: 3.0)", + ) parser.add_argument("--openwiki-timeout", type=int, default=5400) + parser.add_argument( + "--openwiki-cache-dir", + type=Path, + default=DEFAULT_OPENWIKI_CACHE_DIR, + help="Persistent host cache for generated task wikis", + ) + parser.add_argument( + "--reuse-compatible-wiki-cache", + action=argparse.BooleanOptionalAction, + default=True, + help=( + "Reuse a cache whose recorded task commit and model match even when " + "the packaged OpenWiki implementation changed" + ), + ) + parser.add_argument( + "--require-openwiki-cache", + action="store_true", + help="Fail before wiki generation when no compatible cache exists", + ) parser.add_argument( "--retrieval-embedding-provider", choices=("local", "openai"), @@ -799,8 +848,16 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: subparser = subparsers.add_parser(name) add_common_options(subparser) args = parser.parse_args(argv) - if args.n_tasks <= 0 or args.attempts <= 0 or args.concurrency <= 0: - parser.error("--n-tasks, --attempts, and --concurrency must be positive") + if ( + args.n_tasks <= 0 + or args.attempts <= 0 + or args.concurrency <= 0 + or args.agent_setup_timeout_multiplier <= 0 + ): + parser.error( + "--n-tasks, --attempts, --concurrency, and " + "--agent-setup-timeout-multiplier must be positive" + ) if args.task_suite and args.task: parser.error("--task-suite cannot be combined with --task") return args diff --git a/evals/deepswe/test_analyze_openwiki_usage.py b/evals/deepswe/test_analyze_openwiki_usage.py new file mode 100644 index 00000000..e8cca6f5 --- /dev/null +++ b/evals/deepswe/test_analyze_openwiki_usage.py @@ -0,0 +1,243 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from evals.deepswe import analyze_openwiki_usage as analyzer + + +def event(event_type: str, item: dict) -> str: + return json.dumps({"type": event_type, "item": item}) + + +class AnalyzeOpenWikiUsageTests(unittest.TestCase): + def test_canonical_call_json_and_filesystem_classification(self) -> None: + rendered = analyzer.canonical_call_json("tool", {"z": 1, "a": "two"}) + self.assertEqual('{"args":{"a":"two","z":1},"name":"tool"}', rendered) + self.assertTrue( + analyzer.is_openwiki_filesystem_read( + "sed -n '1,80p' /tmp/openwiki-source/openwiki/quickstart.md" + ) + ) + self.assertTrue(analyzer.is_openwiki_filesystem_read("rg foo openwiki/")) + self.assertFalse(analyzer.is_openwiki_filesystem_read("rm -rf openwiki/")) + self.assertFalse(analyzer.is_openwiki_filesystem_read("rg openwiki src/")) + + def test_analyze_trace_pairs_events_and_counts_direct_overhead(self) -> None: + mcp_started = { + "id": "mcp-1", + "type": "mcp_tool_call", + "server": "openwiki_retrieval", + "tool": "change_surface", + "arguments": {"query": "routing"}, + } + mcp_completed = { + **mcp_started, + "result": { + "content": [ + {"type": "text", "text": "first"}, + {"type": "text", "text": "second"}, + ] + }, + } + shell_started = { + "id": "shell-1", + "type": "command_execution", + "command": "cat openwiki/quickstart.md", + } + shell_completed = { + **shell_started, + "aggregated_output": "wiki text\n", + } + unrelated_started = { + "id": "shell-2", + "type": "command_execution", + "command": "rg routing src tests", + } + unrelated_completed = {**unrelated_started, "aggregated_output": "match\n"} + other_mcp = { + "id": "mcp-2", + "type": "mcp_tool_call", + "server": "other_server", + "tool": "search", + "arguments": {}, + } + file_change = { + "id": "edit-1", + "type": "file_change", + "changes": [{"path": "/app/module.py", "kind": "update"}], + } + todo_list = { + "id": "todo-1", + "type": "todo_list", + "items": [], + } + + lines = [ + "not json", + event("item.started", mcp_started), + event("item.completed", mcp_completed), + event("item.started", shell_started), + event("item.completed", shell_completed), + event("item.started", unrelated_started), + event("item.completed", unrelated_completed), + event("item.started", other_mcp), + event("item.completed", {**other_mcp, "result": "ignored"}), + event("item.started", file_change), + event("item.completed", file_change), + event("item.started", todo_list), + event("item.completed", todo_list), + json.dumps( + { + "type": "turn.completed", + "usage": { + "input_tokens": 1000, + "cached_input_tokens": 600, + "output_tokens": 200, + }, + } + ), + ] + with tempfile.TemporaryDirectory() as temp_dir: + trace = Path(temp_dir) / "codex.txt" + trace.write_text("\n".join(lines), encoding="utf-8") + (trace.parent / "openwiki-agents.md").write_text( + "prefix\nwiki guidance" + "\nsuffix\n", + encoding="utf-8", + ) + result = analyzer.analyze_trace(trace, "owner/task") + + self.assertIsNotNone(result) + assert result is not None + self.assertEqual(6, result.total_tool_calls) + self.assertEqual(1, result.openwiki_mcp_calls) + self.assertEqual(1, result.openwiki_filesystem_calls) + expected_call_chars = len( + analyzer.canonical_call_json( + "openwiki_retrieval.change_surface", {"query": "routing"} + ) + ) + len( + analyzer.canonical_call_json( + "command_execution", {"command": "cat openwiki/quickstart.md"} + ) + ) + self.assertEqual(expected_call_chars, result.call_json_chars) + self.assertEqual(len("firstsecondwiki text\n"), result.result_chars) + self.assertGreater(result.agents_prompt_chars, 0) + self.assertEqual(400, result.usage.uncached_input_tokens) + + def test_collect_trials_includes_complete_usage_even_after_verifier_failure( + self, + ) -> None: + def write_trial(root: Path, name: str, *, valid: bool, complete: bool) -> None: + trial = root / name + (trial / "agent").mkdir(parents=True) + result = { + "task_name": f"owner/{name}", + "exception_info": None if valid else {"message": "failed"}, + "verifier_result": {"rewards": {"reward": 1}}, + } + (trial / "result.json").write_text(json.dumps(result), encoding="utf-8") + trace_lines = [] + if complete: + trace_lines.append( + json.dumps( + { + "type": "turn.completed", + "usage": { + "input_tokens": 20, + "cached_input_tokens": 10, + "output_tokens": 5, + }, + } + ) + ) + (trial / "agent" / "codex.txt").write_text( + "\n".join(trace_lines), encoding="utf-8" + ) + + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + write_trial(root, "valid", valid=True, complete=True) + write_trial(root, "failed", valid=False, complete=True) + write_trial(root, "incomplete", valid=True, complete=False) + trials = analyzer.collect_trials([root]) + + self.assertEqual( + ["owner/failed", "owner/valid"], + [trial.task_name for trial in trials], + ) + + def test_analyze_trace_scopes_reused_item_ids_to_each_completion(self) -> None: + reused_id = "item_1" + mcp_started = { + "id": reused_id, + "type": "mcp_tool_call", + "server": "openwiki_retrieval", + "tool": "change_surface", + "arguments": {"query": "ownership"}, + } + shell_started = { + "id": reused_id, + "type": "command_execution", + "command": "sed -n '1,40p' openwiki/quickstart.md", + } + lines = [ + event("item.started", mcp_started), + event( + "item.completed", + {**mcp_started, "result": {"content": [{"text": "brief"}]}}, + ), + event("item.started", shell_started), + event( + "item.completed", + {**shell_started, "aggregated_output": "quickstart"}, + ), + json.dumps( + { + "type": "turn.completed", + "usage": { + "input_tokens": 100, + "cached_input_tokens": 50, + "output_tokens": 25, + }, + } + ), + ] + + with tempfile.TemporaryDirectory() as temp_dir: + trace = Path(temp_dir) / "codex.txt" + trace.write_text("\n".join(lines), encoding="utf-8") + result = analyzer.analyze_trace(trace, "owner/task") + + self.assertIsNotNone(result) + assert result is not None + self.assertEqual(2, result.total_tool_calls) + self.assertEqual(1, result.openwiki_mcp_calls) + self.assertEqual(1, result.openwiki_filesystem_calls) + self.assertEqual(len("briefquickstart"), result.result_chars) + + def test_summary_subtracts_calls_and_estimated_tokens(self) -> None: + trial = analyzer.TrialMetrics( + task_name="owner/task", + trial_dir="/trial", + total_tool_calls=10, + openwiki_mcp_calls=2, + openwiki_filesystem_calls=1, + call_json_chars=40, + result_chars=80, + agents_prompt_chars=20, + usage=analyzer.Usage(1000, 600, 200), + ) + summary = analyzer.summarize([trial]) + + self.assertEqual(35, summary["openwiki_token_overhead"]["total_tokens"]) + self.assertEqual(7, summary["tool_calls"]["adjusted_without_openwiki"]) + self.assertEqual(975, summary["adjusted_usage"]["input_tokens"]) + self.assertEqual(190, summary["adjusted_usage"]["output_tokens"]) + self.assertEqual(1165, summary["adjusted_usage"]["total_tokens"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/evals/deepswe/test_run.py b/evals/deepswe/test_run.py index 2ff854d2..ad92a9b4 100644 --- a/evals/deepswe/test_run.py +++ b/evals/deepswe/test_run.py @@ -1,7 +1,9 @@ from __future__ import annotations import json +import io import os +import tarfile import tempfile import unittest from pathlib import Path @@ -11,6 +13,7 @@ from requests import HTTPError, Response import deepswe_langsmith +import openwiki_codex import run as deepswe_run @@ -71,6 +74,16 @@ def test_openwiki_install_rebuilds_only_native_sqlite_dependency(self) -> None: self.assertIn("npm rebuild better-sqlite3", adapter) self.assertIn("require('better-sqlite3')", adapter) + def test_agent_install_ignores_broken_nodesource_apt_repository(self) -> None: + adapter = (deepswe_run.EVAL_DIR / "openwiki_codex.py").read_text( + encoding="utf-8" + ) + disable_index = adapter.index("/etc/apt/sources.list.d/*nodesource*") + update_index = adapter.index("apt-get update") + + self.assertLess(disable_index, update_index) + self.assertIn('$source.disabled', adapter) + def test_adapter_captures_committed_patch_for_separate_verifier(self) -> None: adapter = (deepswe_run.EVAL_DIR / "openwiki_codex.py").read_text( encoding="utf-8" @@ -85,14 +98,15 @@ def test_openwiki_treatment_uses_just_in_time_navigation(self) -> None: adapter = (deepswe_run.EVAL_DIR / "openwiki_codex.py").read_text( encoding="utf-8" ) - self.assertIn("generated quickstart", adapter) - self.assertIn("follow the OpenWiki instructions", adapter) self.assertIn("codex mcp add openwiki_retrieval", adapter) - self.assertIn("read-only openwiki_retrieval MCP tools", adapter) - self.assertIn("treat /app as", adapter) - self.assertIn("the source of truth", adapter) - self.assertIn("do not edit", adapter) - self.assertIn("/tmp/openwiki-source", adapter) + self.assertIn("--wiki-root", adapter) + self.assertIn("(_APP_DIR / 'openwiki').as_posix()", adapter) + self.assertIn("ensureCodeModeRepoSetup", adapter) + self.assertIn("/logs/agent/openwiki-agents.md", adapter) + self.assertIn("await super().run(instruction, environment, context)", adapter) + self.assertNotIn("treatment_instruction", adapter) + self.assertIn("':(exclude)AGENTS.md'", adapter) + self.assertIn("':(exclude)openwiki/**'", adapter) def test_eval_defaults_use_terra_without_changing_openwiki_defaults(self) -> None: args = deepswe_run.parse_args(["paired"]) @@ -124,6 +138,7 @@ def test_paired_commands_share_selection_and_agent_settings(self) -> None: "--env", "--n-attempts", "--n-concurrent", + "--agent-setup-timeout-multiplier", "--n-tasks", "--include-task-name", ): @@ -133,11 +148,20 @@ def test_paired_commands_share_selection_and_agent_settings(self) -> None: self.assertIn("openwiki_codex:BaselineCodex", baseline) self.assertIn("openwiki_codex:OpenWikiCodex", treatment) self.assertEqual(2, baseline.count("--agent-kwarg")) - self.assertEqual(6, treatment.count("--agent-kwarg")) + self.assertEqual(9, treatment.count("--agent-kwarg")) self.assertIn("retrieval_embedding_provider=local", treatment) + self.assertIn( + f"openwiki_cache_dir={args.openwiki_cache_dir.resolve()}", treatment + ) + self.assertIn("reuse_compatible_wiki_cache=true", treatment) + self.assertIn("require_openwiki_cache=false", treatment) self.assertIn(f"version={deepswe_run.CODEX_VERSION}", baseline) self.assertIn("gateway.smith.langchain.com", baseline) self.assertIn("api.smith.langchain.com", baseline) + self.assertEqual( + "3.0", + baseline[baseline.index("--agent-setup-timeout-multiplier") + 1], + ) self.assertEqual(1, baseline.count("--plugin")) self.assertEqual( "deepswe_langsmith:DeepSWELangSmithPlugin", @@ -161,6 +185,121 @@ def test_paired_commands_share_selection_and_agent_settings(self) -> None: self.assertIn(host, baseline) self.assertIn(host, treatment) + def test_custom_agent_setup_timeout_multiplier_is_forwarded(self) -> None: + args = deepswe_run.parse_args( + ["baseline", "--agent-setup-timeout-multiplier", "4.5"] + ) + command = deepswe_run.harbor_args(args, condition="baseline") + + self.assertEqual( + "4.5", + command[command.index("--agent-setup-timeout-multiplier") + 1], + ) + + def test_openwiki_cache_key_is_stable_and_configuration_sensitive(self) -> None: + commit = "a" * 40 + first = openwiki_codex._wiki_cache_key(commit, "b" * 64, "model-a") + self.assertEqual( + first, openwiki_codex._wiki_cache_key(commit, "b" * 64, "model-a") + ) + self.assertNotEqual( + first, openwiki_codex._wiki_cache_key(commit, "b" * 64, "model-b") + ) + + def test_wiki_cache_archive_accepts_only_contained_regular_content(self) -> None: + def write_archive(path: Path, entries: list[tuple[str, bytes, str]]) -> None: + with tarfile.open(path, mode="w:gz") as archive: + for name, content, kind in entries: + info = tarfile.TarInfo(name) + if kind == "file": + info.size = len(content) + archive.addfile(info, io.BytesIO(content)) + elif kind == "dir": + info.type = tarfile.DIRTYPE + archive.addfile(info) + else: + info.type = tarfile.SYMTYPE + info.linkname = "../outside" + archive.addfile(info) + + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + valid = root / "valid.tgz" + write_archive( + valid, + [ + ("openwiki", b"", "dir"), + ("openwiki/quickstart.md", b"# Quickstart\n", "file"), + ], + ) + openwiki_codex._validate_wiki_cache_archive(valid) + + invalid_entries = { + "absolute": [("/openwiki/quickstart.md", b"x", "file")], + "traversal": [("openwiki/../outside", b"x", "file")], + "outside": [("outside.txt", b"x", "file")], + "symlink": [ + ("openwiki/quickstart.md", b"", "symlink"), + ], + "missing": [("openwiki/overview.md", b"x", "file")], + } + for label, entries in invalid_entries.items(): + archive_path = root / f"{label}.tgz" + write_archive(archive_path, entries) + with self.subTest(label=label), self.assertRaises(ValueError): + openwiki_codex._validate_wiki_cache_archive(archive_path) + + def test_compatible_cache_requires_exact_commit_and_model_metadata(self) -> None: + def write_cache(path: Path, commit: str, model: str) -> None: + metadata = json.dumps({"gitHead": commit, "model": model}).encode() + with tarfile.open(path, mode="w:gz") as archive: + for name, content in ( + ("openwiki/quickstart.md", b"# Quickstart\n"), + ("openwiki/.last-update.json", metadata), + ): + info = tarfile.TarInfo(name) + info.size = len(content) + archive.addfile(info, io.BytesIO(content)) + + with tempfile.TemporaryDirectory() as temp_dir: + cache_dir = Path(temp_dir) + compatible = cache_dir / "old-package.tgz" + exact = cache_dir / "new-package.tgz" + commit = "a" * 40 + write_cache(compatible, commit, "model-a") + + selected, match = openwiki_codex._find_wiki_cache( + cache_dir, + exact, + base_commit=commit, + model="model-a", + reuse_compatible=True, + ) + self.assertEqual(compatible, selected) + self.assertEqual("compatible", match) + + selected, match = openwiki_codex._find_wiki_cache( + cache_dir, + exact, + base_commit=commit, + model="model-b", + reuse_compatible=True, + ) + self.assertIsNone(selected) + self.assertIsNone(match) + + def test_cache_only_cli_flags_are_forwarded(self) -> None: + args = deepswe_run.parse_args( + ["openwiki", "--require-openwiki-cache", "--dry-run"] + ) + command = deepswe_run.harbor_args( + args, + condition="openwiki", + package_path=args.artifacts_dir / "openwiki-eval.tgz", + ) + self.assertIn("reuse_compatible_wiki_cache=true", command) + self.assertIn("require_openwiki_cache=true", command) + def test_custom_allowed_host_is_validated_and_included(self) -> None: args = deepswe_run.parse_args( ["baseline", "--allow-host", "Gateway.Example.com"] @@ -384,13 +523,20 @@ def test_seeded_task_selection_is_reproducible(self) -> None: def test_named_task_suites_are_independent_and_composable(self) -> None: koota = deepswe_run.TASK_SUITES["koota-5"] stress = deepswe_run.WIKI_STRESS_15_TASKS + doc_leverage = deepswe_run.TASK_SUITES["openwiki-doc-leverage-10"] combined = deepswe_run.TASK_SUITES["openwiki-20"] - self.assertEqual({"koota-5", "openwiki-20"}, set(deepswe_run.TASK_SUITES)) + self.assertEqual( + {"koota-5", "openwiki-20", "openwiki-doc-leverage-10"}, + set(deepswe_run.TASK_SUITES), + ) self.assertEqual(5, len(koota)) self.assertEqual(15, len(stress)) + self.assertEqual(10, len(doc_leverage)) self.assertEqual(20, len(combined)) self.assertTrue(set(koota).isdisjoint(stress)) + self.assertTrue(set(doc_leverage).isdisjoint(combined)) + self.assertEqual(len(doc_leverage), len(set(doc_leverage))) self.assertEqual((*koota, *stress), combined) def test_named_task_suite_selects_every_exact_member(self) -> None: diff --git a/src/agent/prompt.ts b/src/agent/prompt.ts index 382f413a..65ac1738 100644 --- a/src/agent/prompt.ts +++ b/src/agent/prompt.ts @@ -170,6 +170,21 @@ timestamp: - Produce valid YAML. Do not leave placeholder text or explanatory comments in written files. - Preserve all existing producer-defined front matter fields when updating a concept. Unknown extension fields are valid OKF and must survive round trips. Change metadata only when the underlying fact or meaningful content changes. - The description field is especially useful for retrieval tools. When present, make it clear, detailed, and optimized for search. +- In repository mode, use the optional namespaced \`openwiki\` producer extension when source evidence supports it. Keep values concise and omit empty keys: + + +openwiki: + roles: [architecture, domain] # One or more of architecture, delivery, domain, integration, operations, repository, testing, workflow + change_kinds: [lifecycle, public-api] # Short kebab-case routing facets + source_paths: [path/to/canonical-source.ts] + symbols: [PublicSymbol, owningInternalSymbol] + test_paths: [path/to/focused.test.ts] + invariants: [A concise externally observable contract.] + validation_commands: [the narrowest non-destructive check] + + +- Use \`type\` as a free-form human concept kind. Use \`openwiki.roles\` for stable retrieval roles and \`tags\` for specific domain facets; do not use generic shared tags as a substitute for explicit concept links. +- Treat \`source_paths\`, \`test_paths\`, invariants, and validation commands as evidence-backed routing metadata, not exhaustive requirements. Never place secrets, credentials, or commands that expose them in metadata. - When updating an existing Markdown concept, preserve accurate body content and correct its opening front matter only when needed for compliance or accuracy. - OpenWiki repairs front matter deterministically after every run, so a page is never rejected for missing or invalid front matter. If a page's front matter contains \`openwiki_generated: true\`, that metadata was code-derived as a fallback: replace it with an accurate \`type\`, \`title\`, and \`description\` grounded in the page body, then remove the \`openwiki_generated\` field. diff --git a/src/code-mode.ts b/src/code-mode.ts index 79f60425..092b7ffb 100644 --- a/src/code-mode.ts +++ b/src/code-mode.ts @@ -126,19 +126,13 @@ function createCodeModeAgentsSnippet(): string { ## OpenWiki -This repository uses OpenWiki for recurring code documentation. Use \`openwiki/\` as a just-in-time repository index: - -- At task start, read \`openwiki/quickstart.md\`, then search the wiki for the task's concepts and read only the relevant linked sections. -- Before a repository-wide \`rg\`, \`find\`, or exploratory directory scan, check the wiki's source maps. When they name relevant files, symbols, or tests, inspect those paths directly. -- Re-consult the wiki when entering a different subsystem, when source evidence contradicts the current understanding, or when blocked by an unfamiliar test or build failure. Do not reread content already returned by retrieval unless surrounding context is needed. -- Treat source code and tests as authoritative. Verify wiki claims in source before editing. -- Before finishing a public API or cross-package change, trace it from implementation through barrel/package exports, generated or publish mirrors, initialization or registration, and the import path consumers actually use. Consult the relevant wiki integration or delivery guidance and run the narrowest consumer-facing check; internal unit tests alone do not prove the shipped surface works. -- If an \`openwiki_retrieval\` MCP server is available, use \`search\` for focused retrieval. Use \`change_surface\` before public, cross-package, generated-artifact, or runtime-registration edits. After changing public symbols, call \`trace_symbols\` once with all of them and treat missing groups as verification gaps, not automatic requirements. -- Before editing stateful or lifecycle behavior, write a compact state model: list every identity axis that partitions state (such as tracker/factory, entity, relation or predicate, and target), define the relevant transitions and observation/reset window, and choose one canonical owner or ledger for those transitions. Turn each externally observable criterion into an input/event/expected-result oracle row. Cover relevant initial state, false-to-true and true-to-false transitions, unchanged updates, missing dependencies, independent instances, reset/reuse, deferred or re-entrant net effects, and static-plus-temporal composition; map every row to a passing focused test before finishing. -- When behavior is unfamiliar, relevant tests are large, or no analogous focused check is known, call \`search\` with the \`tests\` scope using that behavior matrix, then inspect the cited tests directly. Skip this search when the exact focused test is already known. -- When the repository generates or copies package artifacts, identify the canonical source and repository-supported synchronization command. Do not hand-edit derived output unless the documented workflow explicitly requires it. -- Do not read operations, release, or integration pages unless the task affects those areas. If a requested feature does not exist yet, use the wiki to locate extension points, then inspect source rather than searching the wiki for the implementation. -- Prefer the narrowest quiet validation command available. Suppress successful build/test noise when possible, but preserve complete failure output. +This repository has a generated \`openwiki/\` evidence index. It is optional just-in-time context, not required startup reading. + +- If implementation ownership, behavioral invariants, analogous tests, or shipped surfaces are unclear, call \`openwiki_retrieval.change_surface\` once with the task before broad exploration. Inspect its cited source and tests directly; do not reread the returned wiki pages. +- Use \`openwiki_retrieval.search\` only for a concrete unresolved evidence gap. Reconsult when source contradicts the brief, work enters an uncited subsystem, or an unfamiliar failure reveals a missing contract. +- Treat source code and tests as authoritative. A brief's unknowns and review items are verification gaps, not automatic requirements. +- Before finishing a public or cross-package change, call \`change_surface\` with the task and the repository-relative changed paths. Verify relevant flagged exports, registration, generated surfaces, consumer paths, and focused tests. +- Prefer the narrowest quiet validation that proves the changed behavior. Preserve complete failure output. The scheduled OpenWiki GitHub Actions workflow refreshes the repository wiki. Do not hand-edit generated OpenWiki pages unless explicitly asked; prefer updating source code/docs and letting OpenWiki regenerate. diff --git a/src/retrieval/mcp-server.ts b/src/retrieval/mcp-server.ts index ed546df6..2e504dca 100644 --- a/src/retrieval/mcp-server.ts +++ b/src/retrieval/mcp-server.ts @@ -41,7 +41,7 @@ async function handleLine(line: string): Promise { writeResult(request.id, { capabilities: { tools: { listChanged: false } }, instructions: - "Use search for focused wiki, source_code, or tests retrieval. Use change_surface before public or cross-package edits, and trace_symbols once after changing public symbols. Verify citations in source. All tools are read-only and return bounded excerpts.", + "Use change_surface when ownership, invariants, analogous tests, or shipped surfaces are unclear. Inspect its citations directly instead of rereading wiki pages. Reuse it with changed_paths for a final cross-surface review. Use search only for a concrete unresolved evidence gap.", protocolVersion: "2025-06-18", serverInfo: { name: "openwiki-retrieval", version: OPENWIKI_VERSION }, }); @@ -88,12 +88,7 @@ async function callTool( result = await service.changeSurface( requiredString(args.query, "query"), optionalInteger(args.limit, 6), - ); - break; - case "trace_symbols": - result = await service.traceSymbols( - requiredStrings(args.symbols, "symbols"), - optionalInteger(args.limit, 4), + optionalStringArray(args.changed_paths, "changed_paths"), ); break; default: @@ -138,20 +133,6 @@ function requiredString(value: unknown, name: string): string { return value; } -function requiredStrings(value: unknown, name: string): string[] { - if (!Array.isArray(value) || value.length === 0) { - throw new Error(`${name} must be a non-empty string array.`); - } - const strings: string[] = []; - for (const item of value) { - if (typeof item !== "string" || !item.trim()) { - throw new Error(`${name} must be a non-empty string array.`); - } - strings.push(item); - } - return strings; -} - function optionalInteger(value: unknown, fallback: number): number { return typeof value === "number" && Number.isInteger(value) ? value @@ -167,6 +148,21 @@ function optionalScope(value: unknown): SearchScope { : "all"; } +function optionalStringArray(value: unknown, name: string): string[] { + if (value === undefined) return []; + if (!Array.isArray(value)) { + throw new Error(`${name} must be an array of strings.`); + } + const result: string[] = []; + for (const item of value as unknown[]) { + if (typeof item !== "string") { + throw new Error(`${name} must be an array of strings.`); + } + result.push(item); + } + return result; +} + function isRecord(value: unknown): value is Record { return value !== null && typeof value === "object" && !Array.isArray(value); } diff --git a/src/retrieval/mcp-tools.ts b/src/retrieval/mcp-tools.ts index 590f4f82..641dfa39 100644 --- a/src/retrieval/mcp-tools.ts +++ b/src/retrieval/mcp-tools.ts @@ -43,7 +43,7 @@ function tool( export const RETRIEVAL_TOOL_DEFINITIONS = [ tool( "search", - "Search the wiki, implementation code, or tests with automatic exact, lexical, semantic, and OKF ranking. Use the tests scope only when analogous behavior is needed, and inspect cited source before relying on it.", + "Retrieve focused wiki guidance, implementation evidence, or analogous tests with automatic lexical, semantic, and OKF ranking. Use wiki for contracts and invariants; verify every citation in source.", querySchema({ limit: integerSchema(1, 10, 5), scope: { @@ -57,25 +57,16 @@ export const RETRIEVAL_TOOL_DEFINITIONS = [ ), tool( "change_surface", - "Map a public, stateful, or cross-package change before editing. Returns compact citations for implementation, state-transition producers, exports, publish mirrors, initialization, consumers, and tests.", - querySchema({ limit: integerSchema(1, 12, 7) }), - ), - tool( - "trace_symbols", - "After editing public symbols, re-index once and verify them together across implementation, exports, generated/publish mirrors, initialization, consumers, and tests. Missing groups are verification gaps, not automatic requirements.", - { - additionalProperties: false, - properties: { - limit: integerSchema(1, 6, 4), - symbols: { - items: { maxLength: 200, minLength: 1, type: "string" }, - maxItems: 12, - minItems: 1, - type: "array", - }, + "Build a compact, evidence-backed task brief before broad exploration: likely owners, invariants, analogous tests, conditional delivery surfaces, and narrow validation. Pass changed_paths later to review documented adjacent surfaces.", + querySchema({ + changed_paths: { + description: + "Optional repository-relative paths already changed. When present, the response flags documented adjacent surfaces to verify; flags are evidence gaps, not automatic requirements.", + items: { maxLength: 300, minLength: 1, type: "string" }, + maxItems: 50, + type: "array", }, - required: ["symbols"], - type: "object", - }, + limit: integerSchema(1, 8, 6), + }), ), ] as const satisfies readonly ToolDefinition[]; diff --git a/src/retrieval/ranking.ts b/src/retrieval/ranking.ts index 2c5648af..edf7e175 100644 --- a/src/retrieval/ranking.ts +++ b/src/retrieval/ranking.ts @@ -59,12 +59,13 @@ const SYNONYMS = buildSynonyms(); export function tokenize(value: string): string[] { const separated = value + .replace(/[_-]+/gu, " ") .replace(/([a-z0-9])([A-Z])/gu, "$1 $2") .replace(/([A-Z]+)([A-Z][a-z])/gu, "$1 $2") .toLowerCase(); - const terms = separated.match(/[a-z0-9][a-z0-9_-]*/gu) ?? []; + const terms = separated.match(/[a-z0-9]+/gu) ?? []; return terms - .map((term) => stem(term.replace(/[_-]+/gu, ""))) + .map((term) => stem(term)) .filter((term) => term.length > 1 && !STOP_WORDS.has(term)); } @@ -198,8 +199,11 @@ export function searchableText(chunk: IndexedChunk): string { chunk.path, chunk.title, chunk.heading, + chunk.description, chunk.type, + chunk.roles.join(" "), chunk.tags.join(" "), + chunk.resource, chunk.fields, chunk.text, ] diff --git a/src/retrieval/repository-index.ts b/src/retrieval/repository-index.ts index f740454b..a688c1e2 100644 --- a/src/retrieval/repository-index.ts +++ b/src/retrieval/repository-index.ts @@ -5,9 +5,11 @@ import { splitFrontmatter, } from "../okf/frontmatter.js"; import type { + DocumentRole, IndexedChunk, OkfConcept, OkfRelationship, + OpenWikiMetadata, RepositoryCorpus, } from "./types.js"; @@ -61,6 +63,17 @@ const SECRET_FILE = const MARKDOWN_LINK = /\[([^\]]+)\]\(([^)]+)\)/gu; const TEST_NAME = /\b(?:describe|it|test)(?:\.(?:each|only|skip|todo))?\s*\(\s*(["'`])([^\n]{1,160}?)\1/gu; +const DOCUMENT_ROLES = new Set([ + "architecture", + "delivery", + "domain", + "integration", + "operations", + "reference", + "repository", + "testing", + "workflow", +]); export interface RepositoryIndexOptions { repoRoot: string; @@ -103,6 +116,8 @@ async function readWikiPages(wikiRoot: string): Promise { const type = stringField(fields.type) ?? "Reference"; const resource = stringField(fields.resource); const tags = stringArray(fields.tags); + const metadata = parseOpenWikiMetadata(fields.openwiki); + const roles = inferDocumentRoles(type, tags, relative, metadata.roles); return { chunks: chunkWikiPage({ body, @@ -110,6 +125,8 @@ async function readWikiPages(wikiRoot: string): Promise { description, fields, relative, + resource, + roles, tags, title, type, @@ -117,9 +134,11 @@ async function readWikiPages(wikiRoot: string): Promise { concept: { ...(description ? { description } : {}), incoming: new Set(), + metadata: { ...metadata, roles }, path: conceptPath, relationships: extractRelationships(body, relative), ...(resource ? { resource } : {}), + roles, tags, title, type, @@ -161,6 +180,7 @@ async function readSourceChunks( lineEnd, lineStart, path: relative, + roles: [], scope: "source_code", tags: pathTags(relative), ...(testNames.length > 0 ? { testNames } : {}), @@ -178,6 +198,8 @@ function chunkWikiPage(input: { description?: string; fields: Record; relative: string; + resource?: string; + roles: DocumentRole[]; tags: string[]; title: string; type: string; @@ -193,6 +215,7 @@ function chunkWikiPage(input: { const heading = selected[0]?.replace(/^#{1,3}\s+/u, "").trim(); return { conceptPath: input.conceptPath, + ...(input.description ? { description: input.description } : {}), fields: JSON.stringify(input.fields), ...(heading ? { heading } : {}), id: `wiki:${input.relative}:${start + 1}`, @@ -200,6 +223,8 @@ function chunkWikiPage(input: { lineEnd: Math.max(start + 1, end), lineStart: start + 1, path: input.conceptPath, + ...(input.resource ? { resource: input.resource } : {}), + roles: input.roles, scope: "wiki", tags: input.tags, text: [input.description, selected.join("\n")].filter(Boolean).join("\n"), @@ -241,6 +266,7 @@ function extractRelationships( const offset = match.index ?? 0; relationships.push({ context: relationshipContext(body, offset), + kind: relationshipKind(relationshipContext(body, offset), sourceRelative), target, }); } @@ -351,6 +377,10 @@ function stringField(value: unknown): string | undefined { return typeof value === "string" && value.trim() ? value.trim() : undefined; } +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + function stringArray(value: unknown): string[] { return Array.isArray(value) ? value.filter( @@ -360,6 +390,121 @@ function stringArray(value: unknown): string[] { : []; } +function parseOpenWikiMetadata(value: unknown): OpenWikiMetadata { + const record = isRecord(value) ? value : {}; + return { + changeKinds: slugArray(record.change_kinds, 16), + invariants: boundedStringArray(record.invariants, 16, 400), + roles: boundedStringArray(record.roles, 9, 40).filter( + (role): role is DocumentRole => DOCUMENT_ROLES.has(role as DocumentRole), + ), + sourcePaths: pathArray(record.source_paths, 32), + symbols: boundedStringArray(record.symbols, 48, 120).filter((symbol) => + /^[A-Za-z_$][A-Za-z0-9_$.:-]*$/u.test(symbol), + ), + testPaths: pathArray(record.test_paths, 32), + validationCommands: boundedStringArray(record.validation_commands, 12, 300), + }; +} + +function inferDocumentRoles( + type: string, + tags: string[], + relative: string, + declared: DocumentRole[], +): DocumentRole[] { + const value = `${type} ${tags.join(" ")} ${relative}`.toLowerCase(); + const roles = new Set(declared); + const add = (role: DocumentRole, pattern: RegExp): void => { + if (pattern.test(value)) roles.add(role); + }; + add( + "architecture", + /\b(?:architecture|engine|interface|memory|runtime|storage|system)\b/u, + ); + add("delivery", /\b(?:artifact|build|delivery|package|publish|release)\b/u); + add("domain", /\b(?:concept|data|domain|model|query|schema)\b/u); + add( + "integration", + /\b(?:ecosystem|integration|platform|plugin|provider|react)\b/u, + ); + add( + "operations", + /\b(?:contribution|development|operations|practice|tooling)\b/u, + ); + add("repository", /\b(?:project|quickstart|repository)\b/u); + add("testing", /\b(?:quality|test|testing|validation|verification)\b/u); + add("workflow", /\b(?:automation|ingestion|lifecycle|playbook|workflow)\b/u); + if (roles.size === 0) roles.add("reference"); + return [...roles]; +} + +function relationshipKind( + context: string, + sourceRelative: string, +): OkfRelationship["kind"] { + if (/^(?:quickstart|index)\.md$/u.test(path.posix.basename(sourceRelative))) { + return "navigation"; + } + if ( + /\b(?:export|package|publish|release|ship|surface|bundle|deliver)\w*\b/iu.test( + context, + ) + ) { + return "delivery"; + } + if ( + /\b(?:lifecycle|transition|reset|reuse|before|after|enter|exit)\w*\b/iu.test( + context, + ) + ) { + return "lifecycle"; + } + if ( + /\b(?:call|depend|dispatch|own|share|configure|secure|adapt|consume)\w*\b/iu.test( + context, + ) + ) { + return "dependency"; + } + if (/\b(?:start|navigate|read|see|guide|overview)\w*\b/iu.test(context)) { + return "navigation"; + } + return "related"; +} + +function boundedStringArray( + value: unknown, + maximumItems: number, + maximumLength: number, +): string[] { + if (!Array.isArray(value)) return []; + return [ + ...new Set( + value + .filter((item): item is string => typeof item === "string") + .map((item) => item.replace(/\s+/gu, " ").trim()) + .filter((item) => item.length > 0 && item.length <= maximumLength), + ), + ].slice(0, maximumItems); +} + +function slugArray(value: unknown, maximumItems: number): string[] { + return boundedStringArray(value, maximumItems, 60) + .map((item) => item.toLowerCase()) + .filter((item) => /^[a-z0-9]+(?:-[a-z0-9]+)*$/u.test(item)); +} + +function pathArray(value: unknown, maximumItems: number): string[] { + return boundedStringArray(value, maximumItems, 300).filter( + (item) => + !path.posix.isAbsolute(item) && + !item.includes("\\") && + !item.split("/").some((part) => part === "" || part === "..") && + !item.split("/").some(isSecretName), + ); +} + function toPosix(value: string): string { return value.split(path.sep).join("/"); } diff --git a/src/retrieval/search-service.ts b/src/retrieval/search-service.ts index bb3bbc41..6887f6ce 100644 --- a/src/retrieval/search-service.ts +++ b/src/retrieval/search-service.ts @@ -7,47 +7,32 @@ import { } from "./ranking.js"; import { SemanticRanker, type EmbeddingProvider } from "./semantic.js"; import type { - ChangeSurfaceCategory, + BriefInvariant, ChangeSurfaceResponse, + CoverageReviewItem, + DocumentRole, + EvidenceReference, IndexedChunk, OkfConcept, + OpenWikiMetadata, RankedHit, RepositoryCorpus, SearchResponse, SearchResultItem, SearchScope, - SymbolTraceCategory, - SymbolTraceResult, - SymbolTraceResponse, + SourceSurfaceCategory, + ValidationReference, } from "./types.js"; const DEFAULT_LIMIT = 6; const MAX_SEARCH_LIMIT = 10; -const MAX_SURFACE_LIMIT = 12; -const MAX_TRACE_LIMIT = 6; -const MAX_SYMBOLS = 12; +const MAX_SURFACE_LIMIT = 8; const MAX_QUERY_LENGTH = 500; -const MAX_RELATED_CONCEPTS = 2; const MAX_SNIPPET_LENGTH = 220; -const DOTTED_IDENTIFIER = - /^[A-Za-z_$][A-Za-z0-9_$]{0,99}(?:\.[A-Za-z_$][A-Za-z0-9_$]{0,99}){0,5}$/u; -const TRACE_CATEGORY_ORDER: SymbolTraceCategory[] = [ - "implementation", - "exports", - "publish_generated", - "initialization", - "consumer", - "tests", -]; -const CHANGE_SURFACE_CATEGORY_ORDER: ChangeSurfaceCategory[] = [ - "implementation", - "state_transitions", - "exports", - "publish_generated", - "initialization", - "consumer", - "tests", -]; +const MAX_CHANGED_PATHS = 50; +const MAX_CONCEPTS = 3; +const MAX_INVARIANTS = 4; +const MAX_VALIDATION_COMMANDS = 3; export interface RetrievalServiceOptions { embeddingProvider: EmbeddingProvider; @@ -96,70 +81,150 @@ export class RetrievalService { async changeSurface( query: string, - limit = 7, + limit = 6, + changedPaths: string[] = [], ): Promise { const corpus = await this.corpus(); const validQuery = validateQuery(query); const validLimit = normalizeLimit(limit, MAX_SURFACE_LIMIT, 6); - const concepts = await this.search(validQuery, "wiki", validLimit); - const conceptChunks = conceptHits(corpus.chunks, concepts.results); - const referencedPaths = extractPaths( - conceptChunks.map((chunk) => chunk.text).join("\n"), + const validChangedPaths = validateChangedPaths(changedPaths); + const metadataRoles = inferQueryRoles(validQuery); + const conceptChunks = selectConceptChunks( + corpus, + validQuery, + metadataRoles, + MAX_CONCEPTS, + ); + const selectedConcepts = selectedOkfConcepts(corpus, conceptChunks); + const metadata = mergeMetadata(selectedConcepts); + const referencedPaths = new Set([ + ...extractPaths(conceptChunks.map((chunk) => chunk.text).join("\n")), + ...metadata.sourcePaths, + ...metadata.testPaths, + ]); + const symbols = [...new Set(metadata.symbols)]; + const sourceQuery = [validQuery, ...symbols.slice(0, 10)].join(" "); + const sourceCorpus = sourceChunks(corpus.chunks).filter( + (chunk) => !isRepositoryGuidance(chunk), ); - const symbols = extractSymbols( - `${validQuery}\n${conceptChunks.map((chunk) => chunk.text).join("\n")}`, + const source = boostReferencedPaths( + reciprocalRankFusion([ + { + hits: rankBm25(sourceCorpus, sourceQuery), + name: "bm25", + weight: 1, + }, + { + hits: rankKeyword(sourceCorpus, sourceQuery), + name: "keyword", + weight: 0.9, + }, + ]), + referencedPaths, + ).slice(0, 120); + const invariants = collectInvariants( + selectedConcepts, + conceptChunks, + validQuery, ); - const expandedQuery = [validQuery, ...symbols.slice(0, 24)].join(" "); - const source = reciprocalRankFusion([ + const ownershipHits = uniquePathHits( + source.filter( + (hit) => + !isTestChunk(hit.chunk) && + categorize(hit.chunk).includes("implementation"), + ), + ).slice(0, Math.min(3, validLimit)); + const deliveryRequested = requiresDeliveryReview( + validQuery, + metadataRoles, + validChangedPaths, + ); + const deliveryHits = deliveryRequested + ? uniquePathHits( + source.filter((hit) => + categorize(hit.chunk).some((category) => + [ + "consumer", + "exports", + "initialization", + "publish_generated", + ].includes(category), + ), + ), + ).slice(0, 2) + : []; + const testRankers = [ { - hits: rankBm25(sourceChunks(corpus.chunks), expandedQuery), - name: "bm25", - weight: 1, + hits: rankBm25(sourceCorpus.filter(isTestChunk), validQuery), + name: "test_bm25", + weight: 1.2, }, { - hits: boostReferencedPaths( - rankKeyword(sourceChunks(corpus.chunks), expandedQuery), - referencedPaths, - ), - name: "wiki_paths", - weight: 1.1, + hits: rankKeyword(sourceCorpus.filter(isTestChunk), validQuery), + name: "test_keyword", + weight: 1, }, - ]).slice(0, 160); - const groups = emptyChangeSurfaceGroups(); - const candidates = emptyChangeSurfaceGroups(); - for (const hit of source) { - for (const category of categorize(hit.chunk)) { - candidates[category].push(toResultItem(hit)); - } - if (isStateTransitionProducer(hit.chunk)) { - candidates.state_transitions.push(toResultItem(hit)); - } + ]; + if (invariants.length > 0) { + testRankers.push({ + hits: rankBm25( + sourceCorpus.filter(isTestChunk), + invariants.map((invariant) => invariant.text).join(" "), + ), + name: "invariant_bm25", + weight: 0.35, + }); } - fillGroups( - groups, - candidates, - validLimit, - CHANGE_SURFACE_CATEGORY_ORDER, + const testHits = uniquePathHits( + deduplicateTestMirrors( + boostReferencedPaths( + reciprocalRankFusion(testRankers), + new Set(metadata.testPaths), + 2.5, + ), + ), + ).slice(0, Math.min(3, validLimit)); + const ownership = ownershipHits.map((hit) => + toEvidenceReference(hit, ownershipReason(hit.chunk, referencedPaths)), + ); + const tests = testHits.map((hit) => + toEvidenceReference(hit, "Analogous behavior or regression coverage."), + ); + const delivery = deliveryHits.map((hit) => + toEvidenceReference(hit, deliveryReason(hit.chunk)), + ); + const validation = collectValidation(selectedConcepts, conceptChunks); + const unknowns = collectUnknowns({ + delivery, + deliveryRequested, + invariants, + ownership, + tests, + }); + const review = buildCoverageReview( + validChangedPaths, + [...ownership, ...delivery], + referencedPaths, ); return { - groups, + brief: { + delivery, + invariants, + ownership, + tests, + unknowns, + validation, + }, + provenance: { + changedPaths: validChangedPaths, + metadataRoles, + wikiConceptPaths: [ + ...new Set(conceptChunks.map((chunk) => chunk.path)), + ], + wikiReferencedSourcePaths: [...referencedPaths], + }, query: validQuery, - relatedConcepts: concepts.results.slice(0, MAX_RELATED_CONCEPTS), - }; - } - - async traceSymbols( - symbols: string[], - limit = 4, - ): Promise { - const validSymbols = validateSymbols(symbols); - const validLimit = normalizeLimit(limit, MAX_TRACE_LIMIT, 4); - this.corpusPromise = undefined; - const chunks = sourceChunks((await this.corpus()).chunks); - return { - traces: validSymbols.map((symbol) => - traceSymbol(chunks, symbol, validLimit), - ), + ...(review.length > 0 ? { review } : {}), }; } @@ -194,7 +259,7 @@ function rankOkfGraph( const concept = corpus.concepts.get(conceptPath); if (!concept) continue; const base = scores.get(conceptPath) ?? 0; - for (const neighbor of graphNeighbors(concept, corpus.concepts)) { + for (const neighbor of graphNeighbors(concept, query)) { scores.set(neighbor, (scores.get(neighbor) ?? 0) + base * 0.35); next.add(neighbor); } @@ -210,25 +275,28 @@ function rankOkfGraph( .sort((left, right) => right.score - left.score); } -function graphNeighbors( - concept: OkfConcept, - concepts: Map, -): Set { - const neighbors = new Set([ - ...concept.relationships.map((relationship) => relationship.target), - ...concept.incoming, +function graphNeighbors(concept: OkfConcept, query: string): Set { + const queryTerms = new Set(tokenize(query)); + const desiredKinds = new Set([ + "dependency", + "lifecycle", + "related", ]); - if (concept.tags.length > 0) { - for (const candidate of concepts.values()) { - if ( - candidate.path !== concept.path && - candidate.tags.some((tag) => concept.tags.includes(tag)) - ) { - neighbors.add(candidate.path); - } - } + if (/\b(?:export|package|public|publish|release|ship)\w*\b/iu.test(query)) { + desiredKinds.add("delivery"); } - return neighbors; + return new Set( + concept.relationships + .filter( + (relationship) => + desiredKinds.has(relationship.kind) && + (relationship.kind !== "related" || + tokenize(relationship.context).some((term) => + queryTerms.has(term), + )), + ) + .map((relationship) => relationship.target), + ); } function bestConceptChunk( @@ -244,36 +312,475 @@ function bestConceptChunk( ); } -function conceptHits( - chunks: IndexedChunk[], - results: SearchResultItem[], +function selectConceptChunks( + corpus: RepositoryCorpus, + query: string, + desiredRoles: DocumentRole[], + limit: number, ): IndexedChunk[] { - const keys = new Set(results.map((item) => `${item.path}:${item.lineStart}`)); - return chunks.filter((chunk) => keys.has(`${chunk.path}:${chunk.lineStart}`)); + const wikiChunks = corpus.chunks.filter((chunk) => chunk.scope === "wiki"); + const representatives = [...corpus.concepts.values()] + .map((concept) => { + const base = wikiChunks.find( + (chunk) => chunk.conceptPath === concept.path, + ); + if (!base) return undefined; + return { + ...base, + fields: [ + concept.title, + concept.type, + concept.description, + concept.roles.join(" "), + concept.tags.join(" "), + concept.resource, + concept.metadata.changeKinds.join(" "), + concept.metadata.sourcePaths.join(" "), + concept.metadata.symbols.join(" "), + concept.metadata.testPaths.join(" "), + ] + .filter(Boolean) + .join("\n"), + text: concept.description ?? "", + } satisfies IndexedChunk; + }) + .filter((chunk): chunk is IndexedChunk => chunk !== undefined); + const relevantRepresentatives = representatives.filter((chunk) => + hasDistinctiveMetadataMatch(chunk, query), + ); + if (relevantRepresentatives.length === 0) return []; + const neighborPaths = new Set(); + for (const seed of rankBm25(relevantRepresentatives, query).slice(0, 4)) { + const concept = seed.chunk.conceptPath + ? corpus.concepts.get(seed.chunk.conceptPath) + : undefined; + if (!concept) continue; + for (const neighbor of graphNeighbors(concept, query)) { + neighborPaths.add(neighbor); + } + } + const ranked = reciprocalRankFusion([ + { + hits: rankBm25(relevantRepresentatives, query), + name: "metadata_bm25", + weight: 1, + }, + { + hits: rankKeyword(relevantRepresentatives, query), + name: "metadata_keyword", + weight: 0.9, + }, + ]) + .map((hit) => { + const overlap = hit.chunk.roles.filter((role) => + desiredRoles.includes(role), + ).length; + const repositoryOnly = + hit.chunk.roles.includes("repository") && + hit.chunk.roles.every((role) => + ["repository", "reference"].includes(role), + ); + return { + ...hit, + score: + hit.score * + (1 + overlap * 0.18) * + (neighborPaths.has(hit.chunk.conceptPath ?? "") ? 1.15 : 1) * + (repositoryOnly ? 0.65 : 1), + }; + }) + .sort((left, right) => right.score - left.score); + const selected: RankedHit[] = []; + const coveredRoles = new Set(); + for (const hit of ranked) { + if (selected.length >= limit) break; + if ( + selected.some( + (candidate) => candidate.chunk.conceptPath === hit.chunk.conceptPath, + ) + ) { + continue; + } + const addsRole = hit.chunk.roles.some( + (role) => desiredRoles.includes(role) && !coveredRoles.has(role), + ); + if (selected.length < 2 || addsRole || selected.length + 1 === limit) { + selected.push(hit); + hit.chunk.roles.forEach((role) => coveredRoles.add(role)); + } + } + return selected + .map((hit) => + hit.chunk.conceptPath + ? bestConceptChunk(wikiChunks, hit.chunk.conceptPath, query) + : undefined, + ) + .filter((chunk): chunk is IndexedChunk => chunk !== undefined); +} + +const GENERIC_ROUTING_TERMS = new Set([ + "add", + "agent", + "change", + "cod", + "code", + "implement", + "improve", + "repository", + "task", + "update", +]); + +function hasDistinctiveMetadataMatch( + chunk: IndexedChunk, + query: string, +): boolean { + const queryTerms = new Set( + tokenize(query).filter((term) => !GENERIC_ROUTING_TERMS.has(term)), + ); + if (queryTerms.size === 0) return true; + const metadataTerms = new Set( + tokenize( + [ + chunk.path, + chunk.title, + chunk.description, + chunk.type, + chunk.tags.join(" "), + chunk.roles.join(" "), + chunk.fields, + ] + .filter(Boolean) + .join(" "), + ), + ); + return [...queryTerms].some((term) => metadataTerms.has(term)); +} + +function selectedOkfConcepts( + corpus: RepositoryCorpus, + chunks: IndexedChunk[], +): OkfConcept[] { + return [ + ...new Map( + chunks + .map((chunk) => + chunk.conceptPath + ? corpus.concepts.get(chunk.conceptPath) + : undefined, + ) + .filter((concept): concept is OkfConcept => concept !== undefined) + .map((concept) => [concept.path, concept]), + ).values(), + ]; +} + +function mergeMetadata(concepts: OkfConcept[]): OpenWikiMetadata { + const merge = ( + select: (metadata: OpenWikiMetadata) => string[], + ): string[] => [ + ...new Set(concepts.flatMap((concept) => select(concept.metadata))), + ]; + return { + changeKinds: merge((metadata) => metadata.changeKinds), + invariants: merge((metadata) => metadata.invariants), + roles: [...new Set(concepts.flatMap((concept) => concept.metadata.roles))], + sourcePaths: merge((metadata) => metadata.sourcePaths), + symbols: merge((metadata) => metadata.symbols), + testPaths: merge((metadata) => metadata.testPaths), + validationCommands: merge((metadata) => metadata.validationCommands), + }; +} + +function inferQueryRoles(query: string): DocumentRole[] { + const roles = new Set(["architecture", "domain"]); + const add = (role: DocumentRole, pattern: RegExp): void => { + if (pattern.test(query)) roles.add(role); + }; + add( + "delivery", + /\b(?:api|artifact|build|consumer|export|package|public|publish|release|ship)\w*\b/iu, + ); + add( + "integration", + /\b(?:adapter|integration|middleware|plugin|provider|react|router)\w*\b/iu, + ); + add( + "operations", + /\b(?:ci|cli|configure|deploy|development|install|operations|tooling)\w*\b/iu, + ); + add( + "testing", + /\b(?:behavior|compatibility|invariant|regression|test|validate|verify)\w*\b/iu, + ); + add( + "workflow", + /\b(?:defer|event|lifecycle|reset|rollback|state|transition|workflow)\w*\b/iu, + ); + return [...roles]; +} + +function collectInvariants( + concepts: OkfConcept[], + chunks: IndexedChunk[], + query: string, +): BriefInvariant[] { + const candidates: (BriefInvariant & { score: number })[] = []; + for (const concept of concepts) { + for (const invariant of concept.metadata.invariants) { + candidates.push({ + lineEnd: 1, + lineStart: 1, + path: concept.path, + score: invariantScore(invariant, query) + 8, + text: invariant, + }); + } + } + for (const chunk of chunks) { + const lines = chunk.text + .split(/\r?\n/gu) + .map((line) => line.replace(/^\s*(?:[-*]|\d+\.)\s+/u, "").trim()) + .filter((line) => line.length >= 24 && line.length <= 500); + for (const line of lines) { + if (!isInvariantText(line)) continue; + if (!hasTermOverlap(line, query)) continue; + candidates.push({ + lineEnd: chunk.lineEnd, + lineStart: chunk.lineStart, + path: chunk.path, + score: invariantScore(line, query), + text: compactText(line, 240), + }); + } + } + const seen = new Set(); + return candidates + .sort((left, right) => right.score - left.score) + .filter((candidate) => candidate.score > 0) + .filter((candidate) => { + const key = candidate.text.toLowerCase(); + if (seen.has(key)) return false; + seen.add(key); + return true; + }) + .slice(0, MAX_INVARIANTS) + .map((candidate) => ({ + lineEnd: candidate.lineEnd, + lineStart: candidate.lineStart, + path: candidate.path, + text: candidate.text, + })); +} + +function isInvariantText(value: string): boolean { + return /\b(?:must|should not|do not|don't|never|preserve|remain|only|before|after|unchanged|idempotent|reset|reuse|invariant|required|incomplete)\b/iu.test( + value, + ); +} + +function invariantScore(value: string, query: string): number { + const queryTerms = new Set(tokenize(query)); + const overlap = tokenize(value).filter((term) => queryTerms.has(term)).length; + const force = /\b(?:must|do not|don't|never|required|invariant)\b/iu.test( + value, + ) + ? 4 + : 0; + return overlap * 2 + force; +} + +function hasTermOverlap(value: string, query: string): boolean { + const queryTerms = new Set( + tokenize(query).filter((term) => !GENERIC_ROUTING_TERMS.has(term)), + ); + return tokenize(value).some((term) => queryTerms.has(term)); +} + +function collectValidation( + concepts: OkfConcept[], + chunks: IndexedChunk[], +): ValidationReference[] { + const candidates: ValidationReference[] = []; + for (const concept of concepts) { + for (const command of concept.metadata.validationCommands) { + if (isSafeDisplayedCommand(command)) { + candidates.push({ command, path: concept.path }); + } + } + } + for (const chunk of chunks) { + for (const match of chunk.text.matchAll(/`([^`\n]{3,300})`/gu)) { + const command = match[1]?.trim(); + if (command && looksLikeValidationCommand(command)) { + candidates.push({ command, path: chunk.path }); + } + } + } + const seen = new Set(); + return candidates + .filter((candidate) => { + if (seen.has(candidate.command)) return false; + seen.add(candidate.command); + return true; + }) + .slice(0, MAX_VALIDATION_COMMANDS); +} + +function looksLikeValidationCommand(value: string): boolean { + return ( + isSafeDisplayedCommand(value) && + /^(?:bun|cargo|go|make|npm|npx|pnpm|pytest|python\s+-m\s+pytest|ruff|uv\s+run|yarn)\b/iu.test( + value, + ) && + /\b(?:build|check|lint|test|typecheck|verify|vitest)\b/iu.test(value) + ); +} + +function isSafeDisplayedCommand(value: string): boolean { + return ( + value.length <= 300 && + !/[\r\n\0]/u.test(value) && + !/(?:\.env|credential|private[_-]?key|secret|token)/iu.test(value) + ); +} + +function uniquePathHits(hits: RankedHit[]): RankedHit[] { + const seen = new Set(); + return hits.filter((hit) => { + if (seen.has(hit.chunk.path)) return false; + seen.add(hit.chunk.path); + return true; + }); +} + +function toEvidenceReference( + hit: RankedHit, + reason: string, +): EvidenceReference { + return { + lineEnd: hit.chunk.lineEnd, + lineStart: hit.chunk.lineStart, + path: hit.chunk.path, + reason, + ...(hit.chunk.testNames && hit.chunk.testNames.length > 0 + ? { testNames: hit.chunk.testNames.slice(0, 6) } + : {}), + ...(hit.chunk.title ? { title: hit.chunk.title } : {}), + }; +} + +function ownershipReason(chunk: IndexedChunk, references: Set): string { + return pathMatchesReference(chunk.path, references) + ? "Named by the selected OpenWiki concept as an implementation anchor." + : "Highest-ranked implementation ownership candidate; verify in source."; +} + +function deliveryReason(chunk: IndexedChunk): string { + const categories = categorize(chunk); + if (categories.includes("exports")) + return "Public or package export surface."; + if (categories.includes("publish_generated")) { + return "Generated, packaged, or publish-facing surface."; + } + if (categories.includes("consumer")) return "Consumer-facing usage surface."; + return "Initialization or registration surface."; +} + +function requiresDeliveryReview( + query: string, + roles: DocumentRole[], + changedPaths: string[], +): boolean { + return ( + roles.includes("delivery") || + /\b(?:api|consumer|export|package|public|publish|release|ship)\w*\b/iu.test( + query, + ) || + changedPaths.some((candidate) => + /(?:^|\/)(?:index\.[cm]?[jt]sx?|package\.json|dist|publish)(?:$|\/)/u.test( + candidate, + ), + ) + ); +} + +function collectUnknowns(input: { + delivery: EvidenceReference[]; + deliveryRequested: boolean; + invariants: BriefInvariant[]; + ownership: EvidenceReference[]; + tests: EvidenceReference[]; +}): string[] { + const unknowns: string[] = []; + if (input.ownership.length === 0) { + unknowns.push( + "No implementation owner was established; locate it in source.", + ); + } + if (input.invariants.length === 0) { + unknowns.push("No explicit behavioral invariant was found in the wiki."); + } + if (input.tests.length === 0) { + unknowns.push( + "No analogous focused test was found; add task-specific coverage.", + ); + } + if (input.deliveryRequested && input.delivery.length === 0) { + unknowns.push( + "No shipped-surface evidence was found; verify exports manually.", + ); + } + return unknowns; +} + +function buildCoverageReview( + changedPaths: string[], + evidence: EvidenceReference[], + referencedPaths: Set, +): CoverageReviewItem[] { + if (changedPaths.length === 0) return []; + const normalizedChanges = new Set(changedPaths); + const candidates = [...evidence.map((item) => item.path), ...referencedPaths]; + const seen = new Set(); + return candidates + .filter((candidate) => { + if (seen.has(candidate) || normalizedChanges.has(candidate)) return false; + seen.add(candidate); + return true; + }) + .slice(0, 4) + .map((candidate) => ({ + path: candidate, + reason: + "Documented adjacent surface is absent from changed_paths; verify that it is intentionally unaffected.", + })); } function boostReferencedPaths( hits: RankedHit[], paths: Set, + multiplier = 1.6, ): RankedHit[] { return hits .map((hit) => ({ ...hit, score: hit.score * - ([...paths].some( - (candidate) => - hit.chunk.path === candidate || hit.chunk.path.endsWith(candidate), - ) - ? 2.5 - : 1), + (pathMatchesReference(hit.chunk.path, paths) ? multiplier : 1), })) .sort((left, right) => right.score - left.score); } -function categorize(chunk: IndexedChunk): SymbolTraceCategory[] { +function pathMatchesReference(path: string, references: Set): boolean { + return [...references].some( + (candidate) => path === candidate || path.endsWith(candidate), + ); +} + +function categorize(chunk: IndexedChunk): SourceSurfaceCategory[] { const value = `${chunk.path}\n${chunk.text}`; - const categories = new Set(); + const categories = new Set(); if ( /\b(?:exports|entrypoint|public api)\b/iu.test(value) || /\bexport\s+(?:\*|\{[^}]+\})\s+from\b/iu.test(chunk.text) || @@ -317,57 +824,8 @@ function isTestChunk(chunk: IndexedChunk): boolean { ); } -function isStateTransitionProducer(chunk: IndexedChunk): boolean { - if ( - isTestChunk(chunk) || - /(?:^|\/)query\/(?:modifier|modifiers)(?:\/|$)/u.test(chunk.path) - ) { - return false; - } - const producerPath = - /(?:^|\/)(?:actions?|entity|mutation|relation|store|trait|world)(?:\/|[._-])/u.test( - chunk.path, - ); - const transitionText = - /\b(?:add|change|defer|destroy|emit|flush|remove|replace|reset|trigger|update)(?:d|s|ing)?\b/iu.test( - chunk.text, - ); - return producerPath && transitionText; -} - -function traceSymbol( - chunks: IndexedChunk[], - symbol: string, - limit: number, -): SymbolTraceResult { - const leaf = symbol.split(".").at(-1) ?? symbol; - const fullPattern = symbol.split(".").map(escapeRegExp).join("\\s*\\.\\s*"); - const exactSymbol = new RegExp( - `(?:^|[^A-Za-z0-9_$])${fullPattern}(?:$|[^A-Za-z0-9_$])`, - "u", - ); - const exactLeaf = new RegExp( - `(?:^|[^A-Za-z0-9_$])${escapeRegExp(leaf)}(?:$|[^A-Za-z0-9_$])`, - "u", - ); - const candidates = emptyTraceGroups(); - for (const hit of rankKeyword(chunks, `${symbol} ${leaf}`)) { - if (!exactSymbol.test(hit.chunk.text) && !exactLeaf.test(hit.chunk.text)) { - continue; - } - for (const category of categorize(hit.chunk)) { - candidates[category].push(toResultItem(hit)); - } - } - const groups = emptyTraceGroups(); - fillGroups(groups, candidates, limit, TRACE_CATEGORY_ORDER); - return { - groups, - missing: TRACE_CATEGORY_ORDER.filter( - (category) => candidates[category].length === 0, - ), - symbol, - }; +function isRepositoryGuidance(chunk: IndexedChunk): boolean { + return /(?:^|\/)(?:AGENTS|CLAUDE)\.md$/iu.test(chunk.path); } function deduplicateTestMirrors(hits: RankedHit[]): RankedHit[] { @@ -409,69 +867,6 @@ function extractPaths(value: string): Set { ); } -function extractSymbols(value: string): string[] { - const symbols = new Set(); - for (const match of value.matchAll(/`([A-Za-z_$][A-Za-z0-9_$]{2,})`/gu)) { - if (match[1]) symbols.add(match[1]); - } - for (const term of tokenize(value)) { - if (term.length >= 4) symbols.add(term); - } - return [...symbols]; -} - -function escapeRegExp(value: string): string { - return value.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); -} - -function emptyChangeSurfaceGroups(): Record< - ChangeSurfaceCategory, - SearchResultItem[] -> { - return { - consumer: [], - exports: [], - implementation: [], - initialization: [], - publish_generated: [], - state_transitions: [], - tests: [], - }; -} - -function emptyTraceGroups(): Record { - return { - consumer: [], - exports: [], - implementation: [], - initialization: [], - publish_generated: [], - tests: [], - }; -} - -function fillGroups( - groups: Record, - candidates: Record, - totalLimit: number, - categoryOrder: readonly Category[], -): void { - let remaining = totalLimit; - let index = 0; - while (remaining > 0) { - let added = false; - for (const category of categoryOrder) { - const candidate = candidates[category][index]; - if (!candidate || remaining === 0) continue; - groups[category].push(candidate); - remaining -= 1; - added = true; - } - if (!added) return; - index += 1; - } -} - function response( query: string, hits: RankedHit[], @@ -502,7 +897,11 @@ function toResultItem(hit: RankedHit): SearchResultItem { } function compactSnippet(value: string): string { - return value.replace(/\s+/gu, " ").trim().slice(0, MAX_SNIPPET_LENGTH); + return compactText(value, MAX_SNIPPET_LENGTH); +} + +function compactText(value: string, limit: number): string { + return value.replace(/\s+/gu, " ").trim().slice(0, limit); } function scopedChunks( @@ -547,22 +946,39 @@ function validateQuery(query: string): string { return query.trim(); } -function validateSymbols(symbols: string[]): string[] { - if (!Array.isArray(symbols) || symbols.length === 0) { - throw new Error("symbols must contain at least one identifier."); - } - const unique = [...new Set(symbols.map((symbol) => symbol.trim()))]; - if (unique.length > MAX_SYMBOLS) { - throw new Error(`symbols must contain at most ${MAX_SYMBOLS} identifiers.`); +function validateChangedPaths(paths: string[]): string[] { + if (!Array.isArray(paths)) { + throw new Error( + "changed_paths must be an array of repository-relative paths.", + ); } - for (const symbol of unique) { - if (symbol.length > 200 || !DOTTED_IDENTIFIER.test(symbol)) { - throw new Error( - "each symbol must be a plain or dotted identifier up to 200 characters.", - ); - } + if (paths.length > MAX_CHANGED_PATHS) { + throw new Error( + `changed_paths must contain at most ${MAX_CHANGED_PATHS} paths.`, + ); } - return unique; + return [ + ...new Set( + paths.map((candidate) => { + if ( + typeof candidate !== "string" || + !candidate.trim() || + candidate.length > 300 || + candidate.startsWith("/") || + candidate.includes("\\") || + candidate.split("/").some((part) => part === "" || part === "..") || + /(?:^|\/)(?:\.env(?:\..*)?|credentials\.json|secrets?|tokens?)(?:\/|$)/iu.test( + candidate, + ) + ) { + throw new Error( + "changed_paths must contain safe repository-relative paths.", + ); + } + return candidate.trim(); + }), + ), + ]; } function normalizeLimit( diff --git a/src/retrieval/types.ts b/src/retrieval/types.ts index 8c1916a1..4d82b360 100644 --- a/src/retrieval/types.ts +++ b/src/retrieval/types.ts @@ -4,8 +4,30 @@ export type IndexedScope = "source_code" | "wiki"; export type ChunkKind = "source" | "wiki-section"; +export type DocumentRole = + | "architecture" + | "delivery" + | "domain" + | "integration" + | "operations" + | "reference" + | "repository" + | "testing" + | "workflow"; + +export interface OpenWikiMetadata { + changeKinds: string[]; + invariants: string[]; + roles: DocumentRole[]; + sourcePaths: string[]; + symbols: string[]; + testPaths: string[]; + validationCommands: string[]; +} + export interface IndexedChunk { conceptPath?: string; + description?: string; fields: string; heading?: string; id: string; @@ -13,6 +35,8 @@ export interface IndexedChunk { lineEnd: number; lineStart: number; path: string; + resource?: string; + roles: DocumentRole[]; scope: IndexedScope; tags: string[]; testNames?: string[]; @@ -23,15 +47,18 @@ export interface IndexedChunk { export interface OkfRelationship { context: string; + kind: "dependency" | "delivery" | "lifecycle" | "navigation" | "related"; target: string; } export interface OkfConcept { description?: string; incoming: Set; + metadata: OpenWikiMetadata; path: string; relationships: OkfRelationship[]; resource?: string; + roles: DocumentRole[]; tags: string[]; title: string; type: string; @@ -66,7 +93,7 @@ export interface SearchResponse { scope: SearchScope; } -export type SymbolTraceCategory = +export type SourceSurfaceCategory = | "consumer" | "exports" | "implementation" @@ -74,22 +101,54 @@ export type SymbolTraceCategory = | "publish_generated" | "tests"; -export type ChangeSurfaceCategory = - | SymbolTraceCategory - | "state_transitions"; +export type ChangeSurfaceCategory = SourceSurfaceCategory | "state_transitions"; -export interface ChangeSurfaceResponse { - groups: Record; - query: string; - relatedConcepts: SearchResultItem[]; +export interface EvidenceReference { + lineEnd: number; + lineStart: number; + path: string; + reason: string; + symbols?: string[]; + testNames?: string[]; + title?: string; +} + +export interface ChangeSurfaceProvenance { + changedPaths: string[]; + metadataRoles: DocumentRole[]; + wikiConceptPaths: string[]; + wikiReferencedSourcePaths: string[]; } -export interface SymbolTraceResult { - groups: Record; - missing: SymbolTraceCategory[]; - symbol: string; +export interface BriefInvariant { + lineEnd: number; + lineStart: number; + path: string; + text: string; +} + +export interface ValidationReference { + command: string; + path: string; } -export interface SymbolTraceResponse { - traces: SymbolTraceResult[]; +export interface CoverageReviewItem { + path: string; + reason: string; +} + +export interface ChangeSurfaceBrief { + delivery: EvidenceReference[]; + invariants: BriefInvariant[]; + ownership: EvidenceReference[]; + tests: EvidenceReference[]; + unknowns: string[]; + validation: ValidationReference[]; +} + +export interface ChangeSurfaceResponse { + brief: ChangeSurfaceBrief; + provenance: ChangeSurfaceProvenance; + query: string; + review?: CoverageReviewItem[]; } diff --git a/test/code-mode.test.ts b/test/code-mode.test.ts index 1dc1df6d..c858ae18 100644 --- a/test/code-mode.test.ts +++ b/test/code-mode.test.ts @@ -43,19 +43,16 @@ describe("ensureCodeModeRepoSetup agent files", () => { expect(content).toContain(SNIPPET_START); expect(content).toContain(SNIPPET_END); expect(content).toContain("## OpenWiki"); - expect(content).toContain("just-in-time repository index"); - expect(content).toContain("Before a repository-wide"); - expect(content).toContain("Re-consult the wiki"); - expect(content).toContain("import path consumers actually use"); - expect(content).toContain("internal unit tests alone"); - expect(content).toContain("trace_symbols"); - expect(content).toContain("every identity axis"); - expect(content).toContain("canonical owner or ledger"); - expect(content).toContain("input/event/expected-result oracle row"); - expect(content).toContain("independent instances"); - expect(content).toContain("deferred or re-entrant net effects"); - expect(content).toContain("\`tests\` scope"); + expect(content).toContain("optional just-in-time context"); + expect(content).toContain("not required startup reading"); + expect(content).toContain("openwiki_retrieval.change_surface"); + expect(content).toContain("do not reread the returned wiki pages"); + expect(content).toContain("concrete unresolved evidence gap"); + expect(content).toContain("repository-relative changed paths"); + expect(content).toContain("verification gaps"); + expect(content).not.toContain("trace_symbols"); expect(content).toContain("quiet validation"); + expect(content.length).toBeLessThan(2_500); } }); diff --git a/test/retrieval.test.ts b/test/retrieval.test.ts index 436d805c..96071952 100644 --- a/test/retrieval.test.ts +++ b/test/retrieval.test.ts @@ -6,6 +6,7 @@ import { RETRIEVAL_TOOL_DEFINITIONS, SEARCH_SCOPES, } from "../src/retrieval/mcp-tools.ts"; +import { tokenize } from "../src/retrieval/ranking.ts"; import { RetrievalService } from "../src/retrieval/search-service.ts"; let root = ""; @@ -18,7 +19,9 @@ beforeEach(async () => { wikiRoot = path.join(root, "wiki"); await Promise.all([ mkdir(path.join(repoRoot, "packages/core/src/query"), { recursive: true }), - mkdir(path.join(repoRoot, "packages/core/src/relation"), { recursive: true }), + mkdir(path.join(repoRoot, "packages/core/src/relation"), { + recursive: true, + }), mkdir(path.join(repoRoot, "packages/core/tests"), { recursive: true }), mkdir(path.join(repoRoot, "packages/publish/src"), { recursive: true }), mkdir(path.join(repoRoot, "packages/publish/tests"), { recursive: true }), @@ -47,6 +50,18 @@ type: Architecture title: Query runtime and package contract description: Connects predicate implementation to public exports and consumer tests. tags: [query, package, runtime] +openwiki: + roles: [architecture, testing, delivery] + change_kinds: [public-api, lifecycle] + source_paths: + - packages/core/src/query/predicate.ts + - packages/core/src/index.ts + - packages/publish/src/index.ts + symbols: [createPredicate] + test_paths: [packages/core/tests/predicate.test.ts] + invariants: + - Predicate transitions must remain independent between instances. + validation_commands: [pnpm -F core test predicate.test.ts] --- # Query runtime @@ -56,6 +71,9 @@ Implement predicates in \`packages/core/src/query/predicate.ts\`, export them fr \`packages/publish/src/index.ts\`, and validate consumer imports in \`packages/publish/tests/predicate.test.ts\`. +Unchanged predicate inputs must not emit a transition. Public exports must remain +available from the consumer package. + The [quickstart](../quickstart.md) routes adjacent changes here. `, ), @@ -83,6 +101,14 @@ The [quickstart](../quickstart.md) routes adjacent changes here. path.join(repoRoot, "packages/publish/tests/predicate.test.ts"), "import { createPredicate } from 'koota';\ndescribe('predicate lifecycle', () => {\n test('tracks false-to-true transitions independently', () => createPredicate());\n});\n", ), + writeFile( + path.join(repoRoot, "AGENTS.md"), + "Public API exports initialize register factory predicate relation removal tests.\n", + ), + writeFile( + path.join(repoRoot, "packages/core/tests/unrelated.test.ts"), + "test('generic public package initialization', () => true);\n", + ), writeFile( path.join(repoRoot, ".env"), "SECRET_PREDICATE_SURFACE=never-index-this\n", @@ -107,11 +133,10 @@ function service(): RetrievalService { } describe("OKF-aware repository retrieval", () => { - test("exposes three concise workflow-oriented MCP tools", () => { + test("exposes two concise workflow-oriented MCP tools", () => { expect(RETRIEVAL_TOOL_DEFINITIONS.map((tool) => tool.name)).toEqual([ "search", "change_surface", - "trace_symbols", ]); expect( RETRIEVAL_TOOL_DEFINITIONS.every( @@ -144,6 +169,25 @@ describe("OKF-aware repository retrieval", () => { ).toBe(true); }); + test("matches snake and kebab compounds to camel-case source symbols", async () => { + expect(tokenize("name_mapping HttpApi http-api")).toEqual([ + "name", + "mapp", + "http", + "api", + "http", + "api", + ]); + await writeFile( + path.join(repoRoot, "packages/core/src/query/NameMapping.ts"), + "export class NameMapping {}\n", + ); + + const result = await service().search("name_mapping", "source_code", 5); + + expect(result.results[0]?.path).toContain("NameMapping.ts"); + }); + test("supports distinct wiki, source_code, and tests scopes", async () => { const retrieval = service(); const source = await retrieval.search("createPredicate", "source_code", 10); @@ -181,66 +225,96 @@ describe("OKF-aware repository retrieval", () => { expect(result.results.length).toBeLessThanOrEqual(10); }); - test("change_surface groups cross-package evidence", async () => { + test("change_surface returns a bounded metadata-routed task brief", async () => { const surface = await service().changeSurface( "add createPredicate query API and track relation removal events", 7, ); - expect(surface.relatedConcepts[0]?.path).toContain("openwiki/"); - expect(surface.groups.implementation.length).toBeGreaterThan(0); - expect(surface.groups.state_transitions[0]?.path).toContain("relation"); - expect(surface.groups.exports.length).toBeGreaterThan(0); - expect(surface.groups.publish_generated.length).toBeGreaterThan(0); - expect(surface.groups.consumer.length).toBeGreaterThan(0); - expect(surface.groups.tests.length).toBeGreaterThan(0); - const results = Object.values(surface.groups).flat(); - expect(results).toHaveLength(7); - expect(surface.relatedConcepts.length).toBeLessThanOrEqual(2); - expect( - Math.max(...results.map((result) => result.snippet.length)), - ).toBeLessThanOrEqual(220); + expect(surface.brief.ownership[0]?.path).toContain("predicate.ts"); + expect(surface.brief.ownership[0]?.reason).toContain("OpenWiki"); + expect(surface.brief.invariants.map((item) => item.text).join(" ")).toMatch( + /independent|unchanged/iu, + ); + expect(surface.brief.tests[0]?.path).toContain("predicate.test.ts"); + expect(surface.brief.tests[0]?.testNames).toContain( + "tracks false-to-true transitions independently", + ); + expect(surface.brief.delivery.length).toBeGreaterThan(0); + expect(surface.brief.validation[0]?.command).toContain("pnpm -F core test"); + const evidence = [ + ...surface.brief.ownership, + ...surface.brief.tests, + ...surface.brief.delivery, + ]; + expect(evidence.every((result) => result.path !== "AGENTS.md")).toBe(true); + expect(surface.provenance.wikiConceptPaths).toContain( + "openwiki/architecture/runtime.md", + ); + expect(surface.provenance.metadataRoles).toContain("delivery"); + expect(surface.provenance.wikiReferencedSourcePaths).toContain( + "packages/core/src/query/predicate.ts", + ); expect(JSON.stringify(surface).length).toBeLessThan(5_000); }); - test("trace_symbols reindexes once and accepts batched dotted symbols", async () => { - const retrieval = service(); - await retrieval.search("createMatcher", "source_code", 5); - await writeFile( - path.join(repoRoot, "packages/core/src/query/matcher.ts"), - "export function createMatcher() { return true; }\nexport const Entity = { changed() { return true; } };\n", + test("change_surface reviews changed paths without treating gaps as requirements", async () => { + const surface = await service().changeSurface( + "add a public createPredicate API", + 6, + ["packages/core/src/query/predicate.ts"], ); - const response = await retrieval.traceSymbols( - [ - "createMatcher", - "Entity.changed", - "PUBLIC_PREDICATE_FACTORY", - "createMatcher", - ], - 50, + expect(surface.provenance.changedPaths).toEqual([ + "packages/core/src/query/predicate.ts", + ]); + expect(surface.review?.some((item) => item.path.includes("index.ts"))).toBe( + true, ); - const trace = response.traces[0]; - - expect(trace.groups.implementation[0]?.path).toContain("matcher.ts"); - expect(trace.missing).toContain("consumer"); - expect(trace.missing).toContain("tests"); - expect(trace.missing).toContain("exports"); - expect(Object.values(trace.groups).flat()).toHaveLength(1); - expect(response.traces.map((item) => item.symbol)).toEqual([ - "createMatcher", - "Entity.changed", - "PUBLIC_PREDICATE_FACTORY", + expect( + surface.review?.every((item) => item.reason.includes("verify")), + ).toBe(true); + await expect( + service().changeSurface("public API", 6, ["../.env"]), + ).rejects.toThrow("safe repository-relative paths"); + }); + + test("weak wiki matches do not invent invariants or symbol ownership", async () => { + await Promise.all([ + writeFile( + path.join(wikiRoot, "architecture/noisy-routing.md"), + `--- +type: Architecture +title: Metadata routing notes +description: Notes about OKF metadata routing and retrieval briefs. +tags: [okf, metadata, retrieval] +--- + +# Metadata routing notes + +Personal reminders should not be mixed into work commitments. The unrelated +example helper is \`wrongOwner\`. +`, + ), + writeFile( + path.join(repoRoot, "packages/core/src/query/metadata-router.ts"), + "export function routeOkfMetadataRetrievalBrief() { return true; }\n", + ), + writeFile( + path.join(repoRoot, "packages/core/src/relation/wrong-owner.ts"), + "export function wrongOwner() { return wrongOwner; }\n", + ), ]); - expect(response.traces[1]?.groups.implementation[0]?.path).toContain( - "matcher.ts", + + const surface = await service().changeSurface( + "improve OKF metadata routing retrieval briefs", ); - expect(response.traces[2]?.groups.implementation[0]?.path).toContain( - "predicate.ts", + + expect(surface.brief.ownership[0]?.path).toContain("metadata-router.ts"); + expect(surface.brief.invariants).toEqual([]); + expect(surface.brief.unknowns).toContain( + "No explicit behavioral invariant was found in the wiki.", ); - await expect( - retrieval.traceSymbols(["createMatcher(); rm -rf /"], 6), - ).rejects.toThrow("plain or dotted identifier"); }); test("never indexes secret-like files", async () => { @@ -250,7 +324,14 @@ describe("OKF-aware repository retrieval", () => { 50, ); - expect(result.results).toEqual([]); + expect( + result.results.every( + (hit) => !/\.env|credentials\.json|secrets\//u.test(hit.path), + ), + ).toBe(true); + expect(result.results.map((hit) => hit.snippet).join("\n")).not.toContain( + "never-index-this", + ); }); test("bounds query length", async () => {