diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 415f0d98..2332061d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ concurrency: env: NODE_VERSION: 24 - PNPM_VERSION: 10.33.2 + PNPM_VERSION: 10.34.5 jobs: scope: diff --git a/.github/workflows/clawsweeper-dispatch.yml b/.github/workflows/clawsweeper-dispatch.yml new file mode 100644 index 00000000..5af378a9 --- /dev/null +++ b/.github/workflows/clawsweeper-dispatch.yml @@ -0,0 +1,202 @@ +name: ClawSweeper Dispatch + +on: + issues: + types: [opened, reopened, edited, labeled, unlabeled] + issue_comment: + types: [created, edited] + pull_request_target: # zizmor: ignore[dangerous-triggers] maintainer-owned external dispatch; no checkout or untrusted PR code execution + types: [opened, reopened, synchronize, ready_for_review, edited, labeled, unlabeled] + +permissions: + contents: read + +concurrency: + group: clawsweeper-dispatch-${{ github.repository }}-${{ github.event.issue.number || github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event.action == 'edited' || github.event.action == 'synchronize' || github.event.action == 'ready_for_review' }} + +jobs: + dispatch: + runs-on: ubuntu-latest + if: ${{ !(endsWith(github.actor, '[bot]') && (github.event.action == 'labeled' || github.event.action == 'unlabeled')) }} + env: + HAS_CLAWSWEEPER_APP_PRIVATE_KEY: ${{ secrets.CLAWSWEEPER_APP_PRIVATE_KEY != '' }} + CLAWSWEEPER_APP_CLIENT_ID: Iv23liOECG0slfuhz093 + SUPERSEDES_IN_PROGRESS: ${{ (github.event.action == 'edited' || github.event.action == 'synchronize' || github.event.action == 'ready_for_review') && 'true' || 'false' }} + steps: + - name: Debounce bursty metadata events + if: ${{ github.event.action == 'labeled' || github.event.action == 'unlabeled' }} + run: sleep 20 + + - name: Create ClawSweeper dispatch token + id: token + if: ${{ env.HAS_CLAWSWEEPER_APP_PRIVATE_KEY == 'true' }} + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + client-id: ${{ env.CLAWSWEEPER_APP_CLIENT_ID }} + private-key: ${{ secrets.CLAWSWEEPER_APP_PRIVATE_KEY }} + owner: openclaw + repositories: clawsweeper + permission-contents: write + + - name: Pre-filter ClawSweeper comment + id: comment_filter + if: ${{ github.event_name == 'issue_comment' }} + env: + COMMENT_BODY: ${{ github.event.comment.body }} + run: | + set -euo pipefail + if grep -Eiq '(^|[[:space:]])@(clawsweeper|openclaw-clawsweeper)\b(\[bot\])?|(^|[[:space:]])/(clawsweeper|review|autoclose|auto([[:space:]]+|-)?merge)\b' <<< "$COMMENT_BODY"; then + echo "is_command=true" >> "$GITHUB_OUTPUT" + else + echo "is_command=false" >> "$GITHUB_OUTPUT" + fi + + - name: Create target comment token + id: target_token + if: >- + ${{ + github.event_name == 'issue_comment' && + steps.comment_filter.outputs.is_command == 'true' && + env.HAS_CLAWSWEEPER_APP_PRIVATE_KEY == 'true' + }} + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + client-id: ${{ env.CLAWSWEEPER_APP_CLIENT_ID }} + private-key: ${{ secrets.CLAWSWEEPER_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: ${{ github.event.repository.name }} + permission-issues: write + permission-pull-requests: read + + - name: Dispatch exact ClawSweeper review + if: ${{ github.event_name != 'issue_comment' }} + env: + GH_TOKEN: ${{ steps.token.outputs.token }} + TARGET_REPO: ${{ github.repository }} + ITEM_NUMBER: ${{ github.event.issue.number || github.event.pull_request.number }} + ITEM_KIND: ${{ github.event_name == 'pull_request_target' && 'pull_request' || 'issue' }} + SOURCE_EVENT: ${{ github.event_name }} + SOURCE_ACTION: ${{ github.event.action }} + run: | + if [ -z "$GH_TOKEN" ]; then + echo "::notice::Skipping ClawSweeper dispatch because no dispatch credential is configured." + exit 0 + fi + ingress_fingerprint="$(node <<'NODE' + const crypto = require("node:crypto"); + const fs = require("node:fs"); + const event = JSON.parse(fs.readFileSync(process.env.GITHUB_EVENT_PATH, "utf8")); + const pullRequest = event.pull_request && typeof event.pull_request === "object" + ? event.pull_request + : {}; + const headSha = String(pullRequest.head?.sha || "").trim().toLowerCase(); + const updatedAt = String(pullRequest.updated_at || "").trim(); + if ( + process.env.ITEM_KIND !== "pull_request" || + !/^[0-9a-f]{40}$/.test(headSha) || + !updatedAt + ) { + process.stdout.write(""); + } else { + process.stdout.write( + crypto + .createHash("sha256") + .update( + JSON.stringify({ + version: 1, + target_repo: String(process.env.TARGET_REPO || "").toLowerCase(), + item_number: Number(process.env.ITEM_NUMBER), + action: String(process.env.SOURCE_ACTION || ""), + head_sha: headSha, + updated_at: updatedAt, + body: typeof pullRequest.body === "string" ? pullRequest.body : "", + label: String(event.label?.name || ""), + }), + ) + .digest("hex"), + ); + } + NODE + )" + payload="$(jq -nc \ + --arg target_repo "$TARGET_REPO" \ + --argjson item_number "$ITEM_NUMBER" \ + --arg item_kind "$ITEM_KIND" \ + --arg source_event "$SOURCE_EVENT" \ + --arg source_action "$SOURCE_ACTION" \ + --arg ingress_fingerprint "$ingress_fingerprint" \ + --argjson supersedes_in_progress "$SUPERSEDES_IN_PROGRESS" \ + '{event_type:"clawsweeper_item",client_payload:({target_repo:$target_repo,item_number:$item_number,item_kind:$item_kind,source_event:$source_event,source_action:$source_action,supersedes_in_progress:$supersedes_in_progress} + (if $ingress_fingerprint != "" then {ingress_route:"target_dispatcher",ingress_fingerprint:$ingress_fingerprint} else {} end))}')" + gh api repos/openclaw/clawsweeper/dispatches \ + --method POST \ + --input - <<< "$payload" + + - name: Acknowledge and dispatch ClawSweeper comment + if: >- + ${{ + github.event_name == 'issue_comment' && + steps.comment_filter.outputs.is_command == 'true' + }} + env: + DISPATCH_TOKEN: ${{ steps.token.outputs.token }} + TARGET_TOKEN: ${{ steps.target_token.outputs.token }} + TARGET_REPO: ${{ github.repository }} + ITEM_NUMBER: ${{ github.event.issue.number }} + COMMENT_ID: ${{ github.event.comment.id }} + COMMENT_BODY: ${{ github.event.comment.body }} + AUTHOR_ASSOCIATION: ${{ github.event.comment.author_association }} + SOURCE_ACTION: ${{ github.event.action }} + run: | + if [ -z "$DISPATCH_TOKEN" ]; then + echo "::notice::Skipping ClawSweeper dispatch because no dispatch credential is configured." + exit 0 + fi + body_file="$RUNNER_TEMP/clawsweeper-comment-body.txt" + printf '%s\n' "$COMMENT_BODY" > "$body_file" + if grep -Eiq ')' "$body_file"; then + echo "Ignoring ClawSweeper proof-nudge comment." + exit 0 + fi + if [ -n "$TARGET_TOKEN" ]; then + GH_TOKEN="$TARGET_TOKEN" gh api -X POST \ + -H "Accept: application/vnd.github+json" \ + "repos/$TARGET_REPO/issues/comments/$COMMENT_ID/reactions" \ + -f content="eyes" >/dev/null || true + fi + status_comment_id="" + if [ -n "$TARGET_TOKEN" ]; then + case "$AUTHOR_ASSOCIATION" in + OWNER|MEMBER|COLLABORATOR) + status_body="$(printf '%s\n' \ + "" \ + "🦞👀" \ + "ClawSweeper picked this up." \ + "" \ + "Command router queued. I will update this comment with the next step.")" + status_payload="$(jq -nc --arg body "$status_body" '{body:$body}')" + status_err="$(mktemp)" + if status_response="$(GH_TOKEN="$TARGET_TOKEN" gh api \ + "repos/$TARGET_REPO/issues/$ITEM_NUMBER/comments" \ + --method POST \ + --input - <<< "$status_payload" 2>"$status_err")"; then + status_comment_id="$(jq -r '.id // empty' <<< "$status_response")" + else + cat "$status_err" >&2 + echo "::warning::Could not create ClawSweeper queued status comment; dispatching command router without one." + fi + rm -f "$status_err" + ;; + esac + fi + payload="$(jq -nc \ + --arg target_repo "$TARGET_REPO" \ + --argjson item_number "$ITEM_NUMBER" \ + --argjson comment_id "$COMMENT_ID" \ + --arg status_comment_id "$status_comment_id" \ + --arg source_event "issue_comment" \ + --arg source_action "$SOURCE_ACTION" \ + '{event_type:"clawsweeper_comment",client_payload:({target_repo:$target_repo,item_number:$item_number,comment_id:$comment_id,source_event:$source_event,source_action:$source_action,max_comments:"1"} + (if $status_comment_id != "" then {status_comment_id:($status_comment_id|tonumber)} else {} end))}')" + GH_TOKEN="$DISPATCH_TOKEN" gh api repos/openclaw/clawsweeper/dispatches \ + --method POST \ + --input - <<< "$payload" diff --git a/.github/workflows/conformance-nightly.yml b/.github/workflows/conformance-nightly.yml index 121a0142..52a8d7da 100644 --- a/.github/workflows/conformance-nightly.yml +++ b/.github/workflows/conformance-nightly.yml @@ -22,7 +22,7 @@ concurrency: env: NODE_VERSION: 24 - PNPM_VERSION: 10.33.2 + PNPM_VERSION: 10.34.5 jobs: conformance-mock: diff --git a/.github/workflows/crabbox-hydrate.yml b/.github/workflows/crabbox-hydrate.yml index 4bd56680..fc1c4b62 100644 --- a/.github/workflows/crabbox-hydrate.yml +++ b/.github/workflows/crabbox-hydrate.yml @@ -30,7 +30,7 @@ permissions: contents: read env: - PNPM_VERSION: 10.33.2 + PNPM_VERSION: 10.34.5 jobs: hydrate: diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml index fcbee8ca..ad1d9733 100644 --- a/.github/workflows/release-binaries.yml +++ b/.github/workflows/release-binaries.yml @@ -36,7 +36,7 @@ permissions: contents: read env: - PNPM_VERSION: 10.33.2 + PNPM_VERSION: 10.34.5 # Pinned exactly. The shipped binary *is* this Node build with a blob injected, # so a floating version silently changes the release's largest attack surface # (V8, OpenSSL, zlib) and makes builds non-reproducible. The SBOM records this diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 931e6ae8..4a708a0c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,7 +22,7 @@ concurrency: env: NODE_VERSION: 24 - PNPM_VERSION: 10.33.2 + PNPM_VERSION: 10.34.5 jobs: release: diff --git a/.github/workflows/supply-chain.yml b/.github/workflows/supply-chain.yml index e178aaef..f6b756e2 100644 --- a/.github/workflows/supply-chain.yml +++ b/.github/workflows/supply-chain.yml @@ -19,7 +19,7 @@ permissions: contents: read env: - PNPM_VERSION: 10.33.2 + PNPM_VERSION: 10.34.5 NODE_VERSION: 24.11.0 jobs: @@ -175,6 +175,11 @@ jobs: # Copyleft terms would change acpx's distribution terms, and the # single executable ships every dependency inside one file. deny-licenses: AGPL-3.0, GPL-3.0 + # Dependency Review reported EPL-2.0 OR GPL-3.0-or-later for the + # dev-only replay viewer dependency; acpx elects EPL-2.0. + # This exemption skips every license check for exactly that package version; + # any elkjs upgrade must be reviewed and re-pinned explicitly. + allow-dependencies-licenses: pkg:npm/elkjs@0.12.0 audit: name: Dependency audit @@ -203,8 +208,8 @@ jobs: # the build on one trains people to ignore this job. run: | set -uo pipefail - # Distinguish "found advisories" from "could not run". pnpm 10.33.2 - # crashes on a gzipped audit response with + # Distinguish "found advisories" from "could not run". Some pnpm + # versions crash on a gzipped audit response with # `Unexpected token '\x1f' ... is not valid JSON`, which is a # transport bug, not a finding. Failing the build on it would make a # red audit job mean nothing, and a real advisory would be ignored diff --git a/AGENTS.md b/AGENTS.md index eb19e1c2..f8a06520 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,6 +9,7 @@ Use these files for the other concerns: - [`README.md`](README.md) for user-facing install and usage - [`docs/CLI.md`](docs/CLI.md) for CLI reference +- [`docs/PERFORMANCE.md`](docs/PERFORMANCE.md) for reusable benchmark guidance - [`VISION.md`](VISION.md) for product direction and boundaries - [`CONTRIBUTING.md`](CONTRIBUTING.md) for PR expectations - [`skills/acpx/SKILL.md`](skills/acpx/SKILL.md) for agent-usage guidance @@ -22,9 +23,9 @@ instead of expanding this file into a full technical spec. - npm: `https://www.npmjs.com/package/acpx` - Default branch: `main` - Runtime: Node.js `>=22.13.0` -- Package manager: `pnpm@10.33.2` +- Package manager: `pnpm@10.34.5` - Clean Node 22.13 setups can have stale Corepack signing keys; install pnpm - with `npm install -g pnpm@10.33.2` if `corepack prepare` fails. + with `npm install -g pnpm@10.34.5` if `corepack prepare` fails. ## Product Direction @@ -171,6 +172,7 @@ Harness documentation synchronization policy: - `pnpm run mutate` — Stryker mutation check for the configured target - `pnpm run check` — format, typecheck, lint, build, and coverage tests - `pnpm run check:docs` — docs format and markdown lint +- `pnpm run perf:benchmark` — compare already-built worktrees with paired measurements - `pnpm run perf:report` — performance reporting helper ## Testing And Changelog Guidelines diff --git a/CHANGELOG.md b/CHANGELOG.md index 17cfd490..16d7648b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,75 @@ Repo: https://github.com/openclaw/acpx ### Fixes +- Runtime/agents: terminate owned adapter process groups on POSIX and + best-effort tracked process trees on Windows during normal and failed startup + cleanup so package-exec wrappers do not leave captured descendants running + after ACPX exits; exit-triggered snapshots are observed before cleanup + completes, POSIX descendants are captured before signaling so later session + or group changes cannot escape forced cleanup, owned POSIX group members + created during shutdown are discovered and re-signaled, external process-list + discovery is bounded, Linux process identities come from procfs, other POSIX + snapshots use a fixed locale, adapter wrappers are sampled with bounded + exponential backoff through multi-stage launches, escaped descendants are + extended from identity-validated parents after root exit, transient discovery + failures preserve and still signal captured targets, and Windows parent edges + plus remembered descendants are identity-checked before later signaling. + +- Runtime/queue: publish queue-owner leases atomically, preserve fresh malformed + locks during collisions, serialize cross-process refresh, release, and stale + cleanup through inode-qualified, crash-recoverable claims, revalidate stale + owners while holding cleanup claims, retry explicit cleanup across heartbeat + contention while making unresolved explicit cleanup fail visibly, reacquire + atomically with fresh timestamps after retiring a stale collision, and treat + only a fresh, live lease-before-bind owner as a fast startup miss while + retaining a retry window that covers the full startup grace period, preventing + premature `QUEUE_NOT_ACCEPTING_REQUESTS` errors without masking older + unreachable owners, letting released owners overwrite successors, or leaving + stale dangling-symlink locks unrecoverable. + +## 2026.7.27 (v0.13.0) + +### Highlights + +- Windows agent launches now use structured argv end to end. Unambiguous legacy `command` plus `args` entries migrate automatically; ambiguous/raw commands and `.sh` wrappers must move to `agents..argv`, and existing saved custom-agent sessions without argv must be recreated. +- Built-in Pool and ZeroClaw support makes both native ACP stdio servers available without custom registry configuration. +- The new `--no-fs` flag lets compatible agents use their native filesystem implementation instead of ACP client filesystem methods. +- The dependency and pnpm refresh resolves all four known PostCSS, fast-uri, js-yaml, and brace-expansion advisories. + +### Changes + +- Agents/built-ins: add Pool via `pool acp`. Thanks @dan-roberts-poolside and @osolmaz. + +- Agents/built-ins: add ZeroClaw via `zeroclaw acp`, ZeroClaw's native ACP v1 stdio server. Thanks @JordanTheJet. + +- CLI/ACP: add `--no-fs` to disable advertised ACP file read/write capabilities so compatible agents can use their native filesystem implementation. Thanks @zgxkbtl. + +- CLI/timers: preserve tiny positive timeout and TTL values from flags or config as 1 ms instead of disabling timers, and reject delays beyond Node's supported timer range. Thanks @realmehmetali. + +- Dependencies/tooling: refresh the ACP SDK, runtime and development toolchain, update pnpm to 10.34.5, and resolve the PostCSS, fast-uri, js-yaml, and brace-expansion advisories. + +### Breaking + +- Windows agent launches now require structured `agents..argv`; unambiguous legacy `command` plus `args` entries migrate automatically, while raw, ambiguous, or directly executable `.sh` commands fail with explicit migration guidance instead of lossy parsing or CreateProcess ENOENT. Existing custom-agent sessions without saved argv must be recreated. Fixes #466. Thanks @MarcelCFritsche. + +### Fixes + +- Flows: swallow best-effort heartbeat write failures at the timer boundary so storage errors do not become unhandled promise rejections. Thanks @SebTardif. + +- Runtime/sessions: use collision-resistant temporary paths for concurrent atomic session and index writes. Thanks @henkterharmsel. + +- CLI/status: report a normal cold-start session as `agent starting` while preserving `needs reconnect` for an unreachable live owner. Thanks @guettli. + +## 2026.7.23 (v0.12.1) + +### Changes + +- Agents/built-ins: refresh the default Pi, Codex, Claude, and Mux adapter ranges. Thanks @kelvinschen and @TheAngryPit. + +### Breaking + +### Fixes + - Session queue owner: capture a bounded owner stderr tail and exit status during cold start only (stop retaining at first IPC accept; keep draining the pipe so long-lived owners are not killed by EPIPE) so a dead owner reports the real failure instead of a silent timeout. Thanks @SebTardif. ## 2026.7.4 (v0.12.0) diff --git a/README.md b/README.md index d67105d9..9492bf31 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,7 @@ One command surface for Pi, OpenClaw ACP, Codex, Claude, and other ACP-compatibl - **Session export/import**: move portable session archives between machines - **Local status checks**: `status` reports running/idle/dead/no-session, pid, uptime, last prompt - **Client methods**: stable `fs/*` and `terminal/*` handlers with permission controls and cwd sandboxing +- **Capability opt-outs**: let adapters use their native filesystem or terminal tools with `--no-fs` and `--no-terminal` - **Auth handshake**: stable `authenticate` support via env/config credentials - **Structured output**: typed ACP messages (thinking, tool calls, diffs) instead of ANSI scraping - **Any ACP agent**: built-in registry + `--agent` escape hatch for custom servers @@ -148,7 +149,8 @@ acpx codex --file - "extra context" # explicit stdin + appended args acpx codex --no-wait 'draft test migration plan' # enqueue without waiting if session is busy acpx codex cancel # cooperative cancel of in-flight prompt acpx codex set-mode auto # session/set_mode (adapter-defined mode id) -acpx codex set model 'gpt-5.2[high]' # adapter-advertised model control +acpx codex set model gpt-5.6-sol # select the advertised base model +acpx codex set reasoning_effort max # set the advertised reasoning effort acpx exec 'summarize this repo' # default agent shortcut (codex) acpx codex exec 'what does this repo do?' # one-shot, no saved session @@ -218,6 +220,7 @@ acpx flow run ./my-flow.ts --input-file ./flow-input.json acpx --timeout 1800 flow run ./my-flow.ts acpx --format quiet codex 'final recommendation only' acpx --suppress-reads codex exec 'show tool activity without dumping file bodies' +acpx --no-fs codex exec 'read files without ACP client filesystem delegation' acpx --timeout 90 codex 'investigate intermittent test timeout' acpx --ttl 30 codex 'keep queue owner alive for quick follow-ups' @@ -282,7 +285,7 @@ Supported keys: "timeout": null, "format": "text", "agents": { - "my-custom": { "command": "./bin/my-acp-server", "args": ["acp"] } + "my-custom": { "argv": ["./bin/my-acp-server", "acp"] } }, "auth": { "my_auth_method_id": "credential-value" @@ -291,6 +294,9 @@ Supported keys: ``` Use `acpx config show` to inspect the resolved result and `acpx config init` to create the global template. +Use structured `agents..argv` for custom launches; it is required on Windows. Legacy +`command` plus `args` entries migrate when `command` is an unquoted executable with no whitespace, +while raw command strings remain Unix-only. For ACP `authenticate` handshakes, use either config `auth` entries or explicit `ACPX_AUTH_` environment variables such as `ACPX_AUTH_OPENAI_API_KEY`. @@ -361,11 +367,13 @@ Built-ins: | `kilocode` | `npx -y @kilocode/cli acp` | [Kilocode](https://kilocode.ai) | | `kimi` | native (`kimi acp`) | [Kimi CLI](https://github.com/MoonshotAI/kimi-cli) | | `kiro` | native (`kiro-cli-chat acp`) | [Kiro CLI](https://kiro.dev) | -| `mux` | `npx -y mux@^0.27.0 acp` | [Mux](https://mux.coder.com) | +| `mux` | `mux acp` via an ACPX-owned npm range | [Mux](https://mux.coder.com) | | `opencode` | `npx -y opencode-ai acp` | [OpenCode](https://opencode.ai) | +| `pool` | native (`pool acp`) | [Poolside](https://poolside.ai) | | `qoder` | native (`qodercli --acp`) | [Qoder CLI](https://docs.qoder.com/cli/acp) | | `qwen` | native (`qwen --acp`) | [Qwen Code](https://github.com/QwenLM/qwen-code) | | `trae` | native (`traecli acp serve`) | [Trae CLI](https://docs.trae.cn/cli) | +| `zeroclaw` | native (`zeroclaw acp`) | [ZeroClaw](https://github.com/zeroclaw-labs/zeroclaw) | `factory-droid` and `factorydroid` also resolve to the built-in `droid` adapter. diff --git a/agents/Codex.md b/agents/Codex.md index 5968bf5e..064deebb 100644 --- a/agents/Codex.md +++ b/agents/Codex.md @@ -3,6 +3,7 @@ - Built-in name: `codex` - Default command: `npx -y @agentclientprotocol/codex-acp` - Upstream: https://github.com/agentclientprotocol/codex-acp -- Runtime controls exposed by current codex-acp releases include ACP modes and an advertised model session config option. -- Reasoning effort is encoded in advertised Codex model ids such as `gpt-5.2[high]` when the adapter reports those variants. -- `acpx --model codex ...` and `acpx codex set model ` apply the requested model through the advertised config option; legacy adapters that advertise `models` use `session/set_model`. +- ACPX owns the built-in package range so fresh launches use the repository-selected stable adapter line without requiring a global install. +- Runtime controls exposed by current codex-acp releases include ACP modes plus separate `model` and `reasoning_effort` session config options. +- Use the advertised base model id with `acpx --model codex ...` or `acpx codex set model `, then set reasoning effort separately with `acpx codex set reasoning_effort `. +- Legacy `models` metadata may encode both values in a combined id such as `gpt-5.6-sol[max]`; ACPX uses that form only when the adapter does not advertise the newer model config option. diff --git a/agents/Mux.md b/agents/Mux.md index 233a37f3..343bffc7 100644 --- a/agents/Mux.md +++ b/agents/Mux.md @@ -1,7 +1,7 @@ # Mux - Built-in name: `mux` -- Default command: `npx -y mux@^0.27.0 acp` +- Default entrypoint: `mux acp` via an ACPX-owned npm range - Upstream: https://mux.coder.com/integrations/acp `acpx mux` starts coder/mux through its ACP stdio bridge (`mux acp`). `mux acp` auto-starts an in-process mux server, so a separate `mux server` is not required. diff --git a/agents/Pool.md b/agents/Pool.md new file mode 100644 index 00000000..37de7bff --- /dev/null +++ b/agents/Pool.md @@ -0,0 +1,27 @@ +# Pool + +- Built-in name: `pool` +- Default command: `pool acp` +- Upstream: [Poolside pool](https://github.com/poolsideai/pool) + +`acpx pool` launches the installed `pool` CLI through its ACP stdio entrypoint. Install `pool` using the [official instructions](https://github.com/poolsideai/pool#install), then authenticate the CLI before using it through `acpx`; `pool login` is the normal interactive path. Pool reads its configuration from `~/.config/poolside/`. + +Examples: + +```bash +acpx pool sessions new +acpx pool 'review this branch' +acpx pool exec 'summarize this repository' +``` + +If the binary lives outside `PATH` or needs extra startup arguments, override the built-in argv in `~/.acpx/config.json`: + +```json +{ + "agents": { + "pool": { + "argv": ["/opt/pool/bin/pool", "acp"] + } + } +} +``` diff --git a/agents/Qwen.md b/agents/Qwen.md index 530bbd94..0784218a 100644 --- a/agents/Qwen.md +++ b/agents/Qwen.md @@ -3,3 +3,18 @@ - Built-in name: `qwen` - Default command: `qwen --acp` - Upstream: https://github.com/QwenLM/qwen-code + +## Filesystem delegation + +By default, acpx advertises ACP `fs/read_text_file` and `fs/write_text_file`, so +Qwen delegates its file tools to the acpx filesystem proxy. Use `--no-fs` when +Qwen must use its native filesystem service instead, such as when it needs to +read or write files in its own runtime temporary directories: + +```bash +acpx --no-fs qwen exec 'inspect the runtime artifact and summarize it' +``` + +The flag advertises both filesystem capabilities as `false` for the new ACP +client connection. It is independent of `--no-terminal`; combine both flags +when Qwen should use neither acpx filesystem nor terminal delegation. diff --git a/agents/README.md b/agents/README.md index 42cdcee8..520676a1 100644 --- a/agents/README.md +++ b/agents/README.md @@ -16,11 +16,13 @@ Built-in agents: - `kilocode -> npx -y @kilocode/cli acp` - `kimi -> kimi acp` - `kiro -> kiro-cli-chat acp` -- `mux -> npx -y mux@^0.27.0 acp` +- `mux -> mux acp` via an ACPX-owned npm range - `opencode -> npx -y opencode-ai acp` +- `pool -> pool acp` - `qoder -> qodercli --acp` - `qwen -> qwen --acp` - `trae -> traecli acp serve` +- `zeroclaw -> zeroclaw acp` Harness-specific docs in this directory: @@ -36,8 +38,10 @@ Harness-specific docs in this directory: - [Kilocode](Kilocode.md): built-in `kilocode -> npx -y @kilocode/cli acp` - [Kimi](Kimi.md): built-in `kimi -> kimi acp` - [Kiro](Kiro.md): built-in `kiro -> kiro-cli-chat acp` -- [Mux](Mux.md): built-in `mux -> npx -y mux@^0.27.0 acp` +- [Mux](Mux.md): built-in `mux -> mux acp` via an ACPX-owned npm range - [OpenCode](OpenCode.md): built-in `opencode -> npx -y opencode-ai acp` +- [Pool](Pool.md): built-in `pool -> pool acp` - [Qoder](Qoder.md): built-in `qoder -> qodercli --acp` - [Qwen](Qwen.md): built-in `qwen -> qwen --acp` - [Trae](Trae.md): built-in `trae -> traecli acp serve` +- [ZeroClaw](ZeroClaw.md): built-in `zeroclaw -> zeroclaw acp` diff --git a/agents/ZeroClaw.md b/agents/ZeroClaw.md new file mode 100644 index 00000000..2620f99b --- /dev/null +++ b/agents/ZeroClaw.md @@ -0,0 +1,57 @@ +# ZeroClaw + +- Built-in name: `zeroclaw` +- Default command: `zeroclaw acp` +- Upstream: https://github.com/zeroclaw-labs/zeroclaw + +`acpx zeroclaw` starts ZeroClaw's built-in ACP server (`zeroclaw acp`), a JSON-RPC 2.0 +stdio server that implements Agent Client Protocol v1 (`protocolVersion: 1`, no auth +methods). The `channel-acp-server` feature it needs is included in ZeroClaw's default +build, so a standard `zeroclaw` install works out of the box; a binary compiled without +it exits, reporting that the `channel-acp-server` feature is required. + +## Agent selection + +acpx's `session/new` does not pass a ZeroClaw agent alias, so the server picks the +session's agent from your ZeroClaw config in this order: + +1. `acp.default_agent`, when set. +2. The single `[agents.]` entry, when exactly one agent is configured. + +Single-agent setups therefore need no extra configuration. Multi-agent configs must set +`acp.default_agent`, otherwise `session/new` fails, reporting that it requires an +`agentAlias`. + +## Notes + +- Text prompts only — the server advertises no image, audio, or embedded-context prompt + capability. +- Session resume/close (acpx `ensure` and reuse) is advertised when ZeroClaw's ACP + session store is available. +- MCP tools are opt-in per agent via `[agents.].acp_enable_mcp` (off by default); + the `mcpServers` acpx sends on `session/new` are ignored. +- `zeroclaw acp --max-sessions ` and `--session-timeout ` bound concurrency. + Override the built-in command in acpx config to pass them. + +## Connecting to a running gateway + +To route acpx to an already-running ZeroClaw gateway/daemon instead of spawning a fresh +in-process server, override the built-in command to use `zeroclaw-acp-bridge`, which +bridges stdio to the gateway's ACP-over-WebSocket endpoint: + +```json +{ + "agents": { + "zeroclaw": { + "command": "zeroclaw-acp-bridge" + } + } +} +``` + +The bridge is a separate Cargo binary target and is not included in ZeroClaw's normal +prebuilt archives or installer, so build/install `zeroclaw-acp-bridge` from the ZeroClaw +source tree before using this override. It reads the gateway URL from the local ZeroClaw +config. When gateway pairing is active, run `zeroclaw gateway get-paircode --new`, then +start the bridge once with `zeroclaw-acp-bridge --pair-code ` to cache a token; an +existing token can instead be supplied through `ZEROCLAW_ACP_BRIDGE_TOKEN`. diff --git a/docs/CLI.md b/docs/CLI.md index 0984384c..30e545da 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -118,6 +118,7 @@ All global options: | `--format ` | Output format | `text` (default), `json`, `quiet`. | | `--suppress-reads` | Suppress read file contents | Replaces raw read payloads with `[read output suppressed]`. | | `--json-strict` | Strict JSON mode | Requires `--format json`; suppresses non-JSON stderr output. | +| `--no-fs` | Disable ACP filesystem capabilities | Advertises `clientCapabilities.fs.readTextFile` and `writeTextFile` as `false` during ACP initialize for new agent clients. | | `--no-terminal` | Disable ACP terminal capability | Advertises `clientCapabilities.terminal: false` during ACP initialize for new agent clients. | | `--non-interactive-permissions ` | Non-TTY prompt policy | `deny` (default) or `fail` when approval prompt cannot be shown. | | `--permission-policy ` | Per-tool permission policy | JSON object or file path with `autoApprove`, `autoDeny`, `escalate`, and optional `defaultAction` (`approve`, `deny`, `escalate`). Alias: `--policy`. | @@ -140,6 +141,7 @@ acpx --policy '{"escalate":["execute"],"defaultAction":"deny"}' --format json co acpx --cwd ~/repos/api codex 'review auth middleware' acpx --format json codex exec 'summarize open TODO items' acpx --format json --json-strict codex exec 'machine-safe JSON output' +acpx --no-fs codex exec 'use agent-native file operations' acpx --no-terminal codex exec 'summarize without terminal capability' acpx --timeout 120 codex 'investigate flaky test failures' acpx --ttl 30 codex 'keep queue owner warm for quick follow-up' @@ -429,7 +431,7 @@ Supported keys: "timeout": null, "format": "text", "agents": { - "my-custom": { "command": "./bin/my-acp-server", "args": ["acp"] } + "my-custom": { "argv": ["./bin/my-acp-server", "acp"] } }, "auth": { "my_auth_method_id": "credential-value" @@ -439,6 +441,10 @@ Supported keys: CLI flags always override config values. +Custom agents should use structured `agents..argv`, which is required on Windows. Legacy +`command` plus `args` entries migrate when `command` is an unquoted executable with no whitespace. +Raw command strings, including `--agent`, are supported only on Unix. + For ACP `authenticate` handshakes, use either config `auth` entries or explicit `ACPX_AUTH_` environment variables such as `ACPX_AUTH_OPENAI_API_KEY`. Ambient provider env vars such as `OPENAI_API_KEY` are still passed through to @@ -446,7 +452,8 @@ child agents, but they do not trigger ACP auth-method selection on their own. ## `--agent` escape hatch -`--agent ` sets a raw adapter command explicitly. +`--agent ` sets a raw adapter command explicitly on Unix. On Windows, define a named +agent with `agents..argv` so executable and argument boundaries remain unambiguous. Examples: diff --git a/docs/PERFORMANCE.md b/docs/PERFORMANCE.md new file mode 100644 index 00000000..2b0a4362 --- /dev/null +++ b/docs/PERFORMANCE.md @@ -0,0 +1,120 @@ +# Performance Benchmarking + +Use `perf:benchmark` to compare already-built `acpx` worktrees with paired, +repeatable CLI measurements. The runner does not fetch, check out, install, or +build the target worktrees. + +## Prepare benchmark inputs + +Each input must be a Git worktree with: + +- a resolvable Git `HEAD`; and +- a built `dist/cli.js` produced with that worktree's declared Node.js and + package-manager versions. + +For each target, install from its frozen lockfile and run its build before the +benchmark. The `perf:benchmark` package script builds the shared local +benchmark agent, but it does not rebuild the target CLIs. + +The runner records each label, resolved worktree path, `HEAD` SHA, full Git +dirty state, and the validation-time SHA-256 of `dist/cli.js`. It rechecks that +expected digest immediately before every CLI spawn and after all invocations, +before constructing the final report, and fails if an observed digest differs. +These checks are not an immutable snapshot: a mutation in the narrow interval +between a recheck and process loading remains theoretically possible. It also +does not reject a dirty worktree or prove that the recorded binary was built +from `HEAD`. For reviewable results, use clean worktrees and read-only build +artifacts, rebuild after selecting each commit, and retain the build commands. + +## Run a comparison + +Invoke the benchmark from the worktree that contains the benchmark runner: + +```text +pnpm run perf:benchmark -- \ + --baseline baseline=/path/to/acpx-baseline \ + --candidate candidate=/path/to/acpx-candidate \ + --output /path/to/benchmark-results +``` + +`--baseline` is required once. `--candidate` is required and repeatable. Each +value uses `label=worktree` syntax, and every label must be unique. Relative or +absolute worktree paths are accepted and resolved before execution. + +The remaining options are: + +```text +--scenario Repeat to select scenarios; omit for all defaults +--samples Override measured samples for every scenario +--warmups Override warmups for every scenario +--seed Override the deterministic bootstrap seed +--output Select the report directory +``` + +The default seed is `2886672422`. Without `--output`, the runner creates a +temporary directory. It refuses to overwrite existing `results.json` or +`results.md` files and refuses concurrent use of the same output directory. + +## Default scenarios + +When no `--scenario` option is supplied, scenarios run in this order: + +| Scenario | Samples | Warmups | Measured behavior | +| ---------------- | ------: | ------: | ------------------------------------------------------------ | +| `version` | 100 | 15 | Process startup and `--version` | +| `help` | 80 | 12 | Process startup and help rendering | +| `local-sessions` | 50 | 8 | Empty local session listing without an ACP child | +| `agent-sessions` | 25 | 5 | Cold initialize, session-list RPC, and teardown | +| `exec` | 25 | 5 | Cold initialize, new session, prompt, response, and teardown | + +`--scenario` is repeatable, for example `--scenario version --scenario help`. +Global sample and warmup overrides apply to every selected scenario. + +The runner preflights every variant before warmups. Every CLI invocation gets +a new temporary HOME and working directory, including preflight, warmup, +measured, and diagnostic runs. This keeps session and configuration state from +one invocation out of the next. + +The ACP scenarios use the same local benchmark agent for every variant. They +do not benchmark a live provider, credentials, or network service. Do not +replace that agent with a live or network-dependent adapter: provider and +network variance are outside this benchmark's contract. + +## Interpret paired results + +Each candidate is measured against the baseline in adjacent pairs. Pair order +alternates between baseline-first and candidate-first. A candidate is compared +only with the baseline observation at the same pair index; with multiple +candidates, each candidate receives its own baseline sample series. + +Use the paired delta and confidence interval instead of comparing unpaired +summary means. Negative percentage deltas mean the candidate was faster. +Results describe the recorded host and run, not universal performance. + +The runner supports POSIX platforms only because teardown depends on POSIX +process groups. It also requires permission to enumerate processes so it can +prove descendant ownership during cleanup; it fails closed when enumeration is +unavailable. It rejects Windows before execution. + +## Outputs + +Each successful run writes: + +- `results.json`, the authoritative schema-versioned report with raw paired + samples, input SHAs, dirty states, CLI digests, paths, environment, + configuration, statistics, and diagnostics; and +- `results.md`, a concise human-readable rendering of the same report. + +Diagnostic trace and internal-metrics files may also appear below the selected +output directory. Headline measured runs do not write those diagnostics. The +command prints the path to `results.json` when it finishes. + +## `perf:benchmark` versus `perf:report` + +`pnpm run perf:benchmark -- ...` launches built worktree CLIs, performs paired +measurements, and writes `results.json` plus `results.md`. + +`pnpm run perf:report -- ` only aggregates an existing +`ACPX_PERF_METRICS_FILE` NDJSON stream and prints a JSON summary. It does not +launch worktree variants, create paired samples, or produce benchmark result +files. diff --git a/docs/agents.md b/docs/agents.md index 4990f51e..b03c8573 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -25,11 +25,13 @@ The default agent for top-level commands like `acpx exec …` and `acpx prompt | `kilocode` | `npx -y @kilocode/cli acp` | [Kilocode](https://kilocode.ai) | | `kimi` | `kimi acp` | [Kimi CLI](https://github.com/MoonshotAI/kimi-cli) | | `kiro` | `kiro-cli-chat acp` | [Kiro CLI](https://kiro.dev) | -| `mux` | `npx -y mux@^0.27.0 acp` | [Mux](https://mux.coder.com) | +| `mux` | `mux acp` via an ACPX-owned npm range | [Mux](https://mux.coder.com) | | `opencode` | `npx -y opencode-ai acp` | [OpenCode](https://opencode.ai) | +| `pool` | `pool acp` | [Poolside](https://poolside.ai) | | `qoder` | `qodercli --acp` | [Qoder CLI](https://docs.qoder.com/cli/acp) | | `qwen` | `qwen --acp` | [Qwen Code](https://github.com/QwenLM/qwen-code) | | `trae` | `traecli acp serve` | [Trae CLI](https://docs.trae.cn/cli) | +| `zeroclaw` | `zeroclaw acp` | [ZeroClaw](https://github.com/zeroclaw-labs/zeroclaw) | `factory-droid` and `factorydroid` also resolve to the built-in `droid` adapter. @@ -179,7 +181,7 @@ Configure model/provider settings through fast-agent environment variables, fast ### Mux - Built-in name: `mux` -- Default command: `npx -y mux@^0.27.0 acp` +- Default entrypoint: `mux acp` via an ACPX-owned npm range - Upstream: https://mux.coder.com/integrations/acp `acpx mux` starts coder/mux through its ACP stdio bridge (`mux acp`). `mux acp` auto-starts an in-process mux server, so a separate `mux server` is not required. @@ -192,6 +194,14 @@ Configure at least one model provider before prompting (for example `ANTHROPIC_A - Default command: `npx -y opencode-ai acp` - Upstream: [opencode.ai](https://opencode.ai) +### Pool + +- Built-in name: `pool` +- Default command: `pool acp` +- Upstream: [Poolside](https://poolside.ai) + +`acpx pool` uses the installed `pool` CLI ACP server (`pool acp`). Install and authenticate the CLI first; `pool login` is the normal interactive path. Focused setup notes live in `agents/Pool.md`. + ### Qwen - Built-in name: `qwen` @@ -204,6 +214,15 @@ Configure at least one model provider before prompting (for example `ANTHROPIC_A - Default command: `traecli acp serve` - Upstream: [docs.trae.cn](https://docs.trae.cn/cli) +### ZeroClaw + +- Built-in name: `zeroclaw` +- Default command: `zeroclaw acp` +- Upstream: [zeroclaw-labs/zeroclaw](https://github.com/zeroclaw-labs/zeroclaw) +- `zeroclaw acp` is ZeroClaw's native JSON-RPC 2.0 stdio server for ACP v1 (`protocolVersion: 1`, no auth methods). The `channel-acp-server` feature it needs ships in ZeroClaw's default build. +- `session/new` does not carry a ZeroClaw agent alias, so the server binds the session to `acp.default_agent` when set, otherwise to the sole `[agents.]` entry. Single-agent configs work as-is; multi-agent configs must set `acp.default_agent` or `session/new` fails. +- Text prompts only (no image, audio, or embedded-context capability). Session resume/close is advertised when the ZeroClaw ACP session store is available. MCP tools are opt-in per agent via `[agents.].acp_enable_mcp`. + ## Overriding a built-in Any built-in can be replaced wholesale through config, including `args` for adapter sub-commands: diff --git a/docs/compare.md b/docs/compare.md index b87423e3..27ff7f4b 100644 --- a/docs/compare.md +++ b/docs/compare.md @@ -36,6 +36,7 @@ prompt. | `--timeout ` | Per-agent timeout in seconds. | | `--non-interactive-permissions ` | Non-TTY prompt behavior. | | `--auth-policy ` | ACP authentication behavior. | +| `--no-fs` | Do not advertise filesystem support to agents. | | `--no-terminal` | Do not advertise terminal support to agents. | | `--prompt-retries ` | Retry failed prompt turns before any side effects are observed. | | `--model`, `--allowed-tools`, `--max-turns`, `--system-prompt` | Session creation options forwarded to compatible agents. | diff --git a/docs/config.md b/docs/config.md index be57b4e9..571cfa2a 100644 --- a/docs/config.md +++ b/docs/config.md @@ -46,7 +46,7 @@ acpx config init } ], "agents": { - "my-custom": { "command": "./bin/my-acp-server", "args": ["acp"] } + "my-custom": { "argv": ["./bin/my-acp-server", "acp"] } }, "auth": { "openai_api_key": "sk-…" @@ -88,12 +88,10 @@ Custom agents and overrides live here: { "agents": { "my-agent": { - "command": "./bin/my-acp-server", - "args": ["acp", "--profile", "ci"] + "argv": ["./bin/my-acp-server", "acp", "--profile", "ci"] }, "codex": { - "command": "/usr/local/bin/codex-acp", - "args": ["--mode", "stable"] + "argv": ["/usr/local/bin/codex-acp", "--mode", "stable"] } } } @@ -102,15 +100,18 @@ Custom agents and overrides live here: Rules: - Keys are friendly names you would type at `acpx …`. -- `command` is required; it can be a single executable or include in-string args (`"node ./bin/x.mjs"`). -- `args` is optional. If present, it is appended after the parsed `command` tokens. -- Custom agent `args` arrays are honored — required adapter sub-commands are no longer dropped silently. +- `argv` is the preferred form and is required for custom agent launches on Windows. Its first item is the executable and every remaining item is passed literally as one argument. +- Legacy `{ "command": "…", "args": […] }` entries migrate when `command` is an unquoted executable with no whitespace. Quoted executables and inline arguments are rejected as ambiguous; move the complete launch to `argv`. +- A legacy `command` without `args` remains a raw command string for Unix compatibility. Windows rejects it with migration guidance because inferring argv would corrupt paths and quoting. +- The raw `--agent ` escape hatch is likewise Unix-only. +- Windows cannot execute `.sh` files directly. Name the interpreter explicitly, for example `"argv": ["bash", "C:\\tools\\bin\\agent.sh"]`; acpx does not discover or infer an interpreter. +- On Windows, close and recreate custom-agent sessions created by an older acpx release so their records persist structured argv. Known built-in commands migrate automatically. - An entry that shares a name with a built-in **replaces** the built-in for that name. Project config can shadow global config by re-declaring the same key: ```json -{ "agents": { "codex": { "command": "/usr/local/bin/codex-acp" } } } +{ "agents": { "codex": { "argv": ["/usr/local/bin/codex-acp"] } } } ``` Use this to point a particular repo at a vendored or pinned adapter. diff --git a/docs/install.md b/docs/install.md index 67733a57..ceae8144 100644 --- a/docs/install.md +++ b/docs/install.md @@ -8,13 +8,13 @@ description: Install acpx globally with npm, run it ad-hoc with npx, or build fr ## Requirements - Node.js **22.13 or newer** (see `engines.node` in `package.json`) -- pnpm **10.33.2** for source builds +- pnpm **10.34.5** for source builds - The underlying coding agent CLI you plan to talk to (Codex, Claude, etc.) If pnpm is not installed yet, use npm: ```bash -npm install -g pnpm@10.33.2 +npm install -g pnpm@10.34.5 ``` Some older Corepack builds bundled with supported Node.js versions have stale @@ -74,7 +74,7 @@ For development or to test an unreleased branch: ```bash git clone https://github.com/openclaw/acpx.git cd acpx -npm install -g pnpm@10.33.2 # if pnpm is not already installed +npm install -g pnpm@10.34.5 # if pnpm is not already installed pnpm install pnpm run build node dist/cli.js --help diff --git a/docs/session-control.md b/docs/session-control.md index 3a34ee71..543df5a8 100644 --- a/docs/session-control.md +++ b/docs/session-control.md @@ -54,8 +54,11 @@ Calls ACP `session/set_config_option` with the literal `` and ``. No `set model ` is a special-case interception. `acpx` prefers an advertised model session config option and updates it through `session/set_config_option`. If an adapter explicitly advertises legacy `models` metadata instead, `acpx` preserves compatibility through `session/set_model`. +Current codex-acp releases advertise the base model and reasoning effort as separate config options. + ```bash -acpx codex set model 'gpt-5.2[high]' +acpx codex set model gpt-5.6-sol +acpx codex set reasoning_effort max acpx claude set model claude-sonnet-4-6 ``` diff --git a/docs/superpowers/plans/2026-08-02-three-way-performance-benchmark.md b/docs/superpowers/plans/2026-08-02-three-way-performance-benchmark.md new file mode 100644 index 00000000..3aeb2b3f --- /dev/null +++ b/docs/superpowers/plans/2026-08-02-three-way-performance-benchmark.md @@ -0,0 +1,583 @@ +# Three-Way ACPX Performance Benchmark Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build a reusable, statistically paired benchmark and use it to compare current OpenClaw `main`, OpenClaw PR #478, and Artagon performance PR #2. + +**Architecture:** Functional TypeScript in `scripts/perf/` owns validation, ordering, statistics, eager-graph analysis, and rendering. Statistical transforms remain pure; filesystem reads and process spawning stay at explicit boundaries. An exact-pinned Tinybench task engine executes one ordered adjacent pair at a time, while the outer functional runner retains preflight, warmups, pair identity and order, process isolation, timeouts, diagnostics, statistics, and reporting. A small functional ACP agent supplies one shared diagnostic fixture, while the runner spawns already-built checkouts and writes versioned JSON plus Markdown. Git fetching, worktree creation, dependency installation, and builds remain explicit controller operations outside the benchmark runner. + +**Tech Stack:** Node.js >=22.13.0, strict TypeScript, Node test runner, `tsx`, `tinybench@6.1.2`, ACP SDK, existing Oxlint/Oxfmt gates. + +## Global Constraints + +- Follow the existing repository's modular functional TypeScript style: small exported functions, explicit immutable data types, no application-defined benchmark classes, and side effects confined to executable boundaries. Tinybench's `Bench` class is allowed only inside `benchmark-engine.ts`. +- Do not use explicit `any`, unsafe assignment/calls/member access/returns, unchecked casts, or new dependencies other than exact-pinned `tinybench@6.1.2`. +- Tinybench SHALL be a thin task engine, not the experiment controller. Create one fresh non-concurrent `Bench` per adjacent pair; the outer runner SHALL retain preflight, scenario-specific warmups, pair identity and order, process isolation, timeouts, traces, statistics, and reports. +- Every pair `Bench` SHALL use `concurrency: null`, `time: 0`, `iterations: 1`, `warmup: false`, `retainSamples: true`, and `throws: true`. Register tasks in runner-selected order and execute each task exactly once. +- Each async Tinybench task SHALL return the finite, nonnegative externally measured spawn-to-close duration as `overriddenDuration`; Tinybench callback timing SHALL NOT replace the process measurement. +- Ordered `PairedSample[]` SHALL remain the authoritative raw data. Do not derive pair order or report statistics from Tinybench's sorted retained samples; the core SHALL continue to own p95 and deterministic paired bootstrap confidence intervals. +- The runner SHALL NOT fetch, check out, install, build, merge, rebase, force-push, delete worktrees, or otherwise mutate Git state. +- OpenClaw PR #478 SHALL remain unchanged; benchmark code belongs only to `perf/consolidated-runtime` and Artagon PR #2. +- Every measured variant SHALL use the same benchmark agent built from the benchmark-owning performance worktree. +- Headline measured runs SHALL NOT write lifecycle traces or `ACPX_PERF_METRICS_FILE`; diagnostics use separate samples. +- Pair order SHALL alternate, raw samples SHALL be retained, and paired log-ratio confidence intervals SHALL use a deterministic 10,000-resample bootstrap. +- Reports SHALL record exact SHAs, paths, environment, scenario configuration, seed, and schema version. +- Missing trace events and unsupported internal metrics SHALL be reported as unavailable, never as zero. +- Tests SHALL be written and observed failing before implementation for every new behavior. +- Artagon Node and TypeScript plugin skills are unavailable in this runtime; workers SHALL instead follow this repository's Node/TypeScript rules, type-aware lint, and full-check contract. +- No shell source file is planned. If a worker finds shell unavoidable, it SHALL stop and report `NEEDS_CONTEXT`; any approved shell must use the Artagon shell skills and Google Shell Style Guide. +- Workers are not alone in the codebase. They SHALL preserve others' edits and SHALL NOT revert or overwrite files outside their assigned ownership. + +--- + +### Task 1: Functional benchmark core + +**Files:** + +- Create: `scripts/perf/benchmark-core.ts` +- Create: `test/perf-benchmark-core.test.ts` +- Modify: `tsconfig.test.json` + +**Interfaces:** + +- Consumes: Node `fs`, `path`, `os`, and `zlib` only. +- Produces: + +```ts +export const BENCHMARK_SCHEMA_VERSION = 1; + +export type ScenarioName = "version" | "help" | "local-sessions" | "agent-sessions" | "exec"; + +export type VariantSpec = Readonly<{ + label: string; + worktree: string; +}>; + +export type SampleSummary = Readonly<{ + n: number; + meanMs: number; + medianMs: number; + p95Ms: number; + stddevMs: number; + minMs: number; + maxMs: number; +}>; + +export type PairedDelta = Readonly<{ + meanDeltaPct: number; + medianDeltaPct: number; + geometricMeanDeltaPct: number; + ci95Pct: readonly [number, number]; +}>; + +export type PairedSample = Readonly<{ + sampleIndex: number; + order: "baseline-first" | "candidate-first"; + baselineMs: number; + candidateMs: number; +}>; + +export type EagerGraphSummary = Readonly<{ + chunks: number; + bytes: number; + gzipBytes: number; + externalPackages: readonly string[]; + files: readonly string[]; +}>; + +export function parseVariantSpec(value: string, optionName: string): VariantSpec; +export function createPairOrder(baseline: T, candidate: T, sampleIndex: number): readonly [T, T]; +export function summarizeSamples(values: readonly number[]): SampleSummary; +export function calculatePairedDelta( + candidate: readonly number[], + baseline: readonly number[], + seed: number, + resamples?: number, +): PairedDelta; +export function analyzeEagerGraph(entryPath: string): EagerGraphSummary; +export function renderBenchmarkMarkdown(report: BenchmarkReport): string; +``` + +Define `BenchmarkReport` and its nested report types in the same module. They +must model methodology, environment, variants, eager graphs, scenarios, raw +samples, candidate comparisons, trace summaries, and internal-metrics support +without open-ended `Record` payloads. + +- [ ] **Step 1: Include benchmark TypeScript in the test build** + +Change `tsconfig.test.json` include to: + +```json +["src/**/*.ts", "scripts/perf/**/*.ts", "test/**/*.ts", "examples/flows/pr-triage/review-text.js"] +``` + +- [ ] **Step 2: Write failing core tests** + +Add focused Node tests that assert: + +```ts +assert.deepEqual(parseVariantSpec("main=/repo/main", "--baseline"), { + label: "main", + worktree: "/repo/main", +}); +assert.throws(() => parseVariantSpec("missing-separator", "--candidate"), /--candidate/u); +assert.deepEqual(createPairOrder("main", "pr", 0), ["main", "pr"]); +assert.deepEqual(createPairOrder("main", "pr", 1), ["pr", "main"]); +assert.deepEqual(summarizeSamples([1, 2, 3, 4, 100]), { + n: 5, + meanMs: 22, + medianMs: 3, + p95Ms: 100, + stddevMs: 43.617656975128774, + minMs: 1, + maxMs: 100, +}); +``` + +Use fixed baseline/candidate arrays to prove `calculatePairedDelta` returns the +same result twice with seed `0xac0f2026`, rejects unequal/empty/nonpositive +samples, and reports a negative delta for a faster candidate. Build an isolated +temporary eager graph containing local static imports, side-effect imports, +`node:` imports, and scoped/unscoped packages; assert exact sorted files, +packages, bytes, and gzip bytes. Construct a minimal typed report and assert +the Markdown includes schema version, exact SHAs, scenario headings, median, +p95, paired delta, and confidence interval. + +- [ ] **Step 3: Run the test and verify RED** + +Run: + +```bash +rtk pnpm run build:test +``` + +Expected: FAIL because `scripts/perf/benchmark-core.ts` does not exist or its +exports are missing. + +- [ ] **Step 4: Implement the minimal functional core** + +Use immutable inputs, local `Map`/`Set` accumulation, a seeded xorshift32 +generator, nearest-rank percentiles, sample standard deviation, and recursive +eager-import traversal. Reject non-finite/nonpositive timing samples and paths +that cannot be read. Round only when serializing/rendering; retain full +precision in calculations. + +- [ ] **Step 5: Verify GREEN** + +Run: + +```bash +rtk pnpm run build:test +rtk node --test dist-test/test/perf-benchmark-core.test.js +rtk pnpm run typecheck +rtk pnpm run lint +``` + +Expected: all commands pass with no warnings attributable to changed code. + +- [ ] **Step 6: Commit Task 1** + +```bash +rtk git add tsconfig.test.json scripts/perf/benchmark-core.ts test/perf-benchmark-core.test.ts +rtk git commit -m "feat: add reusable benchmark core" +``` + +### Task 2: Shared functional ACP benchmark agent + +**Files:** + +- Create: `scripts/perf/benchmark-agent.ts` +- Create: `test/perf-benchmark-agent.test.ts` + +**Interfaces:** + +- Consumes: `@agentclientprotocol/sdk`, Node streams, filesystem, crypto, and + `performance.timeOrigin + performance.now()`. +- Produces: + +```ts +export type BenchmarkTraceEventName = + | "agent.process_start" + | "agent.initialize.start" + | "agent.initialize.end" + | "agent.session_list.start" + | "agent.session_list.end" + | "agent.new_session.start" + | "agent.new_session.end" + | "agent.prompt.start" + | "agent.prompt.end" + | "agent.stdin_end" + | "agent.sigterm" + | "agent.exit"; + +export type BenchmarkTraceEvent = Readonly<{ + event: BenchmarkTraceEventName; + pid: number; + timestampMs: number; +}>; + +export function appendBenchmarkTrace( + traceFile: string | undefined, + event: BenchmarkTraceEventName, +): void; + +export function createBenchmarkAgent( + connection: AgentSideConnection, + traceFile: string | undefined, +): Agent; + +export function parseBenchmarkAgentArgs(argv: readonly string[]): Readonly<{ traceFile?: string }>; + +export async function runBenchmarkAgent(argv: readonly string[]): Promise; +``` + +The executable calls `runBenchmarkAgent(process.argv.slice(2))` only when its +module URL matches the invoked entrypoint. It accepts only optional +`--trace-file ` and rejects unknown or missing arguments. With no trace +file, `appendBenchmarkTrace` returns without filesystem work. + +- [ ] **Step 1: Write failing agent tests** + +Assert that `appendBenchmarkTrace(undefined, ...)` creates no file; a real trace +file receives valid newline-delimited JSON with the requested event, current +PID, and finite timestamp; two appends preserve order; and argument parsing +rejects unknown/missing flags. Exercise the built agent through the existing +CLI with `sessions list` and `exec`, then assert the trace contains ordered +initialize plus workload events and a terminal stdin/signal/exit event. + +- [ ] **Step 2: Run the test and verify RED** + +Run: + +```bash +rtk pnpm run build:test +``` + +Expected: FAIL because `scripts/perf/benchmark-agent.ts` does not exist. + +- [ ] **Step 3: Implement the minimal agent** + +Construct an `Agent` object literal from closures rather than a class. Support +initialize, authenticate, new session, session list, prompt, cancel, and the +minimal permission/config methods required by the SDK type. The prompt handler +sends one deterministic assistant text update and returns `end_turn`. Use +synchronous append only when tracing is enabled so exit/signal events are not +lost. + +- [ ] **Step 4: Verify GREEN** + +Run: + +```bash +rtk pnpm run build:test +rtk node --test dist-test/test/perf-benchmark-agent.test.js +rtk pnpm run typecheck +rtk pnpm run lint +``` + +Expected: all commands pass. + +- [ ] **Step 5: Commit Task 2** + +```bash +rtk git add scripts/perf/benchmark-agent.ts test/perf-benchmark-agent.test.ts +rtk git commit -m "test: add shared benchmark ACP agent" +``` + +### Task 3: Tinybench engine, benchmark runner, diagnostics, and package entrypoint + +**Files:** + +- Create: `scripts/perf/benchmark-engine.ts` +- Create: `test/perf-benchmark-engine.test.ts` +- Create: `scripts/perf/benchmark-runner.ts` +- Create: `test/perf-benchmark-runner.test.ts` +- Modify: `package.json` +- Modify: `pnpm-lock.yaml` +- Modify: `test/package-scripts.test.ts` + +**Interfaces:** + +- Consumes: exact-pinned `tinybench@6.1.2`, all Task 1 core exports, and Task 2's compiled agent at + `dist-test/scripts/perf/benchmark-agent.js`. +- Produces: + +```ts +export type BenchmarkEngineTask = Readonly<{ + name: string; + execute: () => Promise; +}>; + +export type BenchmarkEngineResult = Readonly<{ + name: string; + durationMs: number; +}>; + +export async function runBenchmarkPair( + tasks: readonly [BenchmarkEngineTask, BenchmarkEngineTask], +): Promise; + +export type BenchmarkOptions = Readonly<{ + baseline: VariantSpec; + candidates: readonly VariantSpec[]; + scenarios: readonly ScenarioName[]; + samplesOverride?: number; + warmupsOverride?: number; + seed: number; + outputDirectory?: string; +}>; + +export function parseBenchmarkArgs(argv: readonly string[]): BenchmarkOptions; +export async function runBenchmark(options: BenchmarkOptions): Promise; +export async function writeBenchmarkReport( + report: BenchmarkReport, + outputDirectory: string, +): Promise>; +``` + +Scenario names and defaults are exact: + +```ts +version: { samples: 100, warmups: 15 } +help: { samples: 80, warmups: 12 } +local-sessions: { samples: 50, warmups: 8 } +agent-sessions: { samples: 25, warmups: 5 } +exec: { samples: 25, warmups: 5 } +``` + +- [ ] **Step 1: Add the exact Tinybench dependency** + +Run: + +```bash +rtk pnpm add --save-dev --save-exact tinybench@6.1.2 +``` + +Expected: `package.json` and `pnpm-lock.yaml` pin exactly `6.1.2`; no semver +range or second benchmark framework is added. + +- [ ] **Step 2: Write failing engine, runner, and package tests** + +In `test/perf-benchmark-engine.test.ts`, use two async tasks that append their +names to an execution log and return distinct fixed durations. Assert the +engine preserves registration order, invokes each task exactly once, and +returns the exact external durations with their names. Call `runBenchmarkPair` +a second time with different tasks and durations; assert the second result and +execution log contain no state from the first call, proving a fresh `Bench` and +isolated results per pair. Assert zero is accepted and preserved exactly, while +`NaN`, positive infinity, negative infinity, and negative durations are each +rejected. Add a task that throws a sentinel error and assert +`runBenchmarkPair` rejects with that same failure. + +Assert repeated candidates/scenarios parse correctly; labels are unique; +counts and seed are validated; worktrees must have resolvable Git HEAD and +`dist/cli.js`; failed preflight and measured commands include label/scenario +and stderr tail in the error; pair ordering alternates independently for every +candidate; and a fake executable CLI produces versioned JSON/Markdown with raw +samples. Assert package scripts contain exactly: + +```json +"perf:benchmark": "pnpm run build:test && tsx scripts/perf/benchmark-runner.ts" +``` + +- [ ] **Step 3: Run the tests and verify RED** + +Run: + +```bash +rtk pnpm run build:test +``` + +Expected: FAIL because engine and runner exports and the package script do not +exist. + +- [ ] **Step 4: Implement the thin Tinybench engine** + +For every `runBenchmarkPair` call, create a fresh `Bench` with: + +```ts +{ + concurrency: null, + time: 0, + iterations: 1, + warmup: false, + retainSamples: true, + throws: true, +} +``` + +Register tasks in input order. Each async Tinybench callback calls `execute()`, +rejects a non-finite or negative duration, retains the exact duration by task +identity, and returns `{ overriddenDuration: durationMs }`. Run the bench once, +propagate task failures, and return the two retained external durations in +registration order. Zero is a valid duration at this engine boundary. Do not +expose Tinybench task/sample objects to the core or runner. + +- [ ] **Step 5: Implement argument and input validation** + +Use `node:util` `parseArgs`, resolve absolute paths, read Git SHAs by spawning +`git` with `-C`, the resolved worktree path, `rev-parse`, and `HEAD` as separate +argv entries, and validate every variant before creating output state. Do not +invoke a shell. + +- [ ] **Step 6: Implement measured and diagnostic runs** + +For every pair, create fresh per-run HOME and cwd directories before starting +the timer. Measure with `performance.now()` immediately before `spawn` through +the child `close` event. The runner selects alternating order and delegates each +adjacent measured pair to `runBenchmarkPair`; the task's returned duration is +the exact external measurement supplied to Tinybench as `overriddenDuration`. +Ignore measured stdout, drain stderr, and reject nonzero exits. Use the shared +agent command for ACP scenarios. Capture trace and internal metrics only in +separate diagnostic samples and derive stage summaries only when required +events are present. Preflight and warmups stay outside Tinybench. + +- [ ] **Step 7: Implement reports and atomic writes** + +Create the selected output directory without deleting existing data. Refuse to +overwrite existing `results.json` or `results.md`. Write same-directory +temporary files with exclusive creation, then rename. Print only the final JSON +path on stdout; diagnostics go to stderr. + +- [ ] **Step 8: Verify GREEN** + +Run: + +```bash +rtk pnpm run build:test +rtk node --test dist-test/test/perf-benchmark-core.test.js dist-test/test/perf-benchmark-agent.test.js dist-test/test/perf-benchmark-engine.test.js dist-test/test/perf-benchmark-runner.test.js dist-test/test/package-scripts.test.js +rtk pnpm run typecheck +rtk pnpm run lint +rtk pnpm run format:check +``` + +Expected: all commands pass. + +- [ ] **Step 9: Commit Task 3** + +```bash +rtk git add package.json pnpm-lock.yaml scripts/perf/benchmark-engine.ts scripts/perf/benchmark-runner.ts test/perf-benchmark-engine.test.ts test/perf-benchmark-runner.test.ts test/package-scripts.test.ts +rtk git commit -m "feat: add reusable CLI benchmark" +``` + +### Task 4: Full validation and three-way execution + +**Files:** + +- No tracked source files unless validation exposes a defect. +- Output: a unique temporary benchmark directory containing `results.json` and + `results.md`. + +**Interfaces:** + +- Consumes the Task 3 package entrypoint and three built worktrees. +- Produces exact three-way measurements and stage evidence. + +- [ ] **Step 1: Run repository gates** + +```bash +rtk pnpm run check +rtk pnpm run check:docs +rtk git diff --check +``` + +Expected: all pass. + +- [ ] **Step 2: Fetch immutable comparison refs** + +Fetch `upstream/main` and `refs/pull/478/head`, then record both fetched SHAs and +the current Artagon PR #2 head. Do not infer SHAs from old worktrees. + +- [ ] **Step 3: Create detached isolated worktrees** + +Use project-local ignored paths: + +```text +.worktrees/openclaw-main-three-way +.worktrees/openclaw-pr-478-three-way +``` + +Refuse if either path already exists at a different SHA. Do not delete or prune +any current worktree. + +- [ ] **Step 4: Install and build each variant** + +Within each worktree, use its declared package-manager version: + +```bash +rtk pnpm install --frozen-lockfile +rtk pnpm run build +``` + +In the performance worktree also run `rtk pnpm run build:test` for the shared +benchmark agent. + +- [ ] **Step 5: Run the full three-way benchmark** + +```bash +benchmark_output="" +benchmark_output="$(mktemp -d /private/tmp/acpx-three-way-benchmark.XXXXXX)" +readonly benchmark_output + +rtk pnpm run perf:benchmark -- \ + --baseline openclaw-main=../openclaw-main-three-way \ + --candidate openclaw-pr-478=../openclaw-pr-478-three-way \ + --candidate artagon-performance=. \ + --seed 2886672422 \ + --output "${benchmark_output}" +``` + +Expected: `results.json` and `results.md` exist, every scenario contains the +configured raw sample count for all comparisons, SHAs match Git, and all child +runs exit zero. + +- [ ] **Step 6: Interpret the cold path** + +Compare PR #478 with OpenClaw main to isolate lifecycle changes. Compare Artagon +performance with PR #478 to isolate its additional lazy-loading and metrics +stack. Correlate cold ACP deltas with pre-agent, ACP-active, teardown, and +supported internal spans. Run one focused hypothesis experiment before naming +a root cause; otherwise label the result a correlation. + +### Task 5: Review, publication, and PR update + +**Files:** + +- Modify only files required by accepted review findings. +- External output: Artagon PR #2 comment and normal branch push. + +- [ ] **Step 1: Run repository autoreview** + +Run `.agents/skills/autoreview/scripts/autoreview` in branch mode until no +accepted/actionable findings remain. Treat inability to obtain a final verdict +as an unresolved gate, not approval. + +- [ ] **Step 2: Run `art-acpx.sh` performance and testing reviews** + +Use +`/Users/gtrump001c@cable.comcast.com/Projects/Artagon/artagon-scripts/scripts/art-acpx.sh`, +one performance persona and one testing/API persona, against the full branch +range. Require fresh terminal final outputs; +session creation, ACKs, partial traces, or exit zero alone are not review proof. +Instruct every review lane that Artagon Node/TypeScript plugins are unavailable +and the authoritative fallback is this repository's functional TypeScript, +strict lint, and test contract. + +- [ ] **Step 3: Resolve accepted findings through reviewed worker fixes** + +For any accepted finding, dispatch one fresh fix worker with exact file +ownership, require focused tests, then run one scoped independent re-review. +Do not mix benchmark findings with unrelated existing branch changes. + +- [ ] **Step 4: Push normally and publish evidence** + +Push `perf/consolidated-runtime` without force. Comment on Artagon PR #2 with +the exact three SHAs, environment, methodology, result table, cold-path +evidence, and residual uncertainty. Do not put local filesystem paths or memory +citations in the PR comment. + +- [ ] **Step 5: Final completion audit** + +Verify the branch is clean, the remote head equals local HEAD, PR #2 is +mergeable, required checks are current, benchmark artifacts match the reported +SHAs, OpenClaw PR #478 is unchanged, and no unrelated worktree was deleted. diff --git a/docs/superpowers/specs/2026-08-02-three-way-performance-benchmark-design.md b/docs/superpowers/specs/2026-08-02-three-way-performance-benchmark-design.md new file mode 100644 index 00000000..bbdf932f --- /dev/null +++ b/docs/superpowers/specs/2026-08-02-three-way-performance-benchmark-design.md @@ -0,0 +1,233 @@ +# Three-Way ACPX Performance Benchmark Design + +## Goal + +Create a reusable benchmark that compares built ACPX checkouts without changing +the revisions being measured. Use it to compare current OpenClaw `main`, +OpenClaw PR #478, and Artagon performance PR #2, then retain the tool on the +Artagon performance branch. + +The benchmark must answer two separate questions: + +1. How do the three revisions compare for end-to-end CLI latency and eager-load + size? +2. Which lifecycle stage accounts for any cold ACP regression? + +## Ownership and isolation + +The reusable benchmark belongs to `perf/consolidated-runtime`, the branch behind +Artagon PR #2. OpenClaw PR #478 remains unchanged so it can be measured as the +submitted lifecycle fix rather than as a benchmark-modified revision. + +The comparison uses three independent worktrees: + +- a detached worktree pinned to the fetched OpenClaw `main` SHA; +- a detached worktree pinned to OpenClaw PR #478's fetched head SHA; and +- the existing `perf/consolidated-runtime` worktree pinned to Artagon PR #2's + head SHA. + +The runner accepts worktree paths and never fetches, checks out, installs, +builds, or mutates Git state. Preparation remains an explicit outer workflow so +the report can record and verify the exact already-built inputs. + +## Components + +### Benchmark core + +A functional TypeScript module under `scripts/perf/` owns: + +- variant and scenario validation; +- deterministic alternating order generation; +- descriptive statistics; +- paired log-ratio deltas and seeded bootstrap confidence intervals; +- eager-import graph accounting; and +- JSON and Markdown report rendering. + +Keeping these functions separate lets focused tests prove the statistical and +reporting contract without spawning hundreds of processes. Statistical, +ordering, validation, and rendering functions are pure. Eager-graph analysis +is a read-only filesystem boundary and performs no writes. + +The core retains ordered `PairedSample[]` values as the authoritative raw +measurements. It also continues to calculate p95 and the deterministic paired +bootstrap interval. Tinybench sorts its retained per-task samples and does not +provide either the required p95 or paired confidence interval, so its result +arrays are not used as the experiment's statistical record. + +### Tinybench task engine + +An exact-pinned `tinybench@6.1.2` development dependency supplies the narrow +task-execution boundary. `scripts/perf/benchmark-engine.ts` creates one fresh, +non-concurrent `Bench` for each adjacent baseline/candidate pair with: + +```ts +{ + concurrency: null, + time: 0, + iterations: 1, + warmup: false, + retainSamples: true, + throws: true, +} +``` + +The runner selects the alternating order before calling the engine. The engine +registers the two tasks in that exact order, executes each task once, and +propagates failures. Each async task performs the externally timed subprocess +run and returns a finite, nonnegative `overriddenDuration` measured from just +before spawn through child close. The engine returns those external durations +with task identity and registration order intact. + +Tinybench is therefore a task engine, not the experiment controller. It does +not own preflight, warmups, pair identity or order, process isolation, timeouts, +traces, statistics, or report generation. Vitest benchmark mode is not selected +because it would add a test-runner layer around Tinybench while this repository +uses the Node test runner. Mitata is not selected because its in-process +microbenchmark loop does not match the one-shot, externally timed subprocess +task boundary used here. + +### Benchmark runner + +The executable TypeScript entrypoint under `scripts/perf/` accepts: + +```text +--baseline