From 44de6444c2ff9deacd91b3998dc81c2e1fc4ff1c Mon Sep 17 00:00:00 2001 From: Chris Lee Date: Tue, 1 Sep 2026 20:27:35 +1000 Subject: [PATCH] feat(runner): isolate structured workflows in one-attempt Kubernetes Pods A `workflow-run` no longer enters the shared-daemon job protocol. One exact attempt owns one Pod and one capability Secret, and every state mutation is fenced by run ID, attempt ID, owner ID, lease, and command receipt. Runner side: - Add `src/runner/` (entry, WebSocket client, workflow executor, output sanitizer, token deadline, process boundary), replacing the deleted `src/daemon/workflow-executor.ts`. - Add `native/daemon-process-guard.c` and `src/daemon/process-boundary.ts` so a worker cannot outlive its attempt. - The runner deny set rejects App, PAT, database, Valkey, Kubernetes, Context7, global GitHub, and daemon-auth credentials. Controller side: - Add `src/orchestrator/workflow-runner-*.ts` (controller, dispatch, payload, store, result, reconciler, resources, capability, output) and `src/k8s/workflow-runner-spawner.ts`, which owns the per-attempt Secret. - Add `src/shared/workflow-runner-messages.ts`: a separate protocol from the shared-daemon schema, deliberately not shared with it. - Add migration `017_workflow_run_leases.sql`, which introduces the `attempt_id` / `offer_id` columns that `src/orchestrator/history.ts` and the daemon-disconnect fencing path query. - The RPC output scanner is stricter than the general one: disablement, failure, or timeout rejects a command and converts a result to a fixed safe failure rather than failing open. Also in this change: - Orchestrator resilience: liveness reaper, queue worker, ws-server split into `ws-connection.ts`, dispatch outbox, completion reconciler, expiry notifier. - The remaining `src/config.ts` surface, including the runner capability secret and namespace validation that only applies once the runner exists. - Workflow handler and ship-rail updates that consume the Gate-2 policy. - Move the last colocated `src/**/*.test.ts` files under `test/` and tighten `check:test-globs` to reject tests in production source. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01KUPpJPtxAaHWrBsjytRGyM --- .env.example | 22 +- .github/skills/research.md | 2 +- .github/workflows/ci.yml | 24 +- .github/workflows/docker-build.yml | 22 +- .github/workflows/research.yml | 2 +- .gitlab-ci.yml | 241 +++-- CLAUDE.md | 49 +- CONTRIBUTING.md | 7 +- Dockerfile.daemon | 16 +- bun.lock | 20 +- bunfig.toml | 3 - docs/build/architecture.md | 419 +++++++-- docs/operate/configuration.md | 319 +++++-- docs/operate/deployment.md | 248 ++++- docs/operate/observability.md | 204 ++-- docs/operate/runbooks/daemon-fleet.md | 165 +++- docs/operate/runbooks/scheduled-actions.md | 10 +- docs/use/invoking.md | 28 +- docs/use/repo-config.md | 65 +- docs/use/scheduled-actions.md | 10 + docs/use/workflows/implement.md | 2 +- docs/use/workflows/index.md | 10 +- docs/use/workflows/plan.md | 2 +- docs/use/workflows/resolve.md | 8 +- docs/use/workflows/review.md | 46 +- docs/use/workflows/triage.md | 2 +- env-contract.json | 50 + examples/scheduled-actions/research.md | 2 +- examples/workflow-runner-admission.yaml | 438 +++++++++ mkdocs.yml | 1 + native/daemon-process-guard.c | 30 + package.json | 5 +- scripts/build.ts | 39 + scripts/check-docs-sync.ts | 2 +- scripts/check-test-globs.ts | 13 +- scripts/env-contract.ts | 12 +- scripts/test-isolated.sh | 7 +- scripts/test-oauth.ts | 2 +- scripts/test-workflow-runner-admission.sh | 101 ++ scripts/test-workflow-runner-admission.ts | 766 +++++++++++++++ src/app.ts | 2 +- src/config-secret-env.ts | 2 + src/config.ts | 282 ++++-- src/core/pipeline.ts | 14 +- src/core/tracking-comment.ts | 54 +- src/daemon/daemon-id.ts | 16 +- src/daemon/job-executor.ts | 35 +- src/daemon/main.ts | 4 +- src/daemon/process-boundary-smoke.ts | 4 + src/daemon/process-boundary.ts | 48 + src/daemon/scoped-rebase-executor.ts | 2 +- src/daemon/workflow-executor.ts | 447 --------- src/daemon/ws-client.ts | 5 +- src/db/migrations/017_workflow_run_leases.sql | 336 +++++++ src/k8s/ephemeral-daemon-spawner.ts | 34 +- src/k8s/workflow-runner-spawner.ts | 689 ++++++++++++++ src/mcp/servers/inline-comment-dedup.ts | 9 +- src/mcp/servers/inline-comment.ts | 14 +- src/orchestrator/connection-handler.ts | 584 ++++++------ src/orchestrator/history.ts | 230 ++++- src/orchestrator/installation-token.ts | 102 +- src/orchestrator/job-dispatcher.ts | 139 ++- src/orchestrator/job-queue.ts | 90 +- src/orchestrator/liveness-reaper.ts | 426 +++++++-- src/orchestrator/log-fields.ts | 4 +- src/orchestrator/queue-worker.ts | 117 ++- .../repo-knowledge-persistence.ts | 126 +++ src/orchestrator/repo-knowledge.ts | 59 +- src/orchestrator/review-learnings.ts | 8 +- src/orchestrator/workflow-expiry-notifier.ts | 280 ++++++ .../workflow-runner-capability.ts | 92 ++ .../workflow-runner-controller.ts | 621 ++++++++++++ src/orchestrator/workflow-runner-dispatch.ts | 131 +++ src/orchestrator/workflow-runner-output.ts | 183 ++++ src/orchestrator/workflow-runner-payload.ts | 330 +++++++ .../workflow-runner-reconciler.ts | 81 ++ src/orchestrator/workflow-runner-resources.ts | 67 ++ src/orchestrator/workflow-runner-result.ts | 308 ++++++ src/orchestrator/workflow-runner-store.ts | 732 +++++++++++++++ src/orchestrator/ws-connection.ts | 30 + src/orchestrator/ws-server.ts | 116 ++- src/runner/main.ts | 153 +++ src/runner/output-sanitizer.ts | 6 + src/runner/process-boundary.ts | 60 ++ src/runner/token-deadline.ts | 46 + src/runner/workflow-executor.ts | 72 ++ src/runner/ws-client.ts | 469 +++++++++ src/shared/daemon-types.ts | 6 +- src/shared/dispatch-types.ts | 13 +- src/shared/workflow-runner-messages.ts | 222 +++++ src/shared/workflow-runner-provider.ts | 180 ++++ src/shared/workflow-types.ts | 74 ++ src/shared/ws-messages.ts | 38 +- src/utils/bot-identity.ts | 30 +- src/webhook/auto-review-guard.ts | 189 ++++ src/webhook/dispatch-failure.ts | 40 + src/webhook/events/issue-comment.ts | 9 + src/webhook/events/issues.ts | 8 +- src/webhook/events/pull-request.ts | 260 ++++- src/webhook/events/review-comment.ts | 11 + src/workflows/completion-reconciler.ts | 91 ++ src/workflows/discussion-digest.ts | 1 - src/workflows/dispatch-outbox.ts | 133 +++ src/workflows/dispatcher.ts | 507 +++++++--- src/workflows/execution-row.ts | 50 +- src/workflows/handlers/implement.ts | 41 +- src/workflows/handlers/plan.ts | 25 +- src/workflows/handlers/remember.ts | 17 +- src/workflows/handlers/resolve.ts | 37 +- .../handlers/review-learnings-footer.ts | 4 +- src/workflows/handlers/review.ts | 24 +- src/workflows/handlers/ship.ts | 117 +-- src/workflows/handlers/triage.ts | 37 +- src/workflows/intent-classifier.ts | 1 - src/workflows/log-fields.ts | 15 +- src/workflows/orchestrator.ts | 189 ++-- src/workflows/registry.ts | 122 +-- src/workflows/runs-store.ts | 521 ++++++++-- src/workflows/ship/command-dispatch.ts | 176 +++- src/workflows/ship/intent.ts | 30 +- src/workflows/ship/iteration.ts | 139 ++- src/workflows/ship/scoped/chat-thread.ts | 12 +- src/workflows/ship/scoped/dispatch-scoped.ts | 2 - src/workflows/ship/session-runner.ts | 2 +- src/workflows/tracking-mirror.ts | 183 +++- test/config.test.ts | 132 ++- .../core/hooks/forbidden-bash.test.ts | 4 +- test/core/pipeline.test.ts | 74 ++ test/core/tracking-comment.test.ts | 98 ++ {src => test}/core/workspace-events.test.ts | 2 +- test/daemon/daemon-id.test.ts | 40 + test/daemon/job-executor.test.ts | 141 +++ test/daemon/process-boundary.test.ts | 51 + test/daemon/scoped-offer-evaluator.test.ts | 6 +- test/daemon/workflow-executor.test.ts | 200 ---- test/daemon/ws-client.test.ts | 13 + test/db/migrate.test.ts | 570 ++++++++++- test/db/migrations/008.test.ts | 1 + test/fixtures/workflow-runner-kind.yaml | 9 + test/integration/repo-knowledge.test.ts | 91 +- test/integration/review-learnings.test.ts | 9 +- .../scoped-rebase-roundtrip.test.ts | 15 +- test/integration/ship-iteration-loop.test.ts | 2 + test/integration/ship-tickle-resume.test.ts | 2 + test/integration/telemetry-aggregates.test.ts | 31 +- .../workflow-dispatch-wakeup.test.ts | 99 ++ test/k8s/ephemeral-daemon-spawner.test.ts | 62 +- test/k8s/workflow-runner-spawner.test.ts | 866 +++++++++++++++++ test/mcp/servers/inline-comment-dedup.test.ts | 6 +- test/orchestrator/connection-handler.test.ts | 521 +++++++++- .../daemon-disconnect-lifecycle.test.ts | 384 ++++++++ test/orchestrator/history.test.ts | 27 +- test/orchestrator/installation-token.test.ts | 116 +++ test/orchestrator/job-dispatcher.test.ts | 152 ++- test/orchestrator/job-queue.test.ts | 71 ++ .../liveness-reaper-resilience.test.ts | 104 ++ test/orchestrator/liveness-reaper.test.ts | 453 ++++++++- test/orchestrator/log-fields.test.ts | 1 - .../queue-worker-resilience.test.ts | 165 ++++ .../repo-knowledge-persistence.test.ts | 101 ++ .../workflow-expiry-notifier.test.ts | 353 +++++++ .../workflow-runner-capability.test.ts | 138 +++ .../workflow-runner-controller.test.ts | 886 ++++++++++++++++++ .../workflow-runner-dispatch.test.ts | 170 ++++ .../workflow-runner-output.test.ts | 215 +++++ .../workflow-runner-payload.test.ts | 314 +++++++ .../workflow-runner-reconciler.test.ts | 166 ++++ .../workflow-runner-resources.test.ts | 96 ++ .../workflow-runner-result.test.ts | 410 ++++++++ .../workflow-runner-store.test.ts | 549 +++++++++++ test/orchestrator/ws-server.test.ts | 88 +- test/preload.ts | 8 +- test/runner/main.test.ts | 331 +++++++ test/runner/output-sanitizer.test.ts | 69 ++ test/runner/process-boundary.test.ts | 146 +++ test/runner/token-deadline.test.ts | 46 + test/runner/workflow-executor.test.ts | 189 ++++ test/runner/ws-client.test.ts | 434 +++++++++ test/scripts/check-test-globs.test.ts | 8 +- test/shared/dispatch-types.test.ts | 11 +- .../shared/scoped-ws-messages.test.ts | 18 +- test/shared/workflow-runner-messages.test.ts | 264 ++++++ test/shared/workflow-runner-provider.test.ts | 225 +++++ test/shared/ws-messages.test.ts | 93 ++ test/utils/bot-identity.test.ts | 7 +- test/webhook/auto-review-guard.test.ts | 228 +++++ test/webhook/events/dispatch-failure.test.ts | 244 +++++ .../events/issue-comment-cache.test.ts | 8 +- test/webhook/events/issue-comment.test.ts | 16 +- test/webhook/events/issues-cache.test.ts | 8 +- .../events/pull-request-auto-review.test.ts | 360 +++++++ .../webhook/events/pull-request-cache.test.ts | 8 +- .../events/pull-request-config-check.test.ts | 288 ++++++ test/workflows/dispatch-outbox.test.ts | 284 ++++++ test/workflows/dispatcher.test.ts | 488 +++++++++- test/workflows/handlers/implement.test.ts | 124 ++- test/workflows/handlers/plan.test.ts | 184 +++- test/workflows/handlers/remember.test.ts | 135 +++ test/workflows/handlers/resolve.test.ts | 164 +++- test/workflows/handlers/review.test.ts | 137 ++- test/workflows/handlers/ship.test.ts | 440 +++------ test/workflows/handlers/triage.test.ts | 184 +++- test/workflows/orchestrator.test.ts | 368 +++++++- test/workflows/runs-store.test.ts | 549 +++++++++-- test/workflows/ship/cancellation.test.ts | 2 + test/workflows/ship/command-dispatch.test.ts | 322 +++++++ test/workflows/ship/fix-attempts.test.ts | 2 + test/workflows/ship/intent.test.ts | 22 + test/workflows/ship/iteration-cap.test.ts | 2 + test/workflows/ship/iteration.test.ts | 67 +- .../workflows/ship/lifecycle-commands.test.ts | 2 + .../ship/session-runner.resume.test.ts | 1 + test/workflows/ship/session-runner.test.ts | 2 + test/workflows/ship/tickle-scheduler.test.ts | 1 + test/workflows/tracking-mirror.test.ts | 193 +++- 215 files changed, 25973 insertions(+), 3342 deletions(-) create mode 100644 examples/workflow-runner-admission.yaml create mode 100644 native/daemon-process-guard.c create mode 100644 scripts/test-workflow-runner-admission.sh create mode 100644 scripts/test-workflow-runner-admission.ts create mode 100644 src/daemon/process-boundary-smoke.ts create mode 100644 src/daemon/process-boundary.ts delete mode 100644 src/daemon/workflow-executor.ts create mode 100644 src/db/migrations/017_workflow_run_leases.sql create mode 100644 src/k8s/workflow-runner-spawner.ts create mode 100644 src/orchestrator/repo-knowledge-persistence.ts create mode 100644 src/orchestrator/workflow-expiry-notifier.ts create mode 100644 src/orchestrator/workflow-runner-capability.ts create mode 100644 src/orchestrator/workflow-runner-controller.ts create mode 100644 src/orchestrator/workflow-runner-dispatch.ts create mode 100644 src/orchestrator/workflow-runner-output.ts create mode 100644 src/orchestrator/workflow-runner-payload.ts create mode 100644 src/orchestrator/workflow-runner-reconciler.ts create mode 100644 src/orchestrator/workflow-runner-resources.ts create mode 100644 src/orchestrator/workflow-runner-result.ts create mode 100644 src/orchestrator/workflow-runner-store.ts create mode 100644 src/orchestrator/ws-connection.ts create mode 100644 src/runner/main.ts create mode 100644 src/runner/output-sanitizer.ts create mode 100644 src/runner/process-boundary.ts create mode 100644 src/runner/token-deadline.ts create mode 100644 src/runner/workflow-executor.ts create mode 100644 src/runner/ws-client.ts create mode 100644 src/shared/workflow-runner-messages.ts create mode 100644 src/shared/workflow-runner-provider.ts create mode 100644 src/webhook/auto-review-guard.ts create mode 100644 src/webhook/dispatch-failure.ts create mode 100644 src/workflows/completion-reconciler.ts create mode 100644 src/workflows/dispatch-outbox.ts rename {src => test}/core/hooks/forbidden-bash.test.ts (98%) rename {src => test}/core/workspace-events.test.ts (99%) create mode 100644 test/daemon/daemon-id.test.ts create mode 100644 test/daemon/job-executor.test.ts create mode 100644 test/daemon/process-boundary.test.ts delete mode 100644 test/daemon/workflow-executor.test.ts create mode 100644 test/fixtures/workflow-runner-kind.yaml create mode 100644 test/integration/workflow-dispatch-wakeup.test.ts create mode 100644 test/k8s/workflow-runner-spawner.test.ts create mode 100644 test/orchestrator/daemon-disconnect-lifecycle.test.ts create mode 100644 test/orchestrator/installation-token.test.ts create mode 100644 test/orchestrator/liveness-reaper-resilience.test.ts create mode 100644 test/orchestrator/queue-worker-resilience.test.ts create mode 100644 test/orchestrator/repo-knowledge-persistence.test.ts create mode 100644 test/orchestrator/workflow-expiry-notifier.test.ts create mode 100644 test/orchestrator/workflow-runner-capability.test.ts create mode 100644 test/orchestrator/workflow-runner-controller.test.ts create mode 100644 test/orchestrator/workflow-runner-dispatch.test.ts create mode 100644 test/orchestrator/workflow-runner-output.test.ts create mode 100644 test/orchestrator/workflow-runner-payload.test.ts create mode 100644 test/orchestrator/workflow-runner-reconciler.test.ts create mode 100644 test/orchestrator/workflow-runner-resources.test.ts create mode 100644 test/orchestrator/workflow-runner-result.test.ts create mode 100644 test/orchestrator/workflow-runner-store.test.ts create mode 100644 test/runner/main.test.ts create mode 100644 test/runner/output-sanitizer.test.ts create mode 100644 test/runner/process-boundary.test.ts create mode 100644 test/runner/token-deadline.test.ts create mode 100644 test/runner/workflow-executor.test.ts create mode 100644 test/runner/ws-client.test.ts rename src/shared/ws-messages.test.ts => test/shared/scoped-ws-messages.test.ts (91%) create mode 100644 test/shared/workflow-runner-messages.test.ts create mode 100644 test/shared/workflow-runner-provider.test.ts create mode 100644 test/webhook/auto-review-guard.test.ts create mode 100644 test/webhook/events/dispatch-failure.test.ts create mode 100644 test/webhook/events/pull-request-auto-review.test.ts create mode 100644 test/webhook/events/pull-request-config-check.test.ts create mode 100644 test/workflows/dispatch-outbox.test.ts create mode 100644 test/workflows/handlers/remember.test.ts create mode 100644 test/workflows/ship/command-dispatch.test.ts diff --git a/.env.example b/.env.example index a00d010e..37af5063 100644 --- a/.env.example +++ b/.env.example @@ -26,11 +26,11 @@ ANTHROPIC_API_KEY= # CLAUDE_CODE_OAUTH_TOKEN= # Model override. Required when CLAUDE_PROVIDER=bedrock (Bedrock uses a different model ID -# format than the Anthropic API). Optional for anthropic: defaults to claude-opus-4-7 +# format than the Anthropic API). Optional for anthropic: defaults to claude-opus-5 # when unset. # Bedrock example: us.anthropic.claude-sonnet-4-6 -# Anthropic example: claude-opus-4-7 -# CLAUDE_MODEL=claude-opus-4-7 +# Anthropic example: claude-opus-5 +# CLAUDE_MODEL=claude-opus-5 # ────────────────────────────────────────────────────────────────────────────── # Amazon Bedrock (when CLAUDE_PROVIDER=bedrock) @@ -141,6 +141,15 @@ AGENT_JOB_MODE=inline # the primary or this previous token (constant-time). Drop after rolling daemons. # DAEMON_AUTH_TOKEN_PREVIOUS= +# Controller-only HMAC root for deadline-bound workflow-runner capabilities. +# Never mount either value on shared daemons or isolated runners, and never +# reuse either DAEMON_AUTH_TOKEN rotation value. +# Generate at least 32 random bytes (e.g. openssl rand -hex 32). +# WORKFLOW_RUNNER_CAPABILITY_SECRET= +# Optional rotation-window predecessor. Remove after every capability minted +# with the old root has reached its signed expiry. +# WORKFLOW_RUNNER_CAPABILITY_SECRET_PREVIOUS= + # Per-job cost ceiling in USD # JOB_MAX_COST_USD=80 @@ -154,6 +163,8 @@ AGENT_JOB_MODE=inline # STALE_EXECUTION_THRESHOLD_MS=600000 # DAEMON_DRAIN_TIMEOUT_MS=300000 # JOB_MAX_RETRIES=3 +# Maximum wall-clock age for a queued structured-workflow dispatch. +# WORKFLOW_DISPATCH_TIMEOUT_MS=4200000 # OFFER_TIMEOUT_MS=5000 # Daemon-side settings (used by scripts/run-daemon.sh) @@ -203,11 +214,14 @@ AGENT_JOB_MODE=inline # Enable debug logging of LLM prompts. # DEBUG_LLM_PROMPTS=1 +# Per-repo config file, read from each installed repo's DEFAULT BRANCH root only. +# Deprecated alias: SCHEDULER_CONFIG_FILE. +# REPO_CONFIG_FILE=.github-app.yaml + # Scheduled actions (.github-app.yaml). The scheduler also requires # DATABASE_URL and a non-empty ALLOWED_OWNERS to start. # SCHEDULER_ENABLED=false # SCHEDULER_SCAN_INTERVAL_MS=300000 -# SCHEDULER_CONFIG_FILE=.github-app.yaml # Hard kill-switch for unattended auto-merge; per-action auto_merge is AND-ed # with this. Leave false unless you accept LLM-judged merges. # SCHEDULER_ALLOW_AUTO_MERGE=false diff --git a/.github/skills/research.md b/.github/skills/research.md index d9006394..c556220b 100644 --- a/.github/skills/research.md +++ b/.github/skills/research.md @@ -67,7 +67,7 @@ Read the key files for the focus area: | idempotency | src/webhook/router.ts, src/core/tracking-comment.ts | | security | src/utils/, src/config.ts | | observability | src/logger.ts (and grep for logger usage across src/) | -| testing | src/\*\*/\*.test.ts (sample 3-5; do not read all) | +| testing | test/\*\*/\*.test.ts (sample 3-5; do not read all) | | docs | CLAUDE.md, README.md, docs/ | | infrastructure | .github/workflows/, Dockerfile.\*, package.json | | agent-sdk | src/core/prompt-builder.ts, src/core/executor.ts | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9fe08188..a237e933 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -112,9 +112,8 @@ jobs: run: bun run check:runner-pins - name: Test-glob drift guard - # Fails if any `*.test.ts` file is not reachable by the runner glob in - # scripts/test-isolated.sh, so a colocated test outside the globbed - # roots cannot go dark while CI stays green. See issue #201. + # Fails if any `*.test.ts` file is outside the canonical `test/` tree + # consumed by scripts/test-isolated.sh. run: bun run check:test-globs - name: Destructive-action guard (FR-009) @@ -190,3 +189,22 @@ jobs: # build/resolve drift class that shipped in 001990d before the daemon # image hits production. Must run AFTER `bun run build`. run: bun run check:mcp-bundle + + admission-policy: + name: Workflow runner admission + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - name: Checkout source code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version-file: .tool-versions + + - name: Install dependencies + run: bun install --frozen-lockfile + + - name: Validate admission against Kubernetes 1.30 + run: bun run test:admission diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 45861b1a..e98586d7 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -6,7 +6,10 @@ # - chrisleekr/github-app:-daemon (rich, Claude worker) # Orchestrator additionally publishes a bare `` alias so existing # consumers that pull `chrisleekr/github-app:` keep working. -# `:latest` is published on prod releases only, on the orchestrator variant. +# Each variant also gets a mutable `:latest-` alias, so `latest-daemon` +# resolves on Docker Hub the same way it does on the GitLab registry. Both that +# alias and the bare `:latest` (orchestrator only) are prod-release only: a beta +# prerelease must not move a tag prod consumers follow. # # Multi-platform via the documented split-and-merge pattern: # https://docs.docker.com/build/ci/github-actions/multi-platform/#distribute-build-across-multiple-runners @@ -185,6 +188,22 @@ jobs: GIT_HASH=${{ steps.meta.outputs.git_hash }} NODE_ENV=production + - name: Smoke daemon process boundary + if: matrix.variant == 'daemon' + env: + DIGEST: ${{ steps.build.outputs.digest }} + run: | + docker pull "${IMAGE_NAME}@${DIGEST}" + docker run --rm \ + --user 1000:1000 \ + --read-only \ + --network none \ + --cap-drop ALL \ + --security-opt no-new-privileges=true \ + --entrypoint bun \ + "${IMAGE_NAME}@${DIGEST}" \ + run dist/daemon/process-boundary-smoke.js + - name: Export digest env: DIGEST: ${{ steps.build.outputs.digest }} @@ -256,6 +275,7 @@ jobs: type=raw,value=${{ steps.tag.outputs.variant_tag }} type=raw,value=${{ steps.tag.outputs.version }},enable=${{ matrix.variant == 'orchestrator' }} type=raw,value=latest,enable=${{ matrix.variant == 'orchestrator' && inputs.is-dev-release == false }} + type=raw,value=latest-${{ matrix.variant }},enable=${{ inputs.is-dev-release == false }} - name: Create manifest list and push working-directory: ${{ runner.temp }}/digests diff --git a/.github/workflows/research.yml b/.github/workflows/research.yml index e64df7dd..f7c49890 100644 --- a/.github/workflows/research.yml +++ b/.github/workflows/research.yml @@ -195,7 +195,7 @@ jobs: | idempotency | src/webhook/router.ts, src/core/tracking-comment.ts | | security | src/utils/, src/config.ts | | observability | src/logger.ts (and grep for logger usage across src/) | - | testing | src/**/*.test.ts (sample 3-5; do not read all) | + | testing | test/**/*.test.ts (sample 3-5; do not read all) | | docs | CLAUDE.md, top-level README.md if present, docs/ | | infrastructure | .github/workflows/, Dockerfile, package.json | | agent-sdk | src/core/prompt-builder.ts, src/core/executor.ts | diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 043beb4d..239f42d9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,9 +13,11 @@ stages: .main_branch: &main_branch rules: - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - - if: $CI_COMMIT_MESSAGE =~ /^chore\(release\):/ + # Exclusion first: rules stop at the first match. Skips release-please's + # release commit, whose tree already built one commit earlier. + - if: '$CI_COMMIT_MESSAGE =~ /^chore\(\w+\): release /' when: never + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Reusable configurations .cache_config: &cache_config @@ -43,6 +45,9 @@ build: policy: pull script: - bun run build + # Must run AFTER the build: resolves every MCP server registered in + # src/mcp/registry.ts against the emitted dist/ bundles. + - bun run check:mcp-bundle needs: - dependencies artifacts: @@ -50,7 +55,9 @@ build: - dist/ expire_in: 1 hour -lint: +# Quality gates run on every branch, main included. They used to be +# feature-branch-only, which meant main published an image nothing had checked. +.gate: &gate stage: test interruptible: true cache: @@ -58,51 +65,104 @@ lint: policy: pull needs: - dependencies - <<: *feature_branch + +lint: + <<: *gate script: - bun run lint format: - stage: test - interruptible: true - cache: - <<: *cache_config - policy: pull - needs: - - dependencies - <<: *feature_branch + <<: *gate script: - bun run format typecheck: - stage: test - interruptible: true - cache: - <<: *cache_config - policy: pull - needs: - - dependencies - <<: *feature_branch + <<: *gate script: - bun run typecheck -# test-unit: -# stage: test -# interruptible: true -# cache: -# <<: *cache_config -# policy: pull -# needs: -# - dependencies -# script: -# - bun test -# coverage: /All files[^\|]*\|[^\|]*\s+([\d\.]+)/ -# artifacts: -# name: coverage -# when: always -# expire_in: 2 days -# paths: -# - coverage/ +test-unit: + <<: *gate + # Postgres + Valkey match the service containers in .github/workflows/ci.yml. + # Without them the DB-backed suites skip themselves and the gate covers nothing. + services: + - name: pgvector/pgvector:pg17 + alias: postgres + - name: valkey/valkey:9 + alias: valkey + variables: + POSTGRES_USER: bot + POSTGRES_PASSWORD: bot + POSTGRES_DB: github_app + TEST_DATABASE_URL: postgres://bot:bot@postgres:5432/github_app_test + # Read directly by test code (test/preload.ts, liveness-reaper.test.ts), so + # it is set explicitly rather than relying on the preload default. + VALKEY_URL: redis://valkey:6379 + before_script: + # bash: scripts/test-isolated.sh needs globstar. postgresql-client: the + # readiness probe and the seed. git: test/core/checkout.test.ts builds a + # real repo fixture via Bun's $`git ...`. + - apk add --no-cache bash git postgresql-client + # The loop's exit status is `sleep`'s, so re-probe after it to fail clearly. + - | + for _ in $(seq 1 30); do + pg_isready -h postgres -U bot -d github_app && break + sleep 2 + done + pg_isready -h postgres -U bot -d github_app \ + || { echo "postgres did not accept connections within 60s"; exit 1; } + # `services:` cannot mount files, so init-test-db.sql never reaches + # /docker-entrypoint-initdb.d/. Create the test database explicitly. + - PGPASSWORD=bot psql -h postgres -U bot -d github_app -v ON_ERROR_STOP=1 -f scripts/init-test-db.sql + script: + - bun run test + # No `coverage:` regex or artifact: test-isolated.sh runs one bun process per + # file and prints output only on failure, so there is no aggregate to parse. + +admission-policy: + <<: *gate + # kind requires privileged DinD. This installation has no isolated ephemeral + # DinD runner, so never expose this job to branch-controlled pipelines. + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_COMMIT_REF_PROTECTED == "true" + image: oven/bun:1.3.14-alpine@sha256:5acc90a93e91ff07bf72aa90a7c9f0fa189765aec90b47bdbf2152d2196383c0 + services: + - name: docker:29-dind@sha256:12e683a161823b2a839aeea999b9d960e6e1f9a97b1679ad6b441982e2d9cf07 + alias: docker + variables: + DOCKER_HOST: tcp://docker:2376 + DOCKER_TLS_CERTDIR: /certs + DOCKER_TLS_VERIFY: 1 + DOCKER_CERT_PATH: $DOCKER_TLS_CERTDIR/client + before_script: + - apk add --no-cache docker-cli + script: + # kind publishes its API on the DinD host loopback. Run the harness in the + # remote daemon's host network so its kubeconfig points at the right host. + - >- + docker run --rm --network host + --volume /var/run/docker.sock:/var/run/docker.sock + --volume "$CI_PROJECT_DIR:$CI_PROJECT_DIR" + --workdir "$CI_PROJECT_DIR" + oven/bun:1.3.14-alpine@sha256:5acc90a93e91ff07bf72aa90a7c9f0fa189765aec90b47bdbf2152d2196383c0 + sh -ec 'apk add --no-cache bash ca-certificates coreutils curl docker-cli + && bun run test:admission' + +# The invariant guards from .github/workflows/ci.yml. They catch stale generated +# artifacts and drifted pins, which a build alone cannot see. +guards: + <<: *gate + script: + - bun run check:dockerfile-base-sync + - bun run check:action-pins + - bun run check:runner-pins + - bun run check:test-globs + - bun run check:no-destructive + - bun run check:env-contract + - bun run check:config-schema + - bun run audit:ci + # check:docs-sync is absent on purpose: it diffs BASE_SHA...HEAD_SHA, so it + # is PR-scoped and empty on main. .github/workflows/ci.yml owns it. # Publish in branch with dev tag (for testing) publish-dev-npm: @@ -120,14 +180,13 @@ publish-dev-npm: - echo "@chrisleekr:registry=https://${CI_SERVER_HOST}/api/v4/projects/${CI_PROJECT_ID}/packages/npm/" > .npmrc - echo "//${CI_SERVER_HOST}/api/v4/projects/${CI_PROJECT_ID}/packages/npm/:_authToken=${CI_JOB_TOKEN}" >> .npmrc - bunx npm version prerelease --preid=dev-${CI_COMMIT_SHORT_SHA} --no-git-tag-version - - bun publish --registry "https://${CI_SERVER_HOST}/api/v4/projects/${CI_PROJECT_ID}/packages/npm/" --tag dev-${CI_COMMIT_SHORT_SHA} + - bun publish --ignore-scripts --registry "https://${CI_SERVER_HOST}/api/v4/projects/${CI_PROJECT_ID}/packages/npm/" --tag dev-${CI_COMMIT_SHORT_SHA} -publish-dev-docker: +# Images publish from main only, under mutable `latest-*` tags. Feature branches +# build nothing. Versioned releases are cut separately by release-please. +.docker_release: &docker_release stage: release - interruptible: true - needs: - - dependencies - <<: *feature_branch + <<: *main_branch image: docker:29-dind services: - name: docker:29-dind @@ -137,46 +196,72 @@ publish-dev-docker: DOCKER_TLS_CERTDIR: "/certs" DOCKER_TLS_VERIFY: 1 DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client" - # Set reference for this block - before_script: &before_script_docker - - apk add curl git jq + # `needs` is a DAG, not a stage barrier: every gate that must hold before + # publishing has to be listed, `build` included (it runs check:mcp-bundle). + # `artifacts: false` because these jobs build their own dist/ inside the image. + needs: + - job: dependencies + artifacts: false + - job: lint + artifacts: false + - job: format + artifacts: false + - job: typecheck + artifacts: false + - job: test-unit + artifacts: false + - job: admission-policy + artifacts: false + - job: guards + artifacts: false + - job: build + artifacts: false + before_script: + - apk add --no-cache curl git jq - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY - docker context create dind - docker buildx create --driver docker-container --use dind --buildkitd-flags '--allow-insecure-entitlement network.host' - script: - - PACKAGE_VERSION=$(grep -m1 version package.json | cut -c 15- | rev | cut - -c 3- | rev) + # jq, not grep|cut|rev: the old chain assumed a fixed column offset. + - PACKAGE_VERSION=$(jq -r .version package.json) - GIT_HASH=$(git rev-parse --short HEAD) + - export PACKAGE_VERSION GIT_HASH + +# The tag is overwritten on every main build, by design. `--provenance false` is +# carried over from the previous job: an attestation adds unknown/unknown entries +# to the manifest index, which downstream digest resolution has to cope with. +publish-orchestrator-docker: + <<: *docker_release + timeout: 45m + script: - docker buildx build --progress plain --platform linux/amd64,linux/arm64 + --file Dockerfile.orchestrator --allow network.host --provenance false - --build-arg PACKAGE_VERSION=$PACKAGE_VERSION --build-arg GIT_HASH=$GIT_HASH - --build-arg NODE_ENV=production --target production --pull --tag - $CI_REGISTRY/chrisleekr/${CI_PROJECT_NAME}:dev-${CI_COMMIT_SHORT_SHA} --push . -# Versioned releases are cut on GitHub by release-please (see -# .github/workflows/release-please.yml), not from this GitLab mirror, to avoid -# releasing the same repository twice. - -publish-tag-docker: - stage: release - <<: *main_branch - image: docker:29-dind - services: - - name: docker:29-dind - alias: docker - variables: - DOCKER_HOST: tcp://docker:2376 - DOCKER_TLS_CERTDIR: "/certs" - DOCKER_TLS_VERIFY: 1 - DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client" - before_script: *before_script_docker + --build-arg "PACKAGE_VERSION=$PACKAGE_VERSION" --build-arg "GIT_HASH=$GIT_HASH" + --target production --pull --tag + $CI_REGISTRY/chrisleekr/${CI_PROJECT_NAME}:latest-orchestrator --push . + +# amd64 only: Dockerfile.daemon carries a large toolchain, and an emulated arm64 +# leg on an amd64 runner would exceed the job timeout. +publish-daemon-docker: + <<: *docker_release + timeout: 55m script: - - apk add --no-cache git - - git pull origin main - - PACKAGE_VERSION=$(grep -m1 version package.json | cut -c 15- | rev | cut - -c 3- | rev) - - GIT_HASH=$(git rev-parse --short HEAD) - - docker buildx build --progress plain --platform linux/amd64,linux/arm64 + - DAEMON_TEST_IMAGE="github-app-daemon:${CI_COMMIT_SHA}" + - docker buildx build --progress plain --platform linux/amd64 + --file Dockerfile.daemon --allow network.host --provenance false - --build-arg PACKAGE_VERSION=$PACKAGE_VERSION --build-arg GIT_HASH=$GIT_HASH - --build-arg NODE_ENV=production --target production --pull --tag - $CI_REGISTRY/chrisleekr/${CI_PROJECT_NAME}:${PACKAGE_VERSION} --push . + --build-arg "PACKAGE_VERSION=$PACKAGE_VERSION" --build-arg "GIT_HASH=$GIT_HASH" + --target production --pull --tag "$DAEMON_TEST_IMAGE" --load . + - >- + docker run --rm + --user 1000:1000 + --read-only + --network none + --cap-drop ALL + --security-opt no-new-privileges=true + --entrypoint bun + "$DAEMON_TEST_IMAGE" + run dist/daemon/process-boundary-smoke.js + - docker tag "$DAEMON_TEST_IMAGE" + "$CI_REGISTRY/chrisleekr/${CI_PROJECT_NAME}:latest-daemon" + - docker push "$CI_REGISTRY/chrisleekr/${CI_PROJECT_NAME}:latest-daemon" diff --git a/CLAUDE.md b/CLAUDE.md index b5efab20..30d08e39 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -32,9 +32,9 @@ Single HTTP server (`src/app.ts`) using `octokit` App class. Webhook events arri **Event handler** (`src/webhook/events/`): parse event → unified `BotContext` → check for `@chrisleekr-bot` trigger → fire-and-forget `processRequest()` -**Router** (`src/webhook/router.ts`): owner allowlist, concurrency guard, triage, and scale-up decision (the `processRequest` entry point is dev-test-only; production handlers in `events/` own dispatch + idempotency). On heavy/overflow it spawns an ephemeral daemon K8s Pod (`src/k8s/ephemeral-daemon-spawner.ts`). The job is then enqueued for any daemon in the fleet to claim over WebSocket. The webhook server never executes the pipeline in-process. +**Router** (`src/webhook/router.ts`): owner allowlist, concurrency guard, triage, and scale-up decision (the `processRequest` entry point is dev-test-only; production handlers in `events/` own dispatch + idempotency). Structured `workflow-run` jobs are claimed by the controller and run in a one-attempt Kubernetes Pod. Legacy and scoped jobs remain on the shared daemon fleet. The webhook server never executes the pipeline in-process. -**Pipeline** (`src/core/pipeline.ts`, executed by the daemon): +**Pipeline** (`src/core/pipeline.ts`, executed by a shared daemon or isolated workflow runner): 1. Create tracking comment ("Working…") 2. Resolve GitHub credential (App installation token by default; PAT when `GITHUB_PERSONAL_ACCESS_TOKEN` is set, see "Authentication options") @@ -49,23 +49,26 @@ Single HTTP server (`src/app.ts`) using `octokit` App class. Webhook events arri ## Architecture - `src/webhook/`: Event routing (`router.ts`) and per-event handlers (`events/`, one file per event type) -- `src/core/`: Pipeline: context → fetch → format → prompt → checkout → execute. `pipeline.ts` is the single execution path (run inside the daemon, never in-process in the webhook server). +- `src/core/`: Pipeline: context → fetch → format → prompt → checkout → execute. `pipeline.ts` is the single execution path, run inside a worker process and never in the webhook server. - `src/db/`: Database layer (Postgres via `Bun.sql`). Connection singleton (`index.ts`), migration runner (`migrate.ts`), SQL migrations (`migrations/`). Only active when `DATABASE_URL` is configured. -- `src/orchestrator/`: WebSocket server, daemon registry, job queue, job dispatcher, execution history, Valkey client, concurrency tracking, ephemeral-daemon scaler. Embedded in the webhook server process. +- `src/orchestrator/`: WebSocket server, daemon registry, job queue, execution history, Valkey client, workflow-runner admission, attempt leases, result reconciliation, and ephemeral-daemon scaling. Embedded in the webhook server process. - `src/daemon/`: Standalone daemon worker process (persistent or ephemeral). Connects to the orchestrator via WebSocket, discovers local capabilities, accepts/rejects job offers, executes jobs via `src/core/pipeline.ts`. When `DAEMON_EPHEMERAL=true`, exits after `EPHEMERAL_DAEMON_IDLE_TIMEOUT_MS` of idle. Entry: `src/daemon/main.ts`. -- `src/k8s/`: Ephemeral daemon Pod spawner (`ephemeral-daemon-spawner.ts`). Creates a bare Pod running the same daemon image with `DAEMON_EPHEMERAL=true`. -- `src/shared/`: Types shared between server and daemon: WebSocket message schemas (`ws-messages.ts`), daemon capability types (`daemon-types.ts`). +- `src/runner/`: One-attempt structured-workflow process. It receives bounded input and a target-repository installation token over an attempt-scoped WebSocket, then reports commands and a retained terminal result to the controller. +- `src/k8s/`: Bare-Pod spawners for ephemeral shared daemons and isolated workflow runners. The workflow spawner also owns the per-attempt capability Secret. +- `src/shared/`: Types shared across controller and workers, including separate shared-daemon and workflow-runner WebSocket schemas. - `src/mcp/`: MCP server registry and servers (extensible: add new servers). Includes `daemon-capabilities` server for daemon environment awareness. +- `src/repo-config/`: The per-repo `.github-app.yaml` surface. `schema.ts` (zod document schema), `fetcher.ts` (fetch + ETag/negative cache + validate, returning a discriminated `ok` / `absent` / `invalid` result), `effective.ts` (merge `workflows.` over `defaults`, clamp against the server env ceilings, fail open to `DEFAULT_REPO_POLICY`), `pr-check.ts` (the authoring-experience read, see below). **Only the default branch's copy is ever applied**: `fetcher.ts` calls `getContent` with no `ref`, so a config edit inside a pull request is inert for that pull request. Filename comes from `REPO_CONFIG_FILE` (deprecated alias `SCHEDULER_CONFIG_FILE`). `pr-check.ts` is the one module that reads a head-ref copy, and it is read-only by construction: it imports neither `fetchRepoConfig` nor `loadRepoPolicy`, so a head-ref read can never populate the fetcher caches or reach the applied policy (`test/repo-config/pr-check.test.ts` asserts the absence of both symbols in that source file). It fires from `handlePullRequestConfigCheck` on `opened` / `synchronize` / `reopened` behind the claim key `` `${deliveryId}:config-check` `` (distinct from the bare `deliveryId` the `labeled` branch claims, since `claimDelivery` is one-shot per key), skips entirely unless the PR diff touches the config file, refuses to decode a file over 64 KB, and upserts one `` sticky comment via `upsertMarkerComment` with `source: "system"`. - `src/scheduler/`: Internal cron scheduler for the `.github-app.yaml` scheduled-actions feature. Runs in the webhook server: enumerates installations, fetches + validates each repo's config, evaluates cron, and enqueues `scheduled-action` jobs for the daemon fleet. - `src/utils/`: Retry logic, sanitization ## Key Concepts - **Async processing**: Webhook must respond within 10 seconds. All heavy work runs asynchronously after 200 OK. -- **Idempotency**: GitHub webhooks are at-least-once, a delivery (auto-retry or operator redelivery) replays with the SAME `X-GitHub-Delivery` header for up to 3 days. The four side-effecting event handlers (`events/issue-comment.ts`, `events/review-comment.ts`, and the label branches of `events/issues.ts` + `events/pull-request.ts`) call `claimDelivery(deliveryId)` (`src/webhook/idempotency.ts`) at the very top of their dispatch path, before any LLM call, `workflow_runs` insert, or GitHub write. `claimDelivery` is a Valkey `SET key 1 NX EX 259200` claim: it returns `true` exactly once per `deliveryId` within the 3-day window (the redelivery gets `false` and the handler returns early). It is **fail-OPEN**, when Valkey is unconfigured or errors it returns `true`, degrading to at-least-once rather than dropping webhooks. `events/review.ts` is intentionally NOT gated: it fires only an idempotent reactor wake (no dispatch/write). Durable backstop behind the best-effort Valkey layer: the `idx_workflow_runs_inflight` partial-unique index, which makes the dispatcher reject a second in-flight run for the same workflow+target even if the Valkey claim was skipped (fail-open). `claimDelivery` also fails open when Valkey is configured-but-disconnected (gated on `isValkeyHealthy()` so a down connection skips the SET rather than blocking on Bun's offline queue). (The legacy in-memory `Map` + `isAlreadyProcessed()` tracking-comment scan was retired in issue #211; it only ever guarded the dev-test-only `router.ts` `processRequest` path, which production handlers bypass.) -- **Repo checkout**: Each request clones the repo to a unique temp dir. Claude operates on local files via `cwd`. +- **Idempotency**: GitHub webhooks are at-least-once, a delivery (auto-retry or operator redelivery) replays with the SAME `X-GitHub-Delivery` header for up to 3 days. The four side-effecting event handlers (`events/issue-comment.ts`, `events/review-comment.ts`, and the label branches of `events/issues.ts` + `events/pull-request.ts`) call `claimDelivery(deliveryId)` (`src/webhook/idempotency.ts`) at the very top of their dispatch path, before any LLM call, `workflow_runs` insert, or GitHub write. `claimDelivery` is a Valkey `SET key 1 NX EX 259200` claim: it returns `true` exactly once per `deliveryId` within the 3-day window (the redelivery gets `false` and the handler returns early). It is **fail-OPEN**, when Valkey is unconfigured or errors it returns `true`, degrading to at-least-once rather than dropping webhooks. `events/review.ts` is intentionally NOT gated: it fires only an idempotent reactor wake (no dispatch/write). Durable backstop behind the best-effort Valkey layer: the `idx_workflow_runs_inflight` partial-unique index, which makes the dispatcher reject a second in-flight run for the same workflow+target even if the Valkey claim was skipped (fail-open). `claimDelivery` also fails open when Valkey is configured-but-disconnected (gated on `isValkeyHealthy()` so a down connection skips the SET rather than blocking on Bun's offline queue). (The legacy in-memory `Map` + `isAlreadyProcessed()` tracking-comment scan was retired in issue #211; it only ever guarded the dev-test-only `router.ts` `processRequest` path, which production handlers bypass.) Two branches claim **suffixed** keys off the same delivery rather than the bare `deliveryId`, because `claimDelivery` is one-shot per key and would otherwise starve its sibling: `` `${deliveryId}:config-check` `` (`handlePullRequestConfigCheck`) and `` `${deliveryId}:auto-review` `` (`maybeAutoReview`). +- **Repo checkout**: Each execution clones the repo to a unique temp dir. Structured workflows get a one-attempt Pod and workspace. Legacy and scoped jobs use a shared daemon workspace. Claude operates on local files via `cwd`. - **MCP servers**: Comment updates, inline reviews, and Context7 for library docs. Git changes are made via git CLI (Bash tool) on the cloned repo. -- **Scheduled actions**: a repo may ship a `.github-app.yaml` at its default-branch root declaring prompt-based actions on a cron schedule. The internal scheduler (`src/scheduler/`, gated by `SCHEDULER_ENABLED` + `DATABASE_URL` + non-empty `ALLOWED_OWNERS`) enqueues a `scheduled-action` job, a new job kind on the scoped-job rail, that the daemon runs as one agent session via `src/daemon/scheduled-action-executor.ts`. Missed cron slots are skipped, not backfilled. The prompt is owner-trusted config. Cron parsing uses the `cron-parser` dependency. The bot-provided `merge_readiness` MCP tool is exposed only when `SCHEDULER_ALLOW_AUTO_MERGE` env AND per-action `auto_merge` are both true; `allowed_tools` is owner-trusted config, so an action granted a merge-capable Bash tool can still merge regardless. `resolve.ts` FR-017 is untouched. +- **Per-repo configuration**: a repo's `.github-app.yaml` (schema, fetcher, and resolver in `src/repo-config/`) is the single per-repo control surface. **Only the default branch's copy is ever read**: `fetchRepoConfig` calls `repos.getContent` with no `ref`, so a config edit inside a PR is inert for that PR. Do NOT add a `ref` to that call; `test/repo-config/fetcher.test.ts` asserts its absence. The config is consumed at two gates. **Gate 1** (`src/repo-config/gate.ts checkRepoGate`) runs pre-dispatch and decides whether the bot acts at all, before any `workflow_runs` row, label mutex, queue job, or tracking comment exists. It is narrowing only: every rule can refuse, none can permit, so no YAML value can readmit a repo the `ALLOWED_OWNERS` env allowlist rejected. Seven ordered rules; three of them (repo disabled / workflow disabled / not in `triggers.allowed_users`) set `explain: true` and earn a refusal comment, the four passive `triggers.*` filters set `explain: false` and stay silent. `ignore_authors` is checked before `allowed_users` on purpose: a bot login is normally in the former and absent from the latter, so the other order would answer every Renovate event with a public refusal comment. Only the `explain: true` reasons are posted to GitHub and all three are static strings, so there is no injection surface. There are **two** dispatch chokepoints and both call the gate: `src/workflows/dispatcher.ts applyRepoGate` (label + intent + by-name) and `src/workflows/ship/command-dispatch.ts isBlockedByRepoConfig` (the canonical ship rail, which bypasses the dispatcher). A third dispatch path would need it too. The PR-side config-validation comment (`src/webhook/events/pull-request.ts handlePullRequestConfigCheck`) is a GitHub write but not a dispatch, and honours the document-level `enabled: false` master switch ONLY, not the full trigger set, matching the scheduler's scope: the passive `triggers.*` filters would wrongly suppress authoring feedback on draft or `ignore_title_keywords` PRs. `stop`/`abort` are deliberately ungated so a config change cannot strand a mid-flight run, and the comment path gates before the intent classifier so a disabled repo costs no LLM call (rule 2 is skipped there, since the workflow name is not yet known). Trigger facts for rules 5-7 (title, draft, base branch) come from the webhook payload as a separate `TriggerContext`, deliberately NOT as fields on `DispatchTarget` (that object is persisted to `workflow_runs` and logged, and the title is attacker-controlled). Everything fails open: a missing, unreachable, or invalid file yields `DEFAULT_REPO_POLICY`. **Gate 2** (per-workflow model, turn/time limits, extra tools, and review-only path filters/instructions) is resolved by the controller. Structured workflows resolve it in `prepareWorkflowRunnerPayload` before registration and send the bounded result over the isolated runner protocol. Legacy shared-daemon jobs resolve it in `handleAccept` and send it in `job:payload`; scoped jobs retain their own rail. The six structured handlers apply the policy through `src/core/agent-policy.ts applyAgentPolicy`; non-review workflows never receive `instructions`. `workflows.review.auto` remains a dispatch-time knob and is not projected to either worker protocol. `workflows.ship` accepts only `enabled`: it is a composite controller workflow whose children resolve their own policy. +- **Scheduled actions**: a repo may ship a `.github-app.yaml` at its default-branch root declaring prompt-based actions on a cron schedule. The internal scheduler (`src/scheduler/`, gated by `SCHEDULER_ENABLED` + `DATABASE_URL` + non-empty `ALLOWED_OWNERS`) enqueues a `scheduled-action` job, a new job kind on the scoped-job rail, that the daemon runs as one agent session via `src/daemon/scheduled-action-executor.ts`. Missed cron slots are skipped, not backfilled. The document-level `enabled: false` short-circuits both `scanOnce` and `runAction`, so the repo-wide master switch silences unattended cron runs, not just the label and mention surfaces Gate 1 covers. The prompt is owner-trusted config. Cron parsing uses the `cron-parser` dependency. The bot-provided `merge_readiness` MCP tool is exposed only when `SCHEDULER_ALLOW_AUTO_MERGE` env AND per-action `auto_merge` are both true; `allowed_tools` is owner-trusted config, so an action granted a merge-capable Bash tool can still merge regardless. `resolve.ts` FR-017 is untouched. - **Review learnings**: review-policy directives extracted from past PR review pushback, persisted in the `review_learnings` table (migration 014) per-repo with an owner-wide (`scope: 'global'`) option. Loaded by the orchestrator into every job's payload uniformly; only `review` and `resolve` handlers pass `enableReviewLearnings: true` into `runPipeline`, so non-review workflows have `ctx.reviewLearnings` stripped before prompt build. The agent saves new directives via `save_review_learning` / removes via `delete_review_learning` MCP tools on the existing `repo_memory` server. The prompt block is gated to review/resolve, not wrapped in `` (these are sanitised repo policy, not attacker input), and the agent is told to treat applicable directives as overrides of its default review heuristics. Each `review`/`resolve` tracking comment ends with a `🧠 Learnings used` collapsible footer listing every directive that informed the run with full provenance (source PR, author, file glob). Server-side kill-switch: `REVIEW_LEARNINGS_ENABLED` (default true). `global`-scope writes are silently downgraded to `local` when `ALLOWED_OWNERS` has more than one owner. - **Comment-aware workflows**: the five structured workflows (`triage`, `plan`, `implement`, `review`, `resolve`) run `src/workflows/discussion-digest.ts` before the agent. - **What it does**: distills the issue/PR comment thread (issue comments, plus inline review comments for PRs) into a maintainer-guidance digest the prompt consumes in place of the raw thread. @@ -82,7 +85,7 @@ The runtime bot in `src/` supports three authentication modes (see `src/config.t 2. **`CLAUDE_CODE_OAUTH_TOKEN`**, Max/Pro subscription OAuth token (`sk-ant-oat...`, generated via `claude setup-token`). **Requires `ALLOWED_OWNERS`** to be set to a single-tenant value, because the [Agent SDK Note](https://code.claude.com/docs/en/agent-sdk/overview) prohibits serving other users' repos from a personal subscription quota. The token is forwarded to the Claude CLI subprocess via `buildProviderEnv()` in `src/core/executor.ts`; the CLI's own [auth precedence chain](https://code.claude.com/docs/en/authentication#authentication-precedence) picks between credentials if multiple are set. 3. **AWS Bedrock**, full credential chain via `CLAUDE_PROVIDER=bedrock` + `AWS_REGION` + `CLAUDE_MODEL` (Bedrock model ID format). Credential resolution handled by the AWS SDK inside the subprocess. -Default agent execution model when `CLAUDE_MODEL` is unset and `CLAUDE_PROVIDER=anthropic`: `claude-opus-4-7` (Opus 4.7). The Bedrock path still requires an explicit `CLAUDE_MODEL` (Bedrock model IDs differ from Anthropic's). +Default agent execution model when `CLAUDE_MODEL` is unset and `CLAUDE_PROVIDER=anthropic`: `claude-opus-5` (Opus 5). The Bedrock path still requires an explicit `CLAUDE_MODEL` (Bedrock model IDs differ from Anthropic's). The scheduled research workflow in `.github/workflows/research.yml` also uses `CLAUDE_CODE_OAUTH_TOKEN`, but via `anthropics/claude-code-action@v1`: that path is separately sanctioned for CI and is not subject to the `ALLOWED_OWNERS` requirement. @@ -90,7 +93,7 @@ The scheduled research workflow in `.github/workflows/research.yml` also uses `C GitHub-side auth defaults to the App installation token minted on demand from `GITHUB_APP_ID` + `GITHUB_APP_PRIVATE_KEY`. Optional override: -- **`GITHUB_PERSONAL_ACCESS_TOKEN`**: when set, replaces the installation token for every GitHub API call (PR comments, reviews, GraphQL) and `git push` authentication. Those actions are attributed to the PAT owner instead of the App bot. Commit author/committer metadata is **not** affected, `src/core/checkout.ts` hard-codes git `user.name` / `user.email` to `chrisleekr-bot[bot]`, so commit objects still carry the bot identity regardless of the auth token. **Requires `ALLOWED_OWNERS`** to contain exactly one owner, same single-tenant constraint as `CLAUDE_CODE_OAUTH_TOKEN`, because a PAT carries a real human identity and its per-user rate-limit bucket. Resolution happens in `resolveGithubToken()` (`src/core/github-token.ts`); downstream consumers (git credential helper, executor env, MCP servers) accept the resolved string regardless of source. +- **`GITHUB_PERSONAL_ACCESS_TOKEN`**: when set, replaces the App installation token on legacy and scoped GitHub API and git paths. Those actions are attributed to the PAT owner; commit author/committer metadata remains `chrisleekr-bot[bot]`. **Requires `ALLOWED_OWNERS`** to contain exactly one owner. Structured `workflow-run` dispatch fails closed in PAT mode because an isolated runner must receive a short-lived App token restricted to its target repository. ## Code Conventions @@ -102,20 +105,22 @@ GitHub-side auth defaults to the App installation token minted on demand from `G - Strict TypeScript: `exactOptionalPropertyTypes`, `noUncheckedIndexedAccess`, `useUnknownInCatchVariables`, etc. - Pre-commit hooks via Husky + lint-staged (auto-format + lint on staged files). - Conventional commits enforced via commitlint. -- Test files live under `test//foo.test.ts` or colocated as `src/.../foo.test.ts`; both are CI-gated. The runner (`scripts/test-isolated.sh`) globs `test/**/*.test.ts src/**/*.test.ts`, and `bun run check:test-globs` (`scripts/check-test-globs.ts`) fails CI if any `*.test.ts` is not reachable by that glob set, so a colocated test cannot go dark while CI stays green (issue #201). +- Test files live under `test//foo.test.ts`. The isolated runner globs only `test/**/*.test.ts`, and `bun run check:test-globs` fails CI if a test is placed outside that tree. ## CI/CD Pipeline -Four workflow files form the pipeline; each owns one responsibility. +Five pipeline files; each owns one responsibility. | Workflow | Trigger | Owns | | -------------------------------------- | --------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `.github/workflows/ci.yml` | `pull_request` + `push: main` + `workflow_call` | Quality gates only: typecheck, lint, format, audit:ci, test, build | | `.github/workflows/secrets-scan.yml` | `push: branches-ignore: [gh-pages]` + `workflow_dispatch` | Standalone gitleaks secret scan, decoupled from ci.yml so every push (incl. chore/docs/ci/test branches) is gated | | `.github/workflows/release-please.yml` | `push: [main, beta]` | release-please maintains a Release PR per branch; merging it cuts the release (main = stable `v` + `latest` image; beta = `v-beta` prerelease, no `latest`) then calls `docker-build.yml`. On a stable `main` release it also dispatches `github-app-released` to `chrisleekr/helm-charts` (`notify-helm-charts` job, after `docker`) so that repo opens the chart-sync PR | -| `.github/workflows/docker-build.yml` | `workflow_call` + `workflow_dispatch` | Reusable image builder: matrix split-and-merge (amd64 on `ubuntu-24.04` + arm64 on `ubuntu-24.04-arm`), SLSA v1 provenance + SBOM attestations (BuildKit + Sigstore), `gh attestation verify` regression gate, Trivy scan | +| `.github/workflows/docker-build.yml` | `workflow_call` + `workflow_dispatch` | Reusable image builder: matrix split-and-merge (amd64 on `ubuntu-24.04` + arm64 on `ubuntu-24.04-arm`), Trivy scan. SLSA v1 provenance + SBOM attestations and the `gh attestation verify` gate are **currently commented out** (SBOM exceeded 16MB); the code is retained in place for re-enablement | +| `.gitlab-ci.yml` | every branch (gates) + `main` (images) | GitLab CI: the same quality gates on **every** branch, then on `main` publishes `latest-orchestrator` (multi-arch) and `latest-daemon` (amd64-only) to the GitLab container registry | -- **Bun version is single-sourced** via `.tool-versions` (`bun 1.3.14`). All workflows use `oven-sh/setup-bun` with `bun-version-file: .tool-versions`. +- **Two image paths.** release-please → Docker Hub `-orchestrator` / `-daemon` plus the mutable `latest-orchestrator` / `latest-daemon` aliases (prod releases only), versioned and Trivy-scanned. GitLab `main` → GitLab registry `latest-orchestrator` / `latest-daemon`, mutable and unscanned. Neither path feeds the other. **Neither currently produces attestations**: the `provenance`/`sbom` inputs and the `gh attestation verify` gate in `docker-build.yml` are commented out ("SBOM file size is over 16MB, temporary disable"), and the GitLab builds pass `--provenance false`. +- **Bun version is single-sourced** via `.tool-versions` (`bun 1.3.14`). All GitHub workflows use `oven-sh/setup-bun` with `bun-version-file: .tool-versions`; `.gitlab-ci.yml` pins `image: oven/bun:-alpine` by hand, so a `.tool-versions` bump must be applied there too (`check:docs-versions` does not cover it). - **`audit:ci` (`scripts/audit-ci.ts`)** wraps `bun audit --json` to gate on severity: blocks on high+critical, warns on moderate+low, with an inline `IGNORED` GHSA allowlist (each entry must carry an `expires` date). Required because `bun audit` exits 1 on **any** finding regardless of `--audit-level`. - **Releases run on `release-please`** (Google), not semantic-release. Two source-controlled config + manifest pairs, selected by branch at runtime: `release-please-config.json` + `.release-please-manifest.json` on `main` (stable), and `release-please-config.beta.json` + `.release-please-manifest.beta.json` on `beta` (`versioning-strategy: prerelease` + `prerelease: true` + `prerelease-type: beta`). Both pairs live on both branches so the branches never share release-please state. Only `feat`/`fix`/`!`(breaking) commits bump the version; `refactor`/`perf`/`revert` alone do not cut a release (unlike the retired semantic-release rules). - **Releases are cut by merging the Release PR**, not by a manual dispatch. On a push to `main`, release-please opens/updates a stable Release PR (version + CHANGELOG diff); merging it tags `v`, creates the GitHub release, and builds the prod image (`latest`). The `beta` branch works the same way for prereleases. CHANGELOG.md is preserved: release-please prepends new sections and never rewrites existing history. @@ -124,19 +129,20 @@ Four workflow files form the pipeline; each owns one responsibility. - Defense-in-depth on workflow injection: every dynamic input flowing into a `run:` block is passed via `env:` first. - **GitHub Actions are SHA-pinned.** Every third-party `uses:` reference is pinned to a full 40-char commit SHA (with a `# vX.Y.Z` comment), not a mutable tag, so a force-moved upstream tag cannot change the bytes a runner executes. Renovate keeps the SHAs current via the `helpers:pinGitHubActionDigests` preset behind the existing 7-day `minimumReleaseAge` soak. `ci.yml` runs `bun run check:action-pins` (`scripts/check-action-pins.ts`), which fails the build if any third-party `uses:` is on a tag. Local reusable-workflow calls (`uses: ./...`) are exempt. - **Test-glob + destructive-action guards run in CI.** `ci.yml` also runs `bun run check:test-globs` (`scripts/check-test-globs.ts`, fails if any `*.test.ts` is unreachable by the runner glob set, issue #201) and `bun run check:no-destructive` (`scripts/check-no-destructive-actions.ts`, FR-009: fails if `src/workflows/ship/` or a `src/daemon/scoped-*-executor.ts` contains a force-push / `reset --hard` / `gh pr merge` / merge-mutation call outside comments; the scoped-executor set is derived from the filesystem so it cannot go stale, issue #203). Its `FORBIDDEN` pattern set is shared with the paired runtime layer, the `PreToolUse` destructive-Bash hook (`src/core/hooks/forbidden-bash.ts`, security invariant #5), via `src/utils/forbidden-bash.ts` so the static and runtime gates stay in lockstep (issue #222). +- **Config-schema guard runs in CI.** `ci.yml` runs `bun run check:config-schema` (`scripts/gen-config-schema.ts --check`), which fails if the committed `schema/github-app.schema.json` differs by a single byte from `z.toJSONSchema(githubAppConfigSchema, { io: "input" })`. That artifact is what `.github-app.yaml` authors consume via a `# yaml-language-server: $schema=` modeline, so a stale copy would advertise a config surface the runtime no longer accepts. After changing `src/repo-config/schema.ts`, rerun `bun run gen-config-schema` and commit the regenerated file. **The generated schema is structural-only**: zod v4's `toJSONSchema` drops `.refine` / `.superRefine`, so prompt-ref path traversal, IANA-timezone validity, glob safety in `review.path_filters`, and duplicate scheduled-action names remain runtime-only checks that no editor will catch. `scripts/validate-repo-config.ts ` runs the real zod pipeline locally and does cover them. `unrepresentable: "any"` is deliberately NOT passed to `toJSONSchema`, so a future unrepresentable node throws and fails this gate loudly instead of silently degrading that field to `{}`. The artifact is in `.prettierignore`: prettier collapses short arrays that `JSON.stringify(x, null, 2)` always expands, and one formatter has to own a byte-compared file. - **Env-contract guard runs in CI.** `ci.yml` runs `bun run check:env-contract` (`scripts/env-contract.ts --check`), which extracts every env var read by `loadConfig()` in `src/config.ts` and fails if the committed `env-contract.json` is stale, if any var is undocumented in `docs/operate/configuration.md`, or if a credential-shaped name (KEY/TOKEN/SECRET/PASSWORD/CREDENTIAL/BEARER segment, or a URL/DSN suffix that is not allowlisted) is missing from `SECRET_ENV_VARS` (`src/config-secret-env.ts`, the single source of secret-vs-config classification). After changing the env schema, rerun `bun run env-contract` and commit the regenerated `env-contract.json`. That file is the contract the `chrisleekr/helm-charts` `github-app` chart consumes (at `v`) to gate its ConfigMap/Secret parity. ## Security invariants (prompt-injection hardening) -Five contracts contributors MUST preserve when touching the agent execution path or any GitHub-bound write: +Six contracts contributors MUST preserve when touching the agent execution path or any GitHub-bound write: -1. **Subprocess env allowlist** (`src/core/executor.ts buildProviderEnv()`). The agent CLI receives an explicit allowlist + prefix patterns, NOT `...process.env`. If you add a new env var the CLI needs, extend the allowlist. Banned: `GITHUB_APP_PRIVATE_KEY`, `GITHUB_WEBHOOK_SECRET`, `DAEMON_AUTH_TOKEN`, `DATABASE_URL`, `VALKEY_URL`, `REDIS_URL`, `CONTEXT7_API_KEY`, `GITHUB_PERSONAL_ACCESS_TOKEN`. See `docs/operate/configuration.md` § "Subprocess env allowlist". +1. **Subprocess env allowlist** (`src/core/executor.ts buildProviderEnv()`). The agent CLI receives an explicit allowlist + prefix patterns, NOT `...process.env`. If you add a new env var the CLI needs, extend the allowlist. Banned: `GITHUB_APP_PRIVATE_KEY`, `GITHUB_WEBHOOK_SECRET`, `DAEMON_AUTH_TOKEN[_PREVIOUS]`, `WORKFLOW_RUNNER_CAPABILITY_SECRET[_PREVIOUS]`, `DATABASE_URL`, `VALKEY_URL`, `REDIS_URL`, `CONTEXT7_API_KEY`, `GITHUB_PERSONAL_ACCESS_TOKEN`. See `docs/operate/configuration.md` § "Subprocess env allowlist". 2. **Output secret-strip chokepoint** (`src/utils/github-output-guard.ts safePostToGitHub`). Two behaviours: - Regex pass (`redactSecrets()`) silently strips matched bytes. NEVER use the input-side `[REDACTED_X]` marker for output paths: markers leak probing signal to attackers. - - LLM scanner (default ON for `source: "agent"`, fail-open on Bedrock outage) catches encoded/obfuscated secrets the regex misses. The scanner sees attacker-influenced text and may itself be prompt-injected, so its `redacted_body` is NOT trusted verbatim: `safePostToGitHub` accepts the substitution only when it is **deletion-only** (`isDeletionOnly(regexResult.body, redactedBody)` in `src/utils/sanitize.ts`, an O(N) subsequence check). Any body that adds, reorders, or alters bytes the regex pass accepted is rejected, logged as `event: "llm_scanner_substitution_rejected"`, and the regex-only body is posted instead. Do NOT relax this gate to trust the scanner's output bytes (issue #198). + - The general GitHub-output LLM scanner (default ON for `source: "agent"`, fail-open on provider outage) catches encoded or obfuscated secrets the regex misses. The isolated-runner RPC scanner is deliberately stricter: scanner disablement, failure, or timeout rejects a command and converts a result to a fixed safe failure, with `workflow_runner_output_scan_unavailable` logged at `error`. The scanner sees attacker-influenced text and may itself be prompt-injected, so its `redacted_body` is NOT trusted verbatim: `safePostToGitHub` accepts the substitution only when it is **deletion-only** (`isDeletionOnly(regexResult.body, redactedBody)` in `src/utils/sanitize.ts`, an O(N) subsequence check). Any body that adds, reorders, or alters bytes the regex pass accepted is rejected, logged as `event: "llm_scanner_substitution_rejected"`, and the regex-only body is posted instead. Do NOT relax this gate to trust the scanner's output bytes (issue #198). - **Coverage status (Phase 1).** Wired through the chokepoint today: `core/tracking-comment.ts` (create + update), `daemon/scoped-fix-thread-executor.ts`, `workflows/ship/scoped/chat-thread.ts`, `workflows/ship/scoped/fix-thread.ts` (all reply paths via `postReply` helper). Phase 2 (NOT yet wired, tracked separately): `webhook/router.ts` capacity messages, `workflows/ship/tracking-comment.ts`, `workflows/ship/scoped/marker-comment.ts`, `workflows/ship/scoped/open-pr.ts`, `workflows/ship/scoped/rebase.ts`, `workflows/tracking-mirror.ts`, `workflows/ship/lifecycle-commands.ts`, `workflows/ship/session-runner.ts`, `workflows/dispatcher.ts`, `daemon/scoped-open-pr-executor.ts`. When you touch any of those, prefer routing the new write through `safePostToGitHub({ body, source, callsite, log, post })` rather than adding another bypass. + **Coverage status (Phase 1).** Wired through the chokepoint today: `core/tracking-comment.ts` (create + update), `daemon/scoped-fix-thread-executor.ts`, `workflows/ship/scoped/chat-thread.ts`, `workflows/ship/scoped/fix-thread.ts` (all reply paths via `postReply` helper), `workflows/tracking-mirror.ts` (every mirror write, which is what guards the Gate-2 `configWarning` banner). Phase 2 (NOT yet wired, tracked separately): `webhook/router.ts` capacity messages, `workflows/ship/tracking-comment.ts`, `workflows/ship/scoped/marker-comment.ts`, `workflows/ship/scoped/open-pr.ts`, `workflows/ship/scoped/rebase.ts`, `workflows/ship/lifecycle-commands.ts`, `workflows/ship/session-runner.ts`, `workflows/dispatcher.ts`, `daemon/scoped-open-pr-executor.ts`. When you touch any of those, prefer routing the new write through `safePostToGitHub({ body, source, callsite, log, post })` rather than adding another bypass. The MCP servers in `src/mcp/servers/` can't import `safePostToGitHub` directly (no daemon config in subprocess), they apply `redactSecrets()` inline. For logging they use `createMcpLogger(serverName)` (`src/mcp/mcp-logger.ts`), a stderr pino logger that shares `REDACT_PATHS` + `errSerializer` via the config-free `src/utils/log-redaction.ts` (issue #172), instead of raw `console.error`. So a server's structured lines are redacted with parity and carry `server` + `deliveryId`. @@ -146,6 +152,8 @@ Five contracts contributors MUST preserve when touching the agent execution path 5. **Runtime destructive-Bash gate** (`src/core/hooks/forbidden-bash.ts` + `src/utils/forbidden-bash.ts`). The agent subprocess runs under `bypassPermissions` with the Bash tool allowed, so a prompt-injected force-push / `git reset --hard` / `gh pr merge` / GraphQL merge mutation (`mergePullRequest` / `mergeBranch`) would otherwise execute unchecked. A `PreToolUse` hook (wired in `src/core/executor.ts` `queryOptions.hooks`, matcher `Bash`) denies any Bash command matching the shared `FORBIDDEN` pattern set at runtime, backing the prompt-only bans with an enforced gate. The patterns live in `src/utils/forbidden-bash.ts` as the single source of truth, shared with the static `check:no-destructive` CI guard (`scripts/check-no-destructive-actions.ts`) so the build-time and runtime layers cannot drift. A deny emits `event: "agent.hook.denied"` (fields `tool`, `rule`) and NEVER logs the raw command (token-leak risk). This stops literal destructive commands and is backed by remote branch protection plus a least-privilege token; a regex denylist on free-form shell is not a complete sandbox (it cannot defeat obfuscation like variable indirection or `base64|sh`). Issue #222. +6. **Structured-workflow isolation** (`src/k8s/workflow-runner-spawner.ts`, `src/runner/`, `src/orchestrator/workflow-runner-*.ts`). A `workflow-run` must never enter the shared-daemon job protocol. One exact attempt owns one Pod and capability Secret. The runner deny set must continue rejecting App, PAT, database, Valkey, Kubernetes, Context7, global GitHub, and daemon-auth credentials. Every state mutation stays fenced by run ID, attempt ID, owner ID, lease, and command receipt; the terminal payload is stored before projections and ACK. + The `triggerUsername` is rejected (not silently stripped) if it contains whitespace/newline, git commit trailer forging vector. Don't relax that check. ## Documentation @@ -161,7 +169,8 @@ The `docs/` tree is published as a MkDocs Material site at =0.50.3 <1", "@aws-crypto/sha256-js": "^4.0.0", "@aws-sdk/client-bedrock-runtime": "^3.797.0", "@aws-sdk/credential-providers": "^3.796.0", "@smithy/eventstream-serde-node": "^2.0.10", "@smithy/fetch-http-handler": "^5.0.4", "@smithy/protocol-http": "^3.0.6", "@smithy/signature-v4": "^3.1.1", "@smithy/smithy-client": "^2.1.9", "@smithy/types": "^2.3.4", "@smithy/util-base64": "^2.0.0" } }, "sha512-eV3vqNvMEy9C5nBgZMlKFewgXoBcIuV/PERRQJIlot3Vd5lU8dVUS7YqkkbXLw8p/gkDmam3PPoPNYsEAmqivw=="], - "@anthropic-ai/claude-agent-sdk": ["@anthropic-ai/claude-agent-sdk@0.3.146", "", { "optionalDependencies": { "@anthropic-ai/claude-agent-sdk-darwin-arm64": "0.3.146", "@anthropic-ai/claude-agent-sdk-darwin-x64": "0.3.146", "@anthropic-ai/claude-agent-sdk-linux-arm64": "0.3.146", "@anthropic-ai/claude-agent-sdk-linux-arm64-musl": "0.3.146", "@anthropic-ai/claude-agent-sdk-linux-x64": "0.3.146", "@anthropic-ai/claude-agent-sdk-linux-x64-musl": "0.3.146", "@anthropic-ai/claude-agent-sdk-win32-arm64": "0.3.146", "@anthropic-ai/claude-agent-sdk-win32-x64": "0.3.146" }, "peerDependencies": { "@anthropic-ai/sdk": ">=0.93.0", "@modelcontextprotocol/sdk": "^1.29.0", "zod": "^4.0.0" } }, "sha512-hK9/Ng+hOyexUemTxdIUsSWJ9o2LFi2YNWzHwz8/YMCohUYOnFMZkBiENvUAb0WIc5hieOyBZrOIlg5OewuJMg=="], + "@anthropic-ai/claude-agent-sdk": ["@anthropic-ai/claude-agent-sdk@0.3.239", "", { "optionalDependencies": { "@anthropic-ai/claude-agent-sdk-darwin-arm64": "0.3.239", "@anthropic-ai/claude-agent-sdk-darwin-x64": "0.3.239", "@anthropic-ai/claude-agent-sdk-linux-arm64": "0.3.239", "@anthropic-ai/claude-agent-sdk-linux-arm64-musl": "0.3.239", "@anthropic-ai/claude-agent-sdk-linux-x64": "0.3.239", "@anthropic-ai/claude-agent-sdk-linux-x64-musl": "0.3.239", "@anthropic-ai/claude-agent-sdk-win32-arm64": "0.3.239", "@anthropic-ai/claude-agent-sdk-win32-x64": "0.3.239" }, "peerDependencies": { "@anthropic-ai/sdk": ">=0.93.0", "@modelcontextprotocol/sdk": "^1.29.0", "zod": "^4.0.0" } }, "sha512-cIuZhK4u76S5Otq78U890GSA6BFT4SLqOuMqzU/bP/tWRWKhHhNp/3/pvgLwoVGlkdhD7luXWduqXKyLC+VNBQ=="], - "@anthropic-ai/claude-agent-sdk-darwin-arm64": ["@anthropic-ai/claude-agent-sdk-darwin-arm64@0.3.146", "", { "os": "darwin", "cpu": "arm64" }, "sha512-0IIvlEaenq2CRSVx5Bo5BaCtHQXS87GancM35WKEYveGVLn6DI+5G7ikYuTE4AKRPkMnogFtY4BJt6LulWGj+A=="], + "@anthropic-ai/claude-agent-sdk-darwin-arm64": ["@anthropic-ai/claude-agent-sdk-darwin-arm64@0.3.239", "", { "os": "darwin", "cpu": "arm64" }, "sha512-GGVGuCwFEUm6cMlnBX0LTC9JX5NdGzxddbuqWtRxEgo9EetS70SO3FW+reitALlotHghPTfnICQILbBDIRyX+Q=="], - "@anthropic-ai/claude-agent-sdk-darwin-x64": ["@anthropic-ai/claude-agent-sdk-darwin-x64@0.3.146", "", { "os": "darwin", "cpu": "x64" }, "sha512-Dk5xJ03Ff1JXbMRP1t2wc/TyfY6xF/2Ysp31wMhFPjoNiKSPHMWaIg242+T3CHdxLWmJ8plWHL1HL5cyZ/LCkw=="], + "@anthropic-ai/claude-agent-sdk-darwin-x64": ["@anthropic-ai/claude-agent-sdk-darwin-x64@0.3.239", "", { "os": "darwin", "cpu": "x64" }, "sha512-QNbBXz3Pb3pQ7a+Kcbets6t9IrQhStKsfl5D518nYiGFoRMioO7efkZ6zHUcrGDqDC0LIzrs7tY2KNzH4RfwZA=="], - "@anthropic-ai/claude-agent-sdk-linux-arm64": ["@anthropic-ai/claude-agent-sdk-linux-arm64@0.3.146", "", { "os": "linux", "cpu": "arm64" }, "sha512-mzBXDDWWBAC/vDtAYpO1G/dq5QvJtYSPXsqcb+sNdcDhiuf4IYnYp7ytRncYlsUNDkLmX6Gk2jkWAHUUA2Lozg=="], + "@anthropic-ai/claude-agent-sdk-linux-arm64": ["@anthropic-ai/claude-agent-sdk-linux-arm64@0.3.239", "", { "os": "linux", "cpu": "arm64" }, "sha512-RE6tDtzU0xj58tsuxnlXMJO8ckJ4tx/1nUgR+D/fPEQVt84oOyXeVspKt1ffvycogh3Sr3MkCGwZrPmxu/V1nA=="], - "@anthropic-ai/claude-agent-sdk-linux-arm64-musl": ["@anthropic-ai/claude-agent-sdk-linux-arm64-musl@0.3.146", "", { "os": "linux", "cpu": "arm64" }, "sha512-QlCid0ucdrmhUAOewfQjaofN2wlokWcfFTxSFePTSj1umk35JO7TDFP700F7jU49r1fPWIdvJpPwWGyB0DeFPA=="], + "@anthropic-ai/claude-agent-sdk-linux-arm64-musl": ["@anthropic-ai/claude-agent-sdk-linux-arm64-musl@0.3.239", "", { "os": "linux", "cpu": "arm64" }, "sha512-Ajc3cuszVdOwfMZVsGdxrCTmgWOeJpQWAIqu8jNEvERIeNnBxvWfGrjmLPxYT3/LJ9Uj/tFTpp52J4IcmrJE5w=="], - "@anthropic-ai/claude-agent-sdk-linux-x64": ["@anthropic-ai/claude-agent-sdk-linux-x64@0.3.146", "", { "os": "linux", "cpu": "x64" }, "sha512-B2baXU1tCBT5CVlD7jJMKjpC4xdO45NUIWpqImmwuOfKvlM/PITjyTXyTY662mGZf1dBmdqBBsqirwFH/jhi8Q=="], + "@anthropic-ai/claude-agent-sdk-linux-x64": ["@anthropic-ai/claude-agent-sdk-linux-x64@0.3.239", "", { "os": "linux", "cpu": "x64" }, "sha512-q4YaDoPgqh0XM23RM1/Zje7OSKccuCTQE89KoppDFOsyGdRsUj5xr01LTtr5hnYQuZD7dfwAbz9zl1g0MF/7TA=="], - "@anthropic-ai/claude-agent-sdk-linux-x64-musl": ["@anthropic-ai/claude-agent-sdk-linux-x64-musl@0.3.146", "", { "os": "linux", "cpu": "x64" }, "sha512-E3coK1ThQT08KIX80RLcsq7DWXFllCKOzoOe32it/bdtY56TBgPY9xemwXhIJ+cVBHTI9/MpBSIlKBcFCt+yQA=="], + "@anthropic-ai/claude-agent-sdk-linux-x64-musl": ["@anthropic-ai/claude-agent-sdk-linux-x64-musl@0.3.239", "", { "os": "linux", "cpu": "x64" }, "sha512-zIUHiG4Romm/t6m/S9n8x4BKluyRCPk0147hPVo4xxkHkp4Di/7TnDMRKO3Wau4x361wRqlE2i5EpYT+CyJjHg=="], - "@anthropic-ai/claude-agent-sdk-win32-arm64": ["@anthropic-ai/claude-agent-sdk-win32-arm64@0.3.146", "", { "os": "win32", "cpu": "arm64" }, "sha512-CIwQxGX2r/yWpjCJ6ahB3smKXhghWgGTxL98+LGW52TUwqTiBnlNrH9DPqqgv1/+Hyquw6xfLrKU+StyfMgiLw=="], + "@anthropic-ai/claude-agent-sdk-win32-arm64": ["@anthropic-ai/claude-agent-sdk-win32-arm64@0.3.239", "", { "os": "win32", "cpu": "arm64" }, "sha512-RxA29NdX9g3ZbpcXvSeWxpbg/Eoo3wXfO2eA1Vc7qa5JyAYjrl9xu6dIGAWMiyk/gnFxNh1WoFER77J9P6LTiQ=="], - "@anthropic-ai/claude-agent-sdk-win32-x64": ["@anthropic-ai/claude-agent-sdk-win32-x64@0.3.146", "", { "os": "win32", "cpu": "x64" }, "sha512-qmxrsyaqA8s4HShqJls7ZCRjdoqN66Jo/hbjQNB3uHepD8tEO1iD19aPV4+osdLT7feMkhDBfLT07Q30R2NB5w=="], + "@anthropic-ai/claude-agent-sdk-win32-x64": ["@anthropic-ai/claude-agent-sdk-win32-x64@0.3.239", "", { "os": "win32", "cpu": "x64" }, "sha512-ylKIX0DfaK1EgWYbVEvBMYATVFdKjFcWvvypTIv2sJhM3KxJT/0lTzvqsai8jYeZN1FBHWlRNEUQJvJMjO/diA=="], "@anthropic-ai/sdk": ["@anthropic-ai/sdk@0.120.0", "", { "dependencies": { "json-schema-to-ts": "^3.1.1", "standardwebhooks": "^1.0.0" }, "peerDependencies": { "zod": "^3.25.0 || ^4.0.0" }, "optionalPeers": ["zod"], "bin": { "anthropic-ai-sdk": "bin/cli" } }, "sha512-ZlvmNFT/iIF6JD13rxbbMWD8nvGR0RaUp6yMQnoc+4Af0YjVVe/bIdW1XSQQsoxXAtg1NaT6Vak0LKFlJ4d37Q=="], diff --git a/bunfig.toml b/bunfig.toml index c379191d..fab472a1 100644 --- a/bunfig.toml +++ b/bunfig.toml @@ -2,9 +2,6 @@ # Preload test environment setup before each test file preload = ["./test/preload.ts"] -# Test timeout (ms) -- prevent hung tests in CI -timeout = 30000 - # Coverage configuration. # coverageThreshold is applied per-file: every file must meet both minimums. # See https://bun.sh/docs/test/code-coverage#coverage-thresholds diff --git a/docs/build/architecture.md b/docs/build/architecture.md index 79b838c8..6cc34311 100644 --- a/docs/build/architecture.md +++ b/docs/build/architecture.md @@ -1,66 +1,189 @@ # Architecture -A single HTTP server process receives GitHub webhook events, acknowledges within ten seconds, and asynchronously hands each event to a daemon for execution. Every event walks the same path: verify → route → classify → enqueue → daemon claims the job → run the pipeline → finalise the tracking comment. +A single controller process receives GitHub webhook events, acknowledges within ten seconds, and asynchronously dispatches execution. The controller is the control plane and the only component with PostgreSQL, Valkey, GitHub App key, and Kubernetes authority. Repository work runs outside that boundary. Structured `workflow-run` jobs use one-attempt Kubernetes runner Pods. Legacy and scoped jobs use the shared daemon fleet. -## Request flow +## System topology ```mermaid -flowchart TD - GH["GitHub webhook
POST /api/github/webhooks"]:::entry - VERIFY["Verify HMAC-SHA256"]:::guard - ACK["200 OK within 10 seconds"]:::ack - ROUTE["Handler dispatch
delivery claim + allowlist + concurrency"]:::guard - TR["Haiku triage
binary heavy classifier"]:::decide - QUEUE["Orchestrator job queue
Valkey list"]:::store - SCALE{{"Scale-up decision
heavy OR queue >= threshold
AND no persistent slots
AND cooldown elapsed"}}:::fork - SPAWN["K8s API
create bare Pod
DAEMON_EPHEMERAL=true"]:::decide - FLEET["Daemon fleet
persistent + ephemeral
WebSocket connections"]:::target - PIPE["runPipeline
clone + prompt + Claude Agent SDK"]:::work - FIN["Finalise tracking comment
success, error, or cost summary"]:::done - - GH --> VERIFY --> ACK - ACK -. async .-> ROUTE - ROUTE --> TR - TR --> QUEUE - QUEUE --> SCALE - SCALE -->|yes| SPAWN - SPAWN --> FLEET - SCALE -->|no, or cooldown active| FLEET - QUEUE -->|JobOffer| FLEET - FLEET --> PIPE - PIPE --> FIN - - classDef entry fill:#0b5cad,stroke:#083e74,color:#ffffff - classDef guard fill:#164a3a,stroke:#0d2c24,color:#ffffff - classDef ack fill:#2a6f2a,stroke:#1a4d1a,color:#ffffff - classDef decide fill:#8a5a00,stroke:#5c3d00,color:#ffffff - classDef fork fill:#6a2080,stroke:#451454,color:#ffffff - classDef target fill:#114a82,stroke:#0a2f56,color:#ffffff - classDef work fill:#4a2e7a,stroke:#311f50,color:#ffffff - classDef store fill:#5c3d00,stroke:#3d2900,color:#ffffff - classDef done fill:#2a6f2a,stroke:#1a4d1a,color:#ffffff +flowchart TB + GitHub["GitHub
webhooks and API"]:::external + Postgres["PostgreSQL
workflows, executions, leases, state"]:::store + Valkey["Valkey
delivery claims, queue, processing lists, liveness"]:::store + Kubernetes["Kubernetes API
Pod and Secret lifecycle"]:::external + Providers["LLM providers
Anthropic and Bedrock"]:::external + + subgraph ControllerProcess["Controller process
single supported replica"] + HttpIngress["Bun HTTP router
src/app.ts and src/http-router.ts"]:::control + WebhookAdmission["Octokit webhook middleware
signature verification"]:::guard + EventCallback["Event callback
synchronous surface and owner gates"]:::guard + HttpResponse["HTTP 200 or 202
response deadline"]:::guard + WebhookContinuation["Async event continuation
delivery claim, repo policy, dispatch"]:::control + SchedulerSources["Internal schedulers
ship tickles, proposals, scheduled actions"]:::control + DispatchControl["Dispatchers and ship reactor
repo policy, mutex, routing"]:::control + DurableCommit["Durable state boundary
workflow and execution transactions"]:::control + DispatchOutbox["Workflow dispatch outbox
post-commit publication"]:::control + QueueWorker["Queue worker
LMOVE processing lease"]:::control + DaemonControl["Daemon control
fleet selection and offer protocol"]:::control + RunnerControl["Runner control
admission, capability RPC, token minting"]:::control + ResultProjection["Result projection
terminal state, cascade, GitHub output"]:::control + Reconciliation["Reconcilers
outbox, liveness, expiry, resource cleanup"]:::recovery + end + + DaemonFleet["Shared daemon fleet
persistent and ephemeral Pods"]:::daemon + DaemonExecution["Daemon execution boundary
legacy pipeline or scoped executor"]:::work + RunnerPod["One-attempt workflow runner Pod
owned capability Secret"]:::runner + RunnerExecution["Runner execution boundary
named workflow handler"]:::work + + GitHub --> HttpIngress --> WebhookAdmission --> EventCallback --> HttpResponse --> GitHub + EventCallback -.->|"start async continuation"| WebhookContinuation --> DispatchControl + SchedulerSources --> DispatchControl + DispatchControl --> DurableCommit + DurableCommit <--> Postgres + DurableCommit --> DispatchOutbox --> Valkey + Valkey <--> QueueWorker + QueueWorker --> DaemonControl + QueueWorker --> RunnerControl + DaemonControl <--> Postgres + DaemonControl <--> Valkey + RunnerControl <--> Postgres + DaemonControl <--> DaemonFleet + DaemonControl -.->|"scale ephemeral fleet"| Kubernetes + Kubernetes --> DaemonFleet + RunnerControl --> Kubernetes --> RunnerPod + RunnerControl <--> RunnerPod + DaemonFleet --> DaemonExecution + RunnerPod --> RunnerExecution + DaemonExecution --> GitHub + RunnerExecution --> GitHub + DaemonExecution --> Providers + RunnerExecution --> Providers + DaemonExecution --> DaemonControl --> ResultProjection + RunnerExecution --> RunnerControl --> ResultProjection + ResultProjection <--> Postgres + ResultProjection --> GitHub + Reconciliation <--> Postgres + Reconciliation <--> Valkey + Reconciliation --> Kubernetes + + classDef external fill:#2c3e50,stroke:#17202a,color:#ffffff + classDef guard fill:#6c3483,stroke:#4a235a,color:#ffffff + classDef control fill:#1f4e79,stroke:#102a43,color:#ffffff + classDef store fill:#7d6608,stroke:#4d3f05,color:#ffffff + classDef daemon fill:#1e6b3a,stroke:#0f3d21,color:#ffffff + classDef runner fill:#0b7285,stroke:#064653,color:#ffffff + classDef work fill:#8e2c62,stroke:#571b3c,color:#ffffff + classDef recovery fill:#4d5656,stroke:#2c3333,color:#ffffff ``` +Solid arrows show the ordinary control or data path. Dotted arrows show asynchronous dispatch or scaling. PostgreSQL remains the durable authority; Valkey provides bounded idempotency, wake-up, queue, and liveness mechanisms. + +## Request lifecycle + +```mermaid +flowchart TB + WebhookDelivery["GitHub webhook delivery"]:::external + VerifySignature["Verify HMAC signature
route subscribed event"]:::guard + InvokeCallback["Invoke subscribed callback"]:::guard + SynchronousGates["Run synchronous gates
event surface, bot sender, allowed owner"]:::guard + Acknowledge["Return HTTP 200 or 202
within ten seconds"]:::guard + AsyncContinuation["Start fire-and-forget continuation"]:::control + DeliveryClaim["Apply delivery claim on guarded paths
before durable or external work"]:::guard + RouteKind{"Dispatch path"}:::decision + ShipReactor["Ship reactor
wake durable continuation"]:::control + WorkflowCommit["PostgreSQL transaction
workflow_runs plus executions"]:::store + DaemonReceipt["Create execution receipt
legacy or scoped payload"]:::store + LegacyTriage["Legacy triage and scaler
heavy or queue overflow"]:::control + PublishOutbox["Publish committed workflow
retryable dispatch outbox"]:::control + SharedQueue["Valkey queue:jobs"]:::store + ProcessingLease["Queue worker LMOVE
instance processing list"]:::control + QueuedKind{"Queued job kind"}:::decision + + RunnerClaim["Atomic PostgreSQL admission
capacity, attempt, absolute deadline"]:::control + RunnerResources["Create bare Pod
read UID, create owned Secret"]:::control + RunnerRegister["Capability-scoped WSS registration
deliver repository token once"]:::guard + RunnerWork["Named workflow handler
isolated checkout, agent and MCP as required"]:::runner + RunnerResult["workflow-runner:result
validated terminal message"]:::runner + RunnerTerminal["Persist attempt and execution result
before result ACK"]:::store + RunnerCleanup["Delete exact Pod and Secret
with UID preconditions"]:::recovery + + DaemonSelect["Select active daemon
capabilities, draining state, load"]:::control + DaemonOffer["job:offer or scoped-job:offer
offer timeout and retry"]:::control + DaemonAccept["job:accept then job:payload
installation token and context"]:::guard + DaemonWork["Legacy pipeline or scoped executor
unique checkout, agent and MCP as required"]:::daemon + DaemonResult["job:result or scoped-job:completion
validated completion message"]:::daemon + DaemonTerminal["Fence daemon ownership
finalize execution receipt"]:::store + + Projection["Reconcile terminal state
cascade, locks, tracking projection"]:::control + GitHubEffects["GitHub API effects
comments, reviews, commits, pull requests"]:::external + RecoveryLoop["Periodic recovery
outbox, lease expiry, disconnected daemons"]:::recovery + + WebhookDelivery --> VerifySignature --> InvokeCallback --> SynchronousGates + SynchronousGates --> Acknowledge + SynchronousGates -.->|"return void and continue asynchronously"| AsyncContinuation + AsyncContinuation --> DeliveryClaim --> RouteKind + RouteKind -->|"ship wake"| ShipReactor + ShipReactor --> WorkflowCommit + RouteKind -->|"structured workflow"| WorkflowCommit + RouteKind -->|"legacy request"| LegacyTriage + LegacyTriage --> DaemonReceipt + RouteKind -->|"scoped or scheduled job"| DaemonReceipt + WorkflowCommit --> PublishOutbox --> SharedQueue + DaemonReceipt --> SharedQueue + SharedQueue --> ProcessingLease --> QueuedKind + + QueuedKind -->|"workflow-run"| RunnerClaim + RunnerClaim --> RunnerResources --> RunnerRegister --> RunnerWork + RunnerWork --> RunnerResult --> RunnerTerminal --> Projection + RunnerTerminal --> RunnerCleanup + + QueuedKind -->|"legacy or scoped"| DaemonSelect + DaemonSelect --> DaemonOffer --> DaemonAccept --> DaemonWork + DaemonWork --> DaemonResult --> DaemonTerminal --> Projection + + RunnerWork --> GitHubEffects + DaemonWork --> GitHubEffects + Projection --> GitHubEffects + RecoveryLoop -.->|"republish stale outbox"| SharedQueue + RecoveryLoop -.->|"expire fenced attempt"| RunnerTerminal + RecoveryLoop -.->|"fail orphaned execution"| DaemonTerminal + RecoveryLoop -.->|"retry owned cleanup"| RunnerCleanup + + classDef external fill:#2c3e50,stroke:#17202a,color:#ffffff + classDef guard fill:#6c3483,stroke:#4a235a,color:#ffffff + classDef control fill:#1f4e79,stroke:#102a43,color:#ffffff + classDef store fill:#7d6608,stroke:#4d3f05,color:#ffffff + classDef daemon fill:#1e6b3a,stroke:#0f3d21,color:#ffffff + classDef runner fill:#0b7285,stroke:#064653,color:#ffffff + classDef decision fill:#9c640c,stroke:#633f08,color:#ffffff + classDef recovery fill:#4d5656,stroke:#2c3333,color:#ffffff +``` + +The webhook middleware verifies the signature, invokes the subscribed callback, and waits only for the callback's return. Event handlers run their cheap synchronous gates before returning `void`, then start a fire-and-forget continuation for delivery claims and dispatch. The HTTP acknowledgement and that continuation can overlap. Neither branch waits for repository execution. + +The two worker rails share the queue and durable execution accounting, but not execution authority or ordinary GitHub output ownership. A structured runner must transfer recovery authority to its PostgreSQL attempt before the processing-list item is released; the controller then stores and projects its terminal result. A daemon performs its GitHub effects directly and remains tied to its execution receipt and exact daemon incarnation through the offer, payload, and completion sequence. + ## Key concepts -- **Async processing.** The webhook handler responds within ten seconds, so the side-effecting `events/*` handlers fire their dispatch with fire-and-forget semantics after the 200 OK is queued. Every box downstream of `ACK` runs after the HTTP response is on the wire. (`router.ts processRequest` is the equivalent path for the dev-only `/api/test/webhook` endpoint, not production.) -- **Webhook delivery idempotency (issue #202).** GitHub is at-least-once: a delivery (auto-retry or operator redelivery) replays with the same `X-GitHub-Delivery` for up to 3 days. The four side-effecting handlers (`events/issue-comment.ts`, `events/review-comment.ts`, the label branches of `events/issues.ts` + `events/pull-request.ts`) call `claimDelivery(deliveryId)` (`src/webhook/idempotency.ts`) at the top of their dispatch path, before any LLM call, `workflow_runs` insert, or GitHub write. It is a Valkey `SET key 1 NX EX 259200` claim: `true` exactly once per delivery, `false` (and an early return) on a redelivery. It is **fail-open**, when Valkey is unconfigured or disconnected (gated on `isValkeyHealthy()`) it returns `true`, degrading to at-least-once rather than dropping or blocking webhooks. `events/review.ts` is exempt (idempotent reactor wake only). The durable backstop behind the best-effort Valkey layer is the `idx_workflow_runs_inflight` partial-unique index: the dispatcher rejects a second in-flight run for the same workflow+target even when the Valkey claim was skipped. The legacy in-memory `Map` + `isAlreadyProcessed` tracking-comment scan was retired in issue #211 (it only ever guarded the dev-test-only `router.ts processRequest` path, which production handlers bypass). `DATABASE_URL` is required to persist execution / dispatch history and the in-flight guard across restarts. -- **One request, one clone.** Each delivery clones the repo into a unique temp directory under `CLONE_BASE_DIR` **on the daemon host**. Claude operates on local files via `cwd`. On PR events the checkout supplementally fetches `origin/` (when it differs from the head ref) so the agent's `git diff origin/...HEAD` and `git rebase origin/` directives resolve first try. A sibling `${workDir}-artifacts` directory is created outside the checkout and exposed to the agent as `BOT_ARTIFACT_DIR`: workflow summary files (IMPLEMENT.md / REVIEW.md / RESOLVE.md) are written there so they can never be picked up by a `git add` inside the clone. Both directories are removed in the pipeline's `finally` block regardless of outcome. -- **GitHub credential resolution.** `src/core/github-token.ts:resolveGithubToken()` is the single source of the GitHub credential the daemon uses. Default is an App installation token minted just-in-time from the cached `App` singleton in `src/orchestrator/connection-handler.ts`. When `GITHUB_PERSONAL_ACCESS_TOKEN` is set, the helper short-circuits and returns the PAT instead, API/git authentication runs as the PAT owner. Commit author/committer metadata is **not** affected; `src/core/checkout.ts` hard-pins git `user.name`/`user.email` to `chrisleekr-bot[bot]` so commit objects still carry the bot identity. The git credential helper, executor `GH_TOKEN`/`GITHUB_TOKEN` env vars, and MCP server env all consume the resolved string without caring about its source. -- **The webhook server never runs the pipeline.** Only daemons execute `runPipeline`. The webhook server is the orchestrator: it enqueues jobs and optionally spawns ephemeral daemons. -- **Every orchestrator runs a queue worker.** `src/orchestrator/queue-worker.ts` polls `queue:jobs` via `LMOVE` into a per-instance processing list (`queue:processing:{instanceId}`), offers the job to a locally-connected daemon, and atomically re-queues it to the head when no local daemon can take it. Multi-orchestrator HA: `LMOVE` grants exactly-once claim across instances; the offer/accept round-trip stays in-process. Crash recovery is handled by each orchestrator draining its own processing list at startup, plus a cross-instance reaper (`src/orchestrator/valkey-cleanup.ts`) draining processing lists owned by instances whose `orchestrator:{id}:alive` liveness key has expired. +- **Async processing.** The webhook callback runs synchronous event-surface and owner-authorization gates, starts its side-effecting continuation without awaiting it, and returns so the middleware can acknowledge within ten seconds. Delivery claims and dispatch run in that continuation and can overlap the HTTP response. (`router.ts processRequest` is the equivalent path for the dev-only `/api/test/webhook` endpoint, not production.) +- **Webhook delivery idempotency (issue #202).** GitHub is at-least-once: a delivery (auto-retry or operator redelivery) replays with the same `X-GitHub-Delivery` for up to 3 days. The four side-effecting handlers (`events/issue-comment.ts`, `events/review-comment.ts`, the label branches of `events/issues.ts` + `events/pull-request.ts`) call `claimDelivery(deliveryId)` (`src/webhook/idempotency.ts`) at the top of their dispatch path, before any LLM call, `workflow_runs` insert, or GitHub write. It is a Valkey `SET key 1 NX EX 259200` claim: `true` exactly once per delivery, `false` (and an early return) on a redelivery. It is **fail-open**, when Valkey is unconfigured or disconnected (gated on `isValkeyHealthy()`) it returns `true`, degrading to at-least-once rather than dropping or blocking webhooks. `events/review.ts` is exempt (idempotent reactor wake only). The durable backstop behind the best-effort Valkey layer is the `idx_workflow_runs_inflight` partial-unique index: the dispatcher rejects a second in-flight run for the same workflow+target even when the Valkey claim was skipped. The legacy in-memory `Map` + `isAlreadyProcessed` tracking-comment scan was retired in issue #211 (it only ever guarded the dev-test-only `router.ts processRequest` path, which production handlers bypass). `DATABASE_URL` is required to persist execution / dispatch history and the in-flight guard across restarts. Two branches claim a **suffixed** key rather than the bare delivery id, because `claimDelivery` is one-shot per key and `pull_request.synchronize` fans out to more than one consumer: `` `${deliveryId}:config-check` `` for the PR config validator and `` `${deliveryId}:auto-review` `` for the auto-review dispatch. A shared key would let whichever branch ran first starve the other. +- **One request, one clone.** Each execution clones the repo into a unique temp directory under `CLONE_BASE_DIR`, on a one-attempt runner Pod for structured workflows and on a shared daemon for legacy or scoped jobs. Claude operates on local files via `cwd`. A sibling `${workDir}-artifacts` directory holds workflow summaries outside the checkout. Both directories are removed in the pipeline's `finally` block. +- **GitHub credentials are repository-scoped at the runner boundary.** In App mode the controller mints an installation token restricted to the target repository and sends only that token to the runner. The runner never receives the App private key. Structured workflows fail closed when `GITHUB_PERSONAL_ACCESS_TOKEN` is configured because a PAT cannot be narrowed to one repository by the controller. Legacy and scoped shared-daemon jobs retain the existing token-resolution behavior. +- **Auto-review on push.** `pull_request.synchronize` can dispatch `review` with no label and no mention, gated on two keys that must agree: the server's `AUTO_REVIEW_USERS` allowlist (which logins may trigger it) and the repo's `workflows.review.auto` (whether this repo wants it). The env half exists because auto-review _widens_ what the bot does, and the `.github-app.yaml` `triggers:` block is narrowing-only by contract; the repo half exists because the env allowlist is server-wide. It matches the authenticated **pusher** (`payload.sender.login`), deliberately not the commit author the ship reactor resolves alongside it: the author is derived from a settable commit email, and this is an authorization decision. Three further guards keep it from feeding itself, all silent: our own pushes are skipped (`resolve` pushes a commit per fix), pushes whose diff fingerprint is unchanged are skipped (a rebase), and a review already in flight wins via `idx_workflow_runs_inflight` rather than queueing. Dispatch goes through `dispatchWorkflowByName({ auto: true })`, which suppresses every refusal comment and the `bot:*` label mutex, so an auto-trigger never writes to the PR except through the review itself. +- **The controller never runs repository code.** Shared daemons execute legacy and scoped jobs. Structured `workflow-run` jobs execute in one-attempt Pods. The controller owns PostgreSQL, Valkey, GitHub App keys, runner admission, Kubernetes resources, and result projection. +- **The supported topology is one controller replica.** `src/orchestrator/queue-worker.ts` leases queue items with `LMOVE`. Workflow admission and recovery authority are committed in PostgreSQL before the processing-list item is released. Startup recovers this instance's list, and every liveness-reaper pass returns items from lists whose 60-second orchestrator heartbeat has expired. The repository does not implement distributed controller session ownership or a distributed admission semaphore. +- **The daemon image enforces a Linux parent-process boundary.** A compiled preload guard sets `PR_SET_DUMPABLE=0` before Bun application code. Shared daemons and workflow runners fail startup unless an empty-environment same-UID child receives `EACCES` or `EPERM` while reading the parent's `/proc//environ`. The GitHub release workflow runs this exact probe against every pushed daemon image digest on both supported architectures. The GitLab main-branch publisher loads its amd64 image locally, runs the same probe, and pushes only after it passes. - **MCP servers.** Tracking-comment updates, inline PR reviews, scoped review-thread resolves, daemon-capability reports, repo-memory, and (optionally) Context7 library docs are exposed as MCP servers the agent can call. Git changes are made via the Bash tool against the cloned repo, not through a dedicated MCP server. - **Destructive Bash is runtime-gated.** The agent runs under `bypassPermissions` with the Bash tool allowed, so prompt-only bans alone do not stop a prompt-injected force-push or merge. A `PreToolUse` hook (`src/core/hooks/forbidden-bash.ts`, wired in `src/core/executor.ts`) denies any Bash command matching the shared `FORBIDDEN` set (force-push, `git reset --hard`, branch delete, history rewrite, `gh pr merge`, GraphQL merge mutations) before it executes. The pattern set is shared with the static `check:no-destructive` CI guard via `src/utils/forbidden-bash.ts`, so build-time and runtime gates cannot drift. A deny emits an `agent.hook.denied` log line. ## Dispatch flow -Dispatch collapsed to a single target, `daemon`, in migration `004_collapse_dispatch_to_daemon.sql`. Every job is claimed by some daemon in the fleet over WebSocket. The router decides only the **reason** the job lands there and whether to spawn an ephemeral daemon. +Migration `017_workflow_run_leases.sql` records the protocol that actually owns each execution. The queue worker branches before daemon selection: `workflow-run` items go to an isolated runner, while legacy and scoped items retain the shared-daemon offer protocol. -### Single target, four reasons +### Two targets, five reasons Canonical source: `src/shared/dispatch-types.ts`. -- `DispatchTarget` = `"daemon"` (singleton: kept as a field for DB/log stability). +- `DispatchTarget` = `"daemon"` for shared jobs or `"workflow-runner"` for structured workflows. - `DispatchReason` is one of: | Reason | When the router sets it | @@ -69,6 +192,7 @@ Canonical source: `src/shared/dispatch-types.ts`. | `ephemeral-daemon-triage` | Triage flagged the job heavy → orchestrator spawned an ephemeral daemon Pod. | | `ephemeral-daemon-overflow` | Queue length ≥ `EPHEMERAL_DAEMON_SPAWN_QUEUE_THRESHOLD` and persistent pool saturated → spawn drains overflow. | | `ephemeral-spawn-failed` | Spawn was required but the K8s API call failed. Job rejected with a tracking-comment infra error. | +| `workflow-runner` | A structured workflow was committed for one isolated runner attempt. | ### Scale-up model @@ -81,21 +205,46 @@ The fleet is two-tiered, see [`../operate/runbooks/daemon-fleet.md`](../operate/ The newly-spawned ephemeral daemon connects via WebSocket, registers with `isEphemeral: true`, claims the job, runs it, then drains and exits after `EPHEMERAL_DAEMON_IDLE_TIMEOUT_MS`. +Every shared-daemon boot uses a new UUID. On socket close, the controller immediately removes that socket from its local connection, daemon-info, heartbeat, and dispatch state so it cannot receive more work. It then starts a serialized asynchronous cleanup, and same-ID registration waits for that cleanup before becoming current. One PostgreSQL transaction locks the exact daemon row, marks it inactive, fails its attempt-less workflow rows and queued/offered/running execution receipts, creates pending public-failure receipts, and releases matching scheduled-action locks. The controller projects those receipts immediately and retries missed projections from PostgreSQL, then removes the best-effort Valkey registry entry. The liveness reaper applies the same exact-incarnation database cleanup if no close callback arrives, including direct or scoped execution receipts that have no `workflow_runs` row. Controller shutdown stops new connections and drains pending registration and disconnect work before closing the database. + +## Isolated workflow runners + +Workflow producers commit the `workflow_runs` row and matching `executions` row in one PostgreSQL transaction. Queue publication happens after commit. `dispatch_enqueued_at` is the last successful wake-up reconciliation time, not proof that Valkey retained the item. A null or stale timestamp makes the periodic reaper reconstruct the byte-stable job from PostgreSQL and atomically ensure that one matching copy exists in either the shared queue or this controller's processing list. Publication failures increment `dispatch_retry_count`. Exceeding `JOB_MAX_RETRIES` or `WORKFLOW_DISPATCH_TIMEOUT_MS` fails the queued workflow and execution, releases the target lock, and stores a retryable `dispatch-expired` public projection. Capacity deferral moves the queue bytes unchanged and does not consume that publication retry budget. If a controller crashes after `LMOVE`, another live controller returns the processing-list item to the shared queue after the old controller's heartbeat expires. + +The queue worker admits a `workflow-run` only when the database capacity query and exact attempt claim succeed in the same transaction. The attempt ID is the row's `dispatch_generation_id`. A duplicate queue item either finds that same live attempt or is consumed as stale. Capacity deferral returns the exact processing-list item to the queue without increasing `retryCount`. + +The controller validates the provider boundary and digest-pinned `@sha256:` image, creates one bare Pod, reads its UID, then creates the per-attempt Secret as an owned dependent of that exact Pod. The Pod uses `restartPolicy: Never` because the repository token payload is delivered at most once. A process failure therefore becomes a terminal Pod failure for the controller to reconcile instead of restarting without credentials. A 10 GiB `emptyDir` and exact ephemeral-storage request/limit provide scheduling and eviction ceilings; a fixed node selector and taint toleration contain node-disk exhaustion to the dedicated runner pool. The runner Secret contains only an expiring HMAC capability derived from a controller-only root for `(runId, attemptId, expiresAtMs)`. The Pod references exactly one complete provider credential chain from the separately managed `workflow-runner-secrets` Secret. It never imports that Secret with `envFrom`. + +The runner receives: + +- the provider credential selected by the deployment; +- a target-repository GitHub App installation token and its authoritative expiry; +- bounded repository memory, review learnings, policy, and handler-specific prior state; +- the deadline-bound HMAC-scoped WSS controller capability. + +It does not receive PostgreSQL, Valkey, Kubernetes credentials, the GitHub App private key, webhook secrets, a fleet-wide daemon token, or a global GitHub token. PAT mode fails closed for workflow runners. Startup also fails when the shared IPv4, AWS IPv6, or Google Cloud IPv6 metadata endpoint answers. Runner commands and results have an exact-value filter in the runner and deterministic plus encoded-secret scanning in the controller before an effect or durable write. + +The attempt claim writes one immutable 4,200-second PostgreSQL deadline. Heartbeat RPC can renew the lease only up to that deadline, and registration, commands, token minting, and result writes require both the lease and deadline to remain active. The runner aborts at the earlier of the database deadline or five minutes before its installation token expires. Loss of renewal also aborts the Agent SDK query and closes it explicitly. Payload preparation owns the repository token until the registered frame succeeds and attempts best-effort revocation if preparation or delivery fails. The runner attempts best-effort revocation after its final repository operation and before sending an ordinary retryable result. Controller-only reconnect, notification, and result-projection paths independently attempt revocation through GitHub's [token self-revocation endpoint](https://docs.github.com/en/rest/apps/installations#revoke-an-installation-access-token), with a ten-second API timeout. Revocation failure does not block terminal result handling. The token is deliberately not persisted, so a failed revocation, process crash, or node loss can leave its repository-scoped authority live until GitHub's authoritative expiry. Guaranteed revocation would require retaining the exact token and is outside this no-durable-token boundary. The controller stores a terminal result before applying retryable projections and before ACK. A reconnect processes the first stored result rather than rescanning retry bytes. Once both execution rows are terminal, resource reconciliation requests deletion of the exact Pod and Secret with UID preconditions independently of projection success. It records cleanup when Kubernetes accepts the deletes or the resources are already absent, preventing terminating Pods from starving later cleanup batches. Lease or absolute-deadline expiry atomically fails the attempt, its execution receipt, its running composite parent, and any matching scheduled-action lock before Valkey liveness is consulted. + +This is not exactly-once execution. A GitHub API request or git push can complete before the runner is fenced and can repeat after a controller crash during projection. Operators must inspect repository state before retrying an expired or interrupted attempt. + ## WebSocket protocol Schema in `src/shared/ws-messages.ts` (Zod discriminated union). Validation failures close the WebSocket with `POLICY_VIOLATION`. Every message has an envelope with `id` (UUID) and `timestamp` (ms). +The daemon protocol is v2. Message discriminants follow `subject:action`, including `scoped-job:offer` and `scoped-job:completion`. Registration compares the peer's major version with `PROTOCOL_VERSION` before admitting work. During a major-version rollout, deploy the controller first and update the daemon image in the same rollout. The v2 controller gives a v1 daemon an urgent `daemon:update-required` and allows five seconds for acknowledgement without admitting work. An acknowledged socket stays open while the daemon drains, then closes from the daemon side; the controller force-closes it only after the configured drain timeout plus a short scheduling grace. An unacknowledged or otherwise incompatible peer closes with code `4003`, and a v2 daemon treats that close as terminal instead of reconnecting. The update message initiates graceful shutdown; it does not install the new daemon binary. + ### Server → Daemon | Type | Purpose | | ------------------------ | ---------------------------------------------------------------------------------------------------------- | | `daemon:registered` | Handshake response after `daemon:register`; carries `heartbeatIntervalMs`, `offerTimeoutMs`, `maxRetries`. | | `heartbeat:ping` | Periodic liveness ping. | -| `job:offer` | Offer a workflow-run job. | -| `scoped-job-offer` | Offer a scoped job (`scoped-rebase`, `scoped-fix-thread`, `scoped-explain-thread`, `scoped-open-pr`). | +| `job:offer` | Offer a legacy shared-daemon job. | +| `scoped-job:offer` | Offer a scoped job (`scoped-rebase`, `scoped-fix-thread`, `scoped-open-pr`, `scheduled-action`). | | `job:payload` | Full `BotContext` plus overrides (maxTurns, allowedTools, trackingCommentId). Sent after accept. | | `job:cancel` | Abort a running job. | -| `daemon:update-required` | Daemon version mismatch; force exit. | +| `daemon:update-required` | Request graceful daemon shutdown after a version mismatch. | ### Daemon → Server @@ -106,15 +255,17 @@ Schema in `src/shared/ws-messages.ts` (Zod discriminated union). Validation fail | `job:accept` | Claim an offered job. | | `job:reject` | Decline with reason (`scoped-kind-unsupported`, `resource-insufficient`, …). | | `job:status` | Mid-run progress. | -| `job:result` | Workflow-run completion with `ExecutionResult` fields. | +| `job:result` | Legacy completion with `ExecutionResult` fields. | | `scoped-job:completion` | Scoped job result with kind-specific fields. | | `daemon:draining` | Graceful shutdown initiated. | | `daemon:update-acknowledged` | Ack for `daemon:update-required`. | | `error` | Generic error envelope. | +The isolated runner uses a separate schema in `src/shared/workflow-runner-messages.ts` at `/ws/workflow-runner//`. Its messages are register, heartbeat, bounded command, terminal result, and their acknowledgements. HMAC authentication binds every connection to the path identity. Initial registration sets `needsJob: true`; the controller records one durable payload receipt before sending bounded input and the repository token. Transport reconnects set `needsJob: false`, receive no job or credential, and resume retained command or result messages against the same database attempt. + ## PR shepherding bridge -The `bot:ship` lifecycle does **not** own a separate daemon-execution path. It bridges onto the existing `workflow_runs` pipeline so a single executor surface clones, runs the Agent SDK, and pushes, no parallel implementation to drift between. +The `bot:ship` lifecycle does **not** own a second execution path. It bridges onto the existing `workflow_runs` pipeline, whose structured attempts run in isolated workflow-runner Pods. The legacy direct and scoped job rails remain on shared daemons. ```mermaid flowchart LR @@ -126,9 +277,9 @@ flowchart LR WR[("workflow_runs row
state.shipIntentId")]:::store Q[("queue:jobs
kind=workflow-run")]:::store - Daemon["Daemon process"]:::work + Runner["One-attempt runner Pod"]:::work Exec["src/core/pipeline.ts"]:::work - Done["markSucceeded(runId)"]:::work + Done["Store result and terminal state"]:::work Cascade["orchestrator.onStepComplete
maybeEarlyWakeShipIntent"]:::core Tickle[("ship:tickle ZSET
score=0")]:::store @@ -140,7 +291,7 @@ flowchart LR SR --> Cont SR --> Iter --> WR Iter --> Q - Q --> Daemon --> Exec --> Done --> Cascade --> Tickle + Q --> Runner --> Exec --> Done --> Cascade --> Tickle Timer --> Due --> Resume Tickle --> Due Resume -. next iteration .-> Iter @@ -151,11 +302,11 @@ flowchart LR classDef store fill:#0e8a16,stroke:#063d09,color:#ffffff ``` -The reactor (`fanOut`) writes `wake_at = now()` and `ZADD ship:tickle 0 ` so the next cron tick (typically under 30 s) re-enters the runner. This keeps daemon slots free between iterations and gives the bot crash-restart safety: on boot, `tickle-scheduler` reconciles missed wakes from Postgres into Valkey before the periodic timer's first tick. +The reactor (`fanOut`) writes `wake_at = now()` and `ZADD ship:tickle 0 ` so the next cron tick (typically under 30 s) re-enters the session runner. No workflow runner Pod is held between iterations. On boot, `tickle-scheduler` reconciles missed wakes from Postgres into Valkey before the periodic timer's first tick. ## System/user trust boundary -The agent executor (`src/core/executor.ts:208`) supports two prompt-layout strategies, selected by `PROMPT_CACHE_LAYOUT`. The legacy layout passes a single user-role string and the unmodified `claude_code` preset systemPrompt: simple, but the preset embeds dynamic sections (`cwd`, platform, shell, OS) that vary per delivery, so the prompt cache key churns and every job pays the 1-hour TTL cache-write surcharge with zero compensating reads. +The agent executor (`src/core/executor.ts:231#useCacheableLayout`) supports two prompt-layout strategies, selected by `PROMPT_CACHE_LAYOUT`. The legacy layout passes a single user-role string and the unmodified `claude_code` preset systemPrompt: simple, but the preset embeds dynamic sections (`cwd`, platform, shell, OS) that vary per delivery, so the prompt cache key churns and every job pays the 1-hour TTL cache-write surcharge with zero compensating reads. The `cacheable` layout splits the prompt by trust: @@ -200,6 +351,136 @@ The per-call nonce on `` tags lives only in the user message; the a Three handlers ship the split today: the main pipeline (`src/core/pipeline.ts`) reads `config.promptCacheLayout` and conditionally threads `buildPromptParts()` output through; `src/workflows/handlers/triage.ts` and `src/workflows/handlers/plan.ts` do the same with their handler-specific builders. The executor's completion log surfaces `cacheReadInputTokens`, `cacheCreationInputTokens`, and `promptCacheLayout` so operators can verify hits before deciding to roll out further. See [`../operate/configuration.md`](../operate/configuration.md#prompt-cache-layout) for the rollout playbook. +## Per-repo config gates + +A repo's `.github-app.yaml` (see [Repo configuration](../use/repo-config.md)) is +read from the **default branch only**: `src/repo-config/fetcher.ts:126#fetchRepoConfig` +calls `repos.getContent` with no `ref`, so a config edit inside a pull request is +inert for that pull request. `test/repo-config/fetcher.test.ts` asserts the call +carries no `ref`, so the invariant cannot regress silently. + +Exactly one module reads a head-ref copy, and it applies nothing: +`src/repo-config/pr-check.ts:319#runPrConfigCheck` validates the PR's own copy +purely to post an authoring verdict comment. It imports neither +`fetchRepoConfig` nor `loadRepoPolicy`, so the read cannot populate the fetcher +caches or reach the applied policy; `test/repo-config/pr-check.test.ts` asserts +the absence of both symbols in that source file. Threading an optional `ref` +through the fetcher instead was rejected for exactly this reason: it would put +an attacker-chosen commit's config one flag-flip away from the policy the bot +enforces. + +That verdict comment is still a GitHub write, so its handler +(`src/webhook/events/pull-request.ts:138#handlePullRequestConfigCheck`) honours +the repo-wide `enabled: false` master switch via `loadRepoPolicy` before calling +into `pr-check.ts`. Only that switch, never the full Gate-1 rule set: the passive +`triggers.*` filters exist to stop the bot _acting_ on a pull request, and +withholding authoring feedback because the config PR is a draft or its title +matches `ignore_title_keywords` is the opposite of what an author wants. Same +scope the scheduler applies to its unattended runs. The lookup lives in the +handler rather than in `pr-check.ts` so that file stays structurally unable to +reach the applied-policy path. + +The config is consumed at two distinct points. + +**Gate 1, pre-dispatch.** Decides _whether the bot acts at all_. It runs before +any `workflow_runs` row, label mutex, queue job, or tracking comment exists, so a +blocked trigger leaves nothing behind but a log line. +`src/repo-config/gate.ts:82#checkRepoGate` evaluates seven rules in order and +returns the first that blocks. Three rules (repo disabled, workflow disabled, +sender not in `allowed_users`) set `explain: true` and earn a one-line refusal +comment; the four passive `triggers.*` filters set `explain: false` and stay +silent, because a filter configured to keep the bot quiet must stay quiet. + +The order is load-bearing in one place: `ignore_authors` is checked _before_ +`allowed_users`. A bot login is normally in the former and absent from the +latter, so the other order would answer every Renovate event with a public +refusal comment, which is exactly the noise `ignore_authors` exists to prevent. + +Gate 1 is **narrowing only**. Every rule can refuse; none can permit. The +`ALLOWED_OWNERS` env allowlist already ran in the webhook handler and a repo that +failed it never reaches the gate, so no YAML value can readmit it. + +There are two dispatch chokepoints, not one, and both call the gate: + +| Chokepoint | Covers | +| ----------------------------------------------------------------- | --------------------------------------------------------------- | +| `src/workflows/dispatcher.ts:183#applyRepoGate` | `dispatchByLabel`, `dispatchByIntent`, `dispatchWorkflowByName` | +| `src/workflows/ship/command-dispatch.ts:80#isBlockedByRepoConfig` | the canonical ship rail, which bypasses the dispatcher entirely | + +A future third dispatch path would need the gate too. Two deliberate carve-outs: +the `stop` and `abort` ship verbs run with `identityRulesOnly` (a config change +must not strand a run it was meant to end, but `ignore_authors` and +`allowed_users` still decide who may end one), and the comment path gates +_before_ the intent classifier runs, so a disabled repo never costs an LLM call. +Because that path does not yet know the workflow name, rule 2 is skipped there +and re-evaluated downstream. + +Rule 2 needs a registry workflow name, and the canonical rail speaks in +`CommandIntent`s. Two intents collide with registry names, `ship` and `triage`, +so `command-dispatch.ts` maps them through `INTENT_TO_WORKFLOW` before calling +the gate. The mapping matters because the canonical parser runs first in the +event handlers and returns before `dispatchByLabel`: a `bot:triage` label that +did not carry its workflow name would never see the per-workflow toggle at all. +A test fails if a colliding intent is missing from the map. + +The trigger facts rules 5 to 7 need (title, draft flag, base branch) are taken +from the webhook payload and threaded in as a `TriggerContext`, so the gate costs +no extra GitHub round trip. They are deliberately **not** fields on +`DispatchTarget`: that object is persisted to `workflow_runs` and logged on every +dispatch line, and the title is attacker-controlled free text. + +**Gate 2, during controller-owned payload preparation.** Resolves _how_ the +agent runs and ships the result, alongside `reviewLearnings`. + +`prepareWorkflowRunnerPayload` calls `loadRepoPolicy` + `policyForWorkflow` +(`src/repo-config/effective.ts`), which merge `workflows.` over `defaults` +and clamp `max_turns` / `timeout` against `AGENT_MAX_TURNS` / `AGENT_TIMEOUT_MS`. +The legacy direct-job rail performs the same resolution in `handleAccept`. +Neither worker re-reads repository YAML. `toAgentPolicy` projects the result +onto the wire as an optional `policy` object. `max_turns` remains the existing +top-level `maxTurns` payload field, whose env fallback chain is +`AGENT_MAX_TURNS ?? DEFAULT_MAXTURNS`. + +From the payload the object reaches the agent by three rails: + +- **Isolated workflow rail via `runPipeline`.** + `src/runner/workflow-executor.ts` puts it on `WorkflowRunContext.policy`; the + `review`, `resolve`, `implement`, and `remember` handlers forward it into + `runPipeline`'s `policy` override. +- **Workflow rail bypassing `runPipeline`.** `plan` and `triage` own their + prompts and call `executeAgent` directly, so they never enter the pipeline. + They read the same `WorkflowRunContext.policy` and apply it themselves. +- **Direct-pipeline rail.** `src/daemon/job-executor.ts` passes it straight + through to `runPipeline`. + +All three end at the same helper, `src/core/agent-policy.ts applyAgentPolicy`, +which is where `model`, `extraAllowedTools`, the turn cap, and `timeoutMs` are +turned into `executeAgent` options. It composes `timeoutMs` over the caller's +abort signal with `AbortSignal.any` (so a daemon cancel is never swallowed), +using an explicit controller that aborts with a named `Error` rather than +`AbortSignal.timeout`, whose bare `TimeoutError` DOMException would defeat the +executor's error-identity check. Its `dispose()` must be called in a `finally`, +or a live timer keeps the event loop alive for the rest of the deadline. + +Two fields deliberately stay in `runPipeline` rather than moving to the helper, +because both need fetched PR data plus the prompt builder: `pathFilters` +(matches dropped from the fetched changed-file list) and `instructions` (put on +`BotContext.reviewInstructions`). They therefore apply to `review` only, which +is the sole workflow whose schema accepts them. + +Because the deadline is composed at the `executeAgent` call, `timeoutMs` bounds +the agent invocation only: the tracking comment, token resolution, GitHub fetch, +and repo clone run before the timer is armed. `AGENT_TIMEOUT_MS` is the outer +bound over the whole run. + +A repo whose file failed validation still runs, on `DEFAULT_REPO_POLICY`, with +the reason carried as `policy.warning`. Both rails render it, and both survive a +re-render. The workflow rail persists the notice into the run's state under +`CONFIG_NOTICE_KEY`, so `renderCommentBody` re-emits it on every subsequent +mirror write instead of losing it after the first. The direct rail passes it to +`createTrackingComment` and again to `finalizeTrackingComment`, which re-appends +it only when the agent's own output did not already carry it through. + ## Scheduled actions A GitHub App receives no native cron event, so scheduled automation runs on an @@ -216,19 +497,21 @@ comment. Missed slots are skipped, not backfilled. See ## Directory layout -| Directory | Responsibility | -| ------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | -| `src/webhook/` | Event routing (`router.ts`) and per-event handlers (`events/`, one file per event type). | -| `src/core/` | Pipeline: context → fetch → format → prompt → checkout → execute → finalise. `pipeline.ts` is the single execution path (daemon-side). | -| `src/ai/` | Provider-agnostic LLM client (Anthropic + Bedrock) used by triage and the intent / NL classifiers. | -| `src/orchestrator/` | WebSocket server, daemon registry, job queue, dispatcher, triage, ephemeral-daemon scaler. Embedded in the webhook server process. | -| `src/daemon/` | Standalone worker process (persistent or ephemeral). WebSocket client that accepts offers and runs `pipeline.ts`. | -| `src/k8s/` | Ephemeral daemon Pod spawner. | -| `src/mcp/` | MCP server registry. | -| `src/workflows/` | Registry, dispatcher, composite cascade, ship lifecycle (`ship/`), per-workflow handlers (`handlers/`). | -| `src/db/` | Postgres layer. Migrations, connection singleton, observability queries. Active when `DATABASE_URL` is set. | -| `src/shared/` | Types shared between server and daemon (WebSocket messages, dispatch enums). | -| `src/utils/` | Retry, sanitisation, circuit breaker. | +| Directory | Responsibility | +| ------------------- | --------------------------------------------------------------------------------------------------------------------------- | +| `src/webhook/` | Event routing (`router.ts`) and per-event handlers (`events/`, one file per event type). | +| `src/core/` | Pipeline: context → fetch → format → prompt → checkout → execute → finalise. Shared by isolated runners and legacy daemons. | +| `src/ai/` | Provider-agnostic LLM client (Anthropic + Bedrock) used by triage and the intent / NL classifiers. | +| `src/orchestrator/` | Controller authority, durable queues and leases, worker protocols, result reconciliation, daemon registry, and scaling. | +| `src/daemon/` | Shared worker for legacy direct and scoped jobs. | +| `src/runner/` | One-attempt structured-workflow worker and its lease-fenced WSS client. | +| `src/k8s/` | Shared-daemon and isolated workflow-runner Pod spawners. | +| `src/mcp/` | MCP server registry. | +| `src/workflows/` | Registry, dispatcher, composite cascade, ship lifecycle (`ship/`), per-workflow handlers (`handlers/`). | +| `src/db/` | Postgres layer. Migrations, connection singleton, observability queries. Active when `DATABASE_URL` is set. | +| `src/shared/` | Types shared by the controller, daemon, and isolated runner. | +| `src/utils/` | Retry, sanitisation, circuit breaker. | +| `test/` | Test suites mirroring the production source tree. | ## Further reading diff --git a/docs/operate/configuration.md b/docs/operate/configuration.md index 3a94a6d1..08402750 100644 --- a/docs/operate/configuration.md +++ b/docs/operate/configuration.md @@ -17,44 +17,44 @@ Server mode only. If `ORCHESTRATOR_URL` is set, the process runs in daemon mode ## AI provider -| Variable | Default | Required when | Notes | -| ---------------------------- | ----------------------------------------------- | -------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------- | -| `CLAUDE_PROVIDER` | `anthropic` | _none_ | `anthropic` or `bedrock`. | -| `CLAUDE_MODEL` | `claude-opus-4-7` (anthropic); _none_ (bedrock) | Bedrock | Bedrock requires an explicit Bedrock model ID. | -| `ANTHROPIC_API_KEY` | _none_ | Anthropic, unless `CLAUDE_CODE_OAUTH_TOKEN` is set | Console pay-as-you-go. Safe for multi-tenant deploys. | -| `CLAUDE_CODE_OAUTH_TOKEN` | _none_ | Anthropic, unless `ANTHROPIC_API_KEY` is set | Max/Pro subscription token (`sk-ant-oat…`). Requires `ALLOWED_OWNERS`. | -| `AWS_REGION` | _none_ | Bedrock | Resolved by the AWS SDK credential chain. | -| `AWS_PROFILE` | _none_ | Optional (bedrock) | Local SSO profile for dev. | -| `AWS_ACCESS_KEY_ID` | _none_ | Optional (bedrock) | Long-lived credential pair. Prefer profile or OIDC. | -| `AWS_SECRET_ACCESS_KEY` | _none_ | Optional (bedrock) | Paired with `AWS_ACCESS_KEY_ID`. | -| `AWS_SESSION_TOKEN` | _none_ | Optional (bedrock) | Temporary credentials. | -| `AWS_BEARER_TOKEN_BEDROCK` | _none_ | Optional (bedrock, CI) | Set automatically by `aws-actions/configure-aws-credentials` OIDC. | -| `ANTHROPIC_BEDROCK_BASE_URL` | _none_ | Optional (bedrock) | Override Bedrock runtime endpoint (VPC endpoint / proxy). | -| `ALLOWED_OWNERS` | _none_ | OAuth or PAT path | Comma-separated allowlist. Required (single owner) when using `CLAUDE_CODE_OAUTH_TOKEN` or `GITHUB_PERSONAL_ACCESS_TOKEN`. | +| Variable | Default | Required when | Notes | +| ---------------------------- | --------------------------------------------- | -------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------- | +| `CLAUDE_PROVIDER` | `anthropic` | _none_ | `anthropic` or `bedrock`. | +| `CLAUDE_MODEL` | `claude-opus-5` (anthropic); _none_ (bedrock) | Bedrock | Bedrock requires an explicit Bedrock model ID. | +| `ANTHROPIC_API_KEY` | _none_ | Anthropic, unless `CLAUDE_CODE_OAUTH_TOKEN` is set | Console pay-as-you-go. Safe for multi-tenant deploys. | +| `CLAUDE_CODE_OAUTH_TOKEN` | _none_ | Anthropic, unless `ANTHROPIC_API_KEY` is set | Max/Pro subscription token (`sk-ant-oat…`). Requires `ALLOWED_OWNERS`. | +| `AWS_REGION` | _none_ | Bedrock | Resolved by the AWS SDK credential chain. | +| `AWS_PROFILE` | _none_ | Optional (bedrock) | Local SSO profile for dev. | +| `AWS_ACCESS_KEY_ID` | _none_ | Optional (bedrock) | IAM access key. Isolated runners require temporary credentials for a dedicated Bedrock-only principal. | +| `AWS_SECRET_ACCESS_KEY` | _none_ | Optional (bedrock) | Paired with `AWS_ACCESS_KEY_ID`. | +| `AWS_SESSION_TOKEN` | _none_ | Optional (bedrock) | Required when the access-key pair is temporary session authority. | +| `AWS_BEARER_TOKEN_BEDROCK` | _none_ | Optional (bedrock) | Amazon Bedrock API key, distinct from IAM credentials exported by `aws-actions/configure-aws-credentials`. | +| `ANTHROPIC_BEDROCK_BASE_URL` | _none_ | Optional (bedrock) | HTTPS Bedrock runtime endpoint or proxy, without URL credentials, query, or fragment. | +| `ALLOWED_OWNERS` | _none_ | OAuth or PAT path | Comma-separated allowlist. Required (single owner) when using `CLAUDE_CODE_OAUTH_TOKEN` or `GITHUB_PERSONAL_ACCESS_TOKEN`. | ## HTTP server -| Variable | Default | Notes | -| ----------------------------- | ---------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `PORT` | `3000` | HTTP webhook listener. | -| `LOG_LEVEL` | `info` | Pino level: `fatal`, `error`, `warn`, `info`, `debug`, `trace`. `debug` surfaces full webhook payloads. | -| `NODE_ENV` | `production` | `production`, `development`, `test`. | -| `TRIGGER_PHRASE` | `@chrisleekr-bot` | Mention text that triggers the bot. Local dev typically sets `@chrisleekr-bot-dev`. | -| `BOT_APP_LOGIN` | `chrisleekr-bot[bot]` | Bot's GitHub login. Used by the loop-prevention check. | -| `MAX_CONCURRENT_REQUESTS` | `3` | Ceiling on simultaneous Claude executions across the fleet. | -| `MAX_FETCHED_COMMENTS` | `500` | Per-PR/issue cap on comments merged from the GraphQL fetcher (`src/core/fetcher.ts`). When the cap fires the fetcher emits `log.warn({ connection: "comments", … })` and sets `FetchedData.truncated.comments=true`. | -| `MAX_FETCHED_REVIEWS` | `500` | Per-PR cap on reviews merged from the fetcher. Sets `FetchedData.truncated.reviews=true` on cap fire. | -| `MAX_FETCHED_REVIEW_COMMENTS` | `500` | Per-PR cap on inline review comments merged across all reviews (top-level + nested follow-up paginate). Sets `truncated.reviewComments=true`. | -| `MAX_FETCHED_FILES` | `500` | Per-PR cap on changed files merged from the fetcher. Sets `truncated.changedFiles=true` on cap fire. | -| `AGENT_TIMEOUT_MS` | `3600000` | Wall-clock budget for one agent execution (60 min). Lower only when the job is bounded. | -| `AGENT_MAX_TURNS` | unset | Optional Claude SDK turn cap. Unset = no cap. Overrides `DEFAULT_MAXTURNS`. | -| `DEFAULT_MAXTURNS` | unset | Process-wide turn cap. Set only if ops needs a hard ceiling. | -| `CLAUDE_CODE_PATH` | resolved from `node_modules` | Absolute path to the Claude Code CLI `cli.js`. | -| `CLONE_BASE_DIR` | `/tmp/bot-workspaces` | Parent directory for per-delivery clones. | -| `CLONE_DEPTH` | `50` | Shallow-clone depth. Increase for deeply-diverged PRs. | -| `WORKSPACE_STALE_TTL_MS` | `3600000` | TTL before an orphaned per-job workspace triple (clone dir + `.cred.sh` + `-artifacts`) under `CLONE_BASE_DIR` is swept at startup. Reclaims SIGKILL/OOM/eviction orphans. Lower only if you understand the risk. | -| `CONTEXT7_API_KEY` | unset | Lifts Context7 MCP rate limiting. No other effect. | -| `GITHUB_API_SLOW_REQUEST_MS` | `3000` | Latency floor (ms) above which an octokit request emits a `github.api.slow` warn line (`src/utils/octokit-observability.ts`). `duration_ms` is threaded onto every `github.api.*` line regardless. See [`observability.md`](observability.md). | +| Variable | Default | Notes | +| ----------------------------- | ---------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `PORT` | `3000` | HTTP webhook listener. | +| `LOG_LEVEL` | `info` | Pino level: `fatal`, `error`, `warn`, `info`, `debug`, `trace`. `debug` surfaces full webhook payloads. | +| `NODE_ENV` | `production` | `production`, `development`, `test`. | +| `TRIGGER_PHRASE` | `@chrisleekr-bot` | Mention text that triggers the bot. Local dev typically sets `@chrisleekr-bot-dev`. | +| `BOT_APP_LOGIN` | `chrisleekr-bot[bot]` | Bot's GitHub login. Used by the loop-prevention check. | +| `MAX_CONCURRENT_REQUESTS` | `3` | Ceiling on simultaneous Claude executions across the fleet. | +| `MAX_FETCHED_COMMENTS` | `500` | Per-PR/issue cap on comments merged from the GraphQL fetcher (`src/core/fetcher.ts`). When the cap fires the fetcher emits `log.warn({ connection: "comments", … })` and sets `FetchedData.truncated.comments=true`. | +| `MAX_FETCHED_REVIEWS` | `500` | Per-PR cap on reviews merged from the fetcher. Sets `FetchedData.truncated.reviews=true` on cap fire. | +| `MAX_FETCHED_REVIEW_COMMENTS` | `500` | Per-PR cap on inline review comments merged across all reviews (top-level + nested follow-up paginate). Sets `truncated.reviewComments=true`. | +| `MAX_FETCHED_FILES` | `500` | Per-PR cap on changed files merged from the fetcher. Sets `truncated.changedFiles=true` on cap fire. | +| `AGENT_TIMEOUT_MS` | `3600000` | Wall-clock budget for one agent execution (60 min). Lower only when the job is bounded. | +| `AGENT_MAX_TURNS` | unset | Optional Claude SDK turn cap. Unset = no cap. Overrides `DEFAULT_MAXTURNS`. | +| `DEFAULT_MAXTURNS` | unset | Process-wide turn cap. Set only if ops needs a hard ceiling. It reaches the agent as the job payload's `maxTurns`, which the workflow rail previously ignored, so before the Gate-2 wiring this value only took effect on the direct-pipeline rail. It now reaches the `review`, `resolve`, `implement`, and `remember` handlers too. `AGENT_MAX_TURNS` is unchanged: the executor still falls back to it when a job carries no cap (`src/core/executor.ts:366#resolvedMaxTurns`). | +| `CLAUDE_CODE_PATH` | resolved from `node_modules` | Absolute path to the Claude Code CLI `cli.js`. | +| `CLONE_BASE_DIR` | `/tmp/bot-workspaces` | Parent directory for per-delivery clones. | +| `CLONE_DEPTH` | `50` | Shallow-clone depth. Increase for deeply-diverged PRs. | +| `WORKSPACE_STALE_TTL_MS` | `3600000` | TTL before an orphaned per-job workspace triple (clone dir + `.cred.sh` + `-artifacts`) under `CLONE_BASE_DIR` is swept at startup. Reclaims SIGKILL/OOM/eviction orphans. Lower only if you understand the risk. | +| `CONTEXT7_API_KEY` | unset | Lifts Context7 MCP rate limiting. No other effect. | +| `GITHUB_API_SLOW_REQUEST_MS` | `3000` | Latency floor (ms) above which an octokit request emits a `github.api.slow` warn line (`src/utils/octokit-observability.ts`). `duration_ms` is threaded onto every `github.api.*` line regardless. See [`observability.md`](observability.md). | ## Postgres @@ -74,47 +74,73 @@ Required whenever the orchestrator role is active. ## Orchestrator and daemon -| Variable | Default | Notes | -| --------------------------------- | ------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `WS_PORT` | `3002` | Orchestrator WebSocket listener. Must differ from `PORT`. | -| `ORCHESTRATOR_URL` | _none_ | Presence flips the process to daemon mode. Use `wss://` in production; `ws://` emits a warning. | -| `ORCHESTRATOR_PUBLIC_URL` | _none_ | Public WebSocket URL the spawner injects into ephemeral Pods. | -| `DAEMON_AUTH_TOKEN` | _none_ | Shared secret for the daemon ⇄ orchestrator handshake. Required on both sides. Compared in constant time. | -| `DAEMON_AUTH_TOKEN_PREVIOUS` | _none_ | Optional rotation overlap. Orchestrator accepts either the primary or this previous token; daemons always send the primary. See [`runbooks/daemon-fleet.md`](runbooks/daemon-fleet.md#rotating-daemon_auth_token). | -| `HEARTBEAT_INTERVAL_MS` | `30000` | Daemon → orchestrator ping cadence. | -| `HEARTBEAT_TIMEOUT_MS` | `90000` | Eviction threshold. Keep `≥ 2 × HEARTBEAT_INTERVAL_MS`. | -| `FLEET_SNAPSHOT_INTERVAL_MS` | `30000` (clamp 10000-300000; `0` disables) | Cadence of the periodic `fleet.snapshot` gauge log (queue depth / daemon counts / free + busy slots). `0` disables it (inline-mode local dev). See [Fleet snapshot fields](observability.md#fleet-snapshot-fields). | -| `SOCKET_HEALTH_INTERVAL_MS` | `30000` (clamp 5000-300000; `0` disables) | Cadence of the CLOSE_WAIT socket-spin watchdog (issue #265). `0` disables it (e.g. no procfs in local dev). Does not fix #264, it detects and structurally logs the signature. See [Socket health watchdog events](observability.md#socket-health-watchdog-events). | -| `SOCKET_HEALTH_LEAK_SAMPLES` | `3` (clamp 2-100) | Consecutive samples a CLOSE_WAIT socket must survive before it is logged as a leak. Lower is noisier; the floor of 2 stops a transient socket from being flagged. | -| `SOCKET_HEALTH_SELF_HEAL_SAMPLES` | `10` (clamp 2-1000) | Consecutive samples a leak must persist, alongside a pinned core, before it is treated as a spin. | -| `SOCKET_HEALTH_CPU_PERCENT` | `90` (clamp 50-100) | CPU floor for a spin, as a percentage of one core. CPU alone is never sufficient: a 13.5s `scheduler.scan` legitimately burns a core. Only persistent CLOSE_WAIT plus this floor escalates to a spin. | -| `SOCKET_HEALTH_SELF_HEAL_ENABLED` | `false` | When `true`, a suspected spin exits the process with code `75` (EX_TEMPFAIL) so k8s restarts the pod and bounds the burn. The distinct code lets `lastState.terminated.exitCode` tell a self-heal from a real crash. | -| `STALE_EXECUTION_THRESHOLD_MS` | `3600000` | How long a `running` execution may sit before the watcher fails it. Set `≥ AGENT_TIMEOUT_MS`. | -| `DAEMON_DRAIN_TIMEOUT_MS` | `300000` | Post-`SIGTERM` window to finish in-flight work. Raise to `≥ AGENT_TIMEOUT_MS` for zero mid-run kills. | -| `JOB_MAX_RETRIES` | `3` | Retries for transient daemon dispatch failures. | -| `OFFER_TIMEOUT_MS` | `5000` | How long the orchestrator waits for a daemon to claim an offer. | -| `QUEUE_WORKER_BACKOFF_MAX_MS` | `5000` | Upper bound on the queue-worker's sleep when no local daemon can take a job. | -| `LIVENESS_REAPER_INTERVAL_MS` | `30000` (min `20000`) | Cadence of the heartbeat-based reaper. | -| `DAEMON_UPDATE_STRATEGY` | `exit` | `exit`, `pull`, or `notify`. Advisory hint reported in the update response. | -| `DAEMON_UPDATE_DELAY_MS` | `0` | Delay before graceful shutdown after an update signal. | -| `DAEMON_MEMORY_FLOOR_MB` | `512` | Minimum free memory the orchestrator requires before dispatching. | -| `DAEMON_DISK_FLOOR_MB` | `1024` | Minimum free disk the orchestrator requires before dispatching. | +| Variable | Default | Notes | +| -------------------------------------------- | ------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `WS_PORT` | `3002` | Orchestrator WebSocket listener. Must differ from `PORT`. | +| `ORCHESTRATOR_URL` | _none_ | Presence flips the process to daemon mode. Use `wss://` in production; `ws://` emits a warning. | +| `ORCHESTRATOR_PUBLIC_URL` | _none_ | WebSocket URL injected into Kubernetes-spawned workers. Must be credential-free. Isolated workflow runners require `wss://`, or `ws://` to a cluster-local `..svc[.cluster.local]:` name so an in-cluster runner can dial the orchestrator directly. | +| `DAEMON_AUTH_TOKEN` | _none_ | Shared-daemon handshake secret. Required on the controller and shared daemons. It is not used for isolated-runner capabilities. | +| `DAEMON_AUTH_TOKEN_PREVIOUS` | _none_ | Optional shared-daemon rotation overlap. The controller accepts daemon handshakes from either slot; daemons send only the primary. See [`runbooks/daemon-fleet.md`](runbooks/daemon-fleet.md#rotating-daemon_auth_token). | +| `WORKFLOW_RUNNER_CAPABILITY_SECRET` | _none_ | Controller-only HMAC root for deadline-bound, per-attempt runner capabilities. Required on the controller, minimum 32 characters. It must differ from both daemon-auth slots and must never be mounted on a worker. | +| `WORKFLOW_RUNNER_CAPABILITY_SECRET_PREVIOUS` | _none_ | Optional controller-only rotation predecessor. Accepted only for capabilities whose signed expiry has not elapsed; it must also differ from both daemon-auth slots. | +| `HEARTBEAT_INTERVAL_MS` | `30000` | Daemon → orchestrator ping cadence. | +| `HEARTBEAT_TIMEOUT_MS` | `90000` | Eviction threshold. Keep `≥ 2 × HEARTBEAT_INTERVAL_MS`. | +| `FLEET_SNAPSHOT_INTERVAL_MS` | `30000` (clamp 10000-300000; `0` disables) | Cadence of the periodic `fleet.snapshot` gauge log (queue depth / daemon counts / free + busy slots). `0` disables it (inline-mode local dev). See [Fleet snapshot fields](observability.md#fleet-snapshot-fields). | +| `SOCKET_HEALTH_INTERVAL_MS` | `30000` (clamp 5000-300000; `0` disables) | Cadence of the CLOSE_WAIT socket-spin watchdog (issue #265). `0` disables it (e.g. no procfs in local dev). Does not fix #264, it detects and structurally logs the signature. See [Socket health watchdog events](observability.md#socket-health-watchdog-events). | +| `SOCKET_HEALTH_LEAK_SAMPLES` | `3` (clamp 2-100) | Consecutive samples a CLOSE_WAIT socket must survive before it is logged as a leak. Lower is noisier; the floor of 2 stops a transient socket from being flagged. | +| `SOCKET_HEALTH_SELF_HEAL_SAMPLES` | `10` (clamp 2-1000) | Consecutive samples a leak must persist, alongside a pinned core, before it is treated as a spin. | +| `SOCKET_HEALTH_CPU_PERCENT` | `90` (clamp 50-100) | CPU floor for a spin, as a percentage of one core. CPU alone is never sufficient: a 13.5s `scheduler.scan` legitimately burns a core. Only persistent CLOSE_WAIT plus this floor escalates to a spin. | +| `SOCKET_HEALTH_SELF_HEAL_ENABLED` | `false` | When `true`, a suspected spin exits the process with code `75` (EX_TEMPFAIL) so k8s restarts the pod and bounds the burn. The distinct code lets `lastState.terminated.exitCode` tell a self-heal from a real crash. | +| `STALE_EXECUTION_THRESHOLD_MS` | `3600000` | Startup-recovery age threshold for unfenced legacy `offered` or `running` execution receipts whose `offer_id` is null. | +| `DAEMON_DRAIN_TIMEOUT_MS` | `300000` | Post-`SIGTERM` window to finish in-flight work. Raise to `≥ AGENT_TIMEOUT_MS` for zero mid-run kills. | +| `JOB_MAX_RETRIES` | `3` | Retries for transient shared-daemon dispatch and structured-workflow publication failures. | +| `WORKFLOW_DISPATCH_TIMEOUT_MS` | `4200000` | Maximum age of an unclaimed structured-workflow dispatch. Retry-budget or age expiry fails the workflow, releases its in-flight lock, fails its execution receipt, and queues a public failure projection. | +| `OFFER_TIMEOUT_MS` | `5000` | How long the orchestrator waits for a daemon to claim an offer. | +| `QUEUE_WORKER_BACKOFF_MAX_MS` | `5000` | Upper bound on the queue-worker's sleep when no local daemon can take a job. | +| `LIVENESS_REAPER_INTERVAL_MS` | `30000` (min `20000`) | Cadence of lease/deadline expiry, workflow-runner result/resource reconciliation, outbox publication, orphan processing-list recovery, and shared-daemon receipt/heartbeat reaping. | +| `DAEMON_UPDATE_STRATEGY` | `exit` | `exit`, `pull`, or `notify`. Advisory hint reported in the update response. | +| `DAEMON_UPDATE_DELAY_MS` | `0` | Delay before graceful shutdown after an update signal. | +| `DAEMON_MEMORY_FLOOR_MB` | `512` | Minimum free memory the orchestrator requires before dispatching. | +| `DAEMON_DISK_FLOOR_MB` | `1024` | Minimum free disk the orchestrator requires before dispatching. | ## Ephemeral daemons Used when the orchestrator scales daemon capacity on demand. -| Variable | Default | Notes | -| ---------------------------------------- | ----------------- | ------------------------------------------------------------------------------------------------ | -| `DAEMON_EPHEMERAL` | `false` | Set to `true` on ephemeral daemon Pods (injected by the spawner). Controls idle-exit. | -| `EPHEMERAL_DAEMON_IDLE_TIMEOUT_MS` | `120000` | Ephemeral daemon exits after this idle window. | -| `EPHEMERAL_DAEMON_SPAWN_COOLDOWN_MS` | `30000` | Minimum time between ephemeral spawns (orchestrator side). | -| `EPHEMERAL_DAEMON_SPAWN_QUEUE_THRESHOLD` | `3` | Queue length that triggers an `ephemeral-daemon-overflow` spawn. | -| `EPHEMERAL_DAEMON_NAMESPACE` | `default` | Kubernetes namespace for spawned ephemeral Pods. | -| `DAEMON_IMAGE` | auto-detected | K8s image URI override. | -| `KUBECONFIG` | auto (in-cluster) | Kubernetes client config path. The client auto-detects in-cluster via `KUBERNETES_SERVICE_HOST`. | +| Variable | Default | Notes | +| ---------------------------------------- | ----------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `DAEMON_EPHEMERAL` | `false` | Set to `true` on ephemeral daemon Pods (injected by the spawner). Controls idle-exit. | +| `EPHEMERAL_DAEMON_IDLE_TIMEOUT_MS` | `120000` | Ephemeral daemon exits after this idle window. | +| `EPHEMERAL_DAEMON_SPAWN_COOLDOWN_MS` | `30000` | Minimum time between ephemeral spawns (orchestrator side). | +| `EPHEMERAL_DAEMON_SPAWN_QUEUE_THRESHOLD` | `3` | Queue length that triggers an `ephemeral-daemon-overflow` spawn. | +| `EPHEMERAL_DAEMON_NAMESPACE` | `default` | Kubernetes namespace for spawned ephemeral Pods. | +| `EPHEMERAL_DAEMON_SECRET_NAME` | `daemon-secrets` | Existing Secret in `EPHEMERAL_DAEMON_NAMESPACE` that spawned Pods mount via `envFrom`. Point it at an existing daemon Secret to avoid a second copy of the same credentials. | +| `DAEMON_IMAGE` | auto-detected | K8s image URI. Isolated workflow runners reject values that do not end in `@sha256:<64 lowercase hex>`. Shared ephemeral daemons retain their existing image handling. | +| `KUBECONFIG` | auto (in-cluster) | Kubernetes client config path. The client auto-detects in-cluster via `KUBERNETES_SERVICE_HOST`. | -The orchestrator also expects a pre-existing `daemon-secrets` Kubernetes Secret in `EPHEMERAL_DAEMON_NAMESPACE`, mounted into the spawned Pod via `envFrom: secretRef: daemon-secrets`. See [`deployment.md`](deployment.md#ephemeral-daemon-kubernetes-requirements). +The orchestrator also expects a pre-existing Kubernetes Secret in `EPHEMERAL_DAEMON_NAMESPACE`, named by `EPHEMERAL_DAEMON_SECRET_NAME` and mounted into the spawned Pod via `envFrom`. Every key in it becomes an env var on a Pod that runs agent-authored code, so its scope is the deployment's blast-radius decision. See [`deployment.md`](deployment.md#kubernetes-worker-requirements). + +## Isolated workflow runners + +Structured `workflow-run` jobs use one bare Kubernetes Pod per exact attempt. + +| Variable | Default | Notes | +| ----------------------------------- | ----------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `WORKFLOW_RUNNER_NAMESPACE` | `github-app-runners` | Dedicated namespace for runner Pods, per-attempt Secrets, Pod Security Admission, and the runner ValidatingAdmissionPolicy. It must differ from `EPHEMERAL_DAEMON_NAMESPACE`. | +| `WORKFLOW_RUNNER_NODE_LABEL` | `github-app.node-restriction.kubernetes.io/workflow-runner` | Node label key the runner Pod's `nodeSelector` and its `NoSchedule` toleration are both built from, so one setting targets a node pool the cluster already labels and taints. The default prefix is reserved by the NodeRestriction admission plugin, which stops a kubelet assigning it to itself; an unprefixed key gives that protection up. Must match `runnerNodeLabel` in the runner boundary ConfigMap. | +| `WORKFLOW_RUNNER_NODE_VALUE` | `true` | Value paired with `WORKFLOW_RUNNER_NODE_LABEL`. Must match `runnerNodeValue` in the runner boundary ConfigMap. | +| `WORKFLOW_RUNNER_IMAGE_PULL_SECRET` | _(empty)_ | Name of an existing `kubernetes.io/dockerconfigjson` Secret in `WORKFLOW_RUNNER_NAMESPACE` that runner Pods may reference. Must match `runnerImagePullSecret` in the runner boundary ConfigMap. Empty emits no `imagePullSecrets`, which only works against a registry allowing anonymous pull. | + +The following variables are internal to the Pod and are injected by `src/k8s/workflow-runner-spawner.ts`; operators must not set them on the controller or shared daemon Deployment. + +| Variable | Default | Notes | +| ---------------------------- | ------- | -------------------------------------------------------------------------------------------------------------- | +| `WORKFLOW_RUNNER` | `false` | Spawner sets `true`; selects runner startup validation and removes controller/data-layer requirements. | +| `WORKFLOW_RUNNER_RUN_ID` | _none_ | Spawner-injected UUID for the workflow row. | +| `WORKFLOW_RUNNER_ATTEMPT_ID` | _none_ | Spawner-injected UUID for the exact dispatch generation and Pod/Secret identity. | +| `WORKFLOW_RUNNER_TOKEN` | _none_ | Attempt-bound HMAC capability, read from the per-attempt Secret. It is not the fleet-wide `DAEMON_AUTH_TOKEN`. | + +The Pod also receives `ORCHESTRATOR_URL`, derived from `ORCHESTRATOR_PUBLIC_URL`, and `LD_PRELOAD` for the native process-boundary guard. It obtains the target-repository GitHub installation token and its expiry only after its first registration over WSS. That credential payload is recorded and sent at most once; reconnects restore controller access without resending it. The database claim fixes an immutable deadline 4,200 seconds after admission. Heartbeats cannot extend the lease past it, and execution stops at the earlier of that deadline or five minutes before the token expires. The Pod uses `restartPolicy: Never`, so a process failure becomes terminal rather than restarting after the one-time credential delivery. Each Pod has a 10 GiB `emptyDir` workspace and exact ephemeral-storage request/limit values of 2 GiB/10 GiB. ## Triage @@ -164,6 +190,116 @@ Tunables for the conversational scoped-intent path (`src/workflows/ship/scoped/c | `SHIP_FORBIDDEN_TARGET_BRANCHES` | empty | Comma-separated branches the bot refuses to shepherd PRs against. | | `REVIEW_RESOLVE_MAX_ITERATIONS` | `2` | Range `[1, 5]`. Max review/resolve loop iterations in the composite ship flow before the intent yields. | +## Per-repo config file + +Every installed repository may ship a config file at its **default branch** +root. Only that copy is ever applied: a change to the file inside a pull request +does not affect that pull request. Such a pull request does get a read-only +validation comment (see [Checking a file before pushing](../use/repo-config.md#checking-a-file-before-pushing)), +which never feeds the applied policy. The file carries the repo-wide master switch, +per-workflow toggles and agent knobs, pre-dispatch trigger filters, scheduled +actions, and the review-learnings block. See +[Per-repo configuration](../use/repo-config.md) for the schema and the field +reference. Every block is applied today, including the agent knobs on +`workflows.plan.*` and `workflows.triage.*`. + +!!! note "`workflows.ship` takes only `enabled`" + + `workflows.ship.model`, `.max_turns`, `.timeout`, and + `.extra_allowed_tools` are rejected by the schema. They were always a + no-op, ship's handler only enqueues child workflows and never runs an + agent, but they used to parse. Validation is whole-document, so one + rejected key fails the entire file and it falls back to + `DEFAULT_REPO_POLICY`. Put those knobs on `defaults:` or on the per-child + entries (`triage`, `plan`, `implement`, `review`, `resolve`), which is + what ship's steps actually resolve against. + +The env vars below are the operator's half of that surface, and the +`ALLOWED_OWNERS` allowlist gates every repo before any of the file is +consulted, so nothing in a repo's YAML can readmit a repo the server rejected. +Two of the rows are cross-references, not settings of their own: they are the +numeric ceilings a repo's `max_turns` and `timeout` are clamped against, and a +repo can only lower them, never raise them. + +| Variable | Default | Notes | +| -------------------------------------- | ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `REPO_CONFIG_FILE` | `.github-app.yaml` | Filename read from each installed repo's default-branch root. Deprecated alias `SCHEDULER_CONFIG_FILE` is still honoured, with a boot warning. | +| `AGENT_MAX_TURNS` / `DEFAULT_MAXTURNS` | unset | Ceiling for a repo's `max_turns`, resolved as `AGENT_MAX_TURNS ?? DEFAULT_MAXTURNS`, so with the first unset the second is the ceiling. Documented under [HTTP server](#http-server). | +| `AGENT_TIMEOUT_MS` | `3600000` | Ceiling for a repo's `timeout`, and an independent outer bound on the run. Documented under [HTTP server](#http-server). | + +## Auto review + +Runs the `review` workflow automatically when an allowlisted user pushes commits +to an open pull request (`pull_request.synchronize`), with no label and no +mention. Server mode only. + +**Two keys must agree.** `AUTO_REVIEW_USERS` is the operator's half and says +_which logins may trigger it_; `workflows.review.auto` in a repo's +`.github-app.yaml` is the maintainer's half and says _whether this repo wants +it_. Neither alone enables anything. The split exists because `AUTO_REVIEW_USERS` +is server-wide across every repo of the owner, so without the per-repo key, +setting it would switch auto-review on everywhere at once. + +The repo key defaults to `false`, unlike every other toggle in that file, +because `loadRepoPolicy` fails open: a missing, unreachable, or invalid config +yields the built-in defaults. Defaulting it on would let a GitHub outage start +spending tokens on every push. This mirrors `SCHEDULER_ALLOW_AUTO_MERGE` + +`auto_merge`, the other env-AND-repo automatic action. + +| Variable | Default | Notes | +| ------------------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `AUTO_REVIEW_USERS` | unset | Comma-separated GitHub logins whose pushes may trigger an automatic review, matched case-insensitively. Unset or empty disables the feature outright. Requires a non-empty `ALLOWED_OWNERS` and `workflows.review.auto: true` on the repo. | + +Four things narrow it further, all silent: + +- The **pusher**, not the commit author, is what is matched. The commit author is + derived from the commit's author email, which anyone can set; the pusher is + authenticated by GitHub. +- **Our own pushes are skipped.** `resolve` pushes a commit per fix, and that + push would otherwise trigger another review. + +!!! warning "Under `GITHUB_PERSONAL_ACCESS_TOKEN`, do not list the PAT owner" + + The bot's writes are attributed to the PAT owner, so that login's pushes are + indistinguishable from the bot's own. Auto-review therefore skips **every** + push by the PAT owner, logging `auto_review.skipped_self_push`. Listing only + the PAT owner in `AUTO_REVIEW_USERS` makes the feature look configured while + doing nothing. List a different collaborator, or run on App auth, where the + bot's identity is its own account. + +- **Content-free pushes are skipped.** A rebase that leaves the pull request's own + diff unchanged does not trigger a review. +- **A review already in flight wins.** A push landing mid-review is dropped, not + queued, and nothing is posted to the pull request about it. +- **Fork pull requests are skipped.** Checkout resolves the head _branch name_ + against the base repository, so a fork's ref either fails to clone or silently + resolves to a same-named base branch and reviews the wrong tree. +- **Pull requests `ship` is driving are skipped.** Ship runs its own + review → resolve iteration, so a second review would duplicate the spend. + +**`ALLOWED_OWNERS` is required.** Setting `AUTO_REVIEW_USERS` without it fails at +startup. Every other allowlist here narrows; this one widens, because it starts +an unattended agent run with no per-event human action, and `isOwnerAllowed` +permits every owner when `ALLOWED_OWNERS` is unset. Unlike +`CLAUDE_CODE_OAUTH_TOKEN` and `GITHUB_PERSONAL_ACCESS_TOKEN`, which demand +exactly one owner, auto-review only needs the list to be non-empty: it carries no +personal identity and no shared rate-limit bucket. + +!!! warning "Auto-review removes the human-in-the-loop step" + + Every other way to start a `review` requires a deliberate act: a `bot:review` + label or an `@chrisleekr-bot` mention. Auto-review starts the same agent + session (Bash tool, on a clone) from an ordinary push, and the reviewer's + prompt includes the pull request's comment thread, which on a public + repository anyone can write to. The spotlighting, input sanitisation, output + secret-strip, and destructive-Bash denylist all still apply, but the attacker + no longer needs a maintainer to *choose* to run the agent, only to push. + Enable `auto: true` only on repositories where you accept that trade-off. + +Gate 1 still applies, so `enabled: false`, `workflows.review.enabled: false`, and +every `triggers.*` filter keep their veto. Refusals are logged but never +commented, since nobody asked for the run. + ## Scheduled actions Controls the internal scheduler that runs prompt-based actions declared in a @@ -221,7 +357,7 @@ before committing: ## Prompt cache layout -Selects the system/user prompt split the agent executor passes to the Claude Agent SDK. See `src/config.ts:604#promptCacheLayout` for the Zod definition and `src/core/executor.ts:208` for the runtime guard. +Selects the system/user prompt split the agent executor passes to the Claude Agent SDK. See `src/config.ts:675#promptCacheLayout` for the Zod definition and `src/core/executor.ts:231#useCacheableLayout` for the runtime guard. | Variable | Default | Notes | | --------------------- | -------- | ------------------------------------------------------------------------------------------------------------ | @@ -248,21 +384,24 @@ The first job warms the cache (creation tokens dominate); subsequent jobs of the ## Mode matrix: what's required when -| Role | Required | -| --------------------------------------- | ------------------------------------------------------------------------------------------------------------- | -| Orchestrator (webhook server) | GitHub App credentials, one AI provider credential, `VALKEY_URL`, `DATABASE_URL`, `DAEMON_AUTH_TOKEN`. | -| Ephemeral-daemon scale-up | K8s API access + RBAC on `pods` in `EPHEMERAL_DAEMON_NAMESPACE`, `daemon-secrets` Secret. | -| Daemon process (`ORCHESTRATOR_URL` set) | `DAEMON_AUTH_TOKEN`, one AI provider credential. GitHub App credentials and data-layer URLs are NOT required. | +| Role | Required | +| ---------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Controller (webhook server) | GitHub App credentials, one AI provider credential, `VALKEY_URL`, `DATABASE_URL`, `DAEMON_AUTH_TOKEN`, and `WORKFLOW_RUNNER_CAPABILITY_SECRET`; K8s API access, `DAEMON_IMAGE`, and a WSS `ORCHESTRATOR_PUBLIC_URL` for structured workflows. | +| Ephemeral shared-daemon scale-up | RBAC on `pods` in `EPHEMERAL_DAEMON_NAMESPACE` and the `EPHEMERAL_DAEMON_SECRET_NAME` Secret. | +| Shared daemon (`ORCHESTRATOR_URL` set) | `DAEMON_AUTH_TOKEN` and one AI provider credential. GitHub App credentials and data-layer URLs are not required. | +| Isolated workflow runner (spawner-owned) | Named provider-key references from `workflow-runner-secrets`, an injected per-attempt capability, IDs, and WSS URL. It must not receive App, PAT, database, Valkey, Kubernetes, global GitHub, or fleet credentials. | + +Only one controller replica is supported. Queue recovery is durable, but workflow-runner admission does not implement a distributed semaphore or controller-session ownership. ## LLM-based output scanner (defense layer 4) Per-call LLM scan of every agent-generated GitHub-bound body, after the deterministic regex pass in `redactSecrets()`. Catches encoded / obfuscated secrets the regex misses. -| Variable | Default | Notes | -| ------------------------------- | ----------- | ----------------------------------------------------------------------------------------------------------------------------------------- | -| `LLM_OUTPUT_SCANNER_ENABLED` | `true` | Set `false` to disable. Skipping the scan saves ~1–2s and ~$0.0002 per agent reply but loses the encoded-secret backstop. | -| `LLM_OUTPUT_SCANNER_MODEL` | `haiku-3-5` | Operator-friendly alias resolved by `src/ai/llm-client.ts MODEL_MAP`. Cheapest Haiku that emits the structured JSON schema is sufficient. | -| `LLM_OUTPUT_SCANNER_TIMEOUT_MS` | `3000` | Per-call wall-clock cap. On timeout, the helper FAILS OPEN, posts the body that survived the regex pass and emits a `warn` log. | +| Variable | Default | Notes | +| ------------------------------- | ----------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `LLM_OUTPUT_SCANNER_ENABLED` | `true` | Set `false` to disable. General GitHub-bound output keeps the deterministic regex floor and skips the encoded-secret backstop. Structured runner commands are rejected and results become a fixed safe failure while disabled. | +| `LLM_OUTPUT_SCANNER_MODEL` | `haiku-4-5` | Operator-friendly alias resolved by `src/ai/llm-client.ts MODEL_MAP`. A Sonnet alias detects more encoded or obfuscated variants; raise the timeout alongside it, because the isolated-runner boundary treats a slow scan as a rejection. | +| `LLM_OUTPUT_SCANNER_TIMEOUT_MS` | `30000` | Per-call wall-clock cap. General GitHub-bound output fails open to the deterministic regex result. Structured runner RPC fails closed as described above, so the default is sized for that case rather than for comment latency. | System messages (router capacity, marker comments, lifecycle pings) skip the LLM pass, they cannot legitimately contain secrets and the scan is wasted spend. @@ -274,23 +413,25 @@ The allowlist (in `src/core/executor.ts buildProviderEnv()`): - **Allowed exact keys**: `HOME`, `PATH`, `USER`, `LANG`, `LC_ALL`, `TZ`, `TMPDIR`, `NODE_OPTIONS`, `NODE_PATH`, `NODE_NO_WARNINGS`, `NODE_EXTRA_CA_CERTS`, `SSL_CERT_FILE`, `SSL_CERT_DIR`, `HTTP_PROXY`/`HTTPS_PROXY`/`NO_PROXY` (uppercase + lowercase), `NO_COLOR`, `FORCE_COLOR`, `TERM`, `COLORTERM`, `CI`, `GH_TOKEN`, `GITHUB_TOKEN`. - **Allowed prefixes** (forward-compatible for vendor knobs): `CLAUDE_CODE_*`, `ANTHROPIC_*`, `AWS_*`, `GIT_*`, `GH_*`. -- **Denied exact keys** (override allow): `GITHUB_APP_ID`, `GITHUB_APP_PRIVATE_KEY`, `GITHUB_WEBHOOK_SECRET`, `GITHUB_PERSONAL_ACCESS_TOKEN`, `DAEMON_AUTH_TOKEN`, `DAEMON_AUTH_TOKEN_PREVIOUS`, `DATABASE_URL`, `VALKEY_URL`, `REDIS_URL`, `CONTEXT7_API_KEY`. +- **Denied exact keys** (override allow): `GITHUB_APP_ID`, `GITHUB_APP_PRIVATE_KEY`, `GITHUB_WEBHOOK_SECRET`, `GITHUB_PERSONAL_ACCESS_TOKEN`, `DAEMON_AUTH_TOKEN`, `DAEMON_AUTH_TOKEN_PREVIOUS`, `WORKFLOW_RUNNER_CAPABILITY_SECRET`, `WORKFLOW_RUNNER_CAPABILITY_SECRET_PREVIOUS`, `DATABASE_URL`, `VALKEY_URL`, `REDIS_URL`, `CONTEXT7_API_KEY`, `GH_ENTERPRISE_TOKEN`, `GITHUB_ENTERPRISE_TOKEN`, `LD_PRELOAD`, `LD_LIBRARY_PATH`. - **Denied prefixes**: `GITHUB_APP_*`, `GITHUB_WEBHOOK_*`. If you add a new env var the agent CLI needs, extend the allowlist in `buildProviderEnv()`. Anything outside the allowlist is silently dropped, verify by running `bun test test/core/build-provider-env.test.ts` after the change. ## K8s Secret split (defense layer 1b, issue #102) -The Helm chart MUST split secrets into two K8s Secret objects so the daemon Pod's filesystem/environment never carries orchestrator-only credentials, even if the env allowlist above develops a future bug: +The deployment must keep controller authority out of every worker. The runtime expects these Secret boundaries: -| Secret object | Mounted on | Contents | -| ---------------------- | ---------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `orchestrator-secrets` | Orchestrator Pod ONLY | `GITHUB_APP_ID`, `GITHUB_APP_PRIVATE_KEY`, `GITHUB_WEBHOOK_SECRET`, `DATABASE_URL`, `VALKEY_URL`, `CONTEXT7_API_KEY`, `DAEMON_AUTH_TOKEN[_PREVIOUS]` (issuance side). | -| `daemon-secrets` | Daemon Pod (incl. ephemeral) | `ANTHROPIC_API_KEY` or `CLAUDE_CODE_OAUTH_TOKEN`, `AWS_*` chain (Bedrock provider), `DAEMON_AUTH_TOKEN[_PREVIOUS]` (handshake side), `GITHUB_PERSONAL_ACCESS_TOKEN` (PAT mode only). | +| Secret object | Mounted on | Contents | +| -------------------------------- | -------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `orchestrator-secrets` | Controller Pod only | `GITHUB_APP_ID`, `GITHUB_APP_PRIVATE_KEY`, `GITHUB_WEBHOOK_SECRET`, `DATABASE_URL`, `VALKEY_URL`, `CONTEXT7_API_KEY`, `DAEMON_AUTH_TOKEN[_PREVIOUS]`, `WORKFLOW_RUNNER_CAPABILITY_SECRET[_PREVIOUS]`, and the selected provider credential. | +| `daemon-secrets` | Shared daemon Pods | Provider configuration/credential, primary `DAEMON_AUTH_TOKEN`, and `GITHUB_PERSONAL_ACCESS_TOKEN` only for legacy PAT deployments. Renameable via `EPHEMERAL_DAEMON_SECRET_NAME`; the name is a reference, the contents are what this row constrains. | +| `workflow-runner-secrets` | Isolated workflow runner Pods | The spawner references only the named provider keys listed in [`deployment.md`](deployment.md#workflow-runner-secrets-secret). Unexpected keys are not imported. Never place App, PAT, database, Valkey, Kubernetes, Context7, GitHub, or daemon-auth credentials here. | +| `workflow-runner-` | One isolated workflow runner Pod | One deadline-bound `capability` derived for that run and attempt. The controller creates the Pod first, then makes this Secret a Kubernetes-owned dependent of that exact Pod UID. | -The orchestrator mints short-lived GitHub installation tokens and forwards them via the WebSocket, daemons never see the App private key or webhook secret. +The controller mints a short-lived token restricted to the target repository and forwards it after runner registration. Shared daemons retain the legacy credential path. Structured workflows refuse PAT mode because the controller cannot mint a repository-bound App token from a PAT. -A startup warning fires if a daemon process detects orchestrator-only env vars at boot: it does NOT crash (a downed daemon is worse than a degraded posture), but the warning surfaces the misconfiguration in operator logs. +A shared daemon logs a startup warning if it sees controller-only secrets. An isolated workflow runner fails startup when any controller, fleet, PAT, global GitHub, Kubernetes, or Context7 credential is present, or when a cloud metadata endpoint answers. Kubernetes Secrets still require encryption at rest and least-privilege RBAC; base64 storage alone is not encryption. ## Output secret-stripping behavior (defense layer 2) diff --git a/docs/operate/deployment.md b/docs/operate/deployment.md index 4ad331de..1bf78c6d 100644 --- a/docs/operate/deployment.md +++ b/docs/operate/deployment.md @@ -4,12 +4,12 @@ The repository ships **two container images**, an orchestrator and a daemon, bui ## Image topology -| Image | Dockerfile | Role | Outbound network | -| -------------- | ------------------------- | ---------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------- | -| `orchestrator` | `Dockerfile.orchestrator` | Webhook server, WebSocket daemon registry, triage classifier, ephemeral-daemon spawner. | GitHub API, Anthropic / Bedrock, Postgres, Valkey, K8s API. | -| `daemon` | `Dockerfile.daemon` | Worker image with the toolchain Claude shells out to (`kubectl`, `helm`, `terraform`, `aws`, `gcloud`, `docker`, `go`, `rust`, …). | Orchestrator WebSocket (outbound), GitHub API, Anthropic. | +| Image | Dockerfile | Role | Outbound network | +| -------------- | ------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------- | +| `orchestrator` | `Dockerfile.orchestrator` | Webhook server, WebSocket daemon registry, triage classifier, ephemeral-daemon spawner. | GitHub API, Anthropic / Bedrock, Postgres, Valkey, K8s API. | +| `daemon` | `Dockerfile.daemon` | Toolchain image for shared daemons and one-attempt workflow runners (`kubectl`, `helm`, `terraform`, `aws`, `gcloud`, `docker`, `go`, `rust`, …). | Orchestrator WebSocket (outbound), GitHub API, AI provider. | -The `daemon` image additionally bundles `@mermaid-js/mermaid-cli` (`mmdc`) plus a headless Chromium, used by the scheduled `research` action's diagram-validation gate. It is daemon-only because the agent runs on the daemon, not the orchestrator. +The `daemon` image additionally bundles `@mermaid-js/mermaid-cli` (`mmdc`) plus a headless Chromium, used by the scheduled `research` action's diagram-validation gate. Agents run in a shared daemon or an isolated workflow runner built from this image, never in the orchestrator. The two images intentionally diverge after the shared base because their cost and attack surface differ. The shared prefix is enforced byte-identical by `scripts/check-dockerfile-base-sync.ts` (in CI) between the `# --- SHARED-BASE-BEGIN ---` and `# --- SHARED-BASE-END ---` markers. @@ -18,7 +18,7 @@ The two images intentionally diverge after the shared base because their cost an | Stage | Base | Purpose | | ------------- | ----------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- | | `base` | `oven/bun:1.3.14` | Installs Node.js 20 (for the Claude Code CLI), npm 11, `curl`, `git`, `@anthropic-ai/claude-code` globally, plus targeted openssl CVE upgrades. | -| `development` | `base` | `bun install` (all deps) + `bun run build` → `dist/` (app, daemon main, MCP stdio servers). | +| `development` | `base` | `bun install` (all deps) + `bun run build` → `dist/` (app, daemon, workflow runner, process-boundary probe, MCP stdio servers). | | `deps` | `base` | `bun install --production --ignore-scripts` (runtime deps only). | ### Orchestrator-only stage @@ -46,6 +46,8 @@ bun run docker:build # both There is no default `Dockerfile`, always pass `-f`. +The daemon image compiles a native preload guard that sets `PR_SET_DUMPABLE=0` before Bun starts. After each architecture-specific digest is pushed, `docker-build.yml` runs the bundled startup probe from that exact digest as UID/GID 1000 with no network, no capabilities, a read-only container root, and `no-new-privileges`. A digest is not published into the merged manifest unless the real child-to-parent `/proc//environ` read is denied. The GitLab main-branch publisher builds and loads its amd64 image under a commit-local tag, runs the same restricted probe, then tags and pushes `latest-daemon`. + ### Build arguments | Argument | Default | Purpose | @@ -209,11 +211,23 @@ Dominated by what Claude runs inside it (`kubectl`, `terraform plan`, `docker bu The daemon image is ~2 GB unpacked. The same sizing applies to ephemeral daemon Pods spawned by the orchestrator (same image). +### Isolated workflow runner + +Each structured workflow gets one Pod with fixed per-container resources from `src/k8s/workflow-runner-spawner.ts`: + +| Resource | Request | Limit | +| ----------------- | ------- | ------ | +| CPU | 500m | 2 | +| Memory | 1 GiB | 4 GiB | +| Ephemeral storage | 2 GiB | 10 GiB | + +`MAX_CONCURRENT_REQUESTS` is the controller's database admission ceiling for these Pods. The supported deployment has one controller replica; this is not a distributed cluster-wide semaphore. + ### Disk Each job clones the target repo to `CLONE_BASE_DIR` (default `/tmp/bot-workspaces`) with `git clone --depth=${CLONE_DEPTH}` (default `50`). The directory is removed in the pipeline's `finally` block. -Peak disk = `average_repo_size × concurrent_jobs`. For monorepos, mount a dedicated volume: +Peak disk = `average_repo_size × concurrent_jobs`. For shared daemons and the orchestrator's local development path, mount a dedicated volume: ```yaml volumes: @@ -230,9 +244,26 @@ containers: mountPath: /workspaces ``` -## Ephemeral-daemon Kubernetes requirements +Each isolated runner gets one 10 GiB `emptyDir` mounted at `/tmp/bot-workspaces`. The clone and artifacts disappear with the Pod. Keep both the volume limit and the container's 10 GiB ephemeral-storage limit because they cover different accounting surfaces, but do not treat either as a filesystem quota. Kubernetes enforces local-storage excess through eviction, and its default directory scan misses deleted files that a process keeps open. The fixed runner-node placement below contains that failure mode away from control-plane and application nodes. Kubernetes documents the eviction behavior, deleted-open-file gap, and optional quota-based measurement in [Local ephemeral storage](https://kubernetes.io/docs/concepts/storage/ephemeral-storage/). + +## Kubernetes worker requirements + +Structured workflows require the controller to create workflow-runner Pods and Secrets in the dedicated `WORKFLOW_RUNNER_NAMESPACE`. Ephemeral shared-daemon scaling creates Pods in `EPHEMERAL_DAEMON_NAMESPACE`. Controller startup rejects equal namespace values because the runner admission policy validates every Pod in its namespace. + +### Dedicated runner nodes + +Provision a worker pool used only for workflow-runner Pods and required node daemons. Do not place control-plane components, the controller, databases, Valkey, or application workloads on it. Label and taint every node in that pool, and put the same label and taint on replacement-node templates: + +```bash +kubectl label node github-app.node-restriction.kubernetes.io/workflow-runner=true +kubectl taint node github-app.node-restriction.kubernetes.io/workflow-runner=true:NoSchedule +``` + +`WORKFLOW_RUNNER_NODE_LABEL` and `WORKFLOW_RUNNER_NODE_VALUE` change that key/value pair, so a cluster can point runners at a node pool it already labels and taints instead of adding a second pair. One setting drives both the `nodeSelector` and the `NoSchedule` toleration. Whatever pair you configure must also be set as `runnerNodeLabel` / `runnerNodeValue` in the runner boundary ConfigMap, or admission denies every runner Pod. -If you want the orchestrator to spawn ephemeral daemon Pods on demand, two things must exist in `EPHEMERAL_DAEMON_NAMESPACE`. +Enable the Node authorizer and `NodeRestriction` admission plugin before using this pool. Kubernetes prevents kubelets from setting labels in the `node-restriction.kubernetes.io` namespace only when both controls are active. Verify those control-plane settings instead of inferring them from a successful label command. See [Node isolation/restriction](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-isolation-restriction). Overriding `WORKFLOW_RUNNER_NODE_LABEL` to a key outside that reserved namespace gives this protection up: a compromised kubelet could then label its own node and attract runner Pods. + +Drain pre-existing non-runner workloads before treating a node as dedicated. The spawner always selects the configured label and carries only the matching `NoSchedule` toleration. The admission policy requires that exact selector and toleration, so a runner stays Pending if the dedicated pool is absent and cannot be mutated onto a shared or control-plane node. Taints affect scheduling, not existing Pods, and another workload with the same toleration could still enter the pool; restrict who can set that toleration and audit the actual node workload set after rollout and node replacement. ### Orchestrator RBAC @@ -240,17 +271,17 @@ If you want the orchestrator to spawn ephemeral daemon Pods on demand, two thing apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: github-app-ephemeral-spawner + name: github-app-ephemeral-daemon-spawner namespace: ${EPHEMERAL_DAEMON_NAMESPACE} rules: - apiGroups: [""] resources: ["pods"] - verbs: ["create", "get", "delete"] + verbs: ["create"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: github-app-ephemeral-spawner + name: github-app-ephemeral-daemon-spawner namespace: ${EPHEMERAL_DAEMON_NAMESPACE} subjects: - kind: ServiceAccount @@ -258,22 +289,161 @@ subjects: namespace: ${ORCHESTRATOR_NAMESPACE} roleRef: kind: Role - name: github-app-ephemeral-spawner + name: github-app-ephemeral-daemon-spawner + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: github-app-workflow-runner-manager + namespace: ${WORKFLOW_RUNNER_NAMESPACE} +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["create", "get", "delete"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["create", "get", "update", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: github-app-workflow-runner-manager + namespace: ${WORKFLOW_RUNNER_NAMESPACE} +subjects: + - kind: ServiceAccount + name: github-app + namespace: ${ORCHESTRATOR_NAMESPACE} +roleRef: + kind: Role + name: github-app-workflow-runner-manager apiGroup: rbac.authorization.k8s.io ``` -Without these verbs every spawn yields `dispatch_reason=ephemeral-spawn-failed` and the job is rejected with a tracking-comment infra error. +The controller uses `get` plus UID deletion preconditions to avoid deleting a replacement resource with the deterministic attempt name. It creates the Pod first and makes the per-attempt Secret an owned dependent of that exact Pod UID. It uses `update` only to rotate an existing owned Secret after an ambiguous create or controller-secret rotation. No worker Pod receives this ServiceAccount token. ### `daemon-secrets` Secret -Spawned ephemeral Pods get their config via `envFrom: secretRef: daemon-secrets`. Create this Secret once in `EPHEMERAL_DAEMON_NAMESPACE` with at minimum: +Spawned ephemeral Pods get their config via `envFrom: secretRef: `, which defaults to `daemon-secrets`. Create this Secret once in `EPHEMERAL_DAEMON_NAMESPACE` with at minimum: - `DAEMON_AUTH_TOKEN`: daemon ⇄ orchestrator handshake. **Only source.** The spawner does not inline this into the Pod spec, so it cannot leak via `kubectl get pod -o yaml` or the Pod audit log. - `ANTHROPIC_API_KEY` or `CLAUDE_CODE_OAUTH_TOKEN` (and `ALLOWED_OWNERS`) or Bedrock `AWS_*` vars. -- `VALKEY_URL`, `DATABASE_URL`. GitHub App private-key material is **not** placed in this Secret. The orchestrator mints installation tokens and hands them per-job, so blast radius does not need to expand to every ephemeral Pod. `ORCHESTRATOR_URL` is provided inline by the spawner from `ORCHESTRATOR_PUBLIC_URL`. +Never place `DATABASE_URL` or `VALKEY_URL` in `daemon-secrets`; shared daemons reach those services through the controller protocol. + +`EPHEMERAL_DAEMON_SECRET_NAME` exists so a deployment whose persistent daemon pools already mount a suitable Secret can point ephemeral daemons at the same object instead of maintaining a second copy of the same credentials, which drift apart on rotation. It does not relax the contents rule above. Reusing a broader Secret gives every ephemeral Pod every key in it, so weigh that against the rotation cost. Reusing the controller's own Secret is the worst case: it hands short-lived agent Pods the GitHub App private key, the webhook secret, and the workflow-runner capability root. + +### `workflow-runner-secrets` Secret + +Create this provider-only Secret once in `WORKFLOW_RUNNER_NAMESPACE`. Put only the selected credential chain in it: + +- Anthropic: `ANTHROPIC_API_KEY` or `CLAUDE_CODE_OAUTH_TOKEN`, never both. If both are configured on the controller, the runner deliberately receives only `ANTHROPIC_API_KEY`, matching Claude Code's documented precedence. +- Bedrock API key: `AWS_BEARER_TOKEN_BEDROCK` only. +- Bedrock static credentials: `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`, plus `AWS_SESSION_TOKEN` only when the credentials are temporary. + +For production Bedrock runners, use a dedicated IAM principal with only the inference actions and model or inference-profile resources the selected SDK path needs. Do not grant non-Bedrock actions or `sts:AssumeRole`. AWS identifies `bedrock:InvokeModel` and `bedrock:InvokeModelWithResponseStream` as the core inference permissions and documents how to narrow actions and resources in [Prerequisites for running model inference](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-prereq.html). Use temporary IAM credentials, or a short-term Bedrock API key with automated Secret rotation. AWS recommends short-term API keys for production and long-term keys only for exploration in [Amazon Bedrock API keys](https://docs.aws.amazon.com/bedrock/latest/userguide/api-keys.html). `AWS_BEARER_TOKEN_BEDROCK` is a Bedrock API key; `aws-actions/configure-aws-credentials` exports a temporary IAM access-key, secret-key, and session-token chain instead. + +The spawner reads `CLAUDE_PROVIDER`, `CLAUDE_MODEL`, `AWS_REGION`, optional `ANTHROPIC_BEDROCK_BASE_URL`, and `ALLOWED_OWNERS` from validated controller configuration and places those non-secret settings inline. A Bedrock base-URL override must be an absolute HTTPS URL with no username, password, query, or fragment; the attempt fails before its capability Secret is created otherwise. `AWS_PROFILE` is unsupported for isolated runners because they mount no AWS profile files. A Bedrock controller configured only with a profile fails the attempt before a Pod is created. Every selected credential reference is required, so a missing Secret key prevents the container from starting instead of silently selecting another chain. + +Do not place unused provider chains, GitHub App, PAT, database, Valkey, daemon, Kubernetes, or Context7 credentials in this Secret. The Pod references only the selected keys and never uses `envFrom`. Before connecting, the runner rejects dual Anthropic credentials, dual or incomplete Bedrock chains, cross-provider credentials, and controller credentials. `GH_ENTERPRISE_TOKEN` and `GITHUB_ENTERPRISE_TOKEN` are forbidden because [GitHub CLI treats them as authentication for GitHub Enterprise Server hosts](https://cli.github.com/manual/gh_help_environment). + +### Workflow-runner admission boundary + +Post-create reconciliation is not early enough to stop a mutated image or lifecycle hook: the kubelet can start the admitted Pod before the controller reads the create response. Install [`examples/workflow-runner-admission.yaml`](https://github.com/chrisleekr/github-app/blob/main/examples/workflow-runner-admission.yaml) before enabling structured workflows. It requires Kubernetes 1.30 or later, where `ValidatingAdmissionPolicy` is generally available. + +The [`github-app` Helm chart](https://github.com/chrisleekr/helm-charts/tree/main/charts/github-app) packages this file behind `workflowRunner.enabled`, and is the recommended install path. It derives the boundary parameters below from the same values that render the controller's own config, so `runnerImage` cannot drift from `DAEMON_IMAGE`. This example stays the canonical copy of the policy: the chart carries its `spec` verbatim and a chart-side gate fails when the two differ. Apply the steps below by hand only when installing without the chart. + +The example creates a Restricted `github-app-runners` namespace and a fail-closed policy and binding. The binding selects the dedicated namespace and the policy validates every Pod create, Pod update, and ephemeral-container update in it. Before applying it: + +1. Set `workflow-runner-boundary.data.runnerImage` to the exact `@sha256:` image configured as `DAEMON_IMAGE`. Tags, including immutable release tags, are rejected by the controller and policy. +2. Set `workflow-runner-boundary.data.orchestratorOrigin` to the WSS origin used by `ORCHESTRATOR_PUBLIC_URL`, without a path or trailing slash. + Set `runnerNodeLabel` and `runnerNodeValue` to the controller's `WORKFLOW_RUNNER_NODE_LABEL` / `WORKFLOW_RUNNER_NODE_VALUE` values. + Set `runnerImagePullSecret` to the controller's `WORKFLOW_RUNNER_IMAGE_PULL_SECRET` value: the name of an existing `kubernetes.io/dockerconfigjson` Secret in the runner namespace, or an empty string to forbid pull secrets entirely. The runner has no ServiceAccount token, so the kubelet reads this Secret and the container never can. +3. Copy the controller's exact `provider`, `model`, optional `awsRegion`, `anthropicBedrockBaseUrl`, and `allowedOwners` values into the boundary ConfigMap. Use an empty string for an omitted optional setting. +4. Set `providerCredential1..3` to the exact selected Secret-key names in spawner order: one Anthropic key; one Bedrock bearer key; or access key, secret key, and optional session token. Leave unused slots empty. The ConfigMap contains names and non-secret settings, never credential values. +5. Provision and verify the dedicated labeled-and-tainted runner nodes described above. Label the controller namespace `github-app.chrislee.kr/workflow-controller=true`, retain the controller Pod labels from the example, and adapt the DNS selectors if the cluster does not label its DNS Pods `k8s-app=kube-dns`. +6. Set `WORKFLOW_RUNNER_NAMESPACE=github-app-runners`, or consistently rename the Namespace, ConfigMap namespace, binding selector, and parameter namespace. Keep `EPHEMERAL_DAEMON_NAMESPACE` different. +7. Apply the Namespace and ConfigMap first, then the policy and binding. Keep `parameterNotFoundAction: Deny`, `failurePolicy: Fail`, and `validationActions: [Deny, Audit]`. +8. Before enabling workflows, require a server-side dry-run canary to be denied by `github-app-workflow-runner-boundary`. Policy `status.typeChecking` proves expression type checking completed; it does not test whether the binding is already enforcing requests. The CI harness polls this negative canary for up to 30 seconds before testing the production renderer. + +This canary is compatible with Restricted Pod Security, performs no write, and fails if admission returns an unrelated error: + +```bash +RUNNER_NAMESPACE="${WORKFLOW_RUNNER_NAMESPACE:-github-app-runners}" +for attempt in $(seq 1 30); do + if output="$( + kubectl create --dry-run=server --output=name \ + --namespace="${RUNNER_NAMESPACE}" --filename=- 2>&1 <<'YAML' +apiVersion: v1 +kind: Pod +metadata: + name: workflow-runner-policy-canary +spec: + automountServiceAccountToken: false + restartPolicy: Never + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - name: canary + image: registry.invalid/canary@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] +YAML + )"; then + if [ "${attempt}" -eq 30 ]; then + echo "workflow runner admission policy did not enforce within 30 seconds" >&2 + exit 1 + fi + sleep 1 + elif printf '%s\n' "${output}" | grep -Fq \ + "ValidatingAdmissionPolicy 'github-app-workflow-runner-boundary'"; then + echo "workflow runner admission policy is enforcing" + break + else + printf '%s\n' "${output}" >&2 + exit 1 + fi +done +``` + +The policy validates the final Pod after mutation. It binds the digest, controller URL, provider settings, selected credential-key names, and dedicated-node placement. It rejects changed identity labels, extra containers, volumes, annotations, finalizers, lifecycle hooks, security overrides, capability additions, changed environment names, sources, values, or resource budgets, direct node placement, affinity, altered node selectors, priority classes, scheduling/readiness gates, more than one image-pull Secret or one whose name differs from `runnerImagePullSecret`, altered tolerations, and ephemeral-container updates. The controller repeats the boundary check after create and during reconciliation. Kubernetes recommends final-state validation because mutating admission order is not stable: [Admission Webhook Good Practices](https://kubernetes.io/docs/concepts/cluster-administration/admission-webhooks-good-practices/#validate-mutations-before-admission). The [ValidatingAdmissionPolicy reference](https://kubernetes.io/docs/reference/access-authn-authz/validating-admission-policy/) defines the fail policy, match conditions, parameter, binding, and deny behavior used by the example. The [policy status API](https://kubernetes.io/docs/reference/kubernetes-api/policy-resources/validating-admission-policy-v1/#ValidatingAdmissionPolicyStatus) limits `typeChecking` to expression-checking results, which is why rollout also probes a real admission request. + +Treat the boundary ConfigMap, policy, binding, and namespace labels as cluster-security configuration. The controller needs no write access to them. Grant changes only to the deployment administrator, verify the policy status has no `expressionWarnings`, and require the negative canary denial before enabling workflows. Under a chart install these are release-managed, so change them through the values file rather than `kubectl`: a manual edit is reverted on the next sync. `bun run test:admission` installs the production manifest and renderer in a disposable pinned Kubernetes 1.30 cluster, waits for real binding enforcement, then verifies the exact Pod plus prohibited mutations. + +The GitHub admission job runs on GitHub-hosted pull-request runners. The current self-managed GitLab project has no dedicated ephemeral privileged runner, so its kind/DinD job is restricted to the protected default branch and uses digest-pinned job, service, and nested images. Do not enable this job on feature branches through the shared instance runner. GitLab states that privileged jobs can gain root access to the runner host and recommends isolated, ephemeral runners restricted to protected branches: [runner security](https://docs.gitlab.com/runner/security/#reduce-the-security-risk-of-using-privileged-containers). + +The controller separately creates `workflow-runner-` with one deadline-bound HMAC capability. The capability is signed by the controller-only `WORKFLOW_RUNNER_CAPABILITY_SECRET`, not the shared-daemon key, and the Secret has one owner reference to the exact runner Pod UID. The target-repository token is delivered at most once and its GitHub-reported expiry must be no later than the immutable attempt deadline. A transport reconnect carries no job payload or repository credential. Payload preparation attempts best-effort revocation when delivery fails. After delivery, the runner attempts best-effort revocation after its final repository operation. The controller independently attempts revocation for reconnect, notification, and result-projection tokens through GitHub's [token self-revocation endpoint](https://docs.github.com/en/rest/apps/installations#revoke-an-installation-access-token), with a ten-second API timeout. Revocation failure does not block terminal handling. The repository token is not stored in PostgreSQL or Kubernetes, so a failed revocation, process crash, or node loss remains bounded by its single-repository scope and authoritative GitHub expiry. A process crash is terminal for that Pod, so `restartPolicy: Never` lets reconciliation fail the attempt promptly instead of looping a replacement process that cannot safely receive the credential again. Projection retries do not retain runner credentials or compute. Pod deletion uses the normal 30-second grace period. The cleanup receipt means Kubernetes accepted UID-preconditioned deletion, or the exact resource was already absent. A Pod may remain `Terminating` while finalizers run; the owned Secret is also eligible for garbage collection. Do not automate `--force --grace-period=0`: [Kubernetes warns that force deletion does not confirm the Pod processes have stopped](https://kubernetes.io/docs/reference/kubectl/generated/kubectl_delete/). If a node is partitioned, verify the node and process state before an operator force-deletes the exact Pod UID. Enable Kubernetes Secret encryption at rest and restrict Secret RBAC to this controller ServiceAccount. + +### ResourceQuota + +Each active workflow attempt consumes one Pod, one per-attempt Secret, and bounded ephemeral storage. Quota all three so a cleanup or retry defect cannot exhaust API-server or node capacity. This is an example ceiling, not a sizing recommendation: + +```yaml +apiVersion: v1 +kind: ResourceQuota +metadata: + name: github-app-runner-objects + namespace: ${WORKFLOW_RUNNER_NAMESPACE} +spec: + hard: + count/pods: "10" + count/secrets: "12" + requests.ephemeral-storage: 20Gi + limits.ephemeral-storage: 100Gi +``` + +### Runner-node PID boundary + +CPU, memory, and ephemeral-storage limits do not bound process IDs. Kubernetes does not support a Pod-spec PID limit, so every dedicated runner node must have a finite positive kubelet `podPidsLimit` and nonzero PID reservations for both the operating system and Kubernetes daemons. Audit the effective kubelet configuration on every runner node before rollout and after node-pool replacement. The default `podPidsLimit: -1` is unbounded and fails this prerequisite. + +Size the limit and `systemReserved.pid` / `kubeReserved.pid` from the node's `pid_max`, maximum Pod density, and system-daemon demand. Do not copy the test fixture's value into production. Kubernetes documents why fast PID exhaustion can destabilize kubelet and the container runtime, and why eviction alone is not a hard boundary: [Process ID limits and reservations](https://kubernetes.io/docs/concepts/policy/pid-limiting/) and [KubeletConfiguration `podPidsLimit`](https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/). + ### Ephemeral Pod security posture The spawner hardens every ephemeral Pod (see `src/k8s/ephemeral-daemon-spawner.ts`): @@ -283,15 +453,45 @@ The spawner hardens every ephemeral Pod (see `src/k8s/ephemeral-daemon-spawner.t - Container: `allowPrivilegeEscalation: false`, `capabilities.drop: [ALL]`. - `restartPolicy: Never` and `activeDeadlineSeconds: 3600` cap the Pod hard. +Workflow-runner Pods add stricter identity reconciliation (see `src/k8s/workflow-runner-spawner.ts`): + +- one container, one size-limited workspace `emptyDir`, no other volumes, no init/sidecar/ephemeral containers, no service-account token, service links, or host PID/IPC/network namespaces; +- `runAsNonRoot: true`, UID/GID 1000, `seccompProfile: RuntimeDefault`, no privilege escalation, and all Linux capabilities dropped; +- `restartPolicy: Never`, `activeDeadlineSeconds: 4200`, and a 30-second termination grace period; +- digest-pinned image, default service account without a token, exact default scheduler and priority, fixed dedicated-node selector and toleration, and no caller-controlled placement or image-pull credentials; +- exact command, environment sources, security context, and resources verified after create and on every reconcile. + +Admission or service-mesh mutation that changes this boundary is rejected as a permanent runner-start failure. Exclude these Pods and their per-attempt Secrets from mutation, while preserving the labels the controller uses for ownership checks. + +### Workflow-runner egress boundary + +Install `github-app-workflow-runner-egress-boundary` from [`examples/workflow-runner-admission.yaml`](https://github.com/chrisleekr/github-app/blob/main/examples/workflow-runner-admission.yaml), or set `workflowRunner.enabled` in the Helm chart, before enabling workflow dispatch. Once it selects a runner Pod for `Egress`, traffic is denied unless one of its three rules allows it: + +1. UDP/TCP DNS to the selected cluster DNS Pods on port 53. +2. WSS to the selected controller Pods on port 3002. +3. TCP 443 to public IPv4 and IPv6 addresses, excluding private, loopback, link-local, documentation, benchmark, multicast, and reserved ranges. + +Label the controller namespace `github-app.chrislee.kr/workflow-controller=true` and keep the controller Pod labels aligned with the policy. Adapt the DNS selector to the cluster's actual DNS labels. If `ORCHESTRATOR_PUBLIC_URL` uses a public ingress instead of the selected controller Pods, expose it on TCP 443. Verify these paths from the runner namespace before rollout. + +Kubernetes NetworkPolicies are additive. Audit every policy selecting runner Pods because another egress rule can widen this boundary. The portable API identifies destinations by Pod, namespace, or CIDR, not DNS name. The example therefore blocks cluster/private-network reachability and non-HTTPS public traffic, but it cannot distinguish GitHub and the selected AI provider from an attacker-controlled public HTTPS host. Deployments requiring exact public-host allowlisting must add a CNI-specific DNS policy or force runner egress through an allowlisted proxy, limited to the GitHub and provider endpoints the selected configuration needs. + +Private provider endpoints are denied by the example. If the selected provider uses a private endpoint, add only that endpoint's exact Pod, namespace, or CIDR destination and required port. Do not allow an entire private address range. + +The network policy is the primary metadata control. The runner also probes the AWS/Azure IPv4 endpoint and AWS/Google IPv6 endpoints before registration. Any HTTP response, including `401` or `403`, fails startup. A policy drop normally appears as a timeout, so timeouts and connection refusals are accepted as corroborating evidence, not treated as proof by themselves. + +For EKS worker nodes, also require IMDSv2 and set the response hop limit to `1`. AWS documents that setting as the Pod-blocking configuration. For AKS, prefer the platform IMDS restriction where its preview limitations are acceptable; existing clusters also require a node reimage after enabling it. These controls are independent of the policy and startup probe. + +Primary references: [Kubernetes NetworkPolicy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), [AWS EKS identity guidance](https://docs.aws.amazon.com/eks/latest/best-practices/identity-and-access-management.html), [AWS IRSA credential isolation](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html), [AWS IMDS endpoints](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html), [Google Cloud metadata endpoints](https://docs.cloud.google.com/compute/docs/metadata/querying-metadata), [Azure AKS cluster security](https://learn.microsoft.com/en-us/azure/aks/operator-best-practices-cluster-security), and [AKS IMDS restriction](https://learn.microsoft.com/en-us/azure/aks/imds-restriction). + ## Production tunables worth double-checking The full schema lives at [`configuration.md`](configuration.md). At minimum: -| Variable | Production recommendation | -| ------------------------- | ------------------------------------------------------------ | -| `NODE_ENV` | `production` | -| `LOG_LEVEL` | `info` (`debug` exposes webhook payloads) | -| `MAX_CONCURRENT_REQUESTS` | Start at `3`, tune against memory and LLM budget | -| `AGENT_TIMEOUT_MS` | Stay below 3600 s, the GitHub installation-token TTL | -| `CLONE_BASE_DIR` | Override if `/tmp` is small or shared | -| `PORT`, `WS_PORT` | `3000`, `3002` (must match probes and the `WS_PORT` env var) | +| Variable | Production recommendation | +| ------------------------- | ------------------------------------------------------------------------------------------------------------ | +| `NODE_ENV` | `production` | +| `LOG_LEVEL` | `info` (`debug` exposes webhook payloads) | +| `MAX_CONCURRENT_REQUESTS` | Start at `3`, tune against memory and LLM budget | +| `AGENT_TIMEOUT_MS` | Keep within the runner's effective token window; it aborts five minutes before the one-hour App token expiry | +| `CLONE_BASE_DIR` | Override if `/tmp` is small or shared | +| `PORT`, `WS_PORT` | `3000`, `3002` (must match probes and the `WS_PORT` env var) | diff --git a/docs/operate/observability.md b/docs/operate/observability.md index fa5f6ee4..5a45ac8a 100644 --- a/docs/operate/observability.md +++ b/docs/operate/observability.md @@ -1,20 +1,20 @@ # Observability -Structured JSON logs via [pino](https://getpino.io) are the primary signal. Every dispatch decision and every pipeline step carries a `deliveryId` so you can reconstruct a request end-to-end from a single log query, and the webhook event handlers plus the daemon workflow executor additionally carry a canonical `entityNumber` (the issue or PR number) so a request can be reconstructed by entity as well. When `DATABASE_URL` is configured, the same information is persisted to `executions` and `triage_results` for aggregate reporting. +Structured JSON logs via [pino](https://getpino.io) are the primary signal. Every dispatch decision and every pipeline step carries a `deliveryId` so you can reconstruct a request end-to-end from a single log query. Webhook handlers and worker execution loggers also carry a canonical `entityNumber` (the issue or PR number), so a request can be reconstructed by entity. When `DATABASE_URL` is configured, the same information is persisted to `executions` and `triage_results` for aggregate reporting. ## Log redaction The exported `logger` in `src/logger.ts` is the canonical chokepoint for secret scrubbing, every child logger inherits its `redact.paths` list and its custom `err` serializer, so individual call sites do not need to remember to scrub. Two layers run on every emitted line: -1. **Path-based redaction**: the exported `REDACT_PATHS` constant in `src/logger.ts` lists every field pino should replace with `[Redacted]` before the JSON is serialised. Paths covered: `authorization` and its `*.authorization` / `headers.authorization` / `*.headers.authorization` / `req.headers.authorization` / `request.headers.authorization` variants; the webhook signature header `x-hub-signature-256` (also wildcard-prefixed); `response.data.token`; and the named credential fields `token`, `installationToken`, `privateKey`, `webhookSecret`, `anthropicApiKey`, `claudeCodeOauthToken`, `daemonAuthToken`, `awsSecretAccessKey`, `awsSessionToken`, `awsBearerTokenBedrock`, `*.password`. The list is `Object.freeze`d so an accidental `push` from another module cannot silently weaken the policy. +1. **Path-based redaction**: the exported `REDACT_PATHS` constant in `src/logger.ts` lists every field pino should replace with `[Redacted]` before the JSON is serialised. Paths covered: `authorization` and its `*.authorization` / `headers.authorization` / `*.headers.authorization` / `req.headers.authorization` / `request.headers.authorization` variants; the webhook signature header `x-hub-signature-256` (also wildcard-prefixed); `response.data.token`; and the named credential fields `token`, `installationToken`, `privateKey`, `webhookSecret`, `anthropicApiKey`, `claudeCodeOauthToken`, `daemonAuthToken`, `daemonAuthTokenPrevious`, `workflowRunnerCapabilitySecret`, `workflowRunnerCapabilitySecretPrevious`, `awsSecretAccessKey`, `awsSessionToken`, `awsBearerTokenBedrock`, `*.password`. The list is `Object.freeze`d so an accidental `push` from another module cannot silently weaken the policy. -2. **`errSerializer` scrubbing**: the exported `errSerializer` in `src/logger.ts` defers to pino's `stdSerializers.err` and then runs the result's `message`, `stack`, `request.headers`, and `response.data` through `redactGitHubTokens` (`src/utils/sanitize.ts`) plus an inline credential-URL scrubber that mirrors `redactValkeyUrl` (`src/orchestrator/valkey.ts`). The walker recurses through nested objects/arrays and replaces any key matching the sensitive-field-name set wholesale, so `err.response.data.meta.token` and `err.request.headers.forwarded.authorization` are caught at any depth: this is necessary because pino's path-based rules cannot match four-or-more segments deep on `err.*`. It also catches `ghs_…` installation tokens and App JWTs echoed inside `err.message` / `err.stack`. +2. **`errSerializer` scrubbing**: the exported `errSerializer` in `src/logger.ts` defers to pino's `stdSerializers.err` and then runs primitive string errors plus the result's `message`, `stack`, `request.headers`, and `response.data` through the GitHub-token marker, credential-URL scrubber, and full `redactSecrets()` scanner from `src/utils/sanitize.ts`. The walker recurses through nested objects/arrays and replaces any key matching the sensitive-field-name set wholesale, so `err.response.data.meta.token` and `err.request.headers.forwarded.authorization` are caught at any depth. It also removes the raw webhook `event`, `payload`, `signature`, and unserialized aggregate-error carriers copied onto Octokit errors. The serializer operates on a copy, so the original Error instance is never mutated. If you add a new secret-bearing config field to `src/config.ts`, add its property name to `REDACT_PATHS` in the same PR. The point helpers `redactGitHubTokens` and `redactValkeyUrl` remain in place for their non-log call sites (prompt sanitisation and the Valkey startup info log respectively); the logger config is the system-wide default. -The crash path is covered too. `installFatalHandlers(processName)` in `src/logger.ts` registers `uncaughtException` and `unhandledRejection` handlers at both entrypoints (`src/app.ts`, `src/daemon/main.ts`) that log via `logger.fatal({ err })` and then `process.exit(1)`. Without them the runtime's default handler would print a plain `stderr` stack that bypasses `errSerializer`, so a token echoed inside an octokit error would reach the log shipper in cleartext. The default destination flushes synchronously on the process `exit` event, so the fatal line is written before exit; `pino.final` is intentionally not used because it throws when the logger is built with the dev-only `pino-pretty` transport. Crash lines carry `level: 60` (fatal) and a `processName` of `orchestrator` or `daemon`, so an alert on sustained `level:60` flags a crash-looping process. +The crash path is covered too. `installFatalHandlers(processName)` in `src/logger.ts` registers `uncaughtException` and `unhandledRejection` handlers at all three entrypoints (`src/app.ts`, `src/daemon/main.ts`, and `src/runner/main.ts`) that log via `logger.fatal({ err })` and then `process.exit(1)`. Without them the runtime's default handler would print a plain `stderr` stack that bypasses `errSerializer`, so a token echoed inside an octokit error would reach the log shipper in cleartext. The default destination flushes synchronously on the process `exit` event, so the fatal line is written before exit; `pino.final` is intentionally not used because it throws when the logger is built with the dev-only `pino-pretty` transport. Crash lines carry `level: 60` (fatal) and a `processName` of `orchestrator`, `daemon`, or `workflow-runner`, so an alert on sustained `level:60` flags a crash-looping process. ## Common log fields @@ -25,7 +25,7 @@ The crash path is covered too. `installFatalHandlers(processName)` in `src/logge | `installationId` | GitHub App installation id. Emitted by the webhook event handlers (`src/webhook/events/`) and the daemon job executor (`src/daemon/job-executor.ts`) so a per-installation rate-limit (see [GitHub API rate-limit fields](#github-api-rate-limit-fields)) is greppable to its installation. App mode only; absent under a `GITHUB_PERSONAL_ACCESS_TOKEN` (PAT) where there is no per-installation bucket. | | `event` | GitHub event name (`pull_request`, `issue_comment`, …) or canonical event key for ship workflow logs. | | `repo` | `owner/name` of the triggering repo. | -| `dispatch_target` | Always `daemon` (singleton, kept as a field for DB/log stability). | +| `dispatch_target` | Execution protocol: `daemon` for legacy/scoped shared jobs or `workflow-runner` for structured workflows. | | `dispatch_reason` | Why the job landed where it did. See [Dispatch reasons](#dispatch-reasons). | | `isEphemeral` | Present on daemon-originating log lines. `true` if emitted by an ephemeral daemon. | | `triage_fallback_reason` | Only present on triage fallbacks, see [`runbooks/triage.md`](runbooks/triage.md). | @@ -121,17 +121,88 @@ The load-bearing event is `retry.succeeded_after_retry`: it is the only signal i | `idempotency.duplicate_skipped` | info | `deliveryId`. The SET-NX found an existing key (a redelivery); the caller skips. | | `idempotency.failed_open` | warn | `deliveryId`, `reason` (`unavailable` when Valkey is unconfigured/disconnected, `error` when the SET threw), and `err` (the error message, on the `error` branch only). The caller proceeds (at-least-once degradation). | +## Per-repo config log events + +The repo's `.github-app.yaml` (see [Repo configuration](../use/repo-config.md)) is fetched at dispatch time and evaluated by Gate 1, then resolved again for Gate 2 during isolated-runner payload preparation or legacy shared-daemon accept. Every config-load failure mode is **fail-open**: a missing, unreachable, or invalid file yields the built-in defaults, so none of these events mean the bot stopped working. They mean it stopped honouring the repo's config. + +| `event` | Level | Fields | +| ----------------------------------- | ----- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `repo_config.gate_blocked` | info | `reason` (the rule that blocked, human-readable) and `explained` (whether a refusal comment was posted) on every emitter. The dispatcher path (`src/workflows/dispatcher.ts`) adds `workflowName`, `target`, `deliveryId`, `senderLogin`; the two ship-rail emitters in `src/workflows/ship/command-dispatch.ts` carry `owner`, `repo`, `pr_number`, and `senderLogin` (both write `senderLogin` on the line itself, so one query catches all three emitters) but no `workflowName` or `deliveryId`. No run row, no queue job, no tracking comment was created. | +| `repo_config.invalid` | warn | `owner`, `repo`, `kind` (`schema` \| `yaml-parse` \| `not-a-file`), plus `issues` (zod issue list) on the `schema` branch and `err` on `yaml-parse`. The file exists but was ignored; defaults were used. | +| `repo_config.fetch_failed` | warn | `owner`, `repo`, `err`. A non-404 GitHub error reading the file (outage, rate limit, permission). Treated as "no config", and deliberately **not** cached, so the next dispatch retries. | +| `repo_config.gate_error` | error | `err`, plus `owner`/`repo` on the loader path. Something unexpected threw while resolving the policy. The trigger dispatches anyway. | +| `repo_config.policy_applied` | info | `owner`, `repo`, `deliveryId`, `workflow` (resolved name, or `none` for a job with no workflow), `model`, `maxTurns`, `timeoutMs`, `extraAllowedToolCount`, `pathFilterCount`, `hasInstructions`, `warned`. Shared jobs emit it from `handleAccept`. Structured workflows emit it while preparing the isolated-runner payload and add `runId` plus `attemptId`; reconnect preparation can repeat, so deduplicate those lines by `attemptId`. Emitted only when the run resolved a non-empty Gate-2 policy or `max_turns`; a repo with no config emits nothing. | +| `repo_config.path_filters_applied` | info | `filterCount`, `excludedCount`, `keptCount`. Emitted by `runPipeline` (`src/core/pipeline.ts`) when `review.path_filters` actually removed at least one changed file from the prompt. | +| `repo_config.path_filters_rejected` | warn | `rejectedCount`. Emitted once by `runPipeline` (`src/core/pipeline.ts`) when at least one `review.path_filters` glob failed the glob-safety guard and was dropped. The surviving globs still apply. The rejected patterns are deliberately **not** logged, since a glob is repo-authored text. | + +The PR-side validation comment (`src/repo-config/pr-check.ts`) is a separate, read-only surface and uses its own `repo_config.pr_check.*` namespace, so a query on the events above never mixes "what the bot applied" with "what an author was told about a branch copy". + +| `event` | Level | Fields | +| ----------------------------------- | ----- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `repo_config.pr_check.posted` | info | `owner`, `repo`, `prNumber`, `outcome` (`valid` \| `invalid` \| `too-large`). One sticky comment was upserted on the PR. A PR whose diff does not touch the config file emits nothing. | +| `repo_config.pr_check.head_missing` | info | `owner`, `repo`, `prNumber`. The diff listed the config file but `(path, head SHA)` returned 404. Usually the pull request deleted the file; a 404 alone does not prove that, so the event name says only what the status proves. No comment is posted: "invalid" would be a lie. | +| `repo_config.pr_check.read_failed` | info | `owner`, `repo`, `prNumber`. A non-404 error reading the head-ref copy. No comment is posted. Info rather than warn: this surface is advisory, and a failed read costs an author a hint, not the bot a behaviour. | +| `repo_config.pr_check.not_a_file` | warn | `owner`, `repo`, `prNumber`, `reason` (`type` when the path resolves to a directory or a symlink on the head ref, `no-content` when the blob came back with no decodable content). | +| `repo_config.pr_check.disabled` | info | `owner`, `repo`, `prNumber`. The default branch's config sets `enabled: false`, so this GitHub-write surface stayed silent. Only that repo-wide master switch is honoured here; the passive `triggers.*` filters are deliberately not applied, so draft or `ignore_title_keywords` pull requests still get authoring feedback. | + +Unlike the head-ref reads above, a 404 on the **default-branch** fetcher read (`fetchRepoConfig`, the `repo_config.*` table) is **not** logged: having no config is the normal case, and it is negative-cached for 60 seconds so a repo without the file does not pay a REST call per dispatch. + +Two things worth alerting on. Any `repo_config.invalid` means someone merged a broken config to the default branch and the bot is silently running on defaults, so alert on **presence, not rate**: the invalid result is ETag-cached, so a permanently broken file emits the event once per process per file change, not once per dispatch. Any `repo_config.gate_error` at all is a bug, since the fetch path is already total. + +`repo_config.gate_blocked` at `explained: false` is the quiet path (the passive `triggers.*` filters). If a user reports "the bot ignored me", this is the event to grep for first. + +`repo_config.policy_applied` is the Gate-2 counterpart: it answers "did this +run actually honour the repo's knobs?". Two fields carry no text on purpose. +`hasInstructions` is a boolean rather than the instructions themselves (the +string can be 10KB of repo prose), and `warned: true` means the repo's file +failed validation and this run silently fell back to defaults, so it pairs +with the `repo_config.invalid` line for the same repo. A `maxTurns` or +`timeoutMs` lower than the repo asked for is not a bug: the resolver clamps +both against `AGENT_MAX_TURNS` / `AGENT_TIMEOUT_MS`. + +## Auto-review log events + +Emitted by `maybeAutoReview` in `src/webhook/events/pull-request.ts` on +`pull_request.synchronize`, and by the inline-comment MCP server. All carry the +standard `deliveryId` / `owner` / `repo` / `entityNumber` / `senderLogin` fields +except where noted. + +| Event | Level | Meaning | +| --------------------------------------- | ----- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `auto_review.skipped_self_push` | info | The pusher is this bot. Expected after a `resolve` run pushes fixes; this is what stops review → resolve → push → review from looping. | +| `auto_review.skipped_repo_opt_out` | debug | The repo does not set `workflows.review.auto: true`, or its config was missing/unreachable/invalid and the fail-open default applied. The most common reason for "nothing happened". | +| `auto_review.skipped_unchanged_diff` | info | The PR's own diff is byte-identical to the last reviewed one, i.e. a rebase. No review was run. | +| `auto_review.skipped_ship_active` | info | `ship` is already driving this PR and runs its own review -> resolve iteration. Avoids duplicate spend and a race on `idx_workflow_runs_inflight`. | +| `auto_review.ship_intent_check_failed` | warn | The `ship_intents` lookup failed. Fail-open: the review still runs. | +| `auto_review.skipped_fork_pr` | info | The PR's head is on a fork. Checkout resolves the head ref against the base repo, so a fork ref would clone the wrong tree; auto-review declines rather than guess. | +| `auto_review.outcome` | info | Carries `status`: `dispatched` or `refused` (Gate 1 or an in-flight run). A `refused` here never posts to GitHub, and its `explained` is `false`. | +| `auto_review.fingerprint_failed` | warn | `pulls.listFiles` failed. Fail-open: the review still runs, so the only cost is a possible redundant review after a rebase. | +| `auto_review.fingerprint_skipped` | info | The diff has 0 files, or 3000+ (GitHub's `listFiles` cap), so it cannot be fingerprinted. Same fail-open consequence; a rebase on such a PR always re-reviews. | +| `auto_review.fingerprint_read_failed` | warn | Valkey read failed. Fail-open, same consequence as above. | +| `auto_review.fingerprint_write_failed` | warn | Valkey write failed. The next push re-reviews once. | +| `auto_review.failed` | error | Dispatch threw. Carries `err`. The webhook is unaffected; the reactor already fired. | +| `mcp.inline_comment.deduped` | info | A finding was skipped because this bot already has a live comment at that `path` / `line` / `side`. Carries all three plus `pull_number`. | +| `mcp.inline_comment.dedup_check_failed` | warn | The existing-comment lookup failed. Fail-open: the comment is posted, so a duplicate is possible. | + +An auto-review that produces no visible effect is normal. Grep +`auto_review.skipped_repo_opt_out` first, then `auto_review.skipped_unchanged_diff`, +then `repo_config.gate_blocked`. A `refused` `auto_review.outcome` with an +in-flight reason means a push arrived mid-review and was dropped by design. + ## Output secret-guard log events `safePostToGitHub` (`src/utils/github-output-guard.ts`) is the output-side chokepoint for every byte sent to GitHub. It emits structured `warn`/`error` events when the regex pass or the optional LLM scanner acts on a body. Per the logging contract, none of these carry the matched bytes, surrounding context, or a hash, only `kinds`, counts, lengths, `callsite`, and `deliveryId`. -| `event` | Level | When | -| ----------------------------------- | ----- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `secret_redacted` | warn | Regex pass (`scanner: "regex"`) or LLM scanner (`scanner: "llm"`) stripped secret bytes from an outgoing body. | -| `llm_scanner_emptied_body_fallback` | warn | LLM scanner emptied a body the regex pass kept; treated as a false positive, regex-only body posted. | -| `llm_scanner_substitution_rejected` | warn | LLM scanner returned a non-deletion-only body (added/reordered/altered bytes); substitution rejected, regex-only body posted. A prompt-injected scanner is the leading hypothesis. The regex floor still applies, so the body is not guaranteed secret-free beyond it. See issue #198. | -| `llm_scanner_error` | warn | LLM scanner threw (e.g. Bedrock outage); fail-open, body that survived the regex pass is posted. | -| `secret_redaction_emptied_body` | error | Body was whitespace-only after redaction; the GitHub call is skipped entirely (no blank comment). | +| `event` | Level | When | +| ---------------------------------------------- | ----- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `secret_redacted` | warn | Regex pass (`scanner: "regex"`) or LLM scanner (`scanner: "llm"`) stripped secret bytes from an outgoing body. | +| `llm_scanner_emptied_body_fallback` | warn | LLM scanner emptied a body the regex pass kept; treated as a false positive, regex-only body posted. | +| `llm_scanner_substitution_rejected` | warn | LLM scanner returned a non-deletion-only body (added/reordered/altered bytes); substitution rejected, regex-only body posted. A prompt-injected scanner is the leading hypothesis. The regex floor still applies, so the body is not guaranteed secret-free beyond it. See issue #198. | +| `llm_scanner_error` | warn | LLM scanner threw (e.g. Bedrock outage); fail-open, body that survived the regex pass is posted. | +| `secret_redaction_emptied_body` | error | Body was whitespace-only after redaction; the GitHub call is skipped entirely (no blank comment). | +| `workflow_runner_output_rejected` | warn | The controller's encoded-secret scan found a credential in a runner command or result. Carries `scanner: "llm"`, `callsite`, `kinds`, and `matchCount`. The command is rejected; a result becomes a fixed safe failure. | +| `workflow_runner_output_scan_unavailable` | error | The runner-output LLM scanner was disabled, threw, or timed out. Carries `scanner: "llm"`, `callsite`, and a scrubbed `err` on errors. Runner RPC fails closed: commands are rejected and results become a fixed safe failure. General GitHub output retains its documented fail-open behavior. | +| `workflow_runner_terminal_projection_rejected` | warn | GitHub rejected a detailed terminal tracking projection with status 422. Carries `runId`, `attemptId`, and `status`; the controller retries once with fixed operator text and keeps the durable result pending if that fallback fails. | ## MCP server log fields @@ -160,7 +231,7 @@ Alerts worth having: `queue_depth` rising while `persistent_free_slots > 0` for ## Dispatcher log fields -The job dispatcher (`src/orchestrator/job-dispatcher.ts`) and the accept handler in `src/orchestrator/connection-handler.ts` emit the offer lifecycle as structured events. The four `dispatcher.offer.*` keys are pinned per-event by a `z.discriminatedUnion` (`src/orchestrator/log-fields.ts:75#DispatcherOfferLogSchema`), so each event carries exactly its own fields; `dispatcher.no_eligible_daemon` has its own shape. Event-key constants live in `src/orchestrator/log-fields.ts:28#DISPATCHER_LOG_EVENTS`, and the co-located test rejects field drift. +The job dispatcher (`src/orchestrator/job-dispatcher.ts`) and the accept handler in `src/orchestrator/connection-handler.ts` emit the offer lifecycle as structured events. The four `dispatcher.offer.*` keys are pinned per-event by a `z.discriminatedUnion` (`src/orchestrator/log-fields.ts:77#DispatcherOfferLogSchema`), so each event carries exactly its own fields; `dispatcher.no_eligible_daemon` has its own shape. Event-key constants live in `src/orchestrator/log-fields.ts:28#DISPATCHER_LOG_EVENTS`, and the co-located test rejects field drift. | `event` | Level | Meaning | | ------------------------------- | ----- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | @@ -176,7 +247,7 @@ The job dispatcher (`src/orchestrator/job-dispatcher.ts`) and the accept handler ## Daemon heartbeat fields -The orchestrator pings each connected daemon every `HEARTBEAT_INTERVAL_MS` (default 30s) and evicts one that misses pongs past `HEARTBEAT_TIMEOUT_MS` (default 90s). The heartbeat lifecycle in `src/orchestrator/connection-handler.ts` emits three structured events pinned per-event by a `z.discriminatedUnion` (`src/orchestrator/log-fields.ts:142#DaemonHeartbeatLogSchema`), so `missedPongs` is pinned to `pong_missed` and `ttl_refresh_failed` carries its `err`; constants live in `src/orchestrator/log-fields.ts:36#DAEMON_HEARTBEAT_LOG_EVENTS`. +The orchestrator pings each connected daemon every `HEARTBEAT_INTERVAL_MS` (default 30s) and evicts one that misses pongs past `HEARTBEAT_TIMEOUT_MS` (default 90s). The heartbeat lifecycle in `src/orchestrator/connection-handler.ts` emits three structured events pinned per-event by a `z.discriminatedUnion` (`src/orchestrator/log-fields.ts:144#DaemonHeartbeatLogSchema`), so `missedPongs` is pinned to `pong_missed` and `ttl_refresh_failed` carries its `err`; constants live in `src/orchestrator/log-fields.ts:36#DAEMON_HEARTBEAT_LOG_EVENTS`. | `event` | Level | Meaning | | ------------------------------------- | ----- | ---------------------------------------------------------------------------------------------------- | @@ -213,17 +284,17 @@ The schema is the source of truth. Adding or renaming a field requires updating Every shepherding emitter draws its `event` value from the typed `SHIP_LOG_EVENTS` constant in `src/workflows/ship/log-fields.ts`. Operators can grep for these literals deterministically. -| Event key | Where it fires | What it indicates | -| ------------------------------------- | --------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------- | -| `ship.iteration.enqueued` | `iteration.runIteration` after `enqueueJob` | A non-ready verdict bridged into the daemon `workflow_runs` pipeline. One row per iteration. | -| `ship.iteration.terminal_cap` | `iteration.runIteration` cap check | The intent hit `MAX_SHIP_ITERATIONS`. | -| `ship.iteration.terminal_deadline` | `iteration.runIteration` deadline check | The intent's `deadline_at` elapsed. | -| `ship.tickle.started` | `app.ts` boot, after `tickleScheduler.start()` | The cron tickle scheduler is scanning `ship:tickle`. | -| `ship.tickle.due` | `orchestrator.onStepComplete` early-wake **or** `session-runner.resumeShipIntent` | An intent is being re-entered. `source` discriminates `workflow_run_completion` vs scheduler. | -| `ship.tickle.skip_terminal` | `orchestrator.onStepComplete` early-wake | The hook found a `shipIntentId` but the intent is already terminal; the ZADD was skipped. | -| `ship.scoped..enqueued` | `dispatch-scoped.ts` after `enqueueJob` | A scoped command (`rebase`, `fix_thread`, `explain_thread`, `open_pr`) was enqueued. | -| `ship.scoped..daemon.completed` | `connection-handler.handleScopedJobCompletion` and the executor | Daemon reported `succeeded`. | -| `ship.scoped..daemon.failed` | Same | Daemon reported `halted` or `failed`. `reason` carries the structured halt reason. | +| Event key | Where it fires | What it indicates | +| ------------------------------------- | --------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------- | +| `ship.iteration.enqueued` | `iteration.runIteration` after `publishWorkflowRunById` | A non-ready verdict bridged into the isolated `workflow_runs` runner pipeline. One row per iteration. | +| `ship.iteration.terminal_cap` | `iteration.runIteration` cap check | The intent hit `MAX_SHIP_ITERATIONS`. | +| `ship.iteration.terminal_deadline` | `iteration.runIteration` deadline check | The intent's `deadline_at` elapsed. | +| `ship.tickle.started` | `app.ts` boot, after `tickleScheduler.start()` | The cron tickle scheduler is scanning `ship:tickle`. | +| `ship.tickle.due` | `orchestrator.onStepComplete` early-wake **or** `session-runner.resumeShipIntent` | An intent is being re-entered. `source` discriminates `workflow_run_completion` vs scheduler. | +| `ship.tickle.skip_terminal` | `orchestrator.onStepComplete` early-wake | The hook found a `shipIntentId` but the intent is already terminal; the ZADD was skipped. | +| `ship.scoped..enqueued` | `dispatch-scoped.ts` after `enqueueJob` | A scoped command (`rebase`, `fix_thread`, `explain_thread`, `open_pr`) was enqueued. | +| `ship.scoped..daemon.completed` | `connection-handler.handleScopedJobCompletion` and the executor | Daemon reported `succeeded`. | +| `ship.scoped..daemon.failed` | Same | Daemon reported `halted` or `failed`. `reason` carries the structured halt reason. | ### Querying example (Datadog / Loki) @@ -234,14 +305,15 @@ event:"ship.intent.transition" to_status:"human_took_over" terminal_blocker_cate ## Dispatch reasons -Canonical source: `src/shared/dispatch-types.ts`. Four values; all land on `dispatch_target=daemon`. +Canonical source: `src/shared/dispatch-types.ts`. Five values across two execution targets. -| Reason | When the router sets it | -| --------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `persistent-daemon` | Routed to an existing persistent daemon. The default, hot path. Also used during cooldown when a scale-up was warranted but blocked by the cooldown window. | -| `ephemeral-daemon-triage` | Triage returned `heavy=true` and an ephemeral daemon Pod was spawned. | -| `ephemeral-daemon-overflow` | Queue length ≥ `EPHEMERAL_DAEMON_SPAWN_QUEUE_THRESHOLD` **and** the persistent pool has zero free slots; a spawn drains the overflow. | -| `ephemeral-spawn-failed` | A spawn was required but the K8s API call failed. The job is rejected with a tracking-comment infra error. | +| Reason | Target | When the router sets it | +| --------------------------- | ----------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `persistent-daemon` | `daemon` | Routed to an existing persistent daemon. The default, hot path. Also used during cooldown when a scale-up was warranted but blocked by the cooldown window. | +| `ephemeral-daemon-triage` | `daemon` | Triage returned `heavy=true` and an ephemeral daemon Pod was spawned. | +| `ephemeral-daemon-overflow` | `daemon` | Queue length ≥ `EPHEMERAL_DAEMON_SPAWN_QUEUE_THRESHOLD` **and** the persistent pool has zero free slots; a spawn drains the overflow. | +| `ephemeral-spawn-failed` | `daemon` | A spawn was required but the K8s API call failed. The job is rejected with a tracking-comment infra error. | +| `workflow-runner` | `workflow-runner` | A structured workflow was committed for an isolated attempt. | ## Scheduled action log fields @@ -308,32 +380,41 @@ The slow line is independent of rate-limit headers, so a slow response from an e ## GitHub App installation-token mints -The orchestrator mints App installation tokens at six call sites, all routed through `mintInstallationToken` (`src/orchestrator/installation-token.ts#mintInstallationToken`). Each mint emits one structured line. Schema pinned by `GithubAppTokenMintLogSchema` (`src/orchestrator/log-fields.ts#GithubAppTokenMintLogSchema`). `cache_hit` is exact, not a latency heuristic: `@octokit/auth-app` serves cached tokens synchronously and only issues `POST /app/installations/{id}/access_tokens` on a miss. +The orchestrator mints App installation tokens at eight call sites, all routed through `mintInstallationToken` (`src/orchestrator/installation-token.ts#mintInstallationToken`). Each mint emits one structured line. Schema pinned by `GithubAppTokenMintLogSchema` (`src/orchestrator/log-fields.ts#GithubAppTokenMintLogSchema`). `cache_hit` is exact, not a latency heuristic: `@octokit/auth-app` serves cached tokens synchronously and only issues `POST /app/installations/{id}/access_tokens` on a miss. | Event | Level | Fields | Meaning | | --------------------------------- | ----- | ---------------------------------------------------- | -------------------------------------------------------------------------------------- | | `github.app.token.mint.succeeded` | info | `installation_id`, `via`, `cache_hit`, `duration_ms` | A token was returned (cache when `cache_hit:true`, else fresh mint). | | `github.app.token.mint.failed` | warn | `installation_id`, `via`, `duration_ms`, `err` | The mint threw; `duration_ms` distinguishes a fast failure from a GitHub-edge timeout. | -`via` is one of `handleAccept`, `handleScopedAccept`, `postOrphanNotification`, `shipTickleResume`, `proposalPoller`, `schedulerRunAction`. The token, App JWT, and private key are never logged (security invariant 2); `err` is serialized through the secret-scrubbing pino error serializer. +`via` is one of `handleAccept`, `handleScopedAccept`, `shipTickleResume`, `proposalPoller`, `schedulerRunAction`, `notifyExpiredWorkflowAttempts`, `workflowRunnerPayload`, or `workflowRunnerResult`. The token, App JWT, and private key are never logged (security invariant 2); `err` is serialized through the secret-scrubbing pino error serializer. ## Inbound HTTP boundary Structured access-log family for the webhook server's inbound HTTP surface (webhook entry, HMAC verification failure, readiness probe, operator scheduler endpoint). Schema pinned in `src/app-log-fields.ts#HttpLogFieldsSchema`; emit sites in `src/app.ts`. All lines carry bounded metadata only, never the webhook secret, the `X-Hub-Signature-256` bytes, the raw request body, or `Authorization` headers. -| Event | Level | Fields | Meaning | -| -------------------------------------- | ----- | ------------------------------------------- | --------------------------------------------------------------------------------------------------- | -| `http.webhook.received` | info | `deliveryId`, `event_name`, `duration_ms` | A delivery entered the webhook middleware (header values, not body). | -| `http.webhook.error` | warn | `kind`, `deliveryId?`, `event_name?`, `err` | `kind: "signature_mismatch"` (HMAC failure), `"handler_threw"`, or `"other"`. Fact of failure only. | -| `http.readyz.unready` | info | `is_ready`, `valkey_healthy` | `/readyz` returned 503. `/healthz` is intentionally silent. | -| `http.scheduler.run.rejected_disabled` | warn | `status` (404) | Scheduler disabled. | -| `http.scheduler.run.rejected_unauth` | warn | `status` (401) | Bad operator bearer token (never logged). | -| `http.scheduler.run.rejected_payload` | warn | `status` (413 \| 400), `reason` | `body_too_large`, `invalid_json`, `not_object`, `missing_field`. | -| `http.scheduler.run.enqueued` | info | `status` (202 \| 409), `enqueued` | `enqueued:true` → 202 fresh; `false` → 409 dedup. | -| `http.scheduler.run.failed` | error | `status` (500), `err` | Operator endpoint threw; `err` secret-scrubbed. | +| Event | Level | Fields | Meaning | +| -------------------------------------- | ----- | ------------------------------------------- | ------------------------------------------------------------------------------------------------------ | +| `http.webhook.received` | info | `deliveryId`, `event_name`, `duration_ms` | A delivery entered the webhook middleware (header values, not body). | +| `http.webhook.error` | warn | `kind`, `deliveryId?`, `event_name?`, `err` | `kind: "signature_mismatch"` (HMAC failure), `"handler_threw"`, or `"other"`. Fact of failure only. | +| `http.readyz.unready` | info | `is_ready`, `valkey_healthy` | `/readyz` returned 503. `/healthz` is intentionally silent. | +| `http.scheduler.run.rejected_disabled` | warn | `status` (404) | Scheduler disabled. | +| `http.scheduler.run.rejected_unauth` | warn | `status` (401) | Bad operator bearer token (never logged). | +| `http.scheduler.run.rejected_payload` | warn | `status` (413 \| 400), `reason` | `body_too_large`, `invalid_json`, `not_object`, `missing_field`. | +| `http.scheduler.run.enqueued` | info | `status` (202 \| 409), `enqueued` | `enqueued:true` → 202 fresh; `false` → 409 dedup. | +| `http.scheduler.run.failed` | error | `status` (500), `err` | Operator endpoint threw; `err` secret-scrubbed. | +| `http.request.failed` | error | `err` | The router itself threw, caught by the `Bun.serve` `error` hook; answers 500. Carries no request data. | The `kind` discriminator on `http.webhook.error` separates signature-verification failures (botched webhook-secret rotation) from downstream handler exceptions. +`http.request.failed` is the last-resort boundary for any route, distinct from `http.scheduler.run.failed` (one endpoint's own `catch`). It deliberately carries only the secret-scrubbed `err`: an unhandled throw may have been triggered by attacker-shaped input, so no path or header is recorded. + +### Routing contract + +Every path resolves to a response; there is no fallthrough. Unrecognised paths get an explicit `404`. This matters because the pre-`Bun.serve` handler delegated unmatched paths to the octokit webhook middleware, which returns `false` without writing anything, leaving the request unanswered and the socket open until the peer gave up. Routing is matched on a parsed, trailing-slash-normalised pathname, so `/healthz`, `/healthz/`, and `/healthz?x=1` all resolve alike. The router lives in `src/http-router.ts#createFetchHandler`, kept free of side effects so it is testable without booting the process. + +The server sets a 30s `idleTimeout`. That is the enforced backstop the `node:http` compat shim lacked: it accepts `requestTimeout`/`headersTimeout` but does not honour them, so a handler that never responded left the connection open indefinitely. Keep the value above octokit's own 9s webhook timeout. + ## Scheduler scan lifecycle events The scheduled-actions scheduler (`src/scheduler/scheduler.ts#createScheduler`) emits a `scheduler.scan.*` lifecycle on every timer tick. These are scan-level signals (heartbeat, duration, traffic, saturation), orthogonal to the per-action `scheduler.action.*` transitions. Schema pinned in `src/scheduler/log-fields.ts#SCHEDULER_LOG_EVENTS`. Scheduler lines carry no `deliveryId` (the scan is timer-driven, not request-scoped). @@ -349,16 +430,16 @@ The scheduled-actions scheduler (`src/scheduler/scheduler.ts#createScheduler`) e Structured lifecycle events for `workflow_runs` state transitions, emitted at the transition call sites (the `runs-store` mutators stay log-free because they are reused under transactions). Pinned by `src/workflows/log-fields.ts#WorkflowRunLogFieldsSchema`. Common fields: `runId`, `workflowName`, `target` (`{ type, owner, repo, number }`), and `deliveryId` (omitted for system-spawned runs). Terminal events add `duration_ms`. -| `event` | Level | Extra fields | Meaning | -| ------------------------------- | ------------ | --------------------------- | ------------------------------------------------------------------------------------------- | -| `workflow.run.queued` | info | _none_ | A `queued` row was inserted. | -| `workflow.run.running` | info | _none_ | Daemon flipped the row to `running`. | -| `workflow.run.succeeded` | info | `duration_ms` | Terminal success. | -| `workflow.run.incomplete` | warn | `duration_ms`, `reason` | Agent ran cleanly but a handler gate left work outstanding. | -| `workflow.run.failed` | warn / error | `duration_ms`, `reason` | Terminal failure. `warn` for handler-reported, `error` for uncaught throw. | -| `workflow.run.handed_off` | info | `duration_ms`, `childRunId` | Composite parent handed off to a child; row stays `running`. | -| `workflow.run.dispatch_refused` | info | `reason` (no `runId`) | Refused before any row inserted. | -| `workflow.run.enqueue_failed` | error | `reason` | Post-insert enqueue/publish failed; compensating `markFailed` released the in-flight guard. | +| `event` | Level | Extra fields | Meaning | +| ------------------------------- | ------------ | --------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `workflow.run.queued` | info | _none_ | A `queued` row was inserted. | +| `workflow.run.running` | info | _none_ | A worker claim flipped the row to `running`. | +| `workflow.run.succeeded` | info | `duration_ms` | Terminal success. | +| `workflow.run.incomplete` | warn | `duration_ms`, `reason` | Agent ran cleanly but a handler gate left work outstanding. | +| `workflow.run.failed` | warn / error | `duration_ms`, `reason` | Terminal failure. `warn` for handler-reported, `error` for uncaught throw. | +| `workflow.run.handed_off` | info | `duration_ms`, `childRunId` | Composite parent handed off to a child; row stays `running`. | +| `workflow.run.dispatch_refused` | info | `reason` (no `runId`) | Refused before any row inserted. | +| `workflow.run.enqueue_failed` | error | `reason` | Initial post-commit publication failed. The row stays queued with an open outbox generation while retrying. Retry-budget or `WORKFLOW_DISPATCH_TIMEOUT_MS` expiry fails the workflow and execution, releases the target guard, and durably queues a `dispatch-expired` public projection. | ## Workspace lifecycle events @@ -437,12 +518,12 @@ When `SOCKET_HEALTH_SELF_HEAL_ENABLED=true`, a suspected spin exits the process When `DATABASE_URL` is set, helpers in `src/db/queries/dispatch-stats.ts` expose the most operator-relevant aggregates: -| Helper | Returns | -| -------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `eventsPerTarget(days)` | Count of executions grouped by `dispatch_target`. Post-collapse this is always a single `daemon` row, query `dispatch_reason` directly for the per-reason split. | -| `triageRate(days)` | Share of events whose `dispatch_reason` is `ephemeral-daemon-triage`. | -| `avgConfidenceAndFallback(days)` | Mean triage confidence plus fallback counts by reason. | -| `triageSpend(days)` | Cumulative `cost_usd` for triage-reached executions. | +| Helper | Returns | +| -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | +| `eventsPerTarget(days)` | Count of executions grouped by `dispatch_target`, separating shared-daemon jobs from isolated structured workflows. | +| `triageRate(days)` | Share of shared-daemon executions whose `dispatch_reason` is `ephemeral-daemon-triage`; workflow runners are excluded from the denominator. | +| `avgConfidenceAndFallback(days)` | Mean triage confidence plus fallback counts by reason. | +| `triageSpend(days)` | Cumulative `cost_usd` for triage-reached executions. | Call them from an internal admin endpoint, a scheduled job, or `bun repl`. @@ -463,7 +544,8 @@ Call them from an internal admin endpoint, a scheduled job, or `bun repl`. - **Token-mint cache misses.** A drop in `github.app.token.mint.succeeded` `cache_hit:true` rate means a regression is bypassing the cached App singleton and paying a network mint per dispatch. - **Webhook signature failures.** Any sustained `event:"http.webhook.error" AND kind:"signature_mismatch"` rate is a botched `GITHUB_WEBHOOK_SECRET` rotation dropping deliveries. - **Scheduler saturation.** A gap in `scheduler.scan.started` longer than the scan interval means the timer stalled; any `scheduler.scan.skipped_overlap` over a 5m window means scans are saturating the interval (precursor to drifting cron slots). -- **Workflow enqueue failures.** Any `event:"workflow.run.enqueue_failed"` means a row was inserted but never reached a daemon; the compensating `markFailed` ran, but a sustained rate points at a broker outage. +- **Workflow enqueue failures.** Any `event:"workflow.run.enqueue_failed"` means the first publication attempt failed. PostgreSQL owns the queued row and open outbox generation while the reaper retries. Retry-budget or dispatch-age expiry terminalizes the workflow and execution, releases the target guard, and retries the `dispatch-expired` public projection. A sustained rate points at a broker outage. +- **Runner output scanner outage.** Any `event:"workflow_runner_output_scan_unavailable"` is an error and fails runner RPC closed. Restore the configured scanner before retrying affected workflows. - **Daemon reconnect storms.** `event:"daemon.connection.reconnect_scheduled" AND attempt >= 5` flags a daemon stuck in backoff. - **Workspace crashloop fingerprint.** A non-zero `count` on `event:"workspace.cleanup.exit"` means a daemon exited with in-flight workspaces. - **Ephemeral-spawn failures by kind.** Break `dispatch_reason=ephemeral-spawn-failed` down with `event:"k8s.spawn.failed"` `kind`: `infra-absent` (deploy regression), `api-rejected` (RBAC/validation), `api-unavailable` (control-plane). A sustained `k8s.spawn.decision_skipped reason:"cooldown"` rate means the fleet is under-scaled. diff --git a/docs/operate/runbooks/daemon-fleet.md b/docs/operate/runbooks/daemon-fleet.md index ed370a3a..c3792416 100644 --- a/docs/operate/runbooks/daemon-fleet.md +++ b/docs/operate/runbooks/daemon-fleet.md @@ -1,6 +1,6 @@ # Runbook: daemon fleet -A daemon is a standalone worker process that connects to the orchestrator over WebSocket, accepts job offers, and runs each job through `src/core/pipeline.ts`. The webhook server never runs the pipeline in-process, every execution happens on a daemon. +A daemon is a shared worker process that connects to the controller over WebSocket and executes legacy or scoped jobs. Structured `workflow-run` jobs no longer enter this fleet; each one uses a separate one-attempt runner Pod. The webhook server never runs the pipeline in-process. ## Persistent vs ephemeral @@ -21,12 +21,12 @@ flowchart LR Connect["WebSocket connect to ORCHESTRATOR_URL
Bearer DAEMON_AUTH_TOKEN"]:::work Register["daemon register
capabilities + resources + isEphemeral"]:::work Idle["Idle wait"]:::wait - Offer["job offer or scoped-job-offer"]:::work + Offer["job:offer or scoped-job:offer"]:::work Eval{{"Capacity check
memory floor + disk floor + slot free"}}:::fork Accept["job accept"]:::work Reject["job reject
with reason"]:::halt Run["src/core/pipeline.ts
clone -> agent -> push -> cleanup"]:::work - Result["job result or scoped-job completion"]:::work + Result["job:result or scoped-job:completion"]:::work Drain["Drain on SIGTERM
refuse new offers"]:::wait Exit["Exit"]:::done IdleExit["Ephemeral idle exit
after EPHEMERAL_DAEMON_IDLE_TIMEOUT_MS"]:::done @@ -48,21 +48,73 @@ flowchart LR At boot, before connecting to the orchestrator, the daemon sweeps stale workspace triples (clone dir + `.cred.sh` token helper + `-artifacts`) older than `WORKSPACE_STALE_TTL_MS` under `CLONE_BASE_DIR`, reclaiming SIGKILL/OOM/eviction orphans left behind when a prior run skipped its own cleanup. Each sweep emits a single `workspace.sweep` log line with `swept` / `retained` / `durationMs`. +Every boot generates a new daemon ID, including a container restart inside the same Kubernetes Pod. On disconnect, the controller immediately removes the socket from its local heartbeat and dispatch state, then starts serialized asynchronous cleanup. Same-ID registration waits for that cleanup. One PostgreSQL transaction marks the exact incarnation inactive, fails its attempt-less workflow rows and queued/offered/running execution receipts, records pending public-failure projections, and releases matching scheduled-action locks. The controller projects those receipts immediately and retries missed projections from PostgreSQL, then removes the best-effort Valkey registry entry. The liveness reaper is the database fallback when a close callback is lost. Structured workflow rows use the isolated-runner lease path below instead of daemon heartbeat identity. + +## Isolated workflow-runner lifecycle + +```mermaid +flowchart TD + Queue["Lease workflow-run queue item"]:::work + Claim{"PostgreSQL claim and capacity"}:::fork + Defer["Return exact queue item
without retry increment"]:::wait + Reconcile["Create or verify bare Pod,
then its owned Secret"]:::work + Register["Runner registers over WSS
with run and attempt capability"]:::work + Payload["Controller sends bounded input
and target-repo App token"]:::work + Run["Execute one handler and agent"]:::work + Renew{"Lease ACK remains current?"}:::fork + Fence["Abort and close Agent SDK query"]:::halt + Result["Retry terminal result until ACK"]:::wait + Store["Store result and terminal rows
before projections"]:::work + Project["Reconcile cascade, tracking,
outbox, and knowledge actions"]:::work + Cleanup["Request exact Pod and Secret
deletion with UID checks"]:::done + Recover["Periodic controller reconciliation"]:::wait + + Queue --> Claim + Claim -->|capacity full| Defer --> Queue + Claim -->|claimed| Reconcile --> Register --> Payload --> Run --> Renew + Renew -->|yes| Run + Renew -->|no or deadline| Fence --> Recover + Run --> Result --> Store + Store --> Project + Store --> Cleanup + Reconcile -. ambiguous API result .-> Recover --> Reconcile + Store -. controller crash before ACK .-> Recover + Recover --> Project + Recover --> Cleanup + + classDef work fill:#114a82,stroke:#0a2f56,color:#ffffff + classDef fork fill:#6a2080,stroke:#451454,color:#ffffff + classDef wait fill:#5c3d00,stroke:#3d2900,color:#ffffff + classDef halt fill:#852020,stroke:#5a1414,color:#ffffff + classDef done fill:#2a6f2a,stroke:#1a4d1a,color:#ffffff +``` + +The attempt Pod references exactly one selected provider credential chain from `workflow-runner-secrets` and a separate Secret containing only its HMAC capability. The controller signs that deadline-bound capability with `WORKFLOW_RUNNER_CAPABILITY_SECRET`, which is never mounted on a shared daemon or runner, and owns the Secret with the exact Pod UID. Provider settings are inline and unused or cross-provider credentials are omitted. The target-repository installation token crosses the first WSS registration only after its GitHub-reported expiry is proven no later than the immutable attempt deadline. Reconnects carry no job payload or repository credential. The runner receives no App private key, PAT, database, Valkey, Kubernetes, Context7, global GitHub, or fleet-wide daemon credential. If `GITHUB_PERSONAL_ACCESS_TOKEN` is configured, structured workflow dispatch fails closed; legacy and scoped jobs are unchanged. + +The runner renews a PostgreSQL lease through heartbeats. Missing lease ACK, token deadline, explicit cancellation, or the 4,200-second Pod deadline stops local execution. The Pod uses `restartPolicy: Never` because the durable payload receipt prohibits credential reissue after a process crash. Catchable exits and controller-owned token paths attempt best-effort revocation with a ten-second GitHub API timeout, then continue terminal handling even if revocation fails. The exact repository token is not persisted, so a failed revocation, SIGKILL, or node loss leaves a repository-scoped residual until its authoritative expiry. The controller stores the terminal payload and both terminal rows before ACK. Projection and exact resource cleanup then reconcile independently from PostgreSQL, so a failing GitHub projection does not retain runner credentials or compute. GitHub API calls and git pushes remain at-least-once: after an interrupted attempt, inspect the repository before manually re-triggering it. + +One controller replica is the supported topology. The database capacity transaction is safe against duplicate queue items, but there is no distributed controller-session owner or multi-replica admission semaphore. + ## Operational knobs The full list lives at [`../configuration.md`](../configuration.md#orchestrator-and-daemon). The handful you'll actually touch: -| Variable | Default | Notes | -| ---------------------------------- | -------- | ---------------------------------------------------------------------------------- | -| `ORCHESTRATOR_URL` | _none_ | Required. `wss://` in production; `ws://` emits a warning. | -| `DAEMON_AUTH_TOKEN` | _none_ | Shared secret with the orchestrator. | -| `DAEMON_EPHEMERAL` | `false` | `true` on ephemeral daemon Pods (injected by the spawner). Enables idle-exit. | -| `EPHEMERAL_DAEMON_IDLE_TIMEOUT_MS` | `120000` | Ephemeral daemons exit after this idle window. | -| `HEARTBEAT_INTERVAL_MS` | `30000` | Ping cadence. | -| `HEARTBEAT_TIMEOUT_MS` | `90000` | Orchestrator eviction threshold. Keep `≥ 2 × HEARTBEAT_INTERVAL_MS`. | -| `DAEMON_DRAIN_TIMEOUT_MS` | `300000` | Post-`SIGTERM` grace. Raise to `≥ AGENT_TIMEOUT_MS` to guarantee no mid-run kills. | -| `DAEMON_MEMORY_FLOOR_MB` | `512` | Below this, the orchestrator skips the daemon on dispatch. | -| `DAEMON_DISK_FLOOR_MB` | `1024` | Same, for free disk. | +| Variable | Default | Notes | +| -------------------------------------------- | -------------------- | ---------------------------------------------------------------------------------------------------------- | +| `ORCHESTRATOR_URL` | _none_ | Required. `wss://` in production; `ws://` emits a warning. | +| `DAEMON_AUTH_TOKEN` | _none_ | Shared secret with the orchestrator. | +| `WORKFLOW_RUNNER_CAPABILITY_SECRET` | _none_ | Controller-only root for expiring per-attempt capabilities. Minimum 32 characters. | +| `WORKFLOW_RUNNER_CAPABILITY_SECRET_PREVIOUS` | _none_ | Optional controller-only predecessor during capability rotation. | +| `DAEMON_EPHEMERAL` | `false` | `true` on ephemeral daemon Pods (injected by the spawner). Enables idle-exit. | +| `EPHEMERAL_DAEMON_IDLE_TIMEOUT_MS` | `120000` | Ephemeral daemons exit after this idle window. | +| `EPHEMERAL_DAEMON_NAMESPACE` | `default` | Namespace for shared ephemeral daemon Pods. | +| `WORKFLOW_RUNNER_NAMESPACE` | `github-app-runners` | Dedicated namespace for isolated runner Pods and Secrets. It must differ from the shared-daemon namespace. | +| `WORKFLOW_DISPATCH_TIMEOUT_MS` | `4200000` | Fails an unclaimed queued workflow and releases its lock after this age. | +| `HEARTBEAT_INTERVAL_MS` | `30000` | Ping cadence. | +| `HEARTBEAT_TIMEOUT_MS` | `90000` | Orchestrator eviction threshold. Keep `≥ 2 × HEARTBEAT_INTERVAL_MS`. | +| `DAEMON_DRAIN_TIMEOUT_MS` | `300000` | Post-`SIGTERM` grace. Raise to `≥ AGENT_TIMEOUT_MS` to guarantee no mid-run kills. | +| `DAEMON_MEMORY_FLOOR_MB` | `512` | Below this, the orchestrator skips the daemon on dispatch. | +| `DAEMON_DISK_FLOOR_MB` | `1024` | Same, for free disk. | ## Persistent daemon Deployment @@ -83,9 +135,24 @@ spec: app: github-app-daemon spec: terminationGracePeriodSeconds: 300 + automountServiceAccountToken: false + enableServiceLinks: false + hostIPC: false + hostNetwork: false + hostPID: false + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + seccompProfile: + type: RuntimeDefault containers: - name: daemon image: chrisleekr/github-app:latest-daemon + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] envFrom: - secretRef: name: daemon-secrets @@ -123,6 +190,39 @@ On every event the orchestrator evaluates: - `AGENT_TIMEOUT_MS` must stay below the GitHub installation-token TTL (3600 s) so the daemon cannot outlive its credentials. - `EPHEMERAL_DAEMON_IDLE_TIMEOUT_MS` should be longer than typical heartbeat cadence so a short lull between back-to-back jobs does not cause a premature exit. - `terminationGracePeriodSeconds` on the daemon Pod should match `DAEMON_DRAIN_TIMEOUT_MS`. +- Workflow runners require GitHub App mode, a digest-pinned `DAEMON_IMAGE`, an `ORCHESTRATOR_PUBLIC_URL` that is `wss://` or a cluster-local `ws://` Service name, a controller-only `WORKFLOW_RUNNER_CAPABILITY_SECRET` that differs from both daemon-auth slots, a dedicated `WORKFLOW_RUNNER_NAMESPACE`, controller RBAC for Pods and Secrets there, one selected credential chain in `workflow-runner-secrets`, dedicated runner nodes carrying the configured `WORKFLOW_RUNNER_NODE_LABEL` label and matching taint, and the egress controls in [`deployment.md`](../deployment.md#workflow-runner-egress-boundary). +- Install the fail-closed ValidatingAdmissionPolicy from [`examples/workflow-runner-admission.yaml`](https://github.com/chrisleekr/github-app/blob/main/examples/workflow-runner-admission.yaml), or set `workflowRunner.enabled` in the Helm chart, which packages the same policy. Require a negative server-side dry-run canary to name that policy before enabling workflows. It prevents a mutated image, lifecycle hook, container override, placement override, or extra container from starting with runner credentials. Controller reconciliation is the second check. +- All three cloud metadata probes must receive no HTTP response before the runner registers. A policy drop normally surfaces as a timeout, so timeout or connection refusal is accepted as corroboration; any HTTP response is a deployment failure. +- Every eligible runner node must enforce a finite positive kubelet `podPidsLimit` plus system and Kubernetes PID reservations. See [`deployment.md`](../deployment.md#runner-node-pid-boundary). +- Treat ephemeral-storage limits as eviction thresholds, not filesystem quotas. Keep control-plane and application workloads off the dedicated runner node pool so a deleted-open-file attack cannot exhaust their disks. + +## Workflow-runner rollout + +Migration 017 can recover reconstructable queued rows, but it cannot safely adopt arbitrary pre-lease work already executing in a shared daemon. Use a zero-in-flight cutover: + +1. Stop new GitHub webhook traffic while leaving the existing controller and daemon fleet running so current work can drain. +2. Wait until both queries return zero: + + ```sql + SELECT count(*) + FROM executions + WHERE status IN ('offered', 'running'); + + SELECT count(*) + FROM workflow_runs + WHERE status IN ('queued', 'running'); + ``` + +3. Gracefully stop the old controller and wait for the process to exit. This quiesces the ship tickle scheduler, scheduled actions, proposal poller, queue worker, and webhook listener. A webhook-only drain is insufficient because a due ship continuation can create new workflow work without a webhook. +4. Rerun both drain queries after the old controller is fully stopped. This is the cutover gate: no old-code producer may run between this final zero result and migration 017. If either query is non-zero, do not migrate. Restore the old version, drain or inspect that work, and repeat the stop plus final-query gate. +5. Confirm App mode is active, `GITHUB_PERSONAL_ACCESS_TOKEN` is absent, `WORKFLOW_RUNNER_CAPABILITY_SECRET` is controller-only, `DAEMON_IMAGE` ends in the tested daemon-image `@sha256:`, `ORCHESTRATOR_PUBLIC_URL` is WSS, and the RBAC, Secret, ValidatingAdmissionPolicy, ResourceQuota, and egress resources from [`../deployment.md`](../deployment.md#kubernetes-worker-requirements) exist. Require the negative admission canary to be denied by name; policy type-check completion alone is not the rollout gate. +6. Deploy migration 017 and exactly one controller replica. Do not scale horizontally. +7. Trigger one structured workflow. Verify one attempt Pod appears before its Secret, the Secret has one owner reference to that exact Pod UID, all three metadata probes receive no HTTP response, the row lease renews without moving `attempt_deadline_at`, and both terminal rows are stored. The cleanup receipt may be set when UID-preconditioned deletes are accepted; separately verify both objects eventually become absent. Projection may complete before or after cleanup. +8. Restore webhook traffic. + +If either drain query is non-zero, do not roll. Determine whether the work is still active or terminalize it through the existing owner path first. Do not edit lease or generation columns by hand. + +Migration 017 still fails closed if this gate is violated: it terminalizes an active shared-daemon workflow or an unreconstructable queued workflow, releases its lock, and records a retryable `migration-interrupted` public projection. That recovery behavior does not make an in-flight rollout safe, because repository operations may already have completed. ## Rotating `DAEMON_AUTH_TOKEN` @@ -148,25 +248,42 @@ sequenceDiagram Step-by-step: 1. **Generate** a new secret: `openssl rand -hex 32`. Persist it in your secret store next to the existing value. -2. **Stage the overlap.** Update the orchestrator Deployment's `daemon-secrets` to set `DAEMON_AUTH_TOKEN=` **and** `DAEMON_AUTH_TOKEN_PREVIOUS=`. Roll the orchestrator. Existing daemon connections (still presenting the old token) keep authenticating, and any daemons that come up with the new token also pass. +2. **Stage the overlap.** Update the controller Secret to set `DAEMON_AUTH_TOKEN=` **and** `DAEMON_AUTH_TOKEN_PREVIOUS=`. Roll the controller. Existing daemon connections using the old secret keep authenticating, and new daemons use the new primary. 3. **Roll daemons.** Update the daemon Deployment's `daemon-secrets` to set `DAEMON_AUTH_TOKEN=` (no `_PREVIOUS` needed, daemons only ever send the primary). Roll daemons one by one (`kubectl rollout restart deployment/github-app-daemon`). Watch `auth-failed` warn-logs in the orchestrator, they should stay flat. -4. **Drop the previous slot.** Once every connected daemon presents the new token, redeploy the orchestrator with `DAEMON_AUTH_TOKEN_PREVIOUS` removed (or empty). The old secret is now dead. +4. **Drop the previous slot.** Once every connected daemon presents the new token, redeploy the controller with `DAEMON_AUTH_TOKEN_PREVIOUS` removed. 5. **Verify.** A `curl` with the old Bearer should now return `401`; a curl with the new Bearer should hit the upgrade-failed path (`500 WebSocket upgrade failed`). Operational notes: - The previous-token slot is **orchestrator-only**. Daemons always send the value of their own `DAEMON_AUTH_TOKEN`; setting `DAEMON_AUTH_TOKEN_PREVIOUS` on a daemon Pod has no effect. -- Keep the overlap window short (hours, not days). The longer two tokens authenticate, the longer a leaked old token remains usable. +- Keep the overlap no longer than needed for the daemon rollout. The longer two tokens authenticate, the longer a leaked old token remains usable. - The rotation does **not** require restarting daemons simultaneously, but you do need to redeploy the orchestrator twice (once to add `_PREVIOUS`, once to remove it). +## Rotating `WORKFLOW_RUNNER_CAPABILITY_SECRET` + +Rotate the isolated-runner HMAC root independently from `DAEMON_AUTH_TOKEN`: + +1. Generate a new root with at least 32 random bytes and store it beside the current value. +2. Deploy the controller with `WORKFLOW_RUNNER_CAPABILITY_SECRET=` and `WORKFLOW_RUNNER_CAPABILITY_SECRET_PREVIOUS=`. New capabilities use `NEW`; unexpired capabilities signed with `OLD` remain valid. +3. Keep the previous slot until every runner capability minted before the rotation has reached its signed expiry and no pre-rotation runner Pod remains. +4. Redeploy without `WORKFLOW_RUNNER_CAPABILITY_SECRET_PREVIOUS`. Confirm an old capability is rejected and a newly created runner still registers. + +Never copy either capability root into `daemon-secrets`, `workflow-runner-secrets`, or a per-attempt Secret. The per-attempt Secret contains only the derived, expiring capability. + ## Common Day-2 issues -| Symptom | Likely cause | -| -------------------------------------------- | ------------------------------------------------------------------------------------------------------------- | -| Sustained heartbeat eviction | Daemon CPU starvation, network partition, or `HEARTBEAT_TIMEOUT_MS` too low. | -| `dispatch_reason=ephemeral-spawn-failed` | Missing RBAC on `pods` in `EPHEMERAL_DAEMON_NAMESPACE`, missing `daemon-secrets`, or a control-plane issue. | -| Mid-run kills on rolling deploys | `terminationGracePeriodSeconds` < `DAEMON_DRAIN_TIMEOUT_MS`. | -| `executions.status='running'` rows piling up | A daemon died abruptly. The `LIVENESS_REAPER_INTERVAL_MS` reaper flips them to failed; check daemon pod logs. | +| Symptom | Likely cause | +| --------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Sustained heartbeat eviction | Daemon CPU starvation, network partition, or `HEARTBEAT_TIMEOUT_MS` too low. | +| `dispatch_reason=ephemeral-spawn-failed` | Missing RBAC on `pods` in `EPHEMERAL_DAEMON_NAMESPACE`, missing `daemon-secrets`, or a control-plane issue. | +| Mid-run kills on rolling deploys | `terminationGracePeriodSeconds` < `DAEMON_DRAIN_TIMEOUT_MS`. | +| Shared `executions.status='running'` rows piling up | A daemon died abruptly. Its per-boot ID and liveness reaper should fail direct, scoped, and workflow-linked receipts for that incarnation; check controller and daemon logs. | +| Queue item remains in `queue:processing:` | The owner heartbeat may still be live, or Valkey recovery is failing. After the 60-second heartbeat expires, a liveness pass should return the exact item to `queue:jobs`. | +| Workflow lease or attempt deadline expired | Runner lost heartbeat ACK, reached the immutable attempt deadline, reached its token safety margin, or the controller was unavailable past the lease. Inspect GitHub state before retrying. | +| Workflow dispatch deadline or retries expired | Valkey publication or runner capacity stayed unavailable past `JOB_MAX_RETRIES` or `WORKFLOW_DISPATCH_TIMEOUT_MS`. The database fails the workflow and execution, releases the target lock, and retries its public failure projection. | +| `workflow_runner_output_scan_unavailable` | The runner RPC secret scanner was disabled, failed, or timed out. This path fails closed. Restore the configured scanner before retrying the workflow. | +| Workflow runner repeatedly fails to start | Missing Pod/Secret RBAC, provider Secret, WSS reachability, image guard, quota, or an admission mutation rejected by exact reconciliation. | +| Terminal runner row but Pod/Secret remains | Kubernetes accepted deletion but graceful termination or finalizers are still pending, or cleanup is retrying. Check the exact UID and node before intervening. Never automate zero-grace force deletion. | ## Diagnosing ephemeral-spawn failures by kind @@ -181,4 +298,4 @@ When `dispatch_reason=ephemeral-spawn-failed` rises, break it down with the `k8s ## Implementation references -`src/daemon/main.ts`, `src/orchestrator/ws-server.ts`, `src/orchestrator/ephemeral-daemon-scaler.ts`, `src/k8s/ephemeral-daemon-spawner.ts`, `src/core/pipeline.ts`, `src/shared/ws-messages.ts`. +`src/daemon/main.ts`, `src/runner/main.ts`, `src/orchestrator/ws-server.ts`, `src/orchestrator/workflow-runner-dispatch.ts`, `src/orchestrator/workflow-runner-result.ts`, `src/k8s/ephemeral-daemon-spawner.ts`, `src/k8s/workflow-runner-spawner.ts`, `src/core/pipeline.ts`, `src/shared/ws-messages.ts`, `src/shared/workflow-runner-messages.ts`. diff --git a/docs/operate/runbooks/scheduled-actions.md b/docs/operate/runbooks/scheduled-actions.md index 98062bb0..0154a53a 100644 --- a/docs/operate/runbooks/scheduled-actions.md +++ b/docs/operate/runbooks/scheduled-actions.md @@ -18,6 +18,14 @@ scheduler: ALLOWED_OWNERS is unset; ... not starting To disable a single action without touching the bot, set `enabled: false` on that action in the repo's `.github-app.yaml`. +To silence every action in one repo, set the **document-level** `enabled: false` +(top level, not inside an action). It short-circuits both the cron scan and the +manual endpoint, so a repo that opted out of the bot does not keep running +unattended cron work. The manual endpoint reports it as +`the bot is disabled for this repository`, distinct from the per-action +`action "" is disabled`. See +[Per-repo configuration](../../use/repo-config.md). + ## Force a run ```bash @@ -49,7 +57,7 @@ The per-action state lives in the `scheduled_action_state` table ## Stuck `in_flight_job_id` The single-flight lock is taken when a run is claimed. It is normally cleared -the moment the run completes (the scoped-job-completion handler), so a healthy +the moment the run completes (the `scoped-job:completion` handler), so a healthy run releases it immediately. As a backstop it is also **self-healing**: the claim treats a lock older than `2 × AGENT_TIMEOUT_MS` (always longer than the longest possible run) as released, so a daemon that died mid-run does not diff --git a/docs/use/invoking.md b/docs/use/invoking.md index 70607755..de7faa96 100644 --- a/docs/use/invoking.md +++ b/docs/use/invoking.md @@ -25,20 +25,22 @@ flowchart TD NL["Mention + NL classifier
Bedrock single-turn"]:::route LabelEvt["issues.labeled or
pull_request.labeled"]:::input LabelMatch["registry.getByLabel"]:::route - Enqueue["enqueueJob
Valkey queue:jobs"]:::store - Daemon["Daemon claims offer"]:::work + Commit["Commit workflow + execution
with open outbox receipt"]:::store + Enqueue["Publish workflow-run
to Valkey queue:jobs"]:::store + Runner["Controller claims lease
and starts one-attempt Pod"]:::work Pipe["src/core/pipeline.ts"]:::work - Track["Tracking comment finalised"]:::done + Durable["Store terminal result
before projection"]:::store + Track["Tracking comment reconciled"]:::done Cmt --> Verify --> Idem --> Allow --> Router Router --> Lit Router --> NL LabelEvt --> Allow Allow -. label path .-> LabelMatch - Lit --> Enqueue - NL --> Enqueue - LabelMatch --> Enqueue - Enqueue --> Daemon --> Pipe --> Track + Lit --> Commit + NL --> Commit + LabelMatch --> Commit + Commit --> Enqueue --> Runner --> Pipe --> Durable --> Track classDef input fill:#0b5cad,stroke:#083e74,color:#ffffff classDef guard fill:#164a3a,stroke:#0d2c24,color:#ffffff @@ -62,12 +64,12 @@ The legacy in-memory `Map` + tracking-comment marker scan was retired in issue # Comment-driven runs stack four reactions on your trigger comment so the lifecycle is visible at a glance: -| Stage | Reaction | -| -------------------------- | -------- | -| Trigger detected | 👀 | -| Job dispatched to a daemon | 🚀 | -| Workflow succeeded | 🎉 | -| Workflow failed | 😕 | +| Stage | Reaction | +| ------------------ | -------- | +| Trigger detected | 👀 | +| Workflow queued | 🚀 | +| Workflow succeeded | 🎉 | +| Workflow failed | 😕 | Reactions are additive: the combined set is the audit trail. Label-driven runs skip reactions because there is no comment to react on. diff --git a/docs/use/repo-config.md b/docs/use/repo-config.md index 4ea66368..2d3999d6 100644 --- a/docs/use/repo-config.md +++ b/docs/use/repo-config.md @@ -41,40 +41,25 @@ falls back to defaults. Nearly every block now also changes behaviour. run under their own workflow names (`triage`, `plan`, `implement`, `review`, `resolve`) and resolve `workflows..*` over `defaults:`. -| Block | Status | -| ------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | -| `enabled` | **Parsed and validated. Not yet enforced.** Enforcement is Gate 1, wired with the isolated workflow runner. | -| `workflows..enabled` | **Parsed and validated. Not yet enforced.** Same Gate 1 path. | -| `triggers.*` | **Parsed and validated. Not yet enforced.** Same Gate 1 path. | -| `review_learnings`, `scheduled_actions`, `config` | **Applied.** Pre-existing blocks, unchanged by this change. | -| `defaults` + `workflows.` agent knobs | **Resolved and clamped. Not yet on the wire.** The `policy` key exists on the job payload; its producer lands with the runner. | -| `workflows.review.path_filters` / `.instructions` | **Consumer ready. Not yet reachable**, since no producer sets `policy` yet. | -| `workflows.review.auto` | **Not yet applied.** Dispatch-time knob; lands with the auto-review guard. | - -!!! warning "Status of this page" - - Only `review_learnings`, `scheduled_actions` and `config` take effect today. - Everything else on this page is parsed, schema-validated, resolved and - clamped, but has no production call site yet: `checkRepoGate`, - `loadRepoPolicy` and `runPrConfigCheck` are reachable only from tests. The - dispatch chokepoints and the `pull_request` config-check handler that call - them depend on database columns from a later migration, so they ship in the - isolated-workflow-runner change rather than here. Authoring a config file - now is safe and its schema is stable, but do not expect a repo-level - `enabled: false` to stop the bot until that lands. +| Block | Status | +| ------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------- | +| `enabled` | **Applied.** Blocks every trigger for the repo. | +| `workflows..enabled` | **Applied.** Blocks that workflow's triggers. | +| `triggers.*` | **Applied.** Four everywhere, `base_branches` on label and review-comment triggers only (see the caveat under `triggers`). | +| `review_learnings`, `scheduled_actions`, `config` | **Applied.** Pre-existing blocks, unchanged. | +| `defaults` + `workflows.` agent knobs | **Applied.** `model`, `max_turns`, `timeout`, and `extra_allowed_tools` reach the agent run. `workflows.ship` takes none. | +| `workflows.review.path_filters` / `.instructions` | **Applied.** Filtered files are hidden from the prompt; instructions are injected as review policy. | +| `workflows.review.auto` | **Applied.** Runs `review` on a push by an `AUTO_REVIEW_USERS` login. Defaults to `false`; both keys are required. | ### How the agent knobs behave -Resolution is owned by the controller: it merges the workflow block over -`defaults`, clamps the result against the server ceilings, and ships it on the -job payload as a `policy` object. `AgentPolicySchema` in -`src/shared/ws-messages.ts` defines that wire shape, and -`src/core/agent-policy.ts` is the consumer that applies it to an agent run. - -The producing side is not in place yet, so no `policy` key is sent today and -the table below describes intended behaviour rather than current behaviour. A -repo with no config file produces no `policy` key at all and runs exactly as it -did before this file existed. +Resolved once during controller-owned payload preparation for an isolated +workflow runner (`src/orchestrator/workflow-runner-payload.ts`), or when a +shared daemon accepts a legacy direct job (`src/orchestrator/connection-handler.ts`). +The controller merges the workflow block over `defaults`, clamps it against the +server ceilings, and ships it on the job payload as a `policy` object. A repo +with no config file produces no `policy` key at all and runs exactly as it did +before this file existed. | Knob | Effect | | --------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | @@ -242,15 +227,8 @@ only declares `version: 1` and `scheduled_actions:` stays valid. | `review_learnings` | object | on | See [Review learnings](review-learnings.md). | | `scheduled_actions` | array | `[]` | See [Scheduled actions](scheduled-actions.md). | -!!! warning "Not enforced yet" - - Gate 1 has no production call site on this change, so `enabled: false` does - not stop label or mention triggers today. It already silences the scheduler, - which reads the document directly. The rest of this section describes the - behaviour once Gate 1 is wired. - -`enabled: false` will stop the bot doing any work in the repo: no workflow run, -no queue job, no scheduled action. It is not a vow of silence. A deliberate `bot:*` +`enabled: false` stops the bot doing any work in the repo: no workflow run, no +queue job, no scheduled action. It is not a vow of silence. A deliberate `bot:*` label or `@chrisleekr-bot` mention still gets one short reply saying the bot is disabled here, so a teammate who tries is told why instead of being ignored. Passive triggers stay silent. If you need the bot to make no writes at all, @@ -561,13 +539,6 @@ appended: - **too large to validate**: files over 64 KB are never decoded, and none of the file's contents are echoed back. -!!! warning "Not wired yet" - - `runPrConfigCheck` has no production caller on this change, so no verdict - comment is posted on a pull request today. The handler that invokes it - ships with the isolated workflow runner. This section describes the - behaviour once that lands. - Every verdict restates that only the default-branch copy is applied, so the change takes effect on merge. Reading the branch copy here is strictly read-only: it never becomes the policy the bot enforces, and it never enters the diff --git a/docs/use/scheduled-actions.md b/docs/use/scheduled-actions.md index 632a0d48..af235a68 100644 --- a/docs/use/scheduled-actions.md +++ b/docs/use/scheduled-actions.md @@ -17,10 +17,20 @@ Scheduled actions run only when the operator has set: - a non-empty `ALLOWED_OWNERS` (the action prompt is owner-trusted config, so the feature refuses to start without an owner allowlist) +Repo-side, the file's `enabled` key is a master switch: `enabled: false` stops +scheduled actions along with every other trigger, regardless of a per-action +`enabled: true`. An unattended cron run writes to the repo with nobody watching, +so turning the bot off has to silence it too. + See [Configuration](../operate/configuration.md#scheduled-actions). ## `.github-app.yaml` +Scheduled actions are one block in a larger per-repo config file. See +[Repo configuration](repo-config.md) for the whole file, including the feature +toggles and trigger filters; the filename comes from `REPO_CONFIG_FILE` +(deprecated alias: `SCHEDULER_CONFIG_FILE`). + Place the file at your repo's **default-branch root**. ```yaml diff --git a/docs/use/workflows/implement.md b/docs/use/workflows/implement.md index 03f554ca..419905fd 100644 --- a/docs/use/workflows/implement.md +++ b/docs/use/workflows/implement.md @@ -50,4 +50,4 @@ The handler does **not** poll CI or reviewer state: that is `resolve`'s job, aft The handler treats public and operator surfaces separately so the public tracking comment never carries the raw underlying error. Both the `runPipeline` failure path and the outer handler `catch` set the same safe `humanMessage`: - **Public tracking comment**: a safe constant: `"implement pipeline execution failed, see server logs for details."` Octokit error stacks embed the installation token in the request URL, so the bot must never inline `err.message` into a comment body. -- **Operator surfaces (DB + logs)**: the SDK error is propagated as `ExecutionResult.errorMessage` and persisted as `state.failedReason` on the `workflow_runs` row. `pino` logs the full `err` object on the daemon. The orchestrator's quota-detection helper reads `state.failedReason` to decide whether to auto-defer the next ship iteration; see [`ship.md`](./ship.md). +- **Operator surfaces (DB + logs)**: the sanitized SDK reason is persisted as `state.failedReason` on the `workflow_runs` row, with runner and controller `pino` lines. The orchestrator's quota-detection helper reads `state.failedReason` to decide whether to auto-defer the next ship iteration; see [`ship.md`](./ship.md). diff --git a/docs/use/workflows/index.md b/docs/use/workflows/index.md index 5a6775ca..b1b42952 100644 --- a/docs/use/workflows/index.md +++ b/docs/use/workflows/index.md @@ -51,7 +51,15 @@ The classifier prompt distinguishes `review` (proactive, find bugs, post inline ## Label-path dispatch -Both the label trigger and the in-registry classifier verdict run the same seven-step sequence in `src/workflows/dispatcher.ts`: registry lookup → context check → prior-output requirement → label mutex → idempotency insert → job enqueue → return. Prior-output is checked before the mutex, so refusing a workflow that lacks its prerequisite (e.g. `bot:implement` before any `bot:plan`) does not strip unrelated `bot:*` labels. The idempotency insert is the durable in-flight guard: a redelivered or concurrent label event for the same workflow and target is rejected at the database, not just at the best-effort Valkey claim. +Both the label trigger and the in-registry classifier verdict run the same durable sequence in `src/workflows/dispatcher.ts`: registry lookup → [repo-config gate](../repo-config.md) → context check → prior-output requirement → label mutex → atomic `workflow_runs` plus `executions` insert → outbox publication → return. The repo-config gate runs second, immediately after the registry lookup, so a workflow disabled in `.github-app.yaml` leaves no run row, no label mutation, and no queue job. Prior-output is checked before the mutex, so refusing a workflow that lacks its prerequisite (e.g. `bot:implement` before any `bot:plan`) does not strip unrelated `bot:*` labels. The partial unique index is the durable in-flight guard: a redelivered or concurrent trigger for the same workflow and target is rejected by PostgreSQL, not just by the best-effort Valkey delivery claim. Queue publication happens only after the transaction commits. The reaper repairs both a null receipt from a failed publish and a stale receipt whose acknowledged Valkey item was later lost, while an atomic membership check keeps at most one stable copy across the queue and current processing list. + +## Per-workflow execution knobs + +The dispatch gate above only decides whether a workflow runs. The per-workflow `model`, `max_turns`, `timeout`, `extra_allowed_tools`, and (for `review`) `path_filters` / `instructions` knobs are a second gate. Structured workflows resolve it during controller-owned runner-payload preparation in `src/orchestrator/workflow-runner-payload.ts`; legacy direct jobs resolve it at shared-daemon accept in `src/orchestrator/connection-handler.ts`. Both clamp against the server-side env ceilings, so a repo can narrow the defaults but never exceed them. + +Six workflows honour those knobs: `review`, `resolve`, `implement`, `remember`, `plan`, and `triage`. The first four apply them through `runPipeline`; `plan` and `triage` own their prompts and bypass the pipeline, so they apply the same resolved values through the shared helper `src/core/agent-policy.ts` instead. `path_filters` / `instructions` remain pipeline-only, since both need fetched PR data and the prompt builder. + +`workflows.ship` accepts only `enabled`. Its handler is a composite orchestrator that enqueues the child workflows and never invokes an agent, so agent knobs there would be a permanent no-op; the children run under their own workflow names and resolve `workflows..*` over `defaults:`. See [Per-repo configuration](../repo-config.md) for the full schema and clamping rules. ## Conversational `chat-thread` (sub-threshold fallback) diff --git a/docs/use/workflows/plan.md b/docs/use/workflows/plan.md index 8cd70498..dda6ef96 100644 --- a/docs/use/workflows/plan.md +++ b/docs/use/workflows/plan.md @@ -31,7 +31,7 @@ When `PROMPT_CACHE_LAYOUT=cacheable`, the plan prompt is split: the static role ## Stop conditions -The agent writes `PLAN.md`; the pipeline reports success or failure. No turn cap: the agent runs to completion. +The agent writes `PLAN.md`; the pipeline reports success or failure. There is no _default_ turn cap, so the agent runs to completion. Three sources can bind one, and all three are honoured: the repo's `workflows.plan.max_turns` in `.github-app.yaml`, the operator's `AGENT_MAX_TURNS`, and `DEFAULT_MAXTURNS`. With all three unset, which is the stock deployment, the run is uncapped as before. ## Re-trigger semantics diff --git a/docs/use/workflows/resolve.md b/docs/use/workflows/resolve.md index 99ddcfa7..105797ea 100644 --- a/docs/use/workflows/resolve.md +++ b/docs/use/workflows/resolve.md @@ -74,9 +74,9 @@ The shared definition lives in `src/workflows/handlers/checks.ts` so the prologu `incomplete` is a fourth `HandlerResult` variant ("agent ran cleanly but work remains") distinct from `succeeded` / `failed` / `handed-off`: - The DB `workflow_runs.status` column accepts `incomplete` (migration `009_workflow_runs_incomplete.sql`). -- `runs-store.markIncomplete(runId, reason, state)` mirrors `markFailed`, persisting `state.incompleteReason`. -- The daemon executor (`src/daemon/workflow-executor.ts`) reacts `confused` on the trigger comment, mirrors the handler's `humanMessage`, sends `job:result` with `success: false` and an `incomplete:`-prefixed `errorMessage`. -- The orchestrator cascade keeps its binary `succeeded | failed` contract: the executor maps `incomplete` → `failed` for cascade purposes, but the parent's tracking-comment headline reads "ship halted at step N (... → resolve), resolve returned incomplete; see PR tracking comment for outstanding items." instead of the generic failure message. +- The attempt-fenced terminal write persists `status='incomplete'` and `state.incompleteReason` only while the exact run, attempt, owner, lease, and immutable deadline remain current. +- The controller stores the runner's terminal result and marks both durable rows before any GitHub projection. Result reconciliation then reacts `confused` on the trigger comment and mirrors the handler's `humanMessage`. +- The orchestrator cascade keeps its binary `succeeded | failed` contract: `completion-reconciler.ts` maps `incomplete` to a failed completion with an internal `incomplete:` marker, but the parent's tracking-comment headline reads "ship halted at step N (... → resolve), resolve returned incomplete; see PR tracking comment for outstanding items." instead of the generic failure message. ## Stop conditions @@ -89,7 +89,7 @@ The shared definition lives in `src/workflows/handlers/checks.ts` so the prologu Public-comment and operator surfaces are separated so a raw SDK or octokit error never reaches the public PR thread. Both the `runPipeline` failure path and the outer handler `catch` apply the same separation: - **Public tracking comment**: a safe constant: `"resolve pipeline execution failed, see server logs for details."` The actual error string remains internal because octokit error stacks include `https://x-access-token:GHS_xxx@…` in the request URL. -- **Operator surfaces**: `state.failedReason` on the `workflow_runs` row, `pino` log lines on the daemon, and `ExecutionResult.errorMessage` returned to the orchestrator. The orchestrator's transient-quota detector reads `state.failedReason` and auto-defers the ship loop's next iteration when the SDK reports `"You've hit your limit · resets … UTC"`. +- **Operator surfaces**: `state.failedReason` on the `workflow_runs` row plus runner and controller `pino` lines. The controller applies the credential-output boundary before persisting the runner's SDK reason. The transient-quota detector reads `state.failedReason` and auto-defers the ship loop's next iteration when the SDK reports `"You've hit your limit · resets … UTC"`. ## Review learnings diff --git a/docs/use/workflows/review.md b/docs/use/workflows/review.md index 7748882a..6865ba58 100644 --- a/docs/use/workflows/review.md +++ b/docs/use/workflows/review.md @@ -2,15 +2,37 @@ Reads a PR diff in full, cross-references with the rest of the codebase, and posts findings as inline comments. -| Field | Value | -| --------------- | ------------------------------------------------------------------------------------------------------------------------------------- | -| Label | `bot:review` | -| Mention | `@chrisleekr-bot review this PR` · `@chrisleekr-bot do a code review` · `@chrisleekr-bot check for issues` | -| Accepted target | Pull request | -| Requires prior | _none_ | -| Artifact | `$BOT_ARTIFACT_DIR/REVIEW.md` (sibling temp dir, never committed to the repo) | -| Side effects | Inline review comments via `mcp__github_inline_comment__create_inline_comment`; force-push of a clean rebase if branch is behind base | -| Source | `src/workflows/handlers/review.ts` | +| Field | Value | +| --------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Label | `bot:review` | +| Mention | `@chrisleekr-bot review this PR` · `@chrisleekr-bot do a code review` · `@chrisleekr-bot check for issues` | +| Auto-trigger | A push to an open PR by an `AUTO_REVIEW_USERS` login, when the repo sets `workflows.review.auto: true`. See [Auto review](../repo-config.md#auto-review) | +| Accepted target | Pull request | +| Requires prior | _none_ | +| Artifact | `$BOT_ARTIFACT_DIR/REVIEW.md` (sibling temp dir, never committed to the repo) | +| Side effects | Inline review comments via `mcp__github_inline_comment__create_inline_comment` | +| Source | `src/workflows/handlers/review.ts` | + +## Duplicate findings + +Each finding is posted as its own review thread, and re-running the workflow does +not repost a finding that is still open: a new comment is skipped when this bot +already has a live comment on the same file, line, and diff side. + +Matching is by location, not wording, because the agent rewords the same finding +between runs. Two consequences worth knowing: + +- What happens once the code around a finding moves is deliberately not relied + on. The REST reference documents no nulling rule for a review comment's `line`, + so the finding is either re-reported on its new line or stays suppressed while + that thread lives. Settling it means switching the check to GraphQL + `PullRequestReviewThread.isOutdated`. +- A genuinely _different_ second finding on an already-commented line is + suppressed. The prompt already asks for one comment per finding on the most + relevant line, so this is rare. + +This matters most under [auto review](../repo-config.md#auto-review), where the +workflow can run on every push. ## Method @@ -62,8 +84,12 @@ The only push acceptable from `review` is `git push --force-with-lease` after a Every failure path (`runPipeline` failure, the outer handler catch, or any sync/async throw before the pipeline runs) returns `status: "failed"` with two distinct outputs: - **Public tracking comment**: a safe constant: `"review pipeline execution failed, see server logs for details."` Never carries the raw error string, since octokit error stacks include the request URL with the installation token (`https://x-access-token:GHS_xxx@…`). -- **Operator surfaces (DB + logs)**: `state.failedReason` on the `workflow_runs` row, `pino` log line on the daemon, and `ExecutionResult.errorMessage` returned to the caller. These carry the full SDK message so an operator can diagnose without tailing daemon stderr. +- **Operator surfaces (DB + logs)**: `state.failedReason` on the `workflow_runs` row plus runner and controller `pino` lines. The controller applies the credential-output boundary before persisting the runner's SDK reason. ## Review learnings `review` is one of two workflows (with `resolve`) that loads persisted review-policy directives from the `review_learnings` table and renders them into the prompt. Loaded directives whose `file_glob` matches at least one changed file are surfaced verbatim in a `` block ahead of the diff, and their IDs flow back through `appliedReviewLearningIds` on the handler result so the orchestrator can bump `use_count` precisely. The tracking comment ends with a `🧠 Learnings used` collapsible footer when at least one directive applied. See `docs/use/review-learnings.md` for the full feature, gating (`REVIEW_LEARNINGS_ENABLED`, per-repo `.github-app.yaml` opt-out), and operator notes. + +## Per-repo review policy + +A repo's `.github-app.yaml` can shape this workflow through two `workflows.review` keys: `instructions` (owner-trusted review policy injected into the prompt, overriding the agent's default heuristics) and `path_filters` (changed files matching a glob are dropped from the prompt the reviewer sees). Both are resolved during controller-owned runner payload preparation, and `path_filters` is advisory prose only, not an access boundary. See [Per-repo configuration](../repo-config.md) for the schema, ceilings, and trust model. diff --git a/docs/use/workflows/triage.md b/docs/use/workflows/triage.md index 62fe1c07..21abfe50 100644 --- a/docs/use/workflows/triage.md +++ b/docs/use/workflows/triage.md @@ -49,4 +49,4 @@ When `PROMPT_CACHE_LAYOUT=cacheable`, the triage prompt is split: the static rol - `valid = false` → handler returns `failed` and any composite cascade halts here. - Missing markdown, malformed JSON, or an SDK error → `failed` with a specific reason. -There is no turn cap on triage: the agent runs until the verdict is honestly defensible. +There is no _default_ turn cap on triage: the agent runs until the verdict is honestly defensible. Three sources can bind one, and all three are honoured: the repo's `workflows.triage.max_turns` in `.github-app.yaml`, the operator's `AGENT_MAX_TURNS`, and `DEFAULT_MAXTURNS`. With all three unset, which is the stock deployment, the run is uncapped as before. diff --git a/env-contract.json b/env-contract.json index 83f260c0..319164b9 100644 --- a/env-contract.json +++ b/env-contract.json @@ -164,6 +164,11 @@ "group": "Group 5", "kind": "config" }, + { + "env": "AUTO_REVIEW_USERS", + "group": "Group 5", + "kind": "config" + }, { "env": "VALKEY_URL", "group": "Group 6", @@ -189,6 +194,16 @@ "group": "Group 8", "kind": "secret" }, + { + "env": "WORKFLOW_RUNNER_CAPABILITY_SECRET", + "group": "Group 8", + "kind": "secret" + }, + { + "env": "WORKFLOW_RUNNER_CAPABILITY_SECRET_PREVIOUS", + "group": "Group 8", + "kind": "secret" + }, { "env": "ORCHESTRATOR_URL", "group": "Group 8", @@ -249,6 +264,11 @@ "group": "Group 8", "kind": "config" }, + { + "env": "WORKFLOW_DISPATCH_TIMEOUT_MS", + "group": "Group 8", + "kind": "config" + }, { "env": "OFFER_TIMEOUT_MS", "group": "Group 8", @@ -289,6 +309,11 @@ "group": "Group 9", "kind": "config" }, + { + "env": "WORKFLOW_RUNNER", + "group": "Group 9", + "kind": "config" + }, { "env": "EPHEMERAL_DAEMON_IDLE_TIMEOUT_MS", "group": "Group 9", @@ -309,6 +334,31 @@ "group": "Group 9", "kind": "config" }, + { + "env": "EPHEMERAL_DAEMON_SECRET_NAME", + "group": "Group 9", + "kind": "config" + }, + { + "env": "WORKFLOW_RUNNER_NAMESPACE", + "group": "Group 9", + "kind": "config" + }, + { + "env": "WORKFLOW_RUNNER_NODE_LABEL", + "group": "Group 9", + "kind": "config" + }, + { + "env": "WORKFLOW_RUNNER_NODE_VALUE", + "group": "Group 9", + "kind": "config" + }, + { + "env": "WORKFLOW_RUNNER_IMAGE_PULL_SECRET", + "group": "Group 9", + "kind": "config" + }, { "env": "DAEMON_IMAGE", "group": "Group 9", diff --git a/examples/scheduled-actions/research.md b/examples/scheduled-actions/research.md index d9006394..c556220b 100644 --- a/examples/scheduled-actions/research.md +++ b/examples/scheduled-actions/research.md @@ -67,7 +67,7 @@ Read the key files for the focus area: | idempotency | src/webhook/router.ts, src/core/tracking-comment.ts | | security | src/utils/, src/config.ts | | observability | src/logger.ts (and grep for logger usage across src/) | -| testing | src/\*\*/\*.test.ts (sample 3-5; do not read all) | +| testing | test/\*\*/\*.test.ts (sample 3-5; do not read all) | | docs | CLAUDE.md, README.md, docs/ | | infrastructure | .github/workflows/, Dockerfile.\*, package.json | | agent-sdk | src/core/prompt-builder.ts, src/core/executor.ts | diff --git a/examples/workflow-runner-admission.yaml b/examples/workflow-runner-admission.yaml new file mode 100644 index 00000000..a19e356a --- /dev/null +++ b/examples/workflow-runner-admission.yaml @@ -0,0 +1,438 @@ +# Canonical workflow-runner admission boundary. +# +# This file is the source of truth for the ValidatingAdmissionPolicy. It is +# derived from buildWorkflowRunnerPod in src/k8s/workflow-runner-spawner.ts and +# verified against that renderer by `bun run test:admission`, which installs it in +# a disposable Kubernetes 1.30 cluster and asserts the exact Pod is admitted and +# every prohibited mutation is denied. +# +# The github-app Helm chart in chrisleekr/helm-charts packages these resources +# behind workflowRunner.enabled and is the recommended way to install them. It +# carries the policy spec verbatim, and its check-policy-parity.sh gate fails when +# its copy drifts from the version published here at the chart's appVersion. +# +# So: changing the policy below means the chart needs the same change. Until the +# next release tag is cut, that gate cannot see this file and will skip, so pair +# any edit here with a chart update rather than relying on CI to catch it. + +apiVersion: v1 +kind: Namespace +metadata: + name: github-app-runners + labels: + pod-security.kubernetes.io/enforce: restricted + pod-security.kubernetes.io/enforce-version: v1.30 + pod-security.kubernetes.io/audit: restricted + pod-security.kubernetes.io/audit-version: v1.30 + pod-security.kubernetes.io/warn: restricted + pod-security.kubernetes.io/warn-version: v1.30 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: github-app-workflow-runner-egress-boundary + namespace: github-app-runners +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: github-app + app.kubernetes.io/component: workflow-runner + policyTypes: + - Egress + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + - to: + - namespaceSelector: + matchLabels: + github-app.chrislee.kr/workflow-controller: "true" + podSelector: + matchLabels: + app.kubernetes.io/name: github-app + app.kubernetes.io/component: orchestrator + ports: + - protocol: TCP + port: 3002 + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: + - 0.0.0.0/8 + - 10.0.0.0/8 + - 100.64.0.0/10 + - 127.0.0.0/8 + - 169.254.0.0/16 + - 172.16.0.0/12 + - 192.0.0.0/24 + - 192.0.2.0/24 + - 192.168.0.0/16 + - 198.18.0.0/15 + - 198.51.100.0/24 + - 203.0.113.0/24 + - 224.0.0.0/4 + - 240.0.0.0/4 + - ipBlock: + cidr: ::/0 + except: + - ::/128 + - ::1/128 + - 100::/64 + - 2001:db8::/32 + - fc00::/7 + - fe80::/10 + - ff00::/8 + ports: + - protocol: TCP + port: 443 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: workflow-runner-boundary + namespace: github-app-runners +data: + runnerImage: "REPLACE_WITH_EXACT_DAEMON_IMAGE" + orchestratorOrigin: "wss://orchestrator.example.internal:3002" + provider: "REPLACE_WITH_CLAUDE_PROVIDER" + model: "REPLACE_WITH_CLAUDE_MODEL" + awsRegion: "REPLACE_WITH_AWS_REGION_OR_EMPTY" + anthropicBedrockBaseUrl: "REPLACE_WITH_BEDROCK_BASE_URL_OR_EMPTY" + allowedOwners: "REPLACE_WITH_ALLOWED_OWNERS_OR_EMPTY" + providerCredential1: "REPLACE_WITH_PROVIDER_CREDENTIAL_1" + providerCredential2: "REPLACE_WITH_PROVIDER_CREDENTIAL_2_OR_EMPTY" + providerCredential3: "REPLACE_WITH_PROVIDER_CREDENTIAL_3_OR_EMPTY" + runnerNodeLabel: "REPLACE_WITH_WORKFLOW_RUNNER_NODE_LABEL" + runnerNodeValue: "REPLACE_WITH_WORKFLOW_RUNNER_NODE_VALUE" + runnerImagePullSecret: "REPLACE_WITH_RUNNER_IMAGE_PULL_SECRET_OR_EMPTY" +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: github-app-workflow-runner-boundary +spec: + failurePolicy: Fail + paramKind: + apiVersion: v1 + kind: ConfigMap + matchConstraints: + resourceRules: + - apiGroups: [""] + apiVersions: ["v1"] + operations: ["CREATE", "UPDATE"] + resources: ["pods", "pods/ephemeralcontainers"] + variables: + - name: podSpec + expression: object.spec + - name: podSecurity + expression: variables.podSpec.securityContext + - name: runner + expression: variables.podSpec.containers[0] + - name: configuredProviderCredentialNames + expression: >- + [params.data['providerCredential1'], params.data['providerCredential2'], + params.data['providerCredential3']].filter(name, name != '') + - name: configuredProviderSettingNames + expression: >- + ['CLAUDE_PROVIDER', 'CLAUDE_MODEL'] + + (params.data['awsRegion'] == '' ? [] : ['AWS_REGION']) + + (params.data['anthropicBedrockBaseUrl'] == '' ? [] : + ['ANTHROPIC_BEDROCK_BASE_URL']) + + (params.data['allowedOwners'] == '' ? [] : ['ALLOWED_OWNERS']) + - name: controlEnvNames + expression: >- + ['WORKFLOW_RUNNER', 'WORKFLOW_RUNNER_RUN_ID', + 'WORKFLOW_RUNNER_ATTEMPT_ID', 'WORKFLOW_RUNNER_TOKEN', + 'ORCHESTRATOR_URL', 'LD_PRELOAD'] + - name: expectedEnvNames + expression: >- + variables.controlEnvNames + variables.configuredProviderCredentialNames + + variables.configuredProviderSettingNames + validations: + - expression: >- + has(object.metadata.labels) && + size(object.metadata.labels) == 4 && + 'app.kubernetes.io/name' in object.metadata.labels && + object.metadata.labels['app.kubernetes.io/name'] == 'github-app' && + 'app.kubernetes.io/component' in object.metadata.labels && + object.metadata.labels['app.kubernetes.io/component'] == 'workflow-runner' && + 'github-app/workflow-run-id' in object.metadata.labels && + 'github-app/workflow-attempt-id' in object.metadata.labels && + object.metadata.labels['github-app/workflow-run-id'].matches( + '^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$') && + object.metadata.labels['github-app/workflow-attempt-id'].matches( + '^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$') && + object.metadata.name == 'workflow-runner-' + + object.metadata.labels['github-app/workflow-attempt-id'] + message: Workflow runner name and labels must carry one exact v4 attempt identity. + - expression: >- + has(params.data) && + size(params.data) == 13 && + 'runnerImagePullSecret' in params.data && + 'runnerNodeLabel' in params.data && params.data['runnerNodeLabel'] != '' && + 'runnerNodeValue' in params.data && params.data['runnerNodeValue'] != '' && + 'runnerImage' in params.data && + params.data['runnerImage'].matches('^.+@sha256:[0-9a-f]{64}$') && + 'orchestratorOrigin' in params.data && + (params.data['orchestratorOrigin'].startsWith('wss://') || + params.data['orchestratorOrigin'].matches( + '^ws://[a-z0-9-]+[.][a-z0-9-]+[.]svc([.]cluster[.]local)?(:[0-9]{1,5})?$')) && + 'provider' in params.data && + 'model' in params.data && params.data['model'] != '' && + 'awsRegion' in params.data && + 'anthropicBedrockBaseUrl' in params.data && + 'allowedOwners' in params.data && + 'providerCredential1' in params.data && + 'providerCredential2' in params.data && + 'providerCredential3' in params.data && + ((params.data['provider'] == 'anthropic' && + params.data['awsRegion'] == '' && + params.data['anthropicBedrockBaseUrl'] == '' && + params.data['providerCredential1'] in + ['ANTHROPIC_API_KEY', 'CLAUDE_CODE_OAUTH_TOKEN'] && + params.data['providerCredential2'] == '' && + params.data['providerCredential3'] == '') || + (params.data['provider'] == 'bedrock' && + params.data['awsRegion'] != '' && + ((params.data['providerCredential1'] == 'AWS_BEARER_TOKEN_BEDROCK' && + params.data['providerCredential2'] == '' && + params.data['providerCredential3'] == '') || + (params.data['providerCredential1'] == 'AWS_ACCESS_KEY_ID' && + params.data['providerCredential2'] == 'AWS_SECRET_ACCESS_KEY' && + params.data['providerCredential3'] in ['', 'AWS_SESSION_TOKEN'])))) + message: Workflow runner boundary parameters must define one exact deployment environment. + - expression: >- + (!has(object.metadata.annotations) || size(object.metadata.annotations) == 0) && + (!has(object.metadata.finalizers) || size(object.metadata.finalizers) == 0) && + (!has(object.metadata.ownerReferences) || size(object.metadata.ownerReferences) == 0) && + !has(object.metadata.generateName) && + size(variables.podSpec.containers) == 1 && + (!has(variables.podSpec.initContainers) || + size(variables.podSpec.initContainers) == 0) && + (!has(variables.podSpec.ephemeralContainers) || + size(variables.podSpec.ephemeralContainers) == 0) && + size(variables.podSpec.volumes) == 1 && + variables.podSpec.volumes[0].name == 'workspace' && + has(variables.podSpec.volumes[0].emptyDir) && + (!has(variables.podSpec.volumes[0].emptyDir.medium) || + variables.podSpec.volumes[0].emptyDir.medium == '') && + quantity(variables.podSpec.volumes[0].emptyDir.sizeLimit).compareTo( + quantity('10Gi')) == 0 + message: Workflow runner Pods may use only the size-limited workspace volume. + - expression: >- + variables.podSpec.restartPolicy == 'Never' && + variables.podSpec.serviceAccountName == 'default' && + variables.podSpec.serviceAccount == 'default' && + variables.podSpec.dnsPolicy == 'ClusterFirst' && + variables.podSpec.schedulerName == 'default-scheduler' && + variables.podSpec.activeDeadlineSeconds == 4200 && + variables.podSpec.terminationGracePeriodSeconds == 30 && + variables.podSpec.automountServiceAccountToken == false && + variables.podSpec.enableServiceLinks == false && + (!has(variables.podSpec.hostIPC) || variables.podSpec.hostIPC == false) && + (!has(variables.podSpec.hostNetwork) || variables.podSpec.hostNetwork == false) && + (!has(variables.podSpec.hostPID) || variables.podSpec.hostPID == false) && + (!has(variables.podSpec.shareProcessNamespace) || + variables.podSpec.shareProcessNamespace == false) && + !has(variables.podSpec.dnsConfig) && !has(variables.podSpec.hostAliases) && + !has(variables.podSpec.hostUsers) && !has(variables.podSpec.hostname) && + !has(variables.podSpec.hostnameOverride) && + (!has(variables.podSpec.imagePullSecrets) || + size(variables.podSpec.imagePullSecrets) == 0 || + (params.data['runnerImagePullSecret'] != '' && + size(variables.podSpec.imagePullSecrets) == 1 && + variables.podSpec.imagePullSecrets[0].name == + params.data['runnerImagePullSecret'])) && + !has(variables.podSpec.affinity) && !has(variables.podSpec.nodeName) && + size(variables.podSpec.nodeSelector) == 1 && + variables.podSpec.nodeSelector[params.data['runnerNodeLabel']] == params.data['runnerNodeValue'] && + !has(variables.podSpec.os) && !has(variables.podSpec.overhead) && + variables.podSpec.preemptionPolicy == 'PreemptLowerPriority' && + variables.podSpec.priority == 0 && !has(variables.podSpec.priorityClassName) && + (!has(variables.podSpec.readinessGates) || + size(variables.podSpec.readinessGates) == 0) && + !has(variables.podSpec.resourceClaims) && !has(variables.podSpec.resources) && + !has(variables.podSpec.runtimeClassName) && + (!has(variables.podSpec.schedulingGates) || + size(variables.podSpec.schedulingGates) == 0) && + (!has(variables.podSpec.setHostnameAsFQDN) || + variables.podSpec.setHostnameAsFQDN == false) && + !has(variables.podSpec.subdomain) && + (!has(variables.podSpec.topologySpreadConstraints) || + size(variables.podSpec.topologySpreadConstraints) == 0) && + size(variables.podSpec.tolerations) == 3 && + variables.podSpec.tolerations.all(t, + (t.operator == 'Exists' && t.effect == 'NoExecute' && + t.tolerationSeconds == 300 && !has(t.value) && + t.key in ['node.kubernetes.io/not-ready', 'node.kubernetes.io/unreachable']) || + (t.key == params.data['runnerNodeLabel'] && t.operator == 'Equal' && + t.value == params.data['runnerNodeValue'] && t.effect == 'NoSchedule' && + !has(t.tolerationSeconds))) && + variables.podSpec.tolerations.exists(t, t.key == 'node.kubernetes.io/not-ready') && + variables.podSpec.tolerations.exists(t, t.key == 'node.kubernetes.io/unreachable') && + variables.podSpec.tolerations.exists(t, + t.key == params.data['runnerNodeLabel']) && + variables.podSecurity.runAsNonRoot == true && + variables.podSecurity.runAsUser == 1000 && + variables.podSecurity.runAsGroup == 1000 && + variables.podSecurity.seccompProfile.type == 'RuntimeDefault' && + !has(variables.podSecurity.appArmorProfile) && + !has(variables.podSecurity.fsGroup) && + !has(variables.podSecurity.fsGroupChangePolicy) && + !has(variables.podSecurity.seLinuxChangePolicy) && + !has(variables.podSecurity.supplementalGroups) && + !has(variables.podSecurity.supplementalGroupsPolicy) && + !has(variables.podSecurity.sysctls) && + !has(variables.podSecurity.seLinuxOptions) && + !has(variables.podSecurity.windowsOptions) + message: Workflow runner Pod isolation settings are immutable. + - expression: >- + variables.runner.name == 'runner' && + variables.runner.image == params.data['runnerImage'] && + variables.runner.imagePullPolicy == 'IfNotPresent' && + variables.runner.command == ['bun', 'run', 'dist/runner/main.js'] && + variables.runner.terminationMessagePath == '/dev/termination-log' && + variables.runner.terminationMessagePolicy == 'File' && + !has(variables.runner.args) && !has(variables.runner.workingDir) && + !has(variables.runner.lifecycle) && !has(variables.runner.livenessProbe) && + !has(variables.runner.readinessProbe) && !has(variables.runner.startupProbe) && + (!has(variables.runner.envFrom) || size(variables.runner.envFrom) == 0) && + size(variables.runner.volumeMounts) == 1 && + variables.runner.volumeMounts[0].name == 'workspace' && + variables.runner.volumeMounts[0].mountPath == '/tmp/bot-workspaces' && + (!has(variables.runner.volumeMounts[0].readOnly) || + variables.runner.volumeMounts[0].readOnly == false) && + !has(variables.runner.volumeMounts[0].subPath) && + !has(variables.runner.volumeMounts[0].subPathExpr) && + !has(variables.runner.volumeMounts[0].mountPropagation) && + !has(variables.runner.volumeMounts[0].recursiveReadOnly) && + (!has(variables.runner.ports) || size(variables.runner.ports) == 0) && + (!has(variables.runner.resizePolicy) || size(variables.runner.resizePolicy) == 0) && + !has(variables.runner.restartPolicy) && + (!has(variables.runner.restartPolicyRules) || + size(variables.runner.restartPolicyRules) == 0) && + (!has(variables.runner.volumeDevices) || size(variables.runner.volumeDevices) == 0) && + (!has(variables.runner.stdin) || variables.runner.stdin == false) && + (!has(variables.runner.stdinOnce) || variables.runner.stdinOnce == false) && + (!has(variables.runner.tty) || variables.runner.tty == false) + message: Workflow runner container execution fields are immutable. + - expression: >- + variables.runner.securityContext.allowPrivilegeEscalation == false && + (!has(variables.runner.securityContext.privileged) || + variables.runner.securityContext.privileged == false) && + variables.runner.securityContext.capabilities.drop == ['ALL'] && + (!has(variables.runner.securityContext.capabilities.add) || + size(variables.runner.securityContext.capabilities.add) == 0) && + !has(variables.runner.securityContext.runAsNonRoot) && + !has(variables.runner.securityContext.runAsUser) && + !has(variables.runner.securityContext.runAsGroup) && + !has(variables.runner.securityContext.seccompProfile) && + !has(variables.runner.securityContext.procMount) && + !has(variables.runner.securityContext.readOnlyRootFilesystem) && + !has(variables.runner.securityContext.appArmorProfile) && + !has(variables.runner.securityContext.seLinuxOptions) && + !has(variables.runner.securityContext.windowsOptions) + message: Workflow runner container security overrides are forbidden. + - expression: >- + has(variables.runner.resources) && + !has(variables.runner.resources.claims) && + has(variables.runner.resources.requests) && + size(variables.runner.resources.requests) == 3 && + 'cpu' in variables.runner.resources.requests && + quantity(variables.runner.resources.requests['cpu']).compareTo(quantity('500m')) == 0 && + 'memory' in variables.runner.resources.requests && + quantity(variables.runner.resources.requests['memory']).compareTo(quantity('1Gi')) == 0 && + 'ephemeral-storage' in variables.runner.resources.requests && + quantity(variables.runner.resources.requests['ephemeral-storage']).compareTo( + quantity('2Gi')) == 0 && + has(variables.runner.resources.limits) && + size(variables.runner.resources.limits) == 3 && + 'cpu' in variables.runner.resources.limits && + quantity(variables.runner.resources.limits['cpu']).compareTo(quantity('2')) == 0 && + 'memory' in variables.runner.resources.limits && + quantity(variables.runner.resources.limits['memory']).compareTo(quantity('4Gi')) == 0 && + 'ephemeral-storage' in variables.runner.resources.limits && + quantity(variables.runner.resources.limits['ephemeral-storage']).compareTo( + quantity('10Gi')) == 0 + message: Workflow runner resource requests and limits are immutable. + - expression: >- + size(variables.runner.env) == size(variables.expectedEnvNames) && + variables.runner.env.all(e, e.name in variables.expectedEnvNames) && + variables.expectedEnvNames.all(name, + variables.runner.env.exists(e, e.name == name)) && + variables.runner.env.all(e, + variables.runner.env.filter(other, other.name == e.name).size() == 1) && + variables.runner.env.filter(e, + e.name in variables.configuredProviderCredentialNames).all(e, + !has(e.value) && has(e.valueFrom) && has(e.valueFrom.secretKeyRef) && + e.valueFrom.secretKeyRef.name == 'workflow-runner-secrets' && + e.valueFrom.secretKeyRef.key == e.name && + (!has(e.valueFrom.secretKeyRef.optional) || + e.valueFrom.secretKeyRef.optional == false)) && + variables.runner.env.filter(e, + e.name in variables.configuredProviderSettingNames).all(e, + has(e.value) && e.value != '' && !has(e.valueFrom)) + message: Workflow runner provider environment must use the exact selected Secret keys. + - expression: >- + variables.runner.env.exists(e, + e.name == 'CLAUDE_PROVIDER' && e.value == params.data['provider']) && + variables.runner.env.exists(e, + e.name == 'CLAUDE_MODEL' && e.value == params.data['model']) && + (params.data['awsRegion'] == '' || variables.runner.env.exists(e, + e.name == 'AWS_REGION' && e.value == params.data['awsRegion'])) && + (params.data['anthropicBedrockBaseUrl'] == '' || + variables.runner.env.exists(e, + e.name == 'ANTHROPIC_BEDROCK_BASE_URL' && + e.value == params.data['anthropicBedrockBaseUrl'])) && + (params.data['allowedOwners'] == '' || variables.runner.env.exists(e, + e.name == 'ALLOWED_OWNERS' && e.value == params.data['allowedOwners'])) + message: Workflow runner provider settings must match protected boundary parameters. + - expression: >- + variables.runner.env.exists(e, + e.name == 'WORKFLOW_RUNNER' && e.value == 'true') && + variables.runner.env.exists(e, + e.name == 'WORKFLOW_RUNNER_RUN_ID' && + e.value == object.metadata.labels['github-app/workflow-run-id']) && + variables.runner.env.exists(e, + e.name == 'WORKFLOW_RUNNER_ATTEMPT_ID' && + e.value == object.metadata.labels['github-app/workflow-attempt-id']) && + variables.runner.env.exists(e, + e.name == 'WORKFLOW_RUNNER_TOKEN' && !has(e.value) && + e.valueFrom.secretKeyRef.name == object.metadata.name && + e.valueFrom.secretKeyRef.key == 'capability' && + (!has(e.valueFrom.secretKeyRef.optional) || + e.valueFrom.secretKeyRef.optional == false)) && + variables.runner.env.exists(e, + e.name == 'ORCHESTRATOR_URL' && + e.value == params.data['orchestratorOrigin'] + '/ws/workflow-runner/' + + object.metadata.labels['github-app/workflow-run-id'] + '/' + + object.metadata.labels['github-app/workflow-attempt-id']) && + variables.runner.env.exists(e, + e.name == 'LD_PRELOAD' && + e.value == '/usr/local/lib/github-app/daemon-process-guard.so') + message: Workflow runner control environment must match its exact attempt. +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: github-app-workflow-runner-boundary +spec: + policyName: github-app-workflow-runner-boundary + validationActions: [Deny, Audit] + paramRef: + name: workflow-runner-boundary + namespace: github-app-runners + parameterNotFoundAction: Deny + matchResources: + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: github-app-runners diff --git a/mkdocs.yml b/mkdocs.yml index e34bc989..c5d9932d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -93,6 +93,7 @@ markdown_extensions: nav: - Home: index.md - Use the bot: + - Repo configuration: use/repo-config.md - Invoking: use/invoking.md - Workflows: - Catalog: use/workflows/index.md diff --git a/native/daemon-process-guard.c b/native/daemon-process-guard.c new file mode 100644 index 00000000..bb30a157 --- /dev/null +++ b/native/daemon-process-guard.c @@ -0,0 +1,30 @@ +/* + * Loaded with LD_PRELOAD before Bun starts. + * + * Worker environment variables contain credentials that same-UID agent + * children must not read through /proc//environ. Mark the parent + * non-dumpable and fail closed if Linux cannot enforce the boundary. + */ + +#include +#include +#include + +static void fail_guard(const char *message, size_t length) { + (void)write(STDERR_FILENO, message, length); + _exit(78); +} + +__attribute__((constructor)) static void protect_daemon_environment(void) { + static const char set_failed[] = + "github-app daemon guard: PR_SET_DUMPABLE failed\n"; + static const char verify_failed[] = + "github-app daemon guard: process remained dumpable\n"; + + if (prctl(PR_SET_DUMPABLE, 0L, 0L, 0L, 0L) == -1) { + fail_guard(set_failed, sizeof(set_failed) - 1); + } + if (prctl(PR_GET_DUMPABLE, 0L, 0L, 0L, 0L) != 0) { + fail_guard(verify_failed, sizeof(verify_failed) - 1); + } +} diff --git a/package.json b/package.json index a9bfc180..856c7be7 100644 --- a/package.json +++ b/package.json @@ -48,6 +48,7 @@ "test:fast": "bun test", "test:watch": "bun test --watch", "test:coverage": "bun test --coverage", + "test:admission": "bash scripts/test-workflow-runner-admission.sh", "typecheck": "tsc --noEmit", "lint": "eslint .", "lint:fix": "eslint . --fix", @@ -68,7 +69,7 @@ "docker:build:daemon": "docker build -f Dockerfile.daemon -t chrisleekr/github-app:local-daemon . --progress=plain", "docker:build": "bun run docker:build:orchestrator && bun run docker:build:daemon", "docker:run:orchestrator": "bun run docker:build:orchestrator && docker run -p 3000:3000 -p 3002:3002 --env-file .env -e DATABASE_URL=postgres://bot:bot@host.docker.internal:5432/github_app -e VALKEY_URL=redis://host.docker.internal:6379 -v $HOME/.aws:/home/bun/.aws:ro --rm --name github-app chrisleekr/github-app:local-orchestrator", - "docker:run:daemon": "bun run docker:build:daemon && docker run --env-file .env -e ORCHESTRATOR_URL=ws://host.docker.internal:3002/ws -e DATABASE_URL=postgres://bot:bot@host.docker.internal:5432/github_app -e VALKEY_URL=redis://host.docker.internal:6379 -v $HOME/.aws:/home/bun/.aws:ro --rm --name github-app-daemon chrisleekr/github-app:local-daemon", + "docker:run:daemon": "bun run docker:build:daemon && docker run --env-file .env -e ORCHESTRATOR_URL=ws://host.docker.internal:3002/ws -v $HOME/.aws:/home/bun/.aws:ro --rm --name github-app-daemon chrisleekr/github-app:local-daemon", "docker:scan:orchestrator": "trivy image --severity CRITICAL,HIGH --ignore-unfixed --ignorefile .trivyignore.yaml --exit-code 1 --platform linux/$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/') chrisleekr/github-app:local-orchestrator", "docker:scan:daemon": "trivy image --severity CRITICAL,HIGH --ignore-unfixed --ignorefile .trivyignore.yaml --exit-code 0 --platform linux/$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/') chrisleekr/github-app:local-daemon", "docker:scan": "bun run docker:scan:orchestrator && bun run docker:scan:daemon", @@ -90,7 +91,7 @@ "license": "MIT", "dependencies": { "@anthropic-ai/bedrock-sdk": "^0.29.0", - "@anthropic-ai/claude-agent-sdk": "^0.3.0", + "@anthropic-ai/claude-agent-sdk": "^0.3.208", "@huggingface/transformers": "^4.2.0", "@kubernetes/client-node": "^1.4.0", "@modelcontextprotocol/sdk": "^1.29.0", diff --git a/scripts/build.ts b/scripts/build.ts index 5f29f152..7153185c 100644 --- a/scripts/build.ts +++ b/scripts/build.ts @@ -87,4 +87,43 @@ if (!daemonResult.success) { process.exit(1); } +// Build 4: execute the real startup probe against the produced daemon image. +const processBoundarySmokeResult = await Bun.build({ + entrypoints: ["./src/daemon/process-boundary-smoke.ts"], + outdir: "./dist/daemon", + target: "bun", + minify: isProduction, + sourcemap: isProduction ? "external" : "inline", + splitting: false, + naming: "process-boundary-smoke.js", +}); + +if (!processBoundarySmokeResult.success) { + console.error("Build failed (process boundary smoke):"); + for (const log of processBoundarySmokeResult.logs) { + console.error(log); + } + process.exit(1); +} + +// Build 5: one-attempt workflow runner. It ships in the daemon image because +// it needs the same agent CLI/toolchain, but has a separate process boundary. +const runnerResult = await Bun.build({ + entrypoints: ["./src/runner/main.ts"], + outdir: "./dist/runner", + target: "bun", + minify: isProduction, + sourcemap: isProduction ? "external" : "inline", + splitting: false, + naming: "main.js", +}); + +if (!runnerResult.success) { + console.error("Build failed (workflow runner):"); + for (const log of runnerResult.logs) { + console.error(log); + } + process.exit(1); +} + console.log("Build completed successfully"); diff --git a/scripts/check-docs-sync.ts b/scripts/check-docs-sync.ts index 45aceab9..1fc39ad9 100644 --- a/scripts/check-docs-sync.ts +++ b/scripts/check-docs-sync.ts @@ -45,7 +45,7 @@ if (touchedWorkflows.length > 0 && !touchedDoc) { "", "Update the relevant docs/use/workflows/*.md page in this PR, or", "mark the change as test/docs-only by moving it under", - "src/workflows/**/*.test.ts or src/workflows/**/*.md.", + "test/workflows/**/*.test.ts or src/workflows/**/*.md.", ].join("\n"), ); exit(1); diff --git a/scripts/check-test-globs.ts b/scripts/check-test-globs.ts index 617287d2..6d4d83d3 100644 --- a/scripts/check-test-globs.ts +++ b/scripts/check-test-globs.ts @@ -2,12 +2,10 @@ /** * CI guard: every `*.test.ts` file in the repo is reachable by the test * runner's glob set. `bun run test` shells out to `scripts/test-isolated.sh`, - * which runs each match of a hard-coded glob in its own Bun process (per-file + * which runs each match of a hard-coded glob in its own Bun process. Per-file * isolation is required because `mock.module()` is process-global and bleeds - * across files). A test file that lives outside the globbed roots is never - * executed, yet CI stays green because the runner only inspects files it - * already matched. The scheduler PR (#159) widened that dark spot from 1 to 4 - * colocated test files under src/ without anyone noticing. See issue #201. + * across files. The runner accepts only the canonical `test/` tree, so this + * guard also prevents tests from drifting back into production source. * * This guard derives the glob set from `scripts/test-isolated.sh` itself (the * single source of truth), enumerates every `*.test.ts` under the repo, and @@ -132,10 +130,7 @@ function main(): void { for (const f of uncovered) { console.error(` - ${f}`); } - console.error( - "\nThese files never run in CI. Fix: widen the `tests=( ... )` glob in\n" + - "scripts/test-isolated.sh to cover their root, e.g. add `src/**/*.test.ts`.", - ); + console.error("\nThese files never run in CI. Move first-party tests under `test/`."); process.exit(1); } diff --git a/scripts/env-contract.ts b/scripts/env-contract.ts index 3eed0a36..058810eb 100644 --- a/scripts/env-contract.ts +++ b/scripts/env-contract.ts @@ -38,6 +38,13 @@ const CONFIG_DOC = join(repoRoot, "docs/operate/configuration.md"); // (e.g. AWS_BEARER_TOKEN_BEDROCK, DAEMON_AUTH_TOKEN_PREVIOUS). No config-only var // contains any of these words as a segment. const SECRET_NAME_RE = /(?:^|_)(?:KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL|BEARER)(?:_|$)/; +// Names that carry a credential word but hold a reference, not a credential. +// A Secret's own name is public: it renders into a Pod spec and `kubectl get pod +// -o yaml`, so it belongs in the ConfigMap alongside the namespace it lives in. +const NON_SECRET_NAME_ALLOWLIST: ReadonlySet = new Set([ + "EPHEMERAL_DAEMON_SECRET_NAME", + "WORKFLOW_RUNNER_IMAGE_PULL_SECRET", +]); // URL / DSN / connection-string names usually carry embedded credentials, so they // must be classified secret unless explicitly listed as a plain, non-secret // endpoint. This inverts the default to fail-safe for connection-string shapes. @@ -120,12 +127,13 @@ function main(): void { // (URL/DSN) must be classified secret unless explicitly allowlisted. for (const e of entries) { const looksSecret = - SECRET_NAME_RE.test(e.env) || + (SECRET_NAME_RE.test(e.env) && !NON_SECRET_NAME_ALLOWLIST.has(e.env)) || (SECRET_URL_RE.test(e.env) && !NON_SECRET_URL_ALLOWLIST.has(e.env)); if (looksSecret && e.kind !== "secret") { errs.push( `${e.env} looks like a secret; add it to SECRET_ENV_VARS in src/config-secret-env.ts ` + - `(or, if it is a non-secret endpoint URL, add it to NON_SECRET_URL_ALLOWLIST in scripts/env-contract.ts)`, + `(or, if it names a resource rather than holding a credential, add it to ` + + `NON_SECRET_NAME_ALLOWLIST or NON_SECRET_URL_ALLOWLIST in scripts/env-contract.ts)`, ); } } diff --git a/scripts/test-isolated.sh b/scripts/test-isolated.sh index c2211bbb..d779b080 100755 --- a/scripts/test-isolated.sh +++ b/scripts/test-isolated.sh @@ -9,14 +9,15 @@ passed=0 failed=0 failures=() -tests=(test/**/*.test.ts src/**/*.test.ts) +tests=(test/**/*.test.ts) if (( ${#tests[@]} == 0 )); then - echo "No test files matched test/**/*.test.ts or src/**/*.test.ts" >&2 + echo "No test files matched test/**/*.test.ts" >&2 exit 1 fi for f in "${tests[@]}"; do - output=$(bun test "$f" 2>&1) + # Cold database migrations can exceed Bun's 5s lifecycle-hook default. + output=$(bun test --timeout=30000 "$f" 2>&1) has_zero_fail=false has_skip=false # Anchor on the leading-whitespace summary line Bun prints (e.g. " 0 fail") diff --git a/scripts/test-oauth.ts b/scripts/test-oauth.ts index f1dbc257..71c4f9f9 100644 --- a/scripts/test-oauth.ts +++ b/scripts/test-oauth.ts @@ -31,7 +31,7 @@ const client = new Anthropic({ authToken: token }); const MODELS: readonly { label: string; id: string }[] = [ { label: "sonnet-4-6 (alias)", id: "claude-sonnet-4-6" }, - { label: "opus-4-7", id: "claude-opus-4-7" }, + { label: "opus-5 (default)", id: "claude-opus-5" }, { label: "haiku-4-5 (snapshot)", id: "claude-haiku-4-5-20251001" }, ]; diff --git a/scripts/test-workflow-runner-admission.sh b/scripts/test-workflow-runner-admission.sh new file mode 100644 index 00000000..3a46fadf --- /dev/null +++ b/scripts/test-workflow-runner-admission.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +set -euo pipefail + +readonly KIND_VERSION="v0.31.0" +readonly KUBECTL_VERSION="v1.30.13" +readonly KIND_NODE_IMAGE="kindest/node:v1.30.13@sha256:8673291894dc400e0fb4f57243f5fdc6e355ceaa765505e0e73941aa1b6e0b80" +readonly CLUSTER_NAME="github-app-admission" +readonly TOOL_DIR="${PWD}/coverage/admission-tools" +readonly KUBECONFIG_PATH="${PWD}/coverage/admission-kubeconfig" +readonly KIND_CONFIG="${PWD}/test/fixtures/workflow-runner-kind.yaml" + +case "$(uname -s)" in + Darwin) platform_os="darwin" ;; + Linux) platform_os="linux" ;; + *) echo "Unsupported operating system: $(uname -s)" >&2; exit 1 ;; +esac +case "$(uname -m)" in + arm64 | aarch64) platform_arch="arm64" ;; + x86_64 | amd64) platform_arch="amd64" ;; + *) echo "Unsupported architecture: $(uname -m)" >&2; exit 1 ;; +esac +readonly platform_os platform_arch +readonly KIND_ASSET="kind-${platform_os}-${platform_arch}" + +# Reviewed upstream release digests. Update each version and all four targets together. +case "${platform_os}-${platform_arch}" in + linux-amd64) + kind_sha256="eb244cbafcc157dff60cf68693c14c9a75c4e6e6fedaf9cd71c58117cb93e3fa" + kubectl_sha256="b92bd89b27386b671841d5970b926b645c2ae44e5ca0663cff0f1c836a1530ee" + ;; + linux-arm64) + kind_sha256="8e1014e87c34901cc422a1445866835d1e666f2a61301c27e722bdeab5a1f7e4" + kubectl_sha256="afed1753b98ab30812203cb469e013082b25502c864f2889e8a0474aac497064" + ;; + darwin-amd64) + kind_sha256="a8b3cf77b2ad77aec5bf710d1a2589d9117576132af812885cad41e9dede4d4e" + kubectl_sha256="4c51288d7f32eafcbb6762a386b8818ce40b82dc3b99e4f27866317fd7cc9e43" + ;; + darwin-arm64) + kind_sha256="88bf554fe9da6311c9f8c2d082613c002911a476f6b5090e9420b35d84e70c5c" + kubectl_sha256="04962f4182b8f0a7260376a91d3fad0ff82c27a32035e9a3eb93465321970ca6" + ;; +esac +readonly kind_sha256 kubectl_sha256 + +file_sha256() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | awk '{print $1}' + else + shasum -a 256 "$1" | awk '{print $1}' + fi +} + +verify_sha256() { + local file="$1" + local expected="$2" + local actual + actual="$(file_sha256 "${file}")" + if [[ "${actual}" != "${expected}" ]]; then + echo "Checksum mismatch for ${file}" >&2 + return 1 + fi +} + +prepare_executable() { + chmod +x "$1" + if [[ "${platform_os}" == "darwin" ]]; then + codesign --force --sign - "$1" + fi +} + +mkdir -p "${TOOL_DIR}" +curl --fail --location --silent --show-error \ + "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/${KIND_ASSET}" \ + --output "${TOOL_DIR}/kind" +verify_sha256 "${TOOL_DIR}/kind" "${kind_sha256}" +prepare_executable "${TOOL_DIR}/kind" + +curl --fail --location --silent --show-error \ + "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/${platform_os}/${platform_arch}/kubectl" \ + --output "${TOOL_DIR}/kubectl" +verify_sha256 "${TOOL_DIR}/kubectl" "${kubectl_sha256}" +prepare_executable "${TOOL_DIR}/kubectl" + +export PATH="${TOOL_DIR}:${PATH}" +export KUBECONFIG="${KUBECONFIG_PATH}" + +cleanup() { + kind delete cluster --name "${CLUSTER_NAME}" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +kind create cluster \ + --name "${CLUSTER_NAME}" \ + --image "${KIND_NODE_IMAGE}" \ + --config "${KIND_CONFIG}" \ + --kubeconfig "${KUBECONFIG_PATH}" \ + --wait 120s +docker exec "${CLUSTER_NAME}-control-plane" \ + grep --fixed-strings --line-regexp 'podPidsLimit: 256' /var/lib/kubelet/config.yaml +bun run scripts/test-workflow-runner-admission.ts diff --git a/scripts/test-workflow-runner-admission.ts b/scripts/test-workflow-runner-admission.ts new file mode 100644 index 00000000..2e50eed8 --- /dev/null +++ b/scripts/test-workflow-runner-admission.ts @@ -0,0 +1,766 @@ +import type { V1Container, V1EnvVar, V1Pod } from "@kubernetes/client-node"; + +const NAMESPACE = "github-app-runners"; +const POLICY = "github-app-workflow-runner-boundary"; +const BOUNDARY_CONFIG_MAP = "workflow-runner-boundary"; +const EGRESS_POLICY = "github-app-workflow-runner-egress-boundary"; +const IMAGE = `registry.example/github-app@sha256:${"a".repeat(64)}`; +const ORIGIN = "wss://orchestrator.example.internal:3002"; +// The one pull secret the boundary pins. Runner Pods carry no ServiceAccount +// token, so the kubelet reads this while the container never can. +const PULL_SECRET = "runner-registry-credentials"; +const RUN_ID = "11111111-1111-4111-8111-111111111111"; +const ATTEMPT_ID = "22222222-2222-4222-8222-222222222222"; +const PROVIDER_ENV_NAMES = new Set([ + "CLAUDE_PROVIDER", + "CLAUDE_MODEL", + "ANTHROPIC_API_KEY", + "CLAUDE_CODE_OAUTH_TOKEN", + "AWS_REGION", + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + "AWS_BEARER_TOKEN_BEDROCK", + "ANTHROPIC_BEDROCK_BASE_URL", + "ALLOWED_OWNERS", +]); + +interface CommandResult { + readonly exitCode: number; + readonly stdout: string; + readonly stderr: string; +} + +async function kubectl(args: readonly string[], input?: string): Promise { + const process = Bun.spawn(["kubectl", ...args], { + stdin: input === undefined ? undefined : new Blob([input]), + stdout: "pipe", + stderr: "pipe", + }); + const [exitCode, stdout, stderr] = await Promise.all([ + process.exited, + new Response(process.stdout).text(), + new Response(process.stderr).text(), + ]); + return { exitCode, stdout, stderr }; +} + +function requireSuccess(result: CommandResult, action: string): void { + if (result.exitCode === 0) return; + throw new Error(`${action} failed: ${result.stderr || result.stdout}`); +} + +async function requireDenied(pod: V1Pod, caseName: string): Promise { + const result = await kubectl( + ["create", "--dry-run=server", "--output=name", "--filename=-"], + JSON.stringify(pod), + ); + if (result.exitCode === 0 || !`${result.stdout}\n${result.stderr}`.includes(POLICY)) { + throw new Error(`${caseName} was not denied by ${POLICY}: ${result.stderr || result.stdout}`); + } +} + +async function requireAdmitted(pod: V1Pod, caseName: string): Promise { + for (let attempt = 0; attempt < 30; attempt++) { + const result = await kubectl( + ["create", "--dry-run=server", "--output=name", "--filename=-"], + JSON.stringify(pod), + ); + if (result.exitCode === 0) return; + if (!`${result.stdout}\n${result.stderr}`.includes(POLICY)) { + requireSuccess(result, `admit ${caseName}`); + } + // eslint-disable-next-line no-await-in-loop -- ConfigMap admission parameters propagate asynchronously + await Bun.sleep(100); + } + throw new Error(`${caseName} was not admitted after boundary parameter propagation`); +} + +async function requireUpdateDenied(pod: V1Pod, caseName: string): Promise { + const result = await kubectl( + ["replace", "--dry-run=server", "--output=name", "--filename=-"], + JSON.stringify(pod), + ); + if (result.exitCode === 0 || !`${result.stdout}\n${result.stderr}`.includes(POLICY)) { + throw new Error(`${caseName} was not denied by ${POLICY}: ${result.stderr || result.stdout}`); + } +} + +async function requirePodSecurityDenied(pod: V1Pod): Promise { + const result = await kubectl( + ["create", "--dry-run=server", "--output=name", "--filename=-"], + JSON.stringify(pod), + ); + if ( + result.exitCode === 0 || + !`${result.stdout}\n${result.stderr}`.includes('violates PodSecurity "restricted:v1.30"') + ) { + throw new Error(`Restricted Pod Security did not deny host networking: ${result.stderr}`); + } +} + +function runner(pod: V1Pod): V1Container { + const container = pod.spec?.containers[0]; + if (container === undefined) throw new Error("Rendered Pod has no runner container"); + return container; +} + +function providerSetting(name: string, value: string): V1EnvVar { + return { name, value }; +} + +function providerCredential(name: string): V1EnvVar { + return { + name, + valueFrom: { + secretKeyRef: { name: "workflow-runner-secrets", key: name, optional: false }, + }, + }; +} + +function replaceProviderEnvironment(pod: V1Pod, providerEnv: readonly V1EnvVar[]): void { + const container = runner(pod); + container.env = [ + ...(container.env ?? []).filter((entry) => !PROVIDER_ENV_NAMES.has(entry.name)), + ...providerEnv, + ]; +} + +function providerBoundaryData(providerEnv: readonly V1EnvVar[]): Record { + const settings = new Map( + providerEnv.flatMap((entry) => (entry.value === undefined ? [] : [[entry.name, entry.value]])), + ); + const credentials = providerEnv + .filter((entry) => entry.valueFrom?.secretKeyRef !== undefined) + .map((entry) => entry.name); + const provider = settings.get("CLAUDE_PROVIDER"); + const model = settings.get("CLAUDE_MODEL"); + if (provider === undefined || model === undefined || credentials.length === 0) { + throw new Error("Provider environment cannot produce boundary parameters"); + } + return { + provider, + model, + awsRegion: settings.get("AWS_REGION") ?? "", + anthropicBedrockBaseUrl: settings.get("ANTHROPIC_BEDROCK_BASE_URL") ?? "", + allowedOwners: settings.get("ALLOWED_OWNERS") ?? "", + providerCredential1: credentials[0] ?? "", + providerCredential2: credentials[1] ?? "", + providerCredential3: credentials[2] ?? "", + }; +} + +async function configureBoundaryProvider(providerEnv: readonly V1EnvVar[]): Promise { + requireSuccess( + await kubectl([ + "patch", + "configmap", + BOUNDARY_CONFIG_MAP, + "--namespace", + NAMESPACE, + "--type=merge", + "--patch", + JSON.stringify({ data: providerBoundaryData(providerEnv) }), + ]), + "update provider boundary parameters", + ); +} + +async function configureBoundaryOrigin(origin: string): Promise { + requireSuccess( + await kubectl([ + "patch", + "configmap", + BOUNDARY_CONFIG_MAP, + "--namespace", + NAMESPACE, + "--type=merge", + "--patch", + JSON.stringify({ data: { orchestratorOrigin: origin } }), + ]), + "update boundary orchestrator origin", + ); +} + +function podWithOrigin(pod: V1Pod, origin: string): V1Pod { + const clone = structuredClone(pod); + const entry = runner(clone).env?.find((candidate) => candidate.name === "ORCHESTRATOR_URL"); + if (entry?.value === undefined) throw new Error("Pod has no ORCHESTRATOR_URL value"); + entry.value = `${origin}${new URL(entry.value).pathname}`; + return clone; +} + +async function waitForPolicyTypecheck(): Promise { + for (let attempt = 0; attempt < 30; attempt++) { + const result = await kubectl(["get", "validatingadmissionpolicy", POLICY, "--output=json"]); + requireSuccess(result, "read admission policy status"); + const policy = JSON.parse(result.stdout) as { + metadata?: { generation?: number }; + status?: { + observedGeneration?: number; + typeChecking?: { expressionWarnings?: unknown[] }; + }; + }; + if (policy.status?.observedGeneration === policy.metadata?.generation) { + const warnings = policy.status.typeChecking?.expressionWarnings ?? []; + if (warnings.length > 0) { + throw new Error(`Admission policy has CEL warnings: ${JSON.stringify(warnings)}`); + } + return; + } + // eslint-disable-next-line no-await-in-loop -- bounded API status poll + await Bun.sleep(1_000); + } + throw new Error("Admission policy type checking did not observe the current generation"); +} + +async function waitForPolicyEnforcement(pod: V1Pod): Promise { + for (let attempt = 0; attempt < 30; attempt++) { + const result = await kubectl( + ["create", "--dry-run=server", "--output=name", "--filename=-"], + JSON.stringify(pod), + ); + const output = `${result.stdout}\n${result.stderr}`; + if (result.exitCode !== 0 && output.includes(POLICY)) return; + if (result.exitCode !== 0) { + throw new Error(`Admission readiness probe failed outside ${POLICY}: ${output}`); + } + // eslint-disable-next-line no-await-in-loop -- bounded admission propagation poll + await Bun.sleep(1_000); + } + throw new Error(`${POLICY} did not enforce its binding within 30 seconds`); +} + +async function main(): Promise { + process.env["NODE_ENV"] ??= "test"; + process.env["GITHUB_APP_ID"] ??= "1"; + process.env["GITHUB_APP_PRIVATE_KEY"] ??= "admission-test-private-key"; + process.env["GITHUB_WEBHOOK_SECRET"] ??= "admission-test-webhook-secret"; + process.env["CLAUDE_PROVIDER"] = "anthropic"; + process.env["CLAUDE_MODEL"] = "claude-test"; + process.env["ANTHROPIC_API_KEY"] = "admission-test-provider-key"; + Reflect.deleteProperty(process.env, "CLAUDE_CODE_OAUTH_TOKEN"); + process.env["DATABASE_URL"] ??= "postgres://test:test@127.0.0.1/test"; + process.env["VALKEY_URL"] ??= "redis://127.0.0.1:6379"; + process.env["DAEMON_AUTH_TOKEN"] ??= "admission-test-daemon-token"; + process.env["WORKFLOW_RUNNER_CAPABILITY_SECRET"] ??= + "admission-test-workflow-runner-capability-secret"; + process.env["WORKFLOW_RUNNER_NAMESPACE"] = NAMESPACE; + process.env["WORKFLOW_RUNNER_IMAGE_PULL_SECRET"] = PULL_SECRET; + + // Imported here, not at module scope: the spawner pulls in src/config, which + // must not load until the env block above has run. The later import of the + // same module resolves from cache. + const { + WORKFLOW_RUNNER_NODE_LABEL, + WORKFLOW_RUNNER_NODE_VALUE, + WORKFLOW_RUNNER_IMAGE_PULL_SECRET, + } = await import("../src/k8s/workflow-runner-spawner"); + + const source = await Bun.file("examples/workflow-runner-admission.yaml").text(); + const manifest = source + .replace("REPLACE_WITH_EXACT_DAEMON_IMAGE", IMAGE) + .replace("wss://orchestrator.example.internal:3002", ORIGIN) + .replace("REPLACE_WITH_CLAUDE_PROVIDER", "anthropic") + .replace("REPLACE_WITH_CLAUDE_MODEL", "claude-test") + .replace("REPLACE_WITH_AWS_REGION_OR_EMPTY", "") + .replace("REPLACE_WITH_BEDROCK_BASE_URL_OR_EMPTY", "") + .replace("REPLACE_WITH_ALLOWED_OWNERS_OR_EMPTY", "") + .replace("REPLACE_WITH_PROVIDER_CREDENTIAL_1", "ANTHROPIC_API_KEY") + .replace("REPLACE_WITH_PROVIDER_CREDENTIAL_2_OR_EMPTY", "") + .replace("REPLACE_WITH_PROVIDER_CREDENTIAL_3_OR_EMPTY", "") + // Sourced from the spawner rather than hardcoded, so the boundary params can + // never drift from the nodeSelector and toleration the Pod actually carries. + .replace("REPLACE_WITH_WORKFLOW_RUNNER_NODE_LABEL", WORKFLOW_RUNNER_NODE_LABEL) + .replace("REPLACE_WITH_WORKFLOW_RUNNER_NODE_VALUE", WORKFLOW_RUNNER_NODE_VALUE) + .replace("REPLACE_WITH_RUNNER_IMAGE_PULL_SECRET_OR_EMPTY", WORKFLOW_RUNNER_IMAGE_PULL_SECRET); + // A placeholder added to the example without a substitution here installs a + // boundary that silently denies the exact Pod, which is how the two node + // placeholders went unnoticed. Fail on the manifest instead of on the assertion. + const unsubstituted = [...manifest.matchAll(/REPLACE_WITH_[A-Z_0-9]+/g)].map((m) => m[0]); + if (unsubstituted.length > 0) { + throw new Error( + `examples/workflow-runner-admission.yaml has placeholders this harness does not substitute: ${[...new Set(unsubstituted)].join(", ")}`, + ); + } + requireSuccess(await kubectl(["apply", "--filename=-"], manifest), "install admission policy"); + const egress = await kubectl([ + "get", + "networkpolicy", + EGRESS_POLICY, + "--namespace", + NAMESPACE, + "--output=json", + ]); + requireSuccess(egress, "read workflow runner egress policy"); + const egressPolicy = JSON.parse(egress.stdout) as { + spec?: { + policyTypes?: string[]; + egress?: Array<{ + to?: Array<{ ipBlock?: { cidr?: string; except?: string[] } }>; + ports?: Array<{ protocol?: string; port?: number }>; + }>; + }; + }; + const publicRule = egressPolicy.spec?.egress?.find((rule) => + rule.to?.some((peer) => peer.ipBlock?.cidr === "0.0.0.0/0"), + ); + const ipv4Public = publicRule?.to?.find((peer) => peer.ipBlock?.cidr === "0.0.0.0/0"); + const ipv6Public = publicRule?.to?.find((peer) => peer.ipBlock?.cidr === "::/0"); + if ( + !egressPolicy.spec?.policyTypes?.includes("Egress") || + (egressPolicy.spec.egress?.length ?? 0) !== 3 || + publicRule?.ports?.length !== 1 || + publicRule.ports[0]?.protocol !== "TCP" || + publicRule.ports[0]?.port !== 443 || + !ipv4Public?.ipBlock?.except?.includes("10.0.0.0/8") || + !ipv4Public.ipBlock.except.includes("169.254.0.0/16") || + !ipv6Public?.ipBlock?.except?.includes("fc00::/7") || + !ipv6Public.ipBlock.except.includes("fe80::/10") + ) { + throw new Error("Workflow runner egress policy is not the expected private-network boundary"); + } + await waitForPolicyTypecheck(); + + const { buildWorkflowRunnerPod } = await import("../src/k8s/workflow-runner-spawner"); + const attempt = { + runId: RUN_ID, + attemptId: ATTEMPT_ID, + runnerId: `workflow-runner:${ATTEMPT_ID}`, + executionDeliveryId: "admission-test-delivery", + workflowName: "implement" as const, + attemptDeadlineAt: new Date("2026-08-23T04:10:00Z"), + }; + const desired = buildWorkflowRunnerPod(attempt, IMAGE, ORIGIN); + const desiredProviderEnv = (runner(desired).env ?? []).filter((entry) => + PROVIDER_ENV_NAMES.has(entry.name), + ); + await configureBoundaryProvider(desiredProviderEnv); + const readinessProbe = structuredClone(desired); + readinessProbe.spec?.containers.push({ + ...structuredClone(runner(readinessProbe)), + name: "binding-readiness-probe", + }); + await waitForPolicyEnforcement(readinessProbe); + await requireAdmitted(desired, "exact production-rendered Pod"); + + const bedrockBearerProviderEnv = [ + providerSetting("CLAUDE_PROVIDER", "bedrock"), + providerSetting("CLAUDE_MODEL", "claude-test"), + providerSetting("AWS_REGION", "ap-southeast-2"), + providerSetting("ANTHROPIC_BEDROCK_BASE_URL", "https://bedrock.example.test/runtime"), + providerSetting("ALLOWED_OWNERS", "owner-a,owner-b"), + providerCredential("AWS_BEARER_TOKEN_BEDROCK"), + ] as const; + const supportedProviderCases: readonly [string, readonly V1EnvVar[]][] = [ + [ + "Anthropic OAuth Pod", + [ + providerSetting("CLAUDE_PROVIDER", "anthropic"), + providerSetting("CLAUDE_MODEL", "claude-test"), + providerCredential("CLAUDE_CODE_OAUTH_TOKEN"), + ], + ], + ["Bedrock bearer Pod", bedrockBearerProviderEnv], + [ + "Bedrock static Pod", + [ + providerSetting("CLAUDE_PROVIDER", "bedrock"), + providerSetting("CLAUDE_MODEL", "claude-test"), + providerSetting("AWS_REGION", "ap-southeast-2"), + providerCredential("AWS_ACCESS_KEY_ID"), + providerCredential("AWS_SECRET_ACCESS_KEY"), + ], + ], + [ + "Bedrock static session Pod", + [ + providerSetting("CLAUDE_PROVIDER", "bedrock"), + providerSetting("CLAUDE_MODEL", "claude-test"), + providerSetting("AWS_REGION", "ap-southeast-2"), + providerCredential("AWS_ACCESS_KEY_ID"), + providerCredential("AWS_SECRET_ACCESS_KEY"), + providerCredential("AWS_SESSION_TOKEN"), + ], + ], + ]; + for (const [caseName, providerEnv] of supportedProviderCases) { + // eslint-disable-next-line no-await-in-loop -- each provider is a separate protected parameter set + await configureBoundaryProvider(providerEnv); + const pod = structuredClone(desired); + replaceProviderEnvironment(pod, providerEnv); + // eslint-disable-next-line no-await-in-loop -- each provider is a separate server-side policy assertion + await requireAdmitted(pod, caseName); + } + + await configureBoundaryProvider(bedrockBearerProviderEnv); + const exactBedrockPod = structuredClone(desired); + replaceProviderEnvironment(exactBedrockPod, bedrockBearerProviderEnv); + await requireAdmitted(exactBedrockPod, "Bedrock bearer mutation baseline"); + const exactSettingMutations: readonly [string, string, string][] = [ + ["Bedrock model mutation", "CLAUDE_MODEL", "changed-model"], + ["Bedrock region mutation", "AWS_REGION", "us-east-1"], + [ + "Bedrock base URL mutation", + "ANTHROPIC_BEDROCK_BASE_URL", + "https://other.example.test/runtime", + ], + ["Bedrock owner mutation", "ALLOWED_OWNERS", "other-owner"], + ]; + for (const [caseName, name, value] of exactSettingMutations) { + const pod = structuredClone(exactBedrockPod); + const setting = runner(pod).env?.find((entry) => entry.name === name); + if (setting === undefined) throw new Error(`${caseName} setting is missing`); + setting.value = value; + // eslint-disable-next-line no-await-in-loop -- each setting is an independent exact-value assertion + await requireDenied(pod, caseName); + } + + await configureBoundaryProvider(desiredProviderEnv); + await requireAdmitted(desired, "restored Anthropic mutation baseline"); + + // Plaintext is permitted only to a cluster-local Service name, so an in-cluster + // runner can dial the orchestrator directly rather than hairpinning out through + // an ingress VIP. Every other origin must still be wss://. + const clusterLocalOrigin = "ws://github-app.github-app.svc.cluster.local:3002"; + await configureBoundaryOrigin(clusterLocalOrigin); + await requireAdmitted(podWithOrigin(desired, clusterLocalOrigin), "cluster-local ws:// origin"); + const publicPlaintextOrigin = "ws://orchestrator.example.com:3002"; + await configureBoundaryOrigin(publicPlaintextOrigin); + await requireDenied( + podWithOrigin(desired, publicPlaintextOrigin), + "plaintext origin outside the cluster", + ); + await configureBoundaryOrigin(ORIGIN); + await requireAdmitted(desired, "restored wss:// origin baseline"); + + const hostNetworkPod = structuredClone(desired); + if (hostNetworkPod.spec !== undefined) hostNetworkPod.spec.hostNetwork = true; + await requirePodSecurityDenied(hostNetworkPod); + + // Exercise the custom policy independently after proving the production PSA layer. + requireSuccess( + await kubectl([ + "label", + "namespace", + NAMESPACE, + "pod-security.kubernetes.io/enforce=privileged", + "--overwrite", + ]), + "relax disposable namespace Pod Security enforcement", + ); + + const cases: readonly [string, (pod: V1Pod) => void][] = [ + [ + "extra sidecar", + (pod) => pod.spec?.containers.push({ ...structuredClone(runner(pod)), name: "sidecar" }), + ], + [ + "lifecycle hook", + (pod) => (runner(pod).lifecycle = { postStart: { exec: { command: ["env"] } } }), + ], + [ + "envFrom", + (pod) => (runner(pod).envFrom = [{ secretRef: { name: "workflow-runner-secrets" } }]), + ], + [ + "dual Anthropic credential", + (pod) => + runner(pod).env?.push({ + name: "CLAUDE_CODE_OAUTH_TOKEN", + valueFrom: { + secretKeyRef: { + name: "workflow-runner-secrets", + key: "CLAUDE_CODE_OAUTH_TOKEN", + optional: false, + }, + }, + }), + ], + [ + "cross-provider credential", + (pod) => + runner(pod).env?.push({ + name: "AWS_BEARER_TOKEN_BEDROCK", + valueFrom: { + secretKeyRef: { + name: "workflow-runner-secrets", + key: "AWS_BEARER_TOKEN_BEDROCK", + optional: false, + }, + }, + }), + ], + [ + "optional selected credential", + (pod) => { + const selected = runner(pod).env?.find((entry) => entry.name === "ANTHROPIC_API_KEY"); + if (selected?.valueFrom?.secretKeyRef === undefined) { + throw new Error("Rendered Pod has no selected provider credential"); + } + selected.valueFrom.secretKeyRef.optional = true; + }, + ], + [ + "changed provider model", + (pod) => { + const model = runner(pod).env?.find((entry) => entry.name === "CLAUDE_MODEL"); + if (model === undefined) throw new Error("Rendered Pod has no provider model"); + model.value = "changed-model"; + }, + ], + [ + "switched Anthropic credential", + (pod) => + replaceProviderEnvironment(pod, [ + providerSetting("CLAUDE_PROVIDER", "anthropic"), + providerSetting("CLAUDE_MODEL", "claude-test"), + providerCredential("CLAUDE_CODE_OAUTH_TOKEN"), + ]), + ], + [ + "added Bedrock base URL", + (pod) => + runner(pod).env?.push( + providerSetting("ANTHROPIC_BEDROCK_BASE_URL", "https://bedrock.example.test"), + ), + ], + [ + "added owner setting", + (pod) => runner(pod).env?.push(providerSetting("ALLOWED_OWNERS", "other-owner")), + ], + [ + "incomplete Bedrock static chain", + (pod) => + replaceProviderEnvironment(pod, [ + providerSetting("CLAUDE_PROVIDER", "bedrock"), + providerSetting("CLAUDE_MODEL", "claude-test"), + providerSetting("AWS_REGION", "ap-southeast-2"), + providerCredential("AWS_ACCESS_KEY_ID"), + ]), + ], + [ + "mixed Bedrock credential chains", + (pod) => + replaceProviderEnvironment(pod, [ + providerSetting("CLAUDE_PROVIDER", "bedrock"), + providerSetting("CLAUDE_MODEL", "claude-test"), + providerSetting("AWS_REGION", "ap-southeast-2"), + providerCredential("AWS_BEARER_TOKEN_BEDROCK"), + providerCredential("AWS_ACCESS_KEY_ID"), + providerCredential("AWS_SECRET_ACCESS_KEY"), + ]), + ], + [ + "Bedrock session without static chain", + (pod) => + replaceProviderEnvironment(pod, [ + providerSetting("CLAUDE_PROVIDER", "bedrock"), + providerSetting("CLAUDE_MODEL", "claude-test"), + providerSetting("AWS_REGION", "ap-southeast-2"), + providerCredential("AWS_SESSION_TOKEN"), + ]), + ], + [ + "Bedrock bearer without region", + (pod) => + replaceProviderEnvironment(pod, [ + providerSetting("CLAUDE_PROVIDER", "bedrock"), + providerSetting("CLAUDE_MODEL", "claude-test"), + providerCredential("AWS_BEARER_TOKEN_BEDROCK"), + ]), + ], + ["changed image", (pod) => (runner(pod).image = "registry.example/attacker:latest")], + [ + "direct node assignment", + (pod) => { + if (pod.spec !== undefined) pod.spec.nodeName = "github-app-admission-control-plane"; + }, + ], + [ + "node affinity", + (pod) => { + if (pod.spec !== undefined) { + pod.spec.affinity = { + nodeAffinity: { + requiredDuringSchedulingIgnoredDuringExecution: { + nodeSelectorTerms: [ + { matchExpressions: [{ key: "kubernetes.io/hostname", operator: "Exists" }] }, + ], + }, + }, + }; + } + }, + ], + [ + "critical priority class", + (pod) => { + if (pod.spec !== undefined) pod.spec.priorityClassName = "system-node-critical"; + }, + ], + [ + "scheduling gate", + (pod) => { + if (pod.spec !== undefined) { + pod.spec.schedulingGates = [{ name: "attacker.example/hold" }]; + } + }, + ], + [ + "image pull secret naming a Secret the boundary does not pin", + (pod) => { + if (pod.spec !== undefined) { + pod.spec.imagePullSecrets = [{ name: "workflow-runner-secrets" }]; + } + }, + ], + [ + // The boundary allows one entry, so the count is the only thing stopping a + // second Secret riding along with the pinned one. + "second image pull secret alongside the pinned one", + (pod) => { + if (pod.spec !== undefined) { + pod.spec.imagePullSecrets = [{ name: PULL_SECRET }, { name: "workflow-runner-secrets" }]; + } + }, + ], + [ + "node selector", + (pod) => { + if (pod.spec !== undefined) pod.spec.nodeSelector = { "attacker.example/node": "true" }; + }, + ], + [ + "missing dedicated node selector", + (pod) => { + if (pod.spec !== undefined) pod.spec.nodeSelector = {}; + }, + ], + [ + "extra dedicated node selector", + (pod) => { + if (pod.spec !== undefined) { + pod.spec.nodeSelector = { ...pod.spec.nodeSelector, "attacker.example/node": "true" }; + } + }, + ], + [ + "missing dedicated node taint toleration", + (pod) => { + if (pod.spec !== undefined) { + // Keep the two built-in node-condition tolerations, drop the + // dedicated-node one. Derived from the shape rather than a literal + // key, since WORKFLOW_RUNNER_NODE_LABEL is deployment-configurable. + pod.spec.tolerations = pod.spec.tolerations?.filter((entry) => + (entry.key ?? "").startsWith("node.kubernetes.io/"), + ); + } + }, + ], + [ + "custom toleration", + (pod) => { + if (pod.spec !== undefined) { + pod.spec.tolerations = [ + ...(pod.spec.tolerations ?? []), + { key: "attacker.example/taint", operator: "Exists" }, + ]; + } + }, + ], + [ + "readiness gate", + (pod) => { + if (pod.spec !== undefined) { + pod.spec.readinessGates = [{ conditionType: "attacker.example/ready" }]; + } + }, + ], + [ + "host network", + (pod) => { + if (pod.spec !== undefined) pod.spec.hostNetwork = true; + }, + ], + [ + "hostPath workspace", + (pod) => { + if (pod.spec !== undefined) { + pod.spec.volumes = [{ name: "workspace", hostPath: { path: "/" } }]; + } + }, + ], + [ + "resource inflation", + (pod) => { + const resources = runner(pod).resources; + if (resources === undefined) throw new Error("Rendered Pod has no resources"); + resources.limits = { ...resources.limits, cpu: "8" }; + }, + ], + [ + "extra label", + (pod) => { + pod.metadata ??= {}; + pod.metadata.labels = { ...pod.metadata.labels, attacker: "true" }; + }, + ], + ]; + for (const [caseName, mutate] of cases) { + const pod = structuredClone(desired); + mutate(pod); + // eslint-disable-next-line no-await-in-loop -- each denial is an independent API assertion + await requireDenied(pod, caseName); + } + + await requireDenied( + { + apiVersion: "v1", + kind: "Pod", + metadata: { name: "alternate-name", namespace: NAMESPACE }, + spec: { + containers: [{ name: "escape", image: "busybox", securityContext: { privileged: true } }], + }, + }, + "non-runner Pod name", + ); + + requireSuccess( + await kubectl(["create", "--output=name", "--filename=-"], JSON.stringify(desired)), + "create exact Pod for ephemeral-container subresource test", + ); + const podName = desired.metadata?.name ?? ""; + const livePodResult = await kubectl([ + "get", + "pod", + podName, + `--namespace=${NAMESPACE}`, + "--output=json", + ]); + requireSuccess(livePodResult, "read exact Pod for ephemeral-container subresource test"); + const livePod = JSON.parse(livePodResult.stdout) as V1Pod; + if (livePod.spec === undefined) throw new Error("Live Pod has no spec"); + const imageUpdate = structuredClone(livePod); + runner(imageUpdate).image = `registry.example/github-app@sha256:${"b".repeat(64)}`; + await requireUpdateDenied(imageUpdate, "ordinary Pod image update"); + + livePod.spec.ephemeralContainers = [ + { name: "debug", image: "busybox", command: ["sh"], stdin: true, tty: true }, + ]; + const ephemeral = await kubectl( + [ + "replace", + `--raw=/api/v1/namespaces/${NAMESPACE}/pods/${podName}/ephemeralcontainers?dryRun=All`, + "--filename=-", + ], + JSON.stringify(livePod), + ); + if (ephemeral.exitCode === 0 || !`${ephemeral.stdout}\n${ephemeral.stderr}`.includes(POLICY)) { + throw new Error(`ephemeral-container update was not denied: ${ephemeral.stderr}`); + } + + console.log("workflow runner admission policy passed"); +} + +await main(); diff --git a/src/app.ts b/src/app.ts index bd09c091..fc9e1ab4 100644 --- a/src/app.ts +++ b/src/app.ts @@ -744,7 +744,7 @@ function shutdown(signal: string): void { proposalPoller = null; } await stopQueueWorker(); - stopLivenessReaper(); + await stopLivenessReaper(); stopFleetSnapshot(); await stopWebSocketServer(); await stopInstanceHeartbeat(); diff --git a/src/config-secret-env.ts b/src/config-secret-env.ts index 4034f22a..9623496d 100644 --- a/src/config-secret-env.ts +++ b/src/config-secret-env.ts @@ -31,4 +31,6 @@ export const SECRET_ENV_VARS: ReadonlySet = new Set([ "VALKEY_URL", "DAEMON_AUTH_TOKEN", "DAEMON_AUTH_TOKEN_PREVIOUS", + "WORKFLOW_RUNNER_CAPABILITY_SECRET", + "WORKFLOW_RUNNER_CAPABILITY_SECRET_PREVIOUS", ]); diff --git a/src/config.ts b/src/config.ts index 47163899..ae15587a 100644 --- a/src/config.ts +++ b/src/config.ts @@ -15,6 +15,11 @@ const nonEmptyOptionalString = z.preprocess( z.string().optional(), ); +const hmacRootSecret = z.preprocess( + (v) => (typeof v === "string" && v.trim() === "" ? undefined : v), + z.string().min(32).optional(), +); + /** * Duration field that accepts either a positive integer (ms) or a * duration string with `h`/`m`/`s` suffix (e.g. `4h`, `30m`, `1.5h`, @@ -116,7 +121,7 @@ const configSchema = z // Model override, required when provider=bedrock (Bedrock uses different model ID format), // optional input for anthropic. The schema-level .transform below defaults the - // anthropic path to "claude-opus-4-7", so the inferred Config.model is `string` + // anthropic path to "claude-opus-5", so the inferred Config.model is `string` // (not `string | undefined`), consumers do not need to handle the undefined case. model: z.string().min(1).optional(), @@ -157,13 +162,13 @@ const configSchema = z // Local dev: AWS SSO profile (after: le aws login -e dev). // Passed to the Claude Code subprocess env so the AWS SDK credential chain resolves it. awsProfile: z.string().optional(), - // Explicit key credentials, use in CI/CD or non-SSO environments. Prefer - // `awsProfile` locally and `awsBearerTokenBedrock` (OIDC) in GitHub Actions. + // Explicit IAM credentials, use in CI/CD or non-SSO environments. Isolated + // runners require a dedicated Bedrock-only principal and temporary keys. awsAccessKeyId: z.string().optional(), awsSecretAccessKey: z.string().optional(), awsSessionToken: z.string().optional(), - // OIDC bearer token, set automatically by aws-actions/configure-aws-credentials - // in GitHub Actions. Do not hand-set in long-running environments. + // Amazon Bedrock API key. This is distinct from the temporary IAM + // credentials exported by aws-actions/configure-aws-credentials. awsBearerTokenBedrock: z.string().optional(), // Overrides the Bedrock runtime endpoint. Leave unset unless fronting Bedrock // with a VPC endpoint or proxy, otherwise the SDK picks the correct regional URL. @@ -247,11 +252,15 @@ const configSchema = z // not linger across restarts. Set via WORKSPACE_STALE_TTL_MS (issue #221). workspaceStaleTtlMs: z.coerce.number().int().positive().default(3_600_000), - // Override max turns for the Claude Agent SDK, used as a FALLBACK ONLY on - // src/core/executor.ts when invoked without an explicit `maxTurns` - // argument. Since the dispatch-collapse, the orchestrator always passes - // `config.defaultMaxTurns` to the daemon, so this knob only affects - // non-dispatched internal callers. + // Override max turns for the Claude Agent SDK. Two roles: the fallback on + // src/core/executor.ts when invoked without an explicit `maxTurns`, and + // the first env link in the orchestrator's accept-site chain (see the + // `maxTurns` assignment in src/orchestrator/connection-handler.ts, which + // reads `workflowPolicy.maxTurns ?? agentMaxTurns ?? defaultMaxTurns`). + // It is also the FIRST LINK in the ceiling chain `src/repo-config/effective.ts` + // clamps a repo's `max_turns` against (`agentMaxTurns ?? defaultMaxTurns`, so + // with this unset DEFAULT_MAXTURNS is the ceiling). Either way a repo can + // lower the cap but never raise it. agentMaxTurns: z.coerce.number().int().positive().optional(), // Absolute path to the Claude Code CLI entry point (cli.js). @@ -279,12 +288,31 @@ const configSchema = z return parsed.length === 0 ? undefined : parsed; }), - // --- 6. Data layer (mandatory in server mode) --- + // Auto-review allowlist. When one of these logins pushes to an open PR + // (`pull_request.synchronize`), the `review` workflow is dispatched with no + // label and no mention. Unset/empty is the off switch. + // Widening, unlike every other allowlist here, which is why it lives in + // operator env and not repo YAML: `.github-app.yaml` `triggers:` is + // narrowing-only. Layered under ALLOWED_OWNERS and Gate 1, so it cannot + // readmit a repo or a workflow either of those rejected. The repo must also + // opt in via `workflows.review.auto`; neither key alone enables anything. + // Set via AUTO_REVIEW_USERS (comma-separated, e.g. "chrisleekr,acme"). + autoReviewUsers: z + .string() + .optional() + .transform((v): string[] | undefined => { + if (v === undefined || v === "") return undefined; + const parsed = v + .split(",") + .map((s) => s.trim()) + .filter(Boolean); + return parsed.length === 0 ? undefined : parsed; + }), + + // --- 6. Data layer (mandatory in controller mode) --- - // `valkeyUrl` backs the daemon job queue. `databaseUrl` backs the - // `executions` + `triage_results` tables. Both are required in server - // mode (no ORCHESTRATOR_URL) and optional in daemon mode (the daemon - // talks to the orchestrator over WebSocket, not to the data layer). + // The controller owns execution, workflow, and queue state. Shared + // daemons and isolated workflow runners use WebSocket RPC only. valkeyUrl: z.string().optional(), databaseUrl: z.string().optional(), @@ -314,6 +342,14 @@ const configSchema = z // credential after `Bearer ` is prepended. daemonAuthTokenPrevious: nonEmptyOptionalString, + // Controller-only HMAC root for one-attempt workflow runner capabilities. + // It must never be mounted into a shared daemon or isolated runner. + workflowRunnerCapabilitySecret: hmacRootSecret, + + // Optional predecessor accepted only while rotating the controller-only + // capability root. Per-attempt capabilities remain deadline-bound. + workflowRunnerCapabilitySecretPrevious: hmacRootSecret, + // Presence of ORCHESTRATOR_URL flips the process from SERVER mode to // DAEMON mode: the webhook HTTP server does NOT start and GitHub App // credentials are not required. Must be ws:// or wss:// (validated below). @@ -363,11 +399,8 @@ const configSchema = z // Off by default: detection ships first, self-heal is opt-in. socketHealthSelfHealEnabled: z.boolean().default(false), - // How long an execution may sit in status="running" before the watcher treats - // it as abandoned and marks it failed. Should generally be ≥ agentTimeoutMs - // so a legitimate long run isn't reaped mid-flight; the built-in default - // equals agentTimeoutMs (both 3_600_000ms / 60 min), which is the minimum - // safe setting. + // Startup-only recovery threshold for unfenced non-workflow executions. + // It must not be used by the periodic reaper while a daemon may still run. staleExecutionThresholdMs: z.coerce.number().int().positive().default(3_600_000), // Post-SIGTERM window the daemon uses to finish in-flight work before @@ -376,9 +409,14 @@ const configSchema = z // graceful shutdown should raise this to ≥ agentTimeoutMs. daemonDrainTimeoutMs: z.coerce.number().int().positive().default(300_000), - // Retries for TRANSIENT daemon dispatch failures only. + // Retries for transient daemon dispatch and workflow publication failures. jobMaxRetries: z.coerce.number().int().nonnegative().default(3), + // Maximum time a durable workflow may remain queued without acquiring an + // isolated runner lease. This bounds the in-flight target lock even when a + // wake-up is lost or repeatedly rejected as stale. + workflowDispatchTimeoutMs: z.coerce.number().int().positive().default(4_200_000), + // How long the orchestrator waits for a daemon in the fleet to claim a // job offer before re-queueing it for another daemon to pick up. offerTimeoutMs: z.coerce.number().int().positive().default(5_000), @@ -421,6 +459,10 @@ const configSchema = z // persistent daemons. daemonEphemeral: z.boolean().default(false), + // One-attempt Kubernetes runner. It communicates through controller RPC + // and must start without database, Valkey, PAT, or daemon-fleet secrets. + workflowRunner: z.boolean().default(false), + // Idle-exit timeout for ephemeral daemons. After this much wall-clock time // with zero active jobs, the ephemeral daemon shuts down and the Pod is // reclaimed by K8s. Capped below the Pod's default @@ -444,10 +486,37 @@ const configSchema = z // burst headroom. ephemeralDaemonSpawnQueueThreshold: z.coerce.number().int().positive().default(3), - // K8s namespace into which the orchestrator spawns ephemeral-daemon Pods. - // The orchestrator's ServiceAccount must hold `create/get/delete` on pods - // in this namespace. - ephemeralDaemonNamespace: z.string().default("default"), + // K8s namespace for ephemeral shared daemons. + ephemeralDaemonNamespace: z.string().trim().min(1).default("default"), + + // Name of the existing Secret in `ephemeralDaemonNamespace` that spawned + // daemon Pods mount via `envFrom`. Configurable so a deployment can point + // ephemeral daemons at the Secret its persistent daemon pools already use + // instead of provisioning a second copy of the same credentials. + ephemeralDaemonSecretName: z.string().trim().min(1).default("daemon-secrets"), + + // Dedicated namespace for isolated workflow runners. Keeping this separate + // lets admission validate every Pod request in the namespace without + // blocking the distinct shared-daemon Pod shape. + workflowRunnerNamespace: z.string().trim().min(1).default("github-app-runners"), + + // Runner nodeSelector and its NoSchedule toleration both derive from this + // pair, so one setting targets an existing node pool. The default keeps the + // `node-restriction.kubernetes.io/` prefix, which the NodeRestriction + // admission plugin stops a kubelet from assigning to itself; overriding to + // an unprefixed key gives that protection up. + workflowRunnerNodeLabel: z + .string() + .trim() + .min(1) + .default("github-app.node-restriction.kubernetes.io/workflow-runner"), + workflowRunnerNodeValue: z.string().trim().min(1).default("true"), + + // Names a dockerconfigjson Secret that already exists in the runner + // namespace. The admission boundary pins this name, so a runner Pod may + // reference this Secret and no other. Empty means the Pod carries no pull + // secret, which only works against a registry allowing anonymous pull. + workflowRunnerImagePullSecret: z.string().trim().default(""), // Container image the orchestrator launches for ephemeral daemons. Should // match the tag the persistent daemon Deployment is running. Optional at @@ -558,15 +627,17 @@ const configSchema = z // if Bedrock latency or cost is unacceptable for the deployment. llmOutputScannerEnabled: z.boolean().default(true), - // Model alias for the scanner call. Sonnet 4.6 by default, the - // higher-reasoning model materially reduces false-negatives on - // obfuscated/encoded secrets vs. Haiku, at the cost of per-call latency. - // Operators can downgrade to a Haiku alias if budget pressure dominates. - llmOutputScannerModel: z.string().default("sonnet-4-6"), + // Model alias for the scanner call. Haiku 4.5 is the latency floor and + // what the per-call budget below is sized around. A Sonnet alias detects + // more obfuscated/encoded variants; raise the timeout with it, because the + // isolated-runner boundary treats a slow scan as a rejection. + llmOutputScannerModel: z.string().default("haiku-4-5"), - // Hard cap per scanner call. Treated as a fail-open failure (post body - // that survived the regex pass, log a warn) when exceeded. - llmOutputScannerTimeoutMs: z.coerce.number().int().positive().default(3_000), + // Hard cap per scanner call. Fail-open on the GitHub-output path (post the + // body that survived the regex pass, log a warn); fail-CLOSED on the + // isolated-runner boundary, where a timeout discards a completed run. Sized + // for that second case, so it is deliberately generous. + llmOutputScannerTimeoutMs: z.coerce.number().int().positive().default(30_000), // --- 11. Agent maxTurns --- @@ -710,7 +781,7 @@ const configSchema = z // `allowed_users`) would go dark fleet-wide with nothing saying why. repoConfigFile: z .string() - .transform((str) => str.trim()) + .transform((s) => s.trim()) .pipe(z.string().min(1)) .default(".github-app.yaml"), @@ -726,18 +797,19 @@ const configSchema = z validateServerModeCredentials(data, ctx); validateProviderCredentials(data, ctx); validateDataLayerConfig(data, ctx); + validateWorkerNamespaces(data, ctx); }) // Runs only if .superRefine added no issues, so by this point: // - provider=bedrock guarantees data.model is defined // (validateProviderCredentials errors otherwise) // - provider=anthropic falls through with data.model possibly undefined - // We default the anthropic branch to Opus 4.7 here. Doing it in .transform + // We default the anthropic branch to Opus 5 here. Doing it in .transform // narrows the inferred Config type: `model` becomes `string`, not // `string | undefined`, so downstream code drops the defensive `?.` / `??`. // Override via CLAUDE_MODEL when cost-sensitive. .transform((data) => ({ ...data, - model: data.model ?? "claude-opus-4-7", + model: data.model ?? "claude-opus-5", })); /** @@ -825,49 +897,96 @@ function validateProviderCredentials( } } -/** - * After the dispatch-to-daemon collapse, every server-mode process needs - * the data layer (DB + Valkey + DAEMON_AUTH_TOKEN) to orchestrate the - * daemon fleet. Daemon-mode processes only need DAEMON_AUTH_TOKEN for the - * WebSocket handshake. - */ +/** The controller owns PostgreSQL and Valkey; runners use scoped RPC. */ +function sameConfiguredSecret(left: string | undefined, right: string | undefined): boolean { + if (left === undefined || right === undefined) return false; + + return left === right; +} + function validateDataLayerConfig( data: { orchestratorUrl?: string | undefined; databaseUrl?: string | undefined; valkeyUrl?: string | undefined; daemonAuthToken?: string | undefined; + daemonAuthTokenPrevious?: string | undefined; + workflowRunnerCapabilitySecret?: string | undefined; + workflowRunnerCapabilitySecretPrevious?: string | undefined; + workflowRunner: boolean; }, ctx: z.RefinementCtx, ): void { - const isDaemonMode = (data.orchestratorUrl?.trim().length ?? 0) > 0; - + if (data.workflowRunner) return; if ((data.daemonAuthToken?.trim().length ?? 0) === 0) { ctx.addIssue({ code: "custom", - message: "DAEMON_AUTH_TOKEN is required (set on both orchestrator and daemon)", + message: "DAEMON_AUTH_TOKEN is required (set on controller and shared daemons)", path: ["daemonAuthToken"], }); } - if (isDaemonMode) return; + if ((data.orchestratorUrl?.trim().length ?? 0) > 0) return; + + if ((data.workflowRunnerCapabilitySecret?.trim().length ?? 0) === 0) { + ctx.addIssue({ + code: "custom", + message: "WORKFLOW_RUNNER_CAPABILITY_SECRET is required on the controller", + path: ["workflowRunnerCapabilitySecret"], + }); + } + const daemonSecrets = [data.daemonAuthToken, data.daemonAuthTokenPrevious]; + const capabilitySecrets = [ + data.workflowRunnerCapabilitySecret, + data.workflowRunnerCapabilitySecretPrevious, + ]; + if ( + capabilitySecrets.some((capability) => + daemonSecrets.some((daemon) => sameConfiguredSecret(capability, daemon)), + ) + ) { + ctx.addIssue({ + code: "custom", + message: "Workflow runner capability roots must differ from daemon authentication roots", + path: ["workflowRunnerCapabilitySecret"], + }); + } if ((data.databaseUrl?.trim().length ?? 0) === 0) { ctx.addIssue({ code: "custom", - message: "DATABASE_URL is required in server mode", + message: "DATABASE_URL is required for orchestrator workflow state", path: ["databaseUrl"], }); } if ((data.valkeyUrl?.trim().length ?? 0) === 0) { ctx.addIssue({ code: "custom", - message: "VALKEY_URL is required in server mode", + message: "VALKEY_URL is required for orchestrator workflow dispatch", path: ["valkeyUrl"], }); } } +/** The runner admission policy covers every Pod in its namespace. */ +function validateWorkerNamespaces( + data: { + orchestratorUrl?: string | undefined; + workflowRunner: boolean; + ephemeralDaemonNamespace: string; + workflowRunnerNamespace: string; + }, + ctx: z.RefinementCtx, +): void { + const isController = !data.workflowRunner && (data.orchestratorUrl?.trim().length ?? 0) === 0; + if (!isController || data.workflowRunnerNamespace !== data.ephemeralDaemonNamespace) return; + ctx.addIssue({ + code: "custom", + message: "WORKFLOW_RUNNER_NAMESPACE must differ from EPHEMERAL_DAEMON_NAMESPACE", + path: ["workflowRunnerNamespace"], + }); +} + export type Config = z.infer; // Export schema for use in tests (avoids importing the singleton which runs loadConfig()) @@ -911,6 +1030,28 @@ export function assertPatRequiresAllowlist(cfg: Config): void { } } +/** + * Tenancy guard for AUTO_REVIEW_USERS. Every other allowlist in this file + * narrows; this one WIDENS, since it starts an unattended agent run (Bash tool, + * on a clone, spending tokens) with no per-event human action. `isOwnerAllowed` + * returns `{ allowed: true }` for every owner when ALLOWED_OWNERS is unset + * (`src/webhook/authorize.ts`), so without this assertion the whole chain to + * "run an agent on a stranger's repository" is a login-string match plus a key + * that the third-party repo owner controls. + * + * Not `length !== 1` like the two guards above: auto-review carries no personal + * identity or shared rate-limit bucket, so several owners are fine. It just may + * not be unbounded. + */ +export function assertAutoReviewRequiresAllowlist(cfg: Config): void { + if ((cfg.autoReviewUsers?.length ?? 0) > 0 && (cfg.allowedOwners?.length ?? 0) === 0) { + throw new Error( + "ALLOWED_OWNERS must be set when AUTO_REVIEW_USERS is set: auto-review starts " + + "unattended agent runs, so it must be bound to owners you control.", + ); + } +} + /** * Parse a boolean environment variable strictly. * Accepts: true/false, 1/0, yes/no (case-insensitive). @@ -975,7 +1116,10 @@ function loadConfig(): Config { workspaceStaleTtlMs: process.env["WORKSPACE_STALE_TTL_MS"], cloneDepth: process.env["CLONE_DEPTH"], triggerPhrase: process.env["TRIGGER_PHRASE"], - botAppLogin: process.env["BOT_APP_LOGIN"], + // `blankToUndefined`, so a chart rendering an unset optional key as "" + // still gets the zod default. `.default()` only fires on undefined, and an + // empty login now hard-fails the inline-comment MCP server at startup. + botAppLogin: blankToUndefined(process.env["BOT_APP_LOGIN"]), port: process.env["PORT"], logLevel: process.env["LOG_LEVEL"], nodeEnv: process.env.NODE_ENV, @@ -988,6 +1132,7 @@ function loadConfig(): Config { agentMaxTurns: process.env["AGENT_MAX_TURNS"], claudeCodePath: process.env["CLAUDE_CODE_PATH"], allowedOwners: process.env["ALLOWED_OWNERS"], + autoReviewUsers: process.env["AUTO_REVIEW_USERS"], // Group 6, Data layer valkeyUrl: process.env["VALKEY_URL"], @@ -999,6 +1144,9 @@ function loadConfig(): Config { // Group 8, Daemon / Orchestrator WebSocket daemonAuthToken: process.env["DAEMON_AUTH_TOKEN"], daemonAuthTokenPrevious: process.env["DAEMON_AUTH_TOKEN_PREVIOUS"], + workflowRunnerCapabilitySecret: process.env["WORKFLOW_RUNNER_CAPABILITY_SECRET"], + workflowRunnerCapabilitySecretPrevious: + process.env["WORKFLOW_RUNNER_CAPABILITY_SECRET_PREVIOUS"], orchestratorUrl: process.env["ORCHESTRATOR_URL"], heartbeatIntervalMs: process.env["HEARTBEAT_INTERVAL_MS"], heartbeatTimeoutMs: process.env["HEARTBEAT_TIMEOUT_MS"], @@ -1014,6 +1162,7 @@ function loadConfig(): Config { staleExecutionThresholdMs: process.env["STALE_EXECUTION_THRESHOLD_MS"], daemonDrainTimeoutMs: process.env["DAEMON_DRAIN_TIMEOUT_MS"], jobMaxRetries: process.env["JOB_MAX_RETRIES"], + workflowDispatchTimeoutMs: process.env["WORKFLOW_DISPATCH_TIMEOUT_MS"], offerTimeoutMs: process.env["OFFER_TIMEOUT_MS"], queueWorkerBackoffMaxMs: process.env["QUEUE_WORKER_BACKOFF_MAX_MS"], livenessReaperIntervalMs: process.env["LIVENESS_REAPER_INTERVAL_MS"], @@ -1024,10 +1173,16 @@ function loadConfig(): Config { // Group 9, Ephemeral daemon (K8s-spawned scale-up) daemonEphemeral: parseBooleanEnv("DAEMON_EPHEMERAL", process.env["DAEMON_EPHEMERAL"]), + workflowRunner: parseBooleanEnv("WORKFLOW_RUNNER", process.env["WORKFLOW_RUNNER"]), ephemeralDaemonIdleTimeoutMs: process.env["EPHEMERAL_DAEMON_IDLE_TIMEOUT_MS"], ephemeralDaemonSpawnCooldownMs: process.env["EPHEMERAL_DAEMON_SPAWN_COOLDOWN_MS"], ephemeralDaemonSpawnQueueThreshold: process.env["EPHEMERAL_DAEMON_SPAWN_QUEUE_THRESHOLD"], ephemeralDaemonNamespace: process.env["EPHEMERAL_DAEMON_NAMESPACE"], + ephemeralDaemonSecretName: process.env["EPHEMERAL_DAEMON_SECRET_NAME"], + workflowRunnerNamespace: process.env["WORKFLOW_RUNNER_NAMESPACE"], + workflowRunnerNodeLabel: process.env["WORKFLOW_RUNNER_NODE_LABEL"], + workflowRunnerNodeValue: process.env["WORKFLOW_RUNNER_NODE_VALUE"], + workflowRunnerImagePullSecret: process.env["WORKFLOW_RUNNER_IMAGE_PULL_SECRET"], daemonImage: process.env["DAEMON_IMAGE"], orchestratorPublicUrl: process.env["ORCHESTRATOR_PUBLIC_URL"], @@ -1112,6 +1267,7 @@ function loadConfig(): Config { assertOauthRequiresAllowlist(cfg); assertPatRequiresAllowlist(cfg); + assertAutoReviewRequiresAllowlist(cfg); // The file stopped being scheduler-specific once it grew feature toggles. // The old name still works so an upgrade doesn't silently change which @@ -1135,26 +1291,19 @@ function loadConfig(): Config { } // H6: Warn when WebSocket URLs use unencrypted ws:// in production. - // Installation tokens and DAEMON_AUTH_TOKEN are transmitted over this connection. + // Runner capabilities and installation tokens cross this connection. if (cfg.nodeEnv === "production") { if (cfg.orchestratorUrl?.startsWith("ws://") === true) { console.warn( "[config] WARNING: ORCHESTRATOR_URL uses ws:// (unencrypted) in production. " + - "Installation tokens and DAEMON_AUTH_TOKEN are transmitted in cleartext. Use wss:// for production.", + "Runner credentials are transmitted in cleartext. Use wss:// for production.", ); } } - // H7 (issue #102, defense layer 1b): warn when orchestrator-only secrets - // are present in a daemon process. The Helm chart should mount only - // `daemon-secrets` on daemon Pods; `orchestrator-secrets` (App private - // key, webhook secret, DB / Valkey URLs) belong on the orchestrator Pod - // alone. A misconfigured deployment that mounts both bundles wouldn't - // crash, the daemon doesn't read these keys, but it weakens the - // capability-minimization gate in `buildProviderEnv()` (the agent - // subprocess can no longer leak them simply because they aren't there). - // Warn only, never refuse to start, since a downed daemon is worse than - // a degraded security posture. + // H7 (issue #102, defense layer 1b): surface controller-only secrets on a + // worker. Shared daemons warn; workflow-runner main adds a fail-closed deny + // set before it connects. // // Detection heuristic: only daemons set `ORCHESTRATOR_URL` (they connect // TO the orchestrator) or `DAEMON_EPHEMERAL`. The orchestrator/webhook @@ -1164,7 +1313,8 @@ function loadConfig(): Config { // // `console.warn` (not pino) on purpose: config loads before the logger is // built, so this is the only available channel at this point. - const isDaemonProcess = (cfg.orchestratorUrl?.trim().length ?? 0) > 0 || cfg.daemonEphemeral; + const isDaemonProcess = + (cfg.orchestratorUrl?.trim().length ?? 0) > 0 || cfg.daemonEphemeral || cfg.workflowRunner; if (isDaemonProcess) { const leakedKeys: string[] = []; if ((process.env["GITHUB_APP_PRIVATE_KEY"]?.trim().length ?? 0) > 0) @@ -1174,10 +1324,14 @@ function loadConfig(): Config { if ((process.env["DATABASE_URL"]?.trim().length ?? 0) > 0) leakedKeys.push("DATABASE_URL"); if ((process.env["VALKEY_URL"]?.trim().length ?? 0) > 0) leakedKeys.push("VALKEY_URL"); if ((process.env["REDIS_URL"]?.trim().length ?? 0) > 0) leakedKeys.push("REDIS_URL"); + if ((process.env["WORKFLOW_RUNNER_CAPABILITY_SECRET"]?.trim().length ?? 0) > 0) + leakedKeys.push("WORKFLOW_RUNNER_CAPABILITY_SECRET"); + if ((process.env["WORKFLOW_RUNNER_CAPABILITY_SECRET_PREVIOUS"]?.trim().length ?? 0) > 0) + leakedKeys.push("WORKFLOW_RUNNER_CAPABILITY_SECRET_PREVIOUS"); if (leakedKeys.length > 0) { console.warn( - `[config] WARNING: orchestrator-only secret(s) present on daemon process: ${leakedKeys.join(", ")}. ` + - "Mount these only via `orchestrator-secrets` on the orchestrator Pod; the daemon does not need them. " + + `[config] WARNING: orchestrator-only secret(s) present on worker process: ${leakedKeys.join(", ")}. ` + + "Mount these only on the orchestrator Pod. " + "See docs/operate/configuration.md for the K8s Secret split contract.", ); } diff --git a/src/core/pipeline.ts b/src/core/pipeline.ts index 3c9cec3b..5a751cb1 100644 --- a/src/core/pipeline.ts +++ b/src/core/pipeline.ts @@ -379,7 +379,7 @@ export async function runPipeline( ctx.log, "trackingComment.create", () => - retryWithBackoff(() => createTrackingComment(ctx), { + retryWithBackoff(() => createTrackingComment(ctx, overrides.policy?.warning), { maxAttempts: 3, initialDelayMs: 1000, log: ctx.log, @@ -438,13 +438,11 @@ export async function runPipeline( ...ctx, headBranch: data.headBranch ?? ctx.headBranch ?? ctx.defaultBranch, baseBranch: data.baseBranch ?? ctx.baseBranch ?? ctx.defaultBranch, - // No review-only gate here, unlike reviewLearnings below. The repo - // schema only accepts `instructions` under `workflows.review`, so the - // value cannot be authored for another workflow. That gate lives on the - // repo's YAML, not on the wire: `AgentPolicySchema.instructions` is - // workflow-agnostic, so a producer that sets it for a non-review - // workflow would land here unchallenged. reviewLearnings needs its own - // gate below because it is loaded uniformly into every job. + // No review-only gate here, unlike reviewLearnings below. Two upstream + // layers already own it: the schema only accepts `instructions` under + // `workflows.review`, and `stripInstructionsUnlessReview` drops it at + // job accept. reviewLearnings needs its gate here because it is loaded + // uniformly into every job and has no upstream filter. ...(overrides.policy?.instructions !== undefined ? { reviewInstructions: overrides.policy.instructions } : {}), diff --git a/src/core/tracking-comment.ts b/src/core/tracking-comment.ts index 75b5ddcb..5b1b49ea 100644 --- a/src/core/tracking-comment.ts +++ b/src/core/tracking-comment.ts @@ -8,8 +8,7 @@ const SPINNER_HTML = ` { +export async function createTrackingComment( + ctx: BotContext, + configWarning?: string, +): Promise { const { octokit, owner, repo, entityNumber, log } = ctx; // Embed the deliveryId marker so the bot can locate and update its own tracking // comment in place (see the `comment` MCP server). Not an idempotency mechanism // anymore (claimDelivery + idx_workflow_runs_inflight own that, #202; the Map + // marker-scan check were retired in #211). - const body = `${deliveryMarker(ctx.deliveryId)}\n${SPINNER_HTML} **${config.triggerPhrase}** is working on this...\n\n_Analyzing your request..._`; + // Same GitHub alert syntax as the workflow rail's `renderConfigNotice` in + // src/workflows/tracking-mirror.ts, but collapsed to one line: that rail + // splits a multi-line notice into paragraphs, while this rail only ever + // carries the single-line validation warning. + const warningLine = + configWarning !== undefined && configWarning.trim() !== "" + ? `\n\n> [!WARNING]\n> ${collapseWarning(configWarning)}` + : ""; + const body = `${deliveryMarker(ctx.deliveryId)}\n${SPINNER_HTML} **${config.triggerPhrase}** is working on this...\n\n_Analyzing your request..._${warningLine}`; const guarded = await safePostToGitHub({ body, @@ -169,9 +187,23 @@ export async function updateTrackingComment( } } +/** + * Collapse a config notice to a single line. `\s+`, not `\n`: a lone `\r` also + * terminates the `> ` blockquote and orphans the rest of the notice. + */ +function collapseWarning(warning: string): string { + return warning.trim().replace(/\s+/g, " "); +} + /** * Finalize the tracking comment with completion status. * Called after Claude finishes or errors. + * + * `configWarning` is re-appended here because the agent's + * `update_claude_comment` MCP tool replaces the whole comment body, wiping the + * banner `createTrackingComment` posted. Skipped when the original banner + * survived, so a run where the agent never touched the comment does not show + * the notice twice. */ export async function finalizeTrackingComment( ctx: BotContext, @@ -181,9 +213,10 @@ export async function finalizeTrackingComment( durationMs?: number; costUsd?: number; error?: string; + configWarning?: string; }, ): Promise { - const { success, durationMs, costUsd, error } = opts; + const { success, durationMs, costUsd, error, configWarning } = opts; let header: string; if (success) { @@ -217,10 +250,19 @@ export async function finalizeTrackingComment( const errorSection = error !== undefined && error !== "" ? `\n\n---\n**Error:** ${error}` : ""; + const collapsedWarning = + configWarning !== undefined && configWarning.trim() !== "" + ? collapseWarning(configWarning) + : ""; + const warningSection = + collapsedWarning !== "" && !cleanedBody.includes(collapsedWarning) + ? `\n\n> [!WARNING]\n> ${collapsedWarning}` + : ""; + // Re-prepend the delivery marker so the tracking comment keeps its stable hidden marker // even if Claude's update_claude_comment call (which runs sanitizeContent) previously // stripped it. The marker locates the bot's comment, not idempotency (#202/#211). - const finalBody = `${deliveryMarker(ctx.deliveryId)}\n${header}\n\n---\n${cleanedBody}${errorSection}`; + const finalBody = `${deliveryMarker(ctx.deliveryId)}\n${header}${warningSection}\n\n---\n${cleanedBody}${errorSection}`; await updateTrackingComment(ctx, trackingCommentId, finalBody); } diff --git a/src/daemon/daemon-id.ts b/src/daemon/daemon-id.ts index cf1f6001..0293f541 100644 --- a/src/daemon/daemon-id.ts +++ b/src/daemon/daemon-id.ts @@ -1,22 +1,14 @@ +import { randomUUID } from "node:crypto"; import { hostname } from "node:os"; let cached: string | undefined; /** - * Stable identifier for this daemon process. Matches the format used by - * `src/daemon/main.ts` so the value the daemon registers with the - * orchestrator (and publishes its Valkey heartbeat under) is the same value - * `workflow-executor` writes to `workflow_runs.owner_id`. - * - * The liveness reaper resolves the heartbeat key as `daemon:{owner_id}`. + * Stable identifier for this shared daemon process. The daemon registers and + * publishes its Valkey heartbeat with this same value. */ export function getDaemonId(): string { if (cached !== undefined) return cached; - cached = `daemon-${hostname()}-${String(process.pid)}`; + cached = `daemon-${hostname()}-${randomUUID()}`; return cached; } - -/** Test-only: clear the cached value so a fresh hostname/pid is read. */ -export function resetDaemonIdForTests(): void { - cached = undefined; -} diff --git a/src/daemon/job-executor.ts b/src/daemon/job-executor.ts index 4b2b88d1..47aae38c 100644 --- a/src/daemon/job-executor.ts +++ b/src/daemon/job-executor.ts @@ -16,7 +16,6 @@ import { WS_REJECT_REASONS, } from "../shared/ws-messages"; import { DAEMON_JOB_LOG_EVENTS } from "./log-fields"; -import { executeWorkflowRun } from "./workflow-executor"; // Active job tracking (FM-9) @@ -126,7 +125,7 @@ export function evaluateOffer( } /** - * Evaluate a `scoped-job-offer`. Scoped jobs do not declare `requiredTools` + * Evaluate a `scoped-job:offer`. Scoped jobs do not declare `requiredTools` * (the four executors only need git + bun, both baseline) and do not consume * the legacy concurrency slot until `job:accept` is sent. The evaluator's * single responsibility is forward-compat: reject jobKinds this image does @@ -246,7 +245,7 @@ export async function executeJob( // executor uses. if (payload.payload.scoped !== undefined) { // Register an AbortController so handleJobCancel can suppress the - // duplicate `scoped-job-completion` that runScopedJob would otherwise + // duplicate `scoped-job:completion` that runScopedJob would otherwise // send after the cancel path's synthetic `job:result`. Without this, // a cancel during a scoped run double-finalizes the execution and // double-decrements the orchestrator capacity counter. @@ -272,14 +271,6 @@ export async function executeJob( if (!validateJobContext(context, offerId, send)) return; - // Workflow-run jobs route through a registry-driven executor instead of - // the legacy single-shot pipeline. Everything downstream of this branch - // assumes a traditional BotContext pipeline run. - if (payload.payload.workflowRun !== undefined) { - await executeWorkflowRun(payload, send); - return; - } - const { installationToken, installationId, @@ -288,6 +279,7 @@ export async function executeJob( envVars, memory, reviewLearnings, + policy, } = payload.payload; // Abort controller for cancel/execute race prevention (C1) @@ -375,6 +367,9 @@ export async function executeJob( // even if the pipeline does not finish its own cleanup. const result = await runPipeline(fullCtx, { ...(maxTurns !== undefined ? { maxTurns } : {}), + // Gate-2 knobs on the direct rail. The workflow rail threads the same + // object through WorkflowRunContext instead. + ...(policy !== undefined ? { policy } : {}), allowedTools, onWorkDirReady: (wd: string) => { job.workDir = wd; @@ -469,7 +464,7 @@ export async function executeJob( * surfaces a clean halt rather than a silent drop. * * Routes via the Zod-validated `payload.scoped.jobKind` discriminator, - * matches the `scoped-job-offer` schema at the WS boundary, so a misrouted + * matches the `scoped-job:offer` schema at the WS boundary, so a misrouted * payload is impossible by construction. */ async function runScopedJob( @@ -488,10 +483,10 @@ async function runScopedJob( const startedAt = Date.now(); const installationToken = payload.payload.installationToken; - // Wrap `send` so every scoped-job-completion is suppressed after a cancel, + // Wrap `send` so every scoped-job:completion is suppressed after a cancel, // matching the legacy executor's abort-then-skip-result pattern. The cancel // path emits its own synthetic `job:result`; we MUST NOT also emit a - // scoped-job-completion or the orchestrator will double-decrement capacity. + // scoped-job:completion or the orchestrator will double-decrement capacity. const sendIfNotAborted = (msg: unknown): void => { if (signal.aborted) return; send(msg); @@ -541,7 +536,7 @@ async function runScopedJob( } })(); sendIfNotAborted({ - type: "scoped-job-completion", + type: "scoped-job:completion", ...createMessageEnvelope(offerId), payload: { offerId, @@ -565,7 +560,7 @@ async function runScopedJob( triggerCommentId: scoped.triggerCommentId, }); sendIfNotAborted({ - type: "scoped-job-completion", + type: "scoped-job:completion", ...createMessageEnvelope(offerId), payload: { offerId, @@ -595,7 +590,7 @@ async function runScopedJob( verdictSummary: scoped.verdictSummary, }); sendIfNotAborted({ - type: "scoped-job-completion", + type: "scoped-job:completion", ...createMessageEnvelope(offerId), payload: { offerId, @@ -629,7 +624,7 @@ async function runScopedJob( ...(scoped.allowedTools !== undefined ? { allowedTools: scoped.allowedTools } : {}), }); sendIfNotAborted({ - type: "scoped-job-completion", + type: "scoped-job:completion", ...createMessageEnvelope(offerId), payload: { offerId, @@ -659,7 +654,7 @@ async function runScopedJob( "runScopedJob received unknown jobKind", ); sendIfNotAborted({ - type: "scoped-job-completion", + type: "scoped-job:completion", ...createMessageEnvelope(offerId), payload: { offerId, @@ -689,7 +684,7 @@ async function runScopedJob( "scoped-job execution failed", ); sendIfNotAborted({ - type: "scoped-job-completion", + type: "scoped-job:completion", ...createMessageEnvelope(offerId), payload: { offerId, diff --git a/src/daemon/main.ts b/src/daemon/main.ts index 01c36a48..cf303c7f 100644 --- a/src/daemon/main.ts +++ b/src/daemon/main.ts @@ -20,6 +20,7 @@ import { handleJobCancel, registerExitCleanup, } from "./job-executor"; +import { assertDaemonEnvironmentPrivate } from "./process-boundary"; import { discoverCapabilities, getCurrentResources } from "./tool-discovery"; import { DaemonWsClient } from "./ws-client"; @@ -185,7 +186,7 @@ function handleMessage(msg: ServerMessage): void { break; } - case "scoped-job-offer": { + case "scoped-job:offer": { if (draining) { wsClient.send({ type: "job:reject", @@ -345,6 +346,7 @@ function startEphemeralIdleLoop(): void { // Main async function main(): Promise { + assertDaemonEnvironmentPrivate(); logger.info({ daemonId }, "Daemon starting"); // Route crashes through the redacting pino chokepoint instead of the diff --git a/src/daemon/process-boundary-smoke.ts b/src/daemon/process-boundary-smoke.ts new file mode 100644 index 00000000..bca359a8 --- /dev/null +++ b/src/daemon/process-boundary-smoke.ts @@ -0,0 +1,4 @@ +import { assertDaemonEnvironmentPrivate } from "./process-boundary"; + +assertDaemonEnvironmentPrivate(); +process.stdout.write("process boundary smoke passed\n"); diff --git a/src/daemon/process-boundary.ts b/src/daemon/process-boundary.ts new file mode 100644 index 00000000..20095762 --- /dev/null +++ b/src/daemon/process-boundary.ts @@ -0,0 +1,48 @@ +import { platform } from "node:os"; + +const EXPECTED_DENIAL_EXIT_CODE = 77; +const PROCESS_GUARD_PATH = "/usr/local/lib/github-app/daemon-process-guard.so"; + +export function daemonEnvironmentBoundaryFailure(input: { + readonly platform: NodeJS.Platform; + readonly preload: string | undefined; + readonly probeExitCode: number | null; +}): string | null { + if (input.platform !== "linux") return null; + if (input.probeExitCode === EXPECTED_DENIAL_EXIT_CODE) return null; + const guardInstalled = input.preload?.split(/[:\s]+/).includes(PROCESS_GUARD_PATH) === true; + if (!guardInstalled) { + return `daemon process guard is not installed at ${PROCESS_GUARD_PATH}`; + } + if (input.probeExitCode === 0) return "daemon process guard is installed but ineffective"; + return `daemon environment isolation probe failed with exit code ${String(input.probeExitCode)}`; +} + +/** Fail daemon startup if a same-UID child can inspect the parent environment. */ +export function assertDaemonEnvironmentPrivate(): void { + const runtimePlatform = platform(); + if (runtimePlatform !== "linux") return; + + const target = `/proc/${String(process.pid)}/environ`; + const script = ` + const { readFileSync } = require("node:fs"); + try { + readFileSync(${JSON.stringify(target)}); + process.exit(0); + } catch (error) { + process.exit(error?.code === "EACCES" || error?.code === "EPERM" ? 77 : 78); + } + `; + const probe = Bun.spawnSync([process.execPath, "--eval", script], { + env: {}, + stdout: "ignore", + stderr: "ignore", + }); + + const failure = daemonEnvironmentBoundaryFailure({ + platform: runtimePlatform, + preload: process.env["LD_PRELOAD"], + probeExitCode: probe.exitCode, + }); + if (failure !== null) throw new Error(failure); +} diff --git a/src/daemon/scoped-rebase-executor.ts b/src/daemon/scoped-rebase-executor.ts index 15623c01..b5acd9cb 100644 --- a/src/daemon/scoped-rebase-executor.ts +++ b/src/daemon/scoped-rebase-executor.ts @@ -43,7 +43,7 @@ export interface ScopedRebaseExecutorInput { /** * Run the scoped-rebase pipeline end-to-end. Returns the policy-layer * `RebaseOutcome` so the orchestrator-side completion handler can map it - * onto the `scoped-job-completion` payload without re-deriving the result. + * onto the `scoped-job:completion` payload without re-deriving the result. * * @throws when the temp-dir cannot be created or the policy callback * surfaces an unrecoverable git error other than a conflict. diff --git a/src/daemon/workflow-executor.ts b/src/daemon/workflow-executor.ts deleted file mode 100644 index 8e07732c..00000000 --- a/src/daemon/workflow-executor.ts +++ /dev/null @@ -1,447 +0,0 @@ -import { Octokit } from "octokit"; - -import { logger } from "../logger"; -import type { SerializableBotContext } from "../shared/daemon-types"; -import { createMessageEnvelope, type JobPayloadMessage } from "../shared/ws-messages"; -import { addReaction, type ReactionContent } from "../utils/reactions"; -import { - logWorkflowRunFailed, - logWorkflowRunHandedOff, - logWorkflowRunIncomplete, - logWorkflowRunRunning, - logWorkflowRunSucceeded, -} from "../workflows/log-fields"; -import { type CompletionResult, onStepComplete } from "../workflows/orchestrator"; -import { getByName, type WorkflowRunContext } from "../workflows/registry"; -import { - markFailed, - markIncomplete, - markRunning, - markSucceeded, - mergeState, -} from "../workflows/runs-store"; -import { setState } from "../workflows/tracking-mirror"; -import { getDaemonId } from "./daemon-id"; - -/** - * Daemon-side entry point for jobs carrying a `workflowRun` field. Routes by - * job type: - * - * 1. Resolve registry entry by `workflowRun.workflowName`. - * 2. Build `WorkflowRunContext` (logger + octokit + deliveryId + setState). - * 3. `runs-store.markRunning(runId)`. - * 4. Invoke handler. - * 5. Translate `HandlerResult` → `markSucceeded` | `markFailed` plus a final - * `tracking-mirror.setState` write. - * 6. On uncaught throw → `markFailed({ reason: "uncaught: " })`. - * 7. Send `job:result` back to orchestrator. - * - * Structured log bindings (T024): every log line emitted from `log` carries - * `{ workflowRunId, workflowName, target, deliveryId, offerId }`. - */ -export async function executeWorkflowRun( - payload: JobPayloadMessage, - send: (msg: unknown) => void, -): Promise { - const offerId = payload.id; - const startedAt = Date.now(); - const workflowRun = payload.payload.workflowRun; - const context = payload.payload.context as unknown as SerializableBotContext; - const installationToken = payload.payload.installationToken; - const reviewLearnings = payload.payload.reviewLearnings; - - if (workflowRun === undefined) { - // Defensive, `executeJob` already branches on this; if we get here the - // caller routed an ordinary pipeline job to the wrong executor. - logger.error({ offerId }, "executeWorkflowRun called without workflowRun, misrouted payload"); - return; - } - - const target = { - type: context.isPR ? ("pr" as const) : ("issue" as const), - owner: context.owner, - repo: context.repo, - number: context.entityNumber, - }; - - const log = logger.child({ - offerId, - workflowRunId: workflowRun.runId, - workflowName: workflowRun.workflowName, - deliveryId: context.deliveryId, - // Flat canonical field so per-entity grep aligns with the rest of the - // fleet; `target` stays as the workflow envelope (type + owner/repo). - entityNumber: context.entityNumber, - target, - }); - - const octokit = new Octokit({ auth: installationToken }); - - // Best-effort reaction on the user's trigger comment. No-op for child runs - // (commentId === 0 because children inherit nothing from the parent's - // dispatch payload) and for label-triggered runs (no comment exists). - const reactOnTrigger = (content: ReactionContent): void => { - if (context.commentId === 0) return; - void addReaction({ - octokit, - logger: log, - owner: context.owner, - repo: context.repo, - commentId: context.commentId, - eventType: context.eventName, - content, - }); - }; - - try { - const entry = getByName(workflowRun.workflowName); - const daemonId = getDaemonId(); - await markRunning(workflowRun.runId, daemonId); - logWorkflowRunRunning(log, { - runId: workflowRun.runId, - workflowName: workflowRun.workflowName, - target, - deliveryId: context.deliveryId, - }); - - const runCtx: WorkflowRunContext = { - runId: workflowRun.runId, - workflowName: workflowRun.workflowName, - target, - ...(workflowRun.parentRunId !== undefined && workflowRun.parentStepIndex !== undefined - ? { parent: { runId: workflowRun.parentRunId, stepIndex: workflowRun.parentStepIndex } } - : {}), - logger: log, - octokit, - deliveryId: context.deliveryId, - daemonId, - // The review and resolve handlers pull this off ctx and forward into - // botCtx so runPipeline's prompt-builder + MCP server see the data. - // Other handlers ignore. Without this thread, the orchestrator's load - // is wired through the wire and discarded here: silent feature breakage. - ...(reviewLearnings !== undefined ? { reviewLearnings } : {}), - setState: async (state, humanMessage) => { - const patch = - typeof state === "object" && state !== null - ? (state as Record) - : { state }; - await setState({ octokit, logger: log }, { runId: workflowRun.runId, patch, humanMessage }); - }, - }; - - const result = await entry.handler(runCtx); - - if (result.status === "handed-off") { - // Composite parent: merge state, emit tracking comment, but leave - // `status = running`. The orchestrator cascade will flip this row's - // status once the final descendant completes. No `onStepComplete` - // call here: this run has no terminal result to propagate yet. - const handOffState = - typeof result.state === "object" && result.state !== null - ? (result.state as Record) - : {}; - await mergeState(workflowRun.runId, handOffState); - try { - await setState( - { octokit, logger: log }, - { - runId: workflowRun.runId, - patch: {}, - humanMessage: - result.humanMessage ?? `${entry.name} handed off to child ${result.childRunId}`, - }, - ); - } catch (mirrorErr) { - log.warn( - { err: mirrorErr instanceof Error ? mirrorErr.message : String(mirrorErr) }, - "Tracking-mirror update failed after hand-off; DB state is authoritative", - ); - } - - logWorkflowRunHandedOff(log, { - runId: workflowRun.runId, - workflowName: workflowRun.workflowName, - target, - deliveryId: context.deliveryId, - durationMs: Date.now() - startedAt, - childRunId: result.childRunId, - }); - log.info( - { - durationMs: Date.now() - startedAt, - outcome: "handed-off", - childRunId: result.childRunId, - }, - "Workflow run handed off to child", - ); - - send({ - type: "job:result", - ...createMessageEnvelope(offerId), - payload: { - success: true, - deliveryId: context.deliveryId, - durationMs: Date.now() - startedAt, - }, - }); - return; - } - - let completion: CompletionResult; - - if (result.status === "succeeded") { - const state = - typeof result.state === "object" && result.state !== null - ? (result.state as Record) - : {}; - await markSucceeded(workflowRun.runId, state); - try { - await setState( - { octokit, logger: log }, - { - runId: workflowRun.runId, - patch: {}, - humanMessage: result.humanMessage ?? `${entry.name} succeeded`, - }, - ); - } catch (mirrorErr) { - log.warn( - { err: mirrorErr instanceof Error ? mirrorErr.message : String(mirrorErr) }, - "Tracking-mirror update failed after markSucceeded; DB state is authoritative", - ); - } - - logWorkflowRunSucceeded(log, { - runId: workflowRun.runId, - workflowName: workflowRun.workflowName, - target, - deliveryId: context.deliveryId, - durationMs: Date.now() - startedAt, - }); - log.info( - { durationMs: Date.now() - startedAt, outcome: "succeeded" }, - "Workflow run completed", - ); - - reactOnTrigger("hooray"); - - completion = { status: "succeeded" }; - - // Forward applied review-learning IDs (when the handler is review or - // resolve and the pipeline actually rendered any) so the orchestrator - // bumps `use_count`. Direct-pipeline path forwards the same field in - // job-executor.ts; without it here, workflow-dispatched review/resolve - // jobs silently skip the bump even when the prompt block fired. - const appliedReviewLearningIds = result.appliedReviewLearningIds; - send({ - type: "job:result", - ...createMessageEnvelope(offerId), - payload: { - success: true, - deliveryId: context.deliveryId, - durationMs: Date.now() - startedAt, - ...(appliedReviewLearningIds !== undefined && appliedReviewLearningIds.length > 0 - ? { appliedReviewLearningIds } - : {}), - }, - }); - } else if (result.status === "incomplete") { - // Handler returned `incomplete`: the agent ran cleanly but a handler- - // side gate (e.g. resolve's post-pipeline CI re-check) found surviving - // failures. We persist a distinct DB status so operators can tell a - // clean-run-but-blocked outcome from a true pipeline error, but the - // orchestrator cascade still needs a binary outcome, propagate it as - // a non-success completion with the original reason preserved. - const incState = - typeof result.state === "object" && result.state !== null - ? (result.state as Record) - : {}; - await markIncomplete(workflowRun.runId, result.reason, incState); - try { - await setState( - { octokit, logger: log }, - { - runId: workflowRun.runId, - patch: {}, - humanMessage: - result.humanMessage ?? - `${entry.name} incomplete, see tracking comment for outstanding items.`, - }, - ); - } catch (mirrorErr) { - log.warn( - { err: mirrorErr instanceof Error ? mirrorErr.message : String(mirrorErr) }, - "Tracking-mirror update failed after markIncomplete; DB state is authoritative", - ); - } - - logWorkflowRunIncomplete(log, { - runId: workflowRun.runId, - workflowName: workflowRun.workflowName, - target, - deliveryId: context.deliveryId, - durationMs: Date.now() - startedAt, - reason: result.reason, - }); - log.warn( - { - durationMs: Date.now() - startedAt, - outcome: "incomplete", - reason: result.reason, - }, - "Workflow run reported incomplete", - ); - - reactOnTrigger("confused"); - - completion = { status: "failed", reason: `incomplete: ${result.reason}` }; - - // Same bump rationale as the succeeded branch above: even when the - // handler reports `incomplete`, the pipeline did render the directives, - // so the orchestrator should still credit the use_count. - const appliedReviewLearningIds = result.appliedReviewLearningIds; - send({ - type: "job:result", - ...createMessageEnvelope(offerId), - payload: { - success: false, - deliveryId: context.deliveryId, - durationMs: Date.now() - startedAt, - errorMessage: `incomplete: ${result.reason}`, - ...(appliedReviewLearningIds !== undefined && appliedReviewLearningIds.length > 0 - ? { appliedReviewLearningIds } - : {}), - }, - }); - } else { - // Handler returned `failed`. - const failState = - typeof result.state === "object" && result.state !== null - ? (result.state as Record) - : {}; - await markFailed(workflowRun.runId, result.reason, failState); - try { - await setState( - { octokit, logger: log }, - { - runId: workflowRun.runId, - patch: {}, - // Defense-in-depth: never default to `result.reason` for the - // public comment, handlers may put raw error messages in - // `reason` (intended for DB state.failedReason and operator - // logs only). Handlers that want a richer comment must set - // `humanMessage` explicitly. - humanMessage: - result.humanMessage ?? `${entry.name} failed, see server logs for details.`, - }, - ); - } catch (mirrorErr) { - log.warn( - { err: mirrorErr instanceof Error ? mirrorErr.message : String(mirrorErr) }, - "Tracking-mirror update failed after markFailed; DB state is authoritative", - ); - } - - logWorkflowRunFailed(log, { - runId: workflowRun.runId, - workflowName: workflowRun.workflowName, - target, - deliveryId: context.deliveryId, - durationMs: Date.now() - startedAt, - reason: result.reason, - }); - log.warn( - { durationMs: Date.now() - startedAt, outcome: "failed", reason: result.reason }, - "Workflow run reported failure", - ); - - reactOnTrigger("confused"); - - completion = { status: "failed", reason: result.reason }; - - send({ - type: "job:result", - ...createMessageEnvelope(offerId), - payload: { - success: false, - deliveryId: context.deliveryId, - durationMs: Date.now() - startedAt, - errorMessage: result.reason, - }, - }); - } - - // T030: propagate the terminal result up the composite chain. Wrapped - // so a cascade error never masks the original handler outcome, the - // daemon has already ack'd the job above. - try { - await onStepComplete({ octokit, logger: log }, workflowRun.runId, completion); - } catch (cascadeErr) { - log.error({ err: cascadeErr }, "onStepComplete cascade failed"); - } - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - const reason = `uncaught: ${message}`; - - try { - await markFailed(workflowRun.runId, reason, {}); - await setState( - { octokit, logger: log }, - { - runId: workflowRun.runId, - patch: {}, - // Public comment must NOT carry the raw uncaught-throw message. - // octokit error stacks include the request URL with the - // installation token (`https://x-access-token:GHS_xxx@…`); other - // throws may surface file paths or env values. Raw `reason` is - // still persisted to DB state.failedReason via markFailed above. - humanMessage: `${workflowRun.workflowName} failed, see server logs for details.`, - }, - ); - } catch (cleanupErr) { - log.error( - { err: cleanupErr }, - "Failed to persist failure state after uncaught handler throw", - ); - } - - logWorkflowRunFailed( - log, - { - runId: workflowRun.runId, - workflowName: workflowRun.workflowName, - target, - deliveryId: context.deliveryId, - durationMs: Date.now() - startedAt, - reason, - }, - "error", - ); - log.error( - { err, durationMs: Date.now() - startedAt, outcome: "uncaught" }, - "Workflow handler threw", - ); - - reactOnTrigger("confused"); - - try { - await onStepComplete({ octokit, logger: log }, workflowRun.runId, { - status: "failed", - reason, - }); - } catch (cascadeErr) { - log.error({ err: cascadeErr }, "onStepComplete cascade failed after uncaught"); - } - - send({ - type: "job:result", - ...createMessageEnvelope(offerId), - payload: { - success: false, - deliveryId: context.deliveryId, - durationMs: Date.now() - startedAt, - errorMessage: reason, - }, - }); - } -} diff --git a/src/daemon/ws-client.ts b/src/daemon/ws-client.ts index 843d5efb..140f1672 100644 --- a/src/daemon/ws-client.ts +++ b/src/daemon/ws-client.ts @@ -7,6 +7,7 @@ import { PROTOCOL_VERSION, type ServerMessage, serverMessageSchema, + WS_CLOSE_CODES, } from "../shared/ws-messages"; import { redactErrorMessage } from "../utils/log-redaction"; import { DAEMON_CONNECTION_LOG_EVENTS } from "./log-fields"; @@ -153,7 +154,9 @@ export class DaemonWsClient { }, "Disconnected from orchestrator", ); - if (!this.closed) { + if (event.code === WS_CLOSE_CODES.INCOMPATIBLE_PROTOCOL.code) { + this.closed = true; + } else if (!this.closed) { this.scheduleReconnect(); } }; diff --git a/src/db/migrations/017_workflow_run_leases.sql b/src/db/migrations/017_workflow_run_leases.sql new file mode 100644 index 00000000..4230e42a --- /dev/null +++ b/src/db/migrations/017_workflow_run_leases.sql @@ -0,0 +1,336 @@ +-- Add durable receipts and leases for one isolated workflow runner attempt. + +-- Fail quickly instead of waiting behind an unrelated long-running table lock. +SET LOCAL lock_timeout = '5s'; + +ALTER TABLE workflow_runs + ADD COLUMN attempt_id UUID NULL, + ADD COLUMN lease_expires_at TIMESTAMPTZ NULL, + ADD COLUMN attempt_deadline_at TIMESTAMPTZ NULL, + ADD COLUMN attempt_completed_at TIMESTAMPTZ NULL, + ADD COLUMN cascade_completed_at TIMESTAMPTZ NULL, + ADD COLUMN execution_delivery_id TEXT NULL, + ADD COLUMN trigger_body_preview TEXT NOT NULL DEFAULT '', + ADD COLUMN dispatch_enqueued_at TIMESTAMPTZ NULL, + ADD COLUMN dispatch_generation_id UUID NULL, + ADD COLUMN runner_payload_issued_at TIMESTAMPTZ NULL, + ADD COLUMN runner_token_expires_at TIMESTAMPTZ NULL, + ADD COLUMN runner_resources_cleaned_at TIMESTAMPTZ NULL, + ADD COLUMN failure_notified_at TIMESTAMPTZ NULL, + ADD COLUMN dispatch_retry_count INTEGER NOT NULL DEFAULT 0 + CHECK (dispatch_retry_count >= 0); + +-- A volatile ADD COLUMN default rewrites every existing row while ALTER TABLE +-- holds its strongest lock. Backfill explicitly, then make future inserts use +-- the default. +UPDATE workflow_runs + SET dispatch_generation_id = gen_random_uuid() + WHERE dispatch_generation_id IS NULL; + +ALTER TABLE workflow_runs + ALTER COLUMN dispatch_generation_id SET DEFAULT gen_random_uuid(), + ALTER COLUMN dispatch_generation_id SET NOT NULL; + +ALTER TABLE workflow_runs + ADD CONSTRAINT workflow_runs_runner_payload_receipt_check + CHECK ( + (runner_payload_issued_at IS NULL AND runner_token_expires_at IS NULL) + OR ( + runner_payload_issued_at IS NOT NULL + AND runner_token_expires_at IS NOT NULL + AND attempt_deadline_at IS NOT NULL + AND runner_token_expires_at <= attempt_deadline_at + ) + ); + +ALTER TABLE executions + ADD COLUMN offer_id UUID NULL, + ADD COLUMN result_processed_at TIMESTAMPTZ NULL, + ADD COLUMN workflow_result_payload JSONB NULL; + +ALTER TABLE executions + DROP CONSTRAINT executions_dispatch_target_check, + DROP CONSTRAINT executions_dispatch_mode_check, + DROP CONSTRAINT executions_dispatch_reason_check; + +ALTER TABLE executions + ADD CONSTRAINT executions_dispatch_target_check + CHECK (dispatch_target IN ('daemon', 'workflow-runner')), + ADD CONSTRAINT executions_dispatch_mode_check + CHECK (dispatch_mode IN ('daemon', 'workflow-runner')), + ADD CONSTRAINT executions_dispatch_reason_check + CHECK (dispatch_reason IN ( + 'persistent-daemon', + 'ephemeral-daemon-triage', + 'ephemeral-daemon-overflow', + 'ephemeral-spawn-failed', + 'workflow-runner' + )); + +CREATE TABLE workflow_attempt_commands ( + attempt_id UUID NOT NULL, + command_id UUID NOT NULL, + run_id UUID NOT NULL REFERENCES workflow_runs (id) ON DELETE CASCADE, + command_kind TEXT NOT NULL CHECK (command_kind IN ('set-state', 'hand-off-child')), + request JSONB NOT NULL, + response JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (attempt_id, command_id) +); + +-- The execution key can be recovered from every producer shape that existed +-- before this migration: top-level delivery, composite child id, or +-- ship-iteration state. Mark non-queued history as already published. +UPDATE workflow_runs + SET execution_delivery_id = CASE + WHEN parent_run_id IS NOT NULL THEN id::text + WHEN delivery_id IS NOT NULL THEN delivery_id + WHEN state ? 'shipIntentId' AND state ? 'iteration_n' + THEN (state ->> 'shipIntentId') || '::iteration::' || (state ->> 'iteration_n') + ELSE NULL + END, + dispatch_enqueued_at = now(); + +-- Queued executions will run on the isolated protocol after this migration. +-- Historical terminal or already-active rows retain the protocol that +-- actually executed them. +UPDATE executions AS e + SET dispatch_mode = 'workflow-runner', + dispatch_target = 'workflow-runner', + dispatch_reason = 'workflow-runner' + FROM workflow_runs AS wr + WHERE wr.execution_delivery_id = e.delivery_id + AND wr.status = 'queued' + AND e.status = 'queued'; + +-- A queued row may have committed immediately before a crash without reaching +-- Valkey. Reopen every reconstructable row; duplicate copies lose at the exact +-- execution-offer claim and are consumed without affecting the winner. +UPDATE workflow_runs + SET dispatch_enqueued_at = NULL + WHERE status = 'queued' + AND execution_delivery_id IS NOT NULL + AND EXISTS ( + SELECT 1 + FROM executions AS e + WHERE e.delivery_id = workflow_runs.execution_delivery_id + AND e.status = 'queued' + ); + +-- A queued row without its execution half cannot be reconstructed. Fail it so +-- the in-flight target guard does not block a deliberate retry forever. +WITH irrecoverable AS ( + UPDATE workflow_runs AS wr + SET status = 'failed', + state = state || jsonb_build_object( + 'phase', 'migration-interrupted', + 'failedReason', 'workflow dispatch incomplete during lease migration' + ), + owner_kind = NULL, + owner_id = NULL, + attempt_completed_at = now() + WHERE wr.status = 'queued' + AND ( + wr.execution_delivery_id IS NULL + OR NOT EXISTS ( + SELECT 1 + FROM executions AS e + WHERE e.delivery_id = wr.execution_delivery_id + ) + ) + RETURNING wr.id, wr.parent_run_id, wr.parent_step_index +), +irrecoverable_parent_inputs AS ( + SELECT parent_run_id, + min(COALESCE(parent_step_index, -1)) AS failed_step_index + FROM irrecoverable + WHERE parent_run_id IS NOT NULL + GROUP BY parent_run_id +) +UPDATE workflow_runs AS parent + SET status = 'failed', + state = state || jsonb_build_object( + 'failedAtStepIndex', irrecoverable_parent_inputs.failed_step_index, + 'failedReason', 'workflow dispatch incomplete during lease migration' + ) + FROM irrecoverable_parent_inputs + WHERE parent.id = irrecoverable_parent_inputs.parent_run_id + AND parent.status = 'running' + AND NOT EXISTS ( + SELECT 1 FROM irrecoverable AS direct WHERE direct.id = parent.id + ); + +-- A shared daemon cannot hand an active workflow attempt to the isolated +-- runner protocol. Fail only rows whose own execution receipt is active. +-- A running composite parent whose receipt is already complete is preserved, +-- allowing its queued child to resume on an isolated runner. +WITH interrupted_workflows AS ( + UPDATE workflow_runs AS wr + SET status = 'failed', + state = state || jsonb_build_object( + 'phase', 'migration-interrupted', + 'failedReason', 'workflow execution interrupted during isolated-runner migration' + ), + owner_kind = NULL, + owner_id = NULL, + lease_expires_at = NULL, + attempt_completed_at = now() + FROM executions AS e + WHERE e.delivery_id = wr.execution_delivery_id + AND wr.status IN ('queued', 'running') + AND e.status IN ('offered', 'running') + RETURNING wr.id, wr.parent_run_id, wr.parent_step_index, wr.execution_delivery_id +), +interrupted_parent_inputs AS ( + SELECT parent_run_id, + min(COALESCE(parent_step_index, -1)) AS failed_step_index + FROM interrupted_workflows + WHERE parent_run_id IS NOT NULL + GROUP BY parent_run_id +), +failed_parents AS ( + UPDATE workflow_runs AS parent + SET status = 'failed', + state = state || jsonb_build_object( + 'failedAtStepIndex', interrupted_parent_inputs.failed_step_index, + 'failedReason', 'workflow execution interrupted during isolated-runner migration' + ) + FROM interrupted_parent_inputs + WHERE parent.id = interrupted_parent_inputs.parent_run_id + AND parent.status = 'running' + AND NOT EXISTS ( + SELECT 1 FROM interrupted_workflows AS direct WHERE direct.id = parent.id + ) + RETURNING parent.id +), +interrupted_executions AS ( + UPDATE executions AS e + SET status = 'failed', + completed_at = now(), + error_message = 'workflow execution interrupted during isolated-runner migration', + result_processed_at = now() + FROM interrupted_workflows + WHERE e.delivery_id = interrupted_workflows.execution_delivery_id + RETURNING e.delivery_id +) +UPDATE scheduled_action_state + SET in_flight_job_id = NULL, + in_flight_started_at = NULL + WHERE in_flight_job_id IN (SELECT delivery_id FROM interrupted_executions); + +-- A composite parent whose own receipt is already complete remains the +-- durable coordinator for its queued child. It must no longer carry the +-- shared daemon identity that performed the hand-off. +UPDATE workflow_runs AS parent + SET owner_kind = NULL, + owner_id = NULL, + lease_expires_at = NULL + WHERE parent.status = 'running' + AND parent.owner_kind = 'daemon' + AND NOT EXISTS ( + SELECT 1 + FROM executions AS e + WHERE e.delivery_id = parent.execution_delivery_id + AND e.status IN ('offered', 'running') + ) + AND EXISTS ( + SELECT 1 + FROM workflow_runs AS child + WHERE child.parent_run_id = parent.id + AND child.status = 'queued' + ); + +CREATE INDEX idx_workflow_runs_lease_expiry + ON workflow_runs (LEAST(lease_expires_at, attempt_deadline_at)) + WHERE status = 'running' + AND lease_expires_at IS NOT NULL + AND attempt_deadline_at IS NOT NULL; + +CREATE UNIQUE INDEX idx_workflow_runs_attempt_id + ON workflow_runs (attempt_id) + WHERE attempt_id IS NOT NULL; + +CREATE INDEX idx_workflow_runs_dispatch_pending + ON workflow_runs (dispatch_enqueued_at NULLS FIRST, created_at) + WHERE status = 'queued' + AND execution_delivery_id IS NOT NULL; + +ALTER TABLE repo_memory + ADD COLUMN content_sha256 BYTEA NULL; + +UPDATE repo_memory + SET content_sha256 = pg_catalog.sha256(pg_catalog.convert_to(content, 'UTF8')) + WHERE category <> 'env_var'; + +ALTER TABLE repo_memory + ADD CONSTRAINT repo_memory_learning_hash_check + CHECK (category = 'env_var' OR content_sha256 IS NOT NULL); + +CREATE OR REPLACE FUNCTION set_repo_memory_content_sha256() +RETURNS trigger +LANGUAGE plpgsql +SET search_path = pg_catalog +AS $$ +BEGIN + NEW.content_sha256 := CASE + WHEN NEW.category = 'env_var' THEN NULL + ELSE pg_catalog.sha256(pg_catalog.convert_to(NEW.content, 'UTF8')) + END; + RETURN NEW; +END; +$$; + +CREATE TRIGGER repo_memory_content_sha256 +BEFORE INSERT OR UPDATE OF category, content ON repo_memory +FOR EACH ROW +EXECUTE FUNCTION set_repo_memory_content_sha256(); + +-- Result replay can persist the same learning concurrently. Keep the most +-- useful existing copy, then let PostgreSQL enforce the durable invariant. +WITH duplicate_learning AS ( + SELECT id, + row_number() OVER ( + PARTITION BY repo_owner, repo_name, category, content + ORDER BY pinned DESC, updated_at DESC, created_at DESC, id + ) AS duplicate_number + FROM repo_memory + WHERE category <> 'env_var' +) +DELETE FROM repo_memory AS memory + USING duplicate_learning AS duplicate + WHERE memory.id = duplicate.id + AND duplicate.duplicate_number > 1; + +CREATE UNIQUE INDEX idx_repo_memory_learning_unique + ON repo_memory ( + repo_owner, + repo_name, + category, + content_sha256 + ) + WHERE category <> 'env_var'; + +CREATE INDEX idx_workflow_runs_runner_cleanup_pending + ON workflow_runs (attempt_completed_at) + WHERE attempt_id IS NOT NULL + AND attempt_completed_at IS NOT NULL + AND runner_resources_cleaned_at IS NULL; + +CREATE INDEX idx_workflow_runs_failure_notification_pending + ON workflow_runs (attempt_completed_at) + WHERE status = 'failed' + AND attempt_completed_at IS NOT NULL + AND failure_notified_at IS NULL; + +CREATE UNIQUE INDEX idx_executions_offer_id + ON executions (offer_id) + WHERE offer_id IS NOT NULL; + +CREATE INDEX idx_executions_running_daemon + ON executions (daemon_id) + WHERE status = 'running' AND daemon_id IS NOT NULL; + +CREATE INDEX idx_executions_workflow_result_pending + ON executions (completed_at) + WHERE workflow_result_payload IS NOT NULL + AND result_processed_at IS NULL; diff --git a/src/k8s/ephemeral-daemon-spawner.ts b/src/k8s/ephemeral-daemon-spawner.ts index 82c8ddab..2017d325 100644 --- a/src/k8s/ephemeral-daemon-spawner.ts +++ b/src/k8s/ephemeral-daemon-spawner.ts @@ -5,6 +5,8 @@ import { logger } from "../logger"; import { K8S_SPAWN_LOG_EVENTS } from "../orchestrator/k8s-spawn-log-fields"; import { redactErrorMessage } from "../utils/log-redaction"; +const DAEMON_PROCESS_GUARD_PATH = "/usr/local/lib/github-app/daemon-process-guard.so"; + /** * Typed errors the ephemeral-daemon spawner can throw. Distinguishing * `infra-absent` from generic K8s API failures lets the router map them @@ -30,7 +32,7 @@ export class EphemeralSpawnError extends Error { let cachedClient: { core: CoreV1Api } | undefined; -function loadKubernetesClient(): { core: CoreV1Api } { +export function loadKubernetesClient(): { core: CoreV1Api } { if (cachedClient !== undefined) return cachedClient; const kc = new KubeConfig(); @@ -50,10 +52,10 @@ function loadKubernetesClient(): { core: CoreV1Api } { } else { kc.loadFromDefault(); } - } catch (err) { + } catch { throw new EphemeralSpawnError( "auth-load-failed", - `Failed to load Kubernetes config: ${err instanceof Error ? err.message : String(err)}`, + "Kubernetes authentication configuration could not be loaded", ); } @@ -105,9 +107,9 @@ export interface SpawnEphemeralDaemonInput { * - `activeDeadlineSeconds`: belt-and-suspenders: the daemon self-exits * on idle, but a bug in the idle loop must not leak a long-running Pod. * - `DAEMON_EPHEMERAL=true`: flips the daemon into idle-exit mode. - * - `envFrom: secretRef: daemon-secrets`, carries the full runtime - * credential set (GitHub App, Claude, DB, Valkey). The operator is - * expected to provision this Secret out-of-band. + * - `envFrom: secretRef: `, carries provider and + * handshake credentials. The operator provisions this Secret out-of-band and + * chooses its scope. */ // K8s label values must match `([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]` and // be ≤63 chars. Strip invalid chars and trim leading/trailing non-alphanum so a @@ -172,14 +174,13 @@ function buildEphemeralDaemonPodSpec(input: SpawnEphemeralDaemonInput): V1Pod { name: "daemon", image: input.image, command: ["bun", "run", "dist/daemon/main.js"], - // The ephemeral-daemon Pod mounts ONLY the `daemon-secrets` Secret - //, never `orchestrator-secrets`. The orchestrator/daemon split - // (defense layer 1b for prompt-injection hardening, issue #102) is - // enforced by the Helm chart: orchestrator-only credentials - // (`GITHUB_APP_PRIVATE_KEY`, `GITHUB_WEBHOOK_SECRET`, `DATABASE_URL`, - // `VALKEY_URL`, `CONTEXT7_API_KEY`) live in `orchestrator-secrets` - // and never reach a daemon Pod. The daemon needs only: - // - `DAEMON_AUTH_TOKEN[_PREVIOUS]` (WS handshake) + // The Pod mounts exactly one Secret, named by + // `EPHEMERAL_DAEMON_SECRET_NAME` and provisioned out-of-band. Scoping + // it to daemon-only credentials is a deployment decision, not one this + // code can enforce: pointing it at a Secret that also carries GitHub + // App issuance or data-layer credentials hands those to the Pod. The + // daemon needs: + // - `DAEMON_AUTH_TOKEN` (WS handshake) // - `ANTHROPIC_API_KEY` / `CLAUDE_CODE_OAUTH_TOKEN` (Claude auth) // - `AWS_*` chain (Bedrock provider) // - `GITHUB_PERSONAL_ACCESS_TOKEN` (PAT mode only; optional) @@ -189,12 +190,15 @@ function buildEphemeralDaemonPodSpec(input: SpawnEphemeralDaemonInput): V1Pod { env: [ { name: "DAEMON_EPHEMERAL", value: "true" }, { name: "ORCHESTRATOR_URL", value: input.orchestratorUrl }, + // Explicit env entries override envFrom values. Keep the process + // guard pinned if the mounted Secret contains a stale LD_PRELOAD. + { name: "LD_PRELOAD", value: DAEMON_PROCESS_GUARD_PATH }, { name: "EPHEMERAL_DAEMON_IDLE_TIMEOUT_MS", value: String(config.ephemeralDaemonIdleTimeoutMs), }, ], - envFrom: [{ secretRef: { name: "daemon-secrets" } }], + envFrom: [{ secretRef: { name: config.ephemeralDaemonSecretName } }], securityContext: { allowPrivilegeEscalation: false, capabilities: { drop: ["ALL"] }, diff --git a/src/k8s/workflow-runner-spawner.ts b/src/k8s/workflow-runner-spawner.ts new file mode 100644 index 00000000..31eb2ea2 --- /dev/null +++ b/src/k8s/workflow-runner-spawner.ts @@ -0,0 +1,689 @@ +import { isDeepStrictEqual } from "node:util"; + +import type { + V1DeleteOptions, + V1EnvVar, + V1ObjectMeta, + V1Pod, + V1Secret, +} from "@kubernetes/client-node"; + +import { config } from "../config"; +import { logger } from "../logger"; +import { workflowRunnerUrl } from "../orchestrator/workflow-runner-capability"; +import { + WORKFLOW_RUNNER_ATTEMPT_DEADLINE_MS, + type WorkflowRunnerAttempt, +} from "../orchestrator/workflow-runner-store"; +import { + WorkflowRunnerProviderConfigurationError, + workflowRunnerProviderEnv, +} from "../shared/workflow-runner-provider"; +import { loadKubernetesClient } from "./ephemeral-daemon-spawner"; + +const RUNNER_PROVIDER_SECRET = "workflow-runner-secrets"; +const PROCESS_GUARD_PATH = "/usr/local/lib/github-app/daemon-process-guard.so"; +const SHA256_IMAGE_DIGEST = /@sha256:[0-9a-f]{64}$/; +export const WORKFLOW_RUNNER_STORAGE_REQUEST = "2Gi"; +export const WORKFLOW_RUNNER_STORAGE_LIMIT = "10Gi"; +export const WORKFLOW_RUNNER_WORKSPACE_PATH = "/tmp/bot-workspaces"; +// Deployment-configurable so a cluster can target a node pool it already +// labels and taints. Drives the nodeSelector AND the NoSchedule toleration. +export const WORKFLOW_RUNNER_NODE_LABEL = config.workflowRunnerNodeLabel; +export const WORKFLOW_RUNNER_NODE_VALUE = config.workflowRunnerNodeValue; +// Pinned by the admission boundary, so a runner may reference this Secret and no +// other. Empty emits no imagePullSecrets at all, which the boundary also allows. +export const WORKFLOW_RUNNER_IMAGE_PULL_SECRET = config.workflowRunnerImagePullSecret; +interface WorkflowRunnerResourceIdentity { + readonly runId: string; + readonly attemptId: string; +} + +export class WorkflowRunnerResourceError extends Error { + constructor( + readonly kind: "permanent" | "transient", + message: string, + ) { + super(message); + this.name = "WorkflowRunnerResourceError"; + } +} + +function buildProviderEnvironment(): V1EnvVar[] { + try { + return workflowRunnerProviderEnv(config).map((entry) => { + if (entry.secretKey === undefined) { + if (entry.value === undefined) { + throw new WorkflowRunnerProviderConfigurationError( + `Workflow runner provider setting ${entry.name} has no value`, + ); + } + return { name: entry.name, value: entry.value }; + } + return { + name: entry.name, + valueFrom: { + secretKeyRef: { + name: RUNNER_PROVIDER_SECRET, + key: entry.secretKey, + optional: false, + }, + }, + }; + }); + } catch (err) { + if (err instanceof WorkflowRunnerProviderConfigurationError) { + throw new WorkflowRunnerResourceError("permanent", err.message); + } + throw err; + } +} + +export function workflowRunnerResourceNames(attemptId: string): { + readonly podName: string; + readonly secretName: string; +} { + const suffix = attemptId.toLowerCase(); + return { + podName: `workflow-runner-${suffix}`, + secretName: `workflow-runner-${suffix}`, + }; +} + +function labels(attempt: WorkflowRunnerResourceIdentity): Record { + return { + "app.kubernetes.io/name": "github-app", + "app.kubernetes.io/component": "workflow-runner", + "github-app/workflow-run-id": attempt.runId, + "github-app/workflow-attempt-id": attempt.attemptId, + }; +} + +function buildSecret(attempt: WorkflowRunnerAttempt, capability: string, podUid: string): V1Secret { + const { podName, secretName } = workflowRunnerResourceNames(attempt.attemptId); + return { + apiVersion: "v1", + kind: "Secret", + metadata: { + name: secretName, + namespace: config.workflowRunnerNamespace, + labels: labels(attempt), + ownerReferences: [ + { + apiVersion: "v1", + kind: "Pod", + name: podName, + uid: podUid, + controller: true, + blockOwnerDeletion: false, + }, + ], + }, + type: "Opaque", + data: { capability: Buffer.from(capability, "utf8").toString("base64") }, + }; +} + +export function buildWorkflowRunnerPod( + attempt: WorkflowRunnerAttempt, + image: string, + orchestratorUrl: string, +): V1Pod { + assertDigestPinnedRunnerImage(image); + const { podName, secretName } = workflowRunnerResourceNames(attempt.attemptId); + return { + apiVersion: "v1", + kind: "Pod", + metadata: { + name: podName, + namespace: config.workflowRunnerNamespace, + labels: labels(attempt), + }, + spec: { + restartPolicy: "Never", + serviceAccountName: "default", + dnsPolicy: "ClusterFirst", + schedulerName: "default-scheduler", + activeDeadlineSeconds: WORKFLOW_RUNNER_ATTEMPT_DEADLINE_MS / 1_000, + terminationGracePeriodSeconds: 30, + automountServiceAccountToken: false, + enableServiceLinks: false, + hostIPC: false, + hostNetwork: false, + hostPID: false, + shareProcessNamespace: false, + nodeSelector: { [WORKFLOW_RUNNER_NODE_LABEL]: WORKFLOW_RUNNER_NODE_VALUE }, + ...(WORKFLOW_RUNNER_IMAGE_PULL_SECRET === "" + ? {} + : { imagePullSecrets: [{ name: WORKFLOW_RUNNER_IMAGE_PULL_SECRET }] }), + tolerations: [ + { + key: "node.kubernetes.io/not-ready", + operator: "Exists", + effect: "NoExecute", + tolerationSeconds: 300, + }, + { + key: "node.kubernetes.io/unreachable", + operator: "Exists", + effect: "NoExecute", + tolerationSeconds: 300, + }, + { + key: WORKFLOW_RUNNER_NODE_LABEL, + operator: "Equal", + value: WORKFLOW_RUNNER_NODE_VALUE, + effect: "NoSchedule", + }, + ], + securityContext: { + runAsNonRoot: true, + runAsUser: 1000, + runAsGroup: 1000, + seccompProfile: { type: "RuntimeDefault" }, + }, + volumes: [ + { + name: "workspace", + emptyDir: { sizeLimit: WORKFLOW_RUNNER_STORAGE_LIMIT }, + }, + ], + containers: [ + { + name: "runner", + image, + imagePullPolicy: "IfNotPresent", + command: ["bun", "run", "dist/runner/main.js"], + terminationMessagePath: "/dev/termination-log", + terminationMessagePolicy: "File", + env: [ + { name: "WORKFLOW_RUNNER", value: "true" }, + { name: "WORKFLOW_RUNNER_RUN_ID", value: attempt.runId }, + { name: "WORKFLOW_RUNNER_ATTEMPT_ID", value: attempt.attemptId }, + { + name: "WORKFLOW_RUNNER_TOKEN", + valueFrom: { secretKeyRef: { name: secretName, key: "capability" } }, + }, + { + name: "ORCHESTRATOR_URL", + value: workflowRunnerUrl(orchestratorUrl, attempt.runId, attempt.attemptId), + }, + { name: "LD_PRELOAD", value: PROCESS_GUARD_PATH }, + ...buildProviderEnvironment(), + ], + securityContext: { + allowPrivilegeEscalation: false, + capabilities: { drop: ["ALL"] }, + }, + volumeMounts: [ + { + name: "workspace", + mountPath: WORKFLOW_RUNNER_WORKSPACE_PATH, + }, + ], + resources: { + requests: { + cpu: "500m", + memory: "1Gi", + "ephemeral-storage": WORKFLOW_RUNNER_STORAGE_REQUEST, + }, + limits: { + cpu: "2", + memory: "4Gi", + "ephemeral-storage": WORKFLOW_RUNNER_STORAGE_LIMIT, + }, + }, + }, + ], + }, + }; +} + +function statusCode(err: unknown): number | undefined { + if (err === null || typeof err !== "object") return undefined; + const value = err as { + code?: unknown; + statusCode?: unknown; + response?: { statusCode?: unknown }; + }; + if (typeof value.code === "number") return value.code; + if (typeof value.statusCode === "number") return value.statusCode; + return typeof value.response?.statusCode === "number" ? value.response.statusCode : undefined; +} + +function exactResourceMetadata( + metadata: V1ObjectMeta | undefined, + name: string, + attempt: WorkflowRunnerResourceIdentity, +): boolean { + return ( + metadata?.name === name && + metadata.namespace === config.workflowRunnerNamespace && + isDeepStrictEqual(metadata.labels, labels(attempt)) + ); +} + +// The full boundary must reject any security-relevant Secret drift. +// eslint-disable-next-line complexity +function validateSecretIdentity( + secret: V1Secret, + desired: V1Secret, + attempt: WorkflowRunnerAttempt, +): void { + const name = workflowRunnerResourceNames(attempt.attemptId).secretName; + if (secret.metadata?.deletionTimestamp !== undefined) { + throw new WorkflowRunnerResourceError("transient", `Secret ${name} is terminating`); + } + if ( + secret.apiVersion !== desired.apiVersion || + secret.kind !== desired.kind || + !exactResourceMetadata(secret.metadata, name, attempt) || + secret.metadata?.annotations !== undefined || + secret.metadata?.deletionGracePeriodSeconds !== undefined || + secret.metadata?.finalizers !== undefined || + secret.metadata?.generateName !== undefined || + !isDeepStrictEqual( + plainData(secret.metadata?.ownerReferences), + plainData(desired.metadata?.ownerReferences), + ) || + secret.type !== desired.type || + secret.immutable !== desired.immutable || + secret.stringData !== undefined || + Object.keys(secret.data ?? {}).length !== 1 || + typeof secret.data?.["capability"] !== "string" + ) { + throw new WorkflowRunnerResourceError( + "permanent", + `Existing Secret ${name} does not match workflow attempt ${attempt.attemptId}`, + ); + } +} + +// Create, reconcile, and replace each have distinct permanent/transient outcomes. +// eslint-disable-next-line complexity +async function ensureSecret( + attempt: WorkflowRunnerAttempt, + capability: string, + podUid: string, +): Promise { + const client = loadKubernetesClient().core; + const namespace = config.workflowRunnerNamespace; + const desired = buildSecret(attempt, capability, podUid); + const name = desired.metadata?.name; + if (name === undefined) throw new WorkflowRunnerResourceError("permanent", "Secret name missing"); + try { + const created = await client.createNamespacedSecret({ namespace, body: desired }); + validateSecretIdentity(created, desired, attempt); + if (created.data?.["capability"] !== desired.data?.["capability"]) { + throw new WorkflowRunnerResourceError( + "permanent", + `Created Secret ${name} was mutated by admission`, + ); + } + return; + } catch (err) { + if (err instanceof WorkflowRunnerResourceError) throw err; + if (statusCode(err) !== 409) throw classifyResourceError("create runner Secret", err); + } + + let existing: V1Secret; + try { + existing = await client.readNamespacedSecret({ name, namespace }); + } catch (err) { + throw classifyResourceError("read existing runner Secret", err); + } + validateSecretIdentity(existing, desired, attempt); + if (existing.data?.["capability"] === desired.data?.["capability"]) return; + const desiredData = desired.data; + if (desiredData === undefined) { + throw new WorkflowRunnerResourceError("permanent", "Runner Secret data missing"); + } + try { + existing.data = desiredData; + delete existing.stringData; + const replaced = await client.replaceNamespacedSecret({ + name, + namespace, + body: existing, + }); + validateSecretIdentity(replaced, desired, attempt); + if (replaced.data?.["capability"] !== desired.data?.["capability"]) { + throw new WorkflowRunnerResourceError( + "permanent", + `Replaced Secret ${name} was mutated by admission`, + ); + } + } catch (err) { + if (err instanceof WorkflowRunnerResourceError) throw err; + throw classifyResourceError("rotate runner Secret", err); + } +} + +// The Kubernetes client deserializes responses into model class instances while +// the desired objects are plain literals. isDeepStrictEqual compares prototypes, +// so a server response never matches until both sides are re-homed onto plain +// objects. Values are untouched, so real drift is still rejected. +function plainData(value: unknown): unknown { + return value === undefined ? undefined : (JSON.parse(JSON.stringify(value)) as unknown); +} + +// Every checked optional field is intentional. nodeName is server-assigned after create. +// eslint-disable-next-line complexity +function podBoundary(pod: V1Pod): unknown { + const spec = pod.spec; + return { + metadata: { + name: pod.metadata?.name, + namespace: pod.metadata?.namespace, + }, + spec: { + restartPolicy: spec?.restartPolicy, + serviceAccountName: spec?.serviceAccountName, + serviceAccount: spec?.serviceAccount ?? spec?.serviceAccountName, + dnsPolicy: spec?.dnsPolicy, + dnsConfig: spec?.dnsConfig, + schedulerName: spec?.schedulerName, + activeDeadlineSeconds: spec?.activeDeadlineSeconds, + terminationGracePeriodSeconds: spec?.terminationGracePeriodSeconds, + automountServiceAccountToken: spec?.automountServiceAccountToken, + enableServiceLinks: spec?.enableServiceLinks, + hostIPC: spec?.hostIPC ?? false, + hostNetwork: spec?.hostNetwork ?? false, + hostPID: spec?.hostPID ?? false, + shareProcessNamespace: spec?.shareProcessNamespace ?? false, + hostAliases: spec?.hostAliases, + hostUsers: spec?.hostUsers, + hostname: spec?.hostname, + hostnameOverride: spec?.hostnameOverride, + imagePullSecrets: spec?.imagePullSecrets, + affinity: spec?.affinity, + nodeSelector: spec?.nodeSelector, + os: spec?.os, + overhead: spec?.overhead, + preemptionPolicy: spec?.preemptionPolicy ?? "PreemptLowerPriority", + priority: spec?.priority ?? 0, + priorityClassName: spec?.priorityClassName, + readinessGates: spec?.readinessGates, + resourceClaims: spec?.resourceClaims, + resources: spec?.resources, + runtimeClassName: spec?.runtimeClassName, + schedulingGates: spec?.schedulingGates, + setHostnameAsFQDN: spec?.setHostnameAsFQDN ?? false, + subdomain: spec?.subdomain, + tolerations: spec?.tolerations, + topologySpreadConstraints: spec?.topologySpreadConstraints, + initContainerCount: spec?.initContainers?.length ?? 0, + ephemeralContainerCount: spec?.ephemeralContainers?.length ?? 0, + volumes: spec?.volumes, + securityContext: spec?.securityContext, + containers: spec?.containers.map((container) => ({ + name: container.name, + image: container.image, + imagePullPolicy: container.imagePullPolicy, + command: container.command, + env: container.env?.map((entry) => ({ + name: entry.name, + value: entry.value, + secretName: entry.valueFrom?.secretKeyRef?.name, + secretKey: entry.valueFrom?.secretKeyRef?.key, + secretOptional: entry.valueFrom?.secretKeyRef?.optional, + })), + envFrom: container.envFrom?.map((entry) => ({ + secretName: entry.secretRef?.name, + prefix: entry.prefix, + })), + volumeMounts: container.volumeMounts, + args: container.args, + workingDir: container.workingDir, + lifecycle: container.lifecycle, + livenessProbe: container.livenessProbe, + readinessProbe: container.readinessProbe, + startupProbe: container.startupProbe, + stdin: container.stdin, + stdinOnce: container.stdinOnce, + tty: container.tty, + ports: container.ports, + resizePolicy: container.resizePolicy, + restartPolicy: container.restartPolicy, + restartPolicyRules: container.restartPolicyRules, + securityContext: container.securityContext, + terminationMessagePath: container.terminationMessagePath, + terminationMessagePolicy: container.terminationMessagePolicy, + volumeDevices: container.volumeDevices, + resources: { + requests: container.resources?.requests, + limits: container.resources?.limits, + }, + })), + }, + }; +} + +// Every condition is one independent fail-closed part of the Pod identity. +// eslint-disable-next-line complexity +function validateExistingPod(pod: V1Pod, desired: V1Pod, attempt: WorkflowRunnerAttempt): void { + const name = workflowRunnerResourceNames(attempt.attemptId).podName; + if (pod.metadata?.deletionTimestamp !== undefined) { + throw new WorkflowRunnerResourceError("transient", `Pod ${name} is terminating`); + } + if ( + pod.apiVersion !== desired.apiVersion || + pod.kind !== desired.kind || + !exactResourceMetadata(pod.metadata, name, attempt) || + pod.metadata?.annotations !== undefined || + pod.metadata?.deletionGracePeriodSeconds !== undefined || + pod.metadata?.finalizers !== undefined || + pod.metadata?.generateName !== undefined || + pod.metadata?.ownerReferences !== undefined || + !isDeepStrictEqual(plainData(podBoundary(pod)), plainData(podBoundary(desired))) + ) { + throw new WorkflowRunnerResourceError( + "permanent", + `Existing Pod ${name} does not match workflow attempt ${attempt.attemptId}`, + ); + } +} + +async function ensurePod( + attempt: WorkflowRunnerAttempt, + image: string, + orchestratorUrl: string, +): Promise { + const client = loadKubernetesClient().core; + const namespace = config.workflowRunnerNamespace; + const desired = buildWorkflowRunnerPod(attempt, image, orchestratorUrl); + const name = desired.metadata?.name; + if (name === undefined) throw new WorkflowRunnerResourceError("permanent", "Pod name missing"); + try { + const created = await client.createNamespacedPod({ namespace, body: desired }); + validateExistingPod(created, desired, attempt); + if (created.metadata?.uid === undefined || created.metadata.uid === "") { + throw new WorkflowRunnerResourceError("transient", `Created Pod ${name} has no UID`); + } + return created; + } catch (err) { + if (err instanceof WorkflowRunnerResourceError) throw err; + if (statusCode(err) !== 409) throw classifyResourceError("create runner Pod", err); + } + + try { + const existing = await client.readNamespacedPod({ name, namespace }); + validateExistingPod(existing, desired, attempt); + if (existing.metadata?.uid === undefined || existing.metadata.uid === "") { + throw new WorkflowRunnerResourceError("transient", `Existing Pod ${name} has no UID`); + } + return existing; + } catch (err) { + if (err instanceof WorkflowRunnerResourceError) throw err; + throw classifyResourceError("read existing runner Pod", err); + } +} + +function classifyResourceError(operation: string, err: unknown): WorkflowRunnerResourceError { + const status = statusCode(err); + const permanent = + status !== undefined && + status >= 400 && + status < 500 && + status !== 408 && + status !== 409 && + status !== 429; + return new WorkflowRunnerResourceError( + permanent ? "permanent" : "transient", + `${operation} failed${status === undefined ? "" : ` (${String(status)})`}`, + ); +} + +export async function ensureWorkflowRunnerResources(input: { + readonly attempt: WorkflowRunnerAttempt; + readonly capability: string; + readonly image: string; + readonly orchestratorUrl: string; +}): Promise { + assertSecureOrchestratorUrl(input.orchestratorUrl); + assertDigestPinnedRunnerImage(input.image); + // Reject provider drift before the attempt capability Secret exists. + buildProviderEnvironment(); + const pod = await ensurePod(input.attempt, input.image, input.orchestratorUrl); + const podUid = pod.metadata?.uid; + if (podUid === undefined || podUid === "") { + throw new WorkflowRunnerResourceError("transient", "Runner Pod UID missing after reconcile"); + } + await ensureSecret(input.attempt, input.capability, podUid); + logger.info( + { + runId: input.attempt.runId, + attemptId: input.attempt.attemptId, + podName: workflowRunnerResourceNames(input.attempt.attemptId).podName, + }, + "Workflow runner resources reconciled", + ); +} + +function assertDigestPinnedRunnerImage(image: string): void { + if (SHA256_IMAGE_DIGEST.test(image)) return; + throw new WorkflowRunnerResourceError( + "permanent", + "Workflow runner DAEMON_IMAGE must end with an immutable sha256 digest", + ); +} + +// A name that cannot resolve outside the cluster. The admission boundary pins the +// same shape on orchestratorOrigin, so the two rules have to stay identical. +const CLUSTER_LOCAL_SERVICE_HOST = /^[a-z0-9-]+\.[a-z0-9-]+\.svc(\.cluster\.local)?$/; + +function assertSecureOrchestratorUrl(value: string): void { + let url: URL; + try { + url = new URL(value); + } catch { + throw new WorkflowRunnerResourceError( + "permanent", + "ORCHESTRATOR_PUBLIC_URL must be a secure WebSocket URL", + ); + } + // A live installation token crosses this socket, so plaintext is confined to a + // cluster-local Service the runner reaches without leaving the cluster. Dialling + // an ingress VIP instead is what the CNI may classify as host traffic, which an + // egress NetworkPolicy cannot match at all. + const reachableWithoutTls = + url.protocol === "ws:" && CLUSTER_LOCAL_SERVICE_HOST.test(url.hostname); + if ( + (url.protocol !== "wss:" && !reachableWithoutTls) || + url.hostname === "" || + url.username !== "" || + url.password !== "" + ) { + throw new WorkflowRunnerResourceError( + "permanent", + "ORCHESTRATOR_PUBLIC_URL must be wss://, or ws:// to a cluster-local Service name, and carry no credentials", + ); + } +} + +function ownedDeleteOptions( + metadata: V1ObjectMeta | undefined, + kind: "Pod" | "Secret", +): V1DeleteOptions { + const uid = metadata?.uid; + if (uid === undefined || uid === "") { + throw new WorkflowRunnerResourceError( + "permanent", + `${kind} identity is missing deletion preconditions`, + ); + } + return { preconditions: { uid } }; +} + +async function readForDelete( + operation: () => Promise, + kind: "Pod" | "Secret", +): Promise { + try { + return await operation(); + } catch (err) { + if (statusCode(err) === 404) return null; + throw classifyResourceError(`read runner ${kind} for deletion`, err); + } +} + +async function deleteIgnoringNotFound( + operation: () => Promise, + kind: "Pod" | "Secret", +): Promise { + try { + await operation(); + } catch (err) { + if (statusCode(err) !== 404) throw classifyResourceError(`delete runner ${kind}`, err); + } +} + +export async function deleteWorkflowRunnerResources( + attempt: WorkflowRunnerResourceIdentity, +): Promise { + const client = loadKubernetesClient().core; + const namespace = config.workflowRunnerNamespace; + const { podName, secretName } = workflowRunnerResourceNames(attempt.attemptId); + const pod = await readForDelete( + () => client.readNamespacedPod({ name: podName, namespace }), + "Pod", + ); + if (pod !== null) { + validateResourceOwnership(pod.metadata, podName, attempt, "Pod"); + const body = ownedDeleteOptions(pod.metadata, "Pod"); + await deleteIgnoringNotFound( + () => client.deleteNamespacedPod({ name: podName, namespace, body }), + "Pod", + ); + } + + const secret = await readForDelete( + () => client.readNamespacedSecret({ name: secretName, namespace }), + "Secret", + ); + // Resource presence is not sensitive data. + // eslint-disable-next-line security/detect-possible-timing-attacks + if (secret !== null) { + validateResourceOwnership(secret.metadata, secretName, attempt, "Secret"); + const body = ownedDeleteOptions(secret.metadata, "Secret"); + await deleteIgnoringNotFound( + () => client.deleteNamespacedSecret({ name: secretName, namespace, body }), + "Secret", + ); + } + return true; +} + +function validateResourceOwnership( + metadata: V1ObjectMeta | undefined, + name: string, + attempt: WorkflowRunnerResourceIdentity, + kind: "Pod" | "Secret", +): void { + if (!exactResourceMetadata(metadata, name, attempt)) { + throw new WorkflowRunnerResourceError( + "permanent", + `${kind} ${name} does not belong to workflow attempt ${attempt.attemptId}`, + ); + } +} diff --git a/src/mcp/servers/inline-comment-dedup.ts b/src/mcp/servers/inline-comment-dedup.ts index b0fba1e4..7f47c994 100644 --- a/src/mcp/servers/inline-comment-dedup.ts +++ b/src/mcp/servers/inline-comment-dedup.ts @@ -43,16 +43,9 @@ export interface ExistingComment { * suppressed while that thread lives. Both are acceptable; neither is asserted. * Switching to GraphQL `PullRequestReviewThread.isOutdated` would settle it. */ -/** The diff position a finding wants to post at. */ -export interface CommentLocation { - readonly path: string; - readonly line: number; - readonly side: string; -} - export function hasDuplicateAt( existing: readonly ExistingComment[], - target: CommentLocation, + target: { readonly path: string; readonly line: number; readonly side: string }, selfLogin: string | null, ): boolean { return existing.some( diff --git a/src/mcp/servers/inline-comment.ts b/src/mcp/servers/inline-comment.ts index 5e67e34f..c843dab5 100644 --- a/src/mcp/servers/inline-comment.ts +++ b/src/mcp/servers/inline-comment.ts @@ -195,11 +195,10 @@ server.tool( const isSingleLine = startLine === undefined; const locKey = locationKey(path, line, side); - const alreadyPosted = await hasExistingComment(pull_number, path, line, side); - // Re-read `pendingLocations` AFTER the await: a concurrent handler can - // claim this location while `hasExistingComment` is in flight, and both - // would otherwise fall through and post. - if (pendingLocations.has(locKey) || alreadyPosted) { + if ( + pendingLocations.has(locKey) || + (await hasExistingComment(pull_number, path, line, side)) + ) { log.info( { event: "mcp.inline_comment.deduped", pull_number, path, line, side }, "skipped duplicate inline comment", @@ -220,10 +219,6 @@ server.tool( }; } - // Claim before the commit-SHA fetch below, which is another suspension - // point a concurrent handler could interleave with. - pendingLocations.add(locKey); - // Get latest commit SHA if not provided let commitSha = commit_id; if (commitSha === undefined || commitSha === "") { @@ -257,6 +252,7 @@ server.tool( params.line = line; } + pendingLocations.add(locKey); let result; try { result = await retryWithBackoff(() => octokit.rest.pulls.createReviewComment(params), { diff --git a/src/orchestrator/connection-handler.ts b/src/orchestrator/connection-handler.ts index 3c09a913..5dda2960 100644 --- a/src/orchestrator/connection-handler.ts +++ b/src/orchestrator/connection-handler.ts @@ -4,10 +4,8 @@ import { App, Octokit } from "octokit"; import { config } from "../config"; import { clearInFlightByJobId } from "../db/queries/scheduled-actions-store"; import { logger } from "../logger"; +import { loadRepoPolicy, toAgentPolicy } from "../repo-config/effective"; import { observableOctokit } from "../utils/octokit-observability"; -import { addReaction } from "../utils/reactions"; -import { findById, findInflightByOwner, type WorkflowRunRow } from "../workflows/runs-store"; -import { setState } from "../workflows/tracking-mirror"; import { mintInstallationToken } from "./installation-token"; import { DAEMON_HEARTBEAT_LOG_EVENTS, DISPATCHER_LOG_EVENTS } from "./log-fields"; import type { @@ -49,8 +47,10 @@ function isDaemonOutdated(daemon: string, orchestrator: string): boolean { } import type { DaemonInfo, HeartbeatState } from "../shared/daemon-types"; import { + type AgentPolicy, createMessageEnvelope, type DaemonMessage, + PROTOCOL_VERSION, type ScopedJobContext, WS_CLOSE_CODES, WS_ERROR_CODES, @@ -65,6 +65,7 @@ import { registerDaemon, } from "./daemon-registry"; import { + failDisconnectedDaemon, getExecutionState, getOrphanedExecutions, markExecutionCompleted, @@ -78,13 +79,28 @@ import { removePendingOffer, } from "./job-dispatcher"; import { isScopedJob, QueuedJobSchema, type ScopedQueuedJob } from "./job-queue"; -import { sendError, type WsConnectionData } from "./ws-server"; +import { persistRepoKnowledge } from "./repo-knowledge-persistence"; +import { notifyDisconnectedDaemonWorkflows } from "./workflow-expiry-notifier"; +import { sendError, type WsConnectionData } from "./ws-connection"; // In-memory state (per orchestrator process) const connections = new Map>(); const daemonInfoMap = new Map(); const heartbeatTimers = new Map(); +const disconnectCleanups = new Set>(); +const disconnectCleanupsByDaemon = new Map>(); +const registrationTransitionsByDaemon = new Map>(); +const protocolUpdateTransitions = new Map< + ServerWebSocket, + { + readonly daemonId: string; + readonly phase: "awaiting-ack" | "draining"; + readonly timer: Timer; + } +>(); +const PROTOCOL_UPDATE_ACK_TIMEOUT_MS = 5_000; +const PROTOCOL_UPDATE_DRAIN_GRACE_MS = 2_000; /** Daemon IDs that sent daemon:draining, excluded from dispatch. */ const drainingDaemons = new Set(); @@ -131,8 +147,19 @@ export function handleWsClose( _code: number, _reason: string, ): void { + const protocolUpdate = protocolUpdateTransitions.get(ws); + if (protocolUpdate !== undefined) { + clearTimeout(protocolUpdate.timer); + protocolUpdateTransitions.delete(ws); + } + const daemonId = ws.data.daemonId; if (daemonId === undefined) return; + ws.data.daemonId = undefined; + + // A superseded socket must not tear down the newer connection that now owns + // this per-boot daemon ID. + if (connections.get(daemonId) !== ws) return; const hb = heartbeatTimers.get(daemonId); if (hb !== undefined) { @@ -145,8 +172,23 @@ export function handleWsClose( daemonInfoMap.delete(daemonId); drainingDaemons.delete(daemonId); - // Async cleanup: deregister from Valkey/Postgres, handle orphaned executions - void cleanupAfterDisconnect(daemonId); + void startDisconnectCleanup(daemonId); +} + +function startDisconnectCleanup(daemonId: string): Promise { + const pending = disconnectCleanupsByDaemon.get(daemonId); + if (pending !== undefined) return pending; + + const cleanup = cleanupAfterDisconnect(daemonId); + disconnectCleanups.add(cleanup); + disconnectCleanupsByDaemon.set(daemonId, cleanup); + void cleanup.finally(() => { + disconnectCleanups.delete(cleanup); + if (disconnectCleanupsByDaemon.get(daemonId) === cleanup) { + disconnectCleanupsByDaemon.delete(daemonId); + } + }); + return cleanup; } /** @@ -155,181 +197,42 @@ export function handleWsClose( */ async function cleanupAfterDisconnect(daemonId: string): Promise { try { - await deregisterDaemon(daemonId); - - // Scan for orphaned executions - const orphans = await getOrphanedExecutions(daemonId); - for (const orphan of orphans) { - try { - // eslint-disable-next-line no-await-in-loop - await markExecutionFailed(orphan.deliveryId, "daemon disconnected during execution"); - // eslint-disable-next-line no-await-in-loop - await releaseScheduledActionLock(orphan.deliveryId); - } catch (err) { - logger.error( - { err, deliveryId: orphan.deliveryId }, - "Failed to mark orphaned execution as failed", - ); - } - } - - if (orphans.length > 0) { + const failed = await failDisconnectedDaemon(daemonId); + await notifyDisconnectedDaemonWorkflows(failed.workflowRunIds); + if (failed.executionDeliveryIds.length > 0 || failed.workflowRunIds.length > 0) { logger.warn( - { daemonId, orphanCount: orphans.length }, + { + daemonId, + orphanCount: failed.executionDeliveryIds.length, + workflowRunCount: failed.workflowRunIds.length, + }, "Cleaned up orphaned executions after daemon disconnect", ); } - - // User-facing notification: any in-flight workflow_runs owned by this - // daemon will be flipped to 'failed' by the liveness reaper. We update - // the user's tracking comment + react on the trigger comment now so the - // user sees the failure immediately instead of staring at a stale - // "starting…" comment. - await notifyOrphanedWorkflowRuns(daemonId); } catch (err) { - logger.error({ err, daemonId }, "Failed to cleanup after daemon disconnect"); + logger.error({ err, daemonId }, "Failed durable daemon disconnect cleanup"); } -} -/** - * Update the user-facing tracking comment + add a `confused` reaction on the - * originating comment for every in-flight workflow_run owned by the dying - * daemon. Walks the parent chain so a child step's failure shows up on the - * top-level run's surface (the surface the user is actually watching) rather - * than on a per-child comment they may not have noticed. - * - * Best-effort throughout: a missing GitHub App config or a comment-update - * failure must never bubble up and prevent the rest of cleanup from running. - */ -async function notifyOrphanedWorkflowRuns(daemonId: string): Promise { - let inflight: WorkflowRunRow[]; try { - inflight = await findInflightByOwner("daemon", daemonId); + await deregisterDaemon(daemonId); } catch (err) { - logger.error({ err, daemonId }, "Failed to query in-flight workflow_runs for orphan cleanup"); - return; - } - - if (inflight.length === 0) return; - - // Dedupe by ancestor so a single ship cascade (ship → plan → implement) - // only updates one comment + one reaction even if multiple of its rows - // were owned by this daemon at the moment of disconnect. - const ancestorIds = new Set(); - for (const row of inflight) { - // eslint-disable-next-line no-await-in-loop - const ancestor = await findTopAncestor(row); - if (ancestor === null) continue; - if (ancestorIds.has(ancestor.id)) continue; - ancestorIds.add(ancestor.id); - - try { - // eslint-disable-next-line no-await-in-loop - await postOrphanNotification(ancestor); - } catch (err) { - logger.warn( - { - err: err instanceof Error ? err.message : String(err), - ancestorRunId: ancestor.id, - daemonId, - }, - "Orphan notification (comment/reaction) failed", - ); - } + logger.error({ err, daemonId }, "Failed best-effort daemon registry cleanup"); } } -/** - * Walk parent_run_id up to the topmost row. Returns the input row if it has - * no parent, or null if the parent chain is broken (orphaned mid-walk) or - * exceeds the safety cap. Returning the cap-iteration row would be a silent - * bug: it still has a non-null parent_run_id, so we'd update the wrong - * (mid-chain) tracking comment. - */ -async function findTopAncestor(row: WorkflowRunRow): Promise { - let current: WorkflowRunRow | null = row; - // Bound at 8 levels of nesting, defensive cap for a chain that should - // realistically never exceed depth 2 (ship → step). A null parent ends - // the walk naturally. - for (let i = 0; i < 8; i++) { - if (current === null) return null; - if (current.parent_run_id === null) return current; - // eslint-disable-next-line no-await-in-loop - current = await findById(current.parent_run_id); +/** Wait until every registration and close callback has finished its durable work. */ +export async function drainDisconnectCleanups(): Promise { + while (registrationTransitionsByDaemon.size > 0 || disconnectCleanups.size > 0) { + // eslint-disable-next-line no-await-in-loop -- connection transitions can enqueue cleanup while settling + await Promise.allSettled([...registrationTransitionsByDaemon.values(), ...disconnectCleanups]); } - logger.warn( - { startRunId: row.id, lastSeenRunId: current?.id ?? null }, - "findTopAncestor: parent chain exceeded 8 levels, skipping orphan notification to avoid touching the wrong comment", - ); - return null; } -async function postOrphanNotification(ancestor: WorkflowRunRow): Promise { - if (config.appId === undefined || config.privateKey === undefined) { - logger.debug( - { ancestorRunId: ancestor.id }, - "Skipping orphan notification, GitHub App credentials not configured", - ); - return; - } - - // PAT mode short-circuit: when GITHUB_PERSONAL_ACCESS_TOKEN is set, the - // contract is that the PAT replaces the installation token for ALL GitHub - // API calls, orphan-notification comments and reactions included, so the - // operator-visible identity stays consistent with other bot replies. - let octokit: Awaited> | Octokit; - if (config.githubPersonalAccessToken !== undefined) { - octokit = new Octokit({ auth: config.githubPersonalAccessToken }); - } else { - const app = getOrCreateApp(); - const { data: installation } = await app.octokit.rest.apps.getRepoInstallation({ - owner: ancestor.target_owner, - repo: ancestor.target_repo, - }); - octokit = ( - await mintInstallationToken({ - app, - installationId: installation.id, - via: "postOrphanNotification", - log: logger, - }) - ).octokit; - } - - const humanMessage = [ - `❌ **Daemon disconnected during execution**, likely an OOM kill on the workflow pod.`, - ``, - `The in-flight step has been marked failed. Its workflow_run row will be flipped`, - `to \`failed\` by the liveness reaper. To resume, re-trigger the workflow:`, - ``, - `- For \`ship\`: re-apply the \`bot:ship\` label, or comment again. Resume picks up`, - ` from the failed step and reuses prior succeeded steps.`, - `- For standalone workflows: re-comment with the same trigger.`, - ].join("\n"); - - // Re-uses tracking-mirror.setState so the cascade refresh and `_lastHumanMessage` - // bookkeeping stay consistent, and so the parent's composite body picks up the - // failure narrative on the next render. - const installationOctokit = octokit as unknown as Octokit; - await setState( - { octokit: installationOctokit, logger }, - { - runId: ancestor.id, - patch: { phase: "orphaned" }, - humanMessage, - }, - ); - - if (ancestor.trigger_comment_id !== null && ancestor.trigger_event_type !== null) { - await addReaction({ - octokit: installationOctokit, - logger, - owner: ancestor.target_owner, - repo: ancestor.target_repo, - commentId: ancestor.trigger_comment_id, - eventType: ancestor.trigger_event_type, - content: "confused", - }); +/** Start exact-incarnation cleanup before the WebSocket server drain timer. */ +export function beginDaemonConnectionShutdown(): void { + for (const ws of [...connections.values()]) { + handleWsClose(ws, 1001, "orchestrator shutting down"); + ws.close(1001, "orchestrator shutting down"); } } @@ -340,7 +243,7 @@ export function handleDaemonMessage( ): void { switch (msg.type) { case "daemon:register": - void handleRegister(ws, msg); + queueRegistration(ws, msg); break; case "heartbeat:pong": handleHeartbeatPong(ws, msg); @@ -357,7 +260,7 @@ export function handleDaemonMessage( case "job:result": handleJobMessage(ws, msg); break; - case "scoped-job-completion": + case "scoped-job:completion": void handleScopedJobCompletion(ws, msg); break; } @@ -365,12 +268,93 @@ export function handleDaemonMessage( // daemon:register handler (FM-8 reconnection logic) +function queueRegistration( + ws: ServerWebSocket, + msg: Extract, +): void { + const { daemonId } = msg.payload; + const previous = registrationTransitionsByDaemon.get(daemonId); + const transition = (previous?.catch(() => undefined) ?? Promise.resolve()).then(() => + handleRegister(ws, msg), + ); + registrationTransitionsByDaemon.set(daemonId, transition); + void transition + .catch((err: unknown) => { + logger.error({ err, daemonId }, "Unhandled daemon registration failure"); + }) + .finally(() => { + if (registrationTransitionsByDaemon.get(daemonId) === transition) { + registrationTransitionsByDaemon.delete(daemonId); + } + }); +} + async function handleRegister( ws: ServerWebSocket, msg: Extract, ): Promise { const { daemonId } = msg.payload; + // A socket that entered the update transition cannot re-register under a + // different schema while its update acknowledgement is pending. + if (protocolUpdateTransitions.has(ws)) return; + + // Reject mixed schemas before this socket can mutate daemon state. Older + // daemons can acknowledge update-required and drain before we close them. + const ourMajor = PROTOCOL_VERSION.split(".", 1)[0] ?? ""; + const theirMajor = msg.payload.protocolVersion.split(".", 1)[0] ?? ""; + if (theirMajor !== ourMajor) { + const ourMajorNumber = Number(ourMajor); + const theirMajorNumber = Number(theirMajor); + if ( + /^\d+$/.test(ourMajor) && + /^\d+$/.test(theirMajor) && + Number.isInteger(ourMajorNumber) && + Number.isInteger(theirMajorNumber) && + theirMajorNumber < ourMajorNumber + ) { + logger.warn( + { + daemonId, + daemonProtocolVersion: msg.payload.protocolVersion, + orchestratorProtocolVersion: PROTOCOL_VERSION, + }, + "Daemon protocol is outdated, sending daemon:update-required", + ); + const timer = setTimeout(() => { + protocolUpdateTransitions.delete(ws); + ws.close( + WS_CLOSE_CODES.INCOMPATIBLE_PROTOCOL.code, + WS_CLOSE_CODES.INCOMPATIBLE_PROTOCOL.reason, + ); + }, PROTOCOL_UPDATE_ACK_TIMEOUT_MS); + timer.unref(); + protocolUpdateTransitions.set(ws, { daemonId, phase: "awaiting-ack", timer }); + ws.sendText( + JSON.stringify({ + type: "daemon:update-required", + ...createMessageEnvelope(msg.id), + payload: { + targetVersion: ORCHESTRATOR_APP_VERSION, + reason: `Orchestrator protocol is ${PROTOCOL_VERSION}; daemon protocol is ${msg.payload.protocolVersion}`, + urgent: true, + }, + }), + ); + } else { + ws.close( + WS_CLOSE_CODES.INCOMPATIBLE_PROTOCOL.code, + WS_CLOSE_CODES.INCOMPATIBLE_PROTOCOL.reason, + ); + } + return; + } + + // A transport reconnect reuses the same per-boot daemon ID. Finish fencing + // the closed socket before restoring that identity in PostgreSQL or Valkey. + const pendingCleanup = disconnectCleanupsByDaemon.get(daemonId); + if (pendingCleanup !== undefined) await pendingCleanup; + // FM-8: Check for existing connection with same daemon ID const existing = connections.get(daemonId); if (existing !== undefined) { @@ -405,21 +389,13 @@ async function handleRegister( } } - // Version compatibility check (T042, Phase 7, basic check here) - // Major protocol version mismatch -> reject - const ourMajor = "1"; - const theirMajor = msg.payload.protocolVersion.split(".")[0]; - if (theirMajor !== ourMajor) { - ws.close( - WS_CLOSE_CODES.INCOMPATIBLE_PROTOCOL.code, - WS_CLOSE_CODES.INCOMPATIBLE_PROTOCOL.reason, - ); - return; - } - // Register in Valkey + Postgres try { const info = await registerDaemon(msg); + if (ws.readyState !== 1) { + await startDisconnectCleanup(daemonId); + return; + } daemonInfoMap.set(daemonId, info); } catch (err) { logger.error({ err, daemonId }, "Failed to register daemon"); @@ -597,6 +573,34 @@ function handleUpdateAcknowledged( ws: ServerWebSocket, msg: Extract, ): void { + const protocolUpdate = protocolUpdateTransitions.get(ws); + if (protocolUpdate !== undefined) { + if (protocolUpdate.phase === "draining") return; + clearTimeout(protocolUpdate.timer); + const timer = setTimeout(() => { + protocolUpdateTransitions.delete(ws); + ws.close( + WS_CLOSE_CODES.INCOMPATIBLE_PROTOCOL.code, + WS_CLOSE_CODES.INCOMPATIBLE_PROTOCOL.reason, + ); + }, config.daemonDrainTimeoutMs + PROTOCOL_UPDATE_DRAIN_GRACE_MS); + timer.unref(); + protocolUpdateTransitions.set(ws, { + daemonId: protocolUpdate.daemonId, + phase: "draining", + timer, + }); + logger.info( + { + daemonId: protocolUpdate.daemonId, + strategy: msg.payload.strategy, + delayMs: msg.payload.delayMs, + }, + "Outdated daemon acknowledged protocol update", + ); + return; + } + const daemonId = ws.data.daemonId; if (daemonId === undefined) return; @@ -631,7 +635,7 @@ async function releaseScheduledActionLock(deliveryId: string): Promise { // Job message handling (T032-T033) /** - * Server-side bridge for `scoped-job-completion` (T033b). The daemon + * Server-side bridge for `scoped-job:completion` (T033b). The daemon * executor reports the structured outcome here; the orchestrator releases * the pending offer + capacity slot and emits a telemetry log line. The * user-facing Octokit reply is posted by the daemon executor itself @@ -645,7 +649,7 @@ async function releaseScheduledActionLock(deliveryId: string): Promise { */ async function handleScopedJobCompletion( ws: ServerWebSocket, - msg: Extract, + msg: Extract, ): Promise { const daemonId = ws.data.daemonId; const offerId = msg.payload.offerId; @@ -661,7 +665,7 @@ async function handleScopedJobCompletion( // Ownership + late-result guard. The in-memory offer is removed by // handleAccept right after dispatch to handleScopedAccept, so by the - // time a real scoped-job-completion arrives `getPendingOffer(offerId)` + // time a real scoped-job:completion arrives `getPendingOffer(offerId)` // is almost always undefined and an offer-based check is ineffective. // Mirror handleResult: validate against durable execution state so a // replayed/forged completion cannot decrement capacity counters or @@ -677,7 +681,7 @@ async function handleScopedJobCompletion( assignedDaemon: state?.daemonId ?? null, currentStatus: state?.status ?? null, }, - "scoped-job-completion failed ownership/finality validation, ignoring", + "scoped-job:completion failed ownership/finality validation, ignoring", ); return; } @@ -808,6 +812,30 @@ function handleJobMessage(ws: ServerWebSocket, msg: DaemonMess } } +/** + * Drop `instructions` from the Gate-2 policy for every workflow but `review`. + * + * The pipeline renders `instructions` as trusted repo policy that OVERRIDES + * the agent's default heuristics. That surface was designed for `review` + * alone. The schema scopes the field to `workflows.review` today, + * so this is a no-op guard, but keeping the restriction here means a future + * hoist to `defaults` cannot silently hand `implement` (which writes code and + * pushes commits) an override block with no pipeline change to review. + * + * Enforced at the accept site, not in the daemon: the daemon stays dumb about + * which knobs each workflow may see. Exported so the guard can be tested + * directly: the schema makes the bad input unreachable through YAML today, + * which is exactly why the guard needs its own test. + */ +export function stripInstructionsUnlessReview( + policy: AgentPolicy | undefined, + workflowName: string | undefined, +): AgentPolicy | undefined { + if (policy?.instructions === undefined || workflowName === "review") return policy; + const { instructions: _dropped, ...rest } = policy; + return Object.keys(rest).length > 0 ? rest : undefined; +} + async function handleAccept( daemonId: string, msg: Extract, @@ -852,7 +880,7 @@ async function handleAccept( // do not need the legacy `executions.context_json` lookup or the // `BotContext`-shaped allowed-tool resolution. Mint the installation // token directly from `offer.scoped.installationId` and forward the - // scoped payload verbatim. The `scoped-job-completion` handler decrements + // scoped payload verbatim. The `scoped-job:completion` handler decrements // capacity on completion. if (offer.scoped !== undefined) { await handleScopedAccept(daemonId, offerId, offer); @@ -937,11 +965,49 @@ async function handleAccept( })); } - // No turn cap by default: workflows must run end-to-end without losing - // progress to a mid-run cap. `AGENT_MAX_TURNS` and `DEFAULT_MAXTURNS` - // remain as opt-in escape hatches for ops; when both are unset we pass - // `undefined` to the SDK and the agent decides when it's done. - const maxTurns = config.agentMaxTurns ?? config.defaultMaxTurns; + // Gate 2: resolve `.github-app.yaml` into the agent knobs shipped with + // the job. `loadRepoPolicy` is the only entry point used here on purpose, + // it clamps `max_turns` / `timeout` against the server ceilings, so a + // repo cannot raise them by editing YAML. Never throws: a missing, + // unreachable, or invalid file yields DEFAULT_REPO_POLICY. + const repoPolicy = await loadRepoPolicy({ octokit: acceptOctokit, owner, repo, log: logger }); + const executionPolicy = repoPolicy.defaults; + + // Repo value first, then the pre-Gate-2 env chain verbatim. Keeping the + // env tail byte-identical is what makes a repo with no config file behave + // exactly as before. `AGENT_MAX_TURNS` doubles as the CEILING the resolver + // already clamped `workflowPolicy.maxTurns` against, so a repo can only + // lower the cap here, never raise it. + const maxTurns = executionPolicy.maxTurns ?? config.agentMaxTurns ?? config.defaultMaxTurns; + const policy = stripInstructionsUnlessReview( + toAgentPolicy(executionPolicy, repoPolicy.warning), + undefined, + ); + // `maxTurns` is checked separately: `toAgentPolicy` never projects it (it + // rides the top-level payload field), so a repo whose only knob is + // `max_turns` resolves to `policy === undefined` and would otherwise log + // nothing at all, leaving "did this run honour the repo's knobs?" + // unanswerable. + if (policy !== undefined || executionPolicy.maxTurns !== undefined) { + logger.info( + { + event: "repo_config.policy_applied", + owner, + repo, + deliveryId: offer.deliveryId, + workflow: "none", + model: policy?.model, + maxTurns, + timeoutMs: policy?.timeoutMs, + extraAllowedToolCount: policy?.extraAllowedTools?.length ?? 0, + pathFilterCount: policy?.pathFilters?.length ?? 0, + // Text stays out of the log line: it can be 10KB of repo prose. + hasInstructions: policy?.instructions !== undefined, + warned: policy?.warning !== undefined, + }, + "Per-repo agent policy applied", + ); + } const { resolveAllowedTools } = await import("../core/prompt-builder"); // Reconstruct a minimal BotContext-shaped object for resolveAllowedTools. @@ -1002,7 +1068,7 @@ async function handleAccept( })), } : {}), - ...(offer.workflowRun !== undefined ? { workflowRun: offer.workflowRun } : {}), + ...(policy !== undefined ? { policy } : {}), }); } catch (err) { logger.error({ err, offerId, daemonId }, "Failed to mint installation token for job"); @@ -1298,120 +1364,6 @@ async function loadReviewLearningsRag( } } -/** - * Persist learnings and deletions from a daemon execution to repo_memory - * and (for review/resolve workflows) review_learnings. Extracted to reduce - * nesting depth in handleResult. - */ -async function persistRepoKnowledge( - deliveryId: string, - learnings: { category: string; content: string }[] | undefined, - deletions: string[] | undefined, - reviewLearningSaves: - | { - directive: string; - rationale?: string | undefined; - fileGlob?: string | undefined; - scope?: "local" | "global" | undefined; - sourcePr?: number | undefined; - sourceThread?: string | undefined; - sourceAuthor?: string | undefined; - }[] - | undefined, - reviewLearningDeletes: string[] | undefined, - appliedReviewLearningIds: string[] | undefined, -): Promise { - try { - const { saveRepoLearnings, deleteRepoMemories } = await import("./repo-knowledge"); - const { requireDb } = await import("../db"); - const knowledgeDb = requireDb(); - const execRows: { repo_owner: string; repo_name: string }[] = await knowledgeDb` - SELECT repo_owner, repo_name FROM executions WHERE delivery_id = ${deliveryId} - `; - const exec = execRows[0]; - if (exec === undefined) return; - - if (learnings !== undefined && learnings.length > 0) { - const saved = await saveRepoLearnings(exec.repo_owner, exec.repo_name, learnings); - if (saved > 0) { - logger.info({ deliveryId, saved }, "Persisted repo learnings from execution"); - } - } - if (deletions !== undefined && deletions.length > 0) { - const deleted = await deleteRepoMemories(deletions); - if (deleted > 0) { - logger.info({ deliveryId, deleted }, "Deleted outdated repo memories per daemon request"); - } - } - // Kill-switch: when disabled, drop any review-learning actions an agent - // managed to emit (e.g. because the operator flipped the flag mid-run - // or a daemon was started before the flag flipped). Log so the drop is - // visible in audits. - if ( - config.reviewLearningsEnabled && - reviewLearningSaves !== undefined && - reviewLearningSaves.length > 0 - ) { - const { saveReviewLearnings } = await import("./review-learnings"); - const saved = await saveReviewLearnings(exec.repo_owner, exec.repo_name, reviewLearningSaves); - if (saved > 0) { - logger.info({ deliveryId, saved }, "Persisted review learnings from execution"); - } - } else if (reviewLearningSaves !== undefined && reviewLearningSaves.length > 0) { - logger.warn( - { deliveryId, dropped: reviewLearningSaves.length }, - "Dropped review-learning saves: REVIEW_LEARNINGS_ENABLED=false", - ); - } - if ( - config.reviewLearningsEnabled && - reviewLearningDeletes !== undefined && - reviewLearningDeletes.length > 0 - ) { - const { deleteReviewLearnings } = await import("./review-learnings"); - // Owner/repo-scoped delete: an id that doesn't belong to this job's - // (owner, repo) is silently no-opped (1.5.D). The caller-supplied - // exec.repo_owner / exec.repo_name come from the executions row we - // already SELECTed above. - const deleted = await deleteReviewLearnings( - exec.repo_owner, - exec.repo_name, - reviewLearningDeletes, - ); - if (deleted > 0) { - logger.info( - { deliveryId, deleted }, - "Deleted outdated review learnings per daemon request", - ); - } - } else if (reviewLearningDeletes !== undefined && reviewLearningDeletes.length > 0) { - // Symmetric audit log: when the kill-switch is off and the daemon sent - // deletes anyway (e.g. flag flipped mid-run), record the drop just like - // we do for save actions above. - logger.warn( - { deliveryId, dropped: reviewLearningDeletes.length }, - "Dropped review-learning deletes: REVIEW_LEARNINGS_ENABLED=false", - ); - } - // 1.5.E: bump use_count for the IDs the daemon actually applied to a - // prompt this run. Behind the same kill-switch. - if ( - config.reviewLearningsEnabled && - appliedReviewLearningIds !== undefined && - appliedReviewLearningIds.length > 0 - ) { - const { bumpReviewLearningUsage } = await import("./review-learnings"); - await bumpReviewLearningUsage(appliedReviewLearningIds); - logger.info( - { deliveryId, applied: appliedReviewLearningIds.length }, - "Bumped use_count for applied review learnings", - ); - } - } catch (err) { - logger.error({ err, deliveryId }, "Failed to persist repo knowledge"); - } -} - /** Persist execution outcome to the database. */ async function finalizeExecution( deliveryId: string, @@ -1518,14 +1470,20 @@ async function handleResult( (reviewLearningDeletes !== undefined && reviewLearningDeletes.length > 0) || (appliedReviewLearningIds !== undefined && appliedReviewLearningIds.length > 0) ) { - await persistRepoKnowledge( - actualDeliveryId, - learnings, - deletions, - reviewLearningSaves, - reviewLearningDeletes, - appliedReviewLearningIds, - ); + try { + await persistRepoKnowledge({ + deliveryId: actualDeliveryId, + daemonActions: { + learnings: learnings ?? [], + deletions: deletions ?? [], + ...(reviewLearningSaves !== undefined ? { reviewLearningSaves } : {}), + ...(reviewLearningDeletes !== undefined ? { reviewLearningDeletes } : {}), + }, + ...(appliedReviewLearningIds !== undefined ? { appliedReviewLearningIds } : {}), + }); + } catch (err) { + logger.error({ err, deliveryId: actualDeliveryId }, "Failed to persist repo knowledge"); + } } logger.info( diff --git a/src/orchestrator/history.ts b/src/orchestrator/history.ts index 6743b650..8f792fc7 100644 --- a/src/orchestrator/history.ts +++ b/src/orchestrator/history.ts @@ -4,6 +4,7 @@ import { config } from "../config"; import { getDb } from "../db"; import { logger } from "../logger"; import type { SerializableBotContext } from "../shared/daemon-types"; +import type { DispatchTarget } from "../shared/dispatch-types"; import type { ModelUsageEntry } from "../types"; import { decrementDaemonActiveJobs } from "./daemon-registry"; @@ -24,6 +25,8 @@ export interface CreateExecutionParams { eventName: string; triggerUsername: string; dispatchMode: string; + /** Exact persisted execution protocol. Defaults to the shared-daemon rail. */ + dispatchTarget?: DispatchTarget; /** * Dispatch-decision reason. Callers pass the resolved DispatchReason * (e.g. "persistent-daemon", "ephemeral-daemon-triage", "ephemeral-spawn-failed") @@ -51,8 +54,8 @@ export interface CreateExecutionParams { * Create an execution record when a webhook arrives. * Returns the generated UUID. */ -export async function createExecution(params: CreateExecutionParams): Promise { - const db = getDb(); +export async function createExecution(params: CreateExecutionParams, sql?: SQL): Promise { + const db = sql ?? getDb(); if (db === null) throw new Error("Database not configured"); const hasDispatchReason = params.dispatchReason !== undefined; @@ -67,10 +70,7 @@ export async function createExecution(params: CreateExecutionParams): Promise { - const db = getDb(); - if (db === null) return; +export type ExecutionOfferOutcome = "offered" | "daemon-inactive" | "stale"; - await db` - UPDATE executions - SET status = 'offered', daemon_id = ${daemonId} - WHERE delivery_id = ${deliveryId} AND status = 'queued' - `; +/** Assign a queued receipt only while the exact daemon row is still active. */ +export async function markExecutionOffered( + deliveryId: string, + daemonId: string, + sql: SQL | null = getDb(), +): Promise { + if (sql === null) return "offered"; + + return sql.begin(async (tx) => { + const active: { id: string }[] = await tx` + SELECT id + FROM daemons + WHERE id = ${daemonId} + AND status = 'active' + FOR UPDATE + `; + if (active[0] === undefined) return "daemon-inactive"; + const offered: { delivery_id: string }[] = await tx` + UPDATE executions + SET status = 'offered', daemon_id = ${daemonId} + WHERE delivery_id = ${deliveryId} + AND status = 'queued' + RETURNING delivery_id + `; + return offered[0] === undefined ? "stale" : "offered"; + }); } /** @@ -206,15 +225,175 @@ export async function markExecutionFailed(deliveryId: string, errorMessage: stri /** * Re-queue an execution (offered -> queued) after rejection or timeout. */ -export async function requeueExecution(deliveryId: string): Promise { +export async function requeueExecution(deliveryId: string): Promise { const db = getDb(); - if (db === null) return; + if (db === null) return true; - await db` + const rows: { delivery_id: string }[] = await db` UPDATE executions - SET status = 'queued', daemon_id = NULL - WHERE delivery_id = ${deliveryId} AND status = 'offered' + SET status = 'queued', daemon_id = NULL + WHERE delivery_id = ${deliveryId} + AND status = 'offered' + RETURNING delivery_id + `; + return rows[0] !== undefined; +} + +export interface DisconnectedDaemonCleanup { + readonly executionDeliveryIds: readonly string[]; + readonly workflowRunIds: readonly string[]; +} + +export interface FailedDaemonWorkflow { + readonly id: string; + readonly workflowName: string; + readonly ownerId: string; +} + +export interface FailedDaemonOwnership extends DisconnectedDaemonCleanup { + readonly daemonMarkedInactive: boolean; + readonly workflows: readonly FailedDaemonWorkflow[]; +} + +interface DaemonFailure { + readonly workflowReason: string; + readonly executionReason: string; + readonly clearOwner: boolean; +} + +/** Terminalize one shared daemon's durable ownership inside the caller's transaction. */ +export async function failDaemonOwnershipInTransaction( + daemonId: string, + failure: DaemonFailure, + tx: SQL, +): Promise { + const daemonRows: { id: string }[] = await tx` + UPDATE daemons + SET status = 'inactive', last_seen_at = now() + WHERE id = ${daemonId} + AND status = 'active' + RETURNING id `; + const terminalWorkflows: { + id: string; + workflow_name: string; + execution_delivery_id: string | null; + propagated_parent: boolean; + }[] = await tx` + WITH failed_workflows AS ( + UPDATE workflow_runs + SET status = 'failed', + owner_kind = CASE WHEN ${failure.clearOwner}::boolean THEN NULL ELSE owner_kind END, + owner_id = CASE WHEN ${failure.clearOwner}::boolean THEN NULL ELSE owner_id END, + attempt_completed_at = COALESCE(attempt_completed_at, now()), + state = state || jsonb_build_object( + 'phase', 'orphaned', + 'failedReason', ${failure.workflowReason}::text + ) + WHERE owner_kind = 'daemon' + AND owner_id = ${daemonId} + AND attempt_id IS NULL + AND status IN ('queued', 'running') + RETURNING id, workflow_name, execution_delivery_id, parent_run_id, parent_step_index + ), + failed_parent_inputs AS ( + SELECT parent_run_id, + min(COALESCE(parent_step_index, -1)) AS failed_step_index + FROM failed_workflows + WHERE parent_run_id IS NOT NULL + GROUP BY parent_run_id + ), + failed_parents AS ( + UPDATE workflow_runs AS parent + SET status = 'failed', + attempt_completed_at = COALESCE(parent.attempt_completed_at, now()), + state = parent.state || jsonb_build_object( + 'phase', 'orphaned', + 'failedAtStepIndex', failed_parent_inputs.failed_step_index, + 'failedReason', ${failure.workflowReason}::text + ) + FROM failed_parent_inputs + WHERE parent.id = failed_parent_inputs.parent_run_id + AND parent.status = 'running' + AND NOT EXISTS ( + SELECT 1 FROM failed_workflows AS direct WHERE direct.id = parent.id + ) + RETURNING parent.id, parent.workflow_name, parent.execution_delivery_id + ) + SELECT id, workflow_name, execution_delivery_id, false AS propagated_parent + FROM failed_workflows + UNION ALL + SELECT id, workflow_name, execution_delivery_id, true AS propagated_parent + FROM failed_parents + `; + const workflows = terminalWorkflows.filter((row) => !row.propagated_parent); + const workflowDeliveries = workflows.flatMap((row) => + row.execution_delivery_id === null ? [] : [row.execution_delivery_id], + ); + const executions: { delivery_id: string }[] = + workflowDeliveries.length === 0 + ? await tx` + UPDATE executions + SET status = 'failed', + completed_at = now(), + error_message = ${failure.executionReason} + WHERE daemon_id = ${daemonId} + AND offer_id IS NULL + AND status IN ('queued', 'offered', 'running') + RETURNING delivery_id + ` + : await tx` + UPDATE executions + SET status = 'failed', + completed_at = now(), + error_message = ${failure.executionReason} + WHERE (daemon_id = ${daemonId} OR delivery_id IN ${tx(workflowDeliveries)}) + AND offer_id IS NULL + AND status IN ('queued', 'offered', 'running') + RETURNING delivery_id + `; + const deliveryIds = executions.map((row) => row.delivery_id); + if (deliveryIds.length > 0) { + await tx` + UPDATE scheduled_action_state + SET in_flight_job_id = NULL, + in_flight_started_at = NULL + WHERE in_flight_job_id IN ${tx(deliveryIds)} + `; + } + return { + daemonMarkedInactive: daemonRows[0] !== undefined, + executionDeliveryIds: deliveryIds, + workflowRunIds: terminalWorkflows.map((row) => row.id), + workflows: workflows.map((row) => ({ + id: row.id, + workflowName: row.workflow_name, + ownerId: daemonId, + })), + }; +} + +/** Fence one shared-daemon incarnation and terminalize all ownership atomically. */ +export async function failDisconnectedDaemon( + daemonId: string, + sql: SQL | null = getDb(), +): Promise { + if (sql === null) return { executionDeliveryIds: [], workflowRunIds: [] }; + const failed = await sql.begin((tx) => + failDaemonOwnershipInTransaction( + daemonId, + { + workflowReason: "daemon disconnected during execution", + executionReason: "daemon disconnected during execution", + clearOwner: true, + }, + tx, + ), + ); + return { + executionDeliveryIds: failed.executionDeliveryIds, + workflowRunIds: failed.workflowRunIds, + }; } /** @@ -272,8 +451,11 @@ export async function recoverStaleExecutions(db: SQL): Promise { await db` SELECT id, delivery_id, daemon_id, status FROM executions - WHERE (status = 'running' AND started_at < now() - make_interval(secs => ${thresholdMs / 1000})) - OR (status = 'offered' AND created_at < now() - make_interval(secs => ${thresholdMs / 1000})) + WHERE offer_id IS NULL + AND ( + (status = 'running' AND started_at < now() - make_interval(secs => ${thresholdMs / 1000})) + OR (status = 'offered' AND created_at < now() - make_interval(secs => ${thresholdMs / 1000})) + ) `; if (staleRows.length === 0) return; diff --git a/src/orchestrator/installation-token.ts b/src/orchestrator/installation-token.ts index c723e553..6cf2652c 100644 --- a/src/orchestrator/installation-token.ts +++ b/src/orchestrator/installation-token.ts @@ -1,8 +1,7 @@ /** * Observed installation-token mint helper (issue #236). * - * Six call sites mint App installation tokens (handleAccept, handleScopedAccept, - * postOrphanNotification, shipTickleResume, proposalPoller, schedulerRunAction). + * App installation-token call sites share one observed mint path. * None emitted a structured event, so operators had no `cache_hit` signal, no * per-call latency, and no per-installation correlation. This helper wraps * `app.getInstallationOctokit(installationId)` + `resolveGithubToken(octokit)` @@ -24,7 +23,8 @@ * success; the failure line adds the standard pino `err` field, serialized * through the secret-scrubbing `errSerializer` in `src/utils/log-redaction.ts`. */ -import type { App, Octokit } from "octokit"; +import { type App, Octokit } from "octokit"; +import { z } from "zod"; import { resolveGithubToken } from "../core/github-token"; import type { Logger } from "../logger"; @@ -38,6 +38,8 @@ interface MintArgs { readonly installationId: number; readonly via: TokenMintVia; readonly log: Logger; + /** Restrict the token to one repository. Omit for the existing fleet path. */ + readonly repositoryName?: string; } interface MintResult { @@ -45,6 +47,50 @@ interface MintResult { readonly token: string; } +interface ScopedMintResult extends MintResult { + readonly expiresAt: string; +} + +const TOKEN_REVOCATION_TIMEOUT_MS = 10_000; + +/** Revoke the installation token authenticating this client without masking caller failures. */ +export async function revokeInstallationToken( + octokit: Octokit, + log: Logger, + fields: Readonly> = {}, +): Promise { + try { + await octokit.request("DELETE /installation/token", { + request: { signal: AbortSignal.timeout(TOKEN_REVOCATION_TIMEOUT_MS) }, + }); + return true; + } catch (err) { + log.error({ ...fields, err }, "Installation token revocation failed"); + return false; + } +} + +/** Construct and revoke a token client while keeping cleanup best-effort. */ +export async function revokeInstallationTokenValue( + token: string, + log: Logger, + fields: Readonly> = {}, +): Promise { + try { + return await revokeInstallationToken(new Octokit({ auth: token }), log, fields); + } catch (err) { + log.error({ ...fields, err }, "Installation token revocation failed"); + return false; + } +} + +export function mintInstallationToken( + args: MintArgs & { repositoryName: string }, +): Promise; +export function mintInstallationToken( + args: MintArgs & { repositoryName?: undefined }, +): Promise; + /** * Mint (or cache-serve) an installation token, returning both the installation * octokit and the resolved token string so callers that need either get one @@ -57,7 +103,8 @@ export async function mintInstallationToken({ installationId, via, log, -}: MintArgs): Promise { + repositoryName, +}: MintArgs): Promise { // A cache miss routes the access-tokens POST through `app.octokit`. The // before-hook receives the merged-but-unparsed endpoint options, so `url` is // the route template and `installation_id` is a top-level merged param. Match @@ -79,20 +126,22 @@ export async function mintInstallationToken({ const start = Date.now(); app.octokit.hook.before("request", probe); try { - const octokit = (await app.getInstallationOctokit(installationId)) as unknown as Octokit; - const token = await resolveGithubToken(octokit); - const duration_ms = Date.now() - start; - log.info( - { - event: GITHUB_APP_TOKEN_LOG_EVENTS.mintSucceeded, + if (repositoryName === undefined) { + const octokit = (await app.getInstallationOctokit(installationId)) as unknown as Octokit; + const token = await resolveGithubToken(octokit); + logMintSuccess(log, installationId, via, networkMint, start); + return { octokit, token }; + } else { + const response = await app.octokit.rest.apps.createInstallationAccessToken({ installation_id: installationId, - via, - cache_hit: !networkMint, - duration_ms, - }, - "Installation token minted", - ); - return { octokit, token }; + repositories: [repositoryName], + }); + const token = response.data.token; + const expiresAt = z.iso.datetime().parse(response.data.expires_at); + const octokit = new Octokit({ auth: token }); + logMintSuccess(log, installationId, via, networkMint, start); + return { octokit, token, expiresAt }; + } } catch (err) { const duration_ms = Date.now() - start; log.warn( @@ -110,3 +159,22 @@ export async function mintInstallationToken({ app.octokit.hook.remove("request", probe); } } + +function logMintSuccess( + log: Logger, + installationId: number, + via: TokenMintVia, + networkMint: boolean, + start: number, +): void { + log.info( + { + event: GITHUB_APP_TOKEN_LOG_EVENTS.mintSucceeded, + installation_id: installationId, + via, + cache_hit: !networkMint, + duration_ms: Date.now() - start, + }, + "Installation token minted", + ); +} diff --git a/src/orchestrator/job-dispatcher.ts b/src/orchestrator/job-dispatcher.ts index b5b72489..c5218a7f 100644 --- a/src/orchestrator/job-dispatcher.ts +++ b/src/orchestrator/job-dispatcher.ts @@ -1,9 +1,11 @@ import { config } from "../config"; import { logger } from "../logger"; import type { DaemonCapabilities, PendingOffer } from "../shared/daemon-types"; -import type { WorkflowRunRef } from "../shared/workflow-types"; -import { createMessageEnvelope, type ScopedJobContext } from "../shared/ws-messages"; -import { markFailed as markWorkflowRunFailed } from "../workflows/runs-store"; +import { + type AgentPolicy, + createMessageEnvelope, + type ScopedJobContext, +} from "../shared/ws-messages"; import { getConnections, getDaemonInfo, isDaemonDraining } from "./connection-handler"; import { getActiveDaemons, getDaemonActiveJobs } from "./daemon-registry"; import { markExecutionFailed, markExecutionOffered, requeueExecution } from "./history"; @@ -138,11 +140,12 @@ export async function selectDaemon(requiredTools: string[]): Promise { + if (job.kind === "workflow-run") { + throw new Error("workflow-run jobs require an isolated workflow runner"); + } const requiredTools = inferRequiredTools(job.labels, job.triggerBodyPreview); const daemonId = await selectDaemon(requiredTools); const fleetSize = getConnections().size; @@ -184,13 +187,27 @@ export async function dispatchJob(job: QueuedJob): Promise { const offerId = crypto.randomUUID(); - await markExecutionOffered(job.deliveryId, daemonId); + const offerOutcome = await markExecutionOffered(job.deliveryId, daemonId); + if (offerOutcome === "daemon-inactive") return false; + if (offerOutcome === "stale") { + logger.info( + { deliveryId: job.deliveryId, daemonId }, + "Execution receipt was no longer queued; consuming stale queue copy", + ); + return true; + } + + // The close callback removes the socket synchronously before its durable + // cleanup starts. Recheck after the DB await so no offer is assigned to a + // connection that disappeared while the receipt was being fenced. + if (connections.get(daemonId) !== ws) { + const requeued = await requeueExecution(job.deliveryId); + return !requeued; + } - if (isScopedJob(job)) { - ws.sendText(JSON.stringify(buildScopedJobOfferEnvelope(offerId, job))); - } else { - ws.sendText( - JSON.stringify({ + const envelope = isScopedJob(job) + ? buildScopedJobOfferEnvelope(offerId, job) + : { type: "job:offer", ...createMessageEnvelope(offerId), payload: { @@ -205,8 +222,19 @@ export async function dispatchJob(job: QueuedJob): Promise { triggerBodyPreview: job.triggerBodyPreview, requiredTools, }, - }), - ); + }; + let sent: number; + try { + sent = ws.sendText(JSON.stringify(envelope)); + } catch (err) { + logger.warn({ err, daemonId, deliveryId: job.deliveryId }, "Daemon offer send threw"); + const requeued = await requeueExecution(job.deliveryId); + return !requeued; + } + if (sent === 0) { + logger.warn({ daemonId, deliveryId: job.deliveryId }, "Daemon offer frame was dropped"); + const requeued = await requeueExecution(job.deliveryId); + return !requeued; } const timer = setTimeout(() => { @@ -228,7 +256,6 @@ export async function dispatchJob(job: QueuedJob): Promise { triggerUsername: job.triggerUsername, labels: job.labels, triggerBodyPreview: job.triggerBodyPreview, - ...(job.kind === "workflow-run" ? { workflowRun: normalizeWorkflowRun(job.workflowRun) } : {}), ...(isScopedJob(job) ? { scoped: job } : {}), }); @@ -252,32 +279,7 @@ export async function dispatchJob(job: QueuedJob): Promise { } /** - * `WorkflowRunRef` declares optional fields without `| undefined`; the - * Zod-inferred shape includes `| undefined` because Zod surfaces missing - * keys as `undefined`. Strip the explicit `undefined` keys so the value - * fits `exactOptionalPropertyTypes: true` consumers like `PendingOffer`. - */ -function normalizeWorkflowRun(ref: { - runId: string; - workflowName: WorkflowRunRef["workflowName"]; - parentRunId?: string | undefined; - parentStepIndex?: number | undefined; -}): WorkflowRunRef { - const result: WorkflowRunRef = { runId: ref.runId, workflowName: ref.workflowName }; - if (ref.parentRunId !== undefined && ref.parentStepIndex !== undefined) { - return { ...result, parentRunId: ref.parentRunId, parentStepIndex: ref.parentStepIndex }; - } - if (ref.parentRunId !== undefined) { - return { ...result, parentRunId: ref.parentRunId }; - } - if (ref.parentStepIndex !== undefined) { - return { ...result, parentStepIndex: ref.parentStepIndex }; - } - return result; -} - -/** - * Build the `scoped-job-offer` envelope from a scoped queue payload. The + * Build the `scoped-job:offer` envelope from a scoped queue payload. The * shape mirrors `contracts/ws-messages.md`: only the per-kind discriminating * fields are included so the daemon can route via Zod discriminated-union * parse before any executor runs. @@ -287,7 +289,7 @@ function buildScopedJobOfferEnvelope( job: ScopedQueuedJob, ): Record { const base = { - type: "scoped-job-offer" as const, + type: "scoped-job:offer" as const, ...createMessageEnvelope(offerId), }; switch (job.kind) { @@ -423,30 +425,13 @@ function reconstructJobFromOffer(offer: PendingOffer): QueuedJob | null { "PendingOffer.scoped failed re-validation, failing job (do not fall back to legacy reconstruct)", ); // Fail closed: a corrupted scoped offer must NOT be reconstructed as a - // legacy or workflow-run job: that would dispatch the wrong job kind + // legacy job: that would dispatch the wrong job kind // against the same repo/PR. Caller marks the execution failed. return null; } const scoped: ScopedQueuedJob = reparsed.data; return { ...scoped, retryCount: offer.retryCount, enqueuedAt: Date.now() }; } - if (offer.workflowRun !== undefined) { - return { - kind: "workflow-run", - deliveryId: offer.deliveryId, - repoOwner: offer.repoOwner, - repoName: offer.repoName, - entityNumber: offer.entityNumber, - isPR: offer.isPR, - eventName: offer.eventName, - triggerUsername: offer.triggerUsername, - labels: offer.labels, - triggerBodyPreview: offer.triggerBodyPreview, - enqueuedAt: Date.now(), - retryCount: offer.retryCount, - workflowRun: offer.workflowRun, - }; - } return { kind: "legacy", deliveryId: offer.deliveryId, @@ -498,8 +483,10 @@ export interface JobAcceptParams { sourceAuthor: string | null; createdAt?: string | undefined; }[]; - /** Present for workflow-run jobs, forwarded verbatim into `job:payload`. */ - workflowRun?: WorkflowRunRef; + /** Resolved per-repo agent knobs ("Gate 2"). Already clamped by + * `loadRepoPolicy`; forwarded verbatim. Omitted when the repo ships no + * `.github-app.yaml`, which keeps the pre-Gate-2 payload byte-identical. */ + policy?: AgentPolicy; /** Present for scoped jobs, forwarded verbatim into `job:payload` so the * daemon's `runScopedJob` router can dispatch on `scoped.jobKind`. */ scoped?: ScopedJobContext; @@ -517,7 +504,7 @@ export function handleJobAccept({ envVars, memory, reviewLearnings, - workflowRun, + policy, scoped, }: JobAcceptParams): void { // Note: the pending offer is already removed by handleAccept in connection-handler.ts @@ -543,7 +530,7 @@ export function handleJobAccept({ ...(Object.keys(envVars).length > 0 ? { envVars } : {}), ...(memory.length > 0 ? { memory } : {}), ...(reviewLearnings !== undefined && reviewLearnings.length > 0 ? { reviewLearnings } : {}), - ...(workflowRun !== undefined ? { workflowRun } : {}), + ...(policy !== undefined ? { policy } : {}), ...(scoped !== undefined ? { scoped } : {}), }, }), @@ -603,29 +590,11 @@ export async function handleJobReject(offerId: string, reason: string): Promise< } /** - * Terminal failure write that covers both the legacy `executions` row (set by - * `src/webhook/router.ts` for the `@chrisleekr-bot` mention path) and the - * `workflow_runs` row (set by the workflow dispatcher path). Either or both - * may be present for a given job; UPDATEs are no-ops when the row is absent. - * - * Marking the `workflow_runs` row as `failed` is essential: the partial - * unique index `idx_workflow_runs_inflight` prevents future dispatches for - * the same target until this row leaves the queued/running states. + * Mark a shared-daemon queue item terminal after exhausting offer retries. */ export async function markJobTerminallyFailed(job: QueuedJob, reason: string): Promise { - await markExecutionFailed(job.deliveryId, reason); if (job.kind === "workflow-run") { - try { - await markWorkflowRunFailed(job.workflowRun.runId, reason, {}); - } catch (err) { - logger.error( - { - err: err instanceof Error ? err.message : String(err), - runId: job.workflowRun.runId, - deliveryId: job.deliveryId, - }, - "Failed to mark workflow_runs row as failed, in-flight guard may block re-dispatch", - ); - } + throw new Error("workflow-run terminalization belongs to the isolated runner"); } + await markExecutionFailed(job.deliveryId, reason); } diff --git a/src/orchestrator/job-queue.ts b/src/orchestrator/job-queue.ts index 8d68cf05..5c13459a 100644 --- a/src/orchestrator/job-queue.ts +++ b/src/orchestrator/job-queue.ts @@ -2,6 +2,7 @@ import { z } from "zod"; import { config } from "../config"; import { logger } from "../logger"; +import { WorkflowNameSchema } from "../shared/workflow-types"; import { requireValkeyClient } from "./valkey"; /** @@ -12,12 +13,7 @@ import { requireValkeyClient } from "./valkey"; const workflowRunRefSchema = z.object({ runId: z.string().min(1), - // Mirror of registry.ts WorkflowNameSchema. Hardcoded rather than imported - // to avoid the registry → handlers → job-queue cycle (handlers/ship.ts - // and ship/iteration.ts call enqueueJob, so importing the schema back - // here would init-deadlock). When adding a new workflow, extend both - // lists; TypeScript will surface the gap at every enqueueJob call site. - workflowName: z.enum(["triage", "plan", "implement", "review", "resolve", "ship", "remember"]), + workflowName: WorkflowNameSchema, parentRunId: z.string().min(1).optional(), parentStepIndex: z.number().int().nonnegative().optional(), }); @@ -158,6 +154,17 @@ export function isScopedJob(job: QueuedJob): job is ScopedQueuedJob { const QUEUE_KEY = "queue:jobs"; const PROCESSING_KEY_PREFIX = "queue:processing:"; +const ENSURE_WORKFLOW_JOB_LUA = ` + if redis.call('LPOS', KEYS[1], ARGV[1]) then + return 0 + end + if redis.call('LPOS', KEYS[2], ARGV[1]) then + return 0 + end + redis.call('LPUSH', KEYS[1], ARGV[1]) + return 1 +`; + /** Build the per-instance processing-list key. Exposed for the cross-instance reaper. */ export function processingListKey(instanceId: string): string { return `${PROCESSING_KEY_PREFIX}${instanceId}`; @@ -204,6 +211,33 @@ export async function enqueueJob(job: QueuedJob): Promise { ); } +/** Ensure one byte-stable workflow wake-up exists in the supported controller's lists. */ +export async function ensureWorkflowJobQueued( + job: WorkflowRunQueuedJob, + instanceId: string, +): Promise { + const validated = workflowRunJobSchema.parse(job); + const raw = JSON.stringify(validated); + const valkey = requireValkeyClient(); + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment -- Valkey EVAL returns an integer + const inserted: number = await valkey.send("EVAL", [ + ENSURE_WORKFLOW_JOB_LUA, + "2", + QUEUE_KEY, + processingListKey(instanceId), + raw, + ]); + logger.info( + { + deliveryId: validated.deliveryId, + runId: validated.workflowRun.runId, + inserted: inserted === 1, + }, + "Workflow dispatch wake-up reconciled", + ); + return inserted === 1; +} + /** * Return the current queue depth. Used by the ephemeral-daemon scaler * to detect persistent-pool backpressure. Non-blocking; returns 0 on @@ -357,6 +391,50 @@ export async function requeueLeasedJob( return updated.retryCount; } +const DEFER_RECEIPT_TTL_SECONDS = 86_400; +const DEFER_LEASED_JOB_LUA = ` + if redis.call('EXISTS', KEYS[3]) == 1 then + return 2 + end + local removed = redis.call('LREM', KEYS[1], 1, ARGV[1]) + if removed == 0 then + return 0 + end + redis.call('LPUSH', KEYS[2], ARGV[2]) + redis.call('SET', KEYS[3], '1', 'EX', ARGV[3]) + return 1 +`; + +export interface LeasedJobDeferralResult { + readonly status: "moved" | "already-moved" | "missing"; +} + +/** Return a capacity-limited workflow lease without consuming a retry. */ +export async function deferLeasedWorkflowJob( + instanceId: string, + raw: string, + job: WorkflowRunQueuedJob, + deferralId: string, +): Promise { + const valkey = requireValkeyClient(); + const stableRaw = JSON.stringify(workflowRunJobSchema.parse(job)); + if (stableRaw !== raw) { + throw new Error("Workflow queue item changed after its durable publication"); + } + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment -- Valkey EVAL returns an integer receipt + const receipt: number = await valkey.send("EVAL", [ + DEFER_LEASED_JOB_LUA, + "3", + processingListKey(instanceId), + QUEUE_KEY, + `queue:workflow-deferral-receipt:${instanceId}:${deferralId}`, + raw, + raw, + String(DEFER_RECEIPT_TTL_SECONDS), + ]); + return { status: receipt === 1 ? "moved" : receipt === 2 ? "already-moved" : "missing" }; +} + /** * Drain a (possibly-orphaned) processing list back into `queue:jobs`. Called * on this instance's own list at startup, and on dead-instance lists by the diff --git a/src/orchestrator/liveness-reaper.ts b/src/orchestrator/liveness-reaper.ts index 2647600b..e1a48507 100644 --- a/src/orchestrator/liveness-reaper.ts +++ b/src/orchestrator/liveness-reaper.ts @@ -2,35 +2,46 @@ import type { SQL } from "bun"; import { config } from "../config"; import { requireDb } from "../db"; +import { clearInFlightByJobId } from "../db/queries/scheduled-actions-store"; import { logger } from "../logger"; +import { WS_CLOSE_CODES } from "../shared/ws-messages"; +import { reconcilePendingWorkflowCascades } from "../workflows/completion-reconciler"; +import { publishPendingWorkflowRuns } from "../workflows/dispatch-outbox"; +import { + expireQueuedWorkflowDispatches, + expireWorkflowAttempts, + type WorkflowRunRow, +} from "../workflows/runs-store"; +import { getConnections } from "./connection-handler"; +import { failDaemonOwnershipInTransaction } from "./history"; +import { getInstanceId } from "./instance-id"; import { requireValkeyClient } from "./valkey"; +import { reapOrphanProcessingLists } from "./valkey-cleanup"; +import { + notifyExpiredWorkflowAttempts, + notifyExpiredWorkflowDispatches, +} from "./workflow-expiry-notifier"; +import { reconcileWorkflowRunners } from "./workflow-runner-reconciler"; const SCAN_BATCH = 100; const ORCH_KEY_PREFIX = "orchestrator:"; const ORCH_KEY_SUFFIX = ":alive"; let timer: ReturnType | null = null; +let inFlight: Promise | null = null; /** - * Heartbeat-based reaper for `workflow_runs` and the `daemons` table. + * Lease and heartbeat reaper for `workflow_runs` and the `daemons` table. * - * Every in-flight `workflow_runs` row carries `(owner_kind, owner_id)` - * pointing at the process responsible for advancing it. Liveness is read - * from Valkey: + * Isolated workflow runners are fenced by their PostgreSQL lease. Legacy + * orchestrator owners and shared-daemon registry rows use Valkey liveness: * * - `orchestrator` owners → key `orchestrator:{owner_id}:alive` * (published by `instance-liveness.ts`, 60s TTL, refreshed every 20s) - * - `daemon` owners → key `daemon:{owner_id}` - * (published by `daemon-registry.ts`, 90s TTL, refreshed on pong) + * - shared daemon owners → key `daemon:{owner_id}` * - * If the heartbeat key is missing the owner is treated as dead and the row - * is flipped to `'failed'` with a reaped-by reason in `state`. Pre-existing - * rows with NULL ownership (migrated from before column 006) are left - * alone. - * - * Idempotent + race-safe: each pass is a single SQL `UPDATE` per owner kind, - * so multiple orchestrators running concurrently just see zero affected - * rows after the first winner. + * Each transition is conditional on the current owner, attempt, lease, and + * status. Repeated passes therefore converge after the first successful write. */ async function listLiveOrchestratorIds(): Promise { @@ -72,8 +83,91 @@ async function listLiveDaemonIds(): Promise { interface ReapedRow { readonly id: string; readonly workflow_name: string; - readonly owner_kind: "orchestrator" | "daemon"; - readonly owner_id: string; + readonly owner_kind: "orchestrator" | "daemon" | null; + readonly owner_id: string | null; +} + +interface DeadDaemonReapResult { + readonly rows: ReapedRow[]; + readonly daemonsMarkedInactive: number; +} + +async function listDaemonCandidates(sql: SQL): Promise { + const rows: { id: string }[] = await sql` + SELECT DISTINCT id + FROM ( + SELECT id + FROM daemons + WHERE status = 'active' + UNION + SELECT owner_id AS id + FROM workflow_runs + WHERE owner_kind = 'daemon' + AND owner_id IS NOT NULL + AND attempt_id IS NULL + AND status IN ('queued', 'running') + UNION + SELECT daemon_id AS id + FROM executions + WHERE daemon_id IS NOT NULL + AND offer_id IS NULL + AND status IN ('queued', 'offered', 'running') + ) AS candidates + WHERE id IS NOT NULL + `; + return rows.map((row) => row.id); +} + +/** Lock, recheck, and fence one candidate so a reconnect cannot be reaped from a stale snapshot. */ +async function reapDeadDaemonCandidates( + sql: SQL, + initiallyLiveDaemonIds: readonly string[], +): Promise { + const initiallyLive = new Set(initiallyLiveDaemonIds); + const candidates = (await listDaemonCandidates(sql)).filter((id) => !initiallyLive.has(id)); + const rows: ReapedRow[] = []; + let daemonsMarkedInactive = 0; + + for (const daemonId of candidates) { + try { + // eslint-disable-next-line no-await-in-loop -- each owner is fenced in its own short transaction + const reaped = await sql.begin(async (tx) => { + await tx`SELECT id FROM daemons WHERE id = ${daemonId} FOR UPDATE`; + const valkey = requireValkeyClient(); + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment -- Valkey EXISTS returns number + const exists: number = await valkey.send("EXISTS", [`daemon:${daemonId}`]); + if (exists === 1) return null; + + const connection = getConnections().get(daemonId); + connection?.close( + WS_CLOSE_CODES.HEARTBEAT_TIMEOUT.code, + WS_CLOSE_CODES.HEARTBEAT_TIMEOUT.reason, + ); + return failDaemonOwnershipInTransaction( + daemonId, + { + workflowReason: `owner daemon:${daemonId} is no longer alive`, + executionReason: "Owning daemon is no longer alive", + clearOwner: false, + }, + tx, + ); + }); + if (reaped === null) continue; + if (reaped.daemonMarkedInactive) daemonsMarkedInactive++; + rows.push( + ...reaped.workflows.map((workflow) => ({ + id: workflow.id, + workflow_name: workflow.workflowName, + owner_kind: "daemon" as const, + owner_id: workflow.ownerId, + })), + ); + } catch (err) { + logger.error({ err, daemonId }, "Dead daemon recheck failed, leaving ownership unchanged"); + } + } + return { rows, daemonsMarkedInactive }; } export interface ReapResult { @@ -86,85 +180,115 @@ export interface ReapResult { * (e.g. on startup, before the periodic timer kicks in). */ export async function reapOnce(sql: SQL = requireDb()): Promise { - const [orchIds, daemonIds] = await Promise.all([listLiveOrchestratorIds(), listLiveDaemonIds()]); + await reapOrphanProcessingLists(getInstanceId()).catch((err: unknown) => { + logger.warn({ err }, "Orphan processing-list recovery failed"); + }); + const expiredDispatches = await expireQueuedWorkflowExecutions(sql); + await notifyExpiredWorkflowDispatches(expiredDispatches).catch((err: unknown) => { + logger.warn({ err }, "Expired workflow dispatch notification pass failed"); + }); + const expiredAttempts = await expireLeasedWorkflowExecutions(sql); + await notifyExpiredWorkflowAttempts(expiredAttempts).catch((err: unknown) => { + logger.warn({ err }, "Expired workflow notification pass failed"); + }); + await reconcileWorkflowRunners().catch((err: unknown) => { + logger.warn({ err }, "Workflow runner reconciliation failed"); + }); + await reconcilePendingWorkflowCascades(sql).catch((err: unknown) => { + logger.warn({ err }, "Pending workflow cascade reconciliation failed"); + }); + await publishPendingWorkflowRuns(sql).catch((err: unknown) => { + logger.warn({ err }, "Pending workflow dispatch reconciliation failed"); + }); + + let orchIds: string[]; + let daemonIds: string[]; + try { + [orchIds, daemonIds] = await Promise.all([listLiveOrchestratorIds(), listLiveDaemonIds()]); + } catch (err) { + logger.error({ err }, "Valkey liveness read failed after workflow lease expiry"); + return { + workflowRunsReaped: [...expiredDispatches, ...expiredAttempts], + daemonsMarkedInactive: 0, + }; + } const reapedOrch: ReapedRow[] = orchIds.length === 0 ? await sql` - UPDATE workflow_runs - SET status = 'failed', - state = state || jsonb_build_object( - 'failedReason', 'owner orchestrator:' || owner_id || ' is no longer alive', - 'reapedAt', now() - ) - WHERE status IN ('queued', 'running') - AND owner_kind = 'orchestrator' - RETURNING id, workflow_name, owner_kind, owner_id + WITH reaped AS ( + UPDATE workflow_runs + SET status = 'failed', + state = state || jsonb_build_object( + 'failedReason', 'owner orchestrator:' || owner_id || ' is no longer alive', + 'reapedAt', now() + ) + WHERE status IN ('queued', 'running') + AND owner_kind = 'orchestrator' + AND NOT (status = 'queued' AND dispatch_enqueued_at IS NULL) + RETURNING id, workflow_name, owner_kind, owner_id, execution_delivery_id + ), failed_executions AS ( + UPDATE executions AS e + SET status = 'failed', + completed_at = now(), + error_message = 'Owning orchestrator is no longer alive' + FROM reaped + WHERE e.delivery_id = reaped.execution_delivery_id + AND e.status IN ('queued', 'offered', 'running') + RETURNING e.delivery_id + ), released_locks AS ( + UPDATE scheduled_action_state + SET in_flight_job_id = NULL, + in_flight_started_at = NULL + WHERE in_flight_job_id IN (SELECT delivery_id FROM failed_executions) + ) + SELECT id, workflow_name, owner_kind, owner_id FROM reaped ` : await sql` - UPDATE workflow_runs - SET status = 'failed', - state = state || jsonb_build_object( - 'failedReason', 'owner orchestrator:' || owner_id || ' is no longer alive', - 'reapedAt', now() - ) - WHERE status IN ('queued', 'running') - AND owner_kind = 'orchestrator' - AND owner_id NOT IN ${sql(orchIds)} - RETURNING id, workflow_name, owner_kind, owner_id + WITH reaped AS ( + UPDATE workflow_runs + SET status = 'failed', + state = state || jsonb_build_object( + 'failedReason', 'owner orchestrator:' || owner_id || ' is no longer alive', + 'reapedAt', now() + ) + WHERE status IN ('queued', 'running') + AND owner_kind = 'orchestrator' + AND NOT (status = 'queued' AND dispatch_enqueued_at IS NULL) + AND owner_id NOT IN ${sql(orchIds)} + RETURNING id, workflow_name, owner_kind, owner_id, execution_delivery_id + ), failed_executions AS ( + UPDATE executions AS e + SET status = 'failed', + completed_at = now(), + error_message = 'Owning orchestrator is no longer alive' + FROM reaped + WHERE e.delivery_id = reaped.execution_delivery_id + AND e.status IN ('queued', 'offered', 'running') + RETURNING e.delivery_id + ), released_locks AS ( + UPDATE scheduled_action_state + SET in_flight_job_id = NULL, + in_flight_started_at = NULL + WHERE in_flight_job_id IN (SELECT delivery_id FROM failed_executions) + ) + SELECT id, workflow_name, owner_kind, owner_id FROM reaped `; - const reapedDaemon: ReapedRow[] = - daemonIds.length === 0 - ? await sql` - UPDATE workflow_runs - SET status = 'failed', - state = state || jsonb_build_object( - 'failedReason', 'owner daemon:' || owner_id || ' is no longer alive', - 'reapedAt', now() - ) - WHERE status IN ('queued', 'running') - AND owner_kind = 'daemon' - RETURNING id, workflow_name, owner_kind, owner_id - ` - : await sql` - UPDATE workflow_runs - SET status = 'failed', - state = state || jsonb_build_object( - 'failedReason', 'owner daemon:' || owner_id || ' is no longer alive', - 'reapedAt', now() - ) - WHERE status IN ('queued', 'running') - AND owner_kind = 'daemon' - AND owner_id NOT IN ${sql(daemonIds)} - RETURNING id, workflow_name, owner_kind, owner_id - `; + const deadDaemons = await reapDeadDaemonCandidates(sql, daemonIds); + const reapedDaemon = deadDaemons.rows; - // Daemons-table sweep: any daemons row still 'active' whose Valkey - // heartbeat is missing flips to 'inactive'. Replaces the prior - // time-threshold `reapStaleDaemons` (5-minute blind window). - const daemonRowsReaped: { id: string }[] = - daemonIds.length === 0 - ? await sql` - UPDATE daemons - SET status = 'inactive' - WHERE status = 'active' - RETURNING id - ` - : await sql` - UPDATE daemons - SET status = 'inactive' - WHERE status = 'active' - AND id NOT IN ${sql(daemonIds)} - RETURNING id - `; - - const workflowRunsReaped = [...reapedOrch, ...reapedDaemon]; - if (workflowRunsReaped.length > 0 || daemonRowsReaped.length > 0) { + const workflowRunsReaped = [ + ...expiredDispatches, + ...expiredAttempts, + ...reapedOrch, + ...reapedDaemon, + ]; + if (workflowRunsReaped.length > 0 || deadDaemons.daemonsMarkedInactive > 0) { logger.info( { workflowRunsReaped: workflowRunsReaped.length, - daemonsMarkedInactive: daemonRowsReaped.length, + daemonsMarkedInactive: deadDaemons.daemonsMarkedInactive, liveOrchestratorCount: orchIds.length, liveDaemonCount: daemonIds.length, reapedRunIds: workflowRunsReaped.map((r) => r.id), @@ -181,7 +305,110 @@ export async function reapOnce(sql: SQL = requireDb()): Promise { ); } - return { workflowRunsReaped, daemonsMarkedInactive: daemonRowsReaped.length }; + return { workflowRunsReaped, daemonsMarkedInactive: deadDaemons.daemonsMarkedInactive }; +} + +async function expireQueuedWorkflowExecutions(sql: SQL): Promise { + if (typeof sql.begin !== "function") { + return expireQueuedWorkflowDispatches( + config.workflowDispatchTimeoutMs, + config.jobMaxRetries, + sql, + ); + } + + return sql.begin(async (tx) => { + const rows = await expireQueuedWorkflowDispatches( + config.workflowDispatchTimeoutMs, + config.jobMaxRetries, + tx, + ); + for (const row of rows) { + const failureReason = + row.state["failedReason"] === "workflow dispatch retries exhausted" + ? "workflow dispatch retries exhausted" + : "workflow dispatch deadline expired"; + if (row.parent_run_id !== null) { + const parentFailure = { + failedAtStepIndex: row.parent_step_index ?? -1, + failedReason: failureReason, + }; + // eslint-disable-next-line no-await-in-loop -- parent and dispatch expire together + await tx` + UPDATE workflow_runs + SET status = 'failed', state = state || ${parentFailure}::jsonb + WHERE id = ${row.parent_run_id} + AND status = 'running' + `; + } + if (row.execution_delivery_id === null) continue; + // eslint-disable-next-line no-await-in-loop -- execution and dispatch expire together + await tx` + UPDATE executions + SET status = 'failed', + completed_at = now(), + error_message = ${ + failureReason === "workflow dispatch retries exhausted" + ? "Workflow dispatch retries exhausted" + : "Workflow dispatch deadline expired" + }, + result_processed_at = now() + WHERE delivery_id = ${row.execution_delivery_id} + AND offer_id IS NULL + AND status = 'queued' + `; + // eslint-disable-next-line no-await-in-loop -- lock release shares the transaction + await clearInFlightByJobId(row.execution_delivery_id, tx); + } + return rows; + }); +} + +async function expireLeasedWorkflowExecutions(sql: SQL): Promise { + if (typeof sql.begin !== "function") return expireWorkflowAttempts(sql); + + return sql.begin(async (tx) => { + const rows = await expireWorkflowAttempts(tx); + for (const row of rows) { + const failureReason = + row.state["failedReason"] === "workflow execution deadline expired" + ? "workflow execution deadline expired" + : "workflow execution lease expired"; + const executionFailureReason = + failureReason === "workflow execution deadline expired" + ? "Workflow execution deadline expired" + : "Workflow execution lease expired"; + if (row.parent_run_id !== null) { + const parentFailure = { + failedAtStepIndex: row.parent_step_index ?? -1, + failedReason: failureReason, + }; + // eslint-disable-next-line no-await-in-loop -- parent and attempt expire together + await tx` + UPDATE workflow_runs + SET status = 'failed', state = state || ${parentFailure}::jsonb + WHERE id = ${row.parent_run_id} + AND status = 'running' + `; + } + if (row.execution_delivery_id === null || row.attempt_id === null) continue; + // eslint-disable-next-line no-await-in-loop -- execution and attempt expire together + await tx` + UPDATE executions + SET status = 'failed', + completed_at = now(), + error_message = ${executionFailureReason}, + result_processed_at = now() + WHERE delivery_id = ${row.execution_delivery_id} + AND daemon_id = ${row.owner_id} + AND offer_id = ${row.attempt_id} + AND status = 'running' + `; + // eslint-disable-next-line no-await-in-loop -- lock release shares the transaction + await clearInFlightByJobId(row.execution_delivery_id, tx); + } + return rows; + }); } /** @@ -195,20 +422,31 @@ export function startLivenessReaper(): void { if (timer !== null) return; const intervalMs = config.livenessReaperIntervalMs; timer = setInterval(() => { - void reapOnce().catch((err: unknown) => { - logger.error( - { err: err instanceof Error ? err.message : String(err) }, - "Liveness reaper pass threw, will retry on next tick", - ); - }); + if (inFlight !== null) { + logger.warn("Liveness reaper tick skipped because the prior pass is still running"); + return; + } + const pass = reapOnce(); + inFlight = pass; + void pass + .catch((err: unknown) => { + logger.error( + { err: err instanceof Error ? err.message : String(err) }, + "Liveness reaper pass threw, will retry on next tick", + ); + }) + .finally(() => { + if (inFlight === pass) inFlight = null; + }); }, intervalMs); logger.info({ intervalMs }, "Liveness reaper started"); } -/** Stop the periodic reaper. Safe to call before start (no-op). */ -export function stopLivenessReaper(): void { - if (timer === null) return; - clearInterval(timer); +/** Stop future ticks and wait for the active pass before downstream clients close. */ +export async function stopLivenessReaper(): Promise { + if (timer !== null) clearInterval(timer); timer = null; + const active = inFlight; + if (active !== null) await active.catch(() => undefined); logger.info("Liveness reaper stopped"); } diff --git a/src/orchestrator/log-fields.ts b/src/orchestrator/log-fields.ts index b70e2501..d24383c2 100644 --- a/src/orchestrator/log-fields.ts +++ b/src/orchestrator/log-fields.ts @@ -51,10 +51,12 @@ export const GITHUB_APP_TOKEN_LOG_EVENTS = { export const TOKEN_MINT_VIA = [ "handleAccept", "handleScopedAccept", - "postOrphanNotification", "shipTickleResume", "proposalPoller", "schedulerRunAction", + "notifyExpiredWorkflowAttempts", + "workflowRunnerPayload", + "workflowRunnerResult", ] as const; export type TokenMintVia = (typeof TOKEN_MINT_VIA)[number]; diff --git a/src/orchestrator/queue-worker.ts b/src/orchestrator/queue-worker.ts index 3f8c7575..de209707 100644 --- a/src/orchestrator/queue-worker.ts +++ b/src/orchestrator/queue-worker.ts @@ -2,26 +2,38 @@ import { config } from "../config"; import { logger } from "../logger"; import { getInstanceId } from "./instance-id"; import { dispatchJob, markJobTerminallyFailed } from "./job-dispatcher"; -import { leaseJob, releaseLeasedJob, requeueLeasedJob } from "./job-queue"; +import { + deferLeasedWorkflowJob, + ensureWorkflowJobQueued, + leaseJob, + type QueuedJob, + releaseLeasedJob, + requeueLeasedJob, +} from "./job-queue"; +import { dispatchWorkflowRunner } from "./workflow-runner-dispatch"; const EMPTY_POLL_MS = 200; const INITIAL_BACKOFF_MS = 100; +const WORKFLOW_CAPACITY_BACKOFF_MS = 1_000; let running = false; let loopPromise: Promise | null = null; let stopRequested = false; +let loopAbortController: AbortController | null = null; -function sleep(ms: number, abortSignal: { aborted: boolean }): Promise { +function sleep(ms: number, abortSignal: AbortSignal): Promise { return new Promise((resolve) => { if (abortSignal.aborted) { resolve(); return; } - const timer = setTimeout(resolve, ms); - // Best-effort abort, caller sets aborted=true and we poll it on wake. - // The setTimeout still fires; the outer loop checks `stopRequested` - // immediately after resolve and exits. - void timer; + const done = (): void => { + clearTimeout(timer); + abortSignal.removeEventListener("abort", done); + resolve(); + }; + const timer = setTimeout(done, ms); + abortSignal.addEventListener("abort", done, { once: true }); }); } @@ -36,12 +48,62 @@ function backoffFor(retryCount: number): number { return Math.min(doubled, config.queueWorkerBackoffMaxMs); } -async function iterate(instanceId: string, abortSignal: { aborted: boolean }): Promise { +async function deferWorkflow( + instanceId: string, + raw: string, + job: Extract, + abortSignal: AbortSignal, + reason: "capacity" | "dispatch-error", +): Promise { + const deferralId = crypto.randomUUID(); + let retryDelayMs = INITIAL_BACKOFF_MS; + for (;;) { + if (abortSignal.aborted) return; + try { + const result = await deferLeasedWorkflowJob(instanceId, raw, job, deferralId); + if (result.status === "missing") { + await ensureWorkflowJobQueued(job, instanceId); + logger.warn( + { deliveryId: job.deliveryId, runId: job.workflowRun.runId, instanceId }, + "Workflow lease was missing; published a recoverable duplicate", + ); + } + logger.debug( + { + deliveryId: job.deliveryId, + runId: job.workflowRun.runId, + reason, + deferralStatus: result.status, + backoffMs: WORKFLOW_CAPACITY_BACKOFF_MS, + }, + "Deferred isolated workflow runner dispatch", + ); + await sleep(WORKFLOW_CAPACITY_BACKOFF_MS, abortSignal); + return; + } catch (err) { + logger.error( + { + err: err instanceof Error ? err.message : String(err), + deliveryId: job.deliveryId, + runId: job.workflowRun.runId, + instanceId, + retryDelayMs, + }, + "Workflow deferral failed; retaining the processing lease", + ); + await sleep(retryDelayMs, abortSignal); + retryDelayMs = Math.min(retryDelayMs * 2, config.queueWorkerBackoffMaxMs); + } + } +} + +async function iterate(instanceId: string, abortSignal: AbortSignal): Promise { const leased = await leaseJob(instanceId); if (leased === null) { await sleep(EMPTY_POLL_MS, abortSignal); return; } + if (abortSignal.aborted) return; const { job, raw } = leased; @@ -57,6 +119,38 @@ async function iterate(instanceId: string, abortSignal: { aborted: boolean }): P ); let dispatched = false; + if (job.kind === "workflow-run") { + try { + const outcome = await dispatchWorkflowRunner(job); + if (outcome === "capacity") { + await deferWorkflow(instanceId, raw, job, abortSignal, "capacity"); + return; + } + await releaseLeasedJob(instanceId, raw); + logger.debug( + { + deliveryId: job.deliveryId, + runId: job.workflowRun.runId, + outcome, + instanceId, + }, + "Queue worker transferred workflow recovery authority to PostgreSQL", + ); + return; + } catch (err) { + logger.error( + { + err: err instanceof Error ? err.message : String(err), + deliveryId: job.deliveryId, + runId: job.workflowRun.runId, + }, + "Isolated workflow runner dispatch failed before authority transfer", + ); + await deferWorkflow(instanceId, raw, job, abortSignal, "dispatch-error"); + return; + } + } + try { dispatched = await dispatchJob(job); } catch (err) { @@ -109,7 +203,9 @@ export function startQueueWorker(): void { running = true; stopRequested = false; const instanceId = getInstanceId(); - const abortSignal = { aborted: false }; + const abortController = new AbortController(); + const abortSignal = abortController.signal; + loopAbortController = abortController; logger.info({ instanceId }, "Queue worker started"); @@ -128,7 +224,7 @@ export function startQueueWorker(): void { await sleep(EMPTY_POLL_MS * 5, abortSignal); } } - abortSignal.aborted = true; + if (loopAbortController === abortController) loopAbortController = null; logger.info({ instanceId }, "Queue worker stopped"); })(); } @@ -145,6 +241,7 @@ export function startQueueWorker(): void { export async function stopQueueWorker(): Promise { if (!running) return; stopRequested = true; + loopAbortController?.abort(); const pending = loopPromise; loopPromise = null; running = false; diff --git a/src/orchestrator/repo-knowledge-persistence.ts b/src/orchestrator/repo-knowledge-persistence.ts new file mode 100644 index 00000000..82dc3176 --- /dev/null +++ b/src/orchestrator/repo-knowledge-persistence.ts @@ -0,0 +1,126 @@ +import type { SQL } from "bun"; + +import { config } from "../config"; +import { requireDb } from "../db"; +import { logger } from "../logger"; + +interface RepoKnowledgeActions { + readonly learnings: readonly { + readonly category: string; + readonly content: string; + }[]; + readonly deletions: readonly string[]; + readonly reviewLearningSaves?: + | readonly { + readonly directive: string; + readonly rationale?: string | undefined; + readonly fileGlob?: string | undefined; + readonly scope?: "local" | "global" | undefined; + readonly sourcePr?: number | undefined; + readonly sourceThread?: string | undefined; + readonly sourceAuthor?: string | undefined; + }[] + | undefined; + readonly reviewLearningDeletes?: readonly string[] | undefined; +} + +export async function persistRepoKnowledge( + input: { + readonly deliveryId: string; + readonly daemonActions?: RepoKnowledgeActions; + readonly appliedReviewLearningIds?: readonly string[]; + }, + db: SQL = requireDb(), +): Promise { + const rows: { repo_owner: string; repo_name: string }[] = await db` + SELECT repo_owner, repo_name FROM executions WHERE delivery_id = ${input.deliveryId} + `; + const execution = rows[0]; + if (execution === undefined) { + throw new Error(`Execution row missing for repo knowledge: ${input.deliveryId}`); + } + + const actions = input.daemonActions; + if (actions !== undefined && actions.learnings.length > 0) { + const { saveRepoLearnings } = await import("./repo-knowledge"); + const saved = await saveRepoLearnings( + execution.repo_owner, + execution.repo_name, + actions.learnings, + db, + ); + if (saved > 0) logger.info({ deliveryId: input.deliveryId, saved }, "Persisted repo learnings"); + } + if (actions !== undefined && actions.deletions.length > 0) { + const { deleteRepoMemories } = await import("./repo-knowledge"); + const deleted = await deleteRepoMemories( + execution.repo_owner, + execution.repo_name, + actions.deletions, + db, + ); + if (deleted > 0) + logger.info({ deliveryId: input.deliveryId, deleted }, "Deleted repo memories"); + } + + if (actions?.reviewLearningSaves !== undefined && actions.reviewLearningSaves.length > 0) { + if (config.reviewLearningsEnabled) { + const { saveReviewLearnings } = await import("./review-learnings"); + const saved = await saveReviewLearnings( + execution.repo_owner, + execution.repo_name, + actions.reviewLearningSaves, + db, + ); + if (saved > 0) { + logger.info({ deliveryId: input.deliveryId, saved }, "Persisted review learnings"); + } + } else { + logger.warn( + { deliveryId: input.deliveryId, dropped: actions.reviewLearningSaves.length }, + "Dropped review-learning saves: REVIEW_LEARNINGS_ENABLED=false", + ); + } + } + + if (actions?.reviewLearningDeletes !== undefined && actions.reviewLearningDeletes.length > 0) { + if (config.reviewLearningsEnabled) { + const { deleteReviewLearnings } = await import("./review-learnings"); + const deleted = await deleteReviewLearnings( + execution.repo_owner, + execution.repo_name, + actions.reviewLearningDeletes, + db, + ); + if (deleted > 0) { + logger.info({ deliveryId: input.deliveryId, deleted }, "Deleted review learnings"); + } + } else { + logger.warn( + { deliveryId: input.deliveryId, dropped: actions.reviewLearningDeletes.length }, + "Dropped review-learning deletes: REVIEW_LEARNINGS_ENABLED=false", + ); + } + } + + if ( + config.reviewLearningsEnabled && + input.appliedReviewLearningIds !== undefined && + input.appliedReviewLearningIds.length > 0 + ) { + try { + // Usage is approximate and must not make durable result settlement fail. + const { bumpReviewLearningUsage } = await import("./review-learnings"); + await bumpReviewLearningUsage(input.appliedReviewLearningIds, db); + logger.info( + { deliveryId: input.deliveryId, applied: input.appliedReviewLearningIds.length }, + "Bumped use_count for applied review learnings", + ); + } catch (err) { + logger.warn( + { err, deliveryId: input.deliveryId }, + "Failed to bump approximate review-learning usage", + ); + } + } +} diff --git a/src/orchestrator/repo-knowledge.ts b/src/orchestrator/repo-knowledge.ts index 4564deff..d1243c9d 100644 --- a/src/orchestrator/repo-knowledge.ts +++ b/src/orchestrator/repo-knowledge.ts @@ -1,7 +1,6 @@ import type { SQL } from "bun"; import { requireDb } from "../db"; -import { logger } from "../logger"; import { sanitizeRepoMemoryContent } from "../utils/sanitize"; // Types @@ -107,7 +106,7 @@ export async function getRepoMemory( export async function saveRepoLearnings( owner: string, repo: string, - learnings: { category: string; content: string }[], + learnings: readonly { category: string; content: string }[], db: SQL = requireDb(), ): Promise { if (learnings.length === 0) return 0; @@ -123,28 +122,27 @@ export async function saveRepoLearnings( // collapsed to nothing after sanitization. const safeContent = sanitizeRepoMemoryContent(learning.content); if (safeContent === "") continue; - try { - // Upsert: insert if new, bump updated_at if duplicate - // eslint-disable-next-line no-await-in-loop - const result: { id: string }[] = await db` - INSERT INTO repo_memory (repo_owner, repo_name, category, content, pinned) - VALUES (${owner}, ${repo}, ${learning.category}, ${safeContent}, false) - ON CONFLICT DO NOTHING - RETURNING id - `; - if (result.length > 0) { - saved++; - } else { - // Duplicate, bump updated_at to keep it relevant in LRU - // eslint-disable-next-line no-await-in-loop - await db` - UPDATE repo_memory SET updated_at = now() - WHERE repo_owner = ${owner} AND repo_name = ${repo} - AND category = ${learning.category} AND content = ${safeContent} - `; - } - } catch (err) { - logger.warn({ err, owner, repo, category: learning.category }, "Failed to save learning"); + // eslint-disable-next-line no-await-in-loop -- bounded action list preserves result order + const result: { id: string }[] = await db` + INSERT INTO repo_memory (repo_owner, repo_name, category, content, pinned) + VALUES (${owner}, ${repo}, ${learning.category}, ${safeContent}, false) + ON CONFLICT ( + repo_owner, + repo_name, + category, + content_sha256 + ) WHERE category <> 'env_var' DO NOTHING + RETURNING id + `; + if (result.length > 0) { + saved++; + } else { + // eslint-disable-next-line no-await-in-loop -- bounded action list preserves result order + await db` + UPDATE repo_memory SET updated_at = now() + WHERE repo_owner = ${owner} AND repo_name = ${repo} + AND category = ${learning.category} AND content = ${safeContent} + `; } } @@ -155,10 +153,19 @@ export async function saveRepoLearnings( * Delete repo memory entries by ID. * Used when Claude identifies outdated or incorrect memories. */ -export async function deleteRepoMemories(ids: string[], db: SQL = requireDb()): Promise { +export async function deleteRepoMemories( + owner: string, + repo: string, + ids: readonly string[], + db: SQL = requireDb(), +): Promise { if (ids.length === 0) return 0; const deleted: { id: string }[] = await db` - DELETE FROM repo_memory WHERE id IN ${db(ids)} RETURNING id + DELETE FROM repo_memory + WHERE id IN ${db(ids)} + AND repo_owner = ${owner} + AND repo_name = ${repo} + RETURNING id `; return deleted.length; } diff --git a/src/orchestrator/review-learnings.ts b/src/orchestrator/review-learnings.ts index bc9d35ce..558e2114 100644 --- a/src/orchestrator/review-learnings.ts +++ b/src/orchestrator/review-learnings.ts @@ -326,12 +326,6 @@ export async function bumpReviewLearningUsage( } } -/** One row from the review-learnings upsert: the row id and whether it was new. */ -interface ReviewLearningInsertResult { - id: string; - inserted: boolean; -} - /** * Persist review learnings discovered during a review/resolve run. * @@ -392,7 +386,7 @@ export async function saveReviewLearnings( // (the most recent embedding reflects the most recent text), but only // when the incoming save brought one. // eslint-disable-next-line no-await-in-loop -- bounded action list preserves result order - const result: ReviewLearningInsertResult[] = await db` + const result: { id: string; inserted: boolean }[] = await db` INSERT INTO review_learnings (repo_owner, repo_name, scope, file_glob, directive, rationale, source_pr, source_thread, source_author, embedding) diff --git a/src/orchestrator/workflow-expiry-notifier.ts b/src/orchestrator/workflow-expiry-notifier.ts new file mode 100644 index 00000000..17216ad4 --- /dev/null +++ b/src/orchestrator/workflow-expiry-notifier.ts @@ -0,0 +1,280 @@ +import { App, Octokit } from "octokit"; + +import { config } from "../config"; +import { logger } from "../logger"; +import { observableOctokit } from "../utils/octokit-observability"; +import { addReaction } from "../utils/reactions"; +import { + findPendingWorkflowFailureNotifications, + markWorkflowFailureNotified, + type WorkflowRunRow, +} from "../workflows/runs-store"; +import { mintInstallationToken, revokeInstallationToken } from "./installation-token"; + +let cachedApp: InstanceType | null = null; + +function getApp(): InstanceType { + if (cachedApp !== null) return cachedApp; + if (config.appId === undefined || config.privateKey === undefined) { + throw new Error("GitHub App credentials are not configured"); + } + cachedApp = new App({ + appId: config.appId, + privateKey: config.privateKey, + Octokit: observableOctokit(), + }); + return cachedApp; +} + +function canNotify(): boolean { + if (config.nodeEnv === "test") return false; + return ( + config.githubPersonalAccessToken !== undefined || + (config.appId !== undefined && config.privateKey !== undefined) + ); +} + +interface NotificationOctokit { + readonly octokit: Octokit; + readonly ownsInstallationToken: boolean; +} + +async function getOctokit(row: WorkflowRunRow): Promise { + if (config.githubPersonalAccessToken !== undefined) { + return { + octokit: new Octokit({ auth: config.githubPersonalAccessToken }), + ownsInstallationToken: false, + }; + } + const app = getApp(); + const { data: installation } = await app.octokit.rest.apps.getRepoInstallation({ + owner: row.target_owner, + repo: row.target_repo, + }); + const minted = await mintInstallationToken({ + app, + installationId: installation.id, + repositoryName: row.target_repo, + via: "notifyExpiredWorkflowAttempts", + log: logger, + }); + return { + octokit: minted.octokit as unknown as Octokit, + ownsInstallationToken: true, + }; +} + +async function findTopAncestor(row: WorkflowRunRow): Promise { + const { findById } = await import("../workflows/runs-store"); + let current: WorkflowRunRow | null = row; + const visited = new Set(); + while (current !== null) { + if (visited.has(current.id)) { + throw new Error(`Workflow parent cycle detected at ${current.id}`); + } + visited.add(current.id); + if (current.parent_run_id === null) return current; + // eslint-disable-next-line no-await-in-loop + current = await findById(current.parent_run_id); + } + return null; +} + +interface WorkflowFailureNotice { + readonly phase: string | ((row: WorkflowRunRow) => string); + readonly humanMessage: (row: WorkflowRunRow) => string; +} + +const disconnectedDaemonNotice: WorkflowFailureNotice = { + phase: "orphaned", + humanMessage: () => + [ + "❌ **Daemon disconnected during execution**", + "", + "The database marked the in-flight workflow failed and released its target lock.", + "", + "External GitHub or git operations may have completed before the disconnect. Inspect the repository before re-triggering the workflow.", + ].join("\n"), +}; + +const migrationInterruptedNotice: WorkflowFailureNotice = { + phase: "migration-interrupted", + humanMessage: (row) => { + const dispatchIncomplete = + row.state["failedReason"] === "workflow dispatch incomplete during lease migration"; + return [ + dispatchIncomplete + ? "❌ **Workflow dispatch interrupted during migration**" + : "❌ **Workflow execution interrupted during migration**", + "", + dispatchIncomplete + ? "The deployment migration found a queued workflow without its matching execution receipt. The database marked the workflow failed and released its target lock." + : "The deployment could not safely transfer this active shared-daemon workflow to an isolated runner. The database marked the workflow failed and released its target lock.", + "", + "Inspect the repository for partial external operations, then re-trigger the workflow.", + ].join("\n"); + }, +}; + +export async function notifyWorkflowAttemptFailures( + rows: readonly WorkflowRunRow[], + notice: WorkflowFailureNotice, +): Promise { + if (rows.length === 0 || !canNotify()) return; + const notifiedAncestors = new Set(); + + for (const row of rows) { + try { + // eslint-disable-next-line no-await-in-loop + const ancestor = await findTopAncestor(row); + if (ancestor === null) continue; + if (notifiedAncestors.has(ancestor.id)) { + // eslint-disable-next-line no-await-in-loop -- each row needs its own durable receipt + await markWorkflowFailureNotified({ runId: row.id, attemptId: row.attempt_id }); + continue; + } + // eslint-disable-next-line no-await-in-loop + const auth = await getOctokit(ancestor); + try { + // eslint-disable-next-line no-await-in-loop -- loaded only for rows that need projection + const { setState } = await import("../workflows/tracking-mirror"); + // eslint-disable-next-line no-await-in-loop + await setState( + { octokit: auth.octokit, logger }, + { + runId: ancestor.id, + patch: { + phase: typeof notice.phase === "string" ? notice.phase : notice.phase(row), + }, + humanMessage: notice.humanMessage(row), + }, + ); + + if (ancestor.trigger_comment_id !== null && ancestor.trigger_event_type !== null) { + // eslint-disable-next-line no-await-in-loop + await addReaction({ + octokit: auth.octokit, + logger, + owner: ancestor.target_owner, + repo: ancestor.target_repo, + commentId: ancestor.trigger_comment_id, + eventType: ancestor.trigger_event_type, + content: "confused", + }); + } + // eslint-disable-next-line no-await-in-loop -- each row needs its own durable receipt + await markWorkflowFailureNotified({ runId: row.id, attemptId: row.attempt_id }); + notifiedAncestors.add(ancestor.id); + } finally { + if (auth.ownsInstallationToken) { + // eslint-disable-next-line no-await-in-loop -- release each owned token before the next row + await revokeInstallationToken(auth.octokit, logger, { + runId: row.id, + owner: "failure-notification", + }); + } + } + } catch (err) { + logger.warn( + { err: err instanceof Error ? err : new Error(String(err)), runId: row.id }, + "Workflow failure notification failed", + ); + } + } +} + +export async function notifyExpiredWorkflowAttempts( + rows: readonly WorkflowRunRow[], +): Promise { + return notifyWorkflowAttemptFailures(rows, { + phase: (row) => + row.state["failedReason"] === "workflow execution deadline expired" + ? "deadline-expired" + : "lease-expired", + humanMessage: (row) => { + const deadlineExpired = row.state["failedReason"] === "workflow execution deadline expired"; + return [ + deadlineExpired + ? "❌ **Workflow execution deadline expired**" + : "❌ **Workflow execution lease expired**", + "", + deadlineExpired + ? "The immutable attempt deadline elapsed before completion was confirmed. The database marked the workflow failed and released its in-flight lock." + : "The runner stopped renewing this attempt before completion was confirmed. The database marked the workflow failed and released its in-flight lock.", + "", + `External GitHub or git operations may have completed before the ${deadlineExpired ? "deadline" : "lease"} expired. Inspect the repository before re-triggering the workflow.`, + ].join("\n"); + }, + }); +} + +export async function notifyExpiredWorkflowDispatches( + rows: readonly WorkflowRunRow[], +): Promise { + return notifyWorkflowAttemptFailures(rows, { + phase: "dispatch-expired", + humanMessage: (row) => { + const retriesExhausted = row.state["failedReason"] === "workflow dispatch retries exhausted"; + return [ + retriesExhausted + ? "❌ **Workflow dispatch retries exhausted**" + : "❌ **Workflow dispatch deadline expired**", + "", + retriesExhausted + ? "The controller could not durably publish this queued workflow within its retry budget. The database marked it failed and released its in-flight lock." + : "No isolated runner claimed this queued workflow before its dispatch deadline. The database marked it failed and released its in-flight lock.", + "", + "Fix the queue or runner capacity issue, then re-trigger the workflow.", + ].join("\n"); + }, + }); +} + +export async function notifyRunnerStartFailures(rows: readonly WorkflowRunRow[]): Promise { + return notifyWorkflowAttemptFailures(rows, { + phase: "runner-start-failed", + humanMessage: (row) => { + const reason = row.state["failedReason"]; + const detail = typeof reason === "string" ? reason : "Workflow runner configuration failed"; + return [ + "❌ **Workflow runner could not start**", + "", + `${detail}. The database marked the workflow failed and released its in-flight lock.`, + "", + "Fix the runner deployment configuration, then re-trigger the workflow.", + ].join("\n"); + }, + }); +} + +export async function notifyDisconnectedDaemonWorkflows(runIds: readonly string[]): Promise { + const { findById } = await import("../workflows/runs-store"); + const rows = (await Promise.all(runIds.map((runId) => findById(runId)))).filter( + (row): row is WorkflowRunRow => row !== null, + ); + return notifyWorkflowAttemptFailures(rows, disconnectedDaemonNotice); +} + +export async function reconcilePendingWorkflowFailureNotifications(limit = 100): Promise { + const pending = await findPendingWorkflowFailureNotifications(undefined, limit); + const expired = pending + .filter((entry) => entry.phase === "deadline-expired" || entry.phase === "lease-expired") + .map((entry) => entry.row); + const dispatchExpired = pending + .filter((entry) => entry.phase === "dispatch-expired") + .map((entry) => entry.row); + const startFailed = pending + .filter((entry) => entry.phase === "runner-start-failed") + .map((entry) => entry.row); + const disconnected = pending + .filter((entry) => entry.phase === "orphaned") + .map((entry) => entry.row); + const migrationInterrupted = pending + .filter((entry) => entry.phase === "migration-interrupted") + .map((entry) => entry.row); + await notifyExpiredWorkflowAttempts(expired); + await notifyExpiredWorkflowDispatches(dispatchExpired); + await notifyRunnerStartFailures(startFailed); + await notifyWorkflowAttemptFailures(disconnected, disconnectedDaemonNotice); + await notifyWorkflowAttemptFailures(migrationInterrupted, migrationInterruptedNotice); +} diff --git a/src/orchestrator/workflow-runner-capability.ts b/src/orchestrator/workflow-runner-capability.ts new file mode 100644 index 00000000..42923642 --- /dev/null +++ b/src/orchestrator/workflow-runner-capability.ts @@ -0,0 +1,92 @@ +import { createHmac, timingSafeEqual } from "node:crypto"; + +const CAPABILITY_PREFIX = "wfr1."; +const CAPABILITY_PATTERN = /^wfr1\.([1-9][0-9]{12})\.([A-Za-z0-9_-]{43})$/; +const RUNNER_PATH_PREFIX = "/ws/workflow-runner/"; + +function signature(secret: string, runId: string, attemptId: string, expiresAtMs: number): string { + return createHmac("sha256", secret) + .update(`workflow-runner-v1\0${runId}\0${attemptId}\0${String(expiresAtMs)}`, "utf8") + .digest("base64url"); +} + +function expiryMillis(expiresAt: Date | number): number { + const value = expiresAt instanceof Date ? expiresAt.getTime() : expiresAt; + if (!Number.isSafeInteger(value) || value <= 0) { + throw new RangeError("workflow runner capability expiry must be a positive safe integer"); + } + return value; +} + +export function deriveWorkflowRunnerCapability( + secret: string, + runId: string, + attemptId: string, + expiresAt: Date | number, +): string { + const expiresAtMs = expiryMillis(expiresAt); + return `${CAPABILITY_PREFIX}${String(expiresAtMs)}.${signature(secret, runId, attemptId, expiresAtMs)}`; +} + +function constantTimeEqual(left: string, right: string): boolean { + const rightBytes = Buffer.from(right, "utf8"); + const leftLength = Buffer.byteLength(left, "utf8"); + const leftBytes = Buffer.alloc(rightBytes.length); + leftBytes.write(left, 0, rightBytes.length, "utf8"); + const equal = timingSafeEqual(leftBytes, rightBytes); + return equal && leftLength === rightBytes.length; +} + +/** Validate the exact run/attempt capability against both rotation slots. */ +export function isWorkflowRunnerCapabilityValid( + authorization: string | null | undefined, + runId: string, + attemptId: string, + primarySecret: string, + previousSecret?: string, + nowMs = Date.now(), +): boolean { + const actual = authorization?.startsWith("Bearer ") === true ? authorization.slice(7) : ""; + const parsed = CAPABILITY_PATTERN.exec(actual); + if (parsed === null) return false; + const expiresAtMs = Number(parsed[1]); + if (!Number.isSafeInteger(expiresAtMs) || expiresAtMs <= nowMs) return false; + const primary = deriveWorkflowRunnerCapability(primarySecret, runId, attemptId, expiresAtMs); + const previous = deriveWorkflowRunnerCapability( + previousSecret ?? primarySecret, + runId, + attemptId, + expiresAtMs, + ); + const primaryMatch = constantTimeEqual(actual, primary); + const previousMatch = constantTimeEqual(actual, previous); + return primaryMatch || (previousSecret !== undefined && previousMatch); +} + +export function workflowRunnerPath(runId: string, attemptId: string): string { + return `${RUNNER_PATH_PREFIX}${encodeURIComponent(runId)}/${encodeURIComponent(attemptId)}`; +} + +export function parseWorkflowRunnerPath( + pathname: string, +): { runId: string; attemptId: string } | null { + if (!pathname.startsWith(RUNNER_PATH_PREFIX)) return null; + const parts = pathname.slice(RUNNER_PATH_PREFIX.length).split("/"); + if (parts.length !== 2 || parts[0] === "" || parts[1] === "") return null; + try { + return { + runId: decodeURIComponent(parts[0] ?? ""), + attemptId: decodeURIComponent(parts[1] ?? ""), + }; + } catch { + return null; + } +} + +export function workflowRunnerUrl(baseUrl: string, runId: string, attemptId: string): string { + const url = new URL(baseUrl); + url.pathname = workflowRunnerPath(runId, attemptId); + url.search = ""; + url.hash = ""; + return url.toString(); +} diff --git a/src/orchestrator/workflow-runner-controller.ts b/src/orchestrator/workflow-runner-controller.ts new file mode 100644 index 00000000..07897350 --- /dev/null +++ b/src/orchestrator/workflow-runner-controller.ts @@ -0,0 +1,621 @@ +import type { ServerWebSocket } from "bun"; +import { Octokit } from "octokit"; + +import { config } from "../config"; +import { requireDb } from "../db"; +import { logger } from "../logger"; +import { + WORKFLOW_RUNNER_PROTOCOL_VERSION, + type WorkflowRunnerClientMessage, + type WorkflowRunnerCommand, + type WorkflowRunnerPayload, + type WorkflowRunnerResultPayload, +} from "../shared/workflow-runner-messages"; +import { createMessageEnvelope } from "../shared/ws-messages"; +import { redactErrorMessageOrFallback } from "../utils/log-redaction"; +import { publishWorkflowRunById } from "../workflows/dispatch-outbox"; +import { recordWorkflowExecution } from "../workflows/execution-row"; +import { getByName } from "../workflows/registry"; +import { + assertCurrentWorkflowAttempt, + commitAttemptHandOffChild, + findById, + renewWorkflowAttempts, + StaleWorkflowAttemptError, + type WorkflowAttempt, +} from "../workflows/runs-store"; +import { setState } from "../workflows/tracking-mirror"; +import { revokeInstallationToken, revokeInstallationTokenValue } from "./installation-token"; +import { + sanitizeWorkflowRunnerCommand, + sanitizeWorkflowRunnerResult, + WorkflowRunnerOutputRejectedError, +} from "./workflow-runner-output"; +import { + prepareWorkflowRunnerControllerOctokit, + prepareWorkflowRunnerPayload, +} from "./workflow-runner-payload"; +import { + cleanupWorkflowRunnerAttempt, + processWorkflowRunnerResult, +} from "./workflow-runner-result"; +import { + assertMatchingWorkflowRunnerCommand, + findWorkflowRunnerCommandReceipt, + getWorkflowRunnerRegistrationState, + insertWorkflowRunnerCommandReceipt, + recordWorkflowRunnerPayloadIssued, + storeWorkflowRunnerResult, + type WorkflowRunnerAttempt, + WorkflowRunnerCommandConflictError, + type WorkflowRunnerCommandReceipt, +} from "./workflow-runner-store"; +import type { WsConnectionData } from "./ws-connection"; + +const RUNNER_CLIENT_FENCE_MS = Math.max(config.heartbeatTimeoutMs, 3 * config.heartbeatIntervalMs); +const RUNNER_DB_LEASE_MS = 2 * RUNNER_CLIENT_FENCE_MS; +const RUNNER_REGISTRATION_LEASE_MS = Math.max(300_000, RUNNER_DB_LEASE_MS); + +interface RunnerSession { + readonly ws: ServerWebSocket; + readonly attempt: WorkflowRunnerAttempt; + readonly octokit: Octokit; + readonly revokeOctokitOnRelease: boolean; +} + +const sessions = new Map(); +const attemptChains = new Map>(); + +async function revokeUndeliveredRunnerToken(job: WorkflowRunnerPayload): Promise { + await revokeInstallationTokenValue(job.installationToken, logger, { + owner: "undelivered-runner-payload", + }); +} + +async function revokeRunnerSessionToken(session: RunnerSession): Promise { + if (!session.revokeOctokitOnRelease) return; + await revokeInstallationToken(session.octokit, logger, { + attemptId: session.attempt.attemptId, + owner: "controller-session", + }); +} + +async function releaseRunnerSessionToken(session: RunnerSession): Promise { + await (attemptChains.get(session.attempt.attemptId) ?? Promise.resolve()).catch(() => undefined); + await revokeRunnerSessionToken(session); +} + +function send(ws: ServerWebSocket, message: unknown): boolean { + const sent = ws.sendText(JSON.stringify(message)); + if (sent !== 0) return true; + ws.close(1011, "workflow runner control frame delivery failed"); + return false; +} + +function identityMatches( + ws: ServerWebSocket, + payload: { readonly runId: string; readonly attemptId: string }, +): boolean { + return ws.data.runnerRunId === payload.runId && ws.data.runnerAttemptId === payload.attemptId; +} + +function closePolicy(ws: ServerWebSocket, reason: string): void { + ws.close(1008, reason); +} + +export function handleWorkflowRunnerOpen(ws: ServerWebSocket): void { + ws.data.runnerRegistered = false; +} + +export function handleWorkflowRunnerClose(ws: ServerWebSocket): void { + const attemptId = ws.data.runnerAttemptId; + if (attemptId === undefined) return; + const session = sessions.get(attemptId); + if (session?.ws !== ws) return; + sessions.delete(attemptId); + void releaseRunnerSessionToken(session); +} + +export function handleWorkflowRunnerMessage( + ws: ServerWebSocket, + message: WorkflowRunnerClientMessage, +): void { + if (!identityMatches(ws, message.payload)) { + closePolicy(ws, "workflow runner identity mismatch"); + return; + } + if (message.type !== "workflow-runner:register" && ws.data.runnerRegistered !== true) { + closePolicy(ws, "workflow runner is not registered"); + return; + } + + switch (message.type) { + case "workflow-runner:register": + void registerRunner(ws, message).catch((err: unknown) => { + logger.error({ err, attemptId: message.payload.attemptId }, "Runner registration failed"); + ws.close(1011, "workflow runner registration failed"); + }); + break; + case "workflow-runner:heartbeat": + void handleHeartbeat(ws, message).catch((err: unknown) => { + logger.error({ err, attemptId: message.payload.attemptId }, "Runner heartbeat failed"); + ws.close(1011, "workflow runner heartbeat failed"); + }); + break; + case "workflow-runner:command": + queueCommand(ws, message); + break; + case "workflow-runner:result": + queueResult(ws, message); + break; + } +} + +async function registerRunner( + ws: ServerWebSocket, + message: Extract, +): Promise { + if (ws.data.runnerRegistered === true) { + closePolicy(ws, "duplicate workflow runner registration"); + return; + } + if (message.payload.protocolVersion !== WORKFLOW_RUNNER_PROTOCOL_VERSION) { + closePolicy(ws, "incompatible workflow runner protocol"); + return; + } + + const attempt = { runId: message.payload.runId, attemptId: message.payload.attemptId }; + const state = await getWorkflowRunnerRegistrationState(attempt); + if (state.state === "invalid") { + closePolicy(ws, "stale workflow runner attempt"); + return; + } + if (state.state === "completed") { + send(ws, { + type: "workflow-runner:registered", + ...createMessageEnvelope(message.id), + payload: { state: "completed" }, + }); + return; + } + if (state.state === "result-pending") { + await processWorkflowRunnerResult({ + runId: attempt.runId, + attemptId: attempt.attemptId, + executionDeliveryId: state.executionDeliveryId, + payload: state.payload, + }); + send(ws, { + type: "workflow-runner:registered", + ...createMessageEnvelope(message.id), + payload: { state: "completed" }, + }); + return; + } + + const renewal = await renewWorkflowAttempts( + state.attempt.runnerId, + [state.attempt.attemptId], + RUNNER_REGISTRATION_LEASE_MS, + ); + if (!renewal.renewedAttemptIds.includes(state.attempt.attemptId)) { + closePolicy(ws, "workflow runner lease expired"); + return; + } + let job: WorkflowRunnerPayload | undefined; + let octokit: Octokit | undefined; + let revokeOctokitOnRelease = false; + let payloadTransferred = false; + let sessionInstalled = false; + try { + if (message.payload.needsJob) { + if (state.payloadIssuedAt !== null) { + closePolicy(ws, "workflow runner payload was already issued"); + return; + } + job = await prepareWorkflowRunnerPayload(state.attempt); + const recorded = await recordWorkflowRunnerPayloadIssued( + attempt, + new Date(job.installationTokenExpiresAt), + ); + if (!recorded) { + closePolicy(ws, "workflow runner payload delivery was rejected"); + return; + } + octokit = new Octokit({ auth: job.installationToken }); + } else { + if (state.payloadIssuedAt === null || state.tokenExpiresAt === null) { + closePolicy(ws, "workflow runner reconnected before receiving its payload"); + return; + } + octokit = await prepareWorkflowRunnerControllerOctokit(state.attempt); + revokeOctokitOnRelease = true; + } + const renewedAfterPreparation = await renewWorkflowAttempts( + state.attempt.runnerId, + [state.attempt.attemptId], + RUNNER_DB_LEASE_MS, + ); + if (!renewedAfterPreparation.renewedAttemptIds.includes(state.attempt.attemptId)) { + closePolicy(ws, "workflow runner lease expired during payload preparation"); + return; + } + + const old = sessions.get(state.attempt.attemptId); + if (old !== undefined && old.ws !== ws) { + sessions.delete(state.attempt.attemptId); + old.ws.close(4002, "superseded by runner reconnect"); + void releaseRunnerSessionToken(old); + } + const session: RunnerSession = { + ws, + attempt: state.attempt, + octokit, + revokeOctokitOnRelease, + }; + sessions.set(state.attempt.attemptId, session); + sessionInstalled = true; + ws.data.runnerRegistered = true; + if ( + !send(ws, { + type: "workflow-runner:registered", + ...createMessageEnvelope(message.id), + payload: { + state: "ready", + heartbeatIntervalMs: config.heartbeatIntervalMs, + clientFenceMs: RUNNER_CLIENT_FENCE_MS, + dbLeaseMs: RUNNER_DB_LEASE_MS, + ...(job === undefined ? {} : { job }), + }, + }) + ) { + if (sessions.get(state.attempt.attemptId) === session) { + sessions.delete(state.attempt.attemptId); + } + sessionInstalled = false; + ws.data.runnerRegistered = false; + return; + } + payloadTransferred = job !== undefined; + } finally { + if (job !== undefined && !payloadTransferred) await revokeUndeliveredRunnerToken(job); + if (octokit !== undefined && revokeOctokitOnRelease && !sessionInstalled) { + await revokeInstallationToken(octokit, logger, { + attemptId: state.attempt.attemptId, + owner: "controller-registration", + }); + } + } +} + +async function handleHeartbeat( + ws: ServerWebSocket, + message: Extract, +): Promise { + const attemptId = message.payload.attemptId; + const session = sessions.get(attemptId); + if (session?.ws !== ws) { + closePolicy(ws, "workflow runner session superseded"); + return; + } + const renewal = await renewWorkflowAttempts( + session.attempt.runnerId, + [attemptId], + RUNNER_DB_LEASE_MS, + ); + const renewed = renewal.renewedAttemptIds.includes(attemptId); + send(ws, { + type: "workflow-runner:heartbeat-ack", + ...createMessageEnvelope(message.id), + payload: { renewed }, + }); + if (!renewed) closePolicy(ws, "workflow runner attempt fenced"); +} + +function queueCommand( + ws: ServerWebSocket, + message: Extract, +): void { + const attemptId = message.payload.attemptId; + const prior = attemptChains.get(attemptId) ?? Promise.resolve(); + const current = prior + .catch(() => undefined) + .then(() => handleCommand(ws, message)) + .catch((err: unknown) => { + const stale = err instanceof StaleWorkflowAttemptError; + const conflict = err instanceof WorkflowRunnerCommandConflictError; + const rejected = err instanceof WorkflowRunnerOutputRejectedError; + const code = stale + ? "STALE_ATTEMPT" + : conflict || rejected + ? "INVALID_COMMAND" + : "INTERNAL_ERROR"; + logger.error({ err, attemptId, commandId: message.id }, "Workflow runner command failed"); + send(ws, { + type: "workflow-runner:command-result", + ...createMessageEnvelope(message.id), + payload: { + ok: false, + code, + message: stale + ? "workflow attempt is no longer current" + : conflict || rejected + ? "workflow command was rejected" + : "workflow command failed", + }, + }); + if (stale) closePolicy(ws, "workflow runner attempt fenced"); + }); + attemptChains.set(attemptId, current); + void current.finally(() => { + if (attemptChains.get(attemptId) === current) attemptChains.delete(attemptId); + }); +} + +function queueResult( + ws: ServerWebSocket, + message: Extract, +): void { + const attemptId = message.payload.attemptId; + const prior = attemptChains.get(attemptId) ?? Promise.resolve(); + const current = prior + .catch(() => undefined) + .then(() => handleResult(ws, message)) + .catch((err: unknown) => { + logger.error({ err, attemptId }, "Runner result failed"); + if ( + err instanceof StaleWorkflowAttemptError || + err instanceof WorkflowRunnerCommandConflictError + ) { + closePolicy(ws, "workflow runner result rejected"); + } + }); + attemptChains.set(attemptId, current); + void current.finally(() => { + if (attemptChains.get(attemptId) === current) attemptChains.delete(attemptId); + }); +} + +async function handleCommand( + ws: ServerWebSocket, + message: Extract, +): Promise { + const session = sessions.get(message.payload.attemptId); + if (session?.ws !== ws) throw new StaleWorkflowAttemptError(message.payload); + const attempt = { runId: message.payload.runId, attemptId: message.payload.attemptId }; + const command = await sanitizeWorkflowRunnerCommand(message.payload.command); + const existing = await findWorkflowRunnerCommandReceipt(attempt, message.id); + if (existing !== null) { + assertMatchingWorkflowRunnerCommand(existing, message.id, command); + if (command.type === "hand-off-child") { + await settleCommittedHandOff(session); + } + sendCommandSuccess(ws, message.id, existing.response); + return; + } + + const response = + command.type === "set-state" + ? await applySetStateCommand(session, message.id, command) + : await applyHandOffCommand(session, message.id, command); + sendCommandSuccess(ws, message.id, response); +} + +function sendCommandSuccess( + ws: ServerWebSocket, + commandId: string, + response: WorkflowRunnerCommandReceipt["response"], +): void { + send(ws, { + type: "workflow-runner:command-result", + ...createMessageEnvelope(commandId), + payload: { ok: true, result: response }, + }); +} + +async function applySetStateCommand( + session: RunnerSession, + commandId: string, + command: Extract, +): Promise { + const attempt = { runId: session.attempt.runId, attemptId: session.attempt.attemptId }; + await assertCurrentWorkflowAttempt(attempt); + const row = await setState( + { octokit: session.octokit, logger }, + { + runId: attempt.runId, + patch: command.patch, + humanMessage: command.humanMessage, + attempt, + }, + ); + const response = + row.tracking_comment_id === null ? {} : { trackingCommentId: row.tracking_comment_id }; + await insertWorkflowRunnerCommandReceipt(attempt, commandId, command, response, requireDb()); + return response; +} + +async function assertValidHandOff( + attempt: WorkflowAttempt, + command: Extract, +): Promise { + const row = await findById(attempt.runId); + const expectedStep = getByName("ship").steps[command.parentStepIndex]; + if ( + row?.workflow_name !== "ship" || + row.attempt_id !== attempt.attemptId || + expectedStep !== command.workflowName || + row.target_type !== command.target.type || + row.target_owner !== command.target.owner || + row.target_repo !== command.target.repo || + row.target_number !== command.target.number + ) { + throw new WorkflowRunnerCommandConflictError(command.parentStepIndex.toString()); + } +} + +async function applyHandOffCommand( + session: RunnerSession, + commandId: string, + command: Extract, +): Promise { + const attempt = { runId: session.attempt.runId, attemptId: session.attempt.attemptId }; + await assertCurrentWorkflowAttempt(attempt); + await assertValidHandOff(attempt, command); + + const response = { childRunId: commandId }; + const resultPayload = { + runId: attempt.runId, + attemptId: attempt.attemptId, + result: { + status: "handed-off" as const, + state: { ...command.state, handedOffTo: commandId }, + humanMessage: command.humanMessage, + childRunId: commandId, + }, + durationMs: 0, + }; + try { + await requireDb().begin(async (tx) => { + const replay = await findWorkflowRunnerCommandReceipt(attempt, commandId, tx); + if (replay !== null) { + assertMatchingWorkflowRunnerCommand(replay, commandId, command); + return; + } + const child = await commitAttemptHandOffChild( + attempt, + command.state, + { + workflowName: command.workflowName, + target: command.target, + parentStepIndex: command.parentStepIndex, + traceDeliveryId: session.attempt.executionDeliveryId, + childRunId: commandId, + }, + tx, + ); + await recordWorkflowExecution({ + deliveryId: child.id, + target: command.target, + senderLogin: config.botAppLogin, + workflowName: command.workflowName, + runId: child.id, + logger, + sql: tx, + }); + const executionRows: { delivery_id: string }[] = await tx` + UPDATE executions + SET status = 'completed', + completed_at = now(), + duration_ms = 0, + workflow_result_payload = ${resultPayload}::jsonb, + result_processed_at = NULL + WHERE delivery_id = ${session.attempt.executionDeliveryId} + AND daemon_id = ${session.attempt.runnerId} + AND offer_id = ${attempt.attemptId} + AND status = 'running' + RETURNING delivery_id + `; + if (executionRows[0] === undefined) throw new StaleWorkflowAttemptError(attempt); + await insertWorkflowRunnerCommandReceipt(attempt, commandId, command, response, tx); + }); + } catch (err) { + const receipt = await findWorkflowRunnerCommandReceipt(attempt, commandId).catch(() => null); + if (receipt === null) throw err; + assertMatchingWorkflowRunnerCommand(receipt, commandId, command); + } + await publishWorkflowRunById(commandId).catch((err: unknown) => { + logger.warn({ err, childRunId: commandId }, "Committed workflow child awaits outbox retry"); + }); + await settleCommittedHandOff(session); + return response; +} + +async function settleCommittedHandOff(session: RunnerSession): Promise { + const state = await getWorkflowRunnerRegistrationState({ + runId: session.attempt.runId, + attemptId: session.attempt.attemptId, + }); + if (state.state === "completed") return; + if (state.state !== "result-pending") { + throw new Error("Committed hand-off has no durable terminal result"); + } + await processWorkflowRunnerResult({ + runId: session.attempt.runId, + attemptId: session.attempt.attemptId, + executionDeliveryId: state.executionDeliveryId, + payload: state.payload, + }); +} + +function normalizeRunnerFailure(payload: WorkflowRunnerResultPayload): WorkflowRunnerResultPayload { + const result = payload.result; + if (result.status === "succeeded" || result.status === "handed-off") return payload; + return { + ...payload, + result: { + ...result, + reason: redactErrorMessageOrFallback(result.reason, "workflow failed"), + ...(result.humanMessage === undefined + ? {} + : { + humanMessage: redactErrorMessageOrFallback( + result.humanMessage, + "workflow failed, see server logs", + ), + }), + }, + }; +} + +async function handleResult( + ws: ServerWebSocket, + message: Extract, +): Promise { + const session = sessions.get(message.payload.attemptId); + if (session?.ws !== ws) throw new StaleWorkflowAttemptError(message.payload); + const attempt = { runId: message.payload.runId, attemptId: message.payload.attemptId }; + const registration = await getWorkflowRunnerRegistrationState(attempt); + if (registration.state === "invalid") throw new StaleWorkflowAttemptError(attempt); + + if (registration.state === "result-pending") { + await processWorkflowRunnerResult({ + runId: attempt.runId, + attemptId: attempt.attemptId, + executionDeliveryId: registration.executionDeliveryId, + payload: registration.payload, + }); + } else if (registration.state === "ready") { + const payload = normalizeRunnerFailure(await sanitizeWorkflowRunnerResult(message.payload)); + await storeWorkflowRunnerResult(payload); + await processWorkflowRunnerResult({ + runId: payload.runId, + attemptId: payload.attemptId, + executionDeliveryId: session.attempt.executionDeliveryId, + payload, + }); + } + send(ws, { + type: "workflow-runner:result-ack", + ...createMessageEnvelope(attempt.attemptId), + payload: {}, + }); + if (sessions.get(attempt.attemptId)?.ws === ws) { + sessions.delete(attempt.attemptId); + await revokeRunnerSessionToken(session); + } + void cleanupWorkflowRunnerAttempt(attempt).catch((err: unknown) => { + logger.warn({ err, attemptId: attempt.attemptId }, "Runner cleanup will be reconciled"); + }); +} + +export function getWorkflowRunnerConnection( + attemptId: string, +): ServerWebSocket | undefined { + return sessions.get(attemptId)?.ws; +} + +export function resetWorkflowRunnerControllerForTests(): void { + sessions.clear(); + attemptChains.clear(); +} diff --git a/src/orchestrator/workflow-runner-dispatch.ts b/src/orchestrator/workflow-runner-dispatch.ts new file mode 100644 index 00000000..339bc086 --- /dev/null +++ b/src/orchestrator/workflow-runner-dispatch.ts @@ -0,0 +1,131 @@ +import { config } from "../config"; +import { EphemeralSpawnError } from "../k8s/ephemeral-daemon-spawner"; +import { WorkflowRunnerResourceError } from "../k8s/workflow-runner-spawner"; +import { logger } from "../logger"; +import { ensureWorkflowCascadeForOffer } from "../workflows/completion-reconciler"; +import { logWorkflowRunRunning } from "../workflows/log-fields"; +import type { WorkflowRunQueuedJob } from "./job-queue"; +import { notifyRunnerStartFailures } from "./workflow-expiry-notifier"; +import { deriveWorkflowRunnerCapability } from "./workflow-runner-capability"; +import { + cleanupCurrentWorkflowRunnerResources, + ensureCurrentWorkflowRunnerResources, +} from "./workflow-runner-resources"; +import { + claimWorkflowRunnerAttempt, + failWorkflowRunnerAttempt, + type WorkflowRunnerAttempt, +} from "./workflow-runner-store"; + +export const WORKFLOW_RUNNER_STARTUP_LEASE_MS = Math.max(300_000, 4 * config.heartbeatTimeoutMs); + +function requiredRunnerConfig(): { + readonly image: string; + readonly orchestratorUrl: string; + readonly capabilitySecret: string; +} { + if (config.githubPersonalAccessToken !== undefined) { + throw new WorkflowRunnerResourceError( + "permanent", + "Workflow runners require GitHub App mode; PAT mode cannot mint a target-repository token", + ); + } + if (config.daemonImage === undefined || config.daemonImage === "") { + throw new WorkflowRunnerResourceError("permanent", "DAEMON_IMAGE is required for workflows"); + } + if (config.orchestratorPublicUrl === undefined || config.orchestratorPublicUrl === "") { + throw new WorkflowRunnerResourceError( + "permanent", + "ORCHESTRATOR_PUBLIC_URL is required for workflows", + ); + } + if ( + config.workflowRunnerCapabilitySecret === undefined || + config.workflowRunnerCapabilitySecret === "" + ) { + throw new WorkflowRunnerResourceError( + "permanent", + "WORKFLOW_RUNNER_CAPABILITY_SECRET is required to derive workflow runner capabilities", + ); + } + return { + image: config.daemonImage, + orchestratorUrl: config.orchestratorPublicUrl, + capabilitySecret: config.workflowRunnerCapabilitySecret, + }; +} + +function isPermanentResourceFailure(err: unknown): boolean { + if (err instanceof WorkflowRunnerResourceError) return err.kind === "permanent"; + return ( + err instanceof EphemeralSpawnError && + (err.kind === "infra-absent" || err.kind === "auth-load-failed" || err.kind === "api-rejected") + ); +} + +export async function failWorkflowRunnerResourceAttempt( + attempt: WorkflowRunnerAttempt, + reason: string, +): Promise { + const row = await failWorkflowRunnerAttempt(attempt, reason); + await ensureWorkflowCascadeForOffer(attempt.attemptId, logger).catch((err: unknown) => { + logger.warn({ err, attemptId: attempt.attemptId }, "Runner-start cascade will be reconciled"); + }); + await notifyRunnerStartFailures([row]); + await cleanupCurrentWorkflowRunnerResources(attempt).catch((err: unknown) => { + logger.warn({ err, attemptId: attempt.attemptId }, "Runner resource cleanup will be retried"); + }); +} + +/** Admit a workflow queue item into its durable, isolated runner attempt. */ +export async function dispatchWorkflowRunner( + job: WorkflowRunQueuedJob, +): Promise<"accepted" | "stale" | "capacity"> { + const maxActive = config.maxConcurrentRequests; + if (!Number.isInteger(maxActive) || maxActive <= 0) { + throw new RangeError("MAX_CONCURRENT_REQUESTS must be a positive integer"); + } + const claim = await claimWorkflowRunnerAttempt(job, WORKFLOW_RUNNER_STARTUP_LEASE_MS, maxActive); + if (claim.outcome === "stale" || claim.outcome === "capacity") return claim.outcome; + const attempt = claim.attempt; + if (claim.outcome === "claimed") { + logWorkflowRunRunning(logger, { + runId: attempt.runId, + workflowName: attempt.workflowName, + target: { + type: job.isPR ? "pr" : "issue", + owner: job.repoOwner, + repo: job.repoName, + number: job.entityNumber, + }, + deliveryId: job.deliveryId, + }); + } + + try { + const runnerConfig = requiredRunnerConfig(); + const capability = deriveWorkflowRunnerCapability( + runnerConfig.capabilitySecret, + attempt.runId, + attempt.attemptId, + attempt.attemptDeadlineAt, + ); + await ensureCurrentWorkflowRunnerResources({ + attempt, + capability, + image: runnerConfig.image, + orchestratorUrl: runnerConfig.orchestratorUrl, + }); + } catch (err) { + if (isPermanentResourceFailure(err)) { + const reason = err instanceof Error ? err.message : "Workflow runner configuration failed"; + await failWorkflowRunnerResourceAttempt(attempt, reason); + return "accepted"; + } + logger.warn( + { err, runId: attempt.runId, attemptId: attempt.attemptId }, + "Workflow runner resource state is ambiguous; durable reconciliation will retry", + ); + } + return "accepted"; +} diff --git a/src/orchestrator/workflow-runner-output.ts b/src/orchestrator/workflow-runner-output.ts new file mode 100644 index 00000000..b3ef3d4e --- /dev/null +++ b/src/orchestrator/workflow-runner-output.ts @@ -0,0 +1,183 @@ +import { config } from "../config"; +import { logger } from "../logger"; +import { + type WorkflowRunnerCommand, + WorkflowRunnerCommandSchema, + type WorkflowRunnerResultPayload, + WorkflowRunnerResultPayloadSchema, +} from "../shared/workflow-runner-messages"; +import { + configuredCredentialValues, + containsExactCredentialPropertyName, + containsExactCredentialValue, +} from "../utils/exact-credential-redaction"; +import { detectSecretsWithLlm } from "../utils/llm-output-scanner"; +import { redactSecrets } from "../utils/sanitize"; + +export class WorkflowRunnerOutputRejectedError extends Error { + constructor() { + super("workflow runner output was rejected by credential policy"); + this.name = "WorkflowRunnerOutputRejectedError"; + } +} + +interface RedactedValue { + readonly value: unknown; + readonly matchCount: number; + readonly kinds: readonly string[]; + readonly propertyNameMatchCount: number; +} + +function redactStructuredValue(value: unknown): RedactedValue { + if (typeof value === "string") { + const result = redactSecrets(value); + return { + value: result.body, + matchCount: result.matchCount, + kinds: result.kinds, + propertyNameMatchCount: 0, + }; + } + if (Array.isArray(value)) { + const entries = value.map(redactStructuredValue); + return combine( + entries, + entries.map((entry) => entry.value), + ); + } + if (value !== null && typeof value === "object") { + const entries = Object.entries(value).map(([key, entry]) => { + const keyScan = redactSecrets(key); + return { key, keyScan, entry: redactStructuredValue(entry) }; + }); + const combined = combine( + entries.map(({ entry }) => entry), + Object.fromEntries(entries.map(({ key, entry }) => [key, entry.value])), + ); + return { + ...combined, + matchCount: + combined.matchCount + entries.reduce((total, item) => total + item.keyScan.matchCount, 0), + kinds: [...new Set([...combined.kinds, ...entries.flatMap((item) => item.keyScan.kinds)])], + propertyNameMatchCount: + combined.propertyNameMatchCount + + entries.reduce((total, item) => total + item.keyScan.matchCount, 0), + }; + } + return { value, matchCount: 0, kinds: [], propertyNameMatchCount: 0 }; +} + +function combine(entries: readonly RedactedValue[], value: unknown): RedactedValue { + return { + value, + matchCount: entries.reduce((total, entry) => total + entry.matchCount, 0), + kinds: [...new Set(entries.flatMap((entry) => entry.kinds))], + propertyNameMatchCount: entries.reduce( + (total, entry) => total + entry.propertyNameMatchCount, + 0, + ), + }; +} + +async function encodedSecretScanIsSafe(value: unknown, callsite: string): Promise { + if (!config.llmOutputScannerEnabled) { + logger.error( + { event: "workflow_runner_output_scan_unavailable", scanner: "llm", callsite }, + "Workflow runner output scanner is disabled; rejecting output", + ); + return false; + } + try { + // Detect-only: this boundary rejects the whole payload on a hit and never + // reads a redacted body. Asking the model to echo a review-sized payload + // back made the call's wall-clock scale with output size, which timed out + // and discarded completed runs. + const scan = await detectSecretsWithLlm(JSON.stringify(value), { + timeoutMs: config.llmOutputScannerTimeoutMs, + log: logger, + }); + if (scan.containsSecret) { + logger.warn( + { + event: "workflow_runner_output_rejected", + scanner: "llm", + callsite, + kinds: scan.kinds, + matchCount: scan.matchCount, + }, + "Rejected workflow runner output containing encoded credentials", + ); + } + return !scan.containsSecret; + } catch (err) { + logger.error( + { event: "workflow_runner_output_scan_unavailable", scanner: "llm", err, callsite }, + "Workflow runner output scanner failed; rejecting output", + ); + return false; + } +} + +function containsConfiguredCredential(value: unknown): boolean { + const credentials = configuredCredentialValues(); + return ( + containsExactCredentialPropertyName(value, credentials) || + containsExactCredentialValue(value, credentials) + ); +} + +function logDeterministicRedaction(redacted: RedactedValue, callsite: string): void { + if (redacted.matchCount === 0) return; + logger.warn( + { + event: "secret_redacted", + scanner: "regex", + callsite, + kinds: redacted.kinds, + matchCount: redacted.matchCount, + propertyNameMatchCount: redacted.propertyNameMatchCount, + }, + "Redacted credentials from workflow runner output", + ); +} + +export async function sanitizeWorkflowRunnerCommand( + command: WorkflowRunnerCommand, +): Promise { + if (containsConfiguredCredential(command)) throw new WorkflowRunnerOutputRejectedError(); + const redacted = redactStructuredValue(command); + logDeterministicRedaction(redacted, "workflow-runner.command"); + if (redacted.matchCount > 0) throw new WorkflowRunnerOutputRejectedError(); + const parsed = WorkflowRunnerCommandSchema.safeParse(redacted.value); + if (!parsed.success || !(await encodedSecretScanIsSafe(parsed.data, "workflow-runner.command"))) { + throw new WorkflowRunnerOutputRejectedError(); + } + return parsed.data; +} + +export async function sanitizeWorkflowRunnerResult( + payload: WorkflowRunnerResultPayload, +): Promise { + const exactCredential = containsConfiguredCredential(payload); + const redacted = redactStructuredValue(payload); + logDeterministicRedaction(redacted, "workflow-runner.result"); + const parsed = WorkflowRunnerResultPayloadSchema.safeParse(redacted.value); + if ( + !exactCredential && + redacted.propertyNameMatchCount === 0 && + parsed.success && + (await encodedSecretScanIsSafe(parsed.data, "workflow-runner.result")) + ) { + return parsed.data; + } + return { + runId: payload.runId, + attemptId: payload.attemptId, + durationMs: payload.durationMs, + result: { + status: "failed", + reason: "workflow runner output was rejected by credential policy", + humanMessage: "Workflow output was rejected by the credential safety boundary.", + }, + }; +} diff --git a/src/orchestrator/workflow-runner-payload.ts b/src/orchestrator/workflow-runner-payload.ts new file mode 100644 index 00000000..1ed7d13f --- /dev/null +++ b/src/orchestrator/workflow-runner-payload.ts @@ -0,0 +1,330 @@ +import { App, type Octokit } from "octokit"; + +import { config } from "../config"; +import { requireDb } from "../db"; +import { logger } from "../logger"; +import { loadRepoPolicy, policyForWorkflow, toAgentPolicy } from "../repo-config/effective"; +import type { SerializableBotContext } from "../shared/daemon-types"; +import type { WorkflowRunnerPayload } from "../shared/workflow-runner-messages"; +import { + RepoMemoryEntrySchema, + type WorkflowName, + type WorkflowRunSnapshot, +} from "../shared/workflow-types"; +import type { AgentPolicy } from "../shared/ws-messages"; +import { + findLatestForTarget, + findLatestSucceededForTarget, + mergeAttemptState, +} from "../workflows/runs-store"; +import { CONFIG_NOTICE_KEY } from "../workflows/tracking-mirror"; +import { mintInstallationToken, revokeInstallationToken } from "./installation-token"; +import { getRepoMemory } from "./repo-knowledge"; +import { + loadReviewLearnings, + type ReviewLearning, + searchReviewLearningsByEmbedding, +} from "./review-learnings"; +import type { WorkflowRunnerAttempt } from "./workflow-runner-store"; + +let runnerApp: InstanceType | null = null; +export const GITHUB_INSTALLATION_TOKEN_LIFETIME_MS = 60 * 60 * 1_000; + +function getRunnerApp(): InstanceType { + if (runnerApp !== null) return runnerApp; + if (config.appId === undefined || config.privateKey === undefined) { + throw new Error("Workflow runners require GITHUB_APP_ID and GITHUB_APP_PRIVATE_KEY"); + } + runnerApp = new App({ appId: config.appId, privateKey: config.privateKey }); + return runnerApp; +} + +function stripInstructionsUnlessReview( + policy: AgentPolicy | undefined, + workflowName: WorkflowName, +): AgentPolicy | undefined { + if (policy?.instructions === undefined || workflowName === "review") return policy; + const { instructions: _dropped, ...rest } = policy; + return Object.keys(rest).length > 0 ? rest : undefined; +} + +async function loadExecutionContext( + attempt: WorkflowRunnerAttempt, +): Promise { + const db = requireDb(); + const rows: { context_json: Record | null }[] = await db` + SELECT context_json + FROM executions + WHERE delivery_id = ${attempt.executionDeliveryId} + AND daemon_id = ${attempt.runnerId} + AND offer_id = ${attempt.attemptId} + AND status = 'running' + `; + const context = rows[0]?.context_json; + if (context === null || context === undefined) { + throw new Error("Workflow runner execution context is missing"); + } + return context as unknown as SerializableBotContext; +} + +async function mintWorkflowRunnerRepositoryToken(attempt: WorkflowRunnerAttempt): Promise<{ + readonly context: SerializableBotContext; + readonly octokit: Octokit; + readonly token: string; + readonly expiresAt: string; +}> { + const context = await loadExecutionContext(attempt); + const app = getRunnerApp(); + const { data: installation } = await app.octokit.rest.apps.getRepoInstallation({ + owner: context.owner, + repo: context.repo, + }); + const minted = await mintInstallationToken({ + app, + installationId: installation.id, + repositoryName: context.repo, + via: "workflowRunnerPayload", + log: logger, + }); + return { context, ...minted }; +} + +/** Mint controller-only repository authority for a credential-free runner reconnect. */ +export async function prepareWorkflowRunnerControllerOctokit( + attempt: WorkflowRunnerAttempt, +): Promise { + if (config.githubPersonalAccessToken !== undefined) { + throw new Error("Workflow runners do not support GITHUB_PERSONAL_ACCESS_TOKEN mode"); + } + return (await mintWorkflowRunnerRepositoryToken(attempt)).octokit; +} + +async function loadRunnerReviewLearnings( + octokit: Octokit, + context: SerializableBotContext, + policy: Awaited>, +): Promise { + if (!config.reviewLearningsEnabled || !policy.reviewLearnings.enabled) return []; + const filter = { + scope: policy.reviewLearnings.scope, + maxAgeDays: policy.reviewLearnings.max_age_days, + } as const; + if (!config.reviewLearningsRagEnabled || !context.isPR) { + return loadReviewLearnings(context.owner, context.repo, filter); + } + try { + const files = await octokit.paginate(octokit.rest.pulls.listFiles, { + owner: context.owner, + repo: context.repo, + pull_number: context.entityNumber, + per_page: 100, + }); + return await searchReviewLearningsByEmbedding( + context.owner, + context.repo, + files.map((file) => file.filename), + { filter }, + ); + } catch (err) { + logger.warn( + { err, owner: context.owner, repo: context.repo, number: context.entityNumber }, + "Workflow runner RAG load failed; falling back to deterministic review learnings", + ); + return loadReviewLearnings(context.owner, context.repo, filter); + } +} + +function snapshot( + workflowName: WorkflowName, + row: Awaited>, +): WorkflowRunSnapshot | null { + if (row === null) return null; + const state: WorkflowRunSnapshot["state"] = {}; + if ( + workflowName === "triage" && + (row.state["recommendedNext"] === "plan" || row.state["recommendedNext"] === "stop") + ) { + state.recommendedNext = row.state["recommendedNext"]; + } + if ( + workflowName === "implement" && + typeof row.state["pr_number"] === "number" && + Number.isInteger(row.state["pr_number"]) && + row.state["pr_number"] > 0 + ) { + state.pr_number = row.state["pr_number"]; + } + return { + id: row.id, + status: row.status, + state, + createdAt: row.created_at.toISOString(), + }; +} + +async function loadPriorState( + workflowName: WorkflowName, + context: SerializableBotContext, +): Promise> { + const target = { owner: context.owner, repo: context.repo, number: context.entityNumber }; + if (workflowName === "implement") { + const plan = await findLatestSucceededForTarget("plan", target); + const markdown = plan?.state["plan"]; + if (plan === null) return {}; + if (typeof markdown !== "string" || markdown.length === 0 || markdown.length > 100_000) { + throw new Error("Succeeded plan state is missing or exceeds the runner payload limit"); + } + return { priorPlanState: { plan: markdown } }; + } + if (workflowName !== "ship") return {}; + + const names = ["triage", "plan", "implement", "review", "resolve"] as const; + const rows = await Promise.all(names.map((name) => findLatestForTarget(name, target))); + const shipStepRuns: Partial> = {}; + for (const [index, name] of names.entries()) { + const row = snapshot(name, rows[index] ?? null); + if (row !== null) shipStepRuns[name] = row; + } + return { shipStepRuns }; +} + +function configNotice(policy: AgentPolicy | undefined): string | null { + const lines: string[] = []; + if (policy?.warning !== undefined && policy.warning.trim() !== "") lines.push(policy.warning); + if (policy?.pathFilters !== undefined && policy.pathFilters.length > 0) { + const globs = policy.pathFilters.map((glob) => `\`${glob}\``).join(", "); + lines.push( + `Review scope reduced by \`${config.repoConfigFile}\`: files matching ${globs} are excluded.`, + ); + } + return lines.length === 0 ? null : lines.join("\n"); +} + +async function loadRunnerRepoMemory( + owner: string, + repo: string, +): Promise> { + const rows = await getRepoMemory(owner, repo); + const valid: NonNullable = []; + let dropped = 0; + let omitted = 0; + for (const row of rows) { + const parsed = RepoMemoryEntrySchema.safeParse(row); + if (!parsed.success) { + dropped++; + } else if (valid.length < 50) { + valid.push(parsed.data); + } else { + omitted++; + } + } + if (dropped > 0 || omitted > 0) { + logger.warn( + { owner, repo, dropped, omitted }, + "Filtered invalid or excess workflow runner repo memory", + ); + } + return valid; +} + +/** Build the only payload an isolated workflow runner receives. */ +export async function prepareWorkflowRunnerPayload( + attempt: WorkflowRunnerAttempt, +): Promise { + if (config.githubPersonalAccessToken !== undefined) { + throw new Error("Workflow runners do not support GITHUB_PERSONAL_ACCESS_TOKEN mode"); + } + + if (attempt.attemptDeadlineAt.getTime() - Date.now() <= GITHUB_INSTALLATION_TOKEN_LIFETIME_MS) { + throw new Error("Workflow runner attempt has insufficient lifetime for a GitHub token"); + } + + const { context, octokit, token, expiresAt } = await mintWorkflowRunnerRepositoryToken(attempt); + try { + if (new Date(expiresAt).getTime() > attempt.attemptDeadlineAt.getTime()) { + throw new Error("GitHub token expiry exceeds the workflow runner attempt deadline"); + } + const repoPolicy = await loadRepoPolicy({ + octokit, + owner: context.owner, + repo: context.repo, + log: logger, + }); + const workflowPolicy = policyForWorkflow(repoPolicy, attempt.workflowName); + const policy = stripInstructionsUnlessReview( + toAgentPolicy(workflowPolicy, repoPolicy.warning), + attempt.workflowName, + ); + const maxTurns = workflowPolicy.maxTurns ?? config.agentMaxTurns ?? config.defaultMaxTurns; + if (policy !== undefined || workflowPolicy.maxTurns !== undefined) { + logger.info( + { + event: "repo_config.policy_applied", + owner: context.owner, + repo: context.repo, + deliveryId: attempt.executionDeliveryId, + workflow: attempt.workflowName, + runId: attempt.runId, + attemptId: attempt.attemptId, + model: policy?.model, + maxTurns, + timeoutMs: policy?.timeoutMs, + extraAllowedToolCount: policy?.extraAllowedTools?.length ?? 0, + pathFilterCount: policy?.pathFilters?.length ?? 0, + hasInstructions: policy?.instructions !== undefined, + warned: policy?.warning !== undefined, + }, + "Per-repo agent policy applied", + ); + } + const [reviewLearnings, repoMemory, priorState] = await Promise.all([ + loadRunnerReviewLearnings(octokit, context, repoPolicy), + loadRunnerRepoMemory(context.owner, context.repo), + loadPriorState(attempt.workflowName, context), + ]); + + const notice = configNotice(policy); + if (notice !== null) { + await mergeAttemptState( + { runId: attempt.runId, attemptId: attempt.attemptId }, + { [CONFIG_NOTICE_KEY]: notice }, + ); + } + + return { + context: context as unknown as Record, + installationToken: token, + installationTokenExpiresAt: expiresAt, + attemptDeadlineAt: attempt.attemptDeadlineAt.toISOString(), + ...(repoMemory.length > 0 ? { repoMemory } : {}), + ...(maxTurns !== undefined ? { maxTurns } : {}), + ...(reviewLearnings.length > 0 + ? { + reviewLearnings: reviewLearnings.map((learning) => ({ + id: learning.id, + scope: learning.scope, + fileGlob: learning.fileGlob, + directive: learning.directive, + rationale: learning.rationale, + sourcePr: learning.sourcePr, + sourceThread: learning.sourceThread, + sourceAuthor: learning.sourceAuthor, + createdAt: learning.createdAt.toISOString(), + })), + } + : {}), + ...(policy !== undefined ? { policy } : {}), + workflowRun: { + runId: attempt.runId, + workflowName: attempt.workflowName, + }, + ...priorState, + }; + } catch (err) { + await revokeInstallationToken(octokit, logger, { + attemptId: attempt.attemptId, + owner: "payload-preparation", + }); + throw err; + } +} diff --git a/src/orchestrator/workflow-runner-reconciler.ts b/src/orchestrator/workflow-runner-reconciler.ts new file mode 100644 index 00000000..0b0e76f9 --- /dev/null +++ b/src/orchestrator/workflow-runner-reconciler.ts @@ -0,0 +1,81 @@ +import { config } from "../config"; +import { WorkflowRunnerResourceError } from "../k8s/workflow-runner-spawner"; +import { logger } from "../logger"; +import { reconcilePendingWorkflowFailureNotifications } from "./workflow-expiry-notifier"; +import { deriveWorkflowRunnerCapability } from "./workflow-runner-capability"; +import { failWorkflowRunnerResourceAttempt } from "./workflow-runner-dispatch"; +import { ensureCurrentWorkflowRunnerResources } from "./workflow-runner-resources"; +import { + cleanupWorkflowRunnerAttempt, + reconcilePendingWorkflowRunnerResults, +} from "./workflow-runner-result"; +import { + findWorkflowRunnerCleanupCandidates, + listActiveWorkflowRunnerAttempts, +} from "./workflow-runner-store"; + +async function reconcileActiveResources(): Promise { + const image = config.daemonImage; + const orchestratorUrl = config.orchestratorPublicUrl; + const capabilitySecret = config.workflowRunnerCapabilitySecret; + if ( + config.githubPersonalAccessToken !== undefined || + image === undefined || + orchestratorUrl === undefined || + capabilitySecret === undefined + ) { + return; + } + const attempts = await listActiveWorkflowRunnerAttempts(); + for (const attempt of attempts) { + try { + const capability = deriveWorkflowRunnerCapability( + capabilitySecret, + attempt.runId, + attempt.attemptId, + attempt.attemptDeadlineAt, + ); + // eslint-disable-next-line no-await-in-loop -- Kubernetes reconciliation is bounded + await ensureCurrentWorkflowRunnerResources({ attempt, capability, image, orchestratorUrl }); + } catch (err) { + if (err instanceof WorkflowRunnerResourceError && err.kind === "permanent") { + try { + // eslint-disable-next-line no-await-in-loop -- exact attempt failure is ordered + await failWorkflowRunnerResourceAttempt(attempt, err.message); + continue; + } catch (failureErr) { + logger.error( + { failureErr, runId: attempt.runId, attemptId: attempt.attemptId }, + "Workflow runner boundary violation could not be fenced", + ); + } + } + logger.warn( + { err, runId: attempt.runId, attemptId: attempt.attemptId }, + "Workflow runner resource reconciliation failed", + ); + } + } +} + +async function reconcileCleanup(): Promise { + const candidates = await findWorkflowRunnerCleanupCandidates(); + for (const candidate of candidates) { + try { + // eslint-disable-next-line no-await-in-loop -- Kubernetes cleanup is bounded + await cleanupWorkflowRunnerAttempt(candidate); + } catch (err) { + logger.warn( + { err, runId: candidate.runId, attemptId: candidate.attemptId }, + "Workflow runner cleanup reconciliation failed", + ); + } + } +} + +export async function reconcileWorkflowRunners(): Promise { + await reconcilePendingWorkflowRunnerResults(); + await reconcilePendingWorkflowFailureNotifications(); + await reconcileActiveResources(); + await reconcileCleanup(); +} diff --git a/src/orchestrator/workflow-runner-resources.ts b/src/orchestrator/workflow-runner-resources.ts new file mode 100644 index 00000000..5660737b --- /dev/null +++ b/src/orchestrator/workflow-runner-resources.ts @@ -0,0 +1,67 @@ +import { + deleteWorkflowRunnerResources, + ensureWorkflowRunnerResources, +} from "../k8s/workflow-runner-spawner"; +import { + getWorkflowRunnerRegistrationState, + markWorkflowRunnerResourcesCleaned, + type WorkflowRunnerAttempt, +} from "./workflow-runner-store"; + +const resourceChains = new Map>(); + +async function serializeResourceOperation( + attemptId: string, + operation: () => Promise, +): Promise { + const prior = resourceChains.get(attemptId) ?? Promise.resolve(); + const current = prior.catch(() => undefined).then(operation); + resourceChains.set(attemptId, current); + try { + return await current; + } finally { + if (resourceChains.get(attemptId) === current) resourceChains.delete(attemptId); + } +} + +async function deleteAndRecord(attempt: { + readonly runId: string; + readonly attemptId: string; +}): Promise { + if (await deleteWorkflowRunnerResources(attempt)) { + await markWorkflowRunnerResourcesCleaned(attempt); + } +} + +/** Recheck durable ownership under the same per-attempt chain as cleanup. */ +export async function ensureCurrentWorkflowRunnerResources(input: { + readonly attempt: WorkflowRunnerAttempt; + readonly capability: string; + readonly image: string; + readonly orchestratorUrl: string; +}): Promise<"ready" | "result-pending" | "terminal"> { + return serializeResourceOperation(input.attempt.attemptId, async () => { + const before = await getWorkflowRunnerRegistrationState(input.attempt); + if (before.state !== "ready") { + if (before.state !== "result-pending") await deleteAndRecord(input.attempt); + return before.state === "result-pending" ? "result-pending" : "terminal"; + } + + await ensureWorkflowRunnerResources(input); + const after = await getWorkflowRunnerRegistrationState(input.attempt); + if (after.state === "ready") return "ready"; + if (after.state !== "result-pending") await deleteAndRecord(input.attempt); + return after.state === "result-pending" ? "result-pending" : "terminal"; + }); +} + +export async function cleanupCurrentWorkflowRunnerResources(attempt: { + readonly runId: string; + readonly attemptId: string; +}): Promise { + await serializeResourceOperation(attempt.attemptId, () => deleteAndRecord(attempt)); +} + +export function resetWorkflowRunnerResourceChainsForTests(): void { + resourceChains.clear(); +} diff --git a/src/orchestrator/workflow-runner-result.ts b/src/orchestrator/workflow-runner-result.ts new file mode 100644 index 00000000..ffa9375a --- /dev/null +++ b/src/orchestrator/workflow-runner-result.ts @@ -0,0 +1,308 @@ +import { App, type Octokit } from "octokit"; + +import { config } from "../config"; +import { requireDb } from "../db"; +import { logger } from "../logger"; +import { redactErrorMessageOrFallback } from "../utils/log-redaction"; +import { addReaction } from "../utils/reactions"; +import { ensureWorkflowCascadeForOffer } from "../workflows/completion-reconciler"; +import { + logWorkflowRunFailed, + logWorkflowRunHandedOff, + logWorkflowRunIncomplete, + logWorkflowRunSucceeded, +} from "../workflows/log-fields"; +import { findById } from "../workflows/runs-store"; +import { setState } from "../workflows/tracking-mirror"; +import { mintInstallationToken, revokeInstallationToken } from "./installation-token"; +import { persistRepoKnowledge } from "./repo-knowledge-persistence"; +import { cleanupCurrentWorkflowRunnerResources } from "./workflow-runner-resources"; +import { + findPendingWorkflowRunnerResults, + getWorkflowRunnerResultProcessingState, + markWorkflowRunnerResultProcessed, + type PendingWorkflowRunnerResult, +} from "./workflow-runner-store"; + +let resultApp: InstanceType | null = null; +const resultChains = new Map>(); +const workflowProjectionChains = new Map>(); + +function getResultApp(): InstanceType { + if (resultApp !== null) return resultApp; + if (config.appId === undefined || config.privateKey === undefined) { + throw new Error("Workflow result reconciliation requires GitHub App credentials"); + } + resultApp = new App({ appId: config.appId, privateKey: config.privateKey }); + return resultApp; +} + +async function getResultOctokit(owner: string, repo: string): Promise { + const app = getResultApp(); + const { data: installation } = await app.octokit.rest.apps.getRepoInstallation({ owner, repo }); + return ( + await mintInstallationToken({ + app, + installationId: installation.id, + repositoryName: repo, + via: "workflowRunnerResult", + log: logger, + }) + ).octokit; +} + +function terminalHumanMessage( + workflowName: string, + result: PendingWorkflowRunnerResult["payload"]["result"], +): string { + if (result.humanMessage !== undefined) { + return redactErrorMessageOrFallback(result.humanMessage, `${workflowName} completed`); + } + if (result.status === "succeeded") return `${workflowName} succeeded`; + if (result.status === "incomplete") { + return `${workflowName} incomplete, see tracking comment for outstanding items.`; + } + return `${workflowName} failed, see server logs for details.`; +} + +function githubStatus(err: unknown): number | undefined { + if (err === null || typeof err !== "object" || !("status" in err)) return undefined; + const status = (err as { status?: unknown }).status; + return typeof status === "number" ? status : undefined; +} + +async function findProjectionRootId( + row: NonNullable>>, +): Promise { + let current = row; + const visited = new Set(); + while (current.parent_run_id !== null) { + if (visited.has(current.id)) { + throw new Error(`Workflow parent cycle detected at ${current.id}`); + } + visited.add(current.id); + // eslint-disable-next-line no-await-in-loop -- each read follows one durable parent edge + const parent = await findById(current.parent_run_id); + if (parent === null) { + throw new Error(`Workflow parent is missing: ${current.parent_run_id}`); + } + current = parent; + } + return current.id; +} + +async function serializeWorkflowProjection( + row: NonNullable>>, + project: () => Promise, +): Promise { + const key = await findProjectionRootId(row); + const prior = workflowProjectionChains.get(key) ?? Promise.resolve(); + const current = prior.catch(() => undefined).then(project); + workflowProjectionChains.set(key, current); + try { + await current; + } finally { + if (workflowProjectionChains.get(key) === current) workflowProjectionChains.delete(key); + } +} + +async function projectTerminalState( + row: NonNullable>>, + pending: PendingWorkflowRunnerResult, + octokit: Octokit, +): Promise { + try { + await setState( + { octokit, logger }, + { + runId: pending.runId, + patch: {}, + humanMessage: terminalHumanMessage(row.workflow_name, pending.payload.result), + }, + ); + } catch (err) { + if (githubStatus(err) !== 422) throw err; + logger.warn( + { + event: "workflow_runner_terminal_projection_rejected", + runId: pending.runId, + attemptId: pending.attemptId, + status: 422, + }, + "GitHub rejected workflow terminal projection; retrying with fixed fallback", + ); + await setState( + { octokit, logger }, + { + runId: pending.runId, + patch: {}, + humanMessage: `${row.workflow_name} reached a terminal state, but GitHub rejected its detailed status. Inspect controller logs and durable workflow state.`, + }, + ); + } +} + +async function reactToTrigger( + row: NonNullable>>, + octokit: Octokit, + success: boolean, +): Promise { + if (row.trigger_comment_id === null || row.trigger_event_type === null) return; + await addReaction({ + octokit, + logger, + owner: row.target_owner, + repo: row.target_repo, + commentId: row.trigger_comment_id, + eventType: row.trigger_event_type, + content: success ? "hooray" : "confused", + }); +} + +function logTerminalResult( + row: NonNullable>>, + pending: PendingWorkflowRunnerResult, +): void { + const result = pending.payload.result; + const fields = { + runId: pending.runId, + workflowName: row.workflow_name, + target: { + type: row.target_type, + owner: row.target_owner, + repo: row.target_repo, + number: row.target_number, + }, + deliveryId: pending.executionDeliveryId, + durationMs: pending.payload.durationMs, + }; + if (result.status === "succeeded") { + logWorkflowRunSucceeded(logger, fields); + } else if (result.status === "incomplete") { + logWorkflowRunIncomplete(logger, { ...fields, reason: result.reason }); + } else if (result.status === "failed") { + logWorkflowRunFailed(logger, { ...fields, reason: result.reason }); + } else { + logWorkflowRunHandedOff(logger, { ...fields, childRunId: result.childRunId }); + } +} + +async function projectWorkflowRunnerResult( + pending: PendingWorkflowRunnerResult, + suppliedOctokit?: Octokit, +): Promise { + const state = await getWorkflowRunnerResultProcessingState(pending); + if (state === "processed") return; + if (state === "missing") { + throw new Error(`Workflow result is not durably stored: ${pending.attemptId}`); + } + + const row = await findById(pending.runId); + if (row?.attempt_id !== pending.attemptId) { + throw new Error(`Workflow result row is no longer current: ${pending.attemptId}`); + } + const daemonActions = + "daemonActions" in pending.payload.result ? pending.payload.result.daemonActions : undefined; + const learningIds = + "appliedReviewLearningIds" in pending.payload.result + ? pending.payload.result.appliedReviewLearningIds + : undefined; + if (daemonActions !== undefined || (learningIds !== undefined && learningIds.length > 0)) { + await persistRepoKnowledge({ + deliveryId: pending.executionDeliveryId, + ...(daemonActions !== undefined ? { daemonActions } : {}), + ...(learningIds !== undefined ? { appliedReviewLearningIds: learningIds } : {}), + }).catch((err: unknown) => { + logger.warn( + { err, runId: pending.runId, attemptId: pending.attemptId }, + "Failed to persist workflow runner repo knowledge", + ); + }); + } + + const ownsOctokit = suppliedOctokit === undefined; + const octokit = suppliedOctokit ?? (await getResultOctokit(row.target_owner, row.target_repo)); + try { + await serializeWorkflowProjection(row, async () => { + await ensureWorkflowCascadeForOffer(pending.attemptId, logger, requireDb(), octokit); + + if (pending.payload.result.status === "handed-off") { + const current = await findById(pending.runId); + if (current?.attempt_id !== pending.attemptId) { + throw new Error(`Workflow result row is no longer current: ${pending.attemptId}`); + } + if (current.status !== "running") return; + await projectTerminalState(current, pending, octokit); + return; + } + + await projectTerminalState(row, pending, octokit); + }); + + if (pending.payload.result.status !== "handed-off") { + await reactToTrigger(row, octokit, pending.payload.result.status === "succeeded").catch( + (err: unknown) => { + logger.warn({ err, attemptId: pending.attemptId }, "Workflow terminal reaction failed"); + }, + ); + } + + if (!(await markWorkflowRunnerResultProcessed(pending.attemptId))) { + throw new Error(`Workflow result processing receipt was not current: ${pending.attemptId}`); + } + logTerminalResult(row, pending); + } finally { + if (ownsOctokit) { + await revokeInstallationToken(octokit, logger, { + attemptId: pending.attemptId, + owner: "result-projection", + }); + } + } +} + +/** + * Apply retryable projections before ACK. The in-memory chain prevents concurrent + * projection within the supported single orchestrator, but a process crash can replay it. + */ +export async function processWorkflowRunnerResult( + pending: PendingWorkflowRunnerResult, + suppliedOctokit?: Octokit, +): Promise { + const key = `${pending.runId}:${pending.attemptId}`; + const prior = resultChains.get(key) ?? Promise.resolve(); + const current = prior + .catch(() => undefined) + .then(() => projectWorkflowRunnerResult(pending, suppliedOctokit)); + resultChains.set(key, current); + try { + await current; + } finally { + if (resultChains.get(key) === current) resultChains.delete(key); + } +} + +export async function reconcilePendingWorkflowRunnerResults(limit = 100): Promise { + const pending = await findPendingWorkflowRunnerResults(requireDb(), limit); + let processed = 0; + for (const result of pending) { + try { + // eslint-disable-next-line no-await-in-loop -- ordered, bounded durable reconciliation + await processWorkflowRunnerResult(result); + processed++; + } catch (err) { + logger.warn( + { err, runId: result.runId, attemptId: result.attemptId }, + "Pending workflow runner result reconciliation failed", + ); + } + } + return processed; +} + +export async function cleanupWorkflowRunnerAttempt(input: { + readonly runId: string; + readonly attemptId: string; +}): Promise { + await cleanupCurrentWorkflowRunnerResources(input); +} diff --git a/src/orchestrator/workflow-runner-store.ts b/src/orchestrator/workflow-runner-store.ts new file mode 100644 index 00000000..3acb8ce8 --- /dev/null +++ b/src/orchestrator/workflow-runner-store.ts @@ -0,0 +1,732 @@ +import type { SQL } from "bun"; + +import { requireDb } from "../db"; +import { + type WorkflowRunnerCommand, + type WorkflowRunnerResultPayload, + WorkflowRunnerResultPayloadSchema, +} from "../shared/workflow-runner-messages"; +import { workflowRunnerId } from "../shared/workflow-types"; +import { + markAttemptFailed, + markAttemptIncomplete, + markAttemptSucceeded, + StaleWorkflowAttemptError, + type WorkflowAttempt, + type WorkflowRunRow, +} from "../workflows/runs-store"; +import type { WorkflowRunQueuedJob } from "./job-queue"; + +const RUNNER_ID_PREFIX = "workflow-runner:"; +export const WORKFLOW_RUNNER_ATTEMPT_DEADLINE_MS = 4_200_000; + +export { workflowRunnerId }; + +interface ExactRunnerRow { + id: string; + workflow_name: string; + execution_delivery_id: string; + status: string; + attempt_id: string | null; + owner_kind: string | null; + owner_id: string | null; + lease_expires_at: Date | null; + lease_active: boolean; + attempt_deadline_at: Date | null; + attempt_deadline_active: boolean; + attempt_completed_at: Date | null; + dispatch_generation_id: string; + runner_payload_issued_at: Date | null; + runner_token_expires_at: Date | null; + execution_status: string; + execution_daemon_id: string | null; + execution_offer_id: string | null; + result_processed_at: Date | null; + workflow_result_payload: unknown; +} + +export interface WorkflowRunnerAttempt { + readonly runId: string; + readonly attemptId: string; + readonly runnerId: string; + readonly executionDeliveryId: string; + readonly workflowName: WorkflowRunQueuedJob["workflowRun"]["workflowName"]; + readonly attemptDeadlineAt: Date; +} + +export type WorkflowRunnerClaim = + | { readonly outcome: "claimed" | "active"; readonly attempt: WorkflowRunnerAttempt } + | { readonly outcome: "capacity" } + | { readonly outcome: "stale" }; + +function exactJobWhere(job: WorkflowRunQueuedJob): { + readonly targetType: "issue" | "pr"; +} { + return { targetType: job.isPR ? "pr" : "issue" }; +} + +async function loadExactJobRow( + job: WorkflowRunQueuedJob, + sql: SQL, +): Promise { + const { targetType } = exactJobWhere(job); + const rows: ExactRunnerRow[] = await sql` + SELECT wr.id, + wr.workflow_name, + wr.execution_delivery_id, + wr.status, + wr.attempt_id, + wr.owner_kind, + wr.owner_id, + wr.lease_expires_at, + wr.lease_expires_at > now() AS lease_active, + wr.attempt_deadline_at, + wr.attempt_deadline_at > now() AS attempt_deadline_active, + wr.attempt_completed_at, + wr.dispatch_generation_id, + wr.runner_payload_issued_at, + wr.runner_token_expires_at, + e.status AS execution_status, + e.daemon_id AS execution_daemon_id, + e.offer_id AS execution_offer_id, + e.result_processed_at, + e.workflow_result_payload + FROM workflow_runs AS wr + JOIN executions AS e ON e.delivery_id = wr.execution_delivery_id + WHERE wr.id = ${job.workflowRun.runId} + AND wr.workflow_name = ${job.workflowRun.workflowName} + AND wr.execution_delivery_id = ${job.deliveryId} + AND wr.target_type = ${targetType} + AND wr.target_owner = ${job.repoOwner} + AND wr.target_repo = ${job.repoName} + AND wr.target_number = ${job.entityNumber} + `; + return rows[0] ?? null; +} + +function activeAttemptFromRow(row: ExactRunnerRow): WorkflowRunnerAttempt | null { + const attemptId = row.attempt_id; + if ( + row.status !== "running" || + attemptId === null || + row.owner_kind !== "daemon" || + row.owner_id !== workflowRunnerId(attemptId) || + row.execution_status !== "running" || + row.execution_daemon_id !== row.owner_id || + row.execution_offer_id !== attemptId || + row.lease_expires_at === null || + row.attempt_deadline_at === null || + row.attempt_completed_at !== null || + !row.lease_active || + !row.attempt_deadline_active + ) { + return null; + } + return { + runId: row.id, + attemptId, + runnerId: row.owner_id, + executionDeliveryId: row.execution_delivery_id, + workflowName: row.workflow_name as WorkflowRunnerAttempt["workflowName"], + attemptDeadlineAt: row.attempt_deadline_at, + }; +} + +/** Claim the workflow row and execution receipt directly for one runner Pod. */ +export async function claimWorkflowRunnerAttempt( + job: WorkflowRunQueuedJob, + startupLeaseMs: number, + maxActive: number, + sql: SQL = requireDb(), +): Promise { + if (!Number.isInteger(maxActive) || maxActive <= 0) { + throw new RangeError("maxActive must be a positive integer"); + } + try { + const claim = await sql.begin(async (tx) => { + const { targetType } = exactJobWhere(job); + const exactRows: ExactRunnerRow[] = await tx` + SELECT wr.id, + wr.workflow_name, + wr.execution_delivery_id, + wr.status, + wr.attempt_id, + wr.owner_kind, + wr.owner_id, + wr.lease_expires_at, + wr.lease_expires_at > now() AS lease_active, + wr.attempt_deadline_at, + wr.attempt_deadline_at > now() AS attempt_deadline_active, + wr.attempt_completed_at, + wr.dispatch_generation_id, + wr.runner_payload_issued_at, + wr.runner_token_expires_at, + e.status AS execution_status, + e.daemon_id AS execution_daemon_id, + e.offer_id AS execution_offer_id, + e.result_processed_at, + e.workflow_result_payload + FROM workflow_runs AS wr + JOIN executions AS e ON e.delivery_id = wr.execution_delivery_id + WHERE wr.id = ${job.workflowRun.runId} + AND wr.workflow_name = ${job.workflowRun.workflowName} + AND wr.execution_delivery_id = ${job.deliveryId} + AND wr.target_type = ${targetType} + AND wr.target_owner = ${job.repoOwner} + AND wr.target_repo = ${job.repoName} + AND wr.target_number = ${job.entityNumber} + FOR UPDATE OF wr, e + `; + const exact = exactRows[0]; + if (exact === undefined) return null; + const active = activeAttemptFromRow(exact); + if (active !== null) return { outcome: "active" as const, attempt: active }; + if ( + exact.status !== "queued" || + exact.attempt_id !== null || + exact.execution_status !== "queued" + ) { + return null; + } + + // The supported single queue worker serializes admissions. This transaction + // keeps the capacity observation and exact claim mutation indivisible. + const counts: { active_count: number }[] = await tx` + SELECT count(*)::int AS active_count + FROM workflow_runs AS wr + JOIN executions AS e ON e.delivery_id = wr.execution_delivery_id + WHERE wr.status = 'running' + AND wr.owner_kind = 'daemon' + AND wr.attempt_id IS NOT NULL + AND wr.owner_id = 'workflow-runner:' || wr.attempt_id::text + AND wr.lease_expires_at > now() + AND wr.attempt_deadline_at > now() + AND wr.attempt_completed_at IS NULL + AND e.status = 'running' + AND e.daemon_id = wr.owner_id + AND e.offer_id = wr.attempt_id + `; + if ((counts[0]?.active_count ?? 0) >= maxActive) { + return { outcome: "capacity" as const }; + } + + const attemptId = exact.dispatch_generation_id; + const runnerId = workflowRunnerId(attemptId); + const workflowRows: { id: string; attempt_deadline_at: Date }[] = await tx` + UPDATE workflow_runs + SET status = 'running', + owner_kind = 'daemon', + owner_id = ${runnerId}, + attempt_id = ${attemptId}, + attempt_deadline_at = now() + ${WORKFLOW_RUNNER_ATTEMPT_DEADLINE_MS} * interval '1 millisecond', + lease_expires_at = LEAST( + now() + ${startupLeaseMs} * interval '1 millisecond', + now() + ${WORKFLOW_RUNNER_ATTEMPT_DEADLINE_MS} * interval '1 millisecond' + ) + WHERE id = ${exact.id} + AND status = 'queued' + AND attempt_id IS NULL + RETURNING id, attempt_deadline_at + `; + if (workflowRows[0] === undefined) return null; + + const executionRows: { delivery_id: string }[] = await tx` + UPDATE executions + SET status = 'running', + daemon_id = ${runnerId}, + offer_id = ${attemptId}, + started_at = now() + WHERE delivery_id = ${exact.execution_delivery_id} + AND status = 'queued' + RETURNING delivery_id + `; + if (executionRows[0] === undefined) + throw new StaleWorkflowAttemptError({ + runId: exact.id, + attemptId, + }); + + return { + outcome: "claimed" as const, + attempt: { + runId: exact.id, + attemptId, + runnerId, + executionDeliveryId: exact.execution_delivery_id, + workflowName: exact.workflow_name as WorkflowRunnerAttempt["workflowName"], + attemptDeadlineAt: workflowRows[0].attempt_deadline_at, + }, + }; + }); + if (claim !== null) return claim; + } catch (err) { + const row = await loadExactJobRow(job, sql).catch(() => null); + const active = row === null ? null : activeAttemptFromRow(row); + if (active !== null) return { outcome: "active", attempt: active }; + throw err; + } + + const row = await loadExactJobRow(job, sql); + const active = row === null ? null : activeAttemptFromRow(row); + return active === null ? { outcome: "stale" } : { outcome: "active", attempt: active }; +} + +export type WorkflowRunnerRegistrationState = + | { + readonly state: "ready"; + readonly attempt: WorkflowRunnerAttempt; + readonly payloadIssuedAt: Date | null; + readonly tokenExpiresAt: Date | null; + } + | { + readonly state: "result-pending"; + readonly executionDeliveryId: string; + readonly payload: WorkflowRunnerResultPayload; + } + | { readonly state: "completed" } + | { readonly state: "invalid" }; + +/** Resolve registration from durable state, including post-terminal ACK replay. */ +export async function getWorkflowRunnerRegistrationState( + attempt: WorkflowAttempt, + sql: SQL = requireDb(), +): Promise { + const runnerId = workflowRunnerId(attempt.attemptId); + const rows: ExactRunnerRow[] = await sql` + SELECT wr.id, + wr.workflow_name, + wr.execution_delivery_id, + wr.status, + wr.attempt_id, + wr.owner_kind, + wr.owner_id, + wr.lease_expires_at, + wr.lease_expires_at > now() AS lease_active, + wr.attempt_deadline_at, + wr.attempt_deadline_at > now() AS attempt_deadline_active, + wr.attempt_completed_at, + wr.dispatch_generation_id, + wr.runner_payload_issued_at, + wr.runner_token_expires_at, + e.status AS execution_status, + e.daemon_id AS execution_daemon_id, + e.offer_id AS execution_offer_id, + e.result_processed_at, + e.workflow_result_payload + FROM workflow_runs AS wr + JOIN executions AS e ON e.delivery_id = wr.execution_delivery_id + WHERE wr.id = ${attempt.runId} + AND wr.attempt_id = ${attempt.attemptId} + AND wr.owner_id = ${runnerId} + AND e.daemon_id = ${runnerId} + AND e.offer_id = ${attempt.attemptId} + `; + const row = rows[0]; + if (row === undefined) return { state: "invalid" }; + const active = activeAttemptFromRow(row); + if (active !== null) { + return { + state: "ready", + attempt: active, + payloadIssuedAt: row.runner_payload_issued_at, + tokenExpiresAt: row.runner_token_expires_at, + }; + } + if (row.workflow_result_payload !== null) { + const payload = WorkflowRunnerResultPayloadSchema.safeParse(row.workflow_result_payload); + if (!payload.success) return { state: "invalid" }; + return row.result_processed_at === null + ? { + state: "result-pending", + executionDeliveryId: row.execution_delivery_id, + payload: payload.data, + } + : { state: "completed" }; + } + return { state: "invalid" }; +} + +/** Record the only runner credential delivery if its token ends within the attempt. */ +export async function recordWorkflowRunnerPayloadIssued( + attempt: WorkflowAttempt, + tokenExpiresAt: Date, + sql: SQL = requireDb(), +): Promise { + const runnerId = workflowRunnerId(attempt.attemptId); + const rows: { id: string }[] = await sql` + UPDATE workflow_runs + SET runner_payload_issued_at = now(), + runner_token_expires_at = ${tokenExpiresAt} + WHERE id = ${attempt.runId} + AND attempt_id = ${attempt.attemptId} + AND owner_id = ${runnerId} + AND status = 'running' + AND lease_expires_at > now() + AND attempt_deadline_at > now() + AND ${tokenExpiresAt} <= attempt_deadline_at + AND runner_payload_issued_at IS NULL + AND runner_token_expires_at IS NULL + RETURNING id + `; + return rows[0] !== undefined; +} + +export interface WorkflowRunnerCommandReceipt { + readonly commandKind: WorkflowRunnerCommand["type"]; + readonly request: WorkflowRunnerCommand; + readonly response: { readonly trackingCommentId?: number; readonly childRunId?: string }; +} + +export class WorkflowRunnerCommandConflictError extends Error { + constructor(commandId: string) { + super(`workflow runner command id was reused with different content: ${commandId}`); + this.name = "WorkflowRunnerCommandConflictError"; + } +} + +export async function findWorkflowRunnerCommandReceipt( + attempt: WorkflowAttempt, + commandId: string, + sql: SQL = requireDb(), +): Promise { + const rows: { + command_kind: WorkflowRunnerCommand["type"]; + request: WorkflowRunnerCommand; + response: WorkflowRunnerCommandReceipt["response"]; + }[] = await sql` + SELECT command_kind, request, response + FROM workflow_attempt_commands + WHERE attempt_id = ${attempt.attemptId} + AND command_id = ${commandId} + AND run_id = ${attempt.runId} + `; + const row = rows[0]; + return row === undefined + ? null + : { commandKind: row.command_kind, request: row.request, response: row.response }; +} + +function canonicalJson(value: unknown): unknown { + if (Array.isArray(value)) return value.map(canonicalJson); + if (value === null || typeof value !== "object") return value; + return Object.fromEntries( + Object.entries(value as Record) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, entry]) => [key, canonicalJson(entry)]), + ); +} + +function sameJson(left: unknown, right: unknown): boolean { + return JSON.stringify(canonicalJson(left)) === JSON.stringify(canonicalJson(right)); +} + +export function assertMatchingWorkflowRunnerCommand( + receipt: WorkflowRunnerCommandReceipt, + commandId: string, + command: WorkflowRunnerCommand, +): void { + if (receipt.commandKind !== command.type || !sameJson(receipt.request, command)) { + throw new WorkflowRunnerCommandConflictError(commandId); + } +} + +export async function insertWorkflowRunnerCommandReceipt( + attempt: WorkflowAttempt, + commandId: string, + command: WorkflowRunnerCommand, + response: WorkflowRunnerCommandReceipt["response"], + sql: SQL, +): Promise { + await sql` + INSERT INTO workflow_attempt_commands ( + attempt_id, command_id, run_id, command_kind, request, response + ) VALUES ( + ${attempt.attemptId}, ${commandId}, ${attempt.runId}, ${command.type}, + ${command}::jsonb, ${response}::jsonb + ) + ON CONFLICT (attempt_id, command_id) DO NOTHING + `; + const receipt = await findWorkflowRunnerCommandReceipt(attempt, commandId, sql); + if (receipt === null) throw new StaleWorkflowAttemptError(attempt); + assertMatchingWorkflowRunnerCommand(receipt, commandId, command); +} + +export type StoreWorkflowRunnerResultOutcome = "stored" | "already-stored"; + +interface StoredResultRow { + workflow_status: string; + workflow_state: Record; + attempt_completed_at: Date | null; + lease_expires_at: Date | null; + execution_status: string; + workflow_result_payload: unknown; +} + +/** Store the result and both terminal state transitions in one transaction. */ +export async function storeWorkflowRunnerResult( + payload: WorkflowRunnerResultPayload, + sql: SQL = requireDb(), +): Promise { + const attempt = { runId: payload.runId, attemptId: payload.attemptId }; + const runnerId = workflowRunnerId(payload.attemptId); + return sql.begin(async (tx) => { + const rows: StoredResultRow[] = await tx` + SELECT wr.status AS workflow_status, + wr.state AS workflow_state, + wr.attempt_completed_at, + wr.lease_expires_at, + e.status AS execution_status, + e.workflow_result_payload + FROM workflow_runs AS wr + JOIN executions AS e ON e.delivery_id = wr.execution_delivery_id + WHERE wr.id = ${payload.runId} + AND wr.attempt_id = ${payload.attemptId} + AND wr.owner_id = ${runnerId} + AND e.daemon_id = ${runnerId} + AND e.offer_id = ${payload.attemptId} + FOR UPDATE OF wr, e + `; + const row = rows[0]; + if (row === undefined) throw new StaleWorkflowAttemptError(attempt); + if (row.workflow_result_payload !== null) { + if (!sameJson(row.workflow_result_payload, payload)) { + throw new WorkflowRunnerCommandConflictError(payload.attemptId); + } + return "already-stored"; + } + + const result = payload.result; + if (result.status === "handed-off") { + if ( + row.workflow_status !== "running" || + row.attempt_completed_at === null || + row.lease_expires_at !== null || + row.workflow_state["handedOffTo"] !== result.childRunId + ) { + throw new StaleWorkflowAttemptError(attempt); + } + } else { + const state = + typeof result.state === "object" && result.state !== null + ? (result.state as Record) + : {}; + if (result.status === "succeeded") { + await markAttemptSucceeded(attempt, state, tx); + } else if (result.status === "incomplete") { + await markAttemptIncomplete(attempt, result.reason, state, tx); + } else { + await markAttemptFailed(attempt, result.reason, state, tx); + } + } + + const executionSucceeded = result.status === "succeeded" || result.status === "handed-off"; + const errorMessage = + result.status === "failed" || result.status === "incomplete" ? result.reason : null; + const executionRows: { delivery_id: string }[] = await tx` + UPDATE executions + SET status = ${executionSucceeded ? "completed" : "failed"}, + completed_at = now(), + duration_ms = ${payload.durationMs}, + error_message = ${errorMessage}, + workflow_result_payload = ${payload}::jsonb, + result_processed_at = NULL + WHERE daemon_id = ${runnerId} + AND offer_id = ${payload.attemptId} + AND status = 'running' + RETURNING delivery_id + `; + if (executionRows[0] === undefined) throw new StaleWorkflowAttemptError(attempt); + return "stored"; + }); +} + +export interface PendingWorkflowRunnerResult { + readonly runId: string; + readonly attemptId: string; + readonly executionDeliveryId: string; + readonly payload: WorkflowRunnerResultPayload; +} + +export type WorkflowRunnerResultProcessingState = "pending" | "processed" | "missing"; + +/** Read the durable projection receipt for one exact stored result. */ +export async function getWorkflowRunnerResultProcessingState( + pending: PendingWorkflowRunnerResult, + sql: SQL = requireDb(), +): Promise { + const rows: { + workflow_result_payload: unknown; + result_processed_at: Date | null; + }[] = await sql` + SELECT e.workflow_result_payload, e.result_processed_at + FROM executions AS e + JOIN workflow_runs AS wr ON wr.execution_delivery_id = e.delivery_id + WHERE wr.id = ${pending.runId} + AND wr.attempt_id = ${pending.attemptId} + AND wr.execution_delivery_id = ${pending.executionDeliveryId} + AND e.delivery_id = ${pending.executionDeliveryId} + AND e.offer_id = ${pending.attemptId} + `; + const row = rows[0]; + if (row === undefined || row.workflow_result_payload === null) return "missing"; + const stored = WorkflowRunnerResultPayloadSchema.parse(row.workflow_result_payload); + if (!sameJson(stored, pending.payload)) { + throw new WorkflowRunnerCommandConflictError(pending.attemptId); + } + return row.result_processed_at === null ? "pending" : "processed"; +} + +export async function findPendingWorkflowRunnerResults( + sql: SQL = requireDb(), + limit = 100, +): Promise { + const rows: { + run_id: string; + attempt_id: string; + delivery_id: string; + workflow_result_payload: unknown; + }[] = await sql` + SELECT wr.id AS run_id, wr.attempt_id, e.delivery_id, e.workflow_result_payload + FROM executions AS e + JOIN workflow_runs AS wr ON wr.execution_delivery_id = e.delivery_id + WHERE e.workflow_result_payload IS NOT NULL + AND e.result_processed_at IS NULL + AND wr.attempt_id = e.offer_id + ORDER BY e.completed_at + LIMIT ${limit} + `; + return rows.map((row) => ({ + runId: row.run_id, + attemptId: row.attempt_id, + executionDeliveryId: row.delivery_id, + payload: WorkflowRunnerResultPayloadSchema.parse(row.workflow_result_payload), + })); +} + +export async function markWorkflowRunnerResultProcessed( + attemptId: string, + sql: SQL = requireDb(), +): Promise { + const rows: { delivery_id: string }[] = await sql` + UPDATE executions + SET result_processed_at = now() + WHERE offer_id = ${attemptId} + AND workflow_result_payload IS NOT NULL + AND result_processed_at IS NULL + AND status IN ('completed', 'failed') + RETURNING delivery_id + `; + if (rows[0] !== undefined) return true; + const existing: { processed: boolean }[] = await sql` + SELECT result_processed_at IS NOT NULL AS processed + FROM executions + WHERE offer_id = ${attemptId} + AND workflow_result_payload IS NOT NULL + `; + return existing[0]?.processed === true; +} + +export async function findWorkflowRunnerCleanupCandidates( + sql: SQL = requireDb(), + limit = 100, +): Promise<{ runId: string; attemptId: string }[]> { + const rows: { id: string; attempt_id: string }[] = await sql` + SELECT wr.id, wr.attempt_id + FROM workflow_runs AS wr + JOIN executions AS e ON e.delivery_id = wr.execution_delivery_id + WHERE wr.attempt_id IS NOT NULL + AND wr.attempt_completed_at IS NOT NULL + AND wr.runner_resources_cleaned_at IS NULL + AND e.offer_id = wr.attempt_id + AND e.status IN ('completed', 'failed') + ORDER BY wr.attempt_completed_at + LIMIT ${limit} + `; + return rows.map((row) => ({ runId: row.id, attemptId: row.attempt_id })); +} + +export async function markWorkflowRunnerResourcesCleaned( + attempt: WorkflowAttempt, + sql: SQL = requireDb(), +): Promise { + const rows: { id: string }[] = await sql` + UPDATE workflow_runs + SET runner_resources_cleaned_at = now() + WHERE id = ${attempt.runId} + AND attempt_id = ${attempt.attemptId} + AND attempt_completed_at IS NOT NULL + AND runner_resources_cleaned_at IS NULL + RETURNING id + `; + if (rows[0] !== undefined) return true; + const existing: { cleaned: boolean }[] = await sql` + SELECT runner_resources_cleaned_at IS NOT NULL AS cleaned + FROM workflow_runs + WHERE id = ${attempt.runId} + AND attempt_id = ${attempt.attemptId} + `; + return existing[0]?.cleaned === true; +} + +export async function listActiveWorkflowRunnerAttempts( + sql: SQL = requireDb(), + limit = 100, +): Promise { + const rows: { + id: string; + attempt_id: string; + owner_id: string; + execution_delivery_id: string; + workflow_name: WorkflowRunnerAttempt["workflowName"]; + attempt_deadline_at: Date; + }[] = await sql` + SELECT id, attempt_id, owner_id, execution_delivery_id, workflow_name, + attempt_deadline_at + FROM workflow_runs + WHERE status = 'running' + AND attempt_id IS NOT NULL + AND owner_id = ${RUNNER_ID_PREFIX} || attempt_id::text + AND lease_expires_at > now() + AND attempt_deadline_at > now() + ORDER BY updated_at + LIMIT ${limit} + `; + return rows.map((row) => ({ + runId: row.id, + attemptId: row.attempt_id, + runnerId: row.owner_id, + executionDeliveryId: row.execution_delivery_id, + workflowName: row.workflow_name, + attemptDeadlineAt: row.attempt_deadline_at, + })); +} + +/** Fail one claimed runner attempt and its execution receipt atomically. */ +export async function failWorkflowRunnerAttempt( + attempt: WorkflowRunnerAttempt, + reason: string, + sql: SQL = requireDb(), +): Promise { + return sql.begin(async (tx) => { + const row = await markAttemptFailed( + { runId: attempt.runId, attemptId: attempt.attemptId }, + reason, + { phase: "runner-start-failed" }, + tx, + ); + const executions: { delivery_id: string }[] = await tx` + UPDATE executions + SET status = 'failed', + completed_at = now(), + error_message = ${reason}, + result_processed_at = now() + WHERE delivery_id = ${attempt.executionDeliveryId} + AND daemon_id = ${attempt.runnerId} + AND offer_id = ${attempt.attemptId} + AND status = 'running' + RETURNING delivery_id + `; + if (executions[0] === undefined) throw new StaleWorkflowAttemptError(attempt); + return row; + }); +} diff --git a/src/orchestrator/ws-connection.ts b/src/orchestrator/ws-connection.ts new file mode 100644 index 00000000..c7b84a0d --- /dev/null +++ b/src/orchestrator/ws-connection.ts @@ -0,0 +1,30 @@ +import type { ServerWebSocket } from "bun"; + +import { createMessageEnvelope } from "../shared/ws-messages"; + +/** Per-connection data attached during the HTTP WebSocket upgrade. */ +export interface WsConnectionData { + readonly authenticated: boolean; + readonly remoteAddr: string; + daemonId: string | undefined; + kind?: "daemon" | "workflow-runner"; + runnerRunId?: string; + runnerAttemptId?: string; + runnerRegistered?: boolean; +} + +/** Send a protocol error on a daemon WebSocket. */ +export function sendError( + ws: ServerWebSocket, + correlationId: string, + code: string, + message: string, +): void { + ws.sendText( + JSON.stringify({ + type: "error", + ...createMessageEnvelope(correlationId), + payload: { code, message }, + }), + ); +} diff --git a/src/orchestrator/ws-server.ts b/src/orchestrator/ws-server.ts index 585dcaec..c56fc6b0 100644 --- a/src/orchestrator/ws-server.ts +++ b/src/orchestrator/ws-server.ts @@ -1,29 +1,31 @@ import { timingSafeEqual } from "node:crypto"; import type { ServerWebSocket } from "bun"; +import { z } from "zod"; import { config } from "../config"; import { logger } from "../logger"; +import { workflowRunnerClientMessageSchema } from "../shared/workflow-runner-messages"; +import { daemonMessageSchema, WS_CLOSE_CODES, WS_ERROR_CODES } from "../shared/ws-messages"; import { - createMessageEnvelope, - daemonMessageSchema, - WS_CLOSE_CODES, - WS_ERROR_CODES, -} from "../shared/ws-messages"; -import { handleDaemonMessage, handleWsClose, handleWsOpen } from "./connection-handler"; + beginDaemonConnectionShutdown, + drainDisconnectCleanups, + handleDaemonMessage, + handleWsClose, + handleWsOpen, +} from "./connection-handler"; +import { + isWorkflowRunnerCapabilityValid, + parseWorkflowRunnerPath, +} from "./workflow-runner-capability"; +import { + handleWorkflowRunnerClose, + handleWorkflowRunnerMessage, + handleWorkflowRunnerOpen, +} from "./workflow-runner-controller"; +import { sendError, type WsConnectionData } from "./ws-connection"; -/** - * Per-connection data attached to each WebSocket via ws.data. - * Set during the HTTP upgrade in the fetch handler. - */ -export interface WsConnectionData { - /** Authenticated after token check in fetch handler. */ - authenticated: boolean; - /** Remote address for logging. */ - remoteAddr: string; - /** Daemon ID, set after daemon:register is processed. */ - daemonId: string | undefined; -} +export { sendError, type WsConnectionData } from "./ws-connection"; let server: ReturnType> | null = null; @@ -155,6 +157,11 @@ export function startWebSocketServer(): ReturnType({ port: config.wsPort, fetch(req, srv) { const url = new URL(req.url); - if (url.pathname !== "/ws") { + const runnerIdentity = parseWorkflowRunnerPath(url.pathname); + if (url.pathname !== "/ws" && runnerIdentity === null) { return new Response("Not Found", { status: 404 }); } @@ -178,7 +194,22 @@ export function startWebSocketServer(): ReturnType) { logger.info({ remoteAddr: ws.data.remoteAddr }, "WebSocket connection opened"); - handleWsOpen(ws); + if (ws.data.kind === "workflow-runner") handleWorkflowRunnerOpen(ws); + else handleWsOpen(ws); }, message(ws: ServerWebSocket, message: string | Buffer) { @@ -224,6 +264,16 @@ export function startWebSocketServer(): ReturnType, code: number, reason: string) { logger.info({ daemonId: ws.data.daemonId, code, reason }, "WebSocket connection closed"); - handleWsClose(ws, code, reason); + if (ws.data.kind === "workflow-runner") handleWorkflowRunnerClose(ws); + else handleWsClose(ws, code, reason); }, }, }); @@ -270,29 +321,12 @@ export async function stopWebSocketServer(): Promise { if (server !== null) { const stopping = server; server = null; + beginDaemonConnectionShutdown(); await Promise.race([ stopping.stop(true), new Promise((resolve) => setTimeout(resolve, STOP_DRAIN_TIMEOUT_MS)), ]); + await drainDisconnectCleanups(); logger.info("WebSocket server stopped"); } } - -/** - * Send an error message to a daemon WebSocket connection. - */ -export function sendError( - ws: ServerWebSocket, - correlationId: string, - code: string, - message: string, -): void { - const envelope = createMessageEnvelope(correlationId); - ws.sendText( - JSON.stringify({ - type: "error", - ...envelope, - payload: { code, message }, - }), - ); -} diff --git a/src/runner/main.ts b/src/runner/main.ts new file mode 100644 index 00000000..b6816b6a --- /dev/null +++ b/src/runner/main.ts @@ -0,0 +1,153 @@ +import { z } from "zod"; + +import { config } from "../config"; +import { assertDaemonEnvironmentPrivate } from "../daemon/process-boundary"; +import { installFatalHandlers, logger } from "../logger"; +import { revokeInstallationTokenValue } from "../orchestrator/installation-token"; +import type { WorkflowRunnerPayload } from "../shared/workflow-runner-messages"; +import type { HandlerResult } from "../shared/workflow-types"; +import { redactErrorMessageOrFallback } from "../utils/log-redaction"; +import { StaleWorkflowAttemptError } from "../workflows/runs-store"; +import { + assertCloudMetadataUnavailable, + assertWorkflowRunnerEnvironment, +} from "./process-boundary"; +import { createWorkflowRunnerDeadline } from "./token-deadline"; +import { executeWorkflowRunnerJob } from "./workflow-executor"; +import { WorkflowRunnerClient } from "./ws-client"; + +function requiredEnv(value: string | undefined, name: string): string { + value = value?.trim(); + if (value === undefined || value === "") throw new Error(`${name} is required`); + return value; +} + +export function resultForWorkflowRunnerExecutionError(input: { + readonly err: unknown; + readonly shuttingDown: boolean; + readonly tokenDeadlineExpired: boolean; +}): HandlerResult | null { + if (input.shuttingDown) return null; + if (input.tokenDeadlineExpired) { + return { + status: "failed", + reason: "Workflow runner execution deadline reached", + humanMessage: "Workflow runner stopped at its credential or attempt deadline.", + }; + } + return { + status: "failed", + reason: redactErrorMessageOrFallback(input.err, "workflow runner failed"), + humanMessage: "workflow runner failed, see server logs for details.", + }; +} + +export async function executeAndReportWorkflowRunnerJob(input: { + readonly job: WorkflowRunnerPayload; + readonly client: WorkflowRunnerClient; + readonly runId: string; + readonly attemptId: string; +}): Promise { + const { job, client, runId, attemptId } = input; + const startedAt = Date.now(); + const tokenDeadline = createWorkflowRunnerDeadline( + job.installationTokenExpiresAt, + job.attemptDeadlineAt, + ); + const executionSignal = AbortSignal.any([client.signal, tokenDeadline.signal]); + let revocationAttempted = false; + const revokeRunnerToken = async (): Promise => { + if (revocationAttempted) return; + revocationAttempted = true; + await revokeInstallationTokenValue(job.installationToken, logger, { + attemptId, + owner: "workflow-runner", + }); + }; + try { + let result: HandlerResult; + try { + result = await executeWorkflowRunnerJob(job, client, executionSignal); + } catch (err) { + const failure = resultForWorkflowRunnerExecutionError({ + err, + shuttingDown: client.signal.aborted || err instanceof StaleWorkflowAttemptError, + tokenDeadlineExpired: tokenDeadline.signal.aborted, + }); + if (failure === null) { + logger.warn({ runId, attemptId }, "Workflow runner fenced before completion"); + return; + } + result = failure; + } + + if (result.status === "handed-off") { + await revokeRunnerToken(); + return; + } + + await revokeRunnerToken(); + + try { + await client.sendResultUntilAck({ + runId, + attemptId, + result, + durationMs: Date.now() - startedAt, + }); + } catch (err) { + if (!client.signal.aborted) throw err; + logger.warn({ runId, attemptId }, "Workflow runner fenced before result acknowledgement"); + } + } finally { + tokenDeadline.cancel(); + await revokeRunnerToken(); + client.close(); + } +} + +export async function main(): Promise { + assertWorkflowRunnerEnvironment(); + assertDaemonEnvironmentPrivate(); + await assertCloudMetadataUnavailable(); + installFatalHandlers("workflow-runner"); + + if (!config.workflowRunner) throw new Error("WORKFLOW_RUNNER=true is required"); + const runId = z + .uuid() + .parse(requiredEnv(process.env["WORKFLOW_RUNNER_RUN_ID"], "WORKFLOW_RUNNER_RUN_ID")); + const attemptId = z + .uuid() + .parse(requiredEnv(process.env["WORKFLOW_RUNNER_ATTEMPT_ID"], "WORKFLOW_RUNNER_ATTEMPT_ID")); + const token = requiredEnv(process.env["WORKFLOW_RUNNER_TOKEN"], "WORKFLOW_RUNNER_TOKEN"); + const url = requiredEnv(process.env["ORCHESTRATOR_URL"], "ORCHESTRATOR_URL"); + + const client = new WorkflowRunnerClient({ url, token, runId, attemptId }); + const stop = (signal: string): void => { + logger.warn({ signal, runId, attemptId }, "Workflow runner stopping"); + client.cancel(new Error(`${signal} received`)); + }; + process.on("SIGTERM", () => { + stop("SIGTERM"); + }); + process.on("SIGINT", () => { + stop("SIGINT"); + }); + + client.connect(); + const job = await client.waitForJob(); + if (job === null) { + client.close(); + return; + } + client.addSensitiveValue(job.installationToken); + + await executeAndReportWorkflowRunnerJob({ job, client, runId, attemptId }); +} + +if (import.meta.main) { + void main().catch((err: unknown) => { + logger.error({ err }, "Workflow runner startup failed"); + process.exit(1); + }); +} diff --git a/src/runner/output-sanitizer.ts b/src/runner/output-sanitizer.ts new file mode 100644 index 00000000..5f486d13 --- /dev/null +++ b/src/runner/output-sanitizer.ts @@ -0,0 +1,6 @@ +export { + configuredCredentialValues, + containsExactCredentialPropertyName, + containsExactCredentialValue, + redactExactCredentialValues as redactExactValues, +} from "../utils/exact-credential-redaction"; diff --git a/src/runner/process-boundary.ts b/src/runner/process-boundary.ts new file mode 100644 index 00000000..822a07bf --- /dev/null +++ b/src/runner/process-boundary.ts @@ -0,0 +1,60 @@ +import { assertExactWorkflowRunnerProviderEnvironment } from "../shared/workflow-runner-provider"; + +export const FORBIDDEN_RUNNER_ENV = [ + "DATABASE_URL", + "VALKEY_URL", + "REDIS_URL", + "DAEMON_AUTH_TOKEN", + "DAEMON_AUTH_TOKEN_PREVIOUS", + "WORKFLOW_RUNNER_CAPABILITY_SECRET", + "WORKFLOW_RUNNER_CAPABILITY_SECRET_PREVIOUS", + "GITHUB_APP_ID", + "GITHUB_APP_PRIVATE_KEY", + "GITHUB_WEBHOOK_SECRET", + "GITHUB_PERSONAL_ACCESS_TOKEN", + "GH_TOKEN", + "GITHUB_TOKEN", + "GH_ENTERPRISE_TOKEN", + "GITHUB_ENTERPRISE_TOKEN", + "KUBECONFIG", + "CONTEXT7_API_KEY", +] as const; + +const CLOUD_METADATA_ENDPOINTS = [ + "http://169.254.169.254/", + "http://[fd00:ec2::254]/", + "http://[fd20:ce::254]/", +] as const; + +/** Fail closed when the provider-only runner Secret contains controller authority. */ +export function assertWorkflowRunnerEnvironment(): void { + const leaked = FORBIDDEN_RUNNER_ENV.filter((name) => (process.env[name]?.trim().length ?? 0) > 0); + if (leaked.length > 0) { + throw new Error( + `workflow runner received forbidden controller environment: ${leaked.join(", ")}`, + ); + } + assertExactWorkflowRunnerProviderEnvironment(process.env); +} + +/** Fail before registration when a Pod can reach a node metadata service. */ +export async function assertCloudMetadataUnavailable(): Promise { + const reachable = await Promise.all( + CLOUD_METADATA_ENDPOINTS.map(async (endpoint) => { + try { + const response = await fetch(endpoint, { + redirect: "manual", + signal: AbortSignal.timeout(750), + }); + await response.body?.cancel(); + return endpoint; + } catch { + return null; + } + }), + ); + const endpoint = reachable.find((value) => value !== null); + if (endpoint !== undefined) { + throw new Error(`workflow runner can reach forbidden cloud metadata endpoint ${endpoint}`); + } +} diff --git a/src/runner/token-deadline.ts b/src/runner/token-deadline.ts new file mode 100644 index 00000000..066bacf7 --- /dev/null +++ b/src/runner/token-deadline.ts @@ -0,0 +1,46 @@ +export const INSTALLATION_TOKEN_EXPIRY_BUFFER_MS = 5 * 60_000; + +export class WorkflowRunnerDeadlineError extends Error { + constructor() { + super("Workflow runner execution deadline reached"); + this.name = "WorkflowRunnerDeadlineError"; + } +} + +export function workflowRunnerDeadlineDelayMs( + tokenExpiresAt: string, + attemptDeadlineAt: string, + now = Date.now(), +): number { + const tokenExpiresAtMs = Date.parse(tokenExpiresAt); + if (!Number.isFinite(tokenExpiresAtMs)) throw new Error("Invalid installation token expiry"); + const attemptDeadlineAtMs = Date.parse(attemptDeadlineAt); + if (!Number.isFinite(attemptDeadlineAtMs)) throw new Error("Invalid workflow attempt deadline"); + const deadlineAtMs = Math.min( + tokenExpiresAtMs - INSTALLATION_TOKEN_EXPIRY_BUFFER_MS, + attemptDeadlineAtMs, + ); + return Math.max(0, deadlineAtMs - now); +} + +export function createWorkflowRunnerDeadline( + tokenExpiresAt: string, + attemptDeadlineAt: string, + now = Date.now(), +): { readonly signal: AbortSignal; cancel: () => void } { + const controller = new AbortController(); + const delay = workflowRunnerDeadlineDelayMs(tokenExpiresAt, attemptDeadlineAt, now); + if (delay === 0) { + controller.abort(new WorkflowRunnerDeadlineError()); + return { signal: controller.signal, cancel: () => undefined }; + } + const timer = setTimeout(() => { + controller.abort(new WorkflowRunnerDeadlineError()); + }, delay); + return { + signal: controller.signal, + cancel: (): void => { + clearTimeout(timer); + }, + }; +} diff --git a/src/runner/workflow-executor.ts b/src/runner/workflow-executor.ts new file mode 100644 index 00000000..ede55f2d --- /dev/null +++ b/src/runner/workflow-executor.ts @@ -0,0 +1,72 @@ +import { Octokit } from "octokit"; + +import { logger } from "../logger"; +import type { SerializableBotContext } from "../shared/daemon-types"; +import type { WorkflowRunnerPayload } from "../shared/workflow-runner-messages"; +import { + type HandlerResult, + HandlerResultSchema, + workflowRunnerId, +} from "../shared/workflow-types"; +import { getByName, type WorkflowRunContext } from "../workflows/registry"; +import type { WorkflowRunnerClient } from "./ws-client"; + +/** Execute one handler without direct access to PostgreSQL, Valkey, or Kubernetes. */ +export async function executeWorkflowRunnerJob( + job: WorkflowRunnerPayload, + client: WorkflowRunnerClient, + signal: AbortSignal, +): Promise { + const context = job.context as unknown as SerializableBotContext; + const workflowRun = job.workflowRun; + const target = { + type: context.isPR ? ("pr" as const) : ("issue" as const), + owner: context.owner, + repo: context.repo, + number: context.entityNumber, + }; + const log = logger.child({ + workflowRunId: workflowRun.runId, + workflowName: workflowRun.workflowName, + attemptId: client.attemptId, + deliveryId: context.deliveryId, + target, + }); + const runContext: WorkflowRunContext = { + runId: workflowRun.runId, + workflowName: workflowRun.workflowName, + target, + ...(workflowRun.parentRunId !== undefined && workflowRun.parentStepIndex !== undefined + ? { parent: { runId: workflowRun.parentRunId, stepIndex: workflowRun.parentStepIndex } } + : {}), + logger: log, + octokit: new Octokit({ auth: job.installationToken }), + deliveryId: context.deliveryId, + daemonId: workflowRunnerId(client.attemptId), + signal, + handOffChild: async (input) => { + const response = await client.command({ type: "hand-off-child", ...input }); + if (response.childRunId === undefined) { + throw new Error("Controller acknowledged hand-off without a child run id"); + } + return { childRunId: response.childRunId }; + }, + ...(job.reviewLearnings !== undefined ? { reviewLearnings: job.reviewLearnings } : {}), + ...(job.repoMemory !== undefined ? { repoMemory: job.repoMemory } : {}), + ...(job.policy !== undefined ? { policy: job.policy } : {}), + ...(job.maxTurns !== undefined ? { maxTurns: job.maxTurns } : {}), + ...(job.priorPlanState !== undefined ? { priorPlanState: job.priorPlanState } : {}), + ...(job.shipStepRuns !== undefined ? { shipStepRuns: job.shipStepRuns } : {}), + setState: async (state, humanMessage) => { + const patch = + typeof state === "object" && state !== null + ? (state as Record) + : { state }; + return client.command({ type: "set-state", patch, humanMessage }); + }, + }; + signal.throwIfAborted(); + const result = await getByName(workflowRun.workflowName).handler(runContext); + signal.throwIfAborted(); + return HandlerResultSchema.parse(result); +} diff --git a/src/runner/ws-client.ts b/src/runner/ws-client.ts new file mode 100644 index 00000000..41cfde72 --- /dev/null +++ b/src/runner/ws-client.ts @@ -0,0 +1,469 @@ +import { logger } from "../logger"; +import { + WORKFLOW_RUNNER_PROTOCOL_VERSION, + type WorkflowRunnerCommand, + WorkflowRunnerCommandSchema, + type WorkflowRunnerPayload, + type WorkflowRunnerResultPayload, + WorkflowRunnerResultPayloadSchema, + type WorkflowRunnerServerMessage, + workflowRunnerServerMessageSchema, +} from "../shared/workflow-runner-messages"; +import { createMessageEnvelope } from "../shared/ws-messages"; +import { StaleWorkflowAttemptError } from "../workflows/runs-store"; +import { + configuredCredentialValues, + containsExactCredentialPropertyName, + containsExactCredentialValue, + redactExactValues, +} from "./output-sanitizer"; + +const APP_VERSION: string = ((): string => { + try { + // eslint-disable-next-line @typescript-eslint/no-require-imports -- package metadata is synchronous + return (require("../../package.json") as { version: string }).version; + } catch { + return "0.0.0"; + } +})(); + +interface PendingCommand { + readonly message: { + readonly type: "workflow-runner:command"; + readonly id: string; + readonly timestamp: number; + readonly payload: { + readonly runId: string; + readonly attemptId: string; + readonly command: WorkflowRunnerCommand; + }; + }; + readonly resolve: (value: { trackingCommentId?: number; childRunId?: string }) => void; + readonly reject: (reason: Error) => void; + retryTimer: ReturnType | null; +} + +interface PendingResult { + readonly message: { + readonly type: "workflow-runner:result"; + readonly id: string; + readonly timestamp: number; + readonly payload: WorkflowRunnerResultPayload; + }; + readonly resolve: () => void; + readonly reject: (reason: Error) => void; + retryTimer: ReturnType | null; +} + +export interface WorkflowRunnerClientOptions { + readonly url: string; + readonly token: string; + readonly runId: string; + readonly attemptId: string; +} + +export class WorkflowRunnerClient { + private ws: WebSocket | null = null; + private closed = false; + private registered = false; + private reconnectTimer: ReturnType | null = null; + private reconnectMs = 1_000; + private heartbeatTimer: ReturnType | null = null; + private fenceTimer: ReturnType | null = null; + private retryMs = 1_000; + private clientFenceMs = 30_000; + private jobSettled = false; + private readonly pendingCommands = new Map(); + private pendingResult: PendingResult | null = null; + private readonly abortController = new AbortController(); + private readonly sensitiveValues = new Set(configuredCredentialValues()); + private readonly jobPromise: Promise; + private resolveJob: (job: WorkflowRunnerPayload | null) => void = () => undefined; + + constructor(private readonly options: WorkflowRunnerClientOptions) { + this.addSensitiveValue(options.token); + this.jobPromise = new Promise((resolve) => { + this.resolveJob = resolve; + }); + } + + get signal(): AbortSignal { + return this.abortController.signal; + } + + get attemptId(): string { + return this.options.attemptId; + } + + addSensitiveValue(value: string): void { + if (value.length >= 8) this.sensitiveValues.add(value); + } + + connect(): void { + if (this.closed) return; + let socket: WebSocket; + try { + socket = new WebSocket(this.options.url, { + headers: { Authorization: `Bearer ${this.options.token}` }, + }); + } catch (err) { + logger.warn({ err }, "Workflow runner connection creation failed"); + this.scheduleReconnect(); + return; + } + this.ws = socket; + socket.onopen = (): void => { + if (this.ws !== socket) return; + this.registered = false; + this.reconnectMs = 1_000; + this.send({ + type: "workflow-runner:register", + ...createMessageEnvelope(), + payload: { + runId: this.options.runId, + attemptId: this.options.attemptId, + protocolVersion: WORKFLOW_RUNNER_PROTOCOL_VERSION, + appVersion: APP_VERSION, + needsJob: !this.jobSettled, + }, + }); + }; + socket.onmessage = (event: MessageEvent): void => { + if (this.ws !== socket) return; + this.handleRawMessage(typeof event.data === "string" ? event.data : String(event.data)); + }; + socket.onerror = (): void => { + logger.warn({ attemptId: this.options.attemptId }, "Workflow runner WebSocket error"); + }; + socket.onclose = (event: CloseEvent): void => { + if (this.ws !== socket) return; + this.ws = null; + this.registered = false; + this.stopHeartbeatSender(); + if (event.code === 1008) { + this.abort(new Error(`Workflow runner connection rejected: ${event.reason}`)); + } else if (!this.closed) { + this.scheduleReconnect(); + } + }; + } + + waitForJob(): Promise { + return this.jobPromise; + } + + command( + command: WorkflowRunnerCommand, + ): Promise<{ trackingCommentId?: number; childRunId?: string }> { + const unavailable = this.unavailableReason(); + if (unavailable !== null) return Promise.reject(unavailable); + const sensitiveValues = [...this.sensitiveValues]; + if ( + containsExactCredentialPropertyName(command, sensitiveValues) || + containsExactCredentialValue(command, sensitiveValues) + ) { + return Promise.reject(new Error("Workflow runner command was rejected by credential policy")); + } + const parsed = WorkflowRunnerCommandSchema.safeParse( + redactExactValues(command, sensitiveValues), + ); + if (!parsed.success) { + return Promise.reject(new Error("Workflow runner command was rejected by credential policy")); + } + const commandId = crypto.randomUUID(); + const message = { + type: "workflow-runner:command" as const, + ...createMessageEnvelope(commandId), + payload: { + runId: this.options.runId, + attemptId: this.options.attemptId, + command: parsed.data, + }, + }; + return new Promise((resolve, reject) => { + const pending: PendingCommand = { message, resolve, reject, retryTimer: null }; + this.pendingCommands.set(commandId, pending); + this.sendIfReady(message); + }); + } + + sendResultUntilAck(payload: WorkflowRunnerResultPayload): Promise { + const unavailable = this.unavailableReason(); + if (unavailable !== null) return Promise.reject(unavailable); + if (this.pendingResult !== null) { + throw new Error("Workflow runner already has a pending terminal result"); + } + const sensitiveValues = [...this.sensitiveValues]; + const parsed = + containsExactCredentialPropertyName(payload, sensitiveValues) || + containsExactCredentialValue(payload, sensitiveValues) + ? { success: false as const } + : WorkflowRunnerResultPayloadSchema.safeParse(redactExactValues(payload, sensitiveValues)); + const sanitizedPayload: WorkflowRunnerResultPayload = parsed.success + ? parsed.data + : { + runId: payload.runId, + attemptId: payload.attemptId, + durationMs: payload.durationMs, + result: { + status: "failed", + reason: "workflow runner output was rejected by credential policy", + humanMessage: "Workflow output was rejected by the credential safety boundary.", + }, + }; + const message = { + type: "workflow-runner:result" as const, + ...createMessageEnvelope(payload.attemptId), + payload: sanitizedPayload, + }; + return new Promise((resolve, reject) => { + this.pendingResult = { message, resolve, reject, retryTimer: null }; + this.sendIfReady(message); + this.armResultRetry(); + }); + } + + close(): void { + if (this.closed) return; + const reason = new Error("Workflow runner client closed"); + this.closed = true; + this.stopActivity(); + this.settleJob(null); + this.rejectPending(reason); + const socket = this.ws; + this.ws = null; + socket?.close(1000, "workflow runner complete"); + } + + cancel(reason: Error): void { + this.abort(reason); + } + + private handleRawMessage(raw: string): void { + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + this.abort(new Error("Workflow runner received invalid JSON")); + return; + } + const result = workflowRunnerServerMessageSchema.safeParse(parsed); + if (!result.success) { + this.abort(new Error("Workflow runner received a schema-invalid message")); + return; + } + this.handleMessage(result.data); + } + + private handleMessage(message: WorkflowRunnerServerMessage): void { + switch (message.type) { + case "workflow-runner:registered": + this.handleRegistered(message); + break; + case "workflow-runner:heartbeat-ack": + if (!message.payload.renewed) { + this.abort(new StaleWorkflowAttemptError(this.options)); + } else { + this.armFence(); + } + break; + case "workflow-runner:command-result": + this.handleCommandResult(message); + break; + case "workflow-runner:result-ack": + this.completeResult(); + break; + case "workflow-runner:cancel": + this.abort(new Error(message.payload.reason)); + break; + case "workflow-runner:error": + logger.warn( + { code: message.payload.code, message: message.payload.message }, + "Workflow runner controller error", + ); + break; + } + } + + private handleRegistered( + message: Extract, + ): void { + if (message.payload.state === "completed") { + for (const [commandId, pending] of this.pendingCommands) { + if (pending.message.payload.command.type !== "hand-off-child") continue; + if (pending.retryTimer !== null) clearTimeout(pending.retryTimer); + this.pendingCommands.delete(commandId); + pending.resolve({ childRunId: commandId }); + } + if (!this.jobSettled) { + this.jobSettled = true; + this.resolveJob(null); + } + this.completeResult(); + return; + } + + this.registered = true; + this.retryMs = Math.max(1_000, Math.min(message.payload.heartbeatIntervalMs, 10_000)); + this.clientFenceMs = message.payload.clientFenceMs; + this.startHeartbeat(message.payload.heartbeatIntervalMs, message.payload.clientFenceMs); + if (!this.jobSettled) { + if (message.payload.job === undefined) { + this.abort(new Error("Workflow runner controller omitted the initial job payload")); + return; + } + this.jobSettled = true; + this.resolveJob(message.payload.job); + } + for (const pending of this.pendingCommands.values()) this.send(pending.message); + if (this.pendingResult !== null) this.send(this.pendingResult.message); + } + + private handleCommandResult( + message: Extract, + ): void { + const pending = this.pendingCommands.get(message.id); + if (pending === undefined) return; + if (message.payload.ok) { + if (pending.retryTimer !== null) clearTimeout(pending.retryTimer); + this.pendingCommands.delete(message.id); + const response: { trackingCommentId?: number; childRunId?: string } = {}; + if (message.payload.result.trackingCommentId !== undefined) { + response.trackingCommentId = message.payload.result.trackingCommentId; + } + if (message.payload.result.childRunId !== undefined) { + response.childRunId = message.payload.result.childRunId; + } + pending.resolve(response); + return; + } + if (message.payload.code === "INTERNAL_ERROR") { + pending.retryTimer ??= setTimeout(() => { + pending.retryTimer = null; + this.sendIfReady(pending.message); + }, this.retryMs); + return; + } + this.pendingCommands.delete(message.id); + const error = + message.payload.code === "STALE_ATTEMPT" + ? new StaleWorkflowAttemptError(this.options) + : new Error(message.payload.message); + pending.reject(error); + if (message.payload.code === "STALE_ATTEMPT") this.abort(error); + } + + private completeResult(): void { + const pending = this.pendingResult; + if (pending === null) return; + if (pending.retryTimer !== null) clearTimeout(pending.retryTimer); + this.pendingResult = null; + pending.resolve(); + } + + private armResultRetry(): void { + const pending = this.pendingResult; + if (pending?.retryTimer !== null) return; + pending.retryTimer = setTimeout(() => { + pending.retryTimer = null; + if (this.pendingResult !== pending) return; + this.sendIfReady(pending.message); + this.armResultRetry(); + }, this.retryMs); + } + + private startHeartbeat(intervalMs: number, fenceMs: number): void { + this.stopHeartbeatSender(); + this.armFence(fenceMs); + this.heartbeatTimer = setInterval(() => { + this.sendIfReady({ + type: "workflow-runner:heartbeat", + ...createMessageEnvelope(), + payload: { runId: this.options.runId, attemptId: this.options.attemptId }, + }); + }, intervalMs); + } + + private armFence(fenceMs?: number): void { + if (this.fenceTimer !== null) clearTimeout(this.fenceTimer); + const delay = fenceMs ?? this.clientFenceMs; + this.fenceTimer = setTimeout(() => { + this.abort(new Error("Workflow runner lease acknowledgement watchdog expired")); + }, delay); + } + + private stopHeartbeatSender(): void { + if (this.heartbeatTimer !== null) clearInterval(this.heartbeatTimer); + this.heartbeatTimer = null; + } + + private clearFence(): void { + if (this.fenceTimer !== null) clearTimeout(this.fenceTimer); + this.fenceTimer = null; + } + + private stopActivity(): void { + this.stopHeartbeatSender(); + this.clearFence(); + if (this.reconnectTimer !== null) clearTimeout(this.reconnectTimer); + this.reconnectTimer = null; + } + + private sendIfReady(message: unknown): void { + if (this.registered) this.send(message); + } + + private send(message: unknown): boolean { + if (this.ws?.readyState !== WebSocket.OPEN) return false; + this.ws.send(JSON.stringify(message)); + return true; + } + + private scheduleReconnect(): void { + if (this.closed || this.reconnectTimer !== null) return; + const delay = this.reconnectMs; + this.reconnectMs = Math.min(this.reconnectMs * 2, 30_000); + this.reconnectTimer = setTimeout(() => { + this.reconnectTimer = null; + this.connect(); + }, delay); + } + + private abort(reason: Error): void { + if (this.closed) return; + this.closed = true; + this.abortController.abort(reason); + this.stopActivity(); + this.settleJob(null); + this.rejectPending(reason); + const socket = this.ws; + this.ws = null; + socket?.close(1000, "workflow runner aborted"); + } + + private settleJob(job: WorkflowRunnerPayload | null): void { + if (this.jobSettled) return; + this.jobSettled = true; + this.resolveJob(job); + } + + private rejectPending(reason: Error): void { + for (const pending of this.pendingCommands.values()) { + if (pending.retryTimer !== null) clearTimeout(pending.retryTimer); + pending.reject(reason); + } + this.pendingCommands.clear(); + const result = this.pendingResult; + if (result !== null) { + if (result.retryTimer !== null) clearTimeout(result.retryTimer); + this.pendingResult = null; + result.reject(reason); + } + } + + private unavailableReason(): Error | null { + if (!this.closed) return null; + const reason: unknown = this.abortController.signal.reason; + return reason instanceof Error ? reason : new Error("Workflow runner client is closed"); + } +} diff --git a/src/shared/daemon-types.ts b/src/shared/daemon-types.ts index f83c2bb8..8e030cde 100644 --- a/src/shared/daemon-types.ts +++ b/src/shared/daemon-types.ts @@ -1,7 +1,6 @@ import { z } from "zod"; import type { BotContext } from "../types"; -import type { WorkflowRunRef } from "./workflow-types"; // Zod schemas (validated at boundary, types inferred below) @@ -131,13 +130,10 @@ export interface PendingOffer { triggerUsername: string; labels: string[]; triggerBodyPreview: string; - /** Present when the offered job is a workflow run. Forwarded into the - * `job:payload` so the daemon can route to the workflow executor. */ - workflowRun?: WorkflowRunRef; /** Present when the offered job is one of the four scoped variants. Carries * the original `ScopedQueuedJob` payload verbatim so reject/timeout can * reconstruct the queue entry without lossy field copying, and so the - * `scoped-job-completion` handler can format the user-facing reply against + * `scoped-job:completion` handler can format the user-facing reply against * the same context. Typed loosely (`unknown`) here to avoid a circular * import between `shared/daemon-types` and `orchestrator/job-queue`; callers * narrow via a type-guard import from `job-queue`. */ diff --git a/src/shared/dispatch-types.ts b/src/shared/dispatch-types.ts index 9ddfa013..9f0a3130 100644 --- a/src/shared/dispatch-types.ts +++ b/src/shared/dispatch-types.ts @@ -1,15 +1,14 @@ import { z } from "zod"; /** - * DispatchTarget: after the daemon-only collapse, every job goes through the - * daemon WebSocket protocol. The value is retained as a singleton rather than - * removed entirely so DB rows, log lines, and the `ws-messages.ts` schema stay - * stable across future extensions. + * DispatchTarget records the execution protocol selected for an execution. + * Shared jobs use the daemon WebSocket; structured workflows use one isolated + * workflow-runner Pod. * * The Postgres `executions.dispatch_target` and `triage_results.mode` CHECK - * constraints mirror this list (see migration `004_collapse_dispatch_to_daemon.sql`). + * constraints mirror this list (see migration `017_workflow_run_leases.sql`). */ -export const DISPATCH_TARGETS = ["daemon"] as const; +export const DISPATCH_TARGETS = ["daemon", "workflow-runner"] as const; export type DispatchTarget = (typeof DISPATCH_TARGETS)[number]; @@ -45,12 +44,14 @@ export function isDispatchTarget(value: unknown): value is DispatchTarget { * ephemeral-daemon-triage : triage flagged the request as heavy, ephemeral daemon spawned * ephemeral-daemon-overflow: persistent queue at/above threshold, ephemeral daemon spawned * ephemeral-spawn-failed : spawn was required but the K8s API call failed + * workflow-runner : structured workflow claimed by an isolated runner Pod */ export const DISPATCH_REASONS = [ "persistent-daemon", "ephemeral-daemon-triage", "ephemeral-daemon-overflow", "ephemeral-spawn-failed", + "workflow-runner", ] as const; export type DispatchReason = (typeof DISPATCH_REASONS)[number]; diff --git a/src/shared/workflow-runner-messages.ts b/src/shared/workflow-runner-messages.ts new file mode 100644 index 00000000..7fcdc333 --- /dev/null +++ b/src/shared/workflow-runner-messages.ts @@ -0,0 +1,222 @@ +import { z } from "zod"; + +import { + HandlerResultSchema, + PriorPlanStateSchema, + RepoMemoryEntrySchema, + WORKFLOW_RUNNER_HUMAN_MESSAGE_MAX_CHARS, + WorkflowNameSchema, + WorkflowRunRefSchema, + WorkflowRunSnapshotSchema, +} from "./workflow-types"; +import { AgentPolicySchema, ReviewLearningPayloadSchema } from "./ws-messages"; + +const envelope = { + id: z.uuid(), + timestamp: z.number(), +}; + +const attemptIdentity = { + runId: z.uuid(), + attemptId: z.uuid(), +}; + +const CONTROLLER_RESERVED_STATE_KEYS = ["_configNotice", "_lastHumanMessage"] as const; + +function containsControllerReservedState(value: unknown): boolean { + return ( + value !== null && + typeof value === "object" && + !Array.isArray(value) && + CONTROLLER_RESERVED_STATE_KEYS.some((key) => Object.hasOwn(value, key)) + ); +} + +const runnerStatePatchSchema = z + .record(z.string(), z.unknown()) + .refine( + (value) => !containsControllerReservedState(value), + "workflow runner state contains a controller-reserved key", + ); + +export const WORKFLOW_RUNNER_MESSAGE_MAX_BYTES = 900_000; + +function fitsWorkflowRunnerMessageBudget(value: unknown): boolean { + try { + return Buffer.byteLength(JSON.stringify(value), "utf8") <= WORKFLOW_RUNNER_MESSAGE_MAX_BYTES; + } catch { + return false; + } +} + +const targetSchema = z.object({ + type: z.enum(["issue", "pr"]), + owner: z.string().min(1), + repo: z.string().min(1), + number: z.number().int().positive(), +}); + +export const WorkflowRunnerPayloadSchema = z.object({ + context: z.record(z.string(), z.unknown()), + installationToken: z.string().min(1), + installationTokenExpiresAt: z.iso.datetime(), + attemptDeadlineAt: z.iso.datetime(), + repoMemory: z.array(RepoMemoryEntrySchema).max(50).optional(), + maxTurns: z.number().int().positive().optional(), + reviewLearnings: z.array(ReviewLearningPayloadSchema).max(50).optional(), + policy: AgentPolicySchema.optional(), + workflowRun: WorkflowRunRefSchema, + priorPlanState: PriorPlanStateSchema.optional(), + shipStepRuns: z.partialRecord(WorkflowNameSchema, WorkflowRunSnapshotSchema).optional(), +}); +export type WorkflowRunnerPayload = z.infer; + +const runnerRegisteredSchema = z.object({ + type: z.literal("workflow-runner:registered"), + ...envelope, + payload: z.discriminatedUnion("state", [ + z.object({ + state: z.literal("ready"), + heartbeatIntervalMs: z.number().int().positive(), + clientFenceMs: z.number().int().positive(), + dbLeaseMs: z.number().int().positive(), + job: WorkflowRunnerPayloadSchema.optional(), + }), + z.object({ state: z.literal("completed") }), + ]), +}); + +const runnerHeartbeatAckSchema = z.object({ + type: z.literal("workflow-runner:heartbeat-ack"), + ...envelope, + payload: z.object({ renewed: z.boolean() }), +}); + +const runnerCommandResultSchema = z.object({ + type: z.literal("workflow-runner:command-result"), + ...envelope, + payload: z.discriminatedUnion("ok", [ + z.object({ + ok: z.literal(true), + result: z.object({ + trackingCommentId: z.number().int().positive().optional(), + childRunId: z.uuid().optional(), + }), + }), + z.object({ + ok: z.literal(false), + code: z.enum(["STALE_ATTEMPT", "INVALID_COMMAND", "INTERNAL_ERROR"]), + message: z.string().min(1).max(500), + }), + ]), +}); + +const runnerResultAckSchema = z.object({ + type: z.literal("workflow-runner:result-ack"), + ...envelope, + payload: z.object({}), +}); + +const runnerCancelSchema = z.object({ + type: z.literal("workflow-runner:cancel"), + ...envelope, + payload: z.object({ reason: z.string().min(1).max(500) }), +}); + +const runnerErrorSchema = z.object({ + type: z.literal("workflow-runner:error"), + ...envelope, + payload: z.object({ + code: z.string().min(1).max(100), + message: z.string().min(1).max(500), + }), +}); + +export const workflowRunnerServerMessageSchema = z.discriminatedUnion("type", [ + runnerRegisteredSchema, + runnerHeartbeatAckSchema, + runnerCommandResultSchema, + runnerResultAckSchema, + runnerCancelSchema, + runnerErrorSchema, +]); +export type WorkflowRunnerServerMessage = z.infer; + +const runnerRegisterSchema = z.object({ + type: z.literal("workflow-runner:register"), + ...envelope, + payload: z.object({ + ...attemptIdentity, + protocolVersion: z.string().min(1), + appVersion: z.string().min(1), + needsJob: z.boolean(), + }), +}); + +const runnerHeartbeatSchema = z.object({ + type: z.literal("workflow-runner:heartbeat"), + ...envelope, + payload: z.object(attemptIdentity), +}); + +const setStateCommandSchema = z.object({ + type: z.literal("set-state"), + patch: runnerStatePatchSchema, + humanMessage: z.string().min(1).max(WORKFLOW_RUNNER_HUMAN_MESSAGE_MAX_CHARS), +}); + +const handOffChildCommandSchema = z.object({ + type: z.literal("hand-off-child"), + workflowName: WorkflowNameSchema, + target: targetSchema, + parentStepIndex: z.number().int().nonnegative(), + state: runnerStatePatchSchema, + humanMessage: z.string().min(1).max(WORKFLOW_RUNNER_HUMAN_MESSAGE_MAX_CHARS), +}); + +export const WorkflowRunnerCommandSchema = z + .discriminatedUnion("type", [setStateCommandSchema, handOffChildCommandSchema]) + .refine(fitsWorkflowRunnerMessageBudget, "workflow runner command exceeds byte budget"); +export type WorkflowRunnerCommand = z.infer; + +const runnerCommandSchema = z.object({ + type: z.literal("workflow-runner:command"), + ...envelope, + payload: z.object({ + ...attemptIdentity, + command: WorkflowRunnerCommandSchema, + }), +}); + +export const WorkflowRunnerResultPayloadSchema = z + .object({ + ...attemptIdentity, + result: HandlerResultSchema, + durationMs: z.number().int().nonnegative(), + }) + .refine( + (value) => !containsControllerReservedState(value.result.state), + "workflow runner result state contains a controller-reserved key", + ) + .refine(fitsWorkflowRunnerMessageBudget, "workflow runner result exceeds byte budget"); +export type WorkflowRunnerResultPayload = z.infer; + +const runnerResultSchema = z.object({ + type: z.literal("workflow-runner:result"), + ...envelope, + payload: WorkflowRunnerResultPayloadSchema, +}); + +export const workflowRunnerClientMessageSchema = z.discriminatedUnion("type", [ + runnerRegisterSchema, + runnerHeartbeatSchema, + runnerCommandSchema, + runnerResultSchema, +]); +export type WorkflowRunnerClientMessage = z.infer; + +export type WorkflowRunnerRegisteredMessage = z.infer; +export type WorkflowRunnerCommandResultMessage = z.infer; +export type WorkflowRunnerResultMessage = z.infer; + +export const WORKFLOW_RUNNER_PROTOCOL_VERSION = "1.1.0"; diff --git a/src/shared/workflow-runner-provider.ts b/src/shared/workflow-runner-provider.ts new file mode 100644 index 00000000..7bff712f --- /dev/null +++ b/src/shared/workflow-runner-provider.ts @@ -0,0 +1,180 @@ +import type { Config } from "../config"; + +type ProviderConfig = Pick< + Config, + | "provider" + | "model" + | "anthropicApiKey" + | "claudeCodeOauthToken" + | "awsRegion" + | "awsProfile" + | "awsAccessKeyId" + | "awsSecretAccessKey" + | "awsSessionToken" + | "awsBearerTokenBedrock" + | "anthropicBedrockBaseUrl" + | "allowedOwners" +>; + +export interface WorkflowRunnerProviderEnv { + readonly name: string; + readonly value?: string; + readonly secretKey?: string; +} + +export class WorkflowRunnerProviderConfigurationError extends Error { + constructor(message: string) { + super(message); + this.name = "WorkflowRunnerProviderConfigurationError"; + } +} + +function hasValue(value: string | undefined): value is string { + return (value?.trim().length ?? 0) > 0; +} + +function literal(name: string, value: string): WorkflowRunnerProviderEnv { + return { name, value }; +} + +function credential(name: string): WorkflowRunnerProviderEnv { + return { name, secretKey: name }; +} + +function secureBedrockBaseUrl(value: string): string { + if (value !== value.trim()) { + throw new WorkflowRunnerProviderConfigurationError( + "ANTHROPIC_BEDROCK_BASE_URL must not contain surrounding whitespace", + ); + } + let parsed: URL; + try { + parsed = new URL(value); + } catch { + throw new WorkflowRunnerProviderConfigurationError( + "ANTHROPIC_BEDROCK_BASE_URL must be an absolute HTTPS URL", + ); + } + if ( + parsed.protocol !== "https:" || + parsed.hostname === "" || + parsed.username !== "" || + parsed.password !== "" || + parsed.search !== "" || + parsed.hash !== "" + ) { + throw new WorkflowRunnerProviderConfigurationError( + "ANTHROPIC_BEDROCK_BASE_URL must use HTTPS without credentials, query, or fragment", + ); + } + return value; +} + +/** Select one credential chain and omit every credential that chain does not need. */ +export function workflowRunnerProviderEnv(config: ProviderConfig): WorkflowRunnerProviderEnv[] { + const common = [ + literal("CLAUDE_PROVIDER", config.provider), + literal("CLAUDE_MODEL", config.model), + ]; + const owners = + config.allowedOwners === undefined + ? [] + : [literal("ALLOWED_OWNERS", config.allowedOwners.join(","))]; + + if (config.provider === "anthropic") { + const selected = hasValue(config.anthropicApiKey) + ? credential("ANTHROPIC_API_KEY") + : hasValue(config.claudeCodeOauthToken) + ? credential("CLAUDE_CODE_OAUTH_TOKEN") + : null; + if (selected === null) { + throw new WorkflowRunnerProviderConfigurationError( + "Anthropic workflow runners require ANTHROPIC_API_KEY or CLAUDE_CODE_OAUTH_TOKEN", + ); + } + return [...common, selected, ...owners]; + } + + if (!hasValue(config.awsRegion)) { + throw new WorkflowRunnerProviderConfigurationError( + "Bedrock workflow runners require AWS_REGION", + ); + } + const bedrockSettings = [ + literal("AWS_REGION", config.awsRegion), + ...(hasValue(config.anthropicBedrockBaseUrl) + ? [ + literal( + "ANTHROPIC_BEDROCK_BASE_URL", + secureBedrockBaseUrl(config.anthropicBedrockBaseUrl), + ), + ] + : []), + ]; + if (hasValue(config.awsBearerTokenBedrock)) { + return [...common, ...bedrockSettings, credential("AWS_BEARER_TOKEN_BEDROCK"), ...owners]; + } + if (hasValue(config.awsAccessKeyId) && hasValue(config.awsSecretAccessKey)) { + return [ + ...common, + ...bedrockSettings, + credential("AWS_ACCESS_KEY_ID"), + credential("AWS_SECRET_ACCESS_KEY"), + ...(hasValue(config.awsSessionToken) ? [credential("AWS_SESSION_TOKEN")] : []), + ...owners, + ]; + } + const profileDetail = hasValue(config.awsProfile) + ? " AWS_PROFILE cannot be used because runner Pods do not mount AWS profile files." + : ""; + throw new WorkflowRunnerProviderConfigurationError( + `Bedrock workflow runners require AWS_BEARER_TOKEN_BEDROCK or AWS_ACCESS_KEY_ID plus AWS_SECRET_ACCESS_KEY.${profileDetail}`, + ); +} + +/** Reject a mutated Pod before it connects if more than the selected chain reached the process. */ +export function assertExactWorkflowRunnerProviderEnvironment(env: NodeJS.ProcessEnv): void { + const present = (name: string): boolean => hasValue(env[name]); + const provider = env["CLAUDE_PROVIDER"]; + if (provider === "anthropic") { + const anthropicCount = ["ANTHROPIC_API_KEY", "CLAUDE_CODE_OAUTH_TOKEN"].filter(present).length; + const awsCredentials = [ + "AWS_PROFILE", + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + "AWS_BEARER_TOKEN_BEDROCK", + ].filter(present); + if (anthropicCount !== 1 || awsCredentials.length > 0) { + throw new WorkflowRunnerProviderConfigurationError( + "Anthropic workflow runners require exactly one Anthropic credential and no AWS credentials", + ); + } + return; + } + if (provider !== "bedrock") { + throw new WorkflowRunnerProviderConfigurationError( + "CLAUDE_PROVIDER must be anthropic or bedrock in a workflow runner", + ); + } + if (present("ANTHROPIC_API_KEY") || present("CLAUDE_CODE_OAUTH_TOKEN")) { + throw new WorkflowRunnerProviderConfigurationError( + "Bedrock workflow runners must not receive Anthropic credentials", + ); + } + const bearer = present("AWS_BEARER_TOKEN_BEDROCK"); + const access = present("AWS_ACCESS_KEY_ID"); + const secret = present("AWS_SECRET_ACCESS_KEY"); + const session = present("AWS_SESSION_TOKEN"); + const staticChain = access && secret; + if ( + present("AWS_PROFILE") || + bearer === staticChain || + access !== secret || + (session && !staticChain) + ) { + throw new WorkflowRunnerProviderConfigurationError( + "Bedrock workflow runners require exactly one bearer or static AWS credential chain", + ); + } +} diff --git a/src/shared/workflow-types.ts b/src/shared/workflow-types.ts index 5dd0ac15..6b84f721 100644 --- a/src/shared/workflow-types.ts +++ b/src/shared/workflow-types.ts @@ -30,6 +30,13 @@ export const RepoMemoryCategorySchema = z.enum([ "env", "gotchas", ]); +export const RepoMemoryEntrySchema = z.object({ + id: z.uuid(), + category: RepoMemoryCategorySchema, + content: z.string().min(1).max(1000), + pinned: z.boolean(), +}); +export type RepoMemoryEntry = z.infer; const reviewLearningActionSaveSchema = z.object({ directive: z.string().min(1).max(2000), @@ -56,6 +63,73 @@ export const DaemonActionsSchema = z.object({ }); export type DaemonActions = z.infer; +const appliedReviewLearningIdsField = z.array(z.string().max(64)).max(50).optional(); +const daemonActionsField = DaemonActionsSchema.optional(); +export const WORKFLOW_RUNNER_HUMAN_MESSAGE_MAX_CHARS = 50_000; +const boundedHumanMessage = z + .string() + .min(1) + .max(WORKFLOW_RUNNER_HUMAN_MESSAGE_MAX_CHARS) + .optional(); +const boundedFailureReason = z.string().min(1).max(WORKFLOW_RUNNER_HUMAN_MESSAGE_MAX_CHARS); + +/** Result returned by one workflow handler before controller-side settlement. */ +export const HandlerResultSchema = z.discriminatedUnion("status", [ + z.object({ + status: z.literal("succeeded"), + state: z.unknown(), + humanMessage: boundedHumanMessage, + appliedReviewLearningIds: appliedReviewLearningIdsField, + daemonActions: daemonActionsField, + }), + z.object({ + status: z.literal("failed"), + reason: boundedFailureReason, + state: z.unknown().optional(), + humanMessage: boundedHumanMessage, + daemonActions: daemonActionsField, + }), + z.object({ + status: z.literal("incomplete"), + reason: boundedFailureReason, + state: z.unknown().optional(), + humanMessage: boundedHumanMessage, + appliedReviewLearningIds: appliedReviewLearningIdsField, + daemonActions: daemonActionsField, + }), + z.object({ + status: z.literal("handed-off"), + state: z.unknown().optional(), + humanMessage: boundedHumanMessage, + childRunId: z.string().min(1), + daemonActions: z.never().optional(), + }), +]); +export type HandlerResult = z.infer; + +export const PriorPlanStateSchema = z.object({ + plan: z.string().min(1).max(100_000), +}); +export type PriorPlanState = z.infer; + +const WorkflowRunSnapshotStateSchema = z.object({ + recommendedNext: z.enum(["plan", "stop"]).optional(), + pr_number: z.number().int().positive().optional(), +}); + +/** Bounded workflow history projected into a single-attempt runner payload. */ +export const WorkflowRunSnapshotSchema = z.object({ + id: z.uuid(), + status: z.enum(["queued", "running", "succeeded", "failed", "incomplete"]), + state: WorkflowRunSnapshotStateSchema, + createdAt: z.iso.datetime(), +}); +export type WorkflowRunSnapshot = z.infer; + +export function workflowRunnerId(attemptId: string): string { + return `workflow-runner:${attemptId}`; +} + export type { Registry, RegistryEntry, diff --git a/src/shared/ws-messages.ts b/src/shared/ws-messages.ts index 090d70a3..654d913a 100644 --- a/src/shared/ws-messages.ts +++ b/src/shared/ws-messages.ts @@ -63,7 +63,7 @@ const REVIEW_LEARNING_FILE_GLOB_MAX = 500; const REVIEW_LEARNING_SOURCE_THREAD_MAX = 200; const REVIEW_LEARNING_SOURCE_AUTHOR_MAX = 100; -const reviewLearningPayloadSchema = z.object({ +export const ReviewLearningPayloadSchema = z.object({ // UUID; conservatively capped to leave room for tagged formats. id: z.string().max(64), scope: z.enum(["local", "global"]), @@ -89,23 +89,9 @@ const reviewLearningSaveSchema = z.object({ sourceAuthor: z.string().max(REVIEW_LEARNING_SOURCE_AUTHOR_MAX).optional(), }); -/** - * Workflow run reference piggybacking on the existing job payload. Presence - * of this field signals the daemon to route the job through - * `src/daemon/workflow-executor.ts` instead of the legacy pipeline. Kept as - * a pure literal z.enum here to avoid importing the registry module from the - * shared schema layer (registry import pulls handler transitive deps). - */ -const workflowRunRefSchema = z.object({ - runId: z.string().min(1), - workflowName: z.enum(["triage", "plan", "implement", "review", "resolve", "ship"]), - parentRunId: z.string().min(1).optional(), - parentStepIndex: z.number().int().nonnegative().optional(), -}); - /** * Per-kind context passed alongside `job:payload` for scoped jobs. Mirrors - * the discriminator in `scoped-job-offer` so the daemon's executor can route + * the discriminator in `scoped-job:offer` so the daemon's executor can route * via the same Zod parse: discriminator at the schema level, not via runtime * presence checks (per contracts/ws-messages.md validation requirement). */ @@ -222,12 +208,6 @@ const jobPayloadSchema = z.object({ type: z.literal("job:payload"), ...messageEnvelopeBase, payload: z.object({ - /** Per-repo agent knobs resolved by the controller at accept time. - * Declared here so the wire schema matches `AgentPolicy`; the - * producer lands with the isolated workflow runner. A plain - * `z.object` strips unknown keys, so without this the daemon would - * silently drop a policy a future producer sent. */ - policy: AgentPolicySchema.optional(), context: z.record(z.string(), z.unknown()), installationToken: z.string(), /** GitHub App installation id (App mode only; absent in PAT mode). The @@ -242,9 +222,11 @@ const jobPayloadSchema = z.object({ /** Pre-loaded review learnings (all rows for owner+repo plus global rows * for the owner). Daemon-side review/resolve handlers filter by changed * files before injecting into the prompt; other workflows ignore. */ - reviewLearnings: z.array(reviewLearningPayloadSchema).optional(), - workflowRun: workflowRunRefSchema.optional(), - /** Present only when the originating offer was a `scoped-job-offer`. + reviewLearnings: z.array(ReviewLearningPayloadSchema).optional(), + /** Resolved `.github-app.yaml` agent knobs ("Gate 2"). Absent when the + * repo ships no config, which keeps the pre-Gate-2 payload byte-identical. */ + policy: AgentPolicySchema.optional(), + /** Present only when the originating offer was a `scoped-job:offer`. * The daemon's job-executor routes on this field's presence and on the * inner `jobKind` discriminator. */ scoped: scopedJobContextSchema.optional(), @@ -253,7 +235,7 @@ const jobPayloadSchema = z.object({ /** Server-to-daemon scoped job offer (parallel to `job:offer`). */ const scopedJobOfferSchema = z.object({ - type: z.literal("scoped-job-offer"), + type: z.literal("scoped-job:offer"), ...messageEnvelopeBase, payload: scopedJobContextSchema, }); @@ -315,7 +297,7 @@ const scopedJobResultSchema = z.discriminatedUnion("jobKind", [ ]); const scopedJobCompletionSchema = z.object({ - type: z.literal("scoped-job-completion"), + type: z.literal("scoped-job:completion"), ...messageEnvelopeBase, payload: z .object({ @@ -525,7 +507,7 @@ export type DaemonDrainingMessage = z.infer; // WebSocket protocol version /** Current protocol version. Major bump = breaking change = reject connection. */ -export const PROTOCOL_VERSION = "1.0.0"; +export const PROTOCOL_VERSION = "2.0.0"; // Custom WebSocket close codes diff --git a/src/utils/bot-identity.ts b/src/utils/bot-identity.ts index fe8ca7a4..334f1a9b 100644 --- a/src/utils/bot-identity.ts +++ b/src/utils/bot-identity.ts @@ -28,26 +28,22 @@ let selfLogin: Promise | null = null; export function resolveSelfLogin(): Promise { const pat = config.githubPersonalAccessToken; if (pat === undefined) return Promise.resolve(config.botAppLogin); - // Memoise the promise, not the resolved value: that collapses concurrent - // callers onto one request. The assignment stays in this sync body on - // purpose, so there is no read-then-await-then-write window on `selfLogin`. - selfLogin ??= fetchSelfLogin(pat); + // Memoising the promise rather than the resolved value also collapses + // concurrent callers onto one request, and keeps the assignment out of an + // async body (where it would read-then-await-then-write the same variable). + selfLogin ??= new Octokit({ auth: pat }).rest.users.getAuthenticated().then( + (r) => r.data.login, + () => { + // Not cached: a transient failure degrades one call, it does not disable + // the check for the lifetime of the process. Null is the fail-open + // direction for every caller (a redundant review, a duplicate comment). + selfLogin = null; + return null; + }, + ); return selfLogin; } -async function fetchSelfLogin(pat: string): Promise { - try { - const r = await new Octokit({ auth: pat }).rest.users.getAuthenticated(); - return r.data.login; - } catch { - // Not cached: a transient failure degrades one call, it does not disable - // the check for the lifetime of the process. Null is the fail-open - // direction for every caller (a redundant review, a duplicate comment). - selfLogin = null; - return null; - } -} - /** Test-only: drop the memoised login so cases can vary the auth mode. */ export function __resetBotIdentityCache(): void { selfLogin = null; diff --git a/src/webhook/auto-review-guard.ts b/src/webhook/auto-review-guard.ts new file mode 100644 index 00000000..5aa004b2 --- /dev/null +++ b/src/webhook/auto-review-guard.ts @@ -0,0 +1,189 @@ +import { createHash } from "node:crypto"; + +import type { Octokit } from "octokit"; + +import { getDb } from "../db"; +import { findActiveIntent } from "../db/queries/ship"; +import type { Logger } from "../logger"; +import { getValkeyClient, isValkeyHealthy } from "../orchestrator/valkey"; +import { resolveSelfLogin } from "../utils/bot-identity"; +import { type GithubActorLike, isSelfActor } from "../utils/github-actor"; + +/** + * Guards for auto-review-on-push (work item #1). + * + * Every helper here is advisory. On any error each returns the value that lets + * the review proceed, because a missing review is a worse failure than a + * redundant one. The exception is the repo opt-in, which lives in + * `pull-request.ts` and fails the other way: it reads `loadRepoPolicy`, whose + * fail-open default has `auto: false`. + */ + +/** Long enough to outlive a PR's active life; the key is per-PR, not per-repo. */ +const FINGERPRINT_TTL_SECONDS = 60 * 60 * 24 * 30; + +/** + * `pulls.listFiles` truncates above this many files, so the fingerprint would + * silently stop representing the whole diff. Skip the guard instead. + */ +const MAX_FINGERPRINTED_FILES = 3000; + +function fingerprintKey(owner: string, repo: string, prNumber: number): string { + return `autoreview:fp:${owner}/${repo}#${String(prNumber)}`; +} + +/** + * True when this push came from our own credential. + * + * `resolve` fixes review findings, commits, and pushes; that push fires + * `pull_request.synchronize`. Under `GITHUB_PERSONAL_ACCESS_TOKEN` the pusher is + * the PAT owner, who is exactly the human listed in `AUTO_REVIEW_USERS`, so + * without this the loop closes: review -> resolve -> push -> review. This is the + * ONLY thing breaking that loop: `resolve` deliberately does not filter review + * comments by author, because our own findings are ship's review -> resolve + * handoff. + */ +export async function isSelfPush(sender: GithubActorLike): Promise { + return isSelfActor(sender, await resolveSelfLogin()); +} + +/** + * Hash of the PR's own changes: sorted `filename\0status\0blobSha` triples from + * `pulls.listFiles`, which is merge-base-relative by definition, so base commits + * the PR did not author never enter it. + * + * A pure rebase rewrites every commit SHA but leaves those blobs untouched, so + * the hash is stable and the push is recognised as content-free. A rebase that + * pulls a base change into a file the PR also touches DOES alter the head blob, + * which is correct: that content genuinely changed and deserves a review. + * + * Returns null when the fingerprint cannot be trusted, which disables the guard + * and lets the review run. + */ +export async function computeDiffFingerprint( + octokit: Octokit, + owner: string, + repo: string, + prNumber: number, + log: Logger, +): Promise { + try { + const files = await octokit.paginate(octokit.rest.pulls.listFiles, { + owner, + repo, + pull_number: prNumber, + per_page: 100, + }); + if (files.length === 0 || files.length >= MAX_FINGERPRINTED_FILES) { + // Its own event, not `fingerprint_failed`: nothing failed, the diff is + // simply not fingerprintable. An operator asking "why did this PR + // re-review after a rebase?" needs to find this rather than silence. + log.info( + { + event: "auto_review.fingerprint_skipped", + owner, + repo, + prNumber, + fileCount: files.length, + }, + "auto-review: diff not fingerprintable, not skipping", + ); + return null; + } + const hash = createHash("sha256"); + for (const line of files.map((f) => `${f.filename}\0${f.status}\0${f.sha}`).sort()) { + hash.update(line); + hash.update("\n"); + } + return hash.digest("hex"); + } catch (err) { + log.warn( + { err, event: "auto_review.fingerprint_failed", owner, repo, prNumber }, + "auto-review: could not fingerprint the diff, not skipping", + ); + return null; + } +} + +/** + * True when this exact diff was already auto-reviewed. + * + * Gated on `isValkeyHealthy()` for the same reason `claimDelivery` is: Bun's + * RedisClient queues offline commands by default, so issuing GET against a + * disconnected client would block instead of failing open. + */ +export async function matchesLastReviewed( + owner: string, + repo: string, + prNumber: number, + fingerprint: string, + log: Logger, +): Promise { + const client = getValkeyClient(); + if (client === null || !isValkeyHealthy()) return false; + try { + const stored = (await client.send("GET", [fingerprintKey(owner, repo, prNumber)])) as + | string + | null; + return stored === fingerprint; + } catch (err) { + log.warn( + { err, event: "auto_review.fingerprint_read_failed", owner, repo, prNumber }, + "auto-review: fingerprint read failed, not skipping", + ); + return false; + } +} + +/** Record the reviewed diff. Best-effort: a write failure only costs one re-review. */ +export async function recordReviewedFingerprint( + owner: string, + repo: string, + prNumber: number, + fingerprint: string, + log: Logger, +): Promise { + const client = getValkeyClient(); + if (client === null || !isValkeyHealthy()) return; + try { + await client.send("SET", [ + fingerprintKey(owner, repo, prNumber), + fingerprint, + "EX", + String(FINGERPRINT_TTL_SECONDS), + ]); + } catch (err) { + log.warn( + { err, event: "auto_review.fingerprint_write_failed", owner, repo, prNumber }, + "auto-review: fingerprint write failed", + ); + } +} + +/** + * True when `ship` is actively driving this PR. + * + * Ship runs its own `review` -> `resolve` iteration, so auto-reviewing the same + * PR both duplicates the spend and races ship's `insertQueued` on the + * `idx_workflow_runs_inflight` partial-unique index. + * + * Fail-open like every other guard here: no database configured, or a query + * error, answers "no intent" and the review proceeds. + */ +export async function hasActiveShipIntent( + owner: string, + repo: string, + prNumber: number, + log: Logger, +): Promise { + if (getDb() === null) return false; + try { + return (await findActiveIntent(owner, repo, prNumber)) !== null; + } catch (err) { + log.warn( + { err, event: "auto_review.ship_intent_check_failed", owner, repo, prNumber }, + "auto-review: ship-intent lookup failed, not skipping", + ); + return false; + } +} diff --git a/src/webhook/dispatch-failure.ts b/src/webhook/dispatch-failure.ts new file mode 100644 index 00000000..d74e25e9 --- /dev/null +++ b/src/webhook/dispatch-failure.ts @@ -0,0 +1,40 @@ +import type { Octokit } from "octokit"; +import type { Logger } from "pino"; + +import { safePostToGitHub } from "../utils/github-output-guard"; + +const DISPATCH_FAILURE_MESSAGE = "Sorry, I couldn't start that workflow. Please try again."; + +interface DispatchFailureInput { + readonly octokit: Octokit; + readonly log: Logger; + readonly deliveryId: string; + readonly owner: string; + readonly repo: string; + readonly number: number; +} + +/** Tell an explicit requester that dispatch failed without exposing operator details. */ +export async function postDispatchFailure(input: DispatchFailureInput): Promise { + try { + await safePostToGitHub({ + body: DISPATCH_FAILURE_MESSAGE, + source: "system", + callsite: "webhook.dispatch-failure", + log: input.log, + deliveryId: input.deliveryId, + post: (body) => + input.octokit.rest.issues.createComment({ + owner: input.owner, + repo: input.repo, + issue_number: input.number, + body, + }), + }); + } catch (err) { + input.log.warn( + { err: err instanceof Error ? err : new Error(String(err)) }, + "Failed to post workflow dispatch failure comment", + ); + } +} diff --git a/src/webhook/events/issue-comment.ts b/src/webhook/events/issue-comment.ts index 71b65334..5587c954 100644 --- a/src/webhook/events/issue-comment.ts +++ b/src/webhook/events/issue-comment.ts @@ -10,6 +10,7 @@ import { addReaction } from "../../utils/reactions"; import { dispatchByIntent } from "../../workflows/dispatcher"; import { dispatchCommentSurface } from "../../workflows/ship/command-dispatch"; import { isOwnerAllowed } from "../authorize"; +import { postDispatchFailure } from "../dispatch-failure"; import { claimDelivery } from "../idempotency"; /** @@ -76,6 +77,11 @@ export function handleIssueComment( const commentBody = payload.comment.body; const isPR = payload.issue.pull_request !== undefined; const eventSurface = isPR ? "pr-comment" : "issue-comment"; + // Facts the repo-config trigger filters evaluate. `issue.draft` is + // populated when the issue is really a PR, so `ignore_draft_prs` covers + // comments too, not just label events. No base ref on this payload, so + // the `base_branches` rule is the one that skips here. + const trigger = { title: payload.issue.title, draft: isPR ? payload.issue.draft : undefined }; // Trigger-surface dispatch (T028e + T090). PR comments and Issue // comments share the same `issue_comment` event; the @@ -101,6 +107,7 @@ export function handleIssueComment( trigger_comment_id: payload.comment.id, octokit, log: dispatchLog, + trigger, }); } catch (err) { dispatchLog.error({ err }, "ship dispatchCommentSurface threw for issue_comment"); @@ -138,9 +145,11 @@ export function handleIssueComment( deliveryId, triggerCommentId: payload.comment.id, triggerEventType: "issue_comment", + trigger, }); } catch (err) { log.error({ err }, "dispatchByIntent threw for issue_comment"); + await postDispatchFailure({ octokit, log, deliveryId, owner, repo, number: targetNumber }); } // Piggyback proposal-poll on the trigger path too: the early- diff --git a/src/webhook/events/issues.ts b/src/webhook/events/issues.ts index 8e243a92..e1a33629 100644 --- a/src/webhook/events/issues.ts +++ b/src/webhook/events/issues.ts @@ -7,6 +7,7 @@ import { dispatchByLabel } from "../../workflows/dispatcher"; import { dispatchCanonicalCommand } from "../../workflows/ship/command-dispatch"; import { routeTrigger } from "../../workflows/ship/trigger-router"; import { isOwnerAllowed } from "../authorize"; +import { postDispatchFailure } from "../dispatch-failure"; import { claimDelivery } from "../idempotency"; // Permits hyphenated verbs (e.g. `bot:open-pr`, `bot:fix-thread`); the verb @@ -94,6 +95,9 @@ export function handleIssues(octokit: Octokit, payload: IssuesEvent, deliveryId: const repo = payload.repository.name; const issueNumber = payload.issue.number; const installationId = payload.installation?.id; + // Facts the repo-config trigger filters evaluate. A pure issue is never a + // draft PR and has no base branch, so those two rules never apply here. + const trigger = { title: payload.issue.title }; void (async (): Promise => { // Idempotency gate (issue #202): skip a redelivery before any dispatch. @@ -110,7 +114,7 @@ export function handleIssues(octokit: Octokit, payload: IssuesEvent, deliveryId: }, }); if (command !== null) { - dispatchCanonicalCommand(command, { octokit, log }); + dispatchCanonicalCommand(command, { octokit, log, trigger }); return; } } catch (err) { @@ -126,9 +130,11 @@ export function handleIssues(octokit: Octokit, payload: IssuesEvent, deliveryId: target: { type: "issue", owner, repo, number: issueNumber }, senderLogin, deliveryId, + trigger, }); } catch (err) { log.error({ err }, "dispatchByLabel threw for issues.labeled"); + await postDispatchFailure({ octokit, log, deliveryId, owner, repo, number: issueNumber }); } })(); } diff --git a/src/webhook/events/pull-request.ts b/src/webhook/events/pull-request.ts index 7ff4ba67..9052140a 100644 --- a/src/webhook/events/pull-request.ts +++ b/src/webhook/events/pull-request.ts @@ -1,13 +1,24 @@ import type { PullRequestEvent } from "@octokit/webhooks-types"; import type { Octokit } from "octokit"; +import { config } from "../../config"; import { upsertTarget } from "../../db/queries/conversation-store"; import { createChildLogger, logger } from "../../logger"; -import { dispatchByLabel } from "../../workflows/dispatcher"; +import { loadRepoPolicy, policyForWorkflow } from "../../repo-config/effective"; +import { runPrConfigCheck } from "../../repo-config/pr-check"; +import { dispatchByLabel, dispatchWorkflowByName } from "../../workflows/dispatcher"; import { dispatchCanonicalCommand } from "../../workflows/ship/command-dispatch"; import { fireReactor } from "../../workflows/ship/reactor-bridge"; import { routeTrigger } from "../../workflows/ship/trigger-router"; import { isOwnerAllowed } from "../authorize"; +import { + computeDiffFingerprint, + hasActiveShipIntent, + isSelfPush, + matchesLastReviewed, + recordReviewedFingerprint, +} from "../auto-review-guard"; +import { postDispatchFailure } from "../dispatch-failure"; import { claimDelivery } from "../idempotency"; // Permits the documented label shapes: @@ -28,12 +39,18 @@ const BOT_LABEL_PATTERN = /^bot:[a-z][a-z-]*(?:\/deadline=\d+(?:\.\d+)?[hms])?$/ * PR deletion is not a real GitHub action (PRs close, never delete), * so there is no hard-delete branch. See issues #129 and #130. * - * 2. Action-specific dispatch: - * - `opened`: placeholder (trigger detection lands when ready) + * 2. Config validation (`opened` / `synchronize` / `reopened`). When the + * PR touches `.github-app.yaml`, validate the HEAD-ref copy and post a + * sticky verdict comment. Read-only feedback: the copy the bot applies + * is still the default branch's. See `repo-config/pr-check.ts`. + * + * 3. Action-specific dispatch: + * - `opened`: config validation only (trigger detection lands when ready) * - `edited` / `reopened` / `converted_to_draft` / `ready_for_review`: * cache-only, no dispatch * - `labeled`: legacy workflow dispatch + ship reactor label dispatch - * - `synchronize`: ship reactor early-wake / foreign-push detection + * - `synchronize`: ship reactor early-wake / foreign-push detection, + * plus auto-review dispatch when the pusher is allowlisted * - `closed`: ship reactor terminal transition (merged_externally / * pr_closed) * @@ -54,6 +71,17 @@ export function handlePullRequest( logger.warn({ err, deliveryId }, "pull_request: cache write-through failed"); }); + // Config validation is orthogonal to the dispatch branches below, each of + // which returns early for its own action. Invoked here rather than + // duplicated into three of them. + if ( + payload.action === "opened" || + payload.action === "synchronize" || + payload.action === "reopened" + ) { + handlePullRequestConfigCheck(octokit, payload, deliveryId); + } + if (payload.action === "labeled") { handlePullRequestLabeled(octokit, payload, deliveryId); return; @@ -91,6 +119,73 @@ export function handlePullRequest( ); } +/** + * Validate this PR's own copy of `.github-app.yaml` and post a sticky + * verdict comment (issue #3). Shaped like `handlePullRequestLabeled`: + * authorize, then defer the work behind an idempotency claim. + * + * The claim key is suffixed rather than the bare `deliveryId`, because + * `claimDelivery` is one-shot per key and `handlePullRequestLabeled` already + * claims the bare id. A shared key would let whichever branch of a delivery + * ran first silently starve the other. + * + * This is a GitHub-write path, so it honours the repo-wide `enabled: false` + * master switch from the default branch's config (see the comment on the + * `loadRepoPolicy` call for why only that switch, not the full Gate-1 rule + * set). The lookup lives here rather than in `pr-check.ts`, which must stay + * structurally unable to reach the applied-policy path (C4). + */ +export function handlePullRequestConfigCheck( + octokit: Octokit, + payload: PullRequestEvent, + deliveryId: string, +): void { + const senderLogin = payload.sender.login; + const owner = payload.repository.owner.login; + const repo = payload.repository.name; + const prNumber = payload.pull_request.number; + const headSha = payload.pull_request.head.sha; + + const log = createChildLogger({ + deliveryId, + event: `pull_request.${payload.action}`, + senderLogin, + owner, + repo, + entityNumber: prNumber, + ...(payload.installation !== undefined ? { installationId: payload.installation.id } : {}), + }); + + const auth = isOwnerAllowed(senderLogin, log); + if (!auth.allowed) { + log.info({ reason: auth.reason }, "pull_request config check: sender not in ALLOWED_OWNERS"); + return; + } + + void (async (): Promise => { + if (!(await claimDelivery(`${deliveryId}:config-check`, log))) return; + try { + // Only the document-level `enabled` master switch, deliberately NOT the + // full `checkRepoGate` trigger set. The `triggers.*` filters exist to + // stop the bot ACTING on a pull request; suppressing authoring feedback + // because the config PR is a draft, or because its title matches + // `ignore_title_keywords`, is the opposite of what an author wants. Same + // scope the scheduler applies to its unattended runs. + const policy = await loadRepoPolicy({ octokit, owner, repo, log }); + if (!policy.enabled) { + log.info( + { event: "repo_config.pr_check.disabled", owner, repo, prNumber }, + "repo-config PR check: repo disabled by config, staying silent", + ); + return; + } + await runPrConfigCheck({ octokit, owner, repo, prNumber, headSha, deliveryId, log }); + } catch (err) { + log.error({ err }, "pull_request config check failed"); + } + })(); +} + /** * Resolve the actual commit author for the new head SHA before firing * the reactor. `payload.sender.login` is the webhook actor, not the @@ -134,9 +229,155 @@ function handlePullRequestSynchronize( head_sha: headSha, head_author_login: authorLogin, }); + + // `.catch`, not a bare await: this IIFE is fire-and-forget, and + // `installFatalHandlers` (src/logger.ts) answers `unhandledRejection` with + // `process.exit(1)`. One bad webhook must not take the server down. + await maybeAutoReview(octokit, payload, deliveryId).catch((err: unknown) => { + logger.error({ err, deliveryId, event: "auto_review.failed" }, "auto-review threw"); + }); })(); } +/** + * Auto-dispatch `review` when an `AUTO_REVIEW_USERS` login pushes to an open PR + * (work item #1). Two keys must agree: that env allowlist and the repo's own + * `workflows.review.auto`. Either one unset means this returns early. + * + * Reads `payload.sender.login`, deliberately NOT the `authorLogin` resolved + * above. That value comes from the commit's author *email*, which anyone can set + * with `git config user.email`; the reactor wants it because it asks a semantic + * question (did a human take over this branch?). This gate asks an authorization + * question (may we spend tokens on this push?), and only the authenticated + * pusher answers that. Do not "simplify" this to reuse `authorLogin`. + * + * Gate 1 still runs inside `dispatchWorkflowByName`, so `enabled: false`, + * `workflows.review.enabled: false`, and every `triggers.*` filter keep their + * veto. + */ +async function maybeAutoReview( + octokit: Octokit, + payload: PullRequestEvent & { action: "synchronize" }, + deliveryId: string, +): Promise { + const allowlist = config.autoReviewUsers; + if (allowlist === undefined) return; + + const senderLogin = payload.sender.login; + const owner = payload.repository.owner.login; + const repo = payload.repository.name; + const prNumber = payload.pull_request.number; + + const log = createChildLogger({ + deliveryId, + event: "pull_request.synchronize", + senderLogin, + owner, + repo, + entityNumber: prNumber, + ...(payload.installation !== undefined ? { installationId: payload.installation.id } : {}), + }); + + // Two lists, two questions. ALLOWED_OWNERS gates the *repository* (its + // documented meaning, and what `router.ts` passes), AUTO_REVIEW_USERS gates + // the *person* who pushed. Testing the pusher against ALLOWED_OWNERS instead + // would refuse a collaborator the operator deliberately allowlisted here. + if (!isOwnerAllowed(owner, log).allowed) return; + + const normalized = senderLogin.toLowerCase(); + if (!allowlist.some((u) => u.toLowerCase() === normalized)) return; + + // `checkoutRepo` clones the BASE repo and asks for the PR's head *branch name* + // (`src/core/checkout.ts`), so a fork ref either fails to clone or, worse, + // silently resolves to a same-named branch in the base repo and reviews the + // wrong tree. Nobody asked for this run, so skip rather than guess. Matches + // the fork test in `src/workflows/handlers/branch-refresh.ts`. + if (payload.pull_request.head.repo?.full_name !== payload.repository.full_name) { + log.info({ event: "auto_review.skipped_fork_pr" }, "auto-review: head is on a fork"); + return; + } + + try { + // Ordered cheapest-first. The self-push check costs nothing under App auth, + // the policy read is an ETag-cached conditional request, and only then do we + // pay for a paginated listFiles. + if (await isSelfPush(payload.sender)) { + log.info({ event: "auto_review.skipped_self_push" }, "auto-review: our own push"); + return; + } + + // A PR that `ship` is driving already gets reviewed by ship's own + // review -> resolve iteration, so a second review would duplicate the spend + // and race ship's `insertQueued` on `idx_workflow_runs_inflight`. Best + // effort: with no DATABASE_URL there are no intents to collide with. + if (await hasActiveShipIntent(owner, repo, prNumber, log)) { + log.info({ event: "auto_review.skipped_ship_active" }, "auto-review: ship owns this PR"); + return; + } + + // Per-repo opt-in. `loadRepoPolicy` fails open to DEFAULT_REPO_POLICY, where + // `auto` is false, so an unreachable or invalid config means no auto-review. + const policy = await loadRepoPolicy({ octokit, owner, repo, log }); + if (!policyForWorkflow(policy, "review").auto) { + log.debug( + { event: "auto_review.skipped_repo_opt_out" }, + "auto-review: not enabled for this repo", + ); + return; + } + + // A push that leaves the PR's own diff byte-identical (a rebase) bought no + // new code to review, so it buys no review. + const fingerprint = await computeDiffFingerprint(octokit, owner, repo, prNumber, log); + if ( + fingerprint !== null && + (await matchesLastReviewed(owner, repo, prNumber, fingerprint, log)) + ) { + log.info({ event: "auto_review.skipped_unchanged_diff" }, "auto-review: diff unchanged"); + return; + } + + // Suffixed key: `claimDelivery` is one-shot per key and the config-check + // branch already claims one on this same delivery. See the note above + // `handlePullRequestConfigCheck`. + if (!(await claimDelivery(`${deliveryId}:auto-review`, log))) return; + + // `auto: true` => never comments, never touches labels. An in-flight review + // for this PR makes `insertQueued` return a silent `refused`, which is the + // ignore-do-not-queue behaviour this feature wants. + const outcome = await dispatchWorkflowByName({ + octokit, + logger: log, + workflowName: "review", + target: { type: "pr", owner, repo, number: prNumber }, + senderLogin, + deliveryId, + triggerBodyPreview: "", + addRocketReaction: false, + auto: true, + // Reuse the policy loaded above so Gate 1 does not re-fetch it. + repoPolicy: policy, + trigger: { + title: payload.pull_request.title, + draft: payload.pull_request.draft, + baseBranch: payload.pull_request.base.ref, + }, + }); + + // Recorded only on a real dispatch, so a Gate-1 refusal cannot poison the + // fingerprint and suppress the next genuine review. + if (outcome.status === "dispatched" && fingerprint !== null) { + await recordReviewedFingerprint(owner, repo, prNumber, fingerprint, log); + } + log.info( + { event: "auto_review.outcome", status: outcome.status }, + "auto-review: dispatch settled", + ); + } catch (err) { + log.error({ err, event: "auto_review.failed" }, "auto-review: dispatch threw"); + } +} + function handlePullRequestLabeled( octokit: Octokit, payload: PullRequestEvent & { action: "labeled" }, @@ -178,6 +419,13 @@ function handlePullRequestLabeled( const repo = payload.repository.name; const prNumber = payload.pull_request.number; const installationId = payload.installation?.id; + // Facts the repo-config trigger filters evaluate. Taken from the payload + // rather than re-fetched, so the gate costs no extra API call. + const trigger = { + title: payload.pull_request.title, + draft: payload.pull_request.draft, + baseBranch: payload.pull_request.base.ref, + }; void (async (): Promise => { // Idempotency gate (issue #202): skip a redelivery before any dispatch. @@ -194,7 +442,7 @@ function handlePullRequestLabeled( }, }); if (command !== null) { - dispatchCanonicalCommand(command, { octokit, log }); + dispatchCanonicalCommand(command, { octokit, log, trigger }); return; } } catch (err) { @@ -210,9 +458,11 @@ function handlePullRequestLabeled( target: { type: "pr", owner, repo, number: prNumber }, senderLogin, deliveryId, + trigger, }); } catch (err) { log.error({ err }, "dispatchByLabel threw for pull_request.labeled"); + await postDispatchFailure({ octokit, log, deliveryId, owner, repo, number: prNumber }); } })(); } diff --git a/src/webhook/events/review-comment.ts b/src/webhook/events/review-comment.ts index 31cae418..f7fc07bf 100644 --- a/src/webhook/events/review-comment.ts +++ b/src/webhook/events/review-comment.ts @@ -11,6 +11,7 @@ import { dispatchByIntent } from "../../workflows/dispatcher"; import { dispatchCommentSurface } from "../../workflows/ship/command-dispatch"; import { fireReactor } from "../../workflows/ship/reactor-bridge"; import { isOwnerAllowed } from "../authorize"; +import { postDispatchFailure } from "../dispatch-failure"; import { claimDelivery } from "../idempotency"; /** @@ -111,6 +112,13 @@ export function handleReviewComment( const topLevelCommentId = typeof inReplyToIdRaw === "number" ? inReplyToIdRaw : payload.comment.id; const threadId = String(topLevelCommentId); + // Facts the repo-config trigger filters evaluate. Taken from the payload + // rather than re-fetched, so the gate costs no extra API call. + const trigger = { + title: payload.pull_request.title, + draft: payload.pull_request.draft, + baseBranch: payload.pull_request.base.ref, + }; void (async (): Promise => { // Idempotency gate (issue #202): skip a redelivery before any LLM dispatch. @@ -127,6 +135,7 @@ export function handleReviewComment( trigger_comment_id: payload.comment.id, octokit, log: dispatchLog, + trigger, }); } catch (err) { dispatchLog.error({ err }, "ship dispatchCommentSurface threw for review_comment"); @@ -163,9 +172,11 @@ export function handleReviewComment( triggerCommentId: payload.comment.id, triggerEventType: "pull_request_review_comment", ...(typeof inReplyToIdRaw === "number" ? { triggerInReplyToId: inReplyToIdRaw } : {}), + trigger, }); } catch (err) { log.error({ err }, "dispatchByIntent threw for review_comment"); + await postDispatchFailure({ octokit, log, deliveryId, owner, repo, number: prNumber }); } // Piggyback proposal-poll on the trigger path too: the early- diff --git a/src/workflows/completion-reconciler.ts b/src/workflows/completion-reconciler.ts new file mode 100644 index 00000000..547854e5 --- /dev/null +++ b/src/workflows/completion-reconciler.ts @@ -0,0 +1,91 @@ +import type { SQL } from "bun"; +import type { Octokit } from "octokit"; +import type pino from "pino"; + +import { requireDb } from "../db"; +import { logger } from "../logger"; +import { type CompletionResult, onStepComplete } from "./orchestrator"; +import { + findByAttemptId, + markAttemptCascadeCompleted, + type WorkflowAttempt, + type WorkflowRunRow, +} from "./runs-store"; + +function completionFromRow(row: WorkflowRunRow): CompletionResult | null { + if (row.status === "succeeded") return { status: "succeeded" }; + if (row.status === "incomplete") { + const reason = row.state["incompleteReason"]; + return { + status: "failed", + reason: `incomplete: ${typeof reason === "string" ? reason : "workflow incomplete"}`, + }; + } + if (row.status === "failed") { + const reason = row.state["failedReason"]; + return { + status: "failed", + reason: typeof reason === "string" ? reason : "workflow failed", + }; + } + return null; +} + +/** Apply a terminal workflow's parent cascade before its result can be acknowledged. */ +export async function ensureWorkflowCascadeForOffer( + offerId: string, + log: pino.Logger = logger, + sql: SQL = requireDb(), + octokit: Octokit | null = null, +): Promise<"not-workflow" | "pending" | "complete"> { + const row = await findByAttemptId(offerId, sql); + if (row === null) return "not-workflow"; + if (row.attempt_completed_at === null) return "pending"; + const completion = completionFromRow(row); + if (completion === null || row.cascade_completed_at !== null) return "complete"; + + await onStepComplete( + { octokit, logger: log, emitGitHub: octokit !== null, sql }, + row.id, + completion, + ); + const attempt: WorkflowAttempt = { runId: row.id, attemptId: offerId }; + if (!(await markAttemptCascadeCompleted(attempt, sql))) { + throw new Error("Workflow cascade receipt was no longer current"); + } + return "complete"; +} + +/** Retry terminal cascades left pending by a daemon or orchestrator crash. */ +export async function reconcilePendingWorkflowCascades( + sql: SQL = requireDb(), + limit = 100, +): Promise { + const rows: { attempt_id: string }[] = await sql` + SELECT wr.attempt_id + FROM workflow_runs AS wr + JOIN executions AS e ON e.delivery_id = wr.execution_delivery_id + WHERE wr.attempt_id IS NOT NULL + AND wr.attempt_completed_at IS NOT NULL + AND wr.cascade_completed_at IS NULL + AND wr.status IN ('succeeded', 'failed', 'incomplete') + AND NOT (e.workflow_result_payload IS NOT NULL AND e.result_processed_at IS NULL) + ORDER BY wr.attempt_completed_at + LIMIT ${limit} + `; + let completed = 0; + for (const row of rows) { + try { + // eslint-disable-next-line no-await-in-loop -- cascades are ordered and bounded + if ((await ensureWorkflowCascadeForOffer(row.attempt_id, logger, sql)) === "complete") { + completed++; + } + } catch (err) { + logger.warn( + { err: err instanceof Error ? err : new Error(String(err)), attemptId: row.attempt_id }, + "Pending workflow cascade failed", + ); + } + } + return completed; +} diff --git a/src/workflows/discussion-digest.ts b/src/workflows/discussion-digest.ts index 54e59d5a..e1ec6ac3 100644 --- a/src/workflows/discussion-digest.ts +++ b/src/workflows/discussion-digest.ts @@ -334,7 +334,6 @@ async function runDigestCall( system: withStructuredRules(system), messages: [{ role: "user", content: userMessage }], maxTokens: DIGEST_MAX_TOKENS, - temperature: 0, }); } catch (err) { cc.log.warn({ err }, "discussion-digest LLM call failed"); diff --git a/src/workflows/dispatch-outbox.ts b/src/workflows/dispatch-outbox.ts new file mode 100644 index 00000000..6f23ac7e --- /dev/null +++ b/src/workflows/dispatch-outbox.ts @@ -0,0 +1,133 @@ +import type { SQL } from "bun"; + +import { config } from "../config"; +import { requireDb } from "../db"; +import { logger } from "../logger"; +import { getInstanceId } from "../orchestrator/instance-id"; +import { ensureWorkflowJobQueued, type WorkflowRunQueuedJob } from "../orchestrator/job-queue"; +import { + markDispatchEnqueued, + recordWorkflowDispatchPublishFailure, + type WorkflowRunRow, +} from "./runs-store"; + +export const WORKFLOW_DISPATCH_RECONCILE_GRACE_MS = Math.max( + 60_000, + 3 * config.livenessReaperIntervalMs, +); + +interface PendingDispatchRow extends WorkflowRunRow { + execution_context_json: Record | null; + execution_event_name: string; + execution_trigger_username: string; +} + +async function loadPendingDispatch(runId: string, sql: SQL): Promise { + const rows: PendingDispatchRow[] = await sql` + SELECT wr.*, + e.context_json AS execution_context_json, + e.event_name AS execution_event_name, + e.trigger_username AS execution_trigger_username + FROM workflow_runs wr + JOIN executions e ON e.delivery_id = wr.execution_delivery_id + WHERE wr.id = ${runId} + AND wr.status = 'queued' + AND wr.dispatch_retry_count <= ${config.jobMaxRetries} + AND wr.created_at > now() - ${config.workflowDispatchTimeoutMs} * interval '1 millisecond' + AND ( + wr.dispatch_enqueued_at IS NULL + OR wr.dispatch_enqueued_at < now() - ${WORKFLOW_DISPATCH_RECONCILE_GRACE_MS} * interval '1 millisecond' + ) + `; + return rows[0] ?? null; +} + +function queuedJob(row: PendingDispatchRow): WorkflowRunQueuedJob { + if (row.execution_delivery_id === null) { + throw new Error(`Workflow run ${row.id} has no execution delivery id`); + } + const labels = row.execution_context_json?.["labels"]; + return { + kind: "workflow-run", + deliveryId: row.execution_delivery_id, + repoOwner: row.target_owner, + repoName: row.target_repo, + entityNumber: row.target_number, + isPR: row.target_type === "pr", + eventName: row.execution_event_name, + triggerUsername: row.execution_trigger_username || config.botAppLogin, + labels: Array.isArray(labels) + ? labels.filter((label): label is string => typeof label === "string") + : [], + triggerBodyPreview: row.trigger_body_preview, + enqueuedAt: row.created_at.getTime(), + retryCount: row.dispatch_retry_count, + workflowRun: { + runId: row.id, + workflowName: row.workflow_name, + ...(row.parent_run_id !== null && row.parent_step_index !== null + ? { parentRunId: row.parent_run_id, parentStepIndex: row.parent_step_index } + : {}), + }, + }; +} + +/** Publish one committed workflow row. A duplicate publish is safe at the offer CAS. */ +export async function publishWorkflowRunById( + runId: string, + sql: SQL = requireDb(), +): Promise { + const row = await loadPendingDispatch(runId, sql); + if (row?.execution_delivery_id === null || row?.execution_delivery_id === undefined) return false; + try { + await ensureWorkflowJobQueued(queuedJob(row), getInstanceId()); + } catch (err) { + await recordWorkflowDispatchPublishFailure( + row.id, + row.dispatch_generation_id, + row.dispatch_enqueued_at, + sql, + ).catch((receiptErr: unknown) => { + logger.error( + { err: receiptErr, runId: row.id }, + "Workflow dispatch publish failure receipt could not be recorded", + ); + }); + throw err; + } + return markDispatchEnqueued(row.id, row.dispatch_generation_id, row.dispatch_enqueued_at, sql); +} + +/** Retry workflow rows committed before their Valkey publish completed. */ +export async function publishPendingWorkflowRuns( + sql: SQL = requireDb(), + limit = 100, +): Promise { + const rows: { id: string }[] = await sql` + SELECT id + FROM workflow_runs + WHERE status = 'queued' + AND execution_delivery_id IS NOT NULL + AND dispatch_retry_count <= ${config.jobMaxRetries} + AND created_at > now() - ${config.workflowDispatchTimeoutMs} * interval '1 millisecond' + AND ( + dispatch_enqueued_at IS NULL + OR dispatch_enqueued_at < now() - ${WORKFLOW_DISPATCH_RECONCILE_GRACE_MS} * interval '1 millisecond' + ) + ORDER BY created_at + LIMIT ${limit} + `; + let published = 0; + for (const row of rows) { + try { + // eslint-disable-next-line no-await-in-loop -- queue publication is bounded and ordered + if (await publishWorkflowRunById(row.id, sql)) published++; + } catch (err) { + logger.warn( + { err: err instanceof Error ? err : new Error(String(err)), runId: row.id }, + "Pending workflow dispatch publish failed", + ); + } + } + return published; +} diff --git a/src/workflows/dispatcher.ts b/src/workflows/dispatcher.ts index 60fb28ba..2d1444f2 100644 --- a/src/workflows/dispatcher.ts +++ b/src/workflows/dispatcher.ts @@ -4,11 +4,14 @@ import type pino from "pino"; import { type LLMTool, type LLMToolHandler, resolveModelId, runWithTools } from "../ai/llm-client"; import { config } from "../config"; import { getDb } from "../db"; +import { isPostgresUniqueViolation } from "../db/postgres-error"; import { getInstanceId } from "../orchestrator/instance-id"; -import { enqueueJob } from "../orchestrator/job-queue"; +import { type EffectiveRepoPolicy, loadRepoPolicy } from "../repo-config/effective"; +import { checkRepoGate, type TriggerContext } from "../repo-config/gate"; import type { TriggerEventType } from "../shared/dispatch-types"; import { addReaction } from "../utils/reactions"; import { getTriageLLMClient } from "../webhook/triage-client-factory"; +import { publishWorkflowRunById } from "./dispatch-outbox"; import { recordWorkflowExecution } from "./execution-row"; import { classify } from "./intent-classifier"; import { enforceSingleBotLabel } from "./label-mutex"; @@ -18,7 +21,12 @@ import { logWorkflowRunQueued, } from "./log-fields"; import { getByLabel, getByName, type WorkflowName } from "./registry"; -import { findLatestSucceededForTarget, insertQueued, markFailed } from "./runs-store"; +import { + findCommittedWorkflowDispatch, + findLatestSucceededForTarget, + insertQueued, + type WorkflowRunRow, +} from "./runs-store"; import { runChatThread } from "./ship/scoped/chat-thread"; import { postRefusalComment } from "./tracking-mirror"; @@ -36,6 +44,196 @@ export interface DispatchByLabelParams { readonly target: DispatchTarget; readonly senderLogin: string; readonly deliveryId: string; + /** Trigger facts for the repo-config filter rules. See `TriggerContext`. */ + readonly trigger?: TriggerContext; +} + +interface DurableWorkflowDispatchParams { + readonly workflowName: WorkflowName; + readonly target: DispatchTarget; + readonly deliveryId: string; + readonly senderLogin: string; + readonly labels: readonly string[]; + readonly triggerBodyPreview: string; + readonly logger: pino.Logger; + readonly triggerCommentId?: number; + readonly triggerEventType?: TriggerEventType; +} + +const COMMIT_RECONCILIATION_MAX_ATTEMPTS = 8; +const COMMIT_RECONCILIATION_MAX_DELAY_MS = 100; + +async function reconcileCommittedWorkflowDispatch( + params: DurableWorkflowDispatchParams, + db: NonNullable>, +): Promise { + let delayMs = 50; + for (let attempt = 1; attempt <= COMMIT_RECONCILIATION_MAX_ATTEMPTS; attempt++) { + try { + return await findCommittedWorkflowDispatch( + { + workflowName: params.workflowName, + target: params.target, + executionDeliveryId: params.deliveryId, + }, + db, + ); + } catch (err) { + params.logger.warn( + { + err, + deliveryId: params.deliveryId, + attempt, + maxAttempts: COMMIT_RECONCILIATION_MAX_ATTEMPTS, + retryDelayMs: delayMs, + }, + "Workflow commit reconciliation unavailable; retaining trigger ownership", + ); + if (attempt === COMMIT_RECONCILIATION_MAX_ATTEMPTS) return null; + await Bun.sleep(delayMs); + delayMs = Math.min(delayMs * 2, COMMIT_RECONCILIATION_MAX_DELAY_MS); + } + } + return null; +} + +async function commitDurableWorkflowDispatch( + params: DurableWorkflowDispatchParams, +): Promise { + const db = getDb(); + if (db === null) throw new Error("Database not configured"); + try { + return await db.begin(async (tx) => { + const row = await insertQueued( + { + workflowName: params.workflowName, + target: params.target, + deliveryId: params.deliveryId, + triggerBodyPreview: params.triggerBodyPreview, + ownerKind: "orchestrator", + ownerId: getInstanceId(), + ...(params.triggerCommentId !== undefined + ? { triggerCommentId: params.triggerCommentId } + : {}), + ...(params.triggerEventType !== undefined + ? { triggerEventType: params.triggerEventType } + : {}), + }, + tx, + ); + await recordWorkflowExecution({ + deliveryId: params.deliveryId, + target: params.target, + senderLogin: params.senderLogin, + workflowName: params.workflowName, + runId: row.id, + labels: params.labels, + logger: params.logger, + ...(params.triggerCommentId !== undefined + ? { triggerCommentId: params.triggerCommentId } + : {}), + ...(params.triggerEventType !== undefined + ? { triggerEventType: params.triggerEventType } + : {}), + sql: tx, + }); + return row; + }); + } catch (err) { + const committed = await reconcileCommittedWorkflowDispatch(params, db); + if (committed === null) throw err; + params.logger.warn( + { err, runId: committed.id, deliveryId: params.deliveryId }, + "Recovered workflow dispatch after an ambiguous commit response", + ); + return committed; + } +} + +async function publishDurableWorkflowDispatch( + row: WorkflowRunRow, + params: Pick, +): Promise { + try { + await publishWorkflowRunById(row.id); + } catch (err) { + logWorkflowRunEnqueueFailed(params.logger, { + runId: row.id, + workflowName: params.workflowName, + target: params.target, + deliveryId: params.deliveryId, + reason: err instanceof Error ? err.message : String(err), + }); + params.logger.warn( + { err, runId: row.id, workflowName: params.workflowName, target: params.target }, + "Workflow dispatch publication failed; the durable outbox will retry", + ); + } +} + +/** + * Gate 1. Loads the repo's default-branch config and evaluates it against + * this trigger, before any run row, label mutex, queue job, or tracking + * comment exists. + * + * Returns the refusal outcome when blocked, `null` when the dispatch may + * proceed. Fail-open: `loadRepoPolicy` never throws, and a missing or + * broken config yields the permissive default policy. + */ +async function applyRepoGate(input: { + readonly octokit: Octokit; + readonly logger: pino.Logger; + readonly target: DispatchTarget; + readonly senderLogin: string; + readonly deliveryId: string; + readonly workflowName?: WorkflowName; + readonly trigger?: TriggerContext; + /** Reuses a policy already loaded upstream, saving a second fetch. */ + readonly policy?: EffectiveRepoPolicy; + /** See `auto` on `dispatchWorkflowByName`. Suppresses the refusal comment. */ + readonly auto?: boolean; +}): Promise { + const { octokit, logger, target, senderLogin, deliveryId, workflowName } = input; + const policy = + input.policy ?? + (await loadRepoPolicy({ octokit, owner: target.owner, repo: target.repo, log: logger })); + + const verdict = checkRepoGate({ policy, workflowName, senderLogin, trigger: input.trigger }); + if (verdict.allowed) return null; + + const named = workflowName ?? "unknown"; + // A passive filter the owner set to keep the bot quiet must stay quiet; only a + // deliberate label or mention earns a reply. An auto-trigger is not a + // deliberate ask either: a repo with `workflows.review.enabled: false` would + // otherwise be answered on every push. The refusal is still logged. + const explained = verdict.explain && input.auto !== true; + logWorkflowRunDispatchRefused(logger, { + workflowName: named, + target, + deliveryId, + reason: verdict.reason, + }); + logger.info( + { + event: "repo_config.gate_blocked", + workflowName: named, + target, + deliveryId, + senderLogin, + reason: verdict.reason, + explained, + }, + "Workflow dispatch blocked by repo config", + ); + if (explained) { + await postRefusalComment({ octokit, logger }, target, named, verdict.reason); + } + return { + status: "refused", + workflowName: named, + reason: verdict.reason, + explained, + }; } export type DispatchOutcome = @@ -45,6 +243,16 @@ export type DispatchOutcome = readonly status: "refused"; readonly reason: string; readonly workflowName: WorkflowName | "unknown"; + /** + * Whether the refusal was told to the user via `postRefusalComment`. + * Required, not optional: a caller that suppresses its own failure + * message on the assumption the dispatcher already spoke (see + * `WorkflowRefusedByDispatcher` in ship/scoped/chat-thread.ts) would + * otherwise dead-end silently. The repo-config trigger filters refuse + * without commenting on purpose, so this is the only signal that + * distinguishes the two. + */ + readonly explained: boolean; }; /** @@ -66,6 +274,17 @@ export async function dispatchByLabel(params: DispatchByLabelParams): Promise { const { octokit, @@ -360,11 +580,23 @@ export async function dispatchWorkflowByName(input: { deliveryId, triggerCommentId, triggerEventType, - triggerBodyPreview, addRocketReaction, } = input; const entry = getByName(workflowName); + const blocked = await applyRepoGate({ + octokit, + logger, + target, + senderLogin, + deliveryId, + workflowName: entry.name, + ...(input.auto === true ? { auto: true } : {}), + ...(input.trigger !== undefined ? { trigger: input.trigger } : {}), + ...(input.repoPolicy !== undefined ? { policy: input.repoPolicy } : {}), + }); + if (blocked !== null) return blocked; + const contextMatches = entry.context === "both" || (entry.context === "issue" && target.type === "issue") || @@ -373,8 +605,10 @@ export async function dispatchWorkflowByName(input: { if (!contextMatches) { const reason = `workflow '${entry.name}' only accepts ${entry.context} targets (this is a ${target.type})`; logWorkflowRunDispatchRefused(logger, { workflowName: entry.name, target, deliveryId, reason }); - await postRefusalComment({ octokit, logger }, target, entry.name, reason); - return { status: "refused", workflowName: entry.name, reason }; + if (input.auto !== true) { + await postRefusalComment({ octokit, logger }, target, entry.name, reason); + } + return { status: "refused", workflowName: entry.name, reason, explained: input.auto !== true }; } if (entry.requiresPrior !== null) { @@ -387,30 +621,44 @@ export async function dispatchWorkflowByName(input: { deliveryId, reason, }); - await postRefusalComment({ octokit, logger }, target, entry.name, reason); - return { status: "refused", workflowName: entry.name, reason }; + if (input.auto !== true) { + await postRefusalComment({ octokit, logger }, target, entry.name, reason); + } + return { + status: "refused", + workflowName: entry.name, + reason, + explained: input.auto !== true, + }; } } - await enforceSingleBotLabel({ - octokit, - owner: target.owner, - repo: target.repo, - number: target.number, - justApplied: entry.label, - logger, - }); + // Skipped for auto-triggers: no label is being applied, so there is nothing + // to be mutually exclusive with, and the mutex removes every *other* `bot:*` + // label, which would strip a `bot:ship` the user set on their next push. + if (input.auto !== true) { + await enforceSingleBotLabel({ + octokit, + owner: target.owner, + repo: target.repo, + number: target.number, + justApplied: entry.label, + logger, + }); + } - let runRow; + let runRow: WorkflowRunRow; try { - runRow = await insertQueued({ + runRow = await commitDurableWorkflowDispatch({ workflowName: entry.name, target, deliveryId, - ownerKind: "orchestrator", - ownerId: getInstanceId(), - triggerCommentId, - triggerEventType, + senderLogin, + labels: [entry.label], + triggerBodyPreview: input.triggerBodyPreview, + logger, + ...(triggerCommentId !== undefined ? { triggerCommentId } : {}), + ...(triggerEventType !== undefined ? { triggerEventType } : {}), }); logWorkflowRunQueued(logger, { runId: runRow.id, @@ -419,7 +667,7 @@ export async function dispatchWorkflowByName(input: { deliveryId, }); } catch (err) { - if (isInflightCollision(err)) { + if (isPostgresUniqueViolation(err, "idx_workflow_runs_inflight")) { logWorkflowRunDispatchRefused(logger, { workflowName: entry.name, target, @@ -437,63 +685,29 @@ export async function dispatchWorkflowByName(input: { "Workflow dispatch refused, in-flight run already exists", ); const reason = "an in-flight run already exists for this workflow and target"; - await postRefusalComment({ octokit, logger }, target, entry.name, reason); - return { status: "refused", workflowName: entry.name, reason }; + // Auto-triggers ignore rather than queue: a push landing while a review is + // already running is dropped, and dropped silently. + if (input.auto !== true) { + await postRefusalComment({ octokit, logger }, target, entry.name, reason); + } + return { + status: "refused", + workflowName: entry.name, + reason, + explained: input.auto !== true, + }; } throw err; } - try { - await recordWorkflowExecution({ - deliveryId, - target, - senderLogin, - workflowName: entry.name, - runId: runRow.id, - labels: [entry.label], - logger, - triggerCommentId, - triggerEventType, - }); - await enqueueJob({ - kind: "workflow-run", - deliveryId, - repoOwner: target.owner, - repoName: target.repo, - entityNumber: target.number, - isPR: target.type === "pr", - eventName: target.type === "pr" ? "pull_request" : "issues", - triggerUsername: senderLogin, - labels: [entry.label], - triggerBodyPreview, - enqueuedAt: Date.now(), - retryCount: 0, - workflowRun: { runId: runRow.id, workflowName: entry.name }, - }); - } catch (err) { - logWorkflowRunEnqueueFailed(logger, { - runId: runRow.id, - workflowName: entry.name, - target, - deliveryId, - reason: err instanceof Error ? err.message : String(err), - }); - logger.error( - { - runId: runRow.id, - workflowName: entry.name, - target, - deliveryId, - err: err instanceof Error ? err.message : String(err), - reason: "workflow-dispatch-enqueue-failed", - }, - "Workflow dispatch failed during enqueue; clearing in-flight guard", - ); - await markFailed(runRow.id, "enqueue failed", {}); - throw err; - } + await publishDurableWorkflowDispatch(runRow, { + workflowName: entry.name, + target, + deliveryId, + logger, + }); - if (addRocketReaction) { + if (addRocketReaction && triggerCommentId !== undefined && triggerEventType !== undefined) { void addReaction({ octokit, logger, @@ -508,23 +722,6 @@ export async function dispatchWorkflowByName(input: { return { status: "dispatched", runId: runRow.id, workflowName: entry.name }; } -/** - * Detect the Postgres unique-violation on `idx_workflow_runs_inflight` that - * FR-011 relies on to reject a second in-flight row for the same (workflow, - * target). Anything else: transport errors, check violations, permission - * errors: must not be silently converted to "in-flight already exists". - */ -function isInflightCollision(err: unknown): boolean { - if (typeof err !== "object" || err === null) { - return false; - } - const record = err as { code?: unknown; constraint?: unknown }; - if (record.code !== "23505") { - return false; - } - return record.constraint === "idx_workflow_runs_inflight"; -} - /** * Bridge from the legacy intent-classifier dispatcher to the * conversational chat-thread executor. The legacy classifier doesn't @@ -583,7 +780,6 @@ async function runChatThreadFromDispatcher(input: { system: params.systemPrompt, messages: [{ role: "user", content: params.userPrompt }], maxTokens: 1500, - temperature: 0.2, tools: params.tools, onToolCall: params.onToolCall, }); @@ -594,7 +790,6 @@ async function runChatThreadFromDispatcher(input: { system: params.systemPrompt, messages: [{ role: "user", content: params.userPrompt }], maxTokens: 1500, - temperature: 0.2, }); return res.text; }; diff --git a/src/workflows/execution-row.ts b/src/workflows/execution-row.ts index cdc54965..29f2bd3f 100644 --- a/src/workflows/execution-row.ts +++ b/src/workflows/execution-row.ts @@ -1,3 +1,4 @@ +import type { SQL } from "bun"; import type pino from "pino"; import { createExecution } from "../orchestrator/history"; @@ -6,14 +7,13 @@ import type { TriggerEventType } from "../shared/dispatch-types"; import type { DispatchTarget } from "./dispatcher"; /** - * Builds the `context_json` shape the accept handler on - * `src/orchestrator/connection-handler.ts` reads when a daemon claims a - * workflow-dispatched job. Mirrors the fields `buildSyntheticBotContext` + * Builds the `context_json` shape the controller reads while preparing an + * isolated workflow-runner payload. Mirrors the fields `buildSyntheticBotContext` * (in handlers/plan.ts) fills and `serializeBotContext` emits: the only * keys consumed downstream on the workflow branch are `owner`, `repo`, * `isPR`, `labels`, plus the four fields `validateJobContext` hard-requires * (`deliveryId`, `owner`, `repo`, `entityNumber`). All other fields are - * carried for forward-compat parity with the legacy pipeline shape. + * carried for compatibility with the shared pipeline shape. * * `eventName` defaults to `"issue_comment"` for label-triggered runs (no * originating comment) and is set to the real event when the dispatcher @@ -47,12 +47,9 @@ export function buildWorkflowContextJson(params: { } /** - * Persist an `executions` row and take a concurrency slot for a - * workflow-dispatched job. MUST be called before `enqueueJob` so the - * daemon's accept handler can resolve context_json via the delivery_id. - * On failure, the caller must release the slot + unwind the workflow_runs - * row: see `dispatcher.ts` / `orchestrator.ts` for the compensation - * pattern. + * Persist the execution-history half of a workflow dispatch. Callers write + * it in the same transaction as `workflow_runs`, then publish only after the + * transaction commits so the controller never dispatches an incomplete pair. */ export async function recordWorkflowExecution(params: { deliveryId: string; @@ -64,6 +61,7 @@ export async function recordWorkflowExecution(params: { logger: pino.Logger; triggerCommentId?: number; triggerEventType?: TriggerEventType; + sql?: SQL; }): Promise { const { deliveryId, target, senderLogin, workflowName, runId, labels, logger } = params; const { triggerCommentId, triggerEventType } = params; @@ -77,20 +75,24 @@ export async function recordWorkflowExecution(params: { ...(triggerEventType !== undefined ? { triggerEventType } : {}), }); - await createExecution({ - deliveryId, - repoOwner: target.owner, - repoName: target.repo, - entityNumber: target.number, - entityType: target.type === "pr" ? "pull_request" : "issue", - eventName: triggerEventType ?? "issue_comment", - triggerUsername: senderLogin, - dispatchMode: "daemon", - dispatchReason: "persistent-daemon", - contextJson, - ...(triggerCommentId !== undefined ? { triggerCommentId } : {}), - ...(triggerEventType !== undefined ? { triggerEventType } : {}), - }); + await createExecution( + { + deliveryId, + repoOwner: target.owner, + repoName: target.repo, + entityNumber: target.number, + entityType: target.type === "pr" ? "pull_request" : "issue", + eventName: triggerEventType ?? "issue_comment", + triggerUsername: senderLogin, + dispatchMode: "workflow-runner", + dispatchTarget: "workflow-runner", + dispatchReason: "workflow-runner", + contextJson, + ...(triggerCommentId !== undefined ? { triggerCommentId } : {}), + ...(triggerEventType !== undefined ? { triggerEventType } : {}), + }, + params.sql, + ); logger.info( { diff --git a/src/workflows/handlers/implement.ts b/src/workflows/handlers/implement.ts index b3808c05..5130fc31 100644 --- a/src/workflows/handlers/implement.ts +++ b/src/workflows/handlers/implement.ts @@ -1,9 +1,9 @@ import { config } from "../../config"; import { runPipeline } from "../../core/pipeline"; -import type { BotContext } from "../../types"; +import type { BotContext, ExecutionResult } from "../../types"; import { fetchAndBuildDigest, renderDigestSection } from "../discussion-digest"; import type { WorkflowHandler } from "../registry"; -import { findLatestSucceededForTarget } from "../runs-store"; +import { StaleWorkflowAttemptError } from "../runs-store"; /** * `implement` handler (T022): reuses `src/core/pipeline.ts` end-to-end. @@ -20,27 +20,20 @@ import { findLatestSucceededForTarget } from "../runs-store"; */ export const handler: WorkflowHandler = async (ctx) => { const { octokit, target, logger: log, deliveryId, runId } = ctx; + let daemonActions: ExecutionResult["daemonActions"]; try { if (target.type !== "issue") { return { status: "failed", reason: "implement requires issue target" }; } - // Consults succeeded rows only, matches the dispatcher's - // `requiresPrior: 'plan'` gate (which calls `findLatestSucceededForTarget`). - // A later failed `plan` re-run must not shadow an earlier valid plan. - const planRow = await findLatestSucceededForTarget("plan", { - owner: target.owner, - repo: target.repo, - number: target.number, - }); - if (planRow === null) { + // The controller loads the most recent succeeded plan before it gives the + // runner a payload. A later failed plan cannot shadow this snapshot. + const planState = ctx.priorPlanState; + if (planState === undefined) { return { status: "failed", reason: "no succeeded plan row found for target" }; } - const planMarkdown = typeof planRow.state["plan"] === "string" ? planRow.state["plan"] : ""; - if (planMarkdown.length === 0) { - return { status: "failed", reason: "plan row has empty state.plan" }; - } + const planMarkdown = planState.plan; const { data: issue } = await octokit.rest.issues.get({ owner: target.owner, @@ -123,14 +116,19 @@ export const handler: WorkflowHandler = async (ctx) => { defaultBranch, labels: [], skipTrackingComments: true, + ...(ctx.repoMemory !== undefined ? { repoMemory: ctx.repoMemory } : {}), octokit, log, }; const result = await runPipeline(botCtx, { captureFiles: ["IMPLEMENT.md"], + ...(ctx.policy !== undefined ? { policy: ctx.policy } : {}), + ...(ctx.maxTurns !== undefined ? { maxTurns: ctx.maxTurns } : {}), ...(digestSection.length > 0 ? { discussionDigest: digestSection } : {}), + ...(ctx.signal !== undefined ? { signal: ctx.signal } : {}), }); + daemonActions = result.daemonActions; if (!result.success) { // `reason` is internal (DB state.failedReason → orchestrator quota // detection + operator logs); `humanMessage` is the public tracking @@ -139,6 +137,7 @@ export const handler: WorkflowHandler = async (ctx) => { status: "failed", reason: result.errorMessage ?? "implement pipeline execution failed", humanMessage: "implement pipeline execution failed, see server logs for details.", + ...(daemonActions !== undefined ? { daemonActions } : {}), }; } @@ -154,6 +153,7 @@ export const handler: WorkflowHandler = async (ctx) => { return { status: "failed", reason: "implement completed but no PR was found", + ...(daemonActions !== undefined ? { daemonActions } : {}), }; } @@ -179,20 +179,26 @@ export const handler: WorkflowHandler = async (ctx) => { ? `\n\n${report}` : `\n\n_(no IMPLEMENT.md report, agent did not write one)_`; const humanMessage = `🛠️ **Implement complete**, opened PR [#${String(opened.number)}](${opened.url}) on branch \`${opened.branch}\`.${reportSection}${metaLine}`; - await ctx.setState(state, humanMessage); log.info( { prNumber: opened.number, branch: opened.branch, costUsd: result.costUsd }, "implement handler succeeded", ); - return { status: "succeeded", state, humanMessage }; + return { + status: "succeeded", + state, + humanMessage, + ...(daemonActions !== undefined ? { daemonActions } : {}), + }; } catch (err) { + if (err instanceof StaleWorkflowAttemptError) throw err; const message = err instanceof Error ? err.message : String(err); log.warn({ err }, "implement handler caught error"); return { status: "failed", reason: `implement failed: ${message}`, humanMessage: "implement pipeline execution failed, see server logs for details.", + ...(daemonActions !== undefined ? { daemonActions } : {}), }; } }; @@ -296,6 +302,7 @@ async function postStartingComment( try { await ctx.setState({ phase: "starting" }, body); } catch (err) { + if (err instanceof StaleWorkflowAttemptError) throw err; ctx.logger.warn( { err: err instanceof Error ? err.message : String(err) }, "implement starting-comment write failed, continuing without up-front comment", diff --git a/src/workflows/handlers/plan.ts b/src/workflows/handlers/plan.ts index 876c274e..d0c01922 100644 --- a/src/workflows/handlers/plan.ts +++ b/src/workflows/handlers/plan.ts @@ -2,11 +2,13 @@ import { readFile } from "node:fs/promises"; import { join } from "node:path"; import { config } from "../../config"; +import { applyAgentPolicy } from "../../core/agent-policy"; import { checkoutRepo } from "../../core/checkout"; import { executeAgent } from "../../core/executor"; import type { BotContext } from "../../types"; import { fetchAndBuildDigest, renderDigestSection } from "../discussion-digest"; import type { WorkflowHandler } from "../registry"; +import { StaleWorkflowAttemptError } from "../runs-store"; /** * `plan` handler (T021): multi-turn Claude Agent SDK session over the cloned @@ -27,6 +29,7 @@ import type { WorkflowHandler } from "../registry"; export const handler: WorkflowHandler = async (ctx) => { const { octokit, target, logger: log } = ctx; let cleanup: (() => Promise) | undefined; + let disposePolicy: (() => void) | undefined; try { const { data: issue } = await octokit.rest.issues.get({ @@ -80,19 +83,32 @@ export const handler: WorkflowHandler = async (ctx) => { const promptParts = config.promptCacheLayout === "cacheable" ? buildPlanPromptParts(promptInput) : undefined; + // Bypasses `runPipeline` (see header), so it applies the Gate-2 knobs itself. + const applied = applyAgentPolicy({ + baseAllowedTools: ["Read", "Grep", "Glob", "Write", "Bash"], + ...(ctx.policy !== undefined ? { policy: ctx.policy } : {}), + ...(ctx.maxTurns !== undefined ? { maxTurns: ctx.maxTurns } : {}), + ...(ctx.signal !== undefined ? { signal: ctx.signal } : {}), + }); + disposePolicy = applied.dispose; + const result = await executeAgent({ ctx: botCtx, prompt, mcpServers: {}, workDir: checkout.workDir, - allowedTools: ["Read", "Grep", "Glob", "Write", "Bash"], + ...applied.options, ...(promptParts !== undefined ? { promptParts } : {}), }); + applied.options.signal?.throwIfAborted(); if (!result.success) { + // `reason` is internal only; the raw error must not reach the public + // `humanMessage`. Keeping it preserves the per-repo `timeout:` attribution. return { status: "failed", - reason: "plan agent execution failed", + reason: result.errorMessage ?? "plan agent execution failed", + humanMessage: "plan agent execution failed, see server logs for details.", }; } @@ -115,7 +131,6 @@ export const handler: WorkflowHandler = async (ctx) => { meta.push(`duration: ${String(Math.round(result.durationMs / 1000))}s`); const metaLine = meta.length > 0 ? `\n\n_${meta.join(" · ")}_` : ""; const humanMessage = `📋 **Plan ready**, task decomposition below.\n\n${planMarkdown.trim()}${metaLine}`; - await ctx.setState(state, humanMessage); log.info( { planLength: planMarkdown.length, costUsd: result.costUsd }, @@ -123,10 +138,13 @@ export const handler: WorkflowHandler = async (ctx) => { ); return { status: "succeeded", state, humanMessage }; } catch (err) { + if (err instanceof StaleWorkflowAttemptError) throw err; const message = err instanceof Error ? err.message : String(err); log.warn({ err }, "plan handler caught error"); return { status: "failed", reason: `plan failed: ${message}` }; } finally { + // Before the awaited cleanup: a live timer holds Bun's event loop open. + disposePolicy?.(); if (cleanup !== undefined) { await cleanup().catch((err: unknown) => { log.warn({ err }, "plan handler cleanup failed"); @@ -268,6 +286,7 @@ async function postStartingComment( try { await ctx.setState({ phase: "starting" }, body); } catch (err) { + if (err instanceof StaleWorkflowAttemptError) throw err; ctx.logger.warn( { err: err instanceof Error ? err.message : String(err) }, "plan starting-comment write failed, continuing without up-front comment", diff --git a/src/workflows/handlers/remember.ts b/src/workflows/handlers/remember.ts index a5f57b7e..db719198 100644 --- a/src/workflows/handlers/remember.ts +++ b/src/workflows/handlers/remember.ts @@ -1,10 +1,9 @@ import type { Octokit } from "octokit"; import { runPipeline } from "../../core/pipeline"; -import type { BotContext } from "../../types"; +import type { BotContext, ExecutionResult } from "../../types"; import { fetchAndBuildDigest, renderDigestSection } from "../discussion-digest"; import type { WorkflowHandler } from "../registry"; -import { findById } from "../runs-store"; /** * `remember` handler (issue #160 Option A): explicit `@bot remember [...]` @@ -27,6 +26,7 @@ import { findById } from "../runs-store"; */ export const handler: WorkflowHandler = async (ctx) => { const { octokit, target, logger: log, deliveryId, runId } = ctx; + let daemonActions: ExecutionResult["daemonActions"]; try { // Common metadata for both issue + PR targets. The discussion-digest @@ -61,15 +61,14 @@ export const handler: WorkflowHandler = async (ctx) => { }), ); - await ctx.setState( + const seededState = await ctx.setState( { target_type: target.type, target_number: target.number, }, "🧠 **Remember starting**, reading the thread and extracting the directive…", ); - const seededRow = await findById(runId); - const trackingCommentId = seededRow?.tracking_comment_id ?? undefined; + const trackingCommentId = seededState.trackingCommentId; if (trackingCommentId === undefined) { log.warn({ runId }, "remember handler: tracking comment id not found after seed setState"); } @@ -104,13 +103,17 @@ export const handler: WorkflowHandler = async (ctx) => { // an empty universe (which is fine for "save" but breaks dedup via // get_review_learnings). ...(ctx.reviewLearnings !== undefined ? { reviewLearnings: ctx.reviewLearnings } : {}), + ...(ctx.repoMemory !== undefined ? { repoMemory: ctx.repoMemory } : {}), octokit, log, }; const result = await runPipeline(botCtx, { + ...(ctx.policy !== undefined ? { policy: ctx.policy } : {}), + ...(ctx.maxTurns !== undefined ? { maxTurns: ctx.maxTurns } : {}), ...(trackingCommentId !== undefined ? { trackingCommentId } : {}), ...(digestSection.length > 0 ? { discussionDigest: digestSection } : {}), + ...(ctx.signal !== undefined ? { signal: ctx.signal } : {}), // Narrow the agent's tool surface: the only writes this workflow // makes are the save_review_learning round-trip and the tracking // comment update. No code edits, no shell, no commits, and (per @@ -138,12 +141,14 @@ export const handler: WorkflowHandler = async (ctx) => { // directive whose glob does not overlap the current PR. unfilteredReviewLearnings: true, }); + daemonActions = result.daemonActions; if (!result.success) { return { status: "failed", reason: result.errorMessage ?? "remember pipeline execution failed", humanMessage: "remember pipeline execution failed, see server logs for details.", + ...(daemonActions !== undefined ? { daemonActions } : {}), }; } @@ -157,6 +162,7 @@ export const handler: WorkflowHandler = async (ctx) => { return { status: "succeeded", state: { target_number: target.number, target_type: target.type }, + ...(daemonActions !== undefined ? { daemonActions } : {}), }; } catch (err) { log.error({ err, runId, target: target.number }, "remember handler threw"); @@ -164,6 +170,7 @@ export const handler: WorkflowHandler = async (ctx) => { status: "failed", reason: err instanceof Error ? err.message : "unknown error", humanMessage: "remember pipeline execution failed, see server logs for details.", + ...(daemonActions !== undefined ? { daemonActions } : {}), }; } }; diff --git a/src/workflows/handlers/resolve.ts b/src/workflows/handlers/resolve.ts index f4292536..d642cbdb 100644 --- a/src/workflows/handlers/resolve.ts +++ b/src/workflows/handlers/resolve.ts @@ -1,8 +1,7 @@ import { runPipeline } from "../../core/pipeline"; -import type { BotContext } from "../../types"; +import type { BotContext, ExecutionResult } from "../../types"; import { fetchAndBuildDigest, renderDigestSection } from "../discussion-digest"; import type { WorkflowHandler } from "../registry"; -import { findById } from "../runs-store"; import { type BranchStaleness, formatRefreshDirective, getBranchStaleness } from "./branch-refresh"; import { evaluateChecks } from "./checks"; import { parseOutstandingSection } from "./resolve-report"; @@ -45,6 +44,7 @@ export const POLL_WAIT_SECS_CAP = 900; export const handler: WorkflowHandler = async (ctx) => { const { octokit, target, logger: log, deliveryId, runId } = ctx; + let daemonActions: ExecutionResult["daemonActions"]; try { if (target.type !== "pr") { @@ -90,6 +90,12 @@ export const handler: WorkflowHandler = async (ctx) => { pull_number: target.number, per_page: 100, }); + // Deliberately NOT filtered by author. `ship` runs review immediately + // before resolve on the same PR, so our own inline findings ARE the input + // here; dropping them would make ship's resolve step a CI-only fixer. It + // would also discard CodeRabbit / Copilot / Sonar feedback, which is + // `type: "Bot"` too. The review -> resolve -> push -> review loop is broken + // at the trigger instead, by `isSelfPush` in auto-review-guard.ts. const topLevelComments = reviewComments.filter((c) => c.in_reply_to_id === undefined); const staleness = await getBranchStaleness(octokit, target.owner, target.repo, target.number); @@ -113,7 +119,7 @@ export const handler: WorkflowHandler = async (ctx) => { // Seed the tracking comment up front so the agent can post mid-run // progress against it. See review.ts for the same pattern + rationale. - await ctx.setState( + const seededState = await ctx.setState( { pr_number: target.number, failing_checks: failingChecks, @@ -121,8 +127,7 @@ export const handler: WorkflowHandler = async (ctx) => { }, `🔎 **Resolve starting**, ${String(failingChecks.length)} failing checks, ${String(topLevelComments.length)} open comment threads. Refreshing branch and classifying feedback…`, ); - const seededRow = await findById(runId); - const trackingCommentId = seededRow?.tracking_comment_id ?? undefined; + const trackingCommentId = seededState.trackingCommentId; if (trackingCommentId === undefined || trackingCommentId === null) { log.warn({ runId }, "resolve handler: tracking comment id not found after seed setState"); } @@ -155,22 +160,27 @@ export const handler: WorkflowHandler = async (ctx) => { // Mirrors review.ts: workflow-dispatch path needs this thread or the // pipeline sees ctx.reviewLearnings=undefined and the feature no-ops. ...(ctx.reviewLearnings !== undefined ? { reviewLearnings: ctx.reviewLearnings } : {}), + ...(ctx.repoMemory !== undefined ? { repoMemory: ctx.repoMemory } : {}), octokit, log, }; const result = await runPipeline(botCtx, { captureFiles: ["RESOLVE.md"], + ...(ctx.policy !== undefined ? { policy: ctx.policy } : {}), + ...(ctx.maxTurns !== undefined ? { maxTurns: ctx.maxTurns } : {}), ...(trackingCommentId !== undefined && trackingCommentId !== null ? { trackingCommentId } : {}), ...(topLevelComments.length > 0 ? { enableResolveReviewThread: true } : {}), ...(digestSection.length > 0 ? { discussionDigest: digestSection } : {}), + ...(ctx.signal !== undefined ? { signal: ctx.signal } : {}), // Resolve handler may persist new review_learnings when a maintainer // pushback resolves with "intentional, don't flag next time". See // review.ts for the gate's contract. enableReviewLearnings: true, }); + daemonActions = result.daemonActions; if (!result.success) { // `reason` is internal (DB state.failedReason → orchestrator quota // detection + operator logs); `humanMessage` is the public tracking @@ -179,6 +189,7 @@ export const handler: WorkflowHandler = async (ctx) => { status: "failed", reason: result.errorMessage ?? "resolve pipeline execution failed", humanMessage: "resolve pipeline execution failed, see server logs for details.", + ...(daemonActions !== undefined ? { daemonActions } : {}), }; } @@ -276,7 +287,6 @@ export const handler: WorkflowHandler = async (ctx) => { const humanMessage = `${headline}${outstandingSection}${reportSection}${metaLine}${learningsFooter}`; const state = { ...baseState, ci_verified: false }; - await ctx.setState(state, humanMessage); log.warn( { failingChecks: postCheckEvaluation.failingChecks, @@ -286,7 +296,13 @@ export const handler: WorkflowHandler = async (ctx) => { }, "resolve handler returning incomplete, post-pipeline gate caught surviving failures", ); - return { status: "incomplete", reason, state, humanMessage }; + return { + status: "incomplete", + reason, + state, + humanMessage, + ...(daemonActions !== undefined ? { daemonActions } : {}), + }; } const state = { ...baseState, ci_verified: true }; @@ -299,7 +315,6 @@ export const handler: WorkflowHandler = async (ctx) => { const learningsFooter = renderReviewLearningsFooter(result.appliedReviewLearnings); const humanMessage = `${headline}${reportSection}${metaLine}${learningsFooter}`; - await ctx.setState(state, humanMessage); log.info( { failingChecks: failingChecks.length, @@ -308,14 +323,15 @@ export const handler: WorkflowHandler = async (ctx) => { }, "resolve handler succeeded", ); - // Mirrors review.ts: forward applied-learning IDs so orchestrator can - // bump use_count. See workflow-executor.ts for the downstream wire. + // Mirrors review.ts: forward applied-learning IDs so result reconciliation + // can bump use_count after the terminal result is durable. const appliedReviewLearningIds = (result.appliedReviewLearnings ?? []).map((l) => l.id); return { status: "succeeded", state, humanMessage, ...(appliedReviewLearningIds.length > 0 ? { appliedReviewLearningIds } : {}), + ...(daemonActions !== undefined ? { daemonActions } : {}), }; } catch (err) { const message = err instanceof Error ? err.message : String(err); @@ -324,6 +340,7 @@ export const handler: WorkflowHandler = async (ctx) => { status: "failed", reason: `resolve failed: ${message}`, humanMessage: "resolve pipeline execution failed, see server logs for details.", + ...(daemonActions !== undefined ? { daemonActions } : {}), }; } }; diff --git a/src/workflows/handlers/review-learnings-footer.ts b/src/workflows/handlers/review-learnings-footer.ts index c8fd5d3b..9a5832dc 100644 --- a/src/workflows/handlers/review-learnings-footer.ts +++ b/src/workflows/handlers/review-learnings-footer.ts @@ -7,8 +7,8 @@ import type { AppliedReviewLearning } from "../../utils/review-learnings-filter" * * Empty input yields an empty string so the caller can spread the result * unconditionally. Newly-saved learnings (from `save_review_learning`) are - * NOT included here because the persistence happens orchestrator-side after - * the daemon emits the result; the next review will surface them. + * NOT included here because the controller persists them after the worker's + * terminal result; the next review will surface them. * * The block emits one fenced code section per learning so the layout stays * diffable and copy-pasteable, mirroring how operators read CI log blocks. diff --git a/src/workflows/handlers/review.ts b/src/workflows/handlers/review.ts index 195ad451..5e32e13a 100644 --- a/src/workflows/handlers/review.ts +++ b/src/workflows/handlers/review.ts @@ -1,8 +1,7 @@ import { runPipeline } from "../../core/pipeline"; -import type { BotContext } from "../../types"; +import type { BotContext, ExecutionResult } from "../../types"; import { fetchAndBuildDigest, renderDigestSection } from "../discussion-digest"; import type { WorkflowHandler } from "../registry"; -import { findById } from "../runs-store"; import { type BranchStaleness, formatRefreshDirective, getBranchStaleness } from "./branch-refresh"; import { renderReviewLearningsFooter } from "./review-learnings-footer"; @@ -35,6 +34,7 @@ import { renderReviewLearningsFooter } from "./review-learnings-footer"; export const handler: WorkflowHandler = async (ctx) => { const { octokit, target, logger: log, deliveryId, runId } = ctx; + let daemonActions: ExecutionResult["daemonActions"]; try { if (target.type !== "pr") { @@ -77,7 +77,7 @@ export const handler: WorkflowHandler = async (ctx) => { // post mid-run progress against. The orchestrator's setState creates // the comment on first call and reserves its id in the workflow row; // we read that id back and hand it to the pipeline. - await ctx.setState( + const seededState = await ctx.setState( { pr_number: target.number, head_sha: pr.head.sha, @@ -87,8 +87,7 @@ export const handler: WorkflowHandler = async (ctx) => { }, `🔍 **Code review starting**, ${String(pr.changed_files)} files, +${String(pr.additions)}/-${String(pr.deletions)}. Cloning repo and reading changed files…`, ); - const seededRow = await findById(runId); - const trackingCommentId = seededRow?.tracking_comment_id ?? undefined; + const trackingCommentId = seededState.trackingCommentId; if (trackingCommentId === undefined || trackingCommentId === null) { log.warn({ runId }, "review handler: tracking comment id not found after seed setState"); } @@ -123,24 +122,29 @@ export const handler: WorkflowHandler = async (ctx) => { // Forward orchestrator pre-loaded learnings so runPipeline's prompt- // builder and MCP server see them. Without this, the workflow-dispatch // path silently drops the feature even though the direct-pipeline path - // works. Wire upstream: src/daemon/workflow-executor.ts. + // works. The controller preloads it in workflow-runner-payload.ts. ...(ctx.reviewLearnings !== undefined ? { reviewLearnings: ctx.reviewLearnings } : {}), + ...(ctx.repoMemory !== undefined ? { repoMemory: ctx.repoMemory } : {}), octokit, log, }; const result = await runPipeline(botCtx, { captureFiles: ["REVIEW.md"], + ...(ctx.policy !== undefined ? { policy: ctx.policy } : {}), + ...(ctx.maxTurns !== undefined ? { maxTurns: ctx.maxTurns } : {}), ...(trackingCommentId !== undefined && trackingCommentId !== null ? { trackingCommentId } : {}), ...(digestSection.length > 0 ? { discussionDigest: digestSection } : {}), + ...(ctx.signal !== undefined ? { signal: ctx.signal } : {}), // Review handler is one of only two workflows authorised to consume // review_learnings as repo policy (resolve is the other). The orch- // estrator pre-loads them onto ctx for every job; this flag is what // lets the pipeline forward them into the prompt + MCP server. enableReviewLearnings: true, }); + daemonActions = result.daemonActions; if (!result.success) { // `reason` is internal (DB state.failedReason → orchestrator quota // detection + operator logs); `humanMessage` is the public tracking @@ -150,6 +154,7 @@ export const handler: WorkflowHandler = async (ctx) => { status: "failed", reason: result.errorMessage ?? "review pipeline execution failed", humanMessage: "review pipeline execution failed, see server logs for details.", + ...(daemonActions !== undefined ? { daemonActions } : {}), }; } @@ -186,7 +191,6 @@ export const handler: WorkflowHandler = async (ctx) => { const learningsFooter = renderReviewLearningsFooter(result.appliedReviewLearnings); const humanMessage = `${headline}${reportSection}${metaLine}${learningsFooter}`; - await ctx.setState(state, humanMessage); log.info( { changedFiles: pr.changed_files, @@ -200,14 +204,15 @@ export const handler: WorkflowHandler = async (ctx) => { // orchestrator can bump `use_count` + `last_used_at`. Without this, // the workflow-dispatch path silently drops the bump even though the // direct-pipeline path forwards it via `appliedReviewLearningIds` in - // job-executor.ts. workflow-executor.ts reads this field off the - // HandlerResult and writes it into `job:result`. + // job-executor.ts. The isolated runner returns this HandlerResult to the + // controller for durable result reconciliation. const appliedReviewLearningIds = (result.appliedReviewLearnings ?? []).map((l) => l.id); return { status: "succeeded", state, humanMessage, ...(appliedReviewLearningIds.length > 0 ? { appliedReviewLearningIds } : {}), + ...(daemonActions !== undefined ? { daemonActions } : {}), }; } catch (err) { const message = err instanceof Error ? err.message : String(err); @@ -216,6 +221,7 @@ export const handler: WorkflowHandler = async (ctx) => { status: "failed", reason: `review failed: ${message}`, humanMessage: "review pipeline execution failed, see server logs for details.", + ...(daemonActions !== undefined ? { daemonActions } : {}), }; } }; diff --git a/src/workflows/handlers/ship.ts b/src/workflows/handlers/ship.ts index e91026b5..3c66c20d 100644 --- a/src/workflows/handlers/ship.ts +++ b/src/workflows/handlers/ship.ts @@ -1,13 +1,11 @@ -import { requireDb } from "../../db"; -import { enqueueJob } from "../../orchestrator/job-queue"; -import { recordWorkflowExecution } from "../execution-row"; +import type { WorkflowRunSnapshot } from "../../shared/workflow-types"; import { getByName, type WorkflowHandler, type WorkflowName, type WorkflowRunContext, } from "../registry"; -import { findLatestForTarget, type WorkflowRunRow } from "../runs-store"; +import { StaleWorkflowAttemptError } from "../runs-store"; // T028 v2 entry, re-exported for spec-locator parity with the // CanonicalCommand path described in the tasks.md T028 description. The // legacy WorkflowHandler `handler` below covers the workflow_runs @@ -42,7 +40,7 @@ export { runShipFromCommand } from "../ship/session-runner"; * handles the terminal transition. */ export const handler: WorkflowHandler = async (ctx) => { - const { target, logger: log, runId: parentRunId, deliveryId, daemonId } = ctx; + const { target, logger: log } = ctx; try { if (target.type !== "issue") { @@ -56,6 +54,7 @@ export const handler: WorkflowHandler = async (ctx) => { target, octokit: ctx.octokit, logger: log, + stepRuns: ctx.shipStepRuns ?? {}, }); const firstStep = steps[startIndex]; @@ -69,71 +68,39 @@ export const handler: WorkflowHandler = async (ctx) => { return { status: "failed", reason: "ship: computed startIndex out of range" }; } - const child = await insertChildRow({ - parentRunId, - parentStepIndex: startIndex, - workflowName: firstStep, - target, - deliveryId: deliveryId ?? null, - daemonId, - }); - - // First child step uses its own runId as deliveryId so the `executions` - // row doesn't collide with the parent's webhook-scoped deliveryId. - const childDeliveryId = child.id; - await recordWorkflowExecution({ - deliveryId: childDeliveryId, - target, - senderLogin: "chrisleekr-bot[bot]", - workflowName: firstStep, - runId: child.id, - logger: log, - }); - await enqueueJob({ - kind: "workflow-run", - deliveryId: childDeliveryId, - repoOwner: target.owner, - repoName: target.repo, - entityNumber: target.number, - isPR: false, - eventName: "issues", - triggerUsername: "chrisleekr-bot[bot]", - labels: [], - triggerBodyPreview: "", - enqueuedAt: Date.now(), - retryCount: 0, - workflowRun: { - runId: child.id, - workflowName: firstStep, - parentRunId, - parentStepIndex: startIndex, - }, - }); - const state = { currentStepIndex: startIndex, stepRuns: priorRunIds, - handedOffTo: child.id, }; + if (ctx.handOffChild === undefined) { + throw new Error("ship requires an attempt-scoped hand-off operation"); + } const humanMessage = startIndex === 0 ? `ship started, first step \`${firstStep}\` queued.` : `ship resumed at step ${String(startIndex)} (\`${firstStep}\`); ${String(priorRunIds.length)} prior step(s) reused.`; - - await ctx.setState(state, humanMessage); + const handOff = await ctx.handOffChild({ + workflowName: firstStep, + target, + parentStepIndex: startIndex, + state, + humanMessage, + }); + const committedState = { ...state, handedOffTo: handOff.childRunId }; log.info( - { startIndex, firstStep, childRunId: child.id, priorRunIds }, + { startIndex, firstStep, childRunId: handOff.childRunId, priorRunIds }, "ship handler handed off to first child", ); return { status: "handed-off", - state, + state: committedState, humanMessage, - childRunId: child.id, + childRunId: handOff.childRunId, }; } catch (err) { + if (err instanceof StaleWorkflowAttemptError) throw err; const message = err instanceof Error ? err.message : String(err); log.warn({ err }, "ship handler caught error"); return { status: "failed", reason: `ship failed: ${message}` }; @@ -145,23 +112,22 @@ interface ComputeStartIndexParams { readonly target: WorkflowRunContext["target"]; readonly octokit: WorkflowRunContext["octokit"]; readonly logger: WorkflowRunContext["logger"]; + readonly stepRuns: Readonly>>; } async function computeStartIndex(params: ComputeStartIndexParams): Promise<{ startIndex: number; priorRunIds: string[]; }> { - const { steps, target, octokit, logger } = params; + const { steps, target, octokit, logger, stepRuns } = params; const priorRunIds: string[] = []; - const targetKey = { owner: target.owner, repo: target.repo, number: target.number }; let triageCreatedAt: Date | null = null; for (let i = 0; i < steps.length; i++) { const step = steps[i]; if (step === undefined) continue; - // eslint-disable-next-line no-await-in-loop -- sequential by design - const latest = await findLatestForTarget(step, targetKey); + const latest = stepRuns[step] ?? null; // eslint-disable-next-line no-await-in-loop -- sequential by design const fresh = await isFresh({ @@ -179,7 +145,7 @@ async function computeStartIndex(params: ComputeStartIndexParams): Promise<{ if (latest !== null) { priorRunIds.push(latest.id); - if (step === "triage") triageCreatedAt = latest.created_at; + if (step === "triage") triageCreatedAt = new Date(latest.createdAt); } } @@ -191,7 +157,7 @@ async function computeStartIndex(params: ComputeStartIndexParams): Promise<{ interface IsFreshParams { readonly step: WorkflowName; - readonly latest: WorkflowRunRow | null; + readonly latest: WorkflowRunSnapshot | null; readonly triageCreatedAt: Date | null; readonly octokit: WorkflowRunContext["octokit"]; readonly owner: string; @@ -207,16 +173,16 @@ async function isFresh(params: IsFreshParams): Promise { if (step === "review" || step === "resolve") return false; if (step === "triage") { - return latest.state["recommendedNext"] === "plan"; + return latest.state.recommendedNext === "plan"; } if (step === "plan") { if (triageCreatedAt === null) return false; - return new Date(latest.created_at).getTime() > new Date(triageCreatedAt).getTime(); + return new Date(latest.createdAt).getTime() > triageCreatedAt.getTime(); } if (step === "implement") { - const prNumber = latest.state["pr_number"]; + const prNumber = latest.state.pr_number; if (typeof prNumber !== "number") return false; try { const { data: pr } = await octokit.rest.pulls.get({ @@ -236,32 +202,3 @@ async function isFresh(params: IsFreshParams): Promise { return true; } - -interface InsertChildParams { - readonly parentRunId: string; - readonly parentStepIndex: number; - readonly workflowName: WorkflowName; - readonly target: WorkflowRunContext["target"]; - readonly deliveryId: string | null; - readonly daemonId: string; -} - -async function insertChildRow(params: InsertChildParams): Promise { - const sql = requireDb(); - const rows: WorkflowRunRow[] = await sql` - INSERT INTO workflow_runs ( - workflow_name, target_type, target_owner, target_repo, target_number, - parent_run_id, parent_step_index, status, state, delivery_id, - owner_kind, owner_id - ) VALUES ( - ${params.workflowName}, ${params.target.type}, ${params.target.owner}, - ${params.target.repo}, ${params.target.number}, - ${params.parentRunId}, ${params.parentStepIndex}, 'queued', '{}'::jsonb, - ${params.deliveryId}, 'daemon', ${params.daemonId} - ) - RETURNING * - `; - const row = rows[0]; - if (row === undefined) throw new Error("insertChildRow: INSERT returned no row"); - return row; -} diff --git a/src/workflows/handlers/triage.ts b/src/workflows/handlers/triage.ts index 579d38b2..63fed41a 100644 --- a/src/workflows/handlers/triage.ts +++ b/src/workflows/handlers/triage.ts @@ -5,11 +5,13 @@ import { z } from "zod"; import { parseStructuredResponse } from "../../ai/structured-output"; import { config } from "../../config"; +import { applyAgentPolicy } from "../../core/agent-policy"; import { checkoutRepo } from "../../core/checkout"; import { executeAgent } from "../../core/executor"; import type { BotContext } from "../../types"; import { fetchAndBuildDigest, renderDigestSection } from "../discussion-digest"; import type { WorkflowHandler } from "../registry"; +import { StaleWorkflowAttemptError } from "../runs-store"; /** * `triage` handler: code-aware validation of an issue against the actual @@ -32,9 +34,10 @@ import type { WorkflowHandler } from "../registry"; * 5. The full `TRIAGE.md` report is the tracking comment body so the user * sees evidence, reasoning, and reproduction details: not a one-liner. * - * Reproduction: there is NO turn cap. A senior engineer's job is to - * determine whether a reported bug is real; that requires running the code, - * not just reading it. If reproduction is impossible (e.g., production-only, + * Reproduction: determining whether a bug is real requires running the code, + * not just reading it, so there is no default turn cap. A cap arrives only if + * the repo, `AGENT_MAX_TURNS`, or `DEFAULT_MAXTURNS` sets one; all are unset + * by default. If reproduction is impossible (e.g., production-only, * needs external services we lack), the agent reports * `attempted: true, reproduced: null` with honest details, never lies. * Non-bug issues (features, refactors, docs) skip reproduction with @@ -97,6 +100,7 @@ type Verdict = z.infer; export const handler: WorkflowHandler = async (ctx) => { const { octokit, target, logger: log } = ctx; let cleanup: (() => Promise) | undefined; + let disposePolicy: (() => void) | undefined; try { if (target.type !== "issue") { @@ -154,17 +158,33 @@ export const handler: WorkflowHandler = async (ctx) => { const promptParts = config.promptCacheLayout === "cacheable" ? buildTriagePromptParts(promptInput) : undefined; + // Bypasses `runPipeline` (owns its prompt), so it applies the Gate-2 knobs itself. + const applied = applyAgentPolicy({ + baseAllowedTools: ["Read", "Grep", "Glob", "Bash", "Write"], + ...(ctx.policy !== undefined ? { policy: ctx.policy } : {}), + ...(ctx.maxTurns !== undefined ? { maxTurns: ctx.maxTurns } : {}), + ...(ctx.signal !== undefined ? { signal: ctx.signal } : {}), + }); + disposePolicy = applied.dispose; + const result = await executeAgent({ ctx: botCtx, prompt, mcpServers: {}, workDir: checkout.workDir, - allowedTools: ["Read", "Grep", "Glob", "Bash", "Write"], + ...applied.options, ...(promptParts !== undefined ? { promptParts } : {}), }); + applied.options.signal?.throwIfAborted(); if (!result.success) { - return { status: "failed", reason: "triage agent execution failed" }; + // `reason` is internal only; the raw error must not reach the public + // `humanMessage`. Keeping it preserves the per-repo `timeout:` attribution. + return { + status: "failed", + reason: result.errorMessage ?? "triage agent execution failed", + humanMessage: "triage agent execution failed, see server logs for details.", + }; } const reportPath = join(checkout.workDir, "TRIAGE.md"); @@ -232,10 +252,13 @@ export const handler: WorkflowHandler = async (ctx) => { return { status: "succeeded", state, humanMessage }; } catch (err) { + if (err instanceof StaleWorkflowAttemptError) throw err; const message = err instanceof Error ? err.message : String(err); log.warn({ err }, "triage handler caught error"); return { status: "failed", reason: `triage failed: ${message}` }; } finally { + // Before the awaited cleanup: a live timer holds Bun's event loop open. + disposePolicy?.(); if (cleanup !== undefined) { await cleanup().catch((err: unknown) => { log.warn({ err }, "triage handler cleanup failed"); @@ -360,7 +383,8 @@ function buildTriageMethodAndRules(): string { ` \`reproduced=false\` → output contradicts the claim. The issue is wrong (already-fixed, misread, env-only), mark VALID=false.`, ` \`reproduced=null\` → ONLY after walking 3b, 3c, AND 3d and ruling each out. State which you tried and why each failed to either show the defect or pin down a fix-relevant invariant. "Race condition" alone is NOT sufficient, races almost always have an invariant test (3d).`, ``, - ` f. There is NO turn cap. /tmp scratch is fine; do not commit anything.`, + // Byte-stable for the cacheable `append`: states the default, not the run's cap. + ` f. There is no default turn cap, though a repo or the operator may configure one. Work efficiently and record partial findings in TRIAGE.md as you go. /tmp scratch is fine; do not commit anything.`, ``, ` For non-bug classes (feature/refactor/docs/unclear): set \`attempted=false\`, \`reproduced=null\`, \`details\` to a one-liner explaining why reproduction was skipped.`, ``, @@ -476,6 +500,7 @@ async function postStartingComment( try { await ctx.setState({ phase: "starting" }, body); } catch (err) { + if (err instanceof StaleWorkflowAttemptError) throw err; ctx.logger.warn( { err: err instanceof Error ? err.message : String(err) }, "triage starting-comment write failed, continuing without up-front comment", diff --git a/src/workflows/intent-classifier.ts b/src/workflows/intent-classifier.ts index 9204dd54..b93daf73 100644 --- a/src/workflows/intent-classifier.ts +++ b/src/workflows/intent-classifier.ts @@ -140,7 +140,6 @@ export async function classify( system: withStructuredRules(SYSTEM_PROMPT), messages: [{ role: "user", content: buildUserMessage(commentBody) }], maxTokens: config.triageMaxTokens, - temperature: 0, }); rawText = response.text; } catch (err) { diff --git a/src/workflows/log-fields.ts b/src/workflows/log-fields.ts index a843067d..e6a39ab5 100644 --- a/src/workflows/log-fields.ts +++ b/src/workflows/log-fields.ts @@ -21,7 +21,7 @@ * (`duration_ms`) are snake_case. `deliveryId` is optional because system-spawned * runs (ship iteration, orchestrator cascade children) have no originating * webhook delivery. `duration_ms` is carried only on terminal events that own a - * start timestamp (the daemon executor's `startedAt`). + * start timestamp supplied by the execution owner. */ import { z } from "zod"; @@ -62,7 +62,7 @@ export const WorkflowRunLogFieldsSchema = z.union([ target, deliveryId, }), - /** Info: the daemon flipped the row to `running` and took ownership. */ + /** Info: an execution owner flipped the row to `running`. */ z.strictObject({ event: z.literal(WORKFLOW_RUN_LOG_EVENTS.running), runId, @@ -121,9 +121,8 @@ export const WorkflowRunLogFieldsSchema = z.union([ reason: z.string(), }), /** - * Error: the post-insert enqueue/publish failed; the compensating - * `markFailed` released the in-flight guard. `runId` exists (the row was - * inserted then failed). + * Error: the first post-commit publication failed. The queued row and its + * in-flight guard remain durable while the outbox retries. `runId` exists. */ z.strictObject({ event: z.literal(WORKFLOW_RUN_LOG_EVENTS.enqueueFailed), @@ -207,8 +206,8 @@ export function logWorkflowRunIncomplete( /** * Terminal failure. `level` distinguishes a handler-reported failure (`warn`) - * from an uncaught throw (`error`); the daemon executor logs the throw at - * `error` with the standard `err` field, this event mirrors that level. + * from an uncaught throw (`error`); the execution owner logs the throw with + * the standard `err` field, this event mirrors that level. */ export function logWorkflowRunFailed( log: Logger, @@ -258,7 +257,7 @@ export function logWorkflowRunDispatchRefused( ); } -/** Error: post-insert enqueue failed; the in-flight guard was released. */ +/** Error: initial publication failed; the durable outbox retains the queued run. */ export function logWorkflowRunEnqueueFailed( log: Logger, fields: BaseFields & { reason: string }, diff --git a/src/workflows/orchestrator.ts b/src/workflows/orchestrator.ts index 4292b7c2..99ba5037 100644 --- a/src/workflows/orchestrator.ts +++ b/src/workflows/orchestrator.ts @@ -5,18 +5,17 @@ import type pino from "pino"; import { config } from "../config"; import { requireDb } from "../db"; import { getIntentById } from "../db/queries/ship"; -import { getInstanceId } from "../orchestrator/instance-id"; -import { enqueueJob } from "../orchestrator/job-queue"; import { requireValkeyClient } from "../orchestrator/valkey"; import { isSessionTerminalState } from "../shared/ship-types"; import { addReaction, type ReactionContent } from "../utils/reactions"; +import { publishWorkflowRunById } from "./dispatch-outbox"; import { recordWorkflowExecution } from "./execution-row"; import { logWorkflowRunEnqueueFailed } from "./log-fields"; import { getByName, type WorkflowName } from "./registry"; -import { findById, markFailed, type WorkflowRunRow } from "./runs-store"; +import { findById, type WorkflowRunRow } from "./runs-store"; import { TICKLE_KEY } from "./ship/webhook-reactor"; import { extractShipIntentId } from "./ship/workflow-context"; -import { setState } from "./tracking-mirror"; +import { LAST_HUMAN_MESSAGE_KEY, setState } from "./tracking-mirror"; /** * Composite-workflow hand-off engine. Runs as the LAST step of every @@ -42,8 +41,10 @@ export interface CompletionResult { } export interface OnStepCompleteDeps { - readonly octokit: Octokit; + readonly octokit: Octokit | null; readonly logger: pino.Logger; + readonly emitGitHub?: boolean; + readonly sql?: SQL; } export async function onStepComplete( @@ -51,14 +52,14 @@ export async function onStepComplete( childRunId: string, result: CompletionResult, ): Promise { - const db = requireDb(); + const db = deps.sql ?? requireDb(); const { logger } = deps; // Ship-iteration early-wake hook: when the just-completed child carries a // `shipIntentId` in its state JSONB, ZADD `ship:tickle` so the scheduler // re-enters the intent on the next tick. Runs BEFORE the parent-cascade // logic because ship-iteration runs may have no `parent_run_id`. - await maybeEarlyWakeShipIntent(childRunId, logger); + await maybeEarlyWakeShipIntent(childRunId, logger, db); // Capture anything the transaction wants to emit AFTER commit so the DB // is the source of truth for enqueued jobs and GitHub API calls. The @@ -81,9 +82,21 @@ export async function onStepComplete( logger.warn({ childRunId, parentId: child.parent_run_id }, "onStepComplete: parent missing"); return; } + if (parent.status !== "running") { + logger.info( + { childRunId, parentId: parent.id, parentStatus: parent.status }, + "onStepComplete replayed a child of a terminal parent", + ); + const terminal = terminalProjectionFromParent(parent); + if (terminal !== null) { + postCommit = { enqueue: null, parentRunId: parent.id, parentTerminal: terminal }; + } + return; + } const steps = getByName(parent.workflow_name).steps; const childStepIndex = child.parent_step_index ?? -1; + if (extractStepRuns(parent.state).includes(childRunId)) return; if (result.status === "failed") { // The executor edge maps `HandlerResult.status === "incomplete"` to a @@ -97,19 +110,21 @@ export async function onStepComplete( const rawReason = isIncomplete ? (result.reason ?? "").slice("incomplete:".length).trim() : (result.reason ?? "child failed"); + const humanMessage = isIncomplete + ? `ship halted at step ${String(childStepIndex)} (${parent.workflow_name} → ${child.workflow_name}), ${child.workflow_name} returned incomplete; see PR tracking comment for outstanding items.` + : `ship halted at step ${String(childStepIndex)} (${parent.workflow_name} → ${child.workflow_name}), see server logs for details.`; const failPatch = { failedAtStepIndex: childStepIndex, failedReason: rawReason || "child failed", + [LAST_HUMAN_MESSAGE_KEY]: humanMessage, }; await tx` UPDATE workflow_runs SET status = 'failed', state = state || ${failPatch}::jsonb WHERE id = ${parent.id} + AND status = 'running' `; - const humanMessage = isIncomplete - ? `ship halted at step ${String(childStepIndex)} (${parent.workflow_name} → ${child.workflow_name}), ${child.workflow_name} returned incomplete; see PR tracking comment for outstanding items.` - : `ship halted at step ${String(childStepIndex)} (${parent.workflow_name} → ${child.workflow_name}), see server logs for details.`; postCommit = { enqueue: null, parentRunId: parent.id, @@ -118,8 +133,8 @@ export async function onStepComplete( // Do NOT inline `result.reason` for the generic-failed branch, // handlers persist raw error strings in `reason` for DB+log // forensics and that surface is public on GitHub. The - // `incomplete:` prefix is a private executor-internal marker - // (set by `workflow-executor.ts`); the message above does not + // `incomplete:` prefix is a private cascade marker set by + // `completion-reconciler.ts`; the message above does not // include the reason payload itself, only flags the class. humanMessage, }, @@ -173,25 +188,27 @@ export async function onStepComplete( const shouldLoopBackToReview = isShipParent && isResolveChild && reviewIterations < cap; if (reviewClean || (nextIndex >= steps.length && !shouldLoopBackToReview)) { + let humanMessage = `ship complete, all ${String(steps.length)} steps succeeded.`; + if (reviewClean) { + humanMessage = `ship complete, review found no issues after ${String(reviewIterations)} iterations.`; + } else if (isShipParent && reviewIterations >= cap && lastReviewFindings > 0) { + humanMessage = `ship complete, review-${String(reviewIterations)} flagged ${String(lastReviewFindings)} issue${ + lastReviewFindings === 1 ? "" : "s" + }; resolve-${String(reviewIterations)} attempted fixes. Manual re-review recommended.`; + } const successPatch = { currentStepIndex: nextIndex, stepRuns, ...reviewLoopState, + [LAST_HUMAN_MESSAGE_KEY]: humanMessage, }; await tx` UPDATE workflow_runs SET status = 'succeeded', state = state || ${successPatch}::jsonb WHERE id = ${parent.id} + AND status = 'running' `; - let humanMessage = `ship complete, all ${String(steps.length)} steps succeeded.`; - if (reviewClean) { - humanMessage = `ship complete, review found no issues after ${String(reviewIterations)} iterations.`; - } else if (isShipParent && reviewIterations >= cap && lastReviewFindings > 0) { - humanMessage = `ship complete, review-${String(reviewIterations)} flagged ${String(lastReviewFindings)} issue${ - lastReviewFindings === 1 ? "" : "s" - }; resolve-${String(reviewIterations)} attempted fixes. Manual re-review recommended.`; - } postCommit = { enqueue: null, parentRunId: parent.id, @@ -215,39 +232,43 @@ export async function onStepComplete( // loop-back iterations). const targetResult = deriveChildTarget(parent, child, nextStepName); if ("error" in targetResult) { + const humanMessage = `ship halted at step ${String(nextStepIndex)} (${parent.workflow_name} → ${nextStepName}): ${targetResult.error}`; const failPatch = { failedAtStepIndex: nextStepIndex, failedReason: targetResult.error, ...reviewLoopState, + [LAST_HUMAN_MESSAGE_KEY]: humanMessage, }; await tx` UPDATE workflow_runs SET status = 'failed', state = state || ${failPatch}::jsonb WHERE id = ${parent.id} + AND status = 'running' `; postCommit = { enqueue: null, parentRunId: parent.id, parentTerminal: { status: "failed", - humanMessage: `ship halted at step ${String(nextStepIndex)} (${parent.workflow_name} → ${nextStepName}): ${targetResult.error}`, + humanMessage, }, }; return; } const childTarget = targetResult; + const nextChildId = crypto.randomUUID(); const inserted: WorkflowRunRow[] = await tx` INSERT INTO workflow_runs ( - workflow_name, target_type, target_owner, target_repo, target_number, + id, workflow_name, target_type, target_owner, target_repo, target_number, parent_run_id, parent_step_index, status, state, delivery_id, - owner_kind, owner_id + execution_delivery_id, owner_kind, owner_id ) VALUES ( - ${nextStepName}, ${childTarget.type}, ${childTarget.owner}, + ${nextChildId}, ${nextStepName}, ${childTarget.type}, ${childTarget.owner}, ${childTarget.repo}, ${childTarget.number}, ${parent.id}, ${nextStepIndex}, 'queued', '{}'::jsonb, ${parent.delivery_id}, - 'orchestrator', ${getInstanceId()} + ${nextChildId}, NULL, NULL ) RETURNING * `; @@ -255,6 +276,15 @@ export async function onStepComplete( if (nextChild === undefined) { throw new Error("orchestrator: failed to insert next child row"); } + await recordWorkflowExecution({ + deliveryId: nextChild.id, + target: childTarget, + senderLogin: config.botAppLogin, + workflowName: nextStepName, + runId: nextChild.id, + logger, + sql: tx, + }); const progressPatch: Record = { currentStepIndex: nextStepIndex, @@ -269,7 +299,8 @@ export async function onStepComplete( await tx` UPDATE workflow_runs SET state = state || ${progressPatch}::jsonb - WHERE id = ${parent.id} + WHERE id = ${parent.id} + AND status = 'running' `; postCommit = { @@ -288,52 +319,13 @@ export async function onStepComplete( if (postCommit.enqueue !== null) { const job = postCommit.enqueue; - // Cascade steps use `runId` as deliveryId to avoid collision with the - // parent's `executions.delivery_id` (UNIQUE NOT NULL). Parent's original - // webhook deliveryId is retained on the workflow_runs row for traceability - // but is NOT reused as the per-step executions key. - const childDeliveryId = job.runId; try { - await recordWorkflowExecution({ - deliveryId: childDeliveryId, - target: job.target, - senderLogin: config.botAppLogin, - workflowName: job.workflowName, - runId: job.runId, - logger, - }); - await enqueueJob({ - kind: "workflow-run", - deliveryId: childDeliveryId, - repoOwner: job.target.owner, - repoName: job.target.repo, - entityNumber: job.target.number, - isPR: job.target.type === "pr", - eventName: job.target.type === "pr" ? "pull_request" : "issues", - triggerUsername: config.botAppLogin, - labels: [], - triggerBodyPreview: "", - enqueuedAt: Date.now(), - retryCount: 0, - workflowRun: { - runId: job.runId, - workflowName: job.workflowName, - parentRunId: job.parentRunId, - parentStepIndex: job.parentStepIndex, - }, - }); + await publishWorkflowRunById(job.runId, db); logger.info( { nextRunId: job.runId, nextWorkflow: job.workflowName, parentId: job.parentRunId }, "orchestrator enqueued next step", ); } catch (err) { - // Compensation: the transaction committed a `queued` child row and - // mutated the parent's `state`, but Valkey was unreachable (or the - // publish rejected the payload). Without this branch the child would - // sit `queued` forever, and the partial unique index would block any - // retry for the same (workflow, target). Mark both rows `failed` - // BEFORE returning so the index releases and the operator gets a - // breadcrumb on the tracking comment. const reason = `enqueue failed: ${err instanceof Error ? err.message : String(err)}`; logWorkflowRunEnqueueFailed(logger, { runId: job.runId, @@ -344,50 +336,31 @@ export async function onStepComplete( }); logger.error( { err, nextRunId: job.runId, parentId: job.parentRunId }, - "orchestrator post-commit enqueue failed, compensating by marking child and parent failed", + "orchestrator post-commit enqueue failed, reconciliation will retry", ); - await markFailed(job.runId, reason).catch((markErr: unknown) => { - logger.error( - { err: markErr, nextRunId: job.runId }, - "compensation: failed to mark child row as failed", - ); - }); - await markFailed(job.parentRunId, reason, { - failedAtStepIndex: job.parentStepIndex, - }).catch((markErr: unknown) => { - logger.error( - { err: markErr, parentId: job.parentRunId }, - "compensation: failed to mark parent row as failed", - ); - }); - // Surface via the parent's tracking comment so the operator sees - // the failure without needing to tail logs. - postCommit = { - ...postCommit, - parentTerminal: { - status: "failed", - humanMessage: `ship halted at step ${String(job.parentStepIndex)} (${job.workflowName}): enqueue failed, see daemon logs for retry.`, - }, - }; } } - if (postCommit.parentTerminal !== null && postCommit.parentRunId !== null) { + if ( + postCommit.parentTerminal !== null && + postCommit.parentRunId !== null && + deps.emitGitHub !== false && + deps.octokit !== null + ) { const parentRunId = postCommit.parentRunId; const terminal = postCommit.parentTerminal; - await setState(deps, { + const trackingDeps = { octokit: deps.octokit, logger }; + await setState(trackingDeps, { runId: parentRunId, patch: {}, humanMessage: terminal.humanMessage, - }).catch((err: unknown) => { - logger.warn({ err, parentId: parentRunId }, "parent tracking emit failed"); }); // Composite parents (e.g., ship) terminate here, not in the daemon // executor, so this is the right point to react on the user's trigger // comment with the chain's final outcome. await reactOnParentTrigger( - deps, + trackingDeps, parentRunId, terminal.status === "succeeded" ? "hooray" : "confused", ); @@ -407,9 +380,8 @@ export async function onStepComplete( * @param log Pino logger used by the cascade caller. */ /** - * Read `state.failedReason` written by `markFailed` from a workflow_runs - * state blob. Returns `undefined` when the row was not marked failed by - * `markFailed` (e.g. legacy rows or non-failure states). + * Read `state.failedReason` from a terminal workflow_runs state blob. + * Returns `undefined` for legacy rows or non-failure states. */ export function extractFailedReason(state: unknown): string | undefined { if (state === null || typeof state !== "object") return undefined; @@ -493,8 +465,11 @@ export function detectTransientQuotaError( return { retryAtMs: nowMs + QUOTA_FALLBACK_RETRY_DELAY_MS, resetPhrase: "fallback_1h" }; } -async function maybeEarlyWakeShipIntent(childRunId: string, log: pino.Logger): Promise { - const db = requireDb(); +async function maybeEarlyWakeShipIntent( + childRunId: string, + log: pino.Logger, + db: SQL, +): Promise { const rows: { state: Record; status: string }[] = await db` SELECT state, status FROM workflow_runs WHERE id = ${childRunId} `; @@ -610,7 +585,7 @@ async function maybeEarlyWakeShipIntent(childRunId: string, log: pino.Logger): P } async function reactOnParentTrigger( - deps: OnStepCompleteDeps, + deps: { octokit: Octokit; logger: pino.Logger }, parentRunId: string, content: ReactionContent, ): Promise { @@ -648,6 +623,18 @@ interface PostCommitActions { parentTerminal: { status: "succeeded" | "failed"; humanMessage: string } | null; } +function terminalProjectionFromParent(parent: WorkflowRunRow): PostCommitActions["parentTerminal"] { + if (parent.status !== "succeeded" && parent.status !== "failed") return null; + const stored = parent.state[LAST_HUMAN_MESSAGE_KEY]; + const humanMessage = + typeof stored === "string" && stored.trim() !== "" + ? stored + : parent.status === "succeeded" + ? `${parent.workflow_name} succeeded.` + : `${parent.workflow_name} failed, see server logs for details.`; + return { status: parent.status, humanMessage }; +} + function extractStepRuns(state: Record): string[] { const raw = state["stepRuns"]; if (!Array.isArray(raw)) return []; diff --git a/src/workflows/registry.ts b/src/workflows/registry.ts index 728a3369..a56b5fbf 100644 --- a/src/workflows/registry.ts +++ b/src/workflows/registry.ts @@ -3,6 +3,15 @@ import type pino from "pino"; import { z } from "zod"; import type { ReviewLearningPayload } from "../mcp/registry"; +import { + type HandlerResult, + type PriorPlanState, + type RepoMemoryEntry, + type WorkflowName, + WorkflowNameSchema, + type WorkflowRunSnapshot, +} from "../shared/workflow-types"; +import type { AgentPolicy } from "../shared/ws-messages"; import { handler as implementHandler } from "./handlers/implement"; import { handler as planHandler } from "./handlers/plan"; import { handler as rememberHandler } from "./handlers/remember"; @@ -11,79 +20,13 @@ import { handler as reviewHandler } from "./handlers/review"; import { handler as shipHandler } from "./handlers/ship"; import { handler as triageHandler } from "./handlers/triage"; -export const WorkflowNameSchema = z.enum([ - "triage", - "plan", - "implement", - "review", - "resolve", - "ship", - "remember", -]); -export type WorkflowName = z.infer; +export { WorkflowNameSchema }; +export { HandlerResultSchema } from "../shared/workflow-types"; +export type { HandlerResult, WorkflowName }; export const WorkflowContextSchema = z.enum(["issue", "pr", "both"]); export type WorkflowContext = z.infer; -/** - * `humanMessage` lets the handler supply the exact body the executor should - * render into the tracking comment alongside the terminal status header. - * When omitted, the executor falls back to a generic " " - * line. Handlers that already wrote a rich message via `ctx.setState` during - * execution should repeat it here so the final replace-write preserves it. - * - * `handed-off` is the composite-workflow variant: the handler inserted and - * enqueued a child run and wants the parent row to stay `running` until the - * last child completes (FR-006, handoff-protocol.md §Parent status). The - * executor merges `state` into the parent's row but does NOT transition the - * parent's `status`: the orchestrator's cascade does that on the final - * child's completion. - * - * `incomplete` is the "agent ran cleanly but work remains" terminal state - * (issue #93). The pipeline returned `success: true`, but a handler-side - * post-execution gate (e.g., the `resolve` handler's CI re-check) found - * surviving failures: typically when the agent hit `FIX_ATTEMPTS_CAP=3` - * with red CI. Distinct from `failed` so downstream surfaces can tell a - * clean-run-but-blocked outcome from a true pipeline error. - */ -/** - * Review-learning IDs the review/resolve handler's runPipeline actually - * applied to the prompt (post file-glob filter). Forwarded by workflow- - * executor.ts into the `job:result` payload's `appliedReviewLearningIds` - * field so the orchestrator can bump `use_count` + `last_used_at`. Only - * `review` and `resolve` populate this; other handlers leave it omitted. - */ -const appliedReviewLearningIdsField = z.array(z.string().max(64)).max(50).optional(); - -export const HandlerResultSchema = z.discriminatedUnion("status", [ - z.object({ - status: z.literal("succeeded"), - state: z.unknown(), - humanMessage: z.string().min(1).optional(), - appliedReviewLearningIds: appliedReviewLearningIdsField, - }), - z.object({ - status: z.literal("failed"), - reason: z.string().min(1), - state: z.unknown().optional(), - humanMessage: z.string().min(1).optional(), - }), - z.object({ - status: z.literal("incomplete"), - reason: z.string().min(1), - state: z.unknown().optional(), - humanMessage: z.string().min(1).optional(), - appliedReviewLearningIds: appliedReviewLearningIdsField, - }), - z.object({ - status: z.literal("handed-off"), - state: z.unknown().optional(), - humanMessage: z.string().min(1).optional(), - childRunId: z.string().min(1), - }), -]); -export type HandlerResult = z.infer; - export interface WorkflowRunContext { readonly runId: string; readonly workflowName: WorkflowName; @@ -100,14 +43,17 @@ export interface WorkflowRunContext { readonly logger: pino.Logger; readonly octokit: Octokit; readonly deliveryId: string | null; - /** - * Identifier of the daemon process executing this run. Handlers writing - * new `workflow_runs` rows from inside the daemon (e.g., the `ship` - * composite spawning child runs) must set `owner_kind='daemon', - * owner_id=daemonId` so the liveness reaper can detect inserter death - * before the row reaches a downstream handler. - */ + /** Stable isolated-runner identity used for logs and execution context. */ readonly daemonId: string; + readonly signal?: AbortSignal; + /** Commit a composite child and parent hand-off under this attempt's lease. */ + readonly handOffChild?: (input: { + readonly workflowName: WorkflowName; + readonly target: WorkflowRunContext["target"]; + readonly parentStepIndex: number; + readonly state: Record; + readonly humanMessage: string; + }) => Promise<{ childRunId: string }>; /** * Orchestrator pre-loaded review learnings for this job, when any. Only the * `review` and `resolve` handlers read this and forward to `runPipeline` @@ -117,7 +63,29 @@ export interface WorkflowRunContext { * the orchestrator's load returned zero rows. */ readonly reviewLearnings?: ReviewLearningPayload[]; - readonly setState: (state: unknown, humanMessage: string) => Promise; + /** Bounded repository hints loaded by the orchestrator. */ + readonly repoMemory?: RepoMemoryEntry[]; + /** + * Per-repo agent policy from `.github-app.yaml` ("Gate 2"), already clamped + * against the server ceilings by the orchestrator, and applied via + * `applyAgentPolicy`. Undefined when the repo ships no config file. + */ + readonly policy?: AgentPolicy; + /** + * Turn cap for this run, already resolved by the orchestrator as + * `workflows..max_turns ?? AGENT_MAX_TURNS ?? DEFAULT_MAXTURNS` and + * clamped against the server ceiling. Separate from `policy` because it + * rides the isolated runner payload as the single source of truth. + */ + readonly maxTurns?: number; + /** Most recent succeeded plan, preloaded for the implement workflow. */ + readonly priorPlanState?: Readonly; + /** Latest row per ship step, preloaded so the runner never queries PostgreSQL. */ + readonly shipStepRuns?: Readonly>>; + readonly setState: ( + state: unknown, + humanMessage: string, + ) => Promise<{ trackingCommentId?: number }>; } export type WorkflowHandler = (ctx: WorkflowRunContext) => Promise; diff --git a/src/workflows/runs-store.ts b/src/workflows/runs-store.ts index 29867699..8883cad0 100644 --- a/src/workflows/runs-store.ts +++ b/src/workflows/runs-store.ts @@ -29,6 +29,20 @@ export interface WorkflowRunRow { delivery_id: string | null; owner_kind: WorkflowOwnerKind | null; owner_id: string | null; + attempt_id: string | null; + lease_expires_at: Date | null; + attempt_deadline_at: Date | null; + attempt_completed_at: Date | null; + cascade_completed_at: Date | null; + execution_delivery_id: string | null; + trigger_body_preview: string; + dispatch_enqueued_at: Date | null; + dispatch_generation_id: string; + dispatch_retry_count: number; + runner_payload_issued_at: Date | null; + runner_token_expires_at: Date | null; + runner_resources_cleaned_at: Date | null; + failure_notified_at: Date | null; trigger_comment_id: number | null; trigger_event_type: TriggerEventType | null; created_at: Date; @@ -64,6 +78,8 @@ export interface InsertQueuedParams { parentRunId?: string | null; parentStepIndex?: number | null; deliveryId?: string | null; + executionDeliveryId?: string | null; + triggerBodyPreview?: string; initialState?: Record; /** * Identifier of the process responsible for advancing this row. The @@ -80,6 +96,18 @@ export interface InsertQueuedParams { triggerEventType?: TriggerEventType | null; } +export interface WorkflowAttempt { + readonly runId: string; + readonly attemptId: string; +} + +export class StaleWorkflowAttemptError extends Error { + constructor(attempt: WorkflowAttempt) { + super(`workflow attempt is no longer current: ${attempt.attemptId}`); + this.name = "StaleWorkflowAttemptError"; + } +} + /** * Insert a new `queued` row. Throws if the partial unique index rejects the * insert (another in-flight run for the same (workflow, target) exists). @@ -91,6 +119,8 @@ export async function insertQueued( const parentRunId = params.parentRunId ?? null; const parentStepIndex = params.parentStepIndex ?? null; const deliveryId = params.deliveryId ?? null; + const executionDeliveryId = params.executionDeliveryId ?? deliveryId; + const triggerBodyPreview = params.triggerBodyPreview ?? ""; const state = params.initialState ?? {}; const triggerCommentId = params.triggerCommentId ?? null; const triggerEventType = params.triggerEventType ?? null; @@ -99,12 +129,14 @@ export async function insertQueued( INSERT INTO workflow_runs ( workflow_name, target_type, target_owner, target_repo, target_number, parent_run_id, parent_step_index, status, state, delivery_id, - owner_kind, owner_id, trigger_comment_id, trigger_event_type + execution_delivery_id, trigger_body_preview, owner_kind, owner_id, + trigger_comment_id, trigger_event_type ) VALUES ( ${params.workflowName}, ${params.target.type}, ${params.target.owner}, ${params.target.repo}, ${params.target.number}, ${parentRunId}, ${parentStepIndex}, 'queued', ${state}::jsonb, ${deliveryId}, - ${params.ownerKind}, ${params.ownerId}, ${triggerCommentId}, ${triggerEventType} + ${executionDeliveryId}, ${triggerBodyPreview}, ${params.ownerKind}, ${params.ownerId}, + ${triggerCommentId}, ${triggerEventType} ) RETURNING * `; @@ -116,80 +148,376 @@ export async function insertQueued( return normalizeRow(row); } -/** - * Flip a row to `running` and transfer ownership to the executing daemon so - * the liveness reaper tracks the daemon's heartbeat (not the orchestrator's) - * for the duration of the run. No-op if the row is already past `queued`. - */ -export async function markRunning( - runId: string, +function requireAttemptRow(rows: WorkflowRunRow[], attempt: WorkflowAttempt): WorkflowRunRow { + const row = rows[0]; + if (row === undefined) throw new StaleWorkflowAttemptError(attempt); + return normalizeRow(row); +} + +export async function renewWorkflowAttempts( daemonId: string, + attemptIds: readonly string[], + leaseMs: number, + sql: SQL = requireDb(), +): Promise<{ renewedAttemptIds: string[]; fencedAttemptIds: string[] }> { + if (attemptIds.length === 0) return { renewedAttemptIds: [], fencedAttemptIds: [] }; + const requested = [...new Set(attemptIds)]; + const rows: { attempt_id: string }[] = await sql` + UPDATE workflow_runs + SET lease_expires_at = LEAST( + attempt_deadline_at, + now() + ${leaseMs} * interval '1 millisecond' + ) + WHERE status = 'running' + AND owner_kind = 'daemon' + AND owner_id = ${daemonId} + AND attempt_id IN ${sql(requested)} + AND lease_expires_at > now() + AND attempt_deadline_at > now() + RETURNING attempt_id + `; + const renewed = new Set(rows.map((row) => row.attempt_id)); + return { + renewedAttemptIds: requested.filter((attemptId) => renewed.has(attemptId)), + fencedAttemptIds: requested.filter((attemptId) => !renewed.has(attemptId)), + }; +} + +export async function assertCurrentWorkflowAttempt( + attempt: WorkflowAttempt, sql: SQL = requireDb(), ): Promise { - await sql` + const rows: { current: number }[] = await sql` + SELECT 1 AS current + FROM workflow_runs + WHERE id = ${attempt.runId} + AND attempt_id = ${attempt.attemptId} + AND status = 'running' + AND lease_expires_at > now() + AND attempt_deadline_at > now() + `; + if (rows[0] === undefined) throw new StaleWorkflowAttemptError(attempt); +} + +export async function mergeAttemptState( + attempt: WorkflowAttempt, + patch: Record, + sql: SQL = requireDb(), +): Promise { + const rows: WorkflowRunRow[] = await sql` UPDATE workflow_runs - SET status = 'running', - owner_kind = 'daemon', - owner_id = ${daemonId} - WHERE id = ${runId} - AND status = 'queued' + SET state = state || ${patch}::jsonb + WHERE id = ${attempt.runId} + AND attempt_id = ${attempt.attemptId} + AND status = 'running' + AND lease_expires_at > now() + AND attempt_deadline_at > now() + RETURNING * `; + return requireAttemptRow(rows, attempt); } -/** - * Terminal success write. `state` is merged with the existing row's state - * (RHS wins on collision) via Postgres JSONB concat operator. - */ -export async function markSucceeded( - runId: string, +export async function markAttemptSucceeded( + attempt: WorkflowAttempt, state: Record, sql: SQL = requireDb(), -): Promise { - await sql` +): Promise { + const rows: WorkflowRunRow[] = await sql` UPDATE workflow_runs SET status = 'succeeded', - state = state || ${state}::jsonb - WHERE id = ${runId} + state = state || ${state}::jsonb, + lease_expires_at = NULL, + attempt_completed_at = now() + WHERE id = ${attempt.runId} + AND attempt_id = ${attempt.attemptId} + AND status = 'running' + AND lease_expires_at > now() + AND attempt_deadline_at > now() + RETURNING * `; + return requireAttemptRow(rows, attempt); } -/** - * Terminal failure write. Merges `{ reason, ...state }` into state. - */ -export async function markFailed( - runId: string, +export async function markAttemptFailed( + attempt: WorkflowAttempt, reason: string, state: Record = {}, sql: SQL = requireDb(), -): Promise { +): Promise { const merged = { ...state, failedReason: reason }; - await sql` + const rows: WorkflowRunRow[] = await sql` UPDATE workflow_runs SET status = 'failed', - state = state || ${merged}::jsonb - WHERE id = ${runId} + state = state || ${merged}::jsonb, + lease_expires_at = NULL, + attempt_completed_at = now() + WHERE id = ${attempt.runId} + AND attempt_id = ${attempt.attemptId} + AND status = 'running' + AND lease_expires_at > now() + AND attempt_deadline_at > now() + RETURNING * `; + return requireAttemptRow(rows, attempt); } -/** - * Terminal "agent ran cleanly but work remains" write (issue #93). Mirrors - * `markFailed` but flips status to `'incomplete'` and records the reason - * under `state.incompleteReason` (separate from `failedReason` so operator - * tooling can tell a clean-run-but-blocked outcome from a true pipeline error). - */ -export async function markIncomplete( - runId: string, +export async function markAttemptIncomplete( + attempt: WorkflowAttempt, reason: string, state: Record = {}, sql: SQL = requireDb(), -): Promise { +): Promise { const merged = { ...state, incompleteReason: reason }; - await sql` + const rows: WorkflowRunRow[] = await sql` UPDATE workflow_runs SET status = 'incomplete', - state = state || ${merged}::jsonb + state = state || ${merged}::jsonb, + lease_expires_at = NULL, + attempt_completed_at = now() + WHERE id = ${attempt.runId} + AND attempt_id = ${attempt.attemptId} + AND status = 'running' + AND lease_expires_at > now() + AND attempt_deadline_at > now() + RETURNING * + `; + return requireAttemptRow(rows, attempt); +} + +export interface AttemptHandOffChild { + readonly workflowName: WorkflowName; + readonly target: InsertQueuedParams["target"]; + readonly parentStepIndex: number; + readonly traceDeliveryId: string | null; + readonly childRunId?: string; +} + +/** Commit a composite hand-off only while the parent attempt lease is current. */ +export async function commitAttemptHandOffChild( + attempt: WorkflowAttempt, + state: Record, + child: AttemptHandOffChild, + sql: SQL, +): Promise { + const childRunId = child.childRunId ?? crypto.randomUUID(); + const parentState = { ...state, handedOffTo: childRunId }; + const parentRows: WorkflowRunRow[] = await sql` + UPDATE workflow_runs + SET state = state || ${parentState}::jsonb, + lease_expires_at = NULL, + attempt_completed_at = now() + WHERE id = ${attempt.runId} + AND attempt_id = ${attempt.attemptId} + AND status = 'running' + AND lease_expires_at > now() + AND attempt_deadline_at > now() + RETURNING * + `; + requireAttemptRow(parentRows, attempt); + + const rows: WorkflowRunRow[] = await sql` + INSERT INTO workflow_runs ( + id, workflow_name, target_type, target_owner, target_repo, target_number, + parent_run_id, parent_step_index, status, state, delivery_id, + execution_delivery_id, owner_kind, owner_id + ) VALUES ( + ${childRunId}, ${child.workflowName}, ${child.target.type}, ${child.target.owner}, + ${child.target.repo}, ${child.target.number}, ${attempt.runId}, + ${child.parentStepIndex}, 'queued', '{}'::jsonb, ${child.traceDeliveryId}, + ${childRunId}, NULL, NULL + ) + RETURNING * + `; + const row = rows[0]; + if (row === undefined) throw new Error("commitAttemptHandOffChild returned no child row"); + return normalizeRow(row); +} + +export async function expireWorkflowAttempts(sql: SQL = requireDb()): Promise { + const rows: WorkflowRunRow[] = await sql` + UPDATE workflow_runs + SET status = 'failed', + state = state || jsonb_build_object( + 'failedReason', CASE + WHEN attempt_deadline_at <= now() + THEN 'workflow execution deadline expired' + ELSE 'workflow execution lease expired' + END, + 'phase', CASE + WHEN attempt_deadline_at <= now() THEN 'deadline-expired' + ELSE 'lease-expired' + END + ), + lease_expires_at = NULL, + attempt_completed_at = now() + WHERE status = 'running' + AND attempt_id IS NOT NULL + AND lease_expires_at IS NOT NULL + AND attempt_deadline_at IS NOT NULL + AND LEAST(lease_expires_at, attempt_deadline_at) <= now() + RETURNING * + `; + return rows.map(normalizeRow); +} + +/** Fail queued workflow dispatches whose retry or wall-clock budget is exhausted. */ +export async function expireQueuedWorkflowDispatches( + maxAgeMs: number, + maxRetries: number, + sql: SQL = requireDb(), +): Promise { + const rows: WorkflowRunRow[] = await sql` + UPDATE workflow_runs + SET status = 'failed', + state = state || jsonb_build_object( + 'failedReason', CASE + WHEN dispatch_retry_count > ${maxRetries} + THEN 'workflow dispatch retries exhausted' + ELSE 'workflow dispatch deadline expired' + END, + 'phase', 'dispatch-expired' + ), + owner_kind = NULL, + owner_id = NULL, + attempt_completed_at = now() + WHERE status = 'queued' + AND attempt_id IS NULL + AND execution_delivery_id IS NOT NULL + AND ( + dispatch_retry_count > ${maxRetries} + OR created_at <= now() - ${maxAgeMs} * interval '1 millisecond' + ) + RETURNING * + `; + return rows.map(normalizeRow); +} + +export type WorkflowFailureNotificationPhase = + | "deadline-expired" + | "lease-expired" + | "dispatch-expired" + | "runner-start-failed" + | "orphaned" + | "migration-interrupted"; + +export interface PendingWorkflowFailureNotification { + readonly row: WorkflowRunRow; + readonly phase: WorkflowFailureNotificationPhase; +} + +/** Read terminal workflow failures whose public projection has no durable receipt. */ +export async function findPendingWorkflowFailureNotifications( + sql: SQL = requireDb(), + limit = 100, +): Promise { + const rows: WorkflowRunRow[] = await sql` + SELECT * + FROM workflow_runs + WHERE status = 'failed' + AND attempt_completed_at IS NOT NULL + AND failure_notified_at IS NULL + AND state ->> 'phase' IN ( + 'deadline-expired', + 'lease-expired', + 'dispatch-expired', + 'runner-start-failed', + 'orphaned', + 'migration-interrupted' + ) + ORDER BY attempt_completed_at + LIMIT ${limit} + `; + return rows.map((row) => ({ + row: normalizeRow(row), + phase: row.state["phase"] as WorkflowFailureNotificationPhase, + })); +} + +export async function markWorkflowFailureNotified( + receipt: { readonly runId: string; readonly attemptId: string | null }, + sql: SQL = requireDb(), +): Promise { + const rows: { id: string }[] = await sql` + UPDATE workflow_runs + SET failure_notified_at = COALESCE(failure_notified_at, now()) + WHERE id = ${receipt.runId} + AND attempt_id IS NOT DISTINCT FROM ${receipt.attemptId}::uuid + AND status = 'failed' + AND attempt_completed_at IS NOT NULL + RETURNING id + `; + return rows[0] !== undefined; +} + +/** Mark a terminal attempt's parent cascade as durably applied. */ +export async function markAttemptCascadeCompleted( + attempt: WorkflowAttempt, + sql: SQL = requireDb(), +): Promise { + const rows: { id: string }[] = await sql` + UPDATE workflow_runs + SET cascade_completed_at = COALESCE(cascade_completed_at, now()) + WHERE id = ${attempt.runId} + AND attempt_id = ${attempt.attemptId} + AND attempt_completed_at IS NOT NULL + AND status IN ('succeeded', 'failed', 'incomplete') + RETURNING id + `; + return rows[0] !== undefined; +} + +export async function findByAttemptId( + attemptId: string, + sql: SQL = requireDb(), +): Promise { + const rows: WorkflowRunRow[] = await sql` + SELECT * FROM workflow_runs WHERE attempt_id = ${attemptId} + `; + const row = rows[0]; + return row === undefined ? null : normalizeRow(row); +} + +/** Close only the queued dispatch generation that was actually published. */ +export async function markDispatchEnqueued( + runId: string, + expectedGeneration: string, + expectedEnqueuedAt: Date | null, + sql: SQL = requireDb(), +): Promise { + const rows: { id: string }[] = await sql` + UPDATE workflow_runs + SET dispatch_enqueued_at = now(), + owner_kind = NULL, + owner_id = NULL WHERE id = ${runId} + AND status = 'queued' + AND date_trunc('milliseconds', dispatch_enqueued_at) + IS NOT DISTINCT FROM ${expectedEnqueuedAt} + AND dispatch_generation_id = ${expectedGeneration} + RETURNING id `; + return rows[0] !== undefined; +} + +/** Count a failed publish only while the same queued generation is current. */ +export async function recordWorkflowDispatchPublishFailure( + runId: string, + expectedGeneration: string, + expectedEnqueuedAt: Date | null, + sql: SQL = requireDb(), +): Promise { + const rows: { dispatch_retry_count: number }[] = await sql` + UPDATE workflow_runs + SET dispatch_retry_count = dispatch_retry_count + 1 + WHERE id = ${runId} + AND status = 'queued' + AND attempt_id IS NULL + AND dispatch_generation_id = ${expectedGeneration} + AND date_trunc('milliseconds', dispatch_enqueued_at) + IS NOT DISTINCT FROM ${expectedEnqueuedAt} + RETURNING dispatch_retry_count + `; + return rows[0]?.dispatch_retry_count ?? null; } /** @@ -230,26 +558,54 @@ export async function setTrackingCommentId( * stamp their own comment ids. Returns the winning comment id: our `commentId` * if we won, or the pre-existing value if another worker got there first. */ +export function tryReserveTrackingCommentId( + runId: string, + commentId: number, + sql?: SQL, +): Promise<{ won: boolean; trackingCommentId: number }>; +export function tryReserveTrackingCommentId( + runId: string, + commentId: number, + attempt: WorkflowAttempt, + sql?: SQL, +): Promise<{ won: boolean; trackingCommentId: number }>; export async function tryReserveTrackingCommentId( runId: string, commentId: number, - sql: SQL = requireDb(), + sqlOrAttempt: SQL | WorkflowAttempt = requireDb(), + explicitSql?: SQL, ): Promise<{ won: boolean; trackingCommentId: number }> { + const attempt = typeof sqlOrAttempt === "function" ? undefined : sqlOrAttempt; + const sql = typeof sqlOrAttempt === "function" ? sqlOrAttempt : (explicitSql ?? requireDb()); + if (attempt !== undefined && attempt.runId !== runId) { + throw new StaleWorkflowAttemptError(attempt); + } const rows: { tracking_comment_id: number | string }[] = await sql` UPDATE workflow_runs SET tracking_comment_id = ${commentId} WHERE id = ${runId} AND tracking_comment_id IS NULL + ${attempt === undefined ? sql`` : sql`AND attempt_id = ${attempt.attemptId} AND status = 'running' AND lease_expires_at > now() AND attempt_deadline_at > now()`} RETURNING tracking_comment_id `; if (rows[0] !== undefined) { return { won: true, trackingCommentId: coerceCommentId(rows[0].tracking_comment_id) }; } - const existing: { tracking_comment_id: number | string | null }[] = await sql` - SELECT tracking_comment_id FROM workflow_runs WHERE id = ${runId} - `; + const existing: { tracking_comment_id: number | string | null }[] = + attempt === undefined + ? await sql`SELECT tracking_comment_id FROM workflow_runs WHERE id = ${runId}` + : await sql` + SELECT tracking_comment_id + FROM workflow_runs + WHERE id = ${runId} + AND attempt_id = ${attempt.attemptId} + AND status = 'running' + AND lease_expires_at > now() + AND attempt_deadline_at > now() + `; const rawExisting = existing[0]?.tracking_comment_id ?? null; if (rawExisting === null) { + if (attempt !== undefined) throw new StaleWorkflowAttemptError(attempt); throw new Error( `tryReserveTrackingCommentId: run ${runId} has no tracking_comment_id and CAS did not update`, ); @@ -277,6 +633,32 @@ export async function findById( return row === undefined ? null : normalizeRow(row); } +/** Resolve an ambiguously committed workflow/execution pair by exact identity. */ +export async function findCommittedWorkflowDispatch( + input: { + readonly workflowName: WorkflowName; + readonly target: InsertQueuedParams["target"]; + readonly executionDeliveryId: string; + }, + sql: SQL = requireDb(), +): Promise { + const rows: WorkflowRunRow[] = await sql` + SELECT wr.* + FROM workflow_runs AS wr + JOIN executions AS e ON e.delivery_id = wr.execution_delivery_id + WHERE wr.workflow_name = ${input.workflowName} + AND wr.target_type = ${input.target.type} + AND wr.target_owner = ${input.target.owner} + AND wr.target_repo = ${input.target.repo} + AND wr.target_number = ${input.target.number} + AND wr.execution_delivery_id = ${input.executionDeliveryId} + AND e.delivery_id = ${input.executionDeliveryId} + LIMIT 1 + `; + const row = rows[0]; + return row === undefined ? null : normalizeRow(row); +} + /** * Return the in-flight row for (workflow, target) if one exists. The partial * unique index guarantees at most one. @@ -404,23 +786,32 @@ export async function clearTrackingCommentId(runId: string, sql: SQL = requireDb `; } -/** - * In-flight rows owned by a specific (kind, id): used by the disconnect - * cleanup path to find workflow_runs that need a user-facing failure - * notification when their owning daemon dies abruptly. - */ -export async function findInflightByOwner( - ownerKind: WorkflowOwnerKind, - ownerId: string, +/** Clear a prior row only while the current workflow attempt still owns its lease. */ +export async function clearTrackingCommentIdForAttempt( + priorRunId: string, + attempt: WorkflowAttempt, sql: SQL = requireDb(), -): Promise { - const rows = (await sql` - SELECT * FROM workflow_runs - WHERE owner_kind = ${ownerKind} - AND owner_id = ${ownerId} - AND status IN ('queued', 'running') - `) as unknown as WorkflowRunRow[]; - return rows.map(normalizeRow); +): Promise { + const rows: { attempt_current: boolean }[] = await sql` + WITH current_attempt AS MATERIALIZED ( + SELECT 1 + FROM workflow_runs + WHERE id = ${attempt.runId} + AND attempt_id = ${attempt.attemptId} + AND status = 'running' + AND lease_expires_at > now() + AND attempt_deadline_at > now() + ), cleared AS ( + UPDATE workflow_runs + SET tracking_comment_id = NULL + WHERE id = ${priorRunId} + AND EXISTS (SELECT 1 FROM current_attempt) + RETURNING id + ) + SELECT EXISTS (SELECT 1 FROM current_attempt) AS attempt_current + FROM (SELECT count(*) FROM cleared) AS applied + `; + if (rows[0]?.attempt_current !== true) throw new StaleWorkflowAttemptError(attempt); } /** diff --git a/src/workflows/ship/command-dispatch.ts b/src/workflows/ship/command-dispatch.ts index 48cf37a8..cc65c5e5 100644 --- a/src/workflows/ship/command-dispatch.ts +++ b/src/workflows/ship/command-dispatch.ts @@ -11,13 +11,18 @@ import type { Logger } from "pino"; import { resolveModelId } from "../../ai/llm-client"; import { config } from "../../config"; import { logger as rootLogger } from "../../logger"; +import { loadRepoPolicy } from "../../repo-config/effective"; +import { checkRepoGate, type TriggerContext } from "../../repo-config/gate"; import { type CanonicalCommand, type CanonicalCommandPr, + type CommandIntent, isScopedCommandIntent, isShipCommandIntent, } from "../../shared/ship-types"; +import type { WorkflowName } from "../../shared/workflow-types"; import { getTriageLLMClient } from "../../webhook/triage-client-factory"; +import { postRefusalComment } from "../tracking-mirror"; import { runLifecycleCommand } from "./lifecycle-commands"; import { dispatchScopedCommand, type ScopedCommandDeps } from "./scoped/dispatch-scoped"; import { runShipFromCommand } from "./session-runner"; @@ -26,6 +31,103 @@ import { routeTrigger } from "./trigger-router"; export interface DispatchDeps { readonly octokit: Octokit; readonly log?: Logger; + /** Trigger facts for the repo-config filter rules. See `TriggerContext`. */ + readonly trigger?: TriggerContext; +} + +/** + * De-escalating verbs, exempt from Gate 1. If an owner disables the bot + * while an intent is mid-flight, `stop` and `abort` must still land, or the + * config change strands the very run it was meant to end. + * + * `resume` is deliberately NOT here: it re-starts work, so a repo that has + * since been disabled should refuse it. An owner who wants a paused session + * gone still has `abort`. + * + * The exemption is narrow. These verbs skip the two enable toggles and the + * passive trigger filters, not the identity rules, so `allowed_users` and + * `ignore_authors` still decide who may end a run. See `identityRulesOnly` + * in `src/repo-config/gate.ts`. + * + * Typed as `CommandIntent` so renaming a verb is a compile error rather than + * a silently stale literal. + */ +const UNGATED_INTENTS: ReadonlySet = new Set(["stop", "abort"]); + +/** + * Canonical intents that share a name with a registry workflow, so the + * per-workflow `enabled` rule can be evaluated for them. + * + * `ship` and `triage` collide today. Missing an entry is a silent bypass, + * not a type error: the canonical parser runs first in the event handlers + * and returns before `dispatchByLabel`, which is the only other place rule 2 + * is evaluated, so a `bot:triage` label would never see the toggle at all. + * `test/workflows/ship/command-dispatch.test.ts` fails if any `CommandIntent` + * matching a `WorkflowName` is absent here. + * + * Written literally rather than looked up in the registry, which would pull + * every handler's dependency graph into this module. + */ +export const INTENT_TO_WORKFLOW: Partial> = { + ship: "ship", + triage: "triage", +}; + +/** + * Gate 1 for the canonical (ship) rail, which bypasses + * `workflows/dispatcher.ts` entirely and therefore needs its own call. + */ +async function isBlockedByRepoConfig( + command: CanonicalCommand, + deps: DispatchDeps, + log: Logger, +): Promise { + const policy = await loadRepoPolicy({ + octokit: deps.octokit, + owner: command.pr.owner, + repo: command.pr.repo, + log, + }); + const workflowName = INTENT_TO_WORKFLOW[command.intent]; + const verdict = checkRepoGate({ + policy, + identityRulesOnly: UNGATED_INTENTS.has(command.intent), + ...(workflowName !== undefined ? { workflowName } : {}), + senderLogin: command.principal_login, + ...(deps.trigger !== undefined ? { trigger: deps.trigger } : {}), + }); + if (verdict.allowed) return false; + + // A deliberate label or literal command that is refused must be answered, + // same as the dispatcher rail. Nothing else can speak for it: the event + // handlers return as soon as the canonical parser yields a command, and + // `dispatchCommentSurface` returns `true` on the literal branch, so + // `dispatchByLabel` / `dispatchByIntent` never run. Without this the user + // sees only the 👀 reaction. No double-post for the same reason. + if (verdict.explain) { + await postRefusalComment( + { octokit: deps.octokit, logger: log }, + { owner: command.pr.owner, repo: command.pr.repo, number: command.pr.number }, + workflowName ?? command.intent, + verdict.reason, + ); + } + + // `event` overrides the child binding for this line only. `senderLogin` is + // written explicitly even though the child logger already binds the same + // value as `principal_login`, so every `repo_config.gate_blocked` line + // answers "who was refused" under one field name and an operator's triage + // query needs no per-emitter special case. + log.info( + { + event: "repo_config.gate_blocked", + reason: verdict.reason, + explained: verdict.explain, + senderLogin: command.principal_login, + }, + "Ship command blocked by repo config", + ); + return true; } export function dispatchCanonicalCommand(command: CanonicalCommand, deps: DispatchDeps): void { @@ -41,6 +143,25 @@ export function dispatchCanonicalCommand(command: CanonicalCommand, deps: Dispat deadline_ms: command.deadline_ms, }); + // Fire-and-forget, matching how every handler below is already launched. + // The try wraps only the gate, so `routeToHandler` is reachable exactly + // once on both paths. + void (async (): Promise => { + let blocked = false; + try { + blocked = await isBlockedByRepoConfig(command, deps, log); + } catch (err) { + // Fail open: a gate failure must not swallow the command. + log.error( + { event: "repo_config.gate_error", err }, + "repo-config gate threw, dispatching anyway", + ); + } + if (!blocked) routeToHandler(command, deps, log); + })(); +} + +function routeToHandler(command: CanonicalCommand, deps: DispatchDeps, log: Logger): void { if (command.intent === "ship") { void runShipFromCommand({ command, octokit: deps.octokit, log }).catch((err: unknown) => { log.error({ err }, "runShipFromCommand threw"); @@ -104,8 +225,14 @@ export async function dispatchCommentSurface(input: { readonly trigger_comment_id?: number; readonly octokit: Octokit; readonly log?: Logger; + /** Trigger facts for the repo-config filter rules. See `TriggerContext`. */ + readonly trigger?: TriggerContext; }): Promise { - const deps: DispatchDeps = { octokit: input.octokit, ...(input.log ? { log: input.log } : {}) }; + const deps: DispatchDeps = { + octokit: input.octokit, + ...(input.log ? { log: input.log } : {}), + ...(input.trigger !== undefined ? { trigger: input.trigger } : {}), + }; // Wrap parser + classifier in a single guard. The literal parser is // synchronous-ish, but `routeTrigger("nl")` makes a remote LLM call which // can throw on Bedrock outages. Letting that bubble out of the webhook @@ -132,6 +259,52 @@ export async function dispatchCommentSurface(input: { return true; } + // Cheap local pre-check, mirroring the classifier's own FR-025a rule + // (`nl-classifier.ts:86`): a body that does not open with the trigger + // phrase is returned as `null` there regardless. Testing it here keeps a + // disabled repo's ordinary chatter from costing a config fetch and a + // gate_blocked log line per comment. `trimStart`, not `trim`, to match the + // classifier exactly. + if (!input.commentBody.trimStart().startsWith(config.triggerPhrase)) return false; + + // Repo-wide gate, between the two parsers on purpose. The literal parser + // above is local, so running it first preserves the `stop`/`abort` + // carve-out. That carve-out covers the literal `bot:stop` / + // `bot:abort-ship` surface only: an NL-phrased stop reaches the + // classifier, and the gate + // blocks it before the intent is known. Ungating the NL path would mean + // paying an LLM call for every comment in a disabled repo. + // + // Returns `false`, not `true`: the caller falls through to + // `dispatchByIntent`, which re-runs the same gate and owns the + // user-facing refusal comment. Deciding that here would duplicate it. + const policy = await loadRepoPolicy({ + octokit: input.octokit, + owner: input.pr.owner, + repo: input.pr.repo, + log: input.log ?? rootLogger, + }); + const verdict = checkRepoGate({ + policy, + senderLogin: input.principal_login, + ...(input.trigger !== undefined ? { trigger: input.trigger } : {}), + }); + if (!verdict.allowed) { + (input.log ?? rootLogger).info( + { + event: "repo_config.gate_blocked", + reason: verdict.reason, + explained: false, + owner: input.pr.owner, + repo: input.pr.repo, + pr_number: input.pr.number, + senderLogin: input.principal_login, + }, + "Comment surface blocked by repo config before NL classification", + ); + return false; + } + // 2. NL fallback. Mention-prefix gate (FR-025a) lives in classifier. const llm = getTriageLLMClient(); const modelId = resolveModelId(config.triageModel, llm.provider); @@ -144,7 +317,6 @@ export async function dispatchCommentSurface(input: { system: params.systemPrompt, messages: [{ role: "user", content: params.userPrompt }], maxTokens: 256, - temperature: 0, }); return res.text; }; diff --git a/src/workflows/ship/intent.ts b/src/workflows/ship/intent.ts index 7d0e64f7..4f588bbb 100644 --- a/src/workflows/ship/intent.ts +++ b/src/workflows/ship/intent.ts @@ -19,6 +19,7 @@ import type { SQL } from "bun"; import { requireDb } from "../../db"; +import { isPostgresUniqueViolation } from "../../db/postgres-error"; import { appendIteration, type AppendIterationInput, @@ -70,11 +71,7 @@ export async function createIntent( ); return { ok: true, intent }; } catch (err: unknown) { - // Postgres unique_violation == SQLSTATE 23505. Bun.sql exposes the - // raw error fields on the rejection; we prefer structured detection - // over message-substring matching (which breaks if Postgres changes - // wording or the DB driver wraps the error). - if (isUniqueViolation(err, "ship_intents_one_active_per_pr")) { + if (isPostgresUniqueViolation(err, "ship_intents_one_active_per_pr")) { const existing = await dbFindActiveIntent(input.owner, input.repo, input.pr_number, sql); if (existing !== null) { return { ok: false, reason: "already_in_progress", existing }; @@ -84,29 +81,6 @@ export async function createIntent( } } -function isUniqueViolation(err: unknown, constraint: string): boolean { - if (err === null || typeof err !== "object") return false; - const e = err as { - errno?: unknown; - code?: unknown; - constraint?: unknown; - constraint_name?: unknown; - message?: unknown; - }; - // Bun.sql surfaces the SQLSTATE on `errno` ("23505" = unique_violation) - // and a high-level class string on `code` ("ERR_POSTGRES_SERVER_ERROR"). - // The constraint name comes back verbatim on `constraint`. Prefer the - // structured constraint match; fall back to SQLSTATE + message match - // when older drivers omit the constraint field. - if (e.constraint === constraint) return true; - if (e.constraint_name === constraint) return true; - if (e.errno === "23505") { - const message = e.message; - if (typeof message === "string") return message.includes(constraint); - } - return false; -} - export async function getActiveIntent( owner: string, repo: string, diff --git a/src/workflows/ship/iteration.ts b/src/workflows/ship/iteration.ts index 2f0c8f70..297f67ca 100644 --- a/src/workflows/ship/iteration.ts +++ b/src/workflows/ship/iteration.ts @@ -1,17 +1,17 @@ /** * Ship-iteration handler (US1, T012). Bridges a non-ready probe verdict - * onto the existing daemon `workflow_runs` pipeline so a single iteration + * onto the isolated `workflow_runs` runner pipeline so a single iteration * does exactly one of: * * - terminate the intent on cap or deadline, * - leave the intent active because the verdict is already ready * (the caller's terminal-shortcut is responsible for the GraphQL mutation), - * - or insert a `workflow_runs` row + enqueue a daemon job + append a + * - or insert a `workflow_runs` row + publish a runner attempt + append a * `ship_iterations` row so the orchestrator's completion cascade can * early-wake the intent for the next iteration. * * One job per iteration (research.md Q4): the loop runs many iterations - * rather than packing actions, because each daemon job mutates PR state + * rather than packing actions, because each runner attempt mutates PR state * and the next probe is the only way to detect that a fix worked. * * The handler stays inside the `workflow_runs` tree (no new JobKind); @@ -26,7 +26,8 @@ import { config } from "../../config"; import { requireDb } from "../../db"; import { appendIteration, type ShipIntentRow } from "../../db/queries/ship"; import { logger as rootLogger } from "../../logger"; -import { enqueueJob } from "../../orchestrator/job-queue"; +import { getInstanceId } from "../../orchestrator/instance-id"; +import { publishWorkflowRunById } from "../dispatch-outbox"; import { recordWorkflowExecution } from "../execution-row"; import { logWorkflowRunQueued } from "../log-fields"; import type { WorkflowName } from "../registry"; @@ -144,21 +145,11 @@ export async function runIteration(input: RunIterationInput): Promise { + await appendIteration( + { + intent_id: intent.id, + iteration_n: probeIterationN, + kind: "probe", + verdict_json: verdict, + non_readiness_reason: verdict.reason, }, - }, - sql, - ); - logWorkflowRunQueued(log, { runId: run.id, workflowName: nextWorkflowName, target: shipTarget }); + tx, + ); + const committedRun = await insertQueued( + { + workflowName: nextWorkflowName, + target: shipTarget, + ownerKind: "orchestrator", + ownerId: getInstanceId(), + executionDeliveryId: childDeliveryId, + initialState: { + ...serializeShipWorkflowContext(intent.id), + iteration_n: actionIterationN, + }, + }, + tx, + ); - // 7. Persist the `executions` row BEFORE enqueueing so the daemon's - // accept handler can resolve `context_json` via this `deliveryId`. - // Without this, the daemon side rejects the offer with - // `No execution context found, producer did not call createExecution` - // (surfaced by T042 S2 against `@chrisleekr-bot-dev`). The legacy - // workflow dispatcher writes this row before its enqueue too, the - // iteration handler must mirror that contract. - const childDeliveryId = `${intent.id}::iteration::${String(actionIterationN)}`; - await recordWorkflowExecution({ - deliveryId: childDeliveryId, - target: { type: "pr", owner: intent.owner, repo: intent.repo, number: intent.pr_number }, - senderLogin: config.botAppLogin, - workflowName: nextWorkflowName, - runId: run.id, - logger: log, - }); + // Runner admission depends on both rows. Committing only one can strand + // the target behind the in-flight index after a process crash. + await recordWorkflowExecution({ + deliveryId: childDeliveryId, + target: shipTarget, + senderLogin: config.botAppLogin, + workflowName: nextWorkflowName, + runId: committedRun.id, + logger: log, + sql: tx, + }); - // 8. Enqueue the daemon job (workflow-run kind, carrying WorkflowRunRef). - await enqueueJob({ - kind: "workflow-run", - deliveryId: childDeliveryId, - repoOwner: intent.owner, - repoName: intent.repo, - entityNumber: intent.pr_number, - isPR: true, - eventName: "pull_request", - triggerUsername: config.botAppLogin, - labels: [], - triggerBodyPreview: "", - enqueuedAt: Date.now(), - retryCount: 0, - workflowRun: { runId: run.id, workflowName: nextWorkflowName }, + const iterationKind = nextWorkflowName === "review" ? "review" : "resolve"; + await appendIteration( + { + intent_id: intent.id, + iteration_n: actionIterationN, + kind: iterationKind, + runs_store_id: committedRun.id, + }, + tx, + ); + return committedRun; }); + logWorkflowRunQueued(log, { runId: run.id, workflowName: nextWorkflowName, target: shipTarget }); - // 9. Append the action ship_iterations row. `kind=resolve` for fix-shaped - // runs; `kind=review` when the next workflow is `review`. Verdict - // columns are forbidden on non-`probe` kinds (schema CHECK). - const iterationKind = nextWorkflowName === "review" ? "review" : "resolve"; - await appendIteration( - { - intent_id: intent.id, - iteration_n: actionIterationN, - kind: iterationKind, - runs_store_id: run.id, - }, - sql, - ); + // Queue publication follows the durable commit. A crash or Valkey failure + // leaves dispatch_enqueued_at NULL for the periodic outbox retry. + try { + await publishWorkflowRunById(run.id, sql); + } catch (err) { + log.warn( + { + err: err instanceof Error ? err : new Error(String(err)), + runId: run.id, + workflowName: nextWorkflowName, + }, + "Ship iteration publication failed; the durable outbox will retry", + ); + } log.info( { diff --git a/src/workflows/ship/scoped/chat-thread.ts b/src/workflows/ship/scoped/chat-thread.ts index a20add6e..e2b54a8b 100644 --- a/src/workflows/ship/scoped/chat-thread.ts +++ b/src/workflows/ship/scoped/chat-thread.ts @@ -979,10 +979,14 @@ async function runProposalPayload(p: RunProposalPayloadInput): Promise { // FIX #4, surface non-dispatch outcomes as throws so the caller's // catch path posts a failure ack and the proposal does NOT get // marked executed for a workflow that never ran. - if (result.status === "refused") { + if (result.status === "refused" && result.explained) { // Dispatcher already posted a user-facing refusal comment via // postRefusalComment. Throw a sentinel so runPendingApproval // can skip its own failure-ack and avoid double-commenting. + // + // Gated on `explained`: the repo-config trigger filters refuse + // silently by design, and suppressing our ack for those too would + // dead-end an approved proposal with no message at all. throw new WorkflowRefusedByDispatcher( `workflow dispatch refused${result.reason !== undefined ? `, ${result.reason}` : ""}`, ); @@ -1056,8 +1060,10 @@ async function runWorkflowDirectly( }, "chat-thread: high-conf execute-workflow refused by dispatcher", ); - // The dispatcher already posted a refusal comment via - // postRefusalComment, so we don't double-post. + // No follow-up post. When the dispatcher explained the refusal it + // already commented; when a repo-config trigger filter refused it + // stayed silent on purpose, and the reply posted above this call + // means the user is not left with nothing either way. } return replyId !== null ? { replyCommentId: replyId } : {}; } diff --git a/src/workflows/ship/scoped/dispatch-scoped.ts b/src/workflows/ship/scoped/dispatch-scoped.ts index 324981e8..59971bea 100644 --- a/src/workflows/ship/scoped/dispatch-scoped.ts +++ b/src/workflows/ship/scoped/dispatch-scoped.ts @@ -65,7 +65,6 @@ function buildCallLlm(): (input: { system: params.systemPrompt, messages: [{ role: "user", content: params.userPrompt }], maxTokens: 800, - temperature: 0.1, tools: params.tools, onToolCall: params.onToolCall, }); @@ -76,7 +75,6 @@ function buildCallLlm(): (input: { system: params.systemPrompt, messages: [{ role: "user", content: params.userPrompt }], maxTokens: 800, - temperature: 0.1, }); return res.text; }; diff --git a/src/workflows/ship/session-runner.ts b/src/workflows/ship/session-runner.ts index c1794082..eee80e7b 100644 --- a/src/workflows/ship/session-runner.ts +++ b/src/workflows/ship/session-runner.ts @@ -276,7 +276,7 @@ export async function runShipFromCommand(input: RunShipFromCommandInput): Promis return; } - // Non-ready verdict: bridge to the daemon `workflow_runs` pipeline + // Non-ready verdict: bridge to the isolated `workflow_runs` runner pipeline // via runIteration (US1). The orchestrator's completion cascade // (`onStepComplete`) ZADDs `ship:tickle` on the run's terminal write // so the next iteration re-enters via the tickle scheduler (US2). diff --git a/src/workflows/tracking-mirror.ts b/src/workflows/tracking-mirror.ts index ab74112f..b6f86514 100644 --- a/src/workflows/tracking-mirror.ts +++ b/src/workflows/tracking-mirror.ts @@ -1,15 +1,21 @@ import type { Octokit } from "octokit"; import type pino from "pino"; +import type { CommandIntent } from "../shared/ship-types"; import { safePostToGitHub } from "../utils/github-output-guard"; import type { WorkflowName } from "./registry"; import { + assertCurrentWorkflowAttempt, clearTrackingCommentId, + clearTrackingCommentIdForAttempt, findById, findPriorTrackingComments, listChildrenByParent, + mergeAttemptState, mergeState, + StaleWorkflowAttemptError, tryReserveTrackingCommentId, + type WorkflowAttempt, type WorkflowRunRow, } from "./runs-store"; @@ -20,7 +26,40 @@ import { * across child step updates. Underscore prefix marks it as an internal field * not meant for handler-visible state. */ -const LAST_HUMAN_MESSAGE_KEY = "_lastHumanMessage"; +export const LAST_HUMAN_MESSAGE_KEY = "_lastHumanMessage"; + +/** + * State key holding the run's `.github-app.yaml` notice (invalid-config + * fail-open warning and/or the `review.path_filters` scope reduction). + * + * Persisted on the row rather than prepended to one message because + * `renderCommentBody` rebuilds the body from scratch on every write: a + * one-shot prepend is erased by the next progress or terminal update. + * + * Underscore prefix per this file's convention: the key shares the flat + * `workflow_runs.state` namespace that handlers merge into via `ctx.setState`, + * and marks it as internal rather than handler-visible state. + */ +export const CONFIG_NOTICE_KEY = "_configNotice"; + +/** + * Render the persisted config notice as a GitHub alert block. Each stored + * line becomes its own paragraph inside the alert. + * + * Whitespace is collapsed per line: a lone `\r` breaks out of the `> ` line + * and orphans the rest of the notice outside the blockquote. A blank or + * non-string value renders nothing, so a whitespace-only warning is treated + * as absent (same rule as the direct rail's `createTrackingComment`). + */ +function renderConfigNotice(raw: unknown): string { + if (typeof raw !== "string") return ""; + const lines = raw + .split("\n") + .map((line) => line.replace(/\s+/g, " ").trim()) + .filter((line) => line !== ""); + if (lines.length === 0) return ""; + return `> [!WARNING]\n${lines.map((line) => `> ${line}`).join("\n>\n")}\n\n`; +} /** * Hidden HTML-comment marker embedded in every tracking-comment body. Used @@ -57,7 +96,8 @@ export interface TrackingMirrorDeps { */ export function renderCommentBody(row: WorkflowRunRow, humanMessage: string): string { const header = `**bot workflow \`${row.workflow_name}\`**, ${row.status}`; - return `${runMarker(row.id)}\n${header}\n\n${humanMessage}`; + const notice = renderConfigNotice(row.state[CONFIG_NOTICE_KEY]); + return `${runMarker(row.id)}\n${header}\n\n${notice}${humanMessage}`; } /** @@ -91,6 +131,12 @@ export interface SetStateParams { readonly runId: string; readonly patch: Record; readonly humanMessage: string; + readonly attempt?: WorkflowAttempt; +} + +async function assertAttempt(attempt: WorkflowAttempt | undefined): Promise { + if (attempt === undefined) return; + await assertCurrentWorkflowAttempt(attempt); } /** @@ -106,14 +152,22 @@ export async function setState( params: SetStateParams, ): Promise { const { octokit, logger } = deps; - const { runId, patch, humanMessage } = params; + const { runId, patch, humanMessage, attempt } = params; + if (attempt !== undefined && attempt.runId !== runId) { + throw new Error("tracking-mirror.setState received a mismatched workflow attempt"); + } // Persist the human message alongside the caller's patch so the cascade // refresh can re-render the parent's composite body without losing this // run's narrative. - await mergeState(runId, { ...patch, [LAST_HUMAN_MESSAGE_KEY]: humanMessage }); - - const row = await findById(runId); + const mergedPatch = { ...patch, [LAST_HUMAN_MESSAGE_KEY]: humanMessage }; + let row: WorkflowRunRow | null; + if (attempt === undefined) { + await mergeState(runId, mergedPatch); + row = await findById(runId); + } else { + row = await mergeAttemptState(attempt, mergedPatch); + } if (row === null) { throw new Error(`tracking-mirror.setState: run ${runId} not found after merge`); } @@ -122,27 +176,29 @@ export async function setState( let resultRow: WorkflowRunRow; if (row.tracking_comment_id === null) { - resultRow = await createOrAdoptTrackingComment(deps, row, body, humanMessage); + resultRow = await createOrAdoptTrackingComment(deps, row, body, humanMessage, attempt); // First touch for this run: AFTER the new comment is safely created and // reserved, delete this workflow's stale tracking comment(s) from earlier // runs so re-running a workflow does not pile up comments. Ordered after // create so a create failure never leaves the thread with no comment. // Best-effort: never blocks. - await cleanupPriorTrackingComments(deps, resultRow); + await cleanupPriorTrackingComments(deps, resultRow, attempt); } else { const commentId = row.tracking_comment_id; await safePostToGitHub({ body, - source: "system", + source: "agent", callsite: "workflows.tracking-mirror.update", log: logger, - post: (cleanBody) => - octokit.rest.issues.updateComment({ + post: async (cleanBody) => { + await assertAttempt(attempt); + return octokit.rest.issues.updateComment({ owner: row.target_owner, repo: row.target_repo, comment_id: commentId, body: cleanBody, - }), + }); + }, }); resultRow = row; } @@ -153,16 +209,19 @@ export async function setState( // failure must never bubble up because the child's own write already // succeeded. if (resultRow.parent_run_id !== null) { - await refreshParentCompositeBody(deps, resultRow.parent_run_id).catch((err: unknown) => { - logger.warn( - { - err: err instanceof Error ? err.message : String(err), - parentRunId: resultRow.parent_run_id, - childRunId: runId, - }, - "Cascade refresh of parent composite body failed", - ); - }); + await refreshParentCompositeBody(deps, resultRow.parent_run_id, attempt).catch( + (err: unknown) => { + if (err instanceof StaleWorkflowAttemptError) throw err; + logger.warn( + { + err: err instanceof Error ? err.message : String(err), + parentRunId: resultRow.parent_run_id, + childRunId: runId, + }, + "Cascade refresh of parent composite body failed", + ); + }, + ); } return resultRow; @@ -183,6 +242,7 @@ export async function setState( async function cleanupPriorTrackingComments( deps: TrackingMirrorDeps, row: WorkflowRunRow, + attempt: WorkflowAttempt | undefined, ): Promise { const { octokit, logger } = deps; try { @@ -194,6 +254,8 @@ async function cleanupPriorTrackingComments( for (const prior of priors) { if (prior.runId === row.parent_run_id) continue; try { + // eslint-disable-next-line no-await-in-loop + await assertAttempt(attempt); await octokit.rest.issues.deleteComment({ owner: row.target_owner, repo: row.target_repo, @@ -201,7 +263,10 @@ async function cleanupPriorTrackingComments( }); // Clear the prior row's id so a future re-run does not re-list and // re-attempt a 404 delete on this already-removed comment. - await clearTrackingCommentId(prior.runId); + // eslint-disable-next-line no-await-in-loop + if (attempt === undefined) await clearTrackingCommentId(prior.runId); + // eslint-disable-next-line no-await-in-loop + else await clearTrackingCommentIdForAttempt(prior.runId, attempt); logger.info( { runId: row.id, @@ -212,6 +277,7 @@ async function cleanupPriorTrackingComments( "Deleted prior-run tracking comment on workflow re-run", ); } catch (err) { + if (err instanceof StaleWorkflowAttemptError) throw err; logger.warn( { runId: row.id, @@ -224,6 +290,7 @@ async function cleanupPriorTrackingComments( } } } catch (err) { + if (err instanceof StaleWorkflowAttemptError) throw err; logger.warn( { runId: row.id, err: err instanceof Error ? err.message : String(err) }, "Prior tracking-comment lookup failed, skipping re-run cleanup", @@ -251,27 +318,30 @@ async function createOrAdoptTrackingComment( row: WorkflowRunRow, body: string, humanMessage: string, + attempt: WorkflowAttempt | undefined, ): Promise { const { octokit, logger } = deps; const runId = row.id; - const adopted = await tryAdoptExistingMarkerComment(deps, row, humanMessage); + const adopted = await tryAdoptExistingMarkerComment(deps, row, humanMessage, attempt); if (adopted !== null) return adopted; let createErr: unknown = null; try { const guarded = await safePostToGitHub({ body, - source: "system", + source: "agent", callsite: "workflows.tracking-mirror.create", log: deps.logger, - post: (cleanBody) => - octokit.rest.issues.createComment({ + post: async (cleanBody) => { + await assertAttempt(attempt); + return octokit.rest.issues.createComment({ owner: row.target_owner, repo: row.target_repo, issue_number: row.target_number, body: cleanBody, - }), + }); + }, }); // safePostToGitHub returns posted:false when the body is emptied by // secret redaction. Surface that as a synthetic createErr so the @@ -319,7 +389,10 @@ async function createOrAdoptTrackingComment( // its canonical comment silently deleted by us. After CAS, the canonical // id is whatever the row holds, every other marker comment is a true // duplicate and safe to delete. - const reservation = await tryReserveTrackingCommentId(runId, candidate.id); + const reservation = + attempt === undefined + ? await tryReserveTrackingCommentId(runId, candidate.id) + : await tryReserveTrackingCommentId(runId, candidate.id, attempt); const winningId = reservation.trackingCommentId; const losers = matches.filter((m) => m.id !== winningId); @@ -338,12 +411,15 @@ async function createOrAdoptTrackingComment( for (const loser of losers) { try { + // eslint-disable-next-line no-await-in-loop + await assertAttempt(attempt); await octokit.rest.issues.deleteComment({ owner: row.target_owner, repo: row.target_repo, comment_id: loser.id, }); } catch (deleteErr) { + if (deleteErr instanceof StaleWorkflowAttemptError) throw deleteErr; logger.warn( { runId, @@ -368,16 +444,18 @@ async function createOrAdoptTrackingComment( const latest = (await findById(runId)) ?? row; await safePostToGitHub({ body: renderCommentBody(latest, humanMessage), - source: "system", + source: "agent", callsite: "workflows.tracking-mirror.post-create-update", log: deps.logger, - post: (cleanBody) => - octokit.rest.issues.updateComment({ + post: async (cleanBody) => { + await assertAttempt(attempt); + return octokit.rest.issues.updateComment({ owner: latest.target_owner, repo: latest.target_repo, comment_id: winningId, body: cleanBody, - }), + }); + }, }); return { ...latest, tracking_comment_id: winningId }; } @@ -392,6 +470,7 @@ async function tryAdoptExistingMarkerComment( deps: TrackingMirrorDeps, row: WorkflowRunRow, humanMessage: string, + attempt: WorkflowAttempt | undefined, ): Promise { const { octokit, logger } = deps; const matches = await findCommentsByMarker(deps, row); @@ -403,7 +482,10 @@ async function tryAdoptExistingMarkerComment( // CAS first, delete after, see createOrAdoptTrackingComment for the // race that justifies this ordering. The reservation determines the // canonical id; every non-canonical marker comment is then deleted. - const reservation = await tryReserveTrackingCommentId(row.id, candidate.id); + const reservation = + attempt === undefined + ? await tryReserveTrackingCommentId(row.id, candidate.id) + : await tryReserveTrackingCommentId(row.id, candidate.id, attempt); const winningId = reservation.trackingCommentId; const losers = matches.filter((m) => m.id !== winningId); @@ -419,12 +501,15 @@ async function tryAdoptExistingMarkerComment( for (const loser of losers) { try { + // eslint-disable-next-line no-await-in-loop + await assertAttempt(attempt); await octokit.rest.issues.deleteComment({ owner: row.target_owner, repo: row.target_repo, comment_id: loser.id, }); } catch (deleteErr) { + if (deleteErr instanceof StaleWorkflowAttemptError) throw deleteErr; logger.warn( { runId: row.id, @@ -439,16 +524,18 @@ async function tryAdoptExistingMarkerComment( const latest = (await findById(row.id)) ?? row; await safePostToGitHub({ body: renderCommentBody(latest, humanMessage), - source: "system", + source: "agent", callsite: "workflows.tracking-mirror.adopt-update", log: deps.logger, - post: (cleanBody) => - octokit.rest.issues.updateComment({ + post: async (cleanBody) => { + await assertAttempt(attempt); + return octokit.rest.issues.updateComment({ owner: latest.target_owner, repo: latest.target_repo, comment_id: winningId, body: cleanBody, - }), + }); + }, }); return { ...latest, tracking_comment_id: winningId }; } @@ -462,6 +549,7 @@ async function tryAdoptExistingMarkerComment( async function refreshParentCompositeBody( deps: TrackingMirrorDeps, parentRunId: string, + attempt: WorkflowAttempt | undefined, ): Promise { const { octokit } = deps; @@ -475,16 +563,18 @@ async function refreshParentCompositeBody( await safePostToGitHub({ body, - source: "system", + source: "agent", callsite: "workflows.tracking-mirror.composite-refresh", log: deps.logger, - post: (cleanBody) => - octokit.rest.issues.updateComment({ + post: async (cleanBody) => { + await assertAttempt(attempt); + return octokit.rest.issues.updateComment({ owner: parent.target_owner, repo: parent.target_repo, comment_id: commentId, body: cleanBody, - }), + }); + }, }); } @@ -570,18 +660,23 @@ function truncateForComposite(text: string): string { * purely cosmetic: the DB is already authoritative, so a transient GitHub * API blip must not bubble up into `dispatchByLabel` / `dispatchByIntent` and * surface as a webhook 500. + * + * `label` names what the user asked for. The dispatcher passes a registry + * workflow name; the canonical ship rail passes a `CommandIntent`, since most + * of its verbs are scoped actions with no registry entry and echoing + * "unknown" back at someone who typed `bot:summarize` explains nothing. */ export async function postRefusalComment( deps: TrackingMirrorDeps, target: { owner: string; repo: string; number: number }, - workflowName: WorkflowName | "unknown", + workflowName: WorkflowName | CommandIntent | "unknown", reason: string, ): Promise { const body = `**bot workflow \`${workflowName}\`** refused: ${reason}`; try { await safePostToGitHub({ body, - source: "system", + source: "agent", callsite: "workflows.tracking-mirror.postRefusalComment", log: deps.logger, post: (cleanBody) => diff --git a/test/config.test.ts b/test/config.test.ts index 4b5e289e..f475b36e 100644 --- a/test/config.test.ts +++ b/test/config.test.ts @@ -1,8 +1,10 @@ import { describe, expect, it } from "bun:test"; import { + assertAutoReviewRequiresAllowlist, assertOauthRequiresAllowlist, assertPatRequiresAllowlist, + blankToUndefined, type Config, configSchema, parseBooleanEnv, @@ -13,6 +15,7 @@ const BASE = { privateKey: "-----BEGIN RSA PRIVATE KEY-----\ntest\n-----END RSA PRIVATE KEY-----", webhookSecret: "secret", daemonAuthToken: "daemon-token", + workflowRunnerCapabilitySecret: "workflow-runner-capability-root-secret", databaseUrl: "postgres://user:pass@localhost:55432/db", valkeyUrl: "redis://localhost:56379", }; @@ -36,7 +39,7 @@ describe("configSchema: Anthropic provider", () => { expect(result.success).toBe(true); if (result.success) { expect(result.data.provider).toBe("anthropic"); - expect(result.data.model).toBe("claude-opus-4-7"); + expect(result.data.model).toBe("claude-opus-5"); } }); @@ -131,23 +134,37 @@ describe("configSchema: data layer validation", () => { expect(result.success).toBe(false); }); - it("waives DB + Valkey when ORCHESTRATOR_URL is set (daemon mode)", () => { - const result = configSchema.safeParse({ - appId: "123", - privateKey: "-----BEGIN RSA PRIVATE KEY-----\ntest\n-----END RSA PRIVATE KEY-----", - webhookSecret: "secret", + it("requires a dedicated workflow runner capability secret in server mode", () => { + const { workflowRunnerCapabilitySecret, ...withoutCapabilitySecret } = ANTHROPIC_BASE; + expect(workflowRunnerCapabilitySecret).toBeDefined(); + expect(configSchema.safeParse(withoutCapabilitySecret).success).toBe(false); + }); + + it("rejects capability roots reused from either daemon authentication slot", () => { + const shared = "shared-authentication-root-secret-123"; + const cases = [ + { daemonAuthToken: shared, workflowRunnerCapabilitySecret: shared }, + { daemonAuthToken: shared, workflowRunnerCapabilitySecretPrevious: shared }, + { daemonAuthTokenPrevious: shared, workflowRunnerCapabilitySecret: shared }, + { daemonAuthTokenPrevious: shared, workflowRunnerCapabilitySecretPrevious: shared }, + ]; + + for (const reused of cases) { + expect(configSchema.safeParse({ ...ANTHROPIC_BASE, ...reused }).success).toBe(false); + } + }); + + it("does not require DB or Valkey in shared daemon mode", () => { + const daemonBase = { provider: "anthropic", anthropicApiKey: "sk-ant-test", daemonAuthToken: "daemon-token", orchestratorUrl: "wss://orchestrator.example.com", - }); - expect(result.success).toBe(true); + }; + expect(configSchema.safeParse(daemonBase).success).toBe(true); }); it("still requires DAEMON_AUTH_TOKEN in daemon mode (ORCHESTRATOR_URL set)", () => { - // The DB/Valkey waiver must not cascade into waiving daemon auth, - // an orchestrator-connected daemon without a shared token would - // accept unauthenticated connections on restart. const { daemonAuthToken, ...withoutToken } = ANTHROPIC_BASE; expect(daemonAuthToken).toBeDefined(); const result = configSchema.safeParse({ @@ -156,6 +173,16 @@ describe("configSchema: data layer validation", () => { }); expect(result.success).toBe(false); }); + + it("allows an isolated workflow runner without data-layer or daemon credentials", () => { + const result = configSchema.safeParse({ + provider: "anthropic", + anthropicApiKey: "sk-ant-test", + orchestratorUrl: "wss://orchestrator.example.com/ws/workflow-runner/run/attempt", + workflowRunner: true, + }); + expect(result.success).toBe(true); + }); }); describe("configSchema: ephemeral-daemon defaults", () => { @@ -168,6 +195,12 @@ describe("configSchema: ephemeral-daemon defaults", () => { expect(result.data.ephemeralDaemonSpawnCooldownMs).toBe(30_000); expect(result.data.ephemeralDaemonSpawnQueueThreshold).toBe(3); expect(result.data.ephemeralDaemonNamespace).toBe("default"); + expect(result.data.ephemeralDaemonSecretName).toBe("daemon-secrets"); + expect(result.data.workflowRunnerNamespace).toBe("github-app-runners"); + expect(result.data.workflowRunnerNodeLabel).toBe( + "github-app.node-restriction.kubernetes.io/workflow-runner", + ); + expect(result.data.workflowRunnerNodeValue).toBe("true"); } }); @@ -179,6 +212,10 @@ describe("configSchema: ephemeral-daemon defaults", () => { ephemeralDaemonSpawnCooldownMs: 10_000, ephemeralDaemonSpawnQueueThreshold: 5, ephemeralDaemonNamespace: "ops", + ephemeralDaemonSecretName: "github-app-secrets", + workflowRunnerNamespace: "workflow-ops", + workflowRunnerNodeLabel: "node.homelab/class", + workflowRunnerNodeValue: "worker", daemonImage: "ghcr.io/org/daemon:1.2.3", orchestratorPublicUrl: "wss://orchestrator.example.com", }); @@ -187,10 +224,23 @@ describe("configSchema: ephemeral-daemon defaults", () => { expect(result.data.daemonEphemeral).toBe(true); expect(result.data.ephemeralDaemonIdleTimeoutMs).toBe(60_000); expect(result.data.ephemeralDaemonNamespace).toBe("ops"); + expect(result.data.ephemeralDaemonSecretName).toBe("github-app-secrets"); + expect(result.data.workflowRunnerNamespace).toBe("workflow-ops"); + expect(result.data.workflowRunnerNodeLabel).toBe("node.homelab/class"); + expect(result.data.workflowRunnerNodeValue).toBe("worker"); expect(result.data.daemonImage).toBe("ghcr.io/org/daemon:1.2.3"); } }); + it("rejects a controller that shares the workflow-runner namespace", () => { + const result = configSchema.safeParse({ + ...ANTHROPIC_BASE, + ephemeralDaemonNamespace: "workers", + workflowRunnerNamespace: "workers", + }); + expect(result.success).toBe(false); + }); + it("rejects a non-ws URL for orchestratorPublicUrl", () => { const result = configSchema.safeParse({ ...ANTHROPIC_BASE, @@ -319,6 +369,27 @@ describe("parseBooleanEnv", () => { }); }); +describe("blankToUndefined", () => { + it("treats undefined, empty, and whitespace-only as unset", () => { + expect(blankToUndefined(undefined)).toBeUndefined(); + expect(blankToUndefined("")).toBeUndefined(); + expect(blankToUndefined(" ")).toBeUndefined(); + }); + + it("passes a real value through untrimmed", () => { + expect(blankToUndefined(" .github-app.yaml ")).toBe(" .github-app.yaml "); + }); + + it("lets the deprecated alias win only when the new name is blank", () => { + // Mirrors the REPO_CONFIG_FILE ?? SCHEDULER_CONFIG_FILE chain in + // loadConfig. A chart rendering an unset key as "" must not win the + // chain, or the path resolves to the repo root for every repo. + expect(blankToUndefined("") ?? blankToUndefined("legacy.yaml")).toBe("legacy.yaml"); + expect(blankToUndefined("new.yaml") ?? blankToUndefined("legacy.yaml")).toBe("new.yaml"); + expect(blankToUndefined("") ?? blankToUndefined("")).toBeUndefined(); + }); +}); + describe("assertOauthRequiresAllowlist", () => { const baseOauthCfg: Config = configSchema.parse({ ...BASE, @@ -599,3 +670,42 @@ describe("configSchema: PROMPT_CACHE_LAYOUT", () => { expect(result.success).toBe(false); }); }); + +describe("assertAutoReviewRequiresAllowlist", () => { + const withOwners = (owners: string | undefined, users: string | undefined): Config => + configSchema.parse({ + ...ANTHROPIC_BASE, + ...(owners === undefined ? {} : { allowedOwners: owners }), + ...(users === undefined ? {} : { autoReviewUsers: users }), + }); + + it("rejects AUTO_REVIEW_USERS with no ALLOWED_OWNERS", () => { + // Every other allowlist here narrows; this one widens. `isOwnerAllowed` + // permits every owner when ALLOWED_OWNERS is unset, so the whole chain to + // "run an agent on a stranger's repo" would be a login-string match plus a + // key that stranger controls. + expect(() => { + assertAutoReviewRequiresAllowlist(withOwners(undefined, "chrisleekr")); + }).toThrow(/ALLOWED_OWNERS must be set/); + }); + + it("accepts AUTO_REVIEW_USERS bound to one owner", () => { + expect(() => { + assertAutoReviewRequiresAllowlist(withOwners("acme", "chrisleekr")); + }).not.toThrow(); + }); + + it("accepts several owners, unlike the OAuth and PAT guards", () => { + // Auto-review carries no personal identity and no shared rate-limit bucket, + // so it only needs the list to be non-empty, not singular. + expect(() => { + assertAutoReviewRequiresAllowlist(withOwners("acme,other", "chrisleekr")); + }).not.toThrow(); + }); + + it("ignores an unset AUTO_REVIEW_USERS entirely", () => { + expect(() => { + assertAutoReviewRequiresAllowlist(withOwners(undefined, undefined)); + }).not.toThrow(); + }); +}); diff --git a/src/core/hooks/forbidden-bash.test.ts b/test/core/hooks/forbidden-bash.test.ts similarity index 98% rename from src/core/hooks/forbidden-bash.test.ts rename to test/core/hooks/forbidden-bash.test.ts index 9c941c2f..1a32b333 100644 --- a/src/core/hooks/forbidden-bash.test.ts +++ b/test/core/hooks/forbidden-bash.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from "bun:test"; -import type { Logger } from "../../logger"; -import { createForbiddenBashHook } from "./forbidden-bash"; +import { createForbiddenBashHook } from "../../../src/core/hooks/forbidden-bash"; +import type { Logger } from "../../../src/logger"; // Minimal logger stub: records `.warn` calls so a deny can be asserted. The // factory's `log` param is a pino Logger in production; only `.warn` is used diff --git a/test/core/pipeline.test.ts b/test/core/pipeline.test.ts index 202a5902..391ba70c 100644 --- a/test/core/pipeline.test.ts +++ b/test/core/pipeline.test.ts @@ -85,6 +85,25 @@ function lastAgentCall(): ExecuteAgentParams { return call; } +/** + * Depth-bounded search for `needle` among an argument list. Shape-agnostic on + * purpose: the warning may ride on the context or on a dedicated parameter, + * and the acceptance criterion is that it reaches the tracking-comment write, + * not which slot carries it. + */ +function argsContainText(args: readonly unknown[], needle: string): boolean { + const seen = new WeakSet(); + const walk = (value: unknown, depth: number): boolean => { + if (depth > 4) return false; + if (typeof value === "string") return value.includes(needle); + if (typeof value !== "object" || value === null) return false; + if (seen.has(value)) return false; + seen.add(value); + return Object.values(value).some((v) => walk(v, depth + 1)); + }; + return args.some((a) => walk(a, 0)); +} + const PR_FILES: FetchedData["changedFiles"] = [ { filename: "src/a.ts", status: "modified", additions: 5, deletions: 2 }, { filename: "src/__snapshots__/big.snap", status: "modified", additions: 900, deletions: 900 }, @@ -423,3 +442,58 @@ describe("runPipeline: policy.instructions (C6)", () => { expect(lastAgentCall().prompt).toContain("REPO_REVIEW_POLICY_MARKER"); }); }); + +// ─── C7: fail-open warning, direct-pipeline rail ───────────────────────────── + +describe("runPipeline: policy.warning (C7, direct-pipeline rail)", () => { + it("surfaces the invalid-config warning through the tracking comment write", async () => { + const warning = + "`.github-app.yaml` failed validation and was ignored; built-in defaults were used."; + const ctx = makeBotContext({ isPR: false }); + + await runPipeline(ctx, { + allowedTools: ["Read"], + policy: { warning }, + }); + + expect(mockCreateTrackingComment).toHaveBeenCalled(); + const args = mockCreateTrackingComment.mock.calls[0] ?? []; + expect(argsContainText(args, "failed validation")).toBe(true); + }); + + it("still executes the agent with default behaviour despite the warning (C7 fail-open)", async () => { + const ctx = makeBotContext({ isPR: false }); + + await runPipeline(ctx, { + allowedTools: ["Read"], + policy: { warning: "`.github-app.yaml` failed validation and was ignored" }, + }); + + expect(executeAgentCalls).toHaveLength(1); + expect(lastAgentCall().model).toBeUndefined(); + expect(lastAgentCall().allowedTools).toEqual(["Read"]); + }); + + it("hands the warning to finalize so the agent's body rewrite cannot drop it", async () => { + const warning = + "`.github-app.yaml` failed validation and was ignored; built-in defaults were used."; + const ctx = makeBotContext({ isPR: false }); + + await runPipeline(ctx, { allowedTools: ["Read"], policy: { warning } }); + + expect(mockFinalizeTrackingComment).toHaveBeenCalled(); + const opts = mockFinalizeTrackingComment.mock.calls[0]?.[2] as + | { configWarning?: string } + | undefined; + expect(opts?.configWarning).toBe(warning); + }); + + it("writes no warning into the tracking comment when the policy is clean (C8)", async () => { + const ctx = makeBotContext({ isPR: false }); + + await runPipeline(ctx, { allowedTools: ["Read"] }); + + const args = mockCreateTrackingComment.mock.calls[0] ?? []; + expect(argsContainText(args, "failed validation")).toBe(false); + }); +}); diff --git a/test/core/tracking-comment.test.ts b/test/core/tracking-comment.test.ts index 9a822d97..55489813 100644 --- a/test/core/tracking-comment.test.ts +++ b/test/core/tracking-comment.test.ts @@ -46,6 +46,60 @@ describe("createTrackingComment", () => { expect(capturedBody).toContain(""); expect(capturedBody).toContain("@chrisleekr-bot"); }); + + /** Capture the body `createTrackingComment` posts for `configWarning`. */ + async function bodyFor(configWarning?: string): Promise { + const ctx = makeBotContext({ deliveryId: DELIVERY_ID }); + let capturedBody = ""; + ctx.octokit = { + rest: { + issues: { + createComment: mock(({ body }: { body: string }) => { + capturedBody = body; + return Promise.resolve({ data: { id: 999 } }); + }), + }, + }, + } as unknown as Octokit; + + await createTrackingComment(ctx, configWarning); + return capturedBody; + } + + it("renders the invalid-config notice as a GitHub warning alert", async () => { + // A silently ignored config file looks identical to one that took effect, + // so the notice has to ride the first thing the user reads. + const body = await bodyFor("`.github-app.yaml` failed validation and was ignored"); + + expect(body).toContain("> [!WARNING]"); + expect(body).toContain("failed validation"); + // Still a notice, not an error: the run proceeds on built-in defaults. + expect(body).toContain("is working on this..."); + }); + + it("collapses newlines so the blockquote stays a single line", async () => { + const body = await bodyFor("first line\nsecond line"); + + expect(body).toContain("> first line second line"); + // A bare `\n` inside the quote would orphan everything after it. + expect(body).not.toContain("> first line\nsecond line"); + }); + + it("collapses a lone carriage return so the blockquote stays a single line", async () => { + const body = await bodyFor("first line\rsecond line"); + + // GitHub treats a bare `\r` as a line break too, so `\n`-only collapsing + // would orphan the tail outside the `> ` quote. + expect(body).toContain("> first line second line"); + }); + + it("renders no alert block for a whitespace-only warning", async () => { + expect(await bodyFor(" ")).not.toContain("[!WARNING]"); + }); + + it("renders no alert block when no warning is supplied", async () => { + expect(await bodyFor()).not.toContain("[!WARNING]"); + }); }); // ─── updateTrackingComment ──────────────────────────────────────────────────── @@ -136,6 +190,49 @@ describe("finalizeTrackingComment", () => { expect(capturedUpdateBody.startsWith("")).toBe(true); }); + it("re-appends the config warning the agent's comment rewrite erased", async () => { + // `update_claude_comment` replaces the whole body, so the banner + // `createTrackingComment` posted is gone by the time we finalize. + await finalizeTrackingComment(ctx, 1, { + success: true, + configWarning: "`.github-app.yaml` failed validation and was ignored.", + }); + + expect(capturedUpdateBody).toContain("> [!WARNING]"); + expect(capturedUpdateBody).toContain("failed validation"); + }); + + it("does not repeat the warning when the create-time banner survived", async () => { + const warning = "`.github-app.yaml` failed validation and was ignored."; + ctx.octokit = { + rest: { + issues: { + getComment: mock(() => + Promise.resolve({ + data: { + body: `\n**Working...**\n\n> [!WARNING]\n> ${warning}`, + }, + }), + ), + updateComment: mock(({ body }: { body: string }) => { + capturedUpdateBody = body; + return Promise.resolve({ data: { id: 1 } }); + }), + }, + }, + } as unknown as Octokit; + + await finalizeTrackingComment(ctx, 1, { success: true, configWarning: warning }); + + expect(capturedUpdateBody.split("[!WARNING]")).toHaveLength(2); + }); + + it("writes no warning block when the run carried no config warning", async () => { + await finalizeTrackingComment(ctx, 1, { success: true }); + + expect(capturedUpdateBody).not.toContain("[!WARNING]"); + }); + it("falls back gracefully when getComment throws, still calls updateComment", async () => { let updateCalled = false; ctx.octokit = { @@ -186,6 +283,7 @@ describe("renderDispatchReasonLine", () => { expect(renderDispatchReasonLine("ephemeral-spawn-failed", "daemon")).toMatch( /Kubernetes|infrastructure|unavailable/i, ); + expect(renderDispatchReasonLine("workflow-runner", "workflow-runner")).toMatch(/isolated/i); }); it("spawn-failed reason does not use 'Routed' (nothing was routed)", () => { diff --git a/src/core/workspace-events.test.ts b/test/core/workspace-events.test.ts similarity index 99% rename from src/core/workspace-events.test.ts rename to test/core/workspace-events.test.ts index 04d2e8c0..7cbae076 100644 --- a/src/core/workspace-events.test.ts +++ b/test/core/workspace-events.test.ts @@ -4,7 +4,7 @@ import { WORKSPACE_CLEANUP_TARGETS, WORKSPACE_LOG_EVENTS, WorkspaceLogFieldsSchema, -} from "./workspace-events"; +} from "../../src/core/workspace-events"; describe("WORKSPACE_LOG_EVENTS", () => { it("pins the canonical event strings", () => { diff --git a/test/daemon/daemon-id.test.ts b/test/daemon/daemon-id.test.ts new file mode 100644 index 00000000..21e53547 --- /dev/null +++ b/test/daemon/daemon-id.test.ts @@ -0,0 +1,40 @@ +import { hostname } from "node:os"; + +import { describe, expect, it } from "bun:test"; + +import { getDaemonId } from "../../src/daemon/daemon-id"; + +const DAEMON_ID_MODULE = new URL("../../src/daemon/daemon-id.ts", import.meta.url).href; +const UUID_SUFFIX = /[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; + +function getDaemonIdFromFreshProcess(): string { + const script = `import { getDaemonId } from ${JSON.stringify(DAEMON_ID_MODULE)}; process.stdout.write(getDaemonId());`; + const proc = Bun.spawnSync(["bun", "--eval", script], { + stdout: "pipe", + stderr: "pipe", + }); + expect(proc.exitCode).toBe(0); + expect(proc.stderr.toString()).toBe(""); + return proc.stdout.toString(); +} + +describe("getDaemonId", () => { + it("caches one process-incarnation id across calls and reconnects", () => { + const first = getDaemonId(); + + expect(getDaemonId()).toBe(first); + expect(first.startsWith(`daemon-${hostname()}-`)).toBe(true); + expect(first).toMatch(UUID_SUFFIX); + }); + + it("uses a different incarnation id in two fresh daemon processes", () => { + const firstBoot = getDaemonIdFromFreshProcess(); + const secondBoot = getDaemonIdFromFreshProcess(); + + expect(firstBoot).not.toBe(secondBoot); + expect(firstBoot.startsWith(`daemon-${hostname()}-`)).toBe(true); + expect(secondBoot.startsWith(`daemon-${hostname()}-`)).toBe(true); + expect(firstBoot).toMatch(UUID_SUFFIX); + expect(secondBoot).toMatch(UUID_SUFFIX); + }); +}); diff --git a/test/daemon/job-executor.test.ts b/test/daemon/job-executor.test.ts new file mode 100644 index 00000000..f9ffbc62 --- /dev/null +++ b/test/daemon/job-executor.test.ts @@ -0,0 +1,141 @@ +/** + * Wiring-only tests for the direct-pipeline rail's Gate-2 hop. + * + * `executeJob` spreads the wire `policy` into `runPipeline`. The wire schema + * and the pipeline consumer both have their own suites; this hop between them + * had none, so deleting the spread failed nothing. + */ + +import { describe, expect, it, mock } from "bun:test"; + +import type { DaemonCapabilities } from "../../src/shared/daemon-types"; +import { daemonMessageSchema, type JobPayloadMessage } from "../../src/shared/ws-messages"; + +const mockRunPipeline = mock(() => Promise.resolve({ success: true, durationMs: 1, numTurns: 1 })); +const mockGetPr = mock(() => + Promise.resolve({ + data: { state: "closed", merged: false, base: { ref: "main" }, head: { ref: "feat/x" } }, + }), +); +const mockCreateComment = mock(() => Promise.resolve({ data: { id: 4242 } })); + +void mock.module("../../src/core/pipeline", () => ({ + runPipeline: mockRunPipeline, +})); + +void mock.module("octokit", () => ({ + Octokit: class MockOctokit { + rest = { + pulls: { get: mockGetPr }, + issues: { createComment: mockCreateComment }, + }; + }, +})); + +const { executeJob } = await import("../../src/daemon/job-executor"); + +const CAPABILITIES = { + daemonId: "daemon-test", + tools: [], +} as unknown as DaemonCapabilities; + +function buildPayload(policy?: Record): JobPayloadMessage { + return { + id: "offer-1", + timestamp: Date.now(), + payload: { + context: { + owner: "acme", + repo: "widgets", + entityNumber: 42, + isPR: true, + eventName: "pull_request", + commentId: 0, + deliveryId: "delivery-1", + defaultBranch: "main", + }, + installationToken: "tok", + allowedTools: ["Read"], + ...(policy !== undefined ? { policy } : {}), + }, + } as unknown as JobPayloadMessage; +} + +describe("executeJob: per-repo policy forwarding (direct-pipeline rail)", () => { + it("forwards the wire policy into the pipeline overrides", async () => { + mockRunPipeline.mockClear(); + const policy = { + model: "claude-repo-pinned-model", + timeoutMs: 900_000, + pathFilters: ["**/__snapshots__/**"], + }; + + await executeJob(buildPayload(policy), CAPABILITIES, () => {}); + + expect(mockRunPipeline).toHaveBeenCalledTimes(1); + const overrides = mockRunPipeline.mock.calls[0]?.[1] as + | { policy?: Record } + | undefined; + expect(overrides?.policy).toEqual(policy); + }); + + it("passes no policy key when the payload carries none (C8)", async () => { + mockRunPipeline.mockClear(); + + await executeJob(buildPayload(), CAPABILITIES, () => {}); + + const overrides = mockRunPipeline.mock.calls[0]?.[1] as Record | undefined; + expect(overrides).toBeDefined(); + // `exactOptionalPropertyTypes`: absent, not `undefined`-valued. + expect(Object.hasOwn(overrides ?? {}, "policy")).toBe(false); + }); +}); + +describe("executeJob: scoped completion wire contract", () => { + it("emits a schema-valid scoped-job:completion", async () => { + mockGetPr.mockClear(); + mockCreateComment.mockClear(); + const sent: unknown[] = []; + const payload: JobPayloadMessage = { + type: "job:payload", + id: crypto.randomUUID(), + timestamp: Date.now(), + payload: { + context: {}, + installationToken: "tok", + allowedTools: [], + scoped: { + jobKind: "scoped-rebase", + deliveryId: "scoped-delivery-1", + installationId: 123, + owner: "acme", + repo: "widgets", + prNumber: 42, + triggerCommentId: 456, + enqueuedAt: Date.now(), + }, + }, + }; + + await executeJob(payload, CAPABILITIES, (message) => sent.push(message)); + + const completion = sent.find( + (message) => + typeof message === "object" && + message !== null && + (message as { type?: unknown }).type === "scoped-job:completion", + ); + expect(completion).toBeDefined(); + expect(daemonMessageSchema.safeParse(completion).success).toBe(true); + expect(completion).toMatchObject({ + type: "scoped-job:completion", + payload: { + jobKind: "scoped-rebase", + status: "succeeded", + rebaseOutcome: { result: "closed", commentId: 4242 }, + }, + }); + expect(mockGetPr).toHaveBeenCalledTimes(1); + expect(mockCreateComment).toHaveBeenCalledTimes(1); + }); +}); diff --git a/test/daemon/process-boundary.test.ts b/test/daemon/process-boundary.test.ts new file mode 100644 index 00000000..f6b0b2fd --- /dev/null +++ b/test/daemon/process-boundary.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from "bun:test"; + +import { daemonEnvironmentBoundaryFailure } from "../../src/daemon/process-boundary"; + +const guardPath = "/usr/local/lib/github-app/daemon-process-guard.so"; + +describe("daemon process boundary", () => { + it("does not require the Linux preload guard on other platforms", () => { + expect( + daemonEnvironmentBoundaryFailure({ + platform: "darwin", + preload: undefined, + probeExitCode: 0, + }), + ).toBeNull(); + }); + + it("distinguishes a missing guard from an ineffective guard", () => { + expect( + daemonEnvironmentBoundaryFailure({ + platform: "linux", + preload: undefined, + probeExitCode: 0, + }), + ).toContain("not installed"); + expect( + daemonEnvironmentBoundaryFailure({ + platform: "linux", + preload: guardPath, + probeExitCode: 0, + }), + ).toBe("daemon process guard is installed but ineffective"); + }); + + it("accepts only the child probe's permission-denied exit", () => { + expect( + daemonEnvironmentBoundaryFailure({ + platform: "linux", + preload: undefined, + probeExitCode: 77, + }), + ).toBeNull(); + expect( + daemonEnvironmentBoundaryFailure({ + platform: "linux", + preload: guardPath, + probeExitCode: 78, + }), + ).toContain("exit code 78"); + }); +}); diff --git a/test/daemon/scoped-offer-evaluator.test.ts b/test/daemon/scoped-offer-evaluator.test.ts index 411af67b..259b9640 100644 --- a/test/daemon/scoped-offer-evaluator.test.ts +++ b/test/daemon/scoped-offer-evaluator.test.ts @@ -1,9 +1,9 @@ /** - * C1: the daemon's `scoped-job-offer` handler must accept supported jobKinds + * C1: the daemon's `scoped-job:offer` handler must accept supported jobKinds * and reject unknown ones with `WS_REJECT_REASONS.SCOPED_KIND_UNSUPPORTED` * so the orchestrator can re-offer to a capable daemon (FR-021). The * evaluator is exercised via `evaluateScopedOffer` which the daemon's - * `handleMessage` switch invokes for `scoped-job-offer`. + * `handleMessage` switch invokes for `scoped-job:offer`. */ import { describe, expect, it } from "bun:test"; @@ -29,7 +29,7 @@ const baselineCapabilities: DaemonCapabilities = { function rebaseOffer(): ScopedJobOfferMessage { return { - type: "scoped-job-offer", + type: "scoped-job:offer", id: "offer-id", timestamp: Date.now(), payload: { diff --git a/test/daemon/workflow-executor.test.ts b/test/daemon/workflow-executor.test.ts deleted file mode 100644 index d614dea6..00000000 --- a/test/daemon/workflow-executor.test.ts +++ /dev/null @@ -1,200 +0,0 @@ -/** - * Unit tests for the daemon-side workflow-executor `incomplete` branch - * introduced for issue #93. - * - * Mocks the runs-store, tracking-mirror, orchestrator cascade, octokit, and - * the registry handler so the test asserts only the executor's own dispatch - * logic: did it call `markIncomplete`, did it emit a `setState` mirror with - * the supplied `humanMessage`, and did it send a `job:result` envelope with - * `success: false` and an `incomplete:`-prefixed `errorMessage`. - */ - -import { beforeEach, describe, expect, it, mock } from "bun:test"; - -const markIncomplete = mock(async () => Promise.resolve()); -const markFailed = mock(async () => Promise.resolve()); -const markSucceeded = mock(async () => Promise.resolve()); -const markRunning = mock(async () => Promise.resolve()); -const mergeState = mock(async () => Promise.resolve()); - -void mock.module("../../src/workflows/runs-store", () => ({ - markIncomplete, - markFailed, - markSucceeded, - markRunning, - mergeState, - findById: mock(async () => Promise.resolve(null)), - findInflightByOwner: mock(async () => Promise.resolve([])), - findLatestForTarget: mock(async () => Promise.resolve(null)), - findLatestSucceededForTarget: mock(async () => Promise.resolve(null)), - insertQueued: mock(async () => Promise.resolve()), - listChildrenByParent: mock(async () => Promise.resolve([])), - tryReserveTrackingCommentId: mock(async () => - Promise.resolve({ won: true, trackingCommentId: 0 }), - ), -})); - -const setStateMirror = mock(async () => Promise.resolve()); -void mock.module("../../src/workflows/tracking-mirror", () => ({ - setState: setStateMirror, - postRefusalComment: mock(() => Promise.resolve()), -})); - -const onStepComplete = mock(async () => Promise.resolve()); -void mock.module("../../src/workflows/orchestrator", () => ({ - onStepComplete, -})); - -const handlerMock = mock(); -void mock.module("../../src/workflows/registry", () => ({ - getByName: () => ({ - name: "resolve", - label: "bot:resolve", - handler: handlerMock, - }), -})); - -void mock.module("octokit", () => ({ - Octokit: function MockOctokit(this: unknown) { - return this; - }, -})); - -void mock.module("../../src/utils/reactions", () => ({ - addReaction: mock(() => Promise.resolve()), -})); - -void mock.module("../../src/daemon/daemon-id", () => ({ - getDaemonId: () => "daemon-test", -})); - -const { executeWorkflowRun } = await import("../../src/daemon/workflow-executor"); - -interface JobPayload { - id: string; - timestamp: number; - payload: { - context: Record; - installationToken: string; - allowedTools: string[]; - workflowRun: { - runId: string; - workflowName: "resolve"; - parentRunId?: string; - parentStepIndex?: number; - }; - }; -} - -function buildPayload(): JobPayload { - return { - id: "offer-1", - timestamp: Date.now(), - payload: { - context: { - owner: "acme", - repo: "widgets", - entityNumber: 42, - isPR: true, - eventName: "pull_request", - commentId: 0, - deliveryId: "delivery-1", - }, - installationToken: "tok", - allowedTools: [], - workflowRun: { - runId: "run-incomplete", - workflowName: "resolve", - }, - }, - }; -} - -describe("executeWorkflowRun: incomplete branch", () => { - beforeEach(() => { - markIncomplete.mockClear(); - markFailed.mockClear(); - markSucceeded.mockClear(); - setStateMirror.mockClear(); - onStepComplete.mockClear(); - handlerMock.mockReset(); - }); - - it("persists incomplete status, mirrors human message, and sends success=false envelope", async () => { - handlerMock.mockResolvedValueOnce({ - status: "incomplete", - reason: "CI still red after FIX_ATTEMPTS_CAP=3", - state: { post_pipeline: { all_green: false } }, - humanMessage: "🔎 **Resolve incomplete**, typecheck still failing", - }); - - const sent: unknown[] = []; - await executeWorkflowRun(buildPayload() as never, (msg) => { - sent.push(msg); - }); - - expect(markIncomplete).toHaveBeenCalledTimes(1); - const [runId, reason, state] = markIncomplete.mock.calls[0] as unknown as [ - string, - string, - Record, - ]; - expect(runId).toBe("run-incomplete"); - expect(reason).toContain("CI still red"); - expect(state["post_pipeline"]).toBeDefined(); - - expect(markFailed).not.toHaveBeenCalled(); - expect(markSucceeded).not.toHaveBeenCalled(); - - expect(setStateMirror).toHaveBeenCalledTimes(1); - const mirrorArgs = setStateMirror.mock.calls[0] as unknown as [ - unknown, - { humanMessage: string }, - ]; - expect(mirrorArgs[1].humanMessage).toContain("Resolve incomplete"); - - // job:result envelope must be a non-success with an `incomplete:` prefix - // so the orchestrator cascade's failed-branch can detect it. - const result = sent.find( - (m): m is { type: string; payload: { success: boolean; errorMessage?: string } } => - typeof m === "object" && - m !== null && - "type" in m && - (m as { type: string }).type === "job:result", - ); - expect(result).toBeDefined(); - expect(result?.payload.success).toBe(false); - expect(result?.payload.errorMessage).toContain("incomplete:"); - - // Cascade must be invoked once with status=failed (binary contract). - expect(onStepComplete).toHaveBeenCalledTimes(1); - const cascadeArgs = onStepComplete.mock.calls[0] as unknown as [ - unknown, - string, - { status: string; reason: string }, - ]; - expect(cascadeArgs[1]).toBe("run-incomplete"); - expect(cascadeArgs[2].status).toBe("failed"); - expect(cascadeArgs[2].reason).toContain("incomplete:"); - }); - - it("falls back to a generic mirror message when humanMessage is omitted", async () => { - handlerMock.mockResolvedValueOnce({ - status: "incomplete", - reason: "outstanding section non-empty", - }); - - const sent: unknown[] = []; - await executeWorkflowRun(buildPayload() as never, (msg) => { - sent.push(msg); - }); - - expect(markIncomplete).toHaveBeenCalledTimes(1); - expect(setStateMirror).toHaveBeenCalledTimes(1); - const mirrorArgs = setStateMirror.mock.calls[0] as unknown as [ - unknown, - { humanMessage: string }, - ]; - expect(mirrorArgs[1].humanMessage).toContain("incomplete"); - }); -}); diff --git a/test/daemon/ws-client.test.ts b/test/daemon/ws-client.test.ts index fcdb4f51..2f63b005 100644 --- a/test/daemon/ws-client.test.ts +++ b/test/daemon/ws-client.test.ts @@ -20,6 +20,7 @@ void mock.module("../../src/logger", () => ({ import { DaemonWsClient } from "../../src/daemon/ws-client"; import type { DaemonCapabilities } from "../../src/shared/daemon-types"; +import { WS_CLOSE_CODES } from "../../src/shared/ws-messages"; class FakeWebSocket { static readonly CONNECTING = 0; @@ -111,4 +112,16 @@ describe("DaemonWsClient reconnect-timer lifecycle", () => { expect((client as unknown as InternalState).reconnectTimer).toBeNull(); }); + + it("does not reconnect after an incompatible-protocol close", () => { + const client = makeClient(); + client.connect(); + + FakeWebSocket.last?.fireClose( + WS_CLOSE_CODES.INCOMPATIBLE_PROTOCOL.code, + WS_CLOSE_CODES.INCOMPATIBLE_PROTOCOL.reason, + ); + + expect((client as unknown as InternalState).reconnectTimer).toBeNull(); + }); }); diff --git a/test/db/migrate.test.ts b/test/db/migrate.test.ts index a2b8841a..729ce072 100644 --- a/test/db/migrate.test.ts +++ b/test/db/migrate.test.ts @@ -34,6 +34,7 @@ describe.skipIf(sql === null)("runMigrations", () => { beforeAll(async () => { await requireDb().unsafe(` DROP TABLE IF EXISTS _migrations CASCADE; + DROP TABLE IF EXISTS workflow_attempt_commands CASCADE; DROP TABLE IF EXISTS review_learnings CASCADE; DROP TABLE IF EXISTS scheduled_action_state CASCADE; DROP TABLE IF EXISTS comment_cache CASCADE; @@ -54,6 +55,7 @@ describe.skipIf(sql === null)("runMigrations", () => { afterAll(async () => { await requireDb().unsafe(` DROP TABLE IF EXISTS _migrations CASCADE; + DROP TABLE IF EXISTS workflow_attempt_commands CASCADE; DROP TABLE IF EXISTS review_learnings CASCADE; DROP TABLE IF EXISTS scheduled_action_state CASCADE; DROP TABLE IF EXISTS comment_cache CASCADE; @@ -80,7 +82,7 @@ describe.skipIf(sql === null)("runMigrations", () => { const versions: { version: string }[] = await requireDb()` SELECT version FROM _migrations ORDER BY version `; - expect(versions.length).toBe(16); + expect(versions.length).toBe(17); expect(versions[0]?.version).toBe("001_initial"); expect(versions[1]?.version).toBe("002_repo_knowledge"); expect(versions[2]?.version).toBe("003_dispatch_decisions"); @@ -97,6 +99,7 @@ describe.skipIf(sql === null)("runMigrations", () => { expect(versions[13]?.version).toBe("014_review_learnings"); expect(versions[14]?.version).toBe("015_review_learnings_embedding"); expect(versions[15]?.version).toBe("016_executions_tokens"); + expect(versions[16]?.version).toMatch(/^017_/); }); it("is idempotent: second run is a no-op", async () => { @@ -106,7 +109,7 @@ describe.skipIf(sql === null)("runMigrations", () => { const versions: { version: string }[] = await requireDb()` SELECT version FROM _migrations ORDER BY version `; - expect(versions.length).toBe(16); + expect(versions.length).toBe(17); }); it("creates the executions table with expected columns", async () => { @@ -132,6 +135,26 @@ describe.skipIf(sql === null)("runMigrations", () => { expect(names).toContain("cache_read_input_tokens"); expect(names).toContain("cache_creation_input_tokens"); expect(names).toContain("model_usage"); + expect(names).toContain("offer_id"); + expect(names).toContain("result_processed_at"); + expect(names).toContain("workflow_result_payload"); + + const indexes: { indexname: string; indexdef: string }[] = await requireDb()` + SELECT indexname, indexdef FROM pg_indexes WHERE tablename = 'executions' + `; + const offerIndex = indexes.find((index) => index.indexname === "idx_executions_offer_id"); + expect(offerIndex?.indexdef).toContain("UNIQUE"); + expect(offerIndex?.indexdef).toContain("offer_id IS NOT NULL"); + const runningDaemonIndex = indexes.find( + (index) => index.indexname === "idx_executions_running_daemon", + ); + expect(runningDaemonIndex?.indexdef).toContain("daemon_id"); + expect(runningDaemonIndex?.indexdef).toContain("status = 'running'"); + const resultPendingIndex = indexes.find( + (index) => index.indexname === "idx_executions_workflow_result_pending", + ); + expect(resultPendingIndex?.indexdef).toContain("workflow_result_payload IS NOT NULL"); + expect(resultPendingIndex?.indexdef).toContain("result_processed_at IS NULL"); }); it("creates the daemons table with expected columns", async () => { @@ -221,6 +244,20 @@ describe.skipIf(sql === null)("runMigrations", () => { expect(names).toContain("delivery_id"); expect(names).toContain("created_at"); expect(names).toContain("updated_at"); + expect(names).toContain("attempt_id"); + expect(names).toContain("lease_expires_at"); + expect(names).toContain("attempt_deadline_at"); + expect(names).toContain("attempt_completed_at"); + expect(names).toContain("cascade_completed_at"); + expect(names).toContain("execution_delivery_id"); + expect(names).toContain("trigger_body_preview"); + expect(names).toContain("dispatch_enqueued_at"); + expect(names).toContain("dispatch_generation_id"); + expect(names).toContain("runner_payload_issued_at"); + expect(names).toContain("runner_token_expires_at"); + expect(names).toContain("runner_resources_cleaned_at"); + expect(names).toContain("failure_notified_at"); + expect(names).toContain("dispatch_retry_count"); const notNull = columns.filter((c) => c.is_nullable === "NO").map((c) => c.column_name); expect(notNull).toContain("workflow_name"); @@ -230,13 +267,538 @@ describe.skipIf(sql === null)("runMigrations", () => { expect(notNull).toContain("target_number"); expect(notNull).toContain("status"); expect(notNull).toContain("state"); + expect(notNull).toContain("trigger_body_preview"); + expect(notNull).toContain("dispatch_generation_id"); + expect(notNull).toContain("dispatch_retry_count"); + + const [payloadReceiptConstraint] = await requireDb()<{ definition: string }[]>` + SELECT pg_get_constraintdef(oid) AS definition + FROM pg_constraint + WHERE conname = 'workflow_runs_runner_payload_receipt_check' + `; + expect(payloadReceiptConstraint?.definition).toContain("runner_payload_issued_at IS NULL"); + expect(payloadReceiptConstraint?.definition).toContain( + "runner_token_expires_at <= attempt_deadline_at", + ); - const indexes: { indexname: string }[] = await requireDb()` - SELECT indexname FROM pg_indexes WHERE tablename = 'workflow_runs' + const indexes: { indexname: string; indexdef: string }[] = await requireDb()` + SELECT indexname, indexdef FROM pg_indexes WHERE tablename = 'workflow_runs' `; const idxNames = indexes.map((i) => i.indexname); expect(idxNames).toContain("idx_workflow_runs_inflight"); expect(idxNames).toContain("idx_workflow_runs_target"); expect(idxNames).toContain("idx_workflow_runs_parent"); + const leaseIndex = indexes.find((index) => index.indexdef.includes("lease_expires_at")); + expect(leaseIndex?.indexdef).toContain("attempt_deadline_at"); + expect(leaseIndex?.indexdef).toContain("status"); + expect(leaseIndex?.indexdef).toContain("running"); + const attemptIndex = indexes.find( + (index) => index.indexname === "idx_workflow_runs_attempt_id", + ); + expect(attemptIndex?.indexdef).toContain("UNIQUE"); + expect(attemptIndex?.indexdef).toContain("attempt_id IS NOT NULL"); + const dispatchIndex = indexes.find( + (index) => index.indexname === "idx_workflow_runs_dispatch_pending", + ); + expect(dispatchIndex?.indexdef).toContain("execution_delivery_id IS NOT NULL"); + expect(dispatchIndex?.indexdef).toContain("dispatch_enqueued_at"); + expect(dispatchIndex?.indexdef).not.toContain("dispatch_enqueued_at IS NULL"); + const cleanupIndex = indexes.find( + (index) => index.indexname === "idx_workflow_runs_runner_cleanup_pending", + ); + expect(cleanupIndex?.indexdef).toContain("attempt_completed_at IS NOT NULL"); + expect(cleanupIndex?.indexdef).toContain("runner_resources_cleaned_at IS NULL"); + const notificationIndex = indexes.find( + (index) => index.indexname === "idx_workflow_runs_failure_notification_pending", + ); + expect(notificationIndex?.indexdef).toContain("status = 'failed'"); + expect(notificationIndex?.indexdef).toContain("attempt_completed_at IS NOT NULL"); + expect(notificationIndex?.indexdef).toContain("failure_notified_at IS NULL"); + expect(notificationIndex?.indexdef).not.toContain("attempt_id IS NOT NULL"); + + const repoIndexes: { indexname: string; indexdef: string }[] = await requireDb()` + SELECT indexname, indexdef FROM pg_indexes WHERE tablename = 'repo_memory' + `; + const learningIndex = repoIndexes.find( + (index) => index.indexname === "idx_repo_memory_learning_unique", + ); + expect(learningIndex?.indexdef).toContain("UNIQUE"); + expect(learningIndex?.indexdef).toContain("content_sha256"); + expect(learningIndex?.indexdef).toContain("category <> 'env_var'"); + + const repoColumns: { column_name: string }[] = await requireDb()` + SELECT column_name + FROM information_schema.columns + WHERE table_name = 'repo_memory' + `; + expect(repoColumns.map((column) => column.column_name)).toContain("content_sha256"); + const [hashTrigger] = await requireDb()<{ trigger_name: string }[]>` + SELECT trigger_name + FROM information_schema.triggers + WHERE event_object_table = 'repo_memory' + AND trigger_name = 'repo_memory_content_sha256' + `; + expect(hashTrigger?.trigger_name).toBe("repo_memory_content_sha256"); + }); + + it("creates the workflow attempt command receipt table", async () => { + const columns: { column_name: string; is_nullable: string }[] = await requireDb()` + SELECT column_name, is_nullable + FROM information_schema.columns + WHERE table_name = 'workflow_attempt_commands' + ORDER BY ordinal_position + `; + expect(columns.map((column) => column.column_name)).toEqual([ + "attempt_id", + "command_id", + "run_id", + "command_kind", + "request", + "response", + "created_at", + ]); + expect(columns.every((column) => column.is_nullable === "NO")).toBe(true); + + const constraints: { constraint_type: string; definition: string }[] = await requireDb()` + SELECT CASE contype + WHEN 'p' THEN 'PRIMARY KEY' + WHEN 'f' THEN 'FOREIGN KEY' + WHEN 'c' THEN 'CHECK' + ELSE contype::text + END AS constraint_type, + pg_get_constraintdef(oid) AS definition + FROM pg_constraint + WHERE conrelid = 'workflow_attempt_commands'::regclass + `; + expect(constraints).toContainEqual({ + constraint_type: "PRIMARY KEY", + definition: "PRIMARY KEY (attempt_id, command_id)", + }); + expect(constraints).toContainEqual({ + constraint_type: "FOREIGN KEY", + definition: "FOREIGN KEY (run_id) REFERENCES workflow_runs(id) ON DELETE CASCADE", + }); + }); + + it("backfills queued work and fails active shared-daemon workflow attempts", async () => { + await requireDb().unsafe(` + TRUNCATE workflow_runs, executions CASCADE; + DELETE FROM repo_memory; + DELETE FROM _migrations WHERE version = '017_workflow_run_leases'; + DROP TABLE IF EXISTS workflow_attempt_commands CASCADE; + DROP INDEX idx_executions_running_daemon; + DROP INDEX idx_repo_memory_learning_unique; + DROP TRIGGER repo_memory_content_sha256 ON repo_memory; + ALTER TABLE repo_memory + DROP CONSTRAINT repo_memory_learning_hash_check, + DROP COLUMN content_sha256; + ALTER TABLE workflow_runs + DROP COLUMN attempt_id, + DROP COLUMN lease_expires_at, + DROP COLUMN attempt_deadline_at, + DROP COLUMN attempt_completed_at, + DROP COLUMN cascade_completed_at, + DROP COLUMN execution_delivery_id, + DROP COLUMN trigger_body_preview, + DROP COLUMN dispatch_enqueued_at, + DROP COLUMN dispatch_generation_id, + DROP COLUMN runner_payload_issued_at, + DROP COLUMN runner_token_expires_at, + DROP COLUMN runner_resources_cleaned_at, + DROP COLUMN failure_notified_at, + DROP COLUMN dispatch_retry_count; + ALTER TABLE executions + DROP COLUMN offer_id, + DROP COLUMN result_processed_at, + DROP COLUMN workflow_result_payload; + `); + + const [parent] = await requireDb()<{ id: string }[]>`INSERT INTO workflow_runs ( + workflow_name, target_type, target_owner, target_repo, target_number, + status, state, owner_kind, owner_id + ) VALUES ( + 'ship', 'pr', 'acme', 'widgets', 700, + 'running', '{}'::jsonb, 'daemon', 'old-daemon' + ) RETURNING id`; + if (parent === undefined) throw new Error("migration parent seed failed"); + + const [missingChildParent] = await requireDb()<{ id: string }[]>`INSERT INTO workflow_runs ( + workflow_name, target_type, target_owner, target_repo, target_number, + status, state, owner_kind, owner_id + ) VALUES ( + 'ship', 'pr', 'acme', 'widgets', 708, + 'running', '{}'::jsonb, 'daemon', 'old-daemon' + ) RETURNING id`; + const [activeChildParent] = await requireDb()<{ id: string }[]>`INSERT INTO workflow_runs ( + workflow_name, target_type, target_owner, target_repo, target_number, + status, state, owner_kind, owner_id + ) VALUES ( + 'ship', 'pr', 'acme', 'widgets', 710, + 'running', '{}'::jsonb, 'daemon', 'old-daemon' + ) RETURNING id`; + const [overlapParent] = await requireDb()<{ id: string }[]>`INSERT INTO workflow_runs ( + workflow_name, target_type, target_owner, target_repo, target_number, + status, state, delivery_id, owner_kind, owner_id + ) VALUES ( + 'ship', 'pr', 'acme', 'widgets', 712, + 'running', '{}'::jsonb, 'overlap-parent-execution', 'daemon', 'old-daemon' + ) RETURNING id`; + if ( + missingChildParent === undefined || + activeChildParent === undefined || + overlapParent === undefined + ) { + throw new Error("migration failure-propagation parent seed failed"); + } + + const rows: { id: string; target_number: number; updated_at: Date }[] = await requireDb()` + INSERT INTO workflow_runs ( + workflow_name, target_type, target_owner, target_repo, target_number, + parent_run_id, parent_step_index, status, state, delivery_id, + owner_kind, owner_id + ) VALUES + ('review', 'pr', 'acme', 'widgets', 701, + NULL, NULL, 'queued', '{}'::jsonb, 'top-level-delivery', + 'orchestrator', 'old-orchestrator'), + ('review', 'pr', 'acme', 'widgets', 702, + ${parent.id}, 0, 'queued', '{}'::jsonb, 'parent-trace-delivery', + NULL, NULL), + ('resolve', 'pr', 'acme', 'widgets', 703, + NULL, NULL, 'queued', ${{ shipIntentId: "intent-1", iteration_n: 4 }}::jsonb, NULL, + 'orchestrator', 'old-orchestrator'), + ('triage', 'issue', 'acme', 'widgets', 704, + NULL, NULL, 'failed', '{}'::jsonb, 'terminal-delivery', + NULL, NULL), + ('triage', 'issue', 'acme', 'widgets', 706, + NULL, NULL, 'queued', '{}'::jsonb, 'missing-execution', + NULL, NULL), + ('triage', 'issue', 'acme', 'widgets', 705, + NULL, NULL, 'running', '{}'::jsonb, 'pre-017-offer', + 'daemon', 'old-daemon'), + ('triage', 'issue', 'acme', 'widgets', 707, + NULL, NULL, 'running', '{}'::jsonb, 'pre-017-running', + 'daemon', 'old-daemon'), + ('review', 'pr', 'acme', 'widgets', 709, + ${missingChildParent.id}, 2, 'queued', '{}'::jsonb, 'missing-child-execution', + NULL, NULL), + ('review', 'pr', 'acme', 'widgets', 711, + ${activeChildParent.id}, 3, 'running', '{}'::jsonb, 'active-child-execution', + 'daemon', 'old-daemon'), + ('review', 'pr', 'acme', 'widgets', 713, + ${overlapParent.id}, 4, 'running', '{}'::jsonb, 'overlap-child-execution', + 'daemon', 'old-daemon') + RETURNING id, target_number, updated_at + `; + const reconstructableChild = rows.find((row) => row.target_number === 702); + const activeChild = rows.find((row) => row.target_number === 711); + const overlapChild = rows.find((row) => row.target_number === 713); + if ( + reconstructableChild === undefined || + activeChild === undefined || + overlapChild === undefined + ) { + throw new Error("migration child seed failed"); + } + await requireDb()` + INSERT INTO executions ( + delivery_id, repo_owner, repo_name, entity_number, entity_type, + event_name, trigger_username, dispatch_mode, dispatch_target, + dispatch_reason, status, daemon_id + ) VALUES + ('pre-017-offer', 'acme', 'widgets', 705, 'issue', + 'issue_comment', 'user', 'daemon', 'daemon', + 'persistent-daemon', 'offered', 'old-daemon'), + ('pre-017-running', 'acme', 'widgets', 707, 'issue', + 'issue_comment', 'user', 'daemon', 'daemon', + 'persistent-daemon', 'running', 'old-daemon'), + ('top-level-delivery', 'acme', 'widgets', 701, 'pr', + 'pull_request', 'user', 'daemon', 'daemon', + 'persistent-daemon', 'queued', NULL), + (${reconstructableChild.id}, 'acme', 'widgets', 702, 'pr', + 'pull_request', 'user', 'daemon', 'daemon', + 'persistent-daemon', 'queued', NULL), + ('intent-1::iteration::4', 'acme', 'widgets', 703, 'pr', + 'pull_request', 'user', 'daemon', 'daemon', + 'persistent-daemon', 'queued', NULL), + (${activeChild.id}, 'acme', 'widgets', 711, 'pr', + 'pull_request', 'user', 'daemon', 'daemon', + 'persistent-daemon', 'running', 'old-daemon'), + ('overlap-parent-execution', 'acme', 'widgets', 712, 'pr', + 'pull_request', 'user', 'daemon', 'daemon', + 'persistent-daemon', 'running', 'old-daemon'), + (${overlapChild.id}, 'acme', 'widgets', 713, 'pr', + 'pull_request', 'user', 'daemon', 'daemon', + 'persistent-daemon', 'running', 'old-daemon') + `; + await requireDb()` + INSERT INTO scheduled_action_state ( + installation_id, owner, repo, action_name, in_flight_job_id, in_flight_started_at + ) VALUES + (1, 'acme', 'widgets', 'pre-017-offer', 'pre-017-offer', now()), + (1, 'acme', 'widgets', 'pre-017-running', 'pre-017-running', now()) + `; + await requireDb()` + INSERT INTO repo_memory (repo_owner, repo_name, category, content) + VALUES + ('acme', 'widgets', 'gotchas', 'A'), + ('acme', 'widgets', 'gotchas', ${String.raw`\x41`}), + ('acme', 'widgets', 'gotchas', ${String.raw`C:\Users\bin`}) + `; + + const { runMigrations } = await import("../../src/db/migrate"); + await runMigrations(requireDb()); + + const learningHashes = await requireDb()<{ content: string; content_hash: string }[]>` + SELECT content, encode(content_sha256, 'hex') AS content_hash + FROM repo_memory + WHERE repo_owner = 'acme' AND repo_name = 'widgets' + `; + expect(learningHashes.map((row) => row.content).sort()).toEqual( + ["A", String.raw`\x41`, String.raw`C:\Users\bin`].sort(), + ); + expect(new Set(learningHashes.map((row) => row.content_hash)).size).toBe(3); + + const migrated: { + id: string; + status: string; + execution_delivery_id: string | null; + trigger_body_preview: string; + dispatch_enqueued_at: Date | null; + state: Record; + updated_at: Date; + }[] = await requireDb()` + SELECT id, status, state, execution_delivery_id, trigger_body_preview, + dispatch_enqueued_at, updated_at + FROM workflow_runs + WHERE target_number IN (701, 702, 703, 704, 706) + ORDER BY target_number + `; + expect(migrated.map((row) => row.execution_delivery_id)).toEqual([ + "top-level-delivery", + reconstructableChild.id, + "intent-1::iteration::4", + "terminal-delivery", + "missing-execution", + ]); + expect(migrated.slice(0, 3).map((row) => row.dispatch_enqueued_at)).toEqual([null, null, null]); + expect(migrated[3]?.dispatch_enqueued_at?.getTime()).toBe(migrated[3]?.updated_at.getTime()); + expect(migrated[4]?.dispatch_enqueued_at).toBeInstanceOf(Date); + expect(migrated.map((row) => row.status)).toEqual([ + "queued", + "queued", + "queued", + "failed", + "failed", + ]); + expect(migrated.map((row) => row.trigger_body_preview)).toEqual(["", "", "", "", ""]); + expect(migrated[4]?.state["failedReason"]).toBe( + "workflow dispatch incomplete during lease migration", + ); + expect(migrated[4]?.state["phase"]).toBe("migration-interrupted"); + + const [pending] = await requireDb()<{ count: number }[]>` + SELECT count(*)::int AS count + FROM workflow_runs + WHERE status = 'queued' + AND execution_delivery_id IS NOT NULL + AND dispatch_enqueued_at IS NULL + `; + expect(pending?.count).toBe(3); + + const dispatchTelemetry = await requireDb()< + { + delivery_id: string; + dispatch_mode: string; + dispatch_target: string; + dispatch_reason: string; + }[] + >` + SELECT delivery_id, dispatch_mode, dispatch_target, dispatch_reason + FROM executions + WHERE delivery_id IN ( + 'pre-017-offer', 'pre-017-running', 'top-level-delivery', + ${reconstructableChild.id}, 'intent-1::iteration::4' + ) + ORDER BY delivery_id + `; + expect( + dispatchTelemetry + .filter((row) => row.delivery_id.startsWith("pre-017-")) + .map((row) => [row.dispatch_mode, row.dispatch_target, row.dispatch_reason]), + ).toEqual([ + ["daemon", "daemon", "persistent-daemon"], + ["daemon", "daemon", "persistent-daemon"], + ]); + expect( + dispatchTelemetry + .filter((row) => !row.delivery_id.startsWith("pre-017-")) + .map((row) => [row.dispatch_mode, row.dispatch_target, row.dispatch_reason]), + ).toEqual([ + ["workflow-runner", "workflow-runner", "workflow-runner"], + ["workflow-runner", "workflow-runner", "workflow-runner"], + ["workflow-runner", "workflow-runner", "workflow-runner"], + ]); + + const activeWorkflowRows = await requireDb()< + { target_number: number; status: string; state: Record }[] + >` + SELECT target_number, status, state + FROM workflow_runs + WHERE target_number IN (705, 707) + ORDER BY target_number + `; + expect(activeWorkflowRows.map((row) => row.status)).toEqual(["failed", "failed"]); + expect(activeWorkflowRows[0]?.state["failedReason"]).toBe( + "workflow execution interrupted during isolated-runner migration", + ); + expect(activeWorkflowRows.map((row) => row.state["phase"])).toEqual([ + "migration-interrupted", + "migration-interrupted", + ]); + const [preservedParent] = await requireDb()< + { status: string; owner_kind: string | null; owner_id: string | null }[] + >` + SELECT status, owner_kind, owner_id FROM workflow_runs WHERE id = ${parent.id} + `; + expect(preservedParent?.status).toBe("running"); + expect(preservedParent?.owner_kind).toBeNull(); + expect(preservedParent?.owner_id).toBeNull(); + + const propagatedFailures = await requireDb()< + { target_number: number; status: string; state: Record }[] + >` + SELECT target_number, status, state + FROM workflow_runs + WHERE target_number IN (708, 709, 710, 711) + ORDER BY target_number + `; + expect(propagatedFailures.map((row) => row.status)).toEqual([ + "failed", + "failed", + "failed", + "failed", + ]); + expect(propagatedFailures[0]?.state).toMatchObject({ + failedAtStepIndex: 2, + failedReason: "workflow dispatch incomplete during lease migration", + }); + expect(propagatedFailures[1]?.state["failedReason"]).toBe( + "workflow dispatch incomplete during lease migration", + ); + expect(propagatedFailures[2]?.state).toMatchObject({ + failedAtStepIndex: 3, + failedReason: "workflow execution interrupted during isolated-runner migration", + }); + expect(propagatedFailures[3]?.state["failedReason"]).toBe( + "workflow execution interrupted during isolated-runner migration", + ); + const { findPendingWorkflowFailureNotifications } = + await import("../../src/workflows/runs-store"); + const migrationNotifications = await findPendingWorkflowFailureNotifications(requireDb()); + expect( + migrationNotifications + .filter((entry) => entry.phase === "migration-interrupted") + .map((entry) => entry.row.target_number) + .sort((left, right) => left - right), + ).toEqual([705, 706, 707, 709, 711, 712, 713]); + + const overlappingWorkflowRows = await requireDb()< + { + target_number: number; + status: string; + state: Record; + attempt_completed_at: Date | null; + }[] + >` + SELECT target_number, status, state, attempt_completed_at + FROM workflow_runs + WHERE target_number IN (712, 713) + ORDER BY target_number + `; + expect(overlappingWorkflowRows).toEqual([ + { + target_number: 712, + status: "failed", + state: { + phase: "migration-interrupted", + failedReason: "workflow execution interrupted during isolated-runner migration", + }, + attempt_completed_at: expect.any(Date), + }, + { + target_number: 713, + status: "failed", + state: { + phase: "migration-interrupted", + failedReason: "workflow execution interrupted during isolated-runner migration", + }, + attempt_completed_at: expect.any(Date), + }, + ]); + const overlappingExecutions = await requireDb()< + { + delivery_id: string; + entity_number: number; + status: string; + result_processed_at: Date | null; + }[] + >` + SELECT delivery_id, entity_number, status, result_processed_at + FROM executions + WHERE entity_number IN (712, 713) + ORDER BY entity_number + `; + expect(overlappingExecutions).toEqual([ + { + delivery_id: "overlap-parent-execution", + entity_number: 712, + status: "failed", + result_processed_at: expect.any(Date), + }, + { + delivery_id: overlapChild.id, + entity_number: 713, + status: "failed", + result_processed_at: expect.any(Date), + }, + ]); + + const interrupted = await requireDb()< + { + delivery_id: string; + status: string; + error_message: string | null; + result_processed_at: Date | null; + }[] + >` + SELECT delivery_id, status, error_message, result_processed_at + FROM executions + WHERE delivery_id IN ('pre-017-offer', 'pre-017-running') + ORDER BY delivery_id + `; + expect(interrupted).toEqual([ + { + delivery_id: "pre-017-offer", + status: "failed", + error_message: "workflow execution interrupted during isolated-runner migration", + result_processed_at: expect.any(Date), + }, + { + delivery_id: "pre-017-running", + status: "failed", + error_message: "workflow execution interrupted during isolated-runner migration", + result_processed_at: expect.any(Date), + }, + ]); + const locks = await requireDb()<{ action_name: string; in_flight_job_id: string | null }[]>` + SELECT action_name, in_flight_job_id + FROM scheduled_action_state + WHERE action_name IN ('pre-017-offer', 'pre-017-running') + ORDER BY action_name + `; + expect(locks).toEqual([ + { action_name: "pre-017-offer", in_flight_job_id: null }, + { action_name: "pre-017-running", in_flight_job_id: null }, + ]); }); }); diff --git a/test/db/migrations/008.test.ts b/test/db/migrations/008.test.ts index 18a6f20b..d48eb831 100644 --- a/test/db/migrations/008.test.ts +++ b/test/db/migrations/008.test.ts @@ -32,6 +32,7 @@ function requireDb(): SQL { async function dropAll(): Promise { await requireDb().unsafe(` DROP TABLE IF EXISTS _migrations CASCADE; + DROP TABLE IF EXISTS workflow_attempt_commands CASCADE; DROP TABLE IF EXISTS review_learnings CASCADE; DROP TABLE IF EXISTS scheduled_action_state CASCADE; DROP TABLE IF EXISTS comment_cache CASCADE; diff --git a/test/fixtures/workflow-runner-kind.yaml b/test/fixtures/workflow-runner-kind.yaml new file mode 100644 index 00000000..1a2d877c --- /dev/null +++ b/test/fixtures/workflow-runner-kind.yaml @@ -0,0 +1,9 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: + - role: control-plane + kubeadmConfigPatches: + - | + kind: KubeletConfiguration + apiVersion: kubelet.config.k8s.io/v1beta1 + podPidsLimit: 256 diff --git a/test/integration/repo-knowledge.test.ts b/test/integration/repo-knowledge.test.ts index da29acb1..db474504 100644 --- a/test/integration/repo-knowledge.test.ts +++ b/test/integration/repo-knowledge.test.ts @@ -50,6 +50,7 @@ describe.skipIf(sql === null)("repo-knowledge ANY() array binding regression", ( const db = requireSql(); await db.unsafe(` DROP TABLE IF EXISTS _migrations CASCADE; + DROP TABLE IF EXISTS workflow_attempt_commands CASCADE; DROP TABLE IF EXISTS review_learnings CASCADE; DROP TABLE IF EXISTS scheduled_action_state CASCADE; DROP TABLE IF EXISTS comment_cache CASCADE; @@ -127,7 +128,15 @@ describe.skipIf(sql === null)("repo-knowledge ANY() array binding regression", ( const ids = seeded.slice(0, 2).map((r) => r.id); const { deleteRepoMemories } = await import("../../src/orchestrator/repo-knowledge"); - const deletedCount = await deleteRepoMemories(ids, db); + const otherRepo: { id: string }[] = await db` + INSERT INTO repo_memory (repo_owner, repo_name, category, content, pinned) + VALUES (${TEST_OWNER}, 'other-repo', 'gotchas', 'must-survive', false) + RETURNING id + `; + const crossRepoId = otherRepo[0]?.id; + if (crossRepoId === undefined) throw new Error("Expected cross-repo fixture"); + + const deletedCount = await deleteRepoMemories(TEST_OWNER, TEST_REPO, [...ids, crossRepoId], db); expect(deletedCount).toBe(2); const remaining: { content: string }[] = await db` @@ -139,13 +148,18 @@ describe.skipIf(sql === null)("repo-knowledge ANY() array binding regression", ( expect(contents).toContain("keep-me"); expect(contents).not.toContain("to-delete-1"); expect(contents).not.toContain("to-delete-2"); + const otherRows: { content: string }[] = await db` + SELECT content FROM repo_memory + WHERE repo_owner = ${TEST_OWNER} AND repo_name = 'other-repo' + `; + expect(otherRows).toEqual([{ content: "must-survive" }]); }); it("deleteRepoMemories with empty array is a no-op (does not query DB)", async () => { const db = requireSql(); const { deleteRepoMemories } = await import("../../src/orchestrator/repo-knowledge"); - const result = await deleteRepoMemories([], db); + const result = await deleteRepoMemories(TEST_OWNER, TEST_REPO, [], db); expect(result).toBe(0); }); }); @@ -257,4 +271,77 @@ describe.skipIf(sql2 === null)("saveRepoLearnings sanitization at durability bou expect(rows[0]!.content).not.toContain(tok); expect(rows[0]!.content).toContain("[REDACTED_GITHUB_TOKEN]"); }); + + it("hashes text bytes without interpreting bytea escape syntax", async () => { + const db = requireSql2(); + const { saveRepoLearnings } = await import("../../src/orchestrator/repo-knowledge"); + const repo = `${TEST_REPO}-text-hash`; + + expect( + await saveRepoLearnings( + TEST_OWNER, + repo, + [ + { category: "gotchas", content: "A" }, + { category: "gotchas", content: String.raw`\x41` }, + { category: "gotchas", content: String.raw`C:\Users\bin` }, + ], + db, + ), + ).toBe(3); + + const rows: { content: string }[] = await db` + SELECT content + FROM repo_memory + WHERE repo_owner = ${TEST_OWNER} + AND repo_name = ${repo} + AND category = 'gotchas' + ORDER BY content + `; + expect(rows.map((row) => row.content).sort()).toEqual( + ["A", String.raw`\x41`, String.raw`C:\Users\bin`].sort(), + ); + }); + + it("stores one row across duplicate and replayed saves", async () => { + const db = requireSql2(); + const { saveRepoLearnings } = await import("../../src/orchestrator/repo-knowledge"); + const learning = { category: "architecture", content: "One durable controller owns state." }; + + expect(await saveRepoLearnings(TEST_OWNER, TEST_REPO, [learning], db)).toBe(1); + expect(await saveRepoLearnings(TEST_OWNER, TEST_REPO, [learning], db)).toBe(0); + expect(await saveRepoLearnings(TEST_OWNER, TEST_REPO, [learning], db)).toBe(0); + + const rows: { count: number; updated_at: Date }[] = await db` + SELECT count(*)::int AS count, max(updated_at) AS updated_at + FROM repo_memory + WHERE repo_owner = ${TEST_OWNER} + AND repo_name = ${TEST_REPO} + AND category = ${learning.category} + AND content = ${learning.content} + `; + expect(rows[0]?.count).toBe(1); + expect(rows[0]?.updated_at).toBeInstanceOf(Date); + }); + + it("stores one row when identical result projections race", async () => { + const db = requireSql2(); + const { saveRepoLearnings } = await import("../../src/orchestrator/repo-knowledge"); + const learning = { category: "gotchas", content: "Concurrent replay stays idempotent." }; + + const saved = await Promise.all( + Array.from({ length: 10 }, () => saveRepoLearnings(TEST_OWNER, TEST_REPO, [learning], db)), + ); + + expect(saved.reduce((total, count) => total + count, 0)).toBe(1); + const rows: { count: number }[] = await db` + SELECT count(*)::int AS count + FROM repo_memory + WHERE repo_owner = ${TEST_OWNER} + AND repo_name = ${TEST_REPO} + AND category = ${learning.category} + AND content = ${learning.content} + `; + expect(rows[0]?.count).toBe(1); + }); }); diff --git a/test/integration/review-learnings.test.ts b/test/integration/review-learnings.test.ts index 7becb71e..97d7eec0 100644 --- a/test/integration/review-learnings.test.ts +++ b/test/integration/review-learnings.test.ts @@ -35,6 +35,7 @@ describe.skipIf(sql === null)("review-learnings loader and persistence", () => { const db = requireSql(); await db.unsafe(` DROP TABLE IF EXISTS _migrations CASCADE; + DROP TABLE IF EXISTS workflow_attempt_commands CASCADE; DROP TABLE IF EXISTS review_learnings CASCADE; DROP TABLE IF EXISTS scheduled_action_state CASCADE; DROP TABLE IF EXISTS comment_cache CASCADE; @@ -101,11 +102,11 @@ describe.skipIf(sql === null)("review-learnings loader and persistence", () => { const { loadReviewLearnings } = await import("../../src/orchestrator/review-learnings"); // Loader is intentionally file-glob agnostic AND pure-read: the - // orchestrator does not yet know which files the PR touched at - // job-accept time, so the daemon-side prompt-builder applies the filter + // controller does not yet know which files the PR touched during payload + // preparation, so the worker-side prompt-builder applies the filter // via pickApplicableLearnings. Per 1.5.E, the use_count bump moved to - // bumpReviewLearningUsage and fires only after the daemon reports the - // applied ids in job:result. + // bumpReviewLearningUsage and fires only after the worker reports the + // applied ids in its terminal result. const loaded = await loadReviewLearnings(TEST_OWNER, TEST_REPO, {}, db); expect(loaded.length).toBe(1); diff --git a/test/integration/scoped-rebase-roundtrip.test.ts b/test/integration/scoped-rebase-roundtrip.test.ts index 3cc9bdf5..a9424240 100644 --- a/test/integration/scoped-rebase-roundtrip.test.ts +++ b/test/integration/scoped-rebase-roundtrip.test.ts @@ -4,10 +4,10 @@ * Exercises the full WS message contract for one scoped kind end-to-end: * * 1. Build a real `scoped-rebase` `ScopedQueuedJob` shape. - * 2. Construct the `scoped-job-offer` envelope the orchestrator sends to + * 2. Construct the `scoped-job:offer` envelope the orchestrator sends to * a daemon (per `contracts/ws-messages.md`); round-trip through the * `serverMessageSchema` discriminated union. - * 3. Build the daemon's `scoped-job-completion` reply for a `merged` + * 3. Build the daemon's `scoped-job:completion` reply for a `merged` * outcome; round-trip through `daemonMessageSchema`. * 4. Drive that parsed completion message through the orchestrator's * WS-side `handleDaemonMessage` router; assert the side-effects: @@ -89,6 +89,9 @@ void mock.module("../../src/orchestrator/daemon-registry", () => ({ })); void mock.module("../../src/orchestrator/history", () => ({ + failDisconnectedDaemon: mock(() => + Promise.resolve({ executionDeliveryIds: [], workflowRunIds: [] }), + ), markExecutionOffered: mock(() => Promise.resolve()), markExecutionFailed: markExecutionFailedSpy, markExecutionRunning: mock(() => Promise.resolve()), @@ -150,7 +153,7 @@ describe("T028: scoped-rebase WS contract round-trip", () => { // 1. Producer-side: build the offer envelope as the orchestrator does. const offerEnvelope = { - type: "scoped-job-offer" as const, + type: "scoped-job:offer" as const, ...createMessageEnvelope(FAKE_OFFER_ID), payload: { jobKind: "scoped-rebase" as const, @@ -168,11 +171,11 @@ describe("T028: scoped-rebase WS contract round-trip", () => { const offerParsed = serverMessageSchema.safeParse(offerEnvelope); expect(offerParsed.success).toBe(true); if (!offerParsed.success) return; - expect(offerParsed.data.type).toBe("scoped-job-offer"); + expect(offerParsed.data.type).toBe("scoped-job:offer"); // 3. Daemon-side: build the matching completion message for a `merged` rebase. const completionEnvelope = { - type: "scoped-job-completion" as const, + type: "scoped-job:completion" as const, ...createMessageEnvelope(), payload: { offerId: fakePendingOffer.offerId, @@ -193,7 +196,7 @@ describe("T028: scoped-rebase WS contract round-trip", () => { const completionParsed = daemonMessageSchema.safeParse(completionEnvelope); expect(completionParsed.success).toBe(true); if (!completionParsed.success) return; - expect(completionParsed.data.type).toBe("scoped-job-completion"); + expect(completionParsed.data.type).toBe("scoped-job:completion"); // 5. Drive the parsed completion through the orchestrator's WS router. const { ws } = makeMockServerSocket(FAKE_DAEMON_ID); diff --git a/test/integration/ship-iteration-loop.test.ts b/test/integration/ship-iteration-loop.test.ts index f7937843..b146ea65 100644 --- a/test/integration/ship-iteration-loop.test.ts +++ b/test/integration/ship-iteration-loop.test.ts @@ -62,6 +62,7 @@ describe.skipIf(skipSuite)("integration: ship-iteration loop end-to-end", () => const db = requireSql(); await db.unsafe(` DROP TABLE IF EXISTS _migrations CASCADE; + DROP TABLE IF EXISTS workflow_attempt_commands CASCADE; DROP TABLE IF EXISTS review_learnings CASCADE; DROP TABLE IF EXISTS scheduled_action_state CASCADE; DROP TABLE IF EXISTS comment_cache CASCADE; @@ -92,6 +93,7 @@ describe.skipIf(skipSuite)("integration: ship-iteration loop end-to-end", () => if (sql !== null) { await sql.unsafe(` DROP TABLE IF EXISTS _migrations CASCADE; + DROP TABLE IF EXISTS workflow_attempt_commands CASCADE; DROP TABLE IF EXISTS review_learnings CASCADE; DROP TABLE IF EXISTS scheduled_action_state CASCADE; DROP TABLE IF EXISTS comment_cache CASCADE; diff --git a/test/integration/ship-tickle-resume.test.ts b/test/integration/ship-tickle-resume.test.ts index 12e40b8e..f1f98813 100644 --- a/test/integration/ship-tickle-resume.test.ts +++ b/test/integration/ship-tickle-resume.test.ts @@ -56,6 +56,7 @@ describe.skipIf(skipSuite)("integration: ship tickle-scheduler resume", () => { const db = requireSql(); await db.unsafe(` DROP TABLE IF EXISTS _migrations CASCADE; + DROP TABLE IF EXISTS workflow_attempt_commands CASCADE; DROP TABLE IF EXISTS review_learnings CASCADE; DROP TABLE IF EXISTS scheduled_action_state CASCADE; DROP TABLE IF EXISTS comment_cache CASCADE; @@ -83,6 +84,7 @@ describe.skipIf(skipSuite)("integration: ship tickle-scheduler resume", () => { if (sql !== null) { await sql.unsafe(` DROP TABLE IF EXISTS _migrations CASCADE; + DROP TABLE IF EXISTS workflow_attempt_commands CASCADE; DROP TABLE IF EXISTS review_learnings CASCADE; DROP TABLE IF EXISTS scheduled_action_state CASCADE; DROP TABLE IF EXISTS comment_cache CASCADE; diff --git a/test/integration/telemetry-aggregates.test.ts b/test/integration/telemetry-aggregates.test.ts index 295eb031..b156779a 100644 --- a/test/integration/telemetry-aggregates.test.ts +++ b/test/integration/telemetry-aggregates.test.ts @@ -33,6 +33,7 @@ describe.skipIf(sql === null)("FR-014 aggregate queries, dispatch-stats.ts", () const db = requireSql(); await db.unsafe(` DROP TABLE IF EXISTS _migrations CASCADE; + DROP TABLE IF EXISTS workflow_attempt_commands CASCADE; DROP TABLE IF EXISTS review_learnings CASCADE; DROP TABLE IF EXISTS scheduled_action_state CASCADE; DROP TABLE IF EXISTS comment_cache CASCADE; @@ -51,10 +52,9 @@ describe.skipIf(sql === null)("FR-014 aggregate queries, dispatch-stats.ts", () const { runMigrations } = await import("../../src/db/migrate"); await runMigrations(db); - // Post-collapse dispatch values: target always 'daemon'; reason in - // {persistent-daemon, ephemeral-daemon-triage, ephemeral-daemon-overflow, - // ephemeral-spawn-failed}. All rows inside the default 30-day window - // except the '60 days' out-of-window row used for window-filter coverage. + // Shared jobs retain the daemon target and structured workflows use the + // workflow-runner target. All rows are inside the default 30-day window + // except the 60-day row used for window-filter coverage. await db` INSERT INTO executions ( delivery_id, repo_owner, repo_name, entity_number, entity_type, @@ -67,6 +67,7 @@ describe.skipIf(sql === null)("FR-014 aggregate queries, dispatch-stats.ts", () ('d-4', 'o', 'r', 4, 'issue', 'issue_comment', 'u', 'daemon', 'daemon', 'ephemeral-spawn-failed', NULL, NULL, 'queued', NOW() - INTERVAL '2 days'), ('d-5', 'o', 'r', 5, 'issue', 'issue_comment', 'u', 'daemon', 'daemon', 'persistent-daemon', NULL, NULL, 'queued', NOW() - INTERVAL '2 days'), ('d-6', 'o', 'r', 6, 'issue', 'issue_comment', 'u', 'daemon', 'daemon', 'ephemeral-daemon-overflow', NULL, NULL, 'queued', NOW() - INTERVAL '2 days'), + ('d-7', 'o', 'r', 7, 'issue', 'issue_comment', 'u', 'workflow-runner', 'workflow-runner', 'workflow-runner', NULL, NULL, 'queued', NOW() - INTERVAL '6 hours'), ('d-old','o', 'r', 99, 'issue', 'issue_comment', 'u', 'daemon', 'daemon', 'persistent-daemon', NULL, NULL, 'queued', NOW() - INTERVAL '60 days') `; @@ -87,6 +88,7 @@ describe.skipIf(sql === null)("FR-014 aggregate queries, dispatch-stats.ts", () const db = requireSql(); await db.unsafe(` DROP TABLE IF EXISTS _migrations CASCADE; + DROP TABLE IF EXISTS workflow_attempt_commands CASCADE; DROP TABLE IF EXISTS review_learnings CASCADE; DROP TABLE IF EXISTS scheduled_action_state CASCADE; DROP TABLE IF EXISTS comment_cache CASCADE; @@ -105,13 +107,13 @@ describe.skipIf(sql === null)("FR-014 aggregate queries, dispatch-stats.ts", () await db.close(); }); - it("eventsPerTarget, all rows collapse to 'daemon' post-migration", async () => { + it("eventsPerTarget separates shared daemons from isolated workflow runners", async () => { const { eventsPerTarget } = await import("../../src/db/queries/dispatch-stats"); const rows = await eventsPerTarget(30, requireSql()); - expect(rows.length).toBe(1); - expect(rows[0]?.dispatch_target).toBe("daemon"); - // 6 rows in the 30-day window; the 60-day row is excluded. - expect(rows[0]?.events).toBe(6); + expect(rows).toEqual([ + { dispatch_target: "daemon", events: 6 }, + { dispatch_target: "workflow-runner", events: 1 }, + ]); }); it("triageRate: counts ephemeral-daemon-triage as triaged", async () => { @@ -120,7 +122,7 @@ describe.skipIf(sql === null)("FR-014 aggregate queries, dispatch-stats.ts", () const totalTriaged = rows.reduce((acc, r) => acc + r.triaged, 0); const totalAll = rows.reduce((acc, r) => acc + r.total, 0); - // Only d-2 has reason='ephemeral-daemon-triage'; 6 total in-window. + // Only d-2 has reason='ephemeral-daemon-triage'; the runner row is excluded. expect(totalTriaged).toBe(1); expect(totalAll).toBe(6); @@ -155,11 +157,12 @@ describe.skipIf(sql === null)("FR-014 aggregate queries, dispatch-stats.ts", () it("queries honour the `days` parameter, shrinking the window drops rows", async () => { const { eventsPerTarget, triageSpend } = await import("../../src/db/queries/dispatch-stats"); - // 1-day window: only d-1, d-2, d-3 survive (3 rows). + // 1-day window: d-1, d-2, d-3, and d-7 survive. const rows = await eventsPerTarget(1, requireSql()); - expect(rows.length).toBe(1); - expect(rows[0]?.dispatch_target).toBe("daemon"); - expect(rows[0]?.events).toBe(3); + expect(rows).toEqual([ + { dispatch_target: "daemon", events: 3 }, + { dispatch_target: "workflow-runner", events: 1 }, + ]); const spend = await triageSpend(1, requireSql()); expect(spend.total_triage_spend_usd).toBeCloseTo(0.0017, 6); diff --git a/test/integration/workflow-dispatch-wakeup.test.ts b/test/integration/workflow-dispatch-wakeup.test.ts new file mode 100644 index 00000000..6f15e2cc --- /dev/null +++ b/test/integration/workflow-dispatch-wakeup.test.ts @@ -0,0 +1,99 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, it } from "bun:test"; + +import { + deferLeasedWorkflowJob, + ensureWorkflowJobQueued, + leaseJob, + processingListKey, + type WorkflowRunQueuedJob, +} from "../../src/orchestrator/job-queue"; +import { closeValkey, connectValkey, requireValkeyClient } from "../../src/orchestrator/valkey"; + +const instanceId = `outbox-test-${crypto.randomUUID()}`; +const receiptKeys: string[] = []; + +function job(): WorkflowRunQueuedJob { + return { + kind: "workflow-run", + deliveryId: crypto.randomUUID(), + repoOwner: "acme", + repoName: "widgets", + entityNumber: 16, + isPR: false, + eventName: "issue_comment", + triggerUsername: "maintainer", + labels: ["bot:review"], + triggerBodyPreview: "review this", + enqueuedAt: 1_787_519_000_000, + retryCount: 0, + workflowRun: { runId: crypto.randomUUID(), workflowName: "review" }, + }; +} + +async function clearTestQueue(): Promise { + const valkey = requireValkeyClient(); + const receipts = receiptKeys.splice(0); + await valkey.send("DEL", ["queue:jobs", processingListKey(instanceId), ...receipts]); +} + +describe("durable workflow dispatch wake-up", () => { + beforeAll(async () => { + await connectValkey(); + }); + + afterAll(async () => { + await clearTestQueue(); + closeValkey(); + }); + + beforeEach(clearTestQueue); + + it("restores one acknowledged item after Valkey loses it", async () => { + const wake = job(); + const valkey = requireValkeyClient(); + + expect(await ensureWorkflowJobQueued(wake, instanceId)).toBe(true); + await valkey.send("DEL", ["queue:jobs"]); + expect(await ensureWorkflowJobQueued(wake, instanceId)).toBe(true); + expect(await valkey.send("LLEN", ["queue:jobs"])).toBe(1); + }); + + it("does not duplicate an item in either the queue or processing list", async () => { + const wake = job(); + const valkey = requireValkeyClient(); + + const inserts = await Promise.all( + Array.from({ length: 10 }, () => ensureWorkflowJobQueued(wake, instanceId)), + ); + expect(inserts.filter(Boolean)).toHaveLength(1); + expect(await valkey.send("LLEN", ["queue:jobs"])).toBe(1); + + expect(await leaseJob(instanceId)).not.toBeNull(); + expect(await ensureWorkflowJobQueued(wake, instanceId)).toBe(false); + expect(await valkey.send("LLEN", ["queue:jobs"])).toBe(0); + expect(await valkey.send("LLEN", [processingListKey(instanceId)])).toBe(1); + }); + + it("keeps one byte-stable item through repeated capacity deferrals", async () => { + const wake = job(); + const stableRaw = JSON.stringify(wake); + const valkey = requireValkeyClient(); + await ensureWorkflowJobQueued(wake, instanceId); + + for (let index = 0; index < 5; index++) { + // eslint-disable-next-line no-await-in-loop -- each cycle models one capacity-limited lease + const leased = await leaseJob(instanceId); + expect(leased?.raw).toBe(stableRaw); + const deferralId = `capacity-${String(index)}-${crypto.randomUUID()}`; + receiptKeys.push(`queue:workflow-deferral-receipt:${instanceId}:${deferralId}`); + // eslint-disable-next-line no-await-in-loop -- the next lease must observe this exact move + expect(await deferLeasedWorkflowJob(instanceId, stableRaw, wake, deferralId)).toEqual({ + status: "moved", + }); + } + + expect(await valkey.send("LLEN", ["queue:jobs"])).toBe(1); + expect(await valkey.send("LLEN", [processingListKey(instanceId)])).toBe(0); + expect(await valkey.send("LINDEX", ["queue:jobs", "0"])).toBe(stableRaw); + }); +}); diff --git a/test/k8s/ephemeral-daemon-spawner.test.ts b/test/k8s/ephemeral-daemon-spawner.test.ts index c7f738c1..df5a64c7 100644 --- a/test/k8s/ephemeral-daemon-spawner.test.ts +++ b/test/k8s/ephemeral-daemon-spawner.test.ts @@ -21,10 +21,13 @@ interface CapturedCall { } const createNamespacedPod = mock((_args: CapturedCall) => Promise.resolve({ body: {} })); +let loadFromDefaultError: Error | null = null; class MockKubeConfig { loadFromCluster() {} - loadFromDefault() {} + loadFromDefault() { + if (loadFromDefaultError !== null) throw loadFromDefaultError; + } makeApiClient(_apiClass: unknown) { return { createNamespacedPod }; } @@ -48,6 +51,7 @@ beforeEach(() => { _resetK8sClientForTests(); createNamespacedPod.mockClear(); createNamespacedPod.mockImplementation(() => Promise.resolve({ body: {} })); + loadFromDefaultError = null; process.env["KUBERNETES_SERVICE_HOST"] = "10.0.0.1"; delete process.env["KUBECONFIG"]; }); @@ -88,6 +92,7 @@ describe("spawnEphemeralDaemon: Pod spec", () => { const envMap = new Map(container.env.map((e) => [e.name, e.value])); expect(envMap.get("DAEMON_EPHEMERAL")).toBe("true"); expect(envMap.get("ORCHESTRATOR_URL")).toBe("wss://orch.example.com"); + expect(envMap.get("LD_PRELOAD")).toBe("/usr/local/lib/github-app/daemon-process-guard.so"); // Credentials (including DAEMON_AUTH_TOKEN) must come from the // `daemon-secrets` Secret via envFrom, never inline in the Pod spec, // where they'd be readable via `kubectl get pod -o yaml`. @@ -106,6 +111,31 @@ describe("spawnEphemeralDaemon: Pod spec", () => { expect(spec.automountServiceAccountToken).toBe(false); }); + it("runs without CAP_SYS_PTRACE or privilege escalation", async () => { + await spawnEphemeralDaemon({ + deliveryId: "del-security", + image: "ghcr.io/org/daemon:1.0.0", + orchestratorUrl: "wss://orch.example.com", + }); + const call = createNamespacedPod.mock.calls[0] as [CapturedCall]; + const securityContext = ( + call[0].body as { + spec: { + containers: [ + { + securityContext: { + allowPrivilegeEscalation: boolean; + capabilities: { drop: string[] }; + }; + }, + ]; + }; + } + ).spec.containers[0].securityContext; + expect(securityContext.allowPrivilegeEscalation).toBe(false); + expect(securityContext.capabilities.drop).toEqual(["ALL"]); + }); + it("sanitises an unsafe deliveryId into a valid K8s label value", async () => { await spawnEphemeralDaemon({ deliveryId: "-Weird/Delivery.Id!!", @@ -134,7 +164,9 @@ describe("spawnEphemeralDaemon: Pod spec", () => { expect(spec.activeDeadlineSeconds).toBeGreaterThan(0); }); - it("pulls credentials from the daemon-secrets Secret via envFrom", async () => { + // Pins the EPHEMERAL_DAEMON_SECRET_NAME default: a deployment that never sets + // it keeps mounting `daemon-secrets`. + it("pulls credentials from the configured Secret via envFrom", async () => { await spawnEphemeralDaemon({ deliveryId: "del-sec", image: "ghcr.io/org/daemon:1.0.0", @@ -169,6 +201,32 @@ describe("spawnEphemeralDaemon: error kinds", () => { expect((caught as InstanceType).kind).toBe("infra-absent"); }); + it("does not expose kubeconfig parser context when authentication loading fails", async () => { + delete process.env["KUBERNETES_SERVICE_HOST"]; + process.env["KUBECONFIG"] = "/tmp/malformed-kubeconfig"; + loadFromDefaultError = new Error("password: kube-secret\nclient-key-data: private-key"); + _resetK8sClientForTests(); + + let caught: unknown; + try { + await spawnEphemeralDaemon({ + deliveryId: "x", + image: "img", + orchestratorUrl: "wss://x", + }); + } catch (err) { + caught = err; + } + + expect(caught).toBeInstanceOf(EphemeralSpawnError); + expect((caught as InstanceType).kind).toBe("auth-load-failed"); + expect((caught as Error).message).toBe( + "Kubernetes authentication configuration could not be loaded", + ); + expect((caught as Error).message).not.toContain("kube-secret"); + expect((caught as Error).message).not.toContain("private-key"); + }); + it("maps 4xx to api-rejected", async () => { createNamespacedPod.mockImplementation(() => { const err = new Error("forbidden") as Error & { statusCode: number }; diff --git a/test/k8s/workflow-runner-spawner.test.ts b/test/k8s/workflow-runner-spawner.test.ts new file mode 100644 index 00000000..de91a60a --- /dev/null +++ b/test/k8s/workflow-runner-spawner.test.ts @@ -0,0 +1,866 @@ +import { beforeEach, describe, expect, it, mock } from "bun:test"; + +import type { WorkflowRunnerAttempt } from "../../src/orchestrator/workflow-runner-store"; +import { FORBIDDEN_RUNNER_ENV } from "../../src/runner/process-boundary"; +import { expectToReject } from "../utils/assertions"; + +interface ResourceCall { + readonly namespace: string; + readonly body: T; +} + +const createNamespacedSecret = mock((_input: ResourceCall>) => + Promise.resolve({}), +); +const readNamespacedSecret = mock((_input: { name: string; namespace: string }) => + Promise.resolve({}), +); +const replaceNamespacedSecret = mock( + (_input: { name: string; namespace: string; body: Record }) => + Promise.resolve({}), +); +const createNamespacedPod = mock((_input: ResourceCall>) => + Promise.resolve({}), +); +const readNamespacedPod = mock((_input: { name: string; namespace: string }) => + Promise.resolve({}), +); +const deleteNamespacedPod = mock(() => Promise.resolve({})); +const deleteNamespacedSecret = mock(() => Promise.resolve({})); +const core = { + createNamespacedSecret, + readNamespacedSecret, + replaceNamespacedSecret, + createNamespacedPod, + readNamespacedPod, + deleteNamespacedPod, + deleteNamespacedSecret, +}; + +const testConfig = { + workflowRunnerNamespace: "test-ns", + // Deliberately not the schema defaults: the Pod assertions below then prove + // the nodeSelector and toleration follow configuration, not a literal. + workflowRunnerNodeLabel: "node.homelab/class", + workflowRunnerNodeValue: "worker", + provider: "anthropic" as "anthropic" | "bedrock", + model: "claude-test", + anthropicApiKey: "configured", + claudeCodeOauthToken: undefined as string | undefined, + awsRegion: undefined as string | undefined, + awsProfile: undefined as string | undefined, + awsAccessKeyId: undefined as string | undefined, + awsSecretAccessKey: undefined as string | undefined, + awsSessionToken: undefined as string | undefined, + awsBearerTokenBedrock: undefined as string | undefined, + anthropicBedrockBaseUrl: undefined as string | undefined, + allowedOwners: ["acme"] as string[] | undefined, +}; + +void mock.module("../../src/config", () => ({ + config: testConfig, +})); + +void mock.module("../../src/k8s/ephemeral-daemon-spawner", () => ({ + loadKubernetesClient: (): { core: typeof core } => ({ core }), +})); + +void mock.module("../../src/logger", () => ({ + logger: { + info: mock(() => {}), + warn: mock(() => {}), + error: mock(() => {}), + debug: mock(() => {}), + }, +})); + +const { + buildWorkflowRunnerPod, + deleteWorkflowRunnerResources, + ensureWorkflowRunnerResources, + WorkflowRunnerResourceError, +} = await import("../../src/k8s/workflow-runner-spawner"); + +const attempt: WorkflowRunnerAttempt = { + runId: "11111111-1111-4111-8111-111111111111", + attemptId: "22222222-2222-4222-8222-222222222222", + runnerId: "workflow-runner:22222222-2222-4222-8222-222222222222", + executionDeliveryId: "delivery-16", + workflowName: "implement", + attemptDeadlineAt: new Date("2026-08-23T04:10:00Z"), +}; + +const RUNNER_IMAGE = `registry.example/github-app@sha256:${"a".repeat(64)}`; +const input = { + attempt, + capability: "wfr1.test-capability", + image: RUNNER_IMAGE, + orchestratorUrl: "wss://controller.example/ws", +} as const; + +function apiError(code: number): Error & { code: number } { + return Object.assign(new Error(`Kubernetes API ${String(code)}`), { code }); +} + +function createdPod(): Record { + const call = createNamespacedPod.mock.calls[0] as [ResourceCall>]; + return call[0].body; +} + +interface MutableAdmissionPod { + metadata: Record; + spec: { + hostAliases?: unknown; + resourceClaims?: unknown; + volumes?: unknown; + containers: { + env: { + name: string; + valueFrom?: { secretKeyRef?: { optional?: boolean } }; + }[]; + lifecycle?: unknown; + resources?: Record; + securityContext: Record; + terminationMessagePolicy?: string; + }[]; + }; +} + +function runnerContainer( + pod: MutableAdmissionPod, +): MutableAdmissionPod["spec"]["containers"][number] { + const runner = pod.spec.containers[0]; + if (runner === undefined) throw new Error("Expected runner container"); + return runner; +} + +async function expectPermanentUrl(orchestratorUrl: string): Promise { + let caught: unknown; + try { + await ensureWorkflowRunnerResources({ ...input, orchestratorUrl }); + } catch (err) { + caught = err; + } + expect(caught).toBeInstanceOf(WorkflowRunnerResourceError); + expect((caught as InstanceType).kind).toBe("permanent"); +} + +function resourceMetadata( + name: string, + uid?: string, + resourceVersion?: string, +): Record { + return { + name, + namespace: "test-ns", + labels: { + "app.kubernetes.io/name": "github-app", + "app.kubernetes.io/component": "workflow-runner", + "github-app/workflow-run-id": attempt.runId, + "github-app/workflow-attempt-id": attempt.attemptId, + }, + ...(uid === undefined ? {} : { uid }), + ...(resourceVersion === undefined ? {} : { resourceVersion }), + }; +} + +function podOwnerReference(uid = "pod-uid"): Record { + return { + apiVersion: "v1", + kind: "Pod", + name: `workflow-runner-${attempt.attemptId}`, + uid, + controller: true, + blockOwnerDeletion: false, + }; +} + +function admittedPod(body: Record): Record { + const metadata = body["metadata"] as Record; + return { ...body, metadata: { ...metadata, uid: "pod-uid", resourceVersion: "1" } }; +} + +beforeEach(() => { + Object.assign(testConfig, { + provider: "anthropic", + model: "claude-test", + anthropicApiKey: "configured", + claudeCodeOauthToken: undefined, + awsRegion: undefined, + awsProfile: undefined, + awsAccessKeyId: undefined, + awsSecretAccessKey: undefined, + awsSessionToken: undefined, + awsBearerTokenBedrock: undefined, + anthropicBedrockBaseUrl: undefined, + allowedOwners: ["acme"], + }); + createNamespacedSecret.mockReset(); + readNamespacedSecret.mockReset(); + replaceNamespacedSecret.mockReset(); + createNamespacedPod.mockReset(); + readNamespacedPod.mockReset(); + deleteNamespacedPod.mockReset(); + deleteNamespacedSecret.mockReset(); + createNamespacedSecret.mockImplementation(({ body }) => Promise.resolve(body)); + readNamespacedSecret.mockResolvedValue({}); + replaceNamespacedSecret.mockImplementation(({ body }) => Promise.resolve(body)); + createNamespacedPod.mockImplementation(({ body }) => Promise.resolve(admittedPod(body))); + readNamespacedPod.mockResolvedValue({}); + deleteNamespacedPod.mockResolvedValue({}); + deleteNamespacedSecret.mockResolvedValue({}); +}); + +describe("workflow runner Pod boundary", () => { + it("rejects mutable image references before creating credentials", async () => { + for (const image of ["registry.example/github-app", "registry.example/github-app:latest"]) { + // eslint-disable-next-line no-await-in-loop -- each rejected reference is independent + await expectToReject( + ensureWorkflowRunnerResources({ ...input, image }), + "must end with an immutable sha256 digest", + ); + } + expect(createNamespacedSecret).not.toHaveBeenCalled(); + expect(createNamespacedPod).not.toHaveBeenCalled(); + }); + + it("creates one isolated runner with only attempt and provider Secret authority", async () => { + await ensureWorkflowRunnerResources(input); + + const pod = createdPod() as { + spec: { + activeDeadlineSeconds: number; + restartPolicy: string; + automountServiceAccountToken: boolean; + schedulerName: string; + shareProcessNamespace: boolean; + hostIPC: boolean; + hostNetwork: boolean; + hostPID: boolean; + nodeSelector: Record; + tolerations: Record[]; + securityContext: Record; + volumes: Record[]; + containers: { + name: string; + env: Record[]; + envFrom?: Record[]; + securityContext: Record; + terminationMessagePath: string; + terminationMessagePolicy: string; + volumeMounts: Record[]; + resources: { + requests: Record; + limits: Record; + }; + }[]; + }; + }; + expect(pod.spec.containers).toHaveLength(1); + expect(pod.spec.activeDeadlineSeconds).toBe(4_200); + expect(pod.spec.restartPolicy).toBe("Never"); + expect(pod.spec.automountServiceAccountToken).toBe(false); + expect(pod.spec.schedulerName).toBe("default-scheduler"); + expect(pod.spec.nodeSelector).toEqual({ "node.homelab/class": "worker" }); + expect(pod.spec.tolerations).toContainEqual({ + key: "node.homelab/class", + operator: "Equal", + value: "worker", + effect: "NoSchedule", + }); + expect(pod.spec.shareProcessNamespace).toBe(false); + expect([pod.spec.hostIPC, pod.spec.hostNetwork, pod.spec.hostPID]).toEqual([ + false, + false, + false, + ]); + expect(pod.spec.securityContext).toEqual({ + runAsNonRoot: true, + runAsUser: 1000, + runAsGroup: 1000, + seccompProfile: { type: "RuntimeDefault" }, + }); + expect(pod.spec.volumes).toEqual([{ name: "workspace", emptyDir: { sizeLimit: "10Gi" } }]); + const runner = pod.spec.containers[0]; + if (runner === undefined) throw new Error("Expected runner container"); + expect(runner.name).toBe("runner"); + expect(runner.terminationMessagePath).toBe("/dev/termination-log"); + expect(runner.terminationMessagePolicy).toBe("File"); + expect(runner.securityContext).toEqual({ + allowPrivilegeEscalation: false, + capabilities: { drop: ["ALL"] }, + }); + expect(runner.volumeMounts).toEqual([{ name: "workspace", mountPath: "/tmp/bot-workspaces" }]); + expect(runner.resources).toEqual({ + requests: { cpu: "500m", memory: "1Gi", "ephemeral-storage": "2Gi" }, + limits: { cpu: "2", memory: "4Gi", "ephemeral-storage": "10Gi" }, + }); + expect(runner.envFrom).toBeUndefined(); + const env = new Map(runner.env.map((entry) => [entry["name"], entry])); + expect([...env.keys()]).toEqual([ + "WORKFLOW_RUNNER", + "WORKFLOW_RUNNER_RUN_ID", + "WORKFLOW_RUNNER_ATTEMPT_ID", + "WORKFLOW_RUNNER_TOKEN", + "ORCHESTRATOR_URL", + "LD_PRELOAD", + "CLAUDE_PROVIDER", + "CLAUDE_MODEL", + "ANTHROPIC_API_KEY", + "ALLOWED_OWNERS", + ]); + expect(env.get("WORKFLOW_RUNNER_RUN_ID")).toEqual({ + name: "WORKFLOW_RUNNER_RUN_ID", + value: attempt.runId, + }); + expect(env.get("WORKFLOW_RUNNER_ATTEMPT_ID")).toEqual({ + name: "WORKFLOW_RUNNER_ATTEMPT_ID", + value: attempt.attemptId, + }); + expect(env.get("ORCHESTRATOR_URL")).toEqual({ + name: "ORCHESTRATOR_URL", + value: `wss://controller.example/ws/workflow-runner/${attempt.runId}/${attempt.attemptId}`, + }); + expect(env.get("WORKFLOW_RUNNER_TOKEN")).toEqual({ + name: "WORKFLOW_RUNNER_TOKEN", + valueFrom: { + secretKeyRef: { + name: `workflow-runner-${attempt.attemptId}`, + key: "capability", + }, + }, + }); + expect(env.get("ANTHROPIC_API_KEY")).toEqual({ + name: "ANTHROPIC_API_KEY", + valueFrom: { + secretKeyRef: { + name: "workflow-runner-secrets", + key: "ANTHROPIC_API_KEY", + optional: false, + }, + }, + }); + for (const forbidden of FORBIDDEN_RUNNER_ENV) { + expect(env.has(forbidden)).toBe(false); + } + }); + + it("renders only the selected Anthropic credential when both are configured", async () => { + testConfig.claudeCodeOauthToken = "configured-oauth"; + + await ensureWorkflowRunnerResources(input); + + const pod = createdPod() as MutableAdmissionPod; + const names = runnerContainer(pod).env.map((entry) => entry.name); + expect(names).toContain("ANTHROPIC_API_KEY"); + expect(names).not.toContain("CLAUDE_CODE_OAUTH_TOKEN"); + expect(names.some((name) => name.startsWith("AWS_"))).toBe(false); + }); + + it("renders one Bedrock bearer chain without Anthropic or static AWS credentials", async () => { + Object.assign(testConfig, { + provider: "bedrock", + anthropicApiKey: "unselected-controller-key", + awsRegion: "ap-southeast-2", + awsAccessKeyId: "unselected-static-key", + awsSecretAccessKey: "unselected-static-secret", + awsBearerTokenBedrock: "configured-bearer", + }); + + await ensureWorkflowRunnerResources(input); + + const pod = createdPod() as MutableAdmissionPod; + const names = runnerContainer(pod).env.map((entry) => entry.name); + expect(names).toContain("AWS_BEARER_TOKEN_BEDROCK"); + expect(names).toContain("AWS_REGION"); + expect(names).not.toContain("AWS_ACCESS_KEY_ID"); + expect(names).not.toContain("AWS_SECRET_ACCESS_KEY"); + expect(names).not.toContain("ANTHROPIC_API_KEY"); + }); + + it.each([ + "http://bedrock.example.test", + "https://user@bedrock.example.test", + "https://user:secret@bedrock.example.test", + "https://bedrock.example.test?token=secret", + "https://bedrock.example.test#fragment", + " bedrock.example.test ", + ])("rejects unsafe Bedrock base URL %s before creating credentials", async (baseUrl) => { + Object.assign(testConfig, { + provider: "bedrock", + anthropicApiKey: undefined, + awsRegion: "ap-southeast-2", + awsBearerTokenBedrock: "configured-bearer", + anthropicBedrockBaseUrl: baseUrl, + }); + + await expectToReject(ensureWorkflowRunnerResources(input), "ANTHROPIC_BEDROCK_BASE_URL"); + expect(createNamespacedSecret).not.toHaveBeenCalled(); + expect(createNamespacedPod).not.toHaveBeenCalled(); + }); + + it("accepts the exact Pod after create reports an existing name", async () => { + await ensureWorkflowRunnerResources(input); + const existing = admittedPod(structuredClone(createdPod())); + createNamespacedPod.mockRejectedValue(apiError(409)); + readNamespacedPod.mockResolvedValue(existing); + + await ensureWorkflowRunnerResources(input); + + expect(readNamespacedPod).toHaveBeenCalledTimes(1); + }); + + it("accepts Kubernetes omission of false host namespace fields", async () => { + createNamespacedPod.mockImplementation(({ body }) => { + const admitted = structuredClone(body) as { + spec: Record; + }; + Reflect.deleteProperty(admitted.spec, "hostIPC"); + Reflect.deleteProperty(admitted.spec, "hostNetwork"); + Reflect.deleteProperty(admitted.spec, "hostPID"); + return Promise.resolve(admittedPod(admitted as unknown as Record)); + }); + + await ensureWorkflowRunnerResources(input); + }); + + it("rejects an admission-mutated Pod returned by create", async () => { + createNamespacedPod.mockImplementation(({ body }) => { + const mutated = structuredClone(body) as { spec: { hostNetwork: boolean } }; + mutated.spec.hostNetwork = true; + return Promise.resolve(mutated); + }); + + await expectToReject( + ensureWorkflowRunnerResources(input), + `Existing Pod workflow-runner-${attempt.attemptId} does not match`, + ); + }); + + it("rejects insecure or credential-bearing controller URLs before creating resources", async () => { + await expectPermanentUrl("ws://controller.example/ws"); + await expectPermanentUrl("wss://user:secret@controller.example/ws"); + await expectPermanentUrl("not-a-url"); + // A ".svc" label anywhere but the third is a public name, not a cluster one. + await expectPermanentUrl("ws://evil.svc.attacker.example/ws"); + await expectPermanentUrl("ws://github-app.github-app.svc.cluster.local.attacker.example/ws"); + await expectPermanentUrl("ws://user:secret@github-app.github-app.svc.cluster.local:3002/ws"); + expect(createNamespacedSecret).not.toHaveBeenCalled(); + expect(createNamespacedPod).not.toHaveBeenCalled(); + }); + + it("accepts a plaintext controller URL on a cluster-local Service name", async () => { + for (const orchestratorUrl of [ + "ws://github-app.github-app.svc.cluster.local:3002/ws", + "ws://github-app.github-app.svc/ws", + ]) { + await ensureWorkflowRunnerResources({ ...input, orchestratorUrl }); + } + expect(createNamespacedPod).toHaveBeenCalledTimes(2); + }); + + it("rejects a 409-reconciled Pod with a changed isolation field", async () => { + await ensureWorkflowRunnerResources(input); + const existing = structuredClone(createdPod()) as { + spec: { hostPID: boolean }; + }; + existing.spec.hostPID = true; + createNamespacedPod.mockRejectedValue(apiError(409)); + readNamespacedPod.mockResolvedValue(existing); + + await expectToReject( + ensureWorkflowRunnerResources(input), + `Existing Pod workflow-runner-${attempt.attemptId} does not match`, + ); + }); + + it("treats a terminating Pod as transient", async () => { + const existing = admittedPod( + buildWorkflowRunnerPod(attempt, input.image, input.orchestratorUrl) as unknown as Record< + string, + unknown + >, + ); + (existing["metadata"] as Record)["deletionTimestamp"] = "2026-08-24T00:00:00Z"; + createNamespacedPod.mockRejectedValue(apiError(409)); + readNamespacedPod.mockResolvedValue(existing); + + let caught: unknown; + try { + await ensureWorkflowRunnerResources(input); + } catch (err) { + caught = err; + } + expect(caught).toBeInstanceOf(WorkflowRunnerResourceError); + expect((caught as InstanceType).kind).toBe("transient"); + expect((caught as Error).message).toContain("is terminating"); + }); + + for (const [name, mutate] of [ + [ + "container root identity", + (pod: MutableAdmissionPod): void => { + runnerContainer(pod).securityContext["runAsNonRoot"] = false; + runnerContainer(pod).securityContext["runAsUser"] = 0; + }, + ], + [ + "unconfined container seccomp", + (pod: MutableAdmissionPod): void => { + runnerContainer(pod).securityContext["seccompProfile"] = { type: "Unconfined" }; + }, + ], + [ + "added Linux capability", + (pod: MutableAdmissionPod): void => { + const security = runnerContainer(pod).securityContext; + const capabilities = security["capabilities"] as Record; + capabilities["add"] = ["SYS_ADMIN"]; + }, + ], + [ + "unmasked proc mount", + (pod: MutableAdmissionPod): void => { + runnerContainer(pod).securityContext["procMount"] = "Unmasked"; + }, + ], + [ + "credential-reading lifecycle hook", + (pod: MutableAdmissionPod): void => { + runnerContainer(pod).lifecycle = { postStart: { exec: { command: ["env"] } } }; + }, + ], + [ + "changed Secret optionality", + (pod: MutableAdmissionPod): void => { + const entry = runnerContainer(pod).env.find( + (candidate) => candidate.name === "ANTHROPIC_API_KEY", + ); + const secretKeyRef = entry?.valueFrom?.secretKeyRef; + if (secretKeyRef === undefined) throw new Error("Expected provider Secret reference"); + secretKeyRef.optional = true; + }, + ], + [ + "cleanup finalizer", + (pod: MutableAdmissionPod): void => { + pod.metadata["finalizers"] = ["example.invalid/hold"]; + }, + ], + [ + "host alias", + (pod: MutableAdmissionPod): void => { + pod.spec.hostAliases = [{ ip: "127.0.0.1", hostnames: ["controller.example"] }]; + }, + ], + [ + "workspace host path", + (pod: MutableAdmissionPod): void => { + pod.spec.volumes = [{ name: "workspace", hostPath: { path: "/" } }]; + }, + ], + [ + "dynamic resource claim", + (pod: MutableAdmissionPod): void => { + pod.spec.resourceClaims = [{ name: "host-device", resourceClaimName: "host-device" }]; + }, + ], + [ + "resource budget", + (pod: MutableAdmissionPod): void => { + const resources = runnerContainer(pod).resources; + if (resources === undefined) throw new Error("Expected runner resources"); + resources["limits"] = { cpu: "8", memory: "16Gi" }; + }, + ], + [ + "termination message policy", + (pod: MutableAdmissionPod): void => { + runnerContainer(pod).terminationMessagePolicy = "FallbackToLogsOnError"; + }, + ], + ] as const) { + it(`rejects admission-mutated ${name}`, async () => { + createNamespacedPod.mockImplementation(({ body }) => { + const mutated = structuredClone(body); + mutate(mutated); + return Promise.resolve(mutated); + }); + + await expectToReject( + ensureWorkflowRunnerResources(input), + `Existing Pod workflow-runner-${attempt.attemptId} does not match`, + ); + }); + } +}); + +describe("workflow runner Secret boundary", () => { + it("rejects a Secret mutated by admission", async () => { + createNamespacedSecret.mockImplementation(({ body }) => { + const mutated = structuredClone(body) as { data: Record }; + mutated.data["unexpected"] = "value"; + return Promise.resolve(mutated); + }); + + await expectToReject( + ensureWorkflowRunnerResources(input), + `Existing Secret workflow-runner-${attempt.attemptId} does not match`, + ); + expect(createNamespacedPod).toHaveBeenCalledTimes(1); + }); + + it("owns the capability Secret by the exact runner Pod UID", async () => { + await ensureWorkflowRunnerResources(input); + + const create = createNamespacedSecret.mock.calls[0]?.[0]; + expect(create?.body["metadata"]).toEqual( + expect.objectContaining({ ownerReferences: [podOwnerReference()] }), + ); + expect(createNamespacedPod.mock.invocationCallOrder[0]).toBeLessThan( + createNamespacedSecret.mock.invocationCallOrder[0] ?? Number.POSITIVE_INFINITY, + ); + }); + + it("preserves the existing Secret object and resourceVersion while rotating capability", async () => { + class SecretResponse { + readonly responsePrototype = true; + } + const secretName = `workflow-runner-${attempt.attemptId}`; + const existing = Object.assign(new SecretResponse(), { + apiVersion: "v1", + kind: "Secret", + metadata: resourceMetadata(secretName, "secret-uid", "41"), + type: "Opaque", + data: { capability: "stale" }, + }); + existing.metadata["ownerReferences"] = [podOwnerReference()]; + createNamespacedSecret.mockRejectedValue(apiError(409)); + readNamespacedSecret.mockResolvedValue(existing); + + await ensureWorkflowRunnerResources(input); + + const replace = replaceNamespacedSecret.mock.calls[0]?.[0]; + expect(replace?.body).toBe(existing); + expect(replace?.body).toBeInstanceOf(SecretResponse); + expect((replace?.body.metadata as { resourceVersion?: string }).resourceVersion).toBe("41"); + expect(replace?.body.data).toEqual({ + capability: Buffer.from(input.capability, "utf8").toString("base64"), + }); + }); + + it("rejects a Secret mutated by admission during replacement", async () => { + const secretName = `workflow-runner-${attempt.attemptId}`; + createNamespacedSecret.mockRejectedValue(apiError(409)); + readNamespacedSecret.mockResolvedValue({ + apiVersion: "v1", + kind: "Secret", + metadata: { + ...resourceMetadata(secretName, "secret-uid", "41"), + ownerReferences: [podOwnerReference()], + }, + type: "Opaque", + data: { capability: "stale" }, + }); + replaceNamespacedSecret.mockImplementation(({ body }) => { + const mutated = structuredClone(body); + mutated.data = { capability: "admission-mutated" }; + return Promise.resolve(mutated); + }); + + await expectToReject( + ensureWorkflowRunnerResources(input), + `Replaced Secret ${secretName} was mutated by admission`, + ); + expect(createNamespacedPod).toHaveBeenCalledTimes(1); + }); + + it("treats a terminating Secret as transient", async () => { + const secretName = `workflow-runner-${attempt.attemptId}`; + createNamespacedSecret.mockRejectedValue(apiError(409)); + readNamespacedSecret.mockResolvedValue({ + apiVersion: "v1", + kind: "Secret", + metadata: { + ...resourceMetadata(secretName, "secret-uid", "41"), + ownerReferences: [podOwnerReference()], + deletionTimestamp: "2026-08-24T00:00:00Z", + }, + type: "Opaque", + data: { capability: Buffer.from(input.capability, "utf8").toString("base64") }, + }); + + let caught: unknown; + try { + await ensureWorkflowRunnerResources(input); + } catch (err) { + caught = err; + } + expect(caught).toBeInstanceOf(WorkflowRunnerResourceError); + expect((caught as InstanceType).kind).toBe("transient"); + }); +}); + +describe("workflow runner owned cleanup", () => { + it("accepts deletion of exact owned resources with UID preconditions as complete", async () => { + const podName = `workflow-runner-${attempt.attemptId}`; + const secretName = podName; + readNamespacedPod.mockResolvedValue({ + metadata: resourceMetadata(podName, "pod-uid", "17"), + }); + readNamespacedSecret.mockResolvedValue({ + metadata: resourceMetadata(secretName, "secret-uid", "18"), + }); + + expect(await deleteWorkflowRunnerResources(attempt)).toBe(true); + + expect(deleteNamespacedPod).toHaveBeenCalledWith({ + name: podName, + namespace: "test-ns", + body: { + preconditions: { uid: "pod-uid" }, + }, + }); + expect(deleteNamespacedSecret).toHaveBeenCalledWith({ + name: secretName, + namespace: "test-ns", + body: { preconditions: { uid: "secret-uid" } }, + }); + }); + + it("treats missing resources as already cleaned", async () => { + readNamespacedPod.mockRejectedValue(apiError(404)); + readNamespacedSecret.mockRejectedValue(apiError(404)); + + expect(await deleteWorkflowRunnerResources(attempt)).toBe(true); + + expect(deleteNamespacedPod).not.toHaveBeenCalled(); + expect(deleteNamespacedSecret).not.toHaveBeenCalled(); + }); + + it("fails closed for ownership mismatch or missing deletion identity", async () => { + const name = `workflow-runner-${attempt.attemptId}`; + readNamespacedPod.mockResolvedValue({ + metadata: { ...resourceMetadata(name, "pod-uid", "17"), labels: {} }, + }); + await expectToReject(deleteWorkflowRunnerResources(attempt), "does not belong"); + expect(deleteNamespacedPod).not.toHaveBeenCalled(); + + readNamespacedPod.mockResolvedValue({ metadata: resourceMetadata(name) }); + await expectToReject(deleteWorkflowRunnerResources(attempt), "missing deletion preconditions"); + expect(deleteNamespacedPod).not.toHaveBeenCalled(); + }); + + it("classifies a deletion conflict as transient", async () => { + const name = `workflow-runner-${attempt.attemptId}`; + readNamespacedPod.mockResolvedValue({ + metadata: resourceMetadata(name, "pod-uid", "17"), + }); + deleteNamespacedPod.mockRejectedValue(apiError(409)); + + let caught: unknown; + try { + await deleteWorkflowRunnerResources(attempt); + } catch (err) { + caught = err; + } + expect(caught).toBeInstanceOf(WorkflowRunnerResourceError); + expect((caught as InstanceType).kind).toBe("transient"); + }); +}); + +describe("workflow runner Kubernetes error classification", () => { + for (const status of [408, 429, 500, 503]) { + it(`treats Kubernetes ${String(status)} as transient`, async () => { + createNamespacedPod.mockRejectedValue(apiError(status)); + let caught: unknown; + try { + await ensureWorkflowRunnerResources(input); + } catch (err) { + caught = err; + } + expect(caught).toBeInstanceOf(WorkflowRunnerResourceError); + expect((caught as InstanceType).kind).toBe("transient"); + }); + } + + it("treats a Secret resourceVersion update conflict as transient", async () => { + createNamespacedSecret.mockRejectedValue(apiError(409)); + readNamespacedSecret.mockResolvedValue({ + apiVersion: "v1", + kind: "Secret", + metadata: { + ...resourceMetadata(`workflow-runner-${attempt.attemptId}`, "secret-uid", "1"), + ownerReferences: [podOwnerReference()], + }, + type: "Opaque", + data: { capability: "stale" }, + }); + replaceNamespacedSecret.mockRejectedValue(apiError(409)); + + let caught: unknown; + try { + await ensureWorkflowRunnerResources(input); + } catch (err) { + caught = err; + } + expect(caught).toBeInstanceOf(WorkflowRunnerResourceError); + expect((caught as InstanceType).kind).toBe("transient"); + }); + + for (const status of [400, 401, 403, 422]) { + it(`treats Kubernetes ${String(status)} as permanent`, async () => { + createNamespacedPod.mockRejectedValue(apiError(status)); + let caught: unknown; + try { + await ensureWorkflowRunnerResources(input); + } catch (err) { + caught = err; + } + expect(caught).toBeInstanceOf(WorkflowRunnerResourceError); + expect((caught as InstanceType).kind).toBe("permanent"); + }); + } +}); + +// Stands in for V1Toleration and friends: any prototype that is not Object's. +class DeserializedModel { + // A method, not a field: it lives on the prototype, so it never becomes an own + // property and never reaches the value comparison. The prototype is the whole + // point. A field here would make the two sides differ by value instead. + modelName(): string { + return "V1Toleration"; + } +} + +describe("workflow runner Pod drift check against a real client response", () => { + it("accepts a response whose nested values are model class instances", async () => { + // The client deserializes every response through ObjectSerializer, which does + // `new typeMap[type]()`, so the created Pod never shares a prototype with the + // desired literal and isDeepStrictEqual compares prototypes. Modelled with a + // local class rather than importing V1Toleration: the package exposes the + // model classes only to CJS require, so a named ESM import resolves in a + // single-file run and throws once another test file shares the process. + // Guards a regression that failed every attempt permanently with a spurious + // drift error. + createNamespacedPod.mockImplementation(({ body }) => { + // Deep clone first: admittedPod shallow-copies, so mutating its spec in + // place would also convert the desired object and hide the mismatch. + const admitted = admittedPod(JSON.parse(JSON.stringify(body)) as Record); + const spec = admitted["spec"] as { tolerations: Record[] }; + const rehomed: Record[] = []; + for (const entry of spec.tolerations) { + rehomed.push( + Object.assign(new DeserializedModel(), entry) as unknown as Record, + ); + } + spec.tolerations = rehomed; + return Promise.resolve(admitted); + }); + + await ensureWorkflowRunnerResources(input); + + expect(createNamespacedPod).toHaveBeenCalledTimes(1); + expect(deleteNamespacedPod).not.toHaveBeenCalled(); + }); +}); diff --git a/test/mcp/servers/inline-comment-dedup.test.ts b/test/mcp/servers/inline-comment-dedup.test.ts index f34b6d9a..422833a4 100644 --- a/test/mcp/servers/inline-comment-dedup.test.ts +++ b/test/mcp/servers/inline-comment-dedup.test.ts @@ -55,15 +55,15 @@ describe("hasDuplicateAt", () => { }); it("does not match a different line", () => { - expect(hasDuplicateAt([comment({ line: 43 })], TARGET, SELF)).toBe(false); + expect(hasDuplicateAt([comment({ line: 43 })], TARGET, null)).toBe(false); }); it("does not match a different path", () => { - expect(hasDuplicateAt([comment({ path: "src/b.ts" })], TARGET, SELF)).toBe(false); + expect(hasDuplicateAt([comment({ path: "src/b.ts" })], TARGET, null)).toBe(false); }); it("does not match the other side of the diff", () => { - expect(hasDuplicateAt([comment({ side: "LEFT" })], TARGET, SELF)).toBe(false); + expect(hasDuplicateAt([comment({ side: "LEFT" })], TARGET, null)).toBe(false); }); it("finds a match anywhere in the list, not just first", () => { diff --git a/test/orchestrator/connection-handler.test.ts b/test/orchestrator/connection-handler.test.ts index 9c0e835b..0dcaa8ee 100644 --- a/test/orchestrator/connection-handler.test.ts +++ b/test/orchestrator/connection-handler.test.ts @@ -12,10 +12,15 @@ * to avoid mock.module() conflicts when all tests run in the same process. */ -import { beforeEach, describe, expect, it, mock } from "bun:test"; +import { afterEach, beforeEach, describe, expect, it, jest, mock, spyOn } from "bun:test"; import type { DaemonCapabilities, DaemonInfo } from "../../src/shared/daemon-types"; -import type { DaemonMessage } from "../../src/shared/ws-messages"; +import { + type DaemonMessage, + daemonMessageSchema, + PROTOCOL_VERSION, + serverMessageSchema, +} from "../../src/shared/ws-messages"; // ─── Leaf dependency mocks (shared across all orchestrator tests) ───────────── // These MUST be declared and registered before any SUT import. @@ -32,7 +37,7 @@ const mockRegisterDaemon = mock( osVersion: "6.1", capabilities: makeFakeCapabilities(), status: "active", - protocolVersion: "1.0.0", + protocolVersion: PROTOCOL_VERSION, appVersion: "0.1.0", activeJobs: 0, lastSeenAt: Date.now(), @@ -63,14 +68,18 @@ const mockGetOrphanedExecutions = mock( const mockMarkExecutionFailed = mock(() => Promise.resolve()); const mockMarkExecutionRunning = mock(() => Promise.resolve()); const mockMarkExecutionCompleted = mock(() => Promise.resolve()); -const mockMarkExecutionOffered = mock(() => Promise.resolve()); -const mockRequeueExecution = mock(() => Promise.resolve()); +const mockMarkExecutionOffered = mock(() => Promise.resolve("offered" as const)); +const mockRequeueExecution = mock(() => Promise.resolve(true)); +const mockFailDisconnectedDaemon = mock(() => + Promise.resolve({ executionDeliveryIds: [] as string[], workflowRunIds: [] as string[] }), +); const mockGetExecutionState = mock( (): Promise<{ status: string; daemonId: string | null } | null> => Promise.resolve(null), ); void mock.module("../../src/orchestrator/history", () => ({ getOrphanedExecutions: mockGetOrphanedExecutions, + failDisconnectedDaemon: mockFailDisconnectedDaemon, markExecutionFailed: mockMarkExecutionFailed, markExecutionRunning: mockMarkExecutionRunning, markExecutionCompleted: mockMarkExecutionCompleted, @@ -79,6 +88,11 @@ void mock.module("../../src/orchestrator/history", () => ({ requeueExecution: mockRequeueExecution, })); +const mockNotifyDisconnectedDaemonWorkflows = mock(() => Promise.resolve()); +void mock.module("../../src/orchestrator/workflow-expiry-notifier", () => ({ + notifyDisconnectedDaemonWorkflows: mockNotifyDisconnectedDaemonWorkflows, +})); + // job-queue const mockRequeueJob = mock((): Promise => Promise.resolve(true)); const mockEnqueueJob = mock(() => Promise.resolve()); @@ -110,12 +124,48 @@ void mock.module("../../src/orchestrator/concurrency", () => ({ // Mock octokit App const mockGetRepoInstallation = mock(() => Promise.resolve({ data: { id: 123 } })); const mockAuth = mock(() => Promise.resolve({ token: "ghs_fake_token" })); -const mockGetInstallationOctokit = mock(() => Promise.resolve({ auth: mockAuth })); + +/** + * Body served for `.github-app.yaml` by the accept-path octokit. `null` (the + * default) means "no file", so the real `fetchRepoConfig` returns `absent` and + * every pre-existing test keeps its default-policy behaviour. + */ +let repoConfigYaml: string | null = null; + +function makeAcceptOctokit(): unknown { + return { + auth: mockAuth, + rest: { + repos: { + getContent: mock(() => { + if (repoConfigYaml === null) { + const err = new Error("Not Found") as Error & { status: number }; + err.status = 404; + return Promise.reject(err); + } + return Promise.resolve({ + data: { + type: "file", + content: Buffer.from(repoConfigYaml, "utf-8").toString("base64"), + sha: "cfg-sha", + }, + headers: {}, + }); + }), + }, + }, + }; +} + +const mockGetInstallationOctokit = mock(() => Promise.resolve(makeAcceptOctokit())); void mock.module("octokit", () => ({ App: class MockApp { octokit = { rest: { apps: { getRepoInstallation: mockGetRepoInstallation } }, + // `mintInstallationToken` installs a before-hook to detect a cache-miss + // network mint, then removes it in a finally. + hook: { before: mock(() => {}), remove: mock(() => {}) }, }; getInstallationOctokit = mockGetInstallationOctokit; }, @@ -189,10 +239,12 @@ void mock.module("../../src/core/prompt-builder", () => ({ const { handleWsOpen, handleWsClose, + drainDisconnectCleanups, handleDaemonMessage, getConnections, getDaemonInfo, isDaemonDraining, + stripInstructionsUnlessReview, } = await import("../../src/orchestrator/connection-handler"); // ─── Helpers ────────────────────────────────────────────────────────────────── @@ -218,6 +270,7 @@ function makeFakeCapabilities(): DaemonCapabilities { function makeFakeWs(daemonId?: string): { data: { authenticated: boolean; remoteAddr: string; daemonId: string | undefined }; + readyState: number; sendText: ReturnType; close: ReturnType; } { @@ -227,6 +280,7 @@ function makeFakeWs(daemonId?: string): { remoteAddr: "127.0.0.1", daemonId, }, + readyState: 1, sendText: mock(() => {}), close: mock(() => {}), }; @@ -234,7 +288,7 @@ function makeFakeWs(daemonId?: string): { function makeRegisterMsg( daemonId = "daemon-1", - protocolVersion = "1.0.0", + protocolVersion = PROTOCOL_VERSION, ): Extract { return { type: "daemon:register", @@ -336,6 +390,8 @@ function resetAllMocks(): void { mockMarkExecutionOffered, mockGetExecutionState, mockRequeueExecution, + mockFailDisconnectedDaemon, + mockNotifyDisconnectedDaemonWorkflows, mockDecrementActiveCount, mockIncrementDaemonActiveJobs, mockDecrementDaemonActiveJobs, @@ -366,7 +422,7 @@ function resetAllMocks(): void { osVersion: "6.1", capabilities: makeFakeCapabilities(), status: "active", - protocolVersion: "1.0.0", + protocolVersion: PROTOCOL_VERSION, appVersion: "0.1.0", activeJobs: 0, lastSeenAt: Date.now(), @@ -378,9 +434,12 @@ function resetAllMocks(): void { mockMarkExecutionFailed.mockImplementation(() => Promise.resolve()); mockMarkExecutionRunning.mockImplementation(() => Promise.resolve()); mockMarkExecutionCompleted.mockImplementation(() => Promise.resolve()); - mockMarkExecutionOffered.mockImplementation(() => Promise.resolve()); + mockMarkExecutionOffered.mockImplementation(() => Promise.resolve("offered")); mockGetExecutionState.mockImplementation(() => Promise.resolve(null)); - mockRequeueExecution.mockImplementation(() => Promise.resolve()); + mockRequeueExecution.mockImplementation(() => Promise.resolve(true)); + mockFailDisconnectedDaemon.mockImplementation(() => + Promise.resolve({ executionDeliveryIds: [], workflowRunIds: [] }), + ); mockGetActiveDaemons.mockImplementation(() => Promise.resolve([])); mockGetDaemonActiveJobs.mockImplementation(() => Promise.resolve(0)); mockRequeueJob.mockImplementation(() => Promise.resolve(true)); @@ -391,7 +450,8 @@ function resetAllMocks(): void { mockRequireDb.mockImplementation(() => fakeDb); mockGetRepoInstallation.mockImplementation(() => Promise.resolve({ data: { id: 123 } })); mockAuth.mockImplementation(() => Promise.resolve({ token: "ghs_fake_token" })); - mockGetInstallationOctokit.mockImplementation(() => Promise.resolve({ auth: mockAuth })); + repoConfigYaml = null; + mockGetInstallationOctokit.mockImplementation(() => Promise.resolve(makeAcceptOctokit())); mockSaveRepoLearnings.mockImplementation(() => Promise.resolve(0)); mockDeleteRepoMemories.mockImplementation(() => Promise.resolve(0)); } @@ -403,6 +463,10 @@ beforeEach(() => { resetAllMocks(); }); +afterEach(() => { + jest.useRealTimers(); +}); + describe("handleWsOpen", () => { it("is a no-op (does not throw)", () => { const ws = makeFakeWs(); @@ -431,65 +495,57 @@ describe("handleWsClose", () => { expect(getConnections().has("daemon-1")).toBe(false); - await new Promise((r) => setTimeout(r, 30)); + await drainDisconnectCleanups(); expect(mockDeregisterDaemon).toHaveBeenCalledWith("daemon-1"); }); - it("marks orphaned executions as failed during cleanup", async () => { + it("runs the exact durable fencing transaction during cleanup", async () => { const ws = makeFakeWs("daemon-2"); // eslint-disable-next-line @typescript-eslint/no-explicit-any getConnections().set("daemon-2", ws as any); - mockGetOrphanedExecutions.mockImplementation(() => - Promise.resolve([{ deliveryId: "orphan-1", status: "running" }]), - ); + mockFailDisconnectedDaemon.mockResolvedValueOnce({ + executionDeliveryIds: ["orphan-1"], + workflowRunIds: ["workflow-orphan-1"], + }); // eslint-disable-next-line @typescript-eslint/no-explicit-any handleWsClose(ws as any, 1000, "normal"); - await new Promise((r) => setTimeout(r, 30)); - expect(mockMarkExecutionFailed).toHaveBeenCalledWith( - "orphan-1", - "daemon disconnected during execution", - ); + await drainDisconnectCleanups(); + expect(mockFailDisconnectedDaemon).toHaveBeenCalledWith("daemon-2"); + expect(mockNotifyDisconnectedDaemonWorkflows).toHaveBeenCalledWith(["workflow-orphan-1"]); }); - it("handles cleanup errors gracefully", async () => { + it("still runs durable fencing when registry cleanup fails", async () => { const ws = makeFakeWs("daemon-err"); // eslint-disable-next-line @typescript-eslint/no-explicit-any getConnections().set("daemon-err", ws as any); mockDeregisterDaemon.mockImplementation(() => Promise.reject(new Error("Valkey down"))); + mockFailDisconnectedDaemon.mockResolvedValueOnce({ + executionDeliveryIds: ["orphan-valkey"], + workflowRunIds: [], + }); // eslint-disable-next-line @typescript-eslint/no-explicit-any handleWsClose(ws as any, 1000, "normal"); - await new Promise((r) => setTimeout(r, 30)); - // Should not throw + await drainDisconnectCleanups(); + expect(mockFailDisconnectedDaemon).toHaveBeenCalledWith("daemon-err"); }); - it("handles individual orphan failure gracefully", async () => { + it("still attempts registry cleanup when durable fencing fails", async () => { const ws = makeFakeWs("daemon-orf"); // eslint-disable-next-line @typescript-eslint/no-explicit-any getConnections().set("daemon-orf", ws as any); - mockGetOrphanedExecutions.mockImplementation(() => - Promise.resolve([ - { deliveryId: "orf-1", status: "running" }, - { deliveryId: "orf-2", status: "running" }, - ]), - ); - let calls = 0; - mockMarkExecutionFailed.mockImplementation(() => { - calls++; - if (calls === 1) return Promise.reject(new Error("DB error")); - return Promise.resolve(); - }); + mockFailDisconnectedDaemon.mockRejectedValueOnce(new Error("DB error")); // eslint-disable-next-line @typescript-eslint/no-explicit-any handleWsClose(ws as any, 1000, "normal"); - await new Promise((r) => setTimeout(r, 50)); - expect(mockMarkExecutionFailed).toHaveBeenCalledTimes(2); + await drainDisconnectCleanups(); + expect(mockDeregisterDaemon).toHaveBeenCalledWith("daemon-orf"); }); }); @@ -535,9 +591,9 @@ describe("handleDaemonMessage - daemon:register", () => { expect(newWs.data.daemonId).toBe("daemon-rc"); }); - it("rejects incompatible protocol version", async () => { + it("rejects a newer incompatible protocol version", async () => { const ws = makeFakeWs(); - const msg = makeRegisterMsg("daemon-v2", "2.0.0"); + const msg = makeRegisterMsg("daemon-v3", "3.0.0"); // eslint-disable-next-line @typescript-eslint/no-explicit-any handleDaemonMessage(ws as any, msg); @@ -548,6 +604,96 @@ describe("handleDaemonMessage - daemon:register", () => { expect(closeArgs?.[0]).toBe(4003); }); + it("requests an urgent update from an older protocol before closing", async () => { + const currentWs = makeFakeWs("daemon-v1"); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + getConnections().set("daemon-v1", currentWs as any); + const staleWs = makeFakeWs(); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + handleDaemonMessage(staleWs as any, makeRegisterMsg("daemon-v1", "1.0.0")); + await new Promise((resolve) => setTimeout(resolve, 30)); + + expect(mockRegisterDaemon).not.toHaveBeenCalled(); + expect(currentWs.close).not.toHaveBeenCalled(); + expect(staleWs.close).not.toHaveBeenCalled(); + const update = JSON.parse(staleWs.sendText.mock.calls[0]?.[0] as string) as { + id: string; + type: string; + payload: { urgent: boolean }; + }; + expect(update.type).toBe("daemon:update-required"); + expect(update.payload.urgent).toBe(true); + expect(serverMessageSchema.safeParse(update).success).toBe(true); + + const acknowledgement = { + type: "daemon:update-acknowledged", + id: update.id, + timestamp: Date.now(), + payload: { strategy: "exit", delayMs: 0 }, + } satisfies DaemonMessage; + expect(daemonMessageSchema.safeParse(acknowledgement).success).toBe(true); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + handleDaemonMessage(staleWs as any, acknowledgement); + + expect(staleWs.close).not.toHaveBeenCalled(); + expect(getConnections().get("daemon-v1")).toBe(currentWs); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + handleWsClose(staleWs as any, 1000, "graceful shutdown"); + }); + + it("keeps an acknowledged older daemon connected while it drains", async () => { + jest.useFakeTimers(); + const ws = makeFakeWs(); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + handleDaemonMessage(ws as any, makeRegisterMsg("daemon-draining", "1.0.0")); + await drainDisconnectCleanups(); + const update = JSON.parse(ws.sendText.mock.calls[0]?.[0] as string) as { id: string }; + const acknowledgement = { + type: "daemon:update-acknowledged", + id: update.id, + timestamp: Date.now(), + payload: { strategy: "exit", delayMs: 0 }, + } satisfies DaemonMessage; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + handleDaemonMessage(ws as any, acknowledgement); + const { config: currentConfig } = await import("../../src/config"); + + jest.advanceTimersByTime(currentConfig.daemonDrainTimeoutMs + 1_999); + expect(ws.close).not.toHaveBeenCalled(); + jest.advanceTimersByTime(1); + expect(ws.close).toHaveBeenCalledWith(4003, "incompatible protocol version"); + }); + + it("closes an older protocol when the update acknowledgement times out", async () => { + jest.useFakeTimers(); + const ws = makeFakeWs(); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + handleDaemonMessage(ws as any, makeRegisterMsg("daemon-timeout", "1.0.0")); + await drainDisconnectCleanups(); + expect(ws.close).not.toHaveBeenCalled(); + + jest.advanceTimersByTime(5_000); + + expect(ws.close).toHaveBeenCalledWith(4003, "incompatible protocol version"); + }); + + it("cancels the protocol-update timeout when the socket closes", async () => { + jest.useFakeTimers(); + const ws = makeFakeWs(); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + handleDaemonMessage(ws as any, makeRegisterMsg("daemon-closed", "1.0.0")); + await drainDisconnectCleanups(); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + handleWsClose(ws as any, 1006, "transport lost"); + jest.advanceTimersByTime(5_000); + + expect(ws.close).not.toHaveBeenCalled(); + }); + it("sends error on registration failure", async () => { mockRegisterDaemon.mockImplementation(() => Promise.reject(new Error("Valkey down"))); @@ -580,6 +726,80 @@ describe("handleDaemonMessage - daemon:register", () => { "daemon reconnected, previous session orphaned", ); }); + + it("waits for same-daemon disconnect cleanup before re-registering", async () => { + const oldWs = makeFakeWs("daemon-race"); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + getConnections().set("daemon-race", oldWs as any); + let releaseCleanup: + | ((value: { executionDeliveryIds: string[]; workflowRunIds: string[] }) => void) + | undefined; + mockFailDisconnectedDaemon.mockImplementationOnce( + () => + new Promise((resolve) => { + releaseCleanup = resolve; + }), + ); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + handleWsClose(oldWs as any, 1006, "transport lost"); + const newWs = makeFakeWs(); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + handleDaemonMessage(newWs as any, makeRegisterMsg("daemon-race")); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(mockRegisterDaemon).not.toHaveBeenCalled(); + + releaseCleanup?.({ executionDeliveryIds: [], workflowRunIds: [] }); + await drainDisconnectCleanups(); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(mockDeregisterDaemon).toHaveBeenCalledWith("daemon-race"); + expect(mockRegisterDaemon).toHaveBeenCalledTimes(1); + expect(newWs.data.daemonId).toBe("daemon-race"); + }); + + it("serializes simultaneous registrations for the same daemon ID", async () => { + const releases: (() => void)[] = []; + mockRegisterDaemon.mockImplementation( + () => + new Promise((resolve) => { + releases.push(() => { + resolve({ + id: "daemon-simultaneous", + hostname: "host-1", + platform: "linux", + osVersion: "6.1", + capabilities: makeFakeCapabilities(), + status: "active", + protocolVersion: PROTOCOL_VERSION, + appVersion: "0.1.0", + activeJobs: 0, + lastSeenAt: Date.now(), + firstSeenAt: Date.now(), + }); + }); + }), + ); + + const firstWs = makeFakeWs(); + const secondWs = makeFakeWs(); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + handleDaemonMessage(firstWs as any, makeRegisterMsg("daemon-simultaneous")); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + handleDaemonMessage(secondWs as any, makeRegisterMsg("daemon-simultaneous")); + await new Promise((resolve) => setTimeout(resolve, 10)); + + expect(mockRegisterDaemon).toHaveBeenCalledTimes(1); + releases[0]?.(); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(mockRegisterDaemon).toHaveBeenCalledTimes(2); + expect(firstWs.close).toHaveBeenCalled(); + + releases[1]?.(); + await drainDisconnectCleanups(); + expect(firstWs.data.daemonId).toBeUndefined(); + expect(secondWs.data.daemonId).toBe("daemon-simultaneous"); + expect(getConnections().get("daemon-simultaneous")).toBe(secondWs); + }); }); describe("handleDaemonMessage - heartbeat:pong", () => { @@ -1208,7 +1428,7 @@ describe("handleDaemonMessage - job:result", () => { await new Promise((r) => setTimeout(r, 80)); - expect(mockDeleteRepoMemories).toHaveBeenCalledWith(["mem-1", "mem-2"]); + expect(mockDeleteRepoMemories).toHaveBeenCalledWith("o", "r", ["mem-1", "mem-2"], fakeDb); }); it("handles failed result with default error message", async () => { @@ -1338,7 +1558,7 @@ describe("handleDaemonMessage - job:result", () => { // C4: handleScopedJobCompletion must decrement active-count and // daemon-active-jobs for both succeeded and halted/failed branches, // otherwise every scoped run leaks one capacity slot. -describe("handleDaemonMessage - scoped-job-completion (C4)", () => { +describe("handleDaemonMessage - scoped-job:completion (C4)", () => { beforeEach(() => { mockDecrementActiveCount.mockClear(); mockDecrementDaemonActiveJobs.mockClear(); @@ -1356,7 +1576,7 @@ describe("handleDaemonMessage - scoped-job-completion (C4)", () => { ); const completionMsg = { - type: "scoped-job-completion", + type: "scoped-job:completion", id: crypto.randomUUID(), timestamp: Date.now(), payload: { @@ -1389,7 +1609,7 @@ describe("handleDaemonMessage - scoped-job-completion (C4)", () => { ); const completionMsg = { - type: "scoped-job-completion", + type: "scoped-job:completion", id: crypto.randomUUID(), timestamp: Date.now(), payload: { @@ -1425,7 +1645,7 @@ describe("handleDaemonMessage - scoped-job-completion (C4)", () => { ); const completionMsg = { - type: "scoped-job-completion", + type: "scoped-job:completion", id: crypto.randomUUID(), timestamp: Date.now(), payload: { @@ -1463,7 +1683,7 @@ describe("handleDaemonMessage - scoped-job-completion (C4)", () => { ); const completionMsg = { - type: "scoped-job-completion", + type: "scoped-job:completion", id: crypto.randomUUID(), timestamp: Date.now(), payload: { @@ -1496,7 +1716,7 @@ describe("handleDaemonMessage - scoped-job-completion (C4)", () => { ); const completionMsg = { - type: "scoped-job-completion", + type: "scoped-job:completion", id: crypto.randomUUID(), timestamp: Date.now(), payload: { @@ -1519,3 +1739,208 @@ describe("handleDaemonMessage - scoped-job-completion (C4)", () => { expect(mockMarkExecutionCompleted).not.toHaveBeenCalled(); }); }); + +// ─── Per-repo agent policy resolution at accept time (Gate 2) ─────────────── + +const { config } = await import("../../src/config"); +const { logger: realLogger } = await import("../../src/logger"); +const { __resetRepoConfigCaches } = await import("../../src/repo-config/fetcher"); + +describe("handleDaemonMessage - job:accept resolves the per-repo policy", () => { + interface MutableConfig { + agentMaxTurns?: number | undefined; + defaultMaxTurns?: number | undefined; + agentTimeoutMs: number; + reviewLearningsEnabled: boolean; + } + const saved = { + agentMaxTurns: config.agentMaxTurns, + defaultMaxTurns: config.defaultMaxTurns, + agentTimeoutMs: config.agentTimeoutMs, + reviewLearningsEnabled: config.reviewLearningsEnabled, + }; + + beforeEach(() => { + __resetRepoConfigCaches(); + // Ceilings are pinned so the clamp is observable: both turn knobs are + // unset in the test env, which would make every clamp a no-op. + (config as MutableConfig).agentMaxTurns = 100; + (config as MutableConfig).defaultMaxTurns = undefined; + (config as MutableConfig).agentTimeoutMs = 600_000; + // The review-learnings loader hits the same accept-path octokit and the + // mocked DB; off here so the assertions are about policy only. + (config as MutableConfig).reviewLearningsEnabled = false; + }); + + afterEach(() => { + (config as MutableConfig).agentMaxTurns = saved.agentMaxTurns; + (config as MutableConfig).defaultMaxTurns = saved.defaultMaxTurns; + (config as MutableConfig).agentTimeoutMs = saved.agentTimeoutMs; + (config as MutableConfig).reviewLearningsEnabled = saved.reviewLearningsEnabled; + }); + + /** Dispatch and accept one shared-daemon job, then return its payload. */ + async function acceptAndReadPayload(daemonId: string): Promise> { + const ws = makeFakeWs(); + await registerDaemon(ws, daemonId); + + const { dispatchJob: realDispatch } = await import("../../src/orchestrator/job-dispatcher"); + mockGetActiveDaemons.mockImplementation(() => Promise.resolve([daemonId])); + + const base = { + deliveryId: `del-${daemonId}`, + repoOwner: "test-owner", + repoName: "test-repo", + entityNumber: 1, + isPR: true, + eventName: "issue_comment", + triggerUsername: "user1", + labels: [], + triggerBodyPreview: "test body", + enqueuedAt: Date.now(), + retryCount: 0, + }; + const job = { + kind: "legacy" as const, + ...base, + }; + await realDispatch(job as Parameters[0]); + + const acceptMsg: DaemonMessage = { + type: "job:accept", + id: extractOfferId(ws), + timestamp: Date.now(), + payload: {}, + }; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + handleDaemonMessage(ws as any, acceptMsg); + await new Promise((r) => setTimeout(r, 120)); + + const payloadCall = ws.sendText.mock.calls.find((c) => { + const p = JSON.parse(c[0] as string) as { type: string }; + return p.type === "job:payload"; + }); + if (payloadCall === undefined) throw new Error("Expected a job:payload message"); + return (JSON.parse(payloadCall[0] as string) as { payload: Record }).payload; + } + + it("caps maxTurns at the repo's configured value (C2)", async () => { + repoConfigYaml = "version: 1\ndefaults:\n max_turns: 20\n"; + + const payload = await acceptAndReadPayload("daemon-pol-turns"); + + expect(payload.maxTurns).toBe(20); + }); + + it("clamps repo max_turns and timeout down to the server ceilings (C9)", async () => { + repoConfigYaml = "version: 1\ndefaults:\n max_turns: 500\n timeout: 60m\n"; + + const payload = await acceptAndReadPayload("daemon-pol-clamp"); + + // AGENT_MAX_TURNS=100, AGENT_TIMEOUT_MS=600000 both win over the YAML. + expect(payload.maxTurns).toBe(100); + expect((payload.policy as Record | undefined)?.timeoutMs).toBe(600_000); + }); + + it("forwards the repo model and extra tools on the wire (C1/C4)", async () => { + repoConfigYaml = + "version: 1\ndefaults:\n model: claude-repo-pinned-model\n extra_allowed_tools:\n - WebFetch\n"; + + const payload = await acceptAndReadPayload("daemon-pol-model"); + const policy = payload.policy as Record | undefined; + + expect(policy?.model).toBe("claude-repo-pinned-model"); + expect(policy?.extraAllowedTools).toEqual(["WebFetch"]); + }); + + it("carries a fail-open warning when the config fails validation (C7)", async () => { + // `version: 2` is not the schema's `z.literal(1)`. + repoConfigYaml = "version: 2\ndefaults:\n max_turns: 20\n"; + + const payload = await acceptAndReadPayload("daemon-pol-invalid"); + const policy = payload.policy as Record | undefined; + + expect(typeof policy?.warning).toBe("string"); + expect(policy?.warning).toContain("failed validation"); + // Fail-open: the invalid file's max_turns must NOT be honoured. + expect(payload.maxTurns).toBe(100); + }); + + it("keeps the pre-Gate-2 payload when the repo has no config file (C8)", async () => { + repoConfigYaml = null; + + const payload = await acceptAndReadPayload("daemon-pol-absent"); + + // Unchanged fallback: AGENT_MAX_TURNS ?? DEFAULT_MAXTURNS. + expect(payload.maxTurns).toBe(100); + expect(payload.policy).toBeUndefined(); + }); + + /** + * Capture what the REAL emitter writes. Spied rather than module-mocked: + * `src/logger` is shared by the orchestrator modules this file deliberately + * imports unmocked, and `mock.module` is process-wide. + */ + function capturePolicyLogs(): { lines: Record[]; restore: () => void } { + const lines: Record[] = []; + const spy = spyOn(realLogger, "info").mockImplementation(((obj: unknown) => { + if (obj !== null && typeof obj === "object") lines.push(obj as Record); + }) as typeof realLogger.info); + return { + lines, + restore: () => { + spy.mockRestore(); + }, + }; + } + + it("logs repo_config.policy_applied when max_turns is the only resolved knob", async () => { + repoConfigYaml = "version: 1\ndefaults:\n max_turns: 20\n"; + const captured = capturePolicyLogs(); + + try { + const payload = await acceptAndReadPayload("daemon-pol-log-turns"); + + // `toAgentPolicy` never projects max_turns, so `policy` is absent here. + // The event fires anyway because the emitter checks the cap separately, + // otherwise this run would answer nothing about the repo's knobs. + expect(payload.policy).toBeUndefined(); + const line = captured.lines.find((l) => l["event"] === "repo_config.policy_applied"); + expect(line).toBeDefined(); + expect(line?.["maxTurns"]).toBe(20); + } finally { + captured.restore(); + } + }); +}); + +// ─── Gate 2: `instructions` is a review-only knob ─────────────────────────── + +describe("stripInstructionsUnlessReview", () => { + // The schema scopes `instructions` to `workflows.review`, so YAML cannot + // reach a non-review workflow with one today. The guard exists so a future + // hoist to `defaults` cannot silently hand `implement` (which writes code + // and pushes commits) a trusted "this overrides your defaults" block. + const policy = { model: "m", instructions: "reject migrations without a rollback" }; + + it("drops instructions for implement even when the resolved policy carries one", () => { + expect(stripInstructionsUnlessReview(policy, "implement")).toEqual({ model: "m" }); + }); + + it("drops instructions on the direct-pipeline rail, which has no workflow name", () => { + expect(stripInstructionsUnlessReview(policy, undefined)).toEqual({ model: "m" }); + }); + + it("keeps instructions for review", () => { + expect(stripInstructionsUnlessReview(policy, "review")).toEqual(policy); + }); + + it("collapses to undefined when instructions was the only knob", () => { + expect(stripInstructionsUnlessReview({ instructions: "x" }, "implement")).toBeUndefined(); + }); + + it("passes an instruction-free policy through untouched", () => { + const clean = { model: "m", timeoutMs: 1000 }; + expect(stripInstructionsUnlessReview(clean, "implement")).toBe(clean); + }); +}); diff --git a/test/orchestrator/daemon-disconnect-lifecycle.test.ts b/test/orchestrator/daemon-disconnect-lifecycle.test.ts new file mode 100644 index 00000000..bdf1c8e4 --- /dev/null +++ b/test/orchestrator/daemon-disconnect-lifecycle.test.ts @@ -0,0 +1,384 @@ +import { SQL } from "bun"; +import { afterAll, beforeAll, describe, expect, it, mock } from "bun:test"; + +import type { DaemonInfo } from "../../src/shared/daemon-types"; + +const TEST_DATABASE_URL = + process.env["TEST_DATABASE_URL"] ?? "postgres://bot:bot@localhost:55432/github_app_test"; +const originalDatabaseUrl = process.env["DATABASE_URL"]; +process.env["DATABASE_URL"] = TEST_DATABASE_URL; + +const connections = new Map }>(); +const daemonInfo = new Map(); +let activeDaemonIds: string[] = []; + +void mock.module("../../src/orchestrator/connection-handler", () => ({ + getConnections: () => connections, + getDaemonInfo: (id: string) => daemonInfo.get(id), + isDaemonDraining: () => false, +})); +void mock.module("../../src/orchestrator/daemon-registry", () => ({ + getActiveDaemons: () => Promise.resolve(activeDaemonIds), + getDaemonActiveJobs: () => Promise.resolve(0), + decrementDaemonActiveJobs: () => Promise.resolve(), +})); + +let sql: SQL | null = null; +try { + const connection = new SQL(TEST_DATABASE_URL); + await connection`SELECT 1 AS ok`; + sql = connection; +} catch { + sql = null; +} + +function requireSql(): SQL { + if (sql === null) throw new Error("Database not available, test should have been skipped"); + return sql; +} + +async function resetSchema(): Promise { + await requireSql().unsafe(` + DROP TABLE IF EXISTS _migrations CASCADE; + DROP TABLE IF EXISTS review_learnings CASCADE; + DROP TABLE IF EXISTS scheduled_action_state CASCADE; + DROP TABLE IF EXISTS comment_cache CASCADE; + DROP TABLE IF EXISTS target_cache CASCADE; + DROP TABLE IF EXISTS chat_proposals CASCADE; + DROP TABLE IF EXISTS ship_fix_attempts CASCADE; + DROP TABLE IF EXISTS ship_continuations CASCADE; + DROP TABLE IF EXISTS ship_iterations CASCADE; + DROP TABLE IF EXISTS ship_intents CASCADE; + DROP TABLE IF EXISTS workflow_attempt_commands CASCADE; + DROP TABLE IF EXISTS workflow_runs CASCADE; + DROP TABLE IF EXISTS repo_memory CASCADE; + DROP TABLE IF EXISTS triage_results CASCADE; + DROP TABLE IF EXISTS executions CASCADE; + DROP TABLE IF EXISTS daemons CASCADE; + `); +} + +describe.skipIf(sql === null)("daemon disconnect incarnation lifecycle", () => { + beforeAll(async () => { + await resetSchema(); + const { runMigrations } = await import("../../src/db/migrate"); + await runMigrations(requireSql()); + }); + + afterAll(async () => { + const { closeDb } = await import("../../src/db"); + await closeDb(); + await resetSchema(); + await requireSql().close(); + if (originalDatabaseUrl === undefined) Reflect.deleteProperty(process.env, "DATABASE_URL"); + else process.env["DATABASE_URL"] = originalDatabaseUrl; + }); + + it("fences the old Pod incarnation before a same-Pod replacement receives redispatch", async () => { + const oldDaemon = `daemon-github-app-daemon-pod-${crypto.randomUUID()}`; + const newDaemon = `daemon-github-app-daemon-pod-${crypto.randomUUID()}`; + const oldDelivery = crypto.randomUUID(); + const replacementDelivery = crypto.randomUUID(); + const { createExecution, failDisconnectedDaemon } = + await import("../../src/orchestrator/history"); + const { dispatchJob, getPendingOffer, removePendingOffer } = + await import("../../src/orchestrator/job-dispatcher"); + const { findPendingWorkflowFailureNotifications, insertQueued, markWorkflowFailureNotified } = + await import("../../src/workflows/runs-store"); + + await requireSql()` + INSERT INTO daemons ( + id, hostname, platform, os_version, capabilities, resources, status, + first_seen_at, last_seen_at + ) VALUES + (${oldDaemon}, 'github-app-daemon-pod', 'linux', '6', '{}'::jsonb, '{}'::jsonb, + 'active', now(), now()), + (${newDaemon}, 'github-app-daemon-pod', 'linux', '6', '{}'::jsonb, '{}'::jsonb, + 'active', now(), now()) + `; + await createExecution( + { + deliveryId: oldDelivery, + repoOwner: "acme", + repoName: "widgets", + entityNumber: 16, + entityType: "issue", + eventName: "issue_comment", + triggerUsername: "maintainer", + dispatchMode: "daemon", + dispatchTarget: "daemon", + dispatchReason: "persistent-daemon", + }, + requireSql(), + ); + const oldRun = await insertQueued( + { + workflowName: "triage", + target: { type: "issue", owner: "acme", repo: "widgets", number: 16 }, + executionDeliveryId: oldDelivery, + ownerKind: "daemon", + ownerId: oldDaemon, + }, + requireSql(), + ); + await requireSql()` + UPDATE workflow_runs + SET status = 'running' + WHERE id = ${oldRun.id} + `; + await requireSql()` + UPDATE executions + SET status = 'running', daemon_id = ${oldDaemon}, started_at = now() + WHERE delivery_id = ${oldDelivery} + `; + await requireSql()` + INSERT INTO scheduled_action_state ( + installation_id, owner, repo, action_name, in_flight_job_id, in_flight_started_at + ) VALUES (1, 'acme', 'widgets', 'disconnect-test', ${oldDelivery}, now()) + `; + + const failed = await failDisconnectedDaemon(oldDaemon, requireSql()); + expect(failed.executionDeliveryIds).toContain(oldDelivery); + expect(failed.workflowRunIds).toContain(oldRun.id); + expect(await findPendingWorkflowFailureNotifications(requireSql())).toEqual([ + expect.objectContaining({ + phase: "orphaned", + row: expect.objectContaining({ id: oldRun.id, attempt_id: null }), + }), + ]); + expect( + await markWorkflowFailureNotified({ runId: oldRun.id, attemptId: null }, requireSql()), + ).toBe(true); + expect(await findPendingWorkflowFailureNotifications(requireSql())).toEqual([]); + + const activeOld: { count: number }[] = await requireSql()` + SELECT count(*)::int AS count + FROM executions + WHERE daemon_id = ${oldDaemon} + AND status IN ('queued', 'offered', 'running') + `; + expect(activeOld[0]?.count).toBe(0); + const activeRuns: { count: number }[] = await requireSql()` + SELECT count(*)::int AS count + FROM workflow_runs + WHERE owner_id = ${oldDaemon} + AND status IN ('queued', 'running') + `; + expect(activeRuns[0]?.count).toBe(0); + const locks: { in_flight_job_id: string | null }[] = await requireSql()` + SELECT in_flight_job_id FROM scheduled_action_state WHERE action_name = 'disconnect-test' + `; + expect(locks[0]?.in_flight_job_id).toBeNull(); + + await createExecution( + { + deliveryId: replacementDelivery, + repoOwner: "acme", + repoName: "widgets", + entityNumber: 16, + entityType: "issue", + eventName: "issue_comment", + triggerUsername: "maintainer", + dispatchMode: "daemon", + dispatchTarget: "daemon", + dispatchReason: "persistent-daemon", + }, + requireSql(), + ); + const retryRun = await insertQueued( + { + workflowName: "triage", + target: { type: "issue", owner: "acme", repo: "widgets", number: 16 }, + executionDeliveryId: replacementDelivery, + ownerKind: "orchestrator", + ownerId: "replacement-controller", + }, + requireSql(), + ); + expect(retryRun.status).toBe("queued"); + + const sendText = mock(() => 1); + connections.set(newDaemon, { sendText }); + activeDaemonIds = [newDaemon]; + daemonInfo.set(newDaemon, { + id: newDaemon, + hostname: "github-app-daemon-pod", + platform: "linux", + osVersion: "6", + capabilities: { + platform: "linux", + shells: [{ name: "bash", path: "/bin/bash", version: "5", functional: true }], + packageManagers: [ + { name: "bun", path: "/usr/bin/bun", version: "1.3.14", functional: true }, + ], + cliTools: [ + { name: "git", path: "/usr/bin/git", version: "2", functional: true }, + { name: "node", path: "/usr/bin/node", version: "24", functional: true }, + ], + containerRuntime: null, + authContexts: [], + resources: { + cpuCount: 2, + memoryTotalMb: 4096, + memoryFreeMb: 2048, + diskFreeMb: 10_000, + }, + network: { hostname: "github-app-daemon-pod" }, + cachedRepos: [], + ephemeral: false, + maxUptimeMs: null, + }, + status: "active", + protocolVersion: "1.0.0", + appVersion: "test", + activeJobs: 0, + lastSeenAt: Date.now(), + firstSeenAt: Date.now(), + }); + const replacementJob = { + kind: "legacy" as const, + deliveryId: replacementDelivery, + repoOwner: "acme", + repoName: "widgets", + entityNumber: 16, + isPR: false, + eventName: "issue_comment", + triggerUsername: "maintainer", + labels: [], + triggerBodyPreview: "retry after daemon restart", + enqueuedAt: Date.now(), + retryCount: 0, + }; + + expect(await dispatchJob(replacementJob)).toBe(true); + const frame = JSON.parse(String(sendText.mock.calls[0]?.[0])) as { + id: string; + type: string; + payload: { deliveryId: string }; + }; + expect(frame).toMatchObject({ + type: "job:offer", + payload: { deliveryId: replacementDelivery }, + }); + const [replacementReceipt] = await requireSql()< + { status: string; daemon_id: string | null }[] + >`SELECT status, daemon_id FROM executions WHERE delivery_id = ${replacementDelivery}`; + expect(replacementReceipt).toEqual({ status: "offered", daemon_id: newDaemon }); + expect(getPendingOffer(frame.id)?.daemonId).toBe(newDaemon); + removePendingOffer(frame.id); + }); + + it("terminalizes a composite parent owned by another daemon", async () => { + const parentDaemon = `daemon-parent-${crypto.randomUUID()}`; + const childDaemon = `daemon-child-${crypto.randomUUID()}`; + const childDelivery = crypto.randomUUID(); + const { createExecution, failDisconnectedDaemon } = + await import("../../src/orchestrator/history"); + const { findPendingWorkflowFailureNotifications, insertQueued } = + await import("../../src/workflows/runs-store"); + + await requireSql()` + INSERT INTO daemons ( + id, hostname, platform, os_version, capabilities, resources, status, + first_seen_at, last_seen_at + ) VALUES + (${parentDaemon}, 'parent-host', 'linux', '6', '{}'::jsonb, '{}'::jsonb, + 'active', now(), now()), + (${childDaemon}, 'child-host', 'linux', '6', '{}'::jsonb, '{}'::jsonb, + 'active', now(), now()) + `; + const parent = await insertQueued( + { + workflowName: "ship", + target: { type: "issue", owner: "acme", repo: "widgets", number: 17 }, + ownerKind: "daemon", + ownerId: parentDaemon, + }, + requireSql(), + ); + await requireSql()`UPDATE workflow_runs SET status = 'running' WHERE id = ${parent.id}`; + + await createExecution( + { + deliveryId: childDelivery, + repoOwner: "acme", + repoName: "widgets", + entityNumber: 1701, + entityType: "pull_request", + eventName: "issue_comment", + triggerUsername: "maintainer", + dispatchMode: "daemon", + dispatchTarget: "daemon", + dispatchReason: "persistent-daemon", + }, + requireSql(), + ); + const child = await insertQueued( + { + workflowName: "review", + target: { type: "pr", owner: "acme", repo: "widgets", number: 1701 }, + parentRunId: parent.id, + parentStepIndex: 3, + executionDeliveryId: childDelivery, + ownerKind: "daemon", + ownerId: childDaemon, + }, + requireSql(), + ); + await requireSql()` + UPDATE workflow_runs SET status = 'running' WHERE id = ${child.id} + `; + await requireSql()` + UPDATE executions + SET status = 'running', daemon_id = ${childDaemon}, started_at = now() + WHERE delivery_id = ${childDelivery} + `; + + const failed = await failDisconnectedDaemon(childDaemon, requireSql()); + + expect(failed.workflowRunIds).toEqual(expect.arrayContaining([child.id, parent.id])); + const [parentAfter] = await requireSql()< + { status: string; state: Record; attempt_completed_at: Date | null }[] + >` + SELECT status, state, attempt_completed_at FROM workflow_runs WHERE id = ${parent.id} + `; + const [childAfter] = await requireSql()<{ status: string }[]>` + SELECT status FROM workflow_runs WHERE id = ${child.id} + `; + expect(parentAfter).toMatchObject({ + status: "failed", + state: { + phase: "orphaned", + failedAtStepIndex: 3, + failedReason: "daemon disconnected during execution", + }, + attempt_completed_at: expect.any(Date), + }); + expect(childAfter?.status).toBe("failed"); + const pending = await findPendingWorkflowFailureNotifications(requireSql()); + expect(pending).toHaveLength(2); + expect(pending).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + phase: "orphaned", + row: expect.objectContaining({ id: child.id }), + }), + expect.objectContaining({ + phase: "orphaned", + row: expect.objectContaining({ id: parent.id }), + }), + ]), + ); + + const retry = await insertQueued( + { + workflowName: "ship", + target: { type: "issue", owner: "acme", repo: "widgets", number: 17 }, + ownerKind: "orchestrator", + ownerId: "replacement-controller", + }, + requireSql(), + ); + expect(retry.status).toBe("queued"); + }); +}); diff --git a/test/orchestrator/history.test.ts b/test/orchestrator/history.test.ts index a13f2730..ab5b5e5e 100644 --- a/test/orchestrator/history.test.ts +++ b/test/orchestrator/history.test.ts @@ -50,6 +50,10 @@ let mockDbResult: unknown[] = []; const mockDbFn = mock((_strings: TemplateStringsArray, ..._values: unknown[]) => Promise.resolve(mockDbResult), ); +const mockDbBegin = mock( + (callback: (tx: typeof mockDbFn) => Promise): Promise => callback(mockDbFn), +); +Object.assign(mockDbFn, { begin: mockDbBegin }); let dbEnabled = true; void mock.module("../../src/db", () => ({ @@ -123,6 +127,7 @@ function makeCreateParams(overrides: Partial = {}): Creat beforeEach(() => { mockDbFn.mockClear(); + mockDbBegin.mockClear(); mockDbResult = []; dbEnabled = true; mockLoggerInfo.mockClear(); @@ -262,11 +267,11 @@ describe("createExecution", () => { describe("markExecutionOffered", () => { it("updates status to offered with daemon ID when db is available", async () => { - await markExecutionOffered("d-001", "daemon-5"); + mockDbResult = [{ id: "daemon-5", delivery_id: "d-001" }]; + expect(await markExecutionOffered("d-001", "daemon-5")).toBe("offered"); expect(mockDbFn).toHaveBeenCalled(); - const call = firstCall(mockDbFn); - const values = call.slice(1); + const values = mockDbFn.mock.calls.flatMap((call) => call.slice(1)); expect(values).toContain("daemon-5"); expect(values).toContain("d-001"); }); @@ -274,7 +279,7 @@ describe("markExecutionOffered", () => { it("does nothing when db is not available", async () => { dbEnabled = false; - await markExecutionOffered("d-001", "daemon-5"); + expect(await markExecutionOffered("d-001", "daemon-5")).toBe("offered"); expect(mockDbFn).not.toHaveBeenCalled(); }); @@ -356,7 +361,8 @@ describe("markExecutionFailed", () => { describe("requeueExecution", () => { it("updates status back to queued and clears daemon_id", async () => { - await requeueExecution("d-006"); + mockDbResult = [{ delivery_id: "d-006" }]; + expect(await requeueExecution("d-006")).toBe(true); expect(mockDbFn).toHaveBeenCalled(); const call = firstCall(mockDbFn); @@ -367,7 +373,7 @@ describe("requeueExecution", () => { it("does nothing when db is not available", async () => { dbEnabled = false; - await requeueExecution("d-006"); + expect(await requeueExecution("d-006")).toBe(true); expect(mockDbFn).not.toHaveBeenCalled(); }); @@ -596,4 +602,13 @@ describe("recoverStaleExecutions", () => { // thresholdMs / 1000 = 300 expect(interpolatedValues).toContain(300); }); + + it("excludes lease-fenced workflow runner receipts from startup recovery", async () => { + directDbResults = [[]]; + await recoverStaleExecutions(mockDirectDb as unknown as SQL); + + const call = firstCall(mockDirectDb); + const strings = call[0] as unknown as TemplateStringsArray; + expect(strings.join("?")).toContain("offer_id IS NULL"); + }); }); diff --git a/test/orchestrator/installation-token.test.ts b/test/orchestrator/installation-token.test.ts new file mode 100644 index 00000000..db353eb5 --- /dev/null +++ b/test/orchestrator/installation-token.test.ts @@ -0,0 +1,116 @@ +import { beforeEach, describe, expect, it, mock } from "bun:test"; + +import { + mintInstallationToken, + revokeInstallationToken, +} from "../../src/orchestrator/installation-token"; +import { expectToReject } from "../utils/assertions"; + +const createInstallationAccessToken = mock(() => + Promise.resolve({ + data: { + token: "ghs_scoped", + expires_at: "2026-08-23T04:00:00Z", + }, + }), +); +const before = mock(() => undefined); +const remove = mock(() => undefined); +const log = { + info: mock(() => undefined), + warn: mock(() => undefined), + error: mock(() => undefined), + debug: mock(() => undefined), + child: mock(function (this: unknown): unknown { + return this; + }), +}; +const app = { + octokit: { + hook: { before, remove }, + rest: { apps: { createInstallationAccessToken } }, + }, +}; + +describe("scoped installation token mint", () => { + beforeEach(() => { + createInstallationAccessToken.mockReset(); + createInstallationAccessToken.mockResolvedValue({ + data: { token: "ghs_scoped", expires_at: "2026-08-23T04:00:00Z" }, + }); + before.mockClear(); + remove.mockClear(); + log.info.mockClear(); + log.warn.mockClear(); + }); + + it("returns the authoritative expiry and restricts the mint to one repository", async () => { + const result = await mintInstallationToken({ + app: app as never, + installationId: 123, + repositoryName: "widgets", + via: "workflowRunnerPayload", + log: log as never, + }); + + expect(createInstallationAccessToken).toHaveBeenCalledWith({ + installation_id: 123, + repositories: ["widgets"], + }); + expect(result.token).toBe("ghs_scoped"); + expect(result.expiresAt).toBe("2026-08-23T04:00:00Z"); + expect(remove).toHaveBeenCalledTimes(1); + }); + + it("rejects a schema-invalid expiry", async () => { + createInstallationAccessToken.mockResolvedValueOnce({ + data: { token: "ghs_scoped", expires_at: "not-a-timestamp" }, + }); + + await expectToReject( + mintInstallationToken({ + app: app as never, + installationId: 123, + repositoryName: "widgets", + via: "workflowRunnerPayload", + log: log as never, + }), + "Invalid ISO datetime", + ); + expect(log.warn).toHaveBeenCalledTimes(1); + expect(remove).toHaveBeenCalledTimes(1); + }); +}); + +describe("installation token revocation", () => { + beforeEach(() => { + log.error.mockClear(); + }); + + it("uses a bounded request and reports success", async () => { + const request = mock(() => Promise.resolve()); + + expect(await revokeInstallationToken({ request } as never, log as never)).toBe(true); + + expect(request).toHaveBeenCalledWith("DELETE /installation/token", { + request: { signal: expect.any(AbortSignal) }, + }); + expect(log.error).not.toHaveBeenCalled(); + }); + + it("logs a revocation failure without throwing into the owner cleanup path", async () => { + const failure = new Error("GitHub unavailable"); + const request = mock(() => Promise.reject(failure)); + + expect( + await revokeInstallationToken({ request } as never, log as never, { + attemptId: "attempt-1", + }), + ).toBe(false); + + expect(log.error).toHaveBeenCalledWith( + { attemptId: "attempt-1", err: failure }, + "Installation token revocation failed", + ); + }); +}); diff --git a/test/orchestrator/job-dispatcher.test.ts b/test/orchestrator/job-dispatcher.test.ts index 70ac54a6..2d9e7144 100644 --- a/test/orchestrator/job-dispatcher.test.ts +++ b/test/orchestrator/job-dispatcher.test.ts @@ -16,6 +16,8 @@ import { DispatcherOfferLogSchema, } from "../../src/orchestrator/log-fields"; import type { DaemonCapabilities, DaemonInfo } from "../../src/shared/daemon-types"; +import { serverMessageSchema } from "../../src/shared/ws-messages"; +import { expectToReject } from "../utils/assertions"; // ─── Mocks ──────────────────────────────────────────────────────────────────── @@ -50,9 +52,9 @@ void mock.module("../../src/orchestrator/daemon-registry", () => ({ })); // history -const mockMarkExecutionOffered = mock(() => Promise.resolve()); +const mockMarkExecutionOffered = mock(() => Promise.resolve("offered" as const)); const mockMarkExecutionFailed = mock(() => Promise.resolve()); -const mockRequeueExecution = mock(() => Promise.resolve()); +const mockRequeueExecution = mock(() => Promise.resolve(true)); void mock.module("../../src/orchestrator/history", () => ({ markExecutionOffered: mockMarkExecutionOffered, @@ -73,8 +75,9 @@ void mock.module("../../src/orchestrator/job-queue", () => ({ enqueueJob: mock(() => Promise.resolve()), tryDequeueJob: mock(() => Promise.resolve(null)), dequeueJob: mock(() => Promise.resolve(null)), - isScopedJob: () => false, - SCOPED_JOB_KINDS: ["scoped-rebase", "scoped-fix-thread", "scoped-open-pr"], + isScopedJob: (job: QueuedJob) => + ["scoped-rebase", "scoped-fix-thread", "scoped-open-pr", "scheduled-action"].includes(job.kind), + SCOPED_JOB_KINDS: ["scoped-rebase", "scoped-fix-thread", "scoped-open-pr", "scheduled-action"], // C2: job-dispatcher's `reconstructJobFromOffer` re-validates // `offer.scoped` via the discriminated-union schema; the legacy-path // tests in this file never set `offer.scoped`, so a permanently-failing @@ -224,6 +227,8 @@ beforeEach(() => { mockIsDaemonDraining.mockImplementation(() => false); mockGetDaemonInfo.mockImplementation((id: string) => daemonInfoStore.get(id)); mockRequeueJob.mockImplementation(() => Promise.resolve(true)); + mockMarkExecutionOffered.mockImplementation(() => Promise.resolve("offered")); + mockRequeueExecution.mockImplementation(() => Promise.resolve(true)); }); describe("inferRequiredTools", () => { @@ -420,6 +425,21 @@ describe("selectDaemon", () => { }); describe("dispatchJob", () => { + it("rejects workflow jobs before shared-daemon selection", async () => { + const workflowJob: QueuedJob = { + ...makeQueuedJob(), + kind: "workflow-run", + workflowRun: { runId: crypto.randomUUID(), workflowName: "implement" }, + }; + + await expectToReject( + dispatchJob(workflowJob), + "workflow-run jobs require an isolated workflow runner", + ); + expect(mockGetActiveDaemons).not.toHaveBeenCalled(); + expect(mockMarkExecutionOffered).not.toHaveBeenCalled(); + }); + it("returns false when no daemon available", async () => { const result = await dispatchJob(makeQueuedJob()); expect(result).toBe(false); @@ -470,6 +490,67 @@ describe("dispatchJob", () => { expect(typeof offerLog?.["queue_wait_ms"]).toBe("number"); }); + it("emits a schema-valid scoped-job:offer from a scoped queue job", async () => { + const fakeWs = { sendText: mock(() => 1) }; + mockGetActiveDaemons.mockResolvedValue(["d1"]); + daemonInfoStore.set("d1", makeDaemonInfo("d1")); + mockConnections.set("d1", fakeWs); + const job = { + ...makeQueuedJob({ deliveryId: "scoped-dispatch-test" }), + kind: "scoped-rebase", + installationId: 123, + triggerCommentId: 456, + prNumber: 42, + } satisfies QueuedJob; + + expect(await dispatchJob(job)).toBe(true); + + const frame = JSON.parse(fakeWs.sendText.mock.calls[0]?.[0] as string) as { + id: string; + type: string; + payload: { deliveryId: string }; + }; + expect(frame.type).toBe("scoped-job:offer"); + expect(frame.payload.deliveryId).toBe("scoped-dispatch-test"); + expect(serverMessageSchema.safeParse(frame).success).toBe(true); + + removePendingOffer(frame.id); + }); + + it("requeues when the selected socket closes during durable offer assignment", async () => { + const fakeWs = { sendText: mock(() => 1) }; + mockGetActiveDaemons.mockResolvedValue(["d1"]); + daemonInfoStore.set("d1", makeDaemonInfo("d1")); + mockConnections.set("d1", fakeWs); + let resolveOffer: ((value: "offered") => void) | undefined; + mockMarkExecutionOffered.mockImplementationOnce( + () => + new Promise((resolve) => { + resolveOffer = resolve; + }), + ); + + const dispatch = dispatchJob(makeQueuedJob({ deliveryId: "close-race" })); + // eslint-disable-next-line no-await-in-loop -- wait for the controlled offer boundary + while (resolveOffer === undefined) await Promise.resolve(); + mockConnections.delete("d1"); + resolveOffer("offered"); + + expect(await dispatch).toBe(false); + expect(mockRequeueExecution).toHaveBeenCalledWith("close-race"); + expect(fakeWs.sendText).not.toHaveBeenCalled(); + }); + + it("requeues when Bun drops the offer frame", async () => { + const fakeWs = { sendText: mock(() => 0) }; + mockGetActiveDaemons.mockResolvedValue(["d1"]); + daemonInfoStore.set("d1", makeDaemonInfo("d1")); + mockConnections.set("d1", fakeWs); + + expect(await dispatchJob(makeQueuedJob({ deliveryId: "dropped-frame" }))).toBe(false); + expect(mockRequeueExecution).toHaveBeenCalledWith("dropped-frame"); + }); + it("creates pending offer with correct metadata", async () => { const fakeWs = { sendText: mock(() => {}) }; mockGetActiveDaemons.mockImplementation(() => Promise.resolve(["d1"])); @@ -604,6 +685,69 @@ describe("handleJobAccept", () => { expect(parsed.payload.memory).toBeUndefined(); }); + it("forwards the resolved per-repo policy onto the wire (Gate 2)", () => { + const fakeWs = { sendText: mock(() => {}) }; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + mockConnections.set("d-policy", fakeWs as any); + + handleJobAccept({ + offerId: "offer-policy", + daemonId: "d-policy", + deliveryId: "del-policy", + installationToken: "ghs_token", + contextJson: {}, + maxTurns: 42, + allowedTools: ["Read"], + envVars: {}, + memory: [], + policy: { + model: "claude-repo-pinned-model", + timeoutMs: 900_000, + extraAllowedTools: ["WebFetch"], + }, + }); + + const sentText = (fakeWs.sendText as ReturnType).mock.calls[0]?.[0] as string; + const parsed = JSON.parse(sentText) as { payload: Record }; + expect(parsed.payload.policy).toEqual({ + model: "claude-repo-pinned-model", + timeoutMs: 900_000, + extraAllowedTools: ["WebFetch"], + }); + // The per-repo turn cap rides the existing top-level field, not `policy`. + expect(parsed.payload.maxTurns).toBe(42); + }); + + it("emits the pre-Gate-2 key set verbatim when no policy is supplied (C8)", () => { + const fakeWs = { sendText: mock(() => {}) }; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + mockConnections.set("d-nopolicy", fakeWs as any); + + handleJobAccept({ + offerId: "offer-nopolicy", + daemonId: "d-nopolicy", + deliveryId: "del-nopolicy", + installationToken: "ghs_token", + contextJson: { owner: "o", repo: "r" }, + maxTurns: 10, + allowedTools: ["Read"], + envVars: { CUSTOM: "val" }, + memory: [], + }); + + const sentText = (fakeWs.sendText as ReturnType).mock.calls[0]?.[0] as string; + const parsed = JSON.parse(sentText) as { payload: Record }; + // Exact key set: a `policy: undefined` spread leaking an extra key would + // change the serialized envelope every daemon already parses. + expect(Object.keys(parsed.payload).sort()).toEqual([ + "allowedTools", + "context", + "envVars", + "installationToken", + "maxTurns", + ]); + }); + it("returns early when daemon disconnected", () => { // No connection in map expect(() => { diff --git a/test/orchestrator/job-queue.test.ts b/test/orchestrator/job-queue.test.ts index db8c2678..2adf676b 100644 --- a/test/orchestrator/job-queue.test.ts +++ b/test/orchestrator/job-queue.test.ts @@ -64,12 +64,14 @@ void mock.module("../../src/config", () => ({ // Import AFTER mocks const { enqueueJob, + ensureWorkflowJobQueued, tryDequeueJob, dequeueJob, requeueJob, leaseJob, releaseLeasedJob, requeueLeasedJob, + deferLeasedWorkflowJob, recoverProcessingList, processingListKey, } = await import("../../src/orchestrator/job-queue"); @@ -129,6 +131,29 @@ describe("job-queue", () => { }); }); + describe("ensureWorkflowJobQueued", () => { + it("atomically checks the shared and processing lists for one stable payload", async () => { + const job = makeQueuedJob({ + kind: "workflow-run", + workflowRun: { runId: crypto.randomUUID(), workflowName: "review" }, + }); + if (job.kind !== "workflow-run") throw new Error("Expected workflow fixture"); + mockSend.mockResolvedValueOnce(1).mockResolvedValueOnce(0); + + expect(await ensureWorkflowJobQueued(job, "orch-a")).toBe(true); + expect(await ensureWorkflowJobQueued(job, "orch-a")).toBe(false); + + for (const call of mockSend.mock.calls) { + expect(call[0]).toBe("EVAL"); + expect(call[1][0]).toContain("LPOS"); + expect(call[1][1]).toBe("2"); + expect(call[1][2]).toBe("queue:jobs"); + expect(call[1][3]).toBe("queue:processing:orch-a"); + expect(call[1][4]).toBe(JSON.stringify(job)); + } + }); + }); + describe("tryDequeueJob", () => { it("returns null when the queue is empty (RPOP returns null)", async () => { mockSend.mockResolvedValueOnce(null); @@ -362,6 +387,52 @@ describe("job-queue", () => { }); }); + describe("deferLeasedWorkflowJob", () => { + it("moves the exact workflow lease without consuming retryCount", async () => { + const oldEnqueuedAt = Date.now() - 60_000; + const job = makeQueuedJob({ + kind: "workflow-run", + retryCount: 2, + enqueuedAt: oldEnqueuedAt, + workflowRun: { runId: crypto.randomUUID(), workflowName: "implement" }, + }); + if (job.kind !== "workflow-run") throw new Error("Expected workflow fixture"); + const raw = JSON.stringify(job); + mockSend.mockResolvedValueOnce(1); + + expect(await deferLeasedWorkflowJob("orch-a", raw, job, "capacity-1")).toEqual({ + status: "moved", + }); + + const call = mockSend.mock.calls[0]; + expect(call?.[0]).toBe("EVAL"); + expect(call?.[1]?.[0]).toContain("EXISTS"); + expect(call?.[1]?.[1]).toBe("3"); + expect(call?.[1]?.[2]).toBe("queue:processing:orch-a"); + expect(call?.[1]?.[3]).toBe("queue:jobs"); + expect(call?.[1]?.[4]).toBe("queue:workflow-deferral-receipt:orch-a:capacity-1"); + expect(call?.[1]?.[5]).toBe(raw); + expect(call?.[1]?.[6]).toBe(raw); + }); + + it("distinguishes replayed and missing deferral receipts", async () => { + const job = makeQueuedJob({ + kind: "workflow-run", + workflowRun: { runId: crypto.randomUUID(), workflowName: "plan" }, + }); + if (job.kind !== "workflow-run") throw new Error("Expected workflow fixture"); + const raw = JSON.stringify(job); + mockSend.mockResolvedValueOnce(2).mockResolvedValueOnce(0); + + expect(await deferLeasedWorkflowJob("orch-a", raw, job, "capacity-2")).toEqual({ + status: "already-moved", + }); + expect(await deferLeasedWorkflowJob("orch-a", raw, job, "capacity-3")).toEqual({ + status: "missing", + }); + }); + }); + describe("recoverProcessingList", () => { it("drains the processing list back to queue:jobs and returns count", async () => { mockSend.mockResolvedValueOnce("job-a"); diff --git a/test/orchestrator/liveness-reaper-resilience.test.ts b/test/orchestrator/liveness-reaper-resilience.test.ts new file mode 100644 index 00000000..5d6dc61c --- /dev/null +++ b/test/orchestrator/liveness-reaper-resilience.test.ts @@ -0,0 +1,104 @@ +import type { SQL } from "bun"; +import { beforeEach, describe, expect, it, mock } from "bun:test"; + +import { waitFor } from "../utils/assertions"; + +const expiredRun = { + id: crypto.randomUUID(), + workflow_name: "triage", + owner_kind: "daemon" as const, + owner_id: "daemon-expired", +}; +const expireWorkflowAttempts = mock(() => Promise.resolve([expiredRun])); +const expireQueuedWorkflowDispatches = mock(() => Promise.resolve([])); +const fakeSql = (() => Promise.resolve([])) as unknown as SQL; + +void mock.module("../../src/workflows/runs-store", () => ({ + expireQueuedWorkflowDispatches, + expireWorkflowAttempts, +})); +void mock.module("../../src/orchestrator/valkey-cleanup", () => ({ + reapOrphanProcessingLists: mock(() => Promise.resolve(0)), +})); +void mock.module("../../src/db", () => ({ + getDb: () => null, + requireDb: () => fakeSql, +})); +void mock.module("../../src/workflows/completion-reconciler", () => ({ + reconcilePendingWorkflowCascades: mock(() => Promise.resolve(0)), +})); +void mock.module("../../src/workflows/dispatch-outbox", () => ({ + publishPendingWorkflowRuns: mock(() => Promise.resolve(0)), + publishWorkflowRunById: mock(() => Promise.resolve(true)), +})); +void mock.module("../../src/orchestrator/workflow-expiry-notifier", () => ({ + notifyExpiredWorkflowDispatches: mock(() => Promise.resolve()), + notifyExpiredWorkflowAttempts: mock(() => Promise.resolve()), + notifyDisconnectedDaemonWorkflows: mock(() => Promise.resolve()), +})); +void mock.module("../../src/orchestrator/workflow-runner-reconciler", () => ({ + reconcileWorkflowRunners: mock(() => Promise.resolve()), +})); +void mock.module("../../src/db/queries/scheduled-actions-store", () => ({ + clearInFlightByJobId: mock(() => Promise.resolve()), +})); +void mock.module("../../src/orchestrator/valkey", () => ({ + requireValkeyClient: () => { + throw new Error("Valkey unavailable"); + }, +})); + +describe("liveness reaper failure isolation", () => { + beforeEach(() => { + expireQueuedWorkflowDispatches.mockReset(); + expireQueuedWorkflowDispatches.mockResolvedValue([]); + expireWorkflowAttempts.mockReset(); + expireWorkflowAttempts.mockResolvedValue([expiredRun]); + }); + + it("expires database leases when Valkey liveness reads fail", async () => { + const { reapOnce } = await import("../../src/orchestrator/liveness-reaper"); + + const result = await reapOnce(fakeSql); + + expect(expireWorkflowAttempts).toHaveBeenCalledWith(fakeSql); + expect(result.workflowRunsReaped).toEqual([expiredRun]); + expect(result.daemonsMarkedInactive).toBe(0); + }); + + it("coalesces overlapping ticks and waits for the active pass during stop", async () => { + let release!: (rows: never[]) => void; + expireWorkflowAttempts.mockImplementationOnce( + () => + new Promise((resolve) => { + release = resolve; + }), + ); + const { config } = await import("../../src/config"); + const { startLivenessReaper, stopLivenessReaper } = + await import("../../src/orchestrator/liveness-reaper"); + const originalInterval = config.livenessReaperIntervalMs; + (config as { livenessReaperIntervalMs: number }).livenessReaperIntervalMs = 5; + try { + startLivenessReaper(); + await waitFor(() => expireWorkflowAttempts.mock.calls.length === 1); + await Bun.sleep(25); + expect(expireWorkflowAttempts).toHaveBeenCalledTimes(1); + + let stopped = false; + const stopping = stopLivenessReaper().then(() => { + stopped = true; + }); + await Bun.sleep(5); + expect(stopped).toBe(false); + + release([]); + await stopping; + expect(stopped).toBe(true); + } finally { + release?.([]); + await stopLivenessReaper(); + (config as { livenessReaperIntervalMs: number }).livenessReaperIntervalMs = originalInterval; + } + }); +}); diff --git a/test/orchestrator/liveness-reaper.test.ts b/test/orchestrator/liveness-reaper.test.ts index 7d0caa89..05d881a2 100644 --- a/test/orchestrator/liveness-reaper.test.ts +++ b/test/orchestrator/liveness-reaper.test.ts @@ -14,7 +14,15 @@ */ import { SQL } from "bun"; -import { afterAll, beforeAll, beforeEach, describe, expect, it } from "bun:test"; +import { afterAll, beforeAll, beforeEach, describe, expect, it, mock } from "bun:test"; + +void mock.module("../../src/orchestrator/workflow-runner-reconciler", () => ({ + reconcileWorkflowRunners: mock(() => Promise.resolve()), +})); +void mock.module("../../src/workflows/dispatch-outbox", () => ({ + publishPendingWorkflowRuns: mock(() => Promise.resolve(0)), + publishWorkflowRunById: mock(() => Promise.resolve(true)), +})); const TEST_DATABASE_URL = process.env["TEST_DATABASE_URL"] ?? "postgres://bot:bot@localhost:55432/github_app_test"; @@ -42,6 +50,7 @@ describe.skipIf(sql === null)("liveness-reaper", () => { await requireSql().unsafe(` DROP TABLE IF EXISTS _migrations CASCADE; + DROP TABLE IF EXISTS workflow_attempt_commands CASCADE; DROP TABLE IF EXISTS review_learnings CASCADE; DROP TABLE IF EXISTS scheduled_action_state CASCADE; DROP TABLE IF EXISTS comment_cache CASCADE; @@ -66,6 +75,7 @@ describe.skipIf(sql === null)("liveness-reaper", () => { closeValkey(); await requireSql().unsafe(` DROP TABLE IF EXISTS _migrations CASCADE; + DROP TABLE IF EXISTS workflow_attempt_commands CASCADE; DROP TABLE IF EXISTS review_learnings CASCADE; DROP TABLE IF EXISTS scheduled_action_state CASCADE; DROP TABLE IF EXISTS comment_cache CASCADE; @@ -86,6 +96,8 @@ describe.skipIf(sql === null)("liveness-reaper", () => { beforeEach(async () => { await requireSql()`DELETE FROM workflow_runs`; + await requireSql()`DELETE FROM executions`; + await requireSql()`DELETE FROM scheduled_action_state`; await requireSql()`DELETE FROM daemons`; const { requireValkeyClient } = await import("../../src/orchestrator/valkey"); const valkey = requireValkeyClient(); @@ -105,6 +117,22 @@ describe.skipIf(sql === null)("liveness-reaper", () => { orchKeys.push(...result[1]); } while (cursor !== "0"); if (orchKeys.length > 0) await valkey.send("DEL", orchKeys); + const processingKeys: string[] = []; + cursor = "0"; + do { + // eslint-disable-next-line no-await-in-loop -- Valkey SCAN + const result: [string, string[]] = await valkey.send("SCAN", [ + cursor, + "MATCH", + "queue:processing:*", + "COUNT", + "100", + ]); + cursor = result[0]; + processingKeys.push(...result[1]); + } while (cursor !== "0"); + if (processingKeys.length > 0) await valkey.send("DEL", processingKeys); + await valkey.send("DEL", ["queue:jobs"]); const daemonMembers: string[] = await valkey.send("SMEMBERS", ["active_daemons"]); for (const id of daemonMembers) { await valkey.send("DEL", [`daemon:${id}`]); @@ -124,6 +152,11 @@ describe.skipIf(sql === null)("liveness-reaper", () => { await valkey.send("SADD", ["active_daemons", id]); } + async function setDaemonHeartbeatOnly(id: string): Promise { + const { requireValkeyClient } = await import("../../src/orchestrator/valkey"); + await requireValkeyClient().send("SET", [`daemon:${id}`, "{}", "EX", "60"]); + } + async function insertWorkflowRow( target: { number: number }, ownerKind: "orchestrator" | "daemon", @@ -133,10 +166,11 @@ describe.skipIf(sql === null)("liveness-reaper", () => { const rows: { id: string }[] = await requireSql()` INSERT INTO workflow_runs ( workflow_name, target_type, target_owner, target_repo, target_number, - status, state, owner_kind, owner_id + status, state, owner_kind, owner_id, dispatch_enqueued_at ) VALUES ( 'triage', 'issue', 'acme', 'repo', ${target.number}, - ${status}, '{}'::jsonb, ${ownerKind}, ${ownerId} + ${status}, '{}'::jsonb, ${ownerKind}, ${ownerId}, + ${ownerKind === "orchestrator" ? new Date() : null} ) RETURNING id `; @@ -166,6 +200,22 @@ describe.skipIf(sql === null)("liveness-reaper", () => { expect(deadRow?.state["failedReason"]).toContain("orch-dead"); }); + it("preserves an unpublished outbox row after its owner exits", async () => { + const runId = await insertWorkflowRow({ number: 1011 }, "orchestrator", "orch-dead"); + await requireSql()` + UPDATE workflow_runs SET dispatch_enqueued_at = NULL WHERE id = ${runId} + `; + + const { reapOnce } = await import("../../src/orchestrator/liveness-reaper"); + const result = await reapOnce(requireSql()); + + expect(result.workflowRunsReaped.map((row) => row.id)).not.toContain(runId); + const [row] = await requireSql()<{ status: string }[]>` + SELECT status FROM workflow_runs WHERE id = ${runId} + `; + expect(row?.status).toBe("queued"); + }); + it("reaps daemon-owned 'running' rows whose daemon heartbeat is missing", async () => { await setDaemonAlive("daemon-alive"); const liveId = await insertWorkflowRow({ number: 1003 }, "daemon", "daemon-alive", "running"); @@ -182,6 +232,403 @@ describe.skipIf(sql === null)("liveness-reaper", () => { expect(aliveRow?.status).toBe("running"); }); + it("rechecks a candidate heartbeat before changing durable daemon ownership", async () => { + const daemonId = "daemon-registered-after-snapshot"; + const runId = await insertWorkflowRow({ number: 1013 }, "daemon", daemonId, "running"); + await requireSql()` + INSERT INTO daemons ( + id, hostname, platform, os_version, capabilities, resources, + status, first_seen_at, last_seen_at + ) VALUES ( + ${daemonId}, 'host', 'linux', '6', '{}'::jsonb, '{}'::jsonb, + 'active', now(), now() + ) + `; + // The fleet snapshot reads active_daemons, while registration publishes the + // liveness key first. This reproduces that interleaving without a test hook. + await setDaemonHeartbeatOnly(daemonId); + + const { reapOnce } = await import("../../src/orchestrator/liveness-reaper"); + const result = await reapOnce(requireSql()); + + expect(result.workflowRunsReaped.map((row) => row.id)).not.toContain(runId); + expect(result.daemonsMarkedInactive).toBe(0); + const [workflow] = await requireSql()<{ status: string }[]>` + SELECT status FROM workflow_runs WHERE id = ${runId} + `; + const [daemon] = await requireSql()<{ status: string }[]>` + SELECT status FROM daemons WHERE id = ${daemonId} + `; + expect(workflow?.status).toBe("running"); + expect(daemon?.status).toBe("active"); + }); + + it("recovers a crashed instance processing list after its heartbeat expires", async () => { + const oldInstanceId = `orchestrator-old-${crypto.randomUUID()}`; + const raw = JSON.stringify({ deliveryId: crypto.randomUUID() }); + const { requireValkeyClient } = await import("../../src/orchestrator/valkey"); + const valkey = requireValkeyClient(); + await valkey.send("LPUSH", [`queue:processing:${oldInstanceId}`, raw]); + await valkey.send("SET", [`orchestrator:${oldInstanceId}:alive`, "1", "EX", "60"]); + const { reapOnce } = await import("../../src/orchestrator/liveness-reaper"); + + await reapOnce(requireSql()); + expect(await valkey.send("LLEN", [`queue:processing:${oldInstanceId}`])).toBe(1); + + await valkey.send("DEL", [`orchestrator:${oldInstanceId}:alive`]); + await reapOnce(requireSql()); + expect(await valkey.send("LLEN", [`queue:processing:${oldInstanceId}`])).toBe(0); + expect(await valkey.send("LPOP", ["queue:jobs"])).toBe(raw); + }); + + it("fails a dead daemon's standalone execution and releases its lock", async () => { + const deliveryId = crypto.randomUUID(); + const daemonId = `daemon-worker-${crypto.randomUUID()}`; + await requireSql()` + INSERT INTO executions ( + delivery_id, repo_owner, repo_name, entity_number, entity_type, + event_name, trigger_username, dispatch_mode, dispatch_target, dispatch_reason, + daemon_id, status, started_at + ) VALUES ( + ${deliveryId}, 'acme', 'repo', 1012, 'issue', + 'issue_comment', 'user', 'daemon', 'daemon', 'persistent-daemon', + ${daemonId}, 'running', now() + ) + `; + await requireSql()` + INSERT INTO scheduled_action_state ( + installation_id, owner, repo, action_name, in_flight_job_id, in_flight_started_at + ) VALUES (1, 'acme', 'repo', 'standalone-reap', ${deliveryId}, now()) + `; + const { reapOnce } = await import("../../src/orchestrator/liveness-reaper"); + + await reapOnce(requireSql()); + + const [execution] = await requireSql()<{ status: string; error_message: string | null }[]>` + SELECT status, error_message FROM executions WHERE delivery_id = ${deliveryId} + `; + const [lock] = await requireSql()<{ in_flight_job_id: string | null }[]>` + SELECT in_flight_job_id FROM scheduled_action_state WHERE action_name = 'standalone-reap' + `; + expect(execution).toEqual({ + status: "failed", + error_message: "Owning daemon is no longer alive", + }); + expect(lock?.in_flight_job_id).toBeNull(); + }); + + it("does not reap an unexpired runner attempt because Valkey has no runner heartbeat", async () => { + const attemptId = crypto.randomUUID(); + const runId = await insertWorkflowRow( + { number: 1010 }, + "daemon", + `workflow-runner:${attemptId}`, + "running", + ); + await requireSql()` + UPDATE workflow_runs + SET attempt_id = ${attemptId}, + lease_expires_at = now() + interval '2 minutes', + attempt_deadline_at = now() + interval '70 minutes' + WHERE id = ${runId} + `; + + const { reapOnce } = await import("../../src/orchestrator/liveness-reaper"); + const result = await reapOnce(requireSql()); + + expect(result.workflowRunsReaped.map((row) => row.id)).not.toContain(runId); + const [row] = await requireSql()< + { status: string }[] + >`SELECT status FROM workflow_runs WHERE id = ${runId}`; + expect(row?.status).toBe("running"); + }); + + it("atomically fails an expired runner attempt, receipt, parent, and lock", async () => { + const attemptId = crypto.randomUUID(); + const runnerId = `workflow-runner:${attemptId}`; + const executionDeliveryId = crypto.randomUUID(); + const [parent] = await requireSql()<{ id: string }[]>` + INSERT INTO workflow_runs ( + workflow_name, target_type, target_owner, target_repo, target_number, + status, state + ) VALUES ( + 'ship', 'pr', 'acme', 'repo', 1011, 'running', '{}'::jsonb + ) + RETURNING id + `; + if (parent === undefined) throw new Error("Expected parent fixture"); + const [child] = await requireSql()<{ id: string }[]>` + INSERT INTO workflow_runs ( + workflow_name, target_type, target_owner, target_repo, target_number, + parent_run_id, parent_step_index, status, state, + execution_delivery_id, owner_kind, owner_id, attempt_id, lease_expires_at, + attempt_deadline_at + ) VALUES ( + 'triage', 'pr', 'acme', 'repo', 1011, + ${parent.id}, 0, 'running', '{}'::jsonb, + ${executionDeliveryId}, 'daemon', ${runnerId}, ${attemptId}, + now() - interval '1 second', now() + interval '70 minutes' + ) + RETURNING id + `; + if (child === undefined) throw new Error("Expected child fixture"); + await requireSql()` + INSERT INTO executions ( + delivery_id, repo_owner, repo_name, entity_number, entity_type, + event_name, trigger_username, dispatch_mode, dispatch_target, dispatch_reason, + daemon_id, offer_id, status, started_at + ) VALUES ( + ${executionDeliveryId}, 'acme', 'repo', 1011, 'pr', + 'issue_comment', 'user', 'workflow-runner', 'workflow-runner', 'workflow-runner', + ${runnerId}, ${attemptId}, 'running', now() + ) + `; + await requireSql()` + INSERT INTO scheduled_action_state ( + installation_id, owner, repo, action_name, + in_flight_job_id, in_flight_started_at + ) VALUES ( + 1, 'acme', 'repo', 'expiry-test', ${executionDeliveryId}, now() + ) + `; + + const { reapOnce } = await import("../../src/orchestrator/liveness-reaper"); + await reapOnce(requireSql()); + + const [childAfter] = await requireSql()< + { status: string; attempt_completed_at: Date | null }[] + >`SELECT status, attempt_completed_at FROM workflow_runs WHERE id = ${child.id}`; + const [parentAfter] = await requireSql()< + { status: string; state: Record }[] + >`SELECT status, state FROM workflow_runs WHERE id = ${parent.id}`; + const [executionAfter] = await requireSql()< + { status: string; error_message: string | null; result_processed_at: Date | null }[] + >` + SELECT status, error_message, result_processed_at + FROM executions + WHERE delivery_id = ${executionDeliveryId} + `; + const [scheduleAfter] = await requireSql()<{ in_flight_job_id: string | null }[]>` + SELECT in_flight_job_id + FROM scheduled_action_state + WHERE action_name = 'expiry-test' + `; + + expect(childAfter?.status).toBe("failed"); + expect(childAfter?.attempt_completed_at).toBeInstanceOf(Date); + expect(parentAfter?.status).toBe("failed"); + expect(parentAfter?.state["failedAtStepIndex"]).toBe(0); + expect(executionAfter).toMatchObject({ + status: "failed", + error_message: "Workflow execution lease expired", + }); + expect(executionAfter?.result_processed_at).toBeInstanceOf(Date); + expect(scheduleAfter?.in_flight_job_id).toBeNull(); + }); + + it("records the immutable deadline reason on the workflow, parent, and execution", async () => { + const attemptId = crypto.randomUUID(); + const runnerId = `workflow-runner:${attemptId}`; + const executionDeliveryId = crypto.randomUUID(); + const [parent] = await requireSql()<{ id: string }[]>` + INSERT INTO workflow_runs ( + workflow_name, target_type, target_owner, target_repo, target_number, + status, state + ) VALUES ('ship', 'pr', 'acme', 'repo', 1014, 'running', '{}'::jsonb) + RETURNING id + `; + if (parent === undefined) throw new Error("Expected deadline parent fixture"); + const [child] = await requireSql()<{ id: string }[]>` + INSERT INTO workflow_runs ( + workflow_name, target_type, target_owner, target_repo, target_number, + parent_run_id, parent_step_index, status, state, + execution_delivery_id, owner_kind, owner_id, attempt_id, lease_expires_at, + attempt_deadline_at + ) VALUES ( + 'review', 'pr', 'acme', 'repo', 1014, + ${parent.id}, 2, 'running', '{}'::jsonb, + ${executionDeliveryId}, 'daemon', ${runnerId}, ${attemptId}, + now() + interval '2 minutes', now() - interval '1 second' + ) + RETURNING id + `; + if (child === undefined) throw new Error("Expected deadline child fixture"); + await requireSql()` + INSERT INTO executions ( + delivery_id, repo_owner, repo_name, entity_number, entity_type, + event_name, trigger_username, dispatch_mode, dispatch_target, dispatch_reason, + daemon_id, offer_id, status, started_at + ) VALUES ( + ${executionDeliveryId}, 'acme', 'repo', 1014, 'pr', + 'issue_comment', 'user', 'workflow-runner', 'workflow-runner', 'workflow-runner', + ${runnerId}, ${attemptId}, 'running', now() + ) + `; + + const { reapOnce } = await import("../../src/orchestrator/liveness-reaper"); + await reapOnce(requireSql()); + + const [childAfter] = await requireSql()< + { status: string; state: Record }[] + >`SELECT status, state FROM workflow_runs WHERE id = ${child.id}`; + const [parentAfter] = await requireSql()< + { status: string; state: Record }[] + >`SELECT status, state FROM workflow_runs WHERE id = ${parent.id}`; + const [executionAfter] = await requireSql()< + { status: string; error_message: string | null }[] + >`SELECT status, error_message FROM executions WHERE delivery_id = ${executionDeliveryId}`; + expect(childAfter).toMatchObject({ + status: "failed", + state: { + failedReason: "workflow execution deadline expired", + phase: "deadline-expired", + }, + }); + expect(parentAfter).toMatchObject({ + status: "failed", + state: { + failedAtStepIndex: 2, + failedReason: "workflow execution deadline expired", + }, + }); + expect(executionAfter).toEqual({ + status: "failed", + error_message: "Workflow execution deadline expired", + }); + }); + + it("terminalizes an over-age queued dispatch and releases its durable lock", async () => { + const { config } = await import("../../src/config"); + const deliveryId = crypto.randomUUID(); + const [parent] = await requireSql()<{ id: string }[]>` + INSERT INTO workflow_runs ( + workflow_name, target_type, target_owner, target_repo, target_number, + status, state + ) VALUES ('ship', 'issue', 'acme', 'repo', 1015, 'running', '{}'::jsonb) + RETURNING id + `; + if (parent === undefined) throw new Error("Expected queued-expiry parent fixture"); + const [child] = await requireSql()<{ id: string }[]>` + INSERT INTO workflow_runs ( + workflow_name, target_type, target_owner, target_repo, target_number, + parent_run_id, parent_step_index, status, state, execution_delivery_id, + owner_kind, owner_id, created_at + ) VALUES ( + 'review', 'pr', 'acme', 'repo', 1015, + ${parent.id}, 3, 'queued', '{}'::jsonb, ${deliveryId}, + NULL, NULL, + now() - ${config.workflowDispatchTimeoutMs + 1_000} * interval '1 millisecond' + ) + RETURNING id + `; + if (child === undefined) throw new Error("Expected queued-expiry child fixture"); + await requireSql()` + INSERT INTO executions ( + delivery_id, repo_owner, repo_name, entity_number, entity_type, + event_name, trigger_username, dispatch_mode, dispatch_target, dispatch_reason, status + ) VALUES ( + ${deliveryId}, 'acme', 'repo', 1015, 'pr', + 'issue_comment', 'user', 'workflow-runner', 'workflow-runner', 'workflow-runner', 'queued' + ) + `; + await requireSql()` + INSERT INTO scheduled_action_state ( + installation_id, owner, repo, action_name, in_flight_job_id, in_flight_started_at + ) VALUES (1, 'acme', 'repo', 'dispatch-expiry-test', ${deliveryId}, now()) + `; + + const { reapOnce } = await import("../../src/orchestrator/liveness-reaper"); + await reapOnce(requireSql()); + + const [childAfter] = await requireSql()< + { status: string; state: Record; attempt_completed_at: Date | null }[] + >`SELECT status, state, attempt_completed_at FROM workflow_runs WHERE id = ${child.id}`; + const [parentAfter] = await requireSql()< + { status: string; state: Record }[] + >`SELECT status, state FROM workflow_runs WHERE id = ${parent.id}`; + const [executionAfter] = await requireSql()< + { status: string; error_message: string | null; result_processed_at: Date | null }[] + >`SELECT status, error_message, result_processed_at FROM executions WHERE delivery_id = ${deliveryId}`; + const [scheduleAfter] = await requireSql()<{ in_flight_job_id: string | null }[]>` + SELECT in_flight_job_id + FROM scheduled_action_state + WHERE action_name = 'dispatch-expiry-test' + `; + expect(childAfter).toMatchObject({ + status: "failed", + state: { + failedReason: "workflow dispatch deadline expired", + phase: "dispatch-expired", + }, + attempt_completed_at: expect.any(Date), + }); + expect(parentAfter).toMatchObject({ + status: "failed", + state: { + failedAtStepIndex: 3, + failedReason: "workflow dispatch deadline expired", + }, + }); + expect(executionAfter).toMatchObject({ + status: "failed", + error_message: "Workflow dispatch deadline expired", + result_processed_at: expect.any(Date), + }); + expect(scheduleAfter?.in_flight_job_id).toBeNull(); + }); + + it("atomically fails the exact execution and lock for a heartbeat-reaped daemon owner", async () => { + const deliveryId = crypto.randomUUID(); + const daemonId = "daemon-dead-with-receipt"; + const [run] = await requireSql()<{ id: string }[]>` + INSERT INTO workflow_runs ( + workflow_name, target_type, target_owner, target_repo, target_number, + status, state, execution_delivery_id, owner_kind, owner_id + ) VALUES ( + 'review', 'pr', 'acme', 'repo', 1012, + 'running', '{}'::jsonb, ${deliveryId}, 'daemon', ${daemonId} + ) + RETURNING id + `; + if (run === undefined) throw new Error("Expected workflow fixture"); + await requireSql()` + INSERT INTO executions ( + delivery_id, repo_owner, repo_name, entity_number, entity_type, + event_name, trigger_username, dispatch_mode, dispatch_target, dispatch_reason, + daemon_id, status, started_at + ) VALUES ( + ${deliveryId}, 'acme', 'repo', 1012, 'pr', + 'issue_comment', 'user', 'daemon', 'daemon', 'persistent-daemon', + ${daemonId}, 'running', now() + ) + `; + await requireSql()` + INSERT INTO scheduled_action_state ( + installation_id, owner, repo, action_name, + in_flight_job_id, in_flight_started_at + ) VALUES (1, 'acme', 'repo', 'dead-daemon-test', ${deliveryId}, now()) + `; + + const { reapOnce } = await import("../../src/orchestrator/liveness-reaper"); + await reapOnce(requireSql()); + + const [workflow] = await requireSql()<{ status: string }[]>` + SELECT status FROM workflow_runs WHERE id = ${run.id} + `; + const [execution] = await requireSql()< + { status: string; error_message: string | null }[] + >`SELECT status, error_message FROM executions WHERE delivery_id = ${deliveryId}`; + const [schedule] = await requireSql()<{ in_flight_job_id: string | null }[]>` + SELECT in_flight_job_id FROM scheduled_action_state WHERE action_name = 'dead-daemon-test' + `; + expect(workflow?.status).toBe("failed"); + expect(execution).toEqual({ + status: "failed", + error_message: "Owning daemon is no longer alive", + }); + expect(schedule?.in_flight_job_id).toBeNull(); + }); + it("ignores rows in terminal status and rows with NULL owner_kind", async () => { // No live heartbeats at all → would reap everything reapable. await insertWorkflowRow({ number: 1005 }, "orchestrator", "orch-x", "succeeded"); diff --git a/test/orchestrator/log-fields.test.ts b/test/orchestrator/log-fields.test.ts index c1ac8a55..165a77c6 100644 --- a/test/orchestrator/log-fields.test.ts +++ b/test/orchestrator/log-fields.test.ts @@ -330,7 +330,6 @@ describe("GithubAppTokenMintLogSchema (#236)", () => { for (const via of [ "handleAccept", "handleScopedAccept", - "postOrphanNotification", "shipTickleResume", "proposalPoller", "schedulerRunAction", diff --git a/test/orchestrator/queue-worker-resilience.test.ts b/test/orchestrator/queue-worker-resilience.test.ts new file mode 100644 index 00000000..da9c70aa --- /dev/null +++ b/test/orchestrator/queue-worker-resilience.test.ts @@ -0,0 +1,165 @@ +import { beforeEach, describe, expect, it, mock } from "bun:test"; + +import type { QueuedJob } from "../../src/orchestrator/job-queue"; + +const workflowJob: Extract = { + kind: "workflow-run", + deliveryId: "workflow-16", + repoOwner: "acme", + repoName: "widgets", + entityNumber: 42, + isPR: false, + eventName: "issue_comment", + triggerUsername: "user", + labels: [], + triggerBodyPreview: "", + enqueuedAt: Date.now(), + retryCount: 3, + workflowRun: { runId: crypto.randomUUID(), workflowName: "implement" }, +}; +const raw = JSON.stringify(workflowJob); + +const leaseJob = mock(() => Promise.resolve(null as { job: QueuedJob; raw: string } | null)); +const deferLeasedWorkflowJob = mock(() => + Promise.resolve({ status: "moved" as "moved" | "already-moved" | "missing" }), +); +const ensureWorkflowJobQueued = mock(() => Promise.resolve(true)); +const releaseLeasedJob = mock(() => Promise.resolve()); +const requeueLeasedJob = mock(() => Promise.resolve(1)); +const dispatchJob = mock(() => Promise.resolve(false)); +const markJobTerminallyFailed = mock(() => Promise.resolve()); +const dispatchWorkflowRunner = mock(() => + Promise.resolve("accepted" as "accepted" | "stale" | "capacity"), +); + +void mock.module("../../src/orchestrator/job-queue", () => ({ + deferLeasedWorkflowJob, + ensureWorkflowJobQueued, + leaseJob, + releaseLeasedJob, + requeueLeasedJob, +})); +void mock.module("../../src/orchestrator/job-dispatcher", () => ({ + dispatchJob, + markJobTerminallyFailed, +})); +void mock.module("../../src/orchestrator/workflow-runner-dispatch", () => ({ + dispatchWorkflowRunner, +})); +void mock.module("../../src/orchestrator/instance-id", () => ({ + getInstanceId: () => "orchestrator-test", +})); +void mock.module("../../src/config", () => ({ + config: { jobMaxRetries: 3, queueWorkerBackoffMaxMs: 200 }, +})); +void mock.module("../../src/logger", () => ({ + logger: { + info: mock(() => undefined), + warn: mock(() => undefined), + error: mock(() => undefined), + debug: mock(() => undefined), + }, +})); + +const { startQueueWorker, stopQueueWorker } = await import("../../src/orchestrator/queue-worker"); + +describe("isolated workflow queue dispatch", () => { + beforeEach(async () => { + await stopQueueWorker(); + leaseJob.mockReset(); + deferLeasedWorkflowJob.mockReset(); + ensureWorkflowJobQueued.mockReset(); + releaseLeasedJob.mockReset(); + requeueLeasedJob.mockReset(); + dispatchJob.mockReset(); + markJobTerminallyFailed.mockReset(); + dispatchWorkflowRunner.mockReset(); + + let leased = false; + leaseJob.mockImplementation(() => { + if (leased) return Promise.resolve(null); + leased = true; + return Promise.resolve({ job: workflowJob, raw }); + }); + deferLeasedWorkflowJob.mockResolvedValue({ status: "moved" }); + ensureWorkflowJobQueued.mockResolvedValue(true); + releaseLeasedJob.mockResolvedValue(); + requeueLeasedJob.mockResolvedValue(1); + dispatchJob.mockResolvedValue(false); + markJobTerminallyFailed.mockResolvedValue(); + dispatchWorkflowRunner.mockResolvedValue("accepted"); + }); + + it("defers capacity without consuming retry budget or using the shared dispatcher", async () => { + dispatchWorkflowRunner.mockResolvedValue("capacity"); + deferLeasedWorkflowJob.mockResolvedValue({ status: "missing" }); + let duplicatePublished: (() => void) | undefined; + const published = new Promise((resolve) => { + duplicatePublished = resolve; + }); + ensureWorkflowJobQueued.mockImplementation(() => { + duplicatePublished?.(); + return Promise.resolve(); + }); + + startQueueWorker(); + await published; + await stopQueueWorker(); + + expect(deferLeasedWorkflowJob).toHaveBeenCalledWith( + "orchestrator-test", + raw, + workflowJob, + expect.any(String), + ); + expect(ensureWorkflowJobQueued).toHaveBeenCalledWith( + expect.objectContaining({ + kind: "workflow-run", + retryCount: workflowJob.retryCount, + }), + "orchestrator-test", + ); + expect(dispatchJob).not.toHaveBeenCalled(); + expect(releaseLeasedJob).not.toHaveBeenCalled(); + }); + + it("defers a pre-transfer dispatch error instead of falling back to a shared daemon", async () => { + dispatchWorkflowRunner.mockRejectedValue(new Error("database unavailable")); + let deferred: (() => void) | undefined; + const deferralStarted = new Promise((resolve) => { + deferred = resolve; + }); + deferLeasedWorkflowJob.mockImplementation(() => { + deferred?.(); + return Promise.resolve({ status: "moved" }); + }); + + startQueueWorker(); + await deferralStarted; + await stopQueueWorker(); + + expect(dispatchJob).not.toHaveBeenCalled(); + expect(markJobTerminallyFailed).not.toHaveBeenCalled(); + expect(releaseLeasedJob).not.toHaveBeenCalled(); + }); + + it("releases the queue lease after PostgreSQL accepts recovery authority", async () => { + let released: (() => void) | undefined; + const leaseReleased = new Promise((resolve) => { + released = resolve; + }); + releaseLeasedJob.mockImplementation(() => { + released?.(); + return Promise.resolve(); + }); + + startQueueWorker(); + await leaseReleased; + await stopQueueWorker(); + + expect(dispatchWorkflowRunner).toHaveBeenCalledWith(workflowJob); + expect(releaseLeasedJob).toHaveBeenCalledWith("orchestrator-test", raw); + expect(deferLeasedWorkflowJob).not.toHaveBeenCalled(); + expect(dispatchJob).not.toHaveBeenCalled(); + }); +}); diff --git a/test/orchestrator/repo-knowledge-persistence.test.ts b/test/orchestrator/repo-knowledge-persistence.test.ts new file mode 100644 index 00000000..bf5f59db --- /dev/null +++ b/test/orchestrator/repo-knowledge-persistence.test.ts @@ -0,0 +1,101 @@ +import { beforeEach, describe, expect, it, mock } from "bun:test"; + +import { expectToReject } from "../utils/assertions"; + +const db = mock(() => Promise.resolve([{ repo_owner: "acme", repo_name: "widgets" }])); +const saveRepoLearnings = mock(() => Promise.resolve(1)); +const deleteRepoMemories = mock(() => Promise.resolve(1)); +const saveReviewLearnings = mock(() => Promise.resolve(1)); +const deleteReviewLearnings = mock(() => Promise.resolve(1)); +const bumpReviewLearningUsage = mock(() => Promise.resolve()); + +void mock.module("../../src/config", () => ({ config: { reviewLearningsEnabled: true } })); +void mock.module("../../src/db", () => ({ requireDb: (): typeof db => db })); +void mock.module("../../src/logger", () => ({ + logger: { + info: mock(() => undefined), + warn: mock(() => undefined), + error: mock(() => undefined), + debug: mock(() => undefined), + }, +})); +void mock.module("../../src/orchestrator/repo-knowledge", () => ({ + saveRepoLearnings, + deleteRepoMemories, +})); +void mock.module("../../src/orchestrator/review-learnings", () => ({ + saveReviewLearnings, + deleteReviewLearnings, + bumpReviewLearningUsage, +})); + +const { persistRepoKnowledge } = await import("../../src/orchestrator/repo-knowledge-persistence"); + +const actions = { + learnings: [{ category: "setup" as const, content: "Run isolated tests." }], + deletions: ["11111111-1111-4111-8111-111111111111"], + reviewLearningSaves: [{ directive: "Keep tests isolated." }], + reviewLearningDeletes: ["22222222-2222-4222-8222-222222222222"], +}; + +describe("repo knowledge persistence", () => { + beforeEach(() => { + db.mockReset(); + db.mockResolvedValue([{ repo_owner: "acme", repo_name: "widgets" }]); + saveRepoLearnings.mockReset(); + saveRepoLearnings.mockResolvedValue(1); + deleteRepoMemories.mockClear(); + saveReviewLearnings.mockClear(); + deleteReviewLearnings.mockClear(); + bumpReviewLearningUsage.mockClear(); + }); + + it("resolves repository ownership and scopes every durable action", async () => { + await persistRepoKnowledge( + { + deliveryId: "delivery-16", + daemonActions: actions, + appliedReviewLearningIds: ["learning-1"], + }, + db as never, + ); + + expect(saveRepoLearnings).toHaveBeenCalledWith("acme", "widgets", actions.learnings, db); + expect(deleteRepoMemories).toHaveBeenCalledWith("acme", "widgets", actions.deletions, db); + expect(deleteReviewLearnings).toHaveBeenCalledWith( + "acme", + "widgets", + actions.reviewLearningDeletes, + db, + ); + expect(bumpReviewLearningUsage).toHaveBeenCalledWith(["learning-1"], db); + }); + + it("throws for a missing execution row or a load-bearing write failure", async () => { + db.mockResolvedValueOnce([]); + await expectToReject( + persistRepoKnowledge({ deliveryId: "missing", daemonActions: actions }, db as never), + "Execution row missing", + ); + + saveRepoLearnings.mockRejectedValueOnce(new Error("write failed")); + await expectToReject( + persistRepoKnowledge({ deliveryId: "delivery-16", daemonActions: actions }, db as never), + "write failed", + ); + }); + + it("does not block durable settlement when an approximate usage bump fails", async () => { + bumpReviewLearningUsage.mockRejectedValueOnce(new Error("usage unavailable")); + + await persistRepoKnowledge( + { + deliveryId: "delivery-16", + appliedReviewLearningIds: ["learning-1"], + }, + db as never, + ); + + expect(bumpReviewLearningUsage).toHaveBeenCalledWith(["learning-1"], db); + }); +}); diff --git a/test/orchestrator/workflow-expiry-notifier.test.ts b/test/orchestrator/workflow-expiry-notifier.test.ts new file mode 100644 index 00000000..2916e95d --- /dev/null +++ b/test/orchestrator/workflow-expiry-notifier.test.ts @@ -0,0 +1,353 @@ +import { beforeEach, describe, expect, it, mock } from "bun:test"; + +const setState = mock(() => Promise.resolve()); +const addReaction = mock(() => Promise.resolve()); +const findById = mock(() => Promise.resolve(null as unknown)); +const markWorkflowFailureNotified = mock(() => Promise.resolve(true)); +const findPendingWorkflowFailureNotifications = mock(() => Promise.resolve([])); +const getRepoInstallation = mock(() => Promise.resolve({ data: { id: 123 } })); +const mintedOctokit = {}; +const mintInstallationToken = mock(() => Promise.resolve({ octokit: mintedOctokit })); +const revokeInstallationToken = mock(() => Promise.resolve(true)); +const testConfig = { + nodeEnv: "production", + githubPersonalAccessToken: "test-token" as string | undefined, + appId: "test-app" as string | undefined, + privateKey: "test-key" as string | undefined, +}; + +void mock.module("../../src/config", () => ({ + config: testConfig, +})); +void mock.module("../../src/logger", () => ({ + logger: { + info: mock(() => undefined), + warn: mock(() => undefined), + error: mock(() => undefined), + debug: mock(() => undefined), + child: mock(function (this: unknown) { + return this; + }), + }, +})); +void mock.module("octokit", () => ({ + Octokit: function MockOctokit(this: unknown): unknown { + return this; + }, + App: function MockApp(): unknown { + return { octokit: { rest: { apps: { getRepoInstallation } } } }; + }, +})); +void mock.module("../../src/utils/octokit-observability", () => ({ + observableOctokit: () => + function MockOctokit(this: unknown): unknown { + return this; + }, +})); +void mock.module("../../src/utils/reactions", () => ({ addReaction })); +void mock.module("../../src/workflows/tracking-mirror", () => ({ setState })); +void mock.module("../../src/workflows/runs-store", () => ({ + findById, + findPendingWorkflowFailureNotifications, + markWorkflowFailureNotified, +})); +void mock.module("../../src/orchestrator/installation-token", () => ({ + mintInstallationToken, + revokeInstallationToken, +})); + +const { + notifyExpiredWorkflowAttempts, + notifyExpiredWorkflowDispatches, + notifyRunnerStartFailures, + reconcilePendingWorkflowFailureNotifications, +} = await import("../../src/orchestrator/workflow-expiry-notifier"); + +function row( + id: string, + parentRunId: string | null, + failedReason: string, +): Record { + return { + id, + workflow_name: parentRunId === null ? "ship" : "review", + target_type: "pr", + target_owner: "acme", + target_repo: "widgets", + target_number: 42, + parent_run_id: parentRunId, + parent_step_index: parentRunId === null ? null : 3, + status: "failed", + state: { failedReason }, + tracking_comment_id: 100, + delivery_id: "delivery-1", + owner_kind: "daemon", + owner_id: "sensitive-daemon-id", + attempt_id: crypto.randomUUID(), + lease_expires_at: null, + attempt_completed_at: new Date(), + cascade_completed_at: null, + failure_notified_at: null, + execution_delivery_id: "delivery-1", + dispatch_enqueued_at: new Date(), + trigger_comment_id: parentRunId === null ? 200 : null, + trigger_event_type: parentRunId === null ? "issue_comment" : null, + created_at: new Date(), + updated_at: new Date(), + }; +} + +describe("workflow expiry notifier", () => { + beforeEach(() => { + setState.mockClear(); + addReaction.mockClear(); + findById.mockReset(); + markWorkflowFailureNotified.mockClear(); + findPendingWorkflowFailureNotifications.mockReset(); + findPendingWorkflowFailureNotifications.mockResolvedValue([]); + testConfig.githubPersonalAccessToken = "test-token"; + getRepoInstallation.mockClear(); + mintInstallationToken.mockReset(); + mintInstallationToken.mockResolvedValue({ octokit: mintedOctokit }); + revokeInstallationToken.mockReset(); + revokeInstallationToken.mockResolvedValue(true); + }); + + it("notifies one top-level workflow with fixed text for duplicate expired children", async () => { + testConfig.githubPersonalAccessToken = undefined; + const parent = row("parent", null, "parent raw failure"); + const childA = row("child-a", "parent", "secret child A failure"); + const childB = row("child-b", "parent", "secret child B failure"); + findById.mockImplementation((id: string) => Promise.resolve(id === "parent" ? parent : null)); + + await notifyExpiredWorkflowAttempts([childA, childB] as never); + + expect(setState).toHaveBeenCalledTimes(1); + const call = setState.mock.calls[0] as unknown as [ + unknown, + { runId: string; humanMessage: string; patch: Record }, + ]; + expect(call[1].runId).toBe("parent"); + expect(call[1].patch).toEqual({ phase: "lease-expired" }); + expect(call[1].humanMessage).toContain("Workflow execution lease expired"); + expect(call[1].humanMessage).not.toContain("sensitive-daemon-id"); + expect(call[1].humanMessage).not.toContain("secret child"); + expect(addReaction).toHaveBeenCalledTimes(1); + expect(addReaction).toHaveBeenCalledWith( + expect.objectContaining({ + owner: "acme", + repo: "widgets", + commentId: 200, + eventType: "issue_comment", + content: "confused", + }), + ); + expect(markWorkflowFailureNotified).toHaveBeenCalledTimes(2); + expect(getRepoInstallation).toHaveBeenCalledTimes(1); + expect(mintInstallationToken).toHaveBeenCalledTimes(1); + expect(mintInstallationToken).toHaveBeenCalledWith( + expect.objectContaining({ installationId: 123, repositoryName: "widgets" }), + ); + expect(revokeInstallationToken).toHaveBeenCalledTimes(1); + }); + + it("reports an immutable deadline without blaming lease renewal", async () => { + const expired = row("deadline", null, "workflow execution deadline expired"); + + await notifyExpiredWorkflowAttempts([expired] as never); + + const call = setState.mock.calls[0] as unknown as [ + unknown, + { humanMessage: string; patch: Record }, + ]; + expect(call[1].patch).toEqual({ phase: "deadline-expired" }); + expect(call[1].humanMessage).toContain("Workflow execution deadline expired"); + expect(call[1].humanMessage).toContain("immutable attempt deadline elapsed"); + expect(call[1].humanMessage).not.toContain("stopped renewing"); + }); + + it("keeps the durable notification pending when GitHub projection fails", async () => { + testConfig.githubPersonalAccessToken = undefined; + const parent = row("parent", null, "failure"); + setState.mockRejectedValueOnce(new Error("GitHub unavailable")); + + await notifyExpiredWorkflowAttempts([parent] as never); + + expect(markWorkflowFailureNotified).not.toHaveBeenCalled(); + expect(revokeInstallationToken).toHaveBeenCalledWith( + mintedOctokit, + expect.anything(), + expect.objectContaining({ runId: "parent", owner: "failure-notification" }), + ); + }); + + it("keeps the durable notification pending when the trigger reaction fails", async () => { + const parent = row("parent", null, "failure"); + addReaction.mockRejectedValueOnce(new Error("GitHub unavailable")); + + await notifyExpiredWorkflowAttempts([parent] as never); + + expect(setState).toHaveBeenCalledTimes(1); + expect(markWorkflowFailureNotified).not.toHaveBeenCalled(); + }); + + it("projects and receipts a runner-start failure", async () => { + const failed = row("failed", null, "create runner Pod failed (403)"); + + await notifyRunnerStartFailures([failed] as never); + + const call = setState.mock.calls[0] as unknown as [ + unknown, + { humanMessage: string; patch: Record }, + ]; + expect(call[1].patch).toEqual({ phase: "runner-start-failed" }); + expect(call[1].humanMessage).toContain("create runner Pod failed (403)"); + expect(markWorkflowFailureNotified).toHaveBeenCalledTimes(1); + }); + + it("projects and receipts a queued dispatch expiry", async () => { + const failed = row("dispatch-expired", null, "workflow dispatch deadline expired"); + failed.attempt_id = null; + failed.state = { + failedReason: "workflow dispatch deadline expired", + phase: "dispatch-expired", + }; + + await notifyExpiredWorkflowDispatches([failed] as never); + + const call = setState.mock.calls[0] as unknown as [ + unknown, + { humanMessage: string; patch: Record }, + ]; + expect(call[1].patch).toEqual({ phase: "dispatch-expired" }); + expect(call[1].humanMessage).toContain("Workflow dispatch deadline expired"); + expect(markWorkflowFailureNotified).toHaveBeenCalledWith({ + runId: "dispatch-expired", + attemptId: null, + }); + }); + + it("replays a durable missed expiry notification", async () => { + const expired = row("expired", null, "workflow execution lease expired"); + expired.state = { failedReason: "workflow execution lease expired", phase: "lease-expired" }; + findPendingWorkflowFailureNotifications.mockResolvedValueOnce([ + { phase: "lease-expired", row: expired }, + ]); + + await reconcilePendingWorkflowFailureNotifications(); + + expect(setState).toHaveBeenCalledTimes(1); + expect(markWorkflowFailureNotified).toHaveBeenCalledTimes(1); + }); + + it("replays a durable missed deadline notification", async () => { + const expired = row("deadline", null, "workflow execution deadline expired"); + expired.state = { + failedReason: "workflow execution deadline expired", + phase: "deadline-expired", + }; + findPendingWorkflowFailureNotifications.mockResolvedValueOnce([ + { phase: "deadline-expired", row: expired }, + ]); + + await reconcilePendingWorkflowFailureNotifications(); + + const call = setState.mock.calls[0] as unknown as [unknown, { patch: Record }]; + expect(call[1].patch).toEqual({ phase: "deadline-expired" }); + expect(markWorkflowFailureNotified).toHaveBeenCalledTimes(1); + }); + + it("replays a durable missed runner-start notification", async () => { + const failed = row("failed", null, "runner boundary mismatch"); + failed.state = { failedReason: "runner boundary mismatch", phase: "runner-start-failed" }; + findPendingWorkflowFailureNotifications.mockResolvedValueOnce([ + { phase: "runner-start-failed", row: failed }, + ]); + + await reconcilePendingWorkflowFailureNotifications(); + + const call = setState.mock.calls[0] as unknown as [ + unknown, + { humanMessage: string; patch: Record }, + ]; + expect(call[1].patch).toEqual({ phase: "runner-start-failed" }); + expect(call[1].humanMessage).toContain("runner boundary mismatch"); + expect(markWorkflowFailureNotified).toHaveBeenCalledTimes(1); + }); + + it("replays a durable dispatch-expired notification", async () => { + const failed = row("dispatch-expired", null, "workflow dispatch retries exhausted"); + failed.attempt_id = null; + failed.state = { + failedReason: "workflow dispatch retries exhausted", + phase: "dispatch-expired", + }; + findPendingWorkflowFailureNotifications.mockResolvedValueOnce([ + { phase: "dispatch-expired", row: failed }, + ]); + + await reconcilePendingWorkflowFailureNotifications(); + + const call = setState.mock.calls[0] as unknown as [ + unknown, + { humanMessage: string; patch: Record }, + ]; + expect(call[1].patch).toEqual({ phase: "dispatch-expired" }); + expect(call[1].humanMessage).toContain("retries exhausted"); + }); + + it("replays and receipts a durable daemon-disconnect notification", async () => { + const disconnected = row("disconnected", null, "daemon disconnected during execution"); + disconnected.attempt_id = null; + disconnected.state = { + failedReason: "daemon disconnected during execution", + phase: "orphaned", + }; + findPendingWorkflowFailureNotifications.mockResolvedValueOnce([ + { phase: "orphaned", row: disconnected }, + ]); + + await reconcilePendingWorkflowFailureNotifications(); + + const call = setState.mock.calls[0] as unknown as [ + unknown, + { humanMessage: string; patch: Record }, + ]; + expect(call[1].patch).toEqual({ phase: "orphaned" }); + expect(call[1].humanMessage).toContain("Daemon disconnected during execution"); + expect(markWorkflowFailureNotified).toHaveBeenCalledWith({ + runId: "disconnected", + attemptId: null, + }); + }); + + it("replays a migration interruption with migration-specific guidance", async () => { + const interrupted = row( + "migration-interrupted", + null, + "workflow execution interrupted during isolated-runner migration", + ); + interrupted.attempt_id = null; + interrupted.state = { + failedReason: "workflow execution interrupted during isolated-runner migration", + phase: "migration-interrupted", + }; + findPendingWorkflowFailureNotifications.mockResolvedValueOnce([ + { phase: "migration-interrupted", row: interrupted }, + ]); + + await reconcilePendingWorkflowFailureNotifications(); + + const call = setState.mock.calls[0] as unknown as [ + unknown, + { humanMessage: string; patch: Record }, + ]; + expect(call[1].patch).toEqual({ phase: "migration-interrupted" }); + expect(call[1].humanMessage).toContain("execution interrupted during migration"); + expect(call[1].humanMessage).toContain("could not safely transfer"); + expect(markWorkflowFailureNotified).toHaveBeenCalledWith({ + runId: "migration-interrupted", + attemptId: null, + }); + }); +}); diff --git a/test/orchestrator/workflow-runner-capability.test.ts b/test/orchestrator/workflow-runner-capability.test.ts new file mode 100644 index 00000000..083a4019 --- /dev/null +++ b/test/orchestrator/workflow-runner-capability.test.ts @@ -0,0 +1,138 @@ +import { describe, expect, it } from "bun:test"; + +import { + deriveWorkflowRunnerCapability, + isWorkflowRunnerCapabilityValid, + parseWorkflowRunnerPath, + workflowRunnerPath, + workflowRunnerUrl, +} from "../../src/orchestrator/workflow-runner-capability"; + +const runId = "11111111-1111-4111-8111-111111111111"; +const attemptId = "22222222-2222-4222-8222-222222222222"; +const expiresAtMs = 2_000_000_000_000; +const nowMs = 1_900_000_000_000; + +describe("workflow runner attempt capability", () => { + it("binds authentication to the exact run and attempt", () => { + const capability = deriveWorkflowRunnerCapability( + "primary-secret", + runId, + attemptId, + expiresAtMs, + ); + const authorization = `Bearer ${capability}`; + + expect( + isWorkflowRunnerCapabilityValid( + authorization, + runId, + attemptId, + "primary-secret", + undefined, + nowMs, + ), + ).toBe(true); + expect( + isWorkflowRunnerCapabilityValid( + authorization, + crypto.randomUUID(), + attemptId, + "primary-secret", + undefined, + nowMs, + ), + ).toBe(false); + expect( + isWorkflowRunnerCapabilityValid( + authorization, + runId, + crypto.randomUUID(), + "primary-secret", + undefined, + nowMs, + ), + ).toBe(false); + }); + + it("accepts the previous root secret only during an explicit rotation window", () => { + const previous = `Bearer ${deriveWorkflowRunnerCapability( + "old-secret", + runId, + attemptId, + expiresAtMs, + )}`; + expect( + isWorkflowRunnerCapabilityValid( + previous, + runId, + attemptId, + "new-secret", + "old-secret", + nowMs, + ), + ).toBe(true); + expect( + isWorkflowRunnerCapabilityValid(previous, runId, attemptId, "new-secret", undefined, nowMs), + ).toBe(false); + }); + + it("rejects an expired capability before signature comparison", () => { + const expired = deriveWorkflowRunnerCapability("primary-secret", runId, attemptId, nowMs - 1); + expect( + isWorkflowRunnerCapabilityValid( + `Bearer ${expired}`, + runId, + attemptId, + "primary-secret", + undefined, + nowMs, + ), + ).toBe(false); + }); + + it("rejects missing, malformed, and prefix-collision authorization values", () => { + const capability = deriveWorkflowRunnerCapability( + "primary-secret", + runId, + attemptId, + expiresAtMs, + ); + for (const authorization of [ + undefined, + capability, + `Basic ${capability}`, + `Bearer ${capability}extra`, + `Bearer ${capability.slice(0, -1)}`, + ]) { + expect( + isWorkflowRunnerCapabilityValid( + authorization, + runId, + attemptId, + "primary-secret", + undefined, + nowMs, + ), + ).toBe(false); + } + }); + + it("parses only the two-segment runner path and rejects malformed encoding", () => { + const path = workflowRunnerPath(runId, attemptId); + expect(parseWorkflowRunnerPath(path)).toEqual({ runId, attemptId }); + expect(parseWorkflowRunnerPath(`${path}/extra`)).toBeNull(); + expect(parseWorkflowRunnerPath("/ws/workflow-runner/%E0%A4%A/value")).toBeNull(); + expect(parseWorkflowRunnerPath("/ws/workflow-runner//value")).toBeNull(); + expect(parseWorkflowRunnerPath("/ws/workflow-runner/%2F/value")).toEqual({ + runId: "/", + attemptId: "value", + }); + }); + + it("builds a path-only URL without inherited credentials, query, or fragment", () => { + expect( + workflowRunnerUrl("wss://controller.example/base?secret=query#fragment", runId, attemptId), + ).toBe(`wss://controller.example/ws/workflow-runner/${runId}/${attemptId}`); + }); +}); diff --git a/test/orchestrator/workflow-runner-controller.test.ts b/test/orchestrator/workflow-runner-controller.test.ts new file mode 100644 index 00000000..11af6835 --- /dev/null +++ b/test/orchestrator/workflow-runner-controller.test.ts @@ -0,0 +1,886 @@ +import { beforeEach, describe, expect, it, mock } from "bun:test"; + +import type { WorkflowRunnerClientMessage } from "../../src/shared/workflow-runner-messages"; +import { waitFor } from "../utils/assertions"; + +class TestStaleWorkflowAttemptError extends Error {} +class TestWorkflowRunnerCommandConflictError extends Error {} +class TestWorkflowRunnerOutputRejectedError extends Error {} + +const runId = crypto.randomUUID(); +const attemptId = crypto.randomUUID(); +const executionDeliveryId = crypto.randomUUID(); +const runnerId = `workflow-runner:${attemptId}`; +const attempt = { + runId, + attemptId, + executionDeliveryId, + runnerId, + workflowName: "review" as const, + attemptDeadlineAt: new Date("2026-08-23T04:10:00Z"), +}; + +const getWorkflowRunnerRegistrationState = mock(() => + Promise.resolve({ + state: "ready" as const, + attempt, + payloadIssuedAt: null, + tokenExpiresAt: null, + }), +); +const renewWorkflowAttempts = mock(() => + Promise.resolve({ renewedAttemptIds: [attemptId], lostAttemptIds: [] }), +); +const prepareWorkflowRunnerPayload = mock(() => + Promise.resolve({ + context: {}, + installationToken: "installation-token", + installationTokenExpiresAt: "2026-08-23T04:00:00Z", + attemptDeadlineAt: "2026-08-23T04:10:00Z", + workflowRun: { runId, workflowName: "review" as const }, + }), +); +const prepareWorkflowRunnerControllerOctokit = mock(() => Promise.resolve({})); +const recordWorkflowRunnerPayloadIssued = mock(() => Promise.resolve(true)); +const octokitRequest = mock(() => Promise.resolve()); +const revokeInstallationToken = mock((octokit: { request: typeof octokitRequest }) => { + void octokit.request("DELETE /installation/token"); + return Promise.resolve(true); +}); +const revokeInstallationTokenValue = mock(() => { + void octokitRequest("DELETE /installation/token"); + return Promise.resolve(true); +}); +const storeWorkflowRunnerResult = mock(() => Promise.resolve("stored" as const)); +const processWorkflowRunnerResult = mock(() => Promise.resolve()); +const cleanupWorkflowRunnerAttempt = mock(() => Promise.resolve()); +const loggerError = mock(() => undefined); +const assertCurrentWorkflowAttempt = mock(() => Promise.resolve()); +const setState = mock(() => Promise.resolve({ tracking_comment_id: null as number | null })); +const commitAttemptHandOffChild = mock(() => Promise.resolve({ id: crypto.randomUUID() })); +const findById = mock(() => Promise.resolve(null as Record | null)); +const recordWorkflowExecution = mock(() => Promise.resolve()); +const publishWorkflowRunById = mock(() => Promise.resolve()); +const getByName = mock(() => ({ steps: [] as string[] })); +const transactionQuery = mock(() => Promise.resolve([{ delivery_id: executionDeliveryId }])); +const begin = mock( + (callback: (tx: typeof transactionQuery) => Promise): Promise => + callback(transactionQuery), +); +const findWorkflowRunnerCommandReceipt = mock(() => + Promise.resolve( + null as null | { + commandKind: "set-state" | "hand-off-child"; + request: unknown; + response: { trackingCommentId?: number; childRunId?: string }; + }, + ), +); +const insertWorkflowRunnerCommandReceipt = mock(() => Promise.resolve()); +const sanitizeWorkflowRunnerCommand = mock((command: unknown) => Promise.resolve(command)); +const sanitizeWorkflowRunnerResult = mock((payload: unknown) => Promise.resolve(payload)); + +void mock.module("../../src/config", () => ({ + config: { + botAppLogin: "test-bot", + heartbeatIntervalMs: 1_000, + heartbeatTimeoutMs: 3_000, + }, +})); +void mock.module("../../src/db", () => ({ + requireDb: (): { begin: typeof begin } => ({ begin }), +})); +void mock.module("../../src/logger", () => ({ + logger: { + info: mock(() => undefined), + warn: mock(() => undefined), + error: loggerError, + debug: mock(() => undefined), + }, +})); +void mock.module("octokit", () => ({ + Octokit: function MockOctokit(this: unknown): unknown { + return { request: octokitRequest }; + }, +})); +void mock.module("../../src/orchestrator/workflow-runner-output", () => ({ + sanitizeWorkflowRunnerCommand, + sanitizeWorkflowRunnerResult, + WorkflowRunnerOutputRejectedError: TestWorkflowRunnerOutputRejectedError, +})); +void mock.module("../../src/workflows/execution-row", () => ({ + recordWorkflowExecution, +})); +void mock.module("../../src/workflows/registry", () => ({ + getByName, +})); +void mock.module("../../src/workflows/runs-store", () => ({ + assertCurrentWorkflowAttempt, + commitAttemptHandOffChild, + findById, + renewWorkflowAttempts, + StaleWorkflowAttemptError: TestStaleWorkflowAttemptError, +})); +void mock.module("../../src/workflows/tracking-mirror", () => ({ + setState, +})); +void mock.module("../../src/workflows/dispatch-outbox", () => ({ + publishWorkflowRunById, + publishPendingWorkflowRuns: mock(() => Promise.resolve(0)), +})); +void mock.module("../../src/orchestrator/workflow-runner-payload", () => ({ + prepareWorkflowRunnerControllerOctokit, + prepareWorkflowRunnerPayload, +})); +void mock.module("../../src/orchestrator/installation-token", () => ({ + revokeInstallationToken, + revokeInstallationTokenValue, +})); +void mock.module("../../src/orchestrator/workflow-runner-result", () => ({ + cleanupWorkflowRunnerAttempt, + processWorkflowRunnerResult, +})); +void mock.module("../../src/orchestrator/workflow-runner-store", () => ({ + assertMatchingWorkflowRunnerCommand: mock(() => undefined), + findWorkflowRunnerCommandReceipt, + getWorkflowRunnerRegistrationState, + insertWorkflowRunnerCommandReceipt, + recordWorkflowRunnerPayloadIssued, + storeWorkflowRunnerResult, + WorkflowRunnerCommandConflictError: TestWorkflowRunnerCommandConflictError, +})); + +const { + getWorkflowRunnerConnection, + handleWorkflowRunnerClose, + handleWorkflowRunnerMessage, + handleWorkflowRunnerOpen, + resetWorkflowRunnerControllerForTests, +} = await import("../../src/orchestrator/workflow-runner-controller"); + +interface CloseCall { + readonly code: number; + readonly reason: string; +} + +class FakeSocket { + readonly data = { + authenticated: true, + remoteAddr: "127.0.0.1", + daemonId: undefined, + sessionId: undefined, + kind: "workflow-runner" as const, + runnerRunId: runId, + runnerAttemptId: attemptId, + runnerRegistered: false, + }; + readonly messages: unknown[] = []; + readonly closes: CloseCall[] = []; + onSend?: (message: unknown) => void; + sendTextResult = 1; + + sendText(value: string): number { + const message = JSON.parse(value) as unknown; + this.messages.push(message); + this.onSend?.(message); + return this.sendTextResult; + } + + close(code: number, reason: string): void { + this.closes.push({ code, reason }); + } +} + +function mockReadyWithPayloadReceipt(): void { + getWorkflowRunnerRegistrationState.mockResolvedValueOnce({ + state: "ready", + attempt, + payloadIssuedAt: new Date("2026-08-23T03:00:00Z"), + tokenExpiresAt: new Date("2026-08-23T04:00:00Z"), + }); +} + +function registerMessage(needsJob = true): WorkflowRunnerClientMessage { + return { + type: "workflow-runner:register", + id: crypto.randomUUID(), + timestamp: Date.now(), + payload: { + runId, + attemptId, + protocolVersion: "1.1.0", + appVersion: "test", + needsJob, + }, + }; +} + +function resultMessage(): WorkflowRunnerClientMessage { + return { + type: "workflow-runner:result", + id: crypto.randomUUID(), + timestamp: Date.now(), + payload: { + runId, + attemptId, + result: { status: "succeeded", state: { phase: "complete" } }, + durationMs: 50, + }, + }; +} + +function heartbeatMessage(): Extract< + WorkflowRunnerClientMessage, + { type: "workflow-runner:heartbeat" } +> { + return { + type: "workflow-runner:heartbeat", + id: crypto.randomUUID(), + timestamp: Date.now(), + payload: { runId, attemptId }, + }; +} + +function setStateMessage( + id = crypto.randomUUID(), +): Extract { + return { + type: "workflow-runner:command", + id, + timestamp: Date.now(), + payload: { + runId, + attemptId, + command: { + type: "set-state", + patch: { phase: "working" }, + humanMessage: "Still working.", + }, + }, + }; +} + +function handOffMessage( + id = crypto.randomUUID(), +): Extract { + return { + type: "workflow-runner:command", + id, + timestamp: Date.now(), + payload: { + runId, + attemptId, + command: { + type: "hand-off-child", + workflowName: "triage", + target: { type: "pr", owner: "owner", repo: "repo", number: 42 }, + parentStepIndex: 0, + state: { phase: "triage" }, + humanMessage: "Handing off to triage.", + }, + }, + }; +} + +async function waitUntil(predicate: () => boolean): Promise { + await waitFor(predicate); + expect(predicate()).toBe(true); +} + +async function register(socket: FakeSocket, needsJob = true): Promise { + handleWorkflowRunnerOpen(socket as never); + handleWorkflowRunnerMessage(socket as never, registerMessage(needsJob)); + await waitUntil(() => + socket.messages.some( + (message) => + (message as { type?: string; payload?: { state?: string } }).type === + "workflow-runner:registered" && + (message as { payload?: { state?: string } }).payload?.state === "ready", + ), + ); +} + +function deferred(): { promise: Promise; resolve: () => void } { + let resolve!: () => void; + const promise = new Promise((done) => { + resolve = done; + }); + return { promise, resolve }; +} + +async function storedAfter(gate: ReturnType): Promise<"stored"> { + await gate.promise; + return "stored"; +} + +function hasMessage(socket: FakeSocket, type: string): boolean { + return socket.messages.some((message) => (message as { type?: string }).type === type); +} + +async function expectRejectedResultPolicyClose(error: Error): Promise { + resetWorkflowRunnerControllerForTests(); + const socket = new FakeSocket(); + await register(socket); + storeWorkflowRunnerResult.mockRejectedValueOnce(error); + + handleWorkflowRunnerMessage(socket as never, resultMessage()); + await waitUntil(() => socket.closes.length > 0); + expect(socket.closes.at(-1)).toEqual({ + code: 1008, + reason: "workflow runner result rejected", + }); +} + +describe("workflow runner controller", () => { + beforeEach(() => { + resetWorkflowRunnerControllerForTests(); + getWorkflowRunnerRegistrationState.mockReset(); + getWorkflowRunnerRegistrationState.mockImplementation(() => + Promise.resolve({ + state: "ready" as const, + attempt, + payloadIssuedAt: null, + tokenExpiresAt: null, + }), + ); + renewWorkflowAttempts.mockReset(); + renewWorkflowAttempts.mockImplementation(() => + Promise.resolve({ renewedAttemptIds: [attemptId], lostAttemptIds: [] }), + ); + prepareWorkflowRunnerPayload.mockClear(); + prepareWorkflowRunnerControllerOctokit.mockClear(); + recordWorkflowRunnerPayloadIssued.mockReset(); + recordWorkflowRunnerPayloadIssued.mockResolvedValue(true); + octokitRequest.mockClear(); + revokeInstallationToken.mockClear(); + revokeInstallationTokenValue.mockClear(); + storeWorkflowRunnerResult.mockReset(); + storeWorkflowRunnerResult.mockImplementation(() => Promise.resolve("stored" as const)); + processWorkflowRunnerResult.mockReset(); + processWorkflowRunnerResult.mockImplementation(() => Promise.resolve()); + cleanupWorkflowRunnerAttempt.mockClear(); + loggerError.mockClear(); + assertCurrentWorkflowAttempt.mockReset(); + assertCurrentWorkflowAttempt.mockResolvedValue(); + setState.mockReset(); + setState.mockResolvedValue({ tracking_comment_id: null }); + findWorkflowRunnerCommandReceipt.mockReset(); + findWorkflowRunnerCommandReceipt.mockResolvedValue(null); + insertWorkflowRunnerCommandReceipt.mockReset(); + insertWorkflowRunnerCommandReceipt.mockResolvedValue(); + commitAttemptHandOffChild.mockReset(); + commitAttemptHandOffChild.mockImplementation((_attempt, _state, input) => + Promise.resolve({ id: (input as { childRunId: string }).childRunId }), + ); + findById.mockReset(); + findById.mockResolvedValue(null); + recordWorkflowExecution.mockReset(); + recordWorkflowExecution.mockResolvedValue(); + publishWorkflowRunById.mockReset(); + publishWorkflowRunById.mockResolvedValue(); + getByName.mockReset(); + getByName.mockReturnValue({ steps: [] }); + transactionQuery.mockReset(); + transactionQuery.mockResolvedValue([{ delivery_id: executionDeliveryId }]); + begin.mockClear(); + sanitizeWorkflowRunnerCommand.mockReset(); + sanitizeWorkflowRunnerCommand.mockImplementation((command) => Promise.resolve(command)); + sanitizeWorkflowRunnerResult.mockReset(); + sanitizeWorkflowRunnerResult.mockImplementation((payload) => Promise.resolve(payload)); + }); + + it("sends ready for the current registration and policy-closes a stale registration", async () => { + const current = new FakeSocket(); + await register(current); + expect(getWorkflowRunnerConnection(attemptId)).toBe(current as never); + + resetWorkflowRunnerControllerForTests(); + getWorkflowRunnerRegistrationState.mockResolvedValueOnce({ state: "invalid" }); + prepareWorkflowRunnerPayload.mockClear(); + const stale = new FakeSocket(); + handleWorkflowRunnerOpen(stale as never); + handleWorkflowRunnerMessage(stale as never, registerMessage()); + await waitUntil(() => stale.closes.length === 1); + expect(stale.closes).toEqual([{ code: 1008, reason: "stale workflow runner attempt" }]); + expect(prepareWorkflowRunnerPayload).not.toHaveBeenCalled(); + }); + + it("supersedes the old socket on reconnect", async () => { + const oldSocket = new FakeSocket(); + const newSocket = new FakeSocket(); + await register(oldSocket); + getWorkflowRunnerRegistrationState.mockResolvedValueOnce({ + state: "ready", + attempt, + payloadIssuedAt: new Date("2026-08-23T03:00:00Z"), + tokenExpiresAt: new Date("2026-08-23T04:00:00Z"), + }); + await register(newSocket, false); + + expect(oldSocket.closes).toContainEqual({ + code: 4002, + reason: "superseded by runner reconnect", + }); + expect(getWorkflowRunnerConnection(attemptId)).toBe(newSocket as never); + const ready = newSocket.messages.find( + (message) => (message as { type?: string }).type === "workflow-runner:registered", + ) as { payload: Record }; + expect(ready.payload["job"]).toBeUndefined(); + expect(prepareWorkflowRunnerPayload).toHaveBeenCalledTimes(1); + expect(prepareWorkflowRunnerControllerOctokit).toHaveBeenCalledTimes(1); + }); + + it("does not reissue a runner payload after its durable receipt exists", async () => { + mockReadyWithPayloadReceipt(); + const socket = new FakeSocket(); + handleWorkflowRunnerOpen(socket as never); + + handleWorkflowRunnerMessage(socket as never, registerMessage(true)); + await waitUntil(() => socket.closes.length === 1); + + expect(socket.closes[0]).toEqual({ + code: 1008, + reason: "workflow runner payload was already issued", + }); + expect(prepareWorkflowRunnerPayload).not.toHaveBeenCalled(); + expect(recordWorkflowRunnerPayloadIssued).not.toHaveBeenCalled(); + }); + + it("revokes a prepared token when the durable delivery receipt loses the race", async () => { + recordWorkflowRunnerPayloadIssued.mockResolvedValueOnce(false); + const socket = new FakeSocket(); + handleWorkflowRunnerOpen(socket as never); + + handleWorkflowRunnerMessage(socket as never, registerMessage(true)); + await waitUntil(() => socket.closes.length === 1); + + expect(socket.closes[0]).toEqual({ + code: 1008, + reason: "workflow runner payload delivery was rejected", + }); + expect(octokitRequest).toHaveBeenCalledWith("DELETE /installation/token"); + expect(socket.messages).toHaveLength(0); + }); + + it("revokes and never reissues a prepared token when the post-payload renewal loses the lease", async () => { + renewWorkflowAttempts + .mockResolvedValueOnce({ renewedAttemptIds: [attemptId], lostAttemptIds: [] }) + .mockResolvedValueOnce({ renewedAttemptIds: [], lostAttemptIds: [attemptId] }); + const socket = new FakeSocket(); + handleWorkflowRunnerOpen(socket as never); + + handleWorkflowRunnerMessage(socket as never, registerMessage(true)); + await waitUntil(() => socket.closes.length === 1); + + expect(socket.closes[0]).toEqual({ + code: 1008, + reason: "workflow runner lease expired during payload preparation", + }); + expect(octokitRequest).toHaveBeenCalledWith("DELETE /installation/token"); + expect(getWorkflowRunnerConnection(attemptId)).toBeUndefined(); + expect(socket.data.runnerRegistered).toBe(false); + + mockReadyWithPayloadReceipt(); + const retry = new FakeSocket(); + handleWorkflowRunnerOpen(retry as never); + handleWorkflowRunnerMessage(retry as never, registerMessage(true)); + await waitUntil(() => retry.closes.length === 1); + expect(prepareWorkflowRunnerPayload).toHaveBeenCalledTimes(1); + }); + + it("revokes a prepared token when post-payload renewal throws", async () => { + renewWorkflowAttempts + .mockResolvedValueOnce({ renewedAttemptIds: [attemptId], lostAttemptIds: [] }) + .mockRejectedValueOnce(new Error("database unavailable")); + const socket = new FakeSocket(); + handleWorkflowRunnerOpen(socket as never); + + handleWorkflowRunnerMessage(socket as never, registerMessage(true)); + await waitUntil(() => socket.closes.length === 1); + + expect(socket.closes[0]).toEqual({ + code: 1011, + reason: "workflow runner registration failed", + }); + expect(revokeInstallationTokenValue).toHaveBeenCalledTimes(1); + expect(octokitRequest).toHaveBeenCalledWith("DELETE /installation/token"); + expect(getWorkflowRunnerConnection(attemptId)).toBeUndefined(); + }); + + it("revokes and never reissues a prepared token when the registered frame cannot be sent", async () => { + const socket = new FakeSocket(); + socket.sendTextResult = 0; + handleWorkflowRunnerOpen(socket as never); + + handleWorkflowRunnerMessage(socket as never, registerMessage(true)); + await waitUntil(() => octokitRequest.mock.calls.length === 1); + + expect(socket.closes).toContainEqual({ + code: 1011, + reason: "workflow runner control frame delivery failed", + }); + expect(octokitRequest).toHaveBeenCalledWith("DELETE /installation/token"); + expect(getWorkflowRunnerConnection(attemptId)).toBeUndefined(); + expect(socket.data.runnerRegistered).toBe(false); + + mockReadyWithPayloadReceipt(); + const retry = new FakeSocket(); + handleWorkflowRunnerOpen(retry as never); + handleWorkflowRunnerMessage(retry as never, registerMessage(true)); + await waitUntil(() => retry.closes.length === 1); + expect(prepareWorkflowRunnerPayload).toHaveBeenCalledTimes(1); + }); + + it("renews an exact heartbeat and policy-closes a lost lease", async () => { + const socket = new FakeSocket(); + await register(socket); + renewWorkflowAttempts.mockClear(); + + handleWorkflowRunnerMessage(socket as never, heartbeatMessage()); + await waitUntil(() => hasMessage(socket, "workflow-runner:heartbeat-ack")); + expect(renewWorkflowAttempts).toHaveBeenCalledWith(runnerId, [attemptId], 6_000); + + renewWorkflowAttempts.mockResolvedValueOnce({ + renewedAttemptIds: [], + lostAttemptIds: [attemptId], + }); + handleWorkflowRunnerMessage(socket as never, heartbeatMessage()); + await waitUntil(() => socket.closes.length > 0); + expect(socket.closes.at(-1)).toEqual({ + code: 1008, + reason: "workflow runner attempt fenced", + }); + }); + + it("fences, applies, and receipts a state command before replying, then replays it", async () => { + const socket = new FakeSocket(); + await register(socket); + const events: string[] = []; + assertCurrentWorkflowAttempt.mockImplementationOnce(() => { + events.push("fence"); + return Promise.resolve(); + }); + setState.mockImplementationOnce(() => { + events.push("effect"); + return Promise.resolve({ tracking_comment_id: 42 }); + }); + insertWorkflowRunnerCommandReceipt.mockImplementationOnce(() => { + events.push("receipt"); + return Promise.resolve(); + }); + socket.onSend = (message): void => { + if ((message as { type?: string }).type === "workflow-runner:command-result") { + events.push("reply"); + } + }; + const message = setStateMessage(); + + handleWorkflowRunnerMessage(socket as never, message); + await waitUntil(() => events.includes("reply")); + expect(events).toEqual(["fence", "effect", "receipt", "reply"]); + expect(setState).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + runId, + patch: { phase: "working" }, + humanMessage: "Still working.", + attempt: { runId, attemptId }, + }), + ); + + findWorkflowRunnerCommandReceipt.mockResolvedValueOnce({ + commandKind: "set-state", + request: message.payload.command, + response: { trackingCommentId: 42 }, + }); + handleWorkflowRunnerMessage(socket as never, message); + await waitUntil( + () => + socket.messages.filter( + (entry) => (entry as { type?: string }).type === "workflow-runner:command-result", + ).length === 2, + ); + expect(setState).toHaveBeenCalledTimes(1); + expect(insertWorkflowRunnerCommandReceipt).toHaveBeenCalledTimes(1); + }); + + it("commits and processes a hand-off before replying without projecting early state", async () => { + const shipAttempt = { ...attempt, workflowName: "ship" as const }; + getWorkflowRunnerRegistrationState.mockResolvedValueOnce({ + state: "ready" as const, + attempt: shipAttempt, + payloadIssuedAt: null, + tokenExpiresAt: null, + }); + const socket = new FakeSocket(); + await register(socket); + + const message = handOffMessage(); + const resultPayload = { + runId, + attemptId, + result: { + status: "handed-off" as const, + state: { phase: "triage", handedOffTo: message.id }, + humanMessage: "Handing off to triage.", + childRunId: message.id, + }, + durationMs: 0, + }; + const events: string[] = []; + assertCurrentWorkflowAttempt.mockImplementationOnce(() => { + events.push("fence"); + return Promise.resolve(); + }); + getByName.mockReturnValue({ steps: ["triage"] }); + findById.mockResolvedValue({ + workflow_name: "ship", + attempt_id: attemptId, + target_type: "pr", + target_owner: "owner", + target_repo: "repo", + target_number: 42, + }); + commitAttemptHandOffChild.mockImplementationOnce(() => { + events.push("commit-child"); + return Promise.resolve({ id: message.id }); + }); + recordWorkflowExecution.mockImplementationOnce(() => { + events.push("commit-execution"); + return Promise.resolve(); + }); + transactionQuery.mockImplementationOnce(() => { + events.push("commit-parent-result"); + return Promise.resolve([{ delivery_id: executionDeliveryId }]); + }); + insertWorkflowRunnerCommandReceipt.mockImplementationOnce(() => { + events.push("commit-receipt"); + return Promise.resolve(); + }); + publishWorkflowRunById.mockImplementationOnce(() => { + events.push("publish-child"); + return Promise.resolve(); + }); + getWorkflowRunnerRegistrationState.mockResolvedValueOnce({ + state: "result-pending" as const, + executionDeliveryId, + payload: resultPayload, + }); + processWorkflowRunnerResult.mockImplementationOnce(() => { + events.push("process-parent-result"); + return Promise.resolve(); + }); + socket.onSend = (sent): void => { + if ((sent as { type?: string }).type === "workflow-runner:command-result") { + events.push("reply"); + } + }; + + handleWorkflowRunnerMessage(socket as never, message); + await waitUntil(() => events.includes("reply")); + + expect(events).toEqual([ + "fence", + "commit-child", + "commit-execution", + "commit-parent-result", + "commit-receipt", + "publish-child", + "process-parent-result", + "reply", + ]); + expect(begin).toHaveBeenCalledTimes(1); + expect(setState).not.toHaveBeenCalled(); + expect(processWorkflowRunnerResult).toHaveBeenCalledWith({ + runId, + attemptId, + executionDeliveryId, + payload: resultPayload, + }); + }); + + it("waits for an in-flight command before revoking a reconnect-only token", async () => { + mockReadyWithPayloadReceipt(); + prepareWorkflowRunnerControllerOctokit.mockResolvedValueOnce({ + request: octokitRequest, + } as never); + const socket = new FakeSocket(); + await register(socket, false); + const command = deferred(); + setState.mockImplementationOnce(async () => { + await command.promise; + return { tracking_comment_id: null }; + }); + + handleWorkflowRunnerMessage(socket as never, setStateMessage()); + await waitUntil(() => setState.mock.calls.length === 1); + handleWorkflowRunnerClose(socket as never); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(revokeInstallationToken).not.toHaveBeenCalled(); + + command.resolve(); + await waitUntil(() => revokeInstallationToken.mock.calls.length === 1); + expect(octokitRequest).toHaveBeenCalledWith("DELETE /installation/token"); + }); + + it("does not let an old in-flight result evict a reconnected session", async () => { + const oldSocket = new FakeSocket(); + await register(oldSocket); + const stored = deferred(); + storeWorkflowRunnerResult.mockImplementationOnce(() => storedAfter(stored)); + + handleWorkflowRunnerMessage(oldSocket as never, resultMessage()); + await waitUntil(() => storeWorkflowRunnerResult.mock.calls.length === 1); + + const newSocket = new FakeSocket(); + await register(newSocket); + expect(getWorkflowRunnerConnection(attemptId)).toBe(newSocket as never); + + stored.resolve(); + await waitUntil(() => hasMessage(oldSocket, "workflow-runner:result-ack")); + expect(getWorkflowRunnerConnection(attemptId)).toBe(newSocket as never); + }); + + it("policy-closes stale and conflicting results with a fixed reason", async () => { + await expectRejectedResultPolicyClose(new TestStaleWorkflowAttemptError("stale detail")); + await expectRejectedResultPolicyClose( + new TestWorkflowRunnerCommandConflictError("conflicting detail"), + ); + }); + + it("keeps unexpected projection failures retryable", async () => { + const socket = new FakeSocket(); + await register(socket); + processWorkflowRunnerResult.mockRejectedValueOnce(new Error("temporary API failure")); + + handleWorkflowRunnerMessage(socket as never, resultMessage()); + await waitUntil(() => loggerError.mock.calls.length === 1); + expect(socket.closes.some((call) => call.code === 1008)).toBe(false); + expect( + socket.messages.some( + (message) => (message as { type?: string }).type === "workflow-runner:result-ack", + ), + ).toBe(false); + expect(getWorkflowRunnerConnection(attemptId)).toBe(socket as never); + }); + + it("stores and processes a result before acknowledging it", async () => { + const events: string[] = []; + const socket = new FakeSocket(); + socket.onSend = (message): void => { + if ((message as { type?: string }).type === "workflow-runner:result-ack") events.push("ack"); + }; + storeWorkflowRunnerResult.mockImplementationOnce(() => { + events.push("store"); + return Promise.resolve("stored"); + }); + processWorkflowRunnerResult.mockImplementationOnce(() => { + events.push("process"); + return Promise.resolve(); + }); + await register(socket); + + handleWorkflowRunnerMessage(socket as never, resultMessage()); + await waitUntil(() => events.includes("ack")); + expect(events).toEqual(["store", "process", "ack"]); + expect(processWorkflowRunnerResult.mock.calls[0]).toHaveLength(1); + }); + + it("serializes a result behind an earlier command for the same attempt", async () => { + const socket = new FakeSocket(); + await register(socket); + const command = deferred(); + setState.mockImplementationOnce(async () => { + await command.promise; + return { tracking_comment_id: null }; + }); + + handleWorkflowRunnerMessage(socket as never, setStateMessage()); + await waitUntil(() => setState.mock.calls.length === 1); + handleWorkflowRunnerMessage(socket as never, resultMessage()); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(storeWorkflowRunnerResult).not.toHaveBeenCalled(); + + command.resolve(); + await waitUntil(() => hasMessage(socket, "workflow-runner:result-ack")); + expect(storeWorkflowRunnerResult).toHaveBeenCalledTimes(1); + }); + + it("acknowledges the first stored result without rescanning replay bytes", async () => { + const socket = new FakeSocket(); + await register(socket); + const storedPayload = resultMessage().payload; + getWorkflowRunnerRegistrationState.mockResolvedValueOnce({ + state: "result-pending", + executionDeliveryId, + payload: storedPayload, + }); + const replay = resultMessage(); + replay.payload.result = { status: "failed", reason: "different replay bytes" }; + + handleWorkflowRunnerMessage(socket as never, replay); + await waitUntil(() => hasMessage(socket, "workflow-runner:result-ack")); + + expect(sanitizeWorkflowRunnerResult).not.toHaveBeenCalled(); + expect(storeWorkflowRunnerResult).not.toHaveBeenCalled(); + expect(processWorkflowRunnerResult).toHaveBeenCalledWith({ + runId, + attemptId, + executionDeliveryId, + payload: storedPayload, + }); + }); + + it("maps command sanitation rejection before any receipt or effect", async () => { + const socket = new FakeSocket(); + await register(socket); + const message = setStateMessage(); + sanitizeWorkflowRunnerCommand.mockRejectedValueOnce( + new TestWorkflowRunnerOutputRejectedError("credential-bearing command"), + ); + + handleWorkflowRunnerMessage(socket as never, message); + await waitUntil(() => + socket.messages.some( + (entry) => + (entry as { type?: string; payload?: { code?: string } }).type === + "workflow-runner:command-result" && + (entry as { payload?: { code?: string } }).payload?.code === "INVALID_COMMAND", + ), + ); + + expect(sanitizeWorkflowRunnerCommand).toHaveBeenCalledWith(message.payload.command); + expect(findWorkflowRunnerCommandReceipt).not.toHaveBeenCalled(); + expect(setState).not.toHaveBeenCalled(); + expect(commitAttemptHandOffChild).not.toHaveBeenCalled(); + expect(insertWorkflowRunnerCommandReceipt).not.toHaveBeenCalled(); + }); + + it("stores the sanitized result rather than the raw wire payload", async () => { + const socket = new FakeSocket(); + await register(socket); + const raw = resultMessage(); + const safe = { + ...raw.payload, + result: { + status: "failed" as const, + reason: "workflow runner output was rejected by credential policy", + humanMessage: "Workflow output was rejected by the credential safety boundary.", + }, + }; + sanitizeWorkflowRunnerResult.mockResolvedValueOnce(safe); + + handleWorkflowRunnerMessage(socket as never, raw); + await waitUntil(() => hasMessage(socket, "workflow-runner:result-ack")); + + expect(sanitizeWorkflowRunnerResult).toHaveBeenCalledTimes(1); + expect(sanitizeWorkflowRunnerResult).toHaveBeenCalledWith(raw.payload); + expect(storeWorkflowRunnerResult).toHaveBeenCalledWith(safe); + expect(processWorkflowRunnerResult).toHaveBeenCalledWith({ + runId, + attemptId, + executionDeliveryId, + payload: safe, + }); + }); +}); diff --git a/test/orchestrator/workflow-runner-dispatch.test.ts b/test/orchestrator/workflow-runner-dispatch.test.ts new file mode 100644 index 00000000..8094486c --- /dev/null +++ b/test/orchestrator/workflow-runner-dispatch.test.ts @@ -0,0 +1,170 @@ +import { beforeEach, describe, expect, it, mock } from "bun:test"; + +class TestEphemeralSpawnError extends Error { + constructor(readonly kind: string) { + super(kind); + } +} + +class TestWorkflowRunnerResourceError extends Error { + constructor( + readonly kind: "permanent" | "transient", + message: string, + ) { + super(message); + } +} + +const config = { + githubPersonalAccessToken: undefined as string | undefined, + daemonImage: `registry.example/runner@sha256:${"a".repeat(64)}` as string | undefined, + orchestratorPublicUrl: "wss://controller.example/ws" as string | undefined, + workflowRunnerCapabilitySecret: "runner-capability-root-secret" as string | undefined, + heartbeatTimeoutMs: 3_000, + maxConcurrentRequests: 2, + botAppLogin: "test-bot", +}; +const attemptId = crypto.randomUUID(); +const attempt = { + runId: crypto.randomUUID(), + attemptId, + runnerId: `workflow-runner:${attemptId}`, + executionDeliveryId: "delivery-16", + workflowName: "review" as const, + attemptDeadlineAt: new Date("2026-08-23T04:10:00Z"), +}; +const claimWorkflowRunnerAttempt = mock(() => + Promise.resolve({ outcome: "claimed" as const, attempt }), +); +const ensureCurrentWorkflowRunnerResources = mock(() => Promise.resolve("ready" as const)); +const failWorkflowRunnerAttempt = mock(() => Promise.resolve({ id: attempt.runId })); +const ensureWorkflowCascadeForOffer = mock(() => Promise.resolve("complete" as const)); +const notifyRunnerStartFailures = mock(() => Promise.resolve()); +const cleanupCurrentWorkflowRunnerResources = mock(() => Promise.resolve()); +const loggerInfo = mock(() => undefined); + +void mock.module("../../src/config", () => ({ config })); +void mock.module("../../src/k8s/ephemeral-daemon-spawner", () => ({ + EphemeralSpawnError: TestEphemeralSpawnError, +})); +void mock.module("../../src/k8s/workflow-runner-spawner", () => ({ + WorkflowRunnerResourceError: TestWorkflowRunnerResourceError, +})); +void mock.module("../../src/logger", () => ({ + logger: { + info: loggerInfo, + warn: mock(() => undefined), + error: mock(() => undefined), + debug: mock(() => undefined), + }, +})); +void mock.module("../../src/workflows/completion-reconciler", () => ({ + ensureWorkflowCascadeForOffer, +})); +void mock.module("../../src/orchestrator/workflow-expiry-notifier", () => ({ + notifyRunnerStartFailures, +})); +void mock.module("../../src/orchestrator/workflow-runner-capability", () => ({ + deriveWorkflowRunnerCapability: mock(() => "wfr1.capability"), +})); +void mock.module("../../src/orchestrator/workflow-runner-resources", () => ({ + cleanupCurrentWorkflowRunnerResources, + ensureCurrentWorkflowRunnerResources, +})); +void mock.module("../../src/orchestrator/workflow-runner-store", () => ({ + claimWorkflowRunnerAttempt, + failWorkflowRunnerAttempt, +})); + +const { dispatchWorkflowRunner } = await import("../../src/orchestrator/workflow-runner-dispatch"); + +const job = { + kind: "workflow-run" as const, + deliveryId: "delivery-16", + repoOwner: "acme", + repoName: "widgets", + entityNumber: 16, + isPR: false, + eventName: "issues", + triggerUsername: "maintainer", + labels: ["bot:review"], + triggerBodyPreview: "", + enqueuedAt: Date.now(), + retryCount: 0, + workflowRun: { runId: attempt.runId, workflowName: "review" as const }, +}; + +describe("isolated workflow runner dispatch", () => { + beforeEach(() => { + config.githubPersonalAccessToken = undefined; + config.daemonImage = `registry.example/runner@sha256:${"a".repeat(64)}`; + config.orchestratorPublicUrl = "wss://controller.example/ws"; + config.workflowRunnerCapabilitySecret = "runner-capability-root-secret"; + claimWorkflowRunnerAttempt.mockReset(); + claimWorkflowRunnerAttempt.mockResolvedValue({ outcome: "claimed", attempt }); + ensureCurrentWorkflowRunnerResources.mockReset(); + ensureCurrentWorkflowRunnerResources.mockResolvedValue("ready"); + failWorkflowRunnerAttempt.mockClear(); + ensureWorkflowCascadeForOffer.mockClear(); + notifyRunnerStartFailures.mockClear(); + cleanupCurrentWorkflowRunnerResources.mockClear(); + loggerInfo.mockClear(); + }); + + it("does not create Kubernetes resources when admission is at capacity", async () => { + claimWorkflowRunnerAttempt.mockResolvedValueOnce({ outcome: "capacity" }); + expect(await dispatchWorkflowRunner(job)).toBe("capacity"); + expect(ensureCurrentWorkflowRunnerResources).not.toHaveBeenCalled(); + }); + + it("creates the exact claimed attempt with its derived capability", async () => { + expect(await dispatchWorkflowRunner(job)).toBe("accepted"); + expect(ensureCurrentWorkflowRunnerResources).toHaveBeenCalledWith({ + attempt, + capability: "wfr1.capability", + image: `registry.example/runner@sha256:${"a".repeat(64)}`, + orchestratorUrl: "wss://controller.example/ws", + }); + expect(loggerInfo).toHaveBeenCalledWith( + expect.objectContaining({ + event: "workflow.run.running", + runId: attempt.runId, + target: { type: "issue", owner: "acme", repo: "widgets", number: 16 }, + }), + "Workflow run running", + ); + }); + + it("terminalizes a permanent runner configuration failure", async () => { + config.githubPersonalAccessToken = "global-pat"; + expect(await dispatchWorkflowRunner(job)).toBe("accepted"); + expect(failWorkflowRunnerAttempt).toHaveBeenCalledWith( + attempt, + expect.stringContaining("PAT mode"), + ); + expect(ensureWorkflowCascadeForOffer).toHaveBeenCalledWith( + attempt.attemptId, + expect.anything(), + ); + expect(notifyRunnerStartFailures).toHaveBeenCalledTimes(1); + expect(cleanupCurrentWorkflowRunnerResources).toHaveBeenCalledWith(attempt); + }); + + it("terminalizes a missing dedicated capability secret", async () => { + config.workflowRunnerCapabilitySecret = undefined; + expect(await dispatchWorkflowRunner(job)).toBe("accepted"); + expect(failWorkflowRunnerAttempt).toHaveBeenCalledWith( + attempt, + expect.stringContaining("WORKFLOW_RUNNER_CAPABILITY_SECRET"), + ); + }); + + it("retains a claimed attempt for reconciliation after a transient API failure", async () => { + ensureCurrentWorkflowRunnerResources.mockRejectedValueOnce( + new TestWorkflowRunnerResourceError("transient", "Kubernetes unavailable"), + ); + expect(await dispatchWorkflowRunner(job)).toBe("accepted"); + expect(failWorkflowRunnerAttempt).not.toHaveBeenCalled(); + expect(cleanupCurrentWorkflowRunnerResources).not.toHaveBeenCalled(); + }); +}); diff --git a/test/orchestrator/workflow-runner-output.test.ts b/test/orchestrator/workflow-runner-output.test.ts new file mode 100644 index 00000000..7db22f63 --- /dev/null +++ b/test/orchestrator/workflow-runner-output.test.ts @@ -0,0 +1,215 @@ +import { afterAll, beforeEach, describe, expect, it, mock } from "bun:test"; + +import { expectToReject } from "../utils/assertions"; + +const detectSecretsWithLlm = mock(() => + Promise.resolve({ containsSecret: false, matchCount: 0, kinds: [] }), +); +// The redacting entry point must stay unused here: it makes the model restate +// the payload, so its latency scales with body size and a fail-closed boundary +// turns that into a rejected run. +const scanForSecretsWithLlm = mock(() => + Promise.resolve({ containsSecret: false, redactedBody: "", matchCount: 0, kinds: [] }), +); +const config = { llmOutputScannerEnabled: true, llmOutputScannerTimeoutMs: 100 }; +const loggerWarn = mock(() => undefined); +const loggerError = mock(() => undefined); +const originalApiKey = process.env["ANTHROPIC_API_KEY"]; +process.env["ANTHROPIC_API_KEY"] = "controller-test-provider-secret"; + +void mock.module("../../src/config", () => ({ + config, +})); +void mock.module("../../src/logger", () => ({ + logger: { + info: mock(() => undefined), + warn: loggerWarn, + error: loggerError, + debug: mock(() => undefined), + }, +})); +void mock.module("../../src/utils/llm-output-scanner", () => ({ + detectSecretsWithLlm, + scanForSecretsWithLlm, +})); + +const { sanitizeWorkflowRunnerCommand, sanitizeWorkflowRunnerResult } = + await import("../../src/orchestrator/workflow-runner-output"); + +describe("controller workflow runner output boundary", () => { + beforeEach(() => { + config.llmOutputScannerEnabled = true; + detectSecretsWithLlm.mockReset(); + detectSecretsWithLlm.mockResolvedValue({ + containsSecret: false, + matchCount: 0, + kinds: [], + }); + scanForSecretsWithLlm.mockReset(); + loggerWarn.mockClear(); + loggerError.mockClear(); + }); + + it("rejects a command when deterministic redaction detects a credential", async () => { + const token = `ghs_${"a".repeat(36)}`; + await expectToReject( + sanitizeWorkflowRunnerCommand({ + type: "set-state", + patch: { report: `before ${token} after` }, + humanMessage: `done ${token}`, + }), + "credential policy", + ); + }); + + it("rejects a command when the encoded-secret scanner detects a credential", async () => { + detectSecretsWithLlm.mockResolvedValueOnce({ + containsSecret: true, + matchCount: 1, + kinds: ["ENCODED_SECRET"], + }); + + await expectToReject( + sanitizeWorkflowRunnerCommand({ + type: "set-state", + patch: { report: "encoded credential" }, + humanMessage: "done", + }), + "credential policy", + ); + }); + + it("converts a rejected terminal result into a fixed safe failure", async () => { + detectSecretsWithLlm.mockResolvedValueOnce({ + containsSecret: true, + matchCount: 1, + kinds: ["AWS_SECRET_KEY"], + }); + + const payload = await sanitizeWorkflowRunnerResult({ + runId: crypto.randomUUID(), + attemptId: crypto.randomUUID(), + durationMs: 10, + result: { + status: "succeeded", + state: { report: "opaque credential" }, + humanMessage: "complete", + }, + }); + + expect(payload.result).toEqual({ + status: "failed", + reason: "workflow runner output was rejected by credential policy", + humanMessage: "Workflow output was rejected by the credential safety boundary.", + }); + }); + + for (const secret of [ + `ghs_${"b".repeat(36)}`, + "AKIAIOSFODNN7EXAMPLE", + `sk-ant-api03-${"c".repeat(80)}`, + `wfr1.2000000000000.${"d".repeat(43)}`, + ]) { + it(`rejects a credential-bearing property name with deterministic scanning: ${secret.slice(0, 4)}`, async () => { + config.llmOutputScannerEnabled = false; + await expectToReject( + sanitizeWorkflowRunnerCommand({ + type: "set-state", + patch: { nested: { [secret]: "x" } }, + humanMessage: "done", + }), + "credential policy", + ); + + const result = await sanitizeWorkflowRunnerResult({ + runId: crypto.randomUUID(), + attemptId: crypto.randomUUID(), + durationMs: 1, + result: { status: "succeeded", state: { nested: { [secret]: "x" } } }, + }); + expect(result.result.status).toBe("failed"); + }); + } + + it("keeps property-name rejection when the encoded scanner throws", async () => { + const token = `ghs_${"e".repeat(36)}`; + detectSecretsWithLlm.mockRejectedValueOnce(new Error("scanner unavailable")); + const result = await sanitizeWorkflowRunnerResult({ + runId: crypto.randomUUID(), + attemptId: crypto.randomUUID(), + durationMs: 1, + result: { status: "succeeded", state: { [token]: "x" } }, + }); + expect(result.result.status).toBe("failed"); + }); + + it("fails closed when encoded-secret scanning is unavailable", async () => { + detectSecretsWithLlm.mockRejectedValueOnce(new Error("scanner unavailable")); + const result = await sanitizeWorkflowRunnerResult({ + runId: crypto.randomUUID(), + attemptId: crypto.randomUUID(), + durationMs: 1, + result: { status: "succeeded", state: { report: "safe" } }, + }); + expect(result.result.status).toBe("failed"); + expect(loggerError).toHaveBeenCalledWith( + expect.objectContaining({ + event: "workflow_runner_output_scan_unavailable", + scanner: "llm", + callsite: "workflow-runner.result", + }), + "Workflow runner output scanner failed; rejecting output", + ); + }); + + it("fails closed when encoded-secret scanning is disabled", async () => { + config.llmOutputScannerEnabled = false; + await expectToReject( + sanitizeWorkflowRunnerCommand({ + type: "set-state", + patch: { report: "safe" }, + humanMessage: "done", + }), + "credential policy", + ); + const result = await sanitizeWorkflowRunnerResult({ + runId: crypto.randomUUID(), + attemptId: crypto.randomUUID(), + durationMs: 1, + result: { status: "succeeded", state: { report: "safe" } }, + }); + expect(result.result.status).toBe("failed"); + }); + + it("scans with the detect-only entry point so latency stays flat in payload size", async () => { + const payload = await sanitizeWorkflowRunnerResult({ + runId: crypto.randomUUID(), + attemptId: crypto.randomUUID(), + durationMs: 1, + result: { status: "succeeded", state: { report: "x".repeat(50_000) } }, + }); + + expect(payload.result.status).toBe("succeeded"); + expect(detectSecretsWithLlm).toHaveBeenCalledTimes(1); + // Echoing a 50 KB payload back is what exhausted the per-call budget and + // turned a completed run into a credential-policy rejection. + expect(scanForSecretsWithLlm).not.toHaveBeenCalled(); + }); + + it("rejects common reversible encodings of configured credentials", async () => { + const encoded = Buffer.from("controller-test-provider-secret", "utf8").toString("base64"); + await expectToReject( + sanitizeWorkflowRunnerCommand({ + type: "set-state", + patch: { report: encoded }, + humanMessage: "done", + }), + "credential policy", + ); + }); +}); + +afterAll(() => { + if (originalApiKey === undefined) delete process.env["ANTHROPIC_API_KEY"]; + else process.env["ANTHROPIC_API_KEY"] = originalApiKey; +}); diff --git a/test/orchestrator/workflow-runner-payload.test.ts b/test/orchestrator/workflow-runner-payload.test.ts new file mode 100644 index 00000000..2946b427 --- /dev/null +++ b/test/orchestrator/workflow-runner-payload.test.ts @@ -0,0 +1,314 @@ +import { beforeEach, describe, expect, it, mock } from "bun:test"; + +import { expectToReject } from "../utils/assertions"; + +const context = { + owner: "acme", + repo: "widgets", + entityNumber: 16, + isPR: false, + deliveryId: "delivery-16", +}; +const db = mock(() => Promise.resolve([{ context_json: context }])); +const getRepoInstallation = mock(() => Promise.resolve({ data: { id: 123 } })); +const TOKEN_EXPIRES_AT = new Date(Date.now() + 60 * 60 * 1_000).toISOString(); +const ATTEMPT_DEADLINE_AT = new Date(Date.now() + 70 * 60 * 1_000); +const revokeToken = mock(() => Promise.resolve()); +const revokeInstallationToken = mock((octokit: { request: typeof revokeToken }) => { + void octokit.request("DELETE /installation/token"); + return Promise.resolve(true); +}); +const mintInstallationToken = mock(() => + Promise.resolve({ + octokit: { request: revokeToken }, + token: "ghs_scoped", + expiresAt: TOKEN_EXPIRES_AT, + }), +); +const loggerWarn = mock(() => undefined); +const loggerInfo = mock(() => undefined); +const repoMemoryRows = Array.from({ length: 51 }, (_, index) => ({ + id: crypto.randomUUID(), + category: "architecture", + content: `memory-${String(index)}`, + pinned: false, +})); +const invalidMemory = { + id: "not-a-uuid", + category: "secret", + content: "must be dropped", + pinned: false, +}; +const firstMemory = repoMemoryRows[0]; +if (firstMemory === undefined) throw new Error("Expected repo memory fixture"); +const getRepoMemory = mock(() => + Promise.resolve([firstMemory, invalidMemory, ...repoMemoryRows.slice(1)]), +); +const findLatestForTarget = mock(() => Promise.resolve(null)); +const findLatestSucceededForTarget = mock(() => Promise.resolve(null)); +const config = { + appId: "test-app", + privateKey: "test-key", + githubPersonalAccessToken: undefined as string | undefined, + reviewLearningsEnabled: false, + reviewLearningsRagEnabled: false, + agentMaxTurns: undefined as number | undefined, + defaultMaxTurns: 100, + repoConfigFile: ".github/chrisleekr-bot.yml", +}; +const loadRepoPolicy = mock(() => + Promise.resolve({ reviewLearnings: { enabled: false }, warning: undefined }), +); +const policyForWorkflow = mock(() => ({}) as Record); +const toAgentPolicy = mock(() => undefined as undefined | Record); +const mergeAttemptState = mock(() => Promise.resolve()); + +function workflowAttempt( + workflowName: "implement" | "ship" = "implement", + attemptDeadlineAt = ATTEMPT_DEADLINE_AT, +) { + const attemptId = crypto.randomUUID(); + return { + runId: crypto.randomUUID(), + attemptId, + runnerId: `workflow-runner:${attemptId}`, + executionDeliveryId: "delivery-16", + workflowName, + attemptDeadlineAt, + }; +} + +void mock.module("../../src/config", () => ({ config })); +void mock.module("../../src/db", () => ({ requireDb: (): typeof db => db })); +void mock.module("../../src/logger", () => ({ + logger: { + info: loggerInfo, + warn: loggerWarn, + error: mock(() => undefined), + debug: mock(() => undefined), + }, +})); +void mock.module("octokit", () => ({ + App: function MockApp(): unknown { + return { octokit: { rest: { apps: { getRepoInstallation } } } }; + }, +})); +void mock.module("../../src/repo-config/effective", () => ({ + loadRepoPolicy, + policyForWorkflow, + toAgentPolicy, +})); +void mock.module("../../src/workflows/runs-store", () => ({ + findLatestForTarget, + findLatestSucceededForTarget, + mergeAttemptState, +})); +void mock.module("../../src/workflows/tracking-mirror", () => ({ + CONFIG_NOTICE_KEY: "configNotice", +})); +void mock.module("../../src/orchestrator/installation-token", () => ({ + mintInstallationToken, + revokeInstallationToken, +})); +void mock.module("../../src/orchestrator/repo-knowledge", () => ({ getRepoMemory })); +void mock.module("../../src/orchestrator/review-learnings", () => ({ + loadReviewLearnings: mock(() => Promise.resolve([])), + searchReviewLearningsByEmbedding: mock(() => Promise.resolve([])), +})); + +const { prepareWorkflowRunnerPayload } = + await import("../../src/orchestrator/workflow-runner-payload"); + +describe("workflow runner payload", () => { + beforeEach(() => { + config.githubPersonalAccessToken = undefined; + config.agentMaxTurns = undefined; + findLatestForTarget.mockReset(); + findLatestForTarget.mockResolvedValue(null); + findLatestSucceededForTarget.mockReset(); + findLatestSucceededForTarget.mockResolvedValue(null); + loadRepoPolicy.mockReset(); + loadRepoPolicy.mockResolvedValue({ reviewLearnings: { enabled: false }, warning: undefined }); + policyForWorkflow.mockReset(); + policyForWorkflow.mockReturnValue({}); + toAgentPolicy.mockReset(); + toAgentPolicy.mockReturnValue(undefined); + mergeAttemptState.mockClear(); + loggerInfo.mockClear(); + revokeToken.mockClear(); + revokeInstallationToken.mockClear(); + mintInstallationToken.mockReset(); + mintInstallationToken.mockResolvedValue({ + octokit: { request: revokeToken }, + token: "ghs_scoped", + expiresAt: TOKEN_EXPIRES_AT, + }); + }); + + it("propagates the scoped installation token expiry", async () => { + const attempt = { + runId: "11111111-1111-4111-8111-111111111111", + attemptId: "22222222-2222-4222-8222-222222222222", + runnerId: "workflow-runner:22222222-2222-4222-8222-222222222222", + executionDeliveryId: "delivery-16", + workflowName: "implement" as const, + attemptDeadlineAt: ATTEMPT_DEADLINE_AT, + }; + + const payload = await prepareWorkflowRunnerPayload(attempt); + + expect(mintInstallationToken).toHaveBeenCalledWith( + expect.objectContaining({ installationId: 123, repositoryName: "widgets" }), + ); + expect(payload.installationToken).toBe("ghs_scoped"); + expect(payload.installationTokenExpiresAt).toBe(TOKEN_EXPIRES_AT); + expect(payload.attemptDeadlineAt).toBe(ATTEMPT_DEADLINE_AT.toISOString()); + expect(payload.repoMemory).toHaveLength(50); + expect(payload.repoMemory?.map((row) => row.id)).toEqual( + repoMemoryRows.slice(0, 50).map((row) => row.id), + ); + expect(loggerWarn).toHaveBeenCalledWith( + { owner: "acme", repo: "widgets", dropped: 1, omitted: 1 }, + "Filtered invalid or excess workflow runner repo memory", + ); + }); + + it("rejects a late payload request before minting a token", async () => { + await expectToReject( + prepareWorkflowRunnerPayload( + workflowAttempt("implement", new Date(Date.now() + 59 * 60 * 1_000)), + ), + "insufficient lifetime", + ); + + expect(mintInstallationToken).not.toHaveBeenCalled(); + }); + + it("revokes and rejects a token whose authoritative expiry crosses the attempt deadline", async () => { + const attemptDeadlineAt = new Date(Date.now() + 70 * 60 * 1_000); + mintInstallationToken.mockResolvedValueOnce({ + octokit: { request: revokeToken }, + token: "ghs_unsafe", + expiresAt: new Date(attemptDeadlineAt.getTime() + 1_000).toISOString(), + }); + + await expectToReject( + prepareWorkflowRunnerPayload(workflowAttempt("implement", attemptDeadlineAt)), + "expiry exceeds", + ); + + expect(revokeToken).toHaveBeenCalledWith("DELETE /installation/token"); + }); + + it("revokes the minted token when policy loading fails", async () => { + const policyError = new Error("repository policy unavailable"); + loadRepoPolicy.mockRejectedValueOnce(policyError); + + let caught: unknown; + try { + await prepareWorkflowRunnerPayload(workflowAttempt()); + } catch (err) { + caught = err; + } + expect(caught).toBe(policyError); + + expect(revokeInstallationToken).toHaveBeenCalledTimes(1); + expect(revokeToken).toHaveBeenCalledWith("DELETE /installation/token"); + }); + + it("revokes the minted token when the post-load state write fails", async () => { + loadRepoPolicy.mockResolvedValueOnce({ + reviewLearnings: { enabled: false }, + warning: "Repository policy was reduced.", + }); + toAgentPolicy.mockReturnValueOnce({ warning: "Repository policy was reduced." }); + mergeAttemptState.mockRejectedValueOnce(new Error("database unavailable")); + + await expectToReject(prepareWorkflowRunnerPayload(workflowAttempt()), "database unavailable"); + + expect(revokeInstallationToken).toHaveBeenCalledTimes(1); + }); + + it("projects only handler-consumed fields from prior workflow state", async () => { + findLatestSucceededForTarget.mockResolvedValueOnce({ + state: { plan: "## Approved plan", unrelated: "must-not-cross" }, + } as never); + const implementPayload = await prepareWorkflowRunnerPayload(workflowAttempt()); + expect(implementPayload.priorPlanState).toEqual({ plan: "## Approved plan" }); + + const createdAt = new Date("2026-08-23T03:00:00Z"); + findLatestForTarget.mockImplementation((name) => + Promise.resolve({ + id: crypto.randomUUID(), + status: "succeeded", + state: + name === "triage" + ? { recommendedNext: "plan", hidden: "drop" } + : name === "implement" + ? { pr_number: 42, branch: "drop" } + : { hidden: "drop" }, + created_at: createdAt, + } as never), + ); + const shipPayload = await prepareWorkflowRunnerPayload(workflowAttempt("ship")); + expect(shipPayload.shipStepRuns?.triage?.state).toEqual({ recommendedNext: "plan" }); + expect(shipPayload.shipStepRuns?.implement?.state).toEqual({ pr_number: 42 }); + expect(shipPayload.shipStepRuns?.review?.state).toEqual({}); + }); + + it("fails closed before token minting in PAT mode", async () => { + config.githubPersonalAccessToken = "global-pat"; + await expectToReject( + prepareWorkflowRunnerPayload(workflowAttempt()), + "do not support GITHUB_PERSONAL_ACCESS_TOKEN", + ); + }); + + it("applies max-turn precedence and strips review-only instructions", async () => { + const attempt = workflowAttempt(); + loadRepoPolicy.mockResolvedValueOnce({ + reviewLearnings: { enabled: false }, + warning: "Repository policy was reduced.", + }); + policyForWorkflow.mockReturnValueOnce({ maxTurns: 25 }); + toAgentPolicy.mockReturnValueOnce({ + model: "claude-test", + instructions: "Review only.", + pathFilters: ["vendor/**"], + warning: "Repository policy was reduced.", + }); + + const payload = await prepareWorkflowRunnerPayload(attempt); + + expect(payload.maxTurns).toBe(25); + expect(payload.policy).toEqual({ + model: "claude-test", + pathFilters: ["vendor/**"], + warning: "Repository policy was reduced.", + }); + expect(mergeAttemptState).toHaveBeenCalledWith( + { runId: attempt.runId, attemptId: attempt.attemptId }, + { + configNotice: + "Repository policy was reduced.\nReview scope reduced by `.github/chrisleekr-bot.yml`: files matching `vendor/**` are excluded.", + }, + ); + expect(loggerInfo).toHaveBeenCalledWith( + expect.objectContaining({ + event: "repo_config.policy_applied", + owner: "acme", + repo: "widgets", + deliveryId: "delivery-16", + workflow: "implement", + runId: attempt.runId, + attemptId: attempt.attemptId, + model: "claude-test", + maxTurns: 25, + pathFilterCount: 1, + hasInstructions: false, + warned: true, + }), + "Per-repo agent policy applied", + ); + }); +}); diff --git a/test/orchestrator/workflow-runner-reconciler.test.ts b/test/orchestrator/workflow-runner-reconciler.test.ts new file mode 100644 index 00000000..915c345b --- /dev/null +++ b/test/orchestrator/workflow-runner-reconciler.test.ts @@ -0,0 +1,166 @@ +import { beforeEach, describe, expect, it, mock } from "bun:test"; + +const config = { + githubPersonalAccessToken: undefined as string | undefined, + daemonImage: `registry.example/runner@sha256:${"a".repeat(64)}` as string | undefined, + orchestratorPublicUrl: "wss://controller.example/ws" as string | undefined, + workflowRunnerCapabilitySecret: "runner-capability-root-secret" as string | undefined, +}; +const firstAttemptId = crypto.randomUUID(); +const firstAttempt = { + runId: crypto.randomUUID(), + attemptId: firstAttemptId, + runnerId: `workflow-runner:${firstAttemptId}`, + executionDeliveryId: "delivery-first", + workflowName: "plan" as const, + attemptDeadlineAt: new Date("2026-08-23T04:10:00Z"), +}; +const secondAttemptId = crypto.randomUUID(); +const secondAttempt = { + ...firstAttempt, + runId: crypto.randomUUID(), + attemptId: secondAttemptId, + runnerId: `workflow-runner:${secondAttemptId}`, + executionDeliveryId: "delivery-second", +}; +const events: string[] = []; +const reconcilePendingWorkflowRunnerResults = mock(() => { + events.push("results"); + return Promise.resolve(0); +}); +const reconcilePendingWorkflowFailureNotifications = mock(() => { + events.push("notifications"); + return Promise.resolve(); +}); +const ensureCurrentWorkflowRunnerResources = mock((input: { attempt: { attemptId: string } }) => { + events.push(`ensure:${input.attempt.attemptId}`); + return Promise.resolve("ready" as const); +}); +const failWorkflowRunnerResourceAttempt = mock(() => Promise.resolve()); +const cleanupWorkflowRunnerAttempt = mock((input: { attemptId: string }) => { + events.push(`cleanup:${input.attemptId}`); + return Promise.resolve(); +}); +const listActiveWorkflowRunnerAttempts = mock(() => Promise.resolve([firstAttempt, secondAttempt])); +const findWorkflowRunnerCleanupCandidates = mock(() => + Promise.resolve([ + { runId: firstAttempt.runId, attemptId: firstAttempt.attemptId }, + { runId: secondAttempt.runId, attemptId: secondAttempt.attemptId }, + ]), +); + +void mock.module("../../src/config", () => ({ config })); +class TestWorkflowRunnerResourceError extends Error { + constructor(readonly kind: "permanent" | "transient") { + super(kind); + } +} +void mock.module("../../src/k8s/workflow-runner-spawner", () => ({ + WorkflowRunnerResourceError: TestWorkflowRunnerResourceError, +})); +void mock.module("../../src/logger", () => ({ + logger: { + info: mock(() => undefined), + warn: mock(() => undefined), + error: mock(() => undefined), + debug: mock(() => undefined), + }, +})); +void mock.module("../../src/orchestrator/workflow-runner-capability", () => ({ + deriveWorkflowRunnerCapability: mock( + (_secret: string, _runId: string, attemptId: string) => `capability:${attemptId}`, + ), +})); +void mock.module("../../src/orchestrator/workflow-expiry-notifier", () => ({ + reconcilePendingWorkflowFailureNotifications, +})); +void mock.module("../../src/orchestrator/workflow-runner-dispatch", () => ({ + failWorkflowRunnerResourceAttempt, +})); +void mock.module("../../src/orchestrator/workflow-runner-resources", () => ({ + ensureCurrentWorkflowRunnerResources, +})); +void mock.module("../../src/orchestrator/workflow-runner-result", () => ({ + cleanupWorkflowRunnerAttempt, + reconcilePendingWorkflowRunnerResults, +})); +void mock.module("../../src/orchestrator/workflow-runner-store", () => ({ + findWorkflowRunnerCleanupCandidates, + listActiveWorkflowRunnerAttempts, +})); + +const { reconcileWorkflowRunners } = + await import("../../src/orchestrator/workflow-runner-reconciler"); + +describe("workflow runner reconciliation", () => { + beforeEach(() => { + events.length = 0; + config.githubPersonalAccessToken = undefined; + config.daemonImage = `registry.example/runner@sha256:${"a".repeat(64)}`; + config.orchestratorPublicUrl = "wss://controller.example/ws"; + config.workflowRunnerCapabilitySecret = "runner-capability-root-secret"; + reconcilePendingWorkflowRunnerResults.mockClear(); + reconcilePendingWorkflowFailureNotifications.mockClear(); + ensureCurrentWorkflowRunnerResources.mockReset(); + ensureCurrentWorkflowRunnerResources.mockImplementation((input) => { + events.push(`ensure:${input.attempt.attemptId}`); + return Promise.resolve("ready"); + }); + failWorkflowRunnerResourceAttempt.mockClear(); + cleanupWorkflowRunnerAttempt.mockReset(); + cleanupWorkflowRunnerAttempt.mockImplementation((input) => { + events.push(`cleanup:${input.attemptId}`); + return Promise.resolve(); + }); + listActiveWorkflowRunnerAttempts.mockClear(); + findWorkflowRunnerCleanupCandidates.mockClear(); + }); + + it("replays results before repairing active resources and terminal cleanup", async () => { + await reconcileWorkflowRunners(); + expect(events).toEqual([ + "results", + "notifications", + `ensure:${firstAttempt.attemptId}`, + `ensure:${secondAttempt.attemptId}`, + `cleanup:${firstAttempt.attemptId}`, + `cleanup:${secondAttempt.attemptId}`, + ]); + expect(ensureCurrentWorkflowRunnerResources.mock.calls[0]?.[0]).toEqual({ + attempt: firstAttempt, + capability: `capability:${firstAttempt.attemptId}`, + image: `registry.example/runner@sha256:${"a".repeat(64)}`, + orchestratorUrl: "wss://controller.example/ws", + }); + }); + + it("continues with other attempts after one repair and one cleanup fail", async () => { + ensureCurrentWorkflowRunnerResources.mockRejectedValueOnce(new Error("API unavailable")); + cleanupWorkflowRunnerAttempt.mockRejectedValueOnce(new Error("delete pending")); + await reconcileWorkflowRunners(); + expect(events).toContain(`ensure:${secondAttempt.attemptId}`); + expect(events).toContain(`cleanup:${secondAttempt.attemptId}`); + }); + + it("fences a permanent resource violation and continues reconciliation", async () => { + ensureCurrentWorkflowRunnerResources.mockRejectedValueOnce( + new TestWorkflowRunnerResourceError("permanent"), + ); + + await reconcileWorkflowRunners(); + + expect(failWorkflowRunnerResourceAttempt).toHaveBeenCalledWith(firstAttempt, "permanent"); + expect(ensureCurrentWorkflowRunnerResources).toHaveBeenCalledTimes(2); + expect(ensureCurrentWorkflowRunnerResources.mock.calls[1]?.[0]).toEqual( + expect.objectContaining({ attempt: secondAttempt }), + ); + expect(cleanupWorkflowRunnerAttempt).toHaveBeenCalledTimes(2); + }); + + it("skips active resource creation when runner configuration is unsafe", async () => { + config.githubPersonalAccessToken = "global-pat"; + await reconcileWorkflowRunners(); + expect(ensureCurrentWorkflowRunnerResources).not.toHaveBeenCalled(); + expect(cleanupWorkflowRunnerAttempt).toHaveBeenCalledTimes(2); + }); +}); diff --git a/test/orchestrator/workflow-runner-resources.test.ts b/test/orchestrator/workflow-runner-resources.test.ts new file mode 100644 index 00000000..1a3efbcf --- /dev/null +++ b/test/orchestrator/workflow-runner-resources.test.ts @@ -0,0 +1,96 @@ +import { beforeEach, describe, expect, it, mock } from "bun:test"; + +import { waitFor } from "../utils/assertions"; + +const attemptId = crypto.randomUUID(); +const attempt = { + runId: crypto.randomUUID(), + attemptId, + runnerId: `workflow-runner:${attemptId}`, + executionDeliveryId: "delivery-16", + workflowName: "review" as const, + attemptDeadlineAt: new Date("2026-08-23T04:10:00Z"), +}; +const ensureWorkflowRunnerResources = mock(() => Promise.resolve()); +const deleteWorkflowRunnerResources = mock(() => Promise.resolve(true)); +const getWorkflowRunnerRegistrationState = mock(() => + Promise.resolve({ state: "ready" as const, attempt }), +); +const markWorkflowRunnerResourcesCleaned = mock(() => Promise.resolve(true)); + +void mock.module("../../src/k8s/workflow-runner-spawner", () => ({ + deleteWorkflowRunnerResources, + ensureWorkflowRunnerResources, +})); +void mock.module("../../src/orchestrator/workflow-runner-store", () => ({ + getWorkflowRunnerRegistrationState, + markWorkflowRunnerResourcesCleaned, +})); + +const { + cleanupCurrentWorkflowRunnerResources, + ensureCurrentWorkflowRunnerResources, + resetWorkflowRunnerResourceChainsForTests, +} = await import("../../src/orchestrator/workflow-runner-resources"); + +const input = { + attempt, + capability: "wfr1.capability", + image: `registry.example/runner@sha256:${"a".repeat(64)}`, + orchestratorUrl: "wss://controller.example/ws", +}; + +describe("workflow runner resource operation ordering", () => { + beforeEach(() => { + resetWorkflowRunnerResourceChainsForTests(); + ensureWorkflowRunnerResources.mockReset(); + ensureWorkflowRunnerResources.mockResolvedValue(); + deleteWorkflowRunnerResources.mockReset(); + deleteWorkflowRunnerResources.mockResolvedValue(true); + getWorkflowRunnerRegistrationState.mockReset(); + getWorkflowRunnerRegistrationState.mockResolvedValue({ state: "ready", attempt }); + markWorkflowRunnerResourcesCleaned.mockClear(); + }); + + it("does not recreate resources after terminal cleanup won the chain", async () => { + await cleanupCurrentWorkflowRunnerResources(attempt); + getWorkflowRunnerRegistrationState.mockResolvedValueOnce({ state: "completed" }); + + expect(await ensureCurrentWorkflowRunnerResources(input)).toBe("terminal"); + expect(ensureWorkflowRunnerResources).not.toHaveBeenCalled(); + expect(deleteWorkflowRunnerResources).toHaveBeenCalledTimes(2); + }); + + it("cleans resources when terminalization races an in-progress ensure", async () => { + let releaseEnsure!: () => void; + const ensureGate = new Promise((resolve) => { + releaseEnsure = resolve; + }); + ensureWorkflowRunnerResources.mockImplementationOnce(() => ensureGate); + getWorkflowRunnerRegistrationState + .mockResolvedValueOnce({ state: "ready", attempt }) + .mockResolvedValueOnce({ state: "completed" }); + + const ensuring = ensureCurrentWorkflowRunnerResources(input); + await waitFor(() => ensureWorkflowRunnerResources.mock.calls.length > 0); + const cleaning = cleanupCurrentWorkflowRunnerResources(attempt); + releaseEnsure(); + + expect(await ensuring).toBe("terminal"); + await cleaning; + expect(deleteWorkflowRunnerResources).toHaveBeenCalledTimes(2); + expect(markWorkflowRunnerResourcesCleaned).toHaveBeenCalledTimes(2); + }); + + it("leaves resources intact while a durable result awaits projection", async () => { + getWorkflowRunnerRegistrationState.mockResolvedValueOnce({ + state: "result-pending", + executionDeliveryId: attempt.executionDeliveryId, + payload: {}, + }); + + expect(await ensureCurrentWorkflowRunnerResources(input)).toBe("result-pending"); + expect(ensureWorkflowRunnerResources).not.toHaveBeenCalled(); + expect(deleteWorkflowRunnerResources).not.toHaveBeenCalled(); + }); +}); diff --git a/test/orchestrator/workflow-runner-result.test.ts b/test/orchestrator/workflow-runner-result.test.ts new file mode 100644 index 00000000..fea41f85 --- /dev/null +++ b/test/orchestrator/workflow-runner-result.test.ts @@ -0,0 +1,410 @@ +import { beforeEach, describe, expect, it, mock } from "bun:test"; + +import type { PendingWorkflowRunnerResult } from "../../src/orchestrator/workflow-runner-store"; +import { expectToReject, waitFor } from "../utils/assertions"; + +const resultStates = new Map(); +const attemptsByRun = new Map(); +const statusesByRun = new Map(); +const targetNumbersByRun = new Map(); +const targetTypesByRun = new Map(); +const parentIdsByRun = new Map(); +const ensureWorkflowCascadeForOffer = mock(() => Promise.resolve()); +const setState = mock(() => Promise.resolve()); +const persistRepoKnowledge = mock(() => Promise.resolve()); +const addReaction = mock(() => Promise.resolve()); +const findById = mock((runId: string) => + Promise.resolve({ + ...workflowRow(runId), + attempt_id: attemptsByRun.get(runId) ?? null, + status: statusesByRun.get(runId) ?? "succeeded", + }), +); +const getWorkflowRunnerResultProcessingState = mock((pending: PendingWorkflowRunnerResult) => + Promise.resolve(resultStates.get(pending.attemptId) ?? "missing"), +); +const markWorkflowRunnerResultProcessed = mock((attemptId: string) => { + resultStates.set(attemptId, "processed"); + return Promise.resolve(true); +}); +const cleanupCurrentWorkflowRunnerResources = mock(() => Promise.resolve()); +const loggerInfo = mock(() => undefined); +const loggerWarn = mock(() => undefined); +const revokeInstallationToken = mock(() => Promise.resolve(true)); +const mintInstallationToken = mock(() => Promise.resolve({ octokit: {} })); +const getRepoInstallation = mock(() => Promise.resolve({ data: { id: 123 } })); + +void mock.module("../../src/config", () => ({ + config: { + appId: "test-app", + privateKey: "test-key", + reviewLearningsEnabled: true, + }, +})); +void mock.module("../../src/db", () => ({ + requireDb: (): Record => ({}), +})); +void mock.module("../../src/logger", () => ({ + logger: { + info: loggerInfo, + warn: loggerWarn, + error: mock(() => undefined), + debug: mock(() => undefined), + }, +})); +void mock.module("octokit", () => ({ + Octokit: function MockOctokit(this: unknown): unknown { + return this; + }, + App: function MockApp(this: unknown): unknown { + return { octokit: { rest: { apps: { getRepoInstallation } } } }; + }, +})); +void mock.module("../../src/utils/log-redaction", () => ({ + redactErrorMessageOrFallback: (value: string | undefined, fallback: string): string => + value ?? fallback, +})); +void mock.module("../../src/utils/reactions", () => ({ addReaction })); +void mock.module("../../src/workflows/completion-reconciler", () => ({ + ensureWorkflowCascadeForOffer, +})); +void mock.module("../../src/workflows/runs-store", () => ({ findById })); +void mock.module("../../src/workflows/tracking-mirror", () => ({ setState })); +void mock.module("../../src/orchestrator/installation-token", () => ({ + mintInstallationToken, + revokeInstallationToken, +})); +void mock.module("../../src/orchestrator/repo-knowledge-persistence", () => ({ + persistRepoKnowledge, +})); +void mock.module("../../src/orchestrator/workflow-runner-resources", () => ({ + cleanupCurrentWorkflowRunnerResources, +})); +void mock.module("../../src/orchestrator/workflow-runner-store", () => ({ + findPendingWorkflowRunnerResults: mock(() => Promise.resolve([])), + getWorkflowRunnerResultProcessingState, + markWorkflowRunnerResultProcessed, +})); + +const { cleanupWorkflowRunnerAttempt, processWorkflowRunnerResult } = + await import("../../src/orchestrator/workflow-runner-result"); + +function workflowRow(runId: string): Record { + return { + id: runId, + attempt_id: null, + workflow_name: "review", + target_owner: "acme", + target_repo: "widgets", + trigger_comment_id: 42, + trigger_event_type: "issue_comment", + target_type: targetTypesByRun.get(runId) ?? "pr", + target_number: targetNumbersByRun.get(runId) ?? 7, + parent_run_id: parentIdsByRun.get(runId) ?? null, + status: statusesByRun.get(runId) ?? "succeeded", + }; +} + +function pending( + runId = crypto.randomUUID(), + attemptId = crypto.randomUUID(), +): PendingWorkflowRunnerResult { + const result: PendingWorkflowRunnerResult = { + runId, + attemptId, + executionDeliveryId: crypto.randomUUID(), + payload: { + runId, + attemptId, + result: { + status: "succeeded", + state: { phase: "complete" }, + appliedReviewLearningIds: ["learning-1"], + daemonActions: { + learnings: [{ category: "setup", content: "Run isolated tests." }], + deletions: [], + }, + }, + durationMs: 123, + }, + }; + resultStates.set(attemptId, "pending"); + attemptsByRun.set(runId, attemptId); + return result; +} + +function handedOffPending( + runId = crypto.randomUUID(), + attemptId = crypto.randomUUID(), +): PendingWorkflowRunnerResult { + const result: PendingWorkflowRunnerResult = { + runId, + attemptId, + executionDeliveryId: crypto.randomUUID(), + payload: { + runId, + attemptId, + result: { + status: "handed-off", + state: { phase: "child-running" }, + humanMessage: "ship started, first step `triage` queued.", + childRunId: crypto.randomUUID(), + }, + durationMs: 123, + }, + }; + resultStates.set(attemptId, "pending"); + attemptsByRun.set(runId, attemptId); + statusesByRun.set(runId, "running"); + return result; +} + +function deferred(): { promise: Promise; resolve: () => void } { + let resolve!: () => void; + const promise = new Promise((done) => { + resolve = done; + }); + return { promise, resolve }; +} + +async function waitForCalls(fn: { mock: { calls: unknown[][] } }, count: number): Promise { + await waitFor(() => fn.mock.calls.length >= count); + expect(fn.mock.calls.length).toBe(count); +} + +describe("workflow runner result projection", () => { + beforeEach(() => { + resultStates.clear(); + attemptsByRun.clear(); + statusesByRun.clear(); + targetNumbersByRun.clear(); + targetTypesByRun.clear(); + parentIdsByRun.clear(); + ensureWorkflowCascadeForOffer.mockReset(); + ensureWorkflowCascadeForOffer.mockImplementation(() => Promise.resolve()); + setState.mockReset(); + setState.mockResolvedValue(undefined); + persistRepoKnowledge.mockReset(); + persistRepoKnowledge.mockImplementation(() => Promise.resolve()); + addReaction.mockClear(); + findById.mockReset(); + findById.mockImplementation((id: string) => + Promise.resolve({ + ...workflowRow(id), + attempt_id: attemptsByRun.get(id) ?? null, + status: statusesByRun.get(id) ?? "succeeded", + }), + ); + getWorkflowRunnerResultProcessingState.mockClear(); + markWorkflowRunnerResultProcessed.mockClear(); + cleanupCurrentWorkflowRunnerResources.mockClear(); + loggerInfo.mockClear(); + loggerWarn.mockClear(); + mintInstallationToken.mockClear(); + getRepoInstallation.mockClear(); + revokeInstallationToken.mockClear(); + }); + + it("serializes concurrent duplicates and makes processed replay a no-op", async () => { + const result = pending(); + const gate = deferred(); + ensureWorkflowCascadeForOffer.mockImplementation(() => gate.promise); + + const first = processWorkflowRunnerResult(result, {} as never); + const duplicate = processWorkflowRunnerResult(result, {} as never); + await waitForCalls(ensureWorkflowCascadeForOffer, 1); + expect(getWorkflowRunnerResultProcessingState).toHaveBeenCalledTimes(1); + + gate.resolve(); + await Promise.all([first, duplicate]); + expect(ensureWorkflowCascadeForOffer).toHaveBeenCalledTimes(1); + expect(setState).toHaveBeenCalledTimes(1); + expect(persistRepoKnowledge).toHaveBeenCalledTimes(1); + expect(addReaction).toHaveBeenCalledTimes(1); + expect(markWorkflowRunnerResultProcessed).toHaveBeenCalledTimes(1); + expect(loggerInfo).toHaveBeenCalledWith( + expect.objectContaining({ + event: "workflow.run.succeeded", + runId: result.runId, + duration_ms: 123, + }), + "Workflow run succeeded", + ); + + await processWorkflowRunnerResult(result, {} as never); + expect(ensureWorkflowCascadeForOffer).toHaveBeenCalledTimes(1); + expect(setState).toHaveBeenCalledTimes(1); + expect(markWorkflowRunnerResultProcessed).toHaveBeenCalledTimes(1); + expect(loggerInfo).toHaveBeenCalledTimes(1); + expect(revokeInstallationToken).not.toHaveBeenCalled(); + }); + + it("revokes an internally minted projection token after success", async () => { + const result = pending(); + + await processWorkflowRunnerResult(result); + + expect(mintInstallationToken).toHaveBeenCalledTimes(1); + expect(revokeInstallationToken).toHaveBeenCalledWith(expect.anything(), expect.anything(), { + attemptId: result.attemptId, + owner: "result-projection", + }); + }); + + it("revokes an internally minted projection token after a retryable failure", async () => { + const result = pending(); + ensureWorkflowCascadeForOffer.mockRejectedValueOnce(new Error("temporary GitHub failure")); + + await expectToReject(processWorkflowRunnerResult(result), "temporary GitHub failure"); + + expect(revokeInstallationToken).toHaveBeenCalledTimes(1); + expect(resultStates.get(result.attemptId)).toBe("pending"); + }); + + it("leaves a failed projection pending for retry", async () => { + const result = pending(); + ensureWorkflowCascadeForOffer.mockRejectedValueOnce(new Error("temporary GitHub failure")); + + await expectToReject( + processWorkflowRunnerResult(result, {} as never), + "temporary GitHub failure", + ); + expect(resultStates.get(result.attemptId)).toBe("pending"); + expect(markWorkflowRunnerResultProcessed).not.toHaveBeenCalled(); + + await processWorkflowRunnerResult(result, {} as never); + expect(resultStates.get(result.attemptId)).toBe("processed"); + expect(ensureWorkflowCascadeForOffer).toHaveBeenCalledTimes(2); + expect(markWorkflowRunnerResultProcessed).toHaveBeenCalledTimes(1); + }); + + it("receipts the durable result when best-effort knowledge persistence fails", async () => { + const result = pending(); + persistRepoKnowledge.mockRejectedValueOnce(new Error("database unavailable")); + + await processWorkflowRunnerResult(result, {} as never); + + expect(resultStates.get(result.attemptId)).toBe("processed"); + expect(ensureWorkflowCascadeForOffer).toHaveBeenCalledTimes(1); + expect(setState).toHaveBeenCalledTimes(1); + expect(markWorkflowRunnerResultProcessed).toHaveBeenCalledTimes(1); + expect(loggerWarn).toHaveBeenCalledWith( + expect.objectContaining({ runId: result.runId, attemptId: result.attemptId }), + "Failed to persist workflow runner repo knowledge", + ); + }); + + it("dead-letters a GitHub-rejected detail after a fixed fallback succeeds", async () => { + const result = pending(); + setState + .mockRejectedValueOnce(Object.assign(new Error("Validation Failed"), { status: 422 })) + .mockResolvedValueOnce(undefined); + + await processWorkflowRunnerResult(result, {} as never); + + expect(setState).toHaveBeenCalledTimes(2); + expect(setState.mock.calls[1]?.[1]).toEqual( + expect.objectContaining({ + humanMessage: + "review reached a terminal state, but GitHub rejected its detailed status. Inspect controller logs and durable workflow state.", + }), + ); + expect(markWorkflowRunnerResultProcessed).toHaveBeenCalledWith(result.attemptId); + }); + + it("projects incomplete text, a confused reaction, and the incomplete lifecycle event", async () => { + const result = pending(); + result.payload.result = { + status: "incomplete", + reason: "required checks remain", + state: { outstanding: ["fix CI"] }, + humanMessage: "Required checks remain.", + }; + + await processWorkflowRunnerResult(result, {} as never); + + expect(setState).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ humanMessage: "Required checks remain." }), + ); + expect(addReaction).toHaveBeenCalledWith(expect.objectContaining({ content: "confused" })); + expect(loggerWarn).toHaveBeenCalledWith( + expect.objectContaining({ + event: "workflow.run.incomplete", + runId: result.runId, + reason: "required checks remain", + }), + "Workflow run incomplete", + ); + expect(markWorkflowRunnerResultProcessed).toHaveBeenCalledWith(result.attemptId); + }); + + it("keeps a rejected detail pending when the fixed fallback also fails", async () => { + const result = pending(); + setState + .mockRejectedValueOnce(Object.assign(new Error("Validation Failed"), { status: 422 })) + .mockRejectedValueOnce(new Error("fallback unavailable")); + + await expectToReject(processWorkflowRunnerResult(result, {} as never), "fallback unavailable"); + expect(markWorkflowRunnerResultProcessed).not.toHaveBeenCalled(); + }); + + it("does not globally serialize different targets", async () => { + const first = pending(); + const second = pending(); + targetNumbersByRun.set(second.runId, 8); + const firstGate = deferred(); + const secondGate = deferred(); + ensureWorkflowCascadeForOffer.mockImplementation(() => + ensureWorkflowCascadeForOffer.mock.calls.length === 1 + ? firstGate.promise + : secondGate.promise, + ); + + const firstProcess = processWorkflowRunnerResult(first, {} as never); + const secondProcess = processWorkflowRunnerResult(second, {} as never); + await waitForCalls(ensureWorkflowCascadeForOffer, 2); + + firstGate.resolve(); + secondGate.resolve(); + await Promise.all([firstProcess, secondProcess]); + expect(markWorkflowRunnerResultProcessed).toHaveBeenCalledTimes(2); + }); + + it("does not overwrite a terminal parent with a late hand-off projection", async () => { + const parent = handedOffPending(); + const child = pending(); + targetTypesByRun.set(parent.runId, "issue"); + targetTypesByRun.set(child.runId, "pr"); + targetNumbersByRun.set(child.runId, 99); + parentIdsByRun.set(child.runId, parent.runId); + const childProjection = deferred(); + setState.mockImplementationOnce(() => childProjection.promise); + ensureWorkflowCascadeForOffer.mockImplementation((attemptId: string) => { + if (attemptId === child.attemptId) statusesByRun.set(parent.runId, "succeeded"); + return Promise.resolve(); + }); + + const processChild = processWorkflowRunnerResult(child, {} as never); + await waitForCalls(setState, 1); + const processParent = processWorkflowRunnerResult(parent, {} as never); + + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(ensureWorkflowCascadeForOffer).toHaveBeenCalledTimes(1); + expect(setState).toHaveBeenCalledTimes(1); + + childProjection.resolve(); + await Promise.all([processChild, processParent]); + + expect(ensureWorkflowCascadeForOffer).toHaveBeenCalledTimes(2); + expect(setState).toHaveBeenCalledTimes(1); + expect(markWorkflowRunnerResultProcessed).toHaveBeenCalledTimes(2); + }); + + it("delegates cleanup through the serialized resource boundary", async () => { + const attempt = { runId: crypto.randomUUID(), attemptId: crypto.randomUUID() }; + + await cleanupWorkflowRunnerAttempt(attempt); + expect(cleanupCurrentWorkflowRunnerResources).toHaveBeenCalledWith(attempt); + }); +}); diff --git a/test/orchestrator/workflow-runner-store.test.ts b/test/orchestrator/workflow-runner-store.test.ts new file mode 100644 index 00000000..29c30cde --- /dev/null +++ b/test/orchestrator/workflow-runner-store.test.ts @@ -0,0 +1,549 @@ +import { SQL } from "bun"; +import { afterAll, beforeAll, beforeEach, describe, expect, it } from "bun:test"; + +import type { WorkflowRunQueuedJob } from "../../src/orchestrator/job-queue"; +import type { WorkflowRunnerAttempt } from "../../src/orchestrator/workflow-runner-store"; +import type { WorkflowRunnerResultPayload } from "../../src/shared/workflow-runner-messages"; +import { expectToReject } from "../utils/assertions"; + +const TEST_DATABASE_URL = + process.env["TEST_DATABASE_URL"] ?? "postgres://bot:bot@localhost:55432/github_app_test"; + +let sql: SQL | null = null; +try { + const connection = new SQL(TEST_DATABASE_URL); + await connection`SELECT 1 AS ok`; + sql = connection; +} catch { + sql = null; +} + +function requireSql(): SQL { + if (sql === null) throw new Error("Database not available, test should have been skipped"); + return sql; +} + +async function resetSchema(): Promise { + await requireSql().unsafe(` + DROP TABLE IF EXISTS _migrations CASCADE; + DROP TABLE IF EXISTS review_learnings CASCADE; + DROP TABLE IF EXISTS scheduled_action_state CASCADE; + DROP TABLE IF EXISTS comment_cache CASCADE; + DROP TABLE IF EXISTS target_cache CASCADE; + DROP TABLE IF EXISTS chat_proposals CASCADE; + DROP TABLE IF EXISTS ship_fix_attempts CASCADE; + DROP TABLE IF EXISTS ship_continuations CASCADE; + DROP TABLE IF EXISTS ship_iterations CASCADE; + DROP TABLE IF EXISTS ship_intents CASCADE; + DROP TABLE IF EXISTS workflow_attempt_commands CASCADE; + DROP TABLE IF EXISTS workflow_runs CASCADE; + DROP TABLE IF EXISTS repo_memory CASCADE; + DROP TABLE IF EXISTS triage_results CASCADE; + DROP TABLE IF EXISTS executions CASCADE; + DROP TABLE IF EXISTS daemons CASCADE; + `); +} + +async function queuedWorkflow(number: number): Promise { + const deliveryId = crypto.randomUUID(); + const { createExecution } = await import("../../src/orchestrator/history"); + const { insertQueued } = await import("../../src/workflows/runs-store"); + await createExecution( + { + deliveryId, + repoOwner: "acme", + repoName: "widgets", + entityNumber: number, + entityType: "issue", + eventName: "issue_comment", + triggerUsername: "maintainer", + dispatchMode: "workflow-runner", + dispatchTarget: "workflow-runner", + dispatchReason: "workflow-runner", + }, + requireSql(), + ); + const run = await insertQueued( + { + workflowName: "implement", + target: { type: "issue", owner: "acme", repo: "widgets", number }, + deliveryId, + ownerKind: "orchestrator", + ownerId: "orchestrator-test", + }, + requireSql(), + ); + return { + kind: "workflow-run", + deliveryId, + repoOwner: "acme", + repoName: "widgets", + entityNumber: number, + isPR: false, + eventName: "issue_comment", + triggerUsername: "maintainer", + labels: [], + triggerBodyPreview: "", + enqueuedAt: Date.now(), + retryCount: 0, + workflowRun: { runId: run.id, workflowName: "implement" }, + }; +} + +async function claimedWorkflow(number: number): Promise { + const { claimWorkflowRunnerAttempt } = + await import("../../src/orchestrator/workflow-runner-store"); + const job = await queuedWorkflow(number); + const claim = await claimWorkflowRunnerAttempt(job, 60_000, 10, requireSql()); + if (claim.outcome !== "claimed") throw new Error(`Expected claim, received ${claim.outcome}`); + return claim.attempt; +} + +function succeededResult(attempt: WorkflowRunnerAttempt): WorkflowRunnerResultPayload { + return { + runId: attempt.runId, + attemptId: attempt.attemptId, + result: { + status: "succeeded", + state: { phase: "complete" }, + daemonActions: { + learnings: [{ category: "architecture", content: "The controller owns state." }], + deletions: ["11111111-1111-4111-8111-111111111111"], + }, + }, + durationMs: 321, + }; +} + +describe.skipIf(sql === null)("workflow runner admission", () => { + beforeAll(async () => { + await resetSchema(); + const { runMigrations } = await import("../../src/db/migrate"); + await runMigrations(requireSql()); + }); + + beforeEach(async () => { + await requireSql().unsafe("TRUNCATE workflow_runs, executions CASCADE"); + }); + + afterAll(async () => { + await resetSchema(); + await requireSql().close(); + }); + + it("defers queued work at capacity while active duplicates reconcile", async () => { + const { claimWorkflowRunnerAttempt, failWorkflowRunnerAttempt } = + await import("../../src/orchestrator/workflow-runner-store"); + const firstJob = await queuedWorkflow(16); + const secondJob = await queuedWorkflow(17); + + await expectToReject( + claimWorkflowRunnerAttempt(firstJob, 60_000, 0, requireSql()), + "maxActive must be a positive integer", + ); + + const first = await claimWorkflowRunnerAttempt(firstJob, 60_000, 1, requireSql()); + expect(first.outcome).toBe("claimed"); + if (first.outcome !== "claimed") throw new Error("Expected first claim"); + + const duplicate = await claimWorkflowRunnerAttempt(firstJob, 60_000, 1, requireSql()); + expect(duplicate).toEqual({ outcome: "active", attempt: first.attempt }); + + const capacity = await claimWorkflowRunnerAttempt(secondJob, 60_000, 1, requireSql()); + expect(capacity).toEqual({ outcome: "capacity" }); + const queuedRows: { + workflow_status: string; + owner_kind: string | null; + owner_id: string | null; + attempt_id: string | null; + lease_expires_at: Date | null; + execution_status: string; + daemon_id: string | null; + offer_id: string | null; + }[] = await requireSql()` + SELECT wr.status AS workflow_status, + wr.owner_kind, + wr.owner_id, + wr.attempt_id, + wr.lease_expires_at, + e.status AS execution_status, + e.daemon_id, + e.offer_id + FROM workflow_runs AS wr + JOIN executions AS e ON e.delivery_id = wr.execution_delivery_id + WHERE wr.id = ${secondJob.workflowRun.runId} + `; + expect(queuedRows[0]).toEqual({ + workflow_status: "queued", + owner_kind: "orchestrator", + owner_id: "orchestrator-test", + attempt_id: null, + lease_expires_at: null, + execution_status: "queued", + daemon_id: null, + offer_id: null, + }); + + await failWorkflowRunnerAttempt(first.attempt, "test terminal slot", requireSql()); + const second = await claimWorkflowRunnerAttempt(secondJob, 60_000, 1, requireSql()); + expect(second.outcome).toBe("claimed"); + if (second.outcome !== "claimed") throw new Error("Expected second claim"); + + const thirdJob = await queuedWorkflow(18); + await requireSql()` + UPDATE workflow_runs + SET lease_expires_at = now() - interval '1 second' + WHERE id = ${second.attempt.runId} + `; + const third = await claimWorkflowRunnerAttempt(thirdJob, 60_000, 1, requireSql()); + expect(third.outcome).toBe("claimed"); + }); + + it("lists only exact live attempts in reconciliation order", async () => { + const { failWorkflowRunnerAttempt, listActiveWorkflowRunnerAttempts } = + await import("../../src/orchestrator/workflow-runner-store"); + const first = await claimedWorkflow(19); + const terminal = await claimedWorkflow(20); + const expiredLease = await claimedWorkflow(21); + const expiredDeadline = await claimedWorkflow(22); + const wrongOwner = await claimedWorkflow(23); + const second = await claimedWorkflow(24); + + await failWorkflowRunnerAttempt(terminal, "test terminal attempt", requireSql()); + await requireSql()` + UPDATE workflow_runs + SET lease_expires_at = now() - interval '1 second' + WHERE id = ${expiredLease.runId} + `; + await requireSql()` + UPDATE workflow_runs + SET attempt_deadline_at = now() - interval '1 second' + WHERE id = ${expiredDeadline.runId} + `; + await requireSql()` + UPDATE workflow_runs + SET owner_id = ${`workflow-runner:${crypto.randomUUID()}`} + WHERE id = ${wrongOwner.runId} + `; + await requireSql()` + UPDATE workflow_runs + SET updated_at = '2026-01-01T00:00:00Z'::timestamptz + WHERE id = ${first.runId} + `; + await requireSql()` + UPDATE workflow_runs + SET updated_at = '2026-01-02T00:00:00Z'::timestamptz + WHERE id = ${second.runId} + `; + + expect(await listActiveWorkflowRunnerAttempts(requireSql(), 1)).toEqual([first]); + expect(await listActiveWorkflowRunnerAttempts(requireSql())).toEqual([first, second]); + }); + + it("replays stable command receipts and rejects command-id content conflicts", async () => { + const { findWorkflowRunnerCommandReceipt, insertWorkflowRunnerCommandReceipt } = + await import("../../src/orchestrator/workflow-runner-store"); + const attempt = await claimedWorkflow(30); + const commandId = crypto.randomUUID(); + const command = { + type: "set-state" as const, + patch: { phase: "reviewing" }, + humanMessage: "Reviewing.", + }; + + await insertWorkflowRunnerCommandReceipt( + attempt, + commandId, + command, + { trackingCommentId: 301 }, + requireSql(), + ); + await insertWorkflowRunnerCommandReceipt( + attempt, + commandId, + command, + { trackingCommentId: 999 }, + requireSql(), + ); + + expect(await findWorkflowRunnerCommandReceipt(attempt, commandId, requireSql())).toEqual({ + commandKind: "set-state", + request: command, + response: { trackingCommentId: 301 }, + }); + await expectToReject( + insertWorkflowRunnerCommandReceipt( + attempt, + commandId, + { ...command, patch: { phase: "different" } }, + { trackingCommentId: 301 }, + requireSql(), + ), + "command id was reused with different content", + ); + }); + + it("fences registration and renewal at the immutable attempt deadline", async () => { + const { getWorkflowRunnerRegistrationState } = + await import("../../src/orchestrator/workflow-runner-store"); + const { expireWorkflowAttempts, renewWorkflowAttempts } = + await import("../../src/workflows/runs-store"); + const attempt = await claimedWorkflow(34); + await requireSql()` + UPDATE workflow_runs + SET lease_expires_at = now() + interval '1 minute', + attempt_deadline_at = now() - interval '1 second' + WHERE id = ${attempt.runId} + `; + + expect(await getWorkflowRunnerRegistrationState(attempt, requireSql())).toEqual({ + state: "invalid", + }); + expect( + await renewWorkflowAttempts(attempt.runnerId, [attempt.attemptId], 60_000, requireSql()), + ).toEqual({ renewedAttemptIds: [], fencedAttemptIds: [attempt.attemptId] }); + const expired = await expireWorkflowAttempts(requireSql()); + expect(expired).toHaveLength(1); + expect(expired[0]?.state["failedReason"]).toBe("workflow execution deadline expired"); + }); + + it("records one payload delivery only when the token expires within the attempt", async () => { + const safe = await claimedWorkflow(212); + const unsafe = await claimedWorkflow(213); + const { getWorkflowRunnerRegistrationState, recordWorkflowRunnerPayloadIssued } = + await import("../../src/orchestrator/workflow-runner-store"); + const safeExpiry = new Date(safe.attemptDeadlineAt.getTime() - 60_000); + + expect(await recordWorkflowRunnerPayloadIssued(safe, safeExpiry, requireSql())).toBe(true); + expect(await recordWorkflowRunnerPayloadIssued(safe, safeExpiry, requireSql())).toBe(false); + expect( + await recordWorkflowRunnerPayloadIssued( + unsafe, + new Date(unsafe.attemptDeadlineAt.getTime() + 1), + requireSql(), + ), + ).toBe(false); + + expect(await getWorkflowRunnerRegistrationState(safe, requireSql())).toMatchObject({ + state: "ready", + payloadIssuedAt: expect.any(Date), + tokenExpiresAt: safeExpiry, + }); + expect(await getWorkflowRunnerRegistrationState(unsafe, requireSql())).toMatchObject({ + state: "ready", + payloadIssuedAt: null, + tokenExpiresAt: null, + }); + }); + + it("stores one exact terminal result atomically and rejects conflicting replay", async () => { + const { storeWorkflowRunnerResult } = + await import("../../src/orchestrator/workflow-runner-store"); + const attempt = await claimedWorkflow(31); + const payload = succeededResult(attempt); + + expect(await storeWorkflowRunnerResult(payload, requireSql())).toBe("stored"); + const rows: { + workflow_status: string; + workflow_state: Record; + attempt_completed_at: Date | null; + lease_expires_at: Date | null; + execution_status: string; + duration_ms: number | null; + workflow_result_payload: unknown; + result_processed_at: Date | null; + }[] = await requireSql()` + SELECT wr.status AS workflow_status, + wr.state AS workflow_state, + wr.attempt_completed_at, + wr.lease_expires_at, + e.status AS execution_status, + e.duration_ms, + e.workflow_result_payload, + e.result_processed_at + FROM workflow_runs AS wr + JOIN executions AS e ON e.delivery_id = wr.execution_delivery_id + WHERE wr.id = ${attempt.runId} + `; + expect(rows[0]).toEqual({ + workflow_status: "succeeded", + workflow_state: { phase: "complete" }, + attempt_completed_at: expect.any(Date), + lease_expires_at: null, + execution_status: "completed", + duration_ms: 321, + workflow_result_payload: payload, + result_processed_at: null, + }); + expect(await storeWorkflowRunnerResult(payload, requireSql())).toBe("already-stored"); + await expectToReject( + storeWorkflowRunnerResult({ ...payload, durationMs: 322 }, requireSql()), + "id was reused with different content", + ); + }); + + it("stores an incomplete result as incomplete while failing its execution receipt", async () => { + const { storeWorkflowRunnerResult } = + await import("../../src/orchestrator/workflow-runner-store"); + const attempt = await claimedWorkflow(35); + const payload: WorkflowRunnerResultPayload = { + runId: attempt.runId, + attemptId: attempt.attemptId, + result: { + status: "incomplete", + reason: "CI still has required work", + state: { outstanding: ["fix CI"] }, + humanMessage: "CI still has required work.", + }, + durationMs: 456, + }; + + expect(await storeWorkflowRunnerResult(payload, requireSql())).toBe("stored"); + const [row] = await requireSql()< + { + workflow_status: string; + workflow_state: Record; + execution_status: string; + error_message: string | null; + }[] + >` + SELECT wr.status AS workflow_status, + wr.state AS workflow_state, + e.status AS execution_status, + e.error_message + FROM workflow_runs AS wr + JOIN executions AS e ON e.delivery_id = wr.execution_delivery_id + WHERE wr.id = ${attempt.runId} + `; + expect(row).toEqual({ + workflow_status: "incomplete", + workflow_state: { + incompleteReason: "CI still has required work", + outstanding: ["fix CI"], + }, + execution_status: "failed", + error_message: "CI still has required work", + }); + }); + + it("reports registration, pending result, processed result, and cleanup states", async () => { + const { + findPendingWorkflowRunnerResults, + findWorkflowRunnerCleanupCandidates, + getWorkflowRunnerRegistrationState, + getWorkflowRunnerResultProcessingState, + markWorkflowRunnerResourcesCleaned, + markWorkflowRunnerResultProcessed, + storeWorkflowRunnerResult, + } = await import("../../src/orchestrator/workflow-runner-store"); + const attempt = await claimedWorkflow(32); + const payload = succeededResult(attempt); + const pending = { + runId: attempt.runId, + attemptId: attempt.attemptId, + executionDeliveryId: attempt.executionDeliveryId, + payload, + }; + + expect(await getWorkflowRunnerRegistrationState(attempt, requireSql())).toEqual({ + state: "ready", + attempt, + payloadIssuedAt: null, + tokenExpiresAt: null, + }); + expect( + await getWorkflowRunnerRegistrationState( + { runId: attempt.runId, attemptId: crypto.randomUUID() }, + requireSql(), + ), + ).toEqual({ state: "invalid" }); + expect(await getWorkflowRunnerResultProcessingState(pending, requireSql())).toBe("missing"); + + await storeWorkflowRunnerResult(payload, requireSql()); + expect(await getWorkflowRunnerResultProcessingState(pending, requireSql())).toBe("pending"); + expect( + await getWorkflowRunnerResultProcessingState( + { ...pending, executionDeliveryId: crypto.randomUUID() }, + requireSql(), + ), + ).toBe("missing"); + await expectToReject( + getWorkflowRunnerResultProcessingState( + { ...pending, payload: { ...payload, durationMs: payload.durationMs + 1 } }, + requireSql(), + ), + "id was reused with different content", + ); + expect(await getWorkflowRunnerRegistrationState(attempt, requireSql())).toEqual({ + state: "result-pending", + executionDeliveryId: attempt.executionDeliveryId, + payload, + }); + expect(await findPendingWorkflowRunnerResults(requireSql())).toEqual([ + { + runId: attempt.runId, + attemptId: attempt.attemptId, + executionDeliveryId: attempt.executionDeliveryId, + payload, + }, + ]); + expect(await findWorkflowRunnerCleanupCandidates(requireSql())).toEqual([ + { runId: attempt.runId, attemptId: attempt.attemptId }, + ]); + + expect(await markWorkflowRunnerResultProcessed(attempt.attemptId, requireSql())).toBe(true); + expect(await markWorkflowRunnerResultProcessed(attempt.attemptId, requireSql())).toBe(true); + expect(await getWorkflowRunnerResultProcessingState(pending, requireSql())).toBe("processed"); + expect(await getWorkflowRunnerRegistrationState(attempt, requireSql())).toEqual({ + state: "completed", + }); + expect(await findPendingWorkflowRunnerResults(requireSql())).toEqual([]); + expect(await findWorkflowRunnerCleanupCandidates(requireSql())).toEqual([ + { runId: attempt.runId, attemptId: attempt.attemptId }, + ]); + expect(await markWorkflowRunnerResourcesCleaned(attempt, requireSql())).toBe(true); + expect(await markWorkflowRunnerResourcesCleaned(attempt, requireSql())).toBe(true); + expect(await findWorkflowRunnerCleanupCandidates(requireSql())).toEqual([]); + }); + + it("fails the exact workflow and execution rows atomically with the reason", async () => { + const { failWorkflowRunnerAttempt } = + await import("../../src/orchestrator/workflow-runner-store"); + const attempt = await claimedWorkflow(33); + + await failWorkflowRunnerAttempt(attempt, "runner configuration rejected", requireSql()); + + const rows: { + workflow_status: string; + workflow_state: Record; + attempt_completed_at: Date | null; + lease_expires_at: Date | null; + execution_status: string; + error_message: string | null; + result_processed_at: Date | null; + }[] = await requireSql()` + SELECT wr.status AS workflow_status, + wr.state AS workflow_state, + wr.attempt_completed_at, + wr.lease_expires_at, + e.status AS execution_status, + e.error_message, + e.result_processed_at + FROM workflow_runs AS wr + JOIN executions AS e ON e.delivery_id = wr.execution_delivery_id + WHERE wr.id = ${attempt.runId} + `; + expect(rows[0]).toEqual({ + workflow_status: "failed", + workflow_state: { + failedReason: "runner configuration rejected", + phase: "runner-start-failed", + }, + attempt_completed_at: expect.any(Date), + lease_expires_at: null, + execution_status: "failed", + error_message: "runner configuration rejected", + result_processed_at: expect.any(Date), + }); + }); +}); diff --git a/test/orchestrator/ws-server.test.ts b/test/orchestrator/ws-server.test.ts index c388ba50..811c9452 100644 --- a/test/orchestrator/ws-server.test.ts +++ b/test/orchestrator/ws-server.test.ts @@ -9,6 +9,7 @@ import { beforeEach, describe, expect, it, mock } from "bun:test"; +import { deriveWorkflowRunnerCapability } from "../../src/orchestrator/workflow-runner-capability"; import type { DaemonMessage } from "../../src/shared/ws-messages"; // ─── Mocks ──────────────────────────────────────────────────────────────────── @@ -17,8 +18,12 @@ import type { DaemonMessage } from "../../src/shared/ws-messages"; const mockHandleDaemonMessage = mock((_ws: unknown, _msg: DaemonMessage) => {}); const mockHandleWsClose = mock(() => {}); const mockHandleWsOpen = mock(() => {}); +const mockBeginDaemonConnectionShutdown = mock(() => {}); +const mockDrainDisconnectCleanups = mock(() => Promise.resolve()); void mock.module("../../src/orchestrator/connection-handler", () => ({ + beginDaemonConnectionShutdown: mockBeginDaemonConnectionShutdown, + drainDisconnectCleanups: mockDrainDisconnectCleanups, handleDaemonMessage: mockHandleDaemonMessage, handleWsClose: mockHandleWsClose, handleWsOpen: mockHandleWsOpen, @@ -40,13 +45,14 @@ void mock.module("../../src/orchestrator/daemon-registry", () => ({ // history void mock.module("../../src/orchestrator/history", () => ({ + createExecution: mock(() => Promise.resolve("exec-id")), markExecutionOffered: mock(() => Promise.resolve()), markExecutionFailed: mock(() => Promise.resolve()), markExecutionRunning: mock(() => Promise.resolve()), markExecutionCompleted: mock(() => Promise.resolve()), getExecutionState: mock(() => Promise.resolve(null)), getOrphanedExecutions: mock(() => Promise.resolve([])), - requeueExecution: mock(() => Promise.resolve()), + requeueExecution: mock(() => Promise.resolve("released")), })); // job-dispatcher @@ -55,6 +61,7 @@ void mock.module("../../src/orchestrator/job-dispatcher", () => ({ removePendingOffer: mock(() => {}), handleJobAccept: mock(() => {}), handleJobReject: mock(() => Promise.resolve()), + releaseOfferQueueLease: mock(() => Promise.resolve()), inferRequiredTools: mock(() => []), selectDaemon: mock(() => Promise.resolve(null)), dispatchJob: mock(() => Promise.resolve(false)), @@ -116,6 +123,8 @@ beforeEach(() => { mockHandleDaemonMessage.mockClear(); mockHandleWsClose.mockClear(); mockHandleWsOpen.mockClear(); + mockBeginDaemonConnectionShutdown.mockClear(); + mockDrainDisconnectCleanups.mockClear(); }); describe("sendError", () => { @@ -296,12 +305,20 @@ describe("WebSocket auth (constant-time bearer comparator, #76)", () => { const { config } = await import("../../src/config"); const originalToken = config.daemonAuthToken; const originalPrevious = config.daemonAuthTokenPrevious; + const originalRunnerSecret = config.workflowRunnerCapabilitySecret; + const originalRunnerPrevious = config.workflowRunnerCapabilitySecretPrevious; const originalPort = config.wsPort; try { (config as { daemonAuthToken: string | undefined }).daemonAuthToken = primary; (config as { daemonAuthTokenPrevious: string | undefined }).daemonAuthTokenPrevious = previous; + ( + config as { workflowRunnerCapabilitySecret: string | undefined } + ).workflowRunnerCapabilitySecret = primary; + ( + config as { workflowRunnerCapabilitySecretPrevious: string | undefined } + ).workflowRunnerCapabilitySecretPrevious = previous; (config as { wsPort: number }).wsPort = 0; const { stopWebSocketServer, startWebSocketServer } = @@ -318,6 +335,12 @@ describe("WebSocket auth (constant-time bearer comparator, #76)", () => { (config as { daemonAuthToken: string | undefined }).daemonAuthToken = originalToken; (config as { daemonAuthTokenPrevious: string | undefined }).daemonAuthTokenPrevious = originalPrevious; + ( + config as { workflowRunnerCapabilitySecret: string | undefined } + ).workflowRunnerCapabilitySecret = originalRunnerSecret; + ( + config as { workflowRunnerCapabilitySecretPrevious: string | undefined } + ).workflowRunnerCapabilitySecretPrevious = originalRunnerPrevious; (config as { wsPort: number }).wsPort = originalPort; } } @@ -404,6 +427,50 @@ describe("WebSocket auth (constant-time bearer comparator, #76)", () => { }, ); }); + + it("accepts only an exact attempt capability on the workflow-runner path", async () => { + const runId = crypto.randomUUID(); + const attemptId = crypto.randomUUID(); + const path = `/ws/workflow-runner/${runId}/${attemptId}`; + await withServer("runner-root-secret", "old-runner-root", async (port) => { + const expiresAtMs = Date.now() + 60_000; + const current = deriveWorkflowRunnerCapability( + "runner-root-secret", + runId, + attemptId, + expiresAtMs, + ); + const previous = deriveWorkflowRunnerCapability( + "old-runner-root", + runId, + attemptId, + expiresAtMs, + ); + const wrongAttempt = deriveWorkflowRunnerCapability( + "runner-root-secret", + runId, + crypto.randomUUID(), + expiresAtMs, + ); + for (const token of [current, previous]) { + const accepted = await fetch(`http://localhost:${String(port)}${path}`, { + headers: { Authorization: `Bearer ${token}` }, + }); + expect(accepted.status).toBe(500); + } + for (const token of ["runner-root-secret", wrongAttempt]) { + const rejected = await fetch(`http://localhost:${String(port)}${path}`, { + headers: { Authorization: `Bearer ${token}` }, + }); + expect(rejected.status).toBe(401); + } + const encodedSlash = await fetch( + `http://localhost:${String(port)}/ws/workflow-runner/%2F/${attemptId}`, + { headers: { Authorization: `Bearer ${current}` } }, + ); + expect(encodedSlash.status).toBe(401); + }); + }); }); describe("stopWebSocketServer", () => { @@ -413,6 +480,25 @@ describe("stopWebSocketServer", () => { await stopWebSocketServer(); await stopWebSocketServer(); }); + + it("drains daemon disconnect fencing before shutdown completes", async () => { + const { config } = await import("../../src/config"); + const originalToken = config.daemonAuthToken; + const originalPort = config.wsPort; + try { + (config as { daemonAuthToken: string | undefined }).daemonAuthToken = "shutdown-test-token"; + (config as { wsPort: number }).wsPort = 0; + const { startWebSocketServer, stopWebSocketServer } = + await import("../../src/orchestrator/ws-server"); + startWebSocketServer(); + await stopWebSocketServer(); + expect(mockBeginDaemonConnectionShutdown).toHaveBeenCalledTimes(1); + expect(mockDrainDisconnectCleanups).toHaveBeenCalledTimes(1); + } finally { + (config as { daemonAuthToken: string | undefined }).daemonAuthToken = originalToken; + (config as { wsPort: number }).wsPort = originalPort; + } + }); }); describe("WebSocket message handler (integration via real server)", () => { diff --git a/test/preload.ts b/test/preload.ts index 302609cd..fd3a5674 100644 --- a/test/preload.ts +++ b/test/preload.ts @@ -28,6 +28,7 @@ delete process.env["GITHUB_PERSONAL_ACCESS_TOKEN"]; // validateDataLayerConfig). Individual tests that need to exercise missing // auth flip this back to undefined within a save/restore block. process.env["DAEMON_AUTH_TOKEN"] = "test-daemon-token"; +process.env["WORKFLOW_RUNNER_CAPABILITY_SECRET"] = "test-workflow-runner-capability-root-secret"; // A developer's .env may set TRIGGER_PHRASE to a local-dev variant // (e.g. @chrisleekr-bot-dev) which would leak into tests that hardcode @@ -59,5 +60,10 @@ setIfEmpty("ANTHROPIC_API_KEY", "test-anthropic-key"); // Post-dispatch-collapse, validateDataLayerConfig requires DATABASE_URL and // VALKEY_URL in server mode (ORCHESTRATOR_URL unset). Without these, config // load aborts before any test can run in CI, where no .env is present. -setIfEmpty("DATABASE_URL", "postgres://test:test@localhost:55432/test"); +const testDatabaseUrl = process.env["TEST_DATABASE_URL"]; +const databaseUrlFallback = + testDatabaseUrl === undefined || testDatabaseUrl === "" + ? "postgres://test:test@localhost:55432/test" + : testDatabaseUrl; +setIfEmpty("DATABASE_URL", databaseUrlFallback); setIfEmpty("VALKEY_URL", "redis://localhost:56379"); diff --git a/test/runner/main.test.ts b/test/runner/main.test.ts new file mode 100644 index 00000000..1c97c3a6 --- /dev/null +++ b/test/runner/main.test.ts @@ -0,0 +1,331 @@ +import { beforeEach, describe, expect, it, mock } from "bun:test"; + +import { expectToReject } from "../utils/assertions"; + +const executeWorkflowRunnerJob = mock((_job: unknown, _client: unknown, signal: AbortSignal) => { + signal.throwIfAborted(); + return Promise.resolve({ status: "succeeded" as const, state: {} }); +}); +const startupEvents: string[] = []; +const assertDaemonEnvironmentPrivate = mock(() => { + startupEvents.push("daemon-environment"); +}); +const assertWorkflowRunnerEnvironment = mock(() => { + startupEvents.push("runner-environment"); +}); +const assertCloudMetadataUnavailable = mock(() => { + startupEvents.push("metadata"); + return Promise.resolve(); +}); +const installFatalHandlers = mock(() => { + startupEvents.push("fatal-handlers"); +}); +const constructedClients: unknown[] = []; +const revokeInstallationTokenValue = mock(() => Promise.resolve(true)); + +void mock.module("../../src/config", () => ({ config: { workflowRunner: true } })); +void mock.module("../../src/daemon/process-boundary", () => ({ + assertDaemonEnvironmentPrivate, +})); +void mock.module("../../src/logger", () => ({ + installFatalHandlers, + logger: { + info: mock(() => undefined), + warn: mock(() => undefined), + error: mock(() => undefined), + debug: mock(() => undefined), + }, +})); +void mock.module("../../src/orchestrator/installation-token", () => ({ + revokeInstallationTokenValue, +})); +void mock.module("../../src/utils/log-redaction", () => ({ + redactErrorMessageOrFallback: (_err: unknown, fallback: string): string => fallback, +})); +class StaleWorkflowAttemptError extends Error {} +void mock.module("../../src/workflows/runs-store", () => ({ StaleWorkflowAttemptError })); +void mock.module("../../src/runner/process-boundary", () => ({ + assertCloudMetadataUnavailable, + assertWorkflowRunnerEnvironment, +})); +void mock.module("../../src/runner/workflow-executor", () => ({ executeWorkflowRunnerJob })); +void mock.module("../../src/runner/ws-client", () => ({ + WorkflowRunnerClient: class TestWorkflowRunnerClient { + readonly signal = new AbortController().signal; + readonly attemptId = "test-attempt"; + + constructor(options: unknown) { + startupEvents.push("client-constructed"); + constructedClients.push(options); + } + + connect(): void { + startupEvents.push("client-connect"); + } + + waitForJob(): Promise { + startupEvents.push("client-wait"); + return Promise.resolve(null); + } + + close(): void { + startupEvents.push("client-close"); + } + + cancel(): void { + startupEvents.push("client-cancel"); + } + }, +})); + +const { executeAndReportWorkflowRunnerJob, main, resultForWorkflowRunnerExecutionError } = + await import("../../src/runner/main"); + +const runId = "11111111-1111-4111-8111-111111111111"; +const attemptId = "22222222-2222-4222-8222-222222222222"; +const workflowRunnerToken = ["wfr1", "fixture"].join("."); + +function installWorkflowRunnerEnvironment(): () => void { + const original = { + runId: process.env.WORKFLOW_RUNNER_RUN_ID, + attemptId: process.env.WORKFLOW_RUNNER_ATTEMPT_ID, + token: process.env.WORKFLOW_RUNNER_TOKEN, + orchestratorUrl: process.env.ORCHESTRATOR_URL, + }; + process.env.WORKFLOW_RUNNER_RUN_ID = runId; + process.env.WORKFLOW_RUNNER_ATTEMPT_ID = attemptId; + process.env.WORKFLOW_RUNNER_TOKEN = workflowRunnerToken; + process.env.ORCHESTRATOR_URL = "wss://controller.example/ws"; + return () => { + if (original.runId === undefined) Reflect.deleteProperty(process.env, "WORKFLOW_RUNNER_RUN_ID"); + else process.env.WORKFLOW_RUNNER_RUN_ID = original.runId; + if (original.attemptId === undefined) { + Reflect.deleteProperty(process.env, "WORKFLOW_RUNNER_ATTEMPT_ID"); + } else process.env.WORKFLOW_RUNNER_ATTEMPT_ID = original.attemptId; + if (original.token === undefined) Reflect.deleteProperty(process.env, "WORKFLOW_RUNNER_TOKEN"); + else process.env.WORKFLOW_RUNNER_TOKEN = original.token; + if (original.orchestratorUrl === undefined) { + Reflect.deleteProperty(process.env, "ORCHESTRATOR_URL"); + } else process.env.ORCHESTRATOR_URL = original.orchestratorUrl; + }; +} + +function job(expiresAt: string): Record { + return { + context: {}, + installationToken: "ghs_scoped", + installationTokenExpiresAt: expiresAt, + attemptDeadlineAt: new Date(Date.parse(expiresAt) + 10 * 60_000).toISOString(), + workflowRun: { runId, workflowName: "implement" }, + }; +} + +function client(signal: AbortSignal): { + readonly signal: AbortSignal; + readonly attemptId: string; + sendResultUntilAck: ReturnType; + close: ReturnType; +} { + return { + signal, + attemptId, + sendResultUntilAck: mock(() => Promise.resolve()), + close: mock(() => undefined), + }; +} + +describe("workflow runner main result handling", () => { + beforeEach(() => { + startupEvents.length = 0; + constructedClients.length = 0; + assertDaemonEnvironmentPrivate.mockClear(); + assertWorkflowRunnerEnvironment.mockClear(); + assertCloudMetadataUnavailable.mockReset(); + assertCloudMetadataUnavailable.mockImplementation(() => { + startupEvents.push("metadata"); + return Promise.resolve(); + }); + installFatalHandlers.mockClear(); + executeWorkflowRunnerJob.mockReset(); + revokeInstallationTokenValue.mockClear(); + executeWorkflowRunnerJob.mockImplementation( + (_job: unknown, _client: unknown, signal: AbortSignal) => { + signal.throwIfAborted(); + return Promise.resolve({ status: "succeeded" as const, state: {} }); + }, + ); + }); + + it("reports a durable failed result when the token deadline fires", async () => { + const runner = client(new AbortController().signal); + + await executeAndReportWorkflowRunnerJob({ + job: job(new Date(Date.now() + 60_000).toISOString()) as never, + client: runner as never, + runId, + attemptId, + }); + + expect(runner.sendResultUntilAck).toHaveBeenCalledTimes(1); + expect(runner.sendResultUntilAck).toHaveBeenCalledWith( + expect.objectContaining({ + runId, + attemptId, + result: { + status: "failed", + reason: "Workflow runner execution deadline reached", + humanMessage: "Workflow runner stopped at its credential or attempt deadline.", + }, + }), + ); + expect(runner.close).toHaveBeenCalledTimes(1); + expect(revokeInstallationTokenValue).toHaveBeenCalledTimes(1); + }); + + it("keeps client-fence shutdown as a clean exit", async () => { + const controller = new AbortController(); + controller.abort(new Error("client fence expired")); + const runner = client(controller.signal); + + await executeAndReportWorkflowRunnerJob({ + job: job(new Date(Date.now() + 60 * 60_000).toISOString()) as never, + client: runner as never, + runId, + attemptId, + }); + + expect(runner.sendResultUntilAck).not.toHaveBeenCalled(); + expect(runner.close).toHaveBeenCalledTimes(1); + expect(revokeInstallationTokenValue).toHaveBeenCalledTimes(1); + }); + + it("revokes before publishing an ordinary terminal result", async () => { + const events: string[] = []; + revokeInstallationTokenValue.mockImplementationOnce(() => { + events.push("revoke"); + return Promise.resolve(true); + }); + const runner = client(new AbortController().signal); + runner.sendResultUntilAck.mockImplementationOnce(() => { + events.push("result"); + return Promise.resolve(); + }); + + await executeAndReportWorkflowRunnerJob({ + job: job(new Date(Date.now() + 60 * 60_000).toISOString()) as never, + client: runner as never, + runId, + attemptId, + }); + + expect(events).toEqual(["revoke", "result"]); + expect(revokeInstallationTokenValue).toHaveBeenCalledTimes(1); + }); + + it("still reports and closes when token revocation fails", async () => { + revokeInstallationTokenValue.mockResolvedValueOnce(false); + const runner = client(new AbortController().signal); + + await executeAndReportWorkflowRunnerJob({ + job: job(new Date(Date.now() + 60 * 60_000).toISOString()) as never, + client: runner as never, + runId, + attemptId, + }); + + expect(runner.sendResultUntilAck).toHaveBeenCalledTimes(1); + expect(revokeInstallationTokenValue).toHaveBeenCalledTimes(1); + expect(runner.close).toHaveBeenCalledTimes(1); + }); + + it("revokes after a successful hand-off without sending another result", async () => { + executeWorkflowRunnerJob.mockResolvedValueOnce({ + status: "handed-off", + state: { phase: "child-running" }, + childRunId: crypto.randomUUID(), + }); + const runner = client(new AbortController().signal); + + await executeAndReportWorkflowRunnerJob({ + job: job(new Date(Date.now() + 60 * 60_000).toISOString()) as never, + client: runner as never, + runId, + attemptId, + }); + + expect(runner.sendResultUntilAck).not.toHaveBeenCalled(); + expect(revokeInstallationTokenValue).toHaveBeenCalledTimes(1); + expect(runner.close).toHaveBeenCalledTimes(1); + }); + + it("gives permanent client fencing precedence over a simultaneous token deadline", () => { + expect( + resultForWorkflowRunnerExecutionError({ + err: new Error("deadline"), + shuttingDown: true, + tokenDeadlineExpired: true, + }), + ).toBeNull(); + }); + + it("maps a token deadline to a terminal result and a client signal to no result", () => { + expect( + resultForWorkflowRunnerExecutionError({ + err: new Error("deadline"), + shuttingDown: false, + tokenDeadlineExpired: true, + }), + ).toEqual({ + status: "failed", + reason: "Workflow runner execution deadline reached", + humanMessage: "Workflow runner stopped at its credential or attempt deadline.", + }); + expect( + resultForWorkflowRunnerExecutionError({ + err: new Error("SIGTERM"), + shuttingDown: true, + tokenDeadlineExpired: false, + }), + ).toBeNull(); + }); + + it("runs both environment guards and metadata validation before connecting", async () => { + const restoreEnvironment = installWorkflowRunnerEnvironment(); + try { + await main(); + } finally { + restoreEnvironment(); + } + + expect(startupEvents).toEqual([ + "runner-environment", + "daemon-environment", + "metadata", + "fatal-handlers", + "client-constructed", + "client-connect", + "client-wait", + "client-close", + ]); + expect(constructedClients).toEqual([ + { + url: "wss://controller.example/ws", + token: workflowRunnerToken, + runId, + attemptId, + }, + ]); + }); + + it("does not construct a WebSocket client when metadata validation fails", async () => { + const restoreEnvironment = installWorkflowRunnerEnvironment(); + assertCloudMetadataUnavailable.mockRejectedValueOnce(new Error("metadata reachable")); + + try { + await expectToReject(main(), "metadata reachable"); + } finally { + restoreEnvironment(); + } + expect(constructedClients).toEqual([]); + }); +}); diff --git a/test/runner/output-sanitizer.test.ts b/test/runner/output-sanitizer.test.ts new file mode 100644 index 00000000..6ac4cfff --- /dev/null +++ b/test/runner/output-sanitizer.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from "bun:test"; + +import { + configuredCredentialValues, + containsExactCredentialPropertyName, + containsExactCredentialValue, + redactExactValues, +} from "../../src/runner/output-sanitizer"; + +describe("workflow runner exact credential redaction", () => { + it("selects credential values without treating provider settings as secrets", () => { + expect( + configuredCredentialValues({ + AWS_REGION: "ap-southeast-2", + AWS_SECRET_ACCESS_KEY: "opaque-aws-secret-value", + CLAUDE_CODE_OAUTH_TOKEN: "opaque-oauth-token", + CLAUDE_MODEL: "claude-test", + }), + ).toEqual(["opaque-aws-secret-value", "opaque-oauth-token"]); + }); + + it("removes exact raw credentials from nested command and result values", () => { + const secret = "opaque-credential-value"; + expect( + redactExactValues( + { + humanMessage: `before ${secret} after`, + state: { report: secret, rows: [secret, 42] }, + }, + [secret], + ), + ).toEqual({ + humanMessage: "before after", + state: { report: "", rows: ["", 42] }, + }); + }); + + it("detects configured credentials used as nested property names", () => { + const values = [ + `ghs_${"a".repeat(36)}`, + "anthropic-opaque-token", + "aws-opaque-secret", + "wfr1.opaque-capability", + ]; + for (const secret of values) { + expect( + containsExactCredentialPropertyName({ safe: { [`prefix-${secret}-suffix`]: "x" } }, [ + secret, + ]), + ).toBe(true); + } + expect(containsExactCredentialPropertyName({ safe: { nested: "value" } }, values)).toBe(false); + }); + + it("detects common reversible encodings in nested values", () => { + const secret = "opaque-credential-value"; + const encoded = [ + Buffer.from(secret, "utf8").toString("base64"), + Buffer.from(secret, "utf8").toString("base64url"), + Buffer.from(secret, "utf8").toString("hex").toUpperCase(), + encodeURIComponent(secret), + ]; + for (const value of encoded) { + expect(containsExactCredentialValue({ nested: `before ${value} after` }, [secret])).toBe( + true, + ); + } + }); +}); diff --git a/test/runner/process-boundary.test.ts b/test/runner/process-boundary.test.ts new file mode 100644 index 00000000..ccd29624 --- /dev/null +++ b/test/runner/process-boundary.test.ts @@ -0,0 +1,146 @@ +import { afterEach, beforeEach, describe, expect, it, mock } from "bun:test"; + +import { + assertCloudMetadataUnavailable, + assertWorkflowRunnerEnvironment, + FORBIDDEN_RUNNER_ENV, +} from "../../src/runner/process-boundary"; +import { expectToReject } from "../utils/assertions"; + +const originalValues = new Map(); +const originalFetch = globalThis.fetch; +const providerEnv = [ + "CLAUDE_PROVIDER", + "CLAUDE_MODEL", + "ANTHROPIC_API_KEY", + "CLAUDE_CODE_OAUTH_TOKEN", + "AWS_REGION", + "AWS_PROFILE", + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + "AWS_BEARER_TOKEN_BEDROCK", + "ANTHROPIC_BEDROCK_BASE_URL", +] as const; +const allowedRunnerEnv = [...providerEnv, "ALLOWED_OWNERS", "WORKFLOW_RUNNER_TOKEN"] as const; +const metadataEndpoints = [ + "http://169.254.169.254/", + "http://[fd00:ec2::254]/", + "http://[fd20:ce::254]/", +] as const; + +beforeEach(() => { + for (const name of FORBIDDEN_RUNNER_ENV) { + originalValues.set(name, process.env[name]); + Reflect.deleteProperty(process.env, name); + } + for (const name of allowedRunnerEnv) { + originalValues.set(name, process.env[name]); + Reflect.deleteProperty(process.env, name); + } +}); + +afterEach(() => { + globalThis.fetch = originalFetch; + for (const name of [...FORBIDDEN_RUNNER_ENV, ...allowedRunnerEnv]) { + const value = originalValues.get(name); + if (value === undefined) { + Reflect.deleteProperty(process.env, name); + } else { + process.env[name] = value; + } + } + originalValues.clear(); +}); + +describe("workflow runner parent environment boundary", () => { + it("accepts provider-only and runner control variables", () => { + process.env["CLAUDE_PROVIDER"] = "anthropic"; + process.env["CLAUDE_MODEL"] = "claude-test"; + process.env["ANTHROPIC_API_KEY"] = "provider-key"; + process.env["ALLOWED_OWNERS"] = "owner"; + process.env["WORKFLOW_RUNNER_TOKEN"] = "attempt-capability"; + + expect(assertWorkflowRunnerEnvironment).not.toThrow(); + }); + + it("rejects dual Anthropic credentials", () => { + process.env["CLAUDE_PROVIDER"] = "anthropic"; + process.env["CLAUDE_MODEL"] = "claude-test"; + process.env["ANTHROPIC_API_KEY"] = "provider-key"; + process.env["CLAUDE_CODE_OAUTH_TOKEN"] = "provider-oauth"; + + expect(assertWorkflowRunnerEnvironment).toThrow("exactly one Anthropic credential"); + }); + + it("rejects an unselected provider credential", () => { + process.env["CLAUDE_PROVIDER"] = "anthropic"; + process.env["CLAUDE_MODEL"] = "claude-test"; + process.env["ANTHROPIC_API_KEY"] = "provider-key"; + process.env["AWS_BEARER_TOKEN_BEDROCK"] = "unselected-credential"; + + expect(assertWorkflowRunnerEnvironment).toThrow("no AWS credentials"); + }); + + it("accepts one Bedrock static credential chain", () => { + process.env["CLAUDE_PROVIDER"] = "bedrock"; + process.env["CLAUDE_MODEL"] = "bedrock-model"; + process.env["AWS_REGION"] = "ap-southeast-2"; + process.env["AWS_ACCESS_KEY_ID"] = "access"; + process.env["AWS_SECRET_ACCESS_KEY"] = "secret"; + process.env["AWS_SESSION_TOKEN"] = "session"; + + expect(assertWorkflowRunnerEnvironment).not.toThrow(); + }); + + for (const name of FORBIDDEN_RUNNER_ENV) { + it(`rejects ${name}`, () => { + process.env[name] = "must-not-reach-runner"; + expect(assertWorkflowRunnerEnvironment).toThrow(name); + }); + } + + it("does not reject empty forbidden variables", () => { + for (const name of FORBIDDEN_RUNNER_ENV) process.env[name] = " "; + process.env["CLAUDE_PROVIDER"] = "anthropic"; + process.env["CLAUDE_MODEL"] = "claude-test"; + process.env["ANTHROPIC_API_KEY"] = "provider-key"; + expect(assertWorkflowRunnerEnvironment).not.toThrow(); + }); +}); + +describe("workflow runner cloud metadata boundary", () => { + it("accepts a network where metadata endpoints are unreachable", async () => { + const requested: string[] = []; + globalThis.fetch = mock((input: string | URL | Request) => { + requested.push(requestUrl(input)); + return Promise.reject(new TypeError("unreachable")); + }) as typeof fetch; + await assertCloudMetadataUnavailable(); + expect(requested).toEqual(metadataEndpoints); + }); + + it("accepts metadata probes dropped until their timeout", async () => { + globalThis.fetch = mock(() => + Promise.reject(new DOMException("The operation timed out", "TimeoutError")), + ) as typeof fetch; + + await assertCloudMetadataUnavailable(); + }); + + for (const endpoint of metadataEndpoints) { + it(`rejects any HTTP response from ${endpoint}`, async () => { + globalThis.fetch = mock((input: string | URL | Request) => + requestUrl(input) === endpoint + ? Promise.resolve(new Response("", { status: 403 })) + : Promise.reject(new TypeError("unreachable")), + ) as typeof fetch; + + await expectToReject(assertCloudMetadataUnavailable(), endpoint); + }); + } +}); + +function requestUrl(input: string | URL | Request): string { + return typeof input === "string" ? input : input instanceof URL ? input.href : input.url; +} diff --git a/test/runner/token-deadline.test.ts b/test/runner/token-deadline.test.ts new file mode 100644 index 00000000..8b1a5e4f --- /dev/null +++ b/test/runner/token-deadline.test.ts @@ -0,0 +1,46 @@ +import { afterEach, describe, expect, it, jest } from "bun:test"; + +import { + createWorkflowRunnerDeadline, + workflowRunnerDeadlineDelayMs, + WorkflowRunnerDeadlineError, +} from "../../src/runner/token-deadline"; + +describe("installation token execution deadline", () => { + afterEach(() => { + jest.useRealTimers(); + }); + + it("fires five minutes before the authoritative expiry", () => { + jest.useFakeTimers(); + const now = Date.parse("2026-08-23T03:00:00Z"); + const expiresAt = "2026-08-23T04:00:00Z"; + const attemptDeadlineAt = "2026-08-23T04:10:00Z"; + expect(workflowRunnerDeadlineDelayMs(expiresAt, attemptDeadlineAt, now)).toBe(55 * 60_000); + const deadline = createWorkflowRunnerDeadline(expiresAt, attemptDeadlineAt, now); + + jest.advanceTimersByTime(55 * 60_000 - 1); + expect(deadline.signal.aborted).toBe(false); + jest.advanceTimersByTime(1); + expect(deadline.signal.aborted).toBe(true); + expect(deadline.signal.reason).toBeInstanceOf(WorkflowRunnerDeadlineError); + }); + + it("aborts immediately when the reporting buffer has already begun", () => { + const now = Date.parse("2026-08-23T03:56:00Z"); + const deadline = createWorkflowRunnerDeadline( + "2026-08-23T04:00:00Z", + "2026-08-23T04:10:00Z", + now, + ); + expect(deadline.signal.aborted).toBe(true); + expect(deadline.signal.reason).toBeInstanceOf(WorkflowRunnerDeadlineError); + }); + + it("clamps a freshly minted token to the durable attempt deadline", () => { + const now = Date.parse("2026-08-23T03:00:00Z"); + expect(workflowRunnerDeadlineDelayMs("2026-08-23T04:00:00Z", "2026-08-23T03:20:00Z", now)).toBe( + 20 * 60_000, + ); + }); +}); diff --git a/test/runner/workflow-executor.test.ts b/test/runner/workflow-executor.test.ts new file mode 100644 index 00000000..d2b4d9fa --- /dev/null +++ b/test/runner/workflow-executor.test.ts @@ -0,0 +1,189 @@ +import { beforeEach, describe, expect, it, mock } from "bun:test"; + +import { expectToReject } from "../utils/assertions"; + +const handler = mock((_context: unknown) => + Promise.resolve({ + status: "succeeded" as const, + state: { complete: true }, + daemonActions: { + learnings: [{ category: "setup" as const, content: "Run bun test per file." }], + deletions: [], + }, + }), +); + +void mock.module("octokit", () => ({ + Octokit: function MockOctokit(this: unknown): unknown { + return this; + }, +})); +void mock.module("../../src/logger", () => ({ + logger: { + child: mock(() => ({ + info: mock(() => undefined), + warn: mock(() => undefined), + error: mock(() => undefined), + debug: mock(() => undefined), + })), + }, +})); +void mock.module("../../src/workflows/registry", () => ({ + getByName: mock(() => ({ handler })), +})); + +const { executeWorkflowRunnerJob } = await import("../../src/runner/workflow-executor"); + +describe("workflow runner executor", () => { + beforeEach(() => { + handler.mockReset(); + handler.mockResolvedValue({ + status: "succeeded", + state: { complete: true }, + daemonActions: { + learnings: [{ category: "setup", content: "Run bun test per file." }], + deletions: [], + }, + }); + }); + + it("passes bounded repo memory to the handler and retains daemon actions", async () => { + const repoMemory = [ + { + id: "11111111-1111-4111-8111-111111111111", + category: "architecture" as const, + content: "The orchestrator owns durable state.", + pinned: true, + }, + ]; + const result = await executeWorkflowRunnerJob( + { + context: { + owner: "acme", + repo: "widgets", + entityNumber: 16, + isPR: false, + deliveryId: "delivery-16", + }, + installationToken: "ghs_scoped", + installationTokenExpiresAt: "2026-08-23T04:00:00Z", + attemptDeadlineAt: "2026-08-23T04:10:00Z", + repoMemory, + workflowRun: { + runId: "22222222-2222-4222-8222-222222222222", + workflowName: "implement", + }, + }, + { + attemptId: "33333333-3333-4333-8333-333333333333", + command: mock(() => Promise.resolve({})), + } as never, + new AbortController().signal, + ); + + const context = handler.mock.calls[0]?.[0] as { repoMemory?: unknown } | undefined; + expect(context?.repoMemory).toEqual(repoMemory); + expect("daemonActions" in result ? result.daemonActions : undefined).toEqual({ + learnings: [{ category: "setup", content: "Run bun test per file." }], + deletions: [], + }); + }); + + it("maps bounded payload fields and controller commands into the handler context", async () => { + const command = mock((input: { type: string }) => + Promise.resolve(input.type === "hand-off-child" ? { childRunId: crypto.randomUUID() } : {}), + ); + const signal = new AbortController().signal; + handler.mockImplementationOnce(async (context: unknown) => { + const runContext = context as { + setState: (state: unknown, message: string) => Promise; + handOffChild: (input: Record) => Promise<{ childRunId: string }>; + }; + await runContext.setState({ phase: "working" }, "Working."); + await runContext.handOffChild({ + workflowName: "review", + target: { type: "pr", owner: "acme", repo: "widgets", number: 42 }, + parentStepIndex: 3, + state: {}, + humanMessage: "Review queued.", + }); + return { status: "succeeded" as const, state: {} }; + }); + + await executeWorkflowRunnerJob( + { + context: { + owner: "acme", + repo: "widgets", + entityNumber: 16, + isPR: false, + deliveryId: "delivery-16", + }, + installationToken: "ghs_scoped", + installationTokenExpiresAt: "2026-08-23T04:00:00Z", + attemptDeadlineAt: "2026-08-23T04:10:00Z", + maxTurns: 50, + policy: { model: "claude-test" }, + priorPlanState: { plan: "Approved plan" }, + reviewLearnings: [ + { + id: crypto.randomUUID(), + scope: "local", + fileGlob: null, + directive: "Keep the controller authoritative.", + rationale: null, + sourcePr: null, + sourceThread: null, + sourceAuthor: null, + createdAt: "2026-08-23T00:00:00Z", + }, + ], + shipStepRuns: {}, + workflowRun: { + runId: crypto.randomUUID(), + workflowName: "implement", + }, + }, + { attemptId: crypto.randomUUID(), command } as never, + signal, + ); + + const context = handler.mock.calls[0]?.[0] as Record; + expect(context["maxTurns"]).toBe(50); + expect(context["policy"]).toEqual({ model: "claude-test" }); + expect(context["priorPlanState"]).toEqual({ plan: "Approved plan" }); + expect(context["reviewLearnings"]).toHaveLength(1); + expect(context["shipStepRuns"]).toEqual({}); + expect(context["signal"]).toBe(signal); + expect(command.mock.calls.map((call) => call[0]?.type)).toEqual([ + "set-state", + "hand-off-child", + ]); + }); + + it("does not enter a handler after the attempt is fenced", async () => { + const controller = new AbortController(); + controller.abort(new Error("attempt fenced")); + await expectToReject( + executeWorkflowRunnerJob( + { + context: { + owner: "acme", + repo: "widgets", + entityNumber: 16, + isPR: false, + deliveryId: "delivery-16", + }, + installationToken: "ghs_scoped", + installationTokenExpiresAt: "2026-08-23T04:00:00Z", + attemptDeadlineAt: "2026-08-23T04:10:00Z", + workflowRun: { runId: crypto.randomUUID(), workflowName: "implement" }, + }, + { attemptId: crypto.randomUUID(), command: mock(() => Promise.resolve({})) } as never, + controller.signal, + ), + "attempt fenced", + ); + expect(handler).not.toHaveBeenCalled(); + }); +}); diff --git a/test/runner/ws-client.test.ts b/test/runner/ws-client.test.ts new file mode 100644 index 00000000..5aa4893b --- /dev/null +++ b/test/runner/ws-client.test.ts @@ -0,0 +1,434 @@ +import { afterEach, beforeEach, describe, expect, it, jest, mock } from "bun:test"; + +import { expectToReject } from "../utils/assertions"; + +void mock.module("../../src/logger", () => ({ + logger: { + info: mock(() => {}), + warn: mock(() => {}), + error: mock(() => {}), + debug: mock(() => {}), + }, +})); + +class StaleWorkflowAttemptError extends Error {} +void mock.module("../../src/workflows/runs-store", () => ({ StaleWorkflowAttemptError })); + +const { WORKFLOW_RUNNER_MESSAGE_MAX_BYTES } = + await import("../../src/shared/workflow-runner-messages"); +const { WorkflowRunnerClient } = await import("../../src/runner/ws-client"); + +class FakeWebSocket { + static readonly CONNECTING = 0; + static readonly OPEN = 1; + static readonly CLOSING = 2; + static readonly CLOSED = 3; + + static instances: FakeWebSocket[] = []; + + readyState = FakeWebSocket.CONNECTING; + onopen: ((event: Event) => void) | null = null; + onmessage: ((event: MessageEvent) => void) | null = null; + onclose: ((event: CloseEvent) => void) | null = null; + onerror: ((event: Event) => void) | null = null; + readonly sent: string[] = []; + + constructor( + public readonly url: string, + public readonly options?: { headers?: Record }, + ) { + FakeWebSocket.instances.push(this); + } + + send(data: string): void { + this.sent.push(data); + } + + fireOpen(): void { + this.readyState = FakeWebSocket.OPEN; + this.onopen?.({} as Event); + } + + fireMessage(message: unknown): void { + this.onmessage?.({ data: JSON.stringify(message) } as MessageEvent); + } + + fireClose(code = 1006, reason = "abnormal closure"): void { + this.readyState = FakeWebSocket.CLOSED; + this.onclose?.({ code, reason, wasClean: false } as unknown as CloseEvent); + } + + close(): void { + this.readyState = FakeWebSocket.CLOSED; + } +} + +const runId = "11111111-1111-4111-8111-111111111111"; +const attemptId = "22222222-2222-4222-8222-222222222222"; + +function makeClient(): InstanceType { + return new WorkflowRunnerClient({ + url: `ws://controller/ws/workflow-runner/${runId}/${attemptId}`, + token: "test-token", + runId, + attemptId, + }); +} + +function readyMessage(clientFenceMs = 25, heartbeatIntervalMs = 10, includeJob = true): unknown { + return { + type: "workflow-runner:registered", + id: crypto.randomUUID(), + timestamp: Date.now(), + payload: { + state: "ready", + heartbeatIntervalMs, + clientFenceMs, + dbLeaseMs: clientFenceMs * 2, + ...(includeJob + ? { + job: { + context: {}, + installationToken: "installation-token", + installationTokenExpiresAt: "2026-08-23T04:00:00Z", + attemptDeadlineAt: "2026-08-23T04:10:00Z", + workflowRun: { runId, workflowName: "implement" }, + }, + } + : {}), + }, + }; +} + +function sentMessage( + socket: FakeWebSocket, + type: string, +): { type: string; id: string; payload: Record } { + const message = socket.sent + .map((raw) => JSON.parse(raw) as { type: string; id: string; payload: Record }) + .find((candidate) => candidate.type === type); + if (message === undefined) throw new Error(`Expected ${type} message`); + return message; +} + +describe("WorkflowRunnerClient attempt fence", () => { + let originalWebSocket: typeof WebSocket; + + beforeEach(() => { + originalWebSocket = globalThis.WebSocket; + (globalThis as { WebSocket: unknown }).WebSocket = FakeWebSocket; + FakeWebSocket.instances = []; + }); + + afterEach(() => { + jest.useRealTimers(); + (globalThis as { WebSocket: unknown }).WebSocket = originalWebSocket; + }); + + it("keeps the acknowledged fence armed while reconnecting", async () => { + jest.useFakeTimers(); + const client = makeClient(); + client.connect(); + const socket = FakeWebSocket.instances[0]; + if (socket === undefined) throw new Error("Expected workflow runner socket"); + socket.fireOpen(); + expect(sentMessage(socket, "workflow-runner:register").payload["capabilities"]).toBeUndefined(); + expect(sentMessage(socket, "workflow-runner:register").payload["needsJob"]).toBe(true); + socket.fireMessage(readyMessage()); + await client.waitForJob(); + + socket.fireClose(); + jest.advanceTimersByTime(24); + expect(client.signal.aborted).toBe(false); + jest.advanceTimersByTime(1); + expect(client.signal.aborted).toBe(true); + + client.close(); + }); + + it("fails closed when the initial registration omits the job payload", async () => { + const client = makeClient(); + client.connect(); + const socket = FakeWebSocket.instances[0]; + if (socket === undefined) throw new Error("Expected workflow runner socket"); + socket.fireOpen(); + + socket.fireMessage(readyMessage(2_500, 500, false)); + + expect(await client.waitForJob()).toBeNull(); + expect(client.signal.aborted).toBe(true); + }); + + it("replays stable command and result messages after reconnecting before the fence", async () => { + jest.useFakeTimers(); + const client = makeClient(); + client.connect(); + const first = FakeWebSocket.instances[0]; + if (first === undefined) throw new Error("Expected first workflow runner socket"); + first.fireOpen(); + first.fireMessage(readyMessage(2_500, 500)); + await client.waitForJob(); + + const commandPromise = client.command({ + type: "set-state", + patch: { phase: "implementing" }, + humanMessage: "Implementing.", + }); + const originalCommand = sentMessage(first, "workflow-runner:command"); + first.fireClose(); + jest.advanceTimersByTime(1_000); + + const second = FakeWebSocket.instances[1]; + if (second === undefined) throw new Error("Expected reconnected workflow runner socket"); + second.fireOpen(); + expect(sentMessage(second, "workflow-runner:register").payload["needsJob"]).toBe(false); + second.fireMessage(readyMessage(2_500, 500, false)); + const replayedCommand = sentMessage(second, "workflow-runner:command"); + expect(replayedCommand).toEqual(originalCommand); + jest.advanceTimersByTime(1_600); + expect(client.signal.aborted).toBe(false); + second.fireMessage({ + type: "workflow-runner:heartbeat-ack", + id: crypto.randomUUID(), + timestamp: Date.now(), + payload: { renewed: true }, + }); + second.fireMessage({ + type: "workflow-runner:command-result", + id: originalCommand.id, + timestamp: Date.now(), + payload: { ok: true, result: { trackingCommentId: 42 } }, + }); + expect(await commandPromise).toEqual({ trackingCommentId: 42 }); + + const resultPromise = client.sendResultUntilAck({ + runId, + attemptId, + result: { status: "succeeded", state: { phase: "done" } }, + durationMs: 1_000, + }); + const originalResult = sentMessage(second, "workflow-runner:result"); + second.fireClose(); + jest.advanceTimersByTime(1_000); + + const third = FakeWebSocket.instances[2]; + if (third === undefined) throw new Error("Expected second reconnected workflow runner socket"); + third.fireOpen(); + expect(sentMessage(third, "workflow-runner:register").payload["needsJob"]).toBe(false); + third.fireMessage(readyMessage(2_500, 500, false)); + const replayedResult = sentMessage(third, "workflow-runner:result"); + expect(replayedResult).toEqual(originalResult); + third.fireMessage({ + type: "workflow-runner:result-ack", + id: attemptId, + timestamp: Date.now(), + payload: {}, + }); + await resultPromise; + expect(client.signal.aborted).toBe(false); + + client.close(); + }); + + it("rejects exact provider and repository credentials before transport", async () => { + const client = makeClient(); + const secret = "opaque-provider-credential"; + client.addSensitiveValue(secret); + client.connect(); + const socket = FakeWebSocket.instances[0]; + if (socket === undefined) throw new Error("Expected workflow runner socket"); + socket.fireOpen(); + socket.fireMessage(readyMessage(2_500, 500)); + await client.waitForJob(); + + await expectToReject( + client.command({ + type: "set-state", + patch: { report: `before ${secret} after` }, + humanMessage: `safe ${secret} text`, + }), + "credential policy", + ); + + const resultPromise = client.sendResultUntilAck({ + runId, + attemptId, + result: { + status: "succeeded", + state: { report: secret }, + humanMessage: `done ${secret}`, + }, + durationMs: 1, + }); + const result = sentMessage(socket, "workflow-runner:result"); + expect(JSON.stringify(result)).not.toContain(secret); + expect(result.payload).toMatchObject({ + result: { + status: "failed", + reason: "workflow runner output was rejected by credential policy", + }, + }); + socket.fireMessage({ + type: "workflow-runner:result-ack", + id: attemptId, + timestamp: Date.now(), + payload: {}, + }); + await resultPromise; + client.close(); + }); + + it("rejects credential-bearing property names before transport", async () => { + const client = makeClient(); + const secret = "opaque-provider-credential"; + client.addSensitiveValue(secret); + client.connect(); + const socket = FakeWebSocket.instances[0]; + if (socket === undefined) throw new Error("Expected workflow runner socket"); + socket.fireOpen(); + socket.fireMessage(readyMessage(2_500, 500)); + await client.waitForJob(); + + await expectToReject( + client.command({ + type: "set-state", + patch: { nested: { [`credential-${secret}`]: "value" } }, + humanMessage: "Updating state.", + }), + "Workflow runner command was rejected by credential policy", + ); + expect( + socket.sent.some( + (raw) => (JSON.parse(raw) as { type?: unknown }).type === "workflow-runner:command", + ), + ).toBe(false); + + const resultPromise = client.sendResultUntilAck({ + runId, + attemptId, + result: { + status: "succeeded", + state: { nested: { [`credential-${secret}`]: "value" } }, + }, + durationMs: 1, + }); + const result = sentMessage(socket, "workflow-runner:result"); + expect(JSON.stringify(result)).not.toContain(secret); + expect(result.payload).toMatchObject({ + result: { + status: "failed", + reason: "workflow runner output was rejected by credential policy", + }, + }); + socket.fireMessage({ + type: "workflow-runner:result-ack", + id: attemptId, + timestamp: Date.now(), + payload: {}, + }); + await resultPromise; + client.close(); + }); + + it("bounds command and terminal result payloads before transport", async () => { + const client = makeClient(); + client.connect(); + const socket = FakeWebSocket.instances[0]; + if (socket === undefined) throw new Error("Expected workflow runner socket"); + socket.fireOpen(); + socket.fireMessage(readyMessage(2_500, 500)); + await client.waitForJob(); + const oversized = "x".repeat(WORKFLOW_RUNNER_MESSAGE_MAX_BYTES + 1); + + await expectToReject( + client.command({ + type: "set-state", + patch: { report: oversized }, + humanMessage: "Updating state.", + }), + "Workflow runner command was rejected by credential policy", + ); + expect( + socket.sent.some( + (raw) => (JSON.parse(raw) as { type?: unknown }).type === "workflow-runner:command", + ), + ).toBe(false); + + const resultPromise = client.sendResultUntilAck({ + runId, + attemptId, + result: { status: "succeeded", state: { report: oversized } }, + durationMs: 1, + }); + const result = sentMessage(socket, "workflow-runner:result"); + expect(Buffer.byteLength(JSON.stringify(result.payload), "utf8")).toBeLessThanOrEqual( + WORKFLOW_RUNNER_MESSAGE_MAX_BYTES, + ); + expect(result.payload).toMatchObject({ + result: { + status: "failed", + reason: "workflow runner output was rejected by credential policy", + }, + }); + socket.fireMessage({ + type: "workflow-runner:result-ack", + id: attemptId, + timestamp: Date.now(), + payload: {}, + }); + await resultPromise; + client.close(); + }); + + it("treats a policy close before registration as permanent", async () => { + jest.useFakeTimers(); + const client = makeClient(); + client.connect(); + const socket = FakeWebSocket.instances[0]; + if (socket === undefined) throw new Error("Expected workflow runner socket"); + socket.fireOpen(); + + socket.fireClose(1008, "stale workflow runner attempt"); + + expect(await client.waitForJob()).toBeNull(); + expect(client.signal.aborted).toBe(true); + jest.advanceTimersByTime(30_000); + expect(FakeWebSocket.instances).toHaveLength(1); + }); + + it("rejects pending callers and stops activity on a permanent abort", async () => { + jest.useFakeTimers(); + const client = makeClient(); + client.connect(); + const socket = FakeWebSocket.instances[0]; + if (socket === undefined) throw new Error("Expected workflow runner socket"); + socket.fireOpen(); + socket.fireMessage(readyMessage(2_500, 500)); + await client.waitForJob(); + + const commandPromise = client.command({ + type: "set-state", + patch: { phase: "implementing" }, + humanMessage: "Implementing.", + }); + const resultPromise = client.sendResultUntilAck({ + runId, + attemptId, + result: { status: "succeeded", state: { phase: "done" } }, + durationMs: 1_000, + }); + const sentBeforeAbort = socket.sent.length; + + client.cancel(new Error("workflow attempt fenced")); + + await expectToReject(commandPromise, "workflow attempt fenced"); + await expectToReject(resultPromise, "workflow attempt fenced"); + await expectToReject( + client.command({ type: "set-state", patch: {}, humanMessage: "Too late." }), + "workflow attempt fenced", + ); + jest.advanceTimersByTime(30_000); + expect(socket.sent).toHaveLength(sentBeforeAbort); + expect(FakeWebSocket.instances).toHaveLength(1); + }); +}); diff --git a/test/scripts/check-test-globs.test.ts b/test/scripts/check-test-globs.test.ts index 7cf7664b..87373e56 100644 --- a/test/scripts/check-test-globs.test.ts +++ b/test/scripts/check-test-globs.test.ts @@ -53,8 +53,8 @@ afterEach(() => { describe("scripts/check-test-globs.ts", () => { it("exits 0 when every test file is covered by the runner globs", () => { const root = makeFixture({ - runnerGlobs: "test/**/*.test.ts src/**/*.test.ts", - testFiles: ["test/foo.test.ts", "test/nested/bar.test.ts", "src/scheduler/baz.test.ts"], + runnerGlobs: "test/**/*.test.ts", + testFiles: ["test/foo.test.ts", "test/nested/bar.test.ts"], }); fixtures.push(root); const { exitCode, stdout } = runScript(root); @@ -63,8 +63,8 @@ describe("scripts/check-test-globs.ts", () => { }); it("exits 1 and names a test file living outside the globbed roots", () => { - // This is the issue #201 shape: runner globs only `test/`, a colocated - // `src/**/*.test.ts` is silently dark. + // The canonical runner glob covers only `test/`; a colocated source test + // must be reported instead of silently going dark. const root = makeFixture({ runnerGlobs: "test/**/*.test.ts", testFiles: ["test/foo.test.ts", "src/scheduler/due-evaluator.test.ts"], diff --git a/test/shared/dispatch-types.test.ts b/test/shared/dispatch-types.test.ts index d49c7e95..68cac1e6 100644 --- a/test/shared/dispatch-types.test.ts +++ b/test/shared/dispatch-types.test.ts @@ -10,12 +10,13 @@ import { } from "../../src/shared/dispatch-types"; describe("DispatchTarget", () => { - it("exposes the daemon singleton after the dispatch collapse", () => { - expect(DISPATCH_TARGETS).toEqual(["daemon"]); + it("exposes the shared-daemon and isolated-runner protocols", () => { + expect(DISPATCH_TARGETS).toEqual(["daemon", "workflow-runner"]); }); it("Zod schema accepts 'daemon'", () => { expect(DispatchTargetSchema.safeParse("daemon").success).toBe(true); + expect(DispatchTargetSchema.safeParse("workflow-runner").success).toBe(true); }); it("Zod schema rejects removed legacy targets", () => { @@ -30,8 +31,9 @@ describe("DispatchTarget", () => { } }); - it("isDispatchTarget accepts 'daemon' and rejects everything else", () => { + it("isDispatchTarget accepts both current protocols and rejects everything else", () => { expect(isDispatchTarget("daemon")).toBe(true); + expect(isDispatchTarget("workflow-runner")).toBe(true); for (const bogus of ["inline", "shared-runner", "isolated-job", "", 42, null, {}, []]) { expect(isDispatchTarget(bogus)).toBe(false); } @@ -39,12 +41,13 @@ describe("DispatchTarget", () => { }); describe("DispatchReason", () => { - it("exposes exactly the four canonical reasons in documented order", () => { + it("exposes every canonical reason in documented order", () => { expect(DISPATCH_REASONS).toEqual([ "persistent-daemon", "ephemeral-daemon-triage", "ephemeral-daemon-overflow", "ephemeral-spawn-failed", + "workflow-runner", ]); }); diff --git a/src/shared/ws-messages.test.ts b/test/shared/scoped-ws-messages.test.ts similarity index 91% rename from src/shared/ws-messages.test.ts rename to test/shared/scoped-ws-messages.test.ts index 52490a43..895cc1bb 100644 --- a/src/shared/ws-messages.test.ts +++ b/test/shared/scoped-ws-messages.test.ts @@ -7,12 +7,12 @@ import { type ScopedJobOfferMessage, serverMessageSchema, WS_REJECT_REASONS, -} from "./ws-messages"; +} from "../../src/shared/ws-messages"; -describe("scoped-job-offer schema", () => { +describe("scoped-job:offer schema", () => { function buildOffer(overrides?: Partial): unknown { return { - type: "scoped-job-offer", + type: "scoped-job:offer", ...createMessageEnvelope(), payload: { jobKind: "scoped-rebase", @@ -35,7 +35,7 @@ describe("scoped-job-offer schema", () => { it("requires threadRef for scoped-fix-thread", () => { const missing = serverMessageSchema.safeParse({ - type: "scoped-job-offer", + type: "scoped-job:offer", ...createMessageEnvelope(), payload: { jobKind: "scoped-fix-thread", @@ -54,7 +54,7 @@ describe("scoped-job-offer schema", () => { it("requires issueNumber + verdictSummary for scoped-open-pr", () => { const missing = serverMessageSchema.safeParse({ - type: "scoped-job-offer", + type: "scoped-job:offer", ...createMessageEnvelope(), payload: { jobKind: "scoped-open-pr", @@ -72,7 +72,7 @@ describe("scoped-job-offer schema", () => { it("rejects unknown jobKind values at the discriminator", () => { const wrong = serverMessageSchema.safeParse({ - type: "scoped-job-offer", + type: "scoped-job:offer", ...createMessageEnvelope(), payload: { jobKind: "scoped-mystery", @@ -88,12 +88,12 @@ describe("scoped-job-offer schema", () => { }); }); -describe("scoped-job-completion schema", () => { +describe("scoped-job:completion schema", () => { function buildCompletion( payload: ScopedJobCompletionMessage["payload"], ): ScopedJobCompletionMessage { return { - type: "scoped-job-completion", + type: "scoped-job:completion", ...createMessageEnvelope(), payload, }; @@ -126,7 +126,7 @@ describe("scoped-job-completion schema", () => { it("rejects malformed rebaseOutcome (missing commentId)", () => { const wrong = daemonMessageSchema.safeParse({ - type: "scoped-job-completion", + type: "scoped-job:completion", ...createMessageEnvelope(), payload: { offerId: "offer-3", diff --git a/test/shared/workflow-runner-messages.test.ts b/test/shared/workflow-runner-messages.test.ts new file mode 100644 index 00000000..247f7eed --- /dev/null +++ b/test/shared/workflow-runner-messages.test.ts @@ -0,0 +1,264 @@ +import { describe, expect, it } from "bun:test"; + +import { + WORKFLOW_RUNNER_MESSAGE_MAX_BYTES, + WorkflowRunnerCommandSchema, + WorkflowRunnerPayloadSchema, + WorkflowRunnerResultPayloadSchema, +} from "../../src/shared/workflow-runner-messages"; +import { + HandlerResultSchema, + WORKFLOW_RUNNER_HUMAN_MESSAGE_MAX_CHARS, +} from "../../src/shared/workflow-types"; + +function repoMemoryEntries(count: number): Record[] { + return Array.from({ length: count }, () => ({ + id: crypto.randomUUID(), + category: "setup", + content: "Run the focused tests.", + pinned: false, + })); +} + +function repoLearningActions(count: number): Record[] { + return Array.from({ length: count }, () => ({ + category: "setup", + content: "bounded", + })); +} + +function reviewLearnings(count: number): Record[] { + return Array.from({ length: count }, () => ({ + id: crypto.randomUUID(), + scope: "local", + fileGlob: null, + directive: "Bound this list.", + rationale: null, + sourcePr: null, + sourceThread: null, + sourceAuthor: null, + })); +} + +describe("WorkflowRunnerPayloadSchema", () => { + it("retains handler inputs and strips legacy shared-daemon inputs", () => { + const payload = WorkflowRunnerPayloadSchema.parse({ + context: { owner: "acme", repo: "widgets", entityNumber: 16 }, + installationToken: "target-repository-token", + installationTokenExpiresAt: "2026-08-23T04:00:00Z", + attemptDeadlineAt: "2026-08-23T04:10:00Z", + repoMemory: [ + { + id: "11111111-1111-4111-8111-111111111111", + category: "architecture", + content: "Uses a controller and isolated runners.", + pinned: true, + }, + ], + installationId: 123, + allowedTools: ["Read"], + envVars: { REPO_SECRET: "must-not-cross-boundary" }, + memory: [{ id: "memory-1", category: "env", content: "secret", pinned: false }], + maxTurns: 20, + reviewLearnings: [ + { + id: "learning-1", + scope: "local", + fileGlob: null, + directive: "Keep the boundary narrow.", + rationale: null, + sourcePr: null, + sourceThread: null, + sourceAuthor: null, + }, + ], + policy: { model: "provider-model", timeoutMs: 60_000 }, + workflowRun: { runId: crypto.randomUUID(), workflowName: "implement" }, + priorPlanState: { plan: "Implement the approved change." }, + }); + + expect(payload).toEqual({ + context: { owner: "acme", repo: "widgets", entityNumber: 16 }, + installationToken: "target-repository-token", + installationTokenExpiresAt: "2026-08-23T04:00:00Z", + attemptDeadlineAt: "2026-08-23T04:10:00Z", + repoMemory: [ + { + id: "11111111-1111-4111-8111-111111111111", + category: "architecture", + content: "Uses a controller and isolated runners.", + pinned: true, + }, + ], + maxTurns: 20, + reviewLearnings: [ + { + id: "learning-1", + scope: "local", + fileGlob: null, + directive: "Keep the boundary narrow.", + rationale: null, + sourcePr: null, + sourceThread: null, + sourceAuthor: null, + }, + ], + policy: { model: "provider-model", timeoutMs: 60_000 }, + workflowRun: expect.objectContaining({ workflowName: "implement" }), + priorPlanState: { plan: "Implement the approved change." }, + }); + expect("installationId" in payload).toBe(false); + expect("allowedTools" in payload).toBe(false); + expect("envVars" in payload).toBe(false); + expect("memory" in payload).toBe(false); + }); + + it("caps repo memory at 50 entries", () => { + expect(() => + WorkflowRunnerPayloadSchema.parse({ + context: {}, + installationToken: "token", + installationTokenExpiresAt: "2026-08-23T04:00:00Z", + attemptDeadlineAt: "2026-08-23T04:10:00Z", + repoMemory: repoMemoryEntries(51), + workflowRun: { runId: crypto.randomUUID(), workflowName: "implement" }, + }), + ).toThrow(); + }); + + it("bounds review learnings and strips unapproved historical state", () => { + const runId = crypto.randomUUID(); + const base = { + context: {}, + installationToken: "token", + installationTokenExpiresAt: "2026-08-23T04:00:00Z", + attemptDeadlineAt: "2026-08-23T04:10:00Z", + workflowRun: { runId, workflowName: "ship" }, + }; + const parsed = WorkflowRunnerPayloadSchema.parse({ + ...base, + priorPlanState: { plan: "Approved plan.", hidden: "drop-me" }, + shipStepRuns: { + triage: { + id: crypto.randomUUID(), + status: "succeeded", + state: { recommendedNext: "plan", hidden: "drop-me" }, + createdAt: "2026-08-23T04:00:00Z", + }, + }, + }); + expect(parsed.priorPlanState).toEqual({ plan: "Approved plan." }); + expect(parsed.shipStepRuns?.triage?.state).toEqual({ recommendedNext: "plan" }); + + expect(() => + WorkflowRunnerPayloadSchema.parse({ + ...base, + reviewLearnings: reviewLearnings(51), + }), + ).toThrow(); + }); +}); + +describe("HandlerResultSchema daemon actions", () => { + const actions = { + learnings: [{ category: "gotchas" as const, content: "Do not skip isolated tests." }], + deletions: ["22222222-2222-4222-8222-222222222222"], + reviewLearningSaves: [ + { + directive: "Keep result settlement retryable.", + rationale: "Persistence can fail transiently.", + }, + ], + reviewLearningDeletes: ["33333333-3333-4333-8333-333333333333"], + }; + + it("retains bounded daemon actions on terminal results", () => { + const parsed = HandlerResultSchema.parse({ + status: "failed", + reason: "pipeline failed", + daemonActions: actions, + }); + expect("daemonActions" in parsed ? parsed.daemonActions : undefined).toEqual(actions); + }); + + it("rejects actions on hand-off and arrays above 50", () => { + expect(() => + HandlerResultSchema.parse({ + status: "handed-off", + childRunId: crypto.randomUUID(), + daemonActions: actions, + }), + ).toThrow(); + expect(() => + HandlerResultSchema.parse({ + status: "succeeded", + state: {}, + daemonActions: { + learnings: repoLearningActions(51), + deletions: [], + }, + }), + ).toThrow(); + }); +}); + +describe("workflow runner outbound bounds", () => { + it("rejects controller-reserved state keys on every runner-owned state path", () => { + for (const key of ["_configNotice", "_lastHumanMessage"]) { + expect(() => + WorkflowRunnerCommandSchema.parse({ + type: "set-state", + patch: { [key]: "spoofed" }, + humanMessage: "progress", + }), + ).toThrow("controller-reserved"); + expect(() => + WorkflowRunnerCommandSchema.parse({ + type: "hand-off-child", + workflowName: "review", + target: { type: "pr", owner: "acme", repo: "widgets", number: 1 }, + parentStepIndex: 3, + state: { [key]: "spoofed" }, + humanMessage: "handoff", + }), + ).toThrow("controller-reserved"); + expect(() => + WorkflowRunnerResultPayloadSchema.parse({ + runId: crypto.randomUUID(), + attemptId: crypto.randomUUID(), + durationMs: 1, + result: { status: "succeeded", state: { [key]: "spoofed" } }, + }), + ).toThrow("controller-reserved"); + } + }); + + it("rejects human messages that cannot fit safely in a GitHub projection", () => { + expect(() => + HandlerResultSchema.parse({ + status: "succeeded", + state: {}, + humanMessage: "x".repeat(WORKFLOW_RUNNER_HUMAN_MESSAGE_MAX_CHARS + 1), + }), + ).toThrow(); + }); + + it("rejects command and result JSON above the WebSocket budget", () => { + const oversized = "x".repeat(WORKFLOW_RUNNER_MESSAGE_MAX_BYTES); + expect(() => + WorkflowRunnerCommandSchema.parse({ + type: "set-state", + patch: { oversized }, + humanMessage: "progress", + }), + ).toThrow("byte budget"); + expect(() => + WorkflowRunnerResultPayloadSchema.parse({ + runId: crypto.randomUUID(), + attemptId: crypto.randomUUID(), + durationMs: 1, + result: { status: "succeeded", state: { oversized } }, + }), + ).toThrow("byte budget"); + }); +}); diff --git a/test/shared/workflow-runner-provider.test.ts b/test/shared/workflow-runner-provider.test.ts new file mode 100644 index 00000000..c3e8f0b9 --- /dev/null +++ b/test/shared/workflow-runner-provider.test.ts @@ -0,0 +1,225 @@ +import { describe, expect, it } from "bun:test"; + +import { + assertExactWorkflowRunnerProviderEnvironment, + WorkflowRunnerProviderConfigurationError, + workflowRunnerProviderEnv, +} from "../../src/shared/workflow-runner-provider"; + +type RunnerProviderConfig = Parameters[0]; + +const baseConfig: RunnerProviderConfig = { + provider: "anthropic", + model: "claude-test", + anthropicApiKey: undefined, + claudeCodeOauthToken: undefined, + awsRegion: undefined, + awsProfile: undefined, + awsAccessKeyId: undefined, + awsSecretAccessKey: undefined, + awsSessionToken: undefined, + awsBearerTokenBedrock: undefined, + anthropicBedrockBaseUrl: undefined, + allowedOwners: undefined, +}; + +function providerConfig(overrides: Partial): RunnerProviderConfig { + return { ...baseConfig, ...overrides }; +} + +function environment(overrides: NodeJS.ProcessEnv): NodeJS.ProcessEnv { + return { CLAUDE_PROVIDER: "anthropic", ...overrides }; +} + +describe("workflowRunnerProviderEnv", () => { + it("selects the API key when both Anthropic credentials are configured", () => { + expect( + workflowRunnerProviderEnv( + providerConfig({ + anthropicApiKey: "api-key", + claudeCodeOauthToken: "oauth-token", + allowedOwners: ["owner-a", "owner-b"], + }), + ), + ).toEqual([ + { name: "CLAUDE_PROVIDER", value: "anthropic" }, + { name: "CLAUDE_MODEL", value: "claude-test" }, + { name: "ANTHROPIC_API_KEY", secretKey: "ANTHROPIC_API_KEY" }, + { name: "ALLOWED_OWNERS", value: "owner-a,owner-b" }, + ]); + }); + + it("selects Anthropic OAuth when no API key is configured", () => { + expect( + workflowRunnerProviderEnv(providerConfig({ claudeCodeOauthToken: "oauth-token" })), + ).toContainEqual({ + name: "CLAUDE_CODE_OAUTH_TOKEN", + secretKey: "CLAUDE_CODE_OAUTH_TOKEN", + }); + }); + + it("rejects Anthropic without a credential", () => { + expect(() => workflowRunnerProviderEnv(baseConfig)).toThrow( + WorkflowRunnerProviderConfigurationError, + ); + }); + + it("selects a Bedrock bearer token and omits configured static credentials", () => { + expect( + workflowRunnerProviderEnv( + providerConfig({ + provider: "bedrock", + awsRegion: "ap-southeast-2", + awsBearerTokenBedrock: "bearer-token", + awsAccessKeyId: "access-key", + awsSecretAccessKey: "secret-key", + anthropicBedrockBaseUrl: "https://bedrock.example.test", + }), + ), + ).toEqual([ + { name: "CLAUDE_PROVIDER", value: "bedrock" }, + { name: "CLAUDE_MODEL", value: "claude-test" }, + { name: "AWS_REGION", value: "ap-southeast-2" }, + { name: "ANTHROPIC_BEDROCK_BASE_URL", value: "https://bedrock.example.test" }, + { name: "AWS_BEARER_TOKEN_BEDROCK", secretKey: "AWS_BEARER_TOKEN_BEDROCK" }, + ]); + }); + + it("selects a complete Bedrock static session chain", () => { + expect( + workflowRunnerProviderEnv( + providerConfig({ + provider: "bedrock", + awsRegion: "ap-southeast-2", + awsAccessKeyId: "access-key", + awsSecretAccessKey: "secret-key", + awsSessionToken: "session-token", + }), + ).filter((entry) => entry.secretKey !== undefined), + ).toEqual([ + { name: "AWS_ACCESS_KEY_ID", secretKey: "AWS_ACCESS_KEY_ID" }, + { name: "AWS_SECRET_ACCESS_KEY", secretKey: "AWS_SECRET_ACCESS_KEY" }, + { name: "AWS_SESSION_TOKEN", secretKey: "AWS_SESSION_TOKEN" }, + ]); + }); + + it("rejects Bedrock without a region", () => { + expect(() => + workflowRunnerProviderEnv( + providerConfig({ provider: "bedrock", awsBearerTokenBedrock: "bearer-token" }), + ), + ).toThrow("Bedrock workflow runners require AWS_REGION"); + }); + + it.each([ + "http://bedrock.example.test", + "https://user@bedrock.example.test", + "https://user:secret@bedrock.example.test", + "https://bedrock.example.test?token=secret", + "https://bedrock.example.test#fragment", + " bedrock.example.test ", + ])("rejects unsafe Bedrock base URL %s", (baseUrl) => { + expect(() => + workflowRunnerProviderEnv( + providerConfig({ + provider: "bedrock", + awsRegion: "ap-southeast-2", + awsBearerTokenBedrock: "bearer-token", + anthropicBedrockBaseUrl: baseUrl, + }), + ), + ).toThrow(WorkflowRunnerProviderConfigurationError); + }); + + it.each([ + ["access key only", { awsAccessKeyId: "access-key" }], + ["secret key only", { awsSecretAccessKey: "secret-key" }], + ["session token only", { awsSessionToken: "session-token" }], + ["profile only", { awsProfile: "runner-profile" }], + ])("rejects an incomplete Bedrock %s configuration", (_name, credentials) => { + expect(() => + workflowRunnerProviderEnv( + providerConfig({ provider: "bedrock", awsRegion: "ap-southeast-2", ...credentials }), + ), + ).toThrow(WorkflowRunnerProviderConfigurationError); + }); +}); + +describe("assertExactWorkflowRunnerProviderEnvironment", () => { + it.each([ + ["Anthropic API key", environment({ ANTHROPIC_API_KEY: "api-key" })], + ["Anthropic OAuth", environment({ CLAUDE_CODE_OAUTH_TOKEN: "oauth-token" })], + [ + "Bedrock bearer", + environment({ + CLAUDE_PROVIDER: "bedrock", + AWS_BEARER_TOKEN_BEDROCK: "bearer-token", + }), + ], + [ + "Bedrock static", + environment({ + CLAUDE_PROVIDER: "bedrock", + AWS_ACCESS_KEY_ID: "access-key", + AWS_SECRET_ACCESS_KEY: "secret-key", + }), + ], + [ + "Bedrock static session", + environment({ + CLAUDE_PROVIDER: "bedrock", + AWS_ACCESS_KEY_ID: "access-key", + AWS_SECRET_ACCESS_KEY: "secret-key", + AWS_SESSION_TOKEN: "session-token", + }), + ], + ])("accepts exactly one %s chain", (_name, env) => { + expect(() => { + assertExactWorkflowRunnerProviderEnvironment(env); + }).not.toThrow(); + }); + + it.each([ + ["missing provider", {}], + ["unknown provider", { CLAUDE_PROVIDER: "unknown", ANTHROPIC_API_KEY: "api-key" }], + [ + "dual Anthropic", + environment({ ANTHROPIC_API_KEY: "api-key", CLAUDE_CODE_OAUTH_TOKEN: "oauth-token" }), + ], + [ + "Anthropic with AWS", + environment({ ANTHROPIC_API_KEY: "api-key", AWS_ACCESS_KEY_ID: "access-key" }), + ], + [ + "Bedrock with Anthropic", + environment({ + CLAUDE_PROVIDER: "bedrock", + AWS_BEARER_TOKEN_BEDROCK: "bearer-token", + ANTHROPIC_API_KEY: "api-key", + }), + ], + [ + "mixed Bedrock chains", + environment({ + CLAUDE_PROVIDER: "bedrock", + AWS_BEARER_TOKEN_BEDROCK: "bearer-token", + AWS_ACCESS_KEY_ID: "access-key", + AWS_SECRET_ACCESS_KEY: "secret-key", + }), + ], + ["Bedrock access only", environment({ CLAUDE_PROVIDER: "bedrock", AWS_ACCESS_KEY_ID: "key" })], + [ + "Bedrock secret only", + environment({ CLAUDE_PROVIDER: "bedrock", AWS_SECRET_ACCESS_KEY: "secret" }), + ], + [ + "orphan Bedrock session", + environment({ CLAUDE_PROVIDER: "bedrock", AWS_SESSION_TOKEN: "session" }), + ], + ["Bedrock profile", environment({ CLAUDE_PROVIDER: "bedrock", AWS_PROFILE: "runner-profile" })], + ])("rejects %s", (_name, env) => { + expect(() => { + assertExactWorkflowRunnerProviderEnvironment(env); + }).toThrow(WorkflowRunnerProviderConfigurationError); + }); +}); diff --git a/test/shared/ws-messages.test.ts b/test/shared/ws-messages.test.ts index 985016e1..f245c38f 100644 --- a/test/shared/ws-messages.test.ts +++ b/test/shared/ws-messages.test.ts @@ -159,6 +159,99 @@ describe("serverMessageSchema", () => { } }); + it("round-trips the per-repo agent policy on job:payload (Gate 2)", () => { + const msg = { + type: "job:payload", + ...envelope(), + payload: { + context: { owner: "org", repo: "repo" }, + installationToken: "ghs_abc123", + allowedTools: ["Read"], + policy: { + model: "claude-repo-pinned-model", + timeoutMs: 900_000, + extraAllowedTools: ["WebFetch"], + pathFilters: ["**/__snapshots__/**"], + instructions: "reject migrations without a rollback", + warning: "`.github-app.yaml` failed validation and was ignored", + }, + }, + }; + const result = serverMessageSchema.safeParse(msg); + expect(result.success).toBe(true); + if (result.success && result.data.type === "job:payload") { + // z.object strips unknown keys, so an unwired `policy` silently + // vanishes here rather than failing the parse. Assert the values. + expect(result.data.payload.policy).toEqual({ + model: "claude-repo-pinned-model", + timeoutMs: 900_000, + extraAllowedTools: ["WebFetch"], + pathFilters: ["**/__snapshots__/**"], + instructions: "reject migrations without a rollback", + warning: "`.github-app.yaml` failed validation and was ignored", + }); + } + }); + + it("leaves policy undefined when the payload omits it (C8)", () => { + const msg = { + type: "job:payload", + ...envelope(), + payload: { + context: {}, + installationToken: "ghs_abc123", + maxTurns: 10, + allowedTools: [], + }, + }; + const result = serverMessageSchema.safeParse(msg); + expect(result.success).toBe(true); + if (result.success && result.data.type === "job:payload") { + expect(result.data.payload.policy).toBeUndefined(); + // maxTurns stays a top-level field: the per-repo cap reuses it rather + // than adding a second, ambiguous source of truth inside `policy`. + expect(result.data.payload.maxTurns).toBe(10); + } + }); + + it("accepts 100 extraAllowedTools, the most the resolver's union can emit", () => { + // `resolveKnobs` UNIONS `defaults.extra_allowed_tools` with + // `workflows..extra_allowed_tools`, each capped at 50 in + // src/repo-config/schema.ts, so a disjoint pair resolves to 100. A wire + // cap below that drops the job silently in src/daemon/ws-client.ts. + const msg = { + type: "job:payload", + ...envelope(), + payload: { + context: {}, + installationToken: "ghs_abc123", + allowedTools: [], + policy: { + extraAllowedTools: Array.from({ length: 100 }, (_, i) => `Bash(tool-${String(i)}:*)`), + }, + }, + }; + const result = serverMessageSchema.safeParse(msg); + expect(result.success).toBe(true); + if (result.success && result.data.type === "job:payload") { + expect(result.data.payload.policy?.extraAllowedTools).toHaveLength(100); + } + }); + + it("rejects a job:payload policy with a non-positive timeoutMs", () => { + const msg = { + type: "job:payload", + ...envelope(), + payload: { + context: {}, + installationToken: "ghs_abc123", + allowedTools: [], + policy: { timeoutMs: 0 }, + }, + }; + expect(serverMessageSchema.safeParse(msg).success).toBe(false); + }); + it("rejects job:payload with a non-positive installationId (#177)", () => { const msg = { type: "job:payload", diff --git a/test/utils/bot-identity.test.ts b/test/utils/bot-identity.test.ts index 87fed1b7..3175e810 100644 --- a/test/utils/bot-identity.test.ts +++ b/test/utils/bot-identity.test.ts @@ -10,12 +10,7 @@ import { beforeEach, describe, expect, it, mock } from "bun:test"; -interface TestConfig { - botAppLogin: string; - githubPersonalAccessToken?: string; -} - -const testConfig: TestConfig = { +const testConfig: { botAppLogin: string; githubPersonalAccessToken?: string } = { botAppLogin: "chrisleekr-bot[bot]", }; void mock.module("../../src/config", () => ({ diff --git a/test/webhook/auto-review-guard.test.ts b/test/webhook/auto-review-guard.test.ts new file mode 100644 index 00000000..78f02207 --- /dev/null +++ b/test/webhook/auto-review-guard.test.ts @@ -0,0 +1,228 @@ +/** + * Unit tests for the auto-review guards (work item #1). + * + * These four helpers decide whether the bot spends a full agent run on a push. + * The handler suite (`pull-request-auto-review.test.ts`) replaces this whole + * module with `mock.module`, so without this file the real branches never + * execute. Each one is advisory and MUST fail open: a missing review is a worse + * failure than a redundant one, and a guard that wrongly returned "already + * reviewed" would suppress reviews silently and indefinitely. + */ + +import { beforeEach, describe, expect, it, mock } from "bun:test"; +import type { Logger } from "pino"; + +// ─── Mocked downstream surfaces ────────────────────────────────────────── + +let valkeyClient: { send: ReturnType } | null = null; +let valkeyHealthy = true; +void mock.module("../../src/orchestrator/valkey", () => ({ + getValkeyClient: () => valkeyClient, + isValkeyHealthy: () => valkeyHealthy, +})); + +let resolvedSelfLogin: string | null = "chrisleekr-bot[bot]"; +void mock.module("../../src/utils/bot-identity", () => ({ + resolveSelfLogin: () => Promise.resolve(resolvedSelfLogin), +})); + +const { computeDiffFingerprint, isSelfPush, matchesLastReviewed, recordReviewedFingerprint } = + await import("../../src/webhook/auto-review-guard"); + +const log = { + info: () => undefined, + warn: () => undefined, + debug: () => undefined, + error: () => undefined, +} as unknown as Logger; + +/** `paginate` is the only octokit surface these helpers touch. */ +function octokitReturning(files: unknown): OctokitArg { + return { + paginate: mock(() => (files instanceof Error ? Promise.reject(files) : Promise.resolve(files))), + rest: { pulls: { listFiles: () => undefined } }, + } as unknown as OctokitArg; +} + +function file(filename: string, sha: string, status = "modified") { + return { filename, sha, status }; +} + +beforeEach(() => { + valkeyClient = null; + valkeyHealthy = true; + resolvedSelfLogin = "chrisleekr-bot[bot]"; +}); + +describe("isSelfPush", () => { + it("recognises our own push by login", async () => { + // This is the ONLY thing breaking review -> resolve -> push -> review: + // `resolve` deliberately does not filter review comments by author. + expect(await isSelfPush({ login: "chrisleekr-bot[bot]", type: "Bot" })).toBe(true); + }); + + it("does not claim a third-party bot's push as ours", async () => { + expect(await isSelfPush({ login: "renovate[bot]", type: "Bot" })).toBe(false); + }); + + it("does not claim a human's push as ours", async () => { + expect(await isSelfPush({ login: "chrisleekr", type: "User" })).toBe(false); + }); + + it("fails open to 'not us' when the self-login cannot be resolved", async () => { + resolvedSelfLogin = null; + expect(await isSelfPush({ login: "chrisleekr-bot[bot]", type: "Bot" })).toBe(false); + }); +}); + +describe("computeDiffFingerprint", () => { + it("is stable regardless of the order listFiles returns", async () => { + // The `.sort()` is load-bearing: GitHub does not promise an order, and an + // order-sensitive hash would treat every push as a changed diff. + const a = await computeDiffFingerprint( + octokitReturning([file("a.ts", "sha-a"), file("b.ts", "sha-b")]), + "acme", + "widgets", + 7, + log, + ); + const b = await computeDiffFingerprint( + octokitReturning([file("b.ts", "sha-b"), file("a.ts", "sha-a")]), + "acme", + "widgets", + 7, + log, + ); + expect(a).toBe(b); + expect(a).not.toBeNull(); + }); + + it("changes when a blob changes, which is what makes a rebase detectable", async () => { + const before = await computeDiffFingerprint( + octokitReturning([file("a.ts", "sha-a")]), + "acme", + "widgets", + 7, + log, + ); + const after = await computeDiffFingerprint( + octokitReturning([file("a.ts", "sha-DIFFERENT")]), + "acme", + "widgets", + 7, + log, + ); + expect(before).not.toBe(after!); + }); + + it("changes when only a file's status changes", async () => { + const asModified = await computeDiffFingerprint( + octokitReturning([file("a.ts", "sha-a", "modified")]), + "acme", + "widgets", + 7, + log, + ); + const asAdded = await computeDiffFingerprint( + octokitReturning([file("a.ts", "sha-a", "added")]), + "acme", + "widgets", + 7, + log, + ); + expect(asModified).not.toBe(asAdded!); + }); + + it("returns null for an empty diff", async () => { + expect( + await computeDiffFingerprint(octokitReturning([]), "acme", "widgets", 7, log), + ).toBeNull(); + }); + + it("returns null at GitHub's 3000-file listFiles cap, where the list truncates", async () => { + // Above the cap the response no longer represents the whole diff, so a hash + // of it would be a hash of an arbitrary prefix. + const many = Array.from({ length: 3000 }, (_, i) => + file(`f${String(i)}.ts`, `sha-${String(i)}`), + ); + expect( + await computeDiffFingerprint(octokitReturning(many), "acme", "widgets", 7, log), + ).toBeNull(); + }); + + it("returns null when listFiles throws, rather than propagating", async () => { + expect( + await computeDiffFingerprint(octokitReturning(new Error("502")), "acme", "widgets", 7, log), + ).toBeNull(); + }); +}); + +describe("matchesLastReviewed", () => { + it("returns false when Valkey is not configured", async () => { + valkeyClient = null; + expect(await matchesLastReviewed("acme", "widgets", 7, "fp", log)).toBe(false); + }); + + it("returns false when Valkey is configured but disconnected", async () => { + // Gated on health for the same reason `claimDelivery` is: Bun's RedisClient + // queues offline commands, so a GET here would block instead of failing open. + const send = mock(() => Promise.resolve("fp")); + valkeyClient = { send }; + valkeyHealthy = false; + + expect(await matchesLastReviewed("acme", "widgets", 7, "fp", log)).toBe(false); + expect(send).not.toHaveBeenCalled(); + }); + + it("returns true only when the stored fingerprint matches exactly", async () => { + valkeyClient = { send: mock(() => Promise.resolve("fp-current")) }; + expect(await matchesLastReviewed("acme", "widgets", 7, "fp-current", log)).toBe(true); + + valkeyClient = { send: mock(() => Promise.resolve("fp-older")) }; + expect(await matchesLastReviewed("acme", "widgets", 7, "fp-current", log)).toBe(false); + }); + + it("returns false when the read throws, never true", async () => { + // The dangerous direction: a spurious `true` suppresses reviews silently. + valkeyClient = { send: mock(() => Promise.reject(new Error("connection reset"))) }; + expect(await matchesLastReviewed("acme", "widgets", 7, "fp", log)).toBe(false); + }); + + it("keys per pull request, not per repo", async () => { + const send = mock(() => Promise.resolve(null)); + valkeyClient = { send }; + await matchesLastReviewed("acme", "widgets", 7, "fp", log); + + const key = (send.mock.calls[0] as unknown as [string, string[]])[1][0]; + expect(key).toBe("autoreview:fp:acme/widgets#7"); + }); +}); + +describe("recordReviewedFingerprint", () => { + it("writes the fingerprint with a TTL", async () => { + const send = mock(() => Promise.resolve("OK")); + valkeyClient = { send }; + + await recordReviewedFingerprint("acme", "widgets", 7, "fp-current", log); + + const [cmd, args] = send.mock.calls[0] as unknown as [string, string[]]; + expect(cmd).toBe("SET"); + expect(args[0]).toBe("autoreview:fp:acme/widgets#7"); + expect(args[1]).toBe("fp-current"); + expect(args[2]).toBe("EX"); + expect(args[3]).toBe("2592000"); // 30 days + }); + + it("is a no-op when Valkey is unavailable", async () => { + valkeyClient = null; + await recordReviewedFingerprint("acme", "widgets", 7, "fp", log); + // Reaching here without throwing is the assertion. + expect(true).toBe(true); + }); + + it("swallows a write failure, which only costs one re-review", async () => { + valkeyClient = { send: mock(() => Promise.reject(new Error("OOM"))) }; + await recordReviewedFingerprint("acme", "widgets", 7, "fp", log); + expect(true).toBe(true); + }); +}); diff --git a/test/webhook/events/dispatch-failure.test.ts b/test/webhook/events/dispatch-failure.test.ts new file mode 100644 index 00000000..1393d17e --- /dev/null +++ b/test/webhook/events/dispatch-failure.test.ts @@ -0,0 +1,244 @@ +import type { + IssueCommentEvent, + IssuesEvent, + PullRequestEvent, + PullRequestReviewCommentEvent, +} from "@octokit/webhooks-types"; +import { beforeEach, describe, expect, it, mock } from "bun:test"; +import type { Octokit } from "octokit"; + +const FIXED_FAILURE_MESSAGE = "Sorry, I couldn't start that workflow. Please try again."; +const rawError = "postgres://user:secret@internal/db"; + +const testLog = { + info: mock(() => {}), + warn: mock(() => {}), + error: mock(() => {}), + debug: mock(() => {}), + child: mock(function (this: unknown) { + return this; + }), +}; +void mock.module("../../../src/logger", () => ({ + logger: testLog, + createChildLogger: mock(() => testLog), +})); + +const mockDispatchByLabel = mock((_input: unknown) => + Promise.resolve({ status: "dispatched", runId: "run-1", workflowName: "triage" }), +); +const mockDispatchByIntent = mock((_input: unknown) => + Promise.resolve({ status: "dispatched", runId: "run-2", workflowName: "triage" }), +); +void mock.module("../../../src/workflows/dispatcher", () => ({ + dispatchByLabel: mockDispatchByLabel, + dispatchByIntent: mockDispatchByIntent, + dispatchWorkflowByName: mock(() => Promise.resolve({ status: "ignored", reason: "test" })), +})); + +const mockSafePostToGitHub = mock( + async (input: { body: string; source: string; post: (body: string) => Promise }) => { + await input.post(input.body); + return { posted: true, matchCount: 0, kinds: [] }; + }, +); +void mock.module("../../../src/utils/github-output-guard", () => ({ + safePostToGitHub: mockSafePostToGitHub, +})); + +void mock.module("../../../src/core/trigger", () => ({ containsTrigger: (): boolean => true })); +void mock.module("../../../src/webhook/authorize", () => ({ + isOwnerAllowed: (): { allowed: true } => ({ allowed: true }), +})); +void mock.module("../../../src/webhook/idempotency", () => ({ + claimDelivery: (): Promise => Promise.resolve(true), +})); +void mock.module("../../../src/db/queries/conversation-store", () => ({ + deleteTarget: mock(() => Promise.resolve()), + softDeleteComment: mock(() => Promise.resolve()), + upsertComment: mock(() => Promise.resolve()), + upsertTarget: mock(() => Promise.resolve()), +})); +void mock.module("../../../src/orchestrator/proposal-poller", () => ({ + runProposalPollOnce: mock(() => Promise.resolve()), +})); +void mock.module("../../../src/utils/reactions", () => ({ + addReaction: mock(() => Promise.resolve()), +})); +void mock.module("../../../src/workflows/ship/command-dispatch", () => ({ + dispatchCanonicalCommand: mock(() => undefined), + dispatchCommentSurface: mock(() => Promise.resolve(false)), +})); +void mock.module("../../../src/workflows/ship/reactor-bridge", () => ({ + fireReactor: mock(() => undefined), +})); +void mock.module("../../../src/workflows/ship/trigger-router", () => ({ + routeTrigger: mock(() => Promise.resolve(null)), +})); +void mock.module("../../../src/repo-config/effective", () => ({ + loadRepoPolicy: mock(() => Promise.resolve({ enabled: true })), + policyForWorkflow: (): { + enabled: boolean; + auto: boolean; + extraAllowedTools: never[]; + pathFilters: never[]; + } => ({ enabled: true, auto: false, extraAllowedTools: [], pathFilters: [] }), +})); +void mock.module("../../../src/repo-config/pr-check", () => ({ + runPrConfigCheck: mock(() => Promise.resolve()), +})); +void mock.module("../../../src/webhook/auto-review-guard", () => ({ + computeDiffFingerprint: mock(() => Promise.resolve(null)), + hasActiveShipIntent: mock(() => Promise.resolve(false)), + isSelfPush: mock(() => Promise.resolve(false)), + matchesLastReviewed: mock(() => Promise.resolve(false)), + recordReviewedFingerprint: mock(() => Promise.resolve()), +})); +void mock.module("../../../src/config", () => ({ + config: { allowedOwners: ["acme"], logLevel: "silent", nodeEnv: "test" }, +})); + +const { handleIssues } = await import("../../../src/webhook/events/issues"); +const { handlePullRequest } = await import("../../../src/webhook/events/pull-request"); +const { handleIssueComment } = await import("../../../src/webhook/events/issue-comment"); +const { handleReviewComment } = await import("../../../src/webhook/events/review-comment"); + +const createComment = mock((_input: unknown) => Promise.resolve({ data: { id: 1 } })); +const octokit = { rest: { issues: { createComment } } } as unknown as Octokit; + +function issuesPayload(): IssuesEvent { + return { + action: "labeled", + installation: { id: 1 }, + issue: { number: 16, title: "Broken dispatch" }, + label: { name: "bot:triage" }, + repository: { name: "repo", owner: { login: "acme" } }, + sender: { login: "alice" }, + } as unknown as IssuesEvent; +} + +function pullRequestPayload(): PullRequestEvent { + return { + action: "labeled", + installation: { id: 1 }, + label: { name: "bot:triage" }, + pull_request: { + number: 16, + title: "Broken dispatch", + draft: false, + base: { ref: "main" }, + head: { ref: "fix", sha: "a".repeat(40) }, + }, + repository: { name: "repo", owner: { login: "acme" } }, + sender: { login: "alice" }, + } as unknown as PullRequestEvent; +} + +function issueCommentPayload(): IssueCommentEvent { + return { + action: "created", + installation: { id: 1 }, + comment: { id: 1601, body: "@chrisleekr-bot triage", user: { login: "alice", type: "User" } }, + issue: { number: 16, title: "Broken dispatch" }, + repository: { name: "repo", owner: { login: "acme" } }, + } as unknown as IssueCommentEvent; +} + +function reviewCommentPayload(): PullRequestReviewCommentEvent { + return { + action: "created", + installation: { id: 1 }, + comment: { id: 1602, body: "@chrisleekr-bot triage", user: { login: "alice", type: "User" } }, + pull_request: { number: 16, title: "Broken dispatch", draft: false, base: { ref: "main" } }, + repository: { name: "repo", owner: { login: "acme" } }, + } as unknown as PullRequestReviewCommentEvent; +} + +async function flushDispatch(): Promise { + for (let i = 0; i < 10; i++) { + // eslint-disable-next-line no-await-in-loop + await Promise.resolve(); + } + await new Promise((resolve) => setTimeout(resolve, 0)); +} + +describe("user-triggered dispatch failures", () => { + beforeEach(() => { + mockDispatchByLabel.mockClear(); + mockDispatchByIntent.mockClear(); + mockSafePostToGitHub.mockClear(); + createComment.mockClear(); + testLog.error.mockClear(); + }); + + const cases = [ + { + surface: "issue label", + dispatch: mockDispatchByLabel, + fire: (): void => { + handleIssues(octokit, issuesPayload(), "delivery-issue-label"); + }, + }, + { + surface: "pull request label", + dispatch: mockDispatchByLabel, + fire: (): void => { + handlePullRequest(octokit, pullRequestPayload(), "delivery-pr-label"); + }, + }, + { + surface: "issue mention", + dispatch: mockDispatchByIntent, + fire: (): void => { + handleIssueComment(octokit, issueCommentPayload(), "delivery-issue-mention"); + }, + }, + { + surface: "review mention", + dispatch: mockDispatchByIntent, + fire: (): void => { + handleReviewComment(octokit, reviewCommentPayload(), "delivery-review-mention"); + }, + }, + ]; + + for (const testCase of cases) { + it(`${testCase.surface} logs the raw failure and posts only the fixed sanitized reply`, async () => { + testCase.dispatch.mockRejectedValueOnce(new Error(rawError)); + + testCase.fire(); + await flushDispatch(); + + expect(testLog.error).toHaveBeenCalledTimes(1); + expect(mockSafePostToGitHub).toHaveBeenCalledTimes(1); + const safePostInput = mockSafePostToGitHub.mock.calls[0]?.[0]; + expect(safePostInput?.body).toBe(FIXED_FAILURE_MESSAGE); + expect(safePostInput?.source).toBe("system"); + expect(safePostInput?.body).not.toContain(rawError); + const loggedError = testLog.error.mock.calls[0]?.[0] as { err?: unknown } | undefined; + expect(String(loggedError?.err)).toContain(rawError); + expect(createComment.mock.calls[0]?.[0]).toMatchObject({ body: FIXED_FAILURE_MESSAGE }); + }); + } + + it("contains a GitHub post failure without an unhandled rejection", async () => { + const unhandled: unknown[] = []; + const onUnhandled = (reason: unknown): void => { + unhandled.push(reason); + }; + process.on("unhandledRejection", onUnhandled); + try { + mockDispatchByLabel.mockRejectedValueOnce(new Error(rawError)); + createComment.mockRejectedValueOnce(new Error("GitHub unavailable")); + + handleIssues(octokit, issuesPayload(), "delivery-post-failure"); + await flushDispatch(); + await new Promise((resolve) => setTimeout(resolve, 10)); + + expect(mockSafePostToGitHub).toHaveBeenCalledTimes(1); + expect(unhandled).toHaveLength(0); + } finally { + process.off("unhandledRejection", onUnhandled); + } + }); +}); diff --git a/test/webhook/events/issue-comment-cache.test.ts b/test/webhook/events/issue-comment-cache.test.ts index 068178c3..34146d73 100644 --- a/test/webhook/events/issue-comment-cache.test.ts +++ b/test/webhook/events/issue-comment-cache.test.ts @@ -39,11 +39,13 @@ beforeAll(async () => { const db = getDb(); if (db === null) return; try { - await db`SELECT 1 FROM comment_cache LIMIT 1`; - dbAvailable = true; + await db`SELECT 1`; } catch { - dbAvailable = false; + return; } + const { runMigrations } = await import("../../../src/db/migrate"); + await runMigrations(db); + dbAvailable = true; }); const skipIfNoDb = (): boolean => !dbAvailable; diff --git a/test/webhook/events/issue-comment.test.ts b/test/webhook/events/issue-comment.test.ts index 49def26f..d64e6231 100644 --- a/test/webhook/events/issue-comment.test.ts +++ b/test/webhook/events/issue-comment.test.ts @@ -47,9 +47,9 @@ function requireSql(): SQL { // ─── Mocks ─────────────────────────────────────────────────────────────── -const mockEnqueueJob = mock(() => Promise.resolve()); +const mockEnsureWorkflowJobQueued = mock(() => Promise.resolve(true)); void mock.module("../../../src/orchestrator/job-queue", () => ({ - enqueueJob: mockEnqueueJob, + ensureWorkflowJobQueued: mockEnsureWorkflowJobQueued, isScopedJob: () => false, SCOPED_JOB_KINDS: ["scoped-rebase", "scoped-fix-thread", "scoped-open-pr"], })); @@ -118,6 +118,7 @@ describe.skipIf(sql === null)("issue-comment → dispatchByIntent integration (T beforeAll(async () => { await requireSql().unsafe(` DROP TABLE IF EXISTS _migrations CASCADE; + DROP TABLE IF EXISTS workflow_attempt_commands CASCADE; DROP TABLE IF EXISTS review_learnings CASCADE; DROP TABLE IF EXISTS scheduled_action_state CASCADE; DROP TABLE IF EXISTS comment_cache CASCADE; @@ -140,6 +141,7 @@ describe.skipIf(sql === null)("issue-comment → dispatchByIntent integration (T afterAll(async () => { await requireSql().unsafe(` DROP TABLE IF EXISTS _migrations CASCADE; + DROP TABLE IF EXISTS workflow_attempt_commands CASCADE; DROP TABLE IF EXISTS review_learnings CASCADE; DROP TABLE IF EXISTS scheduled_action_state CASCADE; DROP TABLE IF EXISTS comment_cache CASCADE; @@ -206,11 +208,11 @@ describe.skipIf(sql === null)("issue-comment → dispatchByIntent integration (T expect(Object.keys(intentRow.state)).toEqual(Object.keys(labelRow.state)); // Both dispatches enqueued exactly one job each. - expect(mockEnqueueJob).toHaveBeenCalledTimes(2); - const labelCall = mockEnqueueJob.mock.calls[0]?.[0] as + expect(mockEnsureWorkflowJobQueued).toHaveBeenCalledTimes(2); + const labelCall = mockEnsureWorkflowJobQueued.mock.calls[0]?.[0] as | { workflowRun: { workflowName: string }; repoOwner: string; entityNumber: number } | undefined; - const intentCall = mockEnqueueJob.mock.calls[1]?.[0] as + const intentCall = mockEnsureWorkflowJobQueued.mock.calls[1]?.[0] as | { workflowRun: { workflowName: string }; repoOwner: string; entityNumber: number } | undefined; expect(labelCall?.workflowRun.workflowName).toBe("ship"); @@ -223,7 +225,7 @@ describe.skipIf(sql === null)("issue-comment → dispatchByIntent integration (T it("low-confidence intent comment does NOT create a workflow_runs row", async () => { const { dispatchByIntent } = await import("../../../src/workflows/dispatcher"); - mockEnqueueJob.mockClear(); + mockEnsureWorkflowJobQueued.mockClear(); mockClassify.mockClear(); const outcome = await dispatchByIntent({ @@ -237,7 +239,7 @@ describe.skipIf(sql === null)("issue-comment → dispatchByIntent integration (T triggerEventType: "issue_comment", }); expect(outcome.status).toBe("ignored"); - expect(mockEnqueueJob).not.toHaveBeenCalled(); + expect(mockEnsureWorkflowJobQueued).not.toHaveBeenCalled(); const rows = (await requireSql()`SELECT * FROM workflow_runs WHERE target_number = ${403}`) as unknown as { diff --git a/test/webhook/events/issues-cache.test.ts b/test/webhook/events/issues-cache.test.ts index 876a9fdd..c4deb676 100644 --- a/test/webhook/events/issues-cache.test.ts +++ b/test/webhook/events/issues-cache.test.ts @@ -41,11 +41,13 @@ beforeAll(async () => { const db = getDb(); if (db === null) return; try { - await db`SELECT 1 FROM target_cache LIMIT 1`; - dbAvailable = true; + await db`SELECT 1`; } catch { - dbAvailable = false; + return; } + const { runMigrations } = await import("../../../src/db/migrate"); + await runMigrations(db); + dbAvailable = true; }); const skipIfNoDb = (): boolean => !dbAvailable; diff --git a/test/webhook/events/pull-request-auto-review.test.ts b/test/webhook/events/pull-request-auto-review.test.ts new file mode 100644 index 00000000..efbe571e --- /dev/null +++ b/test/webhook/events/pull-request-auto-review.test.ts @@ -0,0 +1,360 @@ +/** + * Handler tests for auto-review on push (work item #1). + * + * `maybeAutoReview` runs off `pull_request.synchronize` and dispatches `review` + * with no label and no mention. What is pinned here is the gate order and the + * refusal-to-act, not the review itself: + * + * - both keys are required: the `AUTO_REVIEW_USERS` env allowlist AND the + * repo's `workflows.review.auto`, and an unreachable config fails to OFF; + * - the allowlist is matched against the authenticated **pusher** + * (`sender.login`), never the `repos.getCommit` author, which is derived + * from a commit email anyone can set; + * - our own pushes are skipped, which is what stops + * review -> resolve -> push -> review from looping; + * - a push whose diff fingerprint is unchanged (a rebase) is skipped; + * - the claim key is `${deliveryId}:auto-review`, so it cannot starve the + * `${deliveryId}:config-check` branch of the same delivery; + * - the fingerprint is recorded only on a real dispatch, so a Gate-1 refusal + * cannot suppress the next genuine review. + * + * Dispatch is fire-and-forget inside a `void (async () => {...})()` IIFE, so + * every case drains the microtask queue before asserting. + */ + +import type { PullRequestEvent } from "@octokit/webhooks-types"; +import { beforeEach, describe, expect, it, mock } from "bun:test"; +import type { Octokit } from "octokit"; + +// ─── Mocked downstream surfaces ────────────────────────────────────────── + +let repoAuto = true; +const mockLoadRepoPolicy = mock((_input: unknown) => Promise.resolve({ enabled: true })); +const mockPolicyForWorkflow = mock((_p: unknown, _n: unknown) => ({ + enabled: true, + extraAllowedTools: [], + pathFilters: [], + auto: repoAuto, +})); +void mock.module("../../../src/repo-config/effective", () => ({ + loadRepoPolicy: mockLoadRepoPolicy, + policyForWorkflow: mockPolicyForWorkflow, +})); + +void mock.module("../../../src/repo-config/pr-check", () => ({ + runPrConfigCheck: mock(() => Promise.resolve()), +})); + +const claimed = new Set(); +const mockClaimDelivery = mock((key: string) => { + if (claimed.has(key)) return Promise.resolve(false); + claimed.add(key); + return Promise.resolve(true); +}); +void mock.module("../../../src/webhook/idempotency", () => ({ + claimDelivery: mockClaimDelivery, +})); + +let dispatchOutcome: { status: string; runId?: string; workflowName?: string; reason?: string } = { + status: "dispatched", + runId: "run-1", + workflowName: "review", +}; +const mockDispatchWorkflowByName = mock((_input: unknown) => Promise.resolve(dispatchOutcome)); +void mock.module("../../../src/workflows/dispatcher", () => ({ + dispatchByLabel: mock(() => Promise.resolve({ status: "ignored", reason: "test" })), + dispatchWorkflowByName: mockDispatchWorkflowByName, +})); + +const mockSafePostToGitHub = mock((_input: unknown) => + Promise.resolve({ posted: true, matchCount: 0, kinds: [] }), +); +void mock.module("../../../src/utils/github-output-guard", () => ({ + safePostToGitHub: mockSafePostToGitHub, +})); + +// The guards are unit-tested separately; here they are swapped so gate ORDER +// and short-circuiting are what the assertions actually pin. +let selfPush = false; +let shipActive = false; +let fingerprint: string | null = "fp-current"; +let lastReviewed: string | null = null; +const mockHasActiveShipIntent = mock(() => Promise.resolve(shipActive)); +const mockIsSelfPush = mock((_o: unknown, _s: unknown) => Promise.resolve(selfPush)); +const mockComputeDiffFingerprint = mock((..._a: unknown[]) => Promise.resolve(fingerprint)); +const mockMatchesLastReviewed = mock((...a: unknown[]) => + Promise.resolve(lastReviewed !== null && a[3] === lastReviewed), +); +const mockRecordReviewedFingerprint = mock((..._a: unknown[]) => Promise.resolve()); +void mock.module("../../../src/webhook/auto-review-guard", () => ({ + isSelfPush: mockIsSelfPush, + hasActiveShipIntent: mockHasActiveShipIntent, + computeDiffFingerprint: mockComputeDiffFingerprint, + matchesLastReviewed: mockMatchesLastReviewed, + recordReviewedFingerprint: mockRecordReviewedFingerprint, +})); + +void mock.module("../../../src/workflows/ship/command-dispatch", () => ({ + dispatchCanonicalCommand: mock(() => undefined), +})); +void mock.module("../../../src/workflows/ship/reactor-bridge", () => ({ + fireReactor: mock(() => undefined), +})); +void mock.module("../../../src/workflows/ship/trigger-router", () => ({ + routeTrigger: mock(() => Promise.resolve(null)), +})); +void mock.module("../../../src/db/queries/conversation-store", () => ({ + upsertTarget: mock(() => Promise.resolve()), +})); + +// Mutable so a case can unset the allowlist without re-importing the handler. +// `acme` for the repo owner (which is what auto-review checks) and +// `chrisleekr` for the sender (which is what the sibling config-check branch +// checks), so both branches of one delivery run and the claim keys can be +// asserted not to collide. +const testConfig: { allowedOwners: string[]; autoReviewUsers?: string[] } = { + allowedOwners: ["acme", "chrisleekr"], + autoReviewUsers: ["chrisleekr"], +}; +// Getters, not a spread: a spread would freeze `autoReviewUsers` at mock- +// definition time and the "unset" case could never be exercised. +void mock.module("../../../src/config", () => ({ + config: { + get allowedOwners() { + return testConfig.allowedOwners; + }, + get autoReviewUsers() { + return testConfig.autoReviewUsers; + }, + logLevel: "silent", + nodeEnv: "test", + }, +})); + +const { handlePullRequest } = await import("../../../src/webhook/events/pull-request"); + +const HEAD_SHA = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"; + +const fakeOctokit = { + rest: { repos: { getCommit: () => Promise.resolve({ data: { author: { login: "someone" } } }) } }, +} as unknown as Octokit; + +async function flushMicrotasks(): Promise { + for (let i = 0; i < 10; i++) await Promise.resolve(); +} + +function syncPayload( + senderLogin = "chrisleekr", + opts?: { headRepoFullName?: string }, +): PullRequestEvent { + return { + action: "synchronize", + number: 77, + before: "1111111111111111111111111111111111111111", + after: HEAD_SHA, + installation: { id: 555 }, + pull_request: { + number: 77, + title: "Add widget", + body: "", + state: "open", + merged: false, + draft: false, + user: { login: "chrisleekr" }, + base: { ref: "main" }, + head: { + ref: "feature", + sha: HEAD_SHA, + repo: { full_name: opts?.headRepoFullName ?? "acme/widgets" }, + }, + }, + repository: { name: "widgets", owner: { login: "acme" }, full_name: "acme/widgets" }, + sender: { login: senderLogin, type: "User" }, + } as unknown as PullRequestEvent; +} + +async function fire(payload: PullRequestEvent, deliveryId = "d-1"): Promise { + handlePullRequest(fakeOctokit, payload, deliveryId); + await flushMicrotasks(); +} + +describe("auto-review on pull_request.synchronize", () => { + beforeEach(() => { + claimed.clear(); + testConfig.allowedOwners = ["acme", "chrisleekr"]; + testConfig.autoReviewUsers = ["chrisleekr"]; + repoAuto = true; + selfPush = false; + shipActive = false; + fingerprint = "fp-current"; + lastReviewed = null; + dispatchOutcome = { status: "dispatched", runId: "run-1", workflowName: "review" }; + mockDispatchWorkflowByName.mockClear(); + mockSafePostToGitHub.mockClear(); + mockClaimDelivery.mockClear(); + mockRecordReviewedFingerprint.mockClear(); + mockComputeDiffFingerprint.mockClear(); + mockPolicyForWorkflow.mockClear(); + }); + + it("dispatches review with auto:true, no rocket, and the trigger context", async () => { + await fire(syncPayload()); + + expect(mockDispatchWorkflowByName).toHaveBeenCalledTimes(1); + const arg = mockDispatchWorkflowByName.mock.calls[0]?.[0] as Record; + expect(arg["workflowName"]).toBe("review"); + expect(arg["auto"]).toBe(true); + expect(arg["addRocketReaction"]).toBe(false); + expect(arg["senderLogin"]).toBe("chrisleekr"); + expect(arg["target"]).toEqual({ type: "pr", owner: "acme", repo: "widgets", number: 77 }); + expect(arg["trigger"]).toEqual({ title: "Add widget", draft: false, baseBranch: "main" }); + // Gate 1 must not re-fetch the policy this handler already loaded. + expect(arg["repoPolicy"]).toBeDefined(); + }); + + it("does nothing when AUTO_REVIEW_USERS is unset", async () => { + testConfig.autoReviewUsers = undefined; + await fire(syncPayload()); + expect(mockDispatchWorkflowByName).not.toHaveBeenCalled(); + }); + + it("does nothing when the pusher is not in the allowlist", async () => { + await fire(syncPayload("someone-else")); + expect(mockDispatchWorkflowByName).not.toHaveBeenCalled(); + }); + + it("matches the allowlist case-insensitively", async () => { + await fire(syncPayload("ChrisLeeKr")); + expect(mockDispatchWorkflowByName).toHaveBeenCalledTimes(1); + }); + + it("gates on the pusher, NOT the commit author", async () => { + // The commit author is allowlisted; the authenticated pusher is not. The + // author comes from a settable commit email, so it must not grant a run. + testConfig.autoReviewUsers = ["someone"]; + await fire(syncPayload("mallory")); + expect(mockDispatchWorkflowByName).not.toHaveBeenCalled(); + }); + + it("skips our own push, breaking the resolve -> review loop", async () => { + selfPush = true; + await fire(syncPayload()); + expect(mockDispatchWorkflowByName).not.toHaveBeenCalled(); + }); + + it("skips when the repo has not opted in", async () => { + repoAuto = false; + await fire(syncPayload()); + expect(mockDispatchWorkflowByName).not.toHaveBeenCalled(); + }); + + it("checks the repo opt-in BEFORE paying for a fingerprint", async () => { + repoAuto = false; + await fire(syncPayload()); + expect(mockPolicyForWorkflow).toHaveBeenCalled(); + expect(mockComputeDiffFingerprint).not.toHaveBeenCalled(); + }); + + it("skips a rebase: same diff fingerprint as the last reviewed one", async () => { + lastReviewed = "fp-current"; + await fire(syncPayload()); + expect(mockDispatchWorkflowByName).not.toHaveBeenCalled(); + }); + + it("dispatches when the fingerprint changed", async () => { + lastReviewed = "fp-old"; + fingerprint = "fp-current"; + await fire(syncPayload()); + expect(mockDispatchWorkflowByName).toHaveBeenCalledTimes(1); + }); + + it("still dispatches when the fingerprint is unavailable (fail-open)", async () => { + fingerprint = null; + await fire(syncPayload()); + expect(mockDispatchWorkflowByName).toHaveBeenCalledTimes(1); + expect(mockRecordReviewedFingerprint).not.toHaveBeenCalled(); + }); + + it("records the fingerprint only on a real dispatch", async () => { + await fire(syncPayload()); + expect(mockRecordReviewedFingerprint).toHaveBeenCalledTimes(1); + }); + + it("does NOT record the fingerprint when the dispatch was refused", async () => { + // Otherwise a Gate-1 refusal would poison the fingerprint and suppress the + // next genuine review of the same diff. + dispatchOutcome = { status: "refused", reason: "disabled", workflowName: "review" }; + await fire(syncPayload()); + expect(mockRecordReviewedFingerprint).not.toHaveBeenCalled(); + }); + + it("claims `:auto-review`, never the bare deliveryId", async () => { + await fire(syncPayload()); + const keys = mockClaimDelivery.mock.calls.map((c) => c[0]); + expect(keys).toContain("d-1:auto-review"); + expect(keys).not.toContain("d-1"); + }); + + it("does not dispatch twice for a replayed delivery", async () => { + await fire(syncPayload()); + await fire(syncPayload()); + expect(mockDispatchWorkflowByName).toHaveBeenCalledTimes(1); + }); + + it("does not starve the config-check branch of the same delivery", async () => { + await fire(syncPayload()); + const keys = mockClaimDelivery.mock.calls.map((c) => c[0]); + expect(keys).toContain("d-1:auto-review"); + expect(keys).toContain("d-1:config-check"); + }); + + it("contains a dispatch failure instead of escalating it", async () => { + mockDispatchWorkflowByName.mockImplementationOnce(() => Promise.reject(new Error("boom"))); + // Must not reject out of the fire-and-forget IIFE. + await fire(syncPayload()); + expect(mockDispatchWorkflowByName).toHaveBeenCalledTimes(1); + expect(mockSafePostToGitHub).not.toHaveBeenCalled(); + }); + + it("ignores non-synchronize actions", async () => { + const opened = { ...syncPayload(), action: "opened" } as unknown as PullRequestEvent; + await fire(opened); + expect(mockDispatchWorkflowByName).not.toHaveBeenCalled(); + }); + + it("does nothing when ALLOWED_OWNERS excludes the repository owner", async () => { + // The outermost authorization boundary for a path that spends tokens with + // nobody asking. Without a case here, deleting the check keeps CI green. + testConfig.allowedOwners = ["someone-else"]; + + await fire(syncPayload()); + + expect(mockDispatchWorkflowByName).not.toHaveBeenCalled(); + expect(mockClaimDelivery).not.toHaveBeenCalled(); + }); + + it("skips a pull request whose head is on a fork", async () => { + // `checkoutRepo` clones the BASE repo and asks for the head *branch name*, + // so a fork ref either fails to clone or silently resolves to a same-named + // base branch and reviews the wrong tree. + await fire(syncPayload("chrisleekr", { headRepoFullName: "outsider/widgets" })); + + expect(mockDispatchWorkflowByName).not.toHaveBeenCalled(); + }); + + it("still reviews a same-repo branch, which is the ordinary case", async () => { + await fire(syncPayload("chrisleekr", { headRepoFullName: "acme/widgets" })); + + expect(mockDispatchWorkflowByName).toHaveBeenCalledTimes(1); + }); + + it("skips a pull request that ship is already driving", async () => { + // Ship runs its own review -> resolve iteration, so a second review would + // duplicate the spend and race ship's insertQueued on the in-flight index. + shipActive = true; + + await fire(syncPayload()); + + expect(mockDispatchWorkflowByName).not.toHaveBeenCalled(); + }); +}); diff --git a/test/webhook/events/pull-request-cache.test.ts b/test/webhook/events/pull-request-cache.test.ts index 20a315a0..5d6de753 100644 --- a/test/webhook/events/pull-request-cache.test.ts +++ b/test/webhook/events/pull-request-cache.test.ts @@ -37,11 +37,13 @@ beforeAll(async () => { const db = getDb(); if (db === null) return; try { - await db`SELECT 1 FROM target_cache LIMIT 1`; - dbAvailable = true; + await db`SELECT 1`; } catch { - dbAvailable = false; + return; } + const { runMigrations } = await import("../../../src/db/migrate"); + await runMigrations(db); + dbAvailable = true; }); const skipIfNoDb = (): boolean => !dbAvailable; diff --git a/test/webhook/events/pull-request-config-check.test.ts b/test/webhook/events/pull-request-config-check.test.ts new file mode 100644 index 00000000..af730e85 --- /dev/null +++ b/test/webhook/events/pull-request-config-check.test.ts @@ -0,0 +1,288 @@ +/** + * Handler tests for the PR-side `.github-app.yaml` validation branch + * (issue #3, C7). + * + * `handlePullRequestConfigCheck` is the thin layer between the + * `pull_request` subscription and `runPrConfigCheck`. What is pinned here is + * the idempotency contract, not the rendering: + * + * - a replayed delivery (same `X-GitHub-Delivery`) performs the GitHub + * write at most once; + * - the claim key is `${deliveryId}:config-check`, NOT the bare + * `deliveryId` that `handlePullRequestLabeled` already claims, so the two + * branches of one delivery can never starve each other; + * - the repo-wide `enabled: false` master switch silences this GitHub-write + * surface, and a `runPrConfigCheck` rejection is contained rather than + * escalated to the process-killing `unhandledRejection` handler; + * - the action guard inside `handlePullRequest` that reaches this handler at + * all (last describe block), which is the feature's actual on-switch. + * + * Dispatch is fire-and-forget inside a `void (async () => {...})()` IIFE whose + * first statement is `await claimDelivery(...)` (issue #202). That await defers + * the `runPrConfigCheck` call past the synchronous test body, so every positive + * case must drain the microtask queue (`flushMicrotasks`) before asserting, + * otherwise the deferred call also leaks into the next test after `mockClear`. + */ + +import type { PullRequestEvent } from "@octokit/webhooks-types"; +import { beforeEach, describe, expect, it, mock } from "bun:test"; +import type { Octokit } from "octokit"; + +// ─── Mocked downstream surfaces ────────────────────────────────────────── + +// Behaviour is swapped through a variable rather than `mockImplementationOnce` +// so the call-args assertions below keep binding to one stable mock. +let runPrConfigCheckImpl: () => Promise = () => Promise.resolve(); +const mockRunPrConfigCheck = mock((_input: unknown) => runPrConfigCheckImpl()); +void mock.module("../../../src/repo-config/pr-check", () => ({ + runPrConfigCheck: mockRunPrConfigCheck, +})); + +// The repo-wide `enabled: false` master switch. Only that switch is honoured +// on this surface, never the full Gate-1 trigger set. +let repoPolicy: { enabled: boolean } = { enabled: true }; +const mockLoadRepoPolicy = mock((_input: unknown) => Promise.resolve(repoPolicy)); +void mock.module("../../../src/repo-config/effective", () => ({ + loadRepoPolicy: mockLoadRepoPolicy, + // `pull-request.ts` also imports this for the auto-review opt-in check. The + // config-check path never reaches it, but the module mock must still export + // it or the import fails at load time. + policyForWorkflow: () => ({ + enabled: true, + extraAllowedTools: [], + pathFilters: [], + auto: false, + }), +})); + +// One-shot claim per key, mirroring the Valkey `SET NX` semantics. +const claimed = new Set(); +const mockClaimDelivery = mock((key: string) => { + if (claimed.has(key)) return Promise.resolve(false); + claimed.add(key); + return Promise.resolve(true); +}); +void mock.module("../../../src/webhook/idempotency", () => ({ + claimDelivery: mockClaimDelivery, +})); + +// The labeled branch's downstream surfaces: stubbed so the shared-delivery +// test can exercise it without touching the dispatcher or the DB. +const mockDispatchByLabel = mock(() => + Promise.resolve({ status: "dispatched" as const, runId: "run-1", workflowName: "triage" }), +); +void mock.module("../../../src/workflows/dispatcher", () => ({ + dispatchByLabel: mockDispatchByLabel, + // Imported by `pull-request.ts` for auto-review. Unreachable here (no + // AUTO_REVIEW_USERS in this suite's env), but the mock must export it. + dispatchWorkflowByName: mock(() => Promise.resolve({ status: "ignored", reason: "test" })), +})); +void mock.module("../../../src/workflows/ship/command-dispatch", () => ({ + dispatchCanonicalCommand: mock(() => undefined), +})); +void mock.module("../../../src/workflows/ship/reactor-bridge", () => ({ + fireReactor: mock(() => undefined), +})); +void mock.module("../../../src/workflows/ship/trigger-router", () => ({ + routeTrigger: mock(() => Promise.resolve(null)), +})); +void mock.module("../../../src/db/queries/conversation-store", () => ({ + upsertTarget: mock(() => Promise.resolve()), +})); + +// Keep `isOwnerAllowed` real; pin ALLOWED_OWNERS to a controlled list. +void mock.module("../../../src/config", () => ({ + config: { + allowedOwners: ["acme"], + logLevel: "silent", + nodeEnv: "test", + }, +})); + +const { handlePullRequest, handlePullRequestConfigCheck } = + await import("../../../src/webhook/events/pull-request"); + +const fakeOctokit = {} as unknown as Octokit; + +const HEAD_SHA = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"; + +// Drain the microtask queue so the fire-and-forget IIFE runs past its leading +// `await claimDelivery(...)` gate (#202) and the `runPrConfigCheck` call lands +// before assertions / the next test's `mockClear`. +async function flushMicrotasks(): Promise { + for (let i = 0; i < 5; i++) await Promise.resolve(); +} + +type PrAction = "opened" | "synchronize" | "reopened" | "labeled" | "closed" | "edited"; + +function prPayload(overrides?: { + action?: PrAction; + labelName?: string; + senderLogin?: string; +}): PullRequestEvent { + return { + action: overrides?.action ?? "opened", + number: 77, + installation: { id: 555 }, + label: overrides?.labelName !== undefined ? { name: overrides.labelName } : undefined, + pull_request: { + number: 77, + title: "Add repo config", + body: "", + state: "open", + merged: false, + draft: false, + created_at: "2026-05-11T00:00:00Z", + updated_at: "2026-05-11T00:00:00Z", + user: { login: "acme" }, + base: { ref: "main" }, + head: { ref: "feature", sha: HEAD_SHA }, + }, + repository: { + name: "widgets", + owner: { login: "acme" }, + }, + sender: { login: overrides?.senderLogin ?? "acme" }, + } as unknown as PullRequestEvent; +} + +describe("handlePullRequestConfigCheck", () => { + beforeEach(() => { + claimed.clear(); + runPrConfigCheckImpl = () => Promise.resolve(); + repoPolicy = { enabled: true }; + mockRunPrConfigCheck.mockClear(); + mockClaimDelivery.mockClear(); + mockDispatchByLabel.mockClear(); + mockLoadRepoPolicy.mockClear(); + }); + + it("C7: claims `:config-check`, never the bare deliveryId", async () => { + handlePullRequestConfigCheck(fakeOctokit, prPayload(), "delivery-a"); + await flushMicrotasks(); + + expect(mockClaimDelivery).toHaveBeenCalledTimes(1); + const key = (mockClaimDelivery.mock.calls[0] as unknown as [string])[0]; + expect(key).toBe("delivery-a:config-check"); + + expect(mockRunPrConfigCheck).toHaveBeenCalledTimes(1); + const args = ( + mockRunPrConfigCheck.mock.calls[0] as unknown as [ + { owner: string; repo: string; prNumber: number; headSha: string; deliveryId: string }, + ] + )[0]; + expect(args.owner).toBe("acme"); + expect(args.repo).toBe("widgets"); + expect(args.prNumber).toBe(77); + expect(args.headSha).toBe(HEAD_SHA); + expect(args.deliveryId).toBe("delivery-a"); + }); + + it("C7: a replayed delivery performs the GitHub write at most once", async () => { + handlePullRequestConfigCheck(fakeOctokit, prPayload(), "delivery-dup"); + await flushMicrotasks(); + handlePullRequestConfigCheck(fakeOctokit, prPayload({ action: "synchronize" }), "delivery-dup"); + await flushMicrotasks(); + + expect(mockClaimDelivery).toHaveBeenCalledTimes(2); + expect(mockRunPrConfigCheck).toHaveBeenCalledTimes(1); + }); + + it("C7: a labeled delivery that claimed the bare deliveryId does not starve the config check", async () => { + // Same X-GitHub-Delivery reaching both branches: the labeled branch claims + // `delivery-shared`, so a shared key would silently drop the validation + // comment. + handlePullRequest( + fakeOctokit, + prPayload({ action: "labeled", labelName: "bot:ship" }), + "delivery-shared", + ); + await flushMicrotasks(); + expect(mockClaimDelivery).toHaveBeenCalledWith("delivery-shared", expect.anything()); + + handlePullRequestConfigCheck(fakeOctokit, prPayload(), "delivery-shared"); + await flushMicrotasks(); + + expect(mockRunPrConfigCheck).toHaveBeenCalledTimes(1); + const keys = mockClaimDelivery.mock.calls.map((c) => (c as unknown as [string])[0]); + expect(keys).toContain("delivery-shared"); + expect(keys).toContain("delivery-shared:config-check"); + }); + + it("drops events whose sender is outside ALLOWED_OWNERS before claiming anything", async () => { + handlePullRequestConfigCheck(fakeOctokit, prPayload({ senderLogin: "stranger" }), "delivery-x"); + await flushMicrotasks(); + + expect(mockClaimDelivery).not.toHaveBeenCalled(); + expect(mockRunPrConfigCheck).not.toHaveBeenCalled(); + }); + + it("stays silent when the default branch's config sets `enabled: false`", async () => { + repoPolicy = { enabled: false }; + handlePullRequestConfigCheck(fakeOctokit, prPayload(), "delivery-disabled"); + await flushMicrotasks(); + + expect(mockLoadRepoPolicy).toHaveBeenCalledTimes(1); + // No comment, and no refusal comment either: this surface carries no + // `explain` obligation. + expect(mockRunPrConfigCheck).not.toHaveBeenCalled(); + }); + + it("contains a runPrConfigCheck rejection instead of leaking an unhandled rejection", async () => { + // `src/logger.ts` installs an `unhandledRejection` handler that calls + // `process.exit(1)`, so losing the handler's try/catch would take the + // whole webhook server down on one bad config read. + const unhandled: unknown[] = []; + const onUnhandled = (reason: unknown): void => { + unhandled.push(reason); + }; + process.on("unhandledRejection", onUnhandled); + try { + runPrConfigCheckImpl = () => Promise.reject(new Error("getContent exploded")); + handlePullRequestConfigCheck(fakeOctokit, prPayload(), "delivery-throw"); + await flushMicrotasks(); + await new Promise((resolve) => setTimeout(resolve, 10)); + + expect(mockRunPrConfigCheck).toHaveBeenCalledTimes(1); + expect(unhandled).toHaveLength(0); + } finally { + process.off("unhandledRejection", onUnhandled); + } + }); +}); + +/** + * The feature's on-switch. Every test above drives `handlePullRequestConfigCheck` + * directly, so the action guard inside `handlePullRequest` that actually reaches + * it is only covered here: deleting that block must fail a test rather than + * silently turning the feature off. + */ +describe("handlePullRequest action gating for the config check", () => { + beforeEach(() => { + claimed.clear(); + runPrConfigCheckImpl = () => Promise.resolve(); + repoPolicy = { enabled: true }; + mockRunPrConfigCheck.mockClear(); + mockClaimDelivery.mockClear(); + mockDispatchByLabel.mockClear(); + mockLoadRepoPolicy.mockClear(); + }); + + const cases: { action: PrAction; fires: boolean }[] = [ + { action: "opened", fires: true }, + { action: "synchronize", fires: true }, + { action: "reopened", fires: true }, + { action: "labeled", fires: false }, + { action: "closed", fires: false }, + { action: "edited", fires: false }, + ]; + + for (const { action, fires } of cases) { + it(`${fires ? "runs" : "skips"} the config check on pull_request.${action}`, async () => { + handlePullRequest(fakeOctokit, prPayload({ action }), `delivery-${action}`); + await flushMicrotasks(); + + expect(mockRunPrConfigCheck).toHaveBeenCalledTimes(fires ? 1 : 0); + }); + } +}); diff --git a/test/workflows/dispatch-outbox.test.ts b/test/workflows/dispatch-outbox.test.ts new file mode 100644 index 00000000..f049265f --- /dev/null +++ b/test/workflows/dispatch-outbox.test.ts @@ -0,0 +1,284 @@ +import { SQL } from "bun"; +import { afterAll, beforeAll, beforeEach, describe, expect, it, mock } from "bun:test"; + +import { expectToReject } from "../utils/assertions"; + +const TEST_DATABASE_URL = + process.env["TEST_DATABASE_URL"] ?? "postgres://bot:bot@localhost:55432/github_app_test"; + +let sql: SQL | null = null; +try { + const connection = new SQL(TEST_DATABASE_URL); + await connection`SELECT 1`; + sql = connection; +} catch { + sql = null; +} + +function requireSql(): SQL { + if (sql === null) throw new Error("Database not available, test should have been skipped"); + return sql; +} + +const ensureWorkflowJobQueued = mock(async () => Promise.resolve(true)); +void mock.module("../../src/orchestrator/job-queue", () => ({ + ensureWorkflowJobQueued, + isScopedJob: () => false, + SCOPED_JOB_KINDS: ["scoped-rebase", "scoped-fix-thread", "scoped-open-pr"], +})); +void mock.module("../../src/orchestrator/instance-id", () => ({ + getInstanceId: () => "orchestrator-test", +})); + +async function resetSchema(): Promise { + await requireSql().unsafe(` + DROP TABLE IF EXISTS _migrations CASCADE; + DROP TABLE IF EXISTS workflow_attempt_commands CASCADE; + DROP TABLE IF EXISTS review_learnings CASCADE; + DROP TABLE IF EXISTS scheduled_action_state CASCADE; + DROP TABLE IF EXISTS comment_cache CASCADE; + DROP TABLE IF EXISTS target_cache CASCADE; + DROP TABLE IF EXISTS chat_proposals CASCADE; + DROP TABLE IF EXISTS ship_fix_attempts CASCADE; + DROP TABLE IF EXISTS ship_continuations CASCADE; + DROP TABLE IF EXISTS ship_iterations CASCADE; + DROP TABLE IF EXISTS ship_intents CASCADE; + DROP TABLE IF EXISTS workflow_runs CASCADE; + DROP TABLE IF EXISTS repo_memory CASCADE; + DROP TABLE IF EXISTS triage_results CASCADE; + DROP TABLE IF EXISTS executions CASCADE; + DROP TABLE IF EXISTS daemons CASCADE; + `); +} + +describe.skipIf(sql === null)("workflow dispatch outbox", () => { + beforeAll(async () => { + await resetSchema(); + const { runMigrations } = await import("../../src/db/migrate"); + await runMigrations(requireSql()); + }); + + afterAll(async () => { + await resetSchema(); + await requireSql().close(); + }); + + beforeEach(() => { + ensureWorkflowJobQueued.mockReset(); + ensureWorkflowJobQueued.mockResolvedValue(true); + }); + + async function seedPendingRun(number: number): Promise<{ runId: string; deliveryId: string }> { + const { insertQueued } = await import("../../src/workflows/runs-store"); + const { recordWorkflowExecution } = await import("../../src/workflows/execution-row"); + const deliveryId = crypto.randomUUID(); + const parent = await insertQueued( + { + workflowName: "ship", + target: { type: "pr", owner: "acme", repo: "widgets", number }, + ownerKind: "orchestrator", + ownerId: "orchestrator-test", + }, + requireSql(), + ); + const row = await insertQueued( + { + workflowName: "review", + target: { type: "pr", owner: "acme", repo: "widgets", number }, + deliveryId: "parent-trace", + executionDeliveryId: deliveryId, + triggerBodyPreview: "run python in a docker container", + parentRunId: parent.id, + parentStepIndex: 3, + ownerKind: "orchestrator", + ownerId: "orchestrator-before-publish", + }, + requireSql(), + ); + await recordWorkflowExecution({ + deliveryId, + target: { type: "pr", owner: "acme", repo: "widgets", number }, + senderLogin: "github-app-test", + workflowName: "review", + runId: row.id, + labels: ["bot:review"], + logger: { info: () => undefined } as never, + sql: requireSql(), + }); + return { runId: row.id, deliveryId }; + } + + it("publishes a committed row once and records the dispatch receipt", async () => { + const { publishWorkflowRunById } = await import("../../src/workflows/dispatch-outbox"); + const { findById, findCommittedWorkflowDispatch } = + await import("../../src/workflows/runs-store"); + const { runId, deliveryId } = await seedPendingRun(501); + + expect( + await findCommittedWorkflowDispatch( + { + workflowName: "review", + target: { type: "pr", owner: "acme", repo: "widgets", number: 501 }, + executionDeliveryId: deliveryId, + }, + requireSql(), + ), + ).toMatchObject({ id: runId }); + expect( + await findCommittedWorkflowDispatch( + { + workflowName: "review", + target: { type: "pr", owner: "acme", repo: "widgets", number: 999 }, + executionDeliveryId: deliveryId, + }, + requireSql(), + ), + ).toBeNull(); + + expect(await publishWorkflowRunById(runId, requireSql())).toBe(true); + expect(await publishWorkflowRunById(runId, requireSql())).toBe(false); + expect(ensureWorkflowJobQueued).toHaveBeenCalledTimes(1); + expect(ensureWorkflowJobQueued).toHaveBeenCalledWith( + expect.objectContaining({ + kind: "workflow-run", + repoOwner: "acme", + repoName: "widgets", + entityNumber: 501, + isPR: true, + labels: ["bot:review"], + triggerBodyPreview: "run python in a docker container", + workflowRun: expect.objectContaining({ + runId, + workflowName: "review", + parentStepIndex: 3, + }), + }), + "orchestrator-test", + ); + const row = await findById(runId, requireSql()); + expect(row?.dispatch_enqueued_at).toBeInstanceOf(Date); + expect(row?.owner_kind).toBeNull(); + expect(row?.owner_id).toBeNull(); + }); + + it("leaves a failed publication pending and retries it on the next pass", async () => { + const { publishPendingWorkflowRuns } = await import("../../src/workflows/dispatch-outbox"); + const { findById } = await import("../../src/workflows/runs-store"); + const { runId } = await seedPendingRun(502); + ensureWorkflowJobQueued.mockRejectedValueOnce(new Error("Valkey unavailable")); + + expect(await publishPendingWorkflowRuns(requireSql())).toBe(0); + expect((await findById(runId, requireSql()))?.dispatch_enqueued_at).toBeNull(); + expect(await publishPendingWorkflowRuns(requireSql())).toBe(1); + expect((await findById(runId, requireSql()))?.dispatch_enqueued_at).toBeInstanceOf(Date); + expect(ensureWorkflowJobQueued).toHaveBeenCalledTimes(2); + }); + + it("durably counts failed publication attempts", async () => { + const { publishWorkflowRunById } = await import("../../src/workflows/dispatch-outbox"); + const { findById } = await import("../../src/workflows/runs-store"); + const { runId } = await seedPendingRun(507); + ensureWorkflowJobQueued.mockRejectedValueOnce(new Error("Valkey unavailable")); + + await expectToReject(publishWorkflowRunById(runId, requireSql()), "Valkey unavailable"); + + expect((await findById(runId, requireSql()))?.dispatch_retry_count).toBe(1); + expect(await publishWorkflowRunById(runId, requireSql())).toBe(true); + }); + + it("does not republish a row after its durable retry budget is exhausted", async () => { + const { config } = await import("../../src/config"); + const { publishWorkflowRunById } = await import("../../src/workflows/dispatch-outbox"); + const { runId } = await seedPendingRun(508); + await requireSql()` + UPDATE workflow_runs + SET dispatch_retry_count = ${config.jobMaxRetries + 1} + WHERE id = ${runId} + `; + + expect(await publishWorkflowRunById(runId, requireSql())).toBe(false); + expect(ensureWorkflowJobQueued).not.toHaveBeenCalled(); + }); + + it("publishes the durable retry count after an orchestrator restart", async () => { + const { publishWorkflowRunById } = await import("../../src/workflows/dispatch-outbox"); + const { runId } = await seedPendingRun(504); + await requireSql()` + UPDATE workflow_runs + SET dispatch_retry_count = 2 + WHERE id = ${runId} + `; + + expect(await publishWorkflowRunById(runId, requireSql())).toBe(true); + expect(ensureWorkflowJobQueued).toHaveBeenCalledWith( + expect.objectContaining({ retryCount: 2 }), + "orchestrator-test", + ); + }); + + it("does not let an old publication close a newer dispatch generation", async () => { + const { publishWorkflowRunById } = await import("../../src/workflows/dispatch-outbox"); + const { findById } = await import("../../src/workflows/runs-store"); + const { runId } = await seedPendingRun(503); + ensureWorkflowJobQueued.mockImplementationOnce(async () => { + await requireSql()` + UPDATE workflow_runs + SET dispatch_generation_id = gen_random_uuid() + WHERE id = ${runId} + `; + }); + + expect(await publishWorkflowRunById(runId, requireSql())).toBe(false); + expect((await findById(runId, requireSql()))?.dispatch_enqueued_at).toBeNull(); + expect(await publishWorkflowRunById(runId, requireSql())).toBe(true); + expect((await findById(runId, requireSql()))?.dispatch_enqueued_at).toBeInstanceOf(Date); + expect(ensureWorkflowJobQueued).toHaveBeenCalledTimes(2); + }); + + it("reconciles an acknowledged wake-up after its grace period without changing its bytes", async () => { + const { publishPendingWorkflowRuns, WORKFLOW_DISPATCH_RECONCILE_GRACE_MS } = + await import("../../src/workflows/dispatch-outbox"); + const { findById } = await import("../../src/workflows/runs-store"); + const { runId } = await seedPendingRun(505); + + expect(await publishPendingWorkflowRuns(requireSql())).toBe(1); + const firstJob = ensureWorkflowJobQueued.mock.calls[0]?.[0]; + expect(await publishPendingWorkflowRuns(requireSql())).toBe(0); + + await requireSql()` + UPDATE workflow_runs + SET dispatch_enqueued_at = now() - ${WORKFLOW_DISPATCH_RECONCILE_GRACE_MS + 1_000} * interval '1 millisecond' + WHERE id = ${runId} + `; + expect(await publishPendingWorkflowRuns(requireSql())).toBe(1); + expect(ensureWorkflowJobQueued.mock.calls[1]?.[0]).toEqual(firstJob); + expect((await findById(runId, requireSql()))?.dispatch_enqueued_at).toBeInstanceOf(Date); + }); + + it("does not let a stale publication overwrite a concurrent claim", async () => { + const { publishPendingWorkflowRuns, WORKFLOW_DISPATCH_RECONCILE_GRACE_MS } = + await import("../../src/workflows/dispatch-outbox"); + const { findById } = await import("../../src/workflows/runs-store"); + const { runId } = await seedPendingRun(506); + await requireSql()` + UPDATE workflow_runs + SET dispatch_enqueued_at = now() - ${WORKFLOW_DISPATCH_RECONCILE_GRACE_MS + 1_000} * interval '1 millisecond' + WHERE id = ${runId} + `; + ensureWorkflowJobQueued.mockImplementationOnce(async () => { + await requireSql()` + UPDATE workflow_runs + SET status = 'failed', owner_kind = NULL, owner_id = NULL + WHERE id = ${runId} + `; + return false; + }); + + expect(await publishPendingWorkflowRuns(requireSql())).toBe(0); + expect(await findById(runId, requireSql())).toMatchObject({ + status: "failed", + owner_kind: null, + owner_id: null, + }); + }); +}); diff --git a/test/workflows/dispatcher.test.ts b/test/workflows/dispatcher.test.ts index 57893537..2e824f56 100644 --- a/test/workflows/dispatcher.test.ts +++ b/test/workflows/dispatcher.test.ts @@ -27,6 +27,23 @@ void mock.module("../../src/workflows/execution-row", () => ({ buildWorkflowContextJson: mock(() => ({})), })); +const transaction = {}; +const mockDbBegin = mock(async (callback: (sql: unknown) => Promise) => + callback(transaction), +); +const fakeDb = { begin: mockDbBegin }; +void mock.module("../../src/db", () => ({ + getDb: () => fakeDb, + requireDb: () => fakeDb, + closeDb: () => Promise.resolve(), +})); + +const mockPublishWorkflowRunById = mock(() => Promise.resolve(true)); +void mock.module("../../src/workflows/dispatch-outbox", () => ({ + publishWorkflowRunById: mockPublishWorkflowRunById, + publishPendingWorkflowRuns: mock(() => Promise.resolve(0)), +})); + void mock.module("../../src/orchestrator/concurrency", () => ({ incrementActiveCount: mock(() => {}), decrementActiveCount: mock(() => {}), @@ -44,12 +61,14 @@ const mockInsertQueued = mock(() => ); const mockFindLatestForTarget = mock(() => Promise.resolve(null as unknown)); const mockFindLatestSucceededForTarget = mock(() => Promise.resolve(null as unknown)); -const mockMarkFailed = mock(() => Promise.resolve()); +const mockFindCommittedWorkflowDispatch = mock(() => Promise.resolve(null as unknown)); +const realRunsStore = await import("../../src/workflows/runs-store"); void mock.module("../../src/workflows/runs-store", () => ({ + ...realRunsStore, insertQueued: mockInsertQueued, findLatestForTarget: mockFindLatestForTarget, findLatestSucceededForTarget: mockFindLatestSucceededForTarget, - markFailed: mockMarkFailed, + findCommittedWorkflowDispatch: mockFindCommittedWorkflowDispatch, // findById is imported by review/resolve handlers (transitively reachable // when registry resolves them). Provide a stub so module loading succeeds, // tests in this file don't exercise that code path. @@ -61,8 +80,40 @@ void mock.module("../../src/workflows/tracking-mirror", () => ({ postRefusalComment: mockPostRefusalComment, })); +// Gate 1's config loader. Stubbed rather than left to fail open: `fakeOctokit` +// below is an empty object, so the real `loadRepoPolicy` throws internally and +// degrades to the permissive default, which would let every gate assertion in +// this file pass even if the call site were deleted. +const realEffective = await import("../../src/repo-config/effective"); +const { githubAppConfigSchema } = await import("../../src/repo-config/schema"); +const mockLoadRepoPolicy = mock(() => Promise.resolve(realEffective.DEFAULT_REPO_POLICY)); +void mock.module("../../src/repo-config/effective", () => ({ + ...realEffective, + loadRepoPolicy: mockLoadRepoPolicy, +})); + +// The LLM call `dispatchByIntent` makes after the gate. Mocked so the gate +// tests can assert it was never reached, which is the whole point of gating +// before classification. +const mockClassify = mock(() => + Promise.resolve({ workflow: "triage" as const, confidence: 0.99, rationale: "test" }), +); +void mock.module("../../src/workflows/intent-classifier", () => ({ + classify: mockClassify, +})); + // Import dispatcher AFTER mocks. -const { dispatchByLabel } = await import("../../src/workflows/dispatcher"); +const { dispatchByIntent, dispatchByLabel, dispatchWorkflowByName } = + await import("../../src/workflows/dispatcher"); + +beforeEach(() => { + mockRecordWorkflowExecution.mockClear(); + mockPublishWorkflowRunById.mockReset(); + mockPublishWorkflowRunById.mockResolvedValue(true); + mockDbBegin.mockClear(); + mockFindCommittedWorkflowDispatch.mockReset(); + mockFindCommittedWorkflowDispatch.mockResolvedValue(null); +}); // ─── Test fixtures ─────────────────────────────────────────────────────── @@ -81,6 +132,11 @@ function silentLog(): pino.Logger { const fakeOctokit = {} as unknown as Octokit; +/** Build a policy from a partial YAML document, exercising the real resolver. */ +function policyFrom(doc: Record): typeof realEffective.DEFAULT_REPO_POLICY { + return realEffective.resolvePolicy(githubAppConfigSchema.parse({ version: 1, ...doc })); +} + function baseParams(overrides: { label: string; targetType: "issue" | "pr"; @@ -109,8 +165,12 @@ describe("dispatchByLabel", () => { mockInsertQueued.mockClear(); mockFindLatestForTarget.mockClear(); mockFindLatestSucceededForTarget.mockClear(); - mockMarkFailed.mockClear(); + mockRecordWorkflowExecution.mockClear(); + mockPublishWorkflowRunById.mockReset(); + mockPublishWorkflowRunById.mockResolvedValue(true); + mockDbBegin.mockClear(); mockPostRefusalComment.mockClear(); + mockLoadRepoPolicy.mockClear(); }); it("returns ignored for unknown labels without touching any downstream surface", async () => { @@ -124,7 +184,7 @@ describe("dispatchByLabel", () => { } expect(mockEnforceSingleBotLabel).not.toHaveBeenCalled(); expect(mockInsertQueued).not.toHaveBeenCalled(); - expect(mockEnqueueJob).not.toHaveBeenCalled(); + expect(mockPublishWorkflowRunById).not.toHaveBeenCalled(); expect(mockPostRefusalComment).not.toHaveBeenCalled(); }); @@ -138,15 +198,14 @@ describe("dispatchByLabel", () => { } expect(mockEnforceSingleBotLabel).toHaveBeenCalledTimes(1); expect(mockInsertQueued).toHaveBeenCalledTimes(1); - expect(mockEnqueueJob).toHaveBeenCalledTimes(1); - - // workflowRun must be threaded into the queued job for daemon routing. - const enqueueCall = mockEnqueueJob.mock.calls[0] as unknown as [ - { workflowRun?: { runId: string; workflowName: string } }, - ]; - const queued = enqueueCall[0]; - expect(queued.workflowRun?.runId).toBe("00000000-0000-0000-0000-000000000001"); - expect(queued.workflowRun?.workflowName).toBe("triage"); + expect(mockRecordWorkflowExecution).toHaveBeenCalledWith( + expect.objectContaining({ + runId: "00000000-0000-0000-0000-000000000001", + workflowName: "triage", + sql: transaction, + }), + ); + expect(mockPublishWorkflowRunById).toHaveBeenCalledWith("00000000-0000-0000-0000-000000000001"); }); it("refuses a known label whose context mismatches (bot:resolve on issue)", async () => { @@ -159,7 +218,7 @@ describe("dispatchByLabel", () => { } expect(mockPostRefusalComment).toHaveBeenCalledTimes(1); expect(mockInsertQueued).not.toHaveBeenCalled(); - expect(mockEnqueueJob).not.toHaveBeenCalled(); + expect(mockPublishWorkflowRunById).not.toHaveBeenCalled(); }); it("refuses when requiresPrior is unsatisfied (bot:plan without a successful triage)", async () => { @@ -175,7 +234,7 @@ describe("dispatchByLabel", () => { expect(mockFindLatestSucceededForTarget).toHaveBeenCalledTimes(1); expect(mockPostRefusalComment).toHaveBeenCalledTimes(1); expect(mockInsertQueued).not.toHaveBeenCalled(); - expect(mockEnqueueJob).not.toHaveBeenCalled(); + expect(mockPublishWorkflowRunById).not.toHaveBeenCalled(); expect(mockEnforceSingleBotLabel).not.toHaveBeenCalled(); }); @@ -183,7 +242,8 @@ describe("dispatchByLabel", () => { const collisionErr = Object.assign( new Error("duplicate key value violates unique constraint"), { - code: "23505", + code: "ERR_POSTGRES_SERVER_ERROR", + errno: "23505", constraint: "idx_workflow_runs_inflight", }, ); @@ -197,11 +257,10 @@ describe("dispatchByLabel", () => { expect(result.reason).toContain("in-flight"); } expect(mockPostRefusalComment).toHaveBeenCalledTimes(1); - expect(mockEnqueueJob).not.toHaveBeenCalled(); - // #211 Part 2: a collision surfaces "in-flight" (refused), it must NOT be - // marked failed. markFailed is reserved for post-insert enqueue failures - // (next test), where it deliberately clears the in-flight guard for retry. - expect(mockMarkFailed).not.toHaveBeenCalled(); + expect(mockPostRefusalComment.mock.calls[0]?.[3]).toBe( + "an in-flight run already exists for this workflow and target", + ); + expect(mockPublishWorkflowRunById).not.toHaveBeenCalled(); }); it("rethrows non-collision insertQueued errors without refusing", async () => { @@ -213,18 +272,393 @@ describe("dispatchByLabel", () => { ); expect(mockPostRefusalComment).not.toHaveBeenCalled(); - expect(mockEnqueueJob).not.toHaveBeenCalled(); + expect(mockPublishWorkflowRunById).not.toHaveBeenCalled(); + }); + + it("continues after an ambiguous commit when the exact durable pair exists", async () => { + mockInsertQueued.mockRejectedValueOnce(new Error("connection lost after commit")); + mockFindCommittedWorkflowDispatch.mockResolvedValueOnce({ + id: "00000000-0000-0000-0000-000000000099", + }); + + const result = await dispatchByLabel(baseParams({ label: "bot:triage", targetType: "issue" })); + + expect(result).toEqual({ + status: "dispatched", + runId: "00000000-0000-0000-0000-000000000099", + workflowName: "triage", + }); + expect(mockFindCommittedWorkflowDispatch).toHaveBeenCalledWith( + { + workflowName: "triage", + target: { type: "issue", owner: "acme", repo: "repo", number: 42 }, + executionDeliveryId: "delivery-abc", + }, + fakeDb, + ); + expect(mockPublishWorkflowRunById).toHaveBeenCalledWith("00000000-0000-0000-0000-000000000099"); + expect(mockPostRefusalComment).not.toHaveBeenCalled(); + }); + + it("retains the trigger until an ambiguous commit can be read", async () => { + mockInsertQueued.mockRejectedValueOnce(new Error("connection lost after commit")); + mockFindCommittedWorkflowDispatch + .mockRejectedValueOnce(new Error("database read unavailable")) + .mockResolvedValueOnce({ + id: "00000000-0000-0000-0000-000000000098", + }); + + const result = await dispatchByLabel(baseParams({ label: "bot:triage", targetType: "issue" })); + + expect(result).toEqual({ + status: "dispatched", + runId: "00000000-0000-0000-0000-000000000098", + workflowName: "triage", + }); + expect(mockFindCommittedWorkflowDispatch).toHaveBeenCalledTimes(2); + expect(mockPostRefusalComment).not.toHaveBeenCalled(); }); - it("clears in-flight guard via markFailed when enqueue fails after insert", async () => { - mockEnqueueJob.mockRejectedValueOnce(new Error("valkey unreachable")); + it("stops ambiguous-commit reconciliation and surfaces the dispatch failure", async () => { + mockInsertQueued.mockRejectedValueOnce(new Error("connection lost after commit")); + mockFindCommittedWorkflowDispatch.mockRejectedValue(new Error("database read unavailable")); await expectToReject( dispatchByLabel(baseParams({ label: "bot:triage", targetType: "issue" })), - "valkey unreachable", + "connection lost after commit", ); - expect(mockMarkFailed).toHaveBeenCalledTimes(1); + expect(mockFindCommittedWorkflowDispatch).toHaveBeenCalledTimes(8); + expect(mockPublishWorkflowRunById).not.toHaveBeenCalled(); + }); + + it("keeps the durable run pending when queue publication fails", async () => { + mockPublishWorkflowRunById.mockRejectedValueOnce(new Error("valkey unreachable")); + + const result = await dispatchByLabel(baseParams({ label: "bot:triage", targetType: "issue" })); + + expect(result.status).toBe("dispatched"); + expect(mockPublishWorkflowRunById).toHaveBeenCalledTimes(1); + expect(mockPostRefusalComment).not.toHaveBeenCalled(); + }); +}); + +// ─── Gate 1 wiring ──────────────────────────────────────────────────────── + +/** + * These assert the gate is *called* and its verdict is *honoured*, not the + * rule semantics themselves (`test/repo-config/gate.test.ts` owns those). + * Deleting the `applyRepoGate` call from `dispatchByLabel` must fail here. + */ +describe("dispatchByLabel repo-config gate", () => { + beforeEach(() => { + mockEnqueueJob.mockClear(); + mockInsertQueued.mockClear(); + mockEnforceSingleBotLabel.mockClear(); + mockPostRefusalComment.mockClear(); + mockLoadRepoPolicy.mockClear(); + mockLoadRepoPolicy.mockResolvedValue(realEffective.DEFAULT_REPO_POLICY); + }); + + it("consults the repo policy on every dispatch", async () => { + await dispatchByLabel(baseParams({ label: "bot:triage", targetType: "issue" })); + expect(mockLoadRepoPolicy).toHaveBeenCalledTimes(1); + }); + + it("refuses with a comment when the repo is disabled, before any side effect", async () => { + mockLoadRepoPolicy.mockResolvedValue({ ...realEffective.DEFAULT_REPO_POLICY, enabled: false }); + + const result = await dispatchByLabel(baseParams({ label: "bot:triage", targetType: "issue" })); + + expect(result.status).toBe("refused"); + if (result.status === "refused") expect(result.reason).toContain("disabled"); + // `explain: true`, so the user who applied the label hears why. + expect(mockPostRefusalComment).toHaveBeenCalledTimes(1); + // Nothing persisted, nothing queued, no label mutation. + expect(mockInsertQueued).not.toHaveBeenCalled(); + expect(mockPublishWorkflowRunById).not.toHaveBeenCalled(); + expect(mockEnforceSingleBotLabel).not.toHaveBeenCalled(); + }); + + it("refuses silently when a passive trigger filter matches", async () => { + mockLoadRepoPolicy.mockResolvedValue({ + ...realEffective.DEFAULT_REPO_POLICY, + triggers: { ...realEffective.DEFAULT_REPO_POLICY.triggers, ignoreAuthors: ["alice"] }, + }); + + const result = await dispatchByLabel(baseParams({ label: "bot:triage", targetType: "issue" })); + + expect(result.status).toBe("refused"); + // `explain: false`: a filter set to keep the bot quiet must stay quiet. + expect(mockPostRefusalComment).not.toHaveBeenCalled(); + expect(mockPublishWorkflowRunById).not.toHaveBeenCalled(); + }); + + it("checks ignore_authors before allowed_users, so a bot author stays silent", async () => { + // Both rules would block. The louder one must not win, or every Renovate + // event earns a public refusal comment. + mockLoadRepoPolicy.mockResolvedValue({ + ...realEffective.DEFAULT_REPO_POLICY, + triggers: { + ...realEffective.DEFAULT_REPO_POLICY.triggers, + ignoreAuthors: ["alice"], + allowedUsers: ["bob"], + }, + }); + + const result = await dispatchByLabel(baseParams({ label: "bot:triage", targetType: "issue" })); + + expect(result.status).toBe("refused"); expect(mockPostRefusalComment).not.toHaveBeenCalled(); }); + + it("honours a per-workflow disable without affecting other workflows", async () => { + mockLoadRepoPolicy.mockResolvedValue(policyFrom({ workflows: { triage: { enabled: false } } })); + + const blocked = await dispatchByLabel(baseParams({ label: "bot:triage", targetType: "issue" })); + expect(blocked.status).toBe("refused"); + if (blocked.status === "refused") expect(blocked.reason).toContain("triage"); + expect(mockPostRefusalComment).toHaveBeenCalledTimes(1); + + // `review` has no entry, so it inherits enabled and still dispatches. + const allowed = await dispatchByLabel(baseParams({ label: "bot:review", targetType: "pr" })); + expect(allowed.status).toBe("dispatched"); + }); +}); + +/** + * `dispatchByIntent` gates before classification, so the assertions here are + * about what did NOT happen: no LLM call, and no second config fetch. + */ +describe("dispatchByIntent repo-config gate", () => { + function intentParams(): Parameters[0] { + return { + octokit: fakeOctokit, + logger: silentLog(), + commentBody: "@chrisleekr-bot please triage this", + target: { type: "issue", owner: "acme", repo: "repo", number: 42 }, + senderLogin: "alice", + deliveryId: "delivery-intent", + triggerCommentId: 7, + triggerEventType: "issue_comment", + }; + } + + beforeEach(() => { + mockEnqueueJob.mockClear(); + mockInsertQueued.mockClear(); + mockPostRefusalComment.mockClear(); + mockClassify.mockClear(); + mockLoadRepoPolicy.mockClear(); + mockLoadRepoPolicy.mockResolvedValue(realEffective.DEFAULT_REPO_POLICY); + }); + + it("refuses without paying for classification when the repo is disabled", async () => { + mockLoadRepoPolicy.mockResolvedValue({ ...realEffective.DEFAULT_REPO_POLICY, enabled: false }); + + const result = await dispatchByIntent(intentParams()); + + expect(result.status).toBe("refused"); + // The gate exists to spend nothing on a repo that opted out. + expect(mockClassify).not.toHaveBeenCalled(); + expect(mockInsertQueued).not.toHaveBeenCalled(); + expect(mockPublishWorkflowRunById).not.toHaveBeenCalled(); + expect(mockPostRefusalComment).toHaveBeenCalledTimes(1); + }); + + it("refuses silently for a filtered author and still skips the LLM", async () => { + mockLoadRepoPolicy.mockResolvedValue({ + ...realEffective.DEFAULT_REPO_POLICY, + triggers: { ...realEffective.DEFAULT_REPO_POLICY.triggers, ignoreAuthors: ["alice"] }, + }); + + const result = await dispatchByIntent(intentParams()); + + expect(result.status).toBe("refused"); + expect(mockClassify).not.toHaveBeenCalled(); + expect(mockPostRefusalComment).not.toHaveBeenCalled(); + }); + + it("loads the policy once and threads it into the per-workflow re-check", async () => { + const result = await dispatchByIntent(intentParams()); + + expect(result.status).toBe("dispatched"); + expect(mockClassify).toHaveBeenCalledTimes(1); + // One fetch for the pre-classification gate, reused by + // `dispatchWorkflowByName` for the per-workflow rule. Two would mean the + // `policy` hand-off regressed to a second REST round trip per comment. + expect(mockLoadRepoPolicy).toHaveBeenCalledTimes(1); + }); + + it("refuses with the fixed comment for Bun's in-flight collision shape", async () => { + mockInsertQueued.mockRejectedValueOnce( + Object.assign(new Error("raw duplicate detail"), { + code: "ERR_POSTGRES_SERVER_ERROR", + errno: "23505", + constraint: "idx_workflow_runs_inflight", + }), + ); + + const result = await dispatchByIntent(intentParams()); + + expect(result).toMatchObject({ + status: "refused", + workflowName: "triage", + reason: "an in-flight run already exists for this workflow and target", + explained: true, + }); + expect(mockPostRefusalComment.mock.calls[0]?.[3]).toBe( + "an in-flight run already exists for this workflow and target", + ); + }); + + it("still refuses a disabled workflow once classification names it", async () => { + mockLoadRepoPolicy.mockResolvedValue(policyFrom({ workflows: { triage: { enabled: false } } })); + + const result = await dispatchByIntent(intentParams()); + + expect(result).toMatchObject({ status: "refused", workflowName: "triage", explained: true }); + // The LLM call is still paid: rule 2 cannot fire before the classifier + // names the workflow. Only rules 1 and 3 to 7 save that cost. + expect(mockClassify).toHaveBeenCalledTimes(1); + expect(mockPublishWorkflowRunById).not.toHaveBeenCalled(); + expect(mockPostRefusalComment).toHaveBeenCalledTimes(1); + // Still one fetch, not two: the post-classification re-check reuses the + // policy the pre-classification gate already loaded. + expect(mockLoadRepoPolicy).toHaveBeenCalledTimes(1); + }); +}); + +// ─── auto-trigger mode (work item #1) ──────────────────────────────────── + +describe("dispatchWorkflowByName auto mode", () => { + function autoParams(overrides: Record = {}) { + return { + octokit: fakeOctokit, + logger: silentLog(), + workflowName: "review" as const, + target: { type: "pr" as const, owner: "acme", repo: "widgets", number: 7 }, + senderLogin: "chrisleekr", + deliveryId: "d-auto", + triggerBodyPreview: "", + addRocketReaction: false, + ...overrides, + }; + } + + function inflightError(): Error { + return Object.assign(new Error("duplicate key value violates unique constraint"), { + code: "ERR_POSTGRES_SERVER_ERROR", + errno: "23505", + constraint: "idx_workflow_runs_inflight", + }); + } + + beforeEach(() => { + mockPostRefusalComment.mockClear(); + mockEnforceSingleBotLabel.mockClear(); + mockInsertQueued.mockClear(); + mockEnqueueJob.mockClear(); + // `mockResolvedValue` persists across suites, so without this the auto-mode + // cases inherit the last policy the preceding describe set. They pass today + // only because that policy disables `triage` and these dispatch `review`. + mockLoadRepoPolicy.mockClear(); + mockLoadRepoPolicy.mockResolvedValue(realEffective.DEFAULT_REPO_POLICY); + }); + + it("stays silent on an in-flight collision", async () => { + // The whole point of "ignore, do not queue": a push landing mid-review must + // not narrate itself on the PR. + mockInsertQueued.mockRejectedValueOnce(inflightError()); + + const result = await dispatchWorkflowByName(autoParams({ auto: true })); + + expect(result.status).toBe("refused"); + expect(mockPostRefusalComment).not.toHaveBeenCalled(); + }); + + it("still comments on an in-flight collision when not an auto-trigger", async () => { + // Regression guard for the label/mention callers, who DID ask and deserve + // an answer. + mockInsertQueued.mockRejectedValueOnce(inflightError()); + + const result = await dispatchWorkflowByName( + autoParams({ triggerCommentId: 1, triggerEventType: "issue_comment" }), + ); + + expect(result.status).toBe("refused"); + expect(mockPostRefusalComment).toHaveBeenCalledTimes(1); + }); + + it("skips the bot-label mutex, which would strip a user's bot:ship", async () => { + await dispatchWorkflowByName(autoParams({ auto: true })); + expect(mockEnforceSingleBotLabel).not.toHaveBeenCalled(); + }); + + it("still runs the mutex for a label/mention dispatch", async () => { + await dispatchWorkflowByName( + autoParams({ triggerCommentId: 1, triggerEventType: "issue_comment" }), + ); + expect(mockEnforceSingleBotLabel).toHaveBeenCalledTimes(1); + }); + + it("omits triggerCommentId entirely when the trigger had no comment", async () => { + // The column is nullable and TriggerEventType has a DB CHECK, so there is no + // synthetic value to pass; the key must be absent, not undefined. + await dispatchWorkflowByName(autoParams({ auto: true })); + + const arg = mockInsertQueued.mock.calls[0]?.[0] as Record; + expect(Object.hasOwn(arg, "triggerCommentId")).toBe(false); + expect(Object.hasOwn(arg, "triggerEventType")).toBe(false); + }); + + it("commits the capability preview for durable outbox publication", async () => { + await dispatchWorkflowByName( + autoParams({ triggerBodyPreview: "run python in a docker container" }), + ); + + expect(mockInsertQueued.mock.calls[0]?.[0]).toEqual( + expect.objectContaining({ triggerBodyPreview: "run python in a docker container" }), + ); + }); + + it("still commits and publishes normally", async () => { + const result = await dispatchWorkflowByName(autoParams({ auto: true })); + expect(result.status).toBe("dispatched"); + expect(mockPublishWorkflowRunById).toHaveBeenCalledTimes(1); + }); + + it("suppresses the Gate-1 refusal comment, and reports explained=false", async () => { + // A repo with `workflows.review.enabled: false` would otherwise be answered + // on every single push. `explained` must follow: it is the signal a caller + // uses to decide whether the dispatcher already spoke. + mockLoadRepoPolicy.mockResolvedValue(policyFrom({ workflows: { review: { enabled: false } } })); + + const result = await dispatchWorkflowByName(autoParams({ auto: true })); + + expect(result.status).toBe("refused"); + if (result.status === "refused") expect(result.explained).toBe(false); + expect(mockPostRefusalComment).not.toHaveBeenCalled(); + }); + + it("still comments on a Gate-1 refusal for a label or mention dispatch", async () => { + // Regression guard: the suppression must be scoped to auto-triggers only. + mockLoadRepoPolicy.mockResolvedValue(policyFrom({ workflows: { review: { enabled: false } } })); + + const result = await dispatchWorkflowByName( + autoParams({ triggerCommentId: 1, triggerEventType: "issue_comment" }), + ); + + expect(result.status).toBe("refused"); + if (result.status === "refused") expect(result.explained).toBe(true); + expect(mockPostRefusalComment).toHaveBeenCalledTimes(1); + }); + + it("reports explained=false on a silently-dropped in-flight collision", async () => { + mockInsertQueued.mockRejectedValueOnce(inflightError()); + + const result = await dispatchWorkflowByName(autoParams({ auto: true })); + + if (result.status === "refused") expect(result.explained).toBe(false); + }); }); diff --git a/test/workflows/handlers/implement.test.ts b/test/workflows/handlers/implement.test.ts index 0b18d3e9..a3279dcc 100644 --- a/test/workflows/handlers/implement.test.ts +++ b/test/workflows/handlers/implement.test.ts @@ -14,6 +14,7 @@ import type { Octokit } from "octokit"; import type pino from "pino"; import type { WorkflowRunContext } from "../../../src/workflows/registry"; +import { expectToReject } from "../../utils/assertions"; const mockConfig: { githubPersonalAccessToken: string | undefined } = { githubPersonalAccessToken: undefined, @@ -27,19 +28,22 @@ let pipelineResult: { numTurns?: number; durationMs?: number; capturedFiles?: Record; + daemonActions?: { + learnings: { category: "setup"; content: string }[]; + deletions: string[]; + }; }; +const mockRunPipeline = mock(() => Promise.resolve(pipelineResult)); + void mock.module("../../../src/core/pipeline", () => ({ - runPipeline: mock(() => Promise.resolve(pipelineResult)), + runPipeline: mockRunPipeline, })); +class StaleWorkflowAttemptError extends Error {} + void mock.module("../../../src/workflows/runs-store", () => ({ - findLatestSucceededForTarget: mock(() => - Promise.resolve({ - id: "plan-row-1", - state: { plan: "## Plan\n\nDo the thing." }, - }), - ), + StaleWorkflowAttemptError, })); // Stub the discussion-digest module so this test (which mocks `config` down @@ -115,7 +119,7 @@ function buildCtx( }, } as unknown as Octokit; - const setStateMock = mock(() => Promise.resolve()); + const setStateMock = mock(() => Promise.resolve({ trackingCommentId: 12345 })); return { runId: "run-1", @@ -125,6 +129,7 @@ function buildCtx( octokit, deliveryId: "delivery-1", daemonId: "daemon-1", + priorPlanState: { plan: "## Plan\n\nDo the thing." }, setState: setStateMock, setStateMock, } as unknown as WorkflowRunContext & { setStateMock: ReturnType }; @@ -146,6 +151,16 @@ describe("implement handler: findRecentOpenedPr", () => { mockConfig.githubPersonalAccessToken = undefined; }); + it("stops before pipeline execution when the starting-comment attempt fence is stale", async () => { + const ctx = buildCtx([]); + ctx.setStateMock.mockImplementation(() => + Promise.reject(new StaleWorkflowAttemptError("workflow attempt is no longer current")), + ); + + await expectToReject(implementHandler(ctx), "workflow attempt is no longer current"); + expect(mockRunPipeline).not.toHaveBeenCalled(); + }); + it("App mode: accepts a PR authored by the App bot", async () => { const ctx = buildCtx([{ number: 107, type: "Bot", login: "chrisleekr-bot[bot]" }]); const result = await implementHandler(ctx); @@ -158,13 +173,44 @@ describe("implement handler: findRecentOpenedPr", () => { } }); + it("passes repo memory into the pipeline and forwards terminal daemon actions", async () => { + const ctx = buildCtx([{ number: 107, type: "Bot", login: "chrisleekr-bot[bot]" }]); + const repoMemory = [ + { + id: "11111111-1111-4111-8111-111111111111", + category: "setup" as const, + content: "Run isolated tests.", + pinned: false, + }, + ]; + (ctx as { repoMemory?: typeof repoMemory }).repoMemory = repoMemory; + pipelineResult.daemonActions = { + learnings: [{ category: "setup", content: "Use Bun." }], + deletions: [], + }; + + const result = await implementHandler(ctx); + + expect((mockRunPipeline.mock.calls.at(-1)?.[0] as BotContext).repoMemory).toEqual(repoMemory); + expect("daemonActions" in result ? result.daemonActions : undefined).toEqual( + pipelineResult.daemonActions, + ); + }); + it("App mode: rejects a PR authored by a User account", async () => { + pipelineResult.daemonActions = { + learnings: [{ category: "setup", content: "Use the App-authored PR." }], + deletions: [], + }; const ctx = buildCtx([{ number: 107, type: "User", login: "chrisleekr" }]); const result = await implementHandler(ctx); expect(result.status).toBe("failed"); if (result.status === "failed") { expect(result.reason).toBe("implement completed but no PR was found"); } + expect("daemonActions" in result ? result.daemonActions : undefined).toEqual( + pipelineResult.daemonActions, + ); }); it("PAT mode: accepts a PR authored by the PAT owner (regression: User type)", async () => { @@ -226,3 +272,65 @@ describe("implement handler: findRecentOpenedPr", () => { } }); }); + +// ─── Per-repo agent policy, `.github-app.yaml` Gate 2 ─────────────────────── + +describe("implement handler: per-repo policy forwarding", () => { + beforeEach(() => { + mockRunPipeline.mockClear(); + pipelineResult = { success: true, capturedFiles: { "IMPLEMENT.md": "## Summary\n\nDone." } }; + }); + + it("forwards the run-context policy into the pipeline overrides", async () => { + const policy = { + model: "claude-repo-pinned-model", + timeoutMs: 900_000, + extraAllowedTools: ["WebFetch"], + }; + const ctx = buildCtx([]); + (ctx as unknown as Record)["policy"] = policy; + + await implementHandler(ctx); + + expect(mockRunPipeline).toHaveBeenCalledTimes(1); + const overrides = mockRunPipeline.mock.calls[0]?.[1] as + | { policy?: Record } + | undefined; + expect(overrides?.policy).toEqual(policy); + }); + + it("forwards the run-context maxTurns into the pipeline overrides", async () => { + // The turn cap rides the top-level payload field, not `policy`, so it + // needs its own hop. Without it `workflows.implement.max_turns` is inert. + const ctx = buildCtx([]); + (ctx as unknown as Record)["maxTurns"] = 12; + + await implementHandler(ctx); + + const overrides = mockRunPipeline.mock.calls[0]?.[1] as { maxTurns?: number } | undefined; + expect(overrides?.maxTurns).toBe(12); + }); + + it("forwards the daemon attempt signal into the pipeline overrides", async () => { + const ctx = buildCtx([]); + const signal = new AbortController().signal; + (ctx as unknown as Record)["signal"] = signal; + + await implementHandler(ctx); + + const overrides = mockRunPipeline.mock.calls[0]?.[1] as { signal?: AbortSignal } | undefined; + expect(overrides?.signal).toBe(signal); + }); + + it("passes no policy or maxTurns key when the run context carries neither (C8)", async () => { + await implementHandler(buildCtx([])); + + const overrides = mockRunPipeline.mock.calls[0]?.[1] as Record | undefined; + expect(overrides).toBeDefined(); + // `exactOptionalPropertyTypes`: absent, not `undefined`-valued. + expect(Object.hasOwn(overrides ?? {}, "policy")).toBe(false); + expect(Object.hasOwn(overrides ?? {}, "maxTurns")).toBe(false); + // Existing overrides must survive the addition. + expect(overrides?.["captureFiles"]).toEqual(["IMPLEMENT.md"]); + }); +}); diff --git a/test/workflows/handlers/plan.test.ts b/test/workflows/handlers/plan.test.ts index 534aeefb..a25804fd 100644 --- a/test/workflows/handlers/plan.test.ts +++ b/test/workflows/handlers/plan.test.ts @@ -13,12 +13,15 @@ import type pino from "pino"; import { config } from "../../../src/config"; import type { WorkflowRunContext } from "../../../src/workflows/registry"; +import { StaleWorkflowAttemptError } from "../../../src/workflows/runs-store"; +import { expectToReject } from "../../utils/assertions"; let agentResult: { success: boolean; costUsd?: number; numTurns?: number; durationMs?: number; + errorMessage?: string; }; let planMd: string; @@ -31,9 +34,18 @@ void mock.module("../../../src/core/checkout", () => ({ ), })); +/** The subset of `ExecuteAgentParams` these tests read back. */ +interface ExecuteAgentArgs { + allowedTools: string[]; + promptParts?: { append: string; userMessage: string }; + model?: string; + maxTurns?: number; + signal?: AbortSignal; +} + // Hoisted so tests can inspect the params the handler forwards (e.g. whether // `promptParts` is threaded through under PROMPT_CACHE_LAYOUT=cacheable). -const executeAgentMock = mock(async () => Promise.resolve(agentResult)); +const executeAgentMock = mock(async (_params: ExecuteAgentArgs) => Promise.resolve(agentResult)); void mock.module("../../../src/core/executor", () => ({ executeAgent: executeAgentMock, })); @@ -106,6 +118,18 @@ afterEach(() => { }); describe("plan handler (SDK-driven)", () => { + it("stops before agent execution when the starting-comment attempt fence is stale", async () => { + const ctx = buildCtx(); + ctx.setState = mock(() => + Promise.reject( + new StaleWorkflowAttemptError({ runId: ctx.runId, attemptId: crypto.randomUUID() }), + ), + ); + + await expectToReject(planHandler(ctx), "workflow attempt is no longer current"); + expect(executeAgentMock).not.toHaveBeenCalled(); + }); + it("returns succeeded with the PLAN.md body as state", async () => { const result = await planHandler(buildCtx()); @@ -129,6 +153,25 @@ describe("plan handler (SDK-driven)", () => { } }); + it("propagates the executor's errorMessage into `reason`, not into the public comment", async () => { + agentResult = { + success: false, + durationMs: 5_000, + errorMessage: + "Claude Code returned an error result: You've hit your limit · resets 6pm (UTC)", + }; + + const result = await planHandler(buildCtx()); + + expect(result.status).toBe("failed"); + if (result.status === "failed") { + // Load-bearing: orchestrator.ts:483 matches /hit your limit/i to defer and retry. + expect(result.reason).toContain("hit your limit"); + expect(result.humanMessage).toContain("see server logs"); + expect(result.humanMessage).not.toContain("hit your limit"); + } + }); + it("fails when PLAN.md is missing", async () => { planMd = ""; @@ -162,3 +205,142 @@ describe("plan handler (SDK-driven)", () => { expect(params.promptParts?.userMessage.length).toBeGreaterThan(0); }); }); + +// ─── Per-repo agent policy, `.github-app.yaml` Gate 2 ─────────────────────── + +/** The tool list the handler owns; per-repo extras may only widen it. */ +const PLAN_BASE_TOOLS = ["Read", "Grep", "Glob", "Write", "Bash"]; + +/** `WorkflowRunContext` fields are readonly, mirrors implement.test.ts. */ +function withPolicy(ctx: WorkflowRunContext, policy: Record): WorkflowRunContext { + (ctx as unknown as Record)["policy"] = policy; + return ctx; +} + +function withMaxTurns(ctx: WorkflowRunContext, maxTurns: number): WorkflowRunContext { + (ctx as unknown as Record)["maxTurns"] = maxTurns; + return ctx; +} + +function withSignal(ctx: WorkflowRunContext, signal: AbortSignal): WorkflowRunContext { + (ctx as unknown as Record)["signal"] = signal; + return ctx; +} + +/** Resolves when `signal` aborts; capped so a dead deadline fails rather than hangs. */ +async function waitForAbort(signal: AbortSignal, capMs: number): Promise { + return new Promise((resolve) => { + const cap = setTimeout(resolve, capMs); + const onAbort = (): void => { + clearTimeout(cap); + resolve(); + }; + signal.addEventListener("abort", onAbort, { once: true }); + }); +} + +function firstExecuteAgentArgs(): Record { + const params = executeAgentMock.mock.calls[0]?.[0]; + expect(params).toBeDefined(); + return params as unknown as Record; +} + +describe("plan handler: per-repo policy forwarding", () => { + it("forwards the policy model to executeAgent (C1)", async () => { + await planHandler(withPolicy(buildCtx(), { model: "claude-repo-pinned" })); + + expect(executeAgentMock).toHaveBeenCalledTimes(1); + expect(firstExecuteAgentArgs()["model"]).toBe("claude-repo-pinned"); + }); + + it("forwards the run-context maxTurns to executeAgent (C2)", async () => { + await planHandler(withMaxTurns(buildCtx(), 7)); + + expect(firstExecuteAgentArgs()["maxTurns"]).toBe(7); + }); + + it("forwards the daemon attempt signal to executeAgent", async () => { + const signal = new AbortController().signal; + + await planHandler(withSignal(buildCtx(), signal)); + + expect(firstExecuteAgentArgs()["signal"]).toBe(signal); + }); + + it("stops after an outer fence when a repository deadline is also configured", async () => { + const outer = new AbortController(); + const ctx = withSignal(withPolicy(buildCtx(), { timeoutMs: 60_000 }), outer.signal); + executeAgentMock.mockImplementationOnce(() => { + outer.abort(new Error("workflow lease fenced")); + return Promise.resolve(agentResult); + }); + + const result = await planHandler(ctx); + + expect(result.status).toBe("failed"); + expect(firstExecuteAgentArgs()["signal"]).not.toBe(outer.signal); + expect((firstExecuteAgentArgs()["signal"] as AbortSignal).aborted).toBe(true); + expect(ctx.setState).toHaveBeenCalledTimes(1); + }); + + it("forwards a per-repo deadline as a named Error, not a bare TimeoutError (C3)", async () => { + // A bare TimeoutError DOMException would fail executeAgent's identity check. + const seen: { hasSignal?: boolean; abortedAtEntry?: boolean; reason?: unknown } = {}; + executeAgentMock.mockImplementationOnce(async (params: ExecuteAgentArgs) => { + const { signal } = params; + seen.hasSignal = signal !== undefined; + seen.abortedAtEntry = signal?.aborted; + if (signal !== undefined) { + await waitForAbort(signal, 3_000); + seen.reason = signal.reason; + } + return agentResult; + }); + + await planHandler(withPolicy(buildCtx(), { timeoutMs: 25 })); + + expect(seen.hasSignal).toBe(true); + expect(seen.abortedAtEntry).toBe(false); + expect(seen.reason).toBeInstanceOf(Error); + const reason = seen.reason as Error; + expect(reason.name).not.toBe("TimeoutError"); + expect(reason.message).toContain(config.repoConfigFile); + expect(reason.message).toMatch(/timeout/i); + }); + + it("unions extraAllowedTools onto the handler's base list (C4)", async () => { + // "Read" duplicates a base tool: extras are deduped, never a replacement. + await planHandler(withPolicy(buildCtx(), { extraAllowedTools: ["WebFetch", "Read"] })); + + const tools = firstExecuteAgentArgs()["allowedTools"] as string[]; + for (const tool of PLAN_BASE_TOOLS) expect(tools).toContain(tool); + expect(tools).toContain("WebFetch"); + expect(new Set(tools).size).toBe(tools.length); + expect(tools).toHaveLength(PLAN_BASE_TOOLS.length + 1); + }); + + it("disposes the deadline timer when the run finishes early (no leaked timer)", async () => { + // C3 waits for the deadline, so its timer is spent; only a normal finish + // catches a missing `disposePolicy?.()`. + await planHandler(withPolicy(buildCtx(), { timeoutMs: 25 })); + + const signal = firstExecuteAgentArgs()["signal"] as AbortSignal; + expect(signal.aborted).toBe(false); + await new Promise((resolve) => { + setTimeout(resolve, 200); + }); + + expect(signal.aborted).toBe(false); + }); + + it("passes no model, maxTurns, or signal key when the run context carries none of them (C9)", async () => { + await planHandler(buildCtx()); + + const params = firstExecuteAgentArgs(); + // `exactOptionalPropertyTypes`: absent, not `undefined`-valued. + expect(Object.hasOwn(params, "model")).toBe(false); + expect(Object.hasOwn(params, "maxTurns")).toBe(false); + expect(Object.hasOwn(params, "signal")).toBe(false); + expect(params["allowedTools"]).toEqual(PLAN_BASE_TOOLS); + }); +}); diff --git a/test/workflows/handlers/remember.test.ts b/test/workflows/handlers/remember.test.ts new file mode 100644 index 00000000..c09f2427 --- /dev/null +++ b/test/workflows/handlers/remember.test.ts @@ -0,0 +1,135 @@ +/** + * Wiring-only tests for the `remember` handler's Gate-2 hop. + * + * The handler's agent behaviour (directive extraction, dedup, save) is the + * agent's job and is not unit-testable here. What IS testable, and what has + * no other guard, is the two-line forward of `ctx.policy` / `ctx.maxTurns` + * into `runPipeline`: delete the spread and nothing else fails. + */ + +import { beforeEach, describe, expect, it, mock } from "bun:test"; +import type { Octokit } from "octokit"; +import type pino from "pino"; + +import type { BotContext } from "../../../src/types"; +import type { WorkflowRunContext } from "../../../src/workflows/registry"; + +const mockRunPipeline = mock(() => Promise.resolve({ success: true })); + +void mock.module("../../../src/core/pipeline", () => ({ + runPipeline: mockRunPipeline, +})); + +// The digest path has its own suite; stub it so this file does not pull the +// real module's config/logger graph. +void mock.module("../../../src/workflows/discussion-digest", () => ({ + fetchAndBuildDigest: mock(() => Promise.resolve({ ok: false, reason: "no-comments" })), + renderDigestSection: mock(() => ""), +})); + +const { handler: rememberHandler } = await import("../../../src/workflows/handlers/remember"); + +function silentLog(): pino.Logger { + return { + info: mock(() => {}), + warn: mock(() => {}), + error: mock(() => {}), + debug: mock(() => {}), + child: mock(function (this: unknown) { + return this; + }), + } as unknown as pino.Logger; +} + +function buildCtx(): WorkflowRunContext { + const octokit = { + rest: { + issues: { + get: mock(() => Promise.resolve({ data: { title: "Some issue", body: "body" } })), + }, + repos: { + get: mock(() => Promise.resolve({ data: { default_branch: "main" } })), + }, + }, + } as unknown as Octokit; + + return { + runId: "run-1", + workflowName: "remember", + target: { type: "issue", owner: "acme", repo: "widgets", number: 7 }, + logger: silentLog(), + octokit, + deliveryId: "delivery-1", + daemonId: "daemon-1", + setState: mock(() => Promise.resolve({ trackingCommentId: 12345 })), + } as unknown as WorkflowRunContext; +} + +describe("remember handler: per-repo policy forwarding", () => { + beforeEach(() => { + mockRunPipeline.mockClear(); + }); + + it("forwards the run-context policy and maxTurns into the pipeline overrides", async () => { + const policy = { model: "claude-repo-pinned-model", timeoutMs: 900_000 }; + const ctx = buildCtx(); + (ctx as unknown as Record)["policy"] = policy; + (ctx as unknown as Record)["maxTurns"] = 12; + + const result = await rememberHandler(ctx); + + expect(result.status).toBe("succeeded"); + expect(mockRunPipeline).toHaveBeenCalledTimes(1); + const overrides = mockRunPipeline.mock.calls[0]?.[1] as + | { policy?: Record; maxTurns?: number } + | undefined; + expect(overrides?.policy).toEqual(policy); + expect(overrides?.maxTurns).toBe(12); + }); + + it("forwards the daemon attempt signal into the pipeline overrides", async () => { + const ctx = buildCtx(); + const signal = new AbortController().signal; + (ctx as unknown as Record)["signal"] = signal; + + await rememberHandler(ctx); + + const overrides = mockRunPipeline.mock.calls[0]?.[1] as { signal?: AbortSignal } | undefined; + expect(overrides?.signal).toBe(signal); + }); + + it("passes repo memory into the pipeline and forwards terminal daemon actions", async () => { + const ctx = buildCtx(); + const repoMemory = [ + { + id: "11111111-1111-4111-8111-111111111111", + category: "conventions" as const, + content: "Keep policies concise.", + pinned: false, + }, + ]; + (ctx as { repoMemory?: typeof repoMemory }).repoMemory = repoMemory; + const daemonActions = { + learnings: [{ category: "conventions" as const, content: "Document policies." }], + deletions: [], + }; + mockRunPipeline.mockResolvedValueOnce({ success: true, daemonActions }); + + const result = await rememberHandler(ctx); + + expect((mockRunPipeline.mock.calls[0]?.[0] as BotContext).repoMemory).toEqual(repoMemory); + expect("daemonActions" in result ? result.daemonActions : undefined).toEqual(daemonActions); + }); + + it("passes no policy or maxTurns key when the run context carries neither (C8)", async () => { + await rememberHandler(buildCtx()); + + const overrides = mockRunPipeline.mock.calls[0]?.[1] as Record | undefined; + expect(overrides).toBeDefined(); + // `exactOptionalPropertyTypes`: absent, not `undefined`-valued. + expect(Object.hasOwn(overrides ?? {}, "policy")).toBe(false); + expect(Object.hasOwn(overrides ?? {}, "maxTurns")).toBe(false); + // Existing overrides must survive the addition. + expect(overrides?.["enableReviewLearnings"]).toBe(true); + }); +}); diff --git a/test/workflows/handlers/resolve.test.ts b/test/workflows/handlers/resolve.test.ts index b6829fff..675f1cca 100644 --- a/test/workflows/handlers/resolve.test.ts +++ b/test/workflows/handlers/resolve.test.ts @@ -13,6 +13,7 @@ import { afterEach, beforeEach, describe, expect, it, mock } from "bun:test"; import type { Octokit } from "octokit"; import type pino from "pino"; +import type { BotContext } from "../../../src/types"; import type { WorkflowRunContext } from "../../../src/workflows/registry"; interface PipelineResultStub { @@ -22,21 +23,18 @@ interface PipelineResultStub { durationMs?: number; errorMessage?: string; capturedFiles?: Record; + daemonActions?: { + learnings: { category: "gotchas"; content: string }[]; + deletions: string[]; + }; } let pipelineResult: PipelineResultStub; -void mock.module("../../../src/core/pipeline", () => ({ - runPipeline: mock(async () => Promise.resolve(pipelineResult)), -})); +const mockRunPipeline = mock(async () => Promise.resolve(pipelineResult)); -void mock.module("../../../src/workflows/runs-store", () => ({ - findById: mock(async () => - Promise.resolve({ - id: "run-1", - tracking_comment_id: 12345, - }), - ), +void mock.module("../../../src/core/pipeline", () => ({ + runPipeline: mockRunPipeline, })); const { handler: resolveHandler } = await import("../../../src/workflows/handlers/resolve"); @@ -71,7 +69,7 @@ interface BuildCtxOptions { * see a "the agent pushed commits" scenario. */ postHeadSha?: string; - reviewComments?: { in_reply_to_id?: number }[]; + reviewComments?: { in_reply_to_id?: number; user?: { login?: string; type?: string } }[]; targetType?: "pr" | "issue"; } @@ -142,7 +140,7 @@ function buildCtx(opts: BuildCtxOptions = {}): WorkflowRunContext & { }, } as unknown as Octokit; - const setStateMock = mock(async () => Promise.resolve()); + const setStateMock = mock(async () => Promise.resolve({ trackingCommentId: 12345 })); return { runId: "run-1", @@ -212,7 +210,7 @@ describe("resolve handler", () => { expect(post["all_green"]).toBe(true); expect(post["failing_checks"]).toEqual([]); } - expect(ctx.setStateMock).toHaveBeenCalledTimes(2); + expect(ctx.setStateMock).toHaveBeenCalledTimes(1); }); it("returns incomplete when post-pipeline CI still has failing checks", async () => { @@ -224,6 +222,10 @@ describe("resolve handler", () => { "RESOLVE.md": "## Summary\n\nGave up.\n\n## Outstanding\n\n- typecheck still red, could not isolate root cause", }, + daemonActions: { + learnings: [{ category: "gotchas", content: "CI needs a local service." }], + deletions: [], + }, }; const ctx = buildCtx({ preChecks: [{ status: "completed", conclusion: "failure", name: "typecheck" }], @@ -242,6 +244,7 @@ describe("resolve handler", () => { expect(post["all_green"]).toBe(false); expect(post["failing_checks"]).toEqual(["typecheck"]); expect(post["outstanding_present"]).toBe(true); + expect(result.daemonActions).toEqual(pipelineResult.daemonActions); } }); @@ -311,15 +314,148 @@ describe("resolve handler", () => { }); it("returns failed when runPipeline reports failure (regression guard)", async () => { - pipelineResult = { success: false, errorMessage: "agent crashed" }; + pipelineResult = { + success: false, + errorMessage: "agent crashed", + daemonActions: { + learnings: [{ category: "gotchas", content: "The agent needs a service." }], + deletions: [], + }, + }; const ctx = buildCtx({ preChecks: [{ status: "completed", conclusion: "failure", name: "test" }], }); + const repoMemory = [ + { + id: "11111111-1111-4111-8111-111111111111", + category: "gotchas" as const, + content: "Start the local service.", + pinned: false, + }, + ]; + (ctx as { repoMemory?: typeof repoMemory }).repoMemory = repoMemory; const result = await resolveHandler(ctx); expect(result.status).toBe("failed"); if (result.status === "failed") { expect(result.reason).toContain("agent crashed"); expect(result.humanMessage).toContain("see server logs"); } + expect((mockRunPipeline.mock.calls.at(-1)?.[0] as BotContext).repoMemory).toEqual(repoMemory); + expect("daemonActions" in result ? result.daemonActions : undefined).toEqual( + pipelineResult.daemonActions, + ); + }); +}); + +// ─── Per-repo agent policy, `.github-app.yaml` Gate 2 ─────────────────────── + +describe("resolve handler: per-repo policy forwarding", () => { + beforeEach(() => { + mockRunPipeline.mockClear(); + pipelineResult = { + success: true, + capturedFiles: { "RESOLVE.md": "## Summary\n\nDone." }, + }; + }); + + it("forwards the run-context policy into the pipeline overrides", async () => { + const policy = { + model: "claude-repo-pinned-model", + timeoutMs: 900_000, + extraAllowedTools: ["WebFetch"], + pathFilters: ["**/__snapshots__/**"], + instructions: "reject migrations without a rollback", + }; + const ctx = buildCtx(); + (ctx as unknown as Record)["policy"] = policy; + + await resolveHandler(ctx); + + expect(mockRunPipeline).toHaveBeenCalledTimes(1); + const overrides = mockRunPipeline.mock.calls[0]?.[1] as + | { policy?: Record } + | undefined; + expect(overrides?.policy).toEqual(policy); + }); + + it("forwards the run-context maxTurns into the pipeline overrides", async () => { + // The turn cap rides the top-level payload field, not `policy`, so it + // needs its own hop. Without it `workflows.resolve.max_turns` is inert. + const ctx = buildCtx(); + (ctx as unknown as Record)["maxTurns"] = 12; + + await resolveHandler(ctx); + + const overrides = mockRunPipeline.mock.calls[0]?.[1] as { maxTurns?: number } | undefined; + expect(overrides?.maxTurns).toBe(12); + }); + + it("forwards the daemon attempt signal into the pipeline overrides", async () => { + const ctx = buildCtx(); + const signal = new AbortController().signal; + (ctx as unknown as Record)["signal"] = signal; + + await resolveHandler(ctx); + + const overrides = mockRunPipeline.mock.calls[0]?.[1] as { signal?: AbortSignal } | undefined; + expect(overrides?.signal).toBe(signal); + }); + + it("passes no policy or maxTurns key when the run context carries neither (C8)", async () => { + await resolveHandler(buildCtx()); + + const overrides = mockRunPipeline.mock.calls[0]?.[1] as Record | undefined; + expect(overrides).toBeDefined(); + // `exactOptionalPropertyTypes`: absent, not `undefined`-valued. + expect(Object.hasOwn(overrides ?? {}, "policy")).toBe(false); + expect(Object.hasOwn(overrides ?? {}, "maxTurns")).toBe(false); + // Existing overrides must survive the addition. + expect(overrides?.["captureFiles"]).toEqual(["RESOLVE.md"]); + expect(overrides?.["enableReviewLearnings"]).toBe(true); + }); +}); + +// ─── review comments are NOT filtered by author (work item #1) ──────────── + +describe("resolve acts on bot-authored review comments", () => { + beforeEach(() => { + mockRunPipeline.mockClear(); + pipelineResult = { + success: true, + capturedFiles: { "RESOLVE.md": "## Summary\n\nDone." }, + }; + }); + + // `enableResolveReviewThread` is set iff at least one top-level review comment + // survives, so it is the observable signal for what `resolve` saw. + function sawOpenThreads(): boolean { + const overrides = mockRunPipeline.mock.calls[0]?.[1] as Record | undefined; + return overrides?.["enableResolveReviewThread"] === true; + } + + it("acts on our own review findings, which is ship's review -> resolve handoff", async () => { + // ship runs review immediately before resolve on the same PR. Filtering our + // own findings out here would make resolve a CI-only fixer and silently drop + // the review it just ran. + await resolveHandler( + buildCtx({ reviewComments: [{ user: { login: "chrisleekr-bot[bot]", type: "Bot" } }] }), + ); + expect(sawOpenThreads()).toBe(true); + }); + + it("acts on a third-party review bot's findings", async () => { + // CodeRabbit / Copilot / Sonar are all `type: "Bot"`. A type-based author + // filter would discard exactly the reviewer feedback resolve exists for. + await resolveHandler( + buildCtx({ reviewComments: [{ user: { login: "coderabbitai[bot]", type: "Bot" } }] }), + ); + expect(sawOpenThreads()).toBe(true); + }); + + it("acts on a human's review comment", async () => { + await resolveHandler( + buildCtx({ reviewComments: [{ user: { login: "someone", type: "User" } }] }), + ); + expect(sawOpenThreads()).toBe(true); }); }); diff --git a/test/workflows/handlers/review.test.ts b/test/workflows/handlers/review.test.ts index a7fc6f75..554896cf 100644 --- a/test/workflows/handlers/review.test.ts +++ b/test/workflows/handlers/review.test.ts @@ -15,6 +15,7 @@ import { afterEach, beforeEach, describe, expect, it, mock } from "bun:test"; import type { Octokit } from "octokit"; import type pino from "pino"; +import type { BotContext } from "../../../src/types"; import type { WorkflowRunContext } from "../../../src/workflows/registry"; let pipelineResult: { @@ -23,22 +24,16 @@ let pipelineResult: { numTurns?: number; durationMs?: number; capturedFiles?: Record; + daemonActions?: { + learnings: { category: "conventions"; content: string }[]; + deletions: string[]; + }; }; -void mock.module("../../../src/core/pipeline", () => ({ - runPipeline: mock(async () => Promise.resolve(pipelineResult)), -})); +const mockRunPipeline = mock(async () => Promise.resolve(pipelineResult)); -// Stub the runs-store DB read, the handler reads back the seeded -// tracking_comment_id after the first setState call. Tests don't run -// against a real DB, so return a row with a deterministic id. -void mock.module("../../../src/workflows/runs-store", () => ({ - findById: mock(async () => - Promise.resolve({ - id: "run-1", - tracking_comment_id: 12345, - }), - ), +void mock.module("../../../src/core/pipeline", () => ({ + runPipeline: mockRunPipeline, })); const { handler: reviewHandler } = await import("../../../src/workflows/handlers/review"); @@ -112,7 +107,7 @@ function buildCtx( }, } as unknown as Octokit; - const setStateMock = mock(async () => Promise.resolve()); + const setStateMock = mock(async () => Promise.resolve({ trackingCommentId: 12345 })); return { runId: "run-1", @@ -188,16 +183,16 @@ describe("review handler", () => { expect(branch["commits_behind_base"]).toBe(0); expect(branch["is_fork"]).toBe(false); } - // Two setState calls: (1) seed before pipeline, (2) finalize after. - expect(ctx.setStateMock).toHaveBeenCalledTimes(2); + expect(ctx.setStateMock).toHaveBeenCalledTimes(1); const seedArgs = ctx.setStateMock.mock.calls[0] as [unknown, string]; expect(seedArgs[1]).toContain("Code review starting"); expect(seedArgs[1]).toContain("5 files"); - const finalArgs = ctx.setStateMock.mock.calls[1] as [unknown, string]; - expect(finalArgs[1]).toContain("Code review complete"); - expect(finalArgs[1]).toContain("5 files"); - expect(finalArgs[1]).toContain("+100/-20"); - expect(finalArgs[1]).toContain("Reviewed 3 files"); + if (result.status === "succeeded") { + expect(result.humanMessage).toContain("Code review complete"); + expect(result.humanMessage).toContain("5 files"); + expect(result.humanMessage).toContain("+100/-20"); + expect(result.humanMessage).toContain("Reviewed 3 files"); + } }); it("records commits_behind_base when the branch is stale", async () => { @@ -233,19 +228,39 @@ describe("review handler", () => { const ctx = buildCtx(); const result = await reviewHandler(ctx); expect(result.status).toBe("succeeded"); - // calls[0] is the seed; the placeholder appears in the finalize call. - const finalArgs = ctx.setStateMock.mock.calls[1] as [unknown, string]; - expect(finalArgs[1]).toContain("no REVIEW.md report"); + if (result.status === "succeeded") { + expect(result.humanMessage).toContain("no REVIEW.md report"); + } + expect(ctx.setStateMock).toHaveBeenCalledTimes(1); }); it("fails when the pipeline reports failure", async () => { - pipelineResult = { success: false }; + pipelineResult = { + success: false, + daemonActions: { + learnings: [{ category: "conventions", content: "Keep review findings focused." }], + deletions: [], + }, + }; const ctx = buildCtx(); + const repoMemory = [ + { + id: "11111111-1111-4111-8111-111111111111", + category: "conventions" as const, + content: "Review full files.", + pinned: true, + }, + ]; + (ctx as { repoMemory?: typeof repoMemory }).repoMemory = repoMemory; const result = await reviewHandler(ctx); expect(result.status).toBe("failed"); if (result.status === "failed") { expect(result.reason).toContain("pipeline"); } + expect((mockRunPipeline.mock.calls.at(-1)?.[0] as BotContext).repoMemory).toEqual(repoMemory); + expect("daemonActions" in result ? result.daemonActions : undefined).toEqual( + pipelineResult.daemonActions, + ); }); }); @@ -282,3 +297,75 @@ Found 4 issues. }); }); }); + +// ─── Per-repo agent policy, `.github-app.yaml` Gate 2 ─────────────────────── + +describe("review handler: per-repo policy forwarding", () => { + beforeEach(() => { + mockRunPipeline.mockClear(); + pipelineResult = { success: true, capturedFiles: { "REVIEW.md": "## Summary\n\nOK" } }; + }); + + it("forwards the run-context policy into the pipeline overrides", async () => { + const policy = { + model: "claude-repo-pinned-model", + timeoutMs: 900_000, + extraAllowedTools: ["WebFetch"], + pathFilters: ["**/__snapshots__/**"], + instructions: "reject migrations without a rollback", + }; + const ctx = buildCtx(); + (ctx as unknown as Record)["policy"] = policy; + + await reviewHandler(ctx); + + expect(mockRunPipeline).toHaveBeenCalledTimes(1); + const overrides = mockRunPipeline.mock.calls[0]?.[1] as + | { policy?: Record } + | undefined; + expect(overrides?.policy).toEqual(policy); + }); + + it("forwards the run-context maxTurns into the pipeline overrides", async () => { + // The turn cap rides the top-level payload field, not `policy`, so it + // needs its own hop. Without it `workflows..max_turns` is inert and + // executeAgent silently falls back to AGENT_MAX_TURNS. + const ctx = buildCtx(); + (ctx as unknown as Record)["maxTurns"] = 12; + + await reviewHandler(ctx); + + const overrides = mockRunPipeline.mock.calls[0]?.[1] as { maxTurns?: number } | undefined; + expect(overrides?.maxTurns).toBe(12); + }); + + it("forwards the daemon attempt signal into the pipeline overrides", async () => { + const ctx = buildCtx(); + const signal = new AbortController().signal; + (ctx as unknown as Record)["signal"] = signal; + + await reviewHandler(ctx); + + const overrides = mockRunPipeline.mock.calls[0]?.[1] as { signal?: AbortSignal } | undefined; + expect(overrides?.signal).toBe(signal); + }); + + it("passes no maxTurns key when the run context carries none", async () => { + await reviewHandler(buildCtx()); + + const overrides = mockRunPipeline.mock.calls[0]?.[1] as Record | undefined; + expect(Object.hasOwn(overrides ?? {}, "maxTurns")).toBe(false); + }); + + it("passes no policy key when the run context carries none (C8)", async () => { + await reviewHandler(buildCtx()); + + const overrides = mockRunPipeline.mock.calls[0]?.[1] as Record | undefined; + expect(overrides).toBeDefined(); + // `exactOptionalPropertyTypes`: absent, not `undefined`-valued. + expect(Object.hasOwn(overrides ?? {}, "policy")).toBe(false); + // Existing overrides must survive the addition. + expect(overrides?.["captureFiles"]).toEqual(["REVIEW.md"]); + expect(overrides?.["enableReviewLearnings"]).toBe(true); + }); +}); diff --git a/test/workflows/handlers/ship.test.ts b/test/workflows/handlers/ship.test.ts index 2cc1b3be..bf6796e1 100644 --- a/test/workflows/handlers/ship.test.ts +++ b/test/workflows/handlers/ship.test.ts @@ -1,78 +1,10 @@ -/** - * Integration tests for the composite `ship` handler (T027, T028). - * - * The handler's core job is deciding `startIndex`: which step of - * `triage → plan → implement → review → resolve` to enqueue first when a `bot:ship` - * parent is launched. This is driven by the staleness rules in - * `contracts/handoff-protocol.md` §Skip-if-output-exists. - * - * T027: Resume after failure: parent failed at step 2 → on re-apply, - * the new parent enqueues at index 2 and carries prior step run - * ids through `state.stepRuns`. - * T028: Open-PR shortcut (FR-020): prior successful `implement` run - * whose recorded PR is still open → skip straight to step 3 - * (`review`). The downstream `resolve` step (index 4) is - * enqueued by the orchestrator after `review` completes, - * covered separately in orchestrator.test.ts. - * - * Downstream writes (`enqueueJob`, `tracking-mirror`) are mocked; the - * staleness check and the `stepRuns` bookkeeping are exercised against a - * real Postgres database to catch any regression in the SQL round-trips. - */ +/** Unit tests for bounded ship-resume state at the isolated runner boundary. */ -import { SQL } from "bun"; -import { afterAll, beforeAll, beforeEach, describe, expect, it, mock } from "bun:test"; +import { describe, expect, it, mock } from "bun:test"; import type { Octokit } from "octokit"; import type pino from "pino"; -const TEST_DATABASE_URL = - process.env["TEST_DATABASE_URL"] ?? "postgres://bot:bot@localhost:55432/github_app_test"; - -let sql: SQL | null = null; -try { - const conn = new SQL(TEST_DATABASE_URL); - await conn`SELECT 1 AS ok`; - sql = conn; -} catch { - sql = null; -} - -function requireSql(): SQL { - if (sql === null) throw new Error("Database not available, test should have been skipped"); - return sql; -} - -// ─── Mocks ─────────────────────────────────────────────────────────────── - -const mockEnqueueJob = mock(() => Promise.resolve()); -void mock.module("../../../src/orchestrator/job-queue", () => ({ - enqueueJob: mockEnqueueJob, - isScopedJob: () => false, - SCOPED_JOB_KINDS: ["scoped-rebase", "scoped-fix-thread", "scoped-open-pr"], -})); - -void mock.module("../../../src/workflows/execution-row", () => ({ - recordWorkflowExecution: mock(() => Promise.resolve()), - buildWorkflowContextJson: mock(() => ({})), -})); - -void mock.module("../../../src/orchestrator/concurrency", () => ({ - incrementActiveCount: mock(() => {}), - decrementActiveCount: mock(() => {}), -})); - -void mock.module("../../../src/workflows/tracking-mirror", () => ({ - setState: mock(() => Promise.resolve()), - postRefusalComment: mock(() => Promise.resolve()), -})); - -void mock.module("../../../src/db", () => ({ - requireDb: () => requireSql(), - getDb: () => requireSql(), - closeDb: () => Promise.resolve(), -})); - -// ─── Helpers ───────────────────────────────────────────────────────────── +import type { WorkflowRunSnapshot } from "../../../src/shared/workflow-types"; function silentLogger(): pino.Logger { return { @@ -91,9 +23,7 @@ function buildOctokit(prState: "open" | "closed" | "throw"): Octokit { rest: { pulls: { get: mock(() => { - if (prState === "throw") { - return Promise.reject(new Error("404 not found")); - } + if (prState === "throw") return Promise.reject(new Error("404 not found")); return Promise.resolve({ data: { state: prState } }); }), }, @@ -101,250 +31,178 @@ function buildOctokit(prState: "open" | "closed" | "throw"): Octokit { } as unknown as Octokit; } -// Warm the registry module graph BEFORE touching ship.ts. There is a -// circular-import trap: `src/workflows/handlers/ship.ts` imports `getByName` -// from `../registry`, and registry.ts eagerly references the ship handler -// in its top-level `rawRegistry`. If the test imports ship.ts first, the -// circular chain re-enters registry while ship.ts's `export const handler` -// binding is still in TDZ, and `RegistrySchema.parse(rawRegistry)` explodes -// with `Cannot access 'shipHandler' before initialization`. Importing -// registry first lets the chain resolve via the production path. -await import("../../../src/workflows/registry"); -const { handler: shipHandler } = await import("../../../src/workflows/handlers/ship"); - -async function seedSucceededRun(params: { - workflowName: "triage" | "plan" | "implement"; - target: { owner: string; repo: string; number: number }; - state: Record; -}): Promise<{ runId: string }> { - const { insertQueued, markSucceeded } = await import("../../../src/workflows/runs-store"); - const row = await insertQueued( - { - workflowName: params.workflowName, - target: { type: "issue" as const, ...params.target }, - initialState: {}, - ownerKind: "orchestrator", - ownerId: "test-orchestrator", - }, - requireSql(), - ); - await markSucceeded(row.id, params.state, requireSql()); - // Small tick so subsequent rows get a strictly-later created_at. - await new Promise((resolve) => setTimeout(resolve, 5)); - return { runId: row.id }; +function snapshot(input: { + readonly status: WorkflowRunSnapshot["status"]; + readonly state?: WorkflowRunSnapshot["state"]; + readonly createdAt: string; +}): WorkflowRunSnapshot { + return { + id: crypto.randomUUID(), + status: input.status, + state: input.state ?? {}, + createdAt: input.createdAt, + }; } -describe.skipIf(sql === null)("ship handler", () => { - beforeAll(async () => { - await requireSql().unsafe(` - DROP TABLE IF EXISTS _migrations CASCADE; - DROP TABLE IF EXISTS review_learnings CASCADE; - DROP TABLE IF EXISTS scheduled_action_state CASCADE; - DROP TABLE IF EXISTS comment_cache CASCADE; - DROP TABLE IF EXISTS target_cache CASCADE; - DROP TABLE IF EXISTS chat_proposals CASCADE; - DROP TABLE IF EXISTS ship_fix_attempts CASCADE; - DROP TABLE IF EXISTS ship_continuations CASCADE; - DROP TABLE IF EXISTS ship_iterations CASCADE; - DROP TABLE IF EXISTS ship_intents CASCADE; - DROP TABLE IF EXISTS workflow_runs CASCADE; - DROP TABLE IF EXISTS repo_memory CASCADE; - DROP TABLE IF EXISTS triage_results CASCADE; - DROP TABLE IF EXISTS executions CASCADE; - DROP TABLE IF EXISTS daemons CASCADE; - `); - const { runMigrations } = await import("../../../src/db/migrate"); - await runMigrations(requireSql()); - }); - - afterAll(async () => { - await requireSql().unsafe(` - DROP TABLE IF EXISTS _migrations CASCADE; - DROP TABLE IF EXISTS review_learnings CASCADE; - DROP TABLE IF EXISTS scheduled_action_state CASCADE; - DROP TABLE IF EXISTS comment_cache CASCADE; - DROP TABLE IF EXISTS target_cache CASCADE; - DROP TABLE IF EXISTS chat_proposals CASCADE; - DROP TABLE IF EXISTS ship_fix_attempts CASCADE; - DROP TABLE IF EXISTS ship_continuations CASCADE; - DROP TABLE IF EXISTS ship_iterations CASCADE; - DROP TABLE IF EXISTS ship_intents CASCADE; - DROP TABLE IF EXISTS workflow_runs CASCADE; - DROP TABLE IF EXISTS repo_memory CASCADE; - DROP TABLE IF EXISTS triage_results CASCADE; - DROP TABLE IF EXISTS executions CASCADE; - DROP TABLE IF EXISTS daemons CASCADE; - `); - await requireSql().close(); - }); - - beforeEach(() => { - mockEnqueueJob.mockClear(); - }); - - it("T027 resume: prior triage + plan succeeded, implement previously failed → startIndex=2 with prior step runs carried forward", async () => { - const { insertQueued, markFailed } = await import("../../../src/workflows/runs-store"); - - const targetOwner = "acme"; - const targetRepo = "repo"; - const targetNumber = 301; +// Warm the registry before importing ship.ts. The registry eagerly references +// the ship handler, so importing the handler first would re-enter its TDZ. +await import("../../../src/workflows/registry"); +const { handler: shipHandler } = await import("../../../src/workflows/handlers/ship"); - // Seed prior succeeded triage + plan. - const { runId: triageRunId } = await seedSucceededRun({ - workflowName: "triage", - target: { owner: targetOwner, repo: targetRepo, number: targetNumber }, - state: { verdict: "bug", recommendedNext: "plan" }, +describe("ship handler", () => { + it("resumes at implement after succeeded triage and plan", async () => { + const triage = snapshot({ + status: "succeeded", + state: { recommendedNext: "plan" }, + createdAt: "2026-08-23T01:00:00.000Z", }); - const { runId: planRunId } = await seedSucceededRun({ - workflowName: "plan", - target: { owner: targetOwner, repo: targetRepo, number: targetNumber }, - state: { planWritten: true }, + const plan = snapshot({ + status: "succeeded", + createdAt: "2026-08-23T01:01:00.000Z", }); - // Seed an implement that FAILED previously (not a succeeded row → stale). - const implementPrior = await insertQueued( - { - workflowName: "implement", - target: { - type: "issue", - owner: targetOwner, - repo: targetRepo, - number: targetNumber, - }, - initialState: {}, - ownerKind: "orchestrator", - ownerId: "test-orchestrator", - }, - requireSql(), - ); - await markFailed(implementPrior.id, "merge conflict", {}, requireSql()); - - // New parent ship row representing the re-apply of `bot:ship`. - const parent = await insertQueued( - { - workflowName: "ship", - target: { - type: "issue", - owner: targetOwner, - repo: targetRepo, - number: targetNumber, - }, - initialState: { currentStepIndex: 0, stepRuns: [] }, - ownerKind: "orchestrator", - ownerId: "test-orchestrator", - }, - requireSql(), - ); + const implement = snapshot({ + status: "failed", + createdAt: "2026-08-23T01:02:00.000Z", + }); + const target = { type: "issue" as const, owner: "acme", repo: "repo", number: 301 }; + const setState = mock(() => Promise.resolve()); + const handOffChild = mock(() => Promise.resolve({ childRunId: "child-implement" })); - const setStateMock = mock(async () => Promise.resolve()); const result = await shipHandler({ - runId: parent.id, + runId: crypto.randomUUID(), workflowName: "ship", - target: { - type: "issue", - owner: targetOwner, - repo: targetRepo, - number: targetNumber, - }, + target, logger: silentLogger(), octokit: buildOctokit("open"), deliveryId: "delivery-301", - setState: setStateMock, + setState, + handOffChild, + shipStepRuns: { triage, plan, implement }, }); - expect(result.status).toBe("handed-off"); - if (result.status !== "handed-off") throw new Error("expected handed-off"); - const state = result.state as { currentStepIndex: number; stepRuns: string[] }; - expect(state.currentStepIndex).toBe(2); - expect(state.stepRuns).toEqual([triageRunId, planRunId]); - - // Enqueue must be for the `implement` step, not triage or plan. - expect(mockEnqueueJob).toHaveBeenCalledTimes(1); - const call = mockEnqueueJob.mock.calls[0]?.[0] as - | { workflowRun: { workflowName: string; parentStepIndex: number } } - | undefined; - expect(call?.workflowRun.workflowName).toBe("implement"); - expect(call?.workflowRun.parentStepIndex).toBe(2); - - expect(setStateMock).toHaveBeenCalledTimes(1); + expect(result).toEqual({ + status: "handed-off", + state: { + currentStepIndex: 2, + stepRuns: [triage.id, plan.id], + handedOffTo: "child-implement", + }, + humanMessage: "ship resumed at step 2 (`implement`); 2 prior step(s) reused.", + childRunId: "child-implement", + }); + expect(handOffChild).toHaveBeenCalledWith({ + workflowName: "implement", + target, + parentStepIndex: 2, + state: { currentStepIndex: 2, stepRuns: [triage.id, plan.id] }, + humanMessage: "ship resumed at step 2 (`implement`); 2 prior step(s) reused.", + }); + expect(setState).not.toHaveBeenCalled(); }); - it("T028 open-PR case (FR-020): prior implement succeeded with an open PR → skip straight to review", async () => { - const { insertQueued } = await import("../../../src/workflows/runs-store"); - - const targetOwner = "acme"; - const targetRepo = "repo"; - const targetNumber = 302; - - const { runId: triageRunId } = await seedSucceededRun({ - workflowName: "triage", - target: { owner: targetOwner, repo: targetRepo, number: targetNumber }, - state: { verdict: "feature", recommendedNext: "plan" }, + it("skips to review when the prior implement PR remains open", async () => { + const triage = snapshot({ + status: "succeeded", + state: { recommendedNext: "plan" }, + createdAt: "2026-08-23T02:00:00.000Z", }); - const { runId: planRunId } = await seedSucceededRun({ - workflowName: "plan", - target: { owner: targetOwner, repo: targetRepo, number: targetNumber }, - state: { planWritten: true }, + const plan = snapshot({ + status: "succeeded", + createdAt: "2026-08-23T02:01:00.000Z", }); - const { runId: implementRunId } = await seedSucceededRun({ - workflowName: "implement", - target: { owner: targetOwner, repo: targetRepo, number: targetNumber }, + const implement = snapshot({ + status: "succeeded", state: { pr_number: 999 }, + createdAt: "2026-08-23T02:02:00.000Z", }); - - const parent = await insertQueued( - { - workflowName: "ship", - target: { - type: "issue", - owner: targetOwner, - repo: targetRepo, - number: targetNumber, - }, - initialState: { currentStepIndex: 0, stepRuns: [] }, - ownerKind: "orchestrator", - ownerId: "test-orchestrator", - }, - requireSql(), - ); - - const octokitOpen = buildOctokit("open"); - const setStateMock = mock(async () => Promise.resolve()); + const target = { type: "issue" as const, owner: "acme", repo: "repo", number: 302 }; + const octokit = buildOctokit("open"); + const handOffChild = mock(() => Promise.resolve({ childRunId: "child-review" })); const result = await shipHandler({ - runId: parent.id, + runId: crypto.randomUUID(), workflowName: "ship", - target: { - type: "issue", - owner: targetOwner, - repo: targetRepo, - number: targetNumber, - }, + target, logger: silentLogger(), - octokit: octokitOpen, + octokit, deliveryId: "delivery-302", - setState: setStateMock, + setState: mock(() => Promise.resolve()), + handOffChild, + shipStepRuns: { triage, plan, implement }, }); - expect(result.status).toBe("handed-off"); - if (result.status !== "handed-off") throw new Error("expected handed-off"); - const state = result.state as { currentStepIndex: number; stepRuns: string[] }; - expect(state.currentStepIndex).toBe(3); - expect(state.stepRuns).toEqual([triageRunId, planRunId, implementRunId]); - - // PR state was verified via the live octokit call. - const pullsGet = (octokitOpen.rest.pulls.get as unknown as ReturnType).mock; - expect(pullsGet.calls.length).toBe(1); - expect(pullsGet.calls[0]?.[0]).toMatchObject({ - owner: targetOwner, - repo: targetRepo, + expect(result).toEqual({ + status: "handed-off", + state: { + currentStepIndex: 3, + stepRuns: [triage.id, plan.id, implement.id], + handedOffTo: "child-review", + }, + humanMessage: "ship resumed at step 3 (`review`); 3 prior step(s) reused.", + childRunId: "child-review", + }); + const pullsGet = octokit.rest.pulls.get as unknown as ReturnType; + expect(pullsGet).toHaveBeenCalledWith({ + owner: target.owner, + repo: target.repo, pull_number: 999, }); - - expect(mockEnqueueJob).toHaveBeenCalledTimes(1); - const call = mockEnqueueJob.mock.calls[0]?.[0] as - | { workflowRun: { workflowName: string; parentStepIndex: number } } - | undefined; - expect(call?.workflowRun.workflowName).toBe("review"); - expect(call?.workflowRun.parentStepIndex).toBe(3); + expect(handOffChild).toHaveBeenCalledWith({ + workflowName: "review", + target, + parentStepIndex: 3, + state: { currentStepIndex: 3, stepRuns: [triage.id, plan.id, implement.id] }, + humanMessage: "ship resumed at step 3 (`review`); 3 prior step(s) reused.", + }); }); + + for (const prState of ["closed", "throw"] as const) { + it(`restarts implement when the prior PR is ${prState === "throw" ? "unverifiable" : "closed"}`, async () => { + const triage = snapshot({ + status: "succeeded", + state: { recommendedNext: "plan" }, + createdAt: "2026-08-23T03:00:00.000Z", + }); + const plan = snapshot({ + status: "succeeded", + createdAt: "2026-08-23T03:01:00.000Z", + }); + const implement = snapshot({ + status: "succeeded", + state: { pr_number: 999 }, + createdAt: "2026-08-23T03:02:00.000Z", + }); + const target = { type: "issue" as const, owner: "acme", repo: "repo", number: 303 }; + const handOffChild = mock(() => Promise.resolve({ childRunId: "child-implement" })); + + const result = await shipHandler({ + runId: crypto.randomUUID(), + workflowName: "ship", + target, + logger: silentLogger(), + octokit: buildOctokit(prState), + deliveryId: "delivery-303", + setState: mock(() => Promise.resolve()), + handOffChild, + shipStepRuns: { triage, plan, implement }, + }); + + expect(result).toMatchObject({ + status: "handed-off", + state: { + currentStepIndex: 2, + stepRuns: [triage.id, plan.id], + handedOffTo: "child-implement", + }, + childRunId: "child-implement", + }); + expect(handOffChild).toHaveBeenCalledWith( + expect.objectContaining({ + workflowName: "implement", + parentStepIndex: 2, + state: { currentStepIndex: 2, stepRuns: [triage.id, plan.id] }, + }), + ); + }); + } }); diff --git a/test/workflows/handlers/triage.test.ts b/test/workflows/handlers/triage.test.ts index e7bd0e18..49adb782 100644 --- a/test/workflows/handlers/triage.test.ts +++ b/test/workflows/handlers/triage.test.ts @@ -13,12 +13,15 @@ import type pino from "pino"; import { config } from "../../../src/config"; import type { WorkflowRunContext } from "../../../src/workflows/registry"; +import { StaleWorkflowAttemptError } from "../../../src/workflows/runs-store"; +import { expectToReject } from "../../utils/assertions"; let agentResult: { success: boolean; costUsd?: number; numTurns?: number; durationMs?: number; + errorMessage?: string; }; let triageMd: string; let triageVerdict: string; @@ -32,9 +35,18 @@ void mock.module("../../../src/core/checkout", () => ({ ), })); +/** The subset of `ExecuteAgentParams` these tests read back. */ +interface ExecuteAgentArgs { + allowedTools: string[]; + promptParts?: { append: string; userMessage: string }; + model?: string; + maxTurns?: number; + signal?: AbortSignal; +} + // Hoisted so tests can inspect the params the handler forwards (e.g. whether // `promptParts` is threaded through under PROMPT_CACHE_LAYOUT=cacheable). -const executeAgentMock = mock(async () => Promise.resolve(agentResult)); +const executeAgentMock = mock(async (_params: ExecuteAgentArgs) => Promise.resolve(agentResult)); void mock.module("../../../src/core/executor", () => ({ executeAgent: executeAgentMock, })); @@ -127,6 +139,18 @@ afterEach(() => { }); describe("triage handler (SDK-driven)", () => { + it("stops before agent execution when the starting-comment attempt fence is stale", async () => { + const ctx = buildCtx(); + ctx.setStateMock.mockImplementation(() => + Promise.reject( + new StaleWorkflowAttemptError({ runId: ctx.runId, attemptId: crypto.randomUUID() }), + ), + ); + + await expectToReject(triageHandler(ctx), "workflow attempt is no longer current"); + expect(executeAgentMock).not.toHaveBeenCalled(); + }); + it("returns succeeded when verdict is valid", async () => { const ctx = buildCtx(); @@ -238,6 +262,25 @@ describe("triage handler (SDK-driven)", () => { } }); + it("propagates the executor's errorMessage into `reason`, not into the public comment", async () => { + agentResult = { + success: false, + durationMs: 5_000, + errorMessage: + "Claude Code returned an error result: You've hit your limit · resets 6pm (UTC)", + }; + + const result = await triageHandler(buildCtx()); + + expect(result.status).toBe("failed"); + if (result.status === "failed") { + // Load-bearing: orchestrator.ts:483 matches /hit your limit/i to defer and retry. + expect(result.reason).toContain("hit your limit"); + expect(result.humanMessage).toContain("see server logs"); + expect(result.humanMessage).not.toContain("hit your limit"); + } + }); + it("fails when TRIAGE.md is missing", async () => { triageMd = ""; const ctx = buildCtx(); @@ -342,3 +385,142 @@ describe("triage handler (SDK-driven)", () => { expect(params.promptParts?.userMessage.length).toBeGreaterThan(0); }); }); + +// ─── Per-repo agent policy, `.github-app.yaml` Gate 2 ─────────────────────── + +/** The tool list the handler owns; per-repo extras may only widen it. */ +const TRIAGE_BASE_TOOLS = ["Read", "Grep", "Glob", "Bash", "Write"]; + +/** `WorkflowRunContext` fields are readonly, mirrors implement.test.ts. */ +function withPolicy(ctx: WorkflowRunContext, policy: Record): WorkflowRunContext { + (ctx as unknown as Record)["policy"] = policy; + return ctx; +} + +function withMaxTurns(ctx: WorkflowRunContext, maxTurns: number): WorkflowRunContext { + (ctx as unknown as Record)["maxTurns"] = maxTurns; + return ctx; +} + +function withSignal(ctx: WorkflowRunContext, signal: AbortSignal): WorkflowRunContext { + (ctx as unknown as Record)["signal"] = signal; + return ctx; +} + +/** Resolves when `signal` aborts; capped so a dead deadline fails rather than hangs. */ +async function waitForAbort(signal: AbortSignal, capMs: number): Promise { + return new Promise((resolve) => { + const cap = setTimeout(resolve, capMs); + const onAbort = (): void => { + clearTimeout(cap); + resolve(); + }; + signal.addEventListener("abort", onAbort, { once: true }); + }); +} + +function firstExecuteAgentArgs(): Record { + const params = executeAgentMock.mock.calls[0]?.[0]; + expect(params).toBeDefined(); + return params as unknown as Record; +} + +describe("triage handler: per-repo policy forwarding", () => { + it("forwards the policy model to executeAgent (C5)", async () => { + await triageHandler(withPolicy(buildCtx(), { model: "claude-repo-pinned" })); + + expect(executeAgentMock).toHaveBeenCalledTimes(1); + expect(firstExecuteAgentArgs()["model"]).toBe("claude-repo-pinned"); + }); + + it("forwards the run-context maxTurns to executeAgent (C6)", async () => { + await triageHandler(withMaxTurns(buildCtx(), 7)); + + expect(firstExecuteAgentArgs()["maxTurns"]).toBe(7); + }); + + it("forwards the daemon attempt signal to executeAgent", async () => { + const signal = new AbortController().signal; + + await triageHandler(withSignal(buildCtx(), signal)); + + expect(firstExecuteAgentArgs()["signal"]).toBe(signal); + }); + + it("stops after an outer fence when a repository deadline is also configured", async () => { + const outer = new AbortController(); + const ctx = withSignal(withPolicy(buildCtx(), { timeoutMs: 60_000 }), outer.signal); + executeAgentMock.mockImplementationOnce(() => { + outer.abort(new Error("workflow lease fenced")); + return Promise.resolve(agentResult); + }); + + const result = await triageHandler(ctx); + + expect(result.status).toBe("failed"); + expect(firstExecuteAgentArgs()["signal"]).not.toBe(outer.signal); + expect((firstExecuteAgentArgs()["signal"] as AbortSignal).aborted).toBe(true); + expect(ctx.setStateMock).toHaveBeenCalledTimes(1); + }); + + it("forwards a per-repo deadline as a named Error, not a bare TimeoutError (C7)", async () => { + // A bare TimeoutError DOMException would fail executeAgent's identity check. + const seen: { hasSignal?: boolean; abortedAtEntry?: boolean; reason?: unknown } = {}; + executeAgentMock.mockImplementationOnce(async (params: ExecuteAgentArgs) => { + const { signal } = params; + seen.hasSignal = signal !== undefined; + seen.abortedAtEntry = signal?.aborted; + if (signal !== undefined) { + await waitForAbort(signal, 3_000); + seen.reason = signal.reason; + } + return agentResult; + }); + + await triageHandler(withPolicy(buildCtx(), { timeoutMs: 25 })); + + expect(seen.hasSignal).toBe(true); + expect(seen.abortedAtEntry).toBe(false); + expect(seen.reason).toBeInstanceOf(Error); + const reason = seen.reason as Error; + expect(reason.name).not.toBe("TimeoutError"); + expect(reason.message).toContain(config.repoConfigFile); + expect(reason.message).toMatch(/timeout/i); + }); + + it("unions extraAllowedTools onto the handler's base list (C8)", async () => { + // "Read" duplicates a base tool: extras are deduped, never a replacement. + await triageHandler(withPolicy(buildCtx(), { extraAllowedTools: ["WebFetch", "Read"] })); + + const tools = firstExecuteAgentArgs()["allowedTools"] as string[]; + for (const tool of TRIAGE_BASE_TOOLS) expect(tools).toContain(tool); + expect(tools).toContain("WebFetch"); + expect(new Set(tools).size).toBe(tools.length); + expect(tools).toHaveLength(TRIAGE_BASE_TOOLS.length + 1); + }); + + it("disposes the deadline timer when the run finishes early (no leaked timer)", async () => { + // C7 waits for the deadline, so its timer is spent; only a normal finish + // catches a missing `disposePolicy?.()`. + await triageHandler(withPolicy(buildCtx(), { timeoutMs: 25 })); + + const signal = firstExecuteAgentArgs()["signal"] as AbortSignal; + expect(signal.aborted).toBe(false); + await new Promise((resolve) => { + setTimeout(resolve, 200); + }); + + expect(signal.aborted).toBe(false); + }); + + it("passes no model, maxTurns, or signal key when the run context carries none of them (C12)", async () => { + await triageHandler(buildCtx()); + + const params = firstExecuteAgentArgs(); + // `exactOptionalPropertyTypes`: absent, not `undefined`-valued. + expect(Object.hasOwn(params, "model")).toBe(false); + expect(Object.hasOwn(params, "maxTurns")).toBe(false); + expect(Object.hasOwn(params, "signal")).toBe(false); + expect(params["allowedTools"]).toEqual(TRIAGE_BASE_TOOLS); + }); +}); diff --git a/test/workflows/orchestrator.test.ts b/test/workflows/orchestrator.test.ts index 365823bb..29d2c3c0 100644 --- a/test/workflows/orchestrator.test.ts +++ b/test/workflows/orchestrator.test.ts @@ -19,6 +19,8 @@ import { SQL } from "bun"; import { afterAll, beforeAll, beforeEach, describe, expect, it, mock, spyOn } from "bun:test"; import type pino from "pino"; +import { expectToReject } from "../utils/assertions"; + const TEST_DATABASE_URL = process.env["TEST_DATABASE_URL"] ?? "postgres://bot:bot@localhost:55432/github_app_test"; @@ -36,6 +38,40 @@ function requireSql(): SQL { return sql; } +async function markTestRunning(runId: string, daemonId: string, db: SQL): Promise { + await db` + UPDATE workflow_runs + SET status = 'running', owner_kind = 'daemon', owner_id = ${daemonId} + WHERE id = ${runId} AND status = 'queued' + `; +} + +async function markTestSucceeded( + runId: string, + state: Record, + db: SQL, +): Promise { + await db` + UPDATE workflow_runs + SET status = 'succeeded', state = workflow_runs.state || ${state}::jsonb + WHERE id = ${runId} + `; +} + +async function markTestFailed( + runId: string, + reason: string, + state: Record, + db: SQL, +): Promise { + const merged = { ...state, failedReason: reason }; + await db` + UPDATE workflow_runs + SET status = 'failed', state = workflow_runs.state || ${merged}::jsonb + WHERE id = ${runId} + `; +} + // ─── Mocked downstream surfaces ────────────────────────────────────────── const mockEnqueueJob = mock(() => Promise.resolve()); @@ -50,6 +86,38 @@ void mock.module("../../src/workflows/execution-row", () => ({ buildWorkflowContextJson: mock(() => ({})), })); +void mock.module("../../src/workflows/dispatch-outbox", () => ({ + publishPendingWorkflowRuns: mock(() => Promise.resolve(0)), + publishWorkflowRunById: async (runId: string, db: SQL) => { + const rows: { + id: string; + workflow_name: string; + parent_run_id: string | null; + parent_step_index: number | null; + target_type: "issue" | "pr"; + target_number: number; + }[] = await db` + SELECT id, workflow_name, parent_run_id, parent_step_index, target_type, target_number + FROM workflow_runs + WHERE id = ${runId} + `; + const row = rows[0]; + if (row === undefined) return false; + await mockEnqueueJob({ + entityNumber: row.target_number, + isPR: row.target_type === "pr", + workflowRun: { + runId: row.id, + workflowName: row.workflow_name, + ...(row.parent_run_id !== null && row.parent_step_index !== null + ? { parentRunId: row.parent_run_id, parentStepIndex: row.parent_step_index } + : {}), + }, + }); + return true; + }, +})); + void mock.module("../../src/orchestrator/concurrency", () => ({ incrementActiveCount: mock(() => {}), decrementActiveCount: mock(() => {}), @@ -57,6 +125,7 @@ void mock.module("../../src/orchestrator/concurrency", () => ({ const mockSetState = mock(() => Promise.resolve()); void mock.module("../../src/workflows/tracking-mirror", () => ({ + LAST_HUMAN_MESSAGE_KEY: "_lastHumanMessage", setState: mockSetState, postRefusalComment: mock(() => Promise.resolve()), })); @@ -101,6 +170,7 @@ describe.skipIf(sql === null)("orchestrator.onStepComplete", () => { beforeAll(async () => { await requireSql().unsafe(` DROP TABLE IF EXISTS _migrations CASCADE; + DROP TABLE IF EXISTS workflow_attempt_commands CASCADE; DROP TABLE IF EXISTS review_learnings CASCADE; DROP TABLE IF EXISTS scheduled_action_state CASCADE; DROP TABLE IF EXISTS comment_cache CASCADE; @@ -123,6 +193,7 @@ describe.skipIf(sql === null)("orchestrator.onStepComplete", () => { afterAll(async () => { await requireSql().unsafe(` DROP TABLE IF EXISTS _migrations CASCADE; + DROP TABLE IF EXISTS workflow_attempt_commands CASCADE; DROP TABLE IF EXISTS review_learnings CASCADE; DROP TABLE IF EXISTS scheduled_action_state CASCADE; DROP TABLE IF EXISTS comment_cache CASCADE; @@ -148,8 +219,7 @@ describe.skipIf(sql === null)("orchestrator.onStepComplete", () => { }); it("T025 success chain: each child success enqueues the next, final child flips parent to succeeded", async () => { - const { insertQueued, findById, markRunning, markSucceeded } = - await import("../../src/workflows/runs-store"); + const { insertQueued, findById } = await import("../../src/workflows/runs-store"); const { onStepComplete } = await import("../../src/workflows/orchestrator"); const issueNumber = 201; @@ -168,7 +238,7 @@ describe.skipIf(sql === null)("orchestrator.onStepComplete", () => { }, requireSql(), ); - await markRunning(parent.id, "test-daemon", requireSql()); + await markTestRunning(parent.id, "test-daemon", requireSql()); const child0 = await insertQueued( { workflowName: "triage", @@ -184,7 +254,7 @@ describe.skipIf(sql === null)("orchestrator.onStepComplete", () => { // ── child 0 (triage) succeeds ──────────────────────────────────────── // Simulate the executor's terminal write before the cascade runs, so // the orchestrator sees the child as already succeeded. - await markSucceeded(child0.id, { verdict: "valid" }, requireSql()); + await markTestSucceeded(child0.id, { verdict: "valid" }, requireSql()); await onStepComplete({ octokit: {} as never, logger: silentLogger() }, child0.id, { status: "succeeded", @@ -206,7 +276,7 @@ describe.skipIf(sql === null)("orchestrator.onStepComplete", () => { expect(child1RunId).not.toBe(""); // ── child 1 (plan) succeeds ────────────────────────────────────────── - await markSucceeded(child1RunId, { planWritten: true }, requireSql()); + await markTestSucceeded(child1RunId, { planWritten: true }, requireSql()); await onStepComplete({ octokit: {} as never, logger: silentLogger() }, child1RunId, { status: "succeeded", }); @@ -223,7 +293,7 @@ describe.skipIf(sql === null)("orchestrator.onStepComplete", () => { const child2RunId = call1?.workflowRun.runId ?? ""; // ── child 2 (implement) succeeds ───────────────────────────────────── - await markSucceeded(child2RunId, { pr_number: 42 }, requireSql()); + await markTestSucceeded(child2RunId, { pr_number: 42 }, requireSql()); await onStepComplete({ octokit: {} as never, logger: silentLogger() }, child2RunId, { status: "succeeded", }); @@ -240,7 +310,7 @@ describe.skipIf(sql === null)("orchestrator.onStepComplete", () => { // ── child 3 (review-1) succeeds with findings → resolve-1 enqueued ── // Phase 1: review must run on the PR target (#42), not the issue. - await markSucceeded( + await markTestSucceeded( child3RunId, { findings: { blocker: 0, major: 1, minor: 0, nit: 0, total: 1 } }, requireSql(), @@ -271,7 +341,7 @@ describe.skipIf(sql === null)("orchestrator.onStepComplete", () => { // ── child 4 (resolve-1) succeeds → loops back to review-2 ─────────── // Phase 2c: review_iterations(1) < cap(2), so a new review child is // inserted at parent_step_index = ship.steps.indexOf("review") = 3. - await markSucceeded(child4RunId, { approved: true }, requireSql()); + await markTestSucceeded(child4RunId, { approved: true }, requireSql()); await onStepComplete({ octokit: {} as never, logger: silentLogger() }, child4RunId, { status: "succeeded", }); @@ -296,7 +366,7 @@ describe.skipIf(sql === null)("orchestrator.onStepComplete", () => { // ── child 5 (review-2) succeeds with no findings → parent succeeded ─ // Phase 2c: review_iterations(2) >= 2 AND total findings == 0, so the // loop short-circuits and the ship parent terminates as succeeded. - await markSucceeded( + await markTestSucceeded( child5RunId, { findings: { blocker: 0, major: 0, minor: 0, nit: 0, total: 0 } }, requireSql(), @@ -324,7 +394,7 @@ describe.skipIf(sql === null)("orchestrator.onStepComplete", () => { }); it("T026 failure cascade: child-2 failure flips parent to failed with failedAtStepIndex=2, no further children", async () => { - const { insertQueued, findById, markRunning, markSucceeded, markFailed, listChildrenByParent } = + const { insertQueued, findById, listChildrenByParent } = await import("../../src/workflows/runs-store"); const { onStepComplete } = await import("../../src/workflows/orchestrator"); @@ -343,7 +413,7 @@ describe.skipIf(sql === null)("orchestrator.onStepComplete", () => { }, requireSql(), ); - await markRunning(parent.id, "test-daemon", requireSql()); + await markTestRunning(parent.id, "test-daemon", requireSql()); const child0 = await insertQueued( { workflowName: "triage", @@ -356,7 +426,7 @@ describe.skipIf(sql === null)("orchestrator.onStepComplete", () => { requireSql(), ); - await markSucceeded(child0.id, { verdict: "valid" }, requireSql()); + await markTestSucceeded(child0.id, { verdict: "valid" }, requireSql()); await onStepComplete({ octokit: {} as never, logger: silentLogger() }, child0.id, { status: "succeeded", }); @@ -365,7 +435,7 @@ describe.skipIf(sql === null)("orchestrator.onStepComplete", () => { | { workflowRun: { runId: string } } | undefined; const child1RunId = child1Enqueue?.workflowRun.runId ?? ""; - await markSucceeded(child1RunId, {}, requireSql()); + await markTestSucceeded(child1RunId, {}, requireSql()); await onStepComplete({ octokit: {} as never, logger: silentLogger() }, child1RunId, { status: "succeeded", }); @@ -379,7 +449,7 @@ describe.skipIf(sql === null)("orchestrator.onStepComplete", () => { mockEnqueueJob.mockClear(); // ── child 2 (implement) FAILS ──────────────────────────────────────── - await markFailed(child2RunId, "merge conflict", {}, requireSql()); + await markTestFailed(child2RunId, "merge conflict", {}, requireSql()); await onStepComplete({ octokit: {} as never, logger: silentLogger() }, child2RunId, { status: "failed", reason: "merge conflict", @@ -399,9 +469,238 @@ describe.skipIf(sql === null)("orchestrator.onStepComplete", () => { expect(mockSetState).toHaveBeenCalled(); }); + it("does not revive a terminal parent when a late child result is replayed", async () => { + const { findById, insertQueued } = await import("../../src/workflows/runs-store"); + const { onStepComplete } = await import("../../src/workflows/orchestrator"); + const shipTarget = { ...target, number: 209 }; + const parent = await insertQueued( + { + workflowName: "ship", + target: shipTarget, + initialState: { currentStepIndex: 0, stepRuns: [] }, + ownerKind: "orchestrator", + ownerId: "test-orchestrator", + }, + requireSql(), + ); + await markTestRunning(parent.id, "test-daemon", requireSql()); + const child = await insertQueued( + { + workflowName: "triage", + target: shipTarget, + parentRunId: parent.id, + parentStepIndex: 0, + ownerKind: "orchestrator", + ownerId: "test-orchestrator", + }, + requireSql(), + ); + await markTestFailed(parent.id, "workflow execution lease expired", {}, requireSql()); + await markTestSucceeded(child.id, { verdict: "valid" }, requireSql()); + + await onStepComplete({ octokit: {} as never, logger: silentLogger() }, child.id, { + status: "succeeded", + }); + + const parentAfter = await findById(parent.id, requireSql()); + expect(parentAfter?.status).toBe("failed"); + expect(parentAfter?.state).toEqual({ + currentStepIndex: 0, + stepRuns: [], + failedReason: "workflow execution lease expired", + }); + expect(mockEnqueueJob).not.toHaveBeenCalled(); + expect(mockSetState).toHaveBeenCalledTimes(1); + }); + + it("records a cascade receipt so a replay cannot enqueue the next child twice", async () => { + const store = await import("../../src/workflows/runs-store"); + const { ensureWorkflowCascadeForOffer } = + await import("../../src/workflows/completion-reconciler"); + const shipTarget = { ...target, number: 210 }; + const parent = await store.insertQueued( + { + workflowName: "ship", + target: shipTarget, + initialState: { currentStepIndex: 0, stepRuns: [] }, + ownerKind: "orchestrator", + ownerId: "test-orchestrator", + }, + requireSql(), + ); + await markTestRunning(parent.id, "test-daemon", requireSql()); + const childExecutionDeliveryId = crypto.randomUUID(); + const child = await store.insertQueued( + { + workflowName: "triage", + target: shipTarget, + parentRunId: parent.id, + parentStepIndex: 0, + executionDeliveryId: childExecutionDeliveryId, + ownerKind: "orchestrator", + ownerId: "test-orchestrator", + }, + requireSql(), + ); + const attempt = { runId: child.id, attemptId: crypto.randomUUID() }; + await requireSql()` + UPDATE workflow_runs + SET status = 'running', + owner_kind = 'daemon', + owner_id = ${`workflow-runner:${attempt.attemptId}`}, + attempt_id = ${attempt.attemptId}, + lease_expires_at = now() + interval '1 minute', + attempt_deadline_at = now() + interval '70 minutes' + WHERE id = ${child.id} + `; + await store.markAttemptSucceeded(attempt, { verdict: "valid" }, requireSql()); + + expect( + await ensureWorkflowCascadeForOffer(attempt.attemptId, silentLogger(), requireSql()), + ).toBe("complete"); + expect( + await ensureWorkflowCascadeForOffer(attempt.attemptId, silentLogger(), requireSql()), + ).toBe("complete"); + + const childAfter = await store.findById(child.id, requireSql()); + const parentAfter = await store.findById(parent.id, requireSql()); + const children = await store.listChildrenByParent(parent.id, requireSql()); + expect(childAfter?.cascade_completed_at).toBeInstanceOf(Date); + expect(parentAfter?.state["stepRuns"]).toEqual([child.id]); + expect(children.map((row) => row.parent_step_index)).toEqual([0, 1]); + expect(mockEnqueueJob).toHaveBeenCalledTimes(1); + }); + + it("retries a terminal parent projection before recording the cascade receipt", async () => { + const store = await import("../../src/workflows/runs-store"); + const { ensureWorkflowCascadeForOffer } = + await import("../../src/workflows/completion-reconciler"); + const shipTarget = { ...target, number: 212 }; + const parent = await store.insertQueued( + { + workflowName: "ship", + target: shipTarget, + initialState: { currentStepIndex: 0, stepRuns: [] }, + ownerKind: "orchestrator", + ownerId: "test-orchestrator", + }, + requireSql(), + ); + await markTestRunning(parent.id, "test-daemon", requireSql()); + const child = await store.insertQueued( + { + workflowName: "triage", + target: shipTarget, + parentRunId: parent.id, + parentStepIndex: 0, + executionDeliveryId: crypto.randomUUID(), + ownerKind: "orchestrator", + ownerId: "test-orchestrator", + }, + requireSql(), + ); + const attempt = { runId: child.id, attemptId: crypto.randomUUID() }; + await requireSql()` + UPDATE workflow_runs + SET status = 'running', + owner_kind = 'daemon', + owner_id = ${`workflow-runner:${attempt.attemptId}`}, + attempt_id = ${attempt.attemptId}, + lease_expires_at = now() + interval '1 minute', + attempt_deadline_at = now() + interval '70 minutes' + WHERE id = ${child.id} + `; + await store.markAttemptFailed(attempt, "child failed", {}, requireSql()); + mockSetState.mockRejectedValueOnce(new Error("GitHub unavailable")); + + await expectToReject( + ensureWorkflowCascadeForOffer(attempt.attemptId, silentLogger(), requireSql(), {} as never), + "GitHub unavailable", + ); + expect((await store.findById(child.id, requireSql()))?.cascade_completed_at).toBeNull(); + expect((await store.findById(parent.id, requireSql()))?.status).toBe("failed"); + + expect( + await ensureWorkflowCascadeForOffer( + attempt.attemptId, + silentLogger(), + requireSql(), + {} as never, + ), + ).toBe("complete"); + expect((await store.findById(child.id, requireSql()))?.cascade_completed_at).toBeInstanceOf( + Date, + ); + expect(mockSetState).toHaveBeenCalledTimes(2); + }); + + it("repairs the cascade receipt after the parent commit succeeds first", async () => { + const store = await import("../../src/workflows/runs-store"); + const { ensureWorkflowCascadeForOffer } = + await import("../../src/workflows/completion-reconciler"); + const { onStepComplete } = await import("../../src/workflows/orchestrator"); + const shipTarget = { ...target, number: 211 }; + const parent = await store.insertQueued( + { + workflowName: "ship", + target: shipTarget, + initialState: { currentStepIndex: 0, stepRuns: [] }, + ownerKind: "orchestrator", + ownerId: "test-orchestrator", + }, + requireSql(), + ); + await markTestRunning(parent.id, "test-daemon", requireSql()); + const childExecutionDeliveryId = crypto.randomUUID(); + const child = await store.insertQueued( + { + workflowName: "triage", + target: shipTarget, + parentRunId: parent.id, + parentStepIndex: 0, + executionDeliveryId: childExecutionDeliveryId, + ownerKind: "orchestrator", + ownerId: "test-orchestrator", + }, + requireSql(), + ); + const attempt = { runId: child.id, attemptId: crypto.randomUUID() }; + await requireSql()` + UPDATE workflow_runs + SET status = 'running', + owner_kind = 'daemon', + owner_id = ${`workflow-runner:${attempt.attemptId}`}, + attempt_id = ${attempt.attemptId}, + lease_expires_at = now() + interval '1 minute', + attempt_deadline_at = now() + interval '70 minutes' + WHERE id = ${child.id} + `; + await store.markAttemptSucceeded(attempt, { verdict: "valid" }, requireSql()); + + // Models a crash after the parent/next-child transaction commits but + // before completion-reconciler records cascade_completed_at. + await onStepComplete( + { octokit: null, logger: silentLogger(), emitGitHub: false, sql: requireSql() }, + child.id, + { status: "succeeded" }, + ); + expect((await store.findById(child.id, requireSql()))?.cascade_completed_at).toBeNull(); + + expect( + await ensureWorkflowCascadeForOffer(attempt.attemptId, silentLogger(), requireSql()), + ).toBe("complete"); + + const childAfter = await store.findById(child.id, requireSql()); + const parentAfter = await store.findById(parent.id, requireSql()); + const children = await store.listChildrenByParent(parent.id, requireSql()); + expect(childAfter?.cascade_completed_at).toBeInstanceOf(Date); + expect(parentAfter?.state["stepRuns"]).toEqual([child.id]); + expect(children.map((row) => row.parent_step_index)).toEqual([0, 1]); + expect(mockEnqueueJob).toHaveBeenCalledTimes(1); + }); + it("T027 cascade retargeting: missing pr_number on implement → review fails the parent with a clear reason", async () => { - const { insertQueued, findById, markRunning, markSucceeded } = - await import("../../src/workflows/runs-store"); + const { insertQueued, findById } = await import("../../src/workflows/runs-store"); const { onStepComplete } = await import("../../src/workflows/orchestrator"); const issueNumber = 203; @@ -416,7 +715,7 @@ describe.skipIf(sql === null)("orchestrator.onStepComplete", () => { }, requireSql(), ); - await markRunning(parent.id, "test-daemon", requireSql()); + await markTestRunning(parent.id, "test-daemon", requireSql()); // Seed an implement child as if the prior cascade reached it. Implement // forgets to write pr_number to its state, simulates a regressed @@ -432,7 +731,7 @@ describe.skipIf(sql === null)("orchestrator.onStepComplete", () => { }, requireSql(), ); - await markSucceeded(implementChild.id, { branch: "feature/foo" }, requireSql()); + await markTestSucceeded(implementChild.id, { branch: "feature/foo" }, requireSql()); await onStepComplete({ octokit: {} as never, logger: silentLogger() }, implementChild.id, { status: "succeeded", @@ -446,8 +745,7 @@ describe.skipIf(sql === null)("orchestrator.onStepComplete", () => { }); it("T028 cap reached: review-2 still has findings → resolve-2 runs → parent succeeds with manual-re-review warning", async () => { - const { insertQueued, findById, markRunning, markSucceeded } = - await import("../../src/workflows/runs-store"); + const { insertQueued, findById } = await import("../../src/workflows/runs-store"); const { onStepComplete } = await import("../../src/workflows/orchestrator"); const issueNumber = 204; @@ -468,7 +766,7 @@ describe.skipIf(sql === null)("orchestrator.onStepComplete", () => { }, requireSql(), ); - await markRunning(parent.id, "test-daemon", requireSql()); + await markTestRunning(parent.id, "test-daemon", requireSql()); // Resolve-1 just succeeded; this triggers loop back to review-2. const resolve1 = await insertQueued( @@ -482,7 +780,7 @@ describe.skipIf(sql === null)("orchestrator.onStepComplete", () => { }, requireSql(), ); - await markSucceeded(resolve1.id, {}, requireSql()); + await markTestSucceeded(resolve1.id, {}, requireSql()); await onStepComplete({ octokit: {} as never, logger: silentLogger() }, resolve1.id, { status: "succeeded", @@ -497,7 +795,7 @@ describe.skipIf(sql === null)("orchestrator.onStepComplete", () => { const review2Id = loopBackCall?.workflowRun.runId ?? ""; // Review-2 still finds blocker issues (cap-reached scenario). - await markSucceeded( + await markTestSucceeded( review2Id, { findings: { blocker: 1, major: 1, minor: 0, nit: 0, total: 2 } }, requireSql(), @@ -514,7 +812,7 @@ describe.skipIf(sql === null)("orchestrator.onStepComplete", () => { expect(resolve2Call?.workflowRun.workflowName).toBe("resolve"); const resolve2Id = resolve2Call?.workflowRun.runId ?? ""; - await markSucceeded(resolve2Id, {}, requireSql()); + await markTestSucceeded(resolve2Id, {}, requireSql()); await onStepComplete({ octokit: {} as never, logger: silentLogger() }, resolve2Id, { status: "succeeded", }); @@ -535,7 +833,7 @@ describe.skipIf(sql === null)("orchestrator.onStepComplete", () => { // ─── Ship-iteration early-wake (T009) ───────────────────────────────── it("ZADDs ship:tickle when a completed workflow_run carries state.shipIntentId and the intent is non-terminal", async () => { - const { insertQueued, markSucceeded } = await import("../../src/workflows/runs-store"); + const { insertQueued } = await import("../../src/workflows/runs-store"); const { onStepComplete } = await import("../../src/workflows/orchestrator"); const { insertIntent } = await import("../../src/db/queries/ship"); @@ -563,7 +861,7 @@ describe.skipIf(sql === null)("orchestrator.onStepComplete", () => { }, requireSql(), ); - await markSucceeded(run.id, { shipIntentId: intent.id }, requireSql()); + await markTestSucceeded(run.id, { shipIntentId: intent.id }, requireSql()); await onStepComplete({ octokit: {} as never, logger: silentLogger() }, run.id, { status: "succeeded", @@ -575,7 +873,7 @@ describe.skipIf(sql === null)("orchestrator.onStepComplete", () => { }); it("does not ZADD ship:tickle when the workflow_run state has no shipIntentId", async () => { - const { insertQueued, markSucceeded } = await import("../../src/workflows/runs-store"); + const { insertQueued } = await import("../../src/workflows/runs-store"); const { onStepComplete } = await import("../../src/workflows/orchestrator"); const run = await insertQueued( @@ -587,7 +885,7 @@ describe.skipIf(sql === null)("orchestrator.onStepComplete", () => { }, requireSql(), ); - await markSucceeded(run.id, { unrelated: true }, requireSql()); + await markTestSucceeded(run.id, { unrelated: true }, requireSql()); await onStepComplete({ octokit: {} as never, logger: silentLogger() }, run.id, { status: "succeeded", @@ -598,7 +896,7 @@ describe.skipIf(sql === null)("orchestrator.onStepComplete", () => { }); it("does not ZADD ship:tickle when the intent is already terminal", async () => { - const { insertQueued, markSucceeded } = await import("../../src/workflows/runs-store"); + const { insertQueued } = await import("../../src/workflows/runs-store"); const { onStepComplete } = await import("../../src/workflows/orchestrator"); const { insertIntent, transitionIntent } = await import("../../src/db/queries/ship"); @@ -627,7 +925,7 @@ describe.skipIf(sql === null)("orchestrator.onStepComplete", () => { }, requireSql(), ); - await markSucceeded(run.id, { shipIntentId: intent.id }, requireSql()); + await markTestSucceeded(run.id, { shipIntentId: intent.id }, requireSql()); await onStepComplete({ octokit: {} as never, logger: silentLogger() }, run.id, { status: "succeeded", @@ -641,7 +939,7 @@ describe.skipIf(sql === null)("orchestrator.onStepComplete", () => { // this guard a permanently broken intent would burn the iteration cap // re-firing on every failure. it("does not ZADD ship:tickle when the child workflow_run failed (H1)", async () => { - const { insertQueued, markFailed } = await import("../../src/workflows/runs-store"); + const { insertQueued } = await import("../../src/workflows/runs-store"); const { onStepComplete } = await import("../../src/workflows/orchestrator"); const { insertIntent } = await import("../../src/db/queries/ship"); @@ -669,7 +967,7 @@ describe.skipIf(sql === null)("orchestrator.onStepComplete", () => { }, requireSql(), ); - await markFailed(run.id, "implement crashed", { shipIntentId: intent.id }, requireSql()); + await markTestFailed(run.id, "implement crashed", { shipIntentId: intent.id }, requireSql()); mockValkeySend.mockClear(); await onStepComplete({ octokit: {} as never, logger: silentLogger() }, run.id, { @@ -689,7 +987,7 @@ describe.skipIf(sql === null)("orchestrator.onStepComplete", () => { // wiring between extractFailedReason → detectTransientQuotaError → // ZADD with a future score. it("ZADDs ship:tickle at the parsed reset time when child failed with a quota error (H2)", async () => { - const { insertQueued, markFailed } = await import("../../src/workflows/runs-store"); + const { insertQueued } = await import("../../src/workflows/runs-store"); const { onStepComplete } = await import("../../src/workflows/orchestrator"); const { insertIntent } = await import("../../src/db/queries/ship"); @@ -728,7 +1026,7 @@ describe.skipIf(sql === null)("orchestrator.onStepComplete", () => { ); const quotaReason = "Claude Code returned an error result: You've hit your limit · resets 6pm (UTC)"; - await markFailed(run.id, quotaReason, { shipIntentId: intent.id }, requireSql()); + await markTestFailed(run.id, quotaReason, { shipIntentId: intent.id }, requireSql()); mockValkeySend.mockClear(); await onStepComplete({ octokit: {} as never, logger: silentLogger() }, run.id, { @@ -755,7 +1053,7 @@ describe.skipIf(sql === null)("orchestrator.onStepComplete", () => { }); describe("orchestrator helpers (pure)", () => { - it("extractFailedReason reads state.failedReason set by markFailed", async () => { + it("extractFailedReason reads a terminal state failure reason", async () => { const { extractFailedReason } = await import("../../src/workflows/orchestrator"); expect(extractFailedReason({ failedReason: "implement crashed" })).toBe("implement crashed"); expect(extractFailedReason({ failedReason: "" })).toBeUndefined(); diff --git a/test/workflows/runs-store.test.ts b/test/workflows/runs-store.test.ts index 93ed0d1c..9739849d 100644 --- a/test/workflows/runs-store.test.ts +++ b/test/workflows/runs-store.test.ts @@ -9,6 +9,8 @@ import { SQL } from "bun"; import { afterAll, beforeAll, describe, expect, it } from "bun:test"; +import type { WorkflowName } from "../../src/workflows/registry"; +import type { WorkflowRunRow } from "../../src/workflows/runs-store"; import { expectToReject } from "../utils/assertions"; const TEST_DATABASE_URL = @@ -28,6 +30,77 @@ function requireSql(): SQL { return sql; } +async function markTestRunning(runId: string, daemonId: string, db: SQL): Promise { + await db` + UPDATE workflow_runs + SET status = 'running', owner_kind = 'daemon', owner_id = ${daemonId} + WHERE id = ${runId} AND status = 'queued' + `; +} + +async function markTestSucceeded( + runId: string, + state: Record, + db: SQL, +): Promise { + await db` + UPDATE workflow_runs + SET status = 'succeeded', state = workflow_runs.state || ${state}::jsonb + WHERE id = ${runId} + `; +} + +async function markTestFailed( + runId: string, + reason: string, + state: Record, + db: SQL, +): Promise { + const merged = { ...state, failedReason: reason }; + await db` + UPDATE workflow_runs + SET status = 'failed', state = workflow_runs.state || ${merged}::jsonb + WHERE id = ${runId} + `; +} + +async function claimTestAttempt( + _store: object, + row: { + id: string; + workflow_name: WorkflowName; + execution_delivery_id: string | null; + }, + input: { attemptId: string; daemonId: string; leaseMs: number; db?: SQL }, +): Promise { + const db = input.db ?? requireSql(); + const executionDeliveryId = row.execution_delivery_id ?? crypto.randomUUID(); + if (row.execution_delivery_id === null) { + await db` + UPDATE workflow_runs + SET execution_delivery_id = ${executionDeliveryId} + WHERE id = ${row.id} + `; + } + const rows: WorkflowRunRow[] = await db` + UPDATE workflow_runs + SET status = 'running', + owner_kind = 'daemon', + owner_id = ${input.daemonId}, + attempt_id = ${input.attemptId}, + lease_expires_at = now() + ${input.leaseMs} * interval '1 millisecond', + attempt_deadline_at = now() + interval '70 minutes' + WHERE id = ${row.id} + AND workflow_name = ${row.workflow_name} + AND execution_delivery_id = ${executionDeliveryId} + AND status = 'queued' + RETURNING * + `; + const claimed = rows[0]; + if (claimed === undefined) throw new Error("Expected workflow attempt fixture to be claimed"); + return claimed; +} + const target = { type: "issue" as const, owner: "acme", repo: "repo", number: 1 }; describe.skipIf(sql === null)("runs-store", () => { @@ -35,6 +108,7 @@ describe.skipIf(sql === null)("runs-store", () => { // Reset to a clean schema so test order is deterministic. await requireSql().unsafe(` DROP TABLE IF EXISTS _migrations CASCADE; + DROP TABLE IF EXISTS workflow_attempt_commands CASCADE; DROP TABLE IF EXISTS review_learnings CASCADE; DROP TABLE IF EXISTS scheduled_action_state CASCADE; DROP TABLE IF EXISTS comment_cache CASCADE; @@ -57,6 +131,7 @@ describe.skipIf(sql === null)("runs-store", () => { afterAll(async () => { await requireSql().unsafe(` DROP TABLE IF EXISTS _migrations CASCADE; + DROP TABLE IF EXISTS workflow_attempt_commands CASCADE; DROP TABLE IF EXISTS review_learnings CASCADE; DROP TABLE IF EXISTS scheduled_action_state CASCADE; DROP TABLE IF EXISTS comment_cache CASCADE; @@ -100,72 +175,11 @@ describe.skipIf(sql === null)("runs-store", () => { expect(row.state).toEqual({ seeded: true }); expect(row.tracking_comment_id).toBeNull(); expect(row.delivery_id).toBe("delivery-100"); + expect(row.trigger_body_preview).toBe(""); expect(typeof row.id).toBe("string"); expect(row.id.length).toBeGreaterThan(0); }); - it("markRunning only flips queued rows", async () => { - const { insertQueued, markRunning, findById, markSucceeded } = - await import("../../src/workflows/runs-store"); - const row = await insertQueued( - { - workflowName: "plan", - target: { ...target, number: 101 }, - ownerKind: "orchestrator", - ownerId: "test-orchestrator", - }, - requireSql(), - ); - - await markRunning(row.id, "test-daemon", requireSql()); - const afterRunning = await findById(row.id, requireSql()); - expect(afterRunning?.status).toBe("running"); - - // Second call must not flip succeeded back to running. - await markSucceeded(row.id, {}, requireSql()); - await markRunning(row.id, "test-daemon", requireSql()); - const afterNoop = await findById(row.id, requireSql()); - expect(afterNoop?.status).toBe("succeeded"); - }); - - it("markSucceeded merges state via JSONB concat", async () => { - const { insertQueued, markSucceeded, findById } = - await import("../../src/workflows/runs-store"); - const row = await insertQueued( - { - workflowName: "implement", - target: { ...target, number: 102 }, - initialState: { a: 1, b: 2 }, - ownerKind: "orchestrator", - ownerId: "test-orchestrator", - }, - requireSql(), - ); - - await markSucceeded(row.id, { b: 99, c: 3 }, requireSql()); - const after = await findById(row.id, requireSql()); - expect(after?.status).toBe("succeeded"); - expect(after?.state).toEqual({ a: 1, b: 99, c: 3 }); - }); - - it("markFailed records the reason inside state", async () => { - const { insertQueued, markFailed, findById } = await import("../../src/workflows/runs-store"); - const row = await insertQueued( - { - workflowName: "resolve", - target: { type: "pr", owner: "acme", repo: "repo", number: 103 }, - ownerKind: "orchestrator", - ownerId: "test-orchestrator", - }, - requireSql(), - ); - - await markFailed(row.id, "handler exploded", { extra: "context" }, requireSql()); - const after = await findById(row.id, requireSql()); - expect(after?.status).toBe("failed"); - expect(after?.state).toEqual({ extra: "context", failedReason: "handler exploded" }); - }); - it("mergeState updates state without changing status", async () => { const { insertQueued, mergeState, findById } = await import("../../src/workflows/runs-store"); const row = await insertQueued( @@ -203,8 +217,7 @@ describe.skipIf(sql === null)("runs-store", () => { }); it("findInflight returns row while queued/running and null once terminal", async () => { - const { insertQueued, findInflight, markRunning, markSucceeded } = - await import("../../src/workflows/runs-store"); + const { insertQueued, findInflight } = await import("../../src/workflows/runs-store"); const t = { owner: "acme", repo: "repo", number: 106 }; const row = await insertQueued( { @@ -219,18 +232,17 @@ describe.skipIf(sql === null)("runs-store", () => { const queuedHit = await findInflight("plan", t, requireSql()); expect(queuedHit?.id).toBe(row.id); - await markRunning(row.id, "test-daemon", requireSql()); + await markTestRunning(row.id, "test-daemon", requireSql()); const runningHit = await findInflight("plan", t, requireSql()); expect(runningHit?.id).toBe(row.id); - await markSucceeded(row.id, {}, requireSql()); + await markTestSucceeded(row.id, {}, requireSql()); const terminalMiss = await findInflight("plan", t, requireSql()); expect(terminalMiss).toBeNull(); }); it("findLatestForTarget orders by created_at DESC", async () => { - const { insertQueued, markSucceeded, findLatestForTarget } = - await import("../../src/workflows/runs-store"); + const { insertQueued, findLatestForTarget } = await import("../../src/workflows/runs-store"); const t = { owner: "acme", repo: "repo", number: 107 }; const first = await insertQueued( { @@ -241,7 +253,7 @@ describe.skipIf(sql === null)("runs-store", () => { }, requireSql(), ); - await markSucceeded(first.id, {}, requireSql()); + await markTestSucceeded(first.id, {}, requireSql()); // Ensure a distinct created_at tick, created_at defaults to now(). await new Promise((resolve) => setTimeout(resolve, 10)); @@ -279,7 +291,7 @@ describe.skipIf(sql === null)("runs-store", () => { }); it("allows a new queued row once the prior one is terminal", async () => { - const { insertQueued, markSucceeded } = await import("../../src/workflows/runs-store"); + const { insertQueued } = await import("../../src/workflows/runs-store"); const first = await insertQueued( { workflowName: "triage", @@ -289,7 +301,7 @@ describe.skipIf(sql === null)("runs-store", () => { }, requireSql(), ); - await markSucceeded(first.id, {}, requireSql()); + await markTestSucceeded(first.id, {}, requireSql()); const second = await insertQueued( { @@ -366,8 +378,58 @@ describe.skipIf(sql === null)("runs-store", () => { expect(second).toEqual({ won: false, trackingCommentId: 11111 }); }); + it("fences tracking-comment reservation by the exact live attempt", async () => { + const store = await import("../../src/workflows/runs-store"); + const current = await store.insertQueued( + { + workflowName: "triage", + target: { ...target, number: 1151 }, + ownerKind: "orchestrator", + ownerId: "test-orchestrator", + }, + requireSql(), + ); + const currentAttempt = { runId: current.id, attemptId: crypto.randomUUID() }; + await claimTestAttempt(store, current, { + attemptId: currentAttempt.attemptId, + daemonId: "daemon-current", + leaseMs: 60_000, + }); + expect( + await store.tryReserveTrackingCommentId(current.id, 33333, currentAttempt, requireSql()), + ).toEqual({ won: true, trackingCommentId: 33333 }); + + const stale = await store.insertQueued( + { + workflowName: "triage", + target: { ...target, number: 1152 }, + ownerKind: "orchestrator", + ownerId: "test-orchestrator", + }, + requireSql(), + ); + const staleAttempt = { runId: stale.id, attemptId: crypto.randomUUID() }; + await claimTestAttempt(store, stale, { + attemptId: staleAttempt.attemptId, + daemonId: "daemon-stale", + leaseMs: 60_000, + }); + await requireSql()` + UPDATE workflow_runs + SET lease_expires_at = now() - interval '1 second' + WHERE id = ${stale.id} + `; + + await expectToReject( + store.tryReserveTrackingCommentId(stale.id, 44444, staleAttempt, requireSql()), + "workflow attempt is no longer current", + ); + expect((await store.findById(stale.id, requireSql()))?.tracking_comment_id).toBeNull(); + await requireSql()`DELETE FROM workflow_runs WHERE id IN ${requireSql()([current.id, stale.id])}`; + }); + it("findLatestSucceededForTarget returns the most recent succeeded row, ignoring later failed rows", async () => { - const { insertQueued, markSucceeded, markFailed, findLatestSucceededForTarget } = + const { insertQueued, findLatestSucceededForTarget } = await import("../../src/workflows/runs-store"); const t = { owner: "acme", repo: "repo", number: 116 }; @@ -381,7 +443,7 @@ describe.skipIf(sql === null)("runs-store", () => { }, requireSql(), ); - await markSucceeded(first.id, { verdict: "valid" }, requireSql()); + await markTestSucceeded(first.id, { verdict: "valid" }, requireSql()); await new Promise((resolve) => setTimeout(resolve, 10)); // Second run: failed (must not shadow the earlier success). @@ -394,14 +456,14 @@ describe.skipIf(sql === null)("runs-store", () => { }, requireSql(), ); - await markFailed(second.id, "intermittent network error", {}, requireSql()); + await markTestFailed(second.id, "intermittent network error", {}, requireSql()); const latestSucceeded = await findLatestSucceededForTarget("triage", t, requireSql()); expect(latestSucceeded?.id).toBe(first.id); }); it("findLatestSucceededForTarget returns null when no succeeded row exists", async () => { - const { insertQueued, markFailed, findLatestSucceededForTarget } = + const { insertQueued, findLatestSucceededForTarget } = await import("../../src/workflows/runs-store"); const t = { owner: "acme", repo: "repo", number: 117 }; @@ -414,9 +476,344 @@ describe.skipIf(sql === null)("runs-store", () => { }, requireSql(), ); - await markFailed(row.id, "no CI yet", {}, requireSql()); + await markTestFailed(row.id, "no CI yet", {}, requireSql()); const latest = await findLatestSucceededForTarget("resolve", t, requireSql()); expect(latest).toBeNull(); }); + + it("renews only current, unexpired attempts owned by the runner", async () => { + const store = await import("../../src/workflows/runs-store"); + const row = await store.insertQueued( + { + workflowName: "plan", + target: { ...target, number: 119 }, + ownerKind: "orchestrator", + ownerId: "orchestrator-a", + }, + requireSql(), + ); + const attemptId = crypto.randomUUID(); + const unknownAttemptId = crypto.randomUUID(); + await claimTestAttempt(store, row, { + attemptId, + daemonId: "daemon-a", + leaseMs: 60_000, + }); + + const renewed = await store.renewWorkflowAttempts( + "daemon-a", + [attemptId, unknownAttemptId], + 120_000, + requireSql(), + ); + expect(renewed).toEqual({ + renewedAttemptIds: [attemptId], + fencedAttemptIds: [unknownAttemptId], + }); + }); + + it("rejects stale progress and terminal writes without changing the row", async () => { + const store = await import("../../src/workflows/runs-store"); + const row = await store.insertQueued( + { + workflowName: "implement", + target: { ...target, number: 120 }, + ownerKind: "orchestrator", + ownerId: "orchestrator-a", + }, + requireSql(), + ); + const attempt = { runId: row.id, attemptId: crypto.randomUUID() }; + await claimTestAttempt(store, row, { + attemptId: attempt.attemptId, + daemonId: "daemon-a", + leaseMs: 60_000, + }); + await requireSql()` + UPDATE workflow_runs + SET lease_expires_at = now() - interval '1 second' + WHERE id = ${row.id} + `; + + await expectToReject( + store.assertCurrentWorkflowAttempt(attempt, requireSql()), + "workflow attempt is no longer current", + ); + await expectToReject( + store.mergeAttemptState(attempt, { staleProgress: true }, requireSql()), + "workflow attempt is no longer current", + ); + await expectToReject( + store.markAttemptSucceeded(attempt, { staleTerminal: true }, requireSql()), + "workflow attempt is no longer current", + ); + + const after = await store.findById(row.id, requireSql()); + expect(after?.status).toBe("running"); + expect(after?.state).toEqual({}); + await requireSql()`DELETE FROM workflow_runs WHERE id = ${row.id}`; + }); + + it("expires only elapsed current attempts and releases the in-flight guard", async () => { + const store = await import("../../src/workflows/runs-store"); + const expired = await store.insertQueued( + { + workflowName: "resolve", + target: { type: "pr", owner: "acme", repo: "repo", number: 122 }, + ownerKind: "orchestrator", + ownerId: "orchestrator-a", + }, + requireSql(), + ); + const live = await store.insertQueued( + { + workflowName: "triage", + target: { ...target, number: 123 }, + ownerKind: "orchestrator", + ownerId: "orchestrator-a", + }, + requireSql(), + ); + const expiredAttemptId = crypto.randomUUID(); + await claimTestAttempt(store, expired, { + attemptId: expiredAttemptId, + daemonId: "daemon-dead", + leaseMs: 60_000, + }); + await claimTestAttempt(store, live, { + attemptId: crypto.randomUUID(), + daemonId: "daemon-live", + leaseMs: 60_000, + }); + await requireSql()` + UPDATE workflow_runs + SET lease_expires_at = now() - interval '1 second' + WHERE id = ${expired.id} + `; + + const transitioned = await store.expireWorkflowAttempts(requireSql()); + expect(transitioned.map((candidate) => candidate.id)).toEqual([expired.id]); + const failedReason = transitioned[0]?.state["failedReason"]; + expect(failedReason).toBeString(); + expect(failedReason as string).toContain("lease expired"); + expect(failedReason as string).not.toContain("daemon-dead"); + expect(transitioned[0]?.state["phase"]).toBe("lease-expired"); + expect(transitioned[0]?.lease_expires_at).toBeNull(); + expect(await store.findPendingWorkflowFailureNotifications(requireSql())).toEqual([ + expect.objectContaining({ + phase: "lease-expired", + row: expect.objectContaining({ id: expired.id, attempt_id: expiredAttemptId }), + }), + ]); + expect( + await store.markWorkflowFailureNotified( + { runId: expired.id, attemptId: expiredAttemptId }, + requireSql(), + ), + ).toBe(true); + expect(await store.findPendingWorkflowFailureNotifications(requireSql())).toEqual([]); + expect((await store.findById(live.id, requireSql()))?.status).toBe("running"); + + const retry = await store.insertQueued( + { + workflowName: "resolve", + target: { type: "pr", owner: "acme", repo: "repo", number: 122 }, + ownerKind: "orchestrator", + ownerId: "orchestrator-b", + }, + requireSql(), + ); + expect(retry.status).toBe("queued"); + }); + + it("expires a fresh queued dispatch after its publication retry budget", async () => { + const store = await import("../../src/workflows/runs-store"); + const row = await store.insertQueued( + { + workflowName: "review", + target: { type: "pr", owner: "acme", repo: "repo", number: 1235 }, + executionDeliveryId: crypto.randomUUID(), + ownerKind: "orchestrator", + ownerId: "orchestrator-a", + }, + requireSql(), + ); + await requireSql()` + UPDATE workflow_runs SET dispatch_retry_count = 4 WHERE id = ${row.id} + `; + + const expired = await store.expireQueuedWorkflowDispatches(4_200_000, 3, requireSql()); + + expect(expired).toHaveLength(1); + expect(expired[0]).toMatchObject({ + id: row.id, + status: "failed", + state: { + failedReason: "workflow dispatch retries exhausted", + phase: "dispatch-expired", + }, + attempt_completed_at: expect.any(Date), + }); + }); + + it("commits the parent hand-off, child row, and execution row in one transaction", async () => { + const store = await import("../../src/workflows/runs-store"); + const { recordWorkflowExecution } = await import("../../src/workflows/execution-row"); + const parent = await store.insertQueued( + { + workflowName: "ship", + target: { ...target, number: 124 }, + deliveryId: "trace-124", + ownerKind: "orchestrator", + ownerId: "orchestrator-a", + }, + requireSql(), + ); + const attempt = { runId: parent.id, attemptId: crypto.randomUUID() }; + await claimTestAttempt(store, parent, { + attemptId: attempt.attemptId, + daemonId: "daemon-a", + leaseMs: 60_000, + }); + + const child = await requireSql().begin(async (tx) => { + const committed = await store.commitAttemptHandOffChild( + attempt, + { currentStepIndex: 0, stepRuns: [] }, + { + workflowName: "triage", + target: { ...target, number: 124 }, + parentStepIndex: 0, + traceDeliveryId: "trace-124", + }, + tx, + ); + await recordWorkflowExecution({ + deliveryId: committed.id, + target: { ...target, number: 124 }, + senderLogin: "github-app-test", + workflowName: "triage", + runId: committed.id, + logger: { info: () => undefined } as never, + sql: tx, + }); + return committed; + }); + + const parentAfter = await store.findById(parent.id, requireSql()); + expect(parentAfter?.status).toBe("running"); + expect(parentAfter?.lease_expires_at).toBeNull(); + expect(parentAfter?.attempt_completed_at).toBeInstanceOf(Date); + expect(parentAfter?.state).toEqual({ + currentStepIndex: 0, + stepRuns: [], + handedOffTo: child.id, + }); + expect(child.parent_run_id).toBe(parent.id); + expect(child.execution_delivery_id).toBe(child.id); + expect(child.owner_kind).toBeNull(); + expect(child.owner_id).toBeNull(); + expect(child.dispatch_enqueued_at).toBeNull(); + + const executionRows: { delivery_id: string; status: string }[] = await requireSql()` + SELECT delivery_id, status FROM executions WHERE delivery_id = ${child.id} + `; + expect(executionRows).toEqual([{ delivery_id: child.id, status: "queued" }]); + }); + + it("rolls back a child hand-off after the parent attempt lease expires", async () => { + const store = await import("../../src/workflows/runs-store"); + const parent = await store.insertQueued( + { + workflowName: "ship", + target: { ...target, number: 125 }, + ownerKind: "orchestrator", + ownerId: "orchestrator-a", + }, + requireSql(), + ); + const attempt = { runId: parent.id, attemptId: crypto.randomUUID() }; + await claimTestAttempt(store, parent, { + attemptId: attempt.attemptId, + daemonId: "daemon-a", + leaseMs: 60_000, + }); + await requireSql()` + UPDATE workflow_runs + SET lease_expires_at = now() - interval '1 second' + WHERE id = ${parent.id} + `; + + await expectToReject( + requireSql().begin(async (tx) => + store.commitAttemptHandOffChild( + attempt, + { currentStepIndex: 0, stepRuns: [] }, + { + workflowName: "triage", + target: { ...target, number: 125 }, + parentStepIndex: 0, + traceDeliveryId: "trace-125", + }, + tx, + ), + ), + "workflow attempt is no longer current", + ); + + const children: { count: number | string }[] = await requireSql()` + SELECT count(*) AS count FROM workflow_runs WHERE parent_run_id = ${parent.id} + `; + expect(Number(children[0]?.count)).toBe(0); + expect((await store.findById(parent.id, requireSql()))?.state).toEqual({}); + }); + + it("clears a prior tracking id only while the current attempt lease is live", async () => { + const store = await import("../../src/workflows/runs-store"); + const current = await store.insertQueued( + { + workflowName: "plan", + target: { ...target, number: 126 }, + ownerKind: "orchestrator", + ownerId: "orchestrator-a", + }, + requireSql(), + ); + const prior = await store.insertQueued( + { + workflowName: "plan", + target: { ...target, number: 127 }, + ownerKind: "orchestrator", + ownerId: "orchestrator-a", + }, + requireSql(), + ); + const attempt = { runId: current.id, attemptId: crypto.randomUUID() }; + await claimTestAttempt(store, current, { + attemptId: attempt.attemptId, + daemonId: "daemon-a", + leaseMs: 60_000, + }); + await requireSql()` + UPDATE workflow_runs SET tracking_comment_id = 8001 WHERE id = ${prior.id} + `; + + await store.clearTrackingCommentIdForAttempt(prior.id, attempt, requireSql()); + expect((await store.findById(prior.id, requireSql()))?.tracking_comment_id).toBeNull(); + + await requireSql()` + UPDATE workflow_runs SET tracking_comment_id = 8002 WHERE id = ${prior.id} + `; + await requireSql()` + UPDATE workflow_runs + SET lease_expires_at = now() - interval '1 second' + WHERE id = ${current.id} + `; + await expectToReject( + store.clearTrackingCommentIdForAttempt(prior.id, attempt, requireSql()), + "workflow attempt is no longer current", + ); + expect((await store.findById(prior.id, requireSql()))?.tracking_comment_id).toBe(8002); + }); }); diff --git a/test/workflows/ship/cancellation.test.ts b/test/workflows/ship/cancellation.test.ts index 2445fa5b..89f70f2a 100644 --- a/test/workflows/ship/cancellation.test.ts +++ b/test/workflows/ship/cancellation.test.ts @@ -168,6 +168,7 @@ describe.skipIf(sql === null)("cancellation-token discipline (T056)", () => { beforeAll(async () => { await requireSql().unsafe(` DROP TABLE IF EXISTS _migrations CASCADE; + DROP TABLE IF EXISTS workflow_attempt_commands CASCADE; DROP TABLE IF EXISTS review_learnings CASCADE; DROP TABLE IF EXISTS scheduled_action_state CASCADE; DROP TABLE IF EXISTS comment_cache CASCADE; @@ -190,6 +191,7 @@ describe.skipIf(sql === null)("cancellation-token discipline (T056)", () => { afterAll(async () => { await requireSql().unsafe(` DROP TABLE IF EXISTS _migrations CASCADE; + DROP TABLE IF EXISTS workflow_attempt_commands CASCADE; DROP TABLE IF EXISTS review_learnings CASCADE; DROP TABLE IF EXISTS scheduled_action_state CASCADE; DROP TABLE IF EXISTS comment_cache CASCADE; diff --git a/test/workflows/ship/command-dispatch.test.ts b/test/workflows/ship/command-dispatch.test.ts new file mode 100644 index 00000000..a6e9baea --- /dev/null +++ b/test/workflows/ship/command-dispatch.test.ts @@ -0,0 +1,322 @@ +/** + * Gate 1 on the canonical ship rail. + * + * `command-dispatch.ts` bypasses `workflows/dispatcher.ts` entirely, so it + * carries its own `checkRepoGate` call. These tests assert that call exists + * and its verdict is honoured; `test/repo-config/gate.test.ts` owns the rule + * semantics. + * + * Dispatch is fire-and-forget (`void (async () => ...)()`), so every + * assertion waits a macrotask tick for the IIFE to settle. + */ + +import { beforeEach, describe, expect, it, mock } from "bun:test"; +import type { Octokit } from "octokit"; +import type pino from "pino"; + +import type { CanonicalCommand, CommandIntent } from "../../../src/shared/ship-types"; + +// ─── Mocked handlers ───────────────────────────────────────────────────── + +const mockRunShipFromCommand = mock(() => Promise.resolve()); +void mock.module("../../../src/workflows/ship/session-runner", () => ({ + runShipFromCommand: mockRunShipFromCommand, +})); + +const mockRunLifecycleCommand = mock(() => Promise.resolve()); +void mock.module("../../../src/workflows/ship/lifecycle-commands", () => ({ + runLifecycleCommand: mockRunLifecycleCommand, +})); + +const mockDispatchScopedCommand = mock(() => Promise.resolve()); +void mock.module("../../../src/workflows/ship/scoped/dispatch-scoped", () => ({ + dispatchScopedCommand: mockDispatchScopedCommand, +})); + +// The ship rail posts its own refusal comment for `explain: true` verdicts; +// nothing else can speak for it once the canonical parser has claimed the +// trigger. Stubbed so the tests can assert it fired without a real Octokit. +const realTrackingMirror = await import("../../../src/workflows/tracking-mirror"); +const mockPostRefusalComment = mock( + (_deps: unknown, _target: unknown, _name: string, _reason: string) => Promise.resolve(), +); +void mock.module("../../../src/workflows/tracking-mirror", () => ({ + ...realTrackingMirror, + postRefusalComment: mockPostRefusalComment, +})); + +// Both parsers behind `dispatchCommentSurface`. Returning null from the +// literal surface is what pushes execution past the gate to the NL fallback, +// which is the path under test. +const mockRouteTrigger = mock((_input: { surface: string }) => Promise.resolve(null)); +void mock.module("../../../src/workflows/ship/trigger-router", () => ({ + routeTrigger: mockRouteTrigger, +})); + +// Gate 1's config loader. Stubbed rather than left to fail open: the fake +// Octokit below is an empty object, so the real loader throws internally and +// degrades to the permissive default, which would let these assertions pass +// even with the gate call deleted. +const realEffective = await import("../../../src/repo-config/effective"); +const mockLoadRepoPolicy = mock(() => Promise.resolve(realEffective.DEFAULT_REPO_POLICY)); +void mock.module("../../../src/repo-config/effective", () => ({ + ...realEffective, + loadRepoPolicy: mockLoadRepoPolicy, +})); + +const { config } = await import("../../../src/config"); +const { githubAppConfigSchema } = await import("../../../src/repo-config/schema"); +const { COMMAND_INTENTS } = await import("../../../src/shared/ship-types"); +const { WorkflowNameSchema } = await import("../../../src/workflows/registry"); +const { dispatchCanonicalCommand, dispatchCommentSurface, INTENT_TO_WORKFLOW } = + await import("../../../src/workflows/ship/command-dispatch"); + +// ─── Fixtures ───────────────────────────────────────────────────────────── + +function silentLog(): pino.Logger { + const log = { + info: mock(() => {}), + warn: mock(() => {}), + error: mock(() => {}), + debug: mock(() => {}), + child: mock(function (this: unknown) { + return this; + }), + } as unknown as pino.Logger; + return log; +} + +const fakeOctokit = {} as unknown as Octokit; + +function command(intent: CommandIntent): CanonicalCommand { + return { + intent, + surface: "literal", + principal_login: "alice", + pr: { owner: "acme", repo: "repo", number: 42, installation_id: 1 }, + }; +} + +/** Let the fire-and-forget IIFE resolve its awaits before asserting. */ +function settle(): Promise { + return new Promise((resolve) => setTimeout(resolve, 0)); +} + +function disabledRepo(): typeof realEffective.DEFAULT_REPO_POLICY { + return { ...realEffective.DEFAULT_REPO_POLICY, enabled: false }; +} + +/** Build a real policy from YAML, so the tests exercise the actual resolver. */ +function policyFrom(doc: unknown): typeof realEffective.DEFAULT_REPO_POLICY { + return realEffective.resolvePolicy(githubAppConfigSchema.parse(doc)); +} + +// ─── Tests ──────────────────────────────────────────────────────────────── + +describe("dispatchCanonicalCommand repo-config gate", () => { + beforeEach(() => { + mockRunShipFromCommand.mockClear(); + mockRunLifecycleCommand.mockClear(); + mockDispatchScopedCommand.mockClear(); + mockLoadRepoPolicy.mockClear(); + mockPostRefusalComment.mockClear(); + mockLoadRepoPolicy.mockResolvedValue(realEffective.DEFAULT_REPO_POLICY); + }); + + it("dispatches ship when the repo policy allows it", async () => { + dispatchCanonicalCommand(command("ship"), { octokit: fakeOctokit, log: silentLog() }); + await settle(); + + expect(mockLoadRepoPolicy).toHaveBeenCalledTimes(1); + expect(mockRunShipFromCommand).toHaveBeenCalledTimes(1); + }); + + it("blocks ship when the repo is disabled", async () => { + mockLoadRepoPolicy.mockResolvedValue(disabledRepo()); + + dispatchCanonicalCommand(command("ship"), { octokit: fakeOctokit, log: silentLog() }); + await settle(); + + expect(mockRunShipFromCommand).not.toHaveBeenCalled(); + }); + + it("blocks a scoped verb when the repo is disabled", async () => { + mockLoadRepoPolicy.mockResolvedValue(disabledRepo()); + + dispatchCanonicalCommand(command("rebase"), { octokit: fakeOctokit, log: silentLog() }); + await settle(); + + expect(mockDispatchScopedCommand).not.toHaveBeenCalled(); + }); + + it.each(["stop", "abort"] as const)("lets '%s' through a disabled repo", async (intent) => { + mockLoadRepoPolicy.mockResolvedValue(disabledRepo()); + + dispatchCanonicalCommand(command(intent), { octokit: fakeOctokit, log: silentLog() }); + await settle(); + + // De-escalating verbs must land, or disabling the bot strands the very + // run the owner was trying to end. + expect(mockRunLifecycleCommand).toHaveBeenCalledTimes(1); + expect(mockPostRefusalComment).not.toHaveBeenCalled(); + }); + + it.each(["stop", "abort"] as const)( + "still blocks '%s' from a sender outside allowed_users", + async (intent) => { + mockLoadRepoPolicy.mockResolvedValue( + policyFrom({ version: 1, triggers: { allowed_users: ["bob"] } }), + ); + + dispatchCanonicalCommand(command(intent), { octokit: fakeOctokit, log: silentLog() }); + await settle(); + + // The carve-out is about config state, not identity: a login the repo + // excluded must not be able to kill someone else's in-flight run. + expect(mockRunLifecycleCommand).not.toHaveBeenCalled(); + expect(mockPostRefusalComment).toHaveBeenCalledTimes(1); + }, + ); + + it.each(["stop", "abort"] as const)( + "lets '%s' through a passive trigger filter that would strand the run", + async (intent) => { + mockLoadRepoPolicy.mockResolvedValue( + policyFrom({ version: 1, triggers: { ignore_title_keywords: ["WIP"] } }), + ); + + dispatchCanonicalCommand(command(intent), { + octokit: fakeOctokit, + log: silentLog(), + trigger: { title: "WIP: something" }, + }); + await settle(); + + expect(mockRunLifecycleCommand).toHaveBeenCalledTimes(1); + }, + ); + + it("blocks ship when only workflows.ship is disabled, leaving scoped verbs alone", async () => { + mockLoadRepoPolicy.mockResolvedValue( + policyFrom({ version: 1, workflows: { ship: { enabled: false } } }), + ); + + dispatchCanonicalCommand(command("ship"), { octokit: fakeOctokit, log: silentLog() }); + await settle(); + expect(mockRunShipFromCommand).not.toHaveBeenCalled(); + + dispatchCanonicalCommand(command("rebase"), { octokit: fakeOctokit, log: silentLog() }); + await settle(); + expect(mockDispatchScopedCommand).toHaveBeenCalledTimes(1); + }); + + it("blocks bot:triage when workflows.triage is disabled", async () => { + // `triage` is both a scoped CommandIntent and a registry workflow. The + // canonical parser claims `bot:triage` before `dispatchByLabel` runs, so + // this rail is the only place the toggle can be enforced. + mockLoadRepoPolicy.mockResolvedValue( + policyFrom({ version: 1, workflows: { triage: { enabled: false } } }), + ); + + dispatchCanonicalCommand(command("triage"), { octokit: fakeOctokit, log: silentLog() }); + await settle(); + + expect(mockDispatchScopedCommand).not.toHaveBeenCalled(); + expect(mockPostRefusalComment).toHaveBeenCalledTimes(1); + }); + + it("maps every intent that collides with a registry workflow name", () => { + // Missing an entry is a silent bypass, not a type error: the intent just + // stops carrying its workflow name into rule 2. + const workflowNames = new Set(WorkflowNameSchema.options); + const colliding = COMMAND_INTENTS.filter((i) => workflowNames.has(i)).sort(); + expect(Object.keys(INTENT_TO_WORKFLOW).sort()).toEqual(colliding); + }); + + it("posts a refusal comment naming the intent for a scoped verb", async () => { + mockLoadRepoPolicy.mockResolvedValue(disabledRepo()); + + dispatchCanonicalCommand(command("summarize"), { octokit: fakeOctokit, log: silentLog() }); + await settle(); + + // Scoped verbs have no registry entry, so the comment names the verb the + // user actually typed rather than echoing "unknown". + expect(mockPostRefusalComment.mock.calls[0]?.[2]).toBe("summarize"); + }); + + it("blocks 'resume' on a disabled repo, unlike stop and abort", async () => { + mockLoadRepoPolicy.mockResolvedValue(disabledRepo()); + + dispatchCanonicalCommand(command("resume"), { octokit: fakeOctokit, log: silentLog() }); + await settle(); + + expect(mockRunLifecycleCommand).not.toHaveBeenCalled(); + }); + + it("fails open and still dispatches when the policy load throws", async () => { + mockLoadRepoPolicy.mockRejectedValueOnce(new Error("github unreachable")); + + dispatchCanonicalCommand(command("ship"), { octokit: fakeOctokit, log: silentLog() }); + await settle(); + + expect(mockRunShipFromCommand).toHaveBeenCalledTimes(1); + }); +}); + +describe("dispatchCommentSurface repo-config gate", () => { + const pr = { owner: "acme", repo: "repo", number: 42, installation_id: 1 }; + + beforeEach(() => { + mockLoadRepoPolicy.mockClear(); + mockLoadRepoPolicy.mockResolvedValue(realEffective.DEFAULT_REPO_POLICY); + mockRouteTrigger.mockClear(); + mockRouteTrigger.mockResolvedValue(null); + }); + + it("returns false and never reaches the NL classifier on a disabled repo", async () => { + mockLoadRepoPolicy.mockResolvedValue(disabledRepo()); + + const handled = await dispatchCommentSurface({ + commentBody: `${config.triggerPhrase} please review this`, + principal_login: "alice", + pr, + octokit: fakeOctokit, + log: silentLog(), + }); + + // `false`, not `true`: the caller falls through to `dispatchByIntent`, + // which re-runs the gate and owns the single refusal comment. + expect(handled).toBe(false); + const surfaces = mockRouteTrigger.mock.calls.map(([arg]) => arg.surface); + expect(surfaces).toEqual(["literal"]); + }); + + it("skips the gate entirely for a comment that does not open with the trigger phrase", async () => { + const handled = await dispatchCommentSurface({ + commentBody: "just a normal review comment, no mention", + principal_login: "alice", + pr, + octokit: fakeOctokit, + log: silentLog(), + }); + + // The classifier would return null for this body anyway (FR-025a), so + // paying a config fetch per comment buys nothing. + expect(handled).toBe(false); + expect(mockLoadRepoPolicy).not.toHaveBeenCalled(); + }); + + it("reaches the NL classifier when the repo policy allows it", async () => { + const handled = await dispatchCommentSurface({ + commentBody: `${config.triggerPhrase} please review this`, + principal_login: "alice", + pr, + octokit: fakeOctokit, + log: silentLog(), + }); + + expect(handled).toBe(false); // both parsers returned null + const surfaces = mockRouteTrigger.mock.calls.map(([arg]) => arg.surface); + expect(surfaces).toEqual(["literal", "nl"]); + }); +}); diff --git a/test/workflows/ship/fix-attempts.test.ts b/test/workflows/ship/fix-attempts.test.ts index ac788aae..a29e0713 100644 --- a/test/workflows/ship/fix-attempts.test.ts +++ b/test/workflows/ship/fix-attempts.test.ts @@ -54,6 +54,7 @@ describe.skipIf(sql === null)("fix-attempts ledger (T033)", () => { beforeAll(async () => { await requireSql().unsafe(` DROP TABLE IF EXISTS _migrations CASCADE; + DROP TABLE IF EXISTS workflow_attempt_commands CASCADE; DROP TABLE IF EXISTS review_learnings CASCADE; DROP TABLE IF EXISTS scheduled_action_state CASCADE; DROP TABLE IF EXISTS comment_cache CASCADE; @@ -76,6 +77,7 @@ describe.skipIf(sql === null)("fix-attempts ledger (T033)", () => { afterAll(async () => { await requireSql().unsafe(` DROP TABLE IF EXISTS _migrations CASCADE; + DROP TABLE IF EXISTS workflow_attempt_commands CASCADE; DROP TABLE IF EXISTS review_learnings CASCADE; DROP TABLE IF EXISTS scheduled_action_state CASCADE; DROP TABLE IF EXISTS comment_cache CASCADE; diff --git a/test/workflows/ship/intent.test.ts b/test/workflows/ship/intent.test.ts index 2478dd03..fa73c8b4 100644 --- a/test/workflows/ship/intent.test.ts +++ b/test/workflows/ship/intent.test.ts @@ -10,6 +10,8 @@ import { SQL } from "bun"; import { afterAll, beforeAll, beforeEach, describe, expect, it } from "bun:test"; +import { expectToReject } from "../../utils/assertions"; + const TEST_DATABASE_URL = process.env["TEST_DATABASE_URL"] ?? "postgres://bot:bot@localhost:55432/github_app_test"; @@ -52,6 +54,26 @@ const baseInput = ( ...overrides, }); +describe("createIntent Postgres error classification", () => { + it("rethrows a non-23505 error even when it names the active-intent constraint", async () => { + let queryCount = 0; + const wrongSqlState = Object.assign(new Error("foreign key violation"), { + code: "ERR_POSTGRES_SERVER_ERROR", + errno: "23503", + constraint: "ship_intents_one_active_per_pr", + }); + const fakeSql = Object.assign((_strings: TemplateStringsArray, ..._values: unknown[]) => { + queryCount++; + if (queryCount === 1) return Promise.reject(wrongSqlState); + return Promise.resolve([{}]); + }, {}) as unknown as SQL; + const { createIntent } = await import("../../../src/workflows/ship/intent"); + + await expectToReject(createIntent(baseInput(), fakeSql), "foreign key violation"); + expect(queryCount).toBe(1); + }); +}); + describe.skipIf(sql === null)("intent.ts state machine", () => { beforeAll(async () => { const { runMigrations } = await import("../../../src/db/migrate"); diff --git a/test/workflows/ship/iteration-cap.test.ts b/test/workflows/ship/iteration-cap.test.ts index ac60f348..3f2e80f6 100644 --- a/test/workflows/ship/iteration-cap.test.ts +++ b/test/workflows/ship/iteration-cap.test.ts @@ -40,6 +40,7 @@ describe.skipIf(sql === null)("iteration-cap (T036a)", () => { beforeAll(async () => { await requireSql().unsafe(` DROP TABLE IF EXISTS _migrations CASCADE; + DROP TABLE IF EXISTS workflow_attempt_commands CASCADE; DROP TABLE IF EXISTS review_learnings CASCADE; DROP TABLE IF EXISTS scheduled_action_state CASCADE; DROP TABLE IF EXISTS comment_cache CASCADE; @@ -62,6 +63,7 @@ describe.skipIf(sql === null)("iteration-cap (T036a)", () => { afterAll(async () => { await requireSql().unsafe(` DROP TABLE IF EXISTS _migrations CASCADE; + DROP TABLE IF EXISTS workflow_attempt_commands CASCADE; DROP TABLE IF EXISTS review_learnings CASCADE; DROP TABLE IF EXISTS scheduled_action_state CASCADE; DROP TABLE IF EXISTS comment_cache CASCADE; diff --git a/test/workflows/ship/iteration.test.ts b/test/workflows/ship/iteration.test.ts index d39f5ad0..1c320d5f 100644 --- a/test/workflows/ship/iteration.test.ts +++ b/test/workflows/ship/iteration.test.ts @@ -34,9 +34,9 @@ function requireSql(): SQL { return sql; } -const mockEnqueueJob = mock(() => Promise.resolve()); +const mockEnsureWorkflowJobQueued = mock(() => Promise.resolve(true)); void mock.module("../../../src/orchestrator/job-queue", () => ({ - enqueueJob: mockEnqueueJob, + ensureWorkflowJobQueued: mockEnsureWorkflowJobQueued, isScopedJob: () => false, SCOPED_JOB_KINDS: ["scoped-rebase", "scoped-fix-thread", "scoped-open-pr"], })); @@ -51,6 +51,7 @@ describe.skipIf(sql === null)("runIteration", () => { beforeAll(async () => { await requireSql().unsafe(` DROP TABLE IF EXISTS _migrations CASCADE; + DROP TABLE IF EXISTS workflow_attempt_commands CASCADE; DROP TABLE IF EXISTS review_learnings CASCADE; DROP TABLE IF EXISTS scheduled_action_state CASCADE; DROP TABLE IF EXISTS comment_cache CASCADE; @@ -73,6 +74,7 @@ describe.skipIf(sql === null)("runIteration", () => { afterAll(async () => { await requireSql().unsafe(` DROP TABLE IF EXISTS _migrations CASCADE; + DROP TABLE IF EXISTS workflow_attempt_commands CASCADE; DROP TABLE IF EXISTS review_learnings CASCADE; DROP TABLE IF EXISTS scheduled_action_state CASCADE; DROP TABLE IF EXISTS comment_cache CASCADE; @@ -92,7 +94,8 @@ describe.skipIf(sql === null)("runIteration", () => { }); beforeEach(() => { - mockEnqueueJob.mockClear(); + mockEnsureWorkflowJobQueued.mockClear(); + mockEnsureWorkflowJobQueued.mockImplementation(() => Promise.resolve(true)); }); async function seedActiveIntent( @@ -145,11 +148,12 @@ describe.skipIf(sql === null)("runIteration", () => { expect(result.outcome).toBe("enqueued"); - expect(mockEnqueueJob).toHaveBeenCalledTimes(1); - const enqueued = mockEnqueueJob.mock.calls[0]?.[0] as + expect(mockEnsureWorkflowJobQueued).toHaveBeenCalledTimes(1); + const enqueued = mockEnsureWorkflowJobQueued.mock.calls[0]?.[0] as | { kind: string; workflowRun: { runId: string; workflowName: string }; + deliveryId: string; repoOwner: string; repoName: string; entityNumber: number; @@ -160,11 +164,23 @@ describe.skipIf(sql === null)("runIteration", () => { expect(enqueued?.repoOwner).toBe(intent.owner); expect(enqueued?.entityNumber).toBe(intent.pr_number); - const runs: { id: string; state: Record }[] = await requireSql()` - SELECT id, state FROM workflow_runs WHERE id = ${enqueued?.workflowRun.runId ?? ""} + const runs: { + id: string; + state: Record; + execution_delivery_id: string | null; + dispatch_enqueued_at: Date | null; + }[] = await requireSql()` + SELECT id, state, execution_delivery_id, dispatch_enqueued_at + FROM workflow_runs WHERE id = ${enqueued?.workflowRun.runId ?? ""} `; expect(runs).toHaveLength(1); expect(runs[0]?.state["shipIntentId"]).toBe(intent.id); + expect(runs[0]?.execution_delivery_id).toBe(enqueued?.deliveryId); + expect(runs[0]?.dispatch_enqueued_at).toBeInstanceOf(Date); + const executions: { delivery_id: string; status: string }[] = await requireSql()` + SELECT delivery_id, status FROM executions WHERE delivery_id = ${enqueued?.deliveryId ?? ""} + `; + expect(executions).toEqual([{ delivery_id: enqueued?.deliveryId, status: "queued" }]); const iterRows: { iteration_n: number; kind: string }[] = await requireSql()` SELECT iteration_n, kind FROM ship_iterations @@ -176,6 +192,35 @@ describe.skipIf(sql === null)("runIteration", () => { expect(iterRows[1]?.kind).toBe("resolve"); }); + it("returns enqueued and leaves the outbox pending when immediate publication fails", async () => { + const { runIteration } = await import("../../../src/workflows/ship/iteration"); + const { getIntentById } = await import("../../../src/db/queries/ship"); + const intent = await seedActiveIntent({ pr_number: 4247 }); + const intentRow = await getIntentById(intent.id, requireSql()); + if (intentRow === null) throw new Error("seed intent missing"); + mockEnsureWorkflowJobQueued.mockImplementationOnce(() => + Promise.reject(new Error("Valkey unavailable")), + ); + + const result = await runIteration({ + intent: intentRow, + probeVerdict: { + ready: false, + reason: "open_threads", + detail: "1 thread unresolved", + checked_at: new Date().toISOString(), + head_sha: "head-sha", + }, + }); + + expect(result.outcome).toBe("enqueued"); + if (result.outcome !== "enqueued") throw new Error("expected enqueued result"); + const rows: { dispatch_enqueued_at: Date | null }[] = await requireSql()` + SELECT dispatch_enqueued_at FROM workflow_runs WHERE id = ${result.runId} + `; + expect(rows).toEqual([{ dispatch_enqueued_at: null }]); + }); + it("transitions intent to deadline_exceeded with iteration-cap blocker when cap is reached", async () => { const { runIteration } = await import("../../../src/workflows/ship/iteration"); const { appendIteration, getIntentById } = await import("../../../src/db/queries/ship"); @@ -216,7 +261,7 @@ describe.skipIf(sql === null)("runIteration", () => { const refreshed = await getIntentById(intent.id, requireSql()); expect(refreshed?.status).toBe("deadline_exceeded"); expect(refreshed?.terminal_blocker_category).toBe("iteration-cap"); - expect(mockEnqueueJob).toHaveBeenCalledTimes(0); + expect(mockEnsureWorkflowJobQueued).toHaveBeenCalledTimes(0); }); it("transitions intent to deadline_exceeded when the wall-clock deadline has elapsed", async () => { @@ -244,7 +289,7 @@ describe.skipIf(sql === null)("runIteration", () => { expect(result.outcome).toBe("terminal-deadline"); const refreshed = await getIntentById(intent.id, requireSql()); expect(refreshed?.status).toBe("deadline_exceeded"); - expect(mockEnqueueJob).toHaveBeenCalledTimes(0); + expect(mockEnsureWorkflowJobQueued).toHaveBeenCalledTimes(0); }); it("returns ready-shortcut without writing any rows when the verdict is ready", async () => { @@ -265,7 +310,7 @@ describe.skipIf(sql === null)("runIteration", () => { }); expect(result.outcome).toBe("ready-shortcut"); - expect(mockEnqueueJob).toHaveBeenCalledTimes(0); + expect(mockEnsureWorkflowJobQueued).toHaveBeenCalledTimes(0); const iterRows: { count: number }[] = await requireSql()` SELECT COUNT(*)::int AS count FROM ship_iterations WHERE intent_id = ${intent.id} @@ -316,7 +361,7 @@ describe.skipIf(sql === null)("runIteration", () => { if (result.outcome === "in-flight") { expect(result.runId).toBe(inflight.id); } - expect(mockEnqueueJob).toHaveBeenCalledTimes(0); + expect(mockEnsureWorkflowJobQueued).toHaveBeenCalledTimes(0); // No new ship_iterations rows either. const iterRows: { count: number }[] = await requireSql()` diff --git a/test/workflows/ship/lifecycle-commands.test.ts b/test/workflows/ship/lifecycle-commands.test.ts index 14c76d8d..28929c86 100644 --- a/test/workflows/ship/lifecycle-commands.test.ts +++ b/test/workflows/ship/lifecycle-commands.test.ts @@ -141,6 +141,7 @@ describe.skipIf(sql === null)("lifecycle commands (T055 + T058a)", () => { delete process.env["ALLOWED_OWNERS"]; await requireSql().unsafe(` DROP TABLE IF EXISTS _migrations CASCADE; + DROP TABLE IF EXISTS workflow_attempt_commands CASCADE; DROP TABLE IF EXISTS review_learnings CASCADE; DROP TABLE IF EXISTS scheduled_action_state CASCADE; DROP TABLE IF EXISTS comment_cache CASCADE; @@ -164,6 +165,7 @@ describe.skipIf(sql === null)("lifecycle commands (T055 + T058a)", () => { if (ORIGINAL_ALLOWED !== undefined) process.env["ALLOWED_OWNERS"] = ORIGINAL_ALLOWED; await requireSql().unsafe(` DROP TABLE IF EXISTS _migrations CASCADE; + DROP TABLE IF EXISTS workflow_attempt_commands CASCADE; DROP TABLE IF EXISTS review_learnings CASCADE; DROP TABLE IF EXISTS scheduled_action_state CASCADE; DROP TABLE IF EXISTS comment_cache CASCADE; diff --git a/test/workflows/ship/session-runner.resume.test.ts b/test/workflows/ship/session-runner.resume.test.ts index 36cd1861..d4fa8fc2 100644 --- a/test/workflows/ship/session-runner.resume.test.ts +++ b/test/workflows/ship/session-runner.resume.test.ts @@ -62,6 +62,7 @@ describe.skipIf(sql === null)("resumeShipIntent", () => { beforeAll(async () => { await requireSql().unsafe(` DROP TABLE IF EXISTS _migrations CASCADE; + DROP TABLE IF EXISTS workflow_attempt_commands CASCADE; DROP TABLE IF EXISTS review_learnings CASCADE; DROP TABLE IF EXISTS scheduled_action_state CASCADE; DROP TABLE IF EXISTS comment_cache CASCADE; diff --git a/test/workflows/ship/session-runner.test.ts b/test/workflows/ship/session-runner.test.ts index 52aefa61..5bf706fa 100644 --- a/test/workflows/ship/session-runner.test.ts +++ b/test/workflows/ship/session-runner.test.ts @@ -243,6 +243,7 @@ describe.skipIf(sql === null)("runShipFromCommand", () => { beforeAll(async () => { await requireSql().unsafe(` DROP TABLE IF EXISTS _migrations CASCADE; + DROP TABLE IF EXISTS workflow_attempt_commands CASCADE; DROP TABLE IF EXISTS review_learnings CASCADE; DROP TABLE IF EXISTS scheduled_action_state CASCADE; DROP TABLE IF EXISTS comment_cache CASCADE; @@ -265,6 +266,7 @@ describe.skipIf(sql === null)("runShipFromCommand", () => { afterAll(async () => { await requireSql().unsafe(` DROP TABLE IF EXISTS _migrations CASCADE; + DROP TABLE IF EXISTS workflow_attempt_commands CASCADE; DROP TABLE IF EXISTS review_learnings CASCADE; DROP TABLE IF EXISTS scheduled_action_state CASCADE; DROP TABLE IF EXISTS comment_cache CASCADE; diff --git a/test/workflows/ship/tickle-scheduler.test.ts b/test/workflows/ship/tickle-scheduler.test.ts index fef0268b..77342cb4 100644 --- a/test/workflows/ship/tickle-scheduler.test.ts +++ b/test/workflows/ship/tickle-scheduler.test.ts @@ -80,6 +80,7 @@ describe.skipIf(sql === null)("createTickleScheduler, round-trip", () => { beforeAll(async () => { await requireSql().unsafe(` DROP TABLE IF EXISTS _migrations CASCADE; + DROP TABLE IF EXISTS workflow_attempt_commands CASCADE; DROP TABLE IF EXISTS review_learnings CASCADE; DROP TABLE IF EXISTS scheduled_action_state CASCADE; DROP TABLE IF EXISTS comment_cache CASCADE; diff --git a/test/workflows/tracking-mirror.test.ts b/test/workflows/tracking-mirror.test.ts index 46d8a1b2..747b73ca 100644 --- a/test/workflows/tracking-mirror.test.ts +++ b/test/workflows/tracking-mirror.test.ts @@ -3,6 +3,7 @@ import type { Octokit } from "octokit"; import pino from "pino"; import type { WorkflowRunRow } from "../../src/workflows/runs-store"; +import { expectToReject } from "../utils/assertions"; // Mock the runs-store DB layer so the tracking-mirror exercises pure logic // without touching Postgres. Each test seeds the mocks by mutating the @@ -21,18 +22,30 @@ const tryReserveMock = mock(() => Promise.resolve(mockReservation)); const listChildrenByParentMock = mock(() => Promise.resolve([] as readonly WorkflowRunRow[])); const findPriorTrackingCommentsMock = mock(() => Promise.resolve(mockPriorComments)); const clearTrackingCommentIdMock = mock(() => Promise.resolve()); +const clearTrackingCommentIdForAttemptMock = mock(() => Promise.resolve()); +const assertCurrentWorkflowAttemptMock = mock(() => Promise.resolve()); +const mergeAttemptStateMock = mock(() => { + if (mockRow === null) return Promise.reject(new Error("run missing")); + return Promise.resolve(mockRow); +}); +class StaleWorkflowAttemptError extends Error {} void mock.module("../../src/workflows/runs-store", () => ({ + assertCurrentWorkflowAttempt: assertCurrentWorkflowAttemptMock, clearTrackingCommentId: clearTrackingCommentIdMock, + clearTrackingCommentIdForAttempt: clearTrackingCommentIdForAttemptMock, findById: findByIdMock, findPriorTrackingComments: findPriorTrackingCommentsMock, listChildrenByParent: listChildrenByParentMock, mergeState: mergeStateMock, + mergeAttemptState: mergeAttemptStateMock, + StaleWorkflowAttemptError, tryReserveTrackingCommentId: tryReserveMock, })); // Import AFTER mock.module so the module-under-test binds to mocks. -const { setState } = await import("../../src/workflows/tracking-mirror"); +const { CONFIG_NOTICE_KEY, renderCommentBody, setState } = + await import("../../src/workflows/tracking-mirror"); const RUN_ID = "11111111-1111-1111-1111-111111111111"; const MARKER = ``; @@ -111,6 +124,9 @@ describe("tracking-mirror.setState: first-touch create/adopt path", () => { listChildrenByParentMock.mockClear(); findPriorTrackingCommentsMock.mockClear(); clearTrackingCommentIdMock.mockClear(); + clearTrackingCommentIdForAttemptMock.mockClear(); + assertCurrentWorkflowAttemptMock.mockClear(); + mergeAttemptStateMock.mockClear(); mockPriorComments = []; }); @@ -149,6 +165,61 @@ describe("tracking-mirror.setState: first-touch create/adopt path", () => { expect(result.tracking_comment_id).toBe(9000); }); + it("reserves a first-touch comment through the exact workflow attempt", async () => { + mockRow = makeRow(); + mockReservation = { won: true, trackingCommentId: 9000 }; + const attempt = { runId: RUN_ID, attemptId: crypto.randomUUID() }; + const { octokit } = makeOctokit({ createCommentResult: { id: 9000 } }); + let scanCallCount = 0; + const listComments = mock(() => { + scanCallCount += 1; + if (scanCallCount === 1) return Promise.resolve({ data: [] }); + return Promise.resolve({ + data: [{ id: 9000, body: `${MARKER}\nstarting`, created_at: "2026-05-08T02:55:43Z" }], + }); + }); + (octokit.rest.issues as unknown as { listComments: typeof listComments }).listComments = + listComments; + + await setState( + { octokit, logger: SILENT_LOGGER }, + { runId: RUN_ID, patch: {}, humanMessage: "starting", attempt }, + ); + + expect(tryReserveMock).toHaveBeenCalledWith(RUN_ID, 9000, attempt); + }); + + it("stops first-touch cleanup and updates when the reservation attempt is stale", async () => { + mockRow = makeRow({ parent_run_id: "parent-run-id" }); + const attempt = { runId: RUN_ID, attemptId: crypto.randomUUID() }; + const { octokit, calls } = makeOctokit({ createCommentResult: { id: 9000 } }); + tryReserveMock.mockRejectedValueOnce(new StaleWorkflowAttemptError("attempt lost")); + let scanCallCount = 0; + const listComments = mock(() => { + scanCallCount += 1; + if (scanCallCount === 1) return Promise.resolve({ data: [] }); + return Promise.resolve({ + data: [{ id: 9000, body: `${MARKER}\nstarting`, created_at: "2026-05-08T02:55:43Z" }], + }); + }); + (octokit.rest.issues as unknown as { listComments: typeof listComments }).listComments = + listComments; + + await expectToReject( + setState( + { octokit, logger: SILENT_LOGGER }, + { runId: RUN_ID, patch: {}, humanMessage: "starting", attempt }, + ), + "attempt lost", + ); + + expect(calls.createComment).toHaveBeenCalledTimes(1); + expect(calls.updateComment).not.toHaveBeenCalled(); + expect(calls.deleteComment).not.toHaveBeenCalled(); + expect(findPriorTrackingCommentsMock).not.toHaveBeenCalled(); + expect(listChildrenByParentMock).not.toHaveBeenCalled(); + }); + it("adopts an existing marker comment without POSTing when pre-scan finds one (pod-restart recovery)", async () => { mockRow = makeRow(); mockReservation = { won: true, trackingCommentId: 7777 }; @@ -384,6 +455,32 @@ describe("tracking-mirror.setState: first-touch create/adopt path", () => { expect(clearTrackingCommentIdMock).toHaveBeenCalledWith("prior-run-id"); }); + it("re-run cleanup: clears a prior row through the current attempt fence", async () => { + mockRow = makeRow(); + mockReservation = { won: true, trackingCommentId: 9000 }; + mockPriorComments = [{ runId: "prior-run-id", trackingCommentId: 8001 }]; + const { octokit } = makeOctokit({ createCommentResult: { id: 9000 } }); + const attempt = { runId: RUN_ID, attemptId: crypto.randomUUID() }; + let scanCallCount = 0; + const listComments = mock(() => { + scanCallCount += 1; + if (scanCallCount === 1) return Promise.resolve({ data: [] }); + return Promise.resolve({ + data: [{ id: 9000, body: `${MARKER}\nstarting`, created_at: "2026-05-08T02:55:43Z" }], + }); + }); + (octokit.rest.issues as unknown as { listComments: typeof listComments }).listComments = + listComments; + + await setState( + { octokit, logger: SILENT_LOGGER }, + { runId: RUN_ID, patch: {}, humanMessage: "starting", attempt }, + ); + + expect(clearTrackingCommentIdMock).not.toHaveBeenCalled(); + expect(clearTrackingCommentIdForAttemptMock).toHaveBeenCalledWith("prior-run-id", attempt); + }); + it("re-run cleanup: a deleteComment failure does not block the new comment (fail-open)", async () => { mockRow = makeRow(); mockReservation = { won: true, trackingCommentId: 9000 }; @@ -457,4 +554,98 @@ describe("tracking-mirror.setState: first-touch create/adopt path", () => { expect(updateArgs?.body).toContain(MARKER); expect(result.tracking_comment_id).toBe(4242); }); + + it("uses the exact workflow attempt for progress state and the GitHub update", async () => { + mockRow = makeRow({ tracking_comment_id: 4242 }); + const { octokit, calls } = makeOctokit({}); + const attempt = { runId: RUN_ID, attemptId: "22222222-2222-4222-8222-222222222222" }; + + await setState( + { octokit, logger: SILENT_LOGGER }, + { + runId: RUN_ID, + patch: { progress: 50 }, + humanMessage: "half complete", + attempt, + }, + ); + + expect(mergeStateMock).not.toHaveBeenCalled(); + expect(mergeAttemptStateMock).toHaveBeenCalledWith( + attempt, + expect.objectContaining({ progress: 50 }), + ); + expect(assertCurrentWorkflowAttemptMock).toHaveBeenCalledWith(attempt); + expect(calls.updateComment).toHaveBeenCalledTimes(1); + }); + + it("does not update GitHub after the progress attempt loses its lease", async () => { + mockRow = makeRow({ tracking_comment_id: 4242 }); + const { octokit, calls } = makeOctokit({}); + const attempt = { runId: RUN_ID, attemptId: "33333333-3333-4333-8333-333333333333" }; + assertCurrentWorkflowAttemptMock.mockRejectedValueOnce( + new StaleWorkflowAttemptError("attempt lost"), + ); + + await expectToReject( + setState( + { octokit, logger: SILENT_LOGGER }, + { runId: RUN_ID, patch: { progress: 50 }, humanMessage: "half complete", attempt }, + ), + "attempt lost", + ); + + expect(calls.updateComment).not.toHaveBeenCalled(); + }); +}); + +// ─── Per-repo config notice, `.github-app.yaml` Gate 2 ────────────────────── + +describe("renderCommentBody: persisted config notice", () => { + it("pins the state key so a rename fails loudly", () => { + // The key is persisted into `workflow_runs.state`. Renaming it after + // release strands the notice on every existing row, and the underscore + // prefix keeps it out of the handler-visible state namespace. + expect(CONFIG_NOTICE_KEY).toBe("_configNotice"); + }); + + it("renders the notice on every body, not just the write that set it", () => { + const row = makeRow({ + state: { [CONFIG_NOTICE_KEY]: "`.github-app.yaml` failed validation and was ignored." }, + }); + + // The body is rebuilt from scratch on every write, so the assertion that + // matters is that a LATER message still carries the notice. + const body = renderCommentBody(row, "third progress update"); + + expect(body).toContain("> [!WARNING]"); + expect(body).toContain("failed validation"); + expect(body).toContain("third progress update"); + }); + + it("collapses whitespace so a stray CR cannot break out of the blockquote", () => { + const row = makeRow({ state: { [CONFIG_NOTICE_KEY]: "config was\rignored" } }); + + const body = renderCommentBody(row, "msg"); + + expect(body).toContain("> config was ignored"); + expect(body).not.toContain("\r"); + }); + + it("renders each stored line as its own paragraph inside one alert", () => { + const row = makeRow({ + state: { [CONFIG_NOTICE_KEY]: "config was ignored\nreview scope reduced" }, + }); + + const body = renderCommentBody(row, "msg"); + + expect(body).toContain("> [!WARNING]\n> config was ignored\n>\n> review scope reduced"); + }); + + it("renders nothing for an absent or whitespace-only notice", () => { + expect(renderCommentBody(makeRow(), "msg")).not.toContain("[!WARNING]"); + expect( + renderCommentBody(makeRow({ state: { [CONFIG_NOTICE_KEY]: " " } }), "msg"), + ).not.toContain("[!WARNING]"); + }); });