From c0d349284514962ab5dd177fce5bc8adb97cff18 Mon Sep 17 00:00:00 2001 From: MAxim Kiktev Date: Thu, 29 Oct 2026 21:31:41 +0000 Subject: [PATCH 1/5] ci: separate fast feedback, candidate preparation and bounded publication --- .github/workflows/ci.yml | 496 ++++++------------------ .github/workflows/codeql.yml | 14 +- .github/workflows/containers.yml | 56 +-- .github/workflows/docs.yml | 33 +- .github/workflows/full-ci.yml | 515 ++++++++++++++++++++++++ .github/workflows/fuzz.yml | 76 +--- .github/workflows/oci-runtimes.yml | 36 +- .github/workflows/packaging.yml | 42 +- .github/workflows/perf.yml | 40 +- .github/workflows/prepare-release.yml | 497 ++++++++++++++++++++++++ .github/workflows/publish-sdks.yml | 2 + .github/workflows/qualification.yml | 84 ++++ .github/workflows/quality.yml | 43 +- .github/workflows/release.yml | 538 ++++---------------------- .github/workflows/request-release.yml | 6 + .github/workflows/security.yml | 61 ++- .github/workflows/web.yml | 23 +- .github/workflows/windows.yml | 11 +- CHANGELOG.md | 5 + docs/architecture/testing.md | 11 +- docs/operations/performance.md | 2 +- docs/operations/releasing.md | 91 ++--- docs/operations/supply-chain.md | 4 +- scripts/ci_changes.py | 83 ++++ scripts/prepare_release.py | 29 ++ scripts/release_candidate.py | 85 ++++ scripts/request_release.py | 87 +++-- scripts/test_ci_policy.py | 149 +++++++ scripts/test_release_candidate.py | 115 ++++++ scripts/test_request_release.py | 103 +++-- scripts/test_web_e2e_workflow.py | 8 +- tests/skip-allowlist.toml | 20 +- 32 files changed, 2047 insertions(+), 1318 deletions(-) create mode 100644 .github/workflows/full-ci.yml create mode 100644 .github/workflows/prepare-release.yml create mode 100644 .github/workflows/qualification.yml create mode 100644 scripts/ci_changes.py create mode 100644 scripts/prepare_release.py create mode 100644 scripts/release_candidate.py create mode 100644 scripts/test_ci_policy.py create mode 100644 scripts/test_release_candidate.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 08365f70..2941ef5f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,9 +1,11 @@ name: CI +# Fast, path-aware feedback for continuous work. Qualification owns the full matrix. on: push: branches: [main] pull_request: + workflow_dispatch: permissions: contents: read @@ -15,117 +17,55 @@ concurrency: env: CARGO_TERM_COLOR: always CARGO_INCREMENTAL: "0" + CARGO_PROFILE_DEV_DEBUG: "0" + CARGO_PROFILE_TEST_DEBUG: "0" jobs: - test: - name: test (${{ matrix.os }}) - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-latest] + changes: + runs-on: ubuntu-24.04 + outputs: + rust: ${{ steps.paths.outputs.rust }} + sdk: ${{ steps.paths.outputs.sdk }} + go: ${{ steps.paths.outputs.go }} + tooling: ${{ steps.paths.outputs.tooling }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: + fetch-depth: 0 persist-credentials: false - - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master - with: - toolchain: stable - - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - # Ubuntu 24.04 runners restrict unprivileged user namespaces with AppArmor, which is what the namespace - # sandbox (`ag-sandbox`) needs. Lift that, and make the sandbox suites *fail* instead of skipping if the - # runner still cannot run them: a green run must mean the isolation was actually exercised. - - name: Allow unprivileged user namespaces - if: runner.os == 'Linux' - run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || true - - name: Build - run: cargo build --workspace --all-targets - - name: Test - run: cargo test --workspace - env: - # Components this job must have: a test that needs one and finds it missing FAILS instead of skipping - # (crates/ag-testkit). The tutorials' commands run against this build of `ag` - # (crates/ag-cli/tests/docs_snippets.rs), so bash, python3 and git are needed; the real-server worker test - # needs the ag-server that `cargo build --workspace` just made. - AG_REQUIRE_COMPONENTS: ${{ runner.os == 'Linux' && 'namespace-sandbox,git,bash,python3,ag-server-binary' || 'git,bash,python3,ag-server-binary' }} - AG_SKIP_REPORT: ${{ runner.temp }}/skips.jsonl - # Every skip is listed; one that tests/skip-allowlist.toml does not cover for this job fails the job. - - name: Skipped tests - if: ${{ !cancelled() }} - run: python3 scripts/check_skips.py --job "ci/test-${{ runner.os == 'Linux' && 'ubuntu' || 'macos' }}" --report "$RUNNER_TEMP/skips.jsonl" - - lint: - name: fmt + clippy + rustdoc - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master - with: - toolchain: stable - components: rustfmt, clippy - - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - - run: cargo fmt --all --check - - run: cargo clippy --workspace --all-targets -- -D warnings - - run: cargo doc --workspace --no-deps - env: - RUSTDOCFLAGS: -D warnings - - msrv: - name: MSRV (1.89) - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - # Select the toolchain with the `toolchain` input rather than an `@1.89.0` ref: - # Dependabot would "update" such a ref and silently move the MSRV. - - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master - with: - toolchain: 1.89.0 - - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - - run: cargo check --workspace --all-targets - - docs-and-demo: - name: generated docs + demo - runs-on: ubuntu-latest + - id: paths + run: python3 scripts/ci_changes.py + + tooling: + name: workflow and release contracts + needs: changes + if: needs.changes.outputs.tooling == 'true' + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master - with: - toolchain: stable - - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - - name: Build release binary - run: cargo build --release -p agentgit-cli - - name: CLI reference is up to date - run: AG=target/release/ag python3 scripts/gen_cli_docs.py --check - - name: README artwork is up to date - run: python3 scripts/gen_readme_assets.py --check - - name: README demo still behaves as documented - run: AG=$PWD/target/release/ag ./examples/quickstart.sh - # Links, test names, `ag` commands, environment variables and the refusal messages quoted in the docs are checked - # against the code. --require-ag: a missing binary fails the step instead of skipping the command check. - - name: Docs match the code - run: AG=$PWD/target/release/ag python3 scripts/check_docs.py --require-ag - # No todo!(), TODO/FIXME/XXX/HACK, "for now", placeholder, stub, mock/fake/dummy or simulation in product source - # (tests excluded); the exceptions, each with a reason, are in scripts/no_stubs_allowlist.txt. - - name: No stubs or simulated code in product source + - run: python3 -m pip install --disable-pip-version-check PyYAML + - name: Offline pipeline and release safety tests run: | - python3 scripts/test_check_no_stubs.py + set -e + python3 scripts/check_workflows.py --strict + python3 scripts/security_exceptions.py check + python3 scripts/check_dependabot.py + python3 scripts/check_deploy.py python3 scripts/check_no_stubs.py - # No test skips by itself: the helper of its language, or an entry in tests/skip-allowlist.toml. - - name: Skip mechanism (tests of the helper and the gate, allow-list, source scan) - run: | - python3 scripts/test_ag_testkit.py - python3 scripts/test_check_skips.py - python3 scripts/check_skips.py --validate --scan + for test in test_ci_policy test_release_candidate test_request_release test_release_scripts test_check_workflows test_web_e2e_workflow test_check_dependabot test_py39_compat; do + python3 "scripts/$test.py" + done - backends: - name: PostgreSQL + S3 backends - runs-on: ubuntu-latest + rust: + name: Linux unit and storage contracts + needs: changes + if: needs.changes.outputs.rust == 'true' || needs.changes.outputs.sdk == 'true' || needs.changes.outputs.go == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 25 + env: + AG_SKIP_REPORT: /tmp/fast-rust-skips.jsonl steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -133,324 +73,128 @@ jobs: - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: toolchain: stable + components: clippy, rustfmt - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - # scripts/pg-test.sh finds the server binaries under /usr/lib/postgresql/*/bin - # (PostgreSQL ships on the runner image); scripts/s3-test.sh installs moto into a venv, or downloads the pinned - # SeaweedFS release (SHA-256 checked). - - name: Make sure the PostgreSQL server binaries are present - run: ls /usr/lib/postgresql/*/bin/initdb || sudo apt-get install -y --no-install-recommends postgresql - # Every suite on PostgreSQL metadata and S3 objects; a missing server fails the run - # instead of silently falling back to SQLite and files. The S3 server here is moto, a Python - # re-implementation of the API that recomputes every request's SigV4 signature. - # The whole workspace runs here, so the sandbox suites do too. Until the skip report existed this job did not - # lift the AppArmor restriction (see the test job) and every namespace-sandbox test silently skipped in it. - - name: Allow unprivileged user namespaces - run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || true - - name: Test on PostgreSQL + S3 (moto) - run: scripts/pg-test.sh scripts/s3-test.sh cargo test --workspace - env: - AG_REQUIRE_COMPONENTS: postgres,s3,namespace-sandbox - AG_SKIP_REPORT: ${{ runner.temp }}/skips-moto.jsonl - - name: Skipped tests (moto) - if: ${{ !cancelled() }} - run: python3 scripts/check_skips.py --job ci/backends-moto --report "$RUNNER_TEMP/skips-moto.jsonl" - # The same object-store suites against a real S3 server: SeaweedFS, the one the Compose stack runs, with a - # configured identity so that the gateway verifies every signature (the script refuses to run otherwise). - # Everything that moves its objects to S3 when AG_TEST_S3_ENDPOINT is set: the store's own suite - # (crates/ag-storage/tests/s3.rs), the engine's lifecycle/GC suites and the server's backend suite. - - name: Test on PostgreSQL + S3 (SeaweedFS, a real S3 server) - run: | - AG_S3_SERVER=seaweedfs scripts/pg-test.sh scripts/s3-test.sh \ - bash -c 'cargo test -p agentgit-storage -p agentgit-integrate && cargo test -p agentgit-server --test storage_backends' - env: - # SeaweedFS serves TLS with a private CA, so the TLS tests of the S3 client must run here - AG_REQUIRE_COMPONENTS: postgres,s3,s3-tls - AG_SKIP_REPORT: ${{ runner.temp }}/skips-seaweedfs.jsonl - - name: Skipped tests (SeaweedFS) - if: ${{ !cancelled() }} - run: python3 scripts/check_skips.py --job ci/backends-seaweedfs --report "$RUNNER_TEMP/skips-seaweedfs.jsonl" - # The multi-replica server (ADR-0007, docs/operations/high-availability.md): 2-5 real ag-server processes on one - # PostgreSQL and one S3 server, driven over sockets - no lost update across replicas, kill -9 of a replica in - # flight, exactly-once job claims, SSE across replicas, bounded revocation, rolling restart, migrate-control. - # The workspace run above executes these too (serialised inside the binary); this step names them in the log, and - # runs them once more against SeaweedFS, a real S3 server. - - name: Multi-replica ag-server on PostgreSQL + S3 (moto) - run: scripts/ha-test.sh - env: - AG_SKIP_REPORT: ${{ runner.temp }}/skips-ha.jsonl - - name: Multi-replica ag-server on PostgreSQL + S3 (SeaweedFS) - run: AG_S3_SERVER=seaweedfs scripts/ha-test.sh - env: - AG_SKIP_REPORT: ${{ runner.temp }}/skips-ha.jsonl - - name: Skipped tests (multi-replica) - if: ${{ !cancelled() }} - run: python3 scripts/check_skips.py --job ci/backends-ha --report "$RUNNER_TEMP/skips-ha.jsonl" - - deploy: - name: deployment files - runs-on: ubuntu-latest - env: - KUSTOMIZE_VERSION: v5.6.0 - KUBECONFORM_VERSION: v0.6.7 - PROMETHEUS_VERSION: 3.2.1 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - persist-credentials: false - - name: Install kustomize, kubeconform, promtool + shared-key: linux-debug + cache-workspace-crates: true + save-if: ${{ github.ref == 'refs/heads/main' }} + - name: Formatting and code lint + if: needs.changes.outputs.rust == 'true' + run: cargo fmt --all --check && cargo clippy --workspace --lib --bins --locked -- -D warnings + - name: Unit tests and core storage contracts + if: needs.changes.outputs.rust == 'true' run: | - set -euo pipefail - mkdir -p "$HOME/bin" - curl -sSL "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2F${KUSTOMIZE_VERSION}/kustomize_${KUSTOMIZE_VERSION}_linux_amd64.tar.gz" | tar xz -C "$HOME/bin" kustomize - curl -sSL "https://github.com/yannh/kubeconform/releases/download/${KUBECONFORM_VERSION}/kubeconform-linux-amd64.tar.gz" | tar xz -C "$HOME/bin" kubeconform - curl -sSL "https://github.com/prometheus/prometheus/releases/download/v${PROMETHEUS_VERSION}/prometheus-${PROMETHEUS_VERSION}.linux-amd64.tar.gz" | tar xz -C "$HOME/bin" --strip-components=1 "prometheus-${PROMETHEUS_VERSION}.linux-amd64/promtool" - echo "$HOME/bin" >> "$GITHUB_PATH" - - run: pip install pyyaml - - name: Manifests, alert rules, Compose file and dashboards - run: python3 scripts/check_deploy.py --strict - - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master - with: - toolchain: stable - - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - - name: Shipped server configurations load with this server - run: cargo test -p agentgit-server --test deploy_configs - - # The container images (both targets, health check, hardened run, end-to-end smoke, compose, kubernetes) are built - # and proven by containers.yml, which runs on every push and pull request that can change an image. They used to - # be built a second time here, with another cache scope and a weaker check. + cargo test --workspace --lib --bins --locked + cargo test -p agentgit-integrate -p agentgit-storage --test '*' --locked + cargo test -p agentgit-server --test audit_regressions --test review_scope --test workspace_creation --locked + - name: Check unexpected skips + if: ${{ !cancelled() && needs.changes.outputs.rust == 'true' }} + run: python3 scripts/check_skips.py --job ci/test-fast --report /tmp/fast-rust-skips.jsonl + - name: Build shared test tools once + if: needs.changes.outputs.sdk == 'true' || needs.changes.outputs.go == 'true' + run: | + cargo build -p agentgit-server -p agentgit-cli --locked + cd target/debug + sha256sum ag ag-server > binaries.sha256 + tar -czf ci-tools.tar.gz ag ag-server binaries.sha256 + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + if: needs.changes.outputs.sdk == 'true' || needs.changes.outputs.go == 'true' + with: + name: ci-tools + path: target/debug/ci-tools.tar.gz + compression-level: 0 + retention-days: 1 sdk: - name: client SDKs (TypeScript, Python) - runs-on: ubuntu-latest + name: SDK contracts on one supported runtime + needs: [changes, rust] + if: needs.changes.outputs.sdk == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 15 env: - AG_SKIP_REPORT: /tmp/ag-skips.jsonl # the runner context is not available in a job-level env - # Harnesses must discover the server even when the caller suppresses INFO logs. + AG_SERVER_BIN: ${{ github.workspace }}/target/debug/ag-server RUST_LOG: error + AG_SKIP_REPORT: /tmp/fast-sdk-skips.jsonl steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: - toolchain: stable - - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - - run: cargo build -p agentgit-server -p agentgit-cli + name: ci-tools + path: target/debug + - name: Restore checked binaries + working-directory: target/debug + run: tar -xzf ci-tools.tar.gz && sha256sum --check binaries.sha256 - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: 22 cache: npm cache-dependency-path: sdk/typescript/package-lock.json - - name: TypeScript SDK (generate:check, types, lint, tests against the real server, package) + - name: TypeScript SDK working-directory: sdk/typescript - run: npm ci && AG_SERVER_BIN=$GITHUB_WORKSPACE/target/debug/ag-server npm run check + run: npm ci && npm run check - uses: astral-sh/setup-uv@18d369b23da25a81e9458561248e1aafff6a70fa # v10.3.1 - - name: Python SDK (tests against the real server, ruff, mypy, generated code) + - name: Python SDK working-directory: sdk/python run: | uv sync --locked - AG_SERVER_BIN=$GITHUB_WORKSPACE/target/debug/ag-server uv run pytest + uv run pytest uv run ruff check . uv run ruff format --check . uv run mypy uv run python scripts/generate.py --check - - name: Code samples in the docs are the code that is tested - run: python3 sdk/tools/sync_doc_samples.py --check - - name: Skipped tests - if: ${{ !cancelled() }} - run: python3 scripts/check_skips.py --job ci/sdk --report /tmp/ag-skips.jsonl - - # The README of each SDK names the runtimes it supports; these legs are what makes that sentence true. The - # `sdk` job above runs the full checks on one interpreter; here only the test suites run on the others. - sdk-runtimes: - name: client SDK tests (${{ matrix.runtime }}) - runs-on: ubuntu-latest + - run: python3 sdk/tools/sync_doc_samples.py --check + - run: python3 scripts/check_skips.py --job ci/sdk --report /tmp/fast-sdk-skips.jsonl + + go: + name: Go executor contracts + needs: [changes, rust] + if: needs.changes.outputs.go == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 20 env: - AG_SKIP_REPORT: /tmp/ag-skips.jsonl # the runner context is not available in a job-level env - RUST_LOG: error - strategy: - fail-fast: false - matrix: - include: - - {runtime: "python 3.10", kind: python, version: "3.10"} - - {runtime: "python 3.11", kind: python, version: "3.11"} - - {runtime: "python 3.12", kind: python, version: "3.12"} - - {runtime: "python 3.13", kind: python, version: "3.13"} - - {runtime: "node 20", kind: node, version: "20"} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master - with: - toolchain: stable - - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - - run: cargo build -p agentgit-server - - if: matrix.kind == 'node' - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 - with: - node-version: ${{ matrix.version }} - cache: npm - cache-dependency-path: sdk/typescript/package-lock.json - - if: matrix.kind == 'node' - name: TypeScript SDK tests - working-directory: sdk/typescript - run: npm ci && AG_SERVER_BIN=$GITHUB_WORKSPACE/target/debug/ag-server npm test - - if: matrix.kind == 'python' - uses: astral-sh/setup-uv@18d369b23da25a81e9458561248e1aafff6a70fa # v10.3.1 - - if: matrix.kind == 'python' - name: Python SDK tests - working-directory: sdk/python - run: | - uv sync --locked --python ${{ matrix.version }} - AG_SERVER_BIN=$GITHUB_WORKSPACE/target/debug/ag-server uv run pytest - - name: Skipped tests - if: ${{ !cancelled() }} - run: python3 scripts/check_skips.py --job "ci/sdk-runtimes-${{ matrix.kind }}-${{ matrix.version }}" --report /tmp/ag-skips.jsonl - - orchestrator: - name: agent orchestrator (Go) - runs-on: ubuntu-latest - env: - AG_LLM_LIVE_KEY: ${{ secrets.OPENROUTER_API_KEY }} - AG_LLM_LIVE_MODEL: ${{ vars.AG_LLM_MODEL }} - # The integration suite runs against the real server: no binary is a failure, not a skip. The live-model tests - # skip here by design (tests/skip-allowlist.toml) and run in the live step. + AG_SERVER_BIN: ${{ github.workspace }}/target/debug/ag-server AG_REQUIRE_COMPONENTS: ag-server-binary - AG_SKIP_REPORT: /tmp/ag-skips.jsonl # the runner context is not available in a job-level env + AG_SKIP_REPORT: /tmp/fast-go-skips.jsonl steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: - toolchain: stable - - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - - run: cargo build -p agentgit-server + name: ci-tools + path: target/debug + - name: Restore checked binaries + working-directory: target/debug + run: tar -xzf ci-tools.tar.gz && sha256sum --check binaries.sha256 - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version-file: services/agent-orchestrator/go.mod cache-dependency-path: services/agent-orchestrator/go.sum - - name: vet, format, race tests (the integration suite runs a real ag-server) + - name: Format, vet and race tests working-directory: services/agent-orchestrator run: | - go vet ./... test -z "$(gofmt -l .)" - AG_SERVER_BIN=$GITHUB_WORKSPACE/target/debug/ag-server go test -race -count=1 ./... - - name: Skipped tests - if: ${{ !cancelled() }} - run: python3 scripts/check_skips.py --job ci/orchestrator --report /tmp/ag-skips.jsonl - # The model-backed advisor, reviewer and index judge against a real endpoint: opt-in (it costs money). Runs on a - # push only when the repository defines the OPENROUTER_API_KEY secret and the AG_LLM_MODEL variable. - - name: model-backed components against a real model (only when a key and a model are configured) - if: ${{ github.event_name == 'push' && env.AG_LLM_LIVE_KEY != '' && env.AG_LLM_LIVE_MODEL != '' }} - working-directory: services/agent-orchestrator - env: - AG_LLM_LIVE: "1" - AG_LLM_LIVE_KEY_ENV: AG_LLM_LIVE_KEY - AG_REQUIRE_COMPONENTS: ag-server-binary,model - AG_SKIP_REPORT: ${{ runner.temp }}/skips-live.jsonl - run: AG_SERVER_BIN=$GITHUB_WORKSPACE/target/debug/ag-server go test -count=1 -v -run 'Live' ./internal/llm ./internal/advisor ./internal/review ./internal/semindex ./internal/app - - name: Skipped tests (live) - if: ${{ github.event_name == 'push' && env.AG_LLM_LIVE_KEY != '' && env.AG_LLM_LIVE_MODEL != '' && !cancelled() }} - run: python3 scripts/check_skips.py --job ci/orchestrator-live --report "$RUNNER_TEMP/skips-live.jsonl" - # The worked example of docs/guides/orchestrator.md is a script: it must keep running against a real ag-server - # (it builds the demo agent, files tasks, runs the orchestrator, plays the people, checks policy refusals). - - name: the worked example runs end to end - working-directory: services/agent-orchestrator - run: | - go build -o "$RUNNER_TEMP/orchestrator" ./cmd/orchestrator - ORCHESTRATOR_BIN="$RUNNER_TEMP/orchestrator" AG_SERVER_BIN="$GITHUB_WORKSPACE/target/debug/ag-server" sh examples/worked-example.sh | tee "$RUNNER_TEMP/worked-example.txt" - grep -q "orchestrator stopped" "$RUNNER_TEMP/worked-example.txt" - # The shipped image: builds, carries the version it was given, and reads the example configuration. - - name: the image builds and starts - run: | - docker build --build-arg VERSION="ci-${GITHUB_SHA}" -t agentgit/agent-orchestrator:ci services/agent-orchestrator - docker run --rm agentgit/agent-orchestrator:ci version | grep -q "ci-${GITHUB_SHA}" - docker run --rm -v "$GITHUB_WORKSPACE/services/agent-orchestrator/examples:/etc/orchestrator:ro" agentgit/agent-orchestrator:ci config -config /etc/orchestrator/orchestrator.yaml | grep -q "docs-bot" - - # The completion scripts must parse in the shells they are for. A missing shell fails the run (AG_REQUIRE_COMPONENTS=shell) - # instead of being skipped; PowerShell is on the runner image, zsh and fish are installed here. - shells: - name: shell completions (bash, zsh, fish, PowerShell) - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master - with: - toolchain: stable - - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - - name: Install zsh and fish - run: sudo apt-get update && sudo apt-get install -y --no-install-recommends zsh fish - - name: The scripts parse; bash completes real commands; the shipped copies are current - run: cargo test -p agentgit-cli --test completions - env: - AG_REQUIRE_COMPONENTS: shell,bash,git - AG_SKIP_REPORT: ${{ runner.temp }}/skips.jsonl - - name: Skipped tests - if: ${{ !cancelled() }} - run: python3 scripts/check_skips.py --job ci/shells --report "$RUNNER_TEMP/skips.jsonl" - - # ag-mcp with an agent in front of it: the scripted client (no model) runs on every push; the run with a real - # model is opt-in (it costs money and CI has no key unless the repository defines the OPENROUTER_API_KEY secret - # and the AG_LLM_MODEL variable; forks and pull requests from forks have neither, so the step is skipped). - mcp: - name: MCP server with an agent (scripted; live model opt-in) - runs-on: ubuntu-latest - env: - AG_LLM_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - AG_LLM_MODEL: ${{ vars.AG_LLM_MODEL }} + go vet ./... + go test -race ./... + - run: python3 scripts/check_skips.py --job ci/orchestrator --report /tmp/fast-go-skips.jsonl + + complete: + name: CI + if: always() + needs: [changes, tooling, rust, sdk, go] + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master - with: - toolchain: stable - - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - - run: cargo build -p agentgit-server -p agentgit-mcp - - name: The examples compile - run: python3 -m py_compile examples/agents/*.py - - name: Scripted agent against a real ag-server (claim, lease, edit, propose, refusals, human integrates) - run: python3 examples/agents/mcp_scripted_agent.py --ag-server target/debug/ag-server --ag-mcp target/debug/ag-mcp - - name: Real model against ag-mcp (only when a key and a model are configured) - if: ${{ env.AG_LLM_API_KEY != '' && env.AG_LLM_MODEL != '' && github.event_name == 'push' }} + - name: Reject failed or cancelled required jobs env: - AG_LIVE_LLM: "1" - run: python3 examples/agents/mcp_llm_agent.py --ag-server target/debug/ag-server --ag-mcp target/debug/ag-mcp - - # The lightweight scripts must keep running on an older Python (the reviewer's laptop had 3.9, and - # `str | None` annotations broke scripts/e2e_smoke.py there). The interpreter comes from uv (already pinned - # above); the scripts that read TOML (tomllib, 3.11+) are tested by the jobs that need them. - scripts-py39: - name: lightweight scripts on Python 3.9 - runs-on: ubuntu-latest - env: - AG_REQUIRE_COMPONENTS: dash - AG_SKIP_REPORT: /tmp/ag-skips.jsonl # the runner context is not available in a job-level env - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - uses: astral-sh/setup-uv@18d369b23da25a81e9458561248e1aafff6a70fa # v10.3.1 - - name: Python 3.9 with PyYAML - run: | - uv venv --python 3.9 "$RUNNER_TEMP/py39" - uv pip install --python "$RUNNER_TEMP/py39/bin/python" pyyaml - "$RUNNER_TEMP/py39/bin/python" --version - - name: Unit tests of the scripts - run: | - set -e - for t in test_py39_compat test_ag_testkit test_check_skips test_check_no_stubs test_doc_snippets \ - test_mkdocs_hooks test_release_scripts test_sbom_release test_sbom_npm test_codeql_summary \ - test_check_workflows test_web_e2e_workflow test_request_release test_check_dependabot; do - echo "== $t" - "$RUNNER_TEMP/py39/bin/python" "scripts/$t.py" - done - - name: The allow-list and the source scan + RESULTS: ${{ toJSON(needs) }} run: | - "$RUNNER_TEMP/py39/bin/python" scripts/check_skips.py --validate --scan - - name: Skipped tests - if: ${{ !cancelled() }} - run: python3 scripts/check_skips.py --job ci/scripts-py39 --report /tmp/ag-skips.jsonl + python3 - <<'PYTHON' + import json, os + results = json.loads(os.environ['RESULTS']) + bad = [name for name, job in results.items() if job['result'] not in ('success', 'skipped')] + if results['changes']['result'] != 'success' or bad: + raise SystemExit('CI failed: ' + ', '.join(bad)) + PYTHON diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 444343c7..f8c71674 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -1,15 +1,8 @@ name: CodeQL -# Static analysis of the code we write, in the four languages of the repository. The findings appear under -# Security -> Code scanning; the job log prints a summary of what the analysis found, so a run can be read without -# the Security tab. What runs where and how to triage an alert: docs/operations/supply-chain.md. - +# Static analysis runs in qualification; regular edits use fast component CI. on: - push: - branches: [main] - pull_request: - schedule: - - cron: "31 3 * * 2" + workflow_call: workflow_dispatch: permissions: @@ -17,7 +10,7 @@ permissions: concurrency: group: codeql-${{ github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + cancel-in-progress: true jobs: analyze: @@ -29,6 +22,7 @@ jobs: security-events: write # upload the SARIF results actions: read strategy: + max-parallel: 2 fail-fast: false matrix: include: diff --git a/.github/workflows/containers.yml b/.github/workflows/containers.yml index 795ae94f..0556fe94 100644 --- a/.github/workflows/containers.yml +++ b/.github/workflows/containers.yml @@ -1,60 +1,8 @@ name: Containers -# Proves that "install it and run it" works, with the files in deploy/ and not just with their syntax: -# -# image builds deploy/docker/Dockerfile (buildx + the gha layer cache; the `ui` stage builds the Web UI that -# ag-server embeds), checks size, user, health check and `--version`, runs it hardened (read-only -# root, no capabilities), checks that it serves the Web UI, and drives the end-to-end smoke -# (scripts/e2e_smoke.py) against it; its artifacts feed the next two jobs -# -# This is the only place images are built on push/PR: ci.yml used to repeat the build in an `images` job (a second -# cache scope and the same two builds), which is why it is gone. The path filters below are what the image is -# made of, so a change that cannot alter an image does not build one. -# cache a fresh runner rebuilds from the gha cache and must find the expensive layers CACHED -# compose `docker compose up` of deploy/compose (server + PostgreSQL + SeaweedFS (S3)), the same smoke against it -# (repository metadata in PostgreSQL, objects in an S3 bucket), a restart, then the observability profile -# kubernetes a kind cluster, deploy/kubernetes (base through the ci overlay), the same smoke through -# `kubectl port-forward`, a rollout restart -# -# The smoke is a real integration: create a repository, grant roles, mint credentials, `ag clone`, snapshot, -# `ag push`, integrate through the server, read the result back over the HTTP API, and assert the metrics. - +# Container build, cache and deployment smoke checks for qualification or explicit runs. on: - push: - branches: [main] - paths: - # A release request requires this workflow on the exact main commit. - - "CHANGELOG.md" - - "deploy/**" - - "crates/**" - - "proto/**" - - "migrations/**" - - "apps/web/**" - - "scripts/build_ui.sh" - - "Cargo.toml" - - "Cargo.lock" - - "LICENSE" - - "NOTICE" - - ".dockerignore" - - "scripts/e2e_*.py" - - "scripts/check_buildx_cache.py" - - ".github/workflows/containers.yml" - pull_request: - paths: - - "deploy/**" - - "crates/**" - - "proto/**" - - "migrations/**" - - "apps/web/**" - - "scripts/build_ui.sh" - - "Cargo.toml" - - "Cargo.lock" - - "LICENSE" - - "NOTICE" - - ".dockerignore" - - "scripts/e2e_*.py" - - "scripts/check_buildx_cache.py" - - ".github/workflows/containers.yml" + workflow_call: workflow_dispatch: permissions: diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index affd4d34..0bd0af40 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -1,28 +1,28 @@ name: Docs -# The documentation site (MkDocs Material, docs/ is the source of truth), the tutorials' commands run against the real -# binaries, and the deploy to GitHub Pages. -# -# site builds the site with `mkdocs build --strict` on every push and pull request: a broken link, a broken -# #anchor or a page missing from the navigation fails the build. -# snippets builds `ag` and `ag-server` and runs every command of the tutorials (scripts/doc_snippets.py), checks -# that the output the pages show is what the commands print, and that the policy documents of the -# cookbook go through the real parser. -# deploy publishes the site from main to GitHub Pages, but only when Pages is enabled for the repository -# (Settings, Pages, Source: GitHub Actions). When it is not, the job says so and succeeds: the build above -# is still checked, nothing is published. It never runs for pull requests. - +# Fast site checks/deployment for documentation edits; executable tutorials run during qualification. on: push: branches: [main] + paths: ["docs/**", "README*", "ROADMAP.md", "SECURITY.md", "CONTRIBUTING.md", "mkdocs.yml", ".github/workflows/docs.yml"] pull_request: + paths: ["docs/**", "README*", "ROADMAP.md", "SECURITY.md", "CONTRIBUTING.md", "mkdocs.yml", ".github/workflows/docs.yml"] workflow_dispatch: + inputs: + full: + type: boolean + default: true + workflow_call: + inputs: + full: + type: boolean + default: true permissions: contents: read concurrency: - group: docs-${{ github.ref }} + group: docs-${{ github.ref }}-${{ inputs.full && 'full' || 'fast' }} cancel-in-progress: true env: @@ -60,6 +60,7 @@ jobs: retention-days: 7 snippets: + if: inputs.full name: tutorials run against the real binaries runs-on: ubuntu-latest steps: @@ -70,6 +71,8 @@ jobs: with: toolchain: stable - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + cache-workspace-crates: true - name: Build ag and ag-server run: cargo build -p agentgit-cli -p agentgit-server - name: The harness has tests of its own @@ -81,8 +84,8 @@ jobs: deploy: name: deploy to GitHub Pages - needs: [site, snippets] - if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' + needs: site + if: github.event_name == 'push' && github.ref == 'refs/heads/main' runs-on: ubuntu-latest permissions: contents: read diff --git a/.github/workflows/full-ci.yml b/.github/workflows/full-ci.yml new file mode 100644 index 00000000..9a10755d --- /dev/null +++ b/.github/workflows/full-ci.yml @@ -0,0 +1,515 @@ +name: Full CI + +on: + workflow_call: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: full-ci-${{ github.ref }} + cancel-in-progress: true + +env: + CARGO_TERM_COLOR: always + CARGO_INCREMENTAL: "0" + CARGO_PROFILE_DEV_DEBUG: "0" + CARGO_PROFILE_TEST_DEBUG: "0" + +jobs: + test: + name: test (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + max-parallel: 2 + fail-fast: false + matrix: + os: [macos-latest] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + with: + toolchain: stable + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + cache-workspace-crates: true + shared-key: linux-debug + # Ubuntu 24.04 runners restrict unprivileged user namespaces with AppArmor, which is what the namespace + # sandbox (`ag-sandbox`) needs. Lift that, and make the sandbox suites *fail* instead of skipping if the + # runner still cannot run them: a green run must mean the isolation was actually exercised. + - name: Allow unprivileged user namespaces + if: runner.os == 'Linux' + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || true + - name: Build + run: cargo build --workspace + - name: Test + run: cargo test --workspace + env: + # Components this job must have: a test that needs one and finds it missing FAILS instead of skipping + # (crates/ag-testkit). The tutorials' commands run against this build of `ag` + # (crates/ag-cli/tests/docs_snippets.rs), so bash, python3 and git are needed; the real-server worker test + # needs the ag-server that `cargo build --workspace` just made. + AG_REQUIRE_COMPONENTS: ${{ runner.os == 'Linux' && 'namespace-sandbox,git,bash,python3,ag-server-binary' || 'git,bash,python3,ag-server-binary' }} + AG_SKIP_REPORT: ${{ runner.temp }}/skips.jsonl + # Every skip is listed; one that tests/skip-allowlist.toml does not cover for this job fails the job. + - name: Skipped tests + if: ${{ !cancelled() }} + run: python3 scripts/check_skips.py --job "ci/test-${{ runner.os == 'Linux' && 'ubuntu' || 'macos' }}" --report "$RUNNER_TEMP/skips.jsonl" + + lint: + name: fmt + clippy + rustdoc + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + with: + toolchain: stable + components: rustfmt, clippy + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + cache-workspace-crates: true + shared-key: linux-debug + - run: cargo fmt --all --check + - run: cargo clippy --workspace --all-targets -- -D warnings + - run: cargo doc --workspace --no-deps + env: + RUSTDOCFLAGS: -D warnings + + msrv: + name: MSRV (1.89) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + # Select the toolchain with the `toolchain` input rather than an `@1.89.0` ref: + # Dependabot would "update" such a ref and silently move the MSRV. + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + with: + toolchain: 1.89.0 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + cache-workspace-crates: true + shared-key: linux-debug + - run: cargo check --workspace --all-targets + + docs-and-demo: + name: generated docs + demo + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + with: + toolchain: stable + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + cache-workspace-crates: true + shared-key: linux-debug + - name: Build release binary + run: cargo build --release -p agentgit-cli + - name: CLI reference is up to date + run: AG=target/release/ag python3 scripts/gen_cli_docs.py --check + - name: README artwork is up to date + run: python3 scripts/gen_readme_assets.py --check + - name: README demo still behaves as documented + run: AG=$PWD/target/release/ag ./examples/quickstart.sh + # Links, test names, `ag` commands, environment variables and the refusal messages quoted in the docs are checked + # against the code. --require-ag: a missing binary fails the step instead of skipping the command check. + - name: Docs match the code + run: AG=$PWD/target/release/ag python3 scripts/check_docs.py --require-ag + # No todo!(), TODO/FIXME/XXX/HACK, "for now", placeholder, stub, mock/fake/dummy or simulation in product source + # (tests excluded); the exceptions, each with a reason, are in scripts/no_stubs_allowlist.txt. + - name: No stubs or simulated code in product source + run: | + python3 scripts/test_check_no_stubs.py + python3 scripts/check_no_stubs.py + # No test skips by itself: the helper of its language, or an entry in tests/skip-allowlist.toml. + - name: Skip mechanism (tests of the helper and the gate, allow-list, source scan) + run: | + python3 scripts/test_ag_testkit.py + python3 scripts/test_check_skips.py + python3 scripts/check_skips.py --validate --scan + + backends: + name: PostgreSQL + S3 backends + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + with: + toolchain: stable + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + cache-workspace-crates: true + shared-key: linux-debug + # scripts/pg-test.sh finds the server binaries under /usr/lib/postgresql/*/bin + # (PostgreSQL ships on the runner image); scripts/s3-test.sh installs moto into a venv, or downloads the pinned + # SeaweedFS release (SHA-256 checked). + - name: Make sure the PostgreSQL server binaries are present + run: ls /usr/lib/postgresql/*/bin/initdb || sudo apt-get install -y --no-install-recommends postgresql + # Every suite on PostgreSQL metadata and S3 objects; a missing server fails the run + # instead of silently falling back to SQLite and files. The S3 server here is moto, a Python + # re-implementation of the API that recomputes every request's SigV4 signature. + # The whole workspace runs here, so the sandbox suites do too. Until the skip report existed this job did not + # lift the AppArmor restriction (see the test job) and every namespace-sandbox test silently skipped in it. + - name: Allow unprivileged user namespaces + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || true + - name: Test on PostgreSQL + S3 (moto) + run: scripts/pg-test.sh scripts/s3-test.sh cargo test --workspace + env: + AG_REQUIRE_COMPONENTS: postgres,s3,namespace-sandbox + AG_SKIP_REPORT: ${{ runner.temp }}/skips-moto.jsonl + - name: Skipped tests (moto) + if: ${{ !cancelled() }} + run: python3 scripts/check_skips.py --job ci/backends-moto --report "$RUNNER_TEMP/skips-moto.jsonl" + # The same object-store suites against a real S3 server: SeaweedFS, the one the Compose stack runs, with a + # configured identity so that the gateway verifies every signature (the script refuses to run otherwise). + # Everything that moves its objects to S3 when AG_TEST_S3_ENDPOINT is set: the store's own suite + # (crates/ag-storage/tests/s3.rs), the engine's lifecycle/GC suites and the server's backend suite. + - name: Test on PostgreSQL + S3 (SeaweedFS, a real S3 server) + run: | + AG_S3_SERVER=seaweedfs scripts/pg-test.sh scripts/s3-test.sh \ + bash -c 'cargo test -p agentgit-storage -p agentgit-integrate && cargo test -p agentgit-server --test storage_backends' + env: + # SeaweedFS serves TLS with a private CA, so the TLS tests of the S3 client must run here + AG_REQUIRE_COMPONENTS: postgres,s3,s3-tls + AG_SKIP_REPORT: ${{ runner.temp }}/skips-seaweedfs.jsonl + - name: Skipped tests (SeaweedFS) + if: ${{ !cancelled() }} + run: python3 scripts/check_skips.py --job ci/backends-seaweedfs --report "$RUNNER_TEMP/skips-seaweedfs.jsonl" + # The multi-replica server (ADR-0007, docs/operations/high-availability.md): 2-5 real ag-server processes on one + # PostgreSQL and one S3 server, driven over sockets - no lost update across replicas, kill -9 of a replica in + # flight, exactly-once job claims, SSE across replicas, bounded revocation, rolling restart, migrate-control. + # The workspace run above executes these too (serialised inside the binary); this step names them in the log, and + # runs them once more against SeaweedFS, a real S3 server. + - name: Multi-replica ag-server on PostgreSQL + S3 (SeaweedFS) + run: AG_S3_SERVER=seaweedfs scripts/ha-test.sh + env: + AG_SKIP_REPORT: ${{ runner.temp }}/skips-ha.jsonl + - name: Skipped tests (multi-replica) + if: ${{ !cancelled() }} + run: python3 scripts/check_skips.py --job ci/backends-ha --report "$RUNNER_TEMP/skips-ha.jsonl" + + deploy: + name: deployment files + runs-on: ubuntu-latest + env: + KUSTOMIZE_VERSION: v5.6.0 + KUBECONFORM_VERSION: v0.6.7 + PROMETHEUS_VERSION: 3.2.1 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Install kustomize, kubeconform, promtool + run: | + set -euo pipefail + mkdir -p "$HOME/bin" + curl -sSL "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2F${KUSTOMIZE_VERSION}/kustomize_${KUSTOMIZE_VERSION}_linux_amd64.tar.gz" | tar xz -C "$HOME/bin" kustomize + curl -sSL "https://github.com/yannh/kubeconform/releases/download/${KUBECONFORM_VERSION}/kubeconform-linux-amd64.tar.gz" | tar xz -C "$HOME/bin" kubeconform + curl -sSL "https://github.com/prometheus/prometheus/releases/download/v${PROMETHEUS_VERSION}/prometheus-${PROMETHEUS_VERSION}.linux-amd64.tar.gz" | tar xz -C "$HOME/bin" --strip-components=1 "prometheus-${PROMETHEUS_VERSION}.linux-amd64/promtool" + echo "$HOME/bin" >> "$GITHUB_PATH" + - run: pip install pyyaml + - name: Manifests, alert rules, Compose file and dashboards + run: python3 scripts/check_deploy.py --strict + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + with: + toolchain: stable + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + cache-workspace-crates: true + shared-key: linux-debug + - name: Shipped server configurations load with this server + run: cargo test -p agentgit-server --test deploy_configs + + # The container images (both targets, health check, hardened run, end-to-end smoke, compose, kubernetes) are built + # and proven by containers.yml, which runs on every push and pull request that can change an image. They used to + # be built a second time here, with another cache scope and a weaker check. + + tools: + name: shared server and agent binaries + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + with: + toolchain: stable + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + shared-key: linux-debug + cache-workspace-crates: true + - name: Build shared tools + run: | + cargo build --locked -p agentgit-server -p agentgit-cli -p agentgit-mcp + cd target/debug + sha256sum ag-server ag ag-mcp > binaries.sha256 + tar -czf ci-tools.tar.gz ag-server ag ag-mcp binaries.sha256 + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: full-ci-tools + path: target/debug/ci-tools.tar.gz + compression-level: 0 + retention-days: 1 + + sdk: + needs: tools + name: client SDKs (TypeScript, Python) + runs-on: ubuntu-latest + env: + AG_SKIP_REPORT: /tmp/ag-skips.jsonl # the runner context is not available in a job-level env + # Harnesses must discover the server even when the caller suppresses INFO logs. + RUST_LOG: error + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: full-ci-tools + path: target/debug + - name: Restore checked tools + working-directory: target/debug + run: tar -xzf ci-tools.tar.gz && sha256sum --check binaries.sha256 + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: 22 + cache: npm + cache-dependency-path: sdk/typescript/package-lock.json + - name: TypeScript SDK (generate:check, types, lint, tests against the real server, package) + working-directory: sdk/typescript + run: npm ci && AG_SERVER_BIN=$GITHUB_WORKSPACE/target/debug/ag-server npm run check + - uses: astral-sh/setup-uv@18d369b23da25a81e9458561248e1aafff6a70fa # v10.3.1 + - name: Python SDK (tests against the real server, ruff, mypy, generated code) + working-directory: sdk/python + run: | + uv sync --locked + AG_SERVER_BIN=$GITHUB_WORKSPACE/target/debug/ag-server uv run pytest + uv run ruff check . + uv run ruff format --check . + uv run mypy + uv run python scripts/generate.py --check + - name: Code samples in the docs are the code that is tested + run: python3 sdk/tools/sync_doc_samples.py --check + - name: Skipped tests + if: ${{ !cancelled() }} + run: python3 scripts/check_skips.py --job ci/sdk --report /tmp/ag-skips.jsonl + + # The README of each SDK names the runtimes it supports; these legs are what makes that sentence true. The + # `sdk` job above runs the full checks on one interpreter; here only the test suites run on the others. + sdk-runtimes: + needs: tools + name: client SDK tests (${{ matrix.runtime }}) + runs-on: ubuntu-latest + env: + AG_SKIP_REPORT: /tmp/ag-skips.jsonl # the runner context is not available in a job-level env + RUST_LOG: error + strategy: + max-parallel: 2 + fail-fast: false + matrix: + include: + - {runtime: "python 3.10", kind: python, version: "3.10"} + - {runtime: "python 3.11", kind: python, version: "3.11"} + - {runtime: "python 3.12", kind: python, version: "3.12"} + - {runtime: "python 3.13", kind: python, version: "3.13"} + - {runtime: "node 20", kind: node, version: "20"} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: full-ci-tools + path: target/debug + - name: Restore checked tools + working-directory: target/debug + run: tar -xzf ci-tools.tar.gz && sha256sum --check binaries.sha256 + - if: matrix.kind == 'node' + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: ${{ matrix.version }} + cache: npm + cache-dependency-path: sdk/typescript/package-lock.json + - if: matrix.kind == 'node' + name: TypeScript SDK tests + working-directory: sdk/typescript + run: npm ci && AG_SERVER_BIN=$GITHUB_WORKSPACE/target/debug/ag-server npm test + - if: matrix.kind == 'python' + uses: astral-sh/setup-uv@18d369b23da25a81e9458561248e1aafff6a70fa # v10.3.1 + - if: matrix.kind == 'python' + name: Python SDK tests + working-directory: sdk/python + run: | + uv sync --locked --python ${{ matrix.version }} + AG_SERVER_BIN=$GITHUB_WORKSPACE/target/debug/ag-server uv run pytest + - name: Skipped tests + if: ${{ !cancelled() }} + run: python3 scripts/check_skips.py --job "ci/sdk-runtimes-${{ matrix.kind }}-${{ matrix.version }}" --report /tmp/ag-skips.jsonl + + orchestrator: + needs: tools + name: agent orchestrator (Go) + runs-on: ubuntu-latest + env: + AG_LLM_LIVE_KEY: ${{ secrets.OPENROUTER_API_KEY }} + AG_LLM_LIVE_MODEL: ${{ vars.AG_LLM_MODEL }} + # The integration suite runs against the real server: no binary is a failure, not a skip. The live-model tests + # skip here by design (tests/skip-allowlist.toml) and run in the live step. + AG_REQUIRE_COMPONENTS: ag-server-binary + AG_SKIP_REPORT: /tmp/ag-skips.jsonl # the runner context is not available in a job-level env + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: full-ci-tools + path: target/debug + - name: Restore checked tools + working-directory: target/debug + run: tar -xzf ci-tools.tar.gz && sha256sum --check binaries.sha256 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: services/agent-orchestrator/go.mod + cache-dependency-path: services/agent-orchestrator/go.sum + - name: vet, format, race tests (the integration suite runs a real ag-server) + working-directory: services/agent-orchestrator + run: | + go vet ./... + test -z "$(gofmt -l .)" + AG_SERVER_BIN=$GITHUB_WORKSPACE/target/debug/ag-server go test -race -count=1 ./... + - name: Skipped tests + if: ${{ !cancelled() }} + run: python3 scripts/check_skips.py --job ci/orchestrator --report /tmp/ag-skips.jsonl + # The model-backed advisor, reviewer and index judge against a real endpoint: opt-in (it costs money). Runs on a + # push only when the repository defines the OPENROUTER_API_KEY secret and the AG_LLM_MODEL variable. + - name: model-backed components against a real model (only when a key and a model are configured) + if: ${{ github.ref == 'refs/heads/main' && env.AG_LLM_LIVE_KEY != '' && env.AG_LLM_LIVE_MODEL != '' }} + working-directory: services/agent-orchestrator + env: + AG_LLM_LIVE: "1" + AG_LLM_LIVE_KEY_ENV: AG_LLM_LIVE_KEY + AG_REQUIRE_COMPONENTS: ag-server-binary,model + AG_SKIP_REPORT: ${{ runner.temp }}/skips-live.jsonl + run: AG_SERVER_BIN=$GITHUB_WORKSPACE/target/debug/ag-server go test -count=1 -v -run 'Live' ./internal/llm ./internal/advisor ./internal/review ./internal/semindex ./internal/app + - name: Skipped tests (live) + if: ${{ github.ref == 'refs/heads/main' && env.AG_LLM_LIVE_KEY != '' && env.AG_LLM_LIVE_MODEL != '' && !cancelled() }} + run: python3 scripts/check_skips.py --job ci/orchestrator-live --report "$RUNNER_TEMP/skips-live.jsonl" + # The worked example of docs/guides/orchestrator.md is a script: it must keep running against a real ag-server + # (it builds the demo agent, files tasks, runs the orchestrator, plays the people, checks policy refusals). + - name: the worked example runs end to end + working-directory: services/agent-orchestrator + run: | + go build -o "$RUNNER_TEMP/orchestrator" ./cmd/orchestrator + ORCHESTRATOR_BIN="$RUNNER_TEMP/orchestrator" AG_SERVER_BIN="$GITHUB_WORKSPACE/target/debug/ag-server" sh examples/worked-example.sh | tee "$RUNNER_TEMP/worked-example.txt" + grep -q "orchestrator stopped" "$RUNNER_TEMP/worked-example.txt" + # The shipped image: builds, carries the version it was given, and reads the example configuration. + - name: the image builds and starts + run: | + docker build --build-arg VERSION="ci-${GITHUB_SHA}" -t agentgit/agent-orchestrator:ci services/agent-orchestrator + docker run --rm agentgit/agent-orchestrator:ci version | grep -q "ci-${GITHUB_SHA}" + docker run --rm -v "$GITHUB_WORKSPACE/services/agent-orchestrator/examples:/etc/orchestrator:ro" agentgit/agent-orchestrator:ci config -config /etc/orchestrator/orchestrator.yaml | grep -q "docs-bot" + + # The completion scripts must parse in the shells they are for. A missing shell fails the run (AG_REQUIRE_COMPONENTS=shell) + # instead of being skipped; PowerShell is on the runner image, zsh and fish are installed here. + shells: + name: shell completions (bash, zsh, fish, PowerShell) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + with: + toolchain: stable + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + cache-workspace-crates: true + shared-key: linux-debug + - name: Install zsh and fish + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends zsh fish + - name: The scripts parse; bash completes real commands; the shipped copies are current + run: cargo test -p agentgit-cli --test completions + env: + AG_REQUIRE_COMPONENTS: shell,bash,git + AG_SKIP_REPORT: ${{ runner.temp }}/skips.jsonl + - name: Skipped tests + if: ${{ !cancelled() }} + run: python3 scripts/check_skips.py --job ci/shells --report "$RUNNER_TEMP/skips.jsonl" + + # ag-mcp with an agent in front of it: the scripted client (no model) runs on every push; the run with a real + # model is opt-in (it costs money and CI has no key unless the repository defines the OPENROUTER_API_KEY secret + # and the AG_LLM_MODEL variable; forks and pull requests from forks have neither, so the step is skipped). + mcp: + needs: tools + name: MCP server with an agent (scripted; live model opt-in) + runs-on: ubuntu-latest + env: + AG_LLM_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + AG_LLM_MODEL: ${{ vars.AG_LLM_MODEL }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: full-ci-tools + path: target/debug + - name: Restore checked tools + working-directory: target/debug + run: tar -xzf ci-tools.tar.gz && sha256sum --check binaries.sha256 + - name: The examples compile + run: python3 -m py_compile examples/agents/*.py + - name: Scripted agent against a real ag-server (claim, lease, edit, propose, refusals, human integrates) + run: python3 examples/agents/mcp_scripted_agent.py --ag-server target/debug/ag-server --ag-mcp target/debug/ag-mcp + - name: Real model against ag-mcp (only when a key and a model are configured) + if: ${{ env.AG_LLM_API_KEY != '' && env.AG_LLM_MODEL != '' && github.event_name == 'push' }} + env: + AG_LIVE_LLM: "1" + run: python3 examples/agents/mcp_llm_agent.py --ag-server target/debug/ag-server --ag-mcp target/debug/ag-mcp + + # The lightweight scripts must keep running on an older Python (the reviewer's laptop had 3.9, and + # `str | None` annotations broke scripts/e2e_smoke.py there). The interpreter comes from uv (already pinned + # above); the scripts that read TOML (tomllib, 3.11+) are tested by the jobs that need them. + scripts-py39: + name: lightweight scripts on Python 3.9 + runs-on: ubuntu-latest + env: + AG_REQUIRE_COMPONENTS: dash + AG_SKIP_REPORT: /tmp/ag-skips.jsonl # the runner context is not available in a job-level env + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@18d369b23da25a81e9458561248e1aafff6a70fa # v10.3.1 + - name: Python 3.9 with PyYAML + run: | + uv venv --python 3.9 "$RUNNER_TEMP/py39" + uv pip install --python "$RUNNER_TEMP/py39/bin/python" pyyaml + "$RUNNER_TEMP/py39/bin/python" --version + - name: Unit tests of the scripts + run: | + set -e + for t in test_py39_compat test_ag_testkit test_check_skips test_check_no_stubs test_doc_snippets \ + test_mkdocs_hooks test_ci_policy test_release_candidate test_release_scripts test_sbom_release test_sbom_npm test_codeql_summary \ + test_check_workflows test_web_e2e_workflow test_request_release test_check_dependabot; do + echo "== $t" + "$RUNNER_TEMP/py39/bin/python" "scripts/$t.py" + done + - name: The allow-list and the source scan + run: | + "$RUNNER_TEMP/py39/bin/python" scripts/check_skips.py --validate --scan + - name: Skipped tests + if: ${{ !cancelled() }} + run: python3 scripts/check_skips.py --job ci/scripts-py39 --report /tmp/ag-skips.jsonl diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index 33044caa..d3e8fc73 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -1,67 +1,12 @@ name: Fuzz -# libFuzzer targets for every parser and decoder that touches untrusted bytes (fuzz/; docs/architecture/security.md -# "Fuzzing"). What runs where: -# -# pull_request / push to main (when the fuzzed crates, the protocol or fuzz/ change) -# build every target once with the address sanitizer, then run each for 60 s from the checked-in seeds plus -# the corpus the nightly run cached. A smoke test: it catches a regression that a seed or a recently -# found input reaches, not a new bug that needs hours. -# schedule (every night) and workflow_dispatch -# every target for 15 minutes (or `seconds`), four jobs in parallel, starting from the corpus of the last night; -# the corpus is minimised and cached for the next run. A crash, a hang over 20 s or 2 GB of memory fails the -# job and uploads the reproducer (fuzz/artifacts) as a workflow artifact. -# -# Reproduce a failure: cargo +nightly fuzz run -s address --features heavy - +# Bounded fuzz qualification plus separate extended nightly/manual campaigns. on: - push: - branches: [main] - paths: - - "fuzz/**" - - "proto/**" - - "Cargo.lock" - - "crates/ag-types/**" - - "crates/ag-crypto/**" - - "crates/ag-object/**" - - "crates/ag-changeset/**" - - "crates/ag-dag/**" - - "crates/ag-diff/**" - - "crates/ag-tree/**" - - "crates/ag-merge/**" - - "crates/ag-policy/**" - - "crates/ag-sandbox/**" - - "crates/ag-storage/**" - - "crates/ag-agent/**" - - "crates/ag-protocol/**" - - "crates/ag-client/**" - - "crates/ag-worker/**" - - "crates/ag-git/**" - - "crates/ag-integrate/**" - - ".github/workflows/fuzz.yml" - pull_request: - paths: - - "fuzz/**" - - "proto/**" - - "Cargo.lock" - - "crates/ag-types/**" - - "crates/ag-crypto/**" - - "crates/ag-object/**" - - "crates/ag-changeset/**" - - "crates/ag-dag/**" - - "crates/ag-diff/**" - - "crates/ag-tree/**" - - "crates/ag-merge/**" - - "crates/ag-policy/**" - - "crates/ag-sandbox/**" - - "crates/ag-storage/**" - - "crates/ag-agent/**" - - "crates/ag-protocol/**" - - "crates/ag-client/**" - - "crates/ag-worker/**" - - "crates/ag-git/**" - - "crates/ag-integrate/**" - - ".github/workflows/fuzz.yml" + workflow_call: + inputs: + extended: + type: boolean + default: false schedule: - cron: "41 2 * * *" workflow_dispatch: @@ -78,7 +23,7 @@ permissions: concurrency: group: fuzz-${{ github.ref }}-${{ github.event_name }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + cancel-in-progress: true env: CARGO_TERM_COLOR: always @@ -104,6 +49,7 @@ jobs: toolchain: ${{ env.FUZZ_NIGHTLY }} - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: + cache-workspace-crates: true workspaces: fuzz key: fuzz-asan - name: cargo-fuzz @@ -134,11 +80,12 @@ jobs: smoke: name: smoke (${{ matrix.group }}, 60 s per target) - if: github.event_name == 'pull_request' || github.event_name == 'push' + if: ${{ !inputs.extended && github.workflow != 'Fuzz' }} needs: build runs-on: ubuntu-latest timeout-minutes: 30 strategy: + max-parallel: 2 fail-fast: false matrix: include: @@ -189,11 +136,12 @@ jobs: nightly: name: nightly (${{ matrix.group }}) - if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + if: inputs.extended || github.workflow == 'Fuzz' needs: build runs-on: ubuntu-latest timeout-minutes: 150 strategy: + max-parallel: 2 fail-fast: false matrix: include: diff --git a/.github/workflows/oci-runtimes.yml b/.github/workflows/oci-runtimes.yml index b660e50f..fbb7ac7c 100644 --- a/.github/workflows/oci-runtimes.yml +++ b/.github/workflows/oci-runtimes.yml @@ -1,35 +1,8 @@ name: OCI runtimes and cgroup v2 -# The OCI sandbox is only worth anything if a real container runtime accepts its flags and the kernel enforces -# what they request; the namespace sandbox's memory and task limits are only exact on a real, delegated cgroup v2 -# hierarchy. Neither can be shown with a fake runtime or a fake directory, so these jobs run the real thing: -# -# * docker, rootful podman and rootless podman, each against a small image pinned by digest and pulled here -# (the sandbox never pulls), asserting from *inside* the container: no network, read-only root, no new -# privileges, no capabilities, memory / pids / CPU limits, no view of the worker's keys, cleanup. -# * the whole `ag-sandbox` suite on a delegated cgroup v2 subtree prepared as root and used by the test user, -# including crates/ag-sandbox/tests/cgroup_real.rs (memory.max, pids.max, memory.peak, OOM kills, removal). -# -# AG_REQUIRE_COMPONENTS (oci:docker / oci:podman, cgroup-v2, namespace-sandbox) turns every skip of what the job -# claims to run into a failure: a runner that cannot do it must not go green. Whatever still skips is listed, and -# fails the job unless tests/skip-allowlist.toml has reviewed it (scripts/check_skips.py). - +# Docker, Podman and cgroup integration checks run during qualification. on: - push: - branches: [main] - paths: - - "crates/ag-sandbox/**" - - "crates/ag-worker/**" - - "crates/ag-client/**" - - ".github/workflows/oci-runtimes.yml" - - "Cargo.lock" - pull_request: - paths: - - "crates/ag-sandbox/**" - - "crates/ag-worker/**" - - "crates/ag-client/**" - - ".github/workflows/oci-runtimes.yml" - - "Cargo.lock" + workflow_call: workflow_dispatch: permissions: @@ -50,6 +23,7 @@ jobs: name: ${{ matrix.name }} runs-on: ubuntu-24.04 strategy: + max-parallel: 2 fail-fast: false matrix: include: @@ -70,6 +44,8 @@ jobs: with: toolchain: stable - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + cache-workspace-crates: true - name: The host run: | uname -r @@ -185,6 +161,8 @@ jobs: with: toolchain: stable - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + cache-workspace-crates: true # Ubuntu 24.04 restricts unprivileged user namespaces with AppArmor (see ci.yml). - name: Allow unprivileged user namespaces run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || true diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml index 65ea4651..ce56b6be 100644 --- a/.github/workflows/packaging.yml +++ b/.github/workflows/packaging.yml @@ -1,44 +1,8 @@ name: Packaging -# Everything between "it works on my machine" and "someone else can install it", checked without publishing anything: -# -# tooling the release/install scripts' tests, shellcheck on install.sh, actionlint on the workflows, the -# Homebrew formula and Scoop manifest rendered from a fixture -# crates `cargo publish --workspace --dry-run` (every publishable crate, packaged and built from its tarball in -# dependency order; docs/operations/releasing.md lists them and the one `publish = false`), the -# metadata checks, and that the copies of proto/ and migrations/ inside the crates are current -# sdks the TypeScript package (`npm pack`) and the Python package (`uv build`, `twine check`) -# -# Publishing is a separate, manual workflow: publish-sdks.yml. - +# Full publication dry runs are qualification checks, not per-commit background work. on: - push: - branches: [main] - paths: - # A release request requires this workflow on the exact main commit. - - "CHANGELOG.md" - - "crates/**" - - "proto/**" - - "migrations/**" - - "apps/web/**" - - "Cargo.toml" - - "Cargo.lock" - - "sdk/**" - - "scripts/**" - - "packaging/**" - - ".github/workflows/**" - pull_request: - paths: - - "crates/**" - - "proto/**" - - "migrations/**" - - "apps/web/**" - - "Cargo.toml" - - "Cargo.lock" - - "sdk/**" - - "scripts/**" - - "packaging/**" - - ".github/workflows/**" + workflow_call: workflow_dispatch: permissions: @@ -103,6 +67,8 @@ jobs: with: toolchain: stable - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + cache-workspace-crates: true - name: Metadata of every package (crates, npm, PyPI) # pipefail: without it `tee` hides the checker's exit status and an incomplete crate passes this step. run: | diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml index c31e27b8..e5b355d9 100644 --- a/.github/workflows/perf.yml +++ b/.github/workflows/perf.yml @@ -1,32 +1,12 @@ name: Performance -# Timings are measurements, not thresholds; functional failures gate (docs/operations/performance.md). The smoke variant -# proves on every change to the engine, the server or the tool that the benchmark and the load generator still run -# end to end; its numbers come from a shared 4-vCPU runner and are uploaded as artifacts, never compared with a -# threshold. The full sizes (1k/10k/100k files; 20 s of load per scenario at 16 writers and 100 subscribers) run -# weekly and by hand, on the same kind of runner, and are the numbers to compare with the published ones. -# -# smoke ag-bench engine --sizes 1000,10000 and ag-bench load --profile smoke. Correctness gates; timings do not. -# full ag-bench engine (1k/10k/100k, memory and disk) and ag-bench load --profile full. Weekly and by hand. - +# Bounded performance qualification plus extended weekly/manual measurements. on: - push: - branches: [main] - paths: - - "crates/ag-bench/**" - - "crates/ag-server/**" - - "crates/ag-integrate/**" - - "crates/ag-merge/**" - - "crates/ag-tree/**" - - ".github/workflows/perf.yml" - pull_request: - paths: - - "crates/ag-bench/**" - - "crates/ag-server/**" - - "crates/ag-integrate/**" - - "crates/ag-merge/**" - - "crates/ag-tree/**" - - ".github/workflows/perf.yml" + workflow_call: + inputs: + extended: + type: boolean + default: false schedule: - cron: "41 4 * * 1" workflow_dispatch: @@ -36,7 +16,7 @@ permissions: concurrency: group: perf-${{ github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + cancel-in-progress: true env: CARGO_TERM_COLOR: always @@ -44,7 +24,7 @@ env: jobs: smoke: name: smoke (correctness) - if: github.event_name != 'schedule' && github.event_name != 'workflow_dispatch' + if: ${{ !inputs.extended && github.workflow != 'Performance' }} runs-on: ubuntu-latest timeout-minutes: 40 # No latency/throughput thresholds: fail only on an unsuccessful run, including integrity violations. @@ -57,6 +37,7 @@ jobs: toolchain: stable - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: + cache-workspace-crates: true key: perf - name: Build the server and the tool (release) run: cargo build --locked --release -p agentgit-server -p agentgit-bench @@ -84,7 +65,7 @@ jobs: full: name: full (weekly, by hand) - if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + if: inputs.extended || github.workflow == 'Performance' runs-on: ubuntu-latest timeout-minutes: 120 steps: @@ -96,6 +77,7 @@ jobs: toolchain: stable - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: + cache-workspace-crates: true key: perf - run: cargo build --locked --release -p agentgit-server -p agentgit-bench - name: Engine benchmark, in memory diff --git a/.github/workflows/prepare-release.yml b/.github/workflows/prepare-release.yml new file mode 100644 index 00000000..a65e8193 --- /dev/null +++ b/.github/workflows/prepare-release.yml @@ -0,0 +1,497 @@ +name: Prepare release + +# Slow preparation runs independently; only ready, tested artifacts can be published. + +on: + workflow_dispatch: + inputs: + force: + description: "Prepare even if this version is already published (validation only)" + type: boolean + default: false + +permissions: + contents: read + +concurrency: + group: prepare-release-${{ github.ref }} + cancel-in-progress: true + +env: + CARGO_TERM_COLOR: always + CARGO_INCREMENTAL: "0" + REGISTRY_IMAGE_SERVER: ghcr.io/paffin/agentgit-server + REGISTRY_IMAGE_CLI: ghcr.io/paffin/agentgit-cli + +jobs: + verify: + name: Candidate version and source + runs-on: ubuntu-latest + outputs: + version: ${{ steps.version.outputs.version }} + cargo_version: ${{ steps.version.outputs.cargo_version }} + prerelease: ${{ steps.version.outputs.prerelease }} + commit_date: ${{ steps.version.outputs.commit_date }} + epoch: ${{ steps.version.outputs.epoch }} + needed: ${{ steps.version.outputs.needed }} + publish: "false" + dry_run: "false" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - id: version + env: + GH_TOKEN: ${{ github.token }} + FORCE: ${{ inputs.force }} + run: python3 scripts/prepare_release.py + + qualify: + needs: verify + if: needs.verify.outputs.needed == 'true' + uses: ./.github/workflows/qualification.yml + permissions: + contents: read + actions: read + security-events: write + pages: write + id-token: write + secrets: inherit + + scripts: + if: needs.verify.outputs.needed == 'true' + name: release tooling tests + needs: verify + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - run: python3 scripts/test_release_scripts.py + + binaries: + if: needs.verify.outputs.needed == 'true' + name: binaries (${{ matrix.target }}) + needs: verify + strategy: + max-parallel: 2 + fail-fast: false + matrix: + include: + - {target: x86_64-unknown-linux-gnu, os: ubuntu-22.04} + - {target: aarch64-unknown-linux-gnu, os: ubuntu-22.04-arm} + - {target: aarch64-apple-darwin, os: macos-14} + - {target: x86_64-apple-darwin, os: macos-15-intel} + runs-on: ${{ matrix.os }} + env: + VERSION: ${{ needs.verify.outputs.version }} + CARGO_VERSION: ${{ needs.verify.outputs.cargo_version }} + SOURCE_DATE_EPOCH: ${{ needs.verify.outputs.epoch }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + with: + toolchain: stable + targets: ${{ matrix.target }} + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + cache-workspace-crates: true + key: release-${{ matrix.target }} + # ag-server serves the Web UI from inside its binary (cargo feature `ui`): the build must exist before cargo + # compiles it, or the archive's ag-server serves a page that says how to build the UI. + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: 22 + cache: npm + cache-dependency-path: apps/web/package-lock.json + - name: Build the Web UI (apps/web/dist, embedded by the build below) + run: scripts/build_ui.sh + - name: Build + env: + # Do not embed the build directory in the binaries. + RUSTFLAGS: --remap-path-prefix=${{ github.workspace }}=/agentgit --remap-path-prefix=${{ runner.temp }}=/tmp + run: cargo build --release --locked --target ${{ matrix.target }} -p agentgit-cli -p agentgit-server -p agentgit-worker -p agentgit-mcp + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: services/agent-orchestrator/go.mod + cache-dependency-path: services/agent-orchestrator/go.sum + - name: Build the native executor + working-directory: services/agent-orchestrator + env: + CGO_ENABLED: "0" + run: >- + go build -trimpath -buildvcs=false + -ldflags "-s -w -X github.com/Paffin/AgentGit/services/agent-orchestrator/internal/app.Version=$CARGO_VERSION" + -o "$GITHUB_WORKSPACE/target/${{ matrix.target }}/release/orchestrator" ./cmd/orchestrator + - name: Archive + run: | + python3 scripts/package_release.py archive --version "$VERSION" --target ${{ matrix.target }} \ + --bin-dir target/${{ matrix.target }}/release --out dist + ls -la dist + - name: Smoke-test what is in the archive (versions, then the real end-to-end flow) + run: | + set -euo pipefail + name="agentgit-${VERSION}-${{ matrix.target }}" + work="$RUNNER_TEMP/unpacked"; mkdir -p "$work" + tar -xzf "dist/$name.tar.gz" -C "$work" + bin="$work/$name" + ls -la "$bin" + for b in ag ag-server ag-worker; do + v=$("$bin/$b" --version); echo "$v" + case "$v" in "$b $CARGO_VERSION ("*") "*|"$b $CARGO_VERSION ("*")") ;; *) echo "::error::$b --version is '$v'"; exit 1;; esac + case "$v" in *"${GITHUB_SHA:0:12}"*) ;; *) echo "::error::$b does not report this commit"; exit 1;; esac + done + "$bin/ag-worker" doctor || true + python3 scripts/onboarding_smoke.py --ag "$bin/ag" --server "$bin/ag-server" + # ag-mcp has no build script, so its version line carries no commit: check the crate version only. + v=$("$bin/ag-mcp" --version); echo "$v" + [ "$v" = "ag-mcp $CARGO_VERSION" ] || { echo "::error::ag-mcp --version is '$v'"; exit 1; } + v=$("$bin/orchestrator" version); echo "$v" + [ "$v" = "agent-orchestrator $CARGO_VERSION" ] || { echo "::error::orchestrator version is '$v'"; exit 1; } + tools=$("$bin/ag-mcp" --list-tools --server http://127.0.0.1:1) + case "$tools" in *'"name": "task_claim"'*) ;; *) echo "::error::ag-mcp lists no task_claim tool"; exit 1;; esac + AG_BOOTSTRAP_TOKEN=$(openssl rand -hex 32) + AG_METRICS_TOKEN=$(openssl rand -hex 32) + export AG_BOOTSTRAP_TOKEN AG_METRICS_TOKEN + "$bin/ag-server" serve --listen 127.0.0.1:18080 --data-dir "$work/data" > "$work/server.log" 2>&1 & + server=$! + trap 'kill $server 2>/dev/null || true' EXIT + python3 scripts/e2e_smoke.py --url http://127.0.0.1:18080 --ag "$bin/ag" \ + --bootstrap-token "$AG_BOOTSTRAP_TOKEN" --metrics-token "$AG_METRICS_TOKEN" || { tail -50 "$work/server.log"; exit 1; } + # The unpacked ag-server carries the Web UI (the server from the smoke above is still running). + page=$(curl -fsS -H 'Accept: text/html' http://127.0.0.1:18080/) + case "$page" in *'
'*) echo "ag-server serves the embedded Web UI";; *) echo "$page" | head -20; echo "::error::the archive's ag-server has no Web UI"; exit 1;; esac + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: archive-${{ matrix.target }} + path: dist/*.tar.gz + if-no-files-found: error + retention-days: 14 + + sbom: + if: needs.verify.outputs.needed == 'true' + name: CycloneDX SBOMs + needs: verify + runs-on: ubuntu-latest + env: + VERSION: ${{ needs.verify.outputs.version }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + with: + toolchain: stable + - uses: taiki-e/install-action@1e29709c0ec63babbe8f266397a465f29a825f91 # v2.87.31 + with: + tool: cargo-cyclonedx@0.5.9 + # The same script the Security workflow runs on every change (so a break shows up there, not at release time). + # The web UI SBOM it can also write is not a release asset yet: the release job below expects four binary inventories. + - name: One SBOM per Rust binary (from Cargo.lock; build-time dependencies included) + run: scripts/sbom.sh --version "$VERSION" --out dist --no-web + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: sbom + path: dist/*.cdx.json + if-no-files-found: error + retention-days: 14 + + image: + if: needs.verify.outputs.needed == 'true' + name: image (${{ matrix.arch }}) + needs: verify + strategy: + max-parallel: 2 + fail-fast: false + matrix: + include: + - {arch: amd64, os: ubuntu-latest} + - {arch: arm64, os: ubuntu-22.04-arm} + runs-on: ${{ matrix.os }} + env: + PUBLISH: "true" + VERSION: ${{ needs.verify.outputs.version }} + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + - uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4.7.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + # Native builds on native runners: an emulated Rust build would take hours. Publishing pushes each + # architecture by digest; a rehearsal exports a tarball instead. The manifest job merges them. + - name: Server image + id: server + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: . + file: deploy/docker/Dockerfile + target: server + platforms: linux/${{ matrix.arch }} + build-args: | + AG_BUILD_COMMIT=${{ github.sha }} + AG_BUILD_DATE=${{ needs.verify.outputs.commit_date }} + labels: | + org.opencontainers.image.source=https://github.com/Paffin/AgentGit + org.opencontainers.image.version=${{ needs.verify.outputs.version }} + org.opencontainers.image.revision=${{ github.sha }} + org.opencontainers.image.licenses=Apache-2.0 + cache-from: | + type=gha,scope=agentgit-image + type=gha,scope=agentgit-image-release-${{ matrix.arch }} + cache-to: type=gha,scope=agentgit-image-release-${{ matrix.arch }},mode=max + provenance: mode=max + sbom: true + outputs: ${{ format('type=image,name={0},push-by-digest=true,name-canonical=true,push=true', env.REGISTRY_IMAGE_SERVER) }} + - name: CLI image + id: cli + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: . + file: deploy/docker/Dockerfile + target: cli + platforms: linux/${{ matrix.arch }} + build-args: | + AG_BUILD_COMMIT=${{ github.sha }} + AG_BUILD_DATE=${{ needs.verify.outputs.commit_date }} + labels: | + org.opencontainers.image.source=https://github.com/Paffin/AgentGit + org.opencontainers.image.version=${{ needs.verify.outputs.version }} + org.opencontainers.image.revision=${{ github.sha }} + org.opencontainers.image.licenses=Apache-2.0 + cache-from: type=gha,scope=agentgit-image-release-${{ matrix.arch }} + provenance: mode=max + sbom: true + outputs: ${{ format('type=image,name={0},push-by-digest=true,name-canonical=true,push=true', env.REGISTRY_IMAGE_CLI) }} + - name: Keep the result for the manifest job + run: | + mkdir -p "$RUNNER_TEMP/handoff" + if [ "$PUBLISH" = true ]; then + digest="${{ steps.server.outputs.digest }}"; touch "$RUNNER_TEMP/handoff/server-${digest#sha256:}" + digest="${{ steps.cli.outputs.digest }}"; touch "$RUNNER_TEMP/handoff/cli-${digest#sha256:}" + else + cp "$RUNNER_TEMP"/server-${{ matrix.arch }}.tar "$RUNNER_TEMP"/cli-${{ matrix.arch }}.tar "$RUNNER_TEMP/handoff/" + fi + ls -la "$RUNNER_TEMP/handoff" + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: image-${{ matrix.arch }} + path: ${{ runner.temp }}/handoff + if-no-files-found: error + retention-days: 3 + + manifest: + if: needs.verify.outputs.needed == 'true' + name: multi-arch manifest + needs: [verify, image] + runs-on: ubuntu-latest + env: + PUBLISH: "true" + VERSION: ${{ needs.verify.outputs.version }} + PRERELEASE: ${{ needs.verify.outputs.prerelease }} + permissions: + contents: read + packages: write + id-token: write + attestations: write + steps: + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: image-* + path: handoff + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + - uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4.7.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Publish - merge the per-architecture digests into one tag + run: | + set -euo pipefail + for kind in server cli; do + image=$([ "$kind" = server ] && echo "$REGISTRY_IMAGE_SERVER" || echo "$REGISTRY_IMAGE_CLI") + tags="-t $image:candidate-$GITHUB_SHA" + refs="" + for f in handoff/image-*/"$kind"-*; do refs="$refs $image@sha256:${f##*/"$kind"-}"; done + # shellcheck disable=SC2086 + docker buildx imagetools create $tags $refs + docker buildx imagetools inspect "$image:candidate-$GITHUB_SHA" + done + - name: Publish - attest the images + id: digests + run: | + set -euo pipefail + for kind in server cli; do + image=$([ "$kind" = server ] && echo "$REGISTRY_IMAGE_SERVER" || echo "$REGISTRY_IMAGE_CLI") + echo "$kind=$(docker buildx imagetools inspect "$image:candidate-$GITHUB_SHA" --format '{{json .Manifest.Digest}}' | tr -d '"')" >> "$GITHUB_OUTPUT" + done + - uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 + with: + subject-name: ${{ env.REGISTRY_IMAGE_SERVER }} + subject-digest: ${{ steps.digests.outputs.server }} + push-to-registry: true + - uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 + with: + subject-name: ${{ env.REGISTRY_IMAGE_CLI }} + subject-digest: ${{ steps.digests.outputs.cli }} + push-to-registry: true + + - name: Record immutable image digests + env: + SERVER_DIGEST: ${{ steps.digests.outputs.server }} + CLI_DIGEST: ${{ steps.digests.outputs.cli }} + run: | + python3 - <<'PYTHON' + import json, os + from pathlib import Path + Path('candidate-images.json').write_text(json.dumps({ + 'sha': os.environ['GITHUB_SHA'], 'version': os.environ['VERSION'], + 'server': os.environ['SERVER_DIGEST'], 'cli': os.environ['CLI_DIGEST'], + })) + PYTHON + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: candidate-images + path: candidate-images.json + retention-days: 14 + + release: + if: needs.verify.outputs.needed == 'true' + name: assemble and attest the candidate + needs: [verify, qualify, scripts, binaries, sbom, manifest] + runs-on: ubuntu-latest + env: + PUBLISH: ${{ needs.verify.outputs.publish }} + VERSION: ${{ needs.verify.outputs.version }} + DRY_RUN: ${{ needs.verify.outputs.dry_run }} + GH_TOKEN: ${{ github.token }} + permissions: + contents: read + id-token: write + attestations: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + path: dist + pattern: "{archive-*,sbom,candidate-images}" + merge-multiple: true + - name: Checksums and the Homebrew formula + run: | + set -euo pipefail + ls -la dist + set -- dist/*.tar.gz; test $# = 4 + set -- dist/*.cdx.json; test $# = 4 + python3 scripts/package_release.py sums dist/*.tar.gz dist/*.cdx.json | tee dist/SHA256SUMS + (cd dist && sha256sum -c SHA256SUMS) + python3 scripts/render_packaging.py --version "$VERSION" --sums dist/SHA256SUMS --out dist --only homebrew + ruby -c dist/agentgit.rb + - name: Candidate manifest and notes + run: | + python3 scripts/check_release.py "v$VERSION" --notes > dist/RELEASE_NOTES.md + python3 scripts/release_candidate.py create dist + - uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 + id: attest + with: + subject-path: dist/* + - name: The attestations verify + run: | + set -euo pipefail + for f in dist/*; do + gh attestation verify "$f" --repo "$GITHUB_REPOSITORY" \ + --signer-workflow "$GITHUB_REPOSITORY/.github/workflows/prepare-release.yml" > /dev/null + echo "attested: $f" + done + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: release-dist + path: dist + retention-days: 14 + - name: Summary + run: | + title="Ready candidate (publication is a separate fast operation)" + [ "$PUBLISH" = true ] && title="Published" + { + echo "## $title" + echo + echo '```' + (cd dist && wc -c -- * | grep -v ' total$') + echo '```' + echo '```' + cat dist/SHA256SUMS + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + install: + if: needs.verify.outputs.needed == 'true' + name: install.sh (${{ matrix.os }}) + needs: [verify, release] + strategy: + max-parallel: 2 + fail-fast: false + matrix: + os: [ubuntu-latest, ubuntu-22.04-arm, macos-latest] + runs-on: ${{ matrix.os }} + env: + VERSION: ${{ needs.verify.outputs.version }} + CARGO_VERSION: ${{ needs.verify.outputs.cargo_version }} + PUBLISHED: ${{ needs.verify.outputs.publish }} + GH_TOKEN: ${{ github.token }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: release-dist + path: dist + - name: Install the version-pinned release, verifying checksum and attestation + run: | + set -euo pipefail + prefix="$RUNNER_TEMP/prefix" + # Real releases exercise the public download path on a fresh runner. Rehearsals use the same layout locally. + base_url="file://$PWD/dist" + if [ "$PUBLISHED" = true ]; then + base_url="https://github.com/Paffin/AgentGit/releases/download/v$VERSION" + fi + uname -a + if [ "${RUNNER_OS}" = macOS ]; then sw_vers; fi + echo "Installing AgentGit $VERSION from $base_url" + sh scripts/install.sh --version "$VERSION" --prefix "$prefix" --base-url "$base_url" --require-attestation + "$prefix/bin/ag" --version + "$prefix/bin/ag-server" --version + "$prefix/bin/ag-worker" --version + "$prefix/bin/ag-mcp" --version + [ "$("$prefix/bin/orchestrator" version)" = "agent-orchestrator $CARGO_VERSION" ] + case "$("$prefix/bin/ag" --version)" in "ag $CARGO_VERSION "*) ;; *) echo "::error::unexpected version"; exit 1;; esac + - name: A corrupted download is refused and installs nothing + run: | + set -euo pipefail + mkdir bad && cp dist/* bad/ + for f in bad/agentgit-*.tar.gz; do printf 'x' >> "$f"; done + if sh scripts/install.sh --version "$VERSION" --prefix "$RUNNER_TEMP/bad-prefix" --base-url "file://$PWD/bad" --no-attestation; then + echo "::error::install.sh accepted a corrupted archive"; exit 1 + fi + test ! -e "$RUNNER_TEMP/bad-prefix/bin" + - name: The installed binaries work end to end + run: | + set -euo pipefail + prefix="$RUNNER_TEMP/prefix" + AG_BOOTSTRAP_TOKEN=$(openssl rand -hex 32) + AG_METRICS_TOKEN=$(openssl rand -hex 32) + export AG_BOOTSTRAP_TOKEN AG_METRICS_TOKEN + "$prefix/bin/ag-server" serve --listen 127.0.0.1:18080 --data-dir "$RUNNER_TEMP/data" > "$RUNNER_TEMP/server.log" 2>&1 & + server=$! + trap 'kill $server 2>/dev/null || true' EXIT + python3 scripts/e2e_smoke.py --url http://127.0.0.1:18080 --ag "$prefix/bin/ag" \ + --bootstrap-token "$AG_BOOTSTRAP_TOKEN" --metrics-token "$AG_METRICS_TOKEN" || { tail -50 "$RUNNER_TEMP/server.log"; exit 1; } diff --git a/.github/workflows/publish-sdks.yml b/.github/workflows/publish-sdks.yml index f48be774..ab19a636 100644 --- a/.github/workflows/publish-sdks.yml +++ b/.github/workflows/publish-sdks.yml @@ -65,6 +65,8 @@ jobs: with: toolchain: stable - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + cache-workspace-crates: true - name: Build the server used by the SDK contract tests run: cargo build --locked -p agentgit-server - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 diff --git a/.github/workflows/qualification.yml b/.github/workflows/qualification.yml new file mode 100644 index 00000000..36a9fe4e --- /dev/null +++ b/.github/workflows/qualification.yml @@ -0,0 +1,84 @@ +name: Qualification + +# One complete matrix per release candidate (or nightly), never per ordinary push. +on: + workflow_call: + workflow_dispatch: + schedule: + - cron: "17 1 * * *" + +permissions: + contents: read + +concurrency: + group: qualification-${{ github.ref }} + cancel-in-progress: true + +jobs: + core: + uses: ./.github/workflows/full-ci.yml + secrets: inherit + + web: + uses: ./.github/workflows/web.yml + with: + full: true + + security: + uses: ./.github/workflows/security.yml + with: + full: true + + codeql: + uses: ./.github/workflows/codeql.yml + permissions: + contents: read + actions: read + security-events: write + + containers: + uses: ./.github/workflows/containers.yml + + docs: + uses: ./.github/workflows/docs.yml + with: + full: true + permissions: + contents: read + pages: write + id-token: write + + packaging: + uses: ./.github/workflows/packaging.yml + + coverage: + uses: ./.github/workflows/quality.yml + + windows: + uses: ./.github/workflows/windows.yml + + fuzz: + uses: ./.github/workflows/fuzz.yml + + performance: + uses: ./.github/workflows/perf.yml + + oci: + uses: ./.github/workflows/oci-runtimes.yml + + complete: + name: Qualified commit + if: always() + needs: [core, web, security, codeql, containers, docs, packaging, coverage, windows, fuzz, performance, oci] + runs-on: ubuntu-latest + steps: + - name: Every qualification component must succeed + env: + RESULTS: ${{ toJSON(needs) }} + run: | + python3 - <<'PYTHON' + import json, os + bad = [name for name, result in json.loads(os.environ['RESULTS']).items() if result['result'] != 'success'] + if bad: + raise SystemExit('Qualification failed: ' + ', '.join(bad)) + PYTHON diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 5c7e474c..65aad20c 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -1,35 +1,12 @@ name: Quality -# Two measurements of how much the tests are worth (docs/architecture/testing.md has the numbers and how to read them): -# -# coverage cargo-llvm-cov over the whole workspace test suite, a table per crate in the job summary and a floor -# per crate (scripts/coverage_floors.toml) below which the job fails. On changes to the Rust code, and -# weekly. The lcov file is uploaded as an artifact. -# mutation cargo-mutants on the crates whose bugs corrupt data silently - ag-object, ag-diff, ag-tree, ag-merge, -# ag-policy: every mutant (a `+` turned into `-`, a condition negated, a body replaced) must be caught by -# the crate's own tests, or sit in .cargo/mutants.toml with the reason it cannot matter. Weekly and by -# hand (an hour or more; not on pull requests). - +# Linux full-suite coverage qualifies correctness. Extended mutation runs remain weekly/manual. on: - push: - branches: [main] - paths: - # A release request requires this workflow on the exact main commit. - - "CHANGELOG.md" - - "crates/**" - - "Cargo.lock" - - "Cargo.toml" - - "tests/**" - - "scripts/coverage_*" - - ".cargo/mutants.toml" - - ".github/workflows/quality.yml" - pull_request: - paths: - - "crates/**" - - "Cargo.lock" - - "Cargo.toml" - - "scripts/coverage_*" - - ".github/workflows/quality.yml" + workflow_call: + inputs: + extended: + type: boolean + default: false schedule: - cron: "17 3 * * 0" workflow_dispatch: @@ -43,7 +20,7 @@ permissions: concurrency: group: quality-${{ github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + cancel-in-progress: true env: CARGO_TERM_COLOR: always @@ -51,7 +28,7 @@ env: jobs: coverage: name: coverage (cargo-llvm-cov) - if: github.event_name != 'workflow_dispatch' || github.event.inputs.crates == '' + if: github.workflow != 'Quality' || github.event_name != 'workflow_dispatch' || github.event.inputs.crates == '' runs-on: ubuntu-latest timeout-minutes: 90 steps: @@ -64,6 +41,7 @@ jobs: components: llvm-tools-preview - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: + cache-workspace-crates: true key: coverage - uses: taiki-e/install-action@1e29709c0ec63babbe8f266397a465f29a825f91 # v2.87.31 with: @@ -95,7 +73,7 @@ jobs: mutation: name: mutation (${{ matrix.crate }}) - if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + if: inputs.extended || github.workflow == 'Quality' runs-on: ubuntu-latest timeout-minutes: 240 strategy: @@ -125,6 +103,7 @@ jobs: - if: steps.pick.outputs.run == 'true' uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: + cache-workspace-crates: true key: mutation-${{ matrix.crate }} - if: steps.pick.outputs.run == 'true' uses: taiki-e/install-action@1e29709c0ec63babbe8f266397a465f29a825f91 # v2.87.31 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 33b3010b..d521475f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,30 +1,18 @@ name: Release -# A tag vX.Y.Z (or vX.Y.Z-rc.N) PUBLISHES a release: GitHub release with archives for four platforms, -# SHA256SUMS, CycloneDX SBOMs and build-provenance attestations, and multi-arch container images on GHCR. -# -# Run it by hand (Actions -> Release -> Run workflow) to REHEARSE the same pipeline without publishing anything: -# -# version what to call the rehearsal, default 0.0.0-rc.dry (any semver; Cargo.toml need not agree) -# publish default false. true is honoured only when the workflow is run on a v* tag whose name matches `version` -# -# A rehearsal builds and smoke-tests every binary, assembles SHA256SUMS, attests and verifies the attestations, -# builds both image architectures natively, merges them into a multi-arch manifest in a throw-away local -# registry, and installs the result with scripts/install.sh. It uploads the would-be release as the workflow -# artifact `release-dist`. What it cannot prove is spelled out in docs/operations/releasing.md. - +# Fast promotion only. Slow builds, full qualification and native smoke tests belong to Prepare release. +# Execution budgets are 3 minutes for publication plus 2 for parallel public-install checks (runner queue excluded). on: push: tags: ["v[0-9]+.[0-9]+.[0-9]+*"] workflow_dispatch: inputs: version: - description: "Version to build, e.g. 0.0.0-rc.dry (a rehearsal) or the tag's version when publishing" + description: "Prepared version, matching this v* tag" required: true - default: 0.0.0-rc.dry type: string publish: - description: "Publish for real (only on a v* tag ref). Leave off to rehearse." + description: "Publish the ready candidate; false verifies it without publishing" required: true default: false type: boolean @@ -33,28 +21,28 @@ permissions: contents: read concurrency: - group: release-${{ github.ref }}-${{ inputs.version }} - # A rehearsal is superseded by the next one; a build of a tag never is. - cancel-in-progress: ${{ github.ref_type != 'tag' }} + group: release-${{ github.ref }} + cancel-in-progress: false env: - CARGO_TERM_COLOR: always - CARGO_INCREMENTAL: "0" REGISTRY_IMAGE_SERVER: ghcr.io/paffin/agentgit-server REGISTRY_IMAGE_CLI: ghcr.io/paffin/agentgit-cli jobs: - verify: - name: what are we releasing + publish: + name: Promote the prepared candidate runs-on: ubuntu-latest + timeout-minutes: 3 + permissions: + contents: write + actions: read + packages: write outputs: version: ${{ steps.v.outputs.version }} cargo_version: ${{ steps.v.outputs.cargo_version }} - prerelease: ${{ steps.v.outputs.prerelease }} - dry_run: ${{ steps.v.outputs.dry_run }} publish: ${{ steps.mode.outputs.publish }} - commit_date: ${{ steps.mode.outputs.commit_date }} - epoch: ${{ steps.mode.outputs.epoch }} + env: + GH_TOKEN: ${{ github.token }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -101,480 +89,93 @@ jobs: flag="" [ "${{ steps.mode.outputs.publish }}" = true ] || flag="--dry-run" python3 scripts/check_release.py "$ref" $flag | tee -a "$GITHUB_OUTPUT" - - name: Summary - run: | - { - echo "## Release: ${{ steps.mode.outputs.publish == 'true' && 'PUBLISH' || 'rehearsal (nothing is published)' }}" - echo - echo "- version: \`$(grep ^version= "$GITHUB_OUTPUT" | cut -d= -f2)\` (Cargo.toml: \`$(grep ^cargo_version= "$GITHUB_OUTPUT" | cut -d= -f2)\`)" - echo "- commit: \`${GITHUB_SHA}\`" - } >> "$GITHUB_STEP_SUMMARY" - - scripts: - name: release tooling tests - needs: verify - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - run: python3 scripts/test_release_scripts.py - - test: - name: tests - needs: verify - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master - with: - toolchain: stable - - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - # As in ci.yml: Ubuntu 24.04 restricts unprivileged user namespaces; the sandbox suites must run, not skip. - - name: Allow unprivileged user namespaces - run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || true - - run: cargo test --workspace --locked - env: - AG_REQUIRE_SANDBOX: "1" - - binaries: - name: binaries (${{ matrix.target }}) - needs: [verify, test] - strategy: - fail-fast: false - matrix: - include: - - {target: x86_64-unknown-linux-gnu, os: ubuntu-22.04} - - {target: aarch64-unknown-linux-gnu, os: ubuntu-22.04-arm} - - {target: aarch64-apple-darwin, os: macos-14} - - {target: x86_64-apple-darwin, os: macos-15-intel} - runs-on: ${{ matrix.os }} - env: - VERSION: ${{ needs.verify.outputs.version }} - CARGO_VERSION: ${{ needs.verify.outputs.cargo_version }} - SOURCE_DATE_EPOCH: ${{ needs.verify.outputs.epoch }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master - with: - toolchain: stable - targets: ${{ matrix.target }} - - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - with: - key: release-${{ matrix.target }} - # ag-server serves the Web UI from inside its binary (cargo feature `ui`): the build must exist before cargo - # compiles it, or the archive's ag-server serves a page that says how to build the UI. - - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 - with: - node-version: 22 - cache: npm - cache-dependency-path: apps/web/package-lock.json - - name: Build the Web UI (apps/web/dist, embedded by the build below) - run: scripts/build_ui.sh - - name: Build - env: - # Do not embed the build directory in the binaries. - RUSTFLAGS: --remap-path-prefix=${{ github.workspace }}=/agentgit --remap-path-prefix=${{ runner.temp }}=/tmp - run: cargo build --release --locked --target ${{ matrix.target }} -p agentgit-cli -p agentgit-server -p agentgit-worker -p agentgit-mcp - - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + - id: ready + name: Find successful preparation for this exact commit + run: python3 scripts/release_candidate.py resolve + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: - go-version-file: services/agent-orchestrator/go.mod - cache-dependency-path: services/agent-orchestrator/go.sum - - name: Build the native executor - working-directory: services/agent-orchestrator + name: release-dist + path: dist + run-id: ${{ steps.ready.outputs.run_id }} + github-token: ${{ github.token }} + - id: images + name: Verify source, version, image digests and every file checksum env: - CGO_ENABLED: "0" - run: >- - go build -trimpath -buildvcs=false - -ldflags "-s -w -X github.com/Paffin/AgentGit/services/agent-orchestrator/internal/app.Version=$CARGO_VERSION" - -o "$GITHUB_WORKSPACE/target/${{ matrix.target }}/release/orchestrator" ./cmd/orchestrator - - name: Archive - run: | - python3 scripts/package_release.py archive --version "$VERSION" --target ${{ matrix.target }} \ - --bin-dir target/${{ matrix.target }}/release --out dist - ls -la dist - - name: Smoke-test what is in the archive (versions, then the real end-to-end flow) - run: | - set -euo pipefail - name="agentgit-${VERSION}-${{ matrix.target }}" - work="$RUNNER_TEMP/unpacked"; mkdir -p "$work" - tar -xzf "dist/$name.tar.gz" -C "$work" - bin="$work/$name" - ls -la "$bin" - for b in ag ag-server ag-worker; do - v=$("$bin/$b" --version); echo "$v" - case "$v" in "$b $CARGO_VERSION ("*") "*|"$b $CARGO_VERSION ("*")") ;; *) echo "::error::$b --version is '$v'"; exit 1;; esac - case "$v" in *"${GITHUB_SHA:0:12}"*) ;; *) echo "::error::$b does not report this commit"; exit 1;; esac + VERSION: ${{ steps.v.outputs.version }} + PREPARE_RUN_ID: ${{ steps.ready.outputs.run_id }} + run: python3 scripts/release_candidate.py verify dist + - name: Verify prepared build provenance + run: | + for file in dist/*; do + gh attestation verify "$file" --repo "$GITHUB_REPOSITORY" \ + --signer-workflow "$GITHUB_REPOSITORY/.github/workflows/prepare-release.yml" \ + --source-digest "$GITHUB_SHA" > /dev/null done - "$bin/ag-worker" doctor || true - python3 scripts/onboarding_smoke.py --ag "$bin/ag" --server "$bin/ag-server" - # ag-mcp has no build script, so its version line carries no commit: check the crate version only. - v=$("$bin/ag-mcp" --version); echo "$v" - [ "$v" = "ag-mcp $CARGO_VERSION" ] || { echo "::error::ag-mcp --version is '$v'"; exit 1; } - v=$("$bin/orchestrator" version); echo "$v" - [ "$v" = "agent-orchestrator $CARGO_VERSION" ] || { echo "::error::orchestrator version is '$v'"; exit 1; } - tools=$("$bin/ag-mcp" --list-tools --server http://127.0.0.1:1) - case "$tools" in *'"name": "task_claim"'*) ;; *) echo "::error::ag-mcp lists no task_claim tool"; exit 1;; esac - AG_BOOTSTRAP_TOKEN=$(openssl rand -hex 32) - AG_METRICS_TOKEN=$(openssl rand -hex 32) - export AG_BOOTSTRAP_TOKEN AG_METRICS_TOKEN - "$bin/ag-server" serve --listen 127.0.0.1:18080 --data-dir "$work/data" > "$work/server.log" 2>&1 & - server=$! - trap 'kill $server 2>/dev/null || true' EXIT - python3 scripts/e2e_smoke.py --url http://127.0.0.1:18080 --ag "$bin/ag" \ - --bootstrap-token "$AG_BOOTSTRAP_TOKEN" --metrics-token "$AG_METRICS_TOKEN" || { tail -50 "$work/server.log"; exit 1; } - # The unpacked ag-server carries the Web UI (the server from the smoke above is still running). - page=$(curl -fsS -H 'Accept: text/html' http://127.0.0.1:18080/) - case "$page" in *'
'*) echo "ag-server serves the embedded Web UI";; *) echo "$page" | head -20; echo "::error::the archive's ag-server has no Web UI"; exit 1;; esac - - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: archive-${{ matrix.target }} - path: dist/*.tar.gz - if-no-files-found: error - retention-days: 14 - - sbom: - name: CycloneDX SBOMs - needs: verify - runs-on: ubuntu-latest - env: - VERSION: ${{ needs.verify.outputs.version }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master - with: - toolchain: stable - - uses: taiki-e/install-action@1e29709c0ec63babbe8f266397a465f29a825f91 # v2.87.31 - with: - tool: cargo-cyclonedx@0.5.9 - # The same script the Security workflow runs on every change (so a break shows up there, not at release time). - # The web UI SBOM it can also write is not a release asset yet: the release job below expects four binary inventories. - - name: One SBOM per Rust binary (from Cargo.lock; build-time dependencies included) - run: scripts/sbom.sh --version "$VERSION" --out dist --no-web - - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: sbom - path: dist/*.cdx.json - if-no-files-found: error - retention-days: 14 - - image: - name: image (${{ matrix.arch }}) - needs: [verify, test] - strategy: - fail-fast: false - matrix: - include: - - {arch: amd64, os: ubuntu-latest} - - {arch: arm64, os: ubuntu-22.04-arm} - runs-on: ${{ matrix.os }} - env: - PUBLISH: ${{ needs.verify.outputs.publish }} - VERSION: ${{ needs.verify.outputs.version }} - permissions: - contents: read - packages: write - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4.7.0 - if: needs.verify.outputs.publish == 'true' + if: steps.mode.outputs.publish == 'true' with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - # Native builds on native runners: an emulated Rust build would take hours. Publishing pushes each - # architecture by digest; a rehearsal exports a tarball instead. The manifest job merges them. - - name: Server image - id: server - uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 - with: - context: . - file: deploy/docker/Dockerfile - target: server - platforms: linux/${{ matrix.arch }} - build-args: | - AG_BUILD_COMMIT=${{ github.sha }} - AG_BUILD_DATE=${{ needs.verify.outputs.commit_date }} - labels: | - org.opencontainers.image.source=https://github.com/Paffin/AgentGit - org.opencontainers.image.version=${{ needs.verify.outputs.version }} - org.opencontainers.image.revision=${{ github.sha }} - org.opencontainers.image.licenses=Apache-2.0 - cache-from: | - type=gha,scope=agentgit-image - type=gha,scope=agentgit-image-release-${{ matrix.arch }} - cache-to: type=gha,scope=agentgit-image-release-${{ matrix.arch }},mode=max - provenance: ${{ needs.verify.outputs.publish == 'true' && 'mode=max' || 'false' }} - sbom: ${{ needs.verify.outputs.publish == 'true' }} - outputs: ${{ needs.verify.outputs.publish == 'true' && format('type=image,name={0},push-by-digest=true,name-canonical=true,push=true', env.REGISTRY_IMAGE_SERVER) || format('type=docker,dest={0}/server-{1}.tar,name=agentgit-server:{2}', runner.temp, matrix.arch, matrix.arch) }} - - name: CLI image - id: cli - uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 - with: - context: . - file: deploy/docker/Dockerfile - target: cli - platforms: linux/${{ matrix.arch }} - build-args: | - AG_BUILD_COMMIT=${{ github.sha }} - AG_BUILD_DATE=${{ needs.verify.outputs.commit_date }} - labels: | - org.opencontainers.image.source=https://github.com/Paffin/AgentGit - org.opencontainers.image.version=${{ needs.verify.outputs.version }} - org.opencontainers.image.revision=${{ github.sha }} - org.opencontainers.image.licenses=Apache-2.0 - cache-from: type=gha,scope=agentgit-image-release-${{ matrix.arch }} - provenance: ${{ needs.verify.outputs.publish == 'true' && 'mode=max' || 'false' }} - sbom: ${{ needs.verify.outputs.publish == 'true' }} - outputs: ${{ needs.verify.outputs.publish == 'true' && format('type=image,name={0},push-by-digest=true,name-canonical=true,push=true', env.REGISTRY_IMAGE_CLI) || format('type=docker,dest={0}/cli-{1}.tar,name=agentgit-cli:{2}', runner.temp, matrix.arch, matrix.arch) }} - - name: Rehearsal - the native image starts, is healthy and reports its version - if: needs.verify.outputs.publish != 'true' - run: | - set -euo pipefail - docker load -i "$RUNNER_TEMP/server-${{ matrix.arch }}.tar" - docker load -i "$RUNNER_TEMP/cli-${{ matrix.arch }}.tar" - docker image inspect agentgit-server:${{ matrix.arch }} --format '{{.Architecture}} {{.Size}}' - docker run --rm agentgit-server:${{ matrix.arch }} --version - docker run --rm agentgit-cli:${{ matrix.arch }} --version - docker run -d --name ag -e AG_BOOTSTRAP_TOKEN="$(openssl rand -hex 32)" agentgit-server:${{ matrix.arch }} - for _ in $(seq 1 60); do - [ "$(docker inspect -f '{{.State.Health.Status}}' ag)" = healthy ] && break - sleep 2 - done - test "$(docker inspect -f '{{.State.Health.Status}}' ag)" = healthy || { docker logs ag; exit 1; } - - name: Keep the result for the manifest job - run: | - mkdir -p "$RUNNER_TEMP/handoff" - if [ "$PUBLISH" = true ]; then - digest="${{ steps.server.outputs.digest }}"; touch "$RUNNER_TEMP/handoff/server-${digest#sha256:}" - digest="${{ steps.cli.outputs.digest }}"; touch "$RUNNER_TEMP/handoff/cli-${digest#sha256:}" - else - cp "$RUNNER_TEMP"/server-${{ matrix.arch }}.tar "$RUNNER_TEMP"/cli-${{ matrix.arch }}.tar "$RUNNER_TEMP/handoff/" - fi - ls -la "$RUNNER_TEMP/handoff" - - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: image-${{ matrix.arch }} - path: ${{ runner.temp }}/handoff - if-no-files-found: error - retention-days: 3 - - manifest: - name: multi-arch manifest - needs: [verify, image] - runs-on: ubuntu-latest - env: - PUBLISH: ${{ needs.verify.outputs.publish }} - VERSION: ${{ needs.verify.outputs.version }} - PRERELEASE: ${{ needs.verify.outputs.prerelease }} - permissions: - contents: read - packages: write - id-token: write - attestations: write - steps: - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - pattern: image-* - path: handoff - - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - - uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4.7.0 - if: needs.verify.outputs.publish == 'true' - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Publish - merge the per-architecture digests into one tag - if: needs.verify.outputs.publish == 'true' - run: | - set -euo pipefail - for kind in server cli; do - image=$([ "$kind" = server ] && echo "$REGISTRY_IMAGE_SERVER" || echo "$REGISTRY_IMAGE_CLI") - tags="-t $image:$VERSION" - [ "$PRERELEASE" = true ] || tags="$tags -t $image:latest" - refs="" - for f in handoff/image-*/"$kind"-*; do refs="$refs $image@sha256:${f##*/"$kind"-}"; done - # shellcheck disable=SC2086 - docker buildx imagetools create $tags $refs - docker buildx imagetools inspect "$image:$VERSION" - done - - name: Rehearsal - the same merge, in a throw-away local registry - if: needs.verify.outputs.publish != 'true' - run: | - set -euo pipefail - docker run -d --name registry -p 127.0.0.1:5000:5000 registry:2 - for _ in $(seq 1 30); do curl -fsS http://127.0.0.1:5000/v2/ > /dev/null && break; sleep 1; done - for kind in server cli; do - refs="" - for arch in amd64 arm64; do - docker load -i "handoff/image-$arch/$kind-$arch.tar" - docker tag "agentgit-$kind:$arch" "localhost:5000/agentgit-$kind:$VERSION-$arch" - docker push "localhost:5000/agentgit-$kind:$VERSION-$arch" - refs="$refs localhost:5000/agentgit-$kind:$VERSION-$arch" - done - # shellcheck disable=SC2086 - docker buildx imagetools create -t "localhost:5000/agentgit-$kind:$VERSION" $refs - docker buildx imagetools inspect "localhost:5000/agentgit-$kind:$VERSION" | tee "inspect-$kind.txt" - grep -q 'linux/amd64' "inspect-$kind.txt" - grep -q 'linux/arm64' "inspect-$kind.txt" - done - echo "the merged manifest lists both architectures" - docker run --rm "localhost:5000/agentgit-server:$VERSION" --version - - name: Publish - attest the images - if: needs.verify.outputs.publish == 'true' - id: digests + - name: Promote immutable container manifests without rebuilding + if: steps.mode.outputs.publish == 'true' + env: + VERSION: ${{ steps.v.outputs.version }} + PRERELEASE: ${{ steps.v.outputs.prerelease }} + SERVER_DIGEST: ${{ steps.images.outputs.server }} + CLI_DIGEST: ${{ steps.images.outputs.cli }} run: | set -euo pipefail for kind in server cli; do - image=$([ "$kind" = server ] && echo "$REGISTRY_IMAGE_SERVER" || echo "$REGISTRY_IMAGE_CLI") - echo "$kind=$(docker buildx imagetools inspect "$image:$VERSION" --format '{{json .Manifest.Digest}}' | tr -d '"')" >> "$GITHUB_OUTPUT" - done - - uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 - if: needs.verify.outputs.publish == 'true' - with: - subject-name: ${{ env.REGISTRY_IMAGE_SERVER }} - subject-digest: ${{ steps.digests.outputs.server }} - push-to-registry: true - - uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 - if: needs.verify.outputs.publish == 'true' - with: - subject-name: ${{ env.REGISTRY_IMAGE_CLI }} - subject-digest: ${{ steps.digests.outputs.cli }} - push-to-registry: true - - release: - name: assemble the release - needs: [verify, scripts, binaries, sbom, manifest] - runs-on: ubuntu-latest - env: - PUBLISH: ${{ needs.verify.outputs.publish }} - VERSION: ${{ needs.verify.outputs.version }} - DRY_RUN: ${{ needs.verify.outputs.dry_run }} - GH_TOKEN: ${{ github.token }} - permissions: - contents: write # only used when publishing - id-token: write - attestations: write - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - path: dist - pattern: "{archive-*,sbom}" - merge-multiple: true - - name: Checksums and the Homebrew formula - run: | - set -euo pipefail - ls -la dist - set -- dist/*.tar.gz; test $# = 4 - set -- dist/*.cdx.json; test $# = 4 - python3 scripts/package_release.py sums dist/*.tar.gz dist/*.cdx.json | tee dist/SHA256SUMS - (cd dist && sha256sum -c SHA256SUMS) - python3 scripts/render_packaging.py --version "$VERSION" --sums dist/SHA256SUMS --out dist --only homebrew - ruby -c dist/agentgit.rb - - uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 - id: attest - with: - subject-path: | - dist/*.tar.gz - dist/*.cdx.json - dist/SHA256SUMS - - name: The attestations verify - run: | - set -euo pipefail - for f in dist/*.tar.gz dist/*.cdx.json dist/SHA256SUMS; do - gh attestation verify "$f" --repo "$GITHUB_REPOSITORY" \ - --signer-workflow "$GITHUB_REPOSITORY/.github/workflows/release.yml" > /dev/null - echo "attested: $f" + if [ "$kind" = server ]; then image="$REGISTRY_IMAGE_SERVER"; digest="$SERVER_DIGEST"; + else image="$REGISTRY_IMAGE_CLI"; digest="$CLI_DIGEST"; fi + docker buildx imagetools create -t "$image:$VERSION" "$image@$digest" + if [ "$PRERELEASE" != true ]; then + docker buildx imagetools create -t "$image:latest" "$image@$digest" + fi done - - name: Release notes from the CHANGELOG - run: | - flag=""; [ "$DRY_RUN" = true ] && flag="--dry-run" - python3 scripts/check_release.py "v${VERSION#v}" $flag --notes > dist/RELEASE_NOTES.md - wc -l dist/RELEASE_NOTES.md - - name: Create the GitHub release + - name: Publish the exact prepared files + if: steps.mode.outputs.publish == 'true' + env: + VERSION: ${{ steps.v.outputs.version }} run: | set -euo pipefail flags="" case "$VERSION" in *-*) flags="--prerelease";; esac - files="$(echo dist/*.tar.gz) $(echo dist/*.cdx.json) dist/SHA256SUMS dist/agentgit.rb" - if [ "$PUBLISH" = true ]; then - # A release page can already exist (the GitHub UI creates one together with a tag): fill it in - # instead of failing on "already exists". - if gh release view "v$VERSION" > /dev/null 2>&1; then - # shellcheck disable=SC2086 - gh release upload "v$VERSION" $files --clobber - # shellcheck disable=SC2086 - gh release edit "v$VERSION" --title "AgentGit v$VERSION" --notes-file dist/RELEASE_NOTES.md --draft=false $flags - else - # shellcheck disable=SC2086 - gh release create "v$VERSION" $files --title "AgentGit v$VERSION" --notes-file dist/RELEASE_NOTES.md $flags --verify-tag - fi + if gh release view "v$VERSION" > /dev/null 2>&1; then + gh release upload "v$VERSION" dist/* --clobber + # shellcheck disable=SC2086 + gh release edit "v$VERSION" --title "AgentGit v$VERSION" --notes-file dist/RELEASE_NOTES.md --draft=false $flags else - echo "REHEARSAL - would run: gh release create v$VERSION --title 'AgentGit v$VERSION' --notes-file RELEASE_NOTES.md $flags " - echo "$files" + # shellcheck disable=SC2086 + gh release create "v$VERSION" dist/* --title "AgentGit v$VERSION" --notes-file dist/RELEASE_NOTES.md $flags --verify-tag fi - - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: release-dist - path: dist - retention-days: 14 - - name: Summary - run: | - title="Rehearsal: the release that would be published (nothing was)" - [ "$PUBLISH" = true ] && title="Published" - { - echo "## $title" - echo - echo '```' - (cd dist && wc -c -- * | grep -v ' total$') - echo '```' - echo '```' - cat dist/SHA256SUMS - echo '```' - } >> "$GITHUB_STEP_SUMMARY" install: + if: needs.publish.outputs.publish == 'true' name: install.sh (${{ matrix.os }}) - needs: [verify, release] + needs: publish + timeout-minutes: 2 strategy: fail-fast: false matrix: os: [ubuntu-latest, ubuntu-22.04-arm, macos-latest] runs-on: ${{ matrix.os }} env: - VERSION: ${{ needs.verify.outputs.version }} - CARGO_VERSION: ${{ needs.verify.outputs.cargo_version }} - PUBLISHED: ${{ needs.verify.outputs.publish }} + VERSION: ${{ needs.publish.outputs.version }} + CARGO_VERSION: ${{ needs.publish.outputs.cargo_version }} + PUBLISHED: ${{ needs.publish.outputs.publish }} GH_TOKEN: ${{ github.token }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: release-dist - path: dist - name: Install the version-pinned release, verifying checksum and attestation run: | set -euo pipefail prefix="$RUNNER_TEMP/prefix" # Real releases exercise the public download path on a fresh runner. Rehearsals use the same layout locally. - base_url="file://$PWD/dist" - if [ "$PUBLISHED" = true ]; then - base_url="https://github.com/Paffin/AgentGit/releases/download/v$VERSION" - fi + base_url="https://github.com/Paffin/AgentGit/releases/download/v$VERSION" uname -a if [ "${RUNNER_OS}" = macOS ]; then sw_vers; fi echo "Installing AgentGit $VERSION from $base_url" @@ -585,15 +186,6 @@ jobs: "$prefix/bin/ag-mcp" --version [ "$("$prefix/bin/orchestrator" version)" = "agent-orchestrator $CARGO_VERSION" ] case "$("$prefix/bin/ag" --version)" in "ag $CARGO_VERSION "*) ;; *) echo "::error::unexpected version"; exit 1;; esac - - name: A corrupted download is refused and installs nothing - run: | - set -euo pipefail - mkdir bad && cp dist/* bad/ - for f in bad/agentgit-*.tar.gz; do printf 'x' >> "$f"; done - if sh scripts/install.sh --version "$VERSION" --prefix "$RUNNER_TEMP/bad-prefix" --base-url "file://$PWD/bad" --no-attestation; then - echo "::error::install.sh accepted a corrupted archive"; exit 1 - fi - test ! -e "$RUNNER_TEMP/bad-prefix/bin" - name: The installed binaries work end to end run: | set -euo pipefail diff --git a/.github/workflows/request-release.yml b/.github/workflows/request-release.yml index e1e02db5..fdc79129 100644 --- a/.github/workflows/request-release.yml +++ b/.github/workflows/request-release.yml @@ -10,6 +10,11 @@ on: - "scripts/request_release.py" - ".github/workflows/request-release.yml" workflow_dispatch: + inputs: + ready_only: + description: "Publish a prepared candidate immediately; fail if it is not ready" + type: boolean + default: true permissions: contents: read @@ -35,4 +40,5 @@ jobs: env: GH_TOKEN: ${{ github.token }} RELEASE_SHA: ${{ github.sha }} + READY_ONLY: ${{ inputs.ready_only }} run: python3 scripts/request_release.py diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index 49725656..bbec5d52 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -1,41 +1,50 @@ name: Security -# Supply-chain and vulnerability gates. They run on every pull request and push to main, and once a day -# (the advisory databases change without any commit of ours). Policy, the exception process and how to reproduce -# each job locally: docs/operations/supply-chain.md. Accepted findings, with an owner and an expiry date: -# security/exceptions.toml. -# -# What each job guards: -# cargo-deny Rust: RustSec advisories (also yanked/unmaintained/unsound), licences against the Apache-2.0 -# policy, duplicate and wildcard dependencies, crates.io as the only source. Both the -# workspace and the fuzz harness (its own workspace). -# npm / pip / go the web UI, the TypeScript SDK, the Python SDK and the Go orchestrator: fail on a high or -# critical advisory (any advisory for Python, whose scanner reports no severity; any REACHABLE -# vulnerability for Go). -# dependency-review the change a pull request makes to the dependency graph (pull requests only). -# sbom the CycloneDX SBOMs the release publishes build and are well-formed (scripts/sbom.sh). -# hygiene workflow pinning/permissions, expiry of accepted exceptions, THIRD_PARTY_LICENSES freshness. - +# Dependency changes receive path-scoped audits. Qualification runs every audit and licence/SBOM check. on: push: branches: [main] pull_request: - schedule: - - cron: "23 4 * * *" workflow_dispatch: + inputs: + full: + type: boolean + default: true + workflow_call: + inputs: + full: + type: boolean + default: true permissions: contents: read concurrency: - group: security-${{ github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + group: security-${{ github.ref }}-${{ inputs.full && 'full' || 'fast' }} + cancel-in-progress: true env: CARGO_TERM_COLOR: always jobs: + changes: + runs-on: ubuntu-latest + outputs: + rust_deps: ${{ steps.paths.outputs.rust_deps }} + npm_deps: ${{ steps.paths.outputs.npm_deps }} + python_deps: ${{ steps.paths.outputs.python_deps }} + go_deps: ${{ steps.paths.outputs.go_deps }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + - id: paths + run: python3 scripts/ci_changes.py + cargo-deny: + needs: changes + if: inputs.full || needs.changes.outputs.rust_deps == 'true' name: cargo-deny (${{ matrix.manifest }}) runs-on: ubuntu-latest timeout-minutes: 15 @@ -54,6 +63,8 @@ jobs: command-arguments: --hide-inclusion-graph npm-audit: + needs: changes + if: inputs.full || needs.changes.outputs.npm_deps == 'true' name: npm audit (${{ matrix.dir }}) runs-on: ubuntu-latest timeout-minutes: 10 @@ -83,6 +94,8 @@ jobs: python3 "$GITHUB_WORKSPACE/scripts/security_exceptions.py" npm "$RUNNER_TEMP/npm-all.json" pip-audit: + needs: changes + if: inputs.full || needs.changes.outputs.python_deps == 'true' name: pip-audit (sdk/python) runs-on: ubuntu-latest timeout-minutes: 10 @@ -100,6 +113,8 @@ jobs: python3 "$GITHUB_WORKSPACE/scripts/security_exceptions.py" pip "$RUNNER_TEMP/pip-audit.json" govulncheck: + needs: changes + if: inputs.full || needs.changes.outputs.go_deps == 'true' name: govulncheck (services/agent-orchestrator) runs-on: ubuntu-latest timeout-minutes: 15 @@ -167,6 +182,7 @@ jobs: comment-summary-in-pr: never sbom: + if: inputs.full name: SBOMs build (scripts/sbom.sh) runs-on: ubuntu-latest timeout-minutes: 20 @@ -178,6 +194,8 @@ jobs: with: toolchain: stable - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + cache-workspace-crates: true - uses: taiki-e/install-action@1e29709c0ec63babbe8f266397a465f29a825f91 # v2.87.31 with: tool: cargo-cyclonedx@0.5.9 @@ -187,6 +205,7 @@ jobs: run: test -z "$(git status --porcelain)" hygiene: + if: inputs.full name: workflows, exceptions, licence file runs-on: ubuntu-latest timeout-minutes: 20 @@ -213,6 +232,8 @@ jobs: with: toolchain: stable - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + cache-workspace-crates: true - uses: taiki-e/install-action@1e29709c0ec63babbe8f266397a465f29a825f91 # v2.87.31 with: tool: cargo-about@0.9.2 diff --git a/.github/workflows/web.yml b/.github/workflows/web.yml index 45ad762e..6b599025 100644 --- a/.github/workflows/web.yml +++ b/.github/workflows/web.yml @@ -1,7 +1,6 @@ name: Web UI -# The Web UI (apps/web) has its own checks. They run when the UI, the API description it is generated from, -# or the server it is tested against change. +# Fast lint/types/unit/build feedback; full browser/identity-provider coverage during qualification. on: push: branches: [main] @@ -26,12 +25,22 @@ on: - "scripts/test_web_e2e_workflow.py" - "tests/skip-allowlist.toml" - ".github/workflows/web.yml" + workflow_call: + inputs: + full: + type: boolean + default: true + workflow_dispatch: + inputs: + full: + type: boolean + default: true permissions: contents: read concurrency: - group: web-${{ github.ref }} + group: web-${{ github.ref }}-${{ inputs.full && 'full' || 'fast' }} cancel-in-progress: true jobs: @@ -60,8 +69,10 @@ jobs: - run: npm test - run: npm run build - name: Keep the checked production UI for the server build + if: inputs.full run: tar -czf "$RUNNER_TEMP/web-dist.tar.gz" dist - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + if: inputs.full with: name: web-production-ui path: ${{ runner.temp }}/web-dist.tar.gz @@ -69,6 +80,7 @@ jobs: retention-days: 3 server-build: + if: inputs.full name: build the embedded server once runs-on: ubuntu-24.04 needs: checks @@ -84,6 +96,7 @@ jobs: toolchain: stable - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: + cache-workspace-crates: true shared-key: web-e2e - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: @@ -112,6 +125,7 @@ jobs: # accessibility specs at tablet and phone size (Chromium). Nothing is quarantined: a failure is a bug or a # documented engine difference (docs/guides/web-ui.md, "Verified where"). e2e: + if: inputs.full name: end-to-end (${{ matrix.project }}) runs-on: ubuntu-24.04 needs: [checks, server-build] @@ -120,6 +134,7 @@ jobs: options: --ipc=host timeout-minutes: 40 strategy: + max-parallel: 2 fail-fast: false matrix: include: @@ -195,6 +210,7 @@ jobs: # provider will not produce on request (a wrong state, a bad nonce, a forged signature); it is a test double and # is documented as one. identity-provider: + if: inputs.full name: real identity provider (Keycloak) runs-on: ubuntu-latest needs: checks @@ -220,6 +236,7 @@ jobs: toolchain: stable - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: + cache-workspace-crates: true shared-key: web-e2e - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 9f1d054a..c44062ea 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -1,13 +1,8 @@ name: Windows -# Windows is a supported platform for the CLI, the server and the clients (see docs/operations/windows.md). -# This is a separate workflow from ci.yml so that it can be iterated on without blocking anything; it becomes a -# required check once it has been green and stable on main. - +# Native Windows qualification; not scheduled by each ordinary edit. on: - push: - branches: [main] - pull_request: + workflow_call: workflow_dispatch: permissions: @@ -33,6 +28,8 @@ jobs: with: toolchain: stable - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + cache-workspace-crates: true - name: Show the toolchain and Git run: | rustc -Vv diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f8c6f0c..ac240f42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ on-disk formats are versioned and refuse to open when newer than the build under ## [Unreleased] +### Improved + +- Use component-specific fast CI for ordinary commits and cancel superseded work. Run the full platform/backend/browser/security matrix during qualification, reuse one server build for SDK runtimes, and retain compiled workspace caches. +- Separate release preparation from publication. Promote the exact prepared and attested archives/images without rebuilding or repeating the Rust suite, with a three-minute publishing job and two-minute parallel public-install checks. Cold preparation and runner queues remain outside the five-minute ready-candidate publication target. + ## [0.3.0-beta.7] - 2026-10-29 **Upgrade:** metadata schema 8 and control schema are unchanged from beta.6. Back up before upgrading. Returning to beta.5 or older requires the pre-upgrade backup. diff --git a/docs/architecture/testing.md b/docs/architecture/testing.md index 8443f836..7358b848 100644 --- a/docs/architecture/testing.md +++ b/docs/architecture/testing.md @@ -1,5 +1,14 @@ # Testing +## Continuous feedback and qualification + +Normal PRs and main commits use path-aware fast CI: Linux Rust unit/core-storage contracts, SDK checks against one shared server build when API/SDK files change, Go contracts when the executor changes, and offline workflow checks when tooling changes. Web changes run lint, types, unit tests and a production build; documentation changes build the site. Unrelated changes do not start Rust or cross-platform builds. Unknown paths or unavailable Git history conservatively select all fast components. + +The full matrix runs through `qualification.yml` on demand, nightly, and during release preparation: macOS, Windows, Linux coverage (which also executes the full Linux suite), PostgreSQL/S3/HA, all SDK runtimes, CLI examples, all browser engines/viewports, CodeQL, fuzzing, OCI runtimes, packaging and performance checks. SDK/runtime jobs share one verified server/CLI/MCP build. Coverage supplies Linux full-suite correctness, so it is not also run as a separate identical Linux test job. + +Stale branch runs are cancelled. Release publication promotes prepared, attested bytes and does not add a third full Rust run. See [Releasing](../operations/releasing.md) for the ready-candidate contract and publication time budget. + + What the tests of AgentGit are, which of them run where, how much of the code they reach, how many wrong versions of the code they notice, and what a green build still does not tell you. The commands are the ones CI runs; every number below was measured on the date given and is reproducible with the command next to it. @@ -177,7 +186,7 @@ remains is listed here, each with the reason it cannot be real, and exists only the mutation runs above show where it was weak and was fixed; nothing here shows the absence of bugs. * **Coverage is measured for Rust only**; the TypeScript, Python and Go code is tested but has no coverage number. Mutation testing covers five crates, not the server, the engine or the CLI. -* **Not run on every push:** the nightly fuzzing, the weekly mutation runs and CodeQL's weekly schedule. A change +* **Not run on every push:** full qualification, fuzzing, mutation testing, cross-platform integration and CodeQL. A change can be merged between two of them. * **Platforms:** Linux and macOS run the whole suite; Windows runs the CLI, client and server ([Windows](../operations/windows.md)). The namespace sandbox is Linux-only. Real Docker and Podman runtimes are exercised only where diff --git a/docs/operations/performance.md b/docs/operations/performance.md index 96e181cf..8be6d989 100644 --- a/docs/operations/performance.md +++ b/docs/operations/performance.md @@ -182,7 +182,7 @@ gRPC clients got through far more refused requests per second in this run; this [`perf.yml`](../../.github/workflows/perf.yml) runs on changes to the server, the engine and the tool: -* **smoke** (every push to `main` and pull request touching the tool, server, integration, merge or tree code): the +* **smoke** (during full qualification): the engine benchmark at 1 000 and 10 000 files and `ag-bench load --profile smoke`, release builds. Failed runs, including integrity-check failures, fail the job; latency and throughput have no thresholds. The results are in the job summary and uploaded as the `perf-smoke` artifact, including on failure when available. diff --git a/docs/operations/releasing.md b/docs/operations/releasing.md index c1fa1228..2735fd20 100644 --- a/docs/operations/releasing.md +++ b/docs/operations/releasing.md @@ -1,77 +1,46 @@ # Releasing -A release is a tag `vX.Y.Z` (or `vX.Y.Z-rc.1`) on `main`. Pushing the tag runs -[`.github/workflows/release.yml`](../../.github/workflows/release.yml), which builds, checks and **publishes** it. -The same workflow can be started by hand as a **rehearsal** that does everything except publish; that is how it -is tested, and how you should test changes to it (see [Rehearsing](#rehearsing-a-release-without-publishing)). +A release has two distinct stages. **Preparation** builds and qualifies a candidate for an exact main commit. **Publication** promotes those already tested bytes; it never compiles or repeats the Rust suite. -## What a tag does +## Prepare a candidate -| Stage | What happens | -|---|---| -| **verify** | `scripts/check_release.py` fails the run unless the tag equals the workspace version in `Cargo.toml` **and** `CHANGELOG.md` has a non-empty `## [X.Y.Z]` section. | -| **release tooling tests** | `scripts/test_release_scripts.py` and `scripts/test_sbom_release.py`: the version check, the archive builder, the Homebrew/Scoop renderer and `install.sh` (with a tampered download, a missing checksum, a wrong platform …). | -| **tests** | `cargo test --workspace --locked` (with unprivileged user namespaces enabled and the sandbox suites required, as in CI). | -| **binaries** | `ag`, `ag-server`, `ag-worker` and `ag-mcp` built `--release --locked` natively on Linux x86_64 (`ubuntu-22.04`, so glibc ≥ 2.35), Linux aarch64 (`ubuntu-22.04-arm`), macOS aarch64 and macOS x86_64, and `orchestrator` built natively with Go (`CGO_ENABLED=0`, version injected at link time), then packed by `scripts/package_release.py` into `agentgit-X.Y.Z-.tar.gz` (the five binaries, `LICENSE`, `NOTICE`, `THIRD_PARTY_LICENSES.md`, `README.md`; the archive is deterministic). Every archive is then **unpacked and run**: the `--version` strings of the first three must carry the version and this commit (`ag-mcp` has no build script: its crate version only; `orchestrator version` must match the workspace version), and the end-to-end smoke ([`scripts/e2e_smoke.py`](../../scripts/e2e_smoke.py): create a repository, grant, mint a credential, `ag clone`, snapshot, `ag push`, integrate, read back over HTTP, check metrics) runs against the unpacked `ag-server`. | -| **sbom** | One CycloneDX 1.5 SBOM per shipped Rust binary (`cargo cyclonedx --describe binaries`, from `Cargo.lock`). The Go executor dependency inventory and license text are included in `THIRD_PARTY_LICENSES.md`; no Go CycloneDX SBOM is claimed. | -| **images** | `ghcr.io/paffin/agentgit-server` and `…/agentgit-cli`: each architecture is built **natively** (`ubuntu-latest` for amd64, `ubuntu-22.04-arm` for arm64; an emulated Rust build would take hours) from [`deploy/docker/Dockerfile`](../../deploy/docker/Dockerfile) and pushed by digest, then **merged into one multi-arch tag** (`:X.Y.Z`, plus `:latest` for a non-prerelease) with SBOM and max-mode provenance attached. | -| **release** | `SHA256SUMS` over the archives and SBOMs, the rendered Homebrew formula (`agentgit.rb`), GitHub build-provenance attestations for every file and image (and a `gh attestation verify` of each in the same run), release notes cut from the CHANGELOG section, and the GitHub release (a pre-release when the tag has a `-suffix`). | -| **install** | `scripts/install.sh` installs the assembled release on Linux x86_64, Linux aarch64 and macOS with `--require-attestation`; a corrupted archive must be refused and install nothing; the installed binaries run the end-to-end smoke. | - -## Rehearsing a release without publishing +[Prepare release](../../.github/workflows/prepare-release.yml) runs the complete [Qualification](../../.github/workflows/qualification.yml) matrix alongside four native Linux/macOS builds and both container architectures. Native archives are unpacked and exercised, SBOMs and checksums are generated, and every file is attested. The installer then runs against the assembled archives on three fresh runners. -*Actions → Release → Run workflow*, or from a terminal: +Successful preparation retains `release-dist` for 14 days. `candidate.json` binds its version, source SHA and preparation run ID; `candidate-images.json` records immutable container manifest digests. The candidate image tags use the source commit, not a public version or `latest`. ```sh -gh workflow run release.yml --ref main -f version=0.0.0-rc.dry -f publish=false +gh workflow run prepare-release.yml --ref main ``` -`version` is only checked for being semver (it need not match `Cargo.toml`; the archives are named after it, the -binaries inside report the `Cargo.toml` version), and release notes fall back to `## [Unreleased]` when the -CHANGELOG has no section of that name. `publish=true` is honoured **only** when the run is on the tag `vX.Y.Z` -that `version` names — on a branch it fails immediately with "Nothing was published". Everything that would -write to the outside world (`docker push` to GHCR, `gh release create`) is behind that one flag, which the -`verify` job computes and every later job reads. +Use `-f force=true` to rehearse preparation of an already published version. This does not replace a GitHub release or move its version tags. Preparation can take tens of minutes on a cold cache, particularly Intel macOS; it runs separately from continuous development checks. -What the rehearsal proves, and how it differs from the real thing: +## Publish a ready candidate -| Step | In a rehearsal | In a real release | -|---|---|---| -| Version, CHANGELOG, workspace tests, four native builds, archives, SBOMs, `SHA256SUMS`, formula rendering | **identical** | identical | -| Attestations | really created and then really verified with `gh attestation verify` (Sigstore, this workflow's identity) | identical | -| Images | built natively for amd64 **and** arm64, started and health-checked; exported as tarballs | pushed by digest to GHCR | -| Multi-arch manifest | merged with `docker buildx imagetools create` in a throw-away **local registry** and inspected: it must list `linux/amd64` and `linux/arm64`, and the merged tag must run | merged on GHCR, then attested | -| GitHub release | the command is printed; all files are uploaded as the workflow artifact `release-dist` | `gh release create` | -| `install.sh` | installs `release-dist` through a `file://` base URL | would download from the release URL | +```sh +# Fails immediately if this exact main commit has no complete, unexpired candidate. +gh workflow run request-release.yml --ref main -f ready_only=true +``` -Evidence (all on `main`'s code, nothing published): see [Evidence](#evidence) at the end of this page. +The request creates the immutable version tag only after validating a successful preparation for that exact commit. [Release](../../.github/workflows/release.yml) downloads that run's immutable artifact, verifies its complete file list, checksums, source/version/run binding and attestations, and promotes the recorded container digests. It uploads the same files to GitHub and verifies installation from the public release URL on Linux x86_64, Linux arm64 and macOS arm64. -## Cutting a release +The publication job has a **3-minute** execution limit and its parallel installation jobs have **2-minute** limits. The target is under five minutes for publication of a **ready** candidate. Runner queue time and cold preparation are separate; a missing candidate or failed check is an error, never an excuse to build silently during publication. A timeout is a failed release run, not a successful five-minute release. + +To verify the fast path without publishing: ```sh -# 1. bump Cargo.toml workspace/package versions, apps/web/package.json, sdk/typescript/package.json -# and sdk/python/pyproject.toml plus sdk/python/src/agentgit/__init__.py; refresh Cargo.lock and npm lock roots -# Set the three default image tags under deploy/ to the same candidate version. -cargo update -w -(cd sdk/python && uv lock) -scripts/third_party_licenses.sh -# 2. move [Unreleased] into a dated [X.Y.Z] section of CHANGELOG.md -# 3. check what the workflow will check -python3 scripts/check_release.py vX.Y.Z --notes -python3 scripts/check_publish_metadata.py -cargo build -p agentgit-cli --locked -AG=target/debug/ag python3 scripts/gen_cli_docs.py -cargo test --workspace --locked -# 4. rehearse on the exact commit you are about to tag -gh workflow run release.yml --ref main -f version=X.Y.Z-rc.dry -f publish=false -# 5. merge the reviewed version bump into main -# Request release waits for that exact commit's checks, then creates the immutable tag and dispatches Release. -# Observe both workflows; only a completed Release with its assets is a published version. +gh workflow run release.yml --ref main -f version=X.Y.Z -f publish=false ``` -Once a version tag is pinned, its release continues from that commit while main can advance. Later requests for the same version report the pinned commit without moving the tag or waiting for unrelated checks; bump the version to request another release. The publication check expects the four Rust SBOMs actually shipped; the native Go executor is inside each platform archive. +This downloads and verifies the prepared candidate but does not move public image tags, create a release, or run public-install checks. For actual publication, `publish=true` is accepted only on the matching `vX.Y.Z` tag. Directly pushing an unprepared tag fails closed. -Use a prerelease suffix for release candidates and betas. The published beta remains the installation target until the next release has completed; see [What is not proven](#what-is-not-proven). +## Cutting a release + +1. Update the workspace, npm and Python versions and lock files; set deployment defaults, regenerate CLI/protocol documentation and dependency notices. +2. Add a nonempty, dated version section in `CHANGELOG.md` and merge the reviewed candidate to `main`. +3. The automatic Request release workflow starts or reuses preparation in the background, then publishes when ready. Ordinary commits continue through fast, component-specific CI. If main moves before the tag is made, the old request stops. +4. Observe preparation and publication separately. Only a complete Release run, including public-install checks, is a verified release. + +A manually requested `ready_only=true` publication never waits for preparation. Published versions are skipped and existing tags are never moved. An expired artifact must be prepared again. Retry a failed publication on its existing tag using the same prepared source commit. ## Verifying what you downloaded @@ -118,7 +87,7 @@ no token is stored in the repository or the workflow. One-time setup on each reg repository `AgentGit`, workflow `publish-sdks.yml`, environment `pypi`. Both names were free on 2026-10-17 (`scripts/check_publish_metadata.py --online`). The -[`Packaging`](../../.github/workflows/packaging.yml) workflow runs the build-and-check half on every change. +[`Packaging`](../../.github/workflows/packaging.yml) workflow runs the build-and-check half during qualification. ## crates.io @@ -267,13 +236,7 @@ runs themselves (`Release`, `Containers`, `Packaging`) are in the Actions tab wi ## Release requests from main -A reviewed version bump on `main`, with matching package versions and a nonempty changelog section, requests the next release automatically. `Request release` waits for successful CI, CodeQL, Containers, Docs, Packaging, Quality, Security, Web UI and Windows runs for that exact push, plus any other workflows triggered by it. A newer main commit cancels the request; a failed or timed-out check creates no tag. - -After the checks pass, the request creates `v` at the checked commit and dispatches the existing Release workflow on that tag. That workflow still builds, tests, attests and verifies the actual archives and images before publication. The request itself is not a successful release. Existing published versions are skipped; an existing tag is never moved. If a request failed after creating its tag, rerun Request release on the same main commit. If main has moved, run Release manually on the existing tag with its matching version. Rehearsals remain available through Release with `publish=false`. - -The request uses a write token only on trusted main pushes or a manual run on main; it has no pull-request or workflow-run trigger and does not persist checkout credentials. - -Required checks are identified by their workflow file paths, so another workflow using the same display name cannot hide a failed or unfinished check. Optional workflows triggered by the same commit must finish successfully too. +A reviewed version bump on main starts or reuses Prepare release for that exact commit. Preparation includes the full qualification matrix, native builds, provenance and installation tests. The request stops if main moves, and cannot create a tag after a failed preparation. Publication only consumes an immutable successful candidate; it does not repeat the full test suite or rebuild archives. Manual publication accepts a version with or without its leading `v`, but validates the actual tag selected in the workflow. A branch or a different version is refused before publishing. Tests execute the workflow’s mode-selection shell block and pass its result to the real version/changelog validator. diff --git a/docs/operations/supply-chain.md b/docs/operations/supply-chain.md index 2c9c35fd..3ae12955 100644 --- a/docs/operations/supply-chain.md +++ b/docs/operations/supply-chain.md @@ -2,7 +2,7 @@ What AgentGit checks about the code it depends on, where each check runs, how to reproduce it on a laptop, and what is *not* covered. The workflow is [`.github/workflows/security.yml`](https://github.com/Paffin/AgentGit/blob/main/.github/workflows/security.yml); -it runs on every push to `main`, on every pull request and once a day (the advisory databases change without any +dependency changes get immediate, path-scoped audits; full audits run in qualification and nightly (the advisory databases change without any commit of ours), and can be started by hand. ## The gates @@ -71,7 +71,7 @@ is pinned to a commit of `master` and selects the toolchain with its `toolchain: [`.github/workflows/codeql.yml`](https://github.com/Paffin/AgentGit/blob/main/.github/workflows/codeql.yml) analyses the four languages of the repository (Rust, JavaScript/TypeScript, Python, Go) with the `security-extended` query suite -on every push to `main`, every pull request and once a week (the queries change without any commit of ours). Rust, +during full qualification and nightly (the queries change without any commit of ours). Rust, TypeScript and Python are analysed without a build; the Go module is built by hand because it sits in a subdirectory. Generated files are excluded (`.github/codeql/codeql-config.yml`), tests are not. diff --git a/scripts/ci_changes.py b/scripts/ci_changes.py new file mode 100644 index 00000000..a3dbcc73 --- /dev/null +++ b/scripts/ci_changes.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Select fast CI jobs from the complete Git diff; unknown/missing history runs all jobs.""" +from __future__ import annotations + +import json +import os +import re +from pathlib import Path +import subprocess + +FLAGS = ('rust', 'sdk', 'go', 'tooling', 'rust_deps', 'npm_deps', 'python_deps', 'go_deps') + + +def classify(paths): + selected = dict.fromkeys(FLAGS, False) + for path in paths: + if path.startswith('.github/'): + return dict.fromkeys(FLAGS, True) + known = False + if path.startswith(('crates/', 'proto/', 'migrations/', '.cargo/')) or path in ('Cargo.toml', 'Cargo.lock'): + selected['rust'] = True + known = True + if path.startswith(('sdk/', 'proto/', 'crates/ag-server/', 'crates/ag-protocol/')) or path == 'docs/protocol/openapi.json': + selected['sdk'] = True + known = True + if path.startswith('services/agent-orchestrator/'): + selected['go'] = True + known = True + if path.startswith(('scripts/', 'tests/', 'security/', 'deploy/', 'examples/')): + selected['tooling'] = True + known = True + if path.endswith(('Cargo.toml', 'Cargo.lock')) or path == 'deny.toml': + selected['rust_deps'] = True + if path.endswith(('package.json', 'package-lock.json')): + selected['npm_deps'] = True + if path.startswith('sdk/python/') and path.endswith(('pyproject.toml', 'uv.lock')): + selected['python_deps'] = True + if path.startswith('services/agent-orchestrator/') and path.endswith(('go.mod', 'go.sum', 'Dockerfile')): + selected['go_deps'] = True + if path.startswith(('apps/web/', 'docs/', 'deploy/', 'packaging/', 'examples/')): + known = True # Web and Docs have their own path-filtered workflows. + if path.endswith(('.md', '.txt')) or path in ('.gitignore', '.dockerignore', 'LICENSE', 'NOTICE', 'mkdocs.yml'): + known = True + if not known: + return dict.fromkeys(FLAGS, True) + return selected + + +def changed_paths(event, name): + if name == 'pull_request': + base = event['pull_request']['base']['sha'] + if not re.fullmatch(r'[0-9a-f]{40}', base): + return None + revision = base + '...HEAD' + elif name == 'push' and event.get('before', '').strip('0'): + if not re.fullmatch(r'[0-9a-f]{40}', event['before']): + return None + revision = event['before'] + '..HEAD' + else: + return None + try: + result = subprocess.run(['git', 'diff', '--no-renames', '--name-only', '-z', revision, '--'], check=True, capture_output=True) + except (OSError, subprocess.CalledProcessError): + return None + return result.stdout.decode('utf-8', errors='surrogateescape').rstrip('\0').split('\0') if result.stdout else [] + + +def main(): + try: + event = json.loads(Path(os.environ['GITHUB_EVENT_PATH']).read_text()) + paths = changed_paths(event, os.environ.get('GITHUB_EVENT_NAME', '')) + except (OSError, KeyError, ValueError): + paths = None + flags = dict.fromkeys(FLAGS, True) if paths is None else classify(paths) + output = ''.join(f'{key}={str(value).lower()}\n' for key, value in flags.items()) + print(output, end='') + if os.environ.get('GITHUB_OUTPUT'): + with open(os.environ['GITHUB_OUTPUT'], 'a') as stream: + stream.write(output) + + +if __name__ == '__main__': + main() diff --git a/scripts/prepare_release.py b/scripts/prepare_release.py new file mode 100644 index 00000000..8887773a --- /dev/null +++ b/scripts/prepare_release.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +"""Decide whether a release candidate needs preparing; never publishes a version.""" +from __future__ import annotations + +import os +import subprocess + +import check_release +from request_release import GitHub, is_published + + +def main(): + version = check_release.workspace_version() + check_release.resolve('v' + version, False) + api = GitHub(os.environ['GH_TOKEN']) + release = api.optional('/repos/' + os.environ['GITHUB_REPOSITORY'] + '/releases/tags/v' + version) + needed = os.environ.get('FORCE') == 'true' or not is_published(release, version) + # check_release.resolve returns the release facts; the CLI is the stable output contract. + output = subprocess.check_output(['python3', 'scripts/check_release.py', 'v' + version], text=True) + output += 'needed=' + str(needed).lower() + '\n' + output += 'commit_date=' + subprocess.check_output(['git', 'show', '-s', '--format=%cs', 'HEAD'], text=True) + output += 'epoch=' + subprocess.check_output(['git', 'show', '-s', '--format=%ct', 'HEAD'], text=True) + with open(os.environ['GITHUB_OUTPUT'], 'a') as stream: + stream.write(output) + print('Preparing ' + version if needed else version + ' is already published; nothing to prepare') + + +if __name__ == '__main__': + main() diff --git a/scripts/release_candidate.py b/scripts/release_candidate.py new file mode 100644 index 00000000..4ac8ea89 --- /dev/null +++ b/scripts/release_candidate.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Manifest and byte-level verification for prebuilt, immutable release candidates.""" +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +import re +import sys + +from request_release import GitHub, required_assets, resolve_candidate, validate_request + +EXTRA = {'candidate.json', 'candidate-images.json', 'RELEASE_NOTES.md'} + + +def validate(directory, sha, version, run_id): + validate_request('candidate/source', sha, version) + if type(run_id) not in (str, int) or not str(run_id).isdigit() or int(run_id) <= 0: + raise ValueError('invalid preparation run') + directory = Path(directory) + expected = required_assets(version) | EXTRA + actual = {p.name for p in directory.iterdir() if p.is_file()} + if actual != expected or any(p.is_symlink() or not p.is_file() for p in directory.iterdir()): + raise ValueError('candidate files differ from the required release set') + metadata = json.loads((directory / 'candidate.json').read_text()) + if type(metadata.get('run_id')) is not int or metadata != {'sha': sha, 'version': version, 'run_id': int(run_id)}: + raise ValueError('candidate source, version or preparation run does not match') + images = json.loads((directory / 'candidate-images.json').read_text()) + if images.get('sha') != sha or images.get('version') != version: + raise ValueError('container images belong to another candidate') + if set(images) != {'sha', 'version', 'server', 'cli'}: + raise ValueError('unexpected image metadata') + for kind in ('server', 'cli'): + if not re.fullmatch(r'sha256:[0-9a-f]{64}', images[kind]): + raise ValueError('invalid container digest') + sums = {} + for line in (directory / 'SHA256SUMS').read_text().splitlines(): + match = re.fullmatch(r'([0-9a-f]{64}) ([A-Za-z0-9_.-]+)', line) + if not match or match[2] in sums: + raise ValueError('invalid or duplicate checksum entry') + sums[match[2]] = match[1] + if set(sums) != expected - {'SHA256SUMS'}: + raise ValueError('checksums do not cover the complete candidate') + for name, digest in sums.items(): + content = (directory / name).read_bytes() + if not content or hashlib.sha256(content).hexdigest() != digest: + raise ValueError('candidate checksum mismatch: ' + name) + return images + + +def create(directory, sha, version, run_id): + validate_request(os.environ.get('GITHUB_REPOSITORY', 'test/repo'), sha, version) + directory = Path(directory) + (directory / 'candidate.json').write_text(json.dumps({'sha': sha, 'version': version, 'run_id': int(run_id)})) + expected = required_assets(version) | EXTRA + if {p.name for p in directory.iterdir()} != expected: + raise ValueError('incomplete candidate') + sums = ''.join(hashlib.sha256((directory / name).read_bytes()).hexdigest() + ' ' + name + '\n' + for name in sorted(expected - {'SHA256SUMS'})) + (directory / 'SHA256SUMS').write_text(sums) + validate(directory, sha, version, run_id) + + +def main(): + mode = sys.argv[1] + if mode == 'resolve': + run, _ = resolve_candidate(GitHub(os.environ['GH_TOKEN']), os.environ['GITHUB_REPOSITORY'], os.environ['GITHUB_SHA']) + with open(os.environ['GITHUB_OUTPUT'], 'a') as stream: + stream.write('run_id=' + str(run['id']) + '\n') + return + args = (sys.argv[2], os.environ['GITHUB_SHA'], os.environ['VERSION'], os.environ.get('PREPARE_RUN_ID', os.environ['GITHUB_RUN_ID'])) + if mode == 'create': + create(*args) + elif mode == 'verify': + images = validate(*args) + with open(os.environ['GITHUB_OUTPUT'], 'a') as stream: + for kind in ('server', 'cli'): + stream.write(kind + '=' + images[kind] + '\n') + else: + raise ValueError('unknown candidate operation') + + +if __name__ == '__main__': + main() diff --git a/scripts/request_release.py b/scripts/request_release.py index cba0c4dc..2456ff7b 100644 --- a/scripts/request_release.py +++ b/scripts/request_release.py @@ -1,4 +1,4 @@ -"""Request the existing release pipeline only for a fully checked main commit.""" +"""Prepare a main commit in the background, or publish its ready candidate immediately.""" from __future__ import annotations import json @@ -8,18 +8,7 @@ import urllib.error import urllib.request -REQUIRED = { - '.github/workflows/ci.yml': 'CI', - '.github/workflows/codeql.yml': 'CodeQL', - '.github/workflows/containers.yml': 'Containers', - '.github/workflows/docs.yml': 'Docs', - '.github/workflows/packaging.yml': 'Packaging', - '.github/workflows/quality.yml': 'Quality', - '.github/workflows/security.yml': 'Security', - '.github/workflows/web.yml': 'Web UI', - '.github/workflows/windows.yml': 'Windows', -} -IGNORED = frozenset(('.github/workflows/request-release.yml', '.github/workflows/release.yml')) +PREPARE_WORKFLOW = '.github/workflows/prepare-release.yml' TARGETS = ('x86_64-unknown-linux-gnu', 'aarch64-unknown-linux-gnu', 'aarch64-apple-darwin', 'x86_64-apple-darwin') @@ -55,26 +44,31 @@ def post(self, path, body): return self.request('POST', path, body) -def checks_pending(runs): - latest = {} - for run in sorted(runs, key=lambda item: item['id'], reverse=True): - if run['path'] not in IGNORED: - latest.setdefault(run['path'], run) - failed = [run['name'] + ' (' + path + ')' for path, run in latest.items() - if run['status'] == 'completed' and run['conclusion'] != 'success'] - if failed: - raise RuntimeError('release blocked by checks: ' + ', '.join(sorted(failed))) - pending = set(REQUIRED).difference(latest) - pending = pending.union(path for path, run in latest.items() if run['status'] != 'completed') - return sorted(REQUIRED[path] if path in REQUIRED else latest[path]['name'] + ' (' + path + ')' - for path in pending) +def candidate_run(runs, sha=None): + matches = [run for run in runs + if run.get('path') == PREPARE_WORKFLOW + and run.get('event') in ('workflow_dispatch', 'schedule') + and run.get('head_branch') == 'main' + and (sha is None or run.get('head_sha') == sha)] + return max(matches, key=lambda run: run['id'], default=None) + + +def checks_pending(runs, sha=None): + run = candidate_run(runs, sha) + if run is None: + return ['Prepared release candidate'] + if run['status'] != 'completed': + return ['Prepared release candidate'] + if run['conclusion'] != 'success': + raise RuntimeError('release blocked by candidate preparation: ' + str(run['conclusion'])) + return [] def workflow_runs(api, base, sha): runs = [] page = 1 while True: - result = api.get(f'{base}/actions/runs?head_sha={sha}&event=push&per_page=100&page={page}') + result = api.get(f'{base}/actions/runs?head_sha={sha}&per_page=100&page={page}') batch = result['workflow_runs'] runs.extend(batch) if len(batch) < 100: @@ -88,16 +82,38 @@ def is_current(api, base, sha): def wait_checks(api, base, sha, *, timeout=5400, sleep=time.sleep, clock=time.monotonic): deadline = clock() + timeout + runs = workflow_runs(api, base, sha) + if candidate_run(runs, sha) is None: + api.post(base + '/actions/workflows/prepare-release.yml/dispatches', {'ref': 'main', 'inputs': {'force': 'false'}}) while is_current(api, base, sha): - pending = checks_pending(workflow_runs(api, base, sha)) - if not pending: + if not checks_pending(workflow_runs(api, base, sha), sha): + resolve_candidate(api, base.removeprefix('/repos/'), sha) return if clock() >= deadline: - raise RuntimeError('release checks did not finish: ' + ', '.join(pending)) - print('Waiting for: ' + ', '.join(pending), flush=True) + raise RuntimeError('release preparation did not finish') + print('Preparing release candidate for ' + sha[:12], flush=True) sleep(20) +def resolve_candidate(api, repo, sha): + validate_request(repo, sha, '0.0.0') + base = '/repos/' + repo + runs = workflow_runs(api, base, sha) + if checks_pending(runs, sha): + raise RuntimeError('No ready candidate for this exact commit; run Prepare release first') + run = candidate_run(runs, sha) + page = 1 + while True: + batch = api.get(f"{base}/actions/runs/{run['id']}/artifacts?per_page=100&page={page}")['artifacts'] + for artifact in batch: + if artifact['name'] == 'release-dist' and not artifact.get('expired', True) and artifact.get('size_in_bytes', 0) > 0: + return run, artifact + if len(batch) < 100: + break + page += 1 + raise RuntimeError('Prepared candidate is missing or expired; prepare it again') + + def tag_commit(api, base, ref): obj = ref['object'] for _ in range(8): @@ -142,7 +158,7 @@ def is_published(release, version): return required_assets(version).issubset(uploaded) -def request_release(api, repo, sha, version, wait=None): +def request_release(api, repo, sha, version, wait=None, *, ready_only=False): validate_request(repo, sha, version) base, tag = '/repos/' + repo, 'v' + version release = api.optional(base + '/releases/tags/' + tag) @@ -155,7 +171,10 @@ def request_release(api, repo, sha, version, wait=None): return f'{tag} already pinned to {pinned[:12]}; bump the version to release another commit' if not is_current(api, base, sha): return 'main moved; no release requested' - (wait or wait_checks)(api, base, sha) + if ready_only: + resolve_candidate(api, repo, sha) + else: + (wait or wait_checks)(api, base, sha) if not is_current(api, base, sha): return 'main moved; no release requested' ensure_tag(api, base, tag, sha) @@ -170,7 +189,7 @@ def main(): version = check_release.workspace_version() check_release.resolve('v' + version, False) result = request_release(GitHub(os.environ['GH_TOKEN']), os.environ['GITHUB_REPOSITORY'], - os.environ['RELEASE_SHA'], version) + os.environ['RELEASE_SHA'], version, ready_only=os.environ.get('READY_ONLY') == 'true') print(result) diff --git a/scripts/test_ci_policy.py b/scripts/test_ci_policy.py new file mode 100644 index 00000000..47cfd37e --- /dev/null +++ b/scripts/test_ci_policy.py @@ -0,0 +1,149 @@ +"""Safety and scope contracts for fast CI and exact-commit release qualification.""" +from __future__ import annotations + +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +import yaml + +from ci_changes import FLAGS, changed_paths, classify + +ROOT = Path(__file__).resolve().parent.parent + + +def workflow(name): + return yaml.safe_load((ROOT / '.github/workflows' / (name + '.yml')).read_text()) + + +class ChangeSelection(unittest.TestCase): + def test_docs_do_not_compile_rust_or_sdks(self): + flags = classify(['README.md', 'docs/guides/agents.md']) + self.assertFalse(any(flags.values())) + + def test_web_code_does_not_launch_backend_compilation(self): + self.assertFalse(any(classify(['apps/web/src/views/HomeView.vue']).values())) + self.assertTrue(classify(['apps/web/package-lock.json'])['npm_deps']) + + def test_api_changes_check_sdk_contracts(self): + flags = classify(['crates/ag-server/src/api/proposals.rs']) + self.assertTrue(flags['rust']) + self.assertTrue(flags['sdk']) + self.assertFalse(flags['go']) + + def test_known_component_changes_are_scoped(self): + self.assertTrue(classify(['services/agent-orchestrator/internal/app/run.go'])['go']) + self.assertTrue(classify(['sdk/python/uv.lock'])['python_deps']) + self.assertTrue(classify(['Cargo.lock'])['rust_deps']) + self.assertTrue(classify(['deploy/compose/compose.yaml'])['tooling']) + + def test_workflow_and_unknown_paths_fail_closed(self): + for path in ('.github/workflows/ci.yml', 'future-runtime/code.rs'): + self.assertEqual(classify([path]), dict.fromkeys(FLAGS, True)) + + def test_deleted_and_renamed_files_are_both_classified(self): + old = os.getcwd() + with tempfile.TemporaryDirectory() as directory: + os.chdir(directory) + try: + def git(*args): + return subprocess.check_output(['git', *args], stderr=subprocess.DEVNULL).decode().strip() + git('init', '-q') + git('config', 'user.email', 'ci@example.invalid') + git('config', 'user.name', 'CI test') + Path('crates/test').mkdir(parents=True) + Path('crates/test/code.rs').write_text('old code') + git('add', '.') + git('commit', '-qm', 'base') + base = git('rev-parse', 'HEAD') + git('mv', 'crates/test/code.rs', 'README.md') + git('commit', '-qm', 'rename') + paths = changed_paths({'before': base}, 'push') + self.assertIn('crates/test/code.rs', paths) + self.assertTrue(classify(paths)['rust']) + self.assertIsNone(changed_paths({'before': '0' * 40}, 'push')) + self.assertIsNone(changed_paths({'before': '--bad'}, 'push')) + self.assertIsNone(changed_paths({'before': 'f' * 40}, 'push')) + finally: + os.chdir(old) + + +class PipelinePolicy(unittest.TestCase): + def test_slow_suites_are_not_triggered_on_every_commit(self): + for name in ('full-ci', 'qualification', 'codeql', 'containers', 'windows', + 'quality', 'packaging', 'oci-runtimes', 'fuzz', 'perf'): + doc = workflow(name) + triggers = doc.get('on', doc.get(True)) + self.assertNotIn('push', triggers, name) + self.assertNotIn('pull_request', triggers, name) + self.assertTrue(doc['concurrency']['cancel-in-progress'], name) + + def test_quick_ci_is_path_aware_and_has_an_always_reported_gate(self): + jobs = workflow('ci')['jobs'] + self.assertIn('ci_changes.py', jobs['changes']['steps'][-1]['run']) + self.assertEqual(jobs['complete']['if'], 'always()') + self.assertEqual(set(jobs['complete']['needs']), set(jobs) - {'complete'}) + self.assertEqual(jobs['rust']['runs-on'], 'ubuntu-24.04') + commands = '\n'.join(step.get('run', '') for step in jobs['rust']['steps']) + self.assertIn('--lib --bins', commands) + self.assertNotIn('cargo test --workspace --locked', commands) + self.assertIn("--test '*'", commands) + + def test_linux_full_tests_are_already_the_coverage_run(self): + self.assertEqual(workflow('full-ci')['jobs']['test']['strategy']['matrix']['os'], ['macos-latest']) + coverage = workflow('quality')['jobs']['coverage'] + self.assertIn('cargo llvm-cov --workspace', '\n'.join(s.get('run', '') for s in coverage['steps'])) + + def test_shared_tools_are_built_once_for_sdk_runtime_matrix(self): + jobs = workflow('full-ci')['jobs'] + for name in ('sdk', 'sdk-runtimes', 'orchestrator', 'mcp'): + self.assertEqual(jobs[name]['needs'], 'tools') + commands = '\n'.join(s.get('run', '') for s in jobs[name]['steps']) + self.assertNotIn('cargo build', commands) + self.assertIn('sha256sum --check', commands) + + def test_browser_matrix_is_preserved_but_explicit(self): + jobs = workflow('web')['jobs'] + for job in ('server-build', 'e2e', 'identity-provider'): + self.assertEqual(jobs[job]['if'], 'inputs.full') + self.assertEqual(len(jobs['e2e']['strategy']['matrix']['include']), 6) + + def test_release_only_promotes_with_a_five_minute_execution_budget(self): + jobs = workflow('release')['jobs'] + self.assertEqual(set(jobs), {'publish', 'install'}) + self.assertEqual(jobs['publish']['timeout-minutes'] + jobs['install']['timeout-minutes'], 5) + commands = '\n'.join(step.get('run', '') for job in jobs.values() for step in job['steps']) + self.assertNotIn('cargo ', commands) + self.assertIn('release_candidate.py resolve', commands) + self.assertIn('release_candidate.py verify', commands) + self.assertIn('--source-digest "$GITHUB_SHA"', commands) + self.assertEqual(jobs['install']['needs'], 'publish') + prepare = workflow('prepare-release')['jobs'] + self.assertIn('qualify', prepare['release']['needs']) + self.assertEqual(prepare['qualify']['uses'], './.github/workflows/qualification.yml') + for job in ('binaries', 'image'): + self.assertEqual(prepare[job]['needs'], 'verify') + + def test_preparation_cannot_publish_public_versions(self): + doc = workflow('prepare-release') + self.assertNotIn('push', doc.get('on', doc.get(True))) + commands = '\n'.join(step.get('run', '') for job in doc['jobs'].values() for step in job.get('steps', [])) + self.assertNotIn('gh release create', commands) + self.assertNotIn('gh release upload', commands) + self.assertNotIn('$image:$VERSION', commands) + self.assertIn('$image:candidate-$GITHUB_SHA', commands) + self.assertTrue(doc['concurrency']['cancel-in-progress']) + + def test_fast_ci_never_publishes_and_release_jobs_are_not_cancelled(self): + ci = workflow('ci') + self.assertEqual(ci['permissions'], {'contents': 'read'}) + self.assertTrue(ci['concurrency']['cancel-in-progress']) + self.assertFalse(workflow('release')['concurrency']['cancel-in-progress']) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/test_release_candidate.py b/scripts/test_release_candidate.py new file mode 100644 index 00000000..5796ab9c --- /dev/null +++ b/scripts/test_release_candidate.py @@ -0,0 +1,115 @@ +"""A ready candidate must be complete, immutable and bound to the exact source/run.""" +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +import tempfile +import unittest + +from release_candidate import create, validate +from request_release import required_assets, resolve_candidate, request_release +from test_request_release import Api, BASE, REPO, SHA, green + +VERSION = '1.2.3' + + +class CandidateFiles(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.root = Path(self.temp.name) + for name in required_assets(VERSION): + (self.root / name).write_bytes(b'checked artifact') + (self.root / 'RELEASE_NOTES.md').write_text('Release notes') + (self.root / 'candidate-images.json').write_text(json.dumps({ + 'sha': SHA, 'version': VERSION, 'server': 'sha256:' + 'a' * 64, 'cli': 'sha256:' + 'b' * 64, + })) + create(self.root, SHA, VERSION, 7) + + def tearDown(self): + self.temp.cleanup() + + def test_complete_candidate_round_trip(self): + result = validate(self.root, SHA, VERSION, 7) + self.assertEqual(result['server'], 'sha256:' + 'a' * 64) + + def test_wrong_source_version_or_run_is_rejected(self): + for args in (('b' * 40, VERSION, 7), (SHA, '1.2.4', 7), (SHA, VERSION, 8)): + with self.assertRaises(ValueError): + validate(self.root, *args) + + def test_tampering_or_missing_archive_is_rejected(self): + archive = next(self.root.glob('*.tar.gz')) + archive.write_bytes(b'tampered') + with self.assertRaisesRegex(ValueError, 'checksum'): + validate(self.root, SHA, VERSION, 7) + archive.unlink() + with self.assertRaises(ValueError): + validate(self.root, SHA, VERSION, 7) + + def test_path_traversal_duplicates_extra_files_and_symlinks_are_rejected(self): + sums = self.root / 'SHA256SUMS' + original = sums.read_text() + for suffix in ('0' * 64 + ' ../outside\n', original.splitlines()[0] + '\n'): + sums.write_text(original + suffix) + with self.assertRaises(ValueError): + validate(self.root, SHA, VERSION, 7) + sums.write_text(original) + (self.root / 'extra.txt').write_text('not part of the candidate') + with self.assertRaises(ValueError): + validate(self.root, SHA, VERSION, 7) + (self.root / 'extra.txt').unlink() + archive = next(self.root.glob('*.tar.gz')) + archive.unlink() + archive.symlink_to(self.root / 'RELEASE_NOTES.md') + with self.assertRaises(ValueError): + validate(self.root, SHA, VERSION, 7) + + def test_invalid_image_reference_is_rejected_even_with_updated_checksum(self): + path = self.root / 'candidate-images.json' + value = json.loads(path.read_text()) + value['server'] = 'untrusted.example/image:latest' + path.write_text(json.dumps(value)) + with self.assertRaisesRegex(ValueError, 'digest'): + create(self.root, SHA, VERSION, 7) + + +class CandidateAPI(unittest.TestCase): + def api(self, artifact=None): + api = Api() + api.values[BASE + '/actions/runs?head_sha=' + SHA + '&per_page=100&page=1'] = {'workflow_runs': green()} + api.values[BASE + '/actions/runs/1/artifacts?per_page=100&page=1'] = { + 'artifacts': [artifact] if artifact else [], + } + return api + + def test_expired_or_missing_artifact_cannot_be_published(self): + for artifact in (None, dict(name='release-dist', expired=True, size_in_bytes=5), + dict(name='release-dist', expired=False, size_in_bytes=0)): + api = self.api(artifact) + with self.assertRaisesRegex(RuntimeError, 'missing or expired'): + request_release(api, REPO, SHA, VERSION, ready_only=True) + self.assertEqual(api.writes, []) + + def test_ready_only_does_not_dispatch_hidden_preparation(self): + api = self.api(dict(name='release-dist', expired=False, size_in_bytes=5)) + request_release(api, REPO, SHA, VERSION, ready_only=True) + self.assertEqual(len(api.writes), 2) # immutable tag, then fast publication + self.assertFalse(any('prepare-release.yml/dispatches' in path for path, _ in api.writes)) + api.values[BASE + '/actions/runs?head_sha=' + SHA + '&per_page=100&page=1'] = {'workflow_runs': []} + api.writes.clear() + with self.assertRaisesRegex(RuntimeError, 'No ready candidate'): + request_release(api, REPO, SHA, VERSION, ready_only=True) + self.assertEqual(api.writes, []) + + def test_artifact_pagination_is_checked(self): + api = self.api() + api.values[BASE + '/actions/runs/1/artifacts?per_page=100&page=1'] = {'artifacts': [dict(name='other')] * 100} + api.values[BASE + '/actions/runs/1/artifacts?per_page=100&page=2'] = { + 'artifacts': [dict(name='release-dist', expired=False, size_in_bytes=5)], + } + self.assertEqual(resolve_candidate(api, REPO, SHA)[0]['id'], 1) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/test_request_release.py b/scripts/test_request_release.py index e263077d..484d0997 100644 --- a/scripts/test_request_release.py +++ b/scripts/test_request_release.py @@ -13,13 +13,10 @@ REPO = 'acme/project' SHA = 'a' * 40 BASE = '/repos/' + REPO -REQUIRED = ['CI', 'CodeQL', 'Containers', 'Docs', 'Packaging', 'Quality', 'Security', 'Web UI', 'Windows'] -PATHS = dict(zip(REQUIRED, ['ci', 'codeql', 'containers', 'docs', 'packaging', 'quality', 'security', 'web', 'windows'])) - - def green(): - return [dict(id=i + 1, name=name, path='.github/workflows/' + PATHS[name] + '.yml', status='completed', conclusion='success') - for i, name in enumerate(REQUIRED)] + return [dict(id=1, name='Prepare release', path='.github/workflows/prepare-release.yml', + event='workflow_dispatch', head_branch='main', head_sha=SHA, + status='completed', conclusion='success')] class Api: @@ -44,14 +41,15 @@ def test_native_archive_binary_does_not_imply_a_separate_rust_sbom(self): self.assertNotIn('agentgit-0.3.0-beta.4-orchestrator.cdx.json', assets) self.assertIn('agentgit-0.3.0-beta.4-ag-mcp.cdx.json', assets) - def test_every_required_workflow_runs_for_a_release_changelog_change(self): - from request_release import REQUIRED as workflows + def test_qualification_includes_all_expensive_suites(self): root = Path(__file__).resolve().parent.parent - for workflow in workflows: - config = yaml.load((root / workflow).read_text(), Loader=yaml.BaseLoader) - push = config["on"]["push"] - if "paths" in push: - self.assertIn("CHANGELOG.md", push["paths"], workflow) + doc = yaml.safe_load((root / '.github/workflows/qualification.yml').read_text()) + self.assertEqual(set(doc.get('on', doc.get(True))), {'workflow_call', 'workflow_dispatch', 'schedule'}) + jobs = doc['jobs'] + required = {'core', 'web', 'security', 'codeql', 'containers', 'docs', 'packaging', + 'coverage', 'windows', 'fuzz', 'performance', 'oci'} + self.assertEqual(set(jobs['complete']['needs']), required) + self.assertIn("!= 'success'", jobs['complete']['steps'][0]['run']) def test_checks_finish_before_tag_and_dispatch(self): api = Api() @@ -137,44 +135,44 @@ def test_annotated_tag_is_resolved_before_retrying(self): request_release(api, REPO, SHA, '0.3.0-beta.2', lambda *_: None) self.assertEqual(len(api.writes), 1) - def test_missing_and_pending_checks_are_not_success(self): - self.assertIn('Windows', checks_pending(green()[:-1])) + def test_missing_and_pending_qualification_are_not_success(self): + self.assertEqual(checks_pending([]), ['Prepared release candidate']) runs = green() runs[0].update(status='in_progress', conclusion=None) - self.assertIn('CI', checks_pending(runs)) - - def test_new_failed_run_cannot_be_hidden_by_old_success(self): - runs = green() + [dict(id=100, name='CI', path='.github/workflows/ci.yml', status='completed', conclusion='failure')] - with self.assertRaisesRegex(RuntimeError, 'CI'): - checks_pending(runs) + self.assertEqual(checks_pending(runs), ['Prepared release candidate']) - def test_all_triggered_workflows_must_pass(self): - runs = green() + [dict(id=100, name='Fuzz', path='.github/workflows/fuzz.yml', status='completed', conclusion='failure')] - with self.assertRaisesRegex(RuntimeError, 'Fuzz'): + def test_latest_failed_attempt_cannot_be_hidden_by_old_success(self): + runs = green() + [dict(green()[0], id=2, conclusion='failure')] + with self.assertRaisesRegex(RuntimeError, 'preparation'): checks_pending(runs) - self.assertEqual(checks_pending(green()), []) - def test_another_workflow_with_the_same_name_cannot_hide_failed_ci(self): - runs = green() - runs[0]['conclusion'] = 'failure' - runs.append(dict(id=100, name='CI', path='.github/workflows/other.yml', status='completed', conclusion='success')) - with self.assertRaisesRegex(RuntimeError, 'CI'): - checks_pending(runs) + def test_only_exact_trusted_qualification_is_accepted(self): + from request_release import resolve_candidate + key = BASE + '/actions/runs?head_sha=' + SHA + '&per_page=100&page=1' + for change in ({'head_sha': 'b' * 40}, {'head_branch': 'feature'}, + {'path': '.github/workflows/pretend.yml'}, {'event': 'pull_request'}, + {'conclusion': 'skipped'}, {'conclusion': 'cancelled'}): + api = Api() + api.values[key] = {'workflow_runs': [dict(green()[0], **change)]} + with self.assertRaises(RuntimeError): + resolve_candidate(api, REPO, SHA) + api = Api() + api.values[key] = {'workflow_runs': green()} + api.values[BASE + '/actions/runs/1/artifacts?per_page=100&page=1'] = {'artifacts': [dict(name='release-dist', size_in_bytes=1, expired=False)]} + resolve_candidate(api, REPO, SHA) - def test_only_the_actual_request_workflow_is_ignored(self): - runs = green() + [dict(id=100, name='Request release', path='.github/workflows/other.yml', status='completed', conclusion='failure')] - with self.assertRaisesRegex(RuntimeError, 'Request release'): - checks_pending(runs) + def test_unrelated_cancelled_fast_run_does_not_block_a_qualified_release(self): + runs = green() + [dict(id=5, path='.github/workflows/ci.yml', status='completed', conclusion='cancelled')] + self.assertEqual(checks_pending(runs, SHA), []) - def test_optional_pending_workflows_are_waited_for_and_identified_by_path(self): - for name in ('Fuzz', 'CI'): - with self.subTest(name=name): - runs = green() + [dict(id=100, name=name, path='.github/workflows/other.yml', status='in_progress', conclusion=None)] - self.assertEqual(checks_pending(runs), [name + ' (.github/workflows/other.yml)']) + def test_qualification_is_reused_without_dispatch(self): + api = Api() + key = BASE + '/actions/runs?head_sha=' + SHA + '&per_page=100&page=1' + api.values[key] = {'workflow_runs': green()} + api.values[BASE + '/actions/runs/1/artifacts?per_page=100&page=1'] = {'artifacts': [dict(name='release-dist', size_in_bytes=1, expired=False)]} + wait_checks(api, BASE, SHA, sleep=lambda _: self.fail('already qualified')) + self.assertEqual(api.writes, []) - def test_request_workflow_does_not_wait_for_itself(self): - runs = green() + [dict(id=100, name='Request release', path='.github/workflows/request-release.yml', status='in_progress', conclusion=None)] - self.assertEqual(checks_pending(runs), []) class ReleaseSafety(unittest.TestCase): @@ -199,32 +197,33 @@ def test_a_draft_release_is_not_treated_as_published(self): def test_pending_checks_timeout_without_creating_a_tag(self): api = Api() - api.values[BASE + '/actions/runs?head_sha=' + SHA + '&event=push&per_page=100&page=1'] = {'workflow_runs': []} + api.values[BASE + '/actions/runs?head_sha=' + SHA + '&per_page=100&page=1'] = {'workflow_runs': []} with self.assertRaisesRegex(RuntimeError, 'did not finish'): wait_checks(api, BASE, SHA, timeout=0, clock=lambda: 0, sleep=lambda _: self.fail('no sleep')) - self.assertEqual(api.writes, []) + self.assertEqual(api.writes, [(BASE + '/actions/workflows/prepare-release.yml/dispatches', {'ref': 'main', 'inputs': {'force': 'false'}})]) def test_all_workflow_pages_are_read(self): api = Api() - prefix = BASE + '/actions/runs?head_sha=' + SHA + '&event=push&per_page=100&page=' - api.values[prefix + '1'] = {'workflow_runs': green() * 11 + [green()[0]]} - api.values[prefix + '2'] = {'workflow_runs': [dict(id=100, name='Fuzz', path='.github/workflows/fuzz.yml', status='completed', conclusion='failure')]} + prefix = BASE + '/actions/runs?head_sha=' + SHA + '&per_page=100&page=' + api.values[prefix + '1'] = {'workflow_runs': green() * 100} + api.values[prefix + '2'] = {'workflow_runs': [dict(green()[0], id=100, conclusion='failure')]} runs = workflow_runs(api, BASE, SHA) self.assertEqual(len(runs), 101) - with self.assertRaisesRegex(RuntimeError, 'Fuzz'): + with self.assertRaisesRegex(RuntimeError, 'preparation'): checks_pending(runs) def test_wait_returns_only_after_all_checks_pass(self): api = Api() - key = BASE + '/actions/runs?head_sha=' + SHA + '&event=push&per_page=100&page=1' + key = BASE + '/actions/runs?head_sha=' + SHA + '&per_page=100&page=1' api.values[key] = {'workflow_runs': []} sleeps = [] visible = io.BytesIO() console = io.TextIOWrapper(visible, encoding='utf-8') def advance(delay): - self.assertIn(b'Waiting for: CI', visible.getvalue(), 'show progress before waiting') + self.assertIn(b'Preparing release candidate', visible.getvalue(), 'show progress before waiting') sleeps.append(delay) api.values[key] = {'workflow_runs': green()} + api.values[BASE + '/actions/runs/1/artifacts?per_page=100&page=1'] = {'artifacts': [dict(name='release-dist', size_in_bytes=1, expired=False)]} with contextlib.redirect_stdout(console): wait_checks(api, BASE, SHA, timeout=30, clock=lambda: 0, sleep=advance) self.assertEqual(sleeps, [20]) @@ -263,7 +262,7 @@ def test_workflow_is_main_only_and_never_checks_out_pull_request_code(self): self.assertEqual(checkout['ref'], '${{ github.sha }}') self.assertFalse(checkout['persist-credentials']) self.assertEqual(job['permissions'], {'contents': 'write', 'actions': 'write'}) - release = yaml.safe_load((root / '.github/workflows/release.yml').read_text()) + release = yaml.safe_load((root / '.github/workflows/prepare-release.yml').read_text()) targets = [row['target'] for row in release['jobs']['binaries']['strategy']['matrix']['include']] self.assertCountEqual(TARGETS, targets) diff --git a/scripts/test_web_e2e_workflow.py b/scripts/test_web_e2e_workflow.py index f3c3924f..bf3d1ce4 100644 --- a/scripts/test_web_e2e_workflow.py +++ b/scripts/test_web_e2e_workflow.py @@ -19,7 +19,7 @@ def setUp(self): def test_the_pipeline_contract_is_executed_by_ci(self): ci = yaml.safe_load((ROOT / '.github/workflows/ci.yml').read_text()) - commands = '\n'.join(step.get('run', '') for job in ci['jobs'].values() for step in job['steps']) + commands = '\n'.join(step.get('run', '') for job in ci['jobs'].values() for step in job.get('steps', [])) self.assertIn('test_web_e2e_workflow', commands) def test_runtime_matches_the_browser_library_and_is_immutable(self): @@ -32,9 +32,9 @@ def test_runtime_matches_the_browser_library_and_is_immutable(self): def test_identity_browser_runs_in_the_same_pinned_runtime(self): job = self.jobs['identity-provider'] self.assertEqual(job['env']['PLAYWRIGHT_IMAGE'], self.browser['container']['image']) - commands = '\n'.join(step.get('run', '') for step in job['steps']) + commands = '\n'.join(step.get('run', '') for step in job.get('steps', [])) self.assertNotIn('playwright install', commands) - execution = next(step['run'] for step in job['steps'] if 'keycloak.spec.ts' in step.get('run', '')) + execution = next(step['run'] for step in job.get('steps', []) if 'keycloak.spec.ts' in step.get('run', '')) with tempfile.TemporaryDirectory(prefix='browser runtime ') as directory: root = Path(directory) docker = root / 'docker' @@ -77,7 +77,7 @@ def test_browser_runtime_has_owned_home_and_external_ui_fixture(self): def test_browser_jobs_use_one_shared_embedded_server_build(self): builds = [name for name, job in self.jobs.items() if name != 'identity-provider' and any('cargo build -p agentgit-server -p agentgit-cli' in step.get('run', '') - for step in job['steps'])] + for step in job.get('steps', []))] self.assertEqual(len(builds), 1) self.assertIn(builds[0], self.browser['needs']) self.assertEqual(self.jobs[builds[0]]['runs-on'], 'ubuntu-24.04') diff --git a/tests/skip-allowlist.toml b/tests/skip-allowlist.toml index 0d9ae938..a5d07378 100644 --- a/tests/skip-allowlist.toml +++ b/tests/skip-allowlist.toml @@ -27,28 +27,28 @@ component = "postgres" jobs = ["ci/test-*", "quality/coverage", "windows/windows"] reason = "The ordinary test jobs have no PostgreSQL server; every suite that needs one runs in the backends job." owner = "storage" -runs_for_real = "ci.yml backends (scripts/pg-test.sh; AG_REQUIRE_COMPONENTS=postgres,s3)" +runs_for_real = "full-ci.yml backends (scripts/pg-test.sh; AG_REQUIRE_COMPONENTS=postgres,s3)" [[skip]] component = "postgres:tls" jobs = ["ci/test-*", "quality/coverage", "windows/windows"] reason = "No PostgreSQL server in this job; the TLS legs need one with ssl=on." owner = "storage" -runs_for_real = "ci.yml backends (scripts/pg-test.sh starts a server with a private CA)" +runs_for_real = "full-ci.yml backends (scripts/pg-test.sh starts a server with a private CA)" [[skip]] component = "s3" jobs = ["ci/test-*", "quality/coverage", "windows/windows"] reason = "The ordinary test jobs have no S3 server; every suite that needs one runs in the backends job." owner = "storage" -runs_for_real = "ci.yml backends (moto, then SeaweedFS; AG_REQUIRE_COMPONENTS=postgres,s3)" +runs_for_real = "full-ci.yml backends (moto, then SeaweedFS; AG_REQUIRE_COMPONENTS=postgres,s3)" [[skip]] component = "s3-tls" jobs = ["ci/test-*", "quality/coverage", "windows/windows", "ci/backends-moto"] reason = "moto serves plain HTTP only; the TLS tests of the S3 client need a server with a private CA." owner = "storage" -runs_for_real = "ci.yml backends, the SeaweedFS legs (AG_REQUIRE_COMPONENTS includes s3-tls there)" +runs_for_real = "full-ci.yml backends, the SeaweedFS legs (AG_REQUIRE_COMPONENTS includes s3-tls there)" [[skip]] component = "keycloak" @@ -76,14 +76,14 @@ component = "shell:*" jobs = ["ci/test-*", "ci/backends-*", "quality/coverage", "windows/windows"] reason = "zsh, fish and PowerShell are not on every runner image; the shells job installs them." owner = "cli" -runs_for_real = "ci.yml shells (AG_REQUIRE_COMPONENTS=shell)" +runs_for_real = "full-ci.yml shells (AG_REQUIRE_COMPONENTS=shell)" [[skip]] component = "namespace-sandbox" jobs = ["ci/test-macos", "windows/windows"] reason = "The namespace sandbox is Linux-only; macOS and Windows have no user namespaces." owner = "sandbox" -runs_for_real = "ci.yml test (ubuntu; AG_REQUIRE_COMPONENTS includes namespace-sandbox) and the cgroup job" +runs_for_real = "full-ci.yml test (ubuntu; AG_REQUIRE_COMPONENTS includes namespace-sandbox) and the cgroup job" [[skip]] component = "mkdocs" @@ -99,7 +99,7 @@ component = "model" jobs = ["ci/orchestrator"] reason = "Live-model tests are opt-in by design: they cost money and need a secret that forks and pull requests do not have." owner = "orchestrator" -runs_for_real = "ci.yml orchestrator, the live step (a push, when OPENROUTER_API_KEY and AG_LLM_MODEL are defined; AG_REQUIRE_COMPONENTS=ag-server-binary,model)" +runs_for_real = "full-ci.yml orchestrator, the live step (a push, when OPENROUTER_API_KEY and AG_LLM_MODEL are defined; AG_REQUIRE_COMPONENTS=ag-server-binary,model)" [[skip]] component = "n/a:host-privilege" @@ -120,7 +120,7 @@ component = "n/a:s3-clock" jobs = ["ci/backends-*"] reason = "S3 objects age on the server's clock, which a test cannot move forward; the guarantee is tested in real time by the _on_s3 counterpart." owner = "storage" -runs_for_real = "ci.yml backends: rewriting_an_old_unreachable_object_saves_it_from_a_concurrent_gc_on_s3" +runs_for_real = "full-ci.yml backends: rewriting_an_old_unreachable_object_saves_it_from_a_concurrent_gc_on_s3" [[skip]] component = "n/a:doc-pages-needing-ag-server" @@ -134,14 +134,14 @@ component = "platform:windows" jobs = ["windows/windows"] reason = "The harnesses drive bash; Windows has the PowerShell demo (the_powershell_demo_script_passes) instead." owner = "cli" -runs_for_real = "ci.yml test (ubuntu, macOS): the same tests with bash; windows.yml runs the PowerShell variants" +runs_for_real = "full-ci.yml test (ubuntu, macOS): the same tests with bash; windows.yml runs the PowerShell variants" [[skip]] component = "n/a:python-version" jobs = ["ci/sdk*"] reason = "typing.get_overloads, which this test inspects, exists from Python 3.11; the SDK supports 3.10." owner = "sdk" -runs_for_real = "ci.yml sdk and sdk-runtimes on Python 3.11 and later" +runs_for_real = "full-ci.yml sdk and sdk-runtimes on Python 3.11 and later" # ---- skip constructs that are not the helper ------------------------------------------------------------------ From 63f0fd72fbcfb9847945c45d1b8fbbc8cd79568a Mon Sep 17 00:00:00 2001 From: MAxim Kiktev Date: Thu, 29 Oct 2026 21:32:41 +0000 Subject: [PATCH 2/5] ci: free superseded runner capacity and retain branch-local build caches --- .github/workflows/ci.yml | 19 +++++++++++- scripts/cancel_stale_ci.py | 49 +++++++++++++++++++++++++++++++ scripts/test_ci_policy.py | 2 +- scripts/test_release_candidate.py | 15 ++++++++++ 4 files changed, 83 insertions(+), 2 deletions(-) create mode 100644 scripts/cancel_stale_ci.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2941ef5f..9edb6882 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,6 +21,24 @@ env: CARGO_PROFILE_TEST_DEBUG: "0" jobs: + superseded: + name: release stale runner capacity + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + runs-on: ubuntu-24.04 + timeout-minutes: 2 + continue-on-error: true + permissions: + contents: read + actions: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Cancel old validation and candidate builds (never Release) + env: + GH_TOKEN: ${{ github.token }} + run: python3 scripts/cancel_stale_ci.py + changes: runs-on: ubuntu-24.04 outputs: @@ -78,7 +96,6 @@ jobs: with: shared-key: linux-debug cache-workspace-crates: true - save-if: ${{ github.ref == 'refs/heads/main' }} - name: Formatting and code lint if: needs.changes.outputs.rust == 'true' run: cargo fmt --all --check && cargo clippy --workspace --lib --bins --locked -- -D warnings diff --git a/scripts/cancel_stale_ci.py b/scripts/cancel_stale_ci.py new file mode 100644 index 00000000..fc0ef855 --- /dev/null +++ b/scripts/cancel_stale_ci.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +"""A trusted main push cancels obsolete validation/preparation, never published releases.""" +from __future__ import annotations + +import os +import urllib.error + +from request_release import GitHub + +CANCELLABLE = frozenset('.github/workflows/' + name + '.yml' for name in ( + 'qualification', 'prepare-release', 'full-ci', 'codeql', 'containers', 'docs', 'fuzz', + 'oci-runtimes', 'packaging', 'perf', 'quality', 'security', 'web', 'windows', +)) + + +def cancel_obsolete(api, repo, sha): + base = '/repos/' + repo + cancelled = [] + seen = set() + for status in ('queued', 'in_progress', 'pending'): + page = 1 + while True: + batch = api.get(f'{base}/actions/runs?branch=main&status={status}&per_page=100&page={page}')['workflow_runs'] + for run in batch: + if (run['id'] in seen or run.get('head_branch') != 'main' or run.get('head_sha') == sha + or run.get('path') not in CANCELLABLE or run.get('status') == 'completed'): + continue + seen.add(run['id']) + try: + api.post(f"{base}/actions/runs/{run['id']}/cancel", {}) + cancelled.append(run['id']) + except urllib.error.HTTPError as error: + if error.code != 409: # already completed between listing and cancellation + raise + if len(batch) < 100: + break + page += 1 + return cancelled + + +def main(): + if os.environ.get('GITHUB_EVENT_NAME') != 'push' or os.environ.get('GITHUB_REF') != 'refs/heads/main': + raise RuntimeError('stale-run cancellation is main-push-only') + result = cancel_obsolete(GitHub(os.environ['GH_TOKEN']), os.environ['GITHUB_REPOSITORY'], os.environ['GITHUB_SHA']) + print('Cancelled obsolete validation/preparation runs:', result) + + +if __name__ == '__main__': + main() diff --git a/scripts/test_ci_policy.py b/scripts/test_ci_policy.py index 47cfd37e..1c5756f3 100644 --- a/scripts/test_ci_policy.py +++ b/scripts/test_ci_policy.py @@ -86,7 +86,7 @@ def test_quick_ci_is_path_aware_and_has_an_always_reported_gate(self): jobs = workflow('ci')['jobs'] self.assertIn('ci_changes.py', jobs['changes']['steps'][-1]['run']) self.assertEqual(jobs['complete']['if'], 'always()') - self.assertEqual(set(jobs['complete']['needs']), set(jobs) - {'complete'}) + self.assertEqual(set(jobs['complete']['needs']), set(jobs) - {'complete', 'superseded'}) self.assertEqual(jobs['rust']['runs-on'], 'ubuntu-24.04') commands = '\n'.join(step.get('run', '') for step in jobs['rust']['steps']) self.assertIn('--lib --bins', commands) diff --git a/scripts/test_release_candidate.py b/scripts/test_release_candidate.py index 5796ab9c..2db28f15 100644 --- a/scripts/test_release_candidate.py +++ b/scripts/test_release_candidate.py @@ -74,6 +74,21 @@ def test_invalid_image_reference_is_rejected_even_with_updated_checksum(self): create(self.root, SHA, VERSION, 7) +class StaleRuns(unittest.TestCase): + def test_only_old_main_validation_is_cancelled(self): + from cancel_stale_ci import cancel_obsolete + api = Api() + def run(number, **kwargs): + return dict(id=number, path='.github/workflows/prepare-release.yml', + head_branch='main', head_sha='b' * 40, status='in_progress', **kwargs) + rows = [run(1), dict(run(2), head_sha=SHA), dict(run(3), head_branch='feature'), + dict(run(4), path='.github/workflows/release.yml'), dict(run(5), status='completed')] + for status in ('queued', 'in_progress', 'pending'): + api.values[BASE + '/actions/runs?branch=main&status=' + status + '&per_page=100&page=1'] = {'workflow_runs': rows} + self.assertEqual(cancel_obsolete(api, REPO, SHA), [1]) + self.assertEqual(api.writes, [(BASE + '/actions/runs/1/cancel', {})]) + + class CandidateAPI(unittest.TestCase): def api(self, artifact=None): api = Api() From 15cf82b6d936c61b64246e5c33cd5eee4ee3cfda Mon Sep 17 00:00:00 2001 From: MAxim Kiktev Date: Thu, 29 Oct 2026 21:33:50 +0000 Subject: [PATCH 3/5] ci: preserve pending checks across cancellations and isolate background validation --- .github/workflows/ci.yml | 6 ++++++ .github/workflows/codeql.yml | 2 +- .github/workflows/containers.yml | 2 +- .github/workflows/docs.yml | 2 +- .github/workflows/full-ci.yml | 2 +- .github/workflows/fuzz.yml | 2 +- .github/workflows/oci-runtimes.yml | 2 +- .github/workflows/packaging.yml | 2 +- .github/workflows/perf.yml | 2 +- .github/workflows/qualification.yml | 2 +- .github/workflows/quality.yml | 2 +- .github/workflows/security.yml | 8 +++++++- .github/workflows/web.yml | 2 +- .github/workflows/windows.yml | 2 +- docs/architecture/testing.md | 2 +- scripts/cancel_stale_ci.py | 4 ++-- scripts/ci_changes.py | 22 +++++++++++++++++++++- scripts/test_ci_policy.py | 14 +++++++++++++- 18 files changed, 62 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9edb6882..4b2e7dee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,6 +40,9 @@ jobs: run: python3 scripts/cancel_stale_ci.py changes: + permissions: + contents: read + actions: read runs-on: ubuntu-24.04 outputs: rust: ${{ steps.paths.outputs.rust }} @@ -52,6 +55,9 @@ jobs: fetch-depth: 0 persist-credentials: false - id: paths + env: + GH_TOKEN: ${{ github.token }} + CI_WORKFLOW: ci.yml run: python3 scripts/ci_changes.py tooling: diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index f8c71674..1332f193 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -9,7 +9,7 @@ permissions: contents: read concurrency: - group: codeql-${{ github.ref }} + group: codeql-${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true jobs: diff --git a/.github/workflows/containers.yml b/.github/workflows/containers.yml index 0556fe94..d9117d3a 100644 --- a/.github/workflows/containers.yml +++ b/.github/workflows/containers.yml @@ -9,7 +9,7 @@ permissions: contents: read concurrency: - group: containers-${{ github.ref }} + group: containers-${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true env: diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 0bd0af40..cf11c67b 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -22,7 +22,7 @@ permissions: contents: read concurrency: - group: docs-${{ github.ref }}-${{ inputs.full && 'full' || 'fast' }} + group: docs-${{ github.ref }}-${{ inputs.full && 'full' || 'fast' }}-${{ github.workflow }} cancel-in-progress: true env: diff --git a/.github/workflows/full-ci.yml b/.github/workflows/full-ci.yml index 9a10755d..60eb5120 100644 --- a/.github/workflows/full-ci.yml +++ b/.github/workflows/full-ci.yml @@ -8,7 +8,7 @@ permissions: contents: read concurrency: - group: full-ci-${{ github.ref }} + group: full-ci-${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true env: diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index d3e8fc73..c14f8dfb 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -22,7 +22,7 @@ permissions: contents: read concurrency: - group: fuzz-${{ github.ref }}-${{ github.event_name }} + group: fuzz-${{ github.ref }}-${{ github.event_name }}-${{ github.workflow }} cancel-in-progress: true env: diff --git a/.github/workflows/oci-runtimes.yml b/.github/workflows/oci-runtimes.yml index fbb7ac7c..74ab622a 100644 --- a/.github/workflows/oci-runtimes.yml +++ b/.github/workflows/oci-runtimes.yml @@ -9,7 +9,7 @@ permissions: contents: read concurrency: - group: oci-runtimes-${{ github.ref }} + group: oci-runtimes-${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true env: diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml index ce56b6be..a1ca5eec 100644 --- a/.github/workflows/packaging.yml +++ b/.github/workflows/packaging.yml @@ -9,7 +9,7 @@ permissions: contents: read concurrency: - group: packaging-${{ github.ref }} + group: packaging-${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true env: diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml index e5b355d9..e2cbe34d 100644 --- a/.github/workflows/perf.yml +++ b/.github/workflows/perf.yml @@ -15,7 +15,7 @@ permissions: contents: read concurrency: - group: perf-${{ github.ref }} + group: perf-${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true env: diff --git a/.github/workflows/qualification.yml b/.github/workflows/qualification.yml index 36a9fe4e..f4f88d3b 100644 --- a/.github/workflows/qualification.yml +++ b/.github/workflows/qualification.yml @@ -11,7 +11,7 @@ permissions: contents: read concurrency: - group: qualification-${{ github.ref }} + group: qualification-${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true jobs: diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 65aad20c..d4abee15 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -19,7 +19,7 @@ permissions: contents: read concurrency: - group: quality-${{ github.ref }} + group: quality-${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true env: diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index bbec5d52..97054cab 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -20,7 +20,7 @@ permissions: contents: read concurrency: - group: security-${{ github.ref }}-${{ inputs.full && 'full' || 'fast' }} + group: security-${{ github.ref }}-${{ inputs.full && 'full' || 'fast' }}-${{ github.workflow }} cancel-in-progress: true env: @@ -28,6 +28,9 @@ env: jobs: changes: + permissions: + contents: read + actions: read runs-on: ubuntu-latest outputs: rust_deps: ${{ steps.paths.outputs.rust_deps }} @@ -40,6 +43,9 @@ jobs: fetch-depth: 0 persist-credentials: false - id: paths + env: + GH_TOKEN: ${{ github.token }} + CI_WORKFLOW: security.yml run: python3 scripts/ci_changes.py cargo-deny: diff --git a/.github/workflows/web.yml b/.github/workflows/web.yml index 6b599025..3c7b1370 100644 --- a/.github/workflows/web.yml +++ b/.github/workflows/web.yml @@ -40,7 +40,7 @@ permissions: contents: read concurrency: - group: web-${{ github.ref }}-${{ inputs.full && 'full' || 'fast' }} + group: web-${{ github.ref }}-${{ inputs.full && 'full' || 'fast' }}-${{ github.workflow }} cancel-in-progress: true jobs: diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index c44062ea..bedf9222 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -9,7 +9,7 @@ permissions: contents: read concurrency: - group: windows-${{ github.ref }} + group: windows-${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true env: diff --git a/docs/architecture/testing.md b/docs/architecture/testing.md index 7358b848..9640b1d0 100644 --- a/docs/architecture/testing.md +++ b/docs/architecture/testing.md @@ -2,7 +2,7 @@ ## Continuous feedback and qualification -Normal PRs and main commits use path-aware fast CI: Linux Rust unit/core-storage contracts, SDK checks against one shared server build when API/SDK files change, Go contracts when the executor changes, and offline workflow checks when tooling changes. Web changes run lint, types, unit tests and a production build; documentation changes build the site. Unrelated changes do not start Rust or cross-platform builds. Unknown paths or unavailable Git history conservatively select all fast components. +Normal PRs and main commits use path-aware fast CI: Linux Rust unit/core-storage contracts, SDK checks against one shared server build when API/SDK files change, Go contracts when the executor changes, and offline workflow checks when tooling changes. Web changes run lint, types, unit tests and a production build; documentation changes build the site. Unrelated changes do not start Rust or cross-platform builds. On main, selection covers all changes since the last successful run, so cancelling an earlier run cannot drop its pending checks. Unknown paths or unavailable Git/API history conservatively select all fast components. The full matrix runs through `qualification.yml` on demand, nightly, and during release preparation: macOS, Windows, Linux coverage (which also executes the full Linux suite), PostgreSQL/S3/HA, all SDK runtimes, CLI examples, all browser engines/viewports, CodeQL, fuzzing, OCI runtimes, packaging and performance checks. SDK/runtime jobs share one verified server/CLI/MCP build. Coverage supplies Linux full-suite correctness, so it is not also run as a separate identical Linux test job. diff --git a/scripts/cancel_stale_ci.py b/scripts/cancel_stale_ci.py index fc0ef855..c00df2c0 100644 --- a/scripts/cancel_stale_ci.py +++ b/scripts/cancel_stale_ci.py @@ -8,8 +8,8 @@ from request_release import GitHub CANCELLABLE = frozenset('.github/workflows/' + name + '.yml' for name in ( - 'qualification', 'prepare-release', 'full-ci', 'codeql', 'containers', 'docs', 'fuzz', - 'oci-runtimes', 'packaging', 'perf', 'quality', 'security', 'web', 'windows', + 'qualification', 'prepare-release', 'full-ci', 'codeql', 'containers', 'fuzz', + 'oci-runtimes', 'packaging', 'perf', 'quality', 'security', 'windows', )) diff --git a/scripts/ci_changes.py b/scripts/ci_changes.py index a3dbcc73..f57168d5 100644 --- a/scripts/ci_changes.py +++ b/scripts/ci_changes.py @@ -8,6 +8,8 @@ from pathlib import Path import subprocess +from request_release import GitHub + FLAGS = ('rust', 'sdk', 'go', 'tooling', 'rust_deps', 'npm_deps', 'python_deps', 'go_deps') @@ -65,10 +67,28 @@ def changed_paths(event, name): return result.stdout.decode('utf-8', errors='surrogateescape').rstrip('\0').split('\0') if result.stdout else [] +def last_successful_base(api, repo, workflow, current=None): + if workflow not in ('ci.yml', 'security.yml'): + return None + try: + runs = api.get('/repos/' + repo + '/actions/workflows/' + workflow + + '/runs?branch=main&event=push&status=success&per_page=50')['workflow_runs'] + sha = next((run['head_sha'] for run in runs if run['head_sha'] != current), None) + return sha if sha and re.fullmatch(r'[0-9a-f]{40}', sha) else None + except Exception: + return None # Missing API/history must broaden the checks, not silently skip them. + + def main(): try: event = json.loads(Path(os.environ['GITHUB_EVENT_PATH']).read_text()) - paths = changed_paths(event, os.environ.get('GITHUB_EVENT_NAME', '')) + name = os.environ.get('GITHUB_EVENT_NAME', '') + if name == 'push' and os.environ.get('GITHUB_REF') == 'refs/heads/main': + base = last_successful_base(GitHub(os.environ.get('GH_TOKEN', '')), + os.environ['GITHUB_REPOSITORY'], os.environ['CI_WORKFLOW'], os.environ.get('GITHUB_SHA')) + paths = changed_paths({'before': base}, name) if base else None + else: + paths = changed_paths(event, name) except (OSError, KeyError, ValueError): paths = None flags = dict.fromkeys(FLAGS, True) if paths is None else classify(paths) diff --git a/scripts/test_ci_policy.py b/scripts/test_ci_policy.py index 1c5756f3..d7c04a0a 100644 --- a/scripts/test_ci_policy.py +++ b/scripts/test_ci_policy.py @@ -11,7 +11,7 @@ import yaml -from ci_changes import FLAGS, changed_paths, classify +from ci_changes import FLAGS, changed_paths, classify, last_successful_base ROOT = Path(__file__).resolve().parent.parent @@ -45,6 +45,18 @@ def test_workflow_and_unknown_paths_fail_closed(self): for path in ('.github/workflows/ci.yml', 'future-runtime/code.rs'): self.assertEqual(classify([path]), dict.fromkeys(FLAGS, True)) + def test_main_covers_changes_since_last_green_run_and_does_not_reuse_itself(self): + class API: + def get(self, path): + self.path = path + return {'workflow_runs': [{'head_sha': 'b' * 40}, {'head_sha': 'a' * 40}]} + api = API() + self.assertEqual(last_successful_base(api, 'org/repo', 'ci.yml', 'b' * 40), 'a' * 40) + self.assertIn('status=success', api.path) + with patch.object(api, 'get', side_effect=RuntimeError('network')): + self.assertIsNone(last_successful_base(api, 'org/repo', 'ci.yml')) + self.assertIsNone(last_successful_base(api, 'org/repo', 'untrusted.yml')) + def test_deleted_and_renamed_files_are_both_classified(self): old = os.getcwd() with tempfile.TemporaryDirectory() as directory: From da97ae6586d4d5eef3fed6c56ea6671ea2cc3107 Mon Sep 17 00:00:00 2001 From: MAxim Kiktev Date: Thu, 29 Oct 2026 21:38:58 +0000 Subject: [PATCH 4/5] ci: enforce fast namespace tests and bound background jobs --- .github/workflows/ci.yml | 8 ++++++++ .github/workflows/full-ci.yml | 13 +++++++++++++ .github/workflows/prepare-release.yml | 8 ++++++++ .github/workflows/qualification.yml | 1 + .github/workflows/web.yml | 1 + docs/architecture/testing.md | 14 +++++++------- docs/operations/supply-chain.md | 2 +- scripts/test_ci_policy.py | 2 ++ 8 files changed, 41 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4b2e7dee..f6d06b43 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,6 +40,7 @@ jobs: run: python3 scripts/cancel_stale_ci.py changes: + timeout-minutes: 3 permissions: contents: read actions: read @@ -61,6 +62,7 @@ jobs: run: python3 scripts/ci_changes.py tooling: + timeout-minutes: 5 name: workflow and release contracts needs: changes if: needs.changes.outputs.tooling == 'true' @@ -90,6 +92,7 @@ jobs: timeout-minutes: 25 env: AG_SKIP_REPORT: /tmp/fast-rust-skips.jsonl + AG_REQUIRE_COMPONENTS: namespace-sandbox steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -102,9 +105,13 @@ jobs: with: shared-key: linux-debug cache-workspace-crates: true + cache-on-failure: true - name: Formatting and code lint if: needs.changes.outputs.rust == 'true' run: cargo fmt --all --check && cargo clippy --workspace --lib --bins --locked -- -D warnings + - name: Enable unprivileged namespace tests + if: needs.changes.outputs.rust == 'true' + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - name: Unit tests and core storage contracts if: needs.changes.outputs.rust == 'true' run: | @@ -205,6 +212,7 @@ jobs: - run: python3 scripts/check_skips.py --job ci/orchestrator --report /tmp/fast-go-skips.jsonl complete: + timeout-minutes: 2 name: CI if: always() needs: [changes, tooling, rust, sdk, go] diff --git a/.github/workflows/full-ci.yml b/.github/workflows/full-ci.yml index 60eb5120..dc1c3e9b 100644 --- a/.github/workflows/full-ci.yml +++ b/.github/workflows/full-ci.yml @@ -19,6 +19,7 @@ env: jobs: test: + timeout-minutes: 45 name: test (${{ matrix.os }}) runs-on: ${{ matrix.os }} strategy: @@ -60,6 +61,7 @@ jobs: run: python3 scripts/check_skips.py --job "ci/test-${{ runner.os == 'Linux' && 'ubuntu' || 'macos' }}" --report "$RUNNER_TEMP/skips.jsonl" lint: + timeout-minutes: 25 name: fmt + clippy + rustdoc runs-on: ubuntu-latest steps: @@ -81,6 +83,7 @@ jobs: RUSTDOCFLAGS: -D warnings msrv: + timeout-minutes: 25 name: MSRV (1.89) runs-on: ubuntu-latest steps: @@ -99,6 +102,7 @@ jobs: - run: cargo check --workspace --all-targets docs-and-demo: + timeout-minutes: 25 name: generated docs + demo runs-on: ubuntu-latest steps: @@ -138,6 +142,7 @@ jobs: python3 scripts/check_skips.py --validate --scan backends: + timeout-minutes: 60 name: PostgreSQL + S3 backends runs-on: ubuntu-latest steps: @@ -200,6 +205,7 @@ jobs: run: python3 scripts/check_skips.py --job ci/backends-ha --report "$RUNNER_TEMP/skips-ha.jsonl" deploy: + timeout-minutes: 20 name: deployment files runs-on: ubuntu-latest env: @@ -236,6 +242,7 @@ jobs: # be built a second time here, with another cache scope and a weaker check. tools: + timeout-minutes: 25 name: shared server and agent binaries runs-on: ubuntu-latest steps: @@ -263,6 +270,7 @@ jobs: retention-days: 1 sdk: + timeout-minutes: 20 needs: tools name: client SDKs (TypeScript, Python) runs-on: ubuntu-latest @@ -308,6 +316,7 @@ jobs: # The README of each SDK names the runtimes it supports; these legs are what makes that sentence true. The # `sdk` job above runs the full checks on one interpreter; here only the test suites run on the others. sdk-runtimes: + timeout-minutes: 20 needs: tools name: client SDK tests (${{ matrix.runtime }}) runs-on: ubuntu-latest @@ -358,6 +367,7 @@ jobs: run: python3 scripts/check_skips.py --job "ci/sdk-runtimes-${{ matrix.kind }}-${{ matrix.version }}" --report /tmp/ag-skips.jsonl orchestrator: + timeout-minutes: 30 needs: tools name: agent orchestrator (Go) runs-on: ubuntu-latest @@ -424,6 +434,7 @@ jobs: # The completion scripts must parse in the shells they are for. A missing shell fails the run (AG_REQUIRE_COMPONENTS=shell) # instead of being skipped; PowerShell is on the runner image, zsh and fish are installed here. shells: + timeout-minutes: 20 name: shell completions (bash, zsh, fish, PowerShell) runs-on: ubuntu-latest steps: @@ -452,6 +463,7 @@ jobs: # model is opt-in (it costs money and CI has no key unless the repository defines the OPENROUTER_API_KEY secret # and the AG_LLM_MODEL variable; forks and pull requests from forks have neither, so the step is skipped). mcp: + timeout-minutes: 20 needs: tools name: MCP server with an agent (scripted; live model opt-in) runs-on: ubuntu-latest @@ -483,6 +495,7 @@ jobs: # `str | None` annotations broke scripts/e2e_smoke.py there). The interpreter comes from uv (already pinned # above); the scripts that read TOML (tomllib, 3.11+) are tested by the jobs that need them. scripts-py39: + timeout-minutes: 10 name: lightweight scripts on Python 3.9 runs-on: ubuntu-latest env: diff --git a/.github/workflows/prepare-release.yml b/.github/workflows/prepare-release.yml index a65e8193..21321932 100644 --- a/.github/workflows/prepare-release.yml +++ b/.github/workflows/prepare-release.yml @@ -25,6 +25,7 @@ env: jobs: verify: + timeout-minutes: 5 name: Candidate version and source runs-on: ubuntu-latest outputs: @@ -59,6 +60,7 @@ jobs: secrets: inherit scripts: + timeout-minutes: 10 if: needs.verify.outputs.needed == 'true' name: release tooling tests needs: verify @@ -70,6 +72,7 @@ jobs: - run: python3 scripts/test_release_scripts.py binaries: + timeout-minutes: 45 if: needs.verify.outputs.needed == 'true' name: binaries (${{ matrix.target }}) needs: verify @@ -171,6 +174,7 @@ jobs: retention-days: 14 sbom: + timeout-minutes: 15 if: needs.verify.outputs.needed == 'true' name: CycloneDX SBOMs needs: verify @@ -199,6 +203,7 @@ jobs: retention-days: 14 image: + timeout-minutes: 45 if: needs.verify.outputs.needed == 'true' name: image (${{ matrix.arch }}) needs: verify @@ -289,6 +294,7 @@ jobs: retention-days: 3 manifest: + timeout-minutes: 10 if: needs.verify.outputs.needed == 'true' name: multi-arch manifest needs: [verify, image] @@ -364,6 +370,7 @@ jobs: retention-days: 14 release: + timeout-minutes: 10 if: needs.verify.outputs.needed == 'true' name: assemble and attest the candidate needs: [verify, qualify, scripts, binaries, sbom, manifest] @@ -433,6 +440,7 @@ jobs: } >> "$GITHUB_STEP_SUMMARY" install: + timeout-minutes: 10 if: needs.verify.outputs.needed == 'true' name: install.sh (${{ matrix.os }}) needs: [verify, release] diff --git a/.github/workflows/qualification.yml b/.github/workflows/qualification.yml index f4f88d3b..0d9d4f41 100644 --- a/.github/workflows/qualification.yml +++ b/.github/workflows/qualification.yml @@ -67,6 +67,7 @@ jobs: uses: ./.github/workflows/oci-runtimes.yml complete: + timeout-minutes: 2 name: Qualified commit if: always() needs: [core, web, security, codeql, containers, docs, packaging, coverage, windows, fuzz, performance, oci] diff --git a/.github/workflows/web.yml b/.github/workflows/web.yml index 3c7b1370..60991661 100644 --- a/.github/workflows/web.yml +++ b/.github/workflows/web.yml @@ -45,6 +45,7 @@ concurrency: jobs: checks: + timeout-minutes: 15 name: lint, types, unit tests, build runs-on: ubuntu-latest defaults: diff --git a/docs/architecture/testing.md b/docs/architecture/testing.md index 9640b1d0..f857d87e 100644 --- a/docs/architecture/testing.md +++ b/docs/architecture/testing.md @@ -17,17 +17,17 @@ every number below was measured on the date given and is reproducible with the c | Layer | What it proves | Run it | Where it runs in CI | |---|---|---|---| -| Unit and property tests | one function or type; invariants over generated inputs (`proptest`: determinism, round trips, "applying the diff gives the target") | `cargo test --workspace` | [CI](https://github.com/Paffin/AgentGit/blob/main/.github/workflows/ci.yml) *test*, Ubuntu and macOS; [Windows](../operations/windows.md) for the CLI, client and server | +| Unit and property tests | one function or type; invariants over generated inputs (`proptest`: determinism, round trips, "applying the diff gives the target") | `cargo test --workspace` | [fast CI](https://github.com/Paffin/AgentGit/blob/main/.github/workflows/ci.yml) for Linux unit/core contracts; full Linux coverage, macOS and [Windows](../operations/windows.md) during qualification | | Crate integration suites | a crate through its public API: the sandbox against the real kernel (`AG_REQUIRE_COMPONENTS=namespace-sandbox`), the engine against a real store, the policy cookbook against the real parser | the same | the same | | Cross-implementation conformance | the byte formats against an independent encoder written in Python (`tests/conformance/`) | the same | the same | | Compatibility | data written by **older builds** still opens and means the same ([Compatibility](compatibility.md)) | `cargo test -p agentgit-integrate --test compat; cargo test -p agentgit-server --test compat_control` | the same | -| Backends | every suite on PostgreSQL 16 and on S3 objects (moto, and SeaweedFS, a real S3 server) | `scripts/pg-test.sh scripts/s3-test.sh cargo test --workspace` | CI *PostgreSQL + S3 backends* | -| End to end | the README demo, every command of the tutorials, the docs against the code | `examples/quickstart.sh`; `python3 scripts/check_docs.py` | CI *generated docs + demo*, *shells* | -| Clients | the TypeScript and Python SDKs and the Go orchestrator against a **real** `ag-server`; the web UI in a real browser | `npm test` in `sdk/typescript`, `pytest` in `sdk/python`, `go test ./...` | CI *client SDKs*, *agent orchestrator*; [Web UI](https://github.com/Paffin/AgentGit/blob/main/.github/workflows/web.yml) | -| Images and manifests | the images build and start, the Compose stack serves, the Kubernetes manifests validate | - | [Containers](https://github.com/Paffin/AgentGit/blob/main/.github/workflows/containers.yml) and CI *deployment files* (path-filtered) | +| Backends | every suite on PostgreSQL 16 and on S3 objects (moto, and SeaweedFS, a real S3 server) | `scripts/pg-test.sh scripts/s3-test.sh cargo test --workspace` | Qualification: Full CI *PostgreSQL + S3 backends* | +| End to end | the README demo, every command of the tutorials, the docs against the code | `examples/quickstart.sh`; `python3 scripts/check_docs.py` | Qualification: Full CI *generated docs + demo*, *shells* | +| Clients | the TypeScript and Python SDKs and the Go orchestrator against a **real** `ag-server`; the web UI in a real browser | `npm test` in `sdk/typescript`, `pytest` in `sdk/python`, `go test ./...` | Fast component checks and qualification SDK/runtime matrix; [Web UI](https://github.com/Paffin/AgentGit/blob/main/.github/workflows/web.yml) | +| Images and manifests | the images build and start, the Compose stack serves, the Kubernetes manifests validate | - | [Containers](https://github.com/Paffin/AgentGit/blob/main/.github/workflows/containers.yml) and Full CI *deployment files* during qualification | | Supply chain | advisories, licences, pinned actions, CodeQL ([Supply chain](../operations/supply-chain.md)) | - | [Security](https://github.com/Paffin/AgentGit/blob/main/.github/workflows/security.yml), [CodeQL](https://github.com/Paffin/AgentGit/blob/main/.github/workflows/codeql.yml) | -| Fuzzing | the parsers and decoders against hostile bytes ([Fuzzing](security.md#fuzzing)) | `fuzz/run.sh 300` | [Fuzz](https://github.com/Paffin/AgentGit/blob/main/.github/workflows/fuzz.yml): 60 s per target on changes, 15 minutes nightly | -| Coverage and mutation | how much of the code the tests reach, and whether they notice a wrong version of it (below) | `cargo llvm-cov --workspace`; `cargo mutants -p agentgit-merge` | [Quality](https://github.com/Paffin/AgentGit/blob/main/.github/workflows/quality.yml): coverage on changes and weekly, mutation weekly and by hand | +| Fuzzing | the parsers and decoders against hostile bytes ([Fuzzing](security.md#fuzzing)) | `fuzz/run.sh 300` | [Fuzz](https://github.com/Paffin/AgentGit/blob/main/.github/workflows/fuzz.yml): 60 s per target during qualification, 15 minutes nightly | +| Coverage and mutation | how much of the code the tests reach, and whether they notice a wrong version of it (below) | `cargo llvm-cov --workspace`; `cargo mutants -p agentgit-merge` | [Quality](https://github.com/Paffin/AgentGit/blob/main/.github/workflows/quality.yml): full-suite coverage during qualification, mutation weekly and by hand | **How many.** `cargo test --workspace` ran **1 036 tests** (and ignored 3, which are measurements run on request, not tests) in the CI *test* job on Ubuntu for commit `148a98c` diff --git a/docs/operations/supply-chain.md b/docs/operations/supply-chain.md index 3ae12955..d0c8a60b 100644 --- a/docs/operations/supply-chain.md +++ b/docs/operations/supply-chain.md @@ -94,7 +94,7 @@ ecosystem the repository has a manifest for: `cargo` (the workspace and `fuzz/`) pull request, major updates one by one. `scripts/check_dependabot.py` (run by the Security workflow) fails when a manifest is not covered by an entry of its ecosystem, or an entry names a directory that does not exist. A Dependabot pull request is a proposal: the -Security, CI and CodeQL runs on it are the evidence, a maintainer merges. +fast CI and dependency audits are immediate evidence; full CodeQL and qualification run before publication. A maintainer merges. Two deliberate gaps: `docs/requirements.txt` is hash-locked by `uv pip compile` and refreshed by hand (a `pip` update would rewrite versions without their hashes), and the toolchain pins in workflows (`toolchain: 1.89.0`, diff --git a/scripts/test_ci_policy.py b/scripts/test_ci_policy.py index d7c04a0a..98081ab7 100644 --- a/scripts/test_ci_policy.py +++ b/scripts/test_ci_policy.py @@ -104,6 +104,8 @@ def test_quick_ci_is_path_aware_and_has_an_always_reported_gate(self): self.assertIn('--lib --bins', commands) self.assertNotIn('cargo test --workspace --locked', commands) self.assertIn("--test '*'", commands) + self.assertIn('kernel.apparmor_restrict_unprivileged_userns=0', commands) + self.assertIn('namespace-sandbox', jobs['rust']['env']['AG_REQUIRE_COMPONENTS']) def test_linux_full_tests_are_already_the_coverage_run(self): self.assertEqual(workflow('full-ci')['jobs']['test']['strategy']['matrix']['os'], ['macos-latest']) From c3d5d8ddbbadca8406fc7989ab509d5ad260f748 Mon Sep 17 00:00:00 2001 From: MAxim Kiktev Date: Thu, 29 Oct 2026 21:41:32 +0000 Subject: [PATCH 5/5] ci: pin release candidates so main development can continue --- .github/workflows/request-release.yml | 17 +++++++---- docs/architecture/testing.md | 2 +- docs/operations/releasing.md | 6 ++-- scripts/cancel_stale_ci.py | 2 +- scripts/release_intent.py | 36 +++++++++++++++++++++++ scripts/request_release.py | 42 +++++++++++++++++++++------ scripts/test_ci_policy.py | 26 +++++++++++++++++ scripts/test_release_candidate.py | 12 +++++++- scripts/test_request_release.py | 25 +++++++++------- 9 files changed, 136 insertions(+), 32 deletions(-) create mode 100644 scripts/release_intent.py diff --git a/.github/workflows/request-release.yml b/.github/workflows/request-release.yml index fdc79129..f2860914 100644 --- a/.github/workflows/request-release.yml +++ b/.github/workflows/request-release.yml @@ -6,9 +6,6 @@ on: branches: [main] paths: - "Cargo.toml" - - "CHANGELOG.md" - - "scripts/request_release.py" - - ".github/workflows/request-release.yml" workflow_dispatch: inputs: ready_only: @@ -20,8 +17,8 @@ permissions: contents: read concurrency: - group: request-release-main - cancel-in-progress: true + group: request-release-${{ github.sha }} + cancel-in-progress: false jobs: request: @@ -35,8 +32,16 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.sha }} + fetch-depth: 0 persist-credentials: false - - name: Wait for the exact main commit, tag it, and request the existing release pipeline + - name: Check whether this push changed the release version + id: intent + env: + EVENT: ${{ github.event_name }} + BEFORE: ${{ github.event.before }} + run: python3 scripts/release_intent.py + - name: Prepare the pinned candidate, then publish without blocking main + if: steps.intent.outputs.release == 'true' env: GH_TOKEN: ${{ github.token }} RELEASE_SHA: ${{ github.sha }} diff --git a/docs/architecture/testing.md b/docs/architecture/testing.md index f857d87e..cea35ca4 100644 --- a/docs/architecture/testing.md +++ b/docs/architecture/testing.md @@ -6,7 +6,7 @@ Normal PRs and main commits use path-aware fast CI: Linux Rust unit/core-storage The full matrix runs through `qualification.yml` on demand, nightly, and during release preparation: macOS, Windows, Linux coverage (which also executes the full Linux suite), PostgreSQL/S3/HA, all SDK runtimes, CLI examples, all browser engines/viewports, CodeQL, fuzzing, OCI runtimes, packaging and performance checks. SDK/runtime jobs share one verified server/CLI/MCP build. Coverage supplies Linux full-suite correctness, so it is not also run as a separate identical Linux test job. -Stale branch runs are cancelled. Release publication promotes prepared, attested bytes and does not add a third full Rust run. See [Releasing](../operations/releasing.md) for the ready-candidate contract and publication time budget. +Stale validation runs are cancelled. Explicitly pinned release candidates use separate refs and continue while main advances. Release publication promotes prepared, attested bytes and does not add a third full Rust run. See [Releasing](../operations/releasing.md) for the ready-candidate contract and publication time budget. What the tests of AgentGit are, which of them run where, how much of the code they reach, how many wrong diff --git a/docs/operations/releasing.md b/docs/operations/releasing.md index 2735fd20..05700806 100644 --- a/docs/operations/releasing.md +++ b/docs/operations/releasing.md @@ -21,7 +21,7 @@ Use `-f force=true` to rehearse preparation of an already published version. Thi gh workflow run request-release.yml --ref main -f ready_only=true ``` -The request creates the immutable version tag only after validating a successful preparation for that exact commit. [Release](../../.github/workflows/release.yml) downloads that run's immutable artifact, verifies its complete file list, checksums, source/version/run binding and attestations, and promotes the recorded container digests. It uploads the same files to GitHub and verifies installation from the public release URL on Linux x86_64, Linux arm64 and macOS arm64. +The request first pins the selected main commit to `ci/release/vX.Y.Z`. This internal candidate branch is never moved by the scripts. The public immutable version tag is created only after successful preparation of that exact source. [Release](../../.github/workflows/release.yml) downloads that run's immutable artifact, verifies its complete file list, checksums, source/version/run binding and attestations, and promotes the recorded container digests. It uploads the same files to GitHub and verifies installation from the public release URL on Linux x86_64, Linux arm64 and macOS arm64. The publication job has a **3-minute** execution limit and its parallel installation jobs have **2-minute** limits. The target is under five minutes for publication of a **ready** candidate. Runner queue time and cold preparation are separate; a missing candidate or failed check is an error, never an excuse to build silently during publication. A timeout is a failed release run, not a successful five-minute release. @@ -37,10 +37,10 @@ This downloads and verifies the prepared candidate but does not move public imag 1. Update the workspace, npm and Python versions and lock files; set deployment defaults, regenerate CLI/protocol documentation and dependency notices. 2. Add a nonempty, dated version section in `CHANGELOG.md` and merge the reviewed candidate to `main`. -3. The automatic Request release workflow starts or reuses preparation in the background, then publishes when ready. Ordinary commits continue through fast, component-specific CI. If main moves before the tag is made, the old request stops. +3. The automatic Request release workflow starts or reuses preparation in the background, then publishes when ready. Ordinary commits continue through fast, component-specific CI. Development on main can continue: candidate preparation and publication keep their selected commit. A dependency-only Cargo edit does not request a release; an actual version change or explicit manual request does. 4. Observe preparation and publication separately. Only a complete Release run, including public-install checks, is a verified release. -A manually requested `ready_only=true` publication never waits for preparation. Published versions are skipped and existing tags are never moved. An expired artifact must be prepared again. Retry a failed publication on its existing tag using the same prepared source commit. +A manually requested `ready_only=true` publication never waits for preparation and uses the already pinned candidate for that version. Published versions are skipped and existing tags are never moved. An expired artifact must be prepared again. Retry a failed publication on its existing tag using the same prepared source commit. ## Verifying what you downloaded diff --git a/scripts/cancel_stale_ci.py b/scripts/cancel_stale_ci.py index c00df2c0..6e648779 100644 --- a/scripts/cancel_stale_ci.py +++ b/scripts/cancel_stale_ci.py @@ -8,7 +8,7 @@ from request_release import GitHub CANCELLABLE = frozenset('.github/workflows/' + name + '.yml' for name in ( - 'qualification', 'prepare-release', 'full-ci', 'codeql', 'containers', 'fuzz', + 'qualification', 'full-ci', 'codeql', 'containers', 'fuzz', 'oci-runtimes', 'packaging', 'perf', 'quality', 'security', 'windows', )) diff --git a/scripts/release_intent.py b/scripts/release_intent.py new file mode 100644 index 00000000..aee22a88 --- /dev/null +++ b/scripts/release_intent.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""Only a version change (or explicit manual request) starts candidate preparation.""" +from __future__ import annotations + +import os +import re +import subprocess + + +def changed_version(event, before): + if event == 'workflow_dispatch': + return True + if not re.fullmatch(r'[0-9a-f]{40}', before or '') or not before.strip('0'): + return True + def manifest(ref): + return subprocess.check_output(['git', 'show', ref + ':Cargo.toml'], text=True) + try: + old = manifest(before) + except subprocess.CalledProcessError: + subprocess.run(['git', 'fetch', '--depth=1', 'origin', before], check=True, capture_output=True) + old = manifest(before) + current = manifest('HEAD') + def version(text): + section = text.split('[workspace.package]', 1)[1].split('\n[', 1)[0] + return re.search(r'^version\s*=\s*"([^"]+)"', section, re.M).group(1) + return version(old) != version(current) + + +def main(): + needed = changed_version(os.environ['EVENT'], os.environ.get('BEFORE', '')) + with open(os.environ['GITHUB_OUTPUT'], 'a') as stream: + stream.write('release=' + str(needed).lower() + '\n') + + +if __name__ == '__main__': + main() diff --git a/scripts/request_release.py b/scripts/request_release.py index 2456ff7b..39b8c93b 100644 --- a/scripts/request_release.py +++ b/scripts/request_release.py @@ -48,7 +48,7 @@ def candidate_run(runs, sha=None): matches = [run for run in runs if run.get('path') == PREPARE_WORKFLOW and run.get('event') in ('workflow_dispatch', 'schedule') - and run.get('head_branch') == 'main' + and (run.get('head_branch') == 'main' or re.fullmatch(r'ci/release/v[0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?', run.get('head_branch') or '')) and (sha is None or run.get('head_sha') == sha)] return max(matches, key=lambda run: run['id'], default=None) @@ -80,12 +80,12 @@ def is_current(api, base, sha): return api.get(base + '/git/ref/heads/main')['object']['sha'] == sha -def wait_checks(api, base, sha, *, timeout=5400, sleep=time.sleep, clock=time.monotonic): +def wait_checks(api, base, sha, *, ref='main', timeout=5400, sleep=time.sleep, clock=time.monotonic): deadline = clock() + timeout runs = workflow_runs(api, base, sha) if candidate_run(runs, sha) is None: - api.post(base + '/actions/workflows/prepare-release.yml/dispatches', {'ref': 'main', 'inputs': {'force': 'false'}}) - while is_current(api, base, sha): + api.post(base + '/actions/workflows/prepare-release.yml/dispatches', {'ref': ref, 'inputs': {'force': 'false'}}) + while True: if not checks_pending(workflow_runs(api, base, sha), sha): resolve_candidate(api, base.removeprefix('/repos/'), sha) return @@ -158,6 +158,27 @@ def is_published(release, version): return required_assets(version).issubset(uploaded) +def pin_candidate(api, base, version, sha): + name = 'ci/release/v' + version + existing = api.optional(base + '/git/ref/heads/' + name) + if existing is not None: + return name, tag_commit(api, base, existing) + if not is_current(api, base, sha): + comparison = api.get(base + '/compare/' + sha + '...main') + if comparison.get('status') not in ('ahead', 'identical'): + raise RuntimeError('candidate commit does not belong to main') + try: + api.post(base + '/git/refs', {'ref': 'refs/heads/' + name, 'sha': sha}) + except urllib.error.HTTPError as error: + if error.code != 422: + raise + existing = api.optional(base + '/git/ref/heads/' + name) + if existing is None: + raise + return name, tag_commit(api, base, existing) + return name, sha + + def request_release(api, repo, sha, version, wait=None, *, ready_only=False): validate_request(repo, sha, version) base, tag = '/repos/' + repo, 'v' + version @@ -169,14 +190,17 @@ def request_release(api, repo, sha, version, wait=None, *, ready_only=False): pinned = tag_commit(api, base, ref) if pinned != sha: return f'{tag} already pinned to {pinned[:12]}; bump the version to release another commit' - if not is_current(api, base, sha): - return 'main moved; no release requested' + candidate_ref = api.optional(base + '/git/ref/heads/ci/release/v' + version) + if candidate_ref is not None: + sha = tag_commit(api, base, candidate_ref) if ready_only: resolve_candidate(api, repo, sha) else: - (wait or wait_checks)(api, base, sha) - if not is_current(api, base, sha): - return 'main moved; no release requested' + branch, sha = pin_candidate(api, base, version, sha) + if wait is not None: + wait(api, base, sha) + else: + wait_checks(api, base, sha, ref=branch) ensure_tag(api, base, tag, sha) api.post(base + '/actions/workflows/release.yml/dispatches', { 'ref': tag, 'inputs': {'version': version, 'publish': 'true'}, diff --git a/scripts/test_ci_policy.py b/scripts/test_ci_policy.py index 98081ab7..24a07177 100644 --- a/scripts/test_ci_policy.py +++ b/scripts/test_ci_policy.py @@ -57,6 +57,32 @@ def get(self, path): self.assertIsNone(last_successful_base(api, 'org/repo', 'ci.yml')) self.assertIsNone(last_successful_base(api, 'org/repo', 'untrusted.yml')) + def test_only_version_changes_or_manual_requests_prepare_a_release(self): + from release_intent import changed_version + old = os.getcwd() + with tempfile.TemporaryDirectory() as directory: + os.chdir(directory) + try: + def git(*args): + return subprocess.check_output(['git', *args], stderr=subprocess.DEVNULL).decode().strip() + git('init', '-q') + git('config', 'user.email', 'ci@example.invalid') + git('config', 'user.name', 'CI test') + path = Path('Cargo.toml') + path.write_text('[workspace.package]\nversion = "1.0.0"\n') + git('add', '.') + git('commit', '-qm', 'base') + base = git('rev-parse', 'HEAD') + path.write_text(path.read_text() + '# dependency edit\n') + git('commit', '-qam', 'dependency') + self.assertFalse(changed_version('push', base)) + path.write_text(path.read_text().replace('1.0.0', '1.0.1')) + git('commit', '-qam', 'version') + self.assertTrue(changed_version('push', base)) + self.assertTrue(changed_version('workflow_dispatch', base)) + finally: + os.chdir(old) + def test_deleted_and_renamed_files_are_both_classified(self): old = os.getcwd() with tempfile.TemporaryDirectory() as directory: diff --git a/scripts/test_release_candidate.py b/scripts/test_release_candidate.py index 2db28f15..42a3ff6d 100644 --- a/scripts/test_release_candidate.py +++ b/scripts/test_release_candidate.py @@ -79,7 +79,7 @@ def test_only_old_main_validation_is_cancelled(self): from cancel_stale_ci import cancel_obsolete api = Api() def run(number, **kwargs): - return dict(id=number, path='.github/workflows/prepare-release.yml', + return dict(id=number, path='.github/workflows/full-ci.yml', head_branch='main', head_sha='b' * 40, status='in_progress', **kwargs) rows = [run(1), dict(run(2), head_sha=SHA), dict(run(3), head_branch='feature'), dict(run(4), path='.github/workflows/release.yml'), dict(run(5), status='completed')] @@ -117,6 +117,16 @@ def test_ready_only_does_not_dispatch_hidden_preparation(self): request_release(api, REPO, SHA, VERSION, ready_only=True) self.assertEqual(api.writes, []) + def test_pinned_candidate_is_ready_while_main_keeps_advancing(self): + api = self.api(dict(name='release-dist', expired=False, size_in_bytes=5)) + key = BASE + '/actions/runs?head_sha=' + SHA + '&per_page=100&page=1' + api.values[key]['workflow_runs'][0]['head_branch'] = 'ci/release/v1.2.3' + api.values[BASE + '/git/ref/heads/ci/release/v1.2.3'] = {'object': {'type': 'commit', 'sha': SHA}} + api.values[BASE + '/git/ref/heads/main'] = {'object': {'sha': 'b' * 40}} + request_release(api, REPO, 'b' * 40, VERSION, ready_only=True) + self.assertEqual(api.writes[0][1]['sha'], SHA) + self.assertEqual(api.writes[1][1]['ref'], 'v1.2.3') + def test_artifact_pagination_is_checked(self): api = self.api() api.values[BASE + '/actions/runs/1/artifacts?per_page=100&page=1'] = {'artifacts': [dict(name='other')] * 100} diff --git a/scripts/test_request_release.py b/scripts/test_request_release.py index 484d0997..33982c1f 100644 --- a/scripts/test_request_release.py +++ b/scripts/test_request_release.py @@ -55,12 +55,13 @@ def test_checks_finish_before_tag_and_dispatch(self): api = Api() seen = [] def wait(client, base, sha): - self.assertEqual(client.writes, []) + self.assertEqual(client.writes, [(BASE + '/git/refs', {'ref': 'refs/heads/ci/release/v0.3.0-beta.2', 'sha': SHA})]) seen.append(sha) result = request_release(api, REPO, SHA, '0.3.0-beta.2', wait) self.assertEqual(seen, [SHA]) self.assertEqual(result, 'requested v0.3.0-beta.2') self.assertEqual(api.writes, [ + (BASE + '/git/refs', {'ref': 'refs/heads/ci/release/v0.3.0-beta.2', 'sha': SHA}), (BASE + '/git/refs', {'ref': 'refs/tags/v0.3.0-beta.2', 'sha': SHA}), (BASE + '/actions/workflows/release.yml/dispatches', {'ref': 'v0.3.0-beta.2', 'inputs': {'version': '0.3.0-beta.2', 'publish': 'true'}}), @@ -90,20 +91,22 @@ def test_unusable_release_assets_do_not_hide_an_incomplete_release(self): api.values[BASE + '/releases/tags/v0.3.0-beta.2'] = {'draft': False, 'assets': assets} api.values[BASE + '/git/ref/tags/v0.3.0-beta.2'] = {'object': {'type': 'commit', 'sha': SHA}} self.assertEqual(request_release(api, REPO, SHA, '0.3.0-beta.2', lambda *_: None), 'requested v0.3.0-beta.2') - self.assertEqual(len(api.writes), 1) + self.assertEqual(len(api.writes), 2) - def test_a_superseded_main_commit_cannot_tag(self): + def test_non_main_commit_cannot_start_preparation(self): api = Api() api.values[BASE + '/git/ref/heads/main'] = {'object': {'sha': 'b' * 40}} - self.assertEqual(request_release(api, REPO, SHA, '0.3.0-beta.2'), 'main moved; no release requested') + api.values[BASE + '/compare/' + SHA + '...main'] = {'status': 'diverged'} + with self.assertRaisesRegex(RuntimeError, 'does not belong'): + request_release(api, REPO, SHA, '0.3.0-beta.2') self.assertEqual(api.writes, []) - def test_main_moving_while_checks_run_cannot_tag(self): + def test_main_can_advance_while_pinned_candidate_is_checked(self): api = Api() def wait(client, *_): client.values[BASE + '/git/ref/heads/main'] = {'object': {'sha': 'b' * 40}} - self.assertEqual(request_release(api, REPO, SHA, '0.3.0-beta.2', wait), 'main moved; no release requested') - self.assertEqual(api.writes, []) + self.assertEqual(request_release(api, REPO, SHA, '0.3.0-beta.2', wait), 'requested v0.3.0-beta.2') + self.assertEqual(api.writes[-2][1], {'ref': 'refs/tags/v0.3.0-beta.2', 'sha': SHA}) def test_failed_checks_cannot_write(self): api = Api() @@ -111,14 +114,14 @@ def wait(*_): raise RuntimeError('Security failed') with self.assertRaisesRegex(RuntimeError, 'Security failed'): request_release(api, REPO, SHA, '0.3.0-beta.2', wait) - self.assertEqual(api.writes, []) + self.assertTrue(all(body.get('ref', '').startswith('refs/heads/ci/release/') for _, body in api.writes)) def test_existing_exact_tag_can_retry_dispatch_without_retagging(self): api = Api() api.values[BASE + '/git/ref/tags/v0.3.0-beta.2'] = {'object': {'type': 'commit', 'sha': SHA}} request_release(api, REPO, SHA, '0.3.0-beta.2', lambda *_: None) - self.assertEqual(len(api.writes), 1) - self.assertTrue(api.writes[0][0].endswith('/dispatches')) + self.assertEqual(len(api.writes), 2) + self.assertTrue(api.writes[-1][0].endswith('/dispatches')) def test_tag_for_another_commit_is_never_moved(self): api = Api() @@ -133,7 +136,7 @@ def test_annotated_tag_is_resolved_before_retrying(self): api.values[BASE + '/git/ref/tags/v0.3.0-beta.2'] = {'object': {'type': 'tag', 'sha': 'c' * 40}} api.values[BASE + '/git/tags/' + 'c' * 40] = {'object': {'type': 'commit', 'sha': SHA}} request_release(api, REPO, SHA, '0.3.0-beta.2', lambda *_: None) - self.assertEqual(len(api.writes), 1) + self.assertEqual(len(api.writes), 2) def test_missing_and_pending_qualification_are_not_success(self): self.assertEqual(checks_pending([]), ['Prepared release candidate'])