bench.yml

name: bench

# The Python benchmark tooling (Terminal-Bench harbor adapter + claim analyzer)
# is not covered by the Rust `ci.yml` gate, so it silently drifted red once
# (adapter suite 224/2 — a posture-hash + stale-fixture drift caught only by hand
# during TB2.1 run prep). This workflow gates those suites so it cannot recur.
#
# It has to be a REQUIRED status check to be worth anything. #659 proved that:
# an automated dependency bump moved `harbor==0.6.1` — an audited constant that
# `secure_launcher` refuses to start without — this workflow ran, reported
# FAILURE, and the PR merged twelve minutes later, because a check that is not
# required does not block a merge. It stayed broken for five days, and only
# #909's attempt to actually run the harness noticed.
#
# A required check must report on EVERY pull request. A workflow filtered with
# `on.pull_request.paths` does not run at all when nothing matches, so its check
# never reports, and a required context that never reports blocks the PR
# forever. That is why the path filter now lives in a step rather than the
# trigger: the job always runs and always reports, and only the expensive part
# is conditional. An ordinary Rust PR pays one runner start-up, not two pytest
# suites.

on:
  pull_request:
  push:
    branches: [main]
  workflow_dispatch:

concurrency:
  group: bench-${{ github.event_name == 'pull_request' && github.ref || github.sha }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

# Checkout + uv sync + pytest only.
permissions:
  contents: read

jobs:
  pytest:
    name: harbor_adapter + analyzer pytest
    runs-on: ubuntu-latest
    timeout-minutes: 25
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          # Needed to diff against the PR base for the scope check below.
          fetch-depth: 0

      - name: Does this change touch the bench tooling?
        id: scope
        env:
          BASE_SHA: ${{ github.event.pull_request.base.sha }}
        run: |
          set -euo pipefail
          # Anything that is not a PR (push to main, manual dispatch) always
          # runs the suites: main is the branch whose green-ness is claimed.
          if [ "${{ github.event_name }}" != "pull_request" ]; then
            echo "changed=true" >> "$GITHUB_OUTPUT"
            exit 0
          fi
          # `crates/stella-model/src/catalog.rs` is in scope because the bench
          # suites READ it: `test_posture.py::TestOutputCeilingParity` parses
          # the shipping models' output ceilings out of that file to prove the
          # frozen posture never caps a model below its own ceiling. A PR that
          # touches only the catalog is exactly the change that ratchet exists
          # to catch, and without this pattern it was the one change that
          # skipped the suite entirely. `test_posture.py` asserts this pattern
          # still names the file it reads, so a crate move cannot silently
          # drop it (the last one left the test's own path literal one segment
          # short with Rust CI all-green).
          #
          # `bench/` is named whole rather than by its four tooling
          # subdirectories, because the adapter-caller sweep
          # (`test_manifest_parity.py`, #2203) derives its subject from the
          # tree instead of a list, and callers had already appeared outside
          # those four: `bench/tb21_preregistration.py` imports the adapter,
          # and `bench/loop-bench/src/main.rs` carries the
          # `--agent-import-path` entry-point spec Harbor resolves at launch,
          # in Rust. Enumerating subdirectories here is the same per-seam
          # pinning that let #2182 through one level up — the sweep asserts
          # this pattern covers every caller it reads, so the next one cannot
          # land outside the filter unnoticed.
          if git diff --name-only "$BASE_SHA"...HEAD \
            | grep -Eq '^(bench/|arenabench/|crates/stella-model/src/catalog\.rs$|\.github/workflows/bench\.yml$)'; then
            echo "changed=true" >> "$GITHUB_OUTPUT"
          else
            echo "changed=false" >> "$GITHUB_OUTPUT"
            echo "No bench-tooling changes in this PR — suites skipped." >> "$GITHUB_STEP_SUMMARY"
            echo "The check still reports, so it can be a required context." >> "$GITHUB_STEP_SUMMARY"
          fi

      - name: Install uv
        if: steps.scope.outputs.changed == 'true'
        uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0

      - name: harbor_adapter — sync (locked) + pytest
        if: steps.scope.outputs.changed == 'true'
        working-directory: bench/harbor_adapter
        run: |
          uv sync --locked --extra dev
          uv run --no-sync pytest -q

      - name: terminal_bench_analysis — sync (locked) + pytest
        if: steps.scope.outputs.changed == 'true'
        working-directory: bench/terminal_bench_analysis
        run: |
          uv sync --locked --extra dev
          uv run --no-sync pytest -q

      # The telemetry store. Same shape as the frontier planner below: no
      # project to sync, because the ingester is standard-library only and
      # loads the price table by path rather than importing the analysis
      # package — the piece that decides what a published cost number means
      # should not need the benchmark harness to be testable.
      - name: telemetry_store — pytest
        if: steps.scope.outputs.changed == 'true'
        run: uv run --with pytest --no-project pytest -q bench/telemetry_store/tests

      # The development-baseline evidence tooling: the scorer, the manifest
      # builder and the witness A/B comparison — the three scripts that decide
      # what a published number *is*. Their tests existed and nothing ran them:
      # the scope filter above named `bench/evidence/frontier/` and no other
      # path under `bench/evidence/`, so this suite gated nothing at all. Same
      # `--no-project` reasoning as the two below — the scripts are
      # standard-library only, which is what makes recomputing a published
      # score from committed evidence possible without the harness installed.
      - name: evidence tooling — pytest
        if: steps.scope.outputs.changed == 'true'
        run: uv run --with pytest --no-project pytest -q bench/evidence/tests

      # The bench-trace triage tool. Same `--no-project` reasoning as its
      # neighbours — it is standard-library only so that the piece deciding what
      # a run says about the product needs no harness to be testable. It is not
      # a `make gate` step (it reads S3 and writes to GitHub), so this workflow
      # is the only thing standing between it and silent rot.
      - name: trace triage — pytest
        if: steps.scope.outputs.changed == 'true'
        run: uv run --with pytest --no-project pytest -q bench/trace_triage/tests

      # The Frontier-Bench run planner. No project to sync and no Harbor to
      # install: plan.py imports nothing outside the standard library, which is
      # deliberate — the piece that decides which tasks are allowed to run
      # should not need the benchmark harness to be testable.
      - name: frontier planner — pytest
        if: steps.scope.outputs.changed == 'true'
        run: uv run --with pytest --no-project pytest -q bench/evidence/frontier/tests

      # ArenaBench is being prepared for ejection into its own repository and
      # carries its own workflow at arenabench/.github/workflows/ci.yml, which
      # becomes active the moment that folder becomes a repo root. Until the
      # split lands, run its suite from here so the gate never lapses — the
      # last time this suite went unwatched, a wrong-field trajectory reader
      # posted $0.00 for every contestant and swept the scoreboard.
      - name: arenabench — pytest (until ejection)
        if: steps.scope.outputs.changed == 'true'
        working-directory: arenabench
        run: uv run --with pytest --no-project pytest -q