stress.yml

# Flake hunter: the whole test suite, 100 times, on all three OSes.
#
# A flaky test harness is worse than none — this workflow is the credibility
# gate. Run it before every release and whenever wait/timing code changes.
#
# Windows joined in 0.10 (#290), when its ci.yml leg became a required check
# (#280) and therefore acquired a flake budget of zero. It is here for one
# reason the other two legs do not have: on Unix "the terminal closed" is a
# kernel fact — EOF on the master means every slave descriptor is closed —
# while on Windows the pseudoconsole's output pipe outlives the child, so
# the harness *manufactures* the edge instead (`EXIT_CLOSES_THE_TERMINAL`,
# `ExitWatch`: reap the child, drain for `DRAIN_GRACE`, call it closed).
# Polling plus a grace window is the shape a one-in-N flake lives in, and
# until now nothing ran it more than once per commit. The second reason is
# `common::fixture_bin`, which shells out to `cargo build -p` from inside
# the tests, on the one filesystem that refuses to relink a running `.exe`.
#
# The 100 are split across SHARDS machines rather than run end to end on one.
# Three things follow, and only the last one is a cost:
#
#   * 100 independent trials are 100 independent trials however they are
#     distributed, so the odds of catching a flake are unchanged;
#   * the shards land on different runners, so a race that only loses on a
#     slow machine gets several rolls at a slow machine instead of one —
#     which is coverage a single-runner loop cannot buy at any depth;
#   * a shard is shallower, so a fault that needs a *long* run on one machine
#     to appear — a leaked descriptor, an unreaped child, a pty device number
#     that only recycles under pressure — has less room to accumulate.
#
# That last point is why the shards are five deep-ish rather than ten shallow
# ones: twenty iterations still stacks a few hundred pty lifecycles onto one
# machine, and five is also the most macOS jobs a free plan will run at once,
# so ten shards would queue into two waves and finish later than five.
#
# The shards do not run the same thing five times. Each takes a different
# `--test-threads`, because concurrency is the axis this suite's faults live
# on and running one point on it five times only samples that point harder:
#
#   * 1 thread is the *fast* end, not the safe one. Nothing contends, so every
#     child starts and exits as quickly as the machine can manage — which is
#     the shape of the instant-exit teardown race, where a program that writes
#     and dies inside a millisecond outruns the reader attaching to it.
#   * 16 threads is the crowded end: sixteen ptys opening and being revoked at
#     once is what surfaced the macOS device-recycling race, and it is also
#     the closest this workflow gets to the descriptor and process pressure a
#     deeper shard used to build up.
#   * 2, 4 and 8 are the middle, where most real suites actually run.
#
# The thread count is in the job name and in the failure, so a flake arrives
# already half-diagnosed: at 1 it is a speed race, at 16 a contention one.
#
# The axis transfers across platforms; the faults at its ends do not. At 1
# thread Windows is hunting the exit synthesis above rather than the macOS
# teardown race — nothing contends, so a child can be reaped before its last
# write has been drained. At 16 it is hunting conhost churn: windows-latest
# is a four-core box, so sixteen pseudoconsoles opening and closing at once
# is genuine oversubscription rather than a figure of speech.
name: stress

on:
  workflow_dispatch:
    inputs:
      iterations:
        description: Full-suite iterations per OS, split across the shards
        required: false
        default: "100"
  # Scheduled runs are disabled. Uncomment the block below to restore them.
  # schedule:
  #   - cron: "17 6 * * 1" # Mondays 06:17 UTC

permissions:
  contents: read

# One stress run per ref at a time; a second dispatch queues rather than
# cancelling a half-finished 100-iteration measurement.
concurrency:
  group: stress-${{ github.ref }}
  cancel-in-progress: false

jobs:
  stress:
    name: stress (${{ matrix.os }}, ${{ matrix.threads }} threads)
    strategy:
      # Never cancel a sibling: which OS and which concurrency a flake landed
      # on is most of the diagnosis, and a cancelled shard reports nothing.
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]
        threads: [1, 2, 4, 8, 16]
        # Balanced by wall time rather than by count. Measured on this suite:
        # a serial iteration costs 43s against 16s at four threads and 14s at
        # sixteen, so an even split would leave four machines idle while the
        # serial one finished, and the whole point of sharding was the clock.
        # The weights are percentages of `iterations` and sum to 100.
        include:
          - threads: 1
            weight: 10
          - threads: 2
            weight: 15
          - threads: 4
            weight: 25
          - threads: 8
            weight: 25
          - threads: 16
            weight: 25
    runs-on: ${{ matrix.os }}
    defaults:
      run:
        # windows-latest defaults to pwsh and the iteration loop below is
        # POSIX shell. Git Bash ships `seq` and arithmetic expansion, so one
        # shell serves all three OSes and the script stays single-sourced.
        # On Unix this only adds `pipefail`, which the loop does not use.
        shell: bash
    # A shard is twenty iterations, which is minutes. Generous, but no longer
    # two hours: a shard that hangs should say so while the run is still worth
    # watching. Windows gets double, because an iteration there costs roughly
    # two to three times a Linux one and the serial shard is the long pole —
    # a timeout that fires on a healthy run is a false alarm that costs more
    # than the minutes it saves.
    timeout-minutes: ${{ matrix.os == 'windows-latest' && 60 || 30 }}
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          persist-credentials: false
      - uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # v1
        with:
          toolchain: stable
      - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2
        with:
          # One cache for every shard rather than ten near-identical ones:
          # they build the same tree, and the default key is per-job.
          shared-key: stress
      - name: Build test binaries once
        run: cargo build --workspace --release --all-targets # bins too: fixture binaries must exist before iteration 1
      - name: Run the suite repeatedly
        env:
          ITERS: ${{ inputs.iterations || '100' }}
          THREADS: ${{ matrix.threads }}
          WEIGHT: ${{ matrix.weight }}
        run: |
          per=$(( WEIGHT * ITERS / 100 ))
          # A tiny `iterations` must still run this shard at all, or a quick
          # ten-iteration check would silently skip the serial one.
          if [ "$per" -lt 1 ]; then per=1; fi
          echo "${THREADS} thread(s): ${per} iterations of ${ITERS}"
          for i in $(seq "$per"); do
            echo "::group::${THREADS} threads, iteration ${i}/${per}"
            cargo test --workspace --release -- --test-threads="$THREADS" \
              || { echo "::error::suite flaked at ${THREADS} thread(s), iteration ${i}/${per}"; exit 1; }
            echo "::endgroup::"
          done