stress #9
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Flake hunter: the whole test suite, many times, on both OSes. | |
| # | |
| # The end-to-end suite drives a real PTY, and a flaky harness is worse than | |
| # none — a test that fails one run in four teaches contributors to re-run CI | |
| # rather than to read it. Run this before a release and whenever anything | |
| # touching frames, waits or the event loop changes. | |
| # | |
| # The iterations are split across SHARDS machines rather than run end to end | |
| # on one. Three things follow, and only the last one is a cost: | |
| # | |
| # * 100 independent trials are 100 independent trials however they are | |
| # distributed, so the odds of catching a flake are unchanged; | |
| # * the shards land on different runners, so a race that only loses on a | |
| # slow machine gets several rolls at a slow machine instead of one — | |
| # coverage a single-runner loop cannot buy at any depth; | |
| # * a shard is shallower, so a fault that needs a *long* run on one machine | |
| # to appear — a leaked descriptor, an unreaped child — has less room to | |
| # accumulate. | |
| # | |
| # That last point is why there are five deep-ish shards rather than ten | |
| # shallow ones; five is also the most macOS jobs a free plan will run at | |
| # once, so ten would queue into two waves and finish later than five. | |
| # | |
| # The shards do not run the same thing five times. Each takes a different | |
| # `--test-threads`, because concurrency is the axis a PTY suite's faults live | |
| # on, and running one point on it five times only samples that point harder. | |
| # One thread is the *fast* end rather than the safe one — nothing contends, so | |
| # every spawned chart starts and exits as quickly as the machine allows — and | |
| # sixteen is the crowded end, sixteen ptys opening and closing at once, which | |
| # is also the closest this workflow now gets to the descriptor pressure a | |
| # deeper shard used to build up. The count is in the job name and in the | |
| # failure, so a flake arrives half-diagnosed. | |
| name: stress | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| iterations: | |
| description: Full-suite iterations per OS, split across the shards | |
| required: false | |
| default: "100" | |
| schedule: | |
| - cron: "23 6 * * 1" # Mondays 06:23 UTC | |
| permissions: | |
| contents: read | |
| # One stress run per ref at a time; a second dispatch queues rather than | |
| # cancelling a half-finished measurement. | |
| concurrency: | |
| group: stress-${{ github.ref }} | |
| cancel-in-progress: false | |
| jobs: | |
| stress: | |
| name: stress (${{ matrix.os }}, ${{ matrix.threads }} threads) | |
| strategy: | |
| # Never cancel a sibling: which OS and which concurrency a flake landed | |
| # on is most of the diagnosis, and a cancelled shard reports nothing. | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest, macos-latest] | |
| threads: [1, 2, 4, 8, 16] | |
| # Balanced by wall time rather than by count. Measured on this suite: | |
| # a serial iteration costs 20s against 8s at four threads and 5s at | |
| # eight, so an even split would leave four machines idle while the | |
| # serial one finished, and the whole point of sharding was the clock. | |
| # The weights are percentages of `iterations` and sum to 100. | |
| include: | |
| - threads: 1 | |
| weight: 10 | |
| - threads: 2 | |
| weight: 15 | |
| - threads: 4 | |
| weight: 25 | |
| - threads: 8 | |
| weight: 25 | |
| - threads: 16 | |
| weight: 25 | |
| runs-on: ${{ matrix.os }} | |
| # A shard is twenty iterations, which is minutes. A shard that hangs | |
| # should say so while the run is still worth watching. | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 | |
| with: | |
| persist-credentials: false | |
| - uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # v1 | |
| with: | |
| toolchain: stable | |
| - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 | |
| with: | |
| # One cache for every shard rather than ten near-identical ones: | |
| # they build the same tree, and the default key is per-job. | |
| shared-key: stress | |
| # The binaries too: the smoke suite spawns `mossaic` itself, so it must | |
| # exist before iteration 1 rather than be built inside the timing loop. | |
| - name: Build once | |
| run: cargo build --release --all-targets | |
| - name: Run the suite repeatedly | |
| env: | |
| ITERS: ${{ inputs.iterations || '100' }} | |
| THREADS: ${{ matrix.threads }} | |
| WEIGHT: ${{ matrix.weight }} | |
| run: | | |
| per=$(( WEIGHT * ITERS / 100 )) | |
| # A tiny `iterations` must still run this shard at all, or a quick | |
| # ten-iteration check would silently skip the serial one. | |
| if [ "$per" -lt 1 ]; then per=1; fi | |
| echo "${THREADS} thread(s): ${per} iterations of ${ITERS}" | |
| for i in $(seq "$per"); do | |
| echo "::group::${THREADS} threads, iteration ${i}/${per}" | |
| cargo test --release -- --test-threads="$THREADS" \ | |
| || { echo "::error::suite flaked at ${THREADS} thread(s), iteration ${i}/${per}"; exit 1; } | |
| echo "::endgroup::" | |
| done |