perf-pr #4217
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: perf-pr | |
| # Measures this pull request and compares it against the commit it branched | |
| # from, then says so in a comment and fails if an instruction count rose beyond | |
| # the gate. | |
| # | |
| # The gate is the point. A chart on a dashboard is something nobody opens; a | |
| # failing check on the pull request that caused the regression is read by the | |
| # person who can still do something about it. | |
| # | |
| # Nothing here writes to refs/notes/tak. A pull request's measurements are | |
| # base and pull-request measurements are recorded locally, used for the comparison, | |
| # and discarded with the runner. Only main contributes to the persistent history — | |
| # a branch's numbers are not the trunk's, and a series that mixes them cannot be read. | |
| on: # zizmor: ignore[dangerous-triggers] validates live PR metadata before any self-hosted job | |
| workflow_run: | |
| workflows: [perf-pr-preflight] | |
| types: [completed] | |
| permissions: | |
| contents: read | |
| pull-requests: read | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.workflow_run.pull_requests[0].number }} | |
| cancel-in-progress: true | |
| env: | |
| MBX_DISABLE: "1" | |
| CARGO_TERM_COLOR: always | |
| CARGO_INCREMENTAL: "0" | |
| TAK_RUNNER: jdx-perf-v1-ubuntu24.04-x64-usage-rust1.97.1-img5263c143 | |
| jobs: | |
| authorize: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| authorized: ${{ steps.pr.outputs.authorized }} | |
| base_ref: ${{ steps.pr.outputs.base_ref }} | |
| head_sha: ${{ steps.pr.outputs.head_sha }} | |
| pr_number: ${{ steps.pr.outputs.pr_number }} | |
| permissions: | |
| contents: read | |
| pull-requests: read | |
| steps: | |
| - id: pr | |
| name: Validate the pull request from the trusted default branch | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| REPOSITORY: ${{ github.repository }} | |
| PR_NUMBER: ${{ github.event.workflow_run.pull_requests[0].number }} | |
| TRIGGER_CONCLUSION: ${{ github.event.workflow_run.conclusion }} | |
| TRIGGER_EVENT: ${{ github.event.workflow_run.event }} | |
| TRIGGER_SHA: ${{ github.event.workflow_run.head_sha }} | |
| run: | | |
| exec 3>>"$GITHUB_OUTPUT" | |
| echo 'authorized=false' >&3 | |
| [ "$TRIGGER_EVENT" = pull_request ] || exit 0 | |
| [ "$TRIGGER_CONCLUSION" = success ] || exit 0 | |
| [[ "$PR_NUMBER" =~ ^[0-9]+$ ]] || exit 0 | |
| pr="$(gh api "repos/$REPOSITORY/pulls/$PR_NUMBER")" | |
| [ "$(jq -r .state <<<"$pr")" = open ] || exit 0 | |
| [ "$(jq -r .user.login <<<"$pr")" = jdx ] || exit 0 | |
| [ "$(jq -r .head.repo.full_name <<<"$pr")" = "$REPOSITORY" ] || exit 0 | |
| [ "$(jq -r .base.repo.full_name <<<"$pr")" = "$REPOSITORY" ] || exit 0 | |
| head_sha="$(jq -r .head.sha <<<"$pr")" | |
| [ "$head_sha" = "$TRIGGER_SHA" ] || exit 0 | |
| echo 'authorized=true' >&3 | |
| echo "base_ref=$(jq -r .base.ref <<<"$pr")" >&3 | |
| echo "head_sha=$head_sha" >&3 | |
| echo "pr_number=$PR_NUMBER" >&3 | |
| measure: | |
| name: Compare instruction counts | |
| needs: authorize | |
| if: needs.authorize.outputs.authorized == 'true' | |
| # Same runner class as perf.yml and perf-backfill.yml. Absolute counts shift | |
| # between machine types by more than a real regression does, so a comparison | |
| # across runner classes is not a comparison — tak refuses to make one, and | |
| # the report would say "nothing was compared" instead of anything useful. | |
| runs-on: [self-hosted, jdx-perf-v1] | |
| container: | |
| image: ghcr.io/jdx/perf-runner@sha256:5263c143b12ce2234e7b0b21d3d6501c2ed05837b42937d0eaeaa191cd5c8e68 | |
| options: --cpuset-cpus 0-7 | |
| timeout-minutes: 40 | |
| permissions: | |
| # Read only. This job builds and runs code from the pull request, so it | |
| # must not hold a token that can write anything — see the `report` job. | |
| contents: read | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| # The branch commit, not the merge commit actions/checkout defaults to | |
| # for a pull_request event. That default is synthetic: it exists only | |
| # for the run, changes whenever main advances, and measuring it would | |
| # attribute a number to a commit nobody can check out. It would also | |
| # make the footer below name a commit the measurement is not of. | |
| ref: ${{ needs.authorize.outputs.head_sha }} | |
| # Needed twice over: to find the merge base, and for the sparkline, | |
| # which walks twenty commits of trunk history. | |
| fetch-depth: 0 | |
| persist-credentials: false | |
| # Compile inside the pinned job container. Restoring target/ could relabel | |
| # a binary built on another runner class as a jdx-perf-v1 measurement. | |
| - uses: jdx/mise-action@c2a87611a18de5b3828c5652fe268e992400cb5c # v4.3.0 | |
| with: | |
| cache: false | |
| env: | |
| MISE_LOCKED: "1" | |
| - name: Install valgrind | |
| timeout-minutes: 3 | |
| run: | | |
| command -v valgrind || { | |
| sudo apt-get update | |
| sudo apt-get install -y --no-install-recommends valgrind | |
| } | |
| valgrind --version | |
| - name: Runner metadata | |
| run: | | |
| { | |
| echo '### Runner metadata' | |
| echo '```text' | |
| echo 'environment-revision: perf-runner@fd66d9b' | |
| echo 'image: ghcr.io/jdx/perf-runner@sha256:5263c143b12ce2234e7b0b21d3d6501c2ed05837b42937d0eaeaa191cd5c8e68' | |
| uname -a | |
| lscpu | |
| mise --version | |
| rustc --version | |
| cargo --version | |
| valgrind --version | |
| for zone in /sys/class/thermal/thermal_zone*/temp; do | |
| [ -r "$zone" ] && printf '%s: %s\n' "$zone" "$(cat "$zone")" | |
| done | |
| echo '```' | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| - name: Find the base commit | |
| id: base | |
| env: | |
| # The generic context resolved to main for a reopened stacked PR even | |
| # though the event and PR both named the stack parent. Read the | |
| # authoritative pull-request payload directly. | |
| BASE_REF: ${{ needs.authorize.outputs.base_ref }} | |
| run: | | |
| git fetch --quiet origin "$BASE_REF" | |
| # The merge base, not the branch tip: comparing against a moving | |
| # target attributes other people's commits to this pull request. | |
| base=$(git merge-base "origin/$BASE_REF" HEAD) | |
| echo "sha=$base" >> "$GITHUB_OUTPUT" | |
| echo "comparing against $base on $BASE_REF" | |
| # Stack-parent commits are intentionally absent from the persistent tak history. Measure | |
| # the exact merge base in a linked worktree so the comparison below has a same-runner | |
| # baseline instead of passing with "nothing was compared". Git notes are shared between | |
| # linked worktrees, but both measurements remain local to this job workspace. | |
| - name: Measure the base commit | |
| env: | |
| BASE_SHA: ${{ steps.base.outputs.sha }} | |
| MISE_TRUSTED_CONFIG_PATHS: /tmp/usage-perf-base | |
| run: | | |
| base_worktree=/tmp/usage-perf-base | |
| git worktree add --detach "$base_worktree" "$BASE_SHA" | |
| trap 'git worktree remove --force "$base_worktree"' EXIT | |
| ( | |
| cd "$base_worktree" | |
| cargo build --release | |
| taskset -c 0-3 tak run --record | |
| ) | |
| # `--record` writes a git note locally and nothing more. There is no | |
| # `tak push` in this workflow and there should never be one. | |
| - name: Measure this pull request | |
| run: | | |
| taskset -c 0-3 mise run perf:record | |
| # Reported, never gated: the shadow is a fixture that grows as the derive learns to | |
| # express more, so a gate on it fires for a bigger fixture rather than a slower | |
| # parser. `|| true` because an informative table is not worth failing a run over. | |
| # The script rather than the task: the release build is already done by the step | |
| # above, and the task exists for running this by hand. | |
| - name: Measure the shadow against clap, argh and bpaf | |
| run: ./tasks/perf-shadow.sh /tmp/shadow-report.md || true | |
| # Deliberately not `continue-on-error`. That would hide a compare that | |
| # died for an unrelated reason behind a green check. The exit code is | |
| # captured, the comment is posted either way, and a later step re-raises | |
| # it — so a regression always arrives with the table that explains it. | |
| # | |
| # No `--gate-pct` here. The threshold belongs to tak's default and tak.toml, | |
| # where a local run reads the same number this does. A flag on this line | |
| # outranks both and, once added, keeps loosening every later pull request. | |
| - name: Compare against the base | |
| env: | |
| BASE_SHA: ${{ steps.base.outputs.sha }} | |
| run: | | |
| set +e | |
| tak compare "$BASE_SHA" > /tmp/tak-report.md | |
| echo $? > /tmp/tak-gate-status | |
| set -e | |
| cat /tmp/tak-report.md | |
| cat /tmp/tak-report.md >> "$GITHUB_STEP_SUMMARY" | |
| - name: Package the report | |
| if: always() | |
| env: | |
| BASE_SHA: ${{ steps.base.outputs.sha }} | |
| HEAD_SHA: ${{ needs.authorize.outputs.head_sha }} | |
| run: | | |
| mkdir -p /tmp/tak-out | |
| # A missing report means an earlier step died. Say so, rather than | |
| # letting the reporting job find nothing and infer a pass. | |
| if [ -f /tmp/tak-report.md ]; then | |
| cp /tmp/tak-report.md /tmp/tak-out/report.md | |
| else | |
| echo "The comparison never ran — an earlier step failed." > /tmp/tak-out/report.md | |
| fi | |
| # 2, not 1: a missing status means the comparison never ran, which is | |
| # a different failure from a regression and must not be reported as | |
| # one. The gate below maps them to different messages. | |
| cp /tmp/tak-gate-status /tmp/tak-out/status 2>/dev/null || echo 2 > /tmp/tak-out/status | |
| # Absent when valgrind is missing or the measurement failed, which the reporting | |
| # job treats as nothing to add rather than as an error. | |
| cp /tmp/shadow-report.md /tmp/tak-out/shadow.md 2>/dev/null || true | |
| printf '%s %s\n' "${HEAD_SHA:0:12}" "${BASE_SHA:0:12}" > /tmp/tak-out/shas | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 | |
| if: always() | |
| with: | |
| name: tak-report | |
| path: /tmp/tak-out | |
| retention-days: 1 | |
| if-no-files-found: error | |
| # A separate job so the write token is never in scope while code from the | |
| # pull request is executing. This one checks out nothing and runs no project | |
| # code: it reads an artifact and talks to the API. | |
| report: | |
| name: Report and gate | |
| needs: [authorize, measure] | |
| # A failed measurement still has something to report — that is the regression case — | |
| # so `always()` stays: without a status-check function this job is skipped whenever | |
| # `measure` fails, which is precisely when its comment matters. What has to go is the | |
| # *cancelled* case: pushing twice to a branch cancels the run in progress, the | |
| # artifact was never uploaded, and a superseded run reads as a failed gate. | |
| if: >- | |
| always() && needs.measure.result != 'cancelled' | |
| && needs.measure.result != 'skipped' | |
| && needs.authorize.outputs.authorized == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| permissions: | |
| checks: write # attach the regression gate to the measured PR head SHA | |
| pull-requests: write # the sticky comment, and nothing else | |
| steps: | |
| - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 | |
| with: | |
| name: tak-report | |
| path: /tmp/tak-out | |
| # Skipped for pull requests from forks, which get a read-only token. The | |
| # comparison and the gate still run there; only the comment is missing. | |
| - name: Comment on the pull request | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| GH_usage: ${{ github.repository }} | |
| PR_NUMBER: ${{ needs.authorize.outputs.pr_number }} | |
| run: | | |
| marker='<!-- usage-perf-pr -->' | |
| read -r head_sha base_sha < /tmp/tak-out/shas | |
| # The backticks below are markdown, not command substitution, and | |
| # single quotes are what keeps them literal. | |
| # shellcheck disable=SC2016 | |
| { | |
| printf '%s\n' "$marker" | |
| printf '### Instruction counts\n\n' | |
| cat /tmp/tak-out/report.md | |
| # The shadow comparison, when there is one: absent on a runner without | |
| # valgrind, and nothing to apologise for when it is. | |
| if [ -f /tmp/tak-out/shadow.md ]; then | |
| printf '\n' | |
| cat /tmp/tak-out/shadow.md | |
| fi | |
| printf '\n<sub>`%s` vs `%s` · measured on the runner, not pushed to the history.</sub>\n' \ | |
| "$head_sha" "$base_sha" | |
| } > /tmp/tak-comment.md | |
| # One comment per pull request, edited in place. A new comment on | |
| # every push buries the conversation under numbers. | |
| existing="$(gh api "repos/$GH_usage/issues/$PR_NUMBER/comments" --paginate \ | |
| --jq ".[] | select(.body | contains(\"$marker\")) | .id" | tail -n1)" | |
| if [ -n "$existing" ]; then | |
| gh api --method PATCH "repos/$GH_usage/issues/comments/$existing" \ | |
| --field body="$(cat /tmp/tak-comment.md)" | |
| else | |
| gh api --method POST "repos/$GH_usage/issues/$PR_NUMBER/comments" \ | |
| --field body="$(cat /tmp/tak-comment.md)" | |
| fi | |
| # `always()` on both this step and the job: a failed step stops the ones | |
| # after it, and the gate is the reason this workflow exists. Without it a | |
| # `gh api` hiccup would skip the gate — red for the wrong reason, and with | |
| # no regression error to read. | |
| - name: Publish the pull request check | |
| if: always() | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| GH_REPO: ${{ github.repository }} | |
| HEAD_SHA: ${{ needs.authorize.outputs.head_sha }} | |
| RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| run: | | |
| status=$(cat /tmp/tak-out/status 2>/dev/null || echo 2) | |
| if [ "$status" -eq 0 ]; then | |
| conclusion=success | |
| title='Instruction counts passed' | |
| else | |
| conclusion=failure | |
| title='Instruction-count regression or measurement failure' | |
| fi | |
| test -f /tmp/tak-out/report.md || echo 'The comparison did not produce a report.' > /tmp/tak-out/report.md | |
| jq -n --arg head_sha "$HEAD_SHA" --arg conclusion "$conclusion" \ | |
| --arg title "$title" --arg details_url "$RUN_URL" \ | |
| --rawfile summary /tmp/tak-out/report.md \ | |
| '{name:"perf / instruction-count",head_sha:$head_sha,status:"completed",conclusion:$conclusion,details_url:$details_url,output:{title:$title,summary:$summary}}' \ | |
| | gh api --method POST "repos/$GH_REPO/check-runs" --input - | |
| - name: Fail on a regression | |
| if: always() | |
| run: | | |
| status=$(cat /tmp/tak-out/status) | |
| case "$status" in | |
| 0) ;; | |
| 2) | |
| echo "::error::the comparison never ran — an earlier step failed, so nothing was gated" | |
| exit 1 | |
| ;; | |
| *) | |
| echo "::error::an instruction count rose beyond the gate — see the comment or the job summary" | |
| exit "$status" | |
| ;; | |
| esac |