From 14bae2cf3c49140fde9671e7dd6d2fb702614768 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Tue, 14 Jul 2026 08:11:03 +0200 Subject: [PATCH 1/3] Trigger benchmarks instead of waiting on it to complete. [only benchmarks] --- .buildkite/pipeline.yml | 31 +++++++++++++++++++++++++++++++ .github/workflows/Benchmark.yml | 20 ++++++++------------ perf/runbenchmarks.jl | 6 +++++- 3 files changed, 44 insertions(+), 13 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 125206f79..d46647a2e 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -162,6 +162,7 @@ steps: continue_on_failure: true - label: ":racehorse: Benchmarks" + key: benchmarks plugins: - JuliaCI/julia#v1: version: "1.12" @@ -183,3 +184,33 @@ steps: build.message !~ /\[only/ && !build.pull_request.draft && build.message !~ /\[skip benchmarks\]/ timeout_in_minutes: 30 + + - label: ":github: Publish benchmarks" + key: publish-benchmarks + depends_on: benchmarks + command: | + if ! buildkite-agent artifact download benchmarkresults.json .; then + echo "No benchmark result found; skipping GitHub dispatch" + exit 0 + fi + if [ -z "$${GITHUB_TOKEN:-}" ]; then + echo "GITHUB_TOKEN is not available" + exit 1 + fi + + payload=$$(jq -cn \ + --arg build_number "$$BUILDKITE_BUILD_NUMBER" \ + --arg commit "$$BUILDKITE_COMMIT" \ + --arg branch "$$BUILDKITE_BRANCH" \ + --arg pull_request "$$BUILDKITE_PULL_REQUEST" \ + '{event_type: "benchmarks_complete", client_payload: $$ARGS.named}') + curl --fail --silent --show-error --request POST \ + --header "Authorization: Bearer $$GITHUB_TOKEN" \ + --header "Accept: application/vnd.github+json" \ + --url "https://api.github.com/repos/JuliaGPU/Metal.jl/dispatches" \ + --data "$$payload" + agents: + queue: "metal" + +env: + SECRET_GITHUB_TOKEN: "AdEayesenNRDVOEDux8opnn4mWyI/BeyTfKKMIG/oeo62sX1OHthjGH90feRw+XMnqUSbWrMHAHDJW45sIyKtK9AMUSeATfLGhA0O4A9qRJNtC3g+/Yy1/FwH2455QV/mDke8nlvq0h6S3ATU+szeggBKOzLnQ7Mc1KrLe0VB2pHdjPdxPdqnxCzf4DcLxTbjp03EAK8YgZE1vsrf+Ygw+rjRM2FN81jHfKtjYjwu7xJf1fcmhVuA3Zc/nfJ2Ocwo+6jMEaoIim2MnkRo6gZ/N0ItO1YNHf+ad+duxmsCqbuZxGwlNq+voUuLsNA5ZbKq5vujtBUSaGXlpZObqo53A==;U2FsdGVkX1/tAsPnUl0uv/EkkPytWTGcbUVHXngVqdy8ns7wFmRPzG6C4Y9DFR6wvlwqnt6K1lGNFa+D64bUqzIGRzRNcIv0Q4vomy1scK1KcH7qCgNG3R7h8TBJb7ppAjtSN2ILjrRholMuVzRGnQ==" diff --git a/.github/workflows/Benchmark.yml b/.github/workflows/Benchmark.yml index 49c4fa40a..d3815fa8a 100644 --- a/.github/workflows/Benchmark.yml +++ b/.github/workflows/Benchmark.yml @@ -1,33 +1,28 @@ name: Benchmarks permissions: - statuses: read # find Buildkite URL from PR status contents: write # update benchmark contents in gh-pages branch pull-requests: write # comment on PR with benchmark results deployments: write # deploy GitHub pages website on: - pull_request_target: - branches: - - main - push: - branches: - - main + repository_dispatch: + types: [benchmarks_complete] jobs: benchmark: runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 + with: + ref: ${{ github.event.client_payload.commit }} - name: Download Buildkite Artifacts id: download uses: EnricoMi/download-buildkite-artifact-action@v1 with: buildkite_token: ${{ secrets.BUILDKITE_TOKEN }} - ignore_build_states: blocked,canceled,skipped,not_run - ignore_job_states: timed_out,failed + buildkite_build_url: https://buildkite.com/julialang/metal-dot-jl/builds/${{ github.event.client_payload.build_number }} output_path: artifacts - poll_interval: 60 # Poll less often to reduce chance of hitting org-wide buildkite limit of 200 requests/sec - name: Locate Benchmarks Artifact id: locate @@ -43,8 +38,9 @@ jobs: output-file-path: ${{ steps.locate.outputs.path }} benchmark-data-dir-path: "bench" github-token: ${{ secrets.GITHUB_TOKEN }} - comment-always: ${{ github.event_name == 'pull_request_target' }} + ref: ${{ github.event.client_payload.commit }} + comment-always: ${{ github.event.client_payload.pull_request != 'false' }} summary-always: true alert-threshold: "125%" fail-on-alert: false - auto-push: ${{ github.event_name != 'pull_request_target' }} + auto-push: ${{ github.event.client_payload.branch == 'main' }} diff --git a/perf/runbenchmarks.jl b/perf/runbenchmarks.jl index 6e8434bc7..050aba23d 100644 --- a/perf/runbenchmarks.jl +++ b/perf/runbenchmarks.jl @@ -54,5 +54,9 @@ results["latency"] = latency_results results["integration"] = integration_results # write out the results +# We report the minimum rather than the median: at the sub-microsecond scale of many +# of these benchmarks, OS scheduler jitter dominates the median and produces 5-15% +# trial-to-trial variance, while the minimum reflects the un-preempted code path +# and is stable to <1% across trials. Real regressions still show up in the minimum. result_file = length(ARGS) >= 1 ? ARGS[1] : "benchmarkresults.json" -BenchmarkTools.save(result_file, median(results)) +BenchmarkTools.save(result_file, minimum(results)) From d14494b73ad14a3c57c03eac723751339c3bb779 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 16 Jul 2026 13:20:29 +0200 Subject: [PATCH 2/3] Report benchmarks on the PR. [ci skip] --- .github/workflows/Benchmark.yml | 59 +++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/.github/workflows/Benchmark.yml b/.github/workflows/Benchmark.yml index d3815fa8a..78236f0eb 100644 --- a/.github/workflows/Benchmark.yml +++ b/.github/workflows/Benchmark.yml @@ -1,6 +1,7 @@ name: Benchmarks permissions: contents: write # update benchmark contents in gh-pages branch + issues: write # comment on PR with benchmark results pull-requests: write # comment on PR with benchmark results deployments: write # deploy GitHub pages website @@ -11,6 +12,9 @@ on: jobs: benchmark: runs-on: ubuntu-latest + concurrency: + group: benchmarks-${{ github.event.client_payload.pull_request != 'false' && format('pr-{0}', github.event.client_payload.pull_request) || github.run_id }} + cancel-in-progress: false steps: - uses: actions/checkout@v7 with: @@ -44,3 +48,58 @@ jobs: alert-threshold: "125%" fail-on-alert: false auto-push: ${{ github.event.client_payload.branch == 'main' }} + + - name: Move Benchmark Results to Pull Request + if: ${{ steps.locate.outputs.path != '' && github.event.client_payload.pull_request != 'false' }} + uses: actions/github-script@v9 + with: + script: | + const {owner, repo} = context.repo; + const commit_sha = context.payload.client_payload.commit; + const issue_number = Number(context.payload.client_payload.pull_request); + const marker = ''; + + const commitComments = await github.paginate( + github.rest.repos.listCommentsForCommit, + {owner, repo, commit_sha, per_page: 100}, + ); + const source = commitComments.filter(comment => comment.body?.startsWith(marker)).at(-1); + if (!source) { + throw new Error(`No benchmark comment found on commit ${commit_sha}`); + } + + const pullRequestComments = await github.paginate( + github.rest.issues.listComments, + {owner, repo, issue_number, per_page: 100}, + ); + const benchmarkComments = pullRequestComments.filter(comment => comment.body?.startsWith(marker)); + const existing = benchmarkComments.at(-1); + if (existing) { + await github.rest.issues.updateComment({ + owner, + repo, + comment_id: existing.id, + body: source.body, + }); + } else { + await github.rest.issues.createComment({ + owner, + repo, + issue_number, + body: source.body, + }); + } + + for (const duplicate of benchmarkComments.slice(0, -1)) { + await github.rest.issues.deleteComment({ + owner, + repo, + comment_id: duplicate.id, + }); + } + + await github.rest.repos.deleteCommitComment({ + owner, + repo, + comment_id: source.id, + }); From c65728b01197a18cc455d297e5136a472a36508f Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 16 Jul 2026 22:29:10 +0200 Subject: [PATCH 3/3] Avoid checked conversions in tensor GEMM kernels --- src/device/intrinsics/tensor.jl | 10 ++++++---- src/gemm.jl | 6 +++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/device/intrinsics/tensor.jl b/src/device/intrinsics/tensor.jl index eaa93004b..c88f751a1 100644 --- a/src/device/intrinsics/tensor.jl +++ b/src/device/intrinsics/tensor.jl @@ -106,7 +106,7 @@ end @device_function @inline function MtlInlineTensor{T, R, AS.Device}( data::MtlDeviceArray{T, <:Any, AS.Device}, extents::NTuple{R, <:Integer}) where {T, R} - e = Int32.(extents) + e = unsafe_trunc.(Int32, extents) storage = Ref{TensorDescriptor}() init_strided_tensor_device!(storage, Int16(R), reinterpret(LLVMPtr{UInt8, AS.Device}, pointer(data)), @@ -117,7 +117,7 @@ end @device_function @inline function MtlInlineTensor{T, R, AS.ThreadGroup}( data::MtlDeviceArray{T, <:Any, AS.ThreadGroup}, extents::NTuple{R, <:Integer}) where {T, R} - e = Int32.(extents) + e = unsafe_trunc.(Int32, extents) storage = Ref{TensorDescriptor}() init_strided_tensor_threadgroup!(storage, Int16(R), reinterpret(LLVMPtr{UInt8, AS.ThreadGroup}, pointer(data)), @@ -133,7 +133,8 @@ end storage = Ref{TensorDescriptor}() init_strided_tensor_device!(storage, Int16(R), reinterpret(LLVMPtr{UInt8, AS.Device}, pointer(data)), - Int32.(extents), Int32.(strides), Int8(0)) + unsafe_trunc.(Int32, extents), + unsafe_trunc.(Int32, strides), Int8(0)) return MtlInlineTensor{T, R, AS.Device}(storage[]) end @@ -144,7 +145,8 @@ end storage = Ref{TensorDescriptor}() init_strided_tensor_threadgroup!(storage, Int16(R), reinterpret(LLVMPtr{UInt8, AS.ThreadGroup}, pointer(data)), - Int32.(extents), Int32.(strides), Int8(0)) + unsafe_trunc.(Int32, extents), + unsafe_trunc.(Int32, strides), Int8(0)) return MtlInlineTensor{T, R, AS.ThreadGroup}(storage[]) end diff --git a/src/gemm.jl b/src/gemm.jl index 57c99b6e2..8aaca5d3d 100644 --- a/src/gemm.jl +++ b/src/gemm.jl @@ -285,8 +285,8 @@ function gemm_tensor_kernel!(C::MtlDeviceArray, A::MtlDeviceArray, B::MtlDeviceA ::Val{TM}, ::Val{TN}, ::Val{TK}, ::Val{NSIMD}) where {TM, TN, TK, NSIMD} tgid = threadgroup_position_in_grid_3d() - m_off = (Int32(tgid.x) - Int32(1)) * Int32(TM) - n_off = (Int32(tgid.y) - Int32(1)) * Int32(TN) + m_off = (unsafe_trunc(Int32, tgid.x) - Int32(1)) * Int32(TM) + n_off = (unsafe_trunc(Int32, tgid.y) - Int32(1)) * Int32(TN) tA = MtlInlineTensor(A, (M, K)) tB = MtlInlineTensor(B, (K, N)) @@ -297,7 +297,7 @@ function gemm_tensor_kernel!(C::MtlDeviceArray, A::MtlDeviceArray, B::MtlDeviceA op = TensorOpsMatmul2D{matmul2d_descriptor(TM, TN, TK; mode = matmul2d_multiply_accumulate), Int32(NSIMD)}() - nslices = Int32(K ÷ UInt32(TK)) + nslices = unsafe_trunc(Int32, K ÷ UInt32(TK)) for s in Int32(0):(nslices - Int32(1)) k_off = s * Int32(TK) mA = view(tA, (m_off + Int32(1), k_off + Int32(1)), (Int32(TM), Int32(TK)))