From f4b6bc32012fbb45d20568d285abce27030240b2 Mon Sep 17 00:00:00 2001 From: Boyuan Lu Date: Fri, 31 Jul 2026 23:18:19 -0700 Subject: [PATCH 1/2] Add GCC vs Clang benchmark comparison workflow The existing Benchmark Compare workflow diffs two branches/refs but always builds each with whatever compiler the runner defaults to, so it can't isolate compiler choice. This adds a workflow_dispatch job that builds a single ref with both GCC and Clang (separate build trees so neither shares generated dependency build files) and diffs the results with benchmark's compare.py. Co-Authored-By: Claude Sonnet 5 --- .../workflows/benchmark-compiler-compare.yml | 113 ++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 .github/workflows/benchmark-compiler-compare.yml diff --git a/.github/workflows/benchmark-compiler-compare.yml b/.github/workflows/benchmark-compiler-compare.yml new file mode 100644 index 0000000..f36b009 --- /dev/null +++ b/.github/workflows/benchmark-compiler-compare.yml @@ -0,0 +1,113 @@ +name: Benchmark Compiler Compare + +on: + workflow_dispatch: + inputs: + ref: + description: "Branch/tag/SHA to build and benchmark" + required: true + default: "main" + benchmark_filter: + description: "Regex passed to --benchmark_filter" + required: false + default: ".*" + repetitions: + description: "Repetitions per benchmark (--benchmark_repetitions)" + required: false + default: "5" + +jobs: + compare: + name: GCC vs Clang @ ${{ inputs.ref }} (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + # Linux runners ship both GCC and upstream Clang, so a single ref can + # be built with each toolchain and compared. macOS/Windows are + # excluded: macOS has no gcc (only Apple Clang), and Windows has no + # native gcc/clang pair to compare against MSVC. + os: [ubuntu-latest, ubuntu-24.04-arm] + defaults: + run: + shell: bash + steps: + - name: Checkout ${{ inputs.ref }} + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + submodules: recursive + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y libomp-dev gcc g++ clang + + - name: Install Python packages + run: python -m pip install --quiet numpy scipy + + - name: Report toolchain versions + run: | + gcc --version | head -1 + clang --version | head -1 + + # Each compiler gets its own build tree (including its own FetchContent + # deps) so GCC and Clang never share generated build files for the same + # dependency - mixing those in place has caused stale/mismatched + # compiler state before. + - name: Build & run benchmark - GCC + run: | + set -euo pipefail + CC=gcc CXX=g++ cmake -B build-gcc -S . \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_BENCHMARK=ON \ + -DBUILD_TESTING=OFF + cmake --build build-gcc --target dsas_benchmark -j + ./build-gcc/dsas_benchmark \ + --benchmark_filter="${{ inputs.benchmark_filter }}" \ + --benchmark_repetitions="${{ inputs.repetitions }}" \ + --benchmark_report_aggregates_only=true \ + --benchmark_out=gcc-result.json \ + --benchmark_out_format=json + + - name: Build & run benchmark - Clang + run: | + set -euo pipefail + CC=clang CXX=clang++ cmake -B build-clang -S . \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_BENCHMARK=ON \ + -DBUILD_TESTING=OFF + cmake --build build-clang --target dsas_benchmark -j + ./build-clang/dsas_benchmark \ + --benchmark_filter="${{ inputs.benchmark_filter }}" \ + --benchmark_repetitions="${{ inputs.repetitions }}" \ + --benchmark_report_aggregates_only=true \ + --benchmark_out=clang-result.json \ + --benchmark_out_format=json + + - name: Compare results + run: | + set -euo pipefail + { + echo "## Benchmark comparison (${{ matrix.os }}) @ \`${{ inputs.ref }}\`: GCC (A) vs Clang (B)" + echo + echo '```text' + python build-gcc/_deps/benchmark-src/tools/compare.py \ + --no-color -a benchmarks \ + gcc-result.json clang-result.json + echo '```' + } | tee -a "$GITHUB_STEP_SUMMARY" + + - name: Upload raw results + uses: actions/upload-artifact@v4 + with: + name: benchmark-compiler-compare-${{ github.run_id }}-${{ matrix.os }} + path: | + gcc-result.json + clang-result.json + retention-days: 90 From e0e5bf8692a693f1e5b3cefbeaf2338dc2db565f Mon Sep 17 00:00:00 2001 From: Boyuan Lu Date: Fri, 31 Jul 2026 23:23:34 -0700 Subject: [PATCH 2/2] Force Clang for branch B in Benchmark Compare workflow Drop the separate new workflow in favor of reusing the existing branch-vs-branch Benchmark Compare workflow: branch B now explicitly builds with clang/clang++ (branch A keeps the runner default, gcc on Linux), so dispatching with branch_a=main, branch_b= profiles compiler choice instead of a source diff. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/benchmark-compare.yml | 5 +- .../workflows/benchmark-compiler-compare.yml | 113 ------------------ 2 files changed, 4 insertions(+), 114 deletions(-) delete mode 100644 .github/workflows/benchmark-compiler-compare.yml diff --git a/.github/workflows/benchmark-compare.yml b/.github/workflows/benchmark-compare.yml index 3af8227..bebbf7c 100644 --- a/.github/workflows/benchmark-compare.yml +++ b/.github/workflows/benchmark-compare.yml @@ -117,7 +117,10 @@ jobs: cmake --build build --target dsas_benchmark --config Release -j EXE=./build/Release/dsas_benchmark.exe else - cmake -B build -S . \ + # Force Clang for branch B (branch A keeps the runner's default + # compiler, gcc on Linux) so this workflow can also be used to + # profile compiler choice, not just source changes. + CC=clang CXX=clang++ cmake -B build -S . \ -DCMAKE_BUILD_TYPE=Release \ -DBUILD_BENCHMARK=ON \ -DBUILD_TESTING=OFF \ diff --git a/.github/workflows/benchmark-compiler-compare.yml b/.github/workflows/benchmark-compiler-compare.yml deleted file mode 100644 index f36b009..0000000 --- a/.github/workflows/benchmark-compiler-compare.yml +++ /dev/null @@ -1,113 +0,0 @@ -name: Benchmark Compiler Compare - -on: - workflow_dispatch: - inputs: - ref: - description: "Branch/tag/SHA to build and benchmark" - required: true - default: "main" - benchmark_filter: - description: "Regex passed to --benchmark_filter" - required: false - default: ".*" - repetitions: - description: "Repetitions per benchmark (--benchmark_repetitions)" - required: false - default: "5" - -jobs: - compare: - name: GCC vs Clang @ ${{ inputs.ref }} (${{ matrix.os }}) - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - # Linux runners ship both GCC and upstream Clang, so a single ref can - # be built with each toolchain and compared. macOS/Windows are - # excluded: macOS has no gcc (only Apple Clang), and Windows has no - # native gcc/clang pair to compare against MSVC. - os: [ubuntu-latest, ubuntu-24.04-arm] - defaults: - run: - shell: bash - steps: - - name: Checkout ${{ inputs.ref }} - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref }} - submodules: recursive - - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: "3.x" - - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y libomp-dev gcc g++ clang - - - name: Install Python packages - run: python -m pip install --quiet numpy scipy - - - name: Report toolchain versions - run: | - gcc --version | head -1 - clang --version | head -1 - - # Each compiler gets its own build tree (including its own FetchContent - # deps) so GCC and Clang never share generated build files for the same - # dependency - mixing those in place has caused stale/mismatched - # compiler state before. - - name: Build & run benchmark - GCC - run: | - set -euo pipefail - CC=gcc CXX=g++ cmake -B build-gcc -S . \ - -DCMAKE_BUILD_TYPE=Release \ - -DBUILD_BENCHMARK=ON \ - -DBUILD_TESTING=OFF - cmake --build build-gcc --target dsas_benchmark -j - ./build-gcc/dsas_benchmark \ - --benchmark_filter="${{ inputs.benchmark_filter }}" \ - --benchmark_repetitions="${{ inputs.repetitions }}" \ - --benchmark_report_aggregates_only=true \ - --benchmark_out=gcc-result.json \ - --benchmark_out_format=json - - - name: Build & run benchmark - Clang - run: | - set -euo pipefail - CC=clang CXX=clang++ cmake -B build-clang -S . \ - -DCMAKE_BUILD_TYPE=Release \ - -DBUILD_BENCHMARK=ON \ - -DBUILD_TESTING=OFF - cmake --build build-clang --target dsas_benchmark -j - ./build-clang/dsas_benchmark \ - --benchmark_filter="${{ inputs.benchmark_filter }}" \ - --benchmark_repetitions="${{ inputs.repetitions }}" \ - --benchmark_report_aggregates_only=true \ - --benchmark_out=clang-result.json \ - --benchmark_out_format=json - - - name: Compare results - run: | - set -euo pipefail - { - echo "## Benchmark comparison (${{ matrix.os }}) @ \`${{ inputs.ref }}\`: GCC (A) vs Clang (B)" - echo - echo '```text' - python build-gcc/_deps/benchmark-src/tools/compare.py \ - --no-color -a benchmarks \ - gcc-result.json clang-result.json - echo '```' - } | tee -a "$GITHUB_STEP_SUMMARY" - - - name: Upload raw results - uses: actions/upload-artifact@v4 - with: - name: benchmark-compiler-compare-${{ github.run_id }}-${{ matrix.os }} - path: | - gcc-result.json - clang-result.json - retention-days: 90