chore: wip #234
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| branches: [main] | |
| concurrency: | |
| group: benchmarks-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| jobs: | |
| benchmarks: | |
| name: run | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup Zig | |
| uses: home-lang/pantry/packages/action@main | |
| - name: Install Linux dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y libgtk-3-dev libwebkit2gtk-4.1-dev | |
| - name: Cache Zig artifacts | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cache/zig | |
| packages/zig/zig-cache | |
| key: bench-${{ runner.os }}-zig-master-${{ hashFiles('packages/zig/build.zig') }} | |
| restore-keys: | | |
| bench-${{ runner.os }}-zig-master- | |
| - name: Build benchmark executable | |
| working-directory: packages/zig | |
| run: | | |
| zig build -Doptimize=ReleaseFast | |
| - name: Run benchmarks | |
| id: bench | |
| working-directory: packages/zig | |
| run: | | |
| mkdir -p benchmark-results | |
| # Run system tray benchmark | |
| if [ -f "zig-out/bin/craft" ]; then | |
| echo "Running startup benchmark..." | |
| # Measure startup time (average of 5 runs) | |
| TOTAL=0 | |
| for i in {1..5}; do | |
| START=$(date +%s%N) | |
| timeout 2 ./zig-out/bin/craft --help 2>/dev/null || true | |
| END=$(date +%s%N) | |
| ELAPSED=$(( (END - START) / 1000000 )) | |
| TOTAL=$((TOTAL + ELAPSED)) | |
| done | |
| AVG_STARTUP=$((TOTAL / 5)) | |
| echo "startup_ms=$AVG_STARTUP" >> $GITHUB_OUTPUT | |
| # Measure memory footprint | |
| MEMORY=$(cat /proc/self/status | grep VmRSS | awk '{print $2}') | |
| echo "memory_kb=${MEMORY:-0}" >> $GITHUB_OUTPUT | |
| fi | |
| # Run Zig benchmark tests | |
| echo "Running benchmark tests..." | |
| zig build test 2>&1 | tee benchmark-results/test-output.txt || true | |
| # Extract benchmark results if available | |
| if grep -q "benchmark" benchmark-results/test-output.txt 2>/dev/null; then | |
| grep "benchmark" benchmark-results/test-output.txt > benchmark-results/benchmarks.txt || true | |
| fi | |
| - name: Download baseline benchmarks | |
| if: github.event_name == 'pull_request' | |
| uses: actions/cache@v4 | |
| with: | |
| path: baseline-benchmarks | |
| key: benchmarks-baseline-${{ github.base_ref }} | |
| restore-keys: | | |
| benchmarks-baseline-main | |
| - name: Compare benchmarks | |
| if: github.event_name == 'pull_request' | |
| id: compare | |
| run: | | |
| CURRENT_STARTUP=${{ steps.bench.outputs.startup_ms }} | |
| if [ -f "baseline-benchmarks/startup_ms.txt" ]; then | |
| BASELINE_STARTUP=$(cat baseline-benchmarks/startup_ms.txt) | |
| DIFF=$((CURRENT_STARTUP - BASELINE_STARTUP)) | |
| PERCENT=$(echo "scale=2; ($DIFF * 100) / $BASELINE_STARTUP" | bc 2>/dev/null || echo "0") | |
| echo "startup_diff=$DIFF" >> $GITHUB_OUTPUT | |
| echo "startup_percent=$PERCENT" >> $GITHUB_OUTPUT | |
| echo "baseline_startup=$BASELINE_STARTUP" >> $GITHUB_OUTPUT | |
| # Fail if startup time regressed by more than 20% | |
| THRESHOLD=$(echo "$BASELINE_STARTUP * 1.2" | bc | cut -d. -f1) | |
| if [ "$CURRENT_STARTUP" -gt "$THRESHOLD" ]; then | |
| echo "regression=true" >> $GITHUB_OUTPUT | |
| echo "::warning::Startup time regressed by ${PERCENT}%" | |
| else | |
| echo "regression=false" >> $GITHUB_OUTPUT | |
| fi | |
| else | |
| echo "No baseline available for comparison" | |
| echo "regression=false" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Generate benchmark report | |
| run: | | |
| echo "## Benchmark Results" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "### Startup Performance" >> $GITHUB_STEP_SUMMARY | |
| echo "| Metric | Value |" >> $GITHUB_STEP_SUMMARY | |
| echo "|--------|-------|" >> $GITHUB_STEP_SUMMARY | |
| echo "| Startup Time | ${{ steps.bench.outputs.startup_ms }}ms |" >> $GITHUB_STEP_SUMMARY | |
| echo "| Memory Usage | ${{ steps.bench.outputs.memory_kb }}KB |" >> $GITHUB_STEP_SUMMARY | |
| if [ "${{ github.event_name }}" == "pull_request" ] && [ -n "${{ steps.compare.outputs.baseline_startup }}" ]; then | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "### Comparison with main" >> $GITHUB_STEP_SUMMARY | |
| echo "| Metric | Baseline | Current | Diff |" >> $GITHUB_STEP_SUMMARY | |
| echo "|--------|----------|---------|------|" >> $GITHUB_STEP_SUMMARY | |
| echo "| Startup | ${{ steps.compare.outputs.baseline_startup }}ms | ${{ steps.bench.outputs.startup_ms }}ms | ${{ steps.compare.outputs.startup_diff }}ms (${{ steps.compare.outputs.startup_percent }}%) |" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| - name: Comment on PR | |
| if: github.event_name == 'pull_request' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const startup = '${{ steps.bench.outputs.startup_ms }}'; | |
| const memory = '${{ steps.bench.outputs.memory_kb }}'; | |
| const diff = '${{ steps.compare.outputs.startup_diff }}' || '0'; | |
| const percent = '${{ steps.compare.outputs.startup_percent }}' || '0'; | |
| const regression = '${{ steps.compare.outputs.regression }}' === 'true'; | |
| let emoji = regression ? '🔴' : '✅'; | |
| let status = regression ? 'Performance regression detected!' : 'Performance looks good'; | |
| const body = `## ${emoji} Benchmark Results | |
| ${status} | |
| | Metric | Value | | |
| |--------|-------| | |
| | Startup Time | ${startup}ms | | |
| | Memory Usage | ${memory}KB | | |
| | Change | ${diff}ms (${percent}%) | | |
| <details> | |
| <summary>Benchmark Details</summary> | |
| - Startup time measured as average of 5 runs | |
| - Regression threshold: 20% slower than baseline | |
| </details>`; | |
| const { data: comments } = await github.rest.issues.listComments({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| }); | |
| const existingComment = comments.find(c => | |
| c.user.login === 'github-actions[bot]' && | |
| c.body.includes('Benchmark Results') | |
| ); | |
| if (existingComment) { | |
| await github.rest.issues.updateComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| comment_id: existingComment.id, | |
| body: body | |
| }); | |
| } else { | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| body: body | |
| }); | |
| } | |
| - name: Save benchmark baseline | |
| if: github.ref == 'refs/heads/main' | |
| run: | | |
| mkdir -p baseline-benchmarks | |
| echo "${{ steps.bench.outputs.startup_ms }}" > baseline-benchmarks/startup_ms.txt | |
| echo "${{ steps.bench.outputs.memory_kb }}" > baseline-benchmarks/memory_kb.txt | |
| - name: Cache benchmark baseline | |
| if: github.ref == 'refs/heads/main' | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: baseline-benchmarks | |
| key: benchmarks-baseline-main-${{ github.sha }} | |
| - name: Save benchmark history | |
| if: github.ref == 'refs/heads/main' | |
| run: | | |
| mkdir -p .github/metrics | |
| DATE=$(date -u +%Y-%m-%d) | |
| COMMIT=$(git rev-parse --short HEAD) | |
| echo "{\"date\": \"$DATE\", \"commit\": \"$COMMIT\", \"startup_ms\": ${{ steps.bench.outputs.startup_ms }}, \"memory_kb\": ${{ steps.bench.outputs.memory_kb }}}" >> .github/metrics/benchmarks.jsonl | |
| - name: Upload benchmark results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark-results | |
| path: | | |
| packages/zig/benchmark-results/ | |
| .github/metrics/ | |
| retention-days: 90 | |
| - name: Fail on regression | |
| if: steps.compare.outputs.regression == 'true' | |
| run: | | |
| echo "::error::Performance regression detected. Startup time increased by ${{ steps.compare.outputs.startup_percent }}%" | |
| exit 1 |