browser-benchmarks.yml

name: Browser Benchmark

on:
  push:
    branches: [master]
    paths:
      - 'benchmarks/browser/**'
      - 'benchmarks/src/util/**'
      - 'benchmarks/src/merge-results.ts'
      - 'package.json'
  schedule:
    - cron: '0 10 * * 5' # Weekly on Friday at 10am UTC (5am CDT)
  workflow_dispatch:
    inputs:
      iterations:
        description: 'Iterations per provider'
        required: false
        default: '100'
      dry_run:
        description: 'Run without committing results'
        required: false
        default: false
        type: boolean

concurrency:
  group: browser-benchmarks
  cancel-in-progress: true

permissions:
  contents: write
  pull-requests: write
  id-token: write

jobs:
  bench:
    name: Bench ${{ matrix.provider }}
    runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe
    # runs-on: self-hosted
    timeout-minutes: 60
    strategy:
      fail-fast: false
      matrix:
        provider:
          - browserbase
          - browseruse
          - hyperbrowser
          - kernel
          - notte
          - steel
          - tilion
          # - anchorbrowser
    steps:
      - uses: actions/checkout@v4
      - uses: pnpm/action-setup@v4
      - uses: actions/setup-node@v4
        with:
          node-version: 24
          cache: 'pnpm'
      - uses: namespacelabs/nscloud-setup@v0
      - name: Install dependencies
        run: |
          if [ "${{ github.event_name }}" = "schedule" ]; then
            pnpm update
          else
            pnpm install --frozen-lockfile
          fi
      - name: Clear stale results from checkout
        run: rm -rf results/browser/
      - name: Run browser benchmark
        run: |
          . benchmarks/scripts/load-vault-secrets.sh '^(BROWSERBASE_API_KEY|BROWSERBASE_PROJECT_ID|BROWSER_USE_API_KEY|HYPERBROWSER_API_KEY|KERNEL_API_KEY|NOTTE_API_KEY|STEEL_API_KEY|TILION_API_KEY|TILION_BASE_URL|COMPUTESDK_ADMIN_API_KEY|BENCHMARKS_PLATFORM_API_KEY)'

          # All matrix jobs for this workflow run share a single platform run. The
          # attempt number is included so re-runs do not rejoin completed runs.
          RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
          npx tsx packages/benchsdk-runner/dist/bin.js run benchmarks/browser/browser.bench.ts \
            --provider ${{ matrix.provider }} \
            --iterations ${{ github.event_name == 'push' && '10' || github.event.inputs.iterations || '100' }} \
            --run-key "$RUN_KEY"
      - name: Upload results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: browser-results-${{ matrix.provider }}
          path: results/browser/
          if-no-files-found: ignore
          retention-days: 7

  collect:
    name: Collect Results
    runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe
    # runs-on: self-hosted
    needs: bench
    if: always()
    steps:
      - uses: actions/checkout@v4
        with:
          # Full history so the rebase-on-push retry below has a merge base.
          # A shallow (depth-1) clone can make `git rebase origin/<branch>`
          # fail when the remote has advanced past the shallow boundary.
          fetch-depth: 0
      - uses: pnpm/action-setup@v4
      - uses: actions/setup-node@v4
        with:
          node-version: 24
          cache: 'pnpm'
      - uses: namespacelabs/nscloud-setup@v0
      - name: Install dependencies
        run: |
          if [ "${{ github.event_name }}" = "schedule" ]; then
            pnpm update
          else
            pnpm install --frozen-lockfile
          fi
      - name: Download all artifacts
        uses: actions/download-artifact@v4
        with:
          path: artifacts/
          pattern: browser-results-*
      - name: Merge results
        run: npx tsx benchmarks/src/merge-results.ts --input artifacts --mode browser
      - run: pnpm run generate-browser-svg
      - name: Upload SVG as artifact
        if: github.event_name == 'push'
        uses: actions/upload-artifact@v4
        with:
          name: browser-benchmark-svg
          path: browser.svg
          if-no-files-found: ignore
          retention-days: 7
      - name: Post results to merged PR
        if: github.event_name == 'push'
        continue-on-error: true
        uses: actions/github-script@v7
        with:
          script: |
            const fs = require('fs');
            const path = require('path');

            const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
            const latestPath = path.join('results', 'browser', 'latest.json');

            let body = '## Browser Benchmark Results\n\n';

            if (!fs.existsSync(latestPath)) {
              body += '> No browser benchmark results were generated.\n\n';
            } else {
              const data = JSON.parse(fs.readFileSync(latestPath, 'utf-8'));
              const results = data.results
                .filter(r => !r.skipped)
                .sort((a, b) => (b.compositeScore || 0) - (a.compositeScore || 0));

              if (results.length === 0) {
                body += '> No browser benchmark results were generated.\n\n';
              } else {
                body += '| # | Provider | Score | Create | Connect | Navigate | Release | Total | Status |\n';
                body += '|---|----------|-------|--------|---------|----------|---------|-------|--------|\n';

                results.forEach((r, i) => {
                  const name = r.provider.charAt(0).toUpperCase() + r.provider.slice(1);
                  const score = r.compositeScore !== undefined ? r.compositeScore.toFixed(1) : '--';
                  const create = (r.summary.createMs.median / 1000).toFixed(2) + 's';
                  const connect = (r.summary.connectMs.median / 1000).toFixed(2) + 's';
                  const navigate = (r.summary.navigateMs.median / 1000).toFixed(2) + 's';
                  const release = (r.summary.releaseMs.median / 1000).toFixed(2) + 's';
                  const total = (r.summary.totalMs.median / 1000).toFixed(2) + 's';
                  const ok = r.iterations.filter(it => !it.error).length;
                  const count = r.iterations.length;
                  body += `| ${i + 1} | ${name} | ${score} | ${create} | ${connect} | ${navigate} | ${release} | ${total} | ${ok}/${count} |\n`;
                });

                body += '\n';
              }
            }

            body += `---\n*[View full run](${runUrl}) · SVG available as [build artifact](${runUrl}#artifacts)*`;

            // This push is a merge to master — post results to the PR it closed.
            // A commit can be associated with several PRs (backports, merge
            // queue), so pick the one merged into the branch we pushed to.
            const target = context.ref.replace('refs/heads/', '');
            const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({
              owner: context.repo.owner,
              repo: context.repo.repo,
              commit_sha: context.sha,
            });
            const pr = prs.find(p => p.merged_at && p.base.ref === target);
            if (!pr) {
              // Direct push with no associated PR — fall back to a commit comment.
              await github.rest.repos.createCommitComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                commit_sha: context.sha,
                body,
              });
              return;
            }

            const marker = '## Browser Benchmark Results';
            const comments = await github.paginate(github.rest.issues.listComments, {
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: pr.number,
              per_page: 100,
            });

            const existing = comments.find(c => c.body.startsWith(marker));

            if (existing) {
              await github.rest.issues.updateComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                comment_id: existing.id,
                body,
              });
            } else {
              await github.rest.issues.createComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                issue_number: pr.number,
                body,
              });
            }
      - name: Commit and push
        if: github.event_name != 'push' && github.event.inputs.dry_run != 'true'
        run: |
          git config user.name "github-actions[bot]"
          git config user.email "github-actions[bot]@users.noreply.github.com"
          git add package.json pnpm-lock.yaml browser.svg results/browser/
          git diff --cached --quiet && echo "No changes to commit" && exit 0
          git commit -m "chore: update browser benchmark results [skip ci]"

          # Remote master can advance during the run (concurrent benchmark
          # workflows push to master too), so a plain push fails non-fast-forward.
          # Rebase onto the latest remote and retry a few times before giving up.
          branch="${GITHUB_REF#refs/heads/}"
          for attempt in 1 2 3 4 5; do
            # Rebase before each attempt (including the last) so every rebase
            # is followed by a push — otherwise the final iteration's rebase
            # would be wasted and a resolvable non-fast-forward could still fail.
            git fetch origin "${branch}"
            git rebase --autostash "origin/${branch}" || { git rebase --abort; exit 1; }
            if git push origin "HEAD:${branch}"; then
              echo "Pushed on attempt ${attempt}"
              exit 0
            fi
            echo "Push rejected (attempt ${attempt}); will rebase and retry"
          done
          echo "Failed to push after multiple attempts" >&2
          exit 1