chore: switch createos to @bhautikin/ccos and isolate in CI #1
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Browser Benchmark | |
| on: | |
| push: | |
| branches: [master] | |
| paths: | |
| - 'benchmarks/browser/**' | |
| - 'benchmarks/src/util/**' | |
| - 'benchmarks/src/merge-results.ts' | |
| - 'package.json' | |
| schedule: | |
| - cron: '0 10 * * 5' # Weekly on Friday at 10am UTC (5am CDT) | |
| workflow_dispatch: | |
| inputs: | |
| iterations: | |
| description: 'Iterations per provider' | |
| required: false | |
| default: '100' | |
| dry_run: | |
| description: 'Run without committing results' | |
| required: false | |
| default: false | |
| type: boolean | |
| concurrency: | |
| group: browser-benchmarks | |
| cancel-in-progress: true | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| id-token: write | |
| jobs: | |
| bench: | |
| name: Bench ${{ matrix.provider }} | |
| runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe | |
| # runs-on: self-hosted | |
| timeout-minutes: 60 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| provider: | |
| - browserbase | |
| - browseruse | |
| - hyperbrowser | |
| - kernel | |
| - notte | |
| - steel | |
| - tilion | |
| # - anchorbrowser | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: pnpm/action-setup@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: 24 | |
| cache: 'pnpm' | |
| - uses: namespacelabs/nscloud-setup@v0 | |
| - name: Install dependencies | |
| run: | | |
| if [ "${{ github.event_name }}" = "schedule" ]; then | |
| pnpm update | |
| else | |
| pnpm install --frozen-lockfile | |
| fi | |
| - name: Clear stale results from checkout | |
| run: rm -rf results/browser/ | |
| - name: Run browser benchmark | |
| run: | | |
| . benchmarks/scripts/load-vault-secrets.sh '^(BROWSERBASE_API_KEY|BROWSERBASE_PROJECT_ID|BROWSER_USE_API_KEY|HYPERBROWSER_API_KEY|KERNEL_API_KEY|NOTTE_API_KEY|STEEL_API_KEY|TILION_API_KEY|TILION_BASE_URL|COMPUTESDK_ADMIN_API_KEY|BENCHMARKS_PLATFORM_API_KEY)' | |
| # All matrix jobs for this workflow run share a single platform run. The | |
| # attempt number is included so re-runs do not rejoin completed runs. | |
| RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" | |
| npx tsx packages/benchsdk-runner/dist/bin.js run benchmarks/browser/browser.bench.ts \ | |
| --provider ${{ matrix.provider }} \ | |
| --iterations ${{ github.event_name == 'push' && '10' || github.event.inputs.iterations || '100' }} \ | |
| --run-key "$RUN_KEY" | |
| - name: Upload results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: browser-results-${{ matrix.provider }} | |
| path: results/browser/ | |
| if-no-files-found: ignore | |
| retention-days: 7 | |
| collect: | |
| name: Collect Results | |
| runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe | |
| # runs-on: self-hosted | |
| needs: bench | |
| if: always() | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| # Full history so the rebase-on-push retry below has a merge base. | |
| # A shallow (depth-1) clone can make `git rebase origin/<branch>` | |
| # fail when the remote has advanced past the shallow boundary. | |
| fetch-depth: 0 | |
| - uses: pnpm/action-setup@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: 24 | |
| cache: 'pnpm' | |
| - uses: namespacelabs/nscloud-setup@v0 | |
| - name: Install dependencies | |
| run: | | |
| if [ "${{ github.event_name }}" = "schedule" ]; then | |
| pnpm update | |
| else | |
| pnpm install --frozen-lockfile | |
| fi | |
| - name: Download all artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: artifacts/ | |
| pattern: browser-results-* | |
| - name: Merge results | |
| run: npx tsx benchmarks/src/merge-results.ts --input artifacts --mode browser | |
| - run: pnpm run generate-browser-svg | |
| - name: Upload SVG as artifact | |
| if: github.event_name == 'push' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: browser-benchmark-svg | |
| path: browser.svg | |
| if-no-files-found: ignore | |
| retention-days: 7 | |
| - name: Post results to merged PR | |
| if: github.event_name == 'push' | |
| continue-on-error: true | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const path = require('path'); | |
| const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; | |
| const latestPath = path.join('results', 'browser', 'latest.json'); | |
| let body = '## Browser Benchmark Results\n\n'; | |
| if (!fs.existsSync(latestPath)) { | |
| body += '> No browser benchmark results were generated.\n\n'; | |
| } else { | |
| const data = JSON.parse(fs.readFileSync(latestPath, 'utf-8')); | |
| const results = data.results | |
| .filter(r => !r.skipped) | |
| .sort((a, b) => (b.compositeScore || 0) - (a.compositeScore || 0)); | |
| if (results.length === 0) { | |
| body += '> No browser benchmark results were generated.\n\n'; | |
| } else { | |
| body += '| # | Provider | Score | Create | Connect | Navigate | Release | Total | Status |\n'; | |
| body += '|---|----------|-------|--------|---------|----------|---------|-------|--------|\n'; | |
| results.forEach((r, i) => { | |
| const name = r.provider.charAt(0).toUpperCase() + r.provider.slice(1); | |
| const score = r.compositeScore !== undefined ? r.compositeScore.toFixed(1) : '--'; | |
| const create = (r.summary.createMs.median / 1000).toFixed(2) + 's'; | |
| const connect = (r.summary.connectMs.median / 1000).toFixed(2) + 's'; | |
| const navigate = (r.summary.navigateMs.median / 1000).toFixed(2) + 's'; | |
| const release = (r.summary.releaseMs.median / 1000).toFixed(2) + 's'; | |
| const total = (r.summary.totalMs.median / 1000).toFixed(2) + 's'; | |
| const ok = r.iterations.filter(it => !it.error).length; | |
| const count = r.iterations.length; | |
| body += `| ${i + 1} | ${name} | ${score} | ${create} | ${connect} | ${navigate} | ${release} | ${total} | ${ok}/${count} |\n`; | |
| }); | |
| body += '\n'; | |
| } | |
| } | |
| body += `---\n*[View full run](${runUrl}) · SVG available as [build artifact](${runUrl}#artifacts)*`; | |
| // This push is a merge to master — post results to the PR it closed. | |
| // A commit can be associated with several PRs (backports, merge | |
| // queue), so pick the one merged into the branch we pushed to. | |
| const target = context.ref.replace('refs/heads/', ''); | |
| const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| commit_sha: context.sha, | |
| }); | |
| const pr = prs.find(p => p.merged_at && p.base.ref === target); | |
| if (!pr) { | |
| // Direct push with no associated PR — fall back to a commit comment. | |
| await github.rest.repos.createCommitComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| commit_sha: context.sha, | |
| body, | |
| }); | |
| return; | |
| } | |
| const marker = '## Browser Benchmark Results'; | |
| const comments = await github.paginate(github.rest.issues.listComments, { | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: pr.number, | |
| per_page: 100, | |
| }); | |
| const existing = comments.find(c => c.body.startsWith(marker)); | |
| if (existing) { | |
| await github.rest.issues.updateComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| comment_id: existing.id, | |
| body, | |
| }); | |
| } else { | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: pr.number, | |
| body, | |
| }); | |
| } | |
| - name: Commit and push | |
| if: github.event_name != 'push' && github.event.inputs.dry_run != 'true' | |
| run: | | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| git add package.json pnpm-lock.yaml browser.svg results/browser/ | |
| git diff --cached --quiet && echo "No changes to commit" && exit 0 | |
| git commit -m "chore: update browser benchmark results [skip ci]" | |
| # Remote master can advance during the run (concurrent benchmark | |
| # workflows push to master too), so a plain push fails non-fast-forward. | |
| # Rebase onto the latest remote and retry a few times before giving up. | |
| branch="${GITHUB_REF#refs/heads/}" | |
| for attempt in 1 2 3 4 5; do | |
| # Rebase before each attempt (including the last) so every rebase | |
| # is followed by a push — otherwise the final iteration's rebase | |
| # would be wasted and a resolvable non-fast-forward could still fail. | |
| git fetch origin "${branch}" | |
| git rebase --autostash "origin/${branch}" || { git rebase --abort; exit 1; } | |
| if git push origin "HEAD:${branch}"; then | |
| echo "Pushed on attempt ${attempt}" | |
| exit 0 | |
| fi | |
| echo "Push rejected (attempt ${attempt}); will rebase and retry" | |
| done | |
| echo "Failed to push after multiple attempts" >&2 | |
| exit 1 |