chore: bump createos benchmark provider #4
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Network-localhost Benchmark | |
| on: | |
| push: | |
| branches: [master] | |
| paths: | |
| - 'benchmarks/sandbox/network-localhost/**' | |
| - 'benchmarks/scripts/smoke.ts' | |
| - 'benchmarks/scripts/build-bundles.ts' | |
| - 'benchmarks/scripts/ensure-bundle.ts' | |
| - 'benchmarks/src/run.ts' | |
| - 'benchmarks/scripts/load-vault-secrets.sh' | |
| - 'package.json' | |
| - '.github/workflows/sandbox-network-localhost.yml' | |
| workflow_dispatch: | |
| inputs: | |
| replicas: | |
| description: 'Replicates per provider' | |
| required: false | |
| default: '3' | |
| provider: | |
| description: 'Comma-separated provider filter. Leave empty for all 23 providers.' | |
| required: false | |
| default: '' | |
| dry_run: | |
| description: 'Run without ingesting or committing results' | |
| required: false | |
| default: false | |
| type: boolean | |
| concurrency: | |
| group: network-localhost-benchmark | |
| cancel-in-progress: true | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| id-token: write | |
| jobs: | |
| bench: | |
| name: ${{ matrix.provider }} | |
| runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe | |
| # runs-on: self-hosted | |
| timeout-minutes: 10 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| provider: | |
| - archil | |
| - beam | |
| - blaxel | |
| - cloud-run | |
| - cloudflare | |
| - codesandbox | |
| - createos | |
| - daytona | |
| - declaw | |
| - e2b | |
| - hopx | |
| - isorun | |
| - lightning | |
| - modal | |
| - mosaic | |
| - northflank | |
| - opencomputer | |
| - run-cloud | |
| - runloop | |
| - sail | |
| - sandbox0 | |
| - superserve | |
| - tenki | |
| - tensorlake | |
| - upstash | |
| - vercel | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: pnpm/action-setup@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: 24 | |
| cache: 'pnpm' | |
| - uses: namespacelabs/nscloud-setup@v0 | |
| - name: Install dependencies | |
| if: | | |
| github.event_name != 'workflow_dispatch' || | |
| (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) | |
| run: | | |
| if [ "${{ github.event_name }}" = 'schedule' ]; then | |
| pnpm update | |
| else | |
| pnpm install --frozen-lockfile | |
| fi | |
| - name: Build bundles | |
| if: | | |
| github.event_name != 'workflow_dispatch' || | |
| (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) | |
| run: pnpm run build:bundles | |
| - name: Clear stale results | |
| if: | | |
| github.event_name != 'workflow_dispatch' || | |
| (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) | |
| run: rm -rf results/ | |
| - name: Run network-localhost benchmark | |
| if: | | |
| github.event_name != 'workflow_dispatch' || | |
| (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) | |
| run: | | |
| . benchmarks/scripts/load-vault-secrets.sh '^(ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID|BEAM_TOKEN|BEAM_WORKSPACE_ID|BL_API_KEY|BL_WORKSPACE|CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET|CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET|CSB_API_KEY|CREATEOS_SANDBOX_API_KEY|DAYTONA_API_KEY|DECLAW_API_KEY|E2B_API_KEY|HOPX_API_KEY|ISORUN_API_KEY|LIGHTNING_API_KEY|MODAL_TOKEN_ID|MODAL_TOKEN_SECRET|MOSAIC_API_URL|MOSAIC_API_TOKEN|NSC_TOKEN|NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID|OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL|RUN_CLOUD_API_KEY|RUNLOOP_API_KEY|SAIL_API_KEY|SANDBOX0_TOKEN|SPRITES_TOKEN|SUPERSERVE_API_KEY|TENKI_API_KEY|TENSORLAKE_API_KEY|UPSTASH_BOX_API_KEY|VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID|BENCHMARKS_PLATFORM_API_KEY)' | |
| pnpm run bench -- \ | |
| --provider ${{ matrix.provider }} \ | |
| --mode network-localhost \ | |
| --replicas ${{ github.event.inputs.replicas || '3' }} | |
| - name: Upload results | |
| if: | | |
| always() && ( | |
| github.event_name != 'workflow_dispatch' || | |
| (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider))) | |
| ) | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: results-${{ matrix.provider }} | |
| path: results/ | |
| if-no-files-found: ignore | |
| retention-days: 14 | |
| collect: | |
| name: Collect Results | |
| runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe | |
| # runs-on: self-hosted | |
| needs: bench | |
| if: always() | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - uses: pnpm/action-setup@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: 24 | |
| cache: 'pnpm' | |
| - uses: namespacelabs/nscloud-setup@v0 | |
| - name: Install dependencies | |
| run: | | |
| if [ "${{ github.event_name }}" = 'schedule' ]; then | |
| pnpm update | |
| else | |
| pnpm install --frozen-lockfile | |
| fi | |
| - name: Download all artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: artifacts/ | |
| pattern: results-* | |
| - name: Merge per-provider results | |
| run: npx tsx benchmarks/src/merge-results.ts --input artifacts --mode benchmark | |
| - name: Generate SVG chart | |
| run: pnpm run generate-network-localhost-svg | |
| - name: Upload SVG as artifact | |
| if: github.event_name == 'push' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: network-localhost-benchmark-svg | |
| path: 'network_localhost.svg' | |
| if-no-files-found: ignore | |
| retention-days: 7 | |
| - name: Ingest results to platform | |
| if: github.event_name != 'push' && github.event.inputs.dry_run != 'true' | |
| continue-on-error: true | |
| run: | | |
| . benchmarks/scripts/load-vault-secrets.sh '^(BENCHMARKS_PLATFORM_API_KEY|BENCHMARKS_PLATFORM_URL)=' | |
| npx tsx benchmarks/src/ingest.ts --type network-localhost | |
| - name: Commit and push | |
| if: github.event_name != 'push' && github.event.inputs.dry_run != 'true' | |
| run: | | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| git add results/ network_localhost.svg | |
| git diff --cached --quiet && echo "No changes to commit" && exit 0 | |
| git commit -m "chore: update network-localhost benchmark results [skip ci]" | |
| branch="${GITHUB_REF#refs/heads/}" | |
| for attempt in 1 2 3 4 5; do | |
| git fetch origin "${branch}" | |
| git rebase --autostash "origin/${branch}" || { git rebase --abort; exit 1; } | |
| if git push origin "HEAD:${branch}"; then | |
| echo "Pushed on attempt ${attempt}" | |
| exit 0 | |
| fi | |
| echo "Push rejected (attempt ${attempt}); will rebase and retry" | |
| done | |
| echo "Failed to push after multiple attempts" >&2 |