sandbox-disk.yml

name: Disk Benchmark

on:
  push:
    branches: [master]
    paths:
      - 'benchmarks/sandbox/disk/**'
      - 'benchmarks/scripts/smoke.ts'
      - 'benchmarks/scripts/build-bundles.ts'
      - 'benchmarks/scripts/ensure-bundle.ts'
      - 'benchmarks/src/run.ts'
      - 'benchmarks/scripts/load-vault-secrets.sh'
      - 'package.json'
      - '.github/workflows/sandbox-disk.yml'
  workflow_dispatch:
    inputs:
      replicas:
        description: 'Replicates per provider'
        required: false
        default: '3'
      provider:
        description: 'Comma-separated provider filter. Leave empty for all 23 providers.'
        required: false
        default: ''
      dry_run:
        description: 'Run without ingesting or committing results'
        required: false
        default: false
        type: boolean

concurrency:
  group: disk-benchmark
  cancel-in-progress: true

permissions:
  contents: write
  pull-requests: write
  id-token: write

jobs:
  bench:
    name: ${{ matrix.provider }}
    runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe
    # runs-on: self-hosted
    timeout-minutes: 15
    strategy:
      fail-fast: false
      matrix:
        provider:
          - archil
          - beam
          - blaxel
          - cloud-run
          - cloudflare
          - codesandbox
          - createos
          - daytona
          - declaw
          - e2b
          - hopx
          - isorun
          - lightning
          - modal
          - mosaic
          - northflank
          - opencomputer
          - run-cloud
          - runloop
          - sail
          - sandbox0
          - superserve
          - tenki
          - tensorlake
          - upstash
          - vercel
    steps:
      - uses: actions/checkout@v4
      - uses: pnpm/action-setup@v4
      - uses: actions/setup-node@v4
        with:
          node-version: 24
          cache: 'pnpm'
      - uses: namespacelabs/nscloud-setup@v0
      - name: Install dependencies
        if: |
          github.event_name != 'workflow_dispatch' ||
          (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider)))
        run: |
          if [ "${{ github.event_name }}" = 'schedule' ]; then
            pnpm update
          else
            pnpm install --frozen-lockfile
          fi
      - name: Build bundles
        if: |
          github.event_name != 'workflow_dispatch' ||
          (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider)))
        run: pnpm run build:bundles
      - name: Clear stale results
        if: |
          github.event_name != 'workflow_dispatch' ||
          (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider)))
        run: rm -rf results/
      - name: Run disk benchmark
        if: |
          github.event_name != 'workflow_dispatch' ||
          (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider)))
        run: |
          . benchmarks/scripts/load-vault-secrets.sh '^(ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID|BEAM_TOKEN|BEAM_WORKSPACE_ID|BL_API_KEY|BL_WORKSPACE|CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET|CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET|CSB_API_KEY|CREATEOS_SANDBOX_API_KEY|DAYTONA_API_KEY|DECLAW_API_KEY|E2B_API_KEY|HOPX_API_KEY|ISORUN_API_KEY|LIGHTNING_API_KEY|MODAL_TOKEN_ID|MODAL_TOKEN_SECRET|MOSAIC_API_URL|MOSAIC_API_TOKEN|NSC_TOKEN|NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID|OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL|RUN_CLOUD_API_KEY|RUNLOOP_API_KEY|SAIL_API_KEY|SANDBOX0_TOKEN|SPRITES_TOKEN|SUPERSERVE_API_KEY|TENKI_API_KEY|TENSORLAKE_API_KEY|UPSTASH_BOX_API_KEY|VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID|BENCHMARKS_PLATFORM_API_KEY)'

          pnpm run bench -- \
            --provider ${{ matrix.provider }} \
            --mode disk \
            --replicas ${{ github.event.inputs.replicas || '3' }}
      - name: Upload results
        if: |
          always() && (
            github.event_name != 'workflow_dispatch' ||
            (github.event.inputs.provider == '' || contains(format(',{0},', github.event.inputs.provider), format(',{0},', matrix.provider)))
          )
        uses: actions/upload-artifact@v4
        with:
          name: results-${{ matrix.provider }}
          path: results/
          if-no-files-found: ignore
          retention-days: 14

  collect:
    name: Collect Results
    runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe
    # runs-on: self-hosted
    needs: bench
    if: always()
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - uses: pnpm/action-setup@v4
      - uses: actions/setup-node@v4
        with:
          node-version: 24
          cache: 'pnpm'
      - uses: namespacelabs/nscloud-setup@v0
      - name: Install dependencies
        run: |
          if [ "${{ github.event_name }}" = 'schedule' ]; then
            pnpm update
          else
            pnpm install --frozen-lockfile
          fi
      - name: Download all artifacts
        uses: actions/download-artifact@v4
        with:
          path: artifacts/
          pattern: results-*
      - name: Merge per-provider results
        run: npx tsx benchmarks/src/merge-results.ts --input artifacts --mode benchmark
      - name: Generate SVG chart
        run: pnpm run generate-disk-svg
      - name: Upload SVG as artifact
        if: github.event_name == 'push'
        uses: actions/upload-artifact@v4
        with:
          name: disk-benchmark-svg
          path: 'disk.svg'
          if-no-files-found: ignore
          retention-days: 7
      - name: Ingest results to platform
        if: github.event_name != 'push' && github.event.inputs.dry_run != 'true'
        continue-on-error: true
        run: |
          . benchmarks/scripts/load-vault-secrets.sh '^(BENCHMARKS_PLATFORM_API_KEY|BENCHMARKS_PLATFORM_URL)='
          npx tsx benchmarks/src/ingest.ts --type disk
      - name: Commit and push
        if: github.event_name != 'push' && github.event.inputs.dry_run != 'true'
        run: |
          git config user.name "github-actions[bot]"
          git config user.email "github-actions[bot]@users.noreply.github.com"
          git add results/ disk.svg
          git diff --cached --quiet && echo "No changes to commit" && exit 0
          git commit -m "chore: update disk benchmark results [skip ci]"

          branch="${GITHUB_REF#refs/heads/}"
          for attempt in 1 2 3 4 5; do
            git fetch origin "${branch}"
            git rebase --autostash "origin/${branch}" || { git rebase --abort; exit 1; }
            if git push origin "HEAD:${branch}"; then
              echo "Pushed on attempt ${attempt}"
              exit 0
            fi
            echo "Push rejected (attempt ${attempt}); will rebase and retry"
          done
          echo "Failed to push after multiple attempts" >&2