Skip to content
Sandbox TTI Benchmark

Sandbox TTI Benchmark #49

Sign in to view logs

Sandbox TTI Benchmark

Sandbox TTI Benchmark #49

Workflow file for this run

.github/workflows/sandbox-tti-benchmarks.yml at fe650fd
name: Sandbox TTI Benchmark
on:
push:
branches: [master]
paths:
- 'benchmarks/sandbox/**'
- 'package.json'
workflow_dispatch:
inputs:
iterations:
description: 'Iterations per provider'
required: false
default: '100'
concurrency:
description: 'Concurrent sandboxes'
required: false
default: '100'
permissions:
contents: write
jobs:
bench:
name: Bench ${{ matrix.provider }}
runs-on: namespace-profile-default
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
provider:
- createos
- isorun
steps:
- uses: actions/checkout@v4
- uses: pnpm/action-setup@v4
- uses: actions/setup-node@v4
with:
node-version: 24
cache: 'pnpm'
- name: Install dependencies
run: pnpm install --frozen-lockfile
- name: Run benchmark
env:
CREATEOS_SANDBOX_API_KEY: ${{ secrets.CREATEOS_SANDBOX_API_KEY }}
ISORUN_API_KEY: ${{ secrets.ISORUN_API_KEY }}
run: |
cat > bench-run.mts << 'BENCH'
import { providers } from './benchmarks/sandbox/providers.ts';
const ITERATIONS = parseInt(process.env.BENCH_ITERATIONS || '100');
const CONCURRENCY = parseInt(process.env.BENCH_CONCURRENCY || '100');
const PROVIDER = process.env.BENCH_PROVIDER || 'createos';
const TIMEOUT = 120_000;
const CMD_TIMEOUT = 30_000;
const DESTROY_TIMEOUT = 15_000;
const provider = providers.find(p => p.name === PROVIDER);
if (!provider) { console.error('Provider not found:', PROVIDER); process.exit(1); }
function withTimeout<T>(p: Promise<T>, ms: number, msg: string): Promise<T> {
return Promise.race([p, new Promise<T>((_, rej) => setTimeout(() => rej(new Error(msg)), ms))]);
}
console.log('Provider:', PROVIDER);
console.log('Iterations:', ITERATIONS, '| Concurrency:', CONCURRENCY);
console.log('');
const results: Array<{ ttiMs: number; error?: string }> = [];
const sandboxes: Array<{ destroy(): Promise<unknown> }> = [];
for (let batch = 0; batch < ITERATIONS; batch += CONCURRENCY) {
const batchSize = Math.min(CONCURRENCY, ITERATIONS - batch);
const promises = Array.from({ length: batchSize }, async (_, j) => {
const i = batch + j;
const start = performance.now();
try {
const compute = provider.createCompute();
const sandbox = await withTimeout(
compute.sandbox.create(provider.sandboxOptions),
provider.timeout ?? TIMEOUT,
'Sandbox creation timed out'
);
sandboxes.push(sandbox);
try {
const result = await withTimeout(
sandbox.runCommand('node -v'),
CMD_TIMEOUT,
'Command timed out'
);
const ttiMs = performance.now() - start;
if (result.exitCode !== 0) throw new Error('exit code ' + result.exitCode);
console.log(` [${i+1}/${ITERATIONS}] ${ttiMs.toFixed(0)} ms`);
return { ttiMs };
} finally {
await withTimeout(sandbox.destroy(), provider.destroyTimeoutMs ?? DESTROY_TIMEOUT, 'Destroy timeout').catch(() => {});
}
} catch (err: any) {
const ttiMs = performance.now() - start;
console.log(` [${i+1}/${ITERATIONS}] FAILED: ${err.message}`);
return { ttiMs, error: err.message };
}
});
results.push(...await Promise.all(promises));
}
// Cleanup sweep: retry destroy on all tracked sandboxes
console.log(`\n Cleanup: destroying ${sandboxes.length} sandboxes...`);
await Promise.allSettled(sandboxes.map(s => s.destroy().catch(() => {})));
console.log(' Cleanup complete.');
const ok = results.filter(r => !r.error).map(r => r.ttiMs).sort((a, b) => a - b);
const pct = (p: number) => ok[Math.min(Math.ceil(p / 100 * ok.length) - 1, ok.length - 1)];
const avg = ok.length > 0 ? ok.reduce((a, b) => a + b, 0) / ok.length : 0;
const successRate = ok.length / results.length;
// Composite score: weighted timing score × success rate (same as scoring.ts)
const CEILING = 10_000;
const scoreMetric = (v: number) => Math.max(0, 100 * (1 - v / CEILING));
let compositeScore = 0;
if (ok.length > 0) {
const timingScore = 0.60 * scoreMetric(pct(50)) + 0.25 * scoreMetric(pct(95)) + 0.15 * scoreMetric(pct(99));
compositeScore = Math.round(timingScore * successRate * 100) / 100;
}
const pad = (s: string, n: number) => s.padEnd(n);
const ms = (v: number) => (v.toFixed(0) + ' ms').padStart(8);
console.log('');
console.log('='.repeat(90));
console.log(` Results: ${PROVIDER} (${ok.length}/${results.length} succeeded)`);
console.log('='.repeat(90));
console.log('');
const hdr = [
pad('#', 3), pad('Provider', 12), pad('Ok/Total', 10), pad('Score', 8),
pad('Min', 10), pad('P50', 10), pad('P90', 10), pad('P95', 10),
pad('P99', 10), pad('Max', 10), pad('Avg', 10),
].join(' | ');
const sep = [
'-'.repeat(3), '-'.repeat(12), '-'.repeat(10), '-'.repeat(8),
'-'.repeat(10), '-'.repeat(10), '-'.repeat(10), '-'.repeat(10),
'-'.repeat(10), '-'.repeat(10), '-'.repeat(10),
].join(' | ');
console.log(` ${hdr}`);
console.log(` ${sep}`);
if (ok.length > 0) {
const row = [
pad('1', 3), pad(PROVIDER, 12), pad(`${ok.length}/${results.length}`, 10),
pad(compositeScore.toFixed(1), 8),
ms(ok[0]), ms(pct(50)), ms(pct(90)), ms(pct(95)), ms(pct(99)),
ms(ok[ok.length - 1]), ms(avg),
].join(' | ');
console.log(` ${row}`);
} else {
console.log(` ${pad('1', 3)} | ${pad(PROVIDER, 12)} | ${pad('0/'+results.length, 10)} | ${pad('0.0', 8)} | ${'FAIL'.padStart(8)} | ${'FAIL'.padStart(8)} | ${'FAIL'.padStart(8)} | ${'FAIL'.padStart(8)} | ${'FAIL'.padStart(8)} | ${'FAIL'.padStart(8)} | ${'FAIL'.padStart(8)}`);
}
console.log('');
process.exit(0);
BENCH
BENCH_ITERATIONS=${{ github.event.inputs.iterations || '100' }} \
BENCH_CONCURRENCY=${{ github.event.inputs.concurrency || '100' }} \
BENCH_PROVIDER=${{ matrix.provider }} \
npx tsx bench-run.mts