perf: share calibrated work across fresh benchmark processes

.github/PERFORMANCE.md

@@ -6,8 +6,10 @@ order for the second pair. Each case gets a fresh app process; installation,
 startup, transport and process restarts are outside timing. There is no automatic
 third pair. Manual reruns are retained as identifiable workflow attempts.
 
-Each case records twenty ordered batch averages after five warmup batches.
-Calibration targets 150 ms of timed work; this target does not establish steady
+A separate base calibration process chooses one per-case iteration and chunk
+plan. Both revisions then use that plan for five warmup batches and twenty ordered
+measurements in fresh processes. Incompatible counts fail rather than silently
+shortening head work. Calibration targets 150 ms of timed work; this target does not establish steady
 state or erase drift. Allocation-heavy cases sum bounded timed chunks with
 explicit cleanup outside timing. Raw `iterations`, `chunkIterations` and ordered
 `samplesNsPerOp` describe the work; timed sample milliseconds are
@@ -27,12 +29,14 @@ failures still fail CI. Turning observed differences into a regression gate need
 empirical validation on unchanged commits and intentional slowdowns on each
 unchanged suite/testbed. No Promise case is permanently exempt. Scheduled/manual
 runs with the same base and head SHA measure baseline variation explicitly.
-Changed benchmark definitions require a new baseline and are not compared.
+Changed benchmark definitions run two head-only measurements as a new baseline,
+without executing the old base app or publishing an invented paired baseline.
+This also handles the first rollout of a new runner protocol.
 
 ## Artifacts and publishing
 
 The canonical artifact is `performance-report-<attempt>`: raw JSON for every
-base/head process plus `performance-report.json` with repository, revisions,
+measured process and discarded calibration plan, plus `performance-report.json` with repository, revisions,
 workflow run and attempt provenance. Artifacts remain available for 30 days.
 The PR comment links its exact immutable artifact ID; downloads require GitHub
 access. An agent can inspect the JSON instead of scraping the rendered table.

.github/workflows/performance.yml

@@ -370,15 +370,20 @@ jobs:
           PR_NUMBER: ${{ needs.prepare.outputs.pr_number }}
         run: |
           mkdir -p performance-report/raw/android performance-report/raw/ios
+          cmp artifacts/android/suite.json artifacts/ios/suite.json
           bun scripts/performance/report.ts \
+            --suite artifacts/android/suite.json \
             --output performance-report/performance-report.json \
             --repository "$REPOSITORY" \
             --event-name "${{ github.event_name }}" \
             --pull-request "$PR_NUMBER" \
             --base-sha "${{ needs.prepare.outputs.base_sha }}" \
             --head-sha "${{ needs.prepare.outputs.head_sha }}"
-          cp artifacts/android/base-*.json artifacts/android/head-*.json performance-report/raw/android/
-          cp artifacts/ios/base-*.json artifacts/ios/head-*.json performance-report/raw/ios/
+          for PLATFORM in android ios; do
+            for RESULT in artifacts/"$PLATFORM"/{base,head,calibration}-*.json; do
+              if [[ -f "$RESULT" ]]; then cp "$RESULT" performance-report/raw/"$PLATFORM"/; fi
+            done
+          done
 
       - name: Upload aggregate report
         if: needs.prepare.outputs.relevant == 'true'

apps/benchmark/README.md

@@ -61,9 +61,10 @@ Local runs do not upload results.
 
 Each metric targets 150 ms of timed work per sample (roughly
 100–200 ms), using round iteration counts with two significant digits, such as
-1,500,000 or 24,000. Calibration can grow or shrink the count and is rechecked
-after five warmup batches. That count is then frozen for twenty measured samples;
-slow samples are retained, not discarded or adaptively shortened.
+1,500,000 or 24,000. Calibration chooses the count in a separate process that is discarded. The
+measurement process performs five warmup batches and twenty samples using those
+fixed counts. In paired runs, both revisions share the base-derived count and
+chunk size for each case. Slow samples are retained without shortening the work.
 
 Allocation-heavy cases split a sample into bounded chunks, collecting garbage
 after each chunk and yielding for native cleanup at most every four chunks,

apps/benchmark/src/benchmarks/BenchmarkApp.tsx

@@ -1,3 +1,4 @@
+import { calibrateBenchmarkDefinitions } from './runner'
 import * as React from 'react'
 import { StyleSheet, Text, View } from 'react-native'
 import {
@@ -23,6 +24,12 @@ function isRunConfiguration(
   if (value == null || typeof value !== 'object') return false
   const candidate = value as Partial<BenchmarkRunConfiguration>
   return (
+    (candidate.calibration === undefined || candidate.calibration === true) &&
+    (candidate.work === undefined ||
+      (candidate.work != null &&
+        typeof candidate.work.id === 'string' &&
+        Number.isSafeInteger(candidate.work.iterations) &&
+        Number.isSafeInteger(candidate.work.chunkIterations))) &&
     typeof candidate.runId === 'string' &&
     typeof candidate.reverse === 'boolean' &&
     typeof candidate.commitSha === 'string' &&
@@ -87,17 +94,35 @@ async function run(): Promise<BenchmarkRunResult> {
         )
   if (selected.length === 0)
     throw new Error('Requested benchmark index is outside the suite.')
-  const metrics = await runBenchmarkDefinitions(selected, {
-    ...RUNNER_OPTIONS,
-    reverse: configuration.reverse,
-  })
+  const runner = configuration.calibration
+    ? { ...RUNNER_OPTIONS, warmupCount: 0, sampleCount: 0 }
+    : RUNNER_OPTIONS
+  const metrics = configuration.calibration
+    ? (
+        await calibrateBenchmarkDefinitions(
+          selected,
+          runner.targetBatchDurationMs
+        )
+      ).map((work, index) => ({
+        ...work,
+        version: selected[index]!.version,
+        family: selected[index]!.family,
+        implementation: selected[index]!.implementation,
+        samplesNsPerOp: [],
+        checksum: 0,
+      }))
+    : await runBenchmarkDefinitions(
+        selected,
+        { ...runner, reverse: configuration.reverse },
+        configuration.work == null ? [] : [configuration.work]
+      )
 
   return {
     schemaVersion: 1,
     suiteVersion: 1,
     configuration,
     environment,
-    runner: RUNNER_OPTIONS,
+    runner,
     startedAt,
     durationMs: performance.now() - start,
     metrics,

apps/benchmark/src/benchmarks/batch.ts

@@ -20,12 +20,12 @@ export const benchmarkRuntime: BenchmarkRuntime = {
 export async function executeBatch(
   definition: BenchmarkDefinition,
   iterations: number,
-  runtime: BenchmarkRuntime
-): Promise<{ durationMs: number; checksum: number }> {
-  const chunkIterations = Math.min(
+  runtime: BenchmarkRuntime,
+  chunkIterations = Math.min(
     iterations,
     definition.maxChunkIterations ?? iterations
   )
+): Promise<{ durationMs: number; checksum: number }> {
   if (!Number.isSafeInteger(chunkIterations) || chunkIterations < 1) {
     throw new Error(`Benchmark ${definition.id} has an invalid chunk size.`)
   }

apps/benchmark/src/benchmarks/calibration.ts

@@ -20,7 +20,7 @@ export async function calibrateIterations(
     if (!Number.isFinite(durationMs) || durationMs < 0) {
       throw new Error('Calibration requires a finite, non-negative duration.')
     }
-    // Aim inside the requested 100–200 ms window, leaving room for drift.
+    // Choose a practical batch duration; this does not establish steady state.
     if (durationMs >= targetMs * 0.8 && durationMs <= targetMs * 1.2) {
       if (++confirmations === 2) return iterations
       continue
@@ -39,5 +39,7 @@ export async function calibrateIterations(
       maximum
     )
   }
-  throw new Error('Batch duration did not stabilize during calibration.')
+  throw new Error(
+    'Calibration could not reach the target duration within its step limit.'
+  )
 }

apps/benchmark/src/benchmarks/runner.ts

@@ -1,84 +1,101 @@
 import { median } from './statistics'
-import { calibrateIterations, roundIterations } from './calibration'
+import { calibrateIterations } from './calibration'
 import { benchmarkRuntime, executeBatch, type BenchmarkRuntime } from './batch'
 import type {
   BenchmarkDefinition,
   BenchmarkMetric,
   BenchmarkRunnerOptions,
+  BenchmarkWork,
 } from './types'
 
+export async function calibrateBenchmarkDefinitions(
+  definitions: readonly BenchmarkDefinition[],
+  targetBatchDurationMs: number,
+  runtime: BenchmarkRuntime = benchmarkRuntime
+): Promise<BenchmarkWork[]> {
+  const work: BenchmarkWork[] = []
+  for (const definition of definitions) {
+    const iterations = await calibrateIterations(
+      async (count) =>
+        (await executeBatch(definition, count, runtime)).durationMs,
+      targetBatchDurationMs,
+      definition.initialIterations,
+      definition.maxIterations
+    )
+    work.push({
+      id: definition.id,
+      iterations,
+      chunkIterations: Math.min(
+        iterations,
+        definition.maxChunkIterations ?? iterations
+      ),
+    })
+  }
+  return work
+}
+
 export async function runBenchmarkDefinitions(
   definitions: readonly BenchmarkDefinition[],
   options: BenchmarkRunnerOptions,
+  work: readonly BenchmarkWork[],
   runtime: BenchmarkRuntime = benchmarkRuntime
 ): Promise<BenchmarkMetric[]> {
   const ordered = options.reverse
     ? [...definitions].reverse()
     : [...definitions]
   const metrics: BenchmarkMetric[] = []
-
   for (const definition of ordered) {
-    let iterations = definition.initialIterations ?? 1_000
-    let checksum = 0
-    for (let attempt = 0; attempt < 3; attempt++) {
-      iterations = await calibrateIterations(
-        async (count) =>
-          (await executeBatch(definition, count, runtime)).durationMs,
-        options.targetBatchDurationMs,
-        iterations,
-        definition.maxIterations
+    const plan = work.find((entry) => entry.id === definition.id)
+    if (
+      plan == null ||
+      !Number.isSafeInteger(plan.iterations) ||
+      plan.iterations < 1 ||
+      plan.iterations > (definition.maxIterations ?? 100_000_000) ||
+      !Number.isSafeInteger(plan.chunkIterations) ||
+      plan.chunkIterations < 1 ||
+      plan.chunkIterations > plan.iterations ||
+      plan.chunkIterations > (definition.maxChunkIterations ?? plan.iterations)
+    ) {
+      throw new Error(
+        `Missing or incompatible work counts for ${definition.id}.`
       )
-      checksum = 0
-      const warmupDurations: number[] = []
-      for (let index = 0; index < options.warmupCount; index++) {
-        const warmup = await executeBatch(definition, iterations, runtime)
-        checksum += warmup.checksum
-        warmupDurations.push(warmup.durationMs)
-      }
-      const warmupMedian = median(warmupDurations)
-      if (
-        warmupMedian >= options.targetBatchDurationMs * (2 / 3) &&
-        warmupMedian <= options.targetBatchDurationMs * (4 / 3)
-      )
-        break
-      if (attempt === 2) {
-        throw new Error(
-          `Benchmark ${definition.id} did not stabilize after warmup.`
+    }
+    let checksum = 0
+    for (let index = 0; index < options.warmupCount; index++) {
+      checksum += (
+        await executeBatch(
+          definition,
+          plan.iterations,
+          runtime,
+          plan.chunkIterations
         )
-      }
-      iterations = roundIterations(
-        (iterations * options.targetBatchDurationMs) / warmupMedian,
-        definition.maxIterations ?? 100_000_000
-      )
+      ).checksum
     }
-
-    // Freeze the count for all measured samples. Do not discard slow samples
-    // or tune iterations from measured results: that would bias the comparison.
-    const samplesNsPerOp = new Array<number>(options.sampleCount)
+    // The same plan is used for both binaries. Preserve every ordered sample,
+    // even if head is much slower than the calibration target.
+    const samplesNsPerOp: number[] = []
     for (let index = 0; index < options.sampleCount; index++) {
-      const sample = await executeBatch(definition, iterations, runtime)
+      const sample = await executeBatch(
+        definition,
+        plan.iterations,
+        runtime,
+        plan.chunkIterations
+      )
       checksum += sample.checksum
-      samplesNsPerOp[index] = (sample.durationMs * 1_000_000) / iterations
+      samplesNsPerOp.push((sample.durationMs * 1_000_000) / plan.iterations)
     }
-
     const metric: BenchmarkMetric = {
-      id: definition.id,
+      ...plan,
       version: definition.version,
       family: definition.family,
       implementation: definition.implementation,
-      iterations,
-      chunkIterations: Math.min(
-        iterations,
-        definition.maxChunkIterations ?? iterations
-      ),
       samplesNsPerOp,
       checksum,
     }
     metrics.push(metric)
     console.info(
-      `[NitroBenchmark] ${metric.id}: ${median(metric.samplesNsPerOp).toFixed(2)} ns/op; ${iterations} ops/sample, chunks of ${metric.chunkIterations}, median timed batch ${((median(metric.samplesNsPerOp) * iterations) / 1_000_000).toFixed(1)} ms`
+      `[NitroBenchmark] ${metric.id}: ${median(samplesNsPerOp).toFixed(2)} ns/op; ${plan.iterations} ops/sample, chunks of ${plan.chunkIterations}`
     )
   }
-
   return metrics
 }

apps/benchmark/src/benchmarks/types.ts

@@ -54,19 +54,25 @@ export interface BenchmarkRunnerOptions {
   reverse: boolean
 }
 
-export interface BenchmarkMetric {
+export interface BenchmarkWork {
   id: string
-  version: number
-  family: BenchmarkFamily
-  implementation: BenchmarkImplementation
   iterations: number
   /** Maximum operations between untimed garbage collections. */
   chunkIterations: number
+}
+
+export interface BenchmarkMetric extends BenchmarkWork {
+  version: number
+  family: BenchmarkFamily
+  implementation: BenchmarkImplementation
   samplesNsPerOp: number[]
   checksum: number
 }
 
 export interface BenchmarkRunConfiguration {
+  /** Calibration is discarded; measurement always uses a fresh process. */
+  calibration?: true
+  work?: BenchmarkWork
   /** Select one case in suite order for a fresh-process measurement. */
   benchmarkIndex?: number
   runId: string

scripts/performance/comparison.test.ts

@@ -100,4 +100,19 @@ describe('performance comparison', () => {
       ).suiteComparable
     ).toBe(false)
   })
+  test('rejects unequal work and calibration data even if timings look identical', () => {
+    const base = run(BASE_SHA, [100, 100])
+    const head = run(HEAD_SHA, [100, 100])
+    head.metrics[0]!.iterations = 9_000
+    expect(() => compareRuns([base], [head])).toThrow('unequal work')
+    head.metrics[0]!.iterations = base.metrics[0]!.iterations
+    head.configuration.calibration = true
+    expect(() => compareRuns([base], [head])).toThrow('Calibration runs')
+    expect(() => validateBenchmarkRun(head)).toThrow(
+      'Calibration must not contain'
+    )
+    head.configuration.calibration = undefined
+    head.metrics[0]!.samplesNsPerOp = []
+    expect(() => validateBenchmarkRun(head)).toThrow('Sample count')
+  })
 })