ci: add benchmark block presets and measured precompile mix

.github/benchmark-blocks.json

@@ -0,0 +1,23 @@
+{
+  "_meta": "Curated benchmark blocks. Percentile roles come from the 16-GPU release benchmark over blocks 24000000..24007199 (July 2026) — re-derive them after major guest program changes. Precompile counts were measured with .github/scripts/precompile_mix.py; every compare run also measures the mix of the block it actually benchmarks, so these notes are documentation, not the source of truth.",
+  "p100": {
+    "block": 24001988,
+    "notes": "Slowest block of the range (123 segments). 60M gas in only 65 txs, precompiles: ecRecover=1 — structurally slow pure-EVM compute, so its tail ranking is robust to precompile accelerations."
+  },
+  "p99": {
+    "block": 24004255,
+    "notes": "p99 latency block (79 segments). 57M gas, clean mix (single bn254 add/mul/pairing). Cheaper CI alternative to p100."
+  },
+  "p50": {
+    "block": 24000855,
+    "notes": "Median block (40 segments). 28M gas, no exotic precompiles. Use to check that a win on tail blocks generalizes to typical blocks."
+  },
+  "p256-heavy": {
+    "block": 23992138,
+    "notes": "P256VERIFY=7, modexp=14, bn254 add/mul=29 each. Default block until July 2026; it over-represents P256VERIFY (~8% of its instructions before the openvm p256 hook), which is why single-block wins on it did not generalize. Kept for precompile-targeted comparisons."
+  },
+  "keccak-heavy": {
+    "block": 24002549,
+    "notes": "Aztec-style rollup block, 109 segments, 1.03B guest instructions. Guest circuit flamegraph (July 2026): KeccakfPermAir is 20.7% of app-proof cells (~4.5M permutations) while all modular/EC/pairing AIRs together are 0.31% - its modexp=277 and bn254 add/mul=65 calls are fully served by the accelerated fast paths and cost nothing. Use to stress keccak cell volume and sheer block size, not field arithmetic."
+  }
+}

.github/scripts/precompile_mix.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+"""Measure a block's precompile mix via debug_traceBlockByNumber (callTracer).
+
+Usage: precompile_mix.py <block_number>
+
+Uses $RPC_URL if set (must support the debug namespace), with public
+trace-capable endpoints as fallback. Prints a markdown summary suitable for
+$GITHUB_STEP_SUMMARY. Exits non-zero if no endpoint could trace the block.
+"""
+
+import json
+import os
+import sys
+import time
+import urllib.request
+
+FALLBACK_RPCS = ["https://eth.drpc.org"]
+
+PRECOMPILES = {
+    "0x0000000000000000000000000000000000000001": "ecRecover",
+    "0x0000000000000000000000000000000000000002": "SHA2-256",
+    "0x0000000000000000000000000000000000000003": "RIPEMD-160",
+    "0x0000000000000000000000000000000000000004": "identity",
+    "0x0000000000000000000000000000000000000005": "modexp",
+    "0x0000000000000000000000000000000000000006": "bn254_add",
+    "0x0000000000000000000000000000000000000007": "bn254_mul",
+    "0x0000000000000000000000000000000000000008": "bn254_pairing",
+    "0x0000000000000000000000000000000000000009": "blake2f",
+    "0x000000000000000000000000000000000000000a": "kzg_point_eval",
+    "0x000000000000000000000000000000000000000b": "bls_g1add",
+    "0x000000000000000000000000000000000000000c": "bls_g1msm",
+    "0x000000000000000000000000000000000000000d": "bls_g2add",
+    "0x000000000000000000000000000000000000000e": "bls_g2msm",
+    "0x000000000000000000000000000000000000000f": "bls_pairing",
+    "0x0000000000000000000000000000000000000010": "bls_map_fp",
+    "0x0000000000000000000000000000000000000011": "bls_map_fp2",
+    "0x0000000000000000000000000000000000000100": "P256VERIFY",
+}
+
+
+def rpc_call(rpcs, method, params):
+    payload = json.dumps({"jsonrpc": "2.0", "id": 1, "method": method, "params": params}).encode()
+    last_err = None
+    for attempt in range(8):
+        rpc = rpcs[attempt % len(rpcs)]
+        req = urllib.request.Request(
+            rpc,
+            data=payload,
+            headers={"Content-Type": "application/json", "User-Agent": "curl/8"},
+        )
+        try:
+            with urllib.request.urlopen(req, timeout=120) as resp:
+                data = json.loads(resp.read())
+            if data.get("result") is not None:
+                return data["result"]
+            last_err = data.get("error")
+        except Exception as e:
+            last_err = str(e)
+        time.sleep(2)
+    raise RuntimeError(f"all RPC attempts failed, last error: {last_err}")
+
+
+def count_calls(node, counts):
+    to = (node.get("to") or "").lower()
+    if to in PRECOMPILES:
+        counts[PRECOMPILES[to]] = counts.get(PRECOMPILES[to], 0) + 1
+    for child in node.get("calls") or []:
+        count_calls(child, counts)
+
+
+def main():
+    block = int(sys.argv[1])
+    rpcs = ([os.environ["RPC_URL"]] if os.environ.get("RPC_URL") else []) + FALLBACK_RPCS
+    trace = rpc_call(rpcs, "debug_traceBlockByNumber", [hex(block), {"tracer": "callTracer", "timeout": "120s"}])
+    counts = {}
+    for tx in trace:
+        count_calls(tx.get("result", {}), counts)
+
+    print(f"### Precompile mix of block {block} (measured)")
+    print()
+    if not counts:
+        print("No precompile calls — pure EVM compute block.")
+    else:
+        print("| precompile | calls |")
+        print("|---|---|")
+        for name, n in sorted(counts.items(), key=lambda kv: -kv[1]):
+            print(f"| {name} | {n} |")
+        print()
+        print("A win concentrated in one precompile only generalizes to blocks that call it — check the mix before extrapolating single-block deltas to fleet percentiles.")
+    print()
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/compare-bench.yml

@@ -1,5 +1,5 @@
 name: "Compare Benchmarks"
-run-name: "Compare Benchmarks (block ${{ inputs.block_number || github.event.inputs.block_number }})"
+run-name: "Compare Benchmarks (block ${{ inputs.block_number || inputs.block_preset }})"
 
 on:
   workflow_dispatch:
@@ -12,11 +12,21 @@ on:
         description: "Target OpenVM revision to compare"
         required: true
         type: string
+      block_preset:
+        type: choice
+        required: false
+        description: Benchmark block preset (see .github/benchmark-blocks.json)
+        options:
+          - p100
+          - p99
+          - p50
+          - keccak-heavy
+          - p256-heavy
+        default: p100
       block_number:
-        type: number
+        type: string
         required: false
-        description: Ethereum block number
-        default: 24001988
+        description: Ethereum block number (overrides block_preset)
       benchmark_mode:
         type: choice
         required: false
@@ -47,14 +57,37 @@ on:
         default: false
 
 jobs:
+  resolve-block:
+    name: "Resolve Benchmark Block"
+    runs-on: ubuntu-latest
+    outputs:
+      block: ${{ steps.resolve.outputs.block }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Resolve block number
+        id: resolve
+        run: |
+          BLOCK="${{ inputs.block_number }}"
+          if [[ -z "$BLOCK" ]]; then
+            BLOCK=$(jq -er --arg preset "${{ inputs.block_preset }}" '.[$preset].block' .github/benchmark-blocks.json)
+          fi
+          [[ "$BLOCK" =~ ^[0-9]+$ ]] || { echo "invalid block number: $BLOCK"; exit 1; }
+          echo "block=$BLOCK" >> "$GITHUB_OUTPUT"
+      - name: Measure precompile mix
+        continue-on-error: true
+        env:
+          RPC_URL: ${{ secrets.RPC_URL_1 }}
+        run: python3 .github/scripts/precompile_mix.py "${{ steps.resolve.outputs.block }}" >> "$GITHUB_STEP_SUMMARY"
+
   run-base-benchmark:
     name: "Run Base Benchmark"
     uses: ./.github/workflows/update-patches.yml
+    needs: resolve-block
     with:
       OPENVM_REV: ${{ github.event.inputs.base_rev }}
       run_benchmark: true
       benchmark_mode: ${{ github.event.inputs.benchmark_mode }}
-      benchmark_block_number: ${{ fromJSON(inputs.block_number || github.event.inputs.block_number || '24001988') }}
+      benchmark_block_number: ${{ fromJSON(needs.resolve-block.outputs.block) }}
       instance_family: ${{ github.event.inputs.instance_family }}
     secrets: inherit
 
@@ -70,10 +103,10 @@ jobs:
   run-target-benchmark:
     name: "Run Target Benchmark"
     uses: ./.github/workflows/reth-benchmark.yml
-    needs: patch-target
+    needs: [patch-target, resolve-block]
     with:
       mode: ${{ github.event.inputs.benchmark_mode }}
-      block_number: ${{ fromJSON(inputs.block_number || github.event.inputs.block_number || '24001988') }}
+      block_number: ${{ fromJSON(needs.resolve-block.outputs.block) }}
       instance_family: ${{ github.event.inputs.instance_family }}
       ref: ${{ needs.patch-target.outputs.branch_name }}
       tag: ${{ needs.patch-target.outputs.tag }}
@@ -84,10 +117,10 @@ jobs:
     name: "Target Host Perf Flamegraph"
     uses: ./.github/workflows/reth-benchmark.yml
     if: ${{ github.event.inputs.host_flamegraph == 'true' }}
-    needs: patch-target
+    needs: [patch-target, resolve-block]
     with:
       mode: ${{ github.event.inputs.benchmark_mode }}
-      block_number: ${{ fromJSON(inputs.block_number || github.event.inputs.block_number || '24001988') }}
+      block_number: ${{ fromJSON(needs.resolve-block.outputs.block) }}
       instance_family: ${{ github.event.inputs.instance_family }}
       ref: ${{ needs.patch-target.outputs.branch_name }}
       tag: ${{ needs.patch-target.outputs.tag }}
@@ -98,10 +131,10 @@ jobs:
     name: "Target Guest Circuit Flamegraphs"
     uses: ./.github/workflows/reth-benchmark.yml
     if: ${{ github.event.inputs.guest_flamegraph == 'true' }}
-    needs: patch-target
+    needs: [patch-target, resolve-block]
     with:
       mode: ${{ github.event.inputs.benchmark_mode }}
-      block_number: ${{ fromJSON(inputs.block_number || github.event.inputs.block_number || '24001988') }}
+      block_number: ${{ fromJSON(needs.resolve-block.outputs.block) }}
       instance_family: ${{ github.event.inputs.instance_family }}
       ref: ${{ needs.patch-target.outputs.branch_name }}
       tag: ${{ needs.patch-target.outputs.tag }}
@@ -110,6 +143,7 @@ jobs:
 
   compare-results:
     needs:
+      - resolve-block
       - run-base-benchmark
       - run-target-benchmark
     runs-on: ubuntu-latest
@@ -152,4 +186,4 @@ jobs:
           echo "Target OpenVM Revision: ${{ github.event.inputs.target_rev }}" >> $GITHUB_STEP_SUMMARY
           echo "Instance Family: ${{ github.event.inputs.instance_family }}" >> $GITHUB_STEP_SUMMARY
           echo "Benchmark Mode: ${{ github.event.inputs.benchmark_mode }}" >> $GITHUB_STEP_SUMMARY
-          echo "Block Number: ${{ github.event.inputs.block_number }}" >> $GITHUB_STEP_SUMMARY
+          echo "Block Number: ${{ needs.resolve-block.outputs.block }}" >> $GITHUB_STEP_SUMMARY

.github/workflows/compare-branches.yml

@@ -1,5 +1,5 @@
 name: "Compare Branches"
-run-name: "Compare Branches: ${{ inputs.base_ref }} vs ${{ inputs.target_ref }} (block ${{ inputs.block_number }})"
+run-name: "Compare Branches: ${{ inputs.base_ref }} vs ${{ inputs.target_ref }} (block ${{ inputs.block_number || inputs.block_preset }})"
 
 on:
   workflow_dispatch:
@@ -13,11 +13,21 @@ on:
         description: "Target branch/ref to compare"
         required: true
         type: string
+      block_preset:
+        type: choice
+        required: false
+        description: Benchmark block preset (see .github/benchmark-blocks.json)
+        options:
+          - p100
+          - p99
+          - p50
+          - keccak-heavy
+          - p256-heavy
+        default: p100
       block_number:
-        type: number
+        type: string
         required: false
-        description: Ethereum block number
-        default: 24001988
+        description: Ethereum block number (overrides block_preset)
       benchmark_mode:
         type: choice
         required: false
@@ -48,24 +58,48 @@ on:
         default: false
 
 jobs:
+  resolve-block:
+    name: "Resolve Benchmark Block"
+    runs-on: ubuntu-latest
+    outputs:
+      block: ${{ steps.resolve.outputs.block }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Resolve block number
+        id: resolve
+        run: |
+          BLOCK="${{ inputs.block_number }}"
+          if [[ -z "$BLOCK" ]]; then
+            BLOCK=$(jq -er --arg preset "${{ inputs.block_preset }}" '.[$preset].block' .github/benchmark-blocks.json)
+          fi
+          [[ "$BLOCK" =~ ^[0-9]+$ ]] || { echo "invalid block number: $BLOCK"; exit 1; }
+          echo "block=$BLOCK" >> "$GITHUB_OUTPUT"
+      - name: Measure precompile mix
+        continue-on-error: true
+        env:
+          RPC_URL: ${{ secrets.RPC_URL_1 }}
+        run: python3 .github/scripts/precompile_mix.py "${{ steps.resolve.outputs.block }}" >> "$GITHUB_STEP_SUMMARY"
+
   run-base-benchmark:
     name: "Run Base Benchmark (${{ inputs.base_ref }})"
     uses: ./.github/workflows/reth-benchmark.yml
+    needs: resolve-block
     with:
       ref: ${{ inputs.base_ref }}
       mode: ${{ inputs.benchmark_mode }}
-      block_number: ${{ fromJSON(inputs.block_number || '24001988') }}
+      block_number: ${{ fromJSON(needs.resolve-block.outputs.block) }}
       instance_family: ${{ inputs.instance_family }}
       profiling: none
     secrets: inherit
 
   run-target-benchmark:
     name: "Run Target Benchmark (${{ inputs.target_ref }})"
     uses: ./.github/workflows/reth-benchmark.yml
+    needs: resolve-block
     with:
       ref: ${{ inputs.target_ref }}
       mode: ${{ inputs.benchmark_mode }}
-      block_number: ${{ fromJSON(inputs.block_number || '24001988') }}
+      block_number: ${{ fromJSON(needs.resolve-block.outputs.block) }}
       instance_family: ${{ inputs.instance_family }}
       profiling: none
     secrets: inherit
@@ -74,10 +108,11 @@ jobs:
     name: "Target Host Perf Flamegraph"
     uses: ./.github/workflows/reth-benchmark.yml
     if: ${{ inputs.host_flamegraph }}
+    needs: resolve-block
     with:
       ref: ${{ inputs.target_ref }}
       mode: ${{ inputs.benchmark_mode }}
-      block_number: ${{ fromJSON(inputs.block_number || '24001988') }}
+      block_number: ${{ fromJSON(needs.resolve-block.outputs.block) }}
       instance_family: ${{ inputs.instance_family }}
       profiling: host
     secrets: inherit
@@ -86,16 +121,18 @@ jobs:
     name: "Target Guest Circuit Flamegraphs"
     uses: ./.github/workflows/reth-benchmark.yml
     if: ${{ inputs.guest_flamegraph }}
+    needs: resolve-block
     with:
       ref: ${{ inputs.target_ref }}
       mode: ${{ inputs.benchmark_mode }}
-      block_number: ${{ fromJSON(inputs.block_number || '24001988') }}
+      block_number: ${{ fromJSON(needs.resolve-block.outputs.block) }}
       instance_family: ${{ inputs.instance_family }}
       profiling: guest
     secrets: inherit
 
   compare-results:
     needs:
+      - resolve-block
       - run-base-benchmark
       - run-target-benchmark
     runs-on: ubuntu-latest
@@ -146,4 +183,4 @@ jobs:
           echo "Target Ref: ${{ inputs.target_ref }}" >> $GITHUB_STEP_SUMMARY
           echo "Instance Family: ${{ inputs.instance_family }}" >> $GITHUB_STEP_SUMMARY
           echo "Benchmark Mode: ${{ inputs.benchmark_mode }}" >> $GITHUB_STEP_SUMMARY
-          echo "Block Number: ${{ inputs.block_number }}" >> $GITHUB_STEP_SUMMARY
+          echo "Block Number: ${{ needs.resolve-block.outputs.block }}" >> $GITHUB_STEP_SUMMARY