diff --git a/.github/benchmark-blocks.json b/.github/benchmark-blocks.json new file mode 100644 index 000000000..78f8e96f4 --- /dev/null +++ b/.github/benchmark-blocks.json @@ -0,0 +1,23 @@ +{ + "_meta": "Curated benchmark blocks. Percentile roles come from the 16-GPU release benchmark over blocks 24000000..24007199 (July 2026) — re-derive them after major guest program changes. Precompile counts were measured with .github/scripts/precompile_mix.py; every compare run also measures the mix of the block it actually benchmarks, so these notes are documentation, not the source of truth.", + "p100": { + "block": 24001988, + "notes": "Slowest block of the range (123 segments). 60M gas in only 65 txs, precompiles: ecRecover=1 — structurally slow pure-EVM compute, so its tail ranking is robust to precompile accelerations." + }, + "p99": { + "block": 24004255, + "notes": "p99 latency block (79 segments). 57M gas, clean mix (single bn254 add/mul/pairing). Cheaper CI alternative to p100." + }, + "p50": { + "block": 24000855, + "notes": "Median block (40 segments). 28M gas, no exotic precompiles. Use to check that a win on tail blocks generalizes to typical blocks." + }, + "p256-heavy": { + "block": 23992138, + "notes": "P256VERIFY=7, modexp=14, bn254 add/mul=29 each. Default block until July 2026; it over-represents P256VERIFY (~8% of its instructions before the openvm p256 hook), which is why single-block wins on it did not generalize. Kept for precompile-targeted comparisons." + }, + "keccak-heavy": { + "block": 24002549, + "notes": "Aztec-style rollup block, 109 segments, 1.03B guest instructions. Guest circuit flamegraph (July 2026): KeccakfPermAir is 20.7% of app-proof cells (~4.5M permutations) while all modular/EC/pairing AIRs together are 0.31% - its modexp=277 and bn254 add/mul=65 calls are fully served by the accelerated fast paths and cost nothing. Use to stress keccak cell volume and sheer block size, not field arithmetic." + } +} diff --git a/.github/scripts/precompile_mix.py b/.github/scripts/precompile_mix.py new file mode 100644 index 000000000..31a0108ce --- /dev/null +++ b/.github/scripts/precompile_mix.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""Measure a block's precompile mix via debug_traceBlockByNumber (callTracer). + +Usage: precompile_mix.py + +Uses $RPC_URL if set (must support the debug namespace), with public +trace-capable endpoints as fallback. Prints a markdown summary suitable for +$GITHUB_STEP_SUMMARY. Exits non-zero if no endpoint could trace the block. +""" + +import json +import os +import sys +import time +import urllib.request + +FALLBACK_RPCS = ["https://eth.drpc.org"] + +PRECOMPILES = { + "0x0000000000000000000000000000000000000001": "ecRecover", + "0x0000000000000000000000000000000000000002": "SHA2-256", + "0x0000000000000000000000000000000000000003": "RIPEMD-160", + "0x0000000000000000000000000000000000000004": "identity", + "0x0000000000000000000000000000000000000005": "modexp", + "0x0000000000000000000000000000000000000006": "bn254_add", + "0x0000000000000000000000000000000000000007": "bn254_mul", + "0x0000000000000000000000000000000000000008": "bn254_pairing", + "0x0000000000000000000000000000000000000009": "blake2f", + "0x000000000000000000000000000000000000000a": "kzg_point_eval", + "0x000000000000000000000000000000000000000b": "bls_g1add", + "0x000000000000000000000000000000000000000c": "bls_g1msm", + "0x000000000000000000000000000000000000000d": "bls_g2add", + "0x000000000000000000000000000000000000000e": "bls_g2msm", + "0x000000000000000000000000000000000000000f": "bls_pairing", + "0x0000000000000000000000000000000000000010": "bls_map_fp", + "0x0000000000000000000000000000000000000011": "bls_map_fp2", + "0x0000000000000000000000000000000000000100": "P256VERIFY", +} + + +def rpc_call(rpcs, method, params): + payload = json.dumps({"jsonrpc": "2.0", "id": 1, "method": method, "params": params}).encode() + last_err = None + for attempt in range(8): + rpc = rpcs[attempt % len(rpcs)] + req = urllib.request.Request( + rpc, + data=payload, + headers={"Content-Type": "application/json", "User-Agent": "curl/8"}, + ) + try: + with urllib.request.urlopen(req, timeout=120) as resp: + data = json.loads(resp.read()) + if data.get("result") is not None: + return data["result"] + last_err = data.get("error") + except Exception as e: + last_err = str(e) + time.sleep(2) + raise RuntimeError(f"all RPC attempts failed, last error: {last_err}") + + +def count_calls(node, counts): + to = (node.get("to") or "").lower() + if to in PRECOMPILES: + counts[PRECOMPILES[to]] = counts.get(PRECOMPILES[to], 0) + 1 + for child in node.get("calls") or []: + count_calls(child, counts) + + +def main(): + block = int(sys.argv[1]) + rpcs = ([os.environ["RPC_URL"]] if os.environ.get("RPC_URL") else []) + FALLBACK_RPCS + trace = rpc_call(rpcs, "debug_traceBlockByNumber", [hex(block), {"tracer": "callTracer", "timeout": "120s"}]) + counts = {} + for tx in trace: + count_calls(tx.get("result", {}), counts) + + print(f"### Precompile mix of block {block} (measured)") + print() + if not counts: + print("No precompile calls — pure EVM compute block.") + else: + print("| precompile | calls |") + print("|---|---|") + for name, n in sorted(counts.items(), key=lambda kv: -kv[1]): + print(f"| {name} | {n} |") + print() + print("A win concentrated in one precompile only generalizes to blocks that call it — check the mix before extrapolating single-block deltas to fleet percentiles.") + print() + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/compare-bench.yml b/.github/workflows/compare-bench.yml index 87ccce288..6dd7c29b8 100644 --- a/.github/workflows/compare-bench.yml +++ b/.github/workflows/compare-bench.yml @@ -1,5 +1,5 @@ name: "Compare Benchmarks" -run-name: "Compare Benchmarks (block ${{ inputs.block_number || github.event.inputs.block_number }})" +run-name: "Compare Benchmarks (block ${{ inputs.block_number || inputs.block_preset }})" on: workflow_dispatch: @@ -12,11 +12,21 @@ on: description: "Target OpenVM revision to compare" required: true type: string + block_preset: + type: choice + required: false + description: Benchmark block preset (see .github/benchmark-blocks.json) + options: + - p100 + - p99 + - p50 + - keccak-heavy + - p256-heavy + default: p100 block_number: - type: number + type: string required: false - description: Ethereum block number - default: 24001988 + description: Ethereum block number (overrides block_preset) benchmark_mode: type: choice required: false @@ -47,14 +57,37 @@ on: default: false jobs: + resolve-block: + name: "Resolve Benchmark Block" + runs-on: ubuntu-latest + outputs: + block: ${{ steps.resolve.outputs.block }} + steps: + - uses: actions/checkout@v4 + - name: Resolve block number + id: resolve + run: | + BLOCK="${{ inputs.block_number }}" + if [[ -z "$BLOCK" ]]; then + BLOCK=$(jq -er --arg preset "${{ inputs.block_preset }}" '.[$preset].block' .github/benchmark-blocks.json) + fi + [[ "$BLOCK" =~ ^[0-9]+$ ]] || { echo "invalid block number: $BLOCK"; exit 1; } + echo "block=$BLOCK" >> "$GITHUB_OUTPUT" + - name: Measure precompile mix + continue-on-error: true + env: + RPC_URL: ${{ secrets.RPC_URL_1 }} + run: python3 .github/scripts/precompile_mix.py "${{ steps.resolve.outputs.block }}" >> "$GITHUB_STEP_SUMMARY" + run-base-benchmark: name: "Run Base Benchmark" uses: ./.github/workflows/update-patches.yml + needs: resolve-block with: OPENVM_REV: ${{ github.event.inputs.base_rev }} run_benchmark: true benchmark_mode: ${{ github.event.inputs.benchmark_mode }} - benchmark_block_number: ${{ fromJSON(inputs.block_number || github.event.inputs.block_number || '24001988') }} + benchmark_block_number: ${{ fromJSON(needs.resolve-block.outputs.block) }} instance_family: ${{ github.event.inputs.instance_family }} secrets: inherit @@ -70,10 +103,10 @@ jobs: run-target-benchmark: name: "Run Target Benchmark" uses: ./.github/workflows/reth-benchmark.yml - needs: patch-target + needs: [patch-target, resolve-block] with: mode: ${{ github.event.inputs.benchmark_mode }} - block_number: ${{ fromJSON(inputs.block_number || github.event.inputs.block_number || '24001988') }} + block_number: ${{ fromJSON(needs.resolve-block.outputs.block) }} instance_family: ${{ github.event.inputs.instance_family }} ref: ${{ needs.patch-target.outputs.branch_name }} tag: ${{ needs.patch-target.outputs.tag }} @@ -84,10 +117,10 @@ jobs: name: "Target Host Perf Flamegraph" uses: ./.github/workflows/reth-benchmark.yml if: ${{ github.event.inputs.host_flamegraph == 'true' }} - needs: patch-target + needs: [patch-target, resolve-block] with: mode: ${{ github.event.inputs.benchmark_mode }} - block_number: ${{ fromJSON(inputs.block_number || github.event.inputs.block_number || '24001988') }} + block_number: ${{ fromJSON(needs.resolve-block.outputs.block) }} instance_family: ${{ github.event.inputs.instance_family }} ref: ${{ needs.patch-target.outputs.branch_name }} tag: ${{ needs.patch-target.outputs.tag }} @@ -98,10 +131,10 @@ jobs: name: "Target Guest Circuit Flamegraphs" uses: ./.github/workflows/reth-benchmark.yml if: ${{ github.event.inputs.guest_flamegraph == 'true' }} - needs: patch-target + needs: [patch-target, resolve-block] with: mode: ${{ github.event.inputs.benchmark_mode }} - block_number: ${{ fromJSON(inputs.block_number || github.event.inputs.block_number || '24001988') }} + block_number: ${{ fromJSON(needs.resolve-block.outputs.block) }} instance_family: ${{ github.event.inputs.instance_family }} ref: ${{ needs.patch-target.outputs.branch_name }} tag: ${{ needs.patch-target.outputs.tag }} @@ -110,6 +143,7 @@ jobs: compare-results: needs: + - resolve-block - run-base-benchmark - run-target-benchmark runs-on: ubuntu-latest @@ -152,4 +186,4 @@ jobs: echo "Target OpenVM Revision: ${{ github.event.inputs.target_rev }}" >> $GITHUB_STEP_SUMMARY echo "Instance Family: ${{ github.event.inputs.instance_family }}" >> $GITHUB_STEP_SUMMARY echo "Benchmark Mode: ${{ github.event.inputs.benchmark_mode }}" >> $GITHUB_STEP_SUMMARY - echo "Block Number: ${{ github.event.inputs.block_number }}" >> $GITHUB_STEP_SUMMARY + echo "Block Number: ${{ needs.resolve-block.outputs.block }}" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/compare-branches.yml b/.github/workflows/compare-branches.yml index af57260e2..ce2eaedaa 100644 --- a/.github/workflows/compare-branches.yml +++ b/.github/workflows/compare-branches.yml @@ -1,5 +1,5 @@ name: "Compare Branches" -run-name: "Compare Branches: ${{ inputs.base_ref }} vs ${{ inputs.target_ref }} (block ${{ inputs.block_number }})" +run-name: "Compare Branches: ${{ inputs.base_ref }} vs ${{ inputs.target_ref }} (block ${{ inputs.block_number || inputs.block_preset }})" on: workflow_dispatch: @@ -13,11 +13,21 @@ on: description: "Target branch/ref to compare" required: true type: string + block_preset: + type: choice + required: false + description: Benchmark block preset (see .github/benchmark-blocks.json) + options: + - p100 + - p99 + - p50 + - keccak-heavy + - p256-heavy + default: p100 block_number: - type: number + type: string required: false - description: Ethereum block number - default: 24001988 + description: Ethereum block number (overrides block_preset) benchmark_mode: type: choice required: false @@ -48,13 +58,36 @@ on: default: false jobs: + resolve-block: + name: "Resolve Benchmark Block" + runs-on: ubuntu-latest + outputs: + block: ${{ steps.resolve.outputs.block }} + steps: + - uses: actions/checkout@v4 + - name: Resolve block number + id: resolve + run: | + BLOCK="${{ inputs.block_number }}" + if [[ -z "$BLOCK" ]]; then + BLOCK=$(jq -er --arg preset "${{ inputs.block_preset }}" '.[$preset].block' .github/benchmark-blocks.json) + fi + [[ "$BLOCK" =~ ^[0-9]+$ ]] || { echo "invalid block number: $BLOCK"; exit 1; } + echo "block=$BLOCK" >> "$GITHUB_OUTPUT" + - name: Measure precompile mix + continue-on-error: true + env: + RPC_URL: ${{ secrets.RPC_URL_1 }} + run: python3 .github/scripts/precompile_mix.py "${{ steps.resolve.outputs.block }}" >> "$GITHUB_STEP_SUMMARY" + run-base-benchmark: name: "Run Base Benchmark (${{ inputs.base_ref }})" uses: ./.github/workflows/reth-benchmark.yml + needs: resolve-block with: ref: ${{ inputs.base_ref }} mode: ${{ inputs.benchmark_mode }} - block_number: ${{ fromJSON(inputs.block_number || '24001988') }} + block_number: ${{ fromJSON(needs.resolve-block.outputs.block) }} instance_family: ${{ inputs.instance_family }} profiling: none secrets: inherit @@ -62,10 +95,11 @@ jobs: run-target-benchmark: name: "Run Target Benchmark (${{ inputs.target_ref }})" uses: ./.github/workflows/reth-benchmark.yml + needs: resolve-block with: ref: ${{ inputs.target_ref }} mode: ${{ inputs.benchmark_mode }} - block_number: ${{ fromJSON(inputs.block_number || '24001988') }} + block_number: ${{ fromJSON(needs.resolve-block.outputs.block) }} instance_family: ${{ inputs.instance_family }} profiling: none secrets: inherit @@ -74,10 +108,11 @@ jobs: name: "Target Host Perf Flamegraph" uses: ./.github/workflows/reth-benchmark.yml if: ${{ inputs.host_flamegraph }} + needs: resolve-block with: ref: ${{ inputs.target_ref }} mode: ${{ inputs.benchmark_mode }} - block_number: ${{ fromJSON(inputs.block_number || '24001988') }} + block_number: ${{ fromJSON(needs.resolve-block.outputs.block) }} instance_family: ${{ inputs.instance_family }} profiling: host secrets: inherit @@ -86,16 +121,18 @@ jobs: name: "Target Guest Circuit Flamegraphs" uses: ./.github/workflows/reth-benchmark.yml if: ${{ inputs.guest_flamegraph }} + needs: resolve-block with: ref: ${{ inputs.target_ref }} mode: ${{ inputs.benchmark_mode }} - block_number: ${{ fromJSON(inputs.block_number || '24001988') }} + block_number: ${{ fromJSON(needs.resolve-block.outputs.block) }} instance_family: ${{ inputs.instance_family }} profiling: guest secrets: inherit compare-results: needs: + - resolve-block - run-base-benchmark - run-target-benchmark runs-on: ubuntu-latest @@ -146,4 +183,4 @@ jobs: echo "Target Ref: ${{ inputs.target_ref }}" >> $GITHUB_STEP_SUMMARY echo "Instance Family: ${{ inputs.instance_family }}" >> $GITHUB_STEP_SUMMARY echo "Benchmark Mode: ${{ inputs.benchmark_mode }}" >> $GITHUB_STEP_SUMMARY - echo "Block Number: ${{ inputs.block_number }}" >> $GITHUB_STEP_SUMMARY + echo "Block Number: ${{ needs.resolve-block.outputs.block }}" >> $GITHUB_STEP_SUMMARY