diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index dd3bbafb412..65d6fc7e40d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,11 +1,11 @@ * @gakonst -crates/chain-state/ @fgimenez @mattsse +crates/chain-state/ @mattsse crates/chainspec/ @Rjected @joshieDo @mattsse crates/cli/ @mattsse @Rjected crates/config/ @shekhirin @mattsse @Rjected crates/consensus/ @mattsse @Rjected -crates/e2e-test-utils/ @mattsse @Rjected @klkvr @fgimenez -crates/engine/ @mattsse @Rjected @mediocregopher @yongkangc +crates/e2e-test-utils/ @mattsse @Rjected @klkvr +crates/engine/ @mattsse @Rjected @mediocregopher crates/era/ @mattsse crates/era-downloader/ @mattsse crates/era-utils/ @mattsse @@ -36,15 +36,15 @@ crates/storage/db/ @joshieDo crates/storage/errors/ @joshieDo crates/storage/libmdbx-rs/ @shekhirin crates/storage/nippy-jar/ @joshieDo @shekhirin -crates/storage/provider/ @joshieDo @shekhirin @yongkangc +crates/storage/provider/ @joshieDo @shekhirin crates/storage/storage-api/ @joshieDo crates/tasks/ @mattsse @DaniPopes crates/tokio-util/ @mattsse crates/tracing/ @mattsse @shekhirin crates/tracing-otlp/ @mattsse @Rjected -crates/transaction-pool/ @mattsse @yongkangc -crates/trie/ @Rjected @shekhirin @mediocregopher @yongkangc +crates/transaction-pool/ @mattsse +crates/trie/ @Rjected @shekhirin @mediocregopher bin/reth/ @mattsse @shekhirin @Rjected -bin/reth-bench-compare/ @mediocregopher @shekhirin @yongkangc +bin/reth-bench-compare/ @mediocregopher @shekhirin etc/ @Rjected @shekhirin .github/ @gakonst @DaniPopes diff --git a/.github/scripts/bench-txgen-run.sh b/.github/scripts/bench-txgen-run.sh index 98ea31f04c3..5631ff70204 100755 --- a/.github/scripts/bench-txgen-run.sh +++ b/.github/scripts/bench-txgen-run.sh @@ -259,6 +259,10 @@ RETH_ARGS=( --no-persist-peers ) +if [ -n "${BENCH_REORG:-}" ]; then + RETH_ARGS+=(--testing.skip-invalid-transactions) +fi + SYNC_STATE_IDLE=false if "$BINARY" node --help 2>/dev/null | grep -qF -- '--debug.startup-sync-state-idle'; then RETH_ARGS+=(--debug.startup-sync-state-idle) diff --git a/.github/scripts/check_wasm.sh b/.github/scripts/check_wasm.sh index 5f47c627cba..8c46933325f 100755 --- a/.github/scripts/check_wasm.sh +++ b/.github/scripts/check_wasm.sh @@ -41,7 +41,6 @@ exclude_crates=( reth-node-metrics reth-rpc reth-rpc-api - reth-rpc-api-testing-util reth-rpc-builder reth-rpc-convert reth-rpc-e2e-tests diff --git a/.github/workflows/bench-benchmarkoor.yml b/.github/workflows/bench-benchmarkoor.yml index 17c70c3682b..f0552a30977 100644 --- a/.github/workflows/bench-benchmarkoor.yml +++ b/.github/workflows/bench-benchmarkoor.yml @@ -26,7 +26,7 @@ on: download_ref: description: "Reth git ref used only for snapshot download" required: false - default: "pull/24027/head" + default: "main" type: string suite: description: "benchmarkoor-replay suite" @@ -149,7 +149,53 @@ name: bench-benchmarkoor permissions: {} jobs: + authorize: + name: authorize self-hosted benchmark refs + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + steps: + - name: Reject fork and hidden pull refs + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + INPUT_PR: ${{ inputs.pr || '' }} + INPUT_FEATURE: ${{ inputs.feature || '' }} + INPUT_BASELINE: ${{ inputs.baseline || 'main' }} + INPUT_DOWNLOAD: ${{ inputs.download_ref || 'main' }} + with: + script: | + const trustedRepository = `${context.repo.owner}/${context.repo.repo}`; + const pullNumber = process.env.INPUT_PR; + if (pullNumber) { + const { data: pr } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: Number(pullNumber), + }); + if (pr.head.repo.full_name !== trustedRepository) { + core.setFailed(`Refusing to run fork PR code from ${pr.head.repo.full_name} on a self-hosted runner`); + return; + } + } + + const unsafeRef = (ref) => { + const value = String(ref || ''); + return /^(refs\/)?pull\//.test(value) || /^[0-9a-f]{40}$/i.test(value); + }; + for (const [name, value] of [ + ['feature', process.env.INPUT_FEATURE], + ['baseline', process.env.INPUT_BASELINE], + ['download_ref', process.env.INPUT_DOWNLOAD], + ]) { + if (unsafeRef(value)) { + core.setFailed(`${name} must be a branch or tag in ${trustedRepository}; pull refs and raw SHAs are not allowed on self-hosted runners`); + return; + } + } + benchmarkoor: + needs: authorize name: bench-benchmarkoor runs-on: [self-hosted, Linux, X64, available] permissions: @@ -330,18 +376,13 @@ jobs: INPUT_PR: ${{ inputs.pr || '' }} INPUT_FEATURE: ${{ inputs.feature || '' }} INPUT_BASELINE: ${{ inputs.baseline || 'main' }} - INPUT_DOWNLOAD: ${{ inputs.download_ref || 'pull/24027/head' }} + INPUT_DOWNLOAD: ${{ inputs.download_ref || 'main' }} run: | set -euo pipefail git fetch origin main --quiet resolve_ref() { local ref="$1" - if [[ "$ref" =~ ^pull/[0-9]+/(head|merge)$ ]]; then - git fetch origin "$ref" --quiet - git rev-parse --verify "FETCH_HEAD^{commit}" - return 0 - fi git fetch origin "$ref" --quiet 2>/dev/null || true if git rev-parse --verify --quiet "${ref}^{commit}" >/dev/null; then git rev-parse --verify "${ref}^{commit}" @@ -364,7 +405,7 @@ jobs: BASELINE_NAME="${INPUT_BASELINE:-main}" BASELINE_REF="$(resolve_ref "$BASELINE_NAME")" - DOWNLOAD_NAME="${INPUT_DOWNLOAD:-pull/24027/head}" + DOWNLOAD_NAME="${INPUT_DOWNLOAD:-main}" DOWNLOAD_REF="$(resolve_ref "$DOWNLOAD_NAME")" { diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index ae38f2cd398..9d12d7c74fd 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -178,7 +178,8 @@ jobs: with: github-token: ${{ secrets.DEREK_BENCH_ACK_TOKEN }} script: | - const org = 'paradigmxyz'; + const org = context.repo.owner; + const trustedRepository = `${context.repo.owner}/${context.repo.repo}`; const checkMembership = async (username) => { try { const { status } = await github.rest.orgs.checkMembershipForUser({ org, username }); @@ -199,6 +200,10 @@ jobs: repo: context.repo.repo, pull_number: context.issue.number, }); + if (pr.head.repo.full_name !== trustedRepository) { + core.setFailed(`Refusing to run fork PR code from ${pr.head.repo.full_name} on a self-hosted runner`); + return; + } const prAuthor = pr.user.login; if (!await checkMembership(prAuthor)) { core.setFailed(`PR author @${prAuthor} is not a member of ${org}`); @@ -322,6 +327,17 @@ jobs: var featureNodeArgs = '${{ github.event.inputs.feature_args }}' || ''; var skipStateRoot = 'false'; + const unsafeRef = (ref) => { + const value = String(ref || ''); + return /^(refs\/)?pull\//.test(value) || /^[0-9a-f]{40}$/i.test(value); + }; + for (const [name, value] of [['baseline', baseline], ['feature', feature]]) { + if (unsafeRef(value)) { + core.setFailed(`${name} must be a branch or tag in ${context.repo.owner}/${context.repo.repo}; pull refs and raw SHAs are not allowed on self-hosted runners`); + return; + } + } + // Find PR for the selected branch const branch = '${{ github.ref_name }}'; const { data: prs } = await github.rest.pulls.list({ @@ -512,6 +528,11 @@ jobs: repo: context.repo.repo, pull_number: parseInt(pr), }); + const trustedRepository = `${context.repo.owner}/${context.repo.repo}`; + if (prData.head.repo.full_name !== trustedRepository) { + core.setFailed(`Refusing to run fork PR code from ${prData.head.repo.full_name} on a self-hosted runner`); + return; + } core.setOutput('pr-head-sha', prData.head.sha); core.setOutput('pr-head-ref', prData.head.ref); core.setOutput('pr-head-repo', prData.head.repo.full_name); @@ -728,7 +749,9 @@ jobs: bench-txgen: needs: bench-ack - if: needs.bench-ack.outputs.command == 'bench' + if: >- + needs.bench-ack.outputs.command == 'bench' && + (needs.bench-ack.outputs.pr == '' || needs.bench-ack.outputs.pr-head-repo == github.repository) name: bench-txgen runs-on: [self-hosted, Linux, X64, available] permissions: @@ -862,7 +885,7 @@ jobs: - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 continue-on-error: true - - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + - uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0 - name: Mask OTLP endpoints uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 diff --git a/.github/workflows/cyclops-audit.yml b/.github/workflows/cyclops-audit.yml new file mode 100644 index 00000000000..d4e3eaef337 --- /dev/null +++ b/.github/workflows/cyclops-audit.yml @@ -0,0 +1,353 @@ +name: Cyclops audit + +on: + pull_request_target: # zizmor: ignore[dangerous-triggers] Cyclops needs privileged PR-label triggers and does not check out PR code. + types: [labeled] + issue_comment: # zizmor: ignore[dangerous-triggers] Cyclops gates comment callers before using secrets. + types: [created] + +permissions: {} + +jobs: + publish-label: + if: github.event_name == 'pull_request_target' && (github.event.label.name == 'cyclops' || github.event.label.name == 'agentic-audit') + runs-on: ubuntu-latest + steps: + - name: Publish event + env: + HAS_EVENTS_ARGS: ${{ secrets.EVENTS_ARGS != '' }} + HAS_EVENTS_KEY: ${{ secrets.EVENTS_KEY != '' }} + HAS_EVENTS_CERT: ${{ secrets.EVENTS_CERT != '' }} + run: | + set -euo pipefail + + [[ "$HAS_EVENTS_ARGS" == "true" ]] || { echo "::error::Missing EVENTS_ARGS secret"; exit 1; } + [[ "$HAS_EVENTS_KEY" == "true" ]] || { echo "::error::Missing EVENTS_KEY secret"; exit 1; } + [[ "$HAS_EVENTS_CERT" == "true" ]] || { echo "::error::Missing EVENTS_CERT secret"; exit 1; } + + printf '%s' '${{ secrets.EVENTS_KEY }}' > "${RUNNER_TEMP}/key" + printf '%s' '${{ secrets.EVENTS_CERT }}' > "${RUNNER_TEMP}/cert" + + # EVENTS_ARGS may contain additional curl arguments, not just a URL. + curl -sf -o /dev/null -X POST ${{ secrets.EVENTS_ARGS }} \ + -H "Content-Type: application/json" \ + --key "${RUNNER_TEMP}/key" \ + --cert "${RUNNER_TEMP}/cert" \ + -d '{ + "repository": "${{ github.repository }}", + "event": "pr_audit", + "data": { + "pr_number": ${{ github.event.pull_request.number }}, + "sha": "${{ github.event.pull_request.head.sha }}" + } + }' + + publish-comment: + if: >- + github.event_name == 'issue_comment' && + github.event.issue.pull_request && + ( + startsWith(github.event.comment.body, 'cyclops audit') || + startsWith(github.event.comment.body, '@decofe cyclops audit') || + startsWith(github.event.comment.body, 'derek audit') + ) + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + steps: + - name: Check commenter permission + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + github-token: ${{ github.token }} + script: | + const allowed = new Set(['OWNER', 'MEMBER', 'COLLABORATOR']); + const commenterAssociation = context.payload.comment.author_association; + if (!allowed.has(commenterAssociation)) { + core.setFailed(`@${context.payload.comment.user.login} is not allowed to trigger Cyclops audits (${commenterAssociation})`); + return; + } + + const trustedPermissions = new Set(['admin', 'maintain', 'write']); + const { data: pr } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number, + }); + if (!allowed.has(pr.author_association)) { + const { data: authorPermission } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: context.repo.owner, + repo: context.repo.repo, + username: pr.user.login, + }); + if (!trustedPermissions.has(authorPermission.permission)) { + core.setFailed(`PR author @${pr.user.login} is not allowed to trigger Cyclops audits (${pr.author_association}, ${authorPermission.permission})`); + return; + } + } + + - name: Parse arguments + id: args + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + github-token: ${{ github.token }} + script: | + const usage = [ + '**Usage:** `cyclops audit [fast] [perf] [iterations=N] [hours=N] [config=pr-review.yaml] ', + '[models="anthropic/claude-opus-4-7,openai/gpt-5.5"] [run-label=LABEL] ', + '[dry-run] [note="per-run audit guidance"]`', + ].join(''); + const body = context.payload.comment.body.trim(); + const prefix = /^(?:@decofe\s+)?(?:cyclops\s+audit|derek\s+audit)\b/i; + const args = body.replace(prefix, '').trim(); + const parts = []; + const argRegex = /(\S+?[=:]"[^"]*"|\S+?[=:]'[^']*'|\S+?[=:]\S+|\S+)/g; + let match; + while ((match = argRegex.exec(args)) !== null) parts.push(match[1]); + + const defaults = { + config: '', + iterations: '', + hours: '', + models: '', + 'run-label': '', + 'dry-run': 'false', + perf: 'false', + note: '', + }; + const intArgs = new Set(['iterations', 'hours']); + const stringArgs = new Set(['config', 'models', 'run-label', 'note']); + const boolArgs = new Set(['dry-run', 'perf']); + const unknown = []; + const invalid = []; + + for (const part of parts) { + if (part === 'fast') { + defaults.iterations = '1'; + continue; + } + + const eq = part.indexOf('='); + const colon = part.indexOf(':'); + const sep = eq === -1 ? colon : colon === -1 ? eq : Math.min(eq, colon); + if (sep === -1) { + if (boolArgs.has(part)) { + defaults[part] = 'true'; + } else { + unknown.push(part); + } + continue; + } + + const key = part.slice(0, sep); + let value = part.slice(sep + 1); + if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) { + value = value.slice(1, -1); + } + + if (intArgs.has(key)) { + if (!/^[1-9]\d*$/.test(value)) { + invalid.push(`\`${key}=${value}\` (must be a positive integer)`); + } else { + defaults[key] = value; + } + } else if (boolArgs.has(key)) { + if (value === 'true' || value === 'false') { + defaults[key] = value; + } else { + invalid.push(`\`${key}=${value}\` (must be true or false)`); + } + } else if (stringArgs.has(key)) { + if (!value) { + invalid.push(`\`${key}=\` (must not be empty)`); + } else { + defaults[key] = value; + } + } else { + unknown.push(key); + } + } + + const errors = []; + if (unknown.length) errors.push(`Unknown argument(s): \`${unknown.join('`, `')}\``); + if (invalid.length) errors.push(`Invalid value(s): ${invalid.join(', ')}`); + if (errors.length) { + const msg = `Invalid cyclops audit command\n\n${errors.join('\n')}\n\n${usage}`; + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: msg, + }); + core.setFailed(msg); + return; + } + + const { data: pr } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number, + }); + + const data = { + pr_number: context.issue.number, + sha: pr.head.sha, + source: 'comment', + actor: context.payload.comment.user.login, + comment_id: context.payload.comment.id, + dry_run: defaults['dry-run'] === 'true', + }; + if (defaults.config) data.config = defaults.config; + if (defaults.iterations) data.max_iterations = Number(defaults.iterations); + if (defaults.hours) data.max_hours = Number(defaults.hours); + if (defaults.models) data.models = defaults.models; + if (defaults['run-label']) data.run_label = defaults['run-label']; + if (defaults.note) data.audit_note_b64 = Buffer.from(defaults.note, 'utf8').toString('base64'); + if (defaults.perf === 'true') data.perf = true; + + const payload = { + repository: `${context.repo.owner}/${context.repo.repo}`, + event: 'pr_audit', + data, + }; + + const summaryParts = [ + defaults.config ? `config: \`${defaults.config}\`` : 'config: `default`', + defaults.iterations ? `iterations: \`${defaults.iterations}\`` : 'iterations: `default`', + defaults.hours ? `hours: \`${defaults.hours}\`` : 'hours: `default`', + ]; + if (defaults.models) summaryParts.push(`models: \`${defaults.models}\``); + if (defaults['run-label']) summaryParts.push(`run-label: \`${defaults['run-label']}\``); + if (defaults['dry-run'] === 'true') summaryParts.push('dry-run: `true`'); + if (defaults.perf === 'true') summaryParts.push('perf: `true`'); + if (defaults.note) { + const note = defaults.note.replace(/`/g, "'").slice(0, 160); + summaryParts.push(`note: \`${note}${defaults.note.length > 160 ? '...' : ''}\``); + } + + core.setOutput('actor', context.payload.comment.user.login); + core.setOutput('payload-b64', Buffer.from(JSON.stringify(payload), 'utf8').toString('base64')); + core.setOutput('summary', `**Config:** ${summaryParts.join(', ')}`); + + - name: Check publisher configuration + id: publisher-config + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + ACTOR: ${{ steps.args.outputs.actor }} + HAS_EVENTS_ARGS: ${{ secrets.EVENTS_ARGS != '' }} + HAS_EVENTS_KEY: ${{ secrets.EVENTS_KEY != '' }} + HAS_EVENTS_CERT: ${{ secrets.EVENTS_CERT != '' }} + with: + github-token: ${{ github.token }} + script: | + const required = { + EVENTS_ARGS: process.env.HAS_EVENTS_ARGS, + EVENTS_KEY: process.env.HAS_EVENTS_KEY, + EVENTS_CERT: process.env.HAS_EVENTS_CERT, + }; + const missing = Object.entries(required).filter(([, present]) => present !== 'true').map(([name]) => name); + if (!missing.length) return; + + const msg = `Cyclops audit is not configured: missing ${missing.map((name) => `\`${name}\``).join(', ')} secret${missing.length === 1 ? '' : 's'}.`; + try { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: `cc @${process.env.ACTOR}\n\n${msg}`, + }); + } catch (error) { + core.warning(`Could not create configuration failure comment: ${error.message}`); + } + core.setFailed(msg); + + - name: Acknowledge request + id: ack + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + ACTOR: ${{ steps.args.outputs.actor }} + SUMMARY: ${{ steps.args.outputs.summary }} + with: + github-token: ${{ github.token }} + script: | + try { + await github.rest.reactions.createForIssueComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: context.payload.comment.id, + content: 'eyes', + }); + } catch (error) { + core.warning(`Could not add acknowledgement reaction: ${error.message}`); + } + + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + try { + const { data: comment } = await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: `cc @${process.env.ACTOR}\n\nCyclops audit event queued. [View workflow run](${runUrl})\n\n${process.env.SUMMARY}`, + }); + core.setOutput('comment-id', String(comment.id)); + } catch (error) { + core.warning(`Could not create acknowledgement comment: ${error.message}`); + core.setOutput('comment-id', ''); + } + + - name: Publish event + id: publish + continue-on-error: true + env: + PAYLOAD_B64: ${{ steps.args.outputs.payload-b64 }} + HAS_EVENTS_ARGS: ${{ secrets.EVENTS_ARGS != '' }} + HAS_EVENTS_KEY: ${{ secrets.EVENTS_KEY != '' }} + HAS_EVENTS_CERT: ${{ secrets.EVENTS_CERT != '' }} + run: | + set -euo pipefail + + [[ "$HAS_EVENTS_ARGS" == "true" ]] || { echo "::error::Missing EVENTS_ARGS secret"; exit 1; } + [[ "$HAS_EVENTS_KEY" == "true" ]] || { echo "::error::Missing EVENTS_KEY secret"; exit 1; } + [[ "$HAS_EVENTS_CERT" == "true" ]] || { echo "::error::Missing EVENTS_CERT secret"; exit 1; } + + printf '%s' '${{ secrets.EVENTS_KEY }}' > "${RUNNER_TEMP}/key" + printf '%s' '${{ secrets.EVENTS_CERT }}' > "${RUNNER_TEMP}/cert" + printf '%s' "$PAYLOAD_B64" | base64 --decode > "${RUNNER_TEMP}/pr-audit-event.json" + + # EVENTS_ARGS may contain additional curl arguments, not just a URL. + curl -sf -o /dev/null -X POST ${{ secrets.EVENTS_ARGS }} \ + -H "Content-Type: application/json" \ + --key "${RUNNER_TEMP}/key" \ + --cert "${RUNNER_TEMP}/cert" \ + -d @"${RUNNER_TEMP}/pr-audit-event.json" + + - name: Update status + if: ${{ always() && steps.args.outcome == 'success' && steps.publisher-config.outcome == 'success' }} + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + COMMENT_ID: ${{ steps.ack.outputs['comment-id'] }} + PUBLISH_OUTCOME: ${{ steps.publish.outcome }} + ACTOR: ${{ steps.args.outputs.actor }} + SUMMARY: ${{ steps.args.outputs.summary }} + with: + github-token: ${{ github.token }} + script: | + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const success = process.env.PUBLISH_OUTCOME === 'success'; + const body = success + ? `cc @${process.env.ACTOR}\n\nCyclops audit event published. [View workflow run](${runUrl})\n\n${process.env.SUMMARY}` + : `cc @${process.env.ACTOR}\n\nCyclops audit event failed to publish. [View workflow run](${runUrl})\n\n${process.env.SUMMARY}`; + + if (process.env.COMMENT_ID) { + try { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: Number(process.env.COMMENT_ID), + body, + }); + } catch (error) { + core.warning(`Could not update acknowledgement comment: ${error.message}`); + } + } + if (!success) core.setFailed('Failed to publish pr_audit event'); diff --git a/.github/workflows/docker-test.yml b/.github/workflows/docker-test.yml index e632deb83b4..5fe20ad6bdf 100644 --- a/.github/workflows/docker-test.yml +++ b/.github/workflows/docker-test.yml @@ -68,11 +68,11 @@ jobs: # Docker build (forks) - name: Set up Docker Buildx if: steps.fork.outputs.is_fork == 'true' - uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 - name: Build reth image (Docker) if: steps.fork.outputs.is_fork == 'true' - uses: docker/bake-action@6614cfa25eff9a0b2b2697efb0b6159e7680d584 # v7.2.0 + uses: docker/bake-action@d3418bd7d0e9324001bca92fa8ba175ea7e6dc9b # v7.3.0 env: VERGEN_GIT_SHA: ${{ steps.git.outputs.sha }} VERGEN_GIT_DESCRIBE: ${{ steps.git.outputs.describe }} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 06d501f6415..3c6ab4c5c82 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -2,7 +2,6 @@ # # Triggers: # - Push tag v*: builds release (RC or latest) -# - Schedule: builds nightly + profiling # - Manual: builds git-sha or nightly name: docker @@ -11,8 +10,6 @@ on: push: tags: - v* - schedule: - - cron: "0 1 * * *" workflow_dispatch: inputs: build_type: diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 638cd653012..b36e330ed35 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -36,7 +36,7 @@ jobs: - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - - uses: taiki-e/install-action@9bcaee1dcae34154180f412e2fa69355a7cda9f6 # v2.82.6 + - uses: taiki-e/install-action@4684b8405694ae9dd42c9f39ba901a70ae83f4a3 # v2.82.9 with: tool: nextest - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 @@ -70,7 +70,7 @@ jobs: - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - - uses: taiki-e/install-action@9bcaee1dcae34154180f412e2fa69355a7cda9f6 # v2.82.6 + - uses: taiki-e/install-action@4684b8405694ae9dd42c9f39ba901a70ae83f4a3 # v2.82.9 with: tool: nextest - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index c4d85b72bf8..1ef91732771 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -44,7 +44,7 @@ jobs: - uses: dtolnay/rust-toolchain@stable - name: Install Geth run: .github/scripts/install_geth.sh - - uses: taiki-e/install-action@9bcaee1dcae34154180f412e2fa69355a7cda9f6 # v2.82.6 + - uses: taiki-e/install-action@4684b8405694ae9dd42c9f39ba901a70ae83f4a3 # v2.82.9 with: tool: nextest - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 @@ -85,7 +85,7 @@ jobs: - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - - uses: taiki-e/install-action@9bcaee1dcae34154180f412e2fa69355a7cda9f6 # v2.82.6 + - uses: taiki-e/install-action@4684b8405694ae9dd42c9f39ba901a70ae83f4a3 # v2.82.9 with: tool: nextest - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index a5b56b90c71..dd1fd4a7226 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -83,7 +83,7 @@ jobs: - uses: dtolnay/rust-toolchain@stable with: target: wasm32-wasip1 - - uses: taiki-e/install-action@9bcaee1dcae34154180f412e2fa69355a7cda9f6 # v2.82.6 + - uses: taiki-e/install-action@4684b8405694ae9dd42c9f39ba901a70ae83f4a3 # v2.82.9 with: tool: cargo-hack - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 @@ -109,7 +109,7 @@ jobs: - uses: dtolnay/rust-toolchain@stable with: target: riscv32imac-unknown-none-elf - - uses: taiki-e/install-action@9bcaee1dcae34154180f412e2fa69355a7cda9f6 # v2.82.6 + - uses: taiki-e/install-action@4684b8405694ae9dd42c9f39ba901a70ae83f4a3 # v2.82.9 with: tool: cargo-hack - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 @@ -137,7 +137,7 @@ jobs: - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - - uses: taiki-e/install-action@9bcaee1dcae34154180f412e2fa69355a7cda9f6 # v2.82.6 + - uses: taiki-e/install-action@4684b8405694ae9dd42c9f39ba901a70ae83f4a3 # v2.82.9 with: tool: cargo-hack - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 @@ -227,7 +227,7 @@ jobs: - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: cache-on-failure: true - - uses: taiki-e/install-action@9bcaee1dcae34154180f412e2fa69355a7cda9f6 # v2.82.6 + - uses: taiki-e/install-action@4684b8405694ae9dd42c9f39ba901a70ae83f4a3 # v2.82.9 with: tool: cargo-udeps - run: cargo udeps --workspace --lib --examples --tests --benches --all-features --locked diff --git a/.github/workflows/phylax-docker.yml b/.github/workflows/phylax-docker.yml index 97d6d719f46..69d02b2e548 100644 --- a/.github/workflows/phylax-docker.yml +++ b/.github/workflows/phylax-docker.yml @@ -54,3 +54,5 @@ jobs: platforms: linux/amd64 push: ${{ !(github.event_name == 'workflow_dispatch' && inputs.dry_run) }} tags: ghcr.io/${{ github.repository }}/reth:sha-${{ steps.git.outputs.short_sha }} + cache-from: type=registry,ref=ghcr.io/${{ github.repository }}/reth:buildcache + cache-to: type=registry,ref=ghcr.io/${{ github.repository }}/reth:buildcache,mode=max diff --git a/.github/workflows/pr-audit.yml b/.github/workflows/pr-audit.yml deleted file mode 100644 index 8bc332f0a40..00000000000 --- a/.github/workflows/pr-audit.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: Pull request audit - -on: - pull_request: - types: [labeled] - -permissions: {} - -jobs: - publish: - runs-on: ubuntu-latest - if: github.event.label.name == 'cyclops' - permissions: {} - steps: - - name: Publish event - env: - EVENTS_KEY: ${{ secrets.EVENTS_KEY }} - EVENTS_CERT: ${{ secrets.EVENTS_CERT }} - EVENTS_ARGS: ${{ secrets.EVENTS_ARGS }} - PR_NUMBER: ${{ github.event.pull_request.number }} - PR_SHA: ${{ github.event.pull_request.head.sha }} - run: | - set -euo pipefail - - echo "$EVENTS_KEY" > "${{ runner.temp }}/key" - echo "$EVENTS_CERT" > "${{ runner.temp }}/cert" - - curl -sf -o /dev/null -X POST $EVENTS_ARGS \ - -H "Content-Type: application/json" \ - --key "${{ runner.temp }}/key" \ - --cert "${{ runner.temp }}/cert" \ - -d '{ - "repository": "${{ github.repository }}", - "event": "pr_audit", - "data": { - "pr_number": '"$PR_NUMBER"', - "sha": "'"$PR_SHA"'" - } - }' diff --git a/.github/workflows/pr-title.yml b/.github/workflows/pr-title.yml index 649fce5ec86..aa526e70210 100644 --- a/.github/workflows/pr-title.yml +++ b/.github/workflows/pr-title.yml @@ -39,7 +39,7 @@ jobs: continue-on-error: true - name: Add PR Comment for Invalid Title if: steps.lint_pr_title.outcome == 'failure' - uses: marocchino/sticky-pull-request-comment@0ea0beb66eb9baf113663a64ec522f60e49231c0 # v3.0.4 + uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5 with: header: pr-title-lint-error message: | @@ -75,7 +75,7 @@ jobs: - name: Remove Comment for Valid Title if: steps.lint_pr_title.outcome == 'success' - uses: marocchino/sticky-pull-request-comment@0ea0beb66eb9baf113663a64ec522f60e49231c0 # v3.0.4 + uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5 with: header: pr-title-lint-error delete: true diff --git a/.github/workflows/release-dist.yml b/.github/workflows/release-dist.yml index 54bc4381364..e9594d9fff5 100644 --- a/.github/workflows/release-dist.yml +++ b/.github/workflows/release-dist.yml @@ -15,7 +15,7 @@ jobs: permissions: {} steps: - name: Update Homebrew formula - uses: dawidd6/action-homebrew-bump-formula@1446dca236b0440c6f02723a3f14f13be2c04ab0 # v7 + uses: dawidd6/action-homebrew-bump-formula@a0e064e08103c01870c6d1b05168d1b726aab119 # v8 with: token: ${{ secrets.HOMEBREW }} no_fork: true diff --git a/.github/workflows/release-reproducible.yml b/.github/workflows/release-reproducible.yml index 0bc0901c0fc..9d5e78e6762 100644 --- a/.github/workflows/release-reproducible.yml +++ b/.github/workflows/release-reproducible.yml @@ -55,7 +55,7 @@ jobs: ref: ${{ needs.extract-version.outputs.VERSION }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 - name: Log in to GitHub Container Registry uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 @@ -71,7 +71,7 @@ jobs: echo "RUST_TOOLCHAIN=$RUST_TOOLCHAIN" >> $GITHUB_OUTPUT - name: Build reproducible artifacts - uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0 + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 id: docker_build with: context: . @@ -85,7 +85,7 @@ jobs: DOCKER_BUILD_RECORD_UPLOAD: false - name: Build and push final image - uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0 + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 with: context: . file: ./Dockerfile.reproducible diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3f50f24cfd2..a6ceda45ffd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,6 +11,10 @@ on: - v* workflow_dispatch: inputs: + ref: + description: "Git ref to release (for example, v2.4.0)" + type: string + required: false dry_run: description: "Enable dry run mode (builds artifacts but skips uploads and release creation)" type: boolean @@ -22,6 +26,7 @@ env: REPRODUCIBLE_IMAGE_NAME: ${{ github.repository_owner }}/reth-reproducible CARGO_TERM_COLOR: always DOCKER_IMAGE_NAME_URL: https://ghcr.io/${{ github.repository_owner }}/reth + RELEASE_REF: ${{ inputs.ref || github.ref }} jobs: dry-run: @@ -42,7 +47,9 @@ jobs: permissions: {} steps: - name: Extract version - run: echo "VERSION=${GITHUB_REF_NAME//\//-}" >> $GITHUB_OUTPUT + env: + REF: ${{ env.RELEASE_REF }} + run: echo "VERSION=${REF#refs/tags/}" >> $GITHUB_OUTPUT id: extract_version outputs: VERSION: ${{ steps.extract_version.outputs.VERSION }} @@ -58,6 +65,7 @@ jobs: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false + ref: ${{ env.RELEASE_REF }} - uses: dtolnay/rust-toolchain@stable - name: Verify crate version matches tag # Check that the Cargo version starts with the tag, @@ -91,11 +99,6 @@ jobs: allow_fail: false rustflags: "" native: true - - target: x86_64-apple-darwin - os: macos-14 - profile: maxperf - allow_fail: false - rustflags: "-C target-cpu=x86-64-v3 -C target-feature=+pclmulqdq" - target: aarch64-apple-darwin os: macos-14 profile: maxperf @@ -108,6 +111,7 @@ jobs: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false + ref: ${{ env.RELEASE_REF }} - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - uses: dtolnay/rust-toolchain@stable with: @@ -193,6 +197,7 @@ jobs: with: persist-credentials: false fetch-depth: 0 + ref: ${{ env.RELEASE_REF }} - name: Download artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - name: Generate full changelog @@ -209,7 +214,7 @@ jobs: # https://github.com/openethereum/openethereum/blob/6c2d392d867b058ff867c4373e40850ca3f96969/.github/workflows/build.yml run: | prerelease_flag="" - if [[ "${GITHUB_REF}" == *-rc* ]]; then + if [[ "${{ env.VERSION }}" == *-rc* ]]; then prerelease_flag="--prerelease" fi @@ -265,7 +270,6 @@ jobs: |:---:|:---:|:---:|:---| | | x86_64 | [reth-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz.asc) | | | aarch64 | [reth-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz.asc) | - | | x86_64 | [reth-${{ env.VERSION }}-x86_64-apple-darwin.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-x86_64-apple-darwin.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-x86_64-apple-darwin.tar.gz.asc) | | | aarch64 | [reth-${{ env.VERSION }}-aarch64-apple-darwin.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-aarch64-apple-darwin.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-aarch64-apple-darwin.tar.gz.asc) | | | Docker | [${{ env.IMAGE_NAME }}](${{ env.DOCKER_IMAGE_NAME_URL }}) | - | ENDBODY diff --git a/.github/workflows/reproducible-build.yml b/.github/workflows/reproducible-build.yml index 37ace9e0cce..70771145da6 100644 --- a/.github/workflows/reproducible-build.yml +++ b/.github/workflows/reproducible-build.yml @@ -31,7 +31,7 @@ jobs: target: x86_64-unknown-linux-gnu - name: Set up Docker Buildx - uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 - name: Build reproducible binary with Docker run: | diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml index dbdc4c5c2d7..7e1350a9553 100644 --- a/.github/workflows/unit.yml +++ b/.github/workflows/unit.yml @@ -47,7 +47,7 @@ jobs: with: cache-targets: false cache-on-failure: true - - uses: taiki-e/install-action@9bcaee1dcae34154180f412e2fa69355a7cda9f6 # v2.82.6 + - uses: taiki-e/install-action@4684b8405694ae9dd42c9f39ba901a70ae83f4a3 # v2.82.9 with: tool: nextest - if: "${{ matrix.type == 'book' }}" @@ -97,7 +97,7 @@ jobs: - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - - uses: taiki-e/install-action@9bcaee1dcae34154180f412e2fa69355a7cda9f6 # v2.82.6 + - uses: taiki-e/install-action@4684b8405694ae9dd42c9f39ba901a70ae83f4a3 # v2.82.9 with: tool: nextest - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 diff --git a/Cargo.lock b/Cargo.lock index 88aa9bde0fc..098f32710b9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -187,9 +187,9 @@ dependencies = [ [[package]] name = "alloy-dyn-abi" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a475bb02d9cef2dbb99065c1664ab3fe1f9352e21d6d5ed3f02cdbfc06ed1abc" +checksum = "9a04eb4abc2b5074a18e687ee63918f407cc7990083cba9b999445f839796060" dependencies = [ "alloy-json-abi", "alloy-primitives", @@ -291,9 +291,9 @@ dependencies = [ [[package]] name = "alloy-evm" -version = "0.37.0" +version = "0.37.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70634e39510c13a4f69f51a474d98030ecb1dcff54b909d2a9a020ee05e44867" +checksum = "acde665074b478c97047dc2a461b4916cac77922d690e5b7415d1941ae7ff7bf" dependencies = [ "alloy-consensus", "alloy-eips", @@ -353,9 +353,9 @@ dependencies = [ [[package]] name = "alloy-json-abi" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c36c9d7f9021601b04bfef14a4b64849f6d73116a4e91e071d7fbfe10247901" +checksum = "6cee30dd4c2f4b23f434fdf675e7bf9681b86768141277266c6f548ef25cba0a" dependencies = [ "alloy-primitives", "alloy-sol-type-parser", @@ -441,9 +441,9 @@ dependencies = [ [[package]] name = "alloy-primitives" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4885c1409b6936c4898e646ef58baf6ec54edaf6d8179f79df805a7b85b7cf3e" +checksum = "f007e257069855bdf21d27762fd3f3705a613f805c9a08309bf353503f081d71" dependencies = [ "alloy-rlp", "arbitrary", @@ -556,7 +556,7 @@ checksum = "9d4311c03125e8a18296504560b9de3d75ecbd0dcda7f71e6cf2a196d57e6fba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -802,41 +802,41 @@ dependencies = [ [[package]] name = "alloy-sol-macro" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "840128ed2b2971d6d4668a553fe403a82683d3acc646c73e75887e7157408033" +checksum = "b5655c38d5f84955bf727b2eeb62fddd91ebb98fd1d7ae6eb77f73ea88f9b9cf" dependencies = [ "alloy-sol-macro-expander", "alloy-sol-macro-input", - "proc-macro-error2", + "proc-macro-error3", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] name = "alloy-sol-macro-expander" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63ec265e5d65d725175f6ca7711c970824c90ef9c0d1f1973711d4150ee612dd" +checksum = "6277c780e07b76951e09a59788dde230d1582612324177d11a43a61e21a6bb83" dependencies = [ "alloy-sol-macro-input", "const-hex", "heck", "indexmap 2.14.0", - "proc-macro-error2", + "proc-macro-error3", "proc-macro2", "quote", "sha3 0.11.0", - "syn 2.0.117", + "syn 2.0.118", "syn-solidity", ] [[package]] name = "alloy-sol-macro-input" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89bf01077f18650876cfa682eb1f949967b5cde03f1a51c955c469d2c9b4aa67" +checksum = "9762b2ad3e5a0c09886de54fe549ab0056681df843cb082e2df7e1c0eb270d30" dependencies = [ "const-hex", "dunce", @@ -844,15 +844,15 @@ dependencies = [ "macro-string", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", "syn-solidity", ] [[package]] name = "alloy-sol-type-parser" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "857b470ecdd2ed38beaf82ad1a38c516a8ff75266750f38b9eeed001d575241b" +checksum = "da4c7130f0f01f4719678bda3db3bc7267fc2f7f9d0565e3bd964cd2bb45050d" dependencies = [ "serde", "winnow 1.0.1", @@ -860,9 +860,9 @@ dependencies = [ [[package]] name = "alloy-sol-types" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "384cf252de0db2dec52821eac037a7f57e2aa33fe5b900ce6fe39973402341f1" +checksum = "d96e74d6213180f78dbdccddce8af02a639c160c94b0a543fa35c77c58b8a7fc" dependencies = [ "alloy-json-abi", "alloy-primitives", @@ -977,7 +977,7 @@ dependencies = [ "darling 0.23.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -1031,7 +1031,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -1042,7 +1042,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -1071,7 +1071,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -1219,7 +1219,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62945a2f7e6de02a31fe400aa489f0e0f5b2502e69f95f853adb82a96c7a6b60" dependencies = [ "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -1257,7 +1257,7 @@ dependencies = [ "num-traits", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -1346,7 +1346,7 @@ checksum = "213888f660fddcca0d257e88e54ac05bca01885f258ccdf695bafd77031bb69d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -1387,9 +1387,9 @@ checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" [[package]] name = "arrayvec" -version = "0.7.6" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" dependencies = [ "serde", ] @@ -1437,7 +1437,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -1448,7 +1448,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -1495,7 +1495,7 @@ checksum = "ffdcb70bdbc4d478427380519163274ac86e52916e10f0a8889adf0f96d3fee7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -1549,7 +1549,7 @@ dependencies = [ "miniz_oxide", "object 0.37.3", "rustc-demangle", - "windows-link 0.2.1", + "windows-link", ] [[package]] @@ -1628,7 +1628,7 @@ dependencies = [ "regex", "rustc-hash", "shlex 1.3.0", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -1729,15 +1729,6 @@ dependencies = [ "hybrid-array", ] -[[package]] -name = "block-padding" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" -dependencies = [ - "generic-array", -] - [[package]] name = "blst" version = "0.3.16" @@ -1855,7 +1846,7 @@ dependencies = [ "cow-utils", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", "synstructure", ] @@ -1907,13 +1898,13 @@ version = "3.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dee98b0db6a962de883bf5d20362dee4d7ca0d12fe39a7c6c73c844e1cd7c1f" dependencies = [ - "darling 0.23.0", + "darling 0.21.3", "ident_case", "prettyplease", "proc-macro2", "quote", "rustversion", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -1937,23 +1928,23 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] name = "boyer-moore-magiclen" -version = "0.2.22" +version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7441b4796eb8a7107d4cd99d829810be75f5573e1081c37faa0e8094169ea0d6" +checksum = "43f0fdabfbc8017645223fd529c6077df2c491277e44e88ed8afd5afe54e715a" dependencies = [ "debug-helper", ] [[package]] name = "brotli" -version = "8.0.3" +version = "8.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8119e4516436f5708bbc474a9d395bf12f1b5395e93a92a56e647ac3388c8610" +checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -2026,7 +2017,7 @@ checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2037,9 +2028,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.11.1" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" dependencies = [ "serde", ] @@ -2056,9 +2047,9 @@ dependencies = [ [[package]] name = "c-kzg" -version = "2.1.7" +version = "2.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6648ed1e4ea8e8a1a4a2c78e1cda29a3fd500bc622899c340d8525ea9a76b24a" +checksum = "38d04308254695569fdb9bfe3bacc1c91837a670d0806605eb82d63748fbd3a6" dependencies = [ "arbitrary", "blst", @@ -2119,9 +2110,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.64" +version = "1.2.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dad887fd958be91b5098c0248def011f4523ab786cd411be668777e55063501f" +checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" dependencies = [ "find-msvc-tools", "jobserver", @@ -2178,7 +2169,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-link 0.2.1", + "windows-link", ] [[package]] @@ -2260,7 +2251,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2790,7 +2781,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2837,7 +2828,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2851,7 +2842,7 @@ dependencies = [ "quote", "serde", "strsim", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2862,7 +2853,7 @@ checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" dependencies = [ "darling_core 0.21.3", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2873,7 +2864,7 @@ checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core 0.23.0", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2915,7 +2906,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ab67060fc6b8ef687992d439ca0fa36e7ed17e9a0b16b25b601e8757df720de" dependencies = [ "data-encoding", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2973,7 +2964,7 @@ checksum = "d08b3a0bcc0d079199cd476b2cae8435016ec11d1c0986c6901c5ac223041534" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2984,7 +2975,7 @@ checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -3006,7 +2997,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version 0.4.1", - "syn 2.0.117", + "syn 2.0.118", "unicode-xid", ] @@ -3107,7 +3098,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -3154,7 +3145,7 @@ checksum = "1ec431cd708430d5029356535259c5d645d60edd3d39c54e5eea9782d46caa7d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -3206,12 +3197,12 @@ dependencies = [ "enum-ordinalize", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] name = "ef-test-runner" -version = "2.3.0" +version = "2.4.1" dependencies = [ "clap", "ef-tests", @@ -3219,7 +3210,7 @@ dependencies = [ [[package]] name = "ef-tests" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -3333,7 +3324,7 @@ checksum = "8ca9601fb2d62598ee17836250842873a413586e5d7ed88b356e38ddbb0ec631" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -3353,7 +3344,7 @@ checksum = "44f23cf4b44bfce11a86ace86f8a73ffdec849c9fd00a386a53d278bd9e81fb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -3369,7 +3360,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3420,7 +3411,7 @@ dependencies = [ "darling 0.23.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -3672,7 +3663,7 @@ dependencies = [ [[package]] name = "example-full-contract-state" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-primitives", "eyre", @@ -3805,7 +3796,7 @@ dependencies = [ [[package]] name = "exex-subscription" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-primitives", "clap", @@ -3953,7 +3944,7 @@ checksum = "6dc7a9cb3326bafb80642c5ce99b39a2c0702d4bfa8ee8a3e773791a6cbe2407" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -4112,7 +4103,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -4171,8 +4162,8 @@ dependencies = [ "libc", "log", "rustversion", - "windows-link 0.2.1", - "windows-result 0.4.1", + "windows-link", + "windows-result", ] [[package]] @@ -4640,9 +4631,9 @@ checksum = "91f255a4535024abf7640cb288260811fc14794f62b063652ed349f9a6c2348e" [[package]] name = "humantime" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" +checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" [[package]] name = "humantime-serde" @@ -4749,7 +4740,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.62.2", + "windows-core", ] [[package]] @@ -4936,7 +4927,7 @@ checksum = "a0eb5a3343abf848c0984fe4604b2b105da9539376e24fc0a3b0007411ae4fd9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -5018,7 +5009,7 @@ checksum = "6cfe97ee860815a90ed17e09639513269e39420a7440f3f4c996f238c514cf8d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -5047,15 +5038,14 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" dependencies = [ - "block-padding", "generic-array", ] [[package]] name = "insta" -version = "1.47.2" +version = "1.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4a6248eb93a4401ed2f37dfe8ea592d3cf05b7cf4f8efa867b6895af7e094e" +checksum = "86f0f8fee8c926415c58d6ae43a08523a26faccb2323f5e6b644fe7dd4ef6b82" dependencies = [ "console 0.16.3", "once_cell", @@ -5073,7 +5063,7 @@ dependencies = [ "indoc", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -5109,7 +5099,7 @@ dependencies = [ "socket2", "widestring", "windows-registry", - "windows-result 0.4.1", + "windows-result", "windows-sys 0.61.2", ] @@ -5130,7 +5120,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -5219,7 +5209,7 @@ dependencies = [ "simd_cesu8", "thiserror 2.0.18", "walkdir", - "windows-link 0.2.1", + "windows-link", ] [[package]] @@ -5232,7 +5222,7 @@ dependencies = [ "quote", "rustc_version 0.4.1", "simd_cesu8", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -5260,7 +5250,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" dependencies = [ "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -5389,7 +5379,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -5612,7 +5602,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" dependencies = [ "cfg-if", - "windows-link 0.2.1", + "windows-link", ] [[package]] @@ -5622,7 +5612,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60" dependencies = [ "cfg-if", - "windows-link 0.2.1", + "windows-link", ] [[package]] @@ -5859,7 +5849,7 @@ checksum = "59a9dbbfc75d2688ed057456ce8a3ee3f48d12eec09229f560f3643b9f275653" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -5883,7 +5873,7 @@ checksum = "757aee279b8bdbb9f9e676796fd459e4207a1f986e87886700abf589f5abf771" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -5937,7 +5927,7 @@ checksum = "161ab904c2c62e7bda0f7562bf22f96440ca35ff79e66c800cbac298f2f4f5ec" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -5968,7 +5958,7 @@ dependencies = [ "once_cell", "procfs", "rlimit", - "windows 0.62.2", + "windows", ] [[package]] @@ -6052,7 +6042,7 @@ checksum = "59b43b4fd69e3437618106f7754f34021b831a514f9e1a98ae863cabcd8d8dad" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -6207,7 +6197,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -6320,7 +6310,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -6491,9 +6481,9 @@ dependencies = [ [[package]] name = "opentelemetry-semantic-conventions" -version = "0.32.0" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ca2f98a0437b427b4b08f19f1caa3c44db885a202bc12cfea13d6c702243d68" +checksum = "c913ac17a6c451661ee255f4625d143e51647ae78ebd969b75e41c4442f4fe47" [[package]] name = "opentelemetry_sdk" @@ -6572,7 +6562,7 @@ dependencies = [ "by_address", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -6601,7 +6591,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -6630,7 +6620,7 @@ dependencies = [ "libc", "redox_syscall 0.5.18", "smallvec", - "windows-link 0.2.1", + "windows-link", ] [[package]] @@ -6722,7 +6712,7 @@ dependencies = [ "phf_shared", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -6751,7 +6741,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -6922,7 +6912,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -6964,6 +6954,16 @@ dependencies = [ "quote", ] +[[package]] +name = "proc-macro-error-attr3" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82366fd7d8b7a440d66d13418820c69df9b3908bcb1a0476d7f5ce5d12f5a04d" +dependencies = [ + "proc-macro2", + "quote", +] + [[package]] name = "proc-macro-error2" version = "2.0.1" @@ -6973,7 +6973,18 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn 2.0.117", +] + +[[package]] +name = "proc-macro-error3" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b511283ea8a74b4b39447b128c5d00f03a356b7424554b13e298a5550100d9ac" +dependencies = [ + "proc-macro-error-attr3", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] @@ -7046,7 +7057,7 @@ checksum = "fb6dc647500e84a25a85b100e76c85b8ace114c209432dc174f20aac11d4ed6c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -7057,7 +7068,7 @@ checksum = "c57924a81864dddafba92e1bf92f9bf82f97096c44489548a60e888e1547549b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -7080,7 +7091,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -7180,9 +7191,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.45" +version = "1.0.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" dependencies = [ "proc-macro2", ] @@ -7314,9 +7325,9 @@ dependencies = [ [[package]] name = "ratatui" -version = "0.30.1" +version = "0.30.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1695748e3a735b34968c887ceea5a380b43545903868ae8f5b666593100f6b68" +checksum = "3274ba0a2c5e1bcad2a2005d20f4dc59dad26b2eb0940fb094500dba4099d57d" dependencies = [ "instability", "ratatui-core", @@ -7327,14 +7338,13 @@ dependencies = [ [[package]] name = "ratatui-core" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d3603f354bba8c595fa47860e60142d7372b7210c27044c6a7d0e1a4336b44" +checksum = "cbb175c433c8e28a809d1f5773a2ae96e68c0ce40db865cbab1020bf33ae479c" dependencies = [ "bitflags 2.13.0", "compact_str", "hashbrown 0.17.0", - "indoc", "itertools 0.14.0", "kasuari", "lru 0.18.0", @@ -7349,9 +7359,9 @@ dependencies = [ [[package]] name = "ratatui-crossterm" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b2867bedcbd6a690ca4f8672a687b730ec07660c79844517b084311b529980c" +checksum = "567584a3b0e6a8203c23de40b4861497266725eb5363dbfd18a1edd603cca9f0" dependencies = [ "cfg-if", "crossterm", @@ -7361,9 +7371,9 @@ dependencies = [ [[package]] name = "ratatui-widgets" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef4f17dd7ac3abf5adc2b920a03c61eee4bfe6a88fa5191936895525371d79c" +checksum = "66e3d19bcc9130ca376277d93b60767ff121ace3be06f5f95f81dd68956407d1" dependencies = [ "bitflags 2.13.0", "hashbrown 0.17.0", @@ -7460,7 +7470,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -7558,7 +7568,7 @@ checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" [[package]] name = "reth" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-node-bindings", "alloy-primitives", @@ -7608,7 +7618,7 @@ dependencies = [ [[package]] name = "reth-basic-payload-builder" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7634,7 +7644,7 @@ dependencies = [ [[package]] name = "reth-bb" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7681,7 +7691,7 @@ dependencies = [ [[package]] name = "reth-chain-state" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7714,7 +7724,7 @@ dependencies = [ [[package]] name = "reth-chainspec" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-chains", "alloy-consensus", @@ -7734,7 +7744,7 @@ dependencies = [ [[package]] name = "reth-cli" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-genesis", "clap", @@ -7746,7 +7756,7 @@ dependencies = [ [[package]] name = "reth-cli-commands" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-chains", "alloy-consensus", @@ -7836,7 +7846,7 @@ dependencies = [ [[package]] name = "reth-cli-runner" -version = "2.3.0" +version = "2.4.1" dependencies = [ "reth-tasks", "tokio", @@ -7845,7 +7855,7 @@ dependencies = [ [[package]] name = "reth-cli-util" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-eips", "alloy-primitives", @@ -7867,9 +7877,9 @@ dependencies = [ [[package]] name = "reth-codecs" -version = "0.5.0" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "312c1817d0e37c6112d30fcc9e92226d8e73cc66ea6bd640c80e0e0df956cc04" +checksum = "eb43f4858fef4e3b42b80954d2edf1ff67c8ad7498347083cdc7d0f6eff2f081" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7888,18 +7898,18 @@ dependencies = [ [[package]] name = "reth-codecs-derive" -version = "0.5.0" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "526ee2bfded1897b7bb43f46410f487ee48a41b7e93cf18031079660b433550d" +checksum = "5798ae4d6b264764bc0d742468bd32c7fb515e774645d4930f95ff6311f3ae14" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] name = "reth-config" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-primitives", "eyre", @@ -7917,7 +7927,7 @@ dependencies = [ [[package]] name = "reth-consensus" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -7930,7 +7940,7 @@ dependencies = [ [[package]] name = "reth-consensus-common" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7944,7 +7954,7 @@ dependencies = [ [[package]] name = "reth-consensus-debug-client" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7968,7 +7978,7 @@ dependencies = [ [[package]] name = "reth-db" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8004,7 +8014,7 @@ dependencies = [ [[package]] name = "reth-db-api" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8032,7 +8042,7 @@ dependencies = [ [[package]] name = "reth-db-common" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-genesis", @@ -8063,7 +8073,7 @@ dependencies = [ [[package]] name = "reth-db-models" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8079,7 +8089,7 @@ dependencies = [ [[package]] name = "reth-discv4" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -8105,7 +8115,7 @@ dependencies = [ [[package]] name = "reth-discv5" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -8130,7 +8140,7 @@ dependencies = [ [[package]] name = "reth-dns-discovery" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-chains", "alloy-primitives", @@ -8158,7 +8168,7 @@ dependencies = [ [[package]] name = "reth-downloaders" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8196,7 +8206,7 @@ dependencies = [ [[package]] name = "reth-e2e-test-utils" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8252,12 +8262,11 @@ dependencies = [ [[package]] name = "reth-ecies" -version = "2.3.0" +version = "2.4.1" dependencies = [ "aes", "alloy-primitives", "alloy-rlp", - "block-padding", "byteorder", "cipher", "concat-kdf", @@ -8279,7 +8288,7 @@ dependencies = [ [[package]] name = "reth-engine-local" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8301,7 +8310,7 @@ dependencies = [ [[package]] name = "reth-engine-primitives" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8313,10 +8322,12 @@ dependencies = [ "reth-errors", "reth-ethereum-primitives", "reth-evm", + "reth-execution-errors", "reth-execution-types", "reth-payload-builder-primitives", "reth-payload-primitives", "reth-primitives-traits", + "reth-storage-api", "reth-trie-common", "serde", "thiserror 2.0.18", @@ -8325,7 +8336,7 @@ dependencies = [ [[package]] name = "reth-engine-tree" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -8394,7 +8405,7 @@ dependencies = [ [[package]] name = "reth-engine-util" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8423,7 +8434,7 @@ dependencies = [ [[package]] name = "reth-era" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8448,7 +8459,7 @@ dependencies = [ [[package]] name = "reth-era-downloader" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-primitives", "bytes", @@ -8466,7 +8477,7 @@ dependencies = [ [[package]] name = "reth-era-utils" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8494,7 +8505,7 @@ dependencies = [ [[package]] name = "reth-errors" -version = "2.3.0" +version = "2.4.1" dependencies = [ "reth-consensus", "reth-execution-errors", @@ -8504,7 +8515,7 @@ dependencies = [ [[package]] name = "reth-eth-wire" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-chains", "alloy-consensus", @@ -8542,7 +8553,7 @@ dependencies = [ [[package]] name = "reth-eth-wire-types" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-chains", "alloy-consensus", @@ -8569,7 +8580,7 @@ dependencies = [ [[package]] name = "reth-ethereum" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-rpc-types-engine", "alloy-rpc-types-eth", @@ -8609,7 +8620,7 @@ dependencies = [ [[package]] name = "reth-ethereum-cli" -version = "2.3.0" +version = "2.4.1" dependencies = [ "clap", "eyre", @@ -8632,7 +8643,7 @@ dependencies = [ [[package]] name = "reth-ethereum-consensus" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8648,7 +8659,7 @@ dependencies = [ [[package]] name = "reth-ethereum-engine-primitives" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8664,7 +8675,7 @@ dependencies = [ [[package]] name = "reth-ethereum-forks" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-eip2124", "alloy-hardforks 0.4.7", @@ -8677,7 +8688,7 @@ dependencies = [ [[package]] name = "reth-ethereum-payload-builder" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8706,7 +8717,7 @@ dependencies = [ [[package]] name = "reth-ethereum-primitives" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8727,7 +8738,7 @@ dependencies = [ [[package]] name = "reth-etl" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-primitives", "rayon", @@ -8737,7 +8748,7 @@ dependencies = [ [[package]] name = "reth-evm" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -8762,7 +8773,7 @@ dependencies = [ [[package]] name = "reth-evm-ethereum" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8787,7 +8798,7 @@ dependencies = [ [[package]] name = "reth-execution-cache" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-primitives", "fixed-cache", @@ -8805,7 +8816,7 @@ dependencies = [ [[package]] name = "reth-execution-errors" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-evm", "alloy-primitives", @@ -8817,7 +8828,7 @@ dependencies = [ [[package]] name = "reth-execution-types" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8838,7 +8849,7 @@ dependencies = [ [[package]] name = "reth-exex" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8883,7 +8894,7 @@ dependencies = [ [[package]] name = "reth-exex-test-utils" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-eips", "eyre", @@ -8914,7 +8925,7 @@ dependencies = [ [[package]] name = "reth-exex-types" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8931,7 +8942,7 @@ dependencies = [ [[package]] name = "reth-fs-util" -version = "2.3.0" +version = "2.4.1" dependencies = [ "serde", "serde_json", @@ -8940,7 +8951,7 @@ dependencies = [ [[package]] name = "reth-invalid-block-hooks" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8971,7 +8982,7 @@ dependencies = [ [[package]] name = "reth-ipc" -version = "2.3.0" +version = "2.4.1" dependencies = [ "bytes", "futures", @@ -8993,7 +9004,7 @@ dependencies = [ [[package]] name = "reth-libmdbx" -version = "2.3.0" +version = "2.4.1" dependencies = [ "bitflags 2.13.0", "byteorder", @@ -9010,7 +9021,7 @@ dependencies = [ [[package]] name = "reth-mdbx-sys" -version = "2.3.0" +version = "2.4.1" dependencies = [ "bindgen", "cc", @@ -9018,7 +9029,7 @@ dependencies = [ [[package]] name = "reth-metrics" -version = "2.3.0" +version = "2.4.1" dependencies = [ "futures", "metrics", @@ -9030,7 +9041,7 @@ dependencies = [ [[package]] name = "reth-net-banlist" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-primitives", "ipnet", @@ -9038,7 +9049,7 @@ dependencies = [ [[package]] name = "reth-net-nat" -version = "2.3.0" +version = "2.4.1" dependencies = [ "futures-util", "if-addrs", @@ -9052,7 +9063,7 @@ dependencies = [ [[package]] name = "reth-network" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9104,6 +9115,7 @@ dependencies = [ "serde", "smallvec", "socket2", + "test-case", "thiserror 2.0.18", "tokio", "tokio-stream", @@ -9114,7 +9126,7 @@ dependencies = [ [[package]] name = "reth-network-api" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9138,7 +9150,7 @@ dependencies = [ [[package]] name = "reth-network-p2p" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -9162,7 +9174,7 @@ dependencies = [ [[package]] name = "reth-network-peers" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -9179,7 +9191,7 @@ dependencies = [ [[package]] name = "reth-network-types" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-eip2124", "humantime-serde", @@ -9192,7 +9204,7 @@ dependencies = [ [[package]] name = "reth-nippy-jar" -version = "2.3.0" +version = "2.4.1" dependencies = [ "anyhow", "bincode", @@ -9210,7 +9222,7 @@ dependencies = [ [[package]] name = "reth-node-api" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-rpc-types-engine", "eyre", @@ -9233,7 +9245,7 @@ dependencies = [ [[package]] name = "reth-node-builder" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9304,7 +9316,7 @@ dependencies = [ [[package]] name = "reth-node-core" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9360,7 +9372,7 @@ dependencies = [ [[package]] name = "reth-node-ethereum" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-contract", @@ -9408,6 +9420,7 @@ dependencies = [ "reth-rpc", "reth-rpc-api", "reth-rpc-builder", + "reth-rpc-engine-api", "reth-rpc-eth-api", "reth-rpc-eth-types", "reth-rpc-layer", @@ -9428,7 +9441,7 @@ dependencies = [ [[package]] name = "reth-node-ethstats" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9451,7 +9464,7 @@ dependencies = [ [[package]] name = "reth-node-events" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9474,7 +9487,7 @@ dependencies = [ [[package]] name = "reth-node-metrics" -version = "2.3.0" +version = "2.4.1" dependencies = [ "bytes", "eyre", @@ -9503,7 +9516,7 @@ dependencies = [ [[package]] name = "reth-node-types" -version = "2.3.0" +version = "2.4.1" dependencies = [ "reth-chainspec", "reth-db-api", @@ -9514,7 +9527,7 @@ dependencies = [ [[package]] name = "reth-payload-builder" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9537,7 +9550,7 @@ dependencies = [ [[package]] name = "reth-payload-builder-primitives" -version = "2.3.0" +version = "2.4.1" dependencies = [ "pin-project", "reth-payload-primitives", @@ -9548,7 +9561,7 @@ dependencies = [ [[package]] name = "reth-payload-primitives" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9572,7 +9585,7 @@ dependencies = [ [[package]] name = "reth-payload-util" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9581,7 +9594,7 @@ dependencies = [ [[package]] name = "reth-payload-validator" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-rpc-types-engine", @@ -9590,9 +9603,9 @@ dependencies = [ [[package]] name = "reth-primitives-traits" -version = "0.5.0" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66169f2c520c1c94192a18d11c5466565f079588def7d76abb631db80d353829" +checksum = "f59c928b2865af5bcdbce94800270b7f7798f27a22ca0aabdd2b7d3e6587c9ce" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9623,7 +9636,7 @@ dependencies = [ [[package]] name = "reth-provider" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -9676,14 +9689,13 @@ dependencies = [ [[package]] name = "reth-prune" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-primitives", "assert_matches", "itertools 0.14.0", "metrics", - "rayon", "reth-config", "reth-db", "reth-db-api", @@ -9706,13 +9718,9 @@ dependencies = [ "tracing", ] -[[package]] -name = "reth-prune-db" -version = "2.3.0" - [[package]] name = "reth-prune-types" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-primitives", "arbitrary", @@ -9732,7 +9740,7 @@ dependencies = [ [[package]] name = "reth-revm" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9748,7 +9756,7 @@ dependencies = [ [[package]] name = "reth-rpc" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-dyn-abi", @@ -9828,7 +9836,7 @@ dependencies = [ [[package]] name = "reth-rpc-api" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-eips", "alloy-genesis", @@ -9856,28 +9864,9 @@ dependencies = [ "tokio", ] -[[package]] -name = "reth-rpc-api-testing-util" -version = "2.3.0" -dependencies = [ - "alloy-eips", - "alloy-primitives", - "alloy-rpc-types-eth", - "alloy-rpc-types-trace", - "futures", - "jsonrpsee", - "jsonrpsee-http-client", - "reth-ethereum-primitives", - "reth-rpc-api", - "reth-rpc-eth-api", - "serde_json", - "similar-asserts", - "tokio", -] - [[package]] name = "reth-rpc-builder" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-eips", "alloy-network", @@ -9935,7 +9924,7 @@ dependencies = [ [[package]] name = "reth-rpc-convert" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-evm", @@ -9955,7 +9944,7 @@ dependencies = [ [[package]] name = "reth-rpc-e2e-tests" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-genesis", "alloy-rpc-types-engine", @@ -9975,7 +9964,7 @@ dependencies = [ [[package]] name = "reth-rpc-engine-api" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-eips", "alloy-primitives", @@ -10011,7 +10000,7 @@ dependencies = [ [[package]] name = "reth-rpc-eth-api" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-dyn-abi", @@ -10057,7 +10046,7 @@ dependencies = [ [[package]] name = "reth-rpc-eth-types" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-chains", "alloy-consensus", @@ -10109,7 +10098,7 @@ dependencies = [ [[package]] name = "reth-rpc-layer" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-rpc-types-engine", "http", @@ -10126,7 +10115,7 @@ dependencies = [ [[package]] name = "reth-rpc-server-types" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-eips", "alloy-primitives", @@ -10141,9 +10130,9 @@ dependencies = [ [[package]] name = "reth-rpc-traits" -version = "0.5.0" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "936ba1aa863f2bbb2a0b5f28ea8a114058154e74969385ab27355421d56f9a03" +checksum = "ace77dbcdd59c014fda4565ebfec76cd1fe6f5d98584b4655e8d22a947b9eee3" dependencies = [ "alloy-consensus", "alloy-network", @@ -10156,7 +10145,7 @@ dependencies = [ [[package]] name = "reth-stages" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10218,7 +10207,7 @@ dependencies = [ [[package]] name = "reth-stages-api" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-eips", "alloy-primitives", @@ -10252,7 +10241,7 @@ dependencies = [ [[package]] name = "reth-stages-types" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-primitives", "arbitrary", @@ -10268,7 +10257,7 @@ dependencies = [ [[package]] name = "reth-static-file" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-primitives", "assert_matches", @@ -10291,7 +10280,7 @@ dependencies = [ [[package]] name = "reth-static-file-types" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-primitives", "clap", @@ -10309,7 +10298,7 @@ dependencies = [ [[package]] name = "reth-storage-api" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -10336,7 +10325,7 @@ dependencies = [ [[package]] name = "reth-storage-errors" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-eips", "alloy-primitives", @@ -10352,7 +10341,7 @@ dependencies = [ [[package]] name = "reth-storage-rpc-provider" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10381,7 +10370,7 @@ dependencies = [ [[package]] name = "reth-tasks" -version = "2.3.0" +version = "2.4.1" dependencies = [ "crossbeam-utils", "dashmap", @@ -10401,7 +10390,7 @@ dependencies = [ [[package]] name = "reth-testing-utils" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10417,7 +10406,7 @@ dependencies = [ [[package]] name = "reth-tokio-util" -version = "2.3.0" +version = "2.4.1" dependencies = [ "tokio", "tokio-stream", @@ -10426,7 +10415,7 @@ dependencies = [ [[package]] name = "reth-tracing" -version = "2.3.0" +version = "2.4.1" dependencies = [ "clap", "eyre", @@ -10445,7 +10434,7 @@ dependencies = [ [[package]] name = "reth-tracing-otlp" -version = "2.3.0" +version = "2.4.1" dependencies = [ "base64 0.22.1", "clap", @@ -10463,7 +10452,7 @@ dependencies = [ [[package]] name = "reth-transaction-pool" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10514,7 +10503,7 @@ dependencies = [ [[package]] name = "reth-trie" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10545,7 +10534,7 @@ dependencies = [ [[package]] name = "reth-trie-common" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10577,7 +10566,7 @@ dependencies = [ [[package]] name = "reth-trie-db" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -10607,7 +10596,7 @@ dependencies = [ [[package]] name = "reth-trie-parallel" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-evm", "alloy-primitives", @@ -10635,7 +10624,7 @@ dependencies = [ [[package]] name = "reth-trie-sparse" -version = "2.3.0" +version = "2.4.1" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -10660,14 +10649,15 @@ dependencies = [ "serde_json", "slotmap", "smallvec", + "strum 0.27.2", "tracing", ] [[package]] name = "reth-zstd-compressors" -version = "0.5.0" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdd4e89ad5d14393bde9b2b152480d4fd7f2399d0f9efe80ea29ec7c0b76bcdc" +checksum = "e1bd4ae4f54a2ba813eef082910bc646bd31cab8ed134308fa71f1068331fb84" dependencies = [ "zstd", ] @@ -10806,9 +10796,9 @@ dependencies = [ [[package]] name = "revm-inspectors" -version = "0.41.0" +version = "0.41.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df57773f1122efc6d02e4126a2482e26936b759692d3b7a0fc649a21e4c893d9" +checksum = "54e22e1aa328f78e8c3dea6a3a860a3a3c3a77c4a7e775e3fdd3dcbb24a152ac" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", @@ -10892,7 +10882,7 @@ dependencies = [ [[package]] name = "revmc" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#7e3536d659d3bdd4b433cff7df201d1605cc8853" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#520462a463523a3bcd0a47226ddbc3200d62232e" dependencies = [ "revm-bytecode", "revm-context-interface", @@ -10912,7 +10902,7 @@ dependencies = [ [[package]] name = "revmc-backend" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#7e3536d659d3bdd4b433cff7df201d1605cc8853" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#520462a463523a3bcd0a47226ddbc3200d62232e" dependencies = [ "eyre", "ruint", @@ -10921,12 +10911,12 @@ dependencies = [ [[package]] name = "revmc-build" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#7e3536d659d3bdd4b433cff7df201d1605cc8853" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#520462a463523a3bcd0a47226ddbc3200d62232e" [[package]] name = "revmc-builtins" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#7e3536d659d3bdd4b433cff7df201d1605cc8853" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#520462a463523a3bcd0a47226ddbc3200d62232e" dependencies = [ "paste", "revm-bytecode", @@ -10941,7 +10931,7 @@ dependencies = [ [[package]] name = "revmc-codegen" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#7e3536d659d3bdd4b433cff7df201d1605cc8853" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#520462a463523a3bcd0a47226ddbc3200d62232e" dependencies = [ "alloy-primitives", "bitflags 2.13.0", @@ -10972,7 +10962,7 @@ dependencies = [ [[package]] name = "revmc-context" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#7e3536d659d3bdd4b433cff7df201d1605cc8853" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#520462a463523a3bcd0a47226ddbc3200d62232e" dependencies = [ "revm-context", "revm-context-interface", @@ -10987,7 +10977,7 @@ dependencies = [ [[package]] name = "revmc-llvm" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#7e3536d659d3bdd4b433cff7df201d1605cc8853" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#520462a463523a3bcd0a47226ddbc3200d62232e" dependencies = [ "alloy-primitives", "cc", @@ -11000,7 +10990,7 @@ dependencies = [ [[package]] name = "revmc-runtime" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#7e3536d659d3bdd4b433cff7df201d1605cc8853" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#520462a463523a3bcd0a47226ddbc3200d62232e" dependencies = [ "alloy-evm", "alloy-primitives", @@ -11189,11 +11179,11 @@ checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" [[package]] name = "rustc-hash" -version = "2.1.2" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" dependencies = [ - "rand 0.8.6", + "rand 0.9.4", ] [[package]] @@ -11239,7 +11229,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -11319,7 +11309,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs 1.0.7", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -11614,7 +11604,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -11681,7 +11671,7 @@ dependencies = [ "darling 0.23.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -11901,9 +11891,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.15.1" +version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" dependencies = [ "arbitrary", "serde", @@ -11941,7 +11931,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -12034,7 +12024,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -12046,7 +12036,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -12074,9 +12064,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.117" +version = "2.0.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" dependencies = [ "proc-macro2", "quote", @@ -12085,14 +12075,14 @@ dependencies = [ [[package]] name = "syn-solidity" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec005042c7d952febc1a3ef5b0f6674e9054aa836877a31c90b20e25b3d31744" +checksum = "083be3061e64d362cbe6ef12cfe1307ba3884326d8856448fe8a120fa2c44ebf" dependencies = [ "paste", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -12112,7 +12102,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -12126,7 +12116,7 @@ dependencies = [ "ntapi", "objc2-core-foundation", "objc2-io-kit", - "windows 0.62.2", + "windows", ] [[package]] @@ -12189,7 +12179,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -12210,7 +12200,7 @@ dependencies = [ "cfg-if", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -12221,7 +12211,7 @@ checksum = "5c89e72a01ed4c579669add59014b9a524d609c0c88c6a585ce37485879f6ffb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", "test-case-core", ] @@ -12261,7 +12251,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -12309,7 +12299,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -12320,21 +12310,21 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] name = "thread-priority" -version = "3.0.0" +version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2210811179577da3d54eb69ab0b50490ee40491a25d95b8c6011ba40771cb721" +checksum = "8d2e834949be5111506bb252643498af1514f600d9e1dceedaa42afae155b67f" dependencies = [ "bitflags 2.13.0", "cfg-if", "libc", "log", "rustversion", - "windows 0.61.3", + "windows", ] [[package]] @@ -12479,7 +12469,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -12740,7 +12730,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -12935,7 +12925,7 @@ dependencies = [ "darling 0.23.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -13203,7 +13193,7 @@ checksum = "d674d135b4a8c1d7e813e2f8d1c9a58308aee4a680323066025e53132218bd91" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -13300,7 +13290,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", "wasm-bindgen-shared", ] @@ -13468,7 +13458,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -13499,20 +13489,7 @@ dependencies = [ "darling 0.21.3", "proc-macro2", "quote", - "syn 2.0.117", -] - -[[package]] -name = "windows" -version = "0.61.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" -dependencies = [ - "windows-collections 0.2.0", - "windows-core 0.61.2", - "windows-future 0.2.1", - "windows-link 0.1.3", - "windows-numerics 0.2.0", + "syn 2.0.118", ] [[package]] @@ -13521,19 +13498,10 @@ version = "0.62.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" dependencies = [ - "windows-collections 0.3.2", - "windows-core 0.62.2", - "windows-future 0.3.2", - "windows-numerics 0.3.1", -] - -[[package]] -name = "windows-collections" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8" -dependencies = [ - "windows-core 0.61.2", + "windows-collections", + "windows-core", + "windows-future", + "windows-numerics", ] [[package]] @@ -13542,20 +13510,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" dependencies = [ - "windows-core 0.62.2", -] - -[[package]] -name = "windows-core" -version = "0.61.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" -dependencies = [ - "windows-implement", - "windows-interface", - "windows-link 0.1.3", - "windows-result 0.3.4", - "windows-strings 0.4.2", + "windows-core", ] [[package]] @@ -13566,20 +13521,9 @@ checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" dependencies = [ "windows-implement", "windows-interface", - "windows-link 0.2.1", - "windows-result 0.4.1", - "windows-strings 0.5.1", -] - -[[package]] -name = "windows-future" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" -dependencies = [ - "windows-core 0.61.2", - "windows-link 0.1.3", - "windows-threading 0.1.0", + "windows-link", + "windows-result", + "windows-strings", ] [[package]] @@ -13588,9 +13532,9 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" dependencies = [ - "windows-core 0.62.2", - "windows-link 0.2.1", - "windows-threading 0.2.1", + "windows-core", + "windows-link", + "windows-threading", ] [[package]] @@ -13601,7 +13545,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -13612,39 +13556,23 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] -[[package]] -name = "windows-link" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" - [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" -[[package]] -name = "windows-numerics" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" -dependencies = [ - "windows-core 0.61.2", - "windows-link 0.1.3", -] - [[package]] name = "windows-numerics" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" dependencies = [ - "windows-core 0.62.2", - "windows-link 0.2.1", + "windows-core", + "windows-link", ] [[package]] @@ -13653,18 +13581,9 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" dependencies = [ - "windows-link 0.2.1", - "windows-result 0.4.1", - "windows-strings 0.5.1", -] - -[[package]] -name = "windows-result" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" -dependencies = [ - "windows-link 0.1.3", + "windows-link", + "windows-result", + "windows-strings", ] [[package]] @@ -13673,16 +13592,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" dependencies = [ - "windows-link 0.2.1", -] - -[[package]] -name = "windows-strings" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" -dependencies = [ - "windows-link 0.1.3", + "windows-link", ] [[package]] @@ -13691,7 +13601,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" dependencies = [ - "windows-link 0.2.1", + "windows-link", ] [[package]] @@ -13736,7 +13646,7 @@ version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ - "windows-link 0.2.1", + "windows-link", ] [[package]] @@ -13776,7 +13686,7 @@ version = "0.53.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" dependencies = [ - "windows-link 0.2.1", + "windows-link", "windows_aarch64_gnullvm 0.53.1", "windows_aarch64_msvc 0.53.1", "windows_i686_gnu 0.53.1", @@ -13787,22 +13697,13 @@ dependencies = [ "windows_x86_64_msvc 0.53.1", ] -[[package]] -name = "windows-threading" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6" -dependencies = [ - "windows-link 0.1.3", -] - [[package]] name = "windows-threading" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" dependencies = [ - "windows-link 0.2.1", + "windows-link", ] [[package]] @@ -13994,7 +13895,7 @@ dependencies = [ "heck", "indexmap 2.14.0", "prettyplease", - "syn 2.0.117", + "syn 2.0.118", "wasm-metadata", "wit-bindgen-core", "wit-component", @@ -14010,7 +13911,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", "wit-bindgen-core", "wit-bindgen-rust", ] @@ -14133,7 +14034,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", "synstructure", ] @@ -14154,7 +14055,7 @@ checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -14174,7 +14075,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", "synstructure", ] @@ -14195,7 +14096,7 @@ checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -14230,7 +14131,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 8df93e9cfe3..6f108fe3cdc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace.package] -version = "2.3.0" +version = "2.4.1" edition = "2024" rust-version = "1.95" license = "MIT OR Apache-2.0" @@ -77,7 +77,6 @@ members = [ "crates/payload/primitives/", "crates/payload/validator/", "crates/payload/util/", - "crates/prune/db", "crates/prune/prune", "crates/prune/types", "crates/revm/", @@ -89,7 +88,6 @@ members = [ "crates/rpc/rpc-eth-types/", "crates/rpc/rpc-layer", "crates/rpc/rpc-server-types/", - "crates/rpc/rpc-testing-util/", "crates/rpc/rpc-e2e-tests/", "crates/rpc/rpc-convert/", "crates/rpc/rpc/", @@ -400,7 +398,6 @@ reth-prune-types = { path = "crates/prune/types", default-features = false } reth-revm = { path = "crates/revm", default-features = false } reth-rpc = { path = "crates/rpc/rpc" } reth-rpc-api = { path = "crates/rpc/rpc-api" } -reth-rpc-api-testing-util = { path = "crates/rpc/rpc-testing-util" } reth-rpc-builder = { path = "crates/rpc/rpc-builder" } reth-rpc-e2e-tests = { path = "crates/rpc/rpc-e2e-tests" } reth-rpc-engine-api = { path = "crates/rpc/rpc-engine-api" } @@ -436,9 +433,9 @@ revmc = { git = "https://github.com/paradigmxyz/revmc", branch = "main", default revm-inspectors = "0.41.0" # eth -alloy-dyn-abi = "1.6.0" -alloy-primitives = { version = "1.6.0", default-features = false, features = ["map-foldhash"] } -alloy-sol-types = { version = "1.6.0", default-features = false } +alloy-dyn-abi = "1.6.1" +alloy-primitives = { version = "1.6.1", default-features = false, features = ["map-foldhash"] } +alloy-sol-types = { version = "1.6.1", default-features = false } alloy-chains = { version = "0.2.33", default-features = false } alloy-eip2124 = { version = "0.2.0", default-features = false } @@ -598,7 +595,7 @@ secp256k1 = { version = "0.30", default-features = false, features = ["global-co rand_08 = { package = "rand", version = "0.8" } # for eip-4844 -c-kzg = "2.1.5" +c-kzg = "2.1.8" # config toml = "0.9" @@ -643,7 +640,6 @@ aes = "0.8.1" ahash = "0.8" anyhow = "1.0" bindgen = { version = "0.72", default-features = false } -block-padding = "0.3" cc = "1.2.62" cipher = "0.4.3" comfy-table = "7.0" diff --git a/crates/chain-state/src/preserved_sparse_trie.rs b/crates/chain-state/src/preserved_sparse_trie.rs index 507ea6f5c85..2d36422283d 100644 --- a/crates/chain-state/src/preserved_sparse_trie.rs +++ b/crates/chain-state/src/preserved_sparse_trie.rs @@ -2,100 +2,170 @@ use alloy_primitives::B256; use reth_trie_sparse::SparseStateTrie; +use std::{ + fmt, + sync::mpsc::{self, Receiver, Sender}, +}; use tracing::debug; /// Type alias for the sparse trie type used in preservation. pub type SparseTrie = SparseStateTrie; -/// Guard that holds the lock on the preserved trie. -/// While held, the next trie take will block. Call `store()` to save the trie before dropping. -#[derive(Debug)] -pub struct PreservedTrieGuard<'a>(parking_lot::MutexGuard<'a, Option>); - -impl<'a> PreservedTrieGuard<'a> { - /// Creates a new guard from the preserved trie lock. - pub(crate) const fn new( - guard: parking_lot::MutexGuard<'a, Option>, - ) -> Self { - PreservedTrieGuard(guard) - } - - /// Stores a preserved trie for later reuse. - pub fn store(&mut self, trie: PreservedSparseTrie) { - self.0.replace(trie); - } +/// A preserved sparse trie that can be reused across payload validations. +pub struct PreservedSparseTrie { + /// The preserved sparse state trie, or a handle to wait for it. + trie: PreservedSparseTrieInner, + /// The state root this trie represents. + /// + /// Used to verify continuity: a new payload's `parent_state_root` must match this before the + /// existing sparse trie nodes can be reused. + state_root: B256, + /// Parent block hash of the earliest overlay state covered by this trie. + anchor_hash: B256, } -/// A preserved sparse trie that can be reused across payload validations. -/// -/// The trie exists in one of two states: -/// - **Anchored**: Has a computed state root and can be reused for payloads whose parent state root -/// matches the anchor. -/// - **Cleared**: Trie data has been cleared but allocations are preserved for reuse. -#[derive(Debug)] -pub enum PreservedSparseTrie { - /// Trie with a computed state root that can be reused for continuation payloads. - Anchored { - /// The sparse state trie anchored to the computed state root. - trie: SparseTrie, - /// The state root this trie represents (computed from the previous block). - /// Used to verify continuity: new payload's `parent_state_root` must match this. - state_root: B256, - }, - /// Cleared trie with preserved allocations, ready for fresh use. - Cleared { - /// The sparse state trie with cleared data but preserved allocations. - trie: SparseTrie, - }, +impl fmt::Debug for PreservedSparseTrie { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PreservedSparseTrie") + .field("state_root", &self.state_root) + .field("anchor_hash", &self.anchor_hash) + .finish_non_exhaustive() + } } impl PreservedSparseTrie { /// Creates a new anchored preserved trie. /// - /// The `state_root` is the computed state root from the trie, which becomes the - /// anchor for determining if subsequent payloads can reuse this trie. - pub const fn anchored(trie: SparseTrie, state_root: B256) -> Self { - Self::Anchored { trie, state_root } + /// The `state_root` is the computed state root from the trie. The `anchor_hash` is the parent + /// block hash of the earliest overlay state covered by the trie. + pub const fn anchored(trie: SparseTrie, state_root: B256, anchor_hash: B256) -> Self { + Self { trie: PreservedSparseTrieInner::Ready(trie), state_root, anchor_hash } + } + + /// Creates a pending preserved trie and a completer that will publish the trie later. + pub fn pending(state_root: B256, anchor_hash: B256) -> (Self, PreservedSparseTrieCompleter) { + let (tx, rx) = mpsc::channel(); + ( + Self { trie: PreservedSparseTrieInner::Pending(rx), state_root, anchor_hash }, + PreservedSparseTrieCompleter { tx }, + ) } - /// Creates a cleared preserved trie (allocations preserved, data cleared). - pub const fn cleared(trie: SparseTrie) -> Self { - Self::Cleared { trie } + /// Returns the state root this trie is anchored to. + pub const fn state_root(&self) -> B256 { + self.state_root } - /// Consumes self and returns the trie for reuse. + /// Returns the parent block hash of the earliest overlay state covered by this trie. + pub const fn anchor_hash(&self) -> B256 { + self.anchor_hash + } + + /// Consumes self and returns the trie if it can be reused for the parent state root. /// - /// If the preserved trie is anchored and the parent state root matches, the preserved - /// trie structure is reused directly. Otherwise, the trie is cleared but allocations - /// are preserved to reduce memory overhead. - pub fn into_trie_for(self, parent_state_root: B256) -> SparseTrie { + /// If the parent state root does not match the preserved trie's state root, this drops the trie + /// and returns `None` so the caller can create a fresh sparse trie. + pub fn into_trie_for( + self, + parent_state_root: B256, + ) -> Result, PreservedSparseTrieError> { + if self.state_root == parent_state_root { + let trie = match self.trie { + PreservedSparseTrieInner::Ready(trie) => trie, + PreservedSparseTrieInner::Pending(rx) => match rx.recv() { + Ok(trie) => trie, + Err(_) => { + return Err(PreservedSparseTrieError::ProducerDropped { + state_root: self.state_root, + }) + } + }, + }; + debug!( + target: "engine::tree::payload_processor", + state_root = %self.state_root, + anchor_hash = %self.anchor_hash, + "Reusing anchored sparse trie for continuation payload" + ); + Ok(Some(trie)) + } else { + debug!( + target: "engine::tree::payload_processor", + anchor_root = %self.state_root, + anchor_hash = %self.anchor_hash, + %parent_state_root, + "Dropping anchored sparse trie - parent state root mismatch" + ); + Ok(None) + } + } +} + +/// Error returned when consuming a preserved sparse trie. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PreservedSparseTrieError { + /// The producer of a pending preserved sparse trie dropped before publishing it. + ProducerDropped { + /// The state root the pending trie was expected to represent. + state_root: B256, + }, +} + +impl fmt::Display for PreservedSparseTrieError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - Self::Anchored { trie, state_root } if state_root == parent_state_root => { - debug!( - target: "engine::tree::payload_processor", - %state_root, - "Reusing anchored sparse trie for continuation payload" - ); - trie - } - Self::Anchored { mut trie, state_root } => { - debug!( - target: "engine::tree::payload_processor", - anchor_root = %state_root, - %parent_state_root, - "Clearing anchored sparse trie - parent state root mismatch" - ); - trie.clear(); - trie - } - Self::Cleared { trie } => { - debug!( - target: "engine::tree::payload_processor", - %parent_state_root, - "Using cleared sparse trie with preserved allocations" - ); - trie + Self::ProducerDropped { state_root } => { + write!(f, "pending preserved sparse trie producer dropped for {state_root}") } } } } + +impl std::error::Error for PreservedSparseTrieError {} + +#[allow(clippy::large_enum_variant)] +enum PreservedSparseTrieInner { + Ready(SparseTrie), + Pending(Receiver), +} + +/// Completes a pending preserved sparse trie. +#[derive(Debug)] +pub struct PreservedSparseTrieCompleter { + tx: Sender, +} + +impl PreservedSparseTrieCompleter { + /// Publishes the trie for a pending preserved sparse trie. + pub fn complete(self, trie: SparseTrie) -> Result<(), SparseTrie> { + self.tx.send(trie).map_err(|err| err.0) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn pending_trie_exposes_state_root_before_completion() { + let state_root = B256::with_last_byte(1); + let anchor_hash = B256::with_last_byte(2); + let (preserved, completer) = PreservedSparseTrie::pending(state_root, anchor_hash); + + assert_eq!(preserved.state_root(), state_root); + assert_eq!(preserved.anchor_hash(), anchor_hash); + completer.complete(SparseTrie::default()).unwrap(); + assert!(preserved.into_trie_for(state_root).unwrap().is_some()); + } + + #[test] + fn pending_trie_with_mismatched_root_does_not_wait() { + let state_root = B256::with_last_byte(1); + let other_state_root = B256::with_last_byte(2); + let anchor_hash = B256::with_last_byte(3); + let (preserved, completer) = PreservedSparseTrie::pending(state_root, anchor_hash); + + assert!(preserved.into_trie_for(other_state_root).unwrap().is_none()); + assert!(completer.complete(SparseTrie::default()).is_err()); + } +} diff --git a/crates/chain-state/src/state_trie_overlay.rs b/crates/chain-state/src/state_trie_overlay.rs index 369599e7b29..a56e1f2a693 100644 --- a/crates/chain-state/src/state_trie_overlay.rs +++ b/crates/chain-state/src/state_trie_overlay.rs @@ -4,7 +4,7 @@ //! parent has not been persisted yet. [`StateTrieOverlayManager`] tracks those in-memory blocks and //! builds reusable flattened state trie overlays on demand. -use crate::{EthPrimitives, ExecutedBlock, PreservedSparseTrie, PreservedTrieGuard}; +use crate::{EthPrimitives, ExecutedBlock, PreservedSparseTrie}; use alloy_primitives::B256; use parking_lot::Mutex; use reth_metrics::{ @@ -86,17 +86,19 @@ impl StateTrieOverlayManager { } } - /// Takes the preserved sparse trie if present, leaving `None` in its place. + /// Takes the preserved sparse trie if present. pub fn take_sparse_trie(&self) -> Option { self.preserved_sparse_trie.lock().take() } - /// Acquires a guard that blocks taking the trie until dropped. - /// - /// Use this before sending the state root result to ensure the next block waits for the trie - /// to be stored. - pub fn lock_sparse_trie(&self) -> PreservedTrieGuard<'_> { - PreservedTrieGuard::new(self.preserved_sparse_trie.lock()) + /// Stores a preserved sparse trie for later reuse. + pub fn store_sparse_trie(&self, trie: PreservedSparseTrie) { + *self.preserved_sparse_trie.lock() = Some(trie); + } + + /// Clears any preserved sparse trie state. + pub fn clear_sparse_trie(&self) { + *self.preserved_sparse_trie.lock() = None; } /// Waits until the sparse trie lock becomes available. @@ -150,54 +152,12 @@ impl StateTrieOverlayManager { } } - // Snapshot matching parent overlays before spawning so DashMap iteration guards are - // dropped. - let cached_parent_overlays = self - .overlays - .iter() - .filter_map(|entry| { - let key = *entry.key(); - (key.tip_hash == parent_hash && entry.value().is_ready()).then_some(key.anchor_hash) - }) - .collect::>(); - debug!( target: "chain_state::state_trie_overlay", %hash, %parent_hash, "inserted block into state trie overlay manager" ); - if cached_parent_overlays.is_empty() { - return - } - - #[cfg(feature = "rayon")] - let Some(worker_pool) = self.worker_pool.clone() else { - return - }; - - #[cfg(not(feature = "rayon"))] - let _ = cached_parent_overlays; - - #[cfg(feature = "rayon")] - { - let parent_span = span; - for anchor_hash in cached_parent_overlays { - let manager = ::clone(self); - let parent_span = parent_span.clone(); - worker_pool.spawn(move || { - let _span = tracing::trace_span!( - target: "chain_state::state_trie_overlay", - parent: parent_span, - "precompute_state_trie_overlay", - tip_hash = %hash, - anchor_hash = %anchor_hash, - ) - .entered(); - let _ = manager.precompute_overlay(hash, anchor_hash); - }); - } - } } /// Removes blocks from the live block graph and prunes cached overlays that can no longer be @@ -268,16 +228,6 @@ impl StateTrieOverlayManager { Ok((Arc::clone(&input.nodes), Arc::clone(&input.state))) } - #[cfg(feature = "rayon")] - fn precompute_overlay( - &self, - tip_hash: B256, - anchor_hash: B256, - ) -> Result<(), StateTrieOverlayError> { - let _ = self.get_overlay_inner(tip_hash, anchor_hash, OverlayLookupMode::Precompute)?; - Ok(()) - } - #[tracing::instrument( level = "trace", target = "chain_state::state_trie_overlay", @@ -295,35 +245,14 @@ impl StateTrieOverlayManager { tip_hash: B256, anchor_hash: B256, ) -> Result, StateTrieOverlayError> { - self.get_overlay_inner(tip_hash, anchor_hash, OverlayLookupMode::Required) - .map(|input| input.expect("required overlay lookups always return an overlay")) - } - - fn get_overlay_inner( - &self, - tip_hash: B256, - anchor_hash: B256, - mode: OverlayLookupMode, - ) -> Result>, StateTrieOverlayError> { let key = OverlayCacheKey { anchor_hash, tip_hash }; let span = tracing::Span::current(); if let Some(entry) = self.overlays.get(&key).map(|entry| entry.value().clone()) { + self.record_overlay_cache_reuse(&span); return Ok(match entry { - OverlayCacheEntry::Ready(input) => { - self.metrics.overlay_cache_reuses.increment(1); - span.record("cache_reused", true); - Some(input) - } - OverlayCacheEntry::Computing(waiter) => { - span.record("cache_reused", true); - if mode == OverlayLookupMode::Precompute { - None - } else { - self.metrics.overlay_cache_reuses.increment(1); - Some(waiter.wait()) - } - } + OverlayCacheEntry::Ready(input) => input, + OverlayCacheEntry::Computing(waiter) => waiter.wait(), }) } span.record("cache_reused", false); @@ -365,26 +294,17 @@ impl StateTrieOverlayManager { Ready(Arc), Wait(Arc), Compute(Arc), - Skip, } let action = match self.overlays.entry(key) { - Entry::Occupied(entry) => match entry.get().clone() { - OverlayCacheEntry::Ready(input) => { - self.metrics.overlay_cache_reuses.increment(1); - span.record("cache_reused", true); - CacheAction::Ready(input) - } - OverlayCacheEntry::Computing(waiter) => { - span.record("cache_reused", true); - if mode == OverlayLookupMode::Precompute { - CacheAction::Skip - } else { - self.metrics.overlay_cache_reuses.increment(1); - CacheAction::Wait(waiter) - } + Entry::Occupied(entry) => { + let entry = entry.get().clone(); + self.record_overlay_cache_reuse(&span); + match entry { + OverlayCacheEntry::Ready(input) => CacheAction::Ready(input), + OverlayCacheEntry::Computing(waiter) => CacheAction::Wait(waiter), } - }, + } Entry::Vacant(entry) => { self.metrics.overlay_cache_fills.increment(1); let waiter = Arc::new(OverlayWaiter::new()); @@ -394,9 +314,8 @@ impl StateTrieOverlayManager { }; match action { - CacheAction::Ready(input) => Ok(Some(input)), - CacheAction::Wait(waiter) => Ok(Some(waiter.wait())), - CacheAction::Skip => Ok(None), + CacheAction::Ready(input) => Ok(input), + CacheAction::Wait(waiter) => Ok(waiter.wait()), CacheAction::Compute(waiter) => { let input = self.compute_overlay(compute_input, anchor_hash, span); waiter.finish(Arc::clone(&input)); @@ -413,11 +332,16 @@ impl StateTrieOverlayManager { } } - Ok(Some(input)) + Ok(input) } } } + fn record_overlay_cache_reuse(&self, span: &tracing::Span) { + self.metrics.overlay_cache_reuses.increment(1); + span.record("cache_reused", true); + } + /// Returns `preferred_anchor` if it is on the parent chain, otherwise the first missing parent. /// /// Returns `None` if `parent_hash` is not `preferred_anchor` and the manager does not contain a @@ -426,6 +350,24 @@ impl StateTrieOverlayManager { Self::anchor_for_parent_in(self.blocks.as_ref(), parent_hash, preferred_anchor) } + /// Returns true if `hash` is in the parent chain segment from `anchor_hash` inclusive to + /// `parent_hash` inclusive. + pub fn contains_hash(&self, parent_hash: B256, anchor_hash: B256, hash: B256) -> bool { + let mut current_hash = parent_hash; + + loop { + if current_hash == hash { + return true + } + if current_hash == anchor_hash { + return false + } + + let Some(block) = self.blocks.get(¤t_hash) else { return false }; + current_hash = block.recovered_block().parent_hash(); + } + } + fn anchor_for_parent_in( blocks: &DashMap>, parent_hash: B256, @@ -505,10 +447,6 @@ enum OverlayCacheEntry { } impl OverlayCacheEntry { - const fn is_ready(&self) -> bool { - matches!(self, Self::Ready(_)) - } - fn ready(&self) -> Option> { match self { Self::Ready(input) => Some(Arc::clone(input)), @@ -517,12 +455,6 @@ impl OverlayCacheEntry { } } -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum OverlayLookupMode { - Required, - Precompute, -} - struct OverlayWaiter { input: OnceLock>, } @@ -673,12 +605,10 @@ fn extend_overlay( #[cfg(test)] mod tests { use super::*; - use crate::{test_utils::TestBlockBuilder, EthPrimitives, ExecutedBlock}; + use crate::{test_utils::TestBlockBuilder, EthPrimitives, ExecutedBlock, SparseTrie}; use alloy_primitives::U256; use reth_primitives_traits::Account; use reth_trie::{updates::TrieUpdatesSorted, ComputedTrieData, HashedPostState, HashedStorage}; - #[cfg(feature = "rayon")] - use std::time::Instant; use std::{ sync::{mpsc, Arc}, thread, @@ -787,6 +717,79 @@ mod tests { ); } + #[test] + fn contains_hash_detects_hashes_from_anchor_to_parent() { + let manager = StateTrieOverlayManager::default(); + let blocks = test_blocks(); + for block in &blocks { + manager.insert_block(block.clone()); + } + + let anchor_hash = blocks[0].recovered_block().parent_hash(); + let parent_hash = blocks[2].recovered_block().hash(); + + assert!(manager.contains_hash(parent_hash, anchor_hash, anchor_hash)); + for block in &blocks { + assert!(manager.contains_hash( + parent_hash, + anchor_hash, + block.recovered_block().hash() + )); + } + assert!(!manager.contains_hash(parent_hash, anchor_hash, B256::random())); + } + + #[test] + fn contains_hash_rejects_hash_before_anchor() { + let manager = StateTrieOverlayManager::default(); + let blocks = test_blocks(); + for block in &blocks { + manager.insert_block(block.clone()); + } + + let parent_hash = blocks[2].recovered_block().hash(); + let anchor_hash = blocks[1].recovered_block().hash(); + let before_anchor_hash = blocks[0].recovered_block().hash(); + + assert!(manager.contains_hash(parent_hash, anchor_hash, parent_hash)); + assert!(manager.contains_hash(parent_hash, anchor_hash, anchor_hash)); + assert!(!manager.contains_hash(parent_hash, anchor_hash, before_anchor_hash)); + } + + #[test] + fn contains_hash_rejects_unknown_anchor() { + let manager = StateTrieOverlayManager::default(); + let blocks = test_blocks(); + for block in &blocks { + manager.insert_block(block.clone()); + } + + let parent_hash = blocks[2].recovered_block().hash(); + let anchor_hash = B256::random(); + + assert!(!manager.contains_hash(parent_hash, anchor_hash, anchor_hash)); + } + + #[test] + fn taking_sparse_trie_removes_it() { + let manager = StateTrieOverlayManager::::default(); + let state_root = B256::with_last_byte(1); + let other_state_root = B256::with_last_byte(2); + let anchor_hash = B256::with_last_byte(3); + + manager.store_sparse_trie(PreservedSparseTrie::anchored( + SparseTrie::default(), + state_root, + anchor_hash, + )); + + let preserved = manager.take_sparse_trie().expect("preserved trie should be available"); + assert_eq!(preserved.state_root(), state_root); + assert_eq!(preserved.anchor_hash(), anchor_hash); + assert!(preserved.into_trie_for(other_state_root).unwrap().is_none()); + assert!(manager.take_sparse_trie().is_none()); + } + #[test] fn required_lookup_waits_for_in_progress_overlay() { let manager = StateTrieOverlayManager::::default(); @@ -815,56 +818,6 @@ mod tests { assert!(state.is_empty()); } - #[cfg(feature = "rayon")] - #[test] - fn precompute_skips_in_progress_overlay() { - let manager = StateTrieOverlayManager::::new(Arc::new(WorkerPool::new( - 1, - "test-ovly", - ))); - let key = OverlayCacheKey { - anchor_hash: B256::with_last_byte(1), - tip_hash: B256::with_last_byte(2), - }; - manager.overlays.insert(key, OverlayCacheEntry::Computing(Arc::new(OverlayWaiter::new()))); - - let (tx, rx) = mpsc::channel(); - thread::spawn(move || { - tx.send(manager.precompute_overlay(key.tip_hash, key.anchor_hash)).unwrap(); - }); - - rx.recv_timeout(Duration::from_secs(1)).unwrap().unwrap(); - } - - #[cfg(feature = "rayon")] - #[test] - fn insert_block_prepares_child_overlay_from_cached_parent() { - let manager = StateTrieOverlayManager::new(Arc::new(WorkerPool::new(2, "test-ovly"))); - let blocks = test_blocks(); - - manager.insert_block(blocks[0].clone()); - - let anchor_hash = blocks[0].recovered_block().parent_hash(); - let parent_hash = blocks[0].recovered_block().hash(); - manager.overlay_for_parent(parent_hash, anchor_hash).unwrap(); - - let child_hash = blocks[1].recovered_block().hash(); - manager.insert_block(blocks[1].clone()); - - let child_key = OverlayCacheKey { anchor_hash, tip_hash: child_hash }; - let deadline = Instant::now() + Duration::from_secs(5); - while !manager.overlays.contains_key(&child_key) { - assert!( - Instant::now() < deadline, - "timed out waiting for optimistically prepared child overlay" - ); - thread::sleep(Duration::from_millis(10)); - } - - let (_, state) = manager.overlay_for_parent(child_hash, anchor_hash).unwrap(); - assert_eq!(state.accounts.len(), 2); - } - #[test] fn prunes_cached_overlays_after_removing_blocks() { let manager = StateTrieOverlayManager::default(); diff --git a/crates/chain-state/src/test_utils.rs b/crates/chain-state/src/test_utils.rs index 577bcefad5e..18fb6b5d43f 100644 --- a/crates/chain-state/src/test_utils.rs +++ b/crates/chain-state/src/test_utils.rs @@ -314,7 +314,6 @@ impl TestBlockBuilder { let trie_data = ComputedTrieData { sorted: SortedTrieData { hashed_state: Arc::new(hashed_state), ..Default::default() }, - ..Default::default() }; let block_hash = recovered.hash(); diff --git a/crates/cli/commands/src/config_cmd.rs b/crates/cli/commands/src/config_cmd.rs index e12f468fac9..571c2087139 100644 --- a/crates/cli/commands/src/config_cmd.rs +++ b/crates/cli/commands/src/config_cmd.rs @@ -24,7 +24,9 @@ impl Command { } else { let path = match self.config.as_ref() { Some(path) => path, - None => bail!("No config file provided. Use --config or pass --default"), + None => { + bail!("No config file provided. Use --config or pass --default"); + } }; if !path.exists() { bail!("Config file does not exist: {}", path.display()); diff --git a/crates/cli/commands/src/download/archive.rs b/crates/cli/commands/src/download/archive.rs index 05ae98166cf..f1189486f29 100644 --- a/crates/cli/commands/src/download/archive.rs +++ b/crates/cli/commands/src/download/archive.rs @@ -203,7 +203,7 @@ impl ArchiveProcessor { "Failed integrity validation after {} attempts for {}", MAX_DOWNLOAD_RETRIES, archive.file_name - ) + ); } } } diff --git a/crates/cli/commands/src/download/manifest.rs b/crates/cli/commands/src/download/manifest.rs index 430f1eeb0d1..cc3f3e1adf9 100644 --- a/crates/cli/commands/src/download/manifest.rs +++ b/crates/cli/commands/src/download/manifest.rs @@ -742,7 +742,7 @@ fn state_source_files(source_datadir: &Path) -> Result> { return collect_files_recursive(source_datadir, Path::new("db")); } - eyre::bail!("Could not find source state DB directory under {}", source_datadir.display()) + eyre::bail!("Could not find source state DB directory under {}", source_datadir.display()); } fn rocksdb_source_files(source_datadir: &Path) -> Result> { diff --git a/crates/cli/commands/src/download/manifest_cmd.rs b/crates/cli/commands/src/download/manifest_cmd.rs index ab34abd2f5b..25e2d0398de 100644 --- a/crates/cli/commands/src/download/manifest_cmd.rs +++ b/crates/cli/commands/src/download/manifest_cmd.rs @@ -126,7 +126,7 @@ fn infer_snapshot_block_from_db(source_datadir: &std::path::Path) -> Result eyre::bail!( "Could not infer --block from source DB (Finish checkpoint missing); pass --block manually" - ) + ); } /// Infers the snapshot block from the highest header static-file range. diff --git a/crates/cli/commands/src/download/mod.rs b/crates/cli/commands/src/download/mod.rs index 5cedbe4ad7e..f0b1fd7e52c 100644 --- a/crates/cli/commands/src/download/mod.rs +++ b/crates/cli/commands/src/download/mod.rs @@ -994,6 +994,11 @@ where } impl DownloadCommand { + /// Returns a reference to the environment arguments. + pub const fn env(&self) -> &EnvironmentArgs { + &self.env + } + /// Returns the underlying chain being used to run this command pub fn chain_spec(&self) -> Option<&Arc> { Some(&self.env.chain) diff --git a/crates/cli/commands/src/p2p/bootnode.rs b/crates/cli/commands/src/p2p/bootnode.rs index 03c98855cfe..cacc6f041ed 100644 --- a/crates/cli/commands/src/p2p/bootnode.rs +++ b/crates/cli/commands/src/p2p/bootnode.rs @@ -202,7 +202,9 @@ impl Command { advertised_ips: vec![first_ip, second_ip], }) } - _ => eyre::bail!("--nat can be provided at most twice"), + _ => { + eyre::bail!("--nat can be provided at most twice"); + } } } @@ -283,7 +285,9 @@ async fn bind_socket(addr: SocketAddr) -> eyre::Result> { fn fixed_external_ip(nat: &NatResolver) -> eyre::Result { match nat { NatResolver::ExternalIp(ip) => Ok(*ip), - _ => eyre::bail!("--nat can only be repeated with extip: values"), + _ => { + eyre::bail!("--nat can only be repeated with extip: values"); + } } } diff --git a/crates/cli/commands/src/p2p/mod.rs b/crates/cli/commands/src/p2p/mod.rs index 8f29fb70cb1..d1f1a83bee8 100644 --- a/crates/cli/commands/src/p2p/mod.rs +++ b/crates/cli/commands/src/p2p/mod.rs @@ -76,7 +76,7 @@ impl eyre::bail!( "Invalid number of bodies received. Expected: 1. Received: {}", result.len() - ) + ); } let body = result.into_iter().next().unwrap(); tracing::info!(target: "reth::cli", ?body, "Successfully downloaded body") @@ -185,7 +185,7 @@ impl DownloadArgs { if config.peers.trusted_nodes.is_empty() && self.network.trusted_only { eyre::bail!( "No trusted nodes. Set trusted peer with `--trusted-peer ` or set `--trusted-only` to `false`" - ) + ); } config.peers.trusted_nodes_only |= self.network.trusted_only; diff --git a/crates/cli/commands/src/stage/run.rs b/crates/cli/commands/src/stage/run.rs index c8601a3cf4b..c99946219d7 100644 --- a/crates/cli/commands/src/stage/run.rs +++ b/crates/cli/commands/src/stage/run.rs @@ -30,8 +30,8 @@ use reth_node_metrics::{ }; use reth_primitives_traits::FastInstant as Instant; use reth_provider::{ - ChainSpecProvider, DBProvider, DatabaseProviderFactory, StageCheckpointReader, - StageCheckpointWriter, + providers::BlockchainProvider, ChainSpecProvider, DBProvider, DatabaseProviderFactory, + StageCheckpointReader, StageCheckpointWriter, }; use reth_stages::{ stages::{ @@ -175,7 +175,7 @@ impl default_peers_path, runtime.clone(), ) - .build(provider_factory.clone()) + .build(BlockchainProvider::new(provider_factory.clone())?) .start_network() .await?; let fetch_client = Arc::new(network.fetch_client().await?); @@ -231,7 +231,7 @@ impl default_peers_path, runtime.clone(), ) - .build(provider_factory.clone()) + .build(BlockchainProvider::new(provider_factory.clone())?) .start_network() .await?; let fetch_client = Arc::new(network.fetch_client().await?); diff --git a/crates/cli/commands/src/stage/unwind.rs b/crates/cli/commands/src/stage/unwind.rs index 2fdda790a0d..b3f6f072c80 100644 --- a/crates/cli/commands/src/stage/unwind.rs +++ b/crates/cli/commands/src/stage/unwind.rs @@ -178,7 +178,7 @@ impl Subcommands { if target > last { eyre::bail!( "Target block number {target} is higher than the latest block number {last}" - ) + ); } Ok(target) } diff --git a/crates/engine/local/src/miner.rs b/crates/engine/local/src/miner.rs index ff131b35152..37d88d6a551 100644 --- a/crates/engine/local/src/miner.rs +++ b/crates/engine/local/src/miner.rs @@ -227,7 +227,7 @@ where let res = self.to_engine.fork_choice_updated(state, None).await?; if !res.is_valid() { - eyre::bail!("Invalid fork choice update {state:?}: {res:?}") + eyre::bail!("Invalid fork choice update {state:?}: {res:?}"); } Ok(()) @@ -245,7 +245,7 @@ where .await?; if !res.is_valid() { - eyre::bail!("Invalid payload status") + eyre::bail!("Invalid payload status"); } let payload_id = res.payload_id.ok_or_eyre("No payload id")?; @@ -257,14 +257,14 @@ where let Some(Ok(payload)) = self.payload_builder.resolve_kind(payload_id, PayloadKind::WaitForPending).await else { - eyre::bail!("No payload") + eyre::bail!("No payload"); }; let header = payload.block().sealed_header().clone(); let res = self.to_engine.new_payload(payload.into()).await?; if !res.is_valid() { - eyre::bail!("Invalid payload") + eyre::bail!("Invalid payload"); } self.last_block_hashes.push_back(header.hash()); diff --git a/crates/engine/primitives/Cargo.toml b/crates/engine/primitives/Cargo.toml index dd14824fb43..11394aa8083 100644 --- a/crates/engine/primitives/Cargo.toml +++ b/crates/engine/primitives/Cargo.toml @@ -13,6 +13,7 @@ workspace = true [dependencies] # reth reth-evm = { workspace = true, optional = true } +reth-execution-errors.workspace = true reth-execution-types.workspace = true reth-payload-primitives.workspace = true reth-payload-builder-primitives = { workspace = true, optional = true } @@ -21,6 +22,7 @@ reth-ethereum-primitives.workspace = true reth-chain-state.workspace = true reth-errors.workspace = true reth-trie-common.workspace = true +reth-storage-api.workspace = true # alloy alloy-primitives.workspace = true @@ -43,10 +45,12 @@ trie-debug = [] std = [ "dep:reth-evm", "dep:reth-payload-builder-primitives", + "reth-execution-errors/std", "reth-execution-types/std", "reth-ethereum-primitives/std", "reth-primitives-traits/std", "reth-trie-common/std", + "reth-storage-api/std", "alloy-primitives/std", "alloy-consensus/std", "alloy-rpc-types-engine/std", diff --git a/crates/engine/primitives/src/config.rs b/crates/engine/primitives/src/config.rs index de7aa199d16..2d5e7abc844 100644 --- a/crates/engine/primitives/src/config.rs +++ b/crates/engine/primitives/src/config.rs @@ -541,12 +541,17 @@ impl TreeConfig { self } - /// Whether or not to use the state root task. + /// Returns whether the host has enough parallelism to run the state root task. + pub const fn has_enough_parallelism(&self) -> bool { + self.has_enough_parallelism + } + + /// Returns whether engine validation should use the state root task. /// /// The state root task requires at least 5 parallel threads, see /// [`has_enough_parallelism`]. pub const fn use_state_root_task(&self) -> bool { - self.has_enough_parallelism + !self.skip_state_root && !self.state_root_fallback && self.has_enough_parallelism } /// Setter for state provider metrics. @@ -777,6 +782,20 @@ impl TreeConfig { mod tests { use super::TreeConfig; + #[test] + fn state_root_task_requires_parallelism_without_overrides() { + assert!(TreeConfig::default().with_has_enough_parallelism(true).use_state_root_task()); + assert!(!TreeConfig::default().with_has_enough_parallelism(false).use_state_root_task()); + assert!(!TreeConfig::default() + .with_has_enough_parallelism(true) + .with_state_root_fallback(true) + .use_state_root_task()); + assert!(!TreeConfig::default() + .with_has_enough_parallelism(true) + .with_skip_state_root(true) + .use_state_root_task()); + } + #[test] #[should_panic( expected = "persistence_backpressure_threshold must be greater than persistence_threshold" diff --git a/crates/engine/primitives/src/error.rs b/crates/engine/primitives/src/error.rs index 4eb6d7d247a..def17707fb9 100644 --- a/crates/engine/primitives/src/error.rs +++ b/crates/engine/primitives/src/error.rs @@ -1,5 +1,7 @@ use alloc::boxed::Box; use alloy_rpc_types_engine::ForkchoiceUpdateError; +use reth_errors::{BlockExecutionError, BlockValidationError, ConsensusError, ProviderError}; +use reth_execution_errors::InternalBlockExecutionError; /// Represents all error cases when handling a new payload. /// @@ -45,3 +47,74 @@ impl BeaconForkChoiceUpdateError { Self::Internal(Box::new(e)) } } + +/// All error variants possible when inserting or validating a block. +#[derive(Debug, thiserror::Error)] +pub enum InsertBlockErrorKind { + /// Block violated consensus rules. + #[error(transparent)] + Consensus(#[from] ConsensusError), + /// Block execution failed. + #[error(transparent)] + Execution(#[from] BlockExecutionError), + /// Provider error. + #[error(transparent)] + Provider(#[from] ProviderError), + /// Other errors. + #[error(transparent)] + Other(#[from] Box), +} + +impl InsertBlockErrorKind { + /// Returns whether the error was caused by an invalid block. + pub const fn is_validation_error(&self) -> bool { + matches!(self, Self::Consensus(_) | Self::Execution(BlockExecutionError::Validation(_))) + } + + /// Returns an [`InsertBlockValidationError`] if the error is caused by an invalid block. + /// + /// Returns an [`InsertBlockFatalError`] if the error is caused by an error that is not + /// validation related or is otherwise fatal. + /// + /// This is intended to be used to determine if we should respond `INVALID` as a response when + /// processing a new block. + pub fn ensure_validation_error( + self, + ) -> Result { + match self { + Self::Consensus(err) => Ok(InsertBlockValidationError::Consensus(err)), + Self::Execution(err) => match err { + BlockExecutionError::Validation(err) => { + Ok(InsertBlockValidationError::Validation(err)) + } + BlockExecutionError::Internal(error) => { + Err(InsertBlockFatalError::BlockExecutionError(error)) + } + }, + Self::Provider(err) => Err(InsertBlockFatalError::Provider(err)), + Self::Other(err) => Err(InternalBlockExecutionError::Other(err).into()), + } + } +} + +/// Error variants that are not caused by invalid blocks. +#[derive(Debug, thiserror::Error)] +pub enum InsertBlockFatalError { + /// A provider error. + #[error(transparent)] + Provider(#[from] ProviderError), + /// An internal or fatal block execution error. + #[error(transparent)] + BlockExecutionError(#[from] InternalBlockExecutionError), +} + +/// Error variants that are caused by invalid blocks. +#[derive(Debug, thiserror::Error)] +pub enum InsertBlockValidationError { + /// Block violated consensus rules. + #[error(transparent)] + Consensus(#[from] ConsensusError), + /// Validation error, transparently wrapping [`BlockValidationError`]. + #[error(transparent)] + Validation(#[from] BlockValidationError), +} diff --git a/crates/engine/primitives/src/lib.rs b/crates/engine/primitives/src/lib.rs index 3010a9cb8a8..7855b803871 100644 --- a/crates/engine/primitives/src/lib.rs +++ b/crates/engine/primitives/src/lib.rs @@ -12,12 +12,12 @@ extern crate alloc; use alloy_consensus::BlockHeader; -use reth_errors::ConsensusError; use reth_payload_primitives::{ EngineApiMessageVersion, EngineObjectValidationError, InvalidPayloadAttributesError, NewPayloadError, PayloadAttributes, PayloadOrAttributes, PayloadTypes, }; -use reth_primitives_traits::{Block, RecoveredBlock, SealedBlock}; +use reth_primitives_traits::{Block, RecoveredBlock, SealedBlock, SealedHeader}; +use reth_storage_api::{errors::ProviderResult, StateProviderBox}; use reth_trie_common::HashedPostState; use serde::{de::DeserializeOwned, Serialize}; @@ -213,11 +213,25 @@ pub trait PayloadValidator: Send + Sync + Unpin + 'static { } /// Verifies payload post-execution w.r.t. hashed state updates. + /// + /// `state_updates` lazily yields the block's hashed post-state; call it only if the + /// implementation needs the executed state changes (the L1 default does not). + /// + /// `parent_header` is the parent header the engine resolved for the block. + /// + /// `parent_state` lazily builds the overlay-aware provider for the block's parent that the + /// engine used for execution — resolving even a not-yet-canonical in-memory parent. It is only + /// built if the implementation needs it (the L1 default does not). fn validate_block_post_execution_with_hashed_state<'a>( &self, - _state_updates: &dyn FnOnce() -> &'a HashedPostState, + _state_updates: impl FnOnce() -> &'a HashedPostState, _block: &RecoveredBlock, - ) -> Result<(), ConsensusError> { + _parent_header: &SealedHeader<::Header>, + _parent_state: impl FnOnce() -> ProviderResult, + ) -> Result<(), InsertBlockErrorKind> + where + Self: Sized, + { // method not used by l1 Ok(()) } diff --git a/crates/engine/tree/src/tree/error.rs b/crates/engine/tree/src/tree/error.rs index de04e4338a2..e0b19c9b6c8 100644 --- a/crates/engine/tree/src/tree/error.rs +++ b/crates/engine/tree/src/tree/error.rs @@ -3,8 +3,10 @@ use crate::tree::payload_processor::bal::BalExecutionError; use alloy_consensus::BlockHeader; use reth_consensus::ConsensusError; -use reth_errors::{BlockExecutionError, BlockValidationError, ProviderError}; -use reth_evm::execute::InternalBlockExecutionError; +pub use reth_engine_primitives::{ + InsertBlockErrorKind, InsertBlockFatalError, InsertBlockValidationError, +}; +use reth_errors::ProviderError; use reth_payload_primitives::NewPayloadError; use reth_primitives_traits::{Block, BlockBody, SealedBlock}; @@ -105,23 +107,6 @@ impl std::fmt::Debug for InsertBlockError { } } -/// All error variants possible when inserting a block -#[derive(Debug, thiserror::Error)] -pub enum InsertBlockErrorKind { - /// Block violated consensus rules. - #[error(transparent)] - Consensus(#[from] ConsensusError), - /// Block execution failed. - #[error(transparent)] - Execution(#[from] BlockExecutionError), - /// Provider error. - #[error(transparent)] - Provider(#[from] ProviderError), - /// Other errors. - #[error(transparent)] - Other(#[from] Box), -} - impl From for InsertBlockErrorKind { fn from(e: BalExecutionError) -> Self { match e { @@ -133,59 +118,6 @@ impl From for InsertBlockErrorKind { } } -impl InsertBlockErrorKind { - /// Returns an [`InsertBlockValidationError`] if the error is caused by an invalid block. - /// - /// Returns an [`InsertBlockFatalError`] if the error is caused by an error that is not - /// validation related or is otherwise fatal. - /// - /// This is intended to be used to determine if we should respond `INVALID` as a response when - /// processing a new block. - pub fn ensure_validation_error( - self, - ) -> Result { - match self { - Self::Consensus(err) => Ok(InsertBlockValidationError::Consensus(err)), - // other execution errors that are considered internal errors - Self::Execution(err) => { - match err { - BlockExecutionError::Validation(err) => { - Ok(InsertBlockValidationError::Validation(err)) - } - // these are internal errors, not caused by an invalid block - BlockExecutionError::Internal(error) => { - Err(InsertBlockFatalError::BlockExecutionError(error)) - } - } - } - Self::Provider(err) => Err(InsertBlockFatalError::Provider(err)), - Self::Other(err) => Err(InternalBlockExecutionError::Other(err).into()), - } - } -} - -/// Error variants that are not caused by invalid blocks -#[derive(Debug, thiserror::Error)] -pub enum InsertBlockFatalError { - /// A provider error - #[error(transparent)] - Provider(#[from] ProviderError), - /// An internal / fatal block execution error - #[error(transparent)] - BlockExecutionError(#[from] InternalBlockExecutionError), -} - -/// Error variants that are caused by invalid blocks -#[derive(Debug, thiserror::Error)] -pub enum InsertBlockValidationError { - /// Block violated consensus rules. - #[error(transparent)] - Consensus(#[from] ConsensusError), - /// Validation error, transparently wrapping [`BlockValidationError`] - #[error(transparent)] - Validation(#[from] BlockValidationError), -} - /// Errors that may occur when inserting a payload. #[derive(Debug, thiserror::Error)] pub enum InsertPayloadError { diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index 0ab383ae944..d78cf0838a5 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -37,7 +37,7 @@ use reth_provider::{ use reth_revm::database::StateProviderDatabase; use reth_stages_api::ControlFlow; use reth_tasks::{spawn_os_thread, utils::increase_thread_priority}; -use reth_trie::{prefix_set::TriePrefixSetsMut, ComputedTrieData}; +use reth_trie::ComputedTrieData; use reth_trie_db::ChangesetCache; use revm::interpreter::debug_unreachable; use state::TreeState; @@ -77,7 +77,7 @@ pub use reth_execution_cache::{ CachedStateCacheMetrics, CachedStateMetrics, CachedStateMetricsSource, CachedStateProvider, ExecutionCache, PayloadExecutionCache, SavedCache, }; -pub use types::{ValidationOutcome, ValidationOutput}; +pub use types::{ExecutionEnv, ValidationOutcome, ValidationOutput}; pub mod state; @@ -142,6 +142,8 @@ where pub struct EngineApiTreeState { /// Tracks the state of the blockchain tree. tree_state: TreeState, + /// Whether the next sparse trie task should attempt cache pruning during trie preservation. + pending_sparse_trie_prune: bool, /// Tracks the forkchoice state updates received by the CL. forkchoice_state_tracker: ForkchoiceStateTracker, /// Buffer of detached blocks. @@ -167,6 +169,7 @@ impl EngineApiTreeState { ), buffer: BlockBuffer::new(block_buffer_limit), tree_state: TreeState::new(canonical_block, engine_kind, state_trie_overlays), + pending_sparse_trie_prune: false, forkchoice_state_tracker: ForkchoiceStateTracker::default(), } } @@ -176,6 +179,39 @@ impl EngineApiTreeState { &self.tree_state } + /// Returns whether sparse trie pruning is pending. + pub const fn pending_sparse_trie_prune(&self) -> bool { + self.pending_sparse_trie_prune + } + + /// Sets whether sparse trie pruning is pending for the next sparse trie task. + pub const fn set_pending_sparse_trie_prune(&mut self, pending: bool) { + self.pending_sparse_trie_prune = pending; + } + + /// Takes a pending sparse trie prune request, if any, and snapshots the in-memory parent chain + /// ending at `parent_hash`. + /// + /// `None` means no prune request is pending. `Some(Vec::new())` means a prune was requested, + /// but no in-memory parent-chain blocks were found for the parent hash; the sparse trie task + /// should still prune using the current block's hashed post state. + pub fn take_sparse_trie_prune_blocks( + &mut self, + parent_hash: B256, + ) -> Option>> { + if !self.pending_sparse_trie_prune { + return None + } + + self.pending_sparse_trie_prune = false; + Some( + self.tree_state + .blocks_by_hash(parent_hash) + .map(|(_, blocks)| blocks) + .unwrap_or_default(), + ) + } + /// Returns true if the block has been marked as invalid. pub fn has_invalid_header(&mut self, hash: &B256) -> bool { self.invalid_headers.get(hash).is_some() @@ -317,9 +353,6 @@ where /// Set when an FCU with payload attributes is received, cleared on the next FCU without. /// Suppresses persistence cycles during payload building. building_payload: bool, - /// Retained paths from the latest persistence cleanup to apply during the next sparse trie - /// cache preservation. - pending_sparse_trie_prune: Option, /// Task runtime for spawning blocking work on named, reusable threads. runtime: reth_tasks::Runtime, } @@ -413,7 +446,6 @@ where changeset_cache, execution_timing_stats: B256Map::default(), building_payload: false, - pending_sparse_trie_prune: None, runtime, } } @@ -1198,7 +1230,7 @@ where /// processing is complete. Returns `None` if the head is not canonical and processing /// should continue. fn handle_canonical_head( - &self, + &mut self, state: ForkchoiceState, attrs: &Option, // Changed to reference ) -> ProviderResult>> { @@ -1365,7 +1397,7 @@ where debug!(target: "engine::tree", ?new_tip_num, last_persisted_block_number=?self.persistence_state.last_persisted_block.number, "Removing blocks using persistence task"); if new_tip_num < self.persistence_state.last_persisted_block.number { debug!(target: "engine::tree", ?new_tip_num, "Starting remove blocks job"); - self.pending_sparse_trie_prune = None; + self.state.set_pending_sparse_trie_prune(false); let (tx, rx) = crossbeam_channel::bounded(1); let _ = self.persistence.remove_blocks_above(new_tip_num, tx); self.persistence_state.start_remove(new_tip_num, rx); @@ -1816,7 +1848,7 @@ where if ctrl.is_unwind() { // the node reset so we need to clear everything above that height so that backfill // height is the new canonical block. - self.pending_sparse_trie_prune = None; + self.state.set_pending_sparse_trie_prune(false); self.state.tree_state.reset(backfill_num_hash) } else { self.state.tree_state.remove_until( @@ -2145,35 +2177,13 @@ where number: self.persistence_state.last_persisted_block.number, hash: self.persistence_state.last_persisted_block.hash, }); - self.pending_sparse_trie_prune = self.sparse_trie_retained_paths_for_in_memory_blocks(); + self.state.set_pending_sparse_trie_prune(self.should_prune_sparse_trie()); Ok(()) } - /// Builds sparse trie retained paths from all blocks still present in the in-memory tree. - fn sparse_trie_retained_paths_for_in_memory_blocks(&self) -> Option { - if self.config.skip_state_root() || - self.config.state_root_fallback() || - !self.config.use_state_root_task() - { - return None - } - - let mut retained_paths = TriePrefixSetsMut::default(); - for block in self.state.tree_state.blocks_by_hash.values() { - let trie_data = block.trie_data(); - let Some(changed_paths) = trie_data.changed_paths.as_deref() else { - // Custom state-root strategies may not track changed paths, so this is an - // expected way to opt out of pruning, not an anomaly. - debug!( - target: "engine::tree", - block = ?block.recovered_block().num_hash(), - "Skipping sparse trie prune because changed paths for in-memory block are unknown" - ); - return None - }; - retained_paths.extend_ref(changed_paths); - } - Some(retained_paths) + /// Returns whether sparse trie pruning should be attempted by the next sparse trie task. + const fn should_prune_sparse_trie(&self) -> bool { + self.config.use_state_root_task() } /// Return an [`ExecutedBlock`] from database or in-memory state by hash. @@ -2700,7 +2710,7 @@ where let old_first = old.first().map(|first| first.recovered_block().num_hash()); trace!(target: "engine::tree", ?new_first, ?old_first, "Reorg detected, new and old first blocks"); - self.pending_sparse_trie_prune = None; + self.state.set_pending_sparse_trie_prune(false); self.update_reorg_metrics(old.len(), old_first); self.reinsert_reorged_blocks(new.clone()); self.reinsert_reorged_blocks(old.clone()); @@ -3014,11 +3024,7 @@ where // as this indicates there's already a canonical block at that height. let is_fork = block_id.block.number <= self.state.tree_state.current_canonical_head.number; - let ctx = TreeCtx::new( - &mut self.state, - &self.canonical_in_memory_state, - &mut self.pending_sparse_trie_prune, - ); + let ctx = TreeCtx::new(&mut self.state, &self.canonical_in_memory_state); let start = Instant::now(); @@ -3282,7 +3288,7 @@ where /// Note: At this point, the fork choice update is considered to be VALID, however, we can still /// return an error if the payload attributes are invalid. fn process_payload_attributes( - &self, + &mut self, attributes: T::PayloadAttributes, head: &N::BlockHeader, state: ForkchoiceState, @@ -3309,7 +3315,7 @@ where state.head_block_hash, head, attributes.timestamp(), - &self.state, + &mut self.state, ); // send the payload to the builder and return the receiver for the pending payload diff --git a/crates/engine/tree/src/tree/payload_processor/mod.rs b/crates/engine/tree/src/tree/payload_processor/mod.rs index 0d5a415b567..b5d8942903f 100644 --- a/crates/engine/tree/src/tree/payload_processor/mod.rs +++ b/crates/engine/tree/src/tree/payload_processor/mod.rs @@ -3,55 +3,44 @@ use super::precompile_cache::PrecompileCacheMap; use crate::tree::{ payload_processor::prewarm::{PrewarmCacheTask, PrewarmContext, PrewarmMode, PrewarmTaskEvent}, - sparse_trie::SparseTrieCacheTask, - CacheWaitDurations, CachedStateCacheMetrics, CachedStateMetrics, CachedStateMetricsSource, - ExecutionCache, PayloadExecutionCache, SavedCache, StateProviderBuilder, TreeConfig, - WaitForCaches, + CachedStateCacheMetrics, CachedStateMetrics, CachedStateMetricsSource, ExecutionCache, + ExecutionEnv, PayloadExecutionCache, SavedCache, StateProviderBuilder, TreeConfig, }; -use alloy_eip7928::bal::DecodedBal; -use alloy_eips::{eip1898::BlockWithParent, eip4895::Withdrawal}; +use alloy_eips::eip1898::BlockWithParent; use alloy_primitives::B256; use crossbeam_channel::{Receiver as CrossbeamReceiver, Sender as CrossbeamSender}; -use multiproof::*; use prewarm::PrewarmMetrics; use rayon::prelude::*; -use reth_chain_state::{PreservedSparseTrie, StateTrieOverlayManager}; use reth_evm::{ block::ExecutableTxParts, execute::{ExecutableTxFor, WithTxEnv}, - ConfigureEvm, ConvertTx, EvmEnvFor, ExecutableTxIterator, ExecutableTxTuple, SpecFor, TxEnvFor, + ConfigureEvm, ConvertTx, ExecutableTxIterator, ExecutableTxTuple, SpecFor, TxEnvFor, }; use reth_primitives_traits::{FastInstant as Instant, NodePrimitives}; -use reth_provider::{ - BlockExecutionOutput, BlockReader, DatabaseProviderROFactory, StateProviderFactory, StateReader, -}; +use reth_provider::{BlockExecutionOutput, BlockReader, StateProviderFactory, StateReader}; use reth_revm::db::BundleState; -use reth_tasks::{utils::increase_thread_priority, Runtime}; -use reth_trie::{ - hashed_cursor::HashedCursorFactory, prefix_set::TriePrefixSetsMut, - trie_cursor::TrieCursorFactory, HashedPostState, -}; -use reth_trie_parallel::{ +use reth_tasks::Runtime; +pub use reth_trie_parallel::{ error::StateRootTaskError, - proof_task::{ProofTaskCtx, ProofWorkerHandle}, + state_root_task::{ + evm_state_to_hashed_post_state, PayloadStateRootHandle, StateAccessHint, + StateRootComputeOutcome, StateRootHandle, StateRootHintStream, StateRootMessage, + StateRootSink, StateRootTaskCancelGuard, StateRootUpdateHook, StateRootUpdateStream, + }, }; -use reth_trie_sparse::{ArenaParallelSparseTrie, RevealableSparseTrie, SparseStateTrie}; use std::{ ops::Not, sync::{ atomic::{AtomicBool, AtomicUsize}, - mpsc::{self, channel}, - Arc, OnceLock, + mpsc, Arc, OnceLock, }, }; -use tracing::{debug, debug_span, instrument, trace, warn, Span}; +use tracing::{debug, instrument, trace, warn, Span}; pub mod bal; pub(crate) mod bal_prewarm_pool; -pub mod multiproof; pub mod prewarm; pub mod receipt_root_task; -pub mod sparse_trie; /// Blocks with fewer transactions than this skip prewarming, since the fixed overhead of spawning /// prewarm workers exceeds the execution time saved. @@ -98,8 +87,6 @@ where cache_metrics: Option, /// Metrics for shared execution cache state. cache_state_metrics: Option, - /// Metrics for trie operations - trie_metrics: MultiProofTaskMetrics, /// Cross-block cache size in bytes. cross_block_cache_size: usize, /// Whether transactions should not be executed on prewarming task. @@ -112,14 +99,6 @@ where precompile_cache_disabled: bool, /// Precompile cache map. precompile_cache_map: PrecompileCacheMap>, - /// State trie overlay manager that owns the preserved sparse trie. - state_trie_overlays: StateTrieOverlayManager, - /// LFU hot-slot capacity: max storage slots retained across prune cycles. - sparse_trie_max_hot_slots: usize, - /// LFU hot-account capacity: max account addresses retained across prune cycles. - sparse_trie_max_hot_accounts: usize, - /// Whether sparse trie cache pruning is fully disabled. - disable_sparse_trie_cache_pruning: bool, /// Whether to disable BAL-driven parallel state root computation. /// Only valid when BAL parallel execution is also disabled. disable_bal_parallel_state_root: bool, @@ -130,43 +109,26 @@ where bal_prewarm_pool: OnceLock>, } -struct SparseTrieTaskOptions { - parent_state_root: B256, - chunk_size: usize, - pending_sparse_trie_prune: Option, -} - impl PayloadProcessor where Evm: ConfigureEvm, { - /// Returns a reference to the workload executor driving payload tasks. - pub const fn executor(&self) -> &Runtime { - &self.executor - } - /// Creates a new payload processor. pub fn new( executor: Runtime, evm_config: Evm, config: &TreeConfig, precompile_cache_map: PrecompileCacheMap>, - state_trie_overlays: StateTrieOverlayManager, ) -> Self { Self { executor, execution_cache: Default::default(), - trie_metrics: Default::default(), cross_block_cache_size: config.cross_block_cache_size(), disable_transaction_prewarming: config.disable_prewarming(), evm_config, disable_state_cache: config.disable_state_cache(), precompile_cache_disabled: config.precompile_cache_disabled(), precompile_cache_map, - state_trie_overlays, - sparse_trie_max_hot_slots: config.sparse_trie_max_hot_slots(), - sparse_trie_max_hot_accounts: config.sparse_trie_max_hot_accounts(), - disable_sparse_trie_cache_pruning: config.disable_sparse_trie_cache_pruning(), cache_metrics: (!config.disable_cache_metrics()) .then(|| CachedStateMetrics::zeroed(CachedStateMetricsSource::Engine)), cache_state_metrics: (!config.disable_cache_metrics()) @@ -186,43 +148,10 @@ where }) .clone() } -} - -impl WaitForCaches for PayloadProcessor -where - Evm: ConfigureEvm, -{ - fn wait_for_caches(&self) -> CacheWaitDurations { - debug!(target: "engine::tree::payload_processor", "Waiting for execution cache and sparse trie locks"); - - let execution_cache = self.execution_cache.clone(); - let state_trie_overlays = self.state_trie_overlays.clone(); - - let (execution_tx, execution_rx) = std::sync::mpsc::channel(); - let (sparse_trie_tx, sparse_trie_rx) = std::sync::mpsc::channel(); - self.executor.spawn_blocking_named("wait-exec-cache", move || { - let _ = execution_tx.send(execution_cache.wait_for_availability()); - }); - self.executor.spawn_blocking_named("wait-sparse-tri", move || { - let _ = sparse_trie_tx.send(state_trie_overlays.wait_for_sparse_trie_availability()); - }); - - let execution_cache_duration = - execution_rx.recv().expect("execution cache wait task failed to send result"); - let sparse_trie_duration = - sparse_trie_rx.recv().expect("sparse trie wait task failed to send result"); - - debug!( - target: "engine::tree::payload_processor", - ?execution_cache_duration, - ?sparse_trie_duration, - "Execution cache and sparse trie locks acquired" - ); - CacheWaitDurations { - execution_cache: execution_cache_duration, - sparse_trie: sparse_trie_duration, - } + /// Returns the shared execution cache handle used for engine backpressure. + pub(crate) fn execution_cache(&self) -> PayloadExecutionCache { + self.execution_cache.clone() } } @@ -238,7 +167,8 @@ where env: ExecutionEnv, transactions: I, provider_builder: StateProviderBuilder, - state_root_streams: StateRootStreams, + hint_stream: Option, + hashed_update_stream: Option, parallel_bal_execution: bool, ) -> IteratorPayloadHandle where @@ -250,78 +180,13 @@ where env, prewarm_rx, provider_builder, - state_root_streams, + hint_stream, + hashed_update_stream, parallel_bal_execution, ); PayloadHandle { prewarm_handle, transactions: execution_rx, _span: Span::current() } } - /// Spawns state root computation pipeline (multiproof + sparse trie tasks). - /// - /// The returned [`StateRootHandle`] provides: - /// - [`StateRootHandle::streams`] — semantic stream views that feed updates into the pipeline, - /// including an execution hook for per-transaction state updates. - /// - [`StateRootHandle::state_root`] — blocks until the state root is computed and returns the - /// state root. - /// - /// The execution hook **must** be dropped after execution to signal the end of state - /// updates. - /// - /// `transaction_count` is the number of transactions in the block when it is known up - /// front. Small blocks halve the proof worker pool, since fewer transactions produce fewer - /// state changes and most workers would be idle. Passing `None` (for example when building - /// a payload, where the transaction count is unknown) uses the full worker pool. - #[instrument(level = "debug", target = "engine::tree::payload_processor", skip_all)] - pub fn spawn_state_root( - &self, - multiproof_provider_factory: F, - parent_state_root: B256, - transaction_count: Option, - config: &TreeConfig, - pending_sparse_trie_prune: Option, - ) -> StateRootHandle - where - F: DatabaseProviderROFactory - + Clone - + Send - + Sync - + 'static, - { - let (updates_tx, from_multi_proof) = crossbeam_channel::unbounded(); - - let task_ctx = ProofTaskCtx::new(multiproof_provider_factory); - #[cfg(feature = "trie-debug")] - let task_ctx = task_ctx.with_proof_jitter(config.proof_jitter()); - let halve_workers = transaction_count - .is_some_and(|count| count <= Self::SMALL_BLOCK_PROOF_WORKER_TX_THRESHOLD); - let proof_handle = ProofWorkerHandle::new(&self.executor, task_ctx, halve_workers); - - let (state_root_tx, state_root_rx) = channel(); - let (hashed_state_tx, hashed_state_rx) = channel(); - - self.spawn_sparse_trie_task( - proof_handle, - state_root_tx, - hashed_state_tx, - from_multi_proof, - SparseTrieTaskOptions { - parent_state_root, - chunk_size: config.multiproof_chunk_size(), - pending_sparse_trie_prune: if self.disable_sparse_trie_cache_pruning { - None - } else { - pending_sparse_trie_prune - }, - }, - ); - - StateRootHandle::new(parent_state_root, updates_tx, state_root_rx, hashed_state_rx) - } - - /// Transaction count threshold below which proof workers are halved, since fewer transactions - /// produce fewer state changes and most workers would be idle overhead. - const SMALL_BLOCK_PROOF_WORKER_TX_THRESHOLD: usize = 30; - /// Transaction count threshold below which sequential conversion is used. /// /// For blocks with fewer than this many transactions, the rayon parallel iterator overhead @@ -466,22 +331,26 @@ where env: ExecutionEnv, transactions: mpsc::Receiver<(usize, impl ExecutableTxFor + Clone + Send + 'static)>, provider_builder: StateProviderBuilder, - state_root_streams: StateRootStreams, + hint_stream: Option, + hashed_update_stream: Option, parallel_bal_execution: bool, ) -> CacheTaskHandle<::Receipt> where P: BlockReader + StateProviderFactory + StateReader + Clone + 'static, { + // Each mode carries the capability its producers use; the rest is dropped here, so + // unused capabilities do not keep the state-root task's update channel open. let mode = if parallel_bal_execution { - PrewarmMode::BlockAccessList( - env.decoded_bal.clone().expect("BAL dispatch implies decoded BAL"), - ) + PrewarmMode::BlockAccessList { + bal: env.decoded_bal.clone().expect("BAL dispatch implies decoded BAL"), + updates: hashed_update_stream, + } } else if self.disable_transaction_prewarming || env.transaction_count < SMALL_BLOCK_TX_THRESHOLD { PrewarmMode::Skipped } else { - PrewarmMode::Transactions(transactions) + PrewarmMode::Transactions { pending: transactions, hints: hint_stream } }; let saved_cache = self.disable_state_cache.not().then(|| self.cache_for(env.parent_hash)); @@ -504,12 +373,8 @@ where disable_bal_batch_io: self.disable_bal_batch_io, }; - let (prewarm_task, to_prewarm_task) = PrewarmCacheTask::new( - self.executor.clone(), - self.execution_cache.clone(), - prewarm_ctx, - state_root_streams, - ); + let (prewarm_task, to_prewarm_task) = + PrewarmCacheTask::new(self.executor.clone(), self.execution_cache.clone(), prewarm_ctx); { let to_prewarm_task = to_prewarm_task.clone(); self.executor.spawn_blocking_named("prewarm", move || { @@ -545,136 +410,6 @@ where } } - /// Spawns the [`SparseTrieCacheTask`] for this payload processor. - /// - /// The trie is preserved when the new payload is a child of the previous one. - fn spawn_sparse_trie_task( - &self, - proof_worker_handle: ProofWorkerHandle, - state_root_tx: mpsc::Sender>, - hashed_state_tx: mpsc::Sender, - from_multi_proof: CrossbeamReceiver, - options: SparseTrieTaskOptions, - ) { - let SparseTrieTaskOptions { parent_state_root, chunk_size, pending_sparse_trie_prune } = - options; - let state_trie_overlays = self.state_trie_overlays.clone(); - let trie_metrics = self.trie_metrics.clone(); - let max_hot_slots = self.sparse_trie_max_hot_slots; - let max_hot_accounts = self.sparse_trie_max_hot_accounts; - let executor = self.executor.clone(); - - let parent_span = Span::current(); - self.executor.spawn_blocking_named("sparse-trie", move || { - reth_tasks::once!(increase_thread_priority); - - let _enter = debug_span!(target: "engine::tree::payload_processor", parent: parent_span, "sparse_trie_task") - .entered(); - - // Reuse a stored SparseStateTrie if available, applying continuation logic. - // If this payload's parent state root matches the preserved trie's anchor, - // we can reuse the preserved trie structure. Otherwise, we clear the trie but - // keep allocations. - let start = Instant::now(); - let preserved = state_trie_overlays.take_sparse_trie(); - trie_metrics - .sparse_trie_cache_wait_duration_histogram - .record(start.elapsed().as_secs_f64()); - - let mut sparse_state_trie = preserved - .map(|preserved| preserved.into_trie_for(parent_state_root)) - .unwrap_or_else(|| { - debug!( - target: "engine::tree::payload_processor", - "Creating new sparse trie - no preserved trie available" - ); - let default_trie = - RevealableSparseTrie::blind_from(ArenaParallelSparseTrie::default()); - SparseStateTrie::default() - .with_accounts_trie(default_trie.clone()) - .with_default_storage_trie(default_trie) - .with_updates(true) - }); - sparse_state_trie.set_changed_paths(true); - sparse_state_trie.set_hot_cache_capacities(max_hot_slots, max_hot_accounts); - - let mut task = SparseTrieCacheTask::new_with_trie( - &executor, - from_multi_proof, - hashed_state_tx, - proof_worker_handle, - trie_metrics.clone(), - sparse_state_trie, - parent_state_root, - chunk_size, - ); - - let result = task.run(); - - // Acquire the guard before sending the result to prevent a race condition: - // Without this, the next block could start after send() but before store(), - // causing take_sparse_trie() to return None and forcing it to create a new empty trie - // instead of reusing the preserved one. Holding the guard ensures the next - // block's take_sparse_trie() blocks until we've stored the trie for reuse. - let mut guard = state_trie_overlays.lock_sparse_trie(); - - let task_result = result.as_ref().ok().cloned(); - // Send state root computation result - next block may start but will block on - // take_sparse_trie(). - if state_root_tx.send(result).is_err() { - // Receiver dropped - payload was likely invalid or cancelled. - // Clear the trie instead of preserving potentially invalid state. - debug!( - target: "engine::tree::payload_processor", - "State root receiver dropped, clearing trie" - ); - let (trie, deferred) = task.into_cleared_trie(); - guard.store(PreservedSparseTrie::cleared(trie)); - drop(guard); - executor.spawn_drop(deferred); - return; - } - - // Only preserve the trie as anchored if computation succeeded. - // A failed computation may have left the trie in a partially updated state. - let _enter = - debug_span!(target: "engine::tree::payload_processor", "preserve").entered(); - let deferred = if let Some(result) = task_result { - let start = Instant::now(); - let (mut trie, deferred) = task.into_trie_for_reuse(); - if let Some(mut retained_paths) = pending_sparse_trie_prune { - let changed_paths = result - .changed_paths - .as_deref() - .expect("sparse trie task always returns changed paths"); - retained_paths.extend_ref(changed_paths); - trie.prune(max_hot_slots, max_hot_accounts, retained_paths); - } - trie_metrics - .into_trie_for_reuse_duration_histogram - .record(start.elapsed().as_secs_f64()); - trie_metrics - .sparse_trie_retained_memory_bytes - .set(trie.memory_size() as f64); - trie_metrics - .sparse_trie_retained_storage_tries - .set(trie.retained_storage_tries_count() as f64); - guard.store(PreservedSparseTrie::anchored(trie, result.state_root)); - deferred - } else { - debug!( - target: "engine::tree::payload_processor", - "State root computation failed, clearing trie" - ); - let (trie, deferred) = task.into_cleared_trie(); - guard.store(PreservedSparseTrie::cleared(trie)); - deferred - }; - drop(guard); - executor.spawn_drop(deferred); - }); - } - /// Updates the execution cache with the post-execution state from an inserted block. /// /// This is used when blocks are inserted directly (e.g., locally built blocks by sequencers) @@ -876,82 +611,20 @@ impl Drop for CacheTaskHandle { } } -/// EVM context required to execute a block. -#[derive(Debug, Clone)] -pub struct ExecutionEnv { - /// Evm environment. - pub evm_env: EvmEnvFor, - /// Hash of the block being executed. - pub hash: B256, - /// Hash of the parent block. - pub parent_hash: B256, - /// State root of the parent block. - /// Used for sparse trie continuation: if the preserved trie's anchor matches this, - /// the trie can be reused directly. - pub parent_state_root: B256, - /// Number of transactions in the block. - /// Used to determine parallel worker count for prewarming. - pub transaction_count: usize, - /// Total gas used by all transactions in the block. - /// Used to adaptively select multiproof chunk size for optimal throughput. - pub gas_used: u64, - /// Withdrawals included in the block. - /// Used to generate prefetch targets for withdrawal addresses. - pub withdrawals: Option>, - /// Optional decoded BAL for the block. - /// Used to validate and optimize execution. - pub decoded_bal: Option>, -} - -impl ExecutionEnv -where - EvmEnvFor: Default, -{ - /// Creates a new [`ExecutionEnv`] with default values for testing. - #[cfg(any(test, feature = "test-utils"))] - pub fn test_default() -> Self { - Self { - evm_env: Default::default(), - hash: Default::default(), - parent_hash: Default::default(), - parent_state_root: Default::default(), - transaction_count: 0, - gas_used: 0, - withdrawals: None, - decoded_bal: None, - } - } -} - #[cfg(test)] mod tests { use crate::tree::{ - payload_processor::{evm_state_to_hashed_post_state, ExecutionEnv, PayloadProcessor}, - precompile_cache::PrecompileCacheMap, - ExecutionCache, PayloadExecutionCache, SavedCache, TreeConfig, + payload_processor::PayloadProcessor, precompile_cache::PrecompileCacheMap, ExecutionCache, + PayloadExecutionCache, SavedCache, TreeConfig, }; use alloy_consensus::constants::KECCAK_EMPTY; use alloy_eips::eip1898::{BlockNumHash, BlockWithParent}; - use alloy_primitives::{map::HashMap, Address, B256, U256}; - use rand::Rng; - use reth_chain_state::StateTrieOverlayManager; + use alloy_primitives::{Address, B256, U256}; use reth_chainspec::ChainSpec; - use reth_db_common::init::init_genesis; - use reth_ethereum_primitives::EthPrimitives; - use reth_evm::OnStateHook; use reth_evm_ethereum::EthEvmConfig; use reth_execution_cache::CachedStatus; - use reth_primitives_traits::{Account, StorageEntry}; - use reth_provider::{ - providers::{BlockchainProvider, OverlayBuilder, OverlayStateProviderFactory}, - test_utils::create_test_provider_factory_with_chain_spec, - ChainSpecProvider, HashingWriter, - }; use reth_revm::db::BundleState; - use reth_testing_utils::generators; - use reth_trie::{test_utils::state_root, HashedPostState}; - use reth_trie_db::ChangesetCache; - use revm::state::{AccountInfo, AccountStatus, EvmState, EvmStorageSlot, TransactionId}; + use revm::state::AccountInfo; use std::sync::Arc; fn make_saved_cache(hash: B256) -> SavedCache { @@ -959,10 +632,6 @@ mod tests { SavedCache::new(hash, execution_cache) } - fn state_trie_overlays() -> StateTrieOverlayManager { - StateTrieOverlayManager::default() - } - #[test] fn execution_cache_allows_single_checkout() { let execution_cache = PayloadExecutionCache::default(); @@ -1046,7 +715,6 @@ mod tests { EthEvmConfig::new(Arc::new(ChainSpec::default())), &TreeConfig::default(), PrecompileCacheMap::default(), - state_trie_overlays(), ); let parent_hash = B256::from([1u8; 32]); @@ -1076,7 +744,6 @@ mod tests { EthEvmConfig::new(Arc::new(ChainSpec::default())), &TreeConfig::default(), PrecompileCacheMap::default(), - state_trie_overlays(), ); // Setup: populate cache with block 1 @@ -1112,7 +779,6 @@ mod tests { EthEvmConfig::new(Arc::new(ChainSpec::default())), &TreeConfig::default(), PrecompileCacheMap::default(), - state_trie_overlays(), ); let parent_hash = B256::from([1u8; 32]); @@ -1165,146 +831,6 @@ mod tests { ); } - fn create_mock_state_updates(num_accounts: usize, updates_per_account: usize) -> Vec { - let mut rng = generators::rng(); - let all_addresses: Vec
= (0..num_accounts).map(|_| rng.random()).collect(); - let mut updates = Vec::with_capacity(updates_per_account); - - for _ in 0..updates_per_account { - let num_accounts_in_update = rng.random_range(1..=num_accounts); - let mut state_update = EvmState::default(); - - let selected_addresses = &all_addresses[0..num_accounts_in_update]; - - for &address in selected_addresses { - let mut storage = HashMap::default(); - if rng.random_bool(0.7) { - for _ in 0..rng.random_range(1..10) { - let slot = U256::from(rng.random::()); - storage.insert( - slot, - EvmStorageSlot::new_changed( - U256::ZERO, - U256::from(rng.random::()), - TransactionId::ZERO, - ), - ); - } - } - - let mut account = revm::state::Account::default(); - account.info = AccountInfo { - balance: U256::from(rng.random::()), - nonce: rng.random::(), - code_hash: KECCAK_EMPTY, - code: Some(Default::default()), - account_id: None, - }; - account.storage = storage; - account.status = AccountStatus::Touched; - account.transaction_id = TransactionId::ZERO; - - state_update.insert(address, account); - } - - updates.push(state_update); - } - - updates - } - - #[test] - fn test_state_root() { - reth_tracing::init_test_tracing(); - - let factory = create_test_provider_factory_with_chain_spec(Arc::new(ChainSpec::default())); - let genesis_hash = init_genesis(&factory).unwrap(); - - let state_updates = create_mock_state_updates(10, 10); - let mut hashed_state = HashedPostState::default(); - let mut accumulated_state: HashMap)> = - HashMap::default(); - - { - let provider_rw = factory.provider_rw().expect("failed to get provider"); - - for update in &state_updates { - let account_updates = update.iter().map(|(address, account)| { - (*address, Some(Account::from_revm_account(account))) - }); - provider_rw - .insert_account_for_hashing(account_updates) - .expect("failed to insert accounts"); - - let storage_updates = update.iter().map(|(address, account)| { - let storage_entries = account.storage.iter().map(|(slot, value)| { - StorageEntry { key: B256::from(*slot), value: value.present_value } - }); - (*address, storage_entries) - }); - provider_rw - .insert_storage_for_hashing(storage_updates) - .expect("failed to insert storage"); - } - provider_rw.commit().expect("failed to commit changes"); - } - - for update in &state_updates { - hashed_state.extend(evm_state_to_hashed_post_state(update.clone())); - - for (address, account) in update { - let storage: HashMap = account - .storage - .iter() - .map(|(k, v)| (B256::from(*k), v.present_value)) - .collect(); - - let entry = accumulated_state.entry(*address).or_default(); - entry.0 = Account::from_revm_account(account); - entry.1.extend(storage); - } - } - - let payload_processor = PayloadProcessor::new( - reth_tasks::Runtime::test(), - EthEvmConfig::new(factory.chain_spec()), - &TreeConfig::default(), - PrecompileCacheMap::default(), - state_trie_overlays(), - ); - - let provider_factory = BlockchainProvider::new(factory).unwrap(); - - let env: ExecutionEnv = ExecutionEnv::test_default(); - let mut state_root_handle = payload_processor.spawn_state_root( - OverlayStateProviderFactory::new( - provider_factory, - OverlayBuilder::::new(genesis_hash, ChangesetCache::new()), - ), - env.parent_state_root, - Some(env.transaction_count), - &TreeConfig::default(), - None, - ); - - let mut streams = state_root_handle.streams(true); - let mut state_hook = - streams.take_execution_stream().expect("execution stream installed").state_hook(); - - for update in state_updates { - state_hook.on_state(update); - } - drop(state_hook); - - let root_from_task = state_root_handle.state_root().expect("task failed").state_root; - let root_from_regular = state_root(accumulated_state); - - assert_eq!( - root_from_task, root_from_regular, - "State root mismatch: task={root_from_task}, base={root_from_regular}" - ); - } - /// Tests the full prewarm lifecycle for a fork block: /// /// 1. Cache is at canonical block 4. diff --git a/crates/engine/tree/src/tree/payload_processor/multiproof.rs b/crates/engine/tree/src/tree/payload_processor/multiproof.rs deleted file mode 100644 index cd75f993fbf..00000000000 --- a/crates/engine/tree/src/tree/payload_processor/multiproof.rs +++ /dev/null @@ -1,88 +0,0 @@ -//! Multiproof task related functionality. - -use metrics::{Gauge, Histogram}; -use reth_metrics::Metrics; - -pub use reth_trie_parallel::{ - error::StateRootTaskError, - state_root_task::{ - evm_state_to_hashed_post_state, PayloadStateRootHandle, StateAccessHint, - StateRootComputeOutcome, StateRootHandle, StateRootHashedUpdateStream, StateRootHintStream, - StateRootMessage, StateRootSink, StateRootStreams, StateRootUpdateHook, - }, -}; - -/// The default max targets, for limiting the number of account and storage proof targets to be -/// fetched by a single worker. If exceeded, chunking is forced regardless of worker availability. -pub(crate) const DEFAULT_MAX_TARGETS_FOR_CHUNKING: usize = 300; - -#[derive(Metrics, Clone)] -#[metrics(scope = "tree.root")] -pub(crate) struct MultiProofTaskMetrics { - /// Histogram of durations spent revealing multiproof results into the sparse trie. - pub sparse_trie_reveal_multiproof_duration_histogram: Histogram, - /// Histogram of durations spent coalescing multiple proof results from the channel. - pub sparse_trie_proof_coalesce_duration_histogram: Histogram, - /// Histogram of durations the event loop spent blocked waiting on channels. - pub sparse_trie_channel_wait_duration_histogram: Histogram, - /// Histogram of durations spent processing trie updates and promoting pending accounts. - pub sparse_trie_process_updates_duration_histogram: Histogram, - /// Histogram of sparse trie final update durations. - pub sparse_trie_final_update_duration_histogram: Histogram, - /// Histogram of sparse trie total durations. - pub sparse_trie_total_duration_histogram: Histogram, - /// Time spent preparing the sparse trie for reuse after state root computation. - pub into_trie_for_reuse_duration_histogram: Histogram, - /// Time spent waiting for preserved sparse trie cache to become available. - pub sparse_trie_cache_wait_duration_histogram: Histogram, - /// Histogram for sparse trie task idle time in seconds (waiting for updates or proof - /// results). Excludes the final wait after the channel is closed. - pub sparse_trie_idle_time_seconds: Histogram, - /// Histogram for hashing task idle time in seconds (waiting for messages from execution). - /// Excludes the final wait after the channel is closed. - pub hashing_task_idle_time_seconds: Histogram, - - /// Number of account leaf updates applied without needing a new proof (cache hits). - pub sparse_trie_account_cache_hits: Histogram, - /// Number of account leaf updates that required a new proof (cache misses). - pub sparse_trie_account_cache_misses: Histogram, - /// Number of storage leaf updates applied without needing a new proof (cache hits). - pub sparse_trie_storage_cache_hits: Histogram, - /// Number of storage leaf updates that required a new proof (cache misses). - pub sparse_trie_storage_cache_misses: Histogram, - - /// Retained memory of the preserved sparse trie cache in bytes. - pub sparse_trie_retained_memory_bytes: Gauge, - /// Number of storage tries retained in the preserved sparse trie cache. - pub sparse_trie_retained_storage_tries: Gauge, -} - -/// Dispatches work items as a single unit or in chunks based on target size and worker -/// availability. -#[expect(clippy::too_many_arguments)] -pub(crate) fn dispatch_with_chunking( - items: T, - chunking_len: usize, - chunk_size: usize, - max_targets_for_chunking: usize, - has_multiple_idle_account_workers: bool, - has_multiple_idle_storage_workers: bool, - chunker: impl FnOnce(T, usize) -> I, - mut dispatch: impl FnMut(T), -) where - I: IntoIterator, -{ - let has_full_chunks = chunking_len >= chunk_size.saturating_mul(2); - let should_chunk = chunking_len > max_targets_for_chunking || - (has_full_chunks && - (has_multiple_idle_account_workers || has_multiple_idle_storage_workers)); - - if should_chunk && chunking_len > chunk_size { - for chunk in chunker(items, chunk_size) { - dispatch(chunk); - } - return; - } - - dispatch(items); -} diff --git a/crates/engine/tree/src/tree/payload_processor/prewarm.rs b/crates/engine/tree/src/tree/payload_processor/prewarm.rs index 2a206fe7ba3..8feda8e1bec 100644 --- a/crates/engine/tree/src/tree/payload_processor/prewarm.rs +++ b/crates/engine/tree/src/tree/payload_processor/prewarm.rs @@ -11,11 +11,8 @@ //! 2. Prewarming tasks execute transactions in parallel using shared caches //! 3. When actual block execution happens, it benefits from the warmed cache -use super::bal_prewarm_pool::BalPrewarmPool; +use super::{bal_prewarm_pool::BalPrewarmPool, StateRootHintStream, StateRootUpdateStream}; use crate::tree::{ - payload_processor::multiproof::{ - StateRootHashedUpdateStream, StateRootHintStream, StateRootStreams, - }, precompile_cache::{CachedPrecompile, PrecompileCacheMap}, CachedStateCacheMetrics, CachedStateMetrics, CachedStateProvider, ExecutionEnv, PayloadExecutionCache, SavedCache, StateProviderBuilder, @@ -44,12 +41,25 @@ use tokio::sync::oneshot; use tracing::{debug, debug_span, instrument, trace, trace_span, warn, Span}; /// Determines the prewarming mode: transaction-based, BAL-based, or skipped. +/// +/// Each variant carries the state-root capability its producers use, so the capability dies +/// with the workers instead of outliving them. #[derive(Debug)] pub enum PrewarmMode { /// Prewarm by executing transactions from a stream, each paired with its block index. - Transactions(Receiver<(usize, Tx)>), + Transactions { + /// Stream of transactions pending prewarm execution. + pending: Receiver<(usize, Tx)>, + /// Best-effort access hints emitted by the prewarm workers. + hints: Option, + }, /// Prewarm by prefetching slots from a Block Access List. - BlockAccessList(Arc), + BlockAccessList { + /// The decoded block access list. + bal: Arc, + /// Authoritative pre-hashed updates derived from the BAL. + updates: Option, + }, /// Transaction prewarming is skipped (e.g. small blocks where the overhead exceeds the /// benefit). No workers are spawned. Skipped, @@ -71,8 +81,6 @@ where execution_cache: PayloadExecutionCache, /// Context provided to execution tasks ctx: PrewarmContext, - /// State-root streams used for prewarm hints and BAL-derived authoritative updates. - state_root_streams: StateRootStreams, /// Receiver for events produced by tx execution actions_rx: Receiver>, /// Parent span for tracing @@ -90,7 +98,6 @@ where executor: Runtime, execution_cache: PayloadExecutionCache, ctx: PrewarmContext, - state_root_streams: StateRootStreams, ) -> (Self, Sender>) { let (actions_tx, actions_rx) = channel(); @@ -102,14 +109,7 @@ where ); ( - Self { - executor, - execution_cache, - ctx, - state_root_streams, - actions_rx, - parent_span: Span::current(), - }, + Self { executor, execution_cache, ctx, actions_rx, parent_span: Span::current() }, actions_tx, ) } @@ -335,11 +335,12 @@ where &self, decoded_bal: Arc, actions_tx: Sender>, + hashed_update_stream: Option, ) { let bal = decoded_bal.as_bal(); if bal.is_empty() { - if let Some(hashed_update_stream) = self.state_root_streams.hashed_update_stream() { - hashed_update_stream.on_updates_finished(); + if let Some(hashed_update_stream) = hashed_update_stream { + hashed_update_stream.finish(); } let _ = actions_tx.send(PrewarmTaskEvent::FinishedTxExecution { executed_transactions: 0 }); @@ -353,7 +354,6 @@ where ); let ctx = self.ctx.clone(); - let hashed_update_stream = self.state_root_streams.hashed_update_stream(); let executor = self.executor.clone(); let parent_span = Span::current(); let stream_parent_span = parent_span; @@ -386,7 +386,7 @@ where }); }); - hashed_update_stream.on_updates_finished(); + hashed_update_stream.finish(); let _ = stream_tx.send(()); }); } else { @@ -452,13 +452,15 @@ where where Tx: ExecutableTxFor + Send + 'static, { - // Spawn execution tasks based on mode + // Spawn execution tasks based on mode. The state-root capabilities arrive inside the + // mode and move into the spawned producers, so they die with the producers instead of + // living for the full lifetime of this task. match mode { - PrewarmMode::Transactions(pending) => { - self.spawn_txs_prewarm(pending, actions_tx, self.state_root_streams.hint_stream()); + PrewarmMode::Transactions { pending, hints } => { + self.spawn_txs_prewarm(pending, actions_tx, hints); } - PrewarmMode::BlockAccessList(bal) => { - self.run_bal_prewarm(bal, actions_tx); + PrewarmMode::BlockAccessList { bal, updates } => { + self.run_bal_prewarm(bal, actions_tx, updates); } PrewarmMode::Skipped => { let _ = actions_tx @@ -640,7 +642,7 @@ where parent_span: &Span, provider: &mut Option>, account_changes: &alloy_eip7928::AccountChanges, - hashed_update_stream: &StateRootHashedUpdateStream, + hashed_update_stream: &StateRootUpdateStream, ) { if self.disable_bal_parallel_state_root { return; @@ -653,6 +655,9 @@ where return; } + // If there are any storage changes we can assume that the resulting account info will be + // non-empty, so the account will exist, and therefore we can pre-emptively send out storage + // changes to start processing them before potentially hitting the db in the next step. if !account_changes.storage_changes.is_empty() { let hashed_address = *hashed_address.get_or_insert_with(|| keccak256(address)); let mut storage_map = reth_trie::HashedStorage::new(false); @@ -707,11 +712,23 @@ where }; let account = account_fields.into_account(existing_account); - let hashed_address = hashed_address.unwrap_or_else(|| keccak256(address)); - let mut hashed_state = reth_trie::HashedPostState::default(); - hashed_state.accounts.insert(hashed_address, Some(account)); + // It is possible for the resulting account info to be empty. This can happen when, in the + // same block: + // * tx1: A new account is funded + // * tx2: CREATE2 is called on the new account, SELFDESTRUCT is called within the init code + // + // In this case the account will have only balance_changes, one for funding and the second + // setting balance back to zero. The resulting account is fully empty, we mark it as None + // with no storage changes to indicate that it should be deleted if nothing else. + // + // We assume that if the account info is all zero then it can't have storage, so we don't + // have to explicitly check for empty storage. + let account = (!account.is_empty()).then_some(account); + + let mut hashed_state = reth_trie::HashedPostState::default(); + hashed_state.accounts.insert(hashed_address, account); hashed_update_stream.on_hashed_state_update(hashed_state); } } diff --git a/crates/engine/tree/src/tree/payload_validator.rs b/crates/engine/tree/src/tree/payload_validator.rs index 7c183ec0c86..c2c3db449a6 100644 --- a/crates/engine/tree/src/tree/payload_validator.rs +++ b/crates/engine/tree/src/tree/payload_validator.rs @@ -98,7 +98,6 @@ use crate::tree::{ error::{InsertBlockError, InsertBlockErrorKind, InsertPayloadError}, instrumented_state::{InstrumentedStateProvider, StateProviderMetrics, StateProviderStats}, - multiproof::{PayloadStateRootHandle, StateRootStreams}, payload_processor::PayloadProcessor, precompile_cache::{CachedPrecompile, CachedPrecompileMetrics, PrecompileCacheMap}, types::{InsertPayloadResult, ValidationOutput}, @@ -118,8 +117,9 @@ use reth_tasks::LazyHandle; use crate::tree::{ payload_processor::receipt_root_task::{IndexedReceipt, ReceiptRootTaskHandle}, state_root_strategy::{ - DefaultStateRootStrategy, PayloadStateRootJobContext, StateRootJobContext, - StateRootStrategy, + DefaultStateRootStrategy, LazyHashedPostState, PayloadStateRootHandle, + PayloadStateRootJobContext, StateRootHintStream, StateRootJobContext, StateRootStrategy, + StateRootUpdateStream, }, }; use alloy_consensus::constants::KECCAK_EMPTY; @@ -154,8 +154,8 @@ use reth_provider::{ }; use reth_revm::db::{states::bundle_state::BundleRetention, BundleAccount, State}; use reth_trie::{ - hashed_cursor::HashedCursorFactory, prefix_set::TriePrefixSetsMut, - trie_cursor::TrieCursorFactory, updates::TrieUpdates, HashedPostState, LazyTrieData, + hashed_cursor::HashedCursorFactory, trie_cursor::TrieCursorFactory, updates::TrieUpdates, + LazyTrieData, }; use reth_trie_db::ChangesetCache; use std::{ @@ -169,9 +169,6 @@ use tracing::{debug, debug_span, error, info, instrument, trace, warn, Level, Sp pub use crate::tree::types::ValidationOutcome; -/// Handle to a [`HashedPostState`] computed on a background thread. -pub type LazyHashedPostState = reth_tasks::LazyHandle>; - /// Multiplier over the parent's gas limit beyond which a block's claimed gas usage cannot be /// legitimate. Gas limit can change by at most 1/1024 per block, so anything over this is rejected /// without entering execution. @@ -193,8 +190,6 @@ pub struct TreeCtx<'a, N: NodePrimitives> { state: &'a mut EngineApiTreeState, /// Reference to the canonical in-memory state canonical_in_memory_state: &'a CanonicalInMemoryState, - /// Pending sparse trie prune request to consume when spawning a sparse trie task. - pending_sparse_trie_prune: &'a mut Option, } impl<'a, N: NodePrimitives> std::fmt::Debug for TreeCtx<'a, N> { @@ -202,7 +197,6 @@ impl<'a, N: NodePrimitives> std::fmt::Debug for TreeCtx<'a, N> { f.debug_struct("TreeCtx") .field("state", &"EngineApiTreeState") .field("canonical_in_memory_state", &self.canonical_in_memory_state) - .field("pending_sparse_trie_prune", &self.pending_sparse_trie_prune.is_some()) .finish() } } @@ -212,9 +206,8 @@ impl<'a, N: NodePrimitives> TreeCtx<'a, N> { pub const fn new( state: &'a mut EngineApiTreeState, canonical_in_memory_state: &'a CanonicalInMemoryState, - pending_sparse_trie_prune: &'a mut Option, ) -> Self { - Self { state, canonical_in_memory_state, pending_sparse_trie_prune } + Self { state, canonical_in_memory_state } } } @@ -233,11 +226,6 @@ impl<'a, N: NodePrimitives> TreeCtx<'a, N> { pub const fn canonical_in_memory_state(&self) -> &'a CanonicalInMemoryState { self.canonical_in_memory_state } - - /// Takes the pending sparse trie prune request, if any. - pub const fn take_sparse_trie_prune(&mut self) -> Option { - self.pending_sparse_trie_prune.take() - } } /// Pauses JIT helper execution while validating imported payloads. @@ -284,7 +272,7 @@ where evm_config: Evm, /// Configuration for the tree. config: TreeConfig, - /// Payload processor for state root computation. + /// Payload processor for transaction conversion, prewarming, and execution caching. payload_processor: PayloadProcessor, /// Precompile cache map. precompile_cache_map: PrecompileCacheMap>, @@ -301,6 +289,8 @@ where changeset_cache: ChangesetCache, /// Task runtime for spawning parallel work. runtime: reth_tasks::Runtime, + /// Shared state trie in-memory overlay data. + state_trie_overlays: StateTrieOverlayManager, /// State-root strategy used to prepare per-block commitment tasks. #[debug(skip)] state_root_strategy: Arc>, @@ -351,7 +341,6 @@ where evm_config.clone(), &config, precompile_cache_map.clone(), - state_trie_overlays, ); Self { provider, @@ -366,7 +355,8 @@ where validator, changeset_cache, runtime, - state_root_strategy: Arc::new(DefaultStateRootStrategy), + state_trie_overlays, + state_root_strategy: Arc::new(DefaultStateRootStrategy::default()), } } @@ -597,20 +587,16 @@ where let parallel_bal_execution = ensure_ok!(self.bal_path_eligible(env.decoded_bal.as_deref())); // Prepare the state-root job before execution so it can provide streaming hooks. - let pending_sparse_trie_prune = (!self.config.skip_state_root() && - !self.config.state_root_fallback() && - self.config.use_state_root_task()) - .then(|| ctx.take_sparse_trie_prune()) - .flatten(); let mut state_root_job = ensure_ok!(self.state_root_strategy.prepare(StateRootJobContext::new( - &self.payload_processor, + &self.runtime, + &self.state_trie_overlays, &env, provider_builder.clone(), overlay_factory, &self.config, parallel_bal_execution, - pending_sparse_trie_prune, + ctx.state_mut(), ))); let state_root_job_name = state_root_job.name(); @@ -620,18 +606,21 @@ where "Prepared state root job" ); - let state_root_streams = state_root_job.streams(); - // Only take the hook on the serial path: on the parallel BAL path it would be dropped - // unused, and the drop would fire a spurious end-of-updates signal into the job. - let execution_state_hook = - (!parallel_bal_execution).then(|| state_root_job.take_execution_hook()).flatten(); + // The hook exists only when `prepare` installed it (serial path); on the parallel BAL + // path the authoritative capability went to the hashed update stream instead. + let execution_state_hook = state_root_job.take_execution_hook(); + // The prewarm capabilities go to the code that produces their messages and are not + // retained anywhere else, so the task's update channel closes when producers finish. + let hint_stream = state_root_job.take_hint_stream(); + let hashed_update_stream = state_root_job.take_hashed_update_stream(); // Spawn transaction conversion and prewarming. let mut handle = ensure_ok!(self.spawn_payload_processor( env.clone(), txs, provider_builder.clone(), - state_root_streams, + hint_stream, + hashed_update_stream, parallel_bal_execution, )); @@ -744,18 +733,17 @@ where let hashed_state_provider = self.provider.clone(); let mut hashed_state_rx = state_root_job.take_hashed_state_rx(); let mut hashed_state: LazyHashedPostState = - self.payload_processor.executor().spawn_blocking_named("hash-post-state", move || { + self.runtime.spawn_blocking_named("hash-post-state", move || { let _span = debug_span!( target: "engine::tree::payload_validator", "hashed_post_state", ) .entered(); - let state = if let Some(Ok(state)) = hashed_state_rx.as_mut().map(|rx| rx.recv()) { + if let Some(Ok(state)) = hashed_state_rx.as_mut().map(|rx| rx.recv()) { state } else { - hashed_state_provider.hashed_post_state(&hashed_state_output.state) - }; - Arc::new(state) + Arc::new(hashed_state_provider.hashed_post_state(&hashed_state_output.state)) + } }); let block = validated_block.try_into_inner().expect("sole handle")?; @@ -797,8 +785,12 @@ where "validate_block_post_execution_with_hashed_state" ) .in_scope(|| { - self.validator - .validate_block_post_execution_with_hashed_state(&|| hashed_state.get(), &block) + self.validator.validate_block_post_execution_with_hashed_state( + || hashed_state.get(), + &block, + &parent_block, + || provider_builder.build(), + ) }); let root_start = Instant::now(); @@ -818,7 +810,6 @@ where let state_root = root_outcome.state_root; let trie_output = root_outcome.trie_updates; - let changed_paths = root_outcome.changed_paths; // A fallback path recomputed the hashed post state. Replace the streaming-derived one // and re-run hashed-state validation against it, since a failed state-root task may @@ -830,15 +821,20 @@ where "validate_block_post_execution_with_hashed_state" ) .in_scope(|| { - self.validator - .validate_block_post_execution_with_hashed_state(&|| hashed_state.get(), &block) + self.validator.validate_block_post_execution_with_hashed_state( + || hashed_state.get(), + &block, + &parent_block, + || provider_builder.build(), + ) }); } if let Err(err) = hashed_state_validate_result { - // call post-block hook - self.on_invalid_block(&parent_block, &block, &output, None, ctx.state_mut()); - return Err(InsertBlockError::new(block.into_sealed_block(), err.into()).into()) + if err.is_validation_error() { + self.on_invalid_block(&parent_block, &block, &output, None, ctx.state_mut()); + } + return Err(InsertBlockError::new(block.into_sealed_block(), err).into()) } self.metrics.block_validation.record_state_root(&trie_output, root_elapsed.as_secs_f64()); @@ -882,13 +878,8 @@ where let _ = valid_block_tx.send(()); } - let executed_block = self.spawn_deferred_trie_task( - Arc::new(block), - output, - hashed_state, - trie_output, - changed_paths, - ); + let executed_block = + self.spawn_deferred_trie_task(Arc::new(block), output, hashed_state, trie_output); let raw_bal = decoded_bal.map(|decoded_bal| decoded_bal.as_raw_bal().clone()); Ok(ValidationOutput::new(executed_block, timing_stats).with_raw_bal(raw_bal)) } @@ -909,7 +900,7 @@ where let validator = self.validator.clone(); let consensus = self.consensus.clone(); let parent_span = Span::current(); - self.payload_processor.executor().spawn_blocking_named("payload-convert", move || { + self.runtime.spawn_blocking_named("payload-convert", move || { let _span = debug_span!( target: "engine::tree::payload_validator", parent: parent_span, @@ -1180,9 +1171,7 @@ where let (receipt_tx, receipt_rx) = crossbeam_channel::unbounded(); let (result_tx, result_rx) = tokio::sync::oneshot::channel(); let task_handle = ReceiptRootTaskHandle::new(receipt_rx, result_tx); - self.payload_processor - .executor() - .spawn_blocking_named("receipt-root", move || task_handle.run(receipts_len)); + self.runtime.spawn_blocking_named("receipt-root", move || task_handle.run(receipts_len)); (receipt_tx, result_rx) } @@ -1342,20 +1331,25 @@ where /// Spawns transaction conversion and cache prewarming for payload validation. /// - /// State-root tasks are prepared before this method and can provide streams that prewarm uses - /// for BAL-derived authoritative updates or transaction-derived hints. + /// State-root tasks are prepared before this method and can provide capabilities that + /// prewarm uses for BAL-derived authoritative updates or transaction-derived hints. #[instrument( level = "debug", target = "engine::tree::payload_validator", skip_all, - fields(has_state_root_streams = !state_root_streams.is_empty(), parallel_bal_execution) + fields( + has_hint_stream = hint_stream.is_some(), + has_hashed_update_stream = hashed_update_stream.is_some(), + parallel_bal_execution + ) )] fn spawn_payload_processor>( &self, env: ExecutionEnv, txs: T, provider_builder: StateProviderBuilder, - state_root_streams: StateRootStreams, + hint_stream: Option, + hashed_update_stream: Option, parallel_bal_execution: bool, ) -> Result< PayloadHandle< @@ -1370,7 +1364,8 @@ where env, txs, provider_builder, - state_root_streams, + hint_stream, + hashed_update_stream, parallel_bal_execution, ); @@ -1454,7 +1449,6 @@ where execution_outcome: Arc>, hashed_state: LazyHashedPostState, trie_output: Arc, - changed_paths: Option>, ) -> ExecutedBlock { // Create deferred handle and task that owns the unsorted inputs. // Resolve the lazy handle into Arc. By this point the hashed state has @@ -1464,7 +1458,7 @@ where Err(handle) => handle.get().clone(), }; let (deferred_trie_data, deferred_trie_task) = - LazyTrieData::pending(hashed_state, trie_output, changed_paths); + LazyTrieData::pending(hashed_state, trie_output); let block_validation_metrics = self.metrics.block_validation.clone(); // Capture block info for tracing. @@ -1495,9 +1489,7 @@ where }; // Spawn task that computes trie data asynchronously. - self.payload_processor - .executor() - .spawn_blocking_named(DEFERRED_TRIE_WORKER_NAME, compute_trie_input_task); + self.runtime.spawn_blocking_named(DEFERRED_TRIE_WORKER_NAME, compute_trie_input_task); ExecutedBlock::with_deferred_trie_data(block, execution_outcome, deferred_trie_data) } @@ -1713,7 +1705,7 @@ pub trait EngineValidator< parent_hash: B256, parent_header: &N::BlockHeader, timestamp: u64, - state: &EngineApiTreeState, + state: &mut EngineApiTreeState, ) -> Option; } @@ -1791,7 +1783,6 @@ where block.execution_output, LazyHashedPostState::ready(block.hashed_state), block.trie_updates, - block.changed_paths, )) } @@ -1804,7 +1795,7 @@ where parent_hash: B256, parent_header: &N::BlockHeader, timestamp: u64, - state: &EngineApiTreeState, + state: &mut EngineApiTreeState, ) -> Option { let provider_builder = match self.state_provider_builder(parent_hash, state) { Ok(Some(provider_builder)) => provider_builder, @@ -1825,10 +1816,12 @@ where ); match self.state_root_strategy.prepare_payload_builder(PayloadStateRootJobContext::new( - &self.payload_processor, + &self.runtime, + &self.state_trie_overlays, parent_hash, parent_header, timestamp, + state, provider_builder, overlay_factory, &self.config, @@ -1852,7 +1845,31 @@ where Evm: ConfigureEvm, { fn wait_for_caches(&self) -> CacheWaitDurations { - self.payload_processor.wait_for_caches() + debug!(target: "engine::tree::payload_validator", "Waiting for execution cache and sparse trie locks"); + + let execution_cache = self.payload_processor.execution_cache(); + let state_trie_overlays = self.state_trie_overlays.clone(); + let (execution_tx, execution_rx) = std::sync::mpsc::channel(); + let (sparse_trie_tx, sparse_trie_rx) = std::sync::mpsc::channel(); + + self.runtime.spawn_blocking_named("wait-exec-cache", move || { + let _ = execution_tx.send(execution_cache.wait_for_availability()); + }); + self.runtime.spawn_blocking_named("wait-sparse-tri", move || { + let _ = sparse_trie_tx.send(state_trie_overlays.wait_for_sparse_trie_availability()); + }); + + let execution_cache = + execution_rx.recv().expect("execution cache wait task failed to send result"); + let sparse_trie = + sparse_trie_rx.recv().expect("sparse trie wait task failed to send result"); + debug!( + target: "engine::tree::payload_validator", + ?execution_cache, + ?sparse_trie, + "Execution cache and sparse trie locks acquired" + ); + CacheWaitDurations { execution_cache, sparse_trie } } } diff --git a/crates/engine/tree/src/tree/state_root_strategy.rs b/crates/engine/tree/src/tree/state_root_strategy.rs deleted file mode 100644 index 005b6944653..00000000000 --- a/crates/engine/tree/src/tree/state_root_strategy.rs +++ /dev/null @@ -1,867 +0,0 @@ -//! State-root strategies for engine-tree block validation. -//! -//! A [`StateRootStrategy`] is installed once per node, via -//! `BasicEngineValidator::with_state_root_strategy`, and consulted for every block that engine -//! validation executes. For each block the strategy prepares a [`StateRootJob`] before execution -//! starts, and validation finishes the job after execution to obtain the state root that is -//! checked against the block header. On every FCU that carries payload attributes, the strategy -//! is also asked through [`StateRootStrategy::prepare_payload_builder`] for an optional -//! [`PayloadStateRootHandle`] that the payload builder uses while building a block. -//! -//! # Job lifecycle -//! -//! 1. [`StateRootStrategy::prepare`] runs before block execution. The job can spawn background work -//! here and can expose hooks that observe execution. -//! 2. Execution runs. Jobs that observe execution receive updates through their hooks. -//! 3. [`StateRootJob::finish`] runs after execution and returns the [`StateRootJobOutcome`]. It -//! must produce a result even if no execution updates were observed, since the full -//! [`BlockExecutionOutput`] is passed to it. -//! -//! Dropping a prepared job without calling `finish` aborts it. Implementations must treat -//! channel disconnects from dropped hooks as cancellation and must not leak background work. -//! -//! # Stream delivery contract -//! -//! A prepared job exposes [`StateRootStreams`] views over its sink. Exactly one authoritative -//! source fires per block, and the job does not control which one: -//! -//! - On the parallel BAL execution path, prewarm converts the block access list and delivers -//! pre-hashed updates through the hashed-update stream, terminated by `on_updates_finished`. -//! - On the serial execution path, per-transaction `EvmState` updates arrive through the execution -//! hook, terminated when the hook is dropped after execution. -//! -//! Which path runs depends on runtime conditions (BAL present, caching and prewarming enabled), -//! so a sink must handle both. Access hints from prewarming are best-effort: they may be -//! missing, duplicated, or stale, and must not be treated as state updates. -//! -//! # Custom strategies -//! -//! Custom implementations can hold a [`DefaultStateRootStrategy`] and forward calls to it for -//! blocks where the default behavior is wanted, for example before a fork activates. See -//! `examples/custom-state-root` for the wiring. -//! -//! Returning empty trie updates in the outcome means the trie tables are no longer maintained: -//! `eth_getProof` and anything else that reads the stored trie will not work for new blocks. -//! Returning no changed paths opts the block out of sparse-trie cache pruning. - -use crate::tree::{ - metrics::BlockValidationMetrics, - multiproof::{ - PayloadStateRootHandle, StateRootComputeOutcome, StateRootHandle, StateRootStreams, - }, - payload_processor::PayloadProcessor, - payload_validator::LazyHashedPostState, - ExecutionEnv, StateProviderBuilder, TreeConfig, -}; -use alloy_primitives::B256; -use reth_errors::ProviderResult; -use reth_evm::{ConfigureEvm, OnStateHook}; -use reth_primitives_traits::{AlloyBlockHeader, NodePrimitives, RecoveredBlock}; -use reth_provider::{ - providers::OverlayStateProviderFactory, BlockExecutionOutput, BlockReader, - DatabaseProviderFactory, DatabaseProviderROFactory, HashedPostStateProvider, ProviderError, - StateProviderFactory, StateReader, StateRootProvider, -}; -use reth_trie::{ - hashed_cursor::HashedCursorFactory, prefix_set::TriePrefixSetsMut, - trie_cursor::TrieCursorFactory, updates::TrieUpdates, HashedPostState, -}; -#[cfg(feature = "trie-debug")] -use reth_trie_sparse::debug_recorder::TrieDebugRecorder; -use std::{ - fmt, - sync::{ - mpsc::{self, RecvTimeoutError}, - Arc, - }, - time::Duration, -}; -use tracing::{debug, warn}; - -/// Strategy used by engine-tree validation to prepare per-block state-root work. -pub trait StateRootStrategy: Send + Sync -where - N: NodePrimitives, - Evm: ConfigureEvm, -{ - /// Prepares a per-block state-root job before execution starts. - fn prepare( - &self, - ctx: StateRootJobContext<'_, N, P, Evm>, - ) -> ProviderResult>; - - /// Prepares the optional payload-builder state-root handle used for FCU-triggered block - /// building. - /// - /// This is consulted on every FCU that carries payload attributes. Returning `None` means the - /// payload builder computes the state root itself; the stock builders fall back to a - /// synchronous MPT state root. The default implementation returns `None`. - fn prepare_payload_builder( - &self, - _ctx: PayloadStateRootJobContext<'_, N, P, Evm>, - ) -> ProviderResult> { - Ok(None) - } -} - -/// Data available while preparing one payload-builder state-root handle. -pub struct PayloadStateRootJobContext<'a, N, P, Evm> -where - N: NodePrimitives, - Evm: ConfigureEvm, -{ - payload_processor: &'a PayloadProcessor, - parent_hash: B256, - parent_header: &'a N::BlockHeader, - timestamp: u64, - provider_builder: StateProviderBuilder, - overlay_factory: OverlayStateProviderFactory, - config: &'a TreeConfig, -} - -impl fmt::Debug for PayloadStateRootJobContext<'_, N, P, Evm> -where - N: NodePrimitives, - Evm: ConfigureEvm, -{ - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("PayloadStateRootJobContext") - .field("parent_hash", &self.parent_hash) - .field("parent_state_root", &self.parent_state_root()) - .field("timestamp", &self.timestamp) - .finish_non_exhaustive() - } -} - -impl<'a, N, P, Evm> PayloadStateRootJobContext<'a, N, P, Evm> -where - N: NodePrimitives, - Evm: ConfigureEvm, -{ - /// Creates a new payload-builder state-root job context. - pub(crate) const fn new( - payload_processor: &'a PayloadProcessor, - parent_hash: B256, - parent_header: &'a N::BlockHeader, - timestamp: u64, - provider_builder: StateProviderBuilder, - overlay_factory: OverlayStateProviderFactory, - config: &'a TreeConfig, - ) -> Self { - Self { - payload_processor, - parent_hash, - parent_header, - timestamp, - provider_builder, - overlay_factory, - config, - } - } - - /// Returns the parent block hash for the payload being built. - pub const fn parent_hash(&self) -> B256 { - self.parent_hash - } - - /// Returns the parent block header for the payload being built. - /// - /// This is the chain's concrete header type, so chain-specific strategies can read - /// chain-specific fields, and number-activated forks can dispatch on the parent number. - pub const fn parent_header(&self) -> &N::BlockHeader { - self.parent_header - } - - /// Returns the parent state root for the payload being built. - pub fn parent_state_root(&self) -> B256 { - self.parent_header.state_root() - } - - /// Returns the timestamp of the payload being built, taken from the payload attributes. - /// - /// Strategies that switch behavior at a fork activation can dispatch on this value. - pub const fn timestamp(&self) -> u64 { - self.timestamp - } - - /// Returns the task runtime used by payload processing. - pub const fn executor(&self) -> &reth_tasks::Runtime { - self.payload_processor.executor() - } - - /// Returns a clone of the state provider builder. - pub fn provider_builder(&self) -> StateProviderBuilder - where - P: Clone, - { - self.provider_builder.clone() - } -} - -/// Data available while preparing one state-root job. -pub struct StateRootJobContext<'a, N, P, Evm> -where - N: NodePrimitives, - Evm: ConfigureEvm, -{ - payload_processor: &'a PayloadProcessor, - env: &'a ExecutionEnv, - provider_builder: StateProviderBuilder, - overlay_factory: OverlayStateProviderFactory, - config: &'a TreeConfig, - parallel_bal_execution: bool, - pending_sparse_trie_prune: Option, -} - -impl fmt::Debug for StateRootJobContext<'_, N, P, Evm> -where - N: NodePrimitives, - Evm: ConfigureEvm, -{ - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("StateRootJobContext") - .field("parallel_bal_execution", &self.parallel_bal_execution) - .field("has_pending_sparse_trie_prune", &self.pending_sparse_trie_prune.is_some()) - .finish_non_exhaustive() - } -} - -impl<'a, N, P, Evm> StateRootJobContext<'a, N, P, Evm> -where - N: NodePrimitives, - Evm: ConfigureEvm, -{ - /// Creates a new state-root job context. - pub(crate) const fn new( - payload_processor: &'a PayloadProcessor, - env: &'a ExecutionEnv, - provider_builder: StateProviderBuilder, - overlay_factory: OverlayStateProviderFactory, - config: &'a TreeConfig, - parallel_bal_execution: bool, - pending_sparse_trie_prune: Option, - ) -> Self { - Self { - payload_processor, - env, - provider_builder, - overlay_factory, - config, - parallel_bal_execution, - pending_sparse_trie_prune, - } - } - - /// Returns the execution environment for the block. - pub const fn env(&self) -> &ExecutionEnv { - self.env - } - - /// Returns the task runtime used by payload processing. - pub const fn executor(&self) -> &reth_tasks::Runtime { - self.payload_processor.executor() - } - - /// Returns true when validation will use the parallel BAL execution path. - pub const fn parallel_bal_execution(&self) -> bool { - self.parallel_bal_execution - } - - /// Returns a clone of the state provider builder. - pub fn provider_builder(&self) -> StateProviderBuilder - where - P: Clone, - { - self.provider_builder.clone() - } -} - -/// Prepared per-block state-root work and its stream wiring. -pub struct PreparedStateRootJob { - job: Box>, - streams: StateRootStreams, - hashed_state_rx: Option>, -} - -impl fmt::Debug for PreparedStateRootJob { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("PreparedStateRootJob") - .field("name", &self.job.name()) - .field("streams", &self.streams) - .field("has_hashed_state_rx", &self.hashed_state_rx.is_some()) - .finish() - } -} - -impl PreparedStateRootJob { - /// Creates a prepared state-root job. - pub const fn new( - job: Box>, - streams: StateRootStreams, - hashed_state_rx: Option>, - ) -> Self { - Self { job, streams, hashed_state_rx } - } - - /// Returns the job name used in logs. - pub fn name(&self) -> &'static str { - self.job.name() - } - - /// Returns stream views used by prewarm. - pub fn streams(&self) -> StateRootStreams { - self.streams.clone() - } - - /// Takes the execution hook, if the job wants normal execution updates. - pub fn take_execution_hook(&mut self) -> Option> { - self.streams - .take_execution_stream() - .map(|stream| Box::new(stream.state_hook()) as Box) - } - - /// Takes the optional hashed-state receiver produced by the job. - /// - /// The sender behind a returned receiver must either deliver one value or be dropped; - /// validation blocks on it while hashing the post state, so a job that keeps the sender - /// alive without sending stalls block validation. - pub const fn take_hashed_state_rx(&mut self) -> Option> { - self.hashed_state_rx.take() - } - - /// Completes the job after execution. - pub fn finish( - &mut self, - block: &RecoveredBlock, - output: Arc>, - hashed_state: &LazyHashedPostState, - ) -> ProviderResult { - self.job.finish(block, output, hashed_state) - } -} - -/// Per-block state-root job prepared before execution and finished after execution. -pub trait StateRootJob: Send { - /// Human-readable strategy name used in logs. - fn name(&self) -> &'static str; - - /// Completes the job after execution. - /// - /// Called at most once per prepared job; implementations may panic if called again. - fn finish( - &mut self, - block: &RecoveredBlock, - output: Arc>, - hashed_state: &LazyHashedPostState, - ) -> ProviderResult; -} - -/// Outcome of a per-block state-root job. -#[derive(Debug)] -pub struct StateRootJobOutcome { - /// Computed state root. - pub state_root: B256, - /// Trie updates associated with the computed state root. - pub trie_updates: Arc, - /// Changed trie node base paths retained while computing the root, if the job tracks them. - pub changed_paths: Option>, - /// Hashed post state recomputed by a fallback path. - /// - /// When set, the root was not derived from the streamed updates, so validation replaces its - /// streaming-derived hashed post state with this one and re-runs hashed-state checks. - pub hashed_state: Option>, -} - -impl StateRootJobOutcome { - /// Creates a state-root job outcome without changed paths. - pub const fn new(state_root: B256, trie_updates: Arc) -> Self { - Self { state_root, trie_updates, changed_paths: None, hashed_state: None } - } - - /// Sets the changed trie node base paths retained while computing the root. - pub fn with_changed_paths(mut self, changed_paths: Option>) -> Self { - self.changed_paths = changed_paths; - self - } - - /// Sets the hashed post state recomputed by a fallback path. - pub fn with_hashed_state(mut self, hashed_state: Option>) -> Self { - self.hashed_state = hashed_state; - self - } -} - -/// Receiver for the raced serial state-root fallback: root, trie updates, and the hashed -/// post state the fallback recomputed. -type SerialFallbackRx = mpsc::Receiver)>>; - -/// Default state-root strategy used by engine-tree validation. -/// -/// Covers the built-in modes: the sparse-trie state-root task, plus the skipped and -/// synchronous modes selected by [`TreeConfig`]. -/// -/// Custom strategies can hold this type and delegate to it for blocks where they want the -/// default behavior. -#[derive(Debug, Default)] -pub struct DefaultStateRootStrategy; - -impl StateRootStrategy for DefaultStateRootStrategy -where - N: NodePrimitives, - P: DatabaseProviderFactory - + BlockReader
- + StateProviderFactory - + StateReader - + Clone - + 'static, - OverlayStateProviderFactory: DatabaseProviderROFactory - + Clone - + Send - + Sync - + 'static, - Evm: ConfigureEvm + 'static, -{ - fn prepare( - &self, - ctx: StateRootJobContext<'_, N, P, Evm>, - ) -> ProviderResult> { - let StateRootJobContext { - payload_processor, - env, - provider_builder, - overlay_factory, - config, - parallel_bal_execution, - pending_sparse_trie_prune, - } = ctx; - - if config.skip_state_root() { - return Ok(PreparedStateRootJob::new( - Box::new(SkippedStateRootJob {}), - StateRootStreams::empty(), - None, - )) - } - - // `state_root_fallback` forces serial computation for tests and debugging. Hosts - // without enough parallelism for the state-root task pipeline also compute the root - // synchronously, since the pipeline's threads can starve each other there; see - // [`TreeConfig::use_state_root_task`]. - if config.state_root_fallback() || !config.use_state_root_task() { - return Ok(PreparedStateRootJob::new( - Box::new(SynchronousStateRootJob { provider_builder }), - StateRootStreams::empty(), - None, - )) - } - - let mut handle = payload_processor.spawn_state_root( - overlay_factory.clone(), - env.parent_state_root, - Some(env.transaction_count), - config, - pending_sparse_trie_prune, - ); - let streams = handle.streams(!parallel_bal_execution); - let hashed_state_rx = Some(handle.take_hashed_state_rx()); - - Ok(PreparedStateRootJob::new( - Box::new(SparseTrieStateRootJob { - handle, - provider_builder, - overlay_factory, - executor: payload_processor.executor().clone(), - timeout: config.state_root_task_timeout(), - compare_trie_updates: config.always_compare_trie_updates(), - metrics: BlockValidationMetrics::default(), - }), - streams, - hashed_state_rx, - )) - } - - fn prepare_payload_builder( - &self, - ctx: PayloadStateRootJobContext<'_, N, P, Evm>, - ) -> ProviderResult> { - let parent_state_root = ctx.parent_state_root(); - let PayloadStateRootJobContext { payload_processor, overlay_factory, config, .. } = ctx; - - // Sharing the engine state-root task with the payload builder is opt-in, and needs a - // host that can run the task pipeline at all. - if !config.share_sparse_trie_with_payload_builder() || - config.skip_state_root() || - !config.use_state_root_task() - { - return Ok(None) - } - - Ok(Some( - payload_processor - .spawn_state_root( - overlay_factory, - parent_state_root, - // Tx count unknown at FCU time (block built incrementally): full proof - // workers. - None, - config, - None, - ) - .into_payload_state_root_handle(), - )) - } -} - -#[derive(Debug)] -struct SkippedStateRootJob {} - -impl StateRootJob for SkippedStateRootJob { - fn name(&self) -> &'static str { - "skipped" - } - - fn finish( - &mut self, - block: &RecoveredBlock, - _output: Arc>, - _hashed_state: &LazyHashedPostState, - ) -> ProviderResult { - Ok(StateRootJobOutcome::new(block.header().state_root(), Arc::new(TrieUpdates::default()))) - } -} - -#[derive(Debug)] -struct SynchronousStateRootJob { - provider_builder: StateProviderBuilder, -} - -impl StateRootJob for SynchronousStateRootJob -where - N: NodePrimitives, - P: BlockReader + StateProviderFactory + StateReader + Clone + Send + Sync + 'static, -{ - fn name(&self) -> &'static str { - "synchronous" - } - - fn finish( - &mut self, - _block: &RecoveredBlock, - _output: Arc>, - hashed_state: &LazyHashedPostState, - ) -> ProviderResult { - let provider = self.provider_builder.clone().build()?; - let (state_root, trie_updates) = - provider.state_root_with_updates(hashed_state.get().as_ref().clone())?; - Ok(StateRootJobOutcome::new(state_root, Arc::new(trie_updates))) - } -} - -#[derive(Debug)] -struct SparseTrieStateRootJob { - handle: StateRootHandle, - provider_builder: StateProviderBuilder, - overlay_factory: OverlayStateProviderFactory, - executor: reth_tasks::Runtime, - timeout: Option, - compare_trie_updates: bool, - metrics: BlockValidationMetrics, -} - -impl SparseTrieStateRootJob -where - N: NodePrimitives, - P: StateProviderFactory + Clone + Send + Sync + 'static, - P: BlockReader + StateReader, - OverlayStateProviderFactory: DatabaseProviderROFactory - + Clone - + Send - + Sync - + 'static, -{ - fn serial_fallback( - executor: &reth_tasks::Runtime, - provider_builder: StateProviderBuilder, - output: Arc>, - ) -> ProviderResult { - let provider = provider_builder.build()?; - let (fallback_tx, fallback_rx) = mpsc::channel(); - executor.spawn_blocking_named("serial-root", move || { - let result = (|| { - let hashed_state = Arc::new(provider.hashed_post_state(&output.state)); - let (root, updates) = - provider.state_root_with_updates(hashed_state.as_ref().clone())?; - Ok((root, updates, hashed_state)) - })(); - let _ = fallback_tx.send(result); - }); - - Ok(fallback_rx) - } - - /// Recomputes the state root serially from the execution output. - /// - /// Used when the state-root task failed or produced a wrong root, so the recomputed hashed - /// post state is returned in the outcome for validation to re-check against. - fn compute_serial( - &self, - output: &BlockExecutionOutput, - ) -> ProviderResult { - let provider = self.provider_builder.clone().build()?; - let hashed_state = Arc::new(provider.hashed_post_state(&output.state)); - let (state_root, trie_updates) = - provider.state_root_with_updates(hashed_state.as_ref().clone())?; - self.metrics.state_root_task_fallback_success_total.increment(1); - Ok(StateRootJobOutcome::new(state_root, Arc::new(trie_updates)) - .with_hashed_state(Some(hashed_state))) - } - - /// Converts a task outcome into a job outcome, recomputing serially when the task returned - /// a root that does not match the block header. A state-root-task bug then costs latency - /// instead of marking a valid block invalid; if the serial root also mismatches, validation - /// rejects the block. - fn verified_sparse_outcome( - &self, - block: &RecoveredBlock, - output: &BlockExecutionOutput, - outcome: StateRootComputeOutcome, - ) -> ProviderResult { - let outcome = self.sparse_outcome(block, output, outcome); - if outcome.state_root == block.header().state_root() { - return Ok(outcome) - } - warn!( - target: "engine::tree::state_root_strategy", - state_root = ?outcome.state_root, - block_state_root = ?block.header().state_root(), - "State root task returned incorrect state root, recomputing serially" - ); - self.compute_serial(output) - } - - fn sparse_outcome( - &self, - _block: &RecoveredBlock, - output: &BlockExecutionOutput, - outcome: StateRootComputeOutcome, - ) -> StateRootJobOutcome { - let StateRootComputeOutcome { - state_root, - trie_updates, - changed_paths, - #[cfg(feature = "trie-debug")] - debug_recorders, - } = outcome; - - if self.compare_trie_updates { - let _has_diff = compare_trie_updates_with_serial( - self.provider_builder.clone(), - self.overlay_factory.clone(), - output, - trie_updates.as_ref().clone(), - ); - #[cfg(feature = "trie-debug")] - if _has_diff { - write_trie_debug_recorders(_block.header().number(), &debug_recorders); - } - } - - #[cfg(feature = "trie-debug")] - if state_root != _block.header().state_root() { - write_trie_debug_recorders(_block.header().number(), &debug_recorders); - } - - StateRootJobOutcome::new(state_root, trie_updates).with_changed_paths(changed_paths) - } -} - -impl StateRootJob for SparseTrieStateRootJob -where - N: NodePrimitives, - P: BlockReader + StateProviderFactory + StateReader + Clone + Send + Sync + 'static, - OverlayStateProviderFactory: DatabaseProviderROFactory - + Clone - + Send - + Sync - + 'static, -{ - fn name(&self) -> &'static str { - "sparse-trie" - } - - fn finish( - &mut self, - block: &RecoveredBlock, - output: Arc>, - _hashed_state: &LazyHashedPostState, - ) -> ProviderResult { - if self.timeout.is_none() { - return match self.handle.state_root() { - Ok(outcome) => self.verified_sparse_outcome(block, &output, outcome), - Err(err) => { - debug!(target: "engine::tree::state_root_strategy", %err, "State root task failed, falling back to serial root"); - self.compute_serial(&output) - } - } - } - - let timeout = self.timeout.expect("checked above"); - let task_rx = self.handle.take_state_root_rx(); - let fallback_rx = match task_rx.recv_timeout(timeout) { - Ok(Ok(outcome)) => return self.verified_sparse_outcome(block, &output, outcome), - Ok(Err(err)) => { - debug!(target: "engine::tree::state_root_strategy", %err, "State root task failed, falling back to serial root"); - Self::serial_fallback( - &self.executor, - self.provider_builder.clone(), - output.clone(), - )? - } - Err(RecvTimeoutError::Timeout) => { - warn!(target: "engine::tree::state_root_strategy", ?timeout, "State root task timed out, racing serial fallback"); - self.metrics.state_root_task_timeout_total.increment(1); - Self::serial_fallback( - &self.executor, - self.provider_builder.clone(), - output.clone(), - )? - } - Err(RecvTimeoutError::Disconnected) => { - debug!(target: "engine::tree::state_root_strategy", "State root task dropped, falling back to serial root"); - Self::serial_fallback( - &self.executor, - self.provider_builder.clone(), - output.clone(), - )? - } - }; - - loop { - if let Ok(Ok(outcome)) = task_rx.try_recv() { - let outcome = self.sparse_outcome(block, &output, outcome); - if outcome.state_root == block.header().state_root() { - return Ok(outcome) - } - // A wrong task root falls through to the serial fallback already racing below. - warn!( - target: "engine::tree::state_root_strategy", - state_root = ?outcome.state_root, - block_state_root = ?block.header().state_root(), - "State root task returned incorrect state root, using serial fallback" - ); - } - - match fallback_rx.try_recv() { - Ok(Ok((state_root, trie_updates, hashed_state))) => { - self.metrics.state_root_task_fallback_success_total.increment(1); - return Ok(StateRootJobOutcome::new(state_root, Arc::new(trie_updates)) - .with_hashed_state(Some(hashed_state))) - } - Ok(Err(err)) => return Err(err), - Err(mpsc::TryRecvError::Empty) => {} - Err(mpsc::TryRecvError::Disconnected) => { - return Err(ProviderError::other(std::io::Error::other( - "serial state root fallback task dropped", - ))) - } - } - - std::thread::sleep(Duration::from_millis(1)); - } - } -} - -fn compare_trie_updates_with_serial( - state_provider_builder: StateProviderBuilder, - overlay_factory: OverlayStateProviderFactory, - output: &BlockExecutionOutput, - task_trie_updates: TrieUpdates, -) -> bool -where - N: NodePrimitives, - P: BlockReader + StateProviderFactory + StateReader + Clone, - OverlayStateProviderFactory: - DatabaseProviderROFactory, -{ - debug!(target: "engine::tree::state_root_strategy", "Comparing trie updates with serial computation"); - - match state_provider_builder.build().and_then(|provider| { - let hashed_state = provider.hashed_post_state(&output.state); - provider.state_root_with_updates(hashed_state) - }) { - Ok((serial_root, serial_trie_updates)) => { - debug!( - target: "engine::tree::state_root_strategy", - ?serial_root, - "Serial state root computation finished for comparison" - ); - - match overlay_factory.database_provider_ro() { - Ok(provider) => match super::trie_updates::compare_trie_updates( - &provider, - task_trie_updates, - serial_trie_updates, - ) { - Ok(has_diff) => return has_diff, - Err(err) => { - warn!( - target: "engine::tree::state_root_strategy", - %err, - "Error comparing trie updates" - ); - return true; - } - }, - Err(err) => { - warn!( - target: "engine::tree::state_root_strategy", - %err, - "Failed to get database provider for trie update comparison" - ); - } - } - } - Err(err) => { - warn!( - target: "engine::tree::state_root_strategy", - %err, - "Failed to compute serial state root for comparison" - ); - } - } - false -} - -/// Writes trie debug recorders to a JSON file for the given block number. -/// -/// The file is written to the current working directory as `trie_debug_block_{block_number}.json`. -#[cfg(feature = "trie-debug")] -fn write_trie_debug_recorders(block_number: u64, recorders: &[(Option, TrieDebugRecorder)]) { - let path = format!("trie_debug_block_{block_number}.json"); - match serde_json::to_string_pretty(recorders) { - Ok(json) => match std::fs::write(&path, json) { - Ok(()) => { - warn!( - target: "engine::tree::state_root_strategy", - %path, - "Wrote trie debug recorders to file" - ); - } - Err(err) => { - warn!( - target: "engine::tree::state_root_strategy", - %err, - %path, - "Failed to write trie debug recorders" - ); - } - }, - Err(err) => { - warn!( - target: "engine::tree::state_root_strategy", - %err, - "Failed to serialize trie debug recorders" - ); - } - } -} diff --git a/crates/engine/tree/src/tree/state_root_strategy/mod.rs b/crates/engine/tree/src/tree/state_root_strategy/mod.rs new file mode 100644 index 00000000000..d778f23de10 --- /dev/null +++ b/crates/engine/tree/src/tree/state_root_strategy/mod.rs @@ -0,0 +1,1534 @@ +//! State-root strategies for engine-tree block validation. +//! +//! A [`StateRootStrategy`] is installed once per node, via +//! `BasicEngineValidator::with_state_root_strategy`, and consulted for every block that engine +//! validation executes. For each block the strategy prepares a [`StateRootJob`] before execution +//! starts, and validation finishes the job after execution to obtain the state root that is +//! checked against the block header. On every FCU that carries payload attributes, the strategy +//! is also asked through [`StateRootStrategy::prepare_payload_builder`] for an optional +//! [`PayloadStateRootHandle`] that the payload builder uses while building a block. +//! +//! # Job lifecycle +//! +//! 1. [`StateRootStrategy::prepare`] runs before block execution. The job can spawn background work +//! here and can expose hooks that observe execution. +//! 2. Execution runs. Jobs that observe execution receive updates through their hooks. +//! 3. [`StateRootJob::finish`] runs after execution and returns the [`StateRootJobOutcome`]. It +//! must produce a result even if no execution updates were observed, since the full +//! [`BlockExecutionOutput`] is passed to it. +//! +//! Dropping a prepared job without calling `finish` aborts it. Implementations must treat +//! channel disconnects from dropped hooks as cancellation and must not leak background work. +//! +//! # Stream delivery contract +//! +//! A prepared job exposes update-stream capabilities over its sink. `prepare` installs exactly +//! one authoritative capability per block, matching the execution mode: +//! +//! - On the parallel BAL execution path, prewarm converts the block access list and delivers +//! pre-hashed updates through the hashed update stream, terminated by +//! [`StateRootUpdateStream::finish`]. +//! - On the serial execution path, per-transaction `EvmState` updates arrive through the execution +//! hook, terminated when the hook is dropped after execution. +//! +//! Which path runs depends on runtime conditions (BAL present, caching and prewarming enabled), +//! so a sink must handle both. Access hints from prewarming are best-effort: they may be +//! missing, duplicated, or stale, and must not be treated as state updates. +//! +//! # Custom strategies +//! +//! Custom implementations can hold a [`DefaultStateRootStrategy`] and forward calls to it for +//! blocks where the default behavior is wanted, for example before a fork activates. See +//! `examples/custom-state-root` for the wiring. +//! +//! Returning empty trie updates in the outcome means the trie tables are no longer maintained: +//! `eth_getProof` and anything else that reads the stored trie will not work for new blocks. +//! Sparse-trie cache pruning derives retained paths from each block's hashed post state. + +mod sparse_trie; + +use self::sparse_trie::{SparseTrieCacheTask, SparseTrieTaskMetrics}; +use crate::tree::{ + metrics::BlockValidationMetrics, EngineApiTreeState, ExecutionEnv, StateProviderBuilder, + TreeConfig, +}; +use alloy_primitives::B256; +use crossbeam_channel::Receiver as CrossbeamReceiver; +use reth_chain_state::{ExecutedBlock, PreservedSparseTrie, StateTrieOverlayManager}; +use reth_errors::ProviderResult; +use reth_evm::{ConfigureEvm, OnStateHook}; +use reth_primitives_traits::{ + AlloyBlockHeader, FastInstant as Instant, NodePrimitives, RecoveredBlock, +}; +use reth_provider::{ + providers::OverlayStateProviderFactory, BlockExecutionOutput, BlockReader, + DatabaseProviderFactory, DatabaseProviderROFactory, HashedPostStateProvider, ProviderError, + StateProviderFactory, StateReader, StateRootProvider, +}; +use reth_tasks::utils::increase_thread_priority; +use reth_trie::{ + hashed_cursor::HashedCursorFactory, prefix_set::TriePrefixSetsMut, + trie_cursor::TrieCursorFactory, updates::TrieUpdates, HashedPostState, +}; +use reth_trie_parallel::proof_task::{ProofTaskCtx, ProofWorkerHandle}; +pub use reth_trie_parallel::{ + error::StateRootTaskError, + state_root_task::{ + evm_state_to_hashed_post_state, PayloadStateRootHandle, StateAccessHint, + StateRootComputeOutcome, StateRootHandle, StateRootHintStream, StateRootMessage, + StateRootSink, StateRootTaskCancelGuard, StateRootUpdateHook, StateRootUpdateStream, + }, +}; +#[cfg(feature = "trie-debug")] +use reth_trie_sparse::debug_recorder::TrieDebugRecorder; +use reth_trie_sparse::{ArenaParallelSparseTrie, RevealableSparseTrie, SparseStateTrie}; +use std::{ + fmt, + sync::{ + mpsc::{self, RecvTimeoutError}, + Arc, + }, + time::Duration, +}; +use tracing::{debug, debug_span, instrument, warn, Span}; + +/// Handle to a [`HashedPostState`] computed on a background thread. +pub type LazyHashedPostState = reth_tasks::LazyHandle>; + +/// Strategy used by engine-tree validation to prepare per-block state-root work. +pub trait StateRootStrategy: Send + Sync +where + N: NodePrimitives, + Evm: ConfigureEvm, +{ + /// Prepares a per-block state-root job before execution starts. + /// + /// A custom strategy that maintains a reusable sparse trie is responsible for consuming the + /// pending prune request from the context when it starts the corresponding job. + fn prepare( + &self, + ctx: StateRootJobContext<'_, N, P, Evm>, + ) -> ProviderResult>; + + /// Prepares the optional payload-builder state-root handle used for FCU-triggered block + /// building. + /// + /// This is consulted on every FCU that carries payload attributes. Returning `None` means the + /// payload builder computes the state root itself; the stock builders fall back to a + /// synchronous MPT state root. The default implementation returns `None`. + fn prepare_payload_builder( + &self, + _ctx: PayloadStateRootJobContext<'_, N, P>, + ) -> ProviderResult> { + Ok(None) + } +} + +/// Data available while preparing one payload-builder state-root handle. +pub struct PayloadStateRootJobContext<'a, N, P> +where + N: NodePrimitives, +{ + executor: &'a reth_tasks::Runtime, + state_trie_overlays: &'a StateTrieOverlayManager, + parent_hash: B256, + parent_header: &'a N::BlockHeader, + timestamp: u64, + state: &'a mut EngineApiTreeState, + provider_builder: StateProviderBuilder, + overlay_factory: OverlayStateProviderFactory, + config: &'a TreeConfig, +} + +impl fmt::Debug for PayloadStateRootJobContext<'_, N, P> +where + N: NodePrimitives, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PayloadStateRootJobContext") + .field("parent_hash", &self.parent_hash) + .field("parent_state_root", &self.parent_state_root()) + .field("timestamp", &self.timestamp) + .field("pending_sparse_trie_prune", &self.state.pending_sparse_trie_prune()) + .finish_non_exhaustive() + } +} + +impl<'a, N, P> PayloadStateRootJobContext<'a, N, P> +where + N: NodePrimitives, +{ + /// Creates a payload-builder state-root job context. + #[expect(clippy::too_many_arguments)] + pub(crate) const fn new( + executor: &'a reth_tasks::Runtime, + state_trie_overlays: &'a StateTrieOverlayManager, + parent_hash: B256, + parent_header: &'a N::BlockHeader, + timestamp: u64, + state: &'a mut EngineApiTreeState, + provider_builder: StateProviderBuilder, + overlay_factory: OverlayStateProviderFactory, + config: &'a TreeConfig, + ) -> Self { + Self { + executor, + state_trie_overlays, + parent_hash, + parent_header, + timestamp, + state, + provider_builder, + overlay_factory, + config, + } + } + + /// Returns the parent block hash for the payload being built. + pub const fn parent_hash(&self) -> B256 { + self.parent_hash + } + + /// Returns the parent block header for the payload being built. + /// + /// This is the chain's concrete header type, so chain-specific strategies can read + /// chain-specific fields, and number-activated forks can dispatch on the parent number. + pub const fn parent_header(&self) -> &N::BlockHeader { + self.parent_header + } + + /// Returns the parent state root for the payload being built. + pub fn parent_state_root(&self) -> B256 { + self.parent_header.state_root() + } + + /// Returns the timestamp of the payload being built, taken from the payload attributes. + /// + /// Strategies that switch behavior at a fork activation can dispatch on this value. + pub const fn timestamp(&self) -> u64 { + self.timestamp + } + + /// Returns the task runtime used by state-root work. + pub const fn executor(&self) -> &reth_tasks::Runtime { + self.executor + } + + /// Returns a clone of the state provider builder. + pub fn provider_builder(&self) -> StateProviderBuilder + where + P: Clone, + { + self.provider_builder.clone() + } + + /// Consumes the pending sparse trie prune request as in-memory parent-chain blocks, if any. + /// + /// Custom strategies that maintain a reusable sparse trie should call this when starting the + /// corresponding job. Strategies that do not use the request should leave it pending. + pub fn take_sparse_trie_prune_blocks(&mut self) -> Option>> { + self.state.take_sparse_trie_prune_blocks(self.parent_hash) + } +} + +/// Data available while preparing one state-root job. +pub struct StateRootJobContext<'a, N, P, Evm> +where + N: NodePrimitives, + Evm: ConfigureEvm, +{ + executor: &'a reth_tasks::Runtime, + state_trie_overlays: &'a StateTrieOverlayManager, + env: &'a ExecutionEnv, + provider_builder: StateProviderBuilder, + overlay_factory: OverlayStateProviderFactory, + config: &'a TreeConfig, + parallel_bal_execution: bool, + state: &'a mut EngineApiTreeState, +} + +impl fmt::Debug for StateRootJobContext<'_, N, P, Evm> +where + N: NodePrimitives, + Evm: ConfigureEvm, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("StateRootJobContext") + .field("parallel_bal_execution", &self.parallel_bal_execution) + .field("has_pending_sparse_trie_prune", &self.state.pending_sparse_trie_prune()) + .finish_non_exhaustive() + } +} + +impl<'a, N, P, Evm> StateRootJobContext<'a, N, P, Evm> +where + N: NodePrimitives, + Evm: ConfigureEvm, +{ + /// Creates a new state-root job context. + #[expect(clippy::too_many_arguments)] + pub(crate) const fn new( + executor: &'a reth_tasks::Runtime, + state_trie_overlays: &'a StateTrieOverlayManager, + env: &'a ExecutionEnv, + provider_builder: StateProviderBuilder, + overlay_factory: OverlayStateProviderFactory, + config: &'a TreeConfig, + parallel_bal_execution: bool, + state: &'a mut EngineApiTreeState, + ) -> Self { + Self { + executor, + state_trie_overlays, + env, + provider_builder, + overlay_factory, + config, + parallel_bal_execution, + state, + } + } + + /// Returns the execution environment for the block. + pub const fn env(&self) -> &ExecutionEnv { + self.env + } + + /// Returns the task runtime used by state-root work. + pub const fn executor(&self) -> &reth_tasks::Runtime { + self.executor + } + + /// Returns true when validation will use the parallel BAL execution path. + pub const fn parallel_bal_execution(&self) -> bool { + self.parallel_bal_execution + } + + /// Returns a clone of the state provider builder. + pub fn provider_builder(&self) -> StateProviderBuilder + where + P: Clone, + { + self.provider_builder.clone() + } + + /// Consumes the pending sparse trie prune request as in-memory parent-chain blocks, if any. + /// + /// Custom strategies that maintain a reusable sparse trie should call this when starting the + /// corresponding job. Strategies that do not use the request should leave it pending. + pub fn take_sparse_trie_prune_blocks(&mut self) -> Option>> { + self.state.take_sparse_trie_prune_blocks(self.env.parent_hash) + } +} + +/// Prepared per-block state-root work and its update-stream capabilities. +/// +/// The capabilities are populated by the strategy's `prepare` according to the execution +/// mode: the execution hook on the serial path, the hashed update stream on the parallel BAL +/// path, never both. Each capability is taken once by the code that produces its messages +/// and is not retained here, so the task's update channel closes when the producers are done. +pub struct PreparedStateRootJob { + job: Box>, + execution_hook: Option, + hint_stream: Option, + hashed_update_stream: Option, + hashed_state_rx: Option>>, +} + +impl fmt::Debug for PreparedStateRootJob { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PreparedStateRootJob") + .field("name", &self.job.name()) + .field("has_execution_hook", &self.execution_hook.is_some()) + .field("has_hint_stream", &self.hint_stream.is_some()) + .field("has_hashed_update_stream", &self.hashed_update_stream.is_some()) + .field("has_hashed_state_rx", &self.hashed_state_rx.is_some()) + .finish() + } +} + +impl PreparedStateRootJob { + /// Creates a prepared state-root job without update-stream capabilities. + pub const fn new( + job: Box>, + hashed_state_rx: Option>>, + ) -> Self { + Self { + job, + execution_hook: None, + hint_stream: None, + hashed_update_stream: None, + hashed_state_rx, + } + } + + /// Attaches the execution hook capability (serial execution path). + pub fn with_execution_hook(mut self, hook: StateRootUpdateHook) -> Self { + self.execution_hook = Some(hook); + self + } + + /// Attaches the hint stream capability. + pub fn with_hint_stream(mut self, hint_stream: StateRootHintStream) -> Self { + self.hint_stream = Some(hint_stream); + self + } + + /// Attaches the hashed update stream capability (parallel BAL path). + pub fn with_hashed_update_stream(mut self, stream: StateRootUpdateStream) -> Self { + self.hashed_update_stream = Some(stream); + self + } + + /// Returns the job name used in logs. + pub fn name(&self) -> &'static str { + self.job.name() + } + + /// Takes the execution hook, present only when the job wants normal execution updates. + pub fn take_execution_hook(&mut self) -> Option> { + self.execution_hook.take().map(|hook| Box::new(hook) as Box) + } + + /// Takes the hint stream for transaction prewarming. + pub const fn take_hint_stream(&mut self) -> Option { + self.hint_stream.take() + } + + /// Takes the hashed update stream, present only on the parallel BAL path. + pub const fn take_hashed_update_stream(&mut self) -> Option { + self.hashed_update_stream.take() + } + + /// Takes the optional hashed-state receiver produced by the job. + /// + /// The sender behind a returned receiver must either deliver one value or be dropped; + /// validation blocks on it while hashing the post state, so a job that keeps the sender + /// alive without sending stalls block validation. + pub const fn take_hashed_state_rx(&mut self) -> Option>> { + self.hashed_state_rx.take() + } + + /// Completes the job after execution. + pub fn finish( + &mut self, + block: &RecoveredBlock, + output: Arc>, + hashed_state: &LazyHashedPostState, + ) -> ProviderResult { + self.job.finish(block, output, hashed_state) + } +} + +/// Per-block state-root job prepared before execution and finished after execution. +pub trait StateRootJob: Send { + /// Human-readable strategy name used in logs. + fn name(&self) -> &'static str; + + /// Completes the job after execution. + /// + /// Called at most once per prepared job; implementations may panic if called again. + fn finish( + &mut self, + block: &RecoveredBlock, + output: Arc>, + hashed_state: &LazyHashedPostState, + ) -> ProviderResult; +} + +/// Outcome of a per-block state-root job. +#[derive(Debug)] +pub struct StateRootJobOutcome { + /// Computed state root. + pub state_root: B256, + /// Trie updates associated with the computed state root. + pub trie_updates: Arc, + /// Hashed post state recomputed by a fallback path. + /// + /// When set, the root was not derived from the streamed updates, so validation replaces its + /// streaming-derived hashed post state with this one and re-runs hashed-state checks. + pub hashed_state: Option>, +} + +impl StateRootJobOutcome { + /// Creates a state-root job outcome. + pub const fn new(state_root: B256, trie_updates: Arc) -> Self { + Self { state_root, trie_updates, hashed_state: None } + } + + /// Sets the hashed post state recomputed by a fallback path. + pub fn with_hashed_state(mut self, hashed_state: Option>) -> Self { + self.hashed_state = hashed_state; + self + } +} + +/// Receiver for the raced serial state-root fallback: root, trie updates, and the hashed +/// post state the fallback recomputed. +type SerialFallbackRx = mpsc::Receiver)>>; + +/// Default state-root strategy used by engine-tree validation. +/// +/// Covers the built-in modes: the sparse-trie state-root task, plus the skipped and +/// synchronous modes selected by [`TreeConfig`]. +/// +/// Custom strategies can hold this type and delegate to it for blocks where they want the +/// default behavior. +#[derive(Default)] +pub struct DefaultStateRootStrategy { + metrics: SparseTrieTaskMetrics, +} + +impl fmt::Debug for DefaultStateRootStrategy { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("DefaultStateRootStrategy").finish_non_exhaustive() + } +} + +impl DefaultStateRootStrategy { + /// Transaction count threshold below which proof workers are halved, since fewer transactions + /// produce fewer state changes and most workers would be idle overhead. + const SMALL_BLOCK_PROOF_WORKER_TX_THRESHOLD: usize = 30; + + /// Spawns the default state-root computation pipeline. + /// + /// The authoritative update capability taken from the returned handle must be dropped or + /// explicitly finished after execution so the task observes the end of the update stream. + /// An unknown transaction count uses the full proof-worker pool. + #[instrument(level = "debug", target = "engine::tree::payload_processor", skip_all)] + fn spawn_state_root( + &self, + executor: &reth_tasks::Runtime, + state_trie_overlays: &StateTrieOverlayManager, + multiproof_provider_factory: F, + options: StateRootTaskOptions<'_, N>, + ) -> StateRootHandle + where + N: NodePrimitives, + F: DatabaseProviderROFactory + + Clone + + Send + + Sync + + 'static, + { + let StateRootTaskOptions { + parent_hash, + parent_state_root, + preserved_sparse_trie, + transaction_count, + config, + pending_sparse_trie_prune_blocks, + } = options; + let (updates_tx, from_multi_proof) = crossbeam_channel::unbounded(); + let (cancel_guard, cancel_rx) = StateRootTaskCancelGuard::channel(); + + let task_ctx = ProofTaskCtx::new(multiproof_provider_factory); + #[cfg(feature = "trie-debug")] + let task_ctx = task_ctx.with_proof_jitter(config.proof_jitter()); + let halve_workers = transaction_count + .is_some_and(|count| count <= Self::SMALL_BLOCK_PROOF_WORKER_TX_THRESHOLD); + let proof_handle = ProofWorkerHandle::new(executor, task_ctx, halve_workers); + + let (state_root_tx, state_root_rx) = mpsc::channel(); + let (hashed_state_tx, hashed_state_rx) = mpsc::channel(); + + self.spawn_sparse_trie_task( + executor, + state_trie_overlays, + proof_handle, + state_root_tx, + hashed_state_tx, + from_multi_proof, + cancel_rx, + SparseTrieTaskOptions { + parent_hash, + parent_state_root, + preserved_sparse_trie, + chunk_size: config.multiproof_chunk_size(), + pending_sparse_trie_prune_blocks: if config.disable_sparse_trie_cache_pruning() { + None + } else { + pending_sparse_trie_prune_blocks + }, + max_hot_slots: config.sparse_trie_max_hot_slots(), + max_hot_accounts: config.sparse_trie_max_hot_accounts(), + }, + ); + + StateRootHandle::new( + parent_state_root, + updates_tx, + cancel_guard, + state_root_rx, + hashed_state_rx, + ) + } + + /// Spawns the sparse-trie task and preserves its trie for the next state-root job. + #[expect(clippy::too_many_arguments)] + fn spawn_sparse_trie_task( + &self, + executor: &reth_tasks::Runtime, + state_trie_overlays: &StateTrieOverlayManager, + proof_worker_handle: ProofWorkerHandle, + state_root_tx: mpsc::Sender>, + hashed_state_tx: mpsc::Sender>, + from_multi_proof: CrossbeamReceiver, + cancel_rx: CrossbeamReceiver<()>, + options: SparseTrieTaskOptions, + ) { + let SparseTrieTaskOptions { + parent_hash, + parent_state_root, + preserved_sparse_trie, + chunk_size, + pending_sparse_trie_prune_blocks, + max_hot_slots, + max_hot_accounts, + } = options; + let state_trie_overlays = state_trie_overlays.clone(); + let trie_metrics = self.metrics.clone(); + let executor = executor.clone(); + + let parent_span = Span::current(); + executor.clone().spawn_blocking_named("sparse-trie", move || { + reth_tasks::once!(increase_thread_priority); + + let _enter = debug_span!( + target: "engine::tree::payload_processor", + parent: parent_span, + "sparse_trie_task" + ) + .entered(); + + let new_sparse_state_trie = || { + debug!( + target: "engine::tree::payload_processor", + "Creating new sparse trie - no preserved trie available" + ); + let default_trie = + RevealableSparseTrie::blind_from(ArenaParallelSparseTrie::default()); + SparseStateTrie::default() + .with_accounts_trie(default_trie.clone()) + .with_default_storage_trie(default_trie) + .with_updates(true) + }; + + let mut sparse_trie_anchor_hash = parent_hash; + let mut reused_preserved_sparse_trie = false; + let mut sparse_state_trie = match preserved_sparse_trie { + Some(preserved) => { + let start = Instant::now(); + let preserved_anchor_hash = preserved.anchor_hash(); + let preserved = preserved.into_trie_for(parent_state_root); + trie_metrics + .sparse_trie_cache_wait_duration_histogram + .record(start.elapsed().as_secs_f64()); + + match preserved { + Ok(Some(trie)) => { + sparse_trie_anchor_hash = preserved_anchor_hash; + reused_preserved_sparse_trie = true; + trie + } + Ok(None) => new_sparse_state_trie(), + Err(err) => { + let _ = + state_root_tx.send(Err(StateRootTaskError::Other(err.to_string()))); + return; + } + } + } + None => new_sparse_state_trie(), + }; + sparse_state_trie.set_hot_cache_capacities(max_hot_slots, max_hot_accounts); + + let mut task = SparseTrieCacheTask::new_with_trie( + &executor, + from_multi_proof, + cancel_rx, + hashed_state_tx, + proof_worker_handle, + trie_metrics.clone(), + sparse_state_trie, + parent_state_root, + chunk_size, + ); + + let result = task.run(); + let task_result = result.as_ref().ok().cloned(); + + // Publish a handle before sending the result so the next block can inspect the + // state root immediately while the trie is finalized for reuse below. + let pending_trie = if let Some(result) = &task_result { + let preserved_anchor_hash = published_sparse_trie_anchor_hash( + sparse_trie_anchor_hash, + reused_preserved_sparse_trie, + pending_sparse_trie_prune_blocks.as_deref(), + ); + let (preserved, completer) = + PreservedSparseTrie::pending(result.state_root, preserved_anchor_hash); + state_trie_overlays.store_sparse_trie(preserved); + Some(completer) + } else { + state_trie_overlays.clear_sparse_trie(); + None + }; + + if state_root_tx.send(result).is_err() { + // A continuation task can take the pending trie during the narrow window between + // publishing it and detecting the abandoned receiver here. Returning drops the + // completer, so the taker wakes with `ProducerDropped` and its state-root consumer + // falls back to serial computation. No partially finalized trie is exposed; the + // worst case is a redundant fallback. + debug!( + target: "engine::tree::payload_processor", + "State root receiver dropped, dropping trie" + ); + let (trie, deferred) = task.into_cleared_trie(); + state_trie_overlays.clear_sparse_trie(); + executor.spawn_drop(trie); + executor.spawn_drop(deferred); + return; + } + + let _enter = + debug_span!(target: "engine::tree::payload_processor", "preserve").entered(); + let mut trie_to_drop = None; + let deferred = if let Some(result) = task_result { + let pending_trie = + pending_trie.expect("pending trie is created for successful task result"); + let start = Instant::now(); + let (mut trie, deferred) = task.into_trie_for_reuse(); + if let Some(prune_blocks) = pending_sparse_trie_prune_blocks { + let retained_paths = + sparse_trie_retained_paths(prune_blocks, result.hashed_state.as_ref()); + trie.prune(max_hot_slots, max_hot_accounts, retained_paths); + } + trie_metrics + .into_trie_for_reuse_duration_histogram + .record(start.elapsed().as_secs_f64()); + trie_metrics.sparse_trie_retained_memory_bytes.set(trie.memory_size() as f64); + trie_metrics + .sparse_trie_retained_storage_tries + .set(trie.retained_storage_tries_count() as f64); + if let Err(trie) = pending_trie.complete(trie) { + trie_to_drop = Some(trie); + } + deferred + } else { + debug!( + target: "engine::tree::payload_processor", + "State root computation failed, dropping trie" + ); + let (trie, deferred) = task.into_cleared_trie(); + trie_to_drop = Some(trie); + deferred + }; + if let Some(trie) = trie_to_drop { + executor.spawn_drop(trie); + } + executor.spawn_drop(deferred); + }); + } +} + +struct SparseTrieTaskOptions { + parent_hash: B256, + parent_state_root: B256, + preserved_sparse_trie: Option, + chunk_size: usize, + /// `None` disables pruning. `Some(Vec::new())` prunes using only the current block's paths. + pending_sparse_trie_prune_blocks: Option>>, + max_hot_slots: usize, + max_hot_accounts: usize, +} + +struct StateRootTaskOptions<'a, N: NodePrimitives> { + parent_hash: B256, + parent_state_root: B256, + preserved_sparse_trie: Option, + transaction_count: Option, + config: &'a TreeConfig, + pending_sparse_trie_prune_blocks: Option>>, +} + +fn sparse_trie_retained_paths( + prune_blocks: Vec>, + current_hashed_state: &HashedPostState, +) -> TriePrefixSetsMut { + let mut retained_paths = TriePrefixSetsMut::default(); + for block in prune_blocks { + let trie_data = block.trie_data(); + retained_paths.extend(trie_data.sorted.hashed_state.construct_prefix_sets()); + } + retained_paths.extend(current_hashed_state.construct_prefix_sets()); + retained_paths +} + +fn published_sparse_trie_anchor_hash( + sparse_trie_anchor_hash: B256, + reused_preserved_sparse_trie: bool, + pending_sparse_trie_prune_blocks: Option<&[ExecutedBlock]>, +) -> B256 { + if !reused_preserved_sparse_trie { + return sparse_trie_anchor_hash + } + + let Some(prune_blocks) = pending_sparse_trie_prune_blocks else { + return sparse_trie_anchor_hash + }; + let Some(oldest_prune_block) = prune_blocks.last() else { return sparse_trie_anchor_hash }; + + // Prune blocks contain the complete in-memory parent chain from newest to oldest, with the + // oldest block's parent being the persisted tip. A fresh trie can be anchored to an in-memory + // block ahead of that tip. If that anchor is still in the prune range, publishing the + // persisted tip as the new anchor would expand the trie's claimed coverage backwards even + // though pruning cannot reveal those paths. + if prune_blocks.iter().any(|block| block.recovered_block().hash() == sparse_trie_anchor_hash) { + return sparse_trie_anchor_hash + } + + oldest_prune_block.recovered_block().parent_hash() +} + +impl StateRootStrategy for DefaultStateRootStrategy +where + N: NodePrimitives, + P: DatabaseProviderFactory + + BlockReader
+ + StateProviderFactory + + StateReader + + Clone + + 'static, + OverlayStateProviderFactory: DatabaseProviderROFactory + + Clone + + Send + + Sync + + 'static, + Evm: ConfigureEvm + 'static, +{ + fn prepare( + &self, + mut ctx: StateRootJobContext<'_, N, P, Evm>, + ) -> ProviderResult> { + if ctx.config.skip_state_root() { + return Ok(PreparedStateRootJob::new(Box::new(SkippedStateRootJob {}), None)) + } + + if !ctx.config.use_state_root_task() { + return Ok(PreparedStateRootJob::new( + Box::new(SynchronousStateRootJob { provider_builder: ctx.provider_builder }), + None, + )) + } + + let pending_sparse_trie_prune_blocks = ctx.take_sparse_trie_prune_blocks(); + let StateRootJobContext { + executor, + state_trie_overlays, + env, + provider_builder, + overlay_factory, + config, + parallel_bal_execution, + state: _, + } = ctx; + + let preserved_sparse_trie = state_trie_overlays.take_sparse_trie(); + let overlay_factory = if let Some(anchor_hash) = preserved_sparse_trie + .as_ref() + .filter(|trie| trie.state_root() == env.parent_state_root) + .map(|trie| trie.anchor_hash()) + { + overlay_factory.with_skip_overlay_for_reused_sparse_trie(anchor_hash) + } else { + overlay_factory + }; + + let mut handle = self.spawn_state_root( + executor, + state_trie_overlays, + overlay_factory.clone(), + StateRootTaskOptions { + parent_hash: env.parent_hash, + parent_state_root: env.parent_state_root, + preserved_sparse_trie, + transaction_count: Some(env.transaction_count), + config, + pending_sparse_trie_prune_blocks, + }, + ); + + // The execution mode decides who finishes the update stream: the execution hook on + // the serial path, the BAL streamer on the parallel path. Both come from one slot in + // the handle, so only one of them can exist. + let (hashed_update_stream, execution_hook): ( + Option, + Option, + ) = match parallel_bal_execution { + true => (Some(handle.take_hashed_update_stream()), None), + false => (None, Some(handle.take_execution_hook())), + }; + let hint_stream = handle.take_hint_stream(); + + let hashed_state_rx = Some(handle.take_hashed_state_rx()); + + let mut prepared = PreparedStateRootJob::new( + Box::new(SparseTrieStateRootJob { + handle, + provider_builder, + overlay_factory, + executor: executor.clone(), + timeout: config.state_root_task_timeout(), + compare_trie_updates: config.always_compare_trie_updates(), + metrics: BlockValidationMetrics::default(), + }), + hashed_state_rx, + ) + .with_hint_stream(hint_stream); + if let Some(hook) = execution_hook { + prepared = prepared.with_execution_hook(hook); + } + if let Some(stream) = hashed_update_stream { + prepared = prepared.with_hashed_update_stream(stream); + } + Ok(prepared) + } + + fn prepare_payload_builder( + &self, + mut ctx: PayloadStateRootJobContext<'_, N, P>, + ) -> ProviderResult> { + // Sharing the engine state-root task with the payload builder is opt-in, and needs a + // host that can run the task pipeline at all. + if !ctx.config.share_sparse_trie_with_payload_builder() || + ctx.config.skip_state_root() || + !ctx.config.has_enough_parallelism() + { + return Ok(None) + } + + let pending_sparse_trie_prune_blocks = ctx.take_sparse_trie_prune_blocks(); + let parent_state_root = ctx.parent_state_root(); + let preserved_sparse_trie = ctx.state_trie_overlays.take_sparse_trie(); + let overlay_factory = if let Some(anchor_hash) = preserved_sparse_trie + .as_ref() + .filter(|trie| trie.state_root() == parent_state_root) + .map(|trie| trie.anchor_hash()) + { + ctx.overlay_factory.clone().with_skip_overlay_for_reused_sparse_trie(anchor_hash) + } else { + ctx.overlay_factory.clone() + }; + Ok(Some( + self.spawn_state_root( + ctx.executor, + ctx.state_trie_overlays, + overlay_factory, + StateRootTaskOptions { + parent_hash: ctx.parent_hash(), + parent_state_root, + preserved_sparse_trie, + // Tx count unknown at FCU time (block built incrementally): full proof workers. + transaction_count: None, + config: ctx.config, + pending_sparse_trie_prune_blocks, + }, + ) + .into_payload_state_root_handle(), + )) + } +} + +#[derive(Debug)] +struct SkippedStateRootJob {} + +impl StateRootJob for SkippedStateRootJob { + fn name(&self) -> &'static str { + "skipped" + } + + fn finish( + &mut self, + block: &RecoveredBlock, + _output: Arc>, + _hashed_state: &LazyHashedPostState, + ) -> ProviderResult { + Ok(StateRootJobOutcome::new(block.header().state_root(), Arc::new(TrieUpdates::default()))) + } +} + +#[derive(Debug)] +struct SynchronousStateRootJob { + provider_builder: StateProviderBuilder, +} + +impl StateRootJob for SynchronousStateRootJob +where + N: NodePrimitives, + P: BlockReader + StateProviderFactory + StateReader + Clone + Send + Sync + 'static, +{ + fn name(&self) -> &'static str { + "synchronous" + } + + fn finish( + &mut self, + _block: &RecoveredBlock, + _output: Arc>, + hashed_state: &LazyHashedPostState, + ) -> ProviderResult { + let provider = self.provider_builder.clone().build()?; + let (state_root, trie_updates) = + provider.state_root_with_updates(hashed_state.get().as_ref().clone())?; + Ok(StateRootJobOutcome::new(state_root, Arc::new(trie_updates))) + } +} + +#[derive(Debug)] +struct SparseTrieStateRootJob { + handle: StateRootHandle, + provider_builder: StateProviderBuilder, + overlay_factory: OverlayStateProviderFactory, + executor: reth_tasks::Runtime, + timeout: Option, + compare_trie_updates: bool, + metrics: BlockValidationMetrics, +} + +impl SparseTrieStateRootJob +where + N: NodePrimitives, + P: StateProviderFactory + Clone + Send + Sync + 'static, + P: BlockReader + StateReader, + OverlayStateProviderFactory: DatabaseProviderROFactory + + Clone + + Send + + Sync + + 'static, +{ + fn serial_fallback( + executor: &reth_tasks::Runtime, + provider_builder: StateProviderBuilder, + output: Arc>, + ) -> ProviderResult { + let provider = provider_builder.build()?; + let (fallback_tx, fallback_rx) = mpsc::channel(); + executor.spawn_blocking_named("serial-root", move || { + let result = (|| { + let hashed_state = Arc::new(provider.hashed_post_state(&output.state)); + let (root, updates) = + provider.state_root_with_updates(hashed_state.as_ref().clone())?; + Ok((root, updates, hashed_state)) + })(); + let _ = fallback_tx.send(result); + }); + + Ok(fallback_rx) + } + + /// Recomputes the state root serially from the execution output. + /// + /// Used when the state-root task failed or produced a wrong root, so the recomputed hashed + /// post state is returned in the outcome for validation to re-check against. + fn compute_serial( + &self, + output: &BlockExecutionOutput, + ) -> ProviderResult { + let provider = self.provider_builder.clone().build()?; + let hashed_state = Arc::new(provider.hashed_post_state(&output.state)); + let (state_root, trie_updates) = + provider.state_root_with_updates(hashed_state.as_ref().clone())?; + self.metrics.state_root_task_fallback_success_total.increment(1); + Ok(StateRootJobOutcome::new(state_root, Arc::new(trie_updates)) + .with_hashed_state(Some(hashed_state))) + } + + /// Converts a task outcome into a job outcome, recomputing serially when the task returned + /// a root that does not match the block header. A state-root-task bug then costs latency + /// instead of marking a valid block invalid; if the serial root also mismatches, validation + /// rejects the block. + fn verified_sparse_outcome( + &self, + block: &RecoveredBlock, + output: &BlockExecutionOutput, + outcome: StateRootComputeOutcome, + ) -> ProviderResult { + let outcome = self.sparse_outcome(block, output, outcome); + if outcome.state_root == block.header().state_root() { + return Ok(outcome) + } + warn!( + target: "engine::tree::state_root_strategy", + state_root = ?outcome.state_root, + block_state_root = ?block.header().state_root(), + "State root task returned incorrect state root, recomputing serially" + ); + self.compute_serial(output) + } + + fn sparse_outcome( + &self, + _block: &RecoveredBlock, + output: &BlockExecutionOutput, + outcome: StateRootComputeOutcome, + ) -> StateRootJobOutcome { + let StateRootComputeOutcome { + state_root, + trie_updates, + hashed_state: _hashed_state, + #[cfg(feature = "trie-debug")] + debug_recorders, + } = outcome; + + if self.compare_trie_updates { + let _has_diff = compare_trie_updates_with_serial( + self.provider_builder.clone(), + self.overlay_factory.clone(), + output, + trie_updates.as_ref().clone(), + ); + #[cfg(feature = "trie-debug")] + if _has_diff { + write_trie_debug_recorders(_block.header().number(), &debug_recorders); + } + } + + #[cfg(feature = "trie-debug")] + if state_root != _block.header().state_root() { + write_trie_debug_recorders(_block.header().number(), &debug_recorders); + } + + StateRootJobOutcome::new(state_root, trie_updates) + } +} + +impl StateRootJob for SparseTrieStateRootJob +where + N: NodePrimitives, + P: BlockReader + StateProviderFactory + StateReader + Clone + Send + Sync + 'static, + OverlayStateProviderFactory: DatabaseProviderROFactory + + Clone + + Send + + Sync + + 'static, +{ + fn name(&self) -> &'static str { + "sparse-trie" + } + + fn finish( + &mut self, + block: &RecoveredBlock, + output: Arc>, + _hashed_state: &LazyHashedPostState, + ) -> ProviderResult { + if self.timeout.is_none() { + return match self.handle.state_root() { + Ok(outcome) => self.verified_sparse_outcome(block, &output, outcome), + Err(err) => { + debug!(target: "engine::tree::state_root_strategy", %err, "State root task failed, falling back to serial root"); + self.compute_serial(&output) + } + } + } + + let timeout = self.timeout.expect("checked above"); + let task_rx = self.handle.take_state_root_rx(); + let fallback_rx = match task_rx.recv_timeout(timeout) { + Ok(Ok(outcome)) => return self.verified_sparse_outcome(block, &output, outcome), + Ok(Err(err)) => { + debug!(target: "engine::tree::state_root_strategy", %err, "State root task failed, falling back to serial root"); + Self::serial_fallback( + &self.executor, + self.provider_builder.clone(), + output.clone(), + )? + } + Err(RecvTimeoutError::Timeout) => { + warn!(target: "engine::tree::state_root_strategy", ?timeout, "State root task timed out, racing serial fallback"); + self.metrics.state_root_task_timeout_total.increment(1); + Self::serial_fallback( + &self.executor, + self.provider_builder.clone(), + output.clone(), + )? + } + Err(RecvTimeoutError::Disconnected) => { + debug!(target: "engine::tree::state_root_strategy", "State root task dropped, falling back to serial root"); + Self::serial_fallback( + &self.executor, + self.provider_builder.clone(), + output.clone(), + )? + } + }; + + loop { + if let Ok(Ok(outcome)) = task_rx.try_recv() { + let outcome = self.sparse_outcome(block, &output, outcome); + if outcome.state_root == block.header().state_root() { + return Ok(outcome) + } + // A wrong task root falls through to the serial fallback already racing below. + warn!( + target: "engine::tree::state_root_strategy", + state_root = ?outcome.state_root, + block_state_root = ?block.header().state_root(), + "State root task returned incorrect state root, using serial fallback" + ); + } + + match fallback_rx.try_recv() { + Ok(Ok((state_root, trie_updates, hashed_state))) => { + self.metrics.state_root_task_fallback_success_total.increment(1); + return Ok(StateRootJobOutcome::new(state_root, Arc::new(trie_updates)) + .with_hashed_state(Some(hashed_state))) + } + Ok(Err(err)) => return Err(err), + Err(mpsc::TryRecvError::Empty) => {} + Err(mpsc::TryRecvError::Disconnected) => { + return Err(ProviderError::other(std::io::Error::other( + "serial state root fallback task dropped", + ))) + } + } + + std::thread::sleep(Duration::from_millis(1)); + } + } +} + +fn compare_trie_updates_with_serial( + state_provider_builder: StateProviderBuilder, + overlay_factory: OverlayStateProviderFactory, + output: &BlockExecutionOutput, + task_trie_updates: TrieUpdates, +) -> bool +where + N: NodePrimitives, + P: BlockReader + StateProviderFactory + StateReader + Clone, + OverlayStateProviderFactory: + DatabaseProviderROFactory, +{ + debug!(target: "engine::tree::state_root_strategy", "Comparing trie updates with serial computation"); + + match state_provider_builder.build().and_then(|provider| { + let hashed_state = provider.hashed_post_state(&output.state); + provider.state_root_with_updates(hashed_state) + }) { + Ok((serial_root, serial_trie_updates)) => { + debug!( + target: "engine::tree::state_root_strategy", + ?serial_root, + "Serial state root computation finished for comparison" + ); + + match overlay_factory.database_provider_ro() { + Ok(provider) => match super::trie_updates::compare_trie_updates( + &provider, + task_trie_updates, + serial_trie_updates, + ) { + Ok(has_diff) => return has_diff, + Err(err) => { + warn!( + target: "engine::tree::state_root_strategy", + %err, + "Error comparing trie updates" + ); + return true; + } + }, + Err(err) => { + warn!( + target: "engine::tree::state_root_strategy", + %err, + "Failed to get database provider for trie update comparison" + ); + } + } + } + Err(err) => { + warn!( + target: "engine::tree::state_root_strategy", + %err, + "Failed to compute serial state root for comparison" + ); + } + } + false +} + +/// Writes trie debug recorders to a JSON file for the given block number. +/// +/// The file is written to the current working directory as `trie_debug_block_{block_number}.json`. +#[cfg(feature = "trie-debug")] +fn write_trie_debug_recorders(block_number: u64, recorders: &[(Option, TrieDebugRecorder)]) { + let path = format!("trie_debug_block_{block_number}.json"); + match serde_json::to_string_pretty(recorders) { + Ok(json) => match std::fs::write(&path, json) { + Ok(()) => { + warn!( + target: "engine::tree::state_root_strategy", + %path, + "Wrote trie debug recorders to file" + ); + } + Err(err) => { + warn!( + target: "engine::tree::state_root_strategy", + %err, + %path, + "Failed to write trie debug recorders" + ); + } + }, + Err(err) => { + warn!( + target: "engine::tree::state_root_strategy", + %err, + "Failed to serialize trie debug recorders" + ); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use alloy_consensus::constants::KECCAK_EMPTY; + use alloy_primitives::{map::HashMap, Address, U256}; + use rand::Rng; + use reth_chain_state::{test_utils::TestBlockBuilder, StateTrieOverlayManager}; + use reth_chainspec::ChainSpec; + use reth_db_common::init::init_genesis; + use reth_ethereum_primitives::EthPrimitives; + use reth_evm::OnStateHook; + use reth_evm_ethereum::EthEvmConfig; + use reth_primitives_traits::{Account, StorageEntry}; + use reth_provider::{ + providers::{BlockchainProvider, OverlayBuilder, OverlayStateProviderFactory}, + test_utils::create_test_provider_factory_with_chain_spec, + HashingWriter, + }; + use reth_testing_utils::generators; + use reth_trie::{test_utils::state_root, HashedPostState, HashedStorage, LazyTrieData}; + use reth_trie_db::ChangesetCache; + use revm::state::{AccountInfo, AccountStatus, EvmState, EvmStorageSlot, TransactionId}; + + fn with_hashed_state( + block: ExecutedBlock, + hashed_state: HashedPostState, + ) -> ExecutedBlock { + let mut trie_data = block.trie_data(); + trie_data.sorted.hashed_state = Arc::new(hashed_state.into_sorted()); + ExecutedBlock::with_deferred_trie_data( + block.recovered_block, + block.execution_output, + LazyTrieData::ready(trie_data), + ) + } + + fn trie_hashed_state(account_path: u8, storage_path: u8) -> HashedPostState { + HashedPostState::default() + .with_accounts([(B256::with_last_byte(account_path), Some(Account::default()))]) + .with_storages([( + B256::with_last_byte(account_path), + HashedStorage::from_iter(false, [(B256::with_last_byte(storage_path), U256::ONE)]), + )]) + } + + #[test] + fn sparse_trie_retained_paths_merges_prune_blocks_with_current_block() { + let blocks: Vec<_> = TestBlockBuilder::eth() + .get_executed_blocks(1..3) + .zip([trie_hashed_state(0x01, 0x02), trie_hashed_state(0x03, 0x04)]) + .map(|(block, hashed_state)| with_hashed_state(block, hashed_state)) + .collect(); + let current_hashed_state = trie_hashed_state(0x05, 0x06); + + let retained_paths = sparse_trie_retained_paths(blocks, ¤t_hashed_state).freeze(); + + assert_eq!(retained_paths.account_prefix_set.len(), 3); + assert_eq!(retained_paths.storage_prefix_sets.len(), 3); + } + + #[test] + fn sparse_trie_retained_paths_uses_hashed_state() { + let blocks: Vec<_> = TestBlockBuilder::eth() + .get_executed_blocks(1..2) + .map(|block| with_hashed_state(block, trie_hashed_state(0x01, 0x02))) + .collect(); + let current_hashed_state = trie_hashed_state(0x03, 0x04); + + let retained_paths = sparse_trie_retained_paths(blocks, ¤t_hashed_state).freeze(); + + assert_eq!(retained_paths.account_prefix_set.len(), 2); + assert_eq!(retained_paths.storage_prefix_sets.len(), 2); + } + + #[test] + fn published_sparse_trie_anchor_advances_to_prune_anchor() { + let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(1..5).collect(); + let reused_anchor_hash = blocks[0].recovered_block().hash(); + let expected_prune_anchor = blocks[1].recovered_block().hash(); + let prune_blocks: Vec<_> = blocks.into_iter().skip(2).rev().collect(); + + assert_eq!( + published_sparse_trie_anchor_hash(reused_anchor_hash, true, Some(&prune_blocks)), + expected_prune_anchor + ); + } + + #[test] + fn published_sparse_trie_anchor_does_not_move_backwards_when_anchor_is_in_prune_range() { + let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(1..5).collect(); + let reused_anchor_hash = blocks[2].recovered_block().hash(); + let mut prune_blocks = blocks; + prune_blocks.reverse(); + let prune_anchor = prune_blocks.last().unwrap().recovered_block().parent_hash(); + + assert_ne!(reused_anchor_hash, prune_anchor); + assert_eq!( + published_sparse_trie_anchor_hash(reused_anchor_hash, true, Some(&prune_blocks)), + reused_anchor_hash + ); + } + + #[test] + fn published_sparse_trie_anchor_keeps_parent_for_fresh_trie() { + let mut blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(1..3).collect(); + blocks.reverse(); + let parent_hash = B256::with_last_byte(0xaa); + + assert_eq!( + published_sparse_trie_anchor_hash(parent_hash, false, Some(&blocks)), + parent_hash + ); + } + + fn create_mock_state_updates(num_accounts: usize, updates_per_account: usize) -> Vec { + let mut rng = generators::rng(); + let all_addresses: Vec
= (0..num_accounts).map(|_| rng.random()).collect(); + let mut updates = Vec::with_capacity(updates_per_account); + + for _ in 0..updates_per_account { + let num_accounts_in_update = rng.random_range(1..=num_accounts); + let mut state_update = EvmState::default(); + + for &address in &all_addresses[0..num_accounts_in_update] { + let mut storage = HashMap::default(); + if rng.random_bool(0.7) { + for _ in 0..rng.random_range(1..10) { + let slot = U256::from(rng.random::()); + storage.insert( + slot, + EvmStorageSlot::new_changed( + U256::ZERO, + U256::from(rng.random::()), + TransactionId::ZERO, + ), + ); + } + } + + let mut account = revm::state::Account::default(); + account.info = AccountInfo { + balance: U256::from(rng.random::()), + nonce: rng.random::(), + code_hash: KECCAK_EMPTY, + code: Some(Default::default()), + account_id: None, + }; + account.storage = storage; + account.status = AccountStatus::Touched; + account.transaction_id = TransactionId::ZERO; + state_update.insert(address, account); + } + + updates.push(state_update); + } + + updates + } + + #[test] + fn state_root_task_matches_serial_root() { + reth_tracing::init_test_tracing(); + + let factory = create_test_provider_factory_with_chain_spec(Arc::new(ChainSpec::default())); + let genesis_hash = init_genesis(&factory).unwrap(); + let state_updates = create_mock_state_updates(10, 10); + let mut accumulated_state: HashMap)> = + HashMap::default(); + + { + let provider_rw = factory.provider_rw().expect("failed to get provider"); + for update in &state_updates { + let account_updates = update.iter().map(|(address, account)| { + (*address, Some(Account::from_revm_account(account))) + }); + provider_rw + .insert_account_for_hashing(account_updates) + .expect("failed to insert accounts"); + + let storage_updates = update.iter().map(|(address, account)| { + let storage_entries = account.storage.iter().map(|(slot, value)| { + StorageEntry { key: B256::from(*slot), value: value.present_value } + }); + (*address, storage_entries) + }); + provider_rw + .insert_storage_for_hashing(storage_updates) + .expect("failed to insert storage"); + } + provider_rw.commit().expect("failed to commit changes"); + } + + for update in &state_updates { + for (address, account) in update { + let storage: HashMap = account + .storage + .iter() + .map(|(key, value)| (B256::from(*key), value.present_value)) + .collect(); + let entry = accumulated_state.entry(*address).or_default(); + entry.0 = Account::from_revm_account(account); + entry.1.extend(storage); + } + } + + let provider_factory = BlockchainProvider::new(factory).unwrap(); + let env: ExecutionEnv = ExecutionEnv::test_default(); + let runtime = reth_tasks::Runtime::test(); + let state_trie_overlays = StateTrieOverlayManager::::default(); + let mut state_root_handle = DefaultStateRootStrategy::default().spawn_state_root( + &runtime, + &state_trie_overlays, + OverlayStateProviderFactory::new( + provider_factory, + OverlayBuilder::::new(genesis_hash, ChangesetCache::new()), + ), + StateRootTaskOptions { + parent_hash: genesis_hash, + parent_state_root: env.parent_state_root, + preserved_sparse_trie: None, + transaction_count: Some(env.transaction_count), + config: &TreeConfig::default(), + pending_sparse_trie_prune_blocks: None, + }, + ); + + let mut state_hook = state_root_handle.take_execution_hook(); + for update in state_updates { + state_hook.on_state(update); + } + drop(state_hook); + + let root_from_task = state_root_handle.state_root().expect("task failed").state_root; + let root_from_regular = state_root(accumulated_state); + assert_eq!(root_from_task, root_from_regular); + } +} diff --git a/crates/engine/tree/src/tree/payload_processor/sparse_trie.rs b/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs similarity index 62% rename from crates/engine/tree/src/tree/payload_processor/sparse_trie.rs rename to crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs index eda038507b0..03f6f839619 100644 --- a/crates/engine/tree/src/tree/payload_processor/sparse_trie.rs +++ b/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs @@ -2,20 +2,16 @@ use std::sync::Arc; -use crate::tree::{ - multiproof::{ - dispatch_with_chunking, evm_state_to_hashed_post_state, StateRootComputeOutcome, - StateRootMessage, DEFAULT_MAX_TARGETS_FOR_CHUNKING, - }, - payload_processor::multiproof::MultiProofTaskMetrics, -}; +use super::{evm_state_to_hashed_post_state, StateRootComputeOutcome, StateRootMessage}; use alloy_primitives::{ map::{hash_map::Entry, B256Map}, B256, }; use alloy_rlp::{Decodable, Encodable}; use crossbeam_channel::{Receiver as CrossbeamReceiver, Sender as CrossbeamSender}; +use metrics::{Gauge, Histogram}; use rayon::iter::{IntoParallelIterator, ParallelIterator}; +use reth_metrics::Metrics; use reth_primitives_traits::{Account, FastInstant as Instant}; use reth_tasks::Runtime; use reth_trie::{ @@ -44,8 +40,12 @@ pub(super) struct SparseTrieCacheTask, /// Receives updates from execution and prewarming. updates: CrossbeamReceiver, + /// Fires (by disconnecting) when the consumer drops its cancel guard, meaning nobody is + /// waiting for the result anymore. This is the teardown path for a task whose pending + /// work never drains, since the updates channel closing is a normal end of stream. + cancel_rx: CrossbeamReceiver<()>, /// Sender half for the channel to send final hashed state to. - final_hashed_state_tx: Option>, + final_hashed_state_tx: Option>>, /// `SparseStateTrie` used for computing the state root. trie: SparseStateTrie, /// The parent block's state root. @@ -104,6 +104,8 @@ pub(super) struct SparseTrieCacheTask SparseTrieCacheTask @@ -128,9 +130,10 @@ where pub(super) fn new_with_trie( executor: &Runtime, updates: CrossbeamReceiver, - final_hashed_state_tx: std::sync::mpsc::Sender, + cancel_rx: CrossbeamReceiver<()>, + final_hashed_state_tx: std::sync::mpsc::Sender>, proof_worker_handle: ProofWorkerHandle, - metrics: MultiProofTaskMetrics, + metrics: SparseTrieTaskMetrics, trie: SparseStateTrie, parent_state_root: B256, chunk_size: usize, @@ -149,6 +152,7 @@ where proof_result_tx, proof_result_rx, updates: hashed_state_rx, + cancel_rx, proof_worker_handle, final_hashed_state_tx: Some(final_hashed_state_tx), trie, @@ -169,6 +173,7 @@ where storage_cache_hits: 0, storage_cache_misses: 0, pending_targets: Default::default(), + in_flight_proof_batches: 0, pending_updates: Default::default(), final_hashed_state: Default::default(), metrics, @@ -180,7 +185,7 @@ where fn run_hashing_task( updates: CrossbeamReceiver, hashed_state_tx: CrossbeamSender, - metrics: MultiProofTaskMetrics, + metrics: SparseTrieTaskMetrics, ) { let mut total_idle_time = std::time::Duration::ZERO; let mut idle_start = Instant::now(); @@ -251,95 +256,74 @@ where let mut total_idle_time = std::time::Duration::ZERO; let mut idle_start = Instant::now(); + let mut done = false; + let mut finalized_hashed_state = None; - loop { + // Streaming phase: updates are still arriving. Ends when the finish marker is + // processed. Only producers hold update senders, so the channel closing before the + // marker means they died without finishing the stream. + while !self.finished_state_updates { let mut t = Instant::now(); crossbeam_channel::select_biased! { recv(self.updates) -> message => { let wake = Instant::now(); - - let update = match message { - Ok(m) => m, - Err(_) => { - return Err(StateRootTaskError::Other( - "updates channel disconnected before state root calculation".to_string(), - )) - } - }; - total_idle_time += wake.duration_since(idle_start); self.metrics .sparse_trie_channel_wait_duration_histogram .record(wake.duration_since(t)); - self.on_message(update); + let update = message.map_err(|_| StateRootTaskError::Other( + "updates channel disconnected before state root calculation".to_string(), + ))?; + if let Some(hashed_state) = self.on_message(update) { + finalized_hashed_state = Some(hashed_state); + } self.pending_updates += 1; } recv(self.proof_result_rx) -> message => { - let phase_end = Instant::now(); - total_idle_time += phase_end.duration_since(idle_start); + let wake = Instant::now(); + total_idle_time += wake.duration_since(idle_start); self.metrics .sparse_trie_channel_wait_duration_histogram - .record(phase_end.duration_since(t)); - t = phase_end; + .record(wake.duration_since(t)); + t = wake; let Ok(result) = message else { unreachable!("we own the sender half") }; - - let mut result = result.result?; - while let Ok(next) = self.proof_result_rx.try_recv() { - let res = next.result?; - result.extend(res); - } - - let phase_end = Instant::now(); - self.metrics - .sparse_trie_proof_coalesce_duration_histogram - .record(phase_end.duration_since(t)); - t = phase_end; - - self.on_proof_result(result)?; - self.metrics - .sparse_trie_reveal_multiproof_duration_histogram - .record(t.elapsed()); + self.on_proof_results(result, &mut t)?; }, + recv(self.cancel_rx) -> _ => return Err(StateRootTaskError::Canceled), } - if self.updates.is_empty() && self.proof_result_rx.is_empty() { - // If we don't have any pending messages, we can spend some time on computing - // storage roots and promoting account updates. - self.dispatch_pending_targets(); - t = Instant::now(); - self.process_new_updates()?; - self.promote_pending_account_updates()?; - self.metrics.sparse_trie_process_updates_duration_histogram.record(t.elapsed()); - - if self.finished_state_updates && - self.account_updates.is_empty() && - self.storage_updates.iter().all(|(_, updates)| updates.is_empty()) - { - break; - } + done = self.make_progress()?; + idle_start = Instant::now(); + } - self.dispatch_pending_targets(); + // Draining phase: the marker is the last message read from the updates channel, so + // after it only proof results and cancellation can occur. The channel closing when + // the producers drop their senders is not observed here, and late best-effort hints + // are ignored: with all updates known, prefetching has nothing left to help. + while !done { + let mut t = Instant::now(); + crossbeam_channel::select_biased! { + recv(self.proof_result_rx) -> message => { + let wake = Instant::now(); + total_idle_time += wake.duration_since(idle_start); + self.metrics + .sparse_trie_channel_wait_duration_histogram + .record(wake.duration_since(t)); + t = wake; - // If there's still no pending updates spend some time pre-computing the account - // trie upper hashes - if self.proof_result_rx.is_empty() { - self.trie.calculate_subtries(); - } - } else if self.updates.is_empty() { - // If we don't have any pending updates, apply them to the trie, - t = Instant::now(); - self.process_new_updates()?; - self.metrics.sparse_trie_process_updates_duration_histogram.record(t.elapsed()); - self.dispatch_pending_targets(); - } else if self.pending_targets.len() > self.chunk_size { - // Make sure to dispatch targets if we've accumulated a lot of them. - self.dispatch_pending_targets(); + let Ok(result) = message else { + unreachable!("we own the sender half") + }; + self.on_proof_results(result, &mut t)?; + }, + recv(self.cancel_rx) -> _ => return Err(StateRootTaskError::Canceled), } + done = self.make_progress()?; idle_start = Instant::now(); } @@ -370,7 +354,6 @@ where #[cfg(feature = "trie-debug")] let debug_recorders = self.trie.take_debug_recorders(); - let changed_paths = Some(Arc::new(self.trie.take_changed_paths().unwrap_or_default())); let end = Instant::now(); self.metrics.sparse_trie_final_update_duration_histogram.record(end.duration_since(start)); @@ -388,26 +371,94 @@ where Ok(StateRootComputeOutcome { state_root, trie_updates: Arc::new(trie_updates), - changed_paths, + hashed_state: finalized_hashed_state + .expect("finished state updates publish the hashed post state"), #[cfg(feature = "trie-debug")] debug_recorders, }) } + /// Handles a received proof result: coalesces everything already queued, reveals the + /// proof in the trie, and records timing metrics. + fn on_proof_results( + &mut self, + message: ProofResultMessage, + t: &mut Instant, + ) -> Result<(), StateRootTaskError> { + let mut result = self.on_proof_result_message(message)?; + while let Ok(next) = self.proof_result_rx.try_recv() { + let res = self.on_proof_result_message(next)?; + result.extend(res); + } + + let phase_end = Instant::now(); + self.metrics + .sparse_trie_proof_coalesce_duration_histogram + .record(phase_end.duration_since(*t)); + *t = phase_end; + + self.on_proof_result(result)?; + self.metrics.sparse_trie_reveal_multiproof_duration_histogram.record(t.elapsed()); + Ok(()) + } + + /// Applies buffered updates to the trie and dispatches proof targets. + /// + /// Messages queued after the finish marker are best-effort hints and are not actionable. + /// Returns `true` once the finish marker was received and all pending trie work is done. + fn make_progress(&mut self) -> Result { + let updates_queued = !self.finished_state_updates && !self.updates.is_empty(); + + if !updates_queued && self.proof_result_rx.is_empty() { + // If we don't have any pending messages, we can spend some time on computing + // storage roots and promoting account updates. + self.dispatch_pending_targets()?; + let t = Instant::now(); + self.process_new_updates()?; + self.promote_pending_account_updates()?; + self.metrics.sparse_trie_process_updates_duration_histogram.record(t.elapsed()); + + if self.finished_state_updates && !self.has_pending_sparse_trie_updates() { + return Ok(true); + } + + self.dispatch_pending_targets()?; + self.ensure_not_stalled(updates_queued)?; + + // If there's still no pending updates spend some time pre-computing the account + // trie upper hashes + if self.proof_result_rx.is_empty() { + self.trie.calculate_subtries(); + } + } else if !updates_queued { + // If we don't have any pending updates, apply them to the trie, + let t = Instant::now(); + self.process_new_updates()?; + self.metrics.sparse_trie_process_updates_duration_histogram.record(t.elapsed()); + self.dispatch_pending_targets()?; + } else if self.pending_targets.len() > self.chunk_size { + // Make sure to dispatch targets if we've accumulated a lot of them. + self.dispatch_pending_targets()?; + } + Ok(false) + } + /// Processes a [`SparseTrieTaskMessage`] from the hashing task. - fn on_message(&mut self, message: SparseTrieTaskMessage) { + fn on_message(&mut self, message: SparseTrieTaskMessage) -> Option> { match message { - SparseTrieTaskMessage::PrefetchProofs(targets) => self.on_prewarm_targets(targets), + SparseTrieTaskMessage::PrefetchProofs(targets) => { + self.on_prewarm_targets(targets); + None + } SparseTrieTaskMessage::HashedState(hashed_state) => { - self.on_hashed_state_update(hashed_state) + self.on_hashed_state_update(hashed_state); + None } SparseTrieTaskMessage::FinishedStateUpdates => { - let _ = self - .final_hashed_state_tx - .take() - .unwrap() - .send(core::mem::take(&mut self.final_hashed_state)); - self.finished_state_updates = true + let hashed_state = Arc::new(core::mem::take(&mut self.final_hashed_state)); + let _ = self.final_hashed_state_tx.take().unwrap().send(Arc::clone(&hashed_state)); + self.finished_state_updates = true; + Some(hashed_state) } } } @@ -501,6 +552,18 @@ where .map_err(|e| StateRootTaskError::Other(format!("could not reveal multiproof: {e:?}"))) } + fn on_proof_result_message( + &mut self, + message: ProofResultMessage, + ) -> Result { + debug_assert!( + self.in_flight_proof_batches > 0, + "received proof result without an in-flight proof batch" + ); + self.in_flight_proof_batches = self.in_flight_proof_batches.saturating_sub(1); + message.result + } + fn process_new_updates(&mut self) -> SparseTrieResult<()> { if self.pending_updates == 0 { return Ok(()); @@ -785,13 +848,14 @@ where Ok(()) } - fn dispatch_pending_targets(&mut self) { + fn dispatch_pending_targets(&mut self) -> Result<(), StateRootTaskError> { if self.pending_targets.is_empty() { - return; + return Ok(()) } let _span = trace_span!("dispatch_pending_targets").entered(); let (targets, chunking_length) = self.pending_targets.take(); + let mut dispatch_error = None; dispatch_with_chunking( targets, chunking_length, @@ -801,21 +865,178 @@ where self.proof_worker_handle.has_multiple_idle_storage_workers(), MultiProofTargetsV2::chunks, |proof_targets| { - if let Err(e) = - self.proof_worker_handle.dispatch_account_multiproof(AccountMultiproofInput { - targets: proof_targets, - proof_result_sender: ProofResultContext::new( - self.proof_result_tx.clone(), - HashedPostState::default(), - Instant::now(), - ), - }) - { - error!("failed to dispatch account multiproof: {e:?}"); + if dispatch_error.is_some() { + return; + } + + match self.proof_worker_handle.dispatch_account_multiproof(AccountMultiproofInput { + targets: proof_targets, + proof_result_sender: ProofResultContext::new( + self.proof_result_tx.clone(), + HashedPostState::default(), + Instant::now(), + ), + }) { + Ok(()) => { + self.in_flight_proof_batches += 1; + } + Err(e) => { + error!("failed to dispatch account multiproof: {e:?}"); + dispatch_error = Some(StateRootTaskError::ProofDispatch(e)); + } } }, ); + + if let Some(error) = dispatch_error { + return Err(error) + } + + Ok(()) } + + fn has_pending_sparse_trie_updates(&self) -> bool { + !self.account_updates.is_empty() || + self.storage_updates.values().any(|updates| !updates.is_empty()) || + !self.pending_account_updates.is_empty() + } + + /// Errors when pending trie updates remain but nothing can deliver them: no update + /// messages are queued, no proof targets are queued or in flight, and no proof results + /// are waiting. + /// + /// `updates_queued` is passed in instead of reading `self.updates` directly, because in + /// the draining phase the updates channel is not read anymore and may hold ignored late + /// hints that must not mask a stall. + fn ensure_not_stalled(&self, updates_queued: bool) -> Result<(), StateRootTaskError> { + if self.finished_state_updates && + !updates_queued && + self.pending_updates == 0 && + self.pending_targets.is_empty() && + self.in_flight_proof_batches == 0 && + self.proof_result_rx.is_empty() && + self.has_pending_sparse_trie_updates() + { + const MAX_STALLED_PROOF_TARGETS_TO_LOG: usize = 5; + + let mut account_targets = self + .account_updates + .keys() + .map(|target| (*target, self.fetched_account_targets.get(target).copied())) + .collect::>(); + account_targets.sort_unstable(); + let account_targets_truncated = + account_targets.len().saturating_sub(MAX_STALLED_PROOF_TARGETS_TO_LOG); + account_targets.truncate(MAX_STALLED_PROOF_TARGETS_TO_LOG); + + let mut storage_targets = self + .storage_updates + .iter() + .flat_map(|(address, updates)| { + let fetched_targets = self.fetched_storage_targets.get(address); + updates.keys().map(move |target| { + ( + *address, + *target, + fetched_targets.and_then(|targets| targets.get(target)).copied(), + ) + }) + }) + .collect::>(); + storage_targets.sort_unstable(); + let storage_targets_truncated = + storage_targets.len().saturating_sub(MAX_STALLED_PROOF_TARGETS_TO_LOG); + storage_targets.truncate(MAX_STALLED_PROOF_TARGETS_TO_LOG); + + error!( + ?account_targets, + account_targets_truncated, + ?storage_targets, + storage_targets_truncated, + "sparse trie task stalled: pending updates remain but no proof targets are queued or in flight" + ); + + return Err(StateRootTaskError::Stalled) + } + + Ok(()) + } +} + +/// Metrics recorded by sparse trie and hashing tasks. +#[derive(Metrics, Clone)] +#[metrics(scope = "tree.root")] +pub(super) struct SparseTrieTaskMetrics { + /// Histogram of durations spent revealing multiproof results into the sparse trie. + pub(super) sparse_trie_reveal_multiproof_duration_histogram: Histogram, + /// Histogram of durations spent coalescing multiple proof results from the channel. + pub(super) sparse_trie_proof_coalesce_duration_histogram: Histogram, + /// Histogram of durations the event loop spent blocked waiting on channels. + pub(super) sparse_trie_channel_wait_duration_histogram: Histogram, + /// Histogram of durations spent processing trie updates and promoting pending accounts. + pub(super) sparse_trie_process_updates_duration_histogram: Histogram, + /// Histogram of sparse trie final update durations. + pub(super) sparse_trie_final_update_duration_histogram: Histogram, + /// Histogram of sparse trie total durations. + pub(super) sparse_trie_total_duration_histogram: Histogram, + /// Time spent preparing the sparse trie for reuse after state root computation. + pub(super) into_trie_for_reuse_duration_histogram: Histogram, + /// Time spent waiting for preserved sparse trie cache to become available. + pub(super) sparse_trie_cache_wait_duration_histogram: Histogram, + /// Histogram for sparse trie task idle time in seconds (waiting for updates or proof + /// results). Excludes the final wait after the channel is closed. + pub(super) sparse_trie_idle_time_seconds: Histogram, + /// Histogram for hashing task idle time in seconds (waiting for messages from execution). + /// Excludes the final wait after the channel is closed. + pub(super) hashing_task_idle_time_seconds: Histogram, + + /// Number of account leaf updates applied without needing a new proof (cache hits). + pub(super) sparse_trie_account_cache_hits: Histogram, + /// Number of account leaf updates that required a new proof (cache misses). + pub(super) sparse_trie_account_cache_misses: Histogram, + /// Number of storage leaf updates applied without needing a new proof (cache hits). + pub(super) sparse_trie_storage_cache_hits: Histogram, + /// Number of storage leaf updates that required a new proof (cache misses). + pub(super) sparse_trie_storage_cache_misses: Histogram, + + /// Retained memory of the preserved sparse trie cache in bytes. + pub(super) sparse_trie_retained_memory_bytes: Gauge, + /// Number of storage tries retained in the preserved sparse trie cache. + pub(super) sparse_trie_retained_storage_tries: Gauge, +} + +/// The default max targets, for limiting the number of account and storage proof targets to be +/// fetched by a single worker. If exceeded, chunking is forced regardless of worker availability. +const DEFAULT_MAX_TARGETS_FOR_CHUNKING: usize = 300; + +/// Dispatches work items as a single unit or in chunks based on target size and worker +/// availability. +#[expect(clippy::too_many_arguments)] +fn dispatch_with_chunking( + items: T, + chunking_len: usize, + chunk_size: usize, + max_targets_for_chunking: usize, + has_multiple_idle_account_workers: bool, + has_multiple_idle_storage_workers: bool, + chunker: impl FnOnce(T, usize) -> I, + mut dispatch: impl FnMut(T), +) where + I: IntoIterator, +{ + let has_full_chunks = chunking_len >= chunk_size.saturating_mul(2); + let should_chunk = chunking_len > max_targets_for_chunking || + (has_full_chunks && + (has_multiple_idle_account_workers || has_multiple_idle_storage_workers)); + + if should_chunk && chunking_len > chunk_size { + for chunk in chunker(items, chunk_size) { + dispatch(chunk); + } + return; + } + + dispatch(items); } /// RLP-encodes the account as a [`TrieAccount`] leaf value, or returns empty for deletions. @@ -918,7 +1139,7 @@ mod tests { SparseTrieCacheTask::::run_hashing_task( updates_rx, hashed_state_tx, - MultiProofTaskMetrics::default(), + SparseTrieTaskMetrics::default(), ); }); @@ -996,12 +1217,14 @@ mod tests { let parent_state_root = B256::from([0x55; 32]); let (updates_tx, updates_rx) = crossbeam_channel::unbounded(); + let (_cancel_guard, cancel_rx) = crossbeam_channel::bounded::<()>(0); let mut task = SparseTrieCacheTask::new_with_trie( &runtime, updates_rx, + cancel_rx, std::sync::mpsc::channel().0, proof_worker_handle, - MultiProofTaskMetrics::default(), + SparseTrieTaskMetrics::default(), trie, parent_state_root, 1, @@ -1016,4 +1239,184 @@ mod tests { assert!(outcome.trie_updates.is_empty()); assert!(task.trie.state_trie_ref().is_none(), "blind trie should not be revealed"); } + + #[test] + fn stall_check_waits_for_in_flight_proofs_then_reports_pending_updates() { + let runtime = reth_tasks::Runtime::test(); + let provider_factory = create_test_provider_factory(); + let anchor_hash = provider_factory.chain_spec().genesis_hash(); + let overlay_factory = OverlayStateProviderFactory::new( + provider_factory, + OverlayBuilder::::new( + anchor_hash, + ChangesetCache::new(), + ), + ); + let proof_worker_handle = + ProofWorkerHandle::new(&runtime, ProofTaskCtx::new(overlay_factory), false); + + let default_trie = RevealableSparseTrie::blind_from(ArenaParallelSparseTrie::default()); + let trie = SparseStateTrie::default() + .with_accounts_trie(default_trie.clone()) + .with_default_storage_trie(default_trie) + .with_updates(true); + + let (updates_tx, updates_rx) = crossbeam_channel::unbounded(); + let (_cancel_guard, cancel_rx) = crossbeam_channel::bounded::<()>(0); + let mut task = SparseTrieCacheTask::new_with_trie( + &runtime, + updates_rx, + cancel_rx, + std::sync::mpsc::channel().0, + proof_worker_handle, + SparseTrieTaskMetrics::default(), + trie, + B256::from([0x55; 32]), + 1, + ); + + drop(updates_tx); + + let account = B256::from([0x11; 32]); + let slot = B256::from([0x22; 32]); + let account_target = B256::from([0x33; 32]); + let storage_target = B256::from([0x44; 32]); + + task.finished_state_updates = true; + task.account_updates.insert(account, LeafUpdate::Touched); + task.storage_updates.entry(account).or_default().insert(slot, LeafUpdate::Touched); + task.pending_account_updates.insert(account, None); + task.fetched_account_targets.insert(account_target, 0); + task.fetched_storage_targets.entry(account).or_default().insert(storage_target, 12); + task.in_flight_proof_batches = 1; + + assert!(task.ensure_not_stalled(false).is_ok()); + + let result = ProofResultMessage { + result: Ok(DecodedMultiProofV2::default()), + elapsed: std::time::Duration::ZERO, + state: HashedPostState::default(), + }; + task.on_proof_result_message(result).expect("proof result should be ok"); + + assert_eq!(task.in_flight_proof_batches, 0); + let error = task.ensure_not_stalled(false).expect_err("task should be stalled"); + assert!(matches!(error, StateRootTaskError::Stalled)); + let error = error.to_string(); + + assert!(error.contains("sparse trie task stalled")); + assert!(!error.contains("account_targets")); + assert!(!error.contains("storage_targets")); + assert!(!error.contains(&format!("{account:?}"))); + assert!(!error.contains(&format!("{account_target:?}"))); + assert!(!error.contains(&format!("{storage_target:?}"))); + assert!(!error.contains("pending_account_leaves")); + assert!(!error.contains("pending_storage_leaves")); + assert!(!error.contains("pending_account_updates")); + assert!(!error.contains(&format!("{slot:?}"))); + } + + #[test] + fn run_errors_when_cancel_guard_drops_before_updates_finish() { + let runtime = reth_tasks::Runtime::test(); + let provider_factory = create_test_provider_factory(); + let anchor_hash = provider_factory.chain_spec().genesis_hash(); + let overlay_factory = OverlayStateProviderFactory::new( + provider_factory, + OverlayBuilder::::new( + anchor_hash, + ChangesetCache::new(), + ), + ); + let proof_worker_handle = + ProofWorkerHandle::new(&runtime, ProofTaskCtx::new(overlay_factory), false); + + let default_trie = RevealableSparseTrie::blind_from(ArenaParallelSparseTrie::default()); + let trie = SparseStateTrie::default() + .with_accounts_trie(default_trie.clone()) + .with_default_storage_trie(default_trie) + .with_updates(true); + + let (updates_tx, updates_rx) = crossbeam_channel::unbounded(); + let (cancel_guard, cancel_rx) = crossbeam_channel::bounded::<()>(0); + let mut task = SparseTrieCacheTask::new_with_trie( + &runtime, + updates_rx, + cancel_rx, + std::sync::mpsc::channel().0, + proof_worker_handle, + SparseTrieTaskMetrics::default(), + trie, + B256::from([0x55; 32]), + 1, + ); + + // The consumer abandons the computation. The updates channel is still open (no finish + // marker was sent), so without the cancel signal the task would wait forever. + drop(cancel_guard); + + let error = task.run().expect_err("canceled task must return an error"); + assert!(matches!(error, StateRootTaskError::Canceled)); + + drop(updates_tx); + } + + #[test] + fn run_ignores_hints_queued_after_updates_finish() { + let runtime = reth_tasks::Runtime::test(); + let provider_factory = create_test_provider_factory(); + let anchor_hash = provider_factory.chain_spec().genesis_hash(); + let overlay_factory = OverlayStateProviderFactory::new( + provider_factory, + OverlayBuilder::::new( + anchor_hash, + ChangesetCache::new(), + ), + ); + let proof_worker_handle = + ProofWorkerHandle::new(&runtime, ProofTaskCtx::new(overlay_factory), false); + + let default_trie = RevealableSparseTrie::blind_from(ArenaParallelSparseTrie::default()); + let trie = SparseStateTrie::default() + .with_accounts_trie(default_trie.clone()) + .with_default_storage_trie(default_trie) + .with_updates(true); + + let (updates_tx, updates_rx) = crossbeam_channel::unbounded(); + let (cancel_guard, cancel_rx) = crossbeam_channel::bounded::<()>(0); + let mut task = SparseTrieCacheTask::new_with_trie( + &runtime, + updates_rx, + cancel_rx, + std::sync::mpsc::channel().0, + proof_worker_handle, + SparseTrieTaskMetrics::default(), + trie, + B256::from([0x55; 32]), + 1, + ); + + updates_tx.send(StateRootMessage::FinishedStateUpdates).unwrap(); + updates_tx.send(StateRootMessage::PrefetchProofs(Default::default())).unwrap(); + + let wait_start = std::time::Instant::now(); + while task.updates.len() < 2 { + assert!( + wait_start.elapsed() < std::time::Duration::from_secs(1), + "hashing task did not queue the test messages" + ); + std::thread::yield_now(); + } + + let (result_tx, result_rx) = std::sync::mpsc::channel(); + let handle = std::thread::spawn(move || { + let _ = result_tx.send(task.run()); + }); + + let result = result_rx.recv_timeout(std::time::Duration::from_secs(1)); + drop(cancel_guard); + handle.join().unwrap(); + + assert!(result.expect("state root task stalled on a late hint").is_ok()); + } } diff --git a/crates/engine/tree/src/tree/tests.rs b/crates/engine/tree/src/tree/tests.rs index 30a95778ad1..6560ae225bc 100644 --- a/crates/engine/tree/src/tree/tests.rs +++ b/crates/engine/tree/src/tree/tests.rs @@ -23,17 +23,14 @@ use reth_chain_state::{test_utils::TestBlockBuilder, BlockState, StateTrieOverla use reth_chainspec::{ChainSpec, HOLESKY, MAINNET}; use reth_engine_primitives::{EngineApiValidator, ForkchoiceStatus, NoopInvalidBlockHook}; use reth_ethereum_consensus::EthBeaconConsensus; -use reth_ethereum_engine_primitives::EthEngineTypes; +use reth_ethereum_engine_primitives::{EthEngineTypes, EthPayloadAttributes}; use reth_ethereum_primitives::{Block, EthPrimitives}; use reth_evm_ethereum::MockEvmConfig; +use reth_payload_builder::PayloadServiceCommand; use reth_primitives_traits::Block as _; use reth_provider::{test_utils::MockEthProvider, BalStoreHandle, InMemoryBalStore, RawBal}; use reth_tasks::spawn_os_thread; -use reth_trie::{ - prefix_set::{PrefixSetMut, TriePrefixSetsMut}, - LazyTrieData, -}; -use reth_trie_common::{ComputedTrieData, Nibbles}; +use reth_trie_common::ComputedTrieData; use std::{ collections::BTreeMap, str::FromStr, @@ -45,48 +42,6 @@ use std::{ }; use tokio::sync::oneshot; -fn with_changed_paths( - block: ExecutedBlock, - changed_paths: TriePrefixSetsMut, -) -> ExecutedBlock { - let mut trie_data = block.trie_data(); - trie_data.changed_paths = Some(Arc::new(changed_paths)); - ExecutedBlock::with_deferred_trie_data( - block.recovered_block, - block.execution_output, - LazyTrieData::ready(trie_data), - ) -} - -fn with_empty_changed_paths(block: ExecutedBlock) -> ExecutedBlock { - with_changed_paths(block, TriePrefixSetsMut::default()) -} - -fn trie_changed_paths( - account_path: Nibbles, - storage_account: B256, - storage_path: Nibbles, -) -> TriePrefixSetsMut { - TriePrefixSetsMut { - account_prefix_set: PrefixSetMut::from([account_path]), - storage_prefix_sets: B256Map::from_iter([( - storage_account, - PrefixSetMut::from([storage_path]), - )]), - destroyed_accounts: Default::default(), - } -} - -fn merged_changed_paths(blocks: &[ExecutedBlock]) -> TriePrefixSetsMut { - let mut merged = TriePrefixSetsMut::default(); - for block in blocks { - let trie_data = block.trie_data(); - let changed_paths = trie_data.changed_paths.as_deref().expect("changed paths are present"); - merged.extend_ref(changed_paths); - } - merged -} - /// Mock engine validator for tests #[derive(Debug, Clone)] struct MockEngineValidator; @@ -198,6 +153,7 @@ struct TestHarness { FromEngine, Block>, >, from_tree_rx: UnboundedReceiver, + payload_command_rx: UnboundedReceiver>, blocks: Vec, action_rx: Receiver, block_builder: TestBlockBuilder, @@ -206,9 +162,13 @@ struct TestHarness { impl TestHarness { fn new(chain_spec: Arc) -> Self { + Self::with_config(chain_spec, TreeConfig::default().with_has_enough_parallelism(true)) + } + + fn with_config(chain_spec: Arc, tree_config: TreeConfig) -> Self { use std::sync::mpsc::channel; let (action_tx, action_rx) = channel(); - Self::with_persistence_channel(chain_spec, action_tx, action_rx) + Self::with_persistence_channel_and_config(chain_spec, action_tx, action_rx, tree_config) } #[expect(dead_code)] @@ -221,6 +181,20 @@ impl TestHarness { chain_spec: Arc, action_tx: Sender, action_rx: Receiver, + ) -> Self { + Self::with_persistence_channel_and_config( + chain_spec, + action_tx, + action_rx, + TreeConfig::default().with_has_enough_parallelism(true), + ) + } + + fn with_persistence_channel_and_config( + chain_spec: Arc, + action_tx: Sender, + action_rx: Receiver, + tree_config: TreeConfig, ) -> Self { let persistence_handle = PersistenceHandle::new(action_tx); @@ -231,7 +205,6 @@ impl TestHarness { let payload_validator = MockEngineValidator; let (from_tree_tx, from_tree_rx) = unbounded_channel(); - let tree_config = TreeConfig::default().with_has_enough_parallelism(true); let runtime = reth_tasks::Runtime::test(); let state_trie_overlays = StateTrieOverlayManager::new(runtime.state_trie_overlay_worker_pool()); @@ -248,7 +221,7 @@ impl TestHarness { ); let canonical_in_memory_state = CanonicalInMemoryState::with_head(header, None, None); - let (to_payload_service, _payload_command_rx) = unbounded_channel(); + let (to_payload_service, payload_command_rx) = unbounded_channel(); let payload_builder = PayloadBuilderHandle::new(to_payload_service); let evm_config = MockEvmConfig::default(); @@ -258,7 +231,7 @@ impl TestHarness { consensus.clone(), evm_config.clone(), payload_validator, - TreeConfig::default(), + tree_config.clone(), Box::new(NoopInvalidBlockHook::default()), changeset_cache.clone(), state_trie_overlays, @@ -287,6 +260,7 @@ impl TestHarness { to_tree_tx: tree.incoming_tx.clone(), tree, from_tree_rx, + payload_command_rx, blocks: vec![], action_rx, block_builder, @@ -502,7 +476,6 @@ impl ValidatorTestHarness { let ctx = TreeCtx::new( &mut self.harness.tree.state, &self.harness.tree.canonical_in_memory_state, - &mut self.harness.tree.pending_sparse_trie_prune, ); let result = self.validator.validate_block(block, ctx); self.metrics.record_validation(result.is_ok()); @@ -625,28 +598,7 @@ async fn test_tree_persist_blocks() { #[test] fn on_new_persisted_block_queues_sparse_trie_prune_request() { - let changed_paths = [ - trie_changed_paths( - Nibbles::from_nibbles([0x01]), - B256::with_last_byte(0x01), - Nibbles::from_nibbles([0x02]), - ), - trie_changed_paths( - Nibbles::from_nibbles([0x03]), - B256::with_last_byte(0x02), - Nibbles::from_nibbles([0x04]), - ), - trie_changed_paths( - Nibbles::from_nibbles([0x05]), - B256::with_last_byte(0x03), - Nibbles::from_nibbles([0x06]), - ), - ]; - let blocks: Vec<_> = TestBlockBuilder::eth() - .get_executed_blocks(1..4) - .zip(changed_paths) - .map(|(block, changed_paths)| with_changed_paths(block, changed_paths)) - .collect(); + let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(1..4).collect(); let mut test_harness = TestHarness::new(MAINNET.clone()).with_blocks(blocks.clone()); test_harness .tree @@ -655,28 +607,11 @@ fn on_new_persisted_block_queues_sparse_trie_prune_request() { test_harness.tree.on_new_persisted_block().unwrap(); - let retained_paths = - test_harness.tree.pending_sparse_trie_prune.as_ref().unwrap().clone().freeze(); - let expected_retained_paths = merged_changed_paths(&blocks[1..]).freeze(); - - assert_eq!( - retained_paths.account_prefix_set.slice(), - expected_retained_paths.account_prefix_set.slice() - ); - assert_eq!( - retained_paths.storage_prefix_sets.len(), - expected_retained_paths.storage_prefix_sets.len() - ); - for (storage_account, expected_slots) in expected_retained_paths.storage_prefix_sets { - assert_eq!( - retained_paths.storage_prefix_sets[&storage_account].slice(), - expected_slots.slice() - ); - } + assert!(test_harness.tree.state.pending_sparse_trie_prune()); } #[test] -fn on_new_persisted_block_skips_sparse_trie_prune_when_changed_paths_unknown() { +fn on_new_persisted_block_queues_sparse_trie_prune_with_in_memory_blocks() { let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(1..4).collect(); let mut test_harness = TestHarness::new(MAINNET.clone()).with_blocks(blocks.clone()); test_harness @@ -686,13 +621,12 @@ fn on_new_persisted_block_skips_sparse_trie_prune_when_changed_paths_unknown() { test_harness.tree.on_new_persisted_block().unwrap(); - assert!(test_harness.tree.pending_sparse_trie_prune.is_none()); + assert!(test_harness.tree.state.pending_sparse_trie_prune()); } #[test] fn on_new_persisted_block_skips_sparse_trie_prune_when_state_root_task_disabled() { - let blocks: Vec<_> = - TestBlockBuilder::eth().get_executed_blocks(1..4).map(with_empty_changed_paths).collect(); + let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(1..4).collect(); let configs = [ TreeConfig::default().with_has_enough_parallelism(false), TreeConfig::default().with_has_enough_parallelism(true).with_state_root_fallback(true), @@ -709,7 +643,7 @@ fn on_new_persisted_block_skips_sparse_trie_prune_when_state_root_task_disabled( test_harness.tree.on_new_persisted_block().unwrap(); - assert!(test_harness.tree.pending_sparse_trie_prune.is_none()); + assert!(!test_harness.tree.state.pending_sparse_trie_prune()); } } @@ -718,11 +652,49 @@ fn remove_blocks_clears_pending_sparse_trie_prune_request() { let mut test_harness = TestHarness::new(MAINNET.clone()); test_harness.tree.persistence_state.last_persisted_block = BlockNumHash { hash: B256::random(), number: 10 }; - test_harness.tree.pending_sparse_trie_prune = Some(Default::default()); + test_harness.tree.state.set_pending_sparse_trie_prune(true); test_harness.tree.remove_blocks(9); - assert!(test_harness.tree.pending_sparse_trie_prune.is_none()); + assert!(!test_harness.tree.state.pending_sparse_trie_prune()); +} + +#[test] +fn process_payload_attributes_shares_sparse_trie_during_validation_fallback() { + let config = TreeConfig::default() + .with_has_enough_parallelism(true) + .with_state_root_fallback(true) + .with_share_sparse_trie_with_payload_builder(true); + let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(1..2).collect(); + let mut test_harness = TestHarness::with_config(MAINNET.clone(), config).with_blocks(blocks); + let head = + test_harness.blocks.last().unwrap().recovered_block().clone_sealed_header().clone_header(); + let head_hash = test_harness.blocks.last().unwrap().recovered_block().hash(); + let state = test_harness.fcu_state(head_hash); + test_harness.tree.state.set_pending_sparse_trie_prune(true); + + let updated = test_harness.tree.process_payload_attributes( + EthPayloadAttributes { + timestamp: head.timestamp() + 1, + prev_randao: B256::ZERO, + suggested_fee_recipient: Default::default(), + withdrawals: None, + parent_beacon_block_root: None, + slot_number: None, + target_gas_limit: None, + }, + &head, + state, + ); + + assert_eq!(updated.forkchoice_status(), ForkchoiceStatus::Valid); + assert!(!test_harness.tree.state.pending_sparse_trie_prune()); + + let command = test_harness.payload_command_rx.try_recv().unwrap(); + let PayloadServiceCommand::BuildNewPayload(input, _, _) = command else { + panic!("expected build new payload command") + }; + assert!(input.state_root_handle.is_some()); } #[tokio::test] diff --git a/crates/engine/tree/src/tree/types.rs b/crates/engine/tree/src/tree/types.rs index eda7c7b1887..2030c13c4d6 100644 --- a/crates/engine/tree/src/tree/types.rs +++ b/crates/engine/tree/src/tree/types.rs @@ -1,9 +1,60 @@ //! Shared types for blockchain tree validation. use crate::tree::error::InsertPayloadError; -use alloy_eip7928::bal::RawBal; +use alloy_eip7928::bal::{DecodedBal, RawBal}; +use alloy_eips::eip4895::Withdrawal; +use alloy_primitives::B256; use reth_chain_state::{ExecutedBlock, ExecutionTimingStats}; +use reth_evm::{ConfigureEvm, EvmEnvFor}; use reth_primitives_traits::{BlockTy, NodePrimitives}; +use std::sync::Arc; + +/// EVM context required to execute a block. +#[derive(Debug, Clone)] +pub struct ExecutionEnv { + /// Evm environment. + pub evm_env: EvmEnvFor, + /// Hash of the block being executed. + pub hash: B256, + /// Hash of the parent block. + pub parent_hash: B256, + /// State root of the parent block. + /// Used for sparse trie continuation: if the preserved trie's anchor matches this, + /// the trie can be reused directly. + pub parent_state_root: B256, + /// Number of transactions in the block. + /// Used to determine parallel worker count for prewarming. + pub transaction_count: usize, + /// Total gas used by all transactions in the block. + /// Used to adaptively select multiproof chunk size for optimal throughput. + pub gas_used: u64, + /// Withdrawals included in the block. + /// Used to generate prefetch targets for withdrawal addresses. + pub withdrawals: Option>, + /// Optional decoded BAL for the block. + /// Used to validate and optimize execution. + pub decoded_bal: Option>, +} + +impl ExecutionEnv +where + EvmEnvFor: Default, +{ + /// Creates a new [`ExecutionEnv`] with default values for testing. + #[cfg(any(test, feature = "test-utils"))] + pub fn test_default() -> Self { + Self { + evm_env: Default::default(), + hash: Default::default(), + parent_hash: Default::default(), + parent_state_root: Default::default(), + transaction_count: 0, + gas_used: 0, + withdrawals: None, + decoded_bal: None, + } + } +} /// Result of block or payload validation. pub type ValidationOutcome>> = Result, E>; diff --git a/crates/ethereum/node/Cargo.toml b/crates/ethereum/node/Cargo.toml index ca5c4198e49..ba86215d317 100644 --- a/crates/ethereum/node/Cargo.toml +++ b/crates/ethereum/node/Cargo.toml @@ -26,6 +26,7 @@ reth-network.workspace = true reth-evm.workspace = true reth-evm-ethereum = { workspace = true, features = ["std"] } reth-rpc.workspace = true +reth-rpc-engine-api.workspace = true reth-rpc-api.workspace = true reth-rpc-eth-api.workspace = true reth-rpc-builder.workspace = true diff --git a/crates/ethereum/node/src/engine_ssz_containers.rs b/crates/ethereum/node/src/engine_ssz_containers.rs index 954687dbdf9..14513d1520f 100644 --- a/crates/ethereum/node/src/engine_ssz_containers.rs +++ b/crates/ethereum/node/src/engine_ssz_containers.rs @@ -2,20 +2,351 @@ //! //! These types intentionally live apart from the legacy JSON-RPC Engine API types because their //! SSZ encodings are not always wire-compatible. This module contains the shared endpoint -//! containers and fork-specific payload containers from -//! [execution-apis PR #793](https://github.com/ethereum/execution-apis/pull/793), plus the -//! experimental payload-with-witness response type that extends the same REST-SSZ model. +//! containers, fork-specific payload containers, blob containers, payload-body containers, and the +//! experimental payload-with-witness response type from the same REST-SSZ model. -use alloy_eips::eip7685::Requests; -use alloy_primitives::{B256, U256}; +use alloy_eips::{ + eip4844::{Blob, BlobAndProofV1, BlobAndProofV2, BlobCellsAndProofsV1, Bytes48}, + eip4895::Withdrawal, + eip7594::Cell, + eip7685::Requests, +}; +use alloy_primitives::{Address, Bytes, B128, B256, U256}; use alloy_rpc_types_engine::{ - BlobsBundleV1, BlobsBundleV2, ExecutionPayloadEnvelopeV2 as LegacyBuiltPayloadShanghai, + BlobsBundleV1, BlobsBundleV2, ExecutionPayloadBodyV1 as LegacyExecutionPayloadBodyV1, + ExecutionPayloadBodyV2 as LegacyExecutionPayloadBodyV2, + ExecutionPayloadEnvelopeV2 as LegacyBuiltPayloadShanghai, ExecutionPayloadEnvelopeV4 as LegacyBuiltPayloadPrague, ExecutionPayloadEnvelopeV5 as LegacyBuiltPayloadOsaka, ExecutionPayloadEnvelopeV6 as LegacyBuiltPayloadAmsterdam, ExecutionPayloadFieldV2, ExecutionPayloadV1, ExecutionPayloadV2, ExecutionPayloadV3, ExecutionPayloadV4, + ForkchoiceState, ForkchoiceUpdated as LegacyForkchoice, + PayloadAttributes as LegacyPayloadAttributes, PayloadId, PayloadStatus as LegacyPayloadStatus, + PayloadStatusEnum, }; +type ErrorBytes = Vec; + +/// Maximum number of blobs in a REST-SSZ blob request or response. +pub const MAX_BLOBS_REQUEST: usize = 128; + +/// Maximum number of payload bodies in a REST-SSZ request or response. +pub const MAX_BODIES_REQUEST: usize = 32; + +/// An Engine API v2 SSZ optional encoded as `List[T, 1]`. +/// +/// This differs from [`Option`]'s `ethereum_ssz` encoding, which uses an SSZ union. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Optional(Vec); + +impl Optional { + /// Creates an absent optional. + pub const fn none() -> Self { + Self(Vec::new()) + } + + /// Creates a present optional. + pub fn some(value: T) -> Self { + Self(vec![value]) + } + + /// Returns the contained value, if present. + pub fn as_ref(&self) -> Option<&T> { + self.0.first() + } + + /// Returns true if no value is present. + pub const fn is_none(&self) -> bool { + self.0.is_empty() + } + + /// Returns true if a value is present. + pub const fn is_some(&self) -> bool { + !self.is_none() + } + + /// Converts into a Rust optional. + pub fn into_option(mut self) -> Option { + self.0.pop() + } +} + +impl From> for Optional { + fn from(value: Option) -> Self { + value.map_or_else(Self::none, Self::some) + } +} + +impl From> for Option { + fn from(value: Optional) -> Self { + value.into_option() + } +} + +impl ssz::Encode for Optional { + fn is_ssz_fixed_len() -> bool { + false + } + + fn ssz_bytes_len(&self) -> usize { + self.0.ssz_bytes_len() + } + + fn ssz_append(&self, buf: &mut Vec) { + self.0.ssz_append(buf); + } +} + +impl ssz::Decode for Optional { + fn is_ssz_fixed_len() -> bool { + false + } + + fn from_ssz_bytes(bytes: &[u8]) -> Result { + let values = Vec::::from_ssz_bytes(bytes)?; + if values.len() > 1 { + return Err(ssz::DecodeError::BytesInvalid("optional has more than one value".into())) + } + Ok(Self(values)) + } +} + +/// Engine API v2 REST-SSZ payload status. +/// +/// This is separate from the legacy status because REST-SSZ uses `Optional` fields instead of +/// zero-value sentinels and legacy byte lists. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PayloadStatus { + /// Payload validation status. + pub status: PayloadStatusEnum, + /// Most recent valid block hash. + pub latest_valid_hash: Optional, + /// Optional payload validation error bytes. + pub validation_error: Optional, +} + +const MAX_ERROR_BYTES: usize = 1024; + +const fn status_code(status: &PayloadStatusEnum) -> u8 { + match status { + PayloadStatusEnum::Valid => 0, + PayloadStatusEnum::Invalid { .. } => 1, + PayloadStatusEnum::Syncing => 2, + PayloadStatusEnum::Accepted => 3, + } +} + +fn status_from_code(code: u8) -> Result { + match code { + 0 => Ok(PayloadStatusEnum::Valid), + 1 => Ok(PayloadStatusEnum::Invalid { validation_error: String::new() }), + 2 => Ok(PayloadStatusEnum::Syncing), + 3 => Ok(PayloadStatusEnum::Accepted), + _ => Err(ssz::DecodeError::BytesInvalid("unknown payload status code".into())), + } +} + +fn legacy_validation_error( + status: &PayloadStatusEnum, +) -> Result, ConversionError> { + match status { + PayloadStatusEnum::Invalid { validation_error } => { + let bytes = validation_error.as_bytes().to_vec(); + if bytes.len() > MAX_ERROR_BYTES { + return Err(ConversionError::ErrorBytesTooLong) + } + Ok(Optional::some(bytes)) + } + _ => Ok(Optional::none()), + } +} + +impl ssz::Encode for PayloadStatus { + fn is_ssz_fixed_len() -> bool { + false + } + + fn ssz_bytes_len(&self) -> usize { + 1 + ssz::BYTES_PER_LENGTH_OFFSET * 2 + + self.latest_valid_hash.ssz_bytes_len() + + self.validation_error.ssz_bytes_len() + } + + fn ssz_append(&self, buf: &mut Vec) { + let mut encoder = ssz::SszEncoder::container(buf, 1 + ssz::BYTES_PER_LENGTH_OFFSET * 2); + encoder.append(&status_code(&self.status)); + encoder.append(&self.latest_valid_hash); + encoder.append(&self.validation_error); + encoder.finalize(); + } +} + +impl ssz::Decode for PayloadStatus { + fn is_ssz_fixed_len() -> bool { + false + } + + fn from_ssz_bytes(bytes: &[u8]) -> Result { + let mut builder = ssz::SszDecoderBuilder::new(bytes); + builder.register_type::()?; + builder.register_type::>()?; + builder.register_type::>()?; + let mut decoder = builder.build()?; + let mut status = status_from_code(decoder.decode_next()?)?; + let latest_valid_hash = decoder.decode_next()?; + let validation_error: Optional = decoder.decode_next()?; + if let PayloadStatusEnum::Invalid { validation_error: error } = &mut status { + *error = match validation_error.as_ref() { + Some(error) => String::from_utf8(error.clone()) + .map_err(|err| ssz::DecodeError::BytesInvalid(err.to_string()))?, + None => String::new(), + }; + if validation_error.as_ref().is_some_and(|error| error.len() > MAX_ERROR_BYTES) { + return Err(ssz::DecodeError::BytesInvalid( + "payload validation error is too long".into(), + )) + } + } else if validation_error.is_some() { + return Err(ssz::DecodeError::BytesInvalid( + "validation error is only valid for INVALID status".into(), + )); + } + Ok(Self { status, latest_valid_hash, validation_error }) + } +} + +/// Error converting legacy Engine API values into v2 REST-SSZ values. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ConversionError { + /// Payload validation error exceeded the REST-SSZ byte bound. + ErrorBytesTooLong, + /// `ACCEPTED` is not permitted in a forkchoice response. + AcceptedForkchoice, + /// A bounded REST-SSZ list exceeded its maximum length. + TooManyItems { + /// Name of the field that exceeded its bound. + field: &'static str, + /// Maximum permitted item count. + max: usize, + /// Actual item count. + actual: usize, + }, +} + +impl core::fmt::Display for ConversionError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::ErrorBytesTooLong => f.write_str("payload validation error is too long"), + Self::AcceptedForkchoice => { + f.write_str("ACCEPTED is not valid in a forkchoice response") + } + Self::TooManyItems { field, max, actual } => { + write!(f, "too many {field}: expected at most {max}, got {actual}") + } + } + } +} + +impl core::error::Error for ConversionError {} + +impl TryFrom for PayloadStatus { + type Error = ConversionError; + + fn try_from(value: LegacyPayloadStatus) -> Result { + let validation_error = legacy_validation_error(&value.status)?; + Ok(Self { + status: value.status, + latest_valid_hash: value.latest_valid_hash.into(), + validation_error, + }) + } +} + +impl From for LegacyPayloadStatus { + fn from(value: PayloadStatus) -> Self { + let status = match value.status { + PayloadStatusEnum::Invalid { .. } => PayloadStatusEnum::Invalid { + validation_error: value + .validation_error + .as_ref() + .and_then(|error| String::from_utf8(error.clone()).ok()) + .unwrap_or_default(), + }, + status => status, + }; + Self { status, latest_valid_hash: value.latest_valid_hash.into() } + } +} + +/// Engine API v2 REST-SSZ forkchoice update response. +/// +/// The REST response is a container of two variable fields, unlike the legacy fixed payload ID. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ForkchoiceUpdateResponse { + /// Restricted payload status; `ACCEPTED` is invalid here. + pub payload_status: PayloadStatus, + /// Opaque server-assigned payload identifier. + pub payload_id: Optional, +} + +impl ssz::Encode for ForkchoiceUpdateResponse { + fn is_ssz_fixed_len() -> bool { + false + } + + fn ssz_bytes_len(&self) -> usize { + ssz::BYTES_PER_LENGTH_OFFSET * 2 + + self.payload_status.ssz_bytes_len() + + self.payload_id.ssz_bytes_len() + } + + fn ssz_append(&self, buf: &mut Vec) { + let mut encoder = ssz::SszEncoder::container(buf, ssz::BYTES_PER_LENGTH_OFFSET * 2); + encoder.append(&self.payload_status); + encoder.append(&self.payload_id); + encoder.finalize(); + } +} + +impl ssz::Decode for ForkchoiceUpdateResponse { + fn is_ssz_fixed_len() -> bool { + false + } + + fn from_ssz_bytes(bytes: &[u8]) -> Result { + let mut builder = ssz::SszDecoderBuilder::new(bytes); + builder.register_type::()?; + builder.register_type::>()?; + let mut decoder = builder.build()?; + let response = + Self { payload_status: decoder.decode_next()?, payload_id: decoder.decode_next()? }; + if matches!(response.payload_status.status, PayloadStatusEnum::Accepted) { + return Err(ssz::DecodeError::BytesInvalid( + "ACCEPTED is not valid in a forkchoice response".into(), + )); + } + Ok(response) + } +} + +impl TryFrom for ForkchoiceUpdateResponse { + type Error = ConversionError; + + fn try_from(value: LegacyForkchoice) -> Result { + let payload_status = PayloadStatus::try_from(value.payload_status)?; + if matches!(payload_status.status, PayloadStatusEnum::Accepted) { + return Err(ConversionError::AcceptedForkchoice) + } + Ok(Self { payload_status, payload_id: value.payload_id.into() }) + } +} + +impl From for LegacyForkchoice { + fn from(value: ForkchoiceUpdateResponse) -> Self { + Self { payload_status: value.payload_status.into(), payload_id: value.payload_id.into() } + } +} + /// Paris execution payload. pub type ExecutionPayloadParis = ExecutionPayloadV1; @@ -34,6 +365,249 @@ pub type ExecutionPayloadOsaka = ExecutionPayloadV3; /// Amsterdam execution payload. pub type ExecutionPayloadAmsterdam = ExecutionPayloadV4; +/// Paris payload attributes. +/// +/// Fork-specific attributes keep later-fork fields out of the SSZ body; the legacy type is a +/// permissive superset. +#[derive(Clone, Debug, Default, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] +pub struct PayloadAttributesParis { + /// Payload timestamp. + pub timestamp: u64, + /// Previous RANDAO value. + pub prev_randao: B256, + /// Suggested fee recipient. + pub suggested_fee_recipient: Address, +} + +/// Shanghai payload attributes. +/// +/// Fork-specific attributes keep later-fork fields out of the SSZ body; the legacy type is a +/// permissive superset. +#[derive(Clone, Debug, Default, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] +pub struct PayloadAttributesShanghai { + /// Payload timestamp. + pub timestamp: u64, + /// Previous RANDAO value. + pub prev_randao: B256, + /// Suggested fee recipient. + pub suggested_fee_recipient: Address, + /// Withdrawals to include in the payload. + pub withdrawals: Vec, +} + +/// Cancun payload attributes. +/// +/// Fork-specific attributes keep later-fork fields out of the SSZ body; the legacy type is a +/// permissive superset. +#[derive(Clone, Debug, Default, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] +pub struct PayloadAttributesCancun { + /// Payload timestamp. + pub timestamp: u64, + /// Previous RANDAO value. + pub prev_randao: B256, + /// Suggested fee recipient. + pub suggested_fee_recipient: Address, + /// Withdrawals to include in the payload. + pub withdrawals: Vec, + /// Root of the parent beacon block. + pub parent_beacon_block_root: B256, +} + +/// Prague uses the Cancun payload-attributes schema. +pub type PayloadAttributesPrague = PayloadAttributesCancun; + +/// Osaka uses the Cancun payload-attributes schema. +pub type PayloadAttributesOsaka = PayloadAttributesCancun; + +/// Amsterdam payload attributes. +/// +/// Fork-specific attributes keep the Amsterdam-only fields in their defined SSZ position. +#[derive(Clone, Debug, Default, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] +pub struct PayloadAttributesAmsterdam { + /// Payload timestamp. + pub timestamp: u64, + /// Previous RANDAO value. + pub prev_randao: B256, + /// Suggested fee recipient. + pub suggested_fee_recipient: Address, + /// Withdrawals to include in the payload. + pub withdrawals: Vec, + /// Root of the parent beacon block. + pub parent_beacon_block_root: B256, + /// Consensus-layer slot number. + pub slot_number: u64, + /// Target gas limit. + pub target_gas_limit: u64, +} + +/// Error converting legacy cross-fork payload attributes into a fork-specific SSZ container. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PayloadAttributesConversionError { + /// A field required by the selected fork is absent. + MissingField(&'static str), + /// A field from a later fork is populated and would be lost. + UnexpectedField(&'static str), +} + +impl core::fmt::Display for PayloadAttributesConversionError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::MissingField(field) => { + write!(f, "missing required payload attributes field: {field}") + } + Self::UnexpectedField(field) => { + write!(f, "unexpected later-fork payload attributes field: {field}") + } + } + } +} + +impl core::error::Error for PayloadAttributesConversionError {} + +const fn ensure_absent( + value: &Option, + field: &'static str, +) -> Result<(), PayloadAttributesConversionError> { + if value.is_some() { + Err(PayloadAttributesConversionError::UnexpectedField(field)) + } else { + Ok(()) + } +} + +fn require( + value: Option, + field: &'static str, +) -> Result { + value.ok_or(PayloadAttributesConversionError::MissingField(field)) +} + +impl From for LegacyPayloadAttributes { + fn from(value: PayloadAttributesParis) -> Self { + Self { + timestamp: value.timestamp, + prev_randao: value.prev_randao, + suggested_fee_recipient: value.suggested_fee_recipient, + withdrawals: None, + parent_beacon_block_root: None, + slot_number: None, + target_gas_limit: None, + } + } +} + +impl TryFrom for PayloadAttributesParis { + type Error = PayloadAttributesConversionError; + + fn try_from(value: LegacyPayloadAttributes) -> Result { + ensure_absent(&value.withdrawals, "withdrawals")?; + ensure_absent(&value.parent_beacon_block_root, "parent_beacon_block_root")?; + ensure_absent(&value.slot_number, "slot_number")?; + ensure_absent(&value.target_gas_limit, "target_gas_limit")?; + Ok(Self { + timestamp: value.timestamp, + prev_randao: value.prev_randao, + suggested_fee_recipient: value.suggested_fee_recipient, + }) + } +} + +impl From for LegacyPayloadAttributes { + fn from(value: PayloadAttributesShanghai) -> Self { + Self { + timestamp: value.timestamp, + prev_randao: value.prev_randao, + suggested_fee_recipient: value.suggested_fee_recipient, + withdrawals: Some(value.withdrawals), + parent_beacon_block_root: None, + slot_number: None, + target_gas_limit: None, + } + } +} + +impl TryFrom for PayloadAttributesShanghai { + type Error = PayloadAttributesConversionError; + + fn try_from(value: LegacyPayloadAttributes) -> Result { + ensure_absent(&value.parent_beacon_block_root, "parent_beacon_block_root")?; + ensure_absent(&value.slot_number, "slot_number")?; + ensure_absent(&value.target_gas_limit, "target_gas_limit")?; + Ok(Self { + timestamp: value.timestamp, + prev_randao: value.prev_randao, + suggested_fee_recipient: value.suggested_fee_recipient, + withdrawals: require(value.withdrawals, "withdrawals")?, + }) + } +} + +impl From for LegacyPayloadAttributes { + fn from(value: PayloadAttributesCancun) -> Self { + Self { + timestamp: value.timestamp, + prev_randao: value.prev_randao, + suggested_fee_recipient: value.suggested_fee_recipient, + withdrawals: Some(value.withdrawals), + parent_beacon_block_root: Some(value.parent_beacon_block_root), + slot_number: None, + target_gas_limit: None, + } + } +} + +impl TryFrom for PayloadAttributesCancun { + type Error = PayloadAttributesConversionError; + + fn try_from(value: LegacyPayloadAttributes) -> Result { + ensure_absent(&value.slot_number, "slot_number")?; + ensure_absent(&value.target_gas_limit, "target_gas_limit")?; + Ok(Self { + timestamp: value.timestamp, + prev_randao: value.prev_randao, + suggested_fee_recipient: value.suggested_fee_recipient, + withdrawals: require(value.withdrawals, "withdrawals")?, + parent_beacon_block_root: require( + value.parent_beacon_block_root, + "parent_beacon_block_root", + )?, + }) + } +} + +impl From for LegacyPayloadAttributes { + fn from(value: PayloadAttributesAmsterdam) -> Self { + Self { + timestamp: value.timestamp, + prev_randao: value.prev_randao, + suggested_fee_recipient: value.suggested_fee_recipient, + withdrawals: Some(value.withdrawals), + parent_beacon_block_root: Some(value.parent_beacon_block_root), + slot_number: Some(value.slot_number), + target_gas_limit: Some(value.target_gas_limit), + } + } +} + +impl TryFrom for PayloadAttributesAmsterdam { + type Error = PayloadAttributesConversionError; + + fn try_from(value: LegacyPayloadAttributes) -> Result { + Ok(Self { + timestamp: value.timestamp, + prev_randao: value.prev_randao, + suggested_fee_recipient: value.suggested_fee_recipient, + withdrawals: require(value.withdrawals, "withdrawals")?, + parent_beacon_block_root: require( + value.parent_beacon_block_root, + "parent_beacon_block_root", + )?, + slot_number: require(value.slot_number, "slot_number")?, + target_gas_limit: require(value.target_gas_limit, "target_gas_limit")?, + }) + } +} + /// This structure maps to the Engine API v2 REST-SSZ payload-build response for Paris. /// /// Unlike the legacy `engine_getPayloadV1` response, this includes the expected block value. @@ -49,6 +623,7 @@ pub struct BuiltPayloadParis { /// /// This follows the legacy `engine_getPayloadV2` payload-build response shape: execution payload /// plus block value only. `should_override_builder` starts at Cancun. +/// The concrete V2 payload prevents the legacy V1/V2 untagged field from accepting a Paris payload. #[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] pub struct BuiltPayloadShanghai { /// Execution payload V2. @@ -82,6 +657,9 @@ pub struct BuiltPayloadPrague { } /// This structure maps to the Engine API v2 REST-SSZ payload-build response for Osaka. +/// +/// It is separate from legacy V5 because REST-SSZ places `execution_requests` before the builder +/// override flag. #[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] pub struct BuiltPayloadOsaka { /// Execution payload V3. @@ -98,6 +676,9 @@ pub struct BuiltPayloadOsaka { } /// This structure maps to the Engine API v2 REST-SSZ payload-build response for Amsterdam. +/// +/// It is separate from legacy V6 because REST-SSZ places `execution_requests` before the builder +/// override flag. #[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] pub struct BuiltPayloadAmsterdam { /// Execution payload V4. @@ -238,6 +819,9 @@ impl From for LegacyBuiltPayloadAmsterdam { /// override hints, or a different field order. /// /// Paris payload-submission request. +/// +/// The single-field container is required by REST-SSZ; the legacy endpoint submitted a bare +/// payload. #[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] pub struct ExecutionPayloadEnvelopeParis { /// Submitted execution payload. @@ -245,6 +829,8 @@ pub struct ExecutionPayloadEnvelopeParis { } /// Shanghai payload-submission request. +/// +/// The single-field container is required by REST-SSZ and fixes the payload fork at decode time. #[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] pub struct ExecutionPayloadEnvelopeShanghai { /// Submitted execution payload. @@ -252,6 +838,8 @@ pub struct ExecutionPayloadEnvelopeShanghai { } /// Cancun payload-submission request. +/// +/// Cancun adds the parent beacon block root to the REST request envelope. #[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] pub struct ExecutionPayloadEnvelopeCancun { /// Submitted execution payload. @@ -261,6 +849,8 @@ pub struct ExecutionPayloadEnvelopeCancun { } /// Prague payload-submission request. +/// +/// Prague adds execution requests to the REST request envelope. #[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] pub struct ExecutionPayloadEnvelopePrague { /// Submitted execution payload. @@ -272,6 +862,8 @@ pub struct ExecutionPayloadEnvelopePrague { } /// Osaka payload-submission request. +/// +/// Osaka keeps the REST envelope shape while selecting the Osaka payload schema. #[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] pub struct ExecutionPayloadEnvelopeOsaka { /// Submitted execution payload. @@ -283,6 +875,8 @@ pub struct ExecutionPayloadEnvelopeOsaka { } /// Amsterdam payload-submission request. +/// +/// Amsterdam selects the V4 payload while retaining the Cancun and Prague envelope fields. #[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] pub struct ExecutionPayloadEnvelopeAmsterdam { /// Submitted execution payload. @@ -347,6 +941,523 @@ impl From<(ExecutionPayloadAmsterdam, B256, Requests)> for ExecutionPayloadEnvel } } +/// Paris forkchoice-update request. +/// +/// REST-SSZ uses an `Optional` field inside one container; legacy FCU used separate RPC +/// parameters and a legacy `Option` encoding. +#[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] +pub struct ForkchoiceUpdateParis { + /// Current forkchoice state. + pub forkchoice_state: ForkchoiceState, + /// Optional Paris payload attributes. + pub payload_attributes: Optional, +} + +/// Shanghai forkchoice-update request. +#[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] +pub struct ForkchoiceUpdateShanghai { + /// Current forkchoice state. + pub forkchoice_state: ForkchoiceState, + /// Optional Shanghai payload attributes. + pub payload_attributes: Optional, +} + +/// Cancun forkchoice-update request. +#[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] +pub struct ForkchoiceUpdateCancun { + /// Current forkchoice state. + pub forkchoice_state: ForkchoiceState, + /// Optional Cancun payload attributes. + pub payload_attributes: Optional, +} + +/// Prague forkchoice-update request. +#[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] +pub struct ForkchoiceUpdatePrague { + /// Current forkchoice state. + pub forkchoice_state: ForkchoiceState, + /// Optional Prague payload attributes. + pub payload_attributes: Optional, +} + +/// Osaka forkchoice-update request. +#[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] +pub struct ForkchoiceUpdateOsaka { + /// Current forkchoice state. + pub forkchoice_state: ForkchoiceState, + /// Optional Osaka payload attributes. + pub payload_attributes: Optional, +} + +/// Amsterdam forkchoice-update request. +#[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] +pub struct ForkchoiceUpdateAmsterdam { + /// Current forkchoice state. + pub forkchoice_state: ForkchoiceState, + /// Optional Amsterdam payload attributes. + pub payload_attributes: Optional, + /// Optional `Bitvector[128]` custody-column selection. + pub custody_columns: Optional, +} + +/// Fork-specific execution payload body for Paris. +/// +/// Paris omits withdrawals entirely; the legacy body keeps them as an optional union field. +#[derive(Clone, Debug, Default, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] +pub struct ExecutionPayloadBodyParis { + /// Enveloped encoded transactions. + pub transactions: Vec, +} + +/// Fork-specific execution payload body for Shanghai. +/// +/// Shanghai makes withdrawals a direct field rather than the legacy optional union. +#[derive(Clone, Debug, Default, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] +pub struct ExecutionPayloadBodyShanghai { + /// Enveloped encoded transactions. + pub transactions: Vec, + /// Withdrawals included in the block. + pub withdrawals: Vec, +} + +/// Cancun uses the Shanghai execution-payload-body schema. +pub type ExecutionPayloadBodyCancun = ExecutionPayloadBodyShanghai; + +/// Prague uses the Shanghai execution-payload-body schema. +pub type ExecutionPayloadBodyPrague = ExecutionPayloadBodyShanghai; + +/// Osaka uses the Shanghai execution-payload-body schema. +pub type ExecutionPayloadBodyOsaka = ExecutionPayloadBodyShanghai; + +/// Fork-specific execution payload body for Amsterdam. +/// +/// Amsterdam adds the block access list as a direct field rather than a legacy optional field. +#[derive(Clone, Debug, Default, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] +pub struct ExecutionPayloadBodyAmsterdam { + /// Enveloped encoded transactions. + pub transactions: Vec, + /// Withdrawals included in the block. + pub withdrawals: Vec, + /// RLP-encoded EIP-7928 block access list. + pub block_access_list: Bytes, +} + +/// Error converting legacy cross-fork execution payload bodies into fork-specific containers. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ExecutionPayloadBodyConversionError { + /// A field required by the selected fork is absent. + MissingField(&'static str), + /// A field from a later fork is populated and would be lost. + UnexpectedField(&'static str), +} + +impl core::fmt::Display for ExecutionPayloadBodyConversionError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::MissingField(field) => { + write!(f, "missing required execution payload body field: {field}") + } + Self::UnexpectedField(field) => { + write!(f, "unexpected later-fork execution payload body field: {field}") + } + } + } +} + +impl core::error::Error for ExecutionPayloadBodyConversionError {} + +impl From for LegacyExecutionPayloadBodyV1 { + fn from(value: ExecutionPayloadBodyParis) -> Self { + Self { transactions: value.transactions, withdrawals: None } + } +} + +impl TryFrom for ExecutionPayloadBodyParis { + type Error = ExecutionPayloadBodyConversionError; + + fn try_from(value: LegacyExecutionPayloadBodyV1) -> Result { + if value.withdrawals.is_some() { + return Err(ExecutionPayloadBodyConversionError::UnexpectedField("withdrawals")) + } + Ok(Self { transactions: value.transactions }) + } +} + +impl From for LegacyExecutionPayloadBodyV1 { + fn from(value: ExecutionPayloadBodyShanghai) -> Self { + Self { transactions: value.transactions, withdrawals: Some(value.withdrawals) } + } +} + +impl TryFrom for ExecutionPayloadBodyShanghai { + type Error = ExecutionPayloadBodyConversionError; + + fn try_from(value: LegacyExecutionPayloadBodyV1) -> Result { + Ok(Self { + transactions: value.transactions, + withdrawals: value + .withdrawals + .ok_or(ExecutionPayloadBodyConversionError::MissingField("withdrawals"))?, + }) + } +} + +impl From for LegacyExecutionPayloadBodyV2 { + fn from(value: ExecutionPayloadBodyAmsterdam) -> Self { + Self { + transactions: value.transactions, + withdrawals: Some(value.withdrawals), + block_access_list: Some(value.block_access_list), + } + } +} + +impl TryFrom for ExecutionPayloadBodyAmsterdam { + type Error = ExecutionPayloadBodyConversionError; + + fn try_from(value: LegacyExecutionPayloadBodyV2) -> Result { + Ok(Self { + transactions: value.transactions, + withdrawals: value + .withdrawals + .ok_or(ExecutionPayloadBodyConversionError::MissingField("withdrawals"))?, + block_access_list: value + .block_access_list + .ok_or(ExecutionPayloadBodyConversionError::MissingField("block_access_list"))?, + }) + } +} + +/// REST-SSZ historical bodies-by-hash request. +/// +/// This is a single-field container, not a bare SSZ list. +#[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] +pub struct BodiesByHashRequest { + /// Requested block hashes. + pub block_hashes: Vec, +} + +/// Historical body response entry with explicit availability. +/// +/// REST-SSZ uses a boolean availability bit instead of the legacy `Option` union. +#[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] +pub struct BodyEntry { + /// Whether the body is available and belongs to the requested fork. + pub available: bool, + /// Fork-specific body, ignored when `available` is false. + pub body: T, +} + +impl BodyEntry { + /// Creates an available body entry. + pub const fn available(body: T) -> Self { + Self { available: true, body } + } +} + +impl BodyEntry { + /// Creates an unavailable body entry. + pub fn unavailable() -> Self { + Self { available: false, body: T::default() } + } +} + +/// REST-SSZ historical bodies response. +/// +/// The response is a one-field SSZ container around the entries list, not a bare list. +#[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] +pub struct BodiesResponse { + /// Body entries in request or range order. + pub entries: Vec>, +} + +impl BodiesResponse { + /// Creates a response from optional legacy bodies. + /// + /// Missing bodies, or bodies that do not convert to the requested fork container, are encoded + /// as unavailable entries. + pub fn from_optional_bodies( + bodies: Vec>, + convert: impl Fn(LegacyBody) -> Option, + ) -> Self { + let entries = bodies + .into_iter() + .map(|body| match body.and_then(&convert) { + Some(body) => BodyEntry::available(body), + None => BodyEntry::unavailable(), + }) + .collect(); + + Self { entries } + } +} + +/// Paris historical bodies response. +pub type BodiesResponseParis = BodiesResponse; + +/// Shanghai historical bodies response. +pub type BodiesResponseShanghai = BodiesResponse; + +/// Cancun historical bodies response. +pub type BodiesResponseCancun = BodiesResponse; + +/// Prague historical bodies response. +pub type BodiesResponsePrague = BodiesResponse; + +/// Osaka historical bodies response. +pub type BodiesResponseOsaka = BodiesResponse; + +/// Amsterdam historical bodies response. +pub type BodiesResponseAmsterdam = BodiesResponse; + +/// V1-V3 blob request container. +/// +/// This single-field container starts with a four-byte SSZ offset and is not wire-equivalent to a +/// top-level list. +#[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] +pub struct BlobsV1Request { + /// Requested versioned blob hashes. + pub versioned_hashes: Vec, +} + +/// V2 uses the V1 request schema. +pub type BlobsV2Request = BlobsV1Request; + +/// V3 uses the V1 request schema. +pub type BlobsV3Request = BlobsV1Request; + +/// V4 blob request container with a packed 128-bit index bitvector. +#[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] +pub struct BlobsV4Request { + /// Requested versioned blob hashes. + pub versioned_hashes: Vec, + /// Requested cell indices, SSZ `Bitvector[128]`. + pub indices_bitarray: B128, +} + +/// Blob response entry with explicit outer availability. +/// +/// REST-SSZ keeps availability separate from the blob contents instead of using a legacy option. +#[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] +pub struct BlobEntry { + /// Whether the complete blob contents are available. + pub available: bool, + /// Complete contents, or valid zero-valued contents when unavailable. + pub contents: T, +} + +/// Bounded blob response container. +/// +/// The outer container and entry availability match the REST-SSZ blob endpoint contract. +#[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] +pub struct BlobsResponse { + /// One response entry per requested hash. + pub entries: Vec>, +} + +/// V1 whole-blob response. +pub type BlobsV1Response = BlobsResponse; + +/// V2 all-or-nothing cell-proof response. +pub type BlobsV2Response = BlobsResponse; + +/// V3 partial cell-proof response. +pub type BlobsV3Response = BlobsResponse; + +/// V4 partial cell-range response. +pub type BlobsV4Response = BlobsResponse; + +/// Blob cells and proofs with REST-SSZ optional cell positions. +/// +/// This uses [`Optional`] (`List[T, 1]`) for per-cell nullability, not Rust [`Option`]'s SSZ +/// union encoding. +#[derive(Clone, Debug, Default, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] +pub struct BlobCellsAndProofs { + /// Requested blob cells. + pub blob_cells: Vec>, + /// KZG proofs for the requested blob cells. + pub proofs: Vec>, +} + +fn zero_blob_v1() -> BlobAndProofV1 { + BlobAndProofV1 { blob: Box::new(Blob::ZERO), proof: Bytes48::ZERO } +} + +fn zero_blob_v2() -> BlobAndProofV2 { + BlobAndProofV2 { blob: Box::new(Blob::ZERO), proofs: Vec::new() } +} + +impl TryFrom>> for BlobsV1Response { + type Error = ConversionError; + + fn try_from(value: Vec>) -> Result { + if value.len() > MAX_BLOBS_REQUEST { + return Err(ConversionError::TooManyItems { + field: "blobs", + max: MAX_BLOBS_REQUEST, + actual: value.len(), + }) + } + + let entries = value + .into_iter() + .map(|value| match value { + Some(contents) => BlobEntry { available: true, contents }, + None => BlobEntry { available: false, contents: zero_blob_v1() }, + }) + .collect(); + Ok(Self { entries }) + } +} + +impl TryFrom> for BlobsV2Response { + type Error = ConversionError; + + fn try_from(value: Vec) -> Result { + if value.len() > MAX_BLOBS_REQUEST { + return Err(ConversionError::TooManyItems { + field: "blobs", + max: MAX_BLOBS_REQUEST, + actual: value.len(), + }) + } + + let entries = + value.into_iter().map(|contents| BlobEntry { available: true, contents }).collect(); + Ok(Self { entries }) + } +} + +impl TryFrom>> for BlobsV3Response { + type Error = ConversionError; + + fn try_from(value: Vec>) -> Result { + if value.len() > MAX_BLOBS_REQUEST { + return Err(ConversionError::TooManyItems { + field: "blobs", + max: MAX_BLOBS_REQUEST, + actual: value.len(), + }) + } + + let entries = value + .into_iter() + .map(|value| match value { + Some(contents) => BlobEntry { available: true, contents }, + None => BlobEntry { available: false, contents: zero_blob_v2() }, + }) + .collect(); + Ok(Self { entries }) + } +} + +impl TryFrom>> for BlobsV4Response { + type Error = ConversionError; + + fn try_from(value: Vec>) -> Result { + if value.len() > MAX_BLOBS_REQUEST { + return Err(ConversionError::TooManyItems { + field: "blobs", + max: MAX_BLOBS_REQUEST, + actual: value.len(), + }) + } + + let entries = value + .into_iter() + .map(|value| match value { + Some(contents) => BlobEntry { + available: true, + contents: BlobCellsAndProofs { + blob_cells: contents.blob_cells.into_iter().map(Optional::from).collect(), + proofs: contents.proofs.into_iter().map(Optional::from).collect(), + }, + }, + None => BlobEntry { available: false, contents: BlobCellsAndProofs::default() }, + }) + .collect(); + Ok(Self { entries }) + } +} + +/// A trie-node byte list in an [`ExecutionWitnessV1`]. +pub type WitnessNodeV1 = Vec; + +/// A contract-code byte list in an [`ExecutionWitnessV1`]. +pub type WitnessCodeV1 = Vec; + +/// An RLP-encoded header byte list in an [`ExecutionWitnessV1`]. +pub type WitnessHeaderV1 = Vec; + +/// Canonical execution witness for `POST /payloads/witness`. +/// +/// `state` and `codes` are produced in lexicographic ascending byte order. `headers` are +/// RLP-encoded and ordered by ascending block number; consecutive headers must be parent-linked. +/// These ordering rules are producer-side requirements from the execution-specs witness builder. +/// +/// This is a REST-SSZ wire container, not the JSON-RPC debug witness shape. +#[derive(Clone, Debug, Default, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] +pub struct ExecutionWitnessV1 { + /// Hashed trie-node preimages required during execution and state-root recomputation. + pub state: Vec, + /// Contract bytecode preimages created or accessed during execution. + pub codes: Vec, + /// RLP-encoded ancestor headers used for pre-state and `BLOCKHASH` correctness proofs. + pub headers: Vec, +} + +/// Canonical execution witness for `POST /payloads/witness`. +pub type ExecutionWitness = ExecutionWitnessV1; + +/// REST-SSZ response for `POST /payloads/witness`. +/// +/// The witness uses the Engine REST-SSZ `Optional[T]` encoding from execution-apis and is present +/// only when the payload status is `VALID`. +#[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode)] +pub struct PayloadStatusWithWitness { + /// Result of processing the submitted payload. + pub payload_status: PayloadStatus, + /// Execution witness produced for a valid payload. + pub witness: Optional, +} + +impl PayloadStatusWithWitness { + /// Creates a response, converting the witness into the REST-SSZ `Optional[T]` representation. + pub fn new(payload_status: PayloadStatus, witness: Option) -> Self { + let witness = match &payload_status.status { + PayloadStatusEnum::Valid => witness.into(), + _ => Optional::none(), + }; + Self { payload_status, witness } + } +} + +/// Backwards-compatible alias for the experimental witness response name. +pub type NewPayloadWithWitnessResponseV1 = PayloadStatusWithWitness; + +impl ssz::Decode for PayloadStatusWithWitness { + fn is_ssz_fixed_len() -> bool { + false + } + + fn from_ssz_bytes(bytes: &[u8]) -> Result { + let mut builder = ssz::SszDecoderBuilder::new(bytes); + builder.register_type::()?; + builder.register_type::>()?; + let mut decoder = builder.build()?; + let response = + Self { payload_status: decoder.decode_next()?, witness: decoder.decode_next()? }; + if response.witness.is_some() && + !matches!(response.payload_status.status, PayloadStatusEnum::Valid) + { + return Err(ssz::DecodeError::BytesInvalid( + "execution witness is only valid for VALID payload status".into(), + )) + } + Ok(response) + } +} + #[cfg(test)] mod tests { use super::*; @@ -389,6 +1500,24 @@ mod tests { } } + fn attributes_cancun() -> PayloadAttributesCancun { + PayloadAttributesCancun { + timestamp: 1, + prev_randao: B256::repeat_byte(2), + suggested_fee_recipient: Address::repeat_byte(3), + withdrawals: vec![Withdrawal::default()], + parent_beacon_block_root: B256::repeat_byte(4), + } + } + + fn state() -> ForkchoiceState { + ForkchoiceState { + head_block_hash: B256::repeat_byte(1), + safe_block_hash: B256::repeat_byte(2), + finalized_block_hash: B256::repeat_byte(3), + } + } + fn assert_roundtrip(value: &T) where T: Encode + Decode + PartialEq + core::fmt::Debug, @@ -540,4 +1669,301 @@ mod tests { assert_eq!(encoded[48], 1); assert_eq!(&encoded[encoded.len() - 2..], &[0xaa, 0xbb]); } + + #[test] + fn forkchoice_updates_roundtrip() { + let paris = PayloadAttributesParis { + timestamp: 1, + prev_randao: B256::repeat_byte(2), + suggested_fee_recipient: Address::repeat_byte(3), + }; + let shanghai = PayloadAttributesShanghai { + timestamp: 1, + prev_randao: B256::repeat_byte(2), + suggested_fee_recipient: Address::repeat_byte(3), + withdrawals: vec![Withdrawal::default()], + }; + let amsterdam = PayloadAttributesAmsterdam { + timestamp: 1, + prev_randao: B256::repeat_byte(2), + suggested_fee_recipient: Address::repeat_byte(3), + withdrawals: vec![Withdrawal::default()], + parent_beacon_block_root: B256::repeat_byte(4), + slot_number: 5, + target_gas_limit: 6, + }; + assert_roundtrip(&ForkchoiceUpdateParis { + forkchoice_state: state(), + payload_attributes: Optional::some(paris), + }); + assert_roundtrip(&ForkchoiceUpdateShanghai { + forkchoice_state: state(), + payload_attributes: Optional::some(shanghai), + }); + assert_roundtrip(&ForkchoiceUpdateCancun { + forkchoice_state: state(), + payload_attributes: Optional::some(attributes_cancun()), + }); + assert_roundtrip(&ForkchoiceUpdatePrague { + forkchoice_state: state(), + payload_attributes: Optional::some(attributes_cancun()), + }); + assert_roundtrip(&ForkchoiceUpdateOsaka { + forkchoice_state: state(), + payload_attributes: Optional::some(attributes_cancun()), + }); + assert_roundtrip(&ForkchoiceUpdateAmsterdam { + forkchoice_state: state(), + payload_attributes: Optional::some(amsterdam), + custody_columns: Optional::some(B128::repeat_byte(0xa5)), + }); + } + + #[test] + fn payload_attributes_legacy_conversions_preserve_fork_shape() { + let cancun = attributes_cancun(); + let legacy = LegacyPayloadAttributes::from(cancun.clone()); + assert_eq!(PayloadAttributesCancun::try_from(legacy).unwrap(), cancun); + + let amsterdam = PayloadAttributesAmsterdam { + timestamp: 1, + prev_randao: B256::repeat_byte(2), + suggested_fee_recipient: Address::repeat_byte(3), + withdrawals: vec![Withdrawal::default()], + parent_beacon_block_root: B256::repeat_byte(4), + slot_number: 5, + target_gas_limit: 6, + }; + let legacy = LegacyPayloadAttributes::from(amsterdam.clone()); + assert_eq!(PayloadAttributesAmsterdam::try_from(legacy).unwrap(), amsterdam); + } + + #[test] + fn payload_attributes_legacy_conversions_reject_loss() { + let mut legacy = LegacyPayloadAttributes::default(); + assert_eq!( + PayloadAttributesShanghai::try_from(legacy.clone()), + Err(PayloadAttributesConversionError::MissingField("withdrawals")) + ); + + legacy.withdrawals = Some(vec![]); + legacy.parent_beacon_block_root = Some(B256::ZERO); + assert_eq!( + PayloadAttributesShanghai::try_from(legacy), + Err(PayloadAttributesConversionError::UnexpectedField("parent_beacon_block_root")) + ); + } + + #[test] + fn every_payload_status_roundtrips() { + for status in [ + PayloadStatusEnum::Valid, + PayloadStatusEnum::Invalid { validation_error: "invalid".into() }, + PayloadStatusEnum::Syncing, + PayloadStatusEnum::Accepted, + ] { + let validation_error = legacy_validation_error(&status).unwrap(); + let value = PayloadStatus { + status, + latest_valid_hash: Optional::some(B256::ZERO), + validation_error, + }; + assert_eq!(PayloadStatus::from_ssz_bytes(&value.as_ssz_bytes()).unwrap(), value); + } + } + + #[test] + fn payload_status_preserves_absent_invalid_validation_error() { + let mut bytes = Vec::new(); + let mut encoder = ssz::SszEncoder::container(&mut bytes, 9); + encoder.append(&1u8); + encoder.append(&Optional::::none()); + encoder.append(&Optional::::none()); + encoder.finalize(); + + let decoded = PayloadStatus::from_ssz_bytes(&bytes).unwrap(); + assert!(decoded.validation_error.is_none()); + assert_eq!(decoded.as_ssz_bytes(), bytes); + } + + #[test] + fn payload_status_rejects_non_invalid_validation_error() { + let mut bytes = Vec::new(); + let mut encoder = ssz::SszEncoder::container(&mut bytes, 9); + encoder.append(&0u8); + encoder.append(&Optional::::none()); + encoder.append(&Optional::some(Vec::::new())); + encoder.finalize(); + assert!(PayloadStatus::from_ssz_bytes(&bytes).is_err()); + } + + #[test] + fn payload_status_legacy_conversion_rejects_oversized_error() { + assert!(PayloadStatus::try_from(LegacyPayloadStatus { + status: PayloadStatusEnum::Invalid { validation_error: "x".repeat(1025) }, + latest_valid_hash: None, + }) + .is_err()); + } + + #[test] + fn forkchoice_response_distinguishes_absent_and_zero_payload_id() { + let status = PayloadStatus { + status: PayloadStatusEnum::Valid, + latest_valid_hash: Optional::none(), + validation_error: Optional::none(), + }; + let none = ForkchoiceUpdateResponse { + payload_status: status.clone(), + payload_id: Optional::none(), + }; + let zero = ForkchoiceUpdateResponse { + payload_status: status, + payload_id: Optional::some(PayloadId::default()), + }; + + assert_ne!(none.as_ssz_bytes(), zero.as_ssz_bytes()); + assert_roundtrip(&none); + assert_roundtrip(&zero); + } + + #[test] + fn forkchoice_conversion_rejects_accepted() { + let legacy = LegacyForkchoice::from_status(PayloadStatusEnum::Accepted); + assert_eq!( + ForkchoiceUpdateResponse::try_from(legacy), + Err(ConversionError::AcceptedForkchoice) + ); + } + + fn blob_v2(byte: u8) -> BlobAndProofV2 { + BlobAndProofV2 { + blob: Box::new(Blob::repeat_byte(byte)), + proofs: vec![Bytes48::repeat_byte(byte)], + } + } + + #[test] + fn blob_requests_are_single_field_containers() { + let request = BlobsV1Request { versioned_hashes: vec![B256::repeat_byte(0x42)] }; + let encoded = request.as_ssz_bytes(); + + assert_eq!(&encoded[..4], &4u32.to_le_bytes()); + assert_eq!(&encoded[4..], B256::repeat_byte(0x42).as_slice()); + assert_eq!(BlobsV1Request::from_ssz_bytes(&encoded).unwrap(), request); + + let _: BlobsV2Request = BlobsV2Request::from_ssz_bytes(&encoded).unwrap(); + let _: BlobsV3Request = BlobsV3Request::from_ssz_bytes(&encoded).unwrap(); + } + + #[test] + fn blob_v4_request_roundtrips_bitvector() { + let request = BlobsV4Request { + versioned_hashes: vec![B256::repeat_byte(0x11)], + indices_bitarray: B128::repeat_byte(0xa5), + }; + + assert_roundtrip(&request); + } + + #[test] + fn blob_response_conversions_preserve_availability_and_order() { + let v1 = BlobsV1Response::try_from(vec![None]).unwrap(); + assert!(!v1.entries[0].available); + assert_eq!(v1.entries[0].contents, zero_blob_v1()); + + let v2 = BlobsV2Response::try_from(vec![blob_v2(1), blob_v2(2)]).unwrap(); + assert!(v2.entries.iter().all(|entry| entry.available)); + + let v3 = BlobsV3Response::try_from(vec![Some(blob_v2(1)), None, Some(blob_v2(3))]).unwrap(); + assert_eq!( + v3.entries.iter().map(|entry| entry.available).collect::>(), + [true, false, true] + ); + assert_eq!(v3.entries[2].contents.blob.as_slice(), Blob::repeat_byte(3).as_slice()); + + let legacy_partial = BlobCellsAndProofsV1 { + blob_cells: vec![Some(Cell::repeat_byte(1)), None], + proofs: vec![Some(Bytes48::repeat_byte(2)), None], + }; + let v4 = BlobsV4Response::try_from(vec![None, Some(legacy_partial)]).unwrap(); + assert!(!v4.entries[0].available); + assert!(v4.entries[1].available); + assert!(v4.entries[1].contents.blob_cells[0].is_some()); + assert!(v4.entries[1].contents.proofs[1].is_none()); + } + + #[test] + fn blob_cells_and_proofs_uses_rest_optional() { + let value = BlobCellsAndProofs { + blob_cells: vec![Optional::some(Cell::repeat_byte(1))], + proofs: vec![Optional::some(Bytes48::repeat_byte(2))], + }; + let encoded = value.as_ssz_bytes(); + + assert_eq!(BlobCellsAndProofs::from_ssz_bytes(&encoded).unwrap(), value); + assert!(!encoded[8..].starts_with(&[1, 0, 0, 0])); + } + + #[test] + fn payload_body_requests_are_single_field_containers() { + let request = BodiesByHashRequest { block_hashes: vec![B256::repeat_byte(0x33)] }; + let encoded = request.as_ssz_bytes(); + + assert_eq!(&encoded[..4], &4u32.to_le_bytes()); + assert_eq!(&encoded[4..], B256::repeat_byte(0x33).as_slice()); + assert_eq!(BodiesByHashRequest::from_ssz_bytes(&encoded).unwrap(), request); + } + + #[test] + fn payload_body_responses_preserve_availability() { + let legacy = LegacyExecutionPayloadBodyV1 { + transactions: vec![Bytes::from_static(&[1, 2, 3])], + withdrawals: Some(vec![Withdrawal::default()]), + }; + let response = + BodiesResponseShanghai::from_optional_bodies(vec![Some(legacy), None], |body| { + ExecutionPayloadBodyShanghai::try_from(body).ok() + }); + + assert!(response.entries[0].available); + assert!(!response.entries[1].available); + assert_roundtrip(&response); + } + + #[test] + fn witness_response_roundtrips_when_status_is_valid() { + let payload_status = PayloadStatus { + status: PayloadStatusEnum::Valid, + latest_valid_hash: Optional::none(), + validation_error: Optional::none(), + }; + let witness = ExecutionWitnessV1 { + state: vec![vec![1, 2, 3]], + codes: vec![vec![4, 5]], + headers: vec![vec![6]], + }; + let response = PayloadStatusWithWitness::new(payload_status, Some(witness)); + + assert_roundtrip(&response); + } + + #[test] + fn witness_response_omits_witness_for_non_valid_status() { + let payload_status = PayloadStatus { + status: PayloadStatusEnum::Syncing, + latest_valid_hash: Optional::none(), + validation_error: Optional::none(), + }; + let response = + PayloadStatusWithWitness::new(payload_status, Some(ExecutionWitnessV1::default())); + + assert!(response.witness.is_none()); + assert_roundtrip(&response); + } + + #[test] + fn optional_rejects_more_than_one_value() { + assert!(Optional::::from_ssz_bytes(&[0; 64]).is_err()); + } } diff --git a/crates/ethereum/node/src/engine_ssz_proxy.rs b/crates/ethereum/node/src/engine_ssz_proxy.rs index 889cbc34584..da1fc821e50 100644 --- a/crates/ethereum/node/src/engine_ssz_proxy.rs +++ b/crates/ethereum/node/src/engine_ssz_proxy.rs @@ -4,16 +4,21 @@ //! //! [EIP-8178]: https://eips.ethereum.org/EIPS/eip-8178 -use alloy_consensus::{Transaction, TxEnvelope}; -use alloy_eips::{ - eip2718::Decodable2718, - eip7685::{Requests, RequestsOrHash}, +use crate::engine_ssz_containers::{ + BuiltPayloadAmsterdam, BuiltPayloadCancun, BuiltPayloadOsaka, BuiltPayloadParis, + BuiltPayloadPrague, BuiltPayloadShanghai, ExecutionPayloadEnvelopeAmsterdam, + ExecutionPayloadEnvelopeCancun, ExecutionPayloadEnvelopeOsaka, ExecutionPayloadEnvelopeParis, + ExecutionPayloadEnvelopePrague, ExecutionPayloadEnvelopeShanghai, ForkchoiceUpdateAmsterdam, + ForkchoiceUpdateCancun, ForkchoiceUpdateOsaka, ForkchoiceUpdateParis, ForkchoiceUpdatePrague, + ForkchoiceUpdateResponse, ForkchoiceUpdateShanghai, Optional, + PayloadStatus as EngineSszPayloadStatus, }; -use alloy_primitives::{Bytes, B128, B256}; +use alloy_consensus::{Transaction, TxEnvelope}; +use alloy_eips::{eip2718::Decodable2718, eip7685::RequestsOrHash}; +use alloy_primitives::{Bytes, B128, B256, B64}; use alloy_rpc_types_engine::{ - CancunPayloadFields, ExecutionData, ExecutionPayload, ExecutionPayloadSidecar, - ExecutionPayloadV1, ExecutionPayloadV2, ExecutionPayloadV3, ExecutionPayloadV4, - ForkchoiceState, PayloadAttributes, PraguePayloadFields, + CancunPayloadFields, ExecutionData, ExecutionPayload, ExecutionPayloadFieldV2, + ExecutionPayloadSidecar, ForkchoiceState, PayloadAttributes, PayloadId, PraguePayloadFields, }; use http_body_util::BodyExt; use jsonrpsee::server::{HttpBody, HttpRequest, HttpResponse}; @@ -22,6 +27,7 @@ use reth_engine_primitives::EngineApiValidator; use reth_ethereum_engine_primitives::EthEngineTypes; use reth_provider::{BalProvider, BlockReader, HeaderProvider, StateProviderFactory}; use reth_rpc::EngineApi; +use reth_rpc_engine_api::EngineApiError; use reth_transaction_pool::TransactionPool; use ssz::Decode; use std::{ @@ -37,6 +43,7 @@ const OCTET_STREAM: &str = "application/octet-stream"; const APPLICATION_JSON: &str = "application/json"; const TEXT_PLAIN: &str = "text/plain"; const CONTENT_TYPE: &str = "content-type"; +const CACHE_CONTROL: &str = "cache-control"; const ETH_EXECUTION_VERSION: &str = "eth-execution-version"; const STATUS_OK: u16 = 200; @@ -228,7 +235,22 @@ where let Some(engine_api) = handle.engine_api().await else { return text_response(STATUS_SERVICE_UNAVAILABLE, "engine api unavailable") }; - handle_new_payload(engine_api, fork.payloads_version(), &body).await + handle_new_payload(engine_api, fork, &body).await + } + EngineSszEndpoint::GetPayload(payload_id) => { + if method != "GET" { + return text_response(STATUS_METHOD_NOT_ALLOWED, "method not allowed") + } + let Ok(payload_id) = payload_id else { + return text_response(STATUS_BAD_REQUEST, "invalid payload id") + }; + let Some(fork) = request_fork(&request) else { + return text_response(STATUS_BAD_REQUEST, "unsupported fork") + }; + let Some(engine_api) = handle.engine_api().await else { + return text_response(STATUS_SERVICE_UNAVAILABLE, "engine api unavailable") + }; + handle_get_payload(engine_api, fork, payload_id).await } EngineSszEndpoint::Forkchoice => { if method != "POST" { @@ -243,7 +265,7 @@ where let Some(engine_api) = handle.engine_api().await else { return text_response(STATUS_SERVICE_UNAVAILABLE, "engine api unavailable") }; - handle_forkchoice_updated(engine_api, fork.forkchoice_version(), &body).await + handle_forkchoice_updated(engine_api, fork, &body).await } EngineSszEndpoint::Blobs(version) => { if method != "POST" { @@ -276,6 +298,10 @@ fn parse_engine_path(path: &str) -> Option { (Some("engine"), Some("v1"), Some("payloads"), None, None) => { Some(EngineSszEndpoint::NewPayload) } + (Some("engine"), Some("v1"), Some("payloads"), Some(payload_id), None) => { + let payload_id = payload_id.parse::().map(PayloadId::from); + Some(EngineSszEndpoint::GetPayload(payload_id)) + } (Some("engine"), Some("v1"), Some("forkchoice"), None, None) => { Some(EngineSszEndpoint::Forkchoice) } @@ -291,6 +317,7 @@ enum EngineSszEndpoint { Capabilities, Identity, NewPayload, + GetPayload(Result::Err>), Forkchoice, Blobs(u8), } @@ -374,9 +401,72 @@ where json_response(vec![engine_api.client_version().clone()]) } +async fn handle_get_payload( + engine_api: EthEngineApi, + fork: EngineSszFork, + payload_id: PayloadId, +) -> HttpResponse +where + Provider: HeaderProvider + BlockReader + StateProviderFactory + BalProvider + 'static, + Pool: TransactionPool + 'static, + Validator: EngineApiValidator, + ChainSpec: EthereumHardforks + Send + Sync + 'static, +{ + match fork { + EngineSszFork::Paris => match engine_api.get_payload_v2_metered(payload_id).await { + Ok(payload) => { + let block_value = payload.block_value; + match payload.execution_payload { + ExecutionPayloadFieldV2::V1(payload) => { + get_payload_response(BuiltPayloadParis { payload, block_value }) + } + ExecutionPayloadFieldV2::V2(_) => { + text_response(STATUS_BAD_REQUEST, "unsupported fork") + } + } + } + Err(err) => get_payload_error_response(err), + }, + EngineSszFork::Shanghai => match engine_api.get_payload_v2_metered(payload_id).await { + Ok(payload) => match BuiltPayloadShanghai::try_from(payload) { + Ok(payload) => get_payload_response(payload), + Err(err) => text_response(STATUS_BAD_REQUEST, err.to_string()), + }, + Err(err) => get_payload_error_response(err), + }, + EngineSszFork::Cancun => match engine_api.get_payload_v3_metered(payload_id).await { + Ok(payload) => get_payload_response(BuiltPayloadCancun::from(payload)), + Err(err) => get_payload_error_response(err), + }, + EngineSszFork::Prague => match engine_api.get_payload_v4_metered(payload_id).await { + Ok(payload) => get_payload_response(BuiltPayloadPrague::from(payload)), + Err(err) => get_payload_error_response(err), + }, + EngineSszFork::Osaka => match engine_api.get_payload_v5_metered(payload_id).await { + Ok(payload) => get_payload_response(BuiltPayloadOsaka::from(payload)), + Err(err) => get_payload_error_response(err), + }, + EngineSszFork::Amsterdam => match engine_api.get_payload_v6_metered(payload_id).await { + Ok(payload) => get_payload_response(BuiltPayloadAmsterdam::from(payload)), + Err(err) => get_payload_error_response(err), + }, + } +} + +fn get_payload_error_response(err: EngineApiError) -> HttpResponse { + let status = match &err { + EngineApiError::UnknownPayload => STATUS_NOT_FOUND, + EngineApiError::EngineObjectValidationError( + reth_payload_primitives::EngineObjectValidationError::UnsupportedFork, + ) => STATUS_BAD_REQUEST, + _ => STATUS_INTERNAL_SERVER_ERROR, + }; + text_response(status, err.to_string()) +} + async fn handle_new_payload( engine_api: EthEngineApi, - version: u8, + fork: EngineSszFork, body: &[u8], ) -> HttpResponse where @@ -385,12 +475,12 @@ where Validator: EngineApiValidator, ChainSpec: EthereumHardforks + Send + Sync + 'static, { - let payload = match decode_new_payload_request(version, body) { + let payload = match decode_new_payload_request(fork, body) { Ok(payload) => payload, Err(err) => return text_response(STATUS_BAD_REQUEST, err), }; - let response = match version { + let response = match fork.payloads_version() { 1 => engine_api.new_payload_v1(payload).await, 2 => engine_api.new_payload_v2(payload).await, 3 => engine_api.new_payload_v3(payload).await, @@ -400,14 +490,17 @@ where }; match response { - Ok(status) => ssz_response(status), + Ok(status) => match EngineSszPayloadStatus::try_from(status) { + Ok(status) => ssz_response(status), + Err(err) => text_response(STATUS_INTERNAL_SERVER_ERROR, err.to_string()), + }, Err(err) => text_response(STATUS_INTERNAL_SERVER_ERROR, err.to_string()), } } async fn handle_forkchoice_updated( engine_api: EthEngineApi, - version: u8, + fork: EngineSszFork, body: &[u8], ) -> HttpResponse where @@ -416,12 +509,12 @@ where Validator: EngineApiValidator, ChainSpec: EthereumHardforks + Send + Sync + 'static, { - let (state, attrs, custody_columns) = match decode_forkchoice_request(version, body) { + let (state, attrs, custody_columns) = match decode_forkchoice_request(fork, body) { Ok(request) => request, Err(err) => return text_response(STATUS_BAD_REQUEST, err), }; - let response = match version { + let response = match fork.forkchoice_version() { 1 => engine_api.fork_choice_updated_v1_metered(state, attrs).await, 2 => engine_api.fork_choice_updated_v2_metered(state, attrs).await, 3 => engine_api.fork_choice_updated_v3_metered(state, attrs).await, @@ -430,7 +523,10 @@ where }; match response { - Ok(updated) => ssz_response(updated), + Ok(updated) => match ForkchoiceUpdateResponse::try_from(updated) { + Ok(updated) => ssz_response(updated), + Err(err) => text_response(STATUS_INTERNAL_SERVER_ERROR, err.to_string()), + }, Err(err) => text_response(STATUS_INTERNAL_SERVER_ERROR, err.to_string()), } } @@ -505,21 +601,27 @@ fn decode_blob_cells_request(body: &[u8]) -> Result<(Vec, B128), &'static <(Vec, B128) as ssz::Decode>::from_ssz_bytes(body).map_err(|_| "invalid ssz") } -fn decode_new_payload_request(version: u8, body: &[u8]) -> Result { - match version { - 1 => { - let execution_payload = - decode_one::(body).map_err(|_| "invalid ssz")?; +fn decode_new_payload_request( + fork: EngineSszFork, + body: &[u8], +) -> Result { + match fork { + EngineSszFork::Paris => { + let ExecutionPayloadEnvelopeParis { payload: execution_payload } = + ExecutionPayloadEnvelopeParis::from_ssz_bytes(body).map_err(|_| "invalid ssz")?; Ok(ExecutionData::new(execution_payload.into(), ExecutionPayloadSidecar::none())) } - 2 => { - let execution_payload = - decode_one::(body).map_err(|_| "invalid ssz")?; + EngineSszFork::Shanghai => { + let ExecutionPayloadEnvelopeShanghai { payload: execution_payload } = + ExecutionPayloadEnvelopeShanghai::from_ssz_bytes(body) + .map_err(|_| "invalid ssz")?; Ok(ExecutionData::new(execution_payload.into(), ExecutionPayloadSidecar::none())) } - 3 => { - let (execution_payload, parent_beacon_block_root) = - <(ExecutionPayloadV3, B256)>::from_ssz_bytes(body).map_err(|_| "invalid ssz")?; + EngineSszFork::Cancun => { + let ExecutionPayloadEnvelopeCancun { + payload: execution_payload, + parent_beacon_block_root, + } = ExecutionPayloadEnvelopeCancun::from_ssz_bytes(body).map_err(|_| "invalid ssz")?; let versioned_hashes = calculate_versioned_hashes( &execution_payload.payload_inner.payload_inner.transactions, )?; @@ -529,37 +631,52 @@ fn decode_new_payload_request(version: u8, body: &[u8]) -> Result { - let (execution_payload, parent_beacon_block_root, execution_requests) = - <(ExecutionPayloadV3, B256, Vec)>::from_ssz_bytes(body) - .map_err(|_| "invalid ssz")?; + EngineSszFork::Prague => { + let ExecutionPayloadEnvelopePrague { + payload: execution_payload, + parent_beacon_block_root, + execution_requests, + } = ExecutionPayloadEnvelopePrague::from_ssz_bytes(body).map_err(|_| "invalid ssz")?; let versioned_hashes = calculate_versioned_hashes( &execution_payload.payload_inner.payload_inner.transactions, )?; let sidecar = ExecutionPayloadSidecar::v4( CancunPayloadFields { parent_beacon_block_root, versioned_hashes }, - PraguePayloadFields::new(RequestsOrHash::Requests(Requests::new( - execution_requests, - ))), + PraguePayloadFields::new(RequestsOrHash::Requests(execution_requests)), ); Ok(ExecutionData::new(execution_payload.into(), sidecar)) } - 5 => { - let (execution_payload, parent_beacon_block_root, execution_requests) = - <(ExecutionPayloadV4, B256, Vec)>::from_ssz_bytes(body) - .map_err(|_| "invalid ssz")?; + EngineSszFork::Osaka => { + let ExecutionPayloadEnvelopeOsaka { + payload: execution_payload, + parent_beacon_block_root, + execution_requests, + } = ExecutionPayloadEnvelopeOsaka::from_ssz_bytes(body).map_err(|_| "invalid ssz")?; + let versioned_hashes = calculate_versioned_hashes( + &execution_payload.payload_inner.payload_inner.transactions, + )?; + let sidecar = ExecutionPayloadSidecar::v4( + CancunPayloadFields { parent_beacon_block_root, versioned_hashes }, + PraguePayloadFields::new(RequestsOrHash::Requests(execution_requests)), + ); + Ok(ExecutionData::new(execution_payload.into(), sidecar)) + } + EngineSszFork::Amsterdam => { + let ExecutionPayloadEnvelopeAmsterdam { + payload: execution_payload, + parent_beacon_block_root, + execution_requests, + } = ExecutionPayloadEnvelopeAmsterdam::from_ssz_bytes(body) + .map_err(|_| "invalid ssz")?; let versioned_hashes = calculate_versioned_hashes( &execution_payload.payload_inner.payload_inner.payload_inner.transactions, )?; let sidecar = ExecutionPayloadSidecar::v4( CancunPayloadFields { parent_beacon_block_root, versioned_hashes }, - PraguePayloadFields::new(RequestsOrHash::Requests(Requests::new( - execution_requests, - ))), + PraguePayloadFields::new(RequestsOrHash::Requests(execution_requests)), ); Ok(ExecutionData::new(ExecutionPayload::V4(execution_payload), sidecar)) } - _ => Err("unsupported payload endpoint version"), } } @@ -577,95 +694,67 @@ fn calculate_versioned_hashes(transactions: &[Bytes]) -> Result, &'sta } fn decode_forkchoice_request( - version: u8, + fork: EngineSszFork, body: &[u8], ) -> Result<(ForkchoiceState, Option, Option), &'static str> { - match version { - 1..=3 => { - let (forkchoice_state, payload_attributes) = - <(ForkchoiceState, Vec)>::from_ssz_bytes(body) - .map_err(|_| "invalid ssz")?; - Ok((forkchoice_state, payload_attrs(version, payload_attributes)?, None)) + match fork { + EngineSszFork::Paris => { + let ForkchoiceUpdateParis { forkchoice_state, payload_attributes } = + ForkchoiceUpdateParis::from_ssz_bytes(body).map_err(|_| "invalid ssz")?; + Ok((forkchoice_state, optional_attrs(payload_attributes), None)) } - 4 => { - let (forkchoice_state, payload_attributes, custody_columns) = - <(ForkchoiceState, Vec, Vec)>::from_ssz_bytes(body) - .map_err(|_| "invalid ssz")?; + EngineSszFork::Shanghai => { + let ForkchoiceUpdateShanghai { forkchoice_state, payload_attributes } = + ForkchoiceUpdateShanghai::from_ssz_bytes(body).map_err(|_| "invalid ssz")?; + Ok((forkchoice_state, optional_attrs(payload_attributes), None)) + } + EngineSszFork::Cancun => { + let ForkchoiceUpdateCancun { forkchoice_state, payload_attributes } = + ForkchoiceUpdateCancun::from_ssz_bytes(body).map_err(|_| "invalid ssz")?; + Ok((forkchoice_state, optional_attrs(payload_attributes), None)) + } + EngineSszFork::Prague => { + let ForkchoiceUpdatePrague { forkchoice_state, payload_attributes } = + ForkchoiceUpdatePrague::from_ssz_bytes(body).map_err(|_| "invalid ssz")?; + Ok((forkchoice_state, optional_attrs(payload_attributes), None)) + } + EngineSszFork::Osaka => { + let ForkchoiceUpdateOsaka { forkchoice_state, payload_attributes } = + ForkchoiceUpdateOsaka::from_ssz_bytes(body).map_err(|_| "invalid ssz")?; + Ok((forkchoice_state, optional_attrs(payload_attributes), None)) + } + EngineSszFork::Amsterdam => { + let ForkchoiceUpdateAmsterdam { forkchoice_state, payload_attributes, custody_columns } = + ForkchoiceUpdateAmsterdam::from_ssz_bytes(body).map_err(|_| "invalid ssz")?; Ok(( forkchoice_state, - payload_attrs(version, payload_attributes)?, - custody_columns_opt(custody_columns)?, + optional_attrs(payload_attributes), + custody_columns.into_option(), )) } - _ => Err("unsupported forkchoice endpoint version"), } } -fn decode_one(body: &[u8]) -> Result { - let mut builder = ssz::SszDecoderBuilder::new(body); - builder.register_type::()?; - let mut decoder = builder.build()?; - decoder.decode_next() -} - -fn payload_attrs( - version: u8, - attrs: Vec, -) -> Result, &'static str> { - if attrs.len() > 1 { - return Err("payload_attributes must contain at most one value") - } - - attrs.into_iter().next().map(|attrs| validate_payload_attrs_version(version, attrs)).transpose() -} - -fn custody_columns_opt(custody_columns: Vec) -> Result, &'static str> { - if custody_columns.len() > 1 { - return Err("invalid params") - } - - Ok(custody_columns.into_iter().next()) +fn optional_attrs(attrs: Optional) -> Option +where + T: Into, +{ + attrs.into_option().map(Into::into) } -fn validate_payload_attrs_version( - version: u8, - attrs: PayloadAttributes, -) -> Result { - let matches_version = match version { - 1 => { - attrs.withdrawals.is_none() && - attrs.parent_beacon_block_root.is_none() && - attrs.slot_number.is_none() - } - 2 => { - attrs.withdrawals.is_some() && - attrs.parent_beacon_block_root.is_none() && - attrs.slot_number.is_none() - } - 3 => { - attrs.withdrawals.is_some() && - attrs.parent_beacon_block_root.is_some() && - attrs.slot_number.is_none() - } - 4 => { - attrs.withdrawals.is_some() && - attrs.parent_beacon_block_root.is_some() && - attrs.slot_number.is_some() - } - _ => false, - }; - - if matches_version { - Ok(attrs) - } else { - Err("payload_attributes version does not match endpoint") - } +fn ssz_response(value: T) -> HttpResponse { + HttpResponse::builder() + .status(STATUS_OK) + .header(CONTENT_TYPE, OCTET_STREAM) + .body(HttpBody::from(value.as_ssz_bytes())) + .expect("valid response") } -fn ssz_response(value: T) -> HttpResponse { +fn get_payload_response(value: T) -> HttpResponse { HttpResponse::builder() .status(STATUS_OK) .header(CONTENT_TYPE, OCTET_STREAM) + .header(CACHE_CONTROL, "no-store") .body(HttpBody::from(value.as_ssz_bytes())) .expect("valid response") } @@ -717,6 +806,23 @@ mod tests { assert_eq!(endpoint, EngineSszEndpoint::NewPayload); } + #[test] + fn parses_fork_scoped_get_payload_endpoint() { + let endpoint = parse_engine_path("/engine/v1/payloads/0x0000000000000001").unwrap(); + assert_eq!( + endpoint, + EngineSszEndpoint::GetPayload(Ok(PayloadId::new([0, 0, 0, 0, 0, 0, 0, 1]))) + ); + } + + #[test] + fn matches_malformed_get_payload_endpoint() { + assert!(matches!( + parse_engine_path("/engine/v1/payloads/0x01"), + Some(EngineSszEndpoint::GetPayload(Err(_))) + )); + } + #[test] fn parses_fork_scoped_forkchoice_endpoint() { let endpoint = parse_engine_path("/engine/v1/forkchoice").unwrap(); @@ -742,32 +848,45 @@ mod tests { safe_block_hash: B256::ZERO, finalized_block_hash: B256::ZERO, }; - let encoded = - (forkchoice_state, Vec::::new(), vec![B128::with_last_byte(1)]) - .as_ssz_bytes(); + let encoded = ForkchoiceUpdateAmsterdam { + forkchoice_state, + payload_attributes: Optional::none(), + custody_columns: Optional::some(B128::with_last_byte(1)), + } + .as_ssz_bytes(); let (decoded_state, decoded_attrs, custody_columns) = - decode_forkchoice_request(4, &encoded).unwrap(); + decode_forkchoice_request(EngineSszFork::Amsterdam, &encoded).unwrap(); assert_eq!(decoded_state, forkchoice_state); assert!(decoded_attrs.is_none()); assert_eq!(custody_columns, Some(B128::with_last_byte(1))); } #[test] - fn rejects_forkchoice_v4_with_multiple_custody_columns() { + fn decodes_forkchoice_cancun_payload_attributes() { let forkchoice_state = ForkchoiceState { head_block_hash: B256::ZERO, safe_block_hash: B256::ZERO, finalized_block_hash: B256::ZERO, }; - let encoded = ( - forkchoice_state, - Vec::::new(), - vec![B128::ZERO, B128::with_last_byte(1)], - ) - .as_ssz_bytes(); + let attrs = crate::engine_ssz_containers::PayloadAttributesCancun { + timestamp: 1, + prev_randao: B256::with_last_byte(2), + suggested_fee_recipient: Default::default(), + withdrawals: Vec::new(), + parent_beacon_block_root: B256::with_last_byte(3), + }; + let encoded = + ForkchoiceUpdateCancun { forkchoice_state, payload_attributes: Optional::some(attrs) } + .as_ssz_bytes(); - let err = decode_forkchoice_request(4, &encoded).unwrap_err(); - assert_eq!(err, "invalid params"); + let (decoded_state, decoded_attrs, custody_columns) = + decode_forkchoice_request(EngineSszFork::Cancun, &encoded).unwrap(); + assert_eq!(decoded_state, forkchoice_state); + let decoded_attrs = decoded_attrs.unwrap(); + assert_eq!(decoded_attrs.timestamp, 1); + assert!(decoded_attrs.withdrawals.as_ref().unwrap().is_empty()); + assert_eq!(decoded_attrs.parent_beacon_block_root, Some(B256::with_last_byte(3))); + assert!(custody_columns.is_none()); } } diff --git a/crates/ethereum/node/src/node.rs b/crates/ethereum/node/src/node.rs index 00b3bb4dc4a..06ff516f6e2 100644 --- a/crates/ethereum/node/src/node.rs +++ b/crates/ethereum/node/src/node.rs @@ -509,6 +509,7 @@ fn jit_runtime_config(jit: &JitArgs) -> RuntimeConfig { dump_dir: default_config.dump_dir, debug_assertions: jit.debug, blocking: jit.blocking, + single_error: default_config.single_error, no_dedup: default_config.no_dedup, no_dse: default_config.no_dse, gas_params: default_config.gas_params, diff --git a/crates/ethereum/node/tests/e2e/eth.rs b/crates/ethereum/node/tests/e2e/eth.rs index f4d1340bb41..b210a9db556 100644 --- a/crates/ethereum/node/tests/e2e/eth.rs +++ b/crates/ethereum/node/tests/e2e/eth.rs @@ -3,8 +3,7 @@ use alloy_eips::{eip4844::BlobAndProofV1, eip7685::RequestsOrHash}; use alloy_genesis::Genesis; use alloy_primitives::{Address, B256}; use alloy_rpc_types_engine::{ - ClientVersionV1, ForkchoiceState, ForkchoiceUpdated, PayloadAttributes, PayloadStatus, - PayloadStatusEnum, + ClientVersionV1, ForkchoiceState, PayloadAttributes, PayloadStatusEnum, }; use jsonrpsee_core::client::ClientT; use reth_chainspec::{ChainSpecBuilder, EthChainSpec, MAINNET}; @@ -19,8 +18,11 @@ use reth_node_core::{ version::{version_metadata, CLIENT_CODE}, }; use reth_node_ethereum::{ - engine_ssz_proxy::EngineSszProxyLayer, EthereumAddOns, EthereumEngineValidatorBuilder, - EthereumNode, + engine_ssz_containers::{ + ForkchoiceUpdateResponse as SszForkchoiceUpdateResponse, PayloadStatus as SszPayloadStatus, + }, + engine_ssz_proxy::EngineSszProxyLayer, + EthereumAddOns, EthereumEngineValidatorBuilder, EthereumNode, }; use reth_provider::BlockNumReader; use reth_rpc_api::TestingBuildBlockRequestV1; @@ -420,7 +422,7 @@ async fn test_engine_ssz_proxy_can_mine_block() -> eyre::Result<()> { .await?; assert_eq!(new_payload_response.status(), reqwest::StatusCode::OK); - let status = PayloadStatus::from_ssz_bytes(&new_payload_response.bytes().await?).unwrap(); + let status = SszPayloadStatus::from_ssz_bytes(&new_payload_response.bytes().await?).unwrap(); assert_eq!(status.status, PayloadStatusEnum::Valid); let fcu_response = client @@ -464,7 +466,7 @@ async fn test_engine_ssz_proxy_can_mine_block() -> eyre::Result<()> { assert_eq!(blobs.len(), versioned_hashes.len()); assert!(blobs.iter().all(Option::is_some)); - let fcu = ForkchoiceUpdated::from_ssz_bytes(&fcu_response.bytes().await?).unwrap(); + let fcu = SszForkchoiceUpdateResponse::from_ssz_bytes(&fcu_response.bytes().await?).unwrap(); assert_eq!(fcu.payload_status.status, PayloadStatusEnum::Valid); node.wait_block(1, block_hash, false).await?; diff --git a/crates/ethereum/payload/src/lib.rs b/crates/ethereum/payload/src/lib.rs index 1892dc0af69..0769c72539b 100644 --- a/crates/ethereum/payload/src/lib.rs +++ b/crates/ethereum/payload/src/lib.rs @@ -161,7 +161,7 @@ where let BuildArguments { mut cached_reads, execution_cache, - state_root_handle, + mut state_root_handle, config, cancel, best_payload, @@ -222,8 +222,8 @@ where let mut total_fees = U256::ZERO; // If we have a state-root task, wire a state hook that streams per-tx state diffs. - if let Some(ref task) = state_root_handle { - builder.evm_mut().db_mut().set_state_hook(Some(Box::new(task.state_hook()))); + if let Some(task) = state_root_handle.as_mut() { + builder.evm_mut().db_mut().set_state_hook(Some(Box::new(task.take_state_hook()))); } builder.apply_pre_execution_changes().map_err(|err| { diff --git a/crates/net/downloaders/src/file_client.rs b/crates/net/downloaders/src/file_client.rs index 0cb4b0df43e..9b11dcbf5c6 100644 --- a/crates/net/downloaders/src/file_client.rs +++ b/crates/net/downloaders/src/file_client.rs @@ -435,7 +435,7 @@ enum FileReader { /// Regular uncompressed file with remaining byte tracking. Plain { file: File, remaining_bytes: u64 }, /// Gzip compressed file. - Gzip(GzipDecoder>), + Gzip { decoder: GzipDecoder>, eof: bool }, } impl FileReader { @@ -443,7 +443,14 @@ impl FileReader { async fn read(&mut self, buf: &mut [u8]) -> Result { match self { Self::Plain { file, .. } => file.read(buf).await, - Self::Gzip(decoder) => decoder.read(buf).await, + Self::Gzip { decoder, .. } => decoder.read(buf).await, + } + } + + const fn is_eof(&self) -> bool { + match self { + Self::Plain { remaining_bytes, .. } => *remaining_bytes == 0, + Self::Gzip { eof, .. } => *eof, } } @@ -456,7 +463,7 @@ impl FileReader { ) -> Result, FileClientError> { match self { Self::Plain { .. } => self.read_plain_chunk(chunk, chunk_byte_len).await, - Self::Gzip(_) => { + Self::Gzip { .. } => { Ok((self.read_gzip_chunk(chunk, chunk_byte_len).await?) .then_some(chunk.len() as u64)) } @@ -521,7 +528,11 @@ impl FileReader { } match self.read(&mut buffer).await { - Ok(0) => return Ok(!chunk.is_empty()), + Ok(0) => { + let Self::Gzip { eof, .. } = self else { unreachable!() }; + *eof = true; + return Ok(!chunk.is_empty()) + } Ok(n) => { chunk.extend_from_slice(&buffer[..n]); } @@ -574,7 +585,7 @@ impl ChunkedFileReader { is_gzip: bool, ) -> Result { let file_reader = if is_gzip { - FileReader::Gzip(GzipDecoder::new(BufReader::new(file))) + FileReader::Gzip { decoder: GzipDecoder::new(BufReader::new(file)), eof: false } } else { let remaining_bytes = file.metadata().await?.len(); FileReader::Plain { file, remaining_bytes } @@ -616,6 +627,10 @@ impl ChunkedFileReader { .build(&self.chunk[..], chunk_len) .await?; + if self.file.is_eof() && !remaining_bytes.is_empty() { + return Err(FileClientError::Rlp(alloy_rlp::Error::InputTooShort, remaining_bytes)) + } + // save left over bytes self.chunk = remaining_bytes; @@ -857,6 +872,25 @@ mod tests { assert!(client.tip().is_none()); } + #[tokio::test] + async fn trailing_transaction_data_at_eof_is_rejected() { + let block = alloy_primitives::hex!( + "f902cef90259a01f1e77fa4e08a5ce98ba78db75ca1a4623c10e832eeff5a4a770e9d5048bfa95a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d4934794000000000000000000000000000000000000c0fea059eb85c4cc1486f67674192abb9fff7ae7f38f2ecdd3c87458ae2b8462eb5ca2a0a79a055a833e5c8e9364a9f6f06e1e01856d25a3cc21bad067cd94eb5cf9c7e9a0f78dfb743fbd92ade140711c8bbc542b5e307f0ab7984eff35d751969fe57efab901000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000080018401c9c3808252080c80a000000000000000000000000000000000000000000000000000000000000000008800000000000000000aa056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b4218080a00000000000000000000000000000000000000000000000000000000000000000a0e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855f86eb86c02f8680180830f4240830f424082520894000000000000000000000000000000000000c0de8080c001a079be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798a03d1613cba75c9e7513aee78156909bdb32050830f2faa3125e8fd64b5778be3000c0c0" + ); + let mut file = File::from_std(tempfile::tempfile().unwrap()); + file.write_all(&block).await.unwrap(); + file.seek(SeekFrom::Start(0)).await.unwrap(); + let mut reader = + ChunkedFileReader::from_file(file, block.len() as u64, false).await.unwrap(); + + let err = reader.next_chunk::(NoopConsensus::arc(), None).await.unwrap_err(); + + assert_matches!( + err, + FileClientError::Rlp(alloy_rlp::Error::InputTooShort, bytes) if bytes == block + ); + } + #[tokio::test] async fn test_chunk_download_headers_from_file() { reth_tracing::init_test_tracing(); diff --git a/crates/net/ecies/Cargo.toml b/crates/net/ecies/Cargo.toml index a094cfa15ba..3d2c6d81795 100644 --- a/crates/net/ecies/Cargo.toml +++ b/crates/net/ecies/Cargo.toml @@ -36,8 +36,7 @@ concat-kdf.workspace = true sha2.workspace = true aes.workspace = true hmac.workspace = true -block-padding.workspace = true -cipher = { workspace = true, features = ["block-padding"] } +cipher.workspace = true [dev-dependencies] tokio = { workspace = true, features = ["net", "rt", "macros"] } diff --git a/crates/net/ecies/src/mac.rs b/crates/net/ecies/src/mac.rs index e16eedbd064..a7feadf8f9a 100644 --- a/crates/net/ecies/src/mac.rs +++ b/crates/net/ecies/src/mac.rs @@ -9,9 +9,8 @@ //! //! For more information, refer to the [Ethereum MAC specification](https://github.com/ethereum/devp2p/blob/master/rlpx.md#mac). -use aes::Aes256Enc; +use aes::{Aes256Enc, Block}; use alloy_primitives::{Keccak256, B128, B256}; -use block_padding::NoPadding; use cipher::BlockEncrypt; use digest::KeyInit; @@ -24,14 +23,22 @@ use digest::KeyInit; /// and is not defined as a general MAC. #[derive(Debug)] pub struct MAC { - secret: B256, + /// AES-256 block cipher keyed with the MAC secret. + /// + /// The secret is fixed for the lifetime of the connection, so the key schedule is expanded + /// once here instead of on every header/body update. + aes: Aes256Enc, hasher: Keccak256, } impl MAC { /// Initialize the MAC with the given secret pub fn new(secret: B256) -> Self { - Self { secret, hasher: Keccak256::new() } + Self { + aes: Aes256Enc::new_from_slice(secret.as_ref()) + .expect("32 bytes is a valid AES-256 key"), + hasher: Keccak256::new(), + } } /// Update the internal keccak256 hasher with the given data @@ -41,28 +48,20 @@ impl MAC { /// Accumulate the given header bytes into the MAC's internal state. pub fn update_header(&mut self, data: &[u8; 16]) { - let aes = Aes256Enc::new_from_slice(self.secret.as_ref()).unwrap(); - let mut encrypted = self.digest().0; + let mut encrypted = self.digest(); - aes.encrypt_padded::(&mut encrypted, B128::len_bytes()).unwrap(); - for i in 0..data.len() { - encrypted[i] ^= data[i]; - } - self.hasher.update(encrypted); + self.aes.encrypt_block(Block::from_mut_slice(encrypted.as_mut_slice())); + self.hasher.update(encrypted ^ B128::from(data)); } /// Accumulate the given message body into the MAC's internal state. pub fn update_body(&mut self, data: &[u8]) { self.hasher.update(data); let prev = self.digest(); - let aes = Aes256Enc::new_from_slice(self.secret.as_ref()).unwrap(); - let mut encrypted = prev.0; + let mut encrypted = prev; - aes.encrypt_padded::(&mut encrypted, B128::len_bytes()).unwrap(); - for i in 0..16 { - encrypted[i] ^= prev[i]; - } - self.hasher.update(encrypted); + self.aes.encrypt_block(Block::from_mut_slice(encrypted.as_mut_slice())); + self.hasher.update(encrypted ^ prev); } /// Produce a digest by finalizing the internal keccak256 hasher and returning the first 128 diff --git a/crates/net/ecies/src/stream.rs b/crates/net/ecies/src/stream.rs index adf4dc7634d..306a32aabe0 100644 --- a/crates/net/ecies/src/stream.rs +++ b/crates/net/ecies/src/stream.rs @@ -26,6 +26,16 @@ use tracing::{instrument, trace}; const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(10); +/// Write buffer size at which the underlying `Framed` transport starts flushing frames to the +/// socket from `poll_ready`, replacing the tokio-util default of 8KiB. +/// +/// `RLPx` callers queue batches of messages (e.g. transaction broadcasts) and drive one flush per +/// batch; the 8KiB default splits such a batch into one write syscall per few messages. A larger +/// boundary batches them into fewer, larger writes. Once grown, the write buffer's capacity is +/// retained for the connection's lifetime, so this also bounds the resident write buffer per +/// connection. +pub const DEFAULT_BACKPRESSURE_BOUNDARY: usize = 64 * 1024; + /// `ECIES` stream over TCP exchanging raw bytes #[derive(Debug)] #[pin_project::pin_project] @@ -70,6 +80,7 @@ where let ecies = ECIESCodec::new_client(secret_key, remote_id)?; let mut transport = ecies.framed(transport); + transport.set_backpressure_boundary(DEFAULT_BACKPRESSURE_BOUNDARY); trace!("sending ecies auth ..."); transport.send(EgressECIESValue::Auth).await?; @@ -101,6 +112,7 @@ where trace!("incoming ecies stream"); let mut transport = ecies.framed(transport); + transport.set_backpressure_boundary(DEFAULT_BACKPRESSURE_BOUNDARY); let msg = transport.try_next().await?; trace!("receiving ecies auth"); @@ -127,6 +139,17 @@ where } } +impl ECIESStream { + /// Sets the write buffer size at which the underlying transport starts flushing to the socket + /// from `poll_ready`, overriding [`DEFAULT_BACKPRESSURE_BOUNDARY`]. + /// + /// A larger boundary batches more frames into a single write syscall when many messages are + /// sent back to back, at the cost of buffering more encrypted data in memory per connection. + pub fn set_backpressure_boundary(&mut self, boundary: usize) { + self.stream.set_backpressure_boundary(boundary); + } +} + impl Stream for ECIESStream where Io: AsyncRead + Unpin, diff --git a/crates/net/eth-wire-types/src/snap.rs b/crates/net/eth-wire-types/src/snap.rs index 7629de3149c..f375499a740 100644 --- a/crates/net/eth-wire-types/src/snap.rs +++ b/crates/net/eth-wire-types/src/snap.rs @@ -68,6 +68,22 @@ pub enum SnapMessageId { BlockAccessLists = 0x09, } +impl SnapMessageId { + /// Returns the message id of the response paired with this request, or `None` if this id is + /// itself a response. + pub const fn response(self) -> Option { + match self { + Self::GetAccountRange => Some(Self::AccountRange), + Self::GetStorageRanges => Some(Self::StorageRanges), + Self::GetByteCodes => Some(Self::ByteCodes), + Self::GetBlockAccessLists => Some(Self::BlockAccessLists), + Self::AccountRange | Self::StorageRanges | Self::ByteCodes | Self::BlockAccessLists => { + None + } + } + } +} + /// Request for a range of accounts from the state trie. // https://github.com/ethereum/devp2p/blob/master/caps/snap.md#getaccountrange-0x00 #[derive(Debug, Clone, PartialEq, Eq, RlpEncodable, RlpDecodable)] diff --git a/crates/net/eth-wire/src/p2pstream.rs b/crates/net/eth-wire/src/p2pstream.rs index dfc5632a9f7..c9eb041ff0b 100644 --- a/crates/net/eth-wire/src/p2pstream.rs +++ b/crates/net/eth-wire/src/p2pstream.rs @@ -69,6 +69,14 @@ const PING_INTERVAL: Duration = Duration::from_secs(60); /// encoded data. const MAX_P2P_CAPACITY: usize = 2; +/// Maximum size of the reusable compression scratch buffer in [`P2PStream`], covering the snappy +/// worst case of typical broadcast messages (soft-capped around 128KiB). +/// +/// Messages with a larger compressed worst case are compressed through a one-off allocation +/// instead, so a single oversized message neither grows the scratch buffer for the connection's +/// lifetime nor causes shrink/regrow churn, see [`compress_frame`]. +const MAX_COMPRESS_SCRATCH_SIZE: usize = 256 * 1024; + /// An un-authenticated [`P2PStream`]. This is consumed and returns a [`P2PStream`] after the /// `Hello` handshake is completed. #[pin_project] @@ -254,6 +262,13 @@ pub struct P2PStream { /// The snappy encoder used for compressing outgoing messages encoder: snap::raw::Encoder, + /// Reusable scratch buffer for compressing outgoing messages, see [`compress_frame`]. + /// + /// Grow-only and capped at [`MAX_COMPRESS_SCRATCH_SIZE`]; kept fully initialized, so + /// zero-initialization is only paid when the buffer grows and each message only copies out + /// the exact compressed size instead of zeroing a worst-case sized buffer per message. + compress_scratch: Vec, + /// The snappy decoder used for decompressing incoming messages decoder: snap::raw::Decoder, @@ -290,6 +305,7 @@ impl P2PStream { Self { inner, encoder: snap::raw::Encoder::new(), + compress_scratch: Vec::new(), decoder: snap::raw::Decoder::new(), pinger: Pinger::new(PING_INTERVAL, PING_TIMEOUT), shared_capabilities, @@ -368,26 +384,20 @@ impl DisconnectP2P for P2PStream { let mut buf = Vec::with_capacity(disconnect.length()); disconnect.encode(&mut buf); - let mut compressed = vec![0u8; 1 + snap::raw::max_compress_len(buf.len() - 1)]; - let compressed_size = - self.encoder.compress(&buf[1..], &mut compressed[1..]).map_err(|err| { - debug!( - %err, - msg=%hex::encode(&buf[1..]), - "error compressing disconnect" - ); - err - })?; - - // truncate the compressed buffer to the actual compressed size (plus one for the message - // id) - compressed.truncate(compressed_size + 1); - // we do not add the capability offset because the disconnect message is a `p2p` reserved // message - compressed[0] = buf[0]; - - self.outgoing_messages.push_back(compressed.into()); + let compressed = + compress_frame(&mut self.encoder, &mut self.compress_scratch, buf[0], &buf[1..]) + .map_err(|err| { + debug!( + %err, + msg=%hex::encode(&buf[1..]), + "error compressing disconnect" + ); + err + })?; + + self.outgoing_messages.push_back(compressed); self.needs_control_flush = true; self.disconnecting = true; Ok(()) @@ -637,25 +647,23 @@ where let this = self.project(); - let mut compressed = BytesMut::zeroed(1 + snap::raw::max_compress_len(item.len() - 1)); - let compressed_size = - this.encoder.compress(&item[1..], &mut compressed[1..]).map_err(|err| { - debug!( - %err, - msg=%hex::encode(&item[1..]), - "error compressing p2p message" - ); - err - })?; - - // truncate the compressed buffer to the actual compressed size (plus one for the message - // id) - compressed.truncate(compressed_size + 1); - // all messages sent in this stream are subprotocol messages, so we need to switch the // message id based on the offset - compressed[0] = item[0] + MAX_RESERVED_MESSAGE_ID + 1; - this.outgoing_messages.push_back(compressed.freeze()); + let compressed = compress_frame( + this.encoder, + this.compress_scratch, + item[0] + MAX_RESERVED_MESSAGE_ID + 1, + &item[1..], + ) + .map_err(|err| { + debug!( + %err, + msg=%hex::encode(&item[1..]), + "error compressing p2p message" + ); + err + })?; + this.outgoing_messages.push_back(compressed); Ok(()) } @@ -827,6 +835,36 @@ impl TryFrom for P2PMessageID { } } +/// Snappy-compresses an id-prefixed `p2p` message payload into a frame carrying the given wire +/// message id. +/// +/// Frames whose worst-case compressed size fits within [`MAX_COMPRESS_SCRATCH_SIZE`] are +/// compressed through the reusable `scratch` buffer and copied out at their exact size; larger +/// frames use a one-off allocation, see [`MAX_COMPRESS_SCRATCH_SIZE`]. +fn compress_frame( + encoder: &mut snap::raw::Encoder, + scratch: &mut Vec, + wire_id: u8, + payload: &[u8], +) -> Result { + let needed = 1 + snap::raw::max_compress_len(payload.len()); + + if needed > MAX_COMPRESS_SCRATCH_SIZE { + let mut compressed = vec![0u8; needed]; + let compressed_size = encoder.compress(payload, &mut compressed[1..])?; + compressed[0] = wire_id; + compressed.truncate(compressed_size + 1); + return Ok(compressed.into()) + } + + if scratch.len() < needed { + scratch.resize(needed, 0); + } + let compressed_size = encoder.compress(payload, &mut scratch[1..])?; + scratch[0] = wire_id; + Ok(Bytes::copy_from_slice(&scratch[..compressed_size + 1])) +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/net/network-api/src/events.rs b/crates/net/network-api/src/events.rs index 69074f57f25..efad700d6fd 100644 --- a/crates/net/network-api/src/events.rs +++ b/crates/net/network-api/src/events.rs @@ -1,14 +1,17 @@ //! API related to listening for network events. use reth_eth_wire_types::{ - message::RequestPair, BlockAccessLists, BlockBodies, BlockHeaders, Capabilities, Cells, - DisconnectReason, EthMessage, EthNetworkPrimitives, EthVersion, GetBlockAccessLists, - GetBlockBodies, GetBlockHeaders, GetCells, GetNodeData, GetPooledTransactions, GetReceipts, - GetReceipts70, NetworkPrimitives, NodeData, PooledTransactions, Receipts, Receipts69, - Receipts70, UnifiedStatus, + message::RequestPair, snap::SnapProtocolMessage, BlockAccessLists, BlockBodies, BlockHeaders, + Capabilities, Cells, DisconnectReason, EthMessage, EthNetworkPrimitives, EthVersion, + GetBlockAccessLists, GetBlockBodies, GetBlockHeaders, GetCells, GetNodeData, + GetPooledTransactions, GetReceipts, GetReceipts70, NetworkPrimitives, NodeData, + PooledTransactions, Receipts, Receipts69, Receipts70, UnifiedStatus, }; use reth_ethereum_forks::ForkId; -use reth_network_p2p::error::{RequestError, RequestResult}; +use reth_network_p2p::{ + error::{RequestError, RequestResult}, + snap::client::SnapResponse, +}; use reth_network_peers::{NodeRecord, PeerId}; use reth_network_types::{PeerAddr, PeerKind}; use reth_tokio_util::EventStream; @@ -271,6 +274,25 @@ pub enum PeerRequest { /// The channel to send the response for cells. response: oneshot::Sender>, }, + /// Requests a `snap/2` (EIP-8189) message from the peer. + /// + /// The response should be sent through the channel. + GetSnap { + /// The `snap/2` request to send. + request: SnapProtocolMessage, + /// The channel to send the response for the request. + response: oneshot::Sender>, + }, +} + +/// The wire message a [`PeerRequest`] resolves to before it's queued for sending: either an `eth` +/// message or a `snap/2` message. +#[derive(Debug)] +pub enum RequestMessage { + /// An `eth` protocol message. + Eth(EthMessage), + /// A `snap/2` (EIP-8189) protocol message. + Snap(SnapProtocolMessage), } // === impl PeerRequest === @@ -293,6 +315,7 @@ impl PeerRequest { Self::GetReceipts70 { response, .. } => response.send(Err(err)).ok(), Self::GetBlockAccessLists { response, .. } => response.send(Err(err)).ok(), Self::GetCells { response, .. } => response.send(Err(err)).ok(), + Self::GetSnap { response, .. } => response.send(Err(err)).ok(), }; } @@ -306,38 +329,61 @@ impl PeerRequest { } } - /// Returns the [`EthMessage`] for this type - pub fn create_request_message(&self, request_id: u64) -> EthMessage { + /// Returns the [`RequestMessage`] for this type. + pub fn create_request_message(&self, request_id: u64) -> RequestMessage { match self { Self::GetBlockHeaders { request, .. } => { - EthMessage::GetBlockHeaders(RequestPair { request_id, message: *request }) + RequestMessage::Eth(EthMessage::GetBlockHeaders(RequestPair { + request_id, + message: *request, + })) } Self::GetBlockBodies { request, .. } => { - EthMessage::GetBlockBodies(RequestPair { request_id, message: request.clone() }) + RequestMessage::Eth(EthMessage::GetBlockBodies(RequestPair { + request_id, + message: request.clone(), + })) } Self::GetPooledTransactions { request, .. } => { - EthMessage::GetPooledTransactions(RequestPair { + RequestMessage::Eth(EthMessage::GetPooledTransactions(RequestPair { request_id, message: request.clone(), - }) + })) } Self::GetNodeData { request, .. } => { - EthMessage::GetNodeData(RequestPair { request_id, message: request.clone() }) + RequestMessage::Eth(EthMessage::GetNodeData(RequestPair { + request_id, + message: request.clone(), + })) } Self::GetReceipts { request, .. } | Self::GetReceipts69 { request, .. } => { - EthMessage::GetReceipts(RequestPair { request_id, message: request.clone() }) + RequestMessage::Eth(EthMessage::GetReceipts(RequestPair { + request_id, + message: request.clone(), + })) } Self::GetReceipts70 { request, .. } => { - EthMessage::GetReceipts70(RequestPair { request_id, message: request.clone() }) + RequestMessage::Eth(EthMessage::GetReceipts70(RequestPair { + request_id, + message: request.clone(), + })) } Self::GetBlockAccessLists { request, .. } => { - EthMessage::GetBlockAccessLists(RequestPair { + RequestMessage::Eth(EthMessage::GetBlockAccessLists(RequestPair { request_id, message: request.clone(), - }) + })) } Self::GetCells { request, .. } => { - EthMessage::GetCells(RequestPair { request_id, message: request.clone() }) + RequestMessage::Eth(EthMessage::GetCells(RequestPair { + request_id, + message: request.clone(), + })) + } + Self::GetSnap { request, .. } => { + let mut message = request.clone(); + message.set_request_id(request_id); + RequestMessage::Snap(message) } } } diff --git a/crates/net/network-api/src/lib.rs b/crates/net/network-api/src/lib.rs index b4ad2236690..5dad5dd90b0 100644 --- a/crates/net/network-api/src/lib.rs +++ b/crates/net/network-api/src/lib.rs @@ -34,7 +34,7 @@ pub use downloaders::BlockDownloaderProvider; pub use error::NetworkError; pub use events::{ DiscoveredEvent, DiscoveryEvent, NetworkEvent, NetworkEventListenerProvider, PeerRequest, - PeerRequestSender, + PeerRequestSender, RequestMessage, }; use reth_eth_wire_types::{ diff --git a/crates/net/network/Cargo.toml b/crates/net/network/Cargo.toml index 298d93abced..951d720b51c 100644 --- a/crates/net/network/Cargo.toml +++ b/crates/net/network/Cargo.toml @@ -39,7 +39,7 @@ reth-network-types = { workspace = true, features = ["serde"] } alloy-consensus.workspace = true alloy-eips.workspace = true alloy-primitives.workspace = true -alloy-rlp.workspace = true +alloy-rlp = { workspace = true, features = ["derive"] } enr = { workspace = true, features = ["serde", "rust-secp256k1"] } discv5.workspace = true @@ -95,6 +95,7 @@ reth-transaction-pool = { workspace = true, features = ["test-utils"] } alloy-genesis.workspace = true # misc +test-case.workspace = true url.workspace = true secp256k1 = { workspace = true, features = ["rand"] } diff --git a/crates/net/network/src/config.rs b/crates/net/network/src/config.rs index 470dd841d00..7f106b9c176 100644 --- a/crates/net/network/src/config.rs +++ b/crates/net/network/src/config.rs @@ -22,6 +22,7 @@ use reth_network_peers::{mainnet_nodes, pk2id, sepolia_nodes, PeerId, TrustedPee use reth_network_types::{PeersConfig, SessionsConfig}; use reth_storage_api::{ noop::NoopProvider, BalProvider, BlockNumReader, BlockReader, HeaderProvider, + StateProviderFactory, StateRangeProviderFactory, }; use reth_tasks::Runtime; use secp256k1::SECP256K1; @@ -163,6 +164,8 @@ impl NetworkConfig where N: NetworkPrimitives, C: BalProvider + + StateProviderFactory + + StateRangeProviderFactory + BlockReader + HeaderProvider + Clone diff --git a/crates/net/network/src/eth_requests.rs b/crates/net/network/src/eth_requests.rs index 9474311d81e..89068af2c93 100644 --- a/crates/net/network/src/eth_requests.rs +++ b/crates/net/network/src/eth_requests.rs @@ -4,20 +4,36 @@ use crate::{ budget::DEFAULT_BUDGET_TRY_DRAIN_DOWNLOADERS, metered_poll_nested_stream_with_budget, metrics::EthRequestHandlerMetrics, }; -use alloy_consensus::{BlockHeader, ReceiptWithBloom}; +use alloy_consensus::{ + constants::{EMPTY_ROOT_HASH, KECCAK_EMPTY}, + BlockHeader, ReceiptWithBloom, +}; use alloy_eips::BlockHashOrNumber; -use alloy_rlp::Encodable; +use alloy_primitives::{Bytes, B256, U256}; +use alloy_rlp::{Encodable, RlpEncodable}; use futures::StreamExt; use reth_eth_wire::{ + snap::{ + AccountData, AccountRangeMessage, BlockAccessListsMessage, ByteCodesMessage, + GetAccountRangeMessage, GetStorageRangesMessage, SnapProtocolMessage, StorageData, + StorageRangesMessage, + }, BlockAccessLists, BlockBodies, BlockHeaders, Cells, EthNetworkPrimitives, GetBlockAccessLists, GetBlockBodies, GetBlockHeaders, GetCells, GetNodeData, GetReceipts, GetReceipts70, HeadersDirection, NetworkPrimitives, NodeData, Receipts, Receipts69, Receipts70, }; use reth_network_api::test_utils::PeersHandle; -use reth_network_p2p::error::RequestResult; +use reth_network_p2p::{ + error::{RequestError, RequestResult}, + snap::client::SnapResponse, +}; use reth_network_peers::PeerId; -use reth_primitives_traits::Block; -use reth_storage_api::{BalProvider, BlockReader, GetBlockAccessListLimit, HeaderProvider}; +use reth_primitives_traits::{Account, Block}; +use reth_storage_api::{ + errors::provider::ProviderResult, BalProvider, BlockReader, BytecodeReader, + GetBlockAccessListLimit, HeaderProvider, RangeEnd, RangeResponse, StateProviderFactory, + StateRangeProviderFactory, +}; use reth_transaction_pool::{blobstore::NoopBlobStore, BlobStore}; use std::{ future::Future, @@ -56,6 +72,14 @@ pub const MAX_BLOCK_ACCESS_LISTS_SERVE: usize = 1024; /// Used to limit lookups. pub const MAX_CELLS_SERVE: usize = 1024; +/// Maximum number of bytecode lookups to serve. +/// +/// Used to limit lookups. +pub const MAX_BYTE_CODES_SERVE: usize = 1024; + +/// Maximum number of storage range account lookups to serve. +pub const MAX_STORAGE_RANGE_ACCOUNTS_SERVE: usize = 1024; + /// Maximum size of replies to data retrievals: 2MB pub const SOFT_RESPONSE_LIMIT: usize = 2 * 1024 * 1024; @@ -381,6 +405,243 @@ where } } +impl EthRequestHandler +where + N: NetworkPrimitives, + C: BalProvider + StateProviderFactory + StateRangeProviderFactory, +{ + /// Handles `snap/2` (EIP-8189) requests. + /// + /// `GetAccountRange`/`GetStorageRanges` are hash-native throughout and served from retained + /// canonical roots via [`StateRangeProviderFactory`]. `GetByteCodes` is content-addressed and + /// independent of any particular state root, so it's served directly. + /// `GetBlockAccessLists` is answered from the same [`BalProvider`] store eth71's + /// `GetBlockAccessLists` uses, since both serve the same underlying data. + fn on_snap_request( + &self, + _peer_id: PeerId, + request: SnapProtocolMessage, + response: oneshot::Sender>, + ) { + self.metrics.snap_requests_received_total.increment(1); + + let result = match request { + SnapProtocolMessage::GetAccountRange(req) => { + let request_id = req.request_id; + let response = self.get_account_range_response(req).unwrap_or_else(|error| { + tracing::debug!(target: "net::snap", %error, "failed to serve account range"); + AccountRangeMessage { request_id, accounts: Vec::new(), proof: Vec::new() } + }); + Ok(SnapResponse::AccountRange(response)) + } + SnapProtocolMessage::GetStorageRanges(req) => { + let request_id = req.request_id; + let response = self.get_storage_ranges_response(req).unwrap_or_else(|error| { + tracing::debug!(target: "net::snap", %error, "failed to serve storage ranges"); + StorageRangesMessage { request_id, slots: Vec::new(), proof: Vec::new() } + }); + Ok(SnapResponse::StorageRanges(response)) + } + SnapProtocolMessage::GetByteCodes(req) => { + let codes = self + .get_byte_codes_response(&req.hashes, req.response_bytes as usize) + .unwrap_or_else(|error| { + tracing::debug!(target: "net::snap", %error, "failed to serve bytecodes"); + Vec::new() + }); + Ok(SnapResponse::ByteCodes(ByteCodesMessage { request_id: req.request_id, codes })) + } + SnapProtocolMessage::GetBlockAccessLists(mut req) => { + req.block_hashes.truncate(MAX_BLOCK_ACCESS_LISTS_SERVE); + let limit = GetBlockAccessListLimit::ResponseSizeSoftLimit( + (req.response_bytes as usize).min(SOFT_RESPONSE_LIMIT), + ); + let block_access_lists = self + .client + .bal_store() + .get_by_hashes_with_limit(&req.block_hashes, limit) + .unwrap_or_default(); + Ok(SnapResponse::BlockAccessLists(BlockAccessListsMessage { + request_id: req.request_id, + block_access_lists: BlockAccessLists(block_access_lists), + })) + } + // The peer sent us a response-shaped message instead of a request; not something we + // asked for. + _ => Err(RequestError::BadResponse), + }; + + let _ = response.send(result); + } + + /// Returns the bytecode for each of `hashes`, skipping hashes with no known code, stopping + /// once `response_bytes` (capped at [`SOFT_RESPONSE_LIMIT`]) is exceeded. + fn get_byte_codes_response( + &self, + hashes: &[B256], + response_bytes: usize, + ) -> ProviderResult> { + let state = self.client.latest()?; + let response_bytes = response_bytes.min(SOFT_RESPONSE_LIMIT); + + let mut codes = Vec::new(); + let mut total_bytes = 0; + for hash in hashes.iter().take(MAX_BYTE_CODES_SERVE) { + let bytes = if *hash == KECCAK_EMPTY { + Bytes::new() + } else { + match state.bytecode_by_hash(hash)? { + Some(bytecode) => bytecode.original_bytes(), + None => continue, + } + }; + total_bytes += bytes.len(); + codes.push(bytes); + + if total_bytes > response_bytes { + break + } + } + Ok(codes) + } + + /// Serves a `GetAccountRange` request via [`StateRangeProviderFactory`]. + /// + /// Fails the request if a storage root or proof becomes unavailable after the range lookup. + /// Always proves the boundary between `starting_hash` and the last returned account, per + /// snap/2's boundary-proof requirement. + fn get_account_range_response( + &self, + req: GetAccountRangeMessage, + ) -> ProviderResult { + let empty = AccountRangeMessage { + request_id: req.request_id, + accounts: Vec::new(), + proof: Vec::new(), + }; + + let response_bytes = (req.response_bytes as usize).min(SOFT_RESPONSE_LIMIT); + let Some(state) = self.client.state_range_provider(req.root_hash)? else { + return Ok(empty) + }; + let RangeResponse { items: accounts, .. } = + state.account_range(req.starting_hash, req.limit_hash, response_bytes)?; + + let boundary_keys = boundary_proof_keys(req.starting_hash, accounts.last()); + + let mut account_data = Vec::with_capacity(accounts.len()); + for (hash, account) in accounts { + let storage_root = state.storage_root_by_hash(hash)?; + account_data + .push(AccountData { hash, body: slim_account_body(&account, storage_root) }); + } + + let proof = state.account_range_proof(&boundary_keys)?; + + Ok(AccountRangeMessage { request_id: req.request_id, accounts: account_data, proof }) + } + + /// Serves a `GetStorageRanges` request via [`StateRangeProviderFactory`]. + /// + /// `starting_hash`/`limit_hash` apply only to the first account. An account unavailable at + /// this root makes the whole response empty, rather than skipping it and shifting later + /// accounts' positions. A proof stops the response at the first account that isn't a + /// complete, zero-origin range. + fn get_storage_ranges_response( + &self, + req: GetStorageRangesMessage, + ) -> ProviderResult { + let empty = StorageRangesMessage { + request_id: req.request_id, + slots: Vec::new(), + proof: Vec::new(), + }; + let Some(state) = self.client.state_range_provider(req.root_hash)? else { + return Ok(empty) + }; + let mut slots = Vec::new(); + let mut proof = Vec::new(); + let mut remaining_bytes = (req.response_bytes as usize).min(SOFT_RESPONSE_LIMIT); + + for (i, &hashed_address) in + req.account_hashes.iter().take(MAX_STORAGE_RANGE_ACCOUNTS_SERVE).enumerate() + { + if remaining_bytes == 0 { + break + } + let origin = if i == 0 { req.starting_hash } else { B256::ZERO }; + let limit = if i == 0 { req.limit_hash } else { B256::repeat_byte(0xff) }; + let Some(RangeResponse { items: account_slots, end }) = + state.storage_range(hashed_address, origin, limit, remaining_bytes)? + else { + return Ok(empty) + }; + + remaining_bytes = remaining_bytes.saturating_sub(account_slots.len() * 64); + let last = account_slots.last().map(|(hash, _)| *hash); + let needs_proof = origin != B256::ZERO || end != RangeEnd::Exhausted; + slots.push( + account_slots + .into_iter() + .map(|(hash, value)| StorageData { + hash, + // snap clients verify proofs against RLP-encoded storage trie leaves. + data: alloy_rlp::encode(value).into(), + }) + .collect(), + ); + + if needs_proof { + let boundary_keys = match last { + Some(last) => vec![origin, last], + None => vec![origin], + }; + proof = state.storage_range_proof(hashed_address, &boundary_keys)?; + break + } + } + + Ok(StorageRangesMessage { request_id: req.request_id, slots, proof }) + } +} + +/// Boundary-proof keys for a range reply: `origin`, plus the last returned item's key if any. +fn boundary_proof_keys(origin: B256, last: Option<&(B256, T)>) -> Vec { + match last { + Some((last, _)) => vec![origin, *last], + None => vec![origin], + } +} + +/// Like the consensus trie account, but the code hash and storage root are empty byte strings +/// rather than [`KECCAK_EMPTY`]/[`EMPTY_ROOT_HASH`] when the account has no code/storage, to +/// avoid transferring the same 32 bytes for every EOA. +#[derive(RlpEncodable)] +struct SlimAccountBody<'a> { + nonce: u64, + balance: U256, + storage_root: &'a [u8], + code_hash: &'a [u8], +} + +/// RLP-encodes `account` in snap/2's slim format; see [`SlimAccountBody`]. +fn slim_account_body(account: &Account, storage_root: B256) -> Bytes { + let storage_root: &[u8] = + if storage_root == EMPTY_ROOT_HASH { &[] } else { storage_root.as_slice() }; + let code_hash: &[u8] = match &account.bytecode_hash { + Some(hash) if *hash != KECCAK_EMPTY => hash.as_slice(), + _ => &[], + }; + + alloy_rlp::encode(SlimAccountBody { + nonce: account.nonce, + balance: account.balance, + storage_root, + code_hash, + }) + .into() +} + /// An endless future. /// /// This should be spawned or used as part of `tokio::select!`. @@ -388,6 +649,8 @@ impl Future for EthRequestHandler where N: NetworkPrimitives, C: BalProvider + + StateProviderFactory + + StateRangeProviderFactory + BlockReader + HeaderProvider
+ Unpin, @@ -430,6 +693,9 @@ where IncomingEthRequest::GetCells { peer_id, request, response } => { this.on_cells_request(peer_id, request, response) } + IncomingEthRequest::GetSnap { peer_id, request, response } => { + this.on_snap_request(peer_id, request, response) + } } }, ); @@ -537,6 +803,17 @@ pub enum IncomingEthRequest { /// The channel sender for the response containing cells. response: oneshot::Sender>, }, + /// Request a `snap/2` message from the peer. + /// + /// The response should be sent through the channel. + GetSnap { + /// The ID of the peer to request from. + peer_id: PeerId, + /// The `snap/2` request. + request: SnapProtocolMessage, + /// The channel sender for the response. + response: oneshot::Sender>, + }, } #[cfg(test)] @@ -546,14 +823,16 @@ mod tests { eip4844::{BlobAndProofV1, BlobAndProofV2, BlobCellsAndProofsV1}, eip7594::{BlobTransactionSidecarVariant, Cell}, }; - use alloy_primitives::{TxHash, B128, B256}; + use alloy_primitives::{keccak256, Address, TxHash, B128}; use reth_network_api::test_utils::PeersHandle; + use reth_provider::test_utils::{ExtendedAccount, MockEthProvider}; use reth_storage_api::noop::NoopProvider; use reth_transaction_pool::blobstore::{BlobStoreCleanupStat, BlobStoreError}; use std::sync::{ atomic::{AtomicUsize, Ordering}, Arc, }; + use test_case::test_case; use tokio::sync::mpsc; #[derive(Debug, Default)] @@ -694,4 +973,441 @@ mod tests { assert!(cells.hashes.is_empty()); assert_eq!(get_cells_calls.load(Ordering::Relaxed), MAX_CELLS_SERVE); } + + /// Creates a request handler backed by the mock provider for snap response tests. + fn snap_handler( + provider: MockEthProvider, + ) -> EthRequestHandler { + let (peers_tx, _) = mpsc::unbounded_channel(); + let (_incoming_tx, incoming_rx) = mpsc::channel(1); + EthRequestHandler::new(provider, PeersHandle::new(peers_tx), incoming_rx) + } + + #[test_case( + SnapProtocolMessage::GetAccountRange(GetAccountRangeMessage { + request_id: 1, + root_hash: B256::ZERO, + starting_hash: B256::ZERO, + limit_hash: B256::repeat_byte(0xff), + response_bytes: SOFT_RESPONSE_LIMIT as u64, + }), + SnapResponse::AccountRange(AccountRangeMessage { + request_id: 1, + accounts: Vec::new(), + proof: Vec::new(), + }); "account range" + )] + #[test_case( + SnapProtocolMessage::GetStorageRanges(GetStorageRangesMessage { + request_id: 2, + root_hash: B256::ZERO, + account_hashes: vec![B256::ZERO], + starting_hash: B256::ZERO, + limit_hash: B256::repeat_byte(0xff), + response_bytes: SOFT_RESPONSE_LIMIT as u64, + }), + SnapResponse::StorageRanges(StorageRangesMessage { + request_id: 2, + slots: Vec::new(), + proof: Vec::new(), + }); "storage ranges" + )] + #[test_case( + SnapProtocolMessage::GetByteCodes(reth_eth_wire::snap::GetByteCodesMessage { + request_id: 3, + hashes: vec![B256::repeat_byte(0x11)], + response_bytes: SOFT_RESPONSE_LIMIT as u64, + }), + SnapResponse::ByteCodes(ByteCodesMessage { request_id: 3, codes: Vec::new() }); "bytecodes" + )] + #[tokio::test] + async fn snap_requests_return_empty_responses_on_provider_errors( + request: SnapProtocolMessage, + expected: SnapResponse, + ) { + let provider = MockEthProvider::default(); + provider.set_snap_state_reads_fail(true); + let handler = snap_handler(provider); + let (response, rx) = oneshot::channel(); + + handler.on_snap_request(PeerId::default(), request, response); + + assert_eq!(rx.await.unwrap(), Ok(expected)); + } + + #[tokio::test] + async fn unavailable_snap_state_returns_empty_response() { + let provider = MockEthProvider::default(); + let handler = snap_handler(provider.clone()); + let (response, rx) = oneshot::channel(); + handler.on_snap_request( + PeerId::default(), + SnapProtocolMessage::GetAccountRange(GetAccountRangeMessage { + request_id: 1, + root_hash: B256::ZERO, + starting_hash: B256::ZERO, + limit_hash: B256::repeat_byte(0xff), + response_bytes: SOFT_RESPONSE_LIMIT as u64, + }), + response, + ); + + assert_eq!( + rx.await.unwrap(), + Ok(SnapResponse::AccountRange(AccountRangeMessage { + request_id: 1, + accounts: Vec::new(), + proof: Vec::new(), + })) + ); + assert_eq!(provider.snap_state_range_resolutions(), 1); + } + + #[tokio::test] + async fn snap_requests_return_empty_responses_on_inconsistent_provider_results() { + let missing_storage_root = MockEthProvider::default(); + missing_storage_root + .set_snap_account_range(vec![(B256::ZERO, Account::default())], RangeEnd::Exhausted); + + let missing_account_proof = MockEthProvider::default(); + missing_account_proof.set_snap_account_range(Vec::new(), RangeEnd::Exhausted); + + let missing_storage_proof = MockEthProvider::default(); + missing_storage_proof + .push_snap_storage_range(vec![(B256::ZERO, U256::from(1))], RangeEnd::ByteLimit); + + let storage_disappears = MockEthProvider::default(); + storage_disappears.push_snap_storage_range(Vec::new(), RangeEnd::Exhausted); + storage_disappears.push_unavailable_snap_storage_range(); + + let account_request = || { + SnapProtocolMessage::GetAccountRange(GetAccountRangeMessage { + request_id: 1, + root_hash: B256::ZERO, + starting_hash: B256::ZERO, + limit_hash: B256::repeat_byte(0xff), + response_bytes: SOFT_RESPONSE_LIMIT as u64, + }) + }; + let storage_request = |account_hashes| { + SnapProtocolMessage::GetStorageRanges(GetStorageRangesMessage { + request_id: 2, + root_hash: B256::ZERO, + account_hashes, + starting_hash: B256::ZERO, + limit_hash: B256::repeat_byte(0xff), + response_bytes: SOFT_RESPONSE_LIMIT as u64, + }) + }; + let empty_accounts = SnapResponse::AccountRange(AccountRangeMessage { + request_id: 1, + accounts: Vec::new(), + proof: Vec::new(), + }); + let empty_storage = SnapResponse::StorageRanges(StorageRangesMessage { + request_id: 2, + slots: Vec::new(), + proof: Vec::new(), + }); + let cases = [ + (missing_storage_root, account_request(), empty_accounts.clone()), + (missing_account_proof, account_request(), empty_accounts), + (missing_storage_proof, storage_request(vec![B256::ZERO]), empty_storage.clone()), + (storage_disappears, storage_request(vec![B256::ZERO, B256::ZERO]), empty_storage), + ]; + + for (provider, request, expected) in cases { + let handler = snap_handler(provider.clone()); + let (response, rx) = oneshot::channel(); + handler.on_snap_request(PeerId::default(), request, response); + assert_eq!(rx.await.unwrap(), Ok(expected)); + assert_eq!(provider.snap_state_range_resolutions(), 1); + } + } + + #[tokio::test] + async fn snap_account_range_response_encodes_accounts_and_proof() { + let provider = MockEthProvider::default(); + let first_hash = B256::repeat_byte(0x01); + let second_hash = B256::repeat_byte(0x02); + let storage_root = B256::repeat_byte(0x11); + let code_hash = B256::repeat_byte(0x22); + let proof = vec![Bytes::from_static(&[0xaa])]; + provider.set_snap_account_range( + vec![ + ( + first_hash, + Account { nonce: 1, balance: U256::from(2), bytecode_hash: Some(code_hash) }, + ), + (second_hash, Account { nonce: 3, balance: U256::from(4), bytecode_hash: None }), + ], + RangeEnd::Exhausted, + ); + provider.set_snap_storage_root(first_hash, storage_root); + provider.set_snap_storage_root(second_hash, EMPTY_ROOT_HASH); + provider.set_snap_account_proof(Some(proof.clone())); + + let mut full_body = vec![0xf8, 0x44, 0x01, 0x02, 0xa0]; + full_body.extend_from_slice(storage_root.as_slice()); + full_body.push(0xa0); + full_body.extend_from_slice(code_hash.as_slice()); + let empty_body = Bytes::from_static(&[0xc4, 0x03, 0x04, 0x80, 0x80]); + + let handler = snap_handler(provider.clone()); + let (response, rx) = oneshot::channel(); + handler.on_snap_request( + PeerId::default(), + SnapProtocolMessage::GetAccountRange(GetAccountRangeMessage { + request_id: 1, + root_hash: B256::ZERO, + starting_hash: B256::ZERO, + limit_hash: B256::repeat_byte(0xff), + response_bytes: SOFT_RESPONSE_LIMIT as u64, + }), + response, + ); + + assert_eq!( + rx.await.unwrap(), + Ok(SnapResponse::AccountRange(AccountRangeMessage { + request_id: 1, + accounts: vec![ + AccountData { hash: first_hash, body: full_body.into() }, + AccountData { hash: second_hash, body: empty_body }, + ], + proof, + })) + ); + assert_eq!(provider.snap_state_range_resolutions(), 1); + } + + #[tokio::test] + async fn snap_storage_range_response_encodes_values_and_proof() { + let provider = MockEthProvider::default(); + let first_hash = B256::repeat_byte(0x01); + let second_hash = B256::repeat_byte(0x02); + let origin = B256::repeat_byte(0x10); + let proof = vec![Bytes::from_static(&[0xbb])]; + provider.push_snap_storage_range( + vec![(first_hash, U256::from(0x0102)), (second_hash, U256::from(0xff))], + RangeEnd::Exhausted, + ); + provider.set_snap_storage_proof(Some(proof.clone())); + + let handler = snap_handler(provider.clone()); + let (response, rx) = oneshot::channel(); + handler.on_snap_request( + PeerId::default(), + SnapProtocolMessage::GetStorageRanges(GetStorageRangesMessage { + request_id: 2, + root_hash: B256::ZERO, + account_hashes: vec![B256::repeat_byte(0x03)], + starting_hash: origin, + limit_hash: B256::repeat_byte(0xff), + response_bytes: SOFT_RESPONSE_LIMIT as u64, + }), + response, + ); + + assert_eq!( + rx.await.unwrap(), + Ok(SnapResponse::StorageRanges(StorageRangesMessage { + request_id: 2, + slots: vec![vec![ + StorageData { hash: first_hash, data: Bytes::from_static(&[0x82, 0x01, 0x02]) }, + StorageData { hash: second_hash, data: Bytes::from_static(&[0x81, 0xff]) }, + ]], + proof, + })) + ); + assert_eq!(provider.snap_storage_range_requests()[0].1, origin); + assert_eq!(provider.snap_state_range_resolutions(), 1); + } + + #[tokio::test] + async fn snap_storage_ranges_only_bound_the_first_account() { + let provider = MockEthProvider::default(); + provider.push_snap_storage_range(Vec::new(), RangeEnd::Exhausted); + provider.push_snap_storage_range(Vec::new(), RangeEnd::Exhausted); + let first_account = B256::repeat_byte(0x01); + let second_account = B256::repeat_byte(0x02); + let origin = B256::ZERO; + let limit = B256::repeat_byte(0x22); + + let handler = snap_handler(provider.clone()); + let (response, rx) = oneshot::channel(); + handler.on_snap_request( + PeerId::default(), + SnapProtocolMessage::GetStorageRanges(GetStorageRangesMessage { + request_id: 3, + root_hash: B256::ZERO, + account_hashes: vec![first_account, second_account], + starting_hash: origin, + limit_hash: limit, + response_bytes: 1_000, + }), + response, + ); + + assert_eq!( + rx.await.unwrap(), + Ok(SnapResponse::StorageRanges(StorageRangesMessage { + request_id: 3, + slots: vec![Vec::new(), Vec::new()], + proof: Vec::new(), + })) + ); + assert_eq!( + provider.snap_storage_range_requests(), + vec![ + (first_account, origin, limit, 1_000), + (second_account, B256::ZERO, B256::repeat_byte(0xff), 1_000), + ] + ); + assert_eq!(provider.snap_state_range_resolutions(), 1); + } + + #[tokio::test] + async fn snap_byte_codes_response_preserves_found_code_order() { + let provider = MockEthProvider::default(); + let code = Bytes::from_static(&[0x60, 0x00]); + let code_hash = keccak256(&code); + let later_code = Bytes::from_static(&[0x60, 0x01]); + let later_code_hash = keccak256(&later_code); + provider.add_account( + Address::repeat_byte(0x01), + ExtendedAccount::new(1, U256::ZERO).with_bytecode(code.clone()), + ); + provider.add_account( + Address::repeat_byte(0x02), + ExtendedAccount::new(1, U256::ZERO).with_bytecode(later_code), + ); + + let handler = snap_handler(provider); + let (response, rx) = oneshot::channel(); + handler.on_snap_request( + PeerId::default(), + SnapProtocolMessage::GetByteCodes(reth_eth_wire::snap::GetByteCodesMessage { + request_id: 3, + hashes: vec![KECCAK_EMPTY, B256::repeat_byte(0xff), code_hash, later_code_hash], + response_bytes: 1, + }), + response, + ); + + assert_eq!( + rx.await.unwrap(), + Ok(SnapResponse::ByteCodes(ByteCodesMessage { + request_id: 3, + codes: vec![Bytes::new(), code], + })) + ); + } + + #[tokio::test] + async fn snap_storage_ranges_limit_account_lookups() { + let provider = MockEthProvider::default(); + for _ in 0..=MAX_STORAGE_RANGE_ACCOUNTS_SERVE { + provider.push_snap_storage_range(Vec::new(), RangeEnd::Exhausted); + } + let handler = snap_handler(provider.clone()); + let (response, rx) = oneshot::channel(); + handler.on_snap_request( + PeerId::default(), + SnapProtocolMessage::GetStorageRanges(GetStorageRangesMessage { + request_id: 4, + root_hash: B256::ZERO, + account_hashes: vec![B256::ZERO; MAX_STORAGE_RANGE_ACCOUNTS_SERVE + 1], + starting_hash: B256::ZERO, + limit_hash: B256::repeat_byte(0xff), + response_bytes: SOFT_RESPONSE_LIMIT as u64, + }), + response, + ); + + assert_eq!( + rx.await.unwrap(), + Ok(SnapResponse::StorageRanges(StorageRangesMessage { + request_id: 4, + slots: vec![Vec::new(); MAX_STORAGE_RANGE_ACCOUNTS_SERVE], + proof: Vec::new(), + })) + ); + assert_eq!(provider.snap_storage_ranges_remaining(), 1); + assert_eq!(provider.snap_state_range_resolutions(), 1); + } + + #[tokio::test] + async fn snap_storage_range_proves_finite_limit_from_zero_origin() { + let provider = MockEthProvider::default(); + let hash = B256::repeat_byte(0x01); + let proof = vec![Bytes::from_static(&[0xcc])]; + // More entries exist beyond `limit_hash`, so the cursor stopped at the hash limit + // rather than exhausting the trie -- a proof is required even though origin is zero. + provider.push_snap_storage_range(vec![(hash, U256::from(1))], RangeEnd::HashLimit); + provider.set_snap_storage_proof(Some(proof.clone())); + + let handler = snap_handler(provider.clone()); + let (response, rx) = oneshot::channel(); + handler.on_snap_request( + PeerId::default(), + SnapProtocolMessage::GetStorageRanges(GetStorageRangesMessage { + request_id: 5, + root_hash: B256::ZERO, + account_hashes: vec![B256::repeat_byte(0x03)], + starting_hash: B256::ZERO, + limit_hash: B256::repeat_byte(0x20), + response_bytes: SOFT_RESPONSE_LIMIT as u64, + }), + response, + ); + + assert_eq!( + rx.await.unwrap(), + Ok(SnapResponse::StorageRanges(StorageRangesMessage { + request_id: 5, + slots: vec![vec![StorageData { hash, data: Bytes::from_static(&[0x01]) }]], + proof, + })) + ); + } + + #[tokio::test] + async fn snap_storage_ranges_are_entirely_empty_when_an_account_is_missing() { + let provider = MockEthProvider::default(); + provider.push_missing_snap_storage_account(); + provider.push_snap_storage_range( + vec![(B256::repeat_byte(0x01), U256::from(1))], + RangeEnd::Exhausted, + ); + let missing_account = B256::repeat_byte(0x01); + let valid_account = B256::repeat_byte(0x02); + + let handler = snap_handler(provider.clone()); + let (response, rx) = oneshot::channel(); + handler.on_snap_request( + PeerId::default(), + SnapProtocolMessage::GetStorageRanges(GetStorageRangesMessage { + request_id: 6, + root_hash: B256::ZERO, + account_hashes: vec![missing_account, valid_account], + starting_hash: B256::ZERO, + limit_hash: B256::repeat_byte(0xff), + response_bytes: SOFT_RESPONSE_LIMIT as u64, + }), + response, + ); + + assert_eq!( + rx.await.unwrap(), + Ok(SnapResponse::StorageRanges(StorageRangesMessage { + request_id: 6, + slots: Vec::new(), + proof: Vec::new(), + })) + ); + // The valid account's queued range is never consumed: the response bails out at the + // first missing account instead of skipping it and shifting later positions. + assert_eq!(provider.snap_storage_ranges_remaining(), 1); + } } diff --git a/crates/net/network/src/fetch/client.rs b/crates/net/network/src/fetch/client.rs index 9fcc0f6067f..345d07c7bdc 100644 --- a/crates/net/network/src/fetch/client.rs +++ b/crates/net/network/src/fetch/client.rs @@ -4,6 +4,10 @@ use crate::{fetch::DownloadRequest, flattened_response::FlattenedResponse}; use alloy_primitives::B256; use futures::{future, future::Either}; use reth_eth_wire::{BlockAccessLists, EthNetworkPrimitives, NetworkPrimitives}; +use reth_eth_wire_types::snap::{ + GetAccountRangeMessage, GetBlockAccessListsMessage, GetByteCodesMessage, + GetStorageRangesMessage, SnapProtocolMessage, +}; use reth_network_api::test_utils::PeersHandle; use reth_network_p2p::{ block_access_lists::client::{BalRequirement, BlockAccessListsClient}, @@ -13,6 +17,7 @@ use reth_network_p2p::{ headers::client::{HeadersClient, HeadersRequest}, priority::Priority, receipts::client::{ReceiptsClient, ReceiptsFut}, + snap::client::{SnapClient, SnapResponse}, BlockClient, }; use reth_network_peers::PeerId; @@ -53,6 +58,23 @@ impl DownloadClient for FetchClient { } } +impl FetchClient { + /// Sends a `snap/2` request to an available peer. + fn send_snap_request( + &self, + request: SnapProtocolMessage, + priority: Priority, + ) -> std::pin::Pin> + Send + Sync>> + { + let (response, rx) = oneshot::channel(); + if self.request_tx.send(DownloadRequest::GetSnap { request, response, priority }).is_ok() { + Box::pin(FlattenedResponse::from(rx)) + } else { + Box::pin(future::err(RequestError::ChannelClosed)) + } + } +} + // The `Output` future of the [HeadersClient] impl of [FetchClient] that either returns a response // or an error. type HeadersClientFuture = Either, future::Ready>; @@ -153,3 +175,54 @@ impl BlockAccessListsClient for FetchClient { } } } + +impl SnapClient for FetchClient { + type Output = + std::pin::Pin> + Send + Sync>>; + + /// Sends a `GetAccountRange` (`snap/2`) request to an available peer. + fn get_account_range_with_priority( + &self, + request: GetAccountRangeMessage, + priority: Priority, + ) -> Self::Output { + self.send_snap_request(SnapProtocolMessage::GetAccountRange(request), priority) + } + + /// Sends a `GetStorageRanges` (`snap/2`) request to an available peer. + fn get_storage_ranges(&self, request: GetStorageRangesMessage) -> Self::Output { + self.get_storage_ranges_with_priority(request, Priority::Normal) + } + + /// Sends a `GetStorageRanges` (`snap/2`) request to an available peer. + fn get_storage_ranges_with_priority( + &self, + request: GetStorageRangesMessage, + priority: Priority, + ) -> Self::Output { + self.send_snap_request(SnapProtocolMessage::GetStorageRanges(request), priority) + } + + /// Sends a `GetByteCodes` (`snap/2`) request to an available peer. + fn get_byte_codes(&self, request: GetByteCodesMessage) -> Self::Output { + self.get_byte_codes_with_priority(request, Priority::Normal) + } + + /// Sends a `GetByteCodes` (`snap/2`) request to an available peer. + fn get_byte_codes_with_priority( + &self, + request: GetByteCodesMessage, + priority: Priority, + ) -> Self::Output { + self.send_snap_request(SnapProtocolMessage::GetByteCodes(request), priority) + } + + /// Sends a `GetBlockAccessLists` (`snap/2`) request to an available peer. + fn get_block_access_lists_with_priority( + &self, + request: GetBlockAccessListsMessage, + priority: Priority, + ) -> Self::Output { + self.send_snap_request(SnapProtocolMessage::GetBlockAccessLists(request), priority) + } +} diff --git a/crates/net/network/src/fetch/mod.rs b/crates/net/network/src/fetch/mod.rs index c160dc76e6d..00eb1a2f1f5 100644 --- a/crates/net/network/src/fetch/mod.rs +++ b/crates/net/network/src/fetch/mod.rs @@ -8,8 +8,8 @@ use crate::{message::BlockRequest, session::BlockRangeInfo}; use alloy_primitives::B256; use futures::StreamExt; use reth_eth_wire::{ - BlockAccessLists, Capabilities, EthNetworkPrimitives, EthVersion, GetBlockAccessLists, - GetBlockBodies, GetBlockHeaders, GetReceipts, NetworkPrimitives, + snap::SnapProtocolMessage, BlockAccessLists, Capabilities, EthNetworkPrimitives, EthVersion, + GetBlockAccessLists, GetBlockBodies, GetBlockHeaders, GetReceipts, NetworkPrimitives, }; use reth_network_api::test_utils::PeersHandle; use reth_network_p2p::{ @@ -18,6 +18,7 @@ use reth_network_p2p::{ headers::client::HeadersRequest, priority::Priority, receipts::client::ReceiptsResponse, + snap::client::SnapResponse, }; use reth_network_peers::PeerId; use reth_network_types::ReputationChangeKind; @@ -37,6 +38,7 @@ type InflightHeadersRequest = Request = Request<(), PeerRequestResult>>; type InflightReceiptsRequest = Request<(), PeerRequestResult>>; type InflightBlockAccessListsRequest = Request<(), PeerRequestResult>; +type InflightSnapRequest = Request<(), PeerRequestResult>; /// Manages data fetching operations. /// @@ -54,6 +56,8 @@ pub struct StateFetcher { inflight_bals_requests: HashMap, /// Currently active `GetReceipts` requests inflight_receipts_requests: HashMap>, + /// Currently active `snap/2` requests + inflight_snap_requests: HashMap, /// The list of _available_ peers for requests. peers: HashMap, /// The handle to the peers manager @@ -78,6 +82,7 @@ impl StateFetcher { inflight_bodies_requests: Default::default(), inflight_bals_requests: Default::default(), inflight_receipts_requests: Default::default(), + inflight_snap_requests: Default::default(), peers: Default::default(), peers_handle, num_active_peers, @@ -88,15 +93,16 @@ impl StateFetcher { } /// Invoked when connected to a new peer. - pub(crate) fn new_active_peer( - &mut self, - peer_id: PeerId, - best_hash: B256, - best_number: u64, - capabilities: Arc, - timeout: Arc, - range_info: Option, - ) { + pub(crate) fn new_active_peer(&mut self, peer: NewPeerInfo) { + let NewPeerInfo { + peer_id, + best_hash, + best_number, + capabilities, + timeout, + range_info, + supports_snap, + } = peer; self.peers.insert( peer_id, Peer { @@ -107,6 +113,7 @@ impl StateFetcher { timeout, last_response_likely_bad: false, range_info, + supports_snap, }, ); } @@ -131,6 +138,9 @@ impl StateFetcher { if let Some(req) = self.inflight_receipts_requests.remove(peer) { let _ = req.response.send(Err(RequestError::ConnectionDropped)); } + if let Some(req) = self.inflight_snap_requests.remove(peer) { + let _ = req.response.send(Err(RequestError::ConnectionDropped)); + } } /// Updates the block information for the peer. @@ -206,15 +216,11 @@ impl StateFetcher { return PollAction::NoRequests } - if self.peers.is_empty() { - return PollAction::NoPeersAvailable - } - let request = self.queued_requests.pop_front().expect("not empty"); let Some(peer_id) = self.next_best_peer(request.best_peer_requirements()) else { - // Optional BAL requests can lose their eth/71 peer while queued; complete them + // Optional BAL/snap requests can lose their capable peer while queued; complete them // instead of waiting for future peer churn. - if request.is_optional_bal() && !self.has_eth71_peer() { + if self.should_fail_fast(&request) { request.send_err_response(RequestError::UnsupportedCapability); } else { // no peer matches this request's requirements; requeue at the back so other @@ -243,9 +249,9 @@ impl StateFetcher { // poll incoming requests match self.download_requests_rx.poll_next_unpin(cx) { Poll::Ready(Some(request)) => { - // Optional BAL requests should not wait for future peer churn if no + // Optional BAL/snap requests should not wait for future peer churn if no // connected peer can serve them right now. - if request.is_optional_bal() && !self.has_eth71_peer() { + if self.should_fail_fast(&request) { request.send_err_response(RequestError::UnsupportedCapability); continue } @@ -279,6 +285,20 @@ impl StateFetcher { } } + /// Returns whether any connected peer negotiated `snap/2`. + fn has_snap_peer(&self) -> bool { + self.peers + .values() + .any(|peer| !matches!(peer.state, PeerState::Closing) && peer.supports_snap) + } + + /// Returns `true` if `request` cannot be served by any currently connected peer and should + /// fail immediately instead of waiting for future peer churn. + fn should_fail_fast(&self, request: &DownloadRequest) -> bool { + (request.is_optional_bal() && !self.has_eth71_peer()) || + (request.is_snap() && !self.has_snap_peer()) + } + /// Handles a new request to a peer. /// /// Caution: this assumes the peer exists and is idle @@ -324,6 +344,11 @@ impl StateFetcher { self.inflight_receipts_requests.insert(peer_id, inflight); BlockRequest::GetReceipts(GetReceipts(request)) } + DownloadRequest::GetSnap { request, response, .. } => { + let inflight = Request { request: (), response }; + self.inflight_snap_requests.insert(peer_id, inflight); + BlockRequest::GetSnap(request) + } } } @@ -454,6 +479,27 @@ impl StateFetcher { None } + /// Called on a `snap/2` response from a peer. + pub(crate) fn on_snap_response( + &mut self, + peer_id: PeerId, + res: RequestResult, + ) -> Option { + let is_likely_bad_response = res.is_err(); + + if let Some(resp) = self.inflight_snap_requests.remove(&peer_id) { + let _ = resp.response.send(res.map(|r| (peer_id, r).into())); + } + if let Some(peer) = self.peers.get_mut(&peer_id) { + peer.last_response_likely_bad = is_likely_bad_response; + + if peer.state.on_request_finished() && !is_likely_bad_response { + return self.followup_request(peer_id) + } + } + None + } + /// Returns a new [`FetchClient`] that can send requests to this type. pub(crate) fn client(&self) -> FetchClient { FetchClient { @@ -471,6 +517,25 @@ enum PollAction { NoPeersAvailable, } +/// Everything [`StateFetcher::new_active_peer`] needs to register a newly connected peer. +#[derive(Debug)] +pub(crate) struct NewPeerInfo { + /// The remote peer's identifier. + pub(crate) peer_id: PeerId, + /// Best known hash that the peer has. + pub(crate) best_hash: B256, + /// The best block number of the peer. + pub(crate) best_number: u64, + /// Capabilities announced by the peer. + pub(crate) capabilities: Arc, + /// The current timeout value to use for the peer. + pub(crate) timeout: Arc, + /// The range info for the peer. + pub(crate) range_info: Option, + /// Whether the connection negotiated `snap/2` and can serve [`DownloadRequest::GetSnap`]. + pub(crate) supports_snap: bool, +} + /// Represents a connected peer #[derive(Debug)] struct Peer { @@ -494,6 +559,8 @@ struct Peer { last_response_likely_bad: bool, /// Tracks the range info for the peer. range_info: Option, + /// Whether the connection negotiated `snap/2` and can serve [`DownloadRequest::GetSnap`]. + supports_snap: bool, } impl Peer { @@ -519,6 +586,7 @@ impl Peer { fn satisfies(&self, requirement: &BestPeerRequirements) -> bool { match requirement { BestPeerRequirements::EthVersion(ver) => self.capabilities.supports_eth_at_least(ver), + BestPeerRequirements::SupportsSnap => self.supports_snap, BestPeerRequirements::None | BestPeerRequirements::FullBlock | BestPeerRequirements::FullBlockRange(_) => true, @@ -573,9 +641,11 @@ impl Peer { match requirement { BestPeerRequirements::FullBlockRange(range) => self.has_better_range(other, range), BestPeerRequirements::FullBlock => self.has_full_history() && !other.has_full_history(), - // Version-based filtering happens in `next_best_peer`, so by the time we get here - // both peers already satisfy the version requirement. - BestPeerRequirements::None | BestPeerRequirements::EthVersion(_) => false, + // Version/capability-based filtering happens in `next_best_peer`, so by the time we + // get here both peers already satisfy the requirement. + BestPeerRequirements::None | + BestPeerRequirements::EthVersion(_) | + BestPeerRequirements::SupportsSnap => false, } } } @@ -593,6 +663,8 @@ enum PeerState { GetBlockAccessLists, /// Peer is handling a `GetReceipts` request. GetReceipts, + /// Peer is handling a `snap/2` request. + GetSnap, /// Peer session is about to close Closing, } @@ -659,6 +731,12 @@ pub(crate) enum DownloadRequest { response: oneshot::Sender>>, priority: Priority, }, + /// Send a `snap/2` request and send response through channel + GetSnap { + request: SnapProtocolMessage, + response: oneshot::Sender>, + priority: Priority, + }, } // === impl DownloadRequest === @@ -671,6 +749,7 @@ impl DownloadRequest { Self::GetBlockBodies { .. } => PeerState::GetBlockBodies, Self::GetBlockAccessLists { .. } => PeerState::GetBlockAccessLists, Self::GetReceipts { .. } => PeerState::GetReceipts, + Self::GetSnap { .. } => PeerState::GetSnap, } } @@ -680,7 +759,8 @@ impl DownloadRequest { Self::GetBlockHeaders { priority, .. } | Self::GetBlockBodies { priority, .. } | Self::GetBlockAccessLists { priority, .. } | - Self::GetReceipts { priority, .. } => priority, + Self::GetReceipts { priority, .. } | + Self::GetSnap { priority, .. } => priority, } } @@ -694,6 +774,11 @@ impl DownloadRequest { matches!(self, Self::GetBlockAccessLists { requirement: BalRequirement::Optional, .. }) } + /// Returns `true` if this is a `snap/2` request. + const fn is_snap(&self) -> bool { + matches!(self, Self::GetSnap { .. }) + } + /// Sends an error response to the waiting caller. fn send_err_response(self, err: RequestError) { let _ = match self { @@ -701,6 +786,7 @@ impl DownloadRequest { Self::GetBlockBodies { response, .. } => response.send(Err(err)).ok(), Self::GetBlockAccessLists { response, .. } => response.send(Err(err)).ok(), Self::GetReceipts { response, .. } => response.send(Err(err)).ok(), + Self::GetSnap { response, .. } => response.send(Err(err)).ok(), }; } @@ -717,6 +803,7 @@ impl DownloadRequest { } } Self::GetReceipts { .. } => BestPeerRequirements::FullBlock, + Self::GetSnap { .. } => BestPeerRequirements::SupportsSnap, } } } @@ -753,6 +840,8 @@ enum BestPeerRequirements { FullBlock, /// Peer must support at least this eth protocol version. EthVersion(EthVersion), + /// Peer must have negotiated `snap/2`. + SupportsSnap, } #[cfg(test)] @@ -762,6 +851,7 @@ mod tests { use alloy_consensus::Header; use alloy_primitives::B512; use reth_eth_wire::Capability; + use reth_eth_wire_types::snap::{AccountRangeMessage, GetAccountRangeMessage}; use std::future::poll_fn; #[tokio::test(flavor = "multi_thread")] @@ -795,22 +885,24 @@ mod tests { let peer1 = B512::random(); let peer2 = B512::random(); let capabilities = Arc::new(Capabilities::from(vec![])); - fetcher.new_active_peer( - peer1, - B256::random(), - 1, - Arc::clone(&capabilities), - Arc::new(AtomicU64::new(1)), - None, - ); - fetcher.new_active_peer( - peer2, - B256::random(), - 2, - Arc::clone(&capabilities), - Arc::new(AtomicU64::new(1)), - None, - ); + fetcher.new_active_peer(NewPeerInfo { + peer_id: peer1, + best_hash: B256::random(), + best_number: 1, + capabilities: Arc::clone(&capabilities), + timeout: Arc::new(AtomicU64::new(1)), + range_info: None, + supports_snap: false, + }); + fetcher.new_active_peer(NewPeerInfo { + peer_id: peer2, + best_hash: B256::random(), + best_number: 2, + capabilities: Arc::clone(&capabilities), + timeout: Arc::new(AtomicU64::new(1)), + range_info: None, + supports_snap: false, + }); let first_peer = fetcher.next_best_peer(BestPeerRequirements::None).unwrap(); assert!(first_peer == peer1 || first_peer == peer2); @@ -838,30 +930,33 @@ mod tests { let peer2_timeout = Arc::new(AtomicU64::new(300)); let capabilities = Arc::new(Capabilities::from(vec![])); - fetcher.new_active_peer( - peer1, - B256::random(), - 1, - Arc::clone(&capabilities), - Arc::new(AtomicU64::new(30)), - None, - ); - fetcher.new_active_peer( - peer2, - B256::random(), - 2, - Arc::clone(&capabilities), - Arc::clone(&peer2_timeout), - None, - ); - fetcher.new_active_peer( - peer3, - B256::random(), - 3, - Arc::clone(&capabilities), - Arc::new(AtomicU64::new(50)), - None, - ); + fetcher.new_active_peer(NewPeerInfo { + peer_id: peer1, + best_hash: B256::random(), + best_number: 1, + capabilities: Arc::clone(&capabilities), + timeout: Arc::new(AtomicU64::new(30)), + range_info: None, + supports_snap: false, + }); + fetcher.new_active_peer(NewPeerInfo { + peer_id: peer2, + best_hash: B256::random(), + best_number: 2, + capabilities: Arc::clone(&capabilities), + timeout: Arc::clone(&peer2_timeout), + range_info: None, + supports_snap: false, + }); + fetcher.new_active_peer(NewPeerInfo { + peer_id: peer3, + best_hash: B256::random(), + best_number: 3, + capabilities: Arc::clone(&capabilities), + timeout: Arc::new(AtomicU64::new(50)), + range_info: None, + supports_snap: false, + }); // Must always get peer1 (lowest timeout) assert_eq!(fetcher.next_best_peer(BestPeerRequirements::None), Some(peer1)); @@ -925,14 +1020,15 @@ mod tests { (req, header) }; - fetcher.new_active_peer( + fetcher.new_active_peer(NewPeerInfo { peer_id, - Default::default(), - Default::default(), - Arc::new(Capabilities::from(vec![])), - Default::default(), - None, - ); + best_hash: Default::default(), + best_number: Default::default(), + capabilities: Arc::new(Capabilities::from(vec![])), + timeout: Default::default(), + range_info: None, + supports_snap: false, + }); let (req, header) = request_pair(); fetcher.inflight_headers_requests.insert(peer_id, req); @@ -971,6 +1067,7 @@ mod tests { timeout: Arc::new(AtomicU64::new(10)), last_response_likely_bad: false, range_info: Some(BlockRangeInfo::new(0, 100, B256::random())), + supports_snap: false, }; let peer2 = Peer { @@ -981,6 +1078,7 @@ mod tests { timeout: Arc::new(AtomicU64::new(20)), last_response_likely_bad: false, range_info: None, + supports_snap: false, }; // With None requirement, is_better should always return false @@ -999,6 +1097,7 @@ mod tests { timeout: Arc::new(AtomicU64::new(10)), last_response_likely_bad: false, range_info: Some(BlockRangeInfo::new(0, 100, B256::random())), + supports_snap: false, }; // Peer without full history (earliest = 50) @@ -1010,6 +1109,7 @@ mod tests { timeout: Arc::new(AtomicU64::new(10)), last_response_likely_bad: false, range_info: Some(BlockRangeInfo::new(50, 100, B256::random())), + supports_snap: false, }; // Peer without range info (treated as full history) @@ -1021,6 +1121,7 @@ mod tests { timeout: Arc::new(AtomicU64::new(10)), last_response_likely_bad: false, range_info: None, + supports_snap: false, }; // Peer with full history is better than peer without @@ -1049,6 +1150,7 @@ mod tests { timeout: Arc::new(AtomicU64::new(10)), last_response_likely_bad: false, range_info: Some(BlockRangeInfo::new(0, 100, B256::random())), + supports_snap: false, }; // Peer that doesn't cover the range (earliest too high) @@ -1060,6 +1162,7 @@ mod tests { timeout: Arc::new(AtomicU64::new(10)), last_response_likely_bad: false, range_info: Some(BlockRangeInfo::new(70, 100, B256::random())), + supports_snap: false, }; // Peer that covers the requested range is better than one that doesn't @@ -1083,6 +1186,7 @@ mod tests { timeout: Arc::new(AtomicU64::new(10)), last_response_likely_bad: false, range_info: Some(BlockRangeInfo::new(0, 50, B256::random())), + supports_snap: false, }; // Peer without full history that also covers the range @@ -1094,6 +1198,7 @@ mod tests { timeout: Arc::new(AtomicU64::new(10)), last_response_likely_bad: false, range_info: Some(BlockRangeInfo::new(30, 50, B256::random())), + supports_snap: false, }; // When both cover the range, prefer none @@ -1115,6 +1220,7 @@ mod tests { timeout: Arc::new(AtomicU64::new(10)), last_response_likely_bad: false, range_info: Some(BlockRangeInfo::new(0, 50, B256::random())), + supports_snap: false, }; // Peer without full history that also covers the range @@ -1126,6 +1232,7 @@ mod tests { timeout: Arc::new(AtomicU64::new(10)), last_response_likely_bad: false, range_info: Some(BlockRangeInfo::new(30, 50, B256::random())), + supports_snap: false, }; // When both cover the range, prefer lower start value @@ -1147,6 +1254,7 @@ mod tests { timeout: Arc::new(AtomicU64::new(10)), last_response_likely_bad: false, range_info: Some(BlockRangeInfo::new(0, 30, B256::random())), + supports_snap: false, }; // Peer without full history that also doesn't cover the range @@ -1158,6 +1266,7 @@ mod tests { timeout: Arc::new(AtomicU64::new(10)), last_response_likely_bad: false, range_info: Some(BlockRangeInfo::new(10, 30, B256::random())), + supports_snap: false, }; // When neither covers the range, prefer full history @@ -1179,6 +1288,7 @@ mod tests { timeout: Arc::new(AtomicU64::new(10)), last_response_likely_bad: false, range_info: Some(BlockRangeInfo::new(30, 100, B256::random())), + supports_snap: false, }; // Peer without range info @@ -1190,6 +1300,7 @@ mod tests { timeout: Arc::new(AtomicU64::new(10)), last_response_likely_bad: false, range_info: None, + supports_snap: false, }; // Peer without range info is not better (we prefer peers with known ranges) @@ -1215,6 +1326,7 @@ mod tests { timeout: Arc::new(AtomicU64::new(10)), last_response_likely_bad: false, range_info: Some(BlockRangeInfo::new(30, 100, B256::random())), + supports_snap: false, }; // Peer without range info (treated as full history with unknown latest) @@ -1226,6 +1338,7 @@ mod tests { timeout: Arc::new(AtomicU64::new(10)), last_response_likely_bad: false, range_info: None, + supports_snap: false, }; // Peer with range that covers is better than peer without range info @@ -1250,6 +1363,7 @@ mod tests { timeout: Arc::new(AtomicU64::new(10)), last_response_likely_bad: false, range_info: Some(BlockRangeInfo::new(70, 100, B256::random())), + supports_snap: false, }; // Peer without range info (treated as full history) @@ -1261,6 +1375,7 @@ mod tests { timeout: Arc::new(AtomicU64::new(10)), last_response_likely_bad: false, range_info: None, + supports_snap: false, }; // Peer with range that doesn't cover is not better @@ -1286,6 +1401,7 @@ mod tests { timeout: Arc::new(AtomicU64::new(10)), last_response_likely_bad: false, range_info: Some(BlockRangeInfo::new(50, 100, B256::random())), + supports_snap: false, }; // Peer that's one block short at the start @@ -1297,6 +1413,7 @@ mod tests { timeout: Arc::new(AtomicU64::new(10)), last_response_likely_bad: false, range_info: Some(BlockRangeInfo::new(51, 100, B256::random())), + supports_snap: false, }; // Peer that's one block short at the end @@ -1308,6 +1425,7 @@ mod tests { timeout: Arc::new(AtomicU64::new(10)), last_response_likely_bad: false, range_info: Some(BlockRangeInfo::new(50, 99, B256::random())), + supports_snap: false, }; // Exact coverage is better than short coverage @@ -1331,14 +1449,15 @@ mod tests { StateFetcher::::new(manager.handle(), Default::default()); let peer_id = B512::random(); - fetcher.new_active_peer( + fetcher.new_active_peer(NewPeerInfo { peer_id, - Default::default(), - Default::default(), - Arc::new(Capabilities::from(vec![])), - Default::default(), - None, - ); + best_hash: Default::default(), + best_number: Default::default(), + capabilities: Arc::new(Capabilities::from(vec![])), + timeout: Default::default(), + range_info: None, + supports_snap: false, + }); (fetcher, peer_id) } @@ -1475,14 +1594,15 @@ mod tests { let peer_71 = B512::random(); let caps_71 = Arc::new(Capabilities::from(vec![Capability::new("eth".into(), 71)])); - fetcher.new_active_peer( - peer_71, - B256::random(), - 100, - caps_71, - Arc::new(AtomicU64::new(10)), - None, - ); + fetcher.new_active_peer(NewPeerInfo { + peer_id: peer_71, + best_hash: B256::random(), + best_number: 100, + capabilities: caps_71, + timeout: Arc::new(AtomicU64::new(10)), + range_info: None, + supports_snap: false, + }); fetcher.peers.get_mut(&peer_71).expect("peer exists").state = PeerState::GetBlockHeaders; let (followup_tx, _followup_rx) = oneshot::channel(); @@ -1514,14 +1634,15 @@ mod tests { let peer_71 = B512::random(); let caps_71 = Arc::new(Capabilities::from(vec![Capability::new("eth".into(), 71)])); - fetcher.new_active_peer( - peer_71, - B256::random(), - 100, - caps_71, - Arc::new(AtomicU64::new(10)), - None, - ); + fetcher.new_active_peer(NewPeerInfo { + peer_id: peer_71, + best_hash: B256::random(), + best_number: 100, + capabilities: caps_71, + timeout: Arc::new(AtomicU64::new(10)), + range_info: None, + supports_snap: false, + }); fetcher.peers.get_mut(&peer_71).expect("peer exists").state = PeerState::GetBlockHeaders; let (bal_tx, _bal_rx) = oneshot::channel(); @@ -1600,14 +1721,15 @@ mod tests { // Capabilities WITHOUT eth71 let capabilities = Arc::new(Capabilities::new(vec![])); - fetcher.new_active_peer( - peer, - B256::random(), - 100, + fetcher.new_active_peer(NewPeerInfo { + peer_id: peer, + best_hash: B256::random(), + best_number: 100, capabilities, - Arc::new(AtomicU64::new(10)), - None, - ); + timeout: Arc::new(AtomicU64::new(10)), + range_info: None, + supports_snap: false, + }); // Should return None because peer doesn't support eth71 assert_eq!( @@ -1627,14 +1749,15 @@ mod tests { // Build capability list that includes Eth71 let capabilities = Arc::new(Capabilities::from(vec![Capability::new("eth".into(), 71)])); - fetcher.new_active_peer( - peer, - B256::random(), - 100, + fetcher.new_active_peer(NewPeerInfo { + peer_id: peer, + best_hash: B256::random(), + best_number: 100, capabilities, - Arc::new(AtomicU64::new(10)), - None, - ); + timeout: Arc::new(AtomicU64::new(10)), + range_info: None, + supports_snap: false, + }); assert_eq!( fetcher.next_best_peer(BestPeerRequirements::EthVersion(EthVersion::Eth71)), @@ -1657,23 +1780,25 @@ mod tests { // Peer with eth71 let caps_71 = Arc::new(Capabilities::from(vec![Capability::new("eth".into(), 71)])); - fetcher.new_active_peer( - peer_no_71, - B256::random(), - 100, - caps_old, - Arc::new(AtomicU64::new(5)), - None, - ); + fetcher.new_active_peer(NewPeerInfo { + peer_id: peer_no_71, + best_hash: B256::random(), + best_number: 100, + capabilities: caps_old, + timeout: Arc::new(AtomicU64::new(5)), + range_info: None, + supports_snap: false, + }); - fetcher.new_active_peer( - peer_with_71, - B256::random(), - 100, - caps_71, - Arc::new(AtomicU64::new(50)), - None, - ); + fetcher.new_active_peer(NewPeerInfo { + peer_id: peer_with_71, + best_hash: B256::random(), + best_number: 100, + capabilities: caps_71, + timeout: Arc::new(AtomicU64::new(50)), + range_info: None, + supports_snap: false, + }); // Even though peer_no_71 has lower timeout, // it must NOT be selected. @@ -1711,14 +1836,15 @@ mod tests { let peer_old = B512::random(); let caps_old = Arc::new(Capabilities::new(vec![])); - fetcher.new_active_peer( - peer_old, - B256::random(), - 100, - caps_old, - Arc::new(AtomicU64::new(10)), - None, - ); + fetcher.new_active_peer(NewPeerInfo { + peer_id: peer_old, + best_hash: B256::random(), + best_number: 100, + capabilities: caps_old, + timeout: Arc::new(AtomicU64::new(10)), + range_info: None, + supports_snap: false, + }); // Still Pending assert!(matches!(fetcher.poll(&mut cx), Poll::Pending)); @@ -1727,14 +1853,15 @@ mod tests { let peer_71 = B512::random(); let caps_71 = Arc::new(Capabilities::from(vec![Capability::new("eth".into(), 71)])); - fetcher.new_active_peer( - peer_71, - B256::random(), - 100, - caps_71, - Arc::new(AtomicU64::new(10)), - None, - ); + fetcher.new_active_peer(NewPeerInfo { + peer_id: peer_71, + best_hash: B256::random(), + best_number: 100, + capabilities: caps_71, + timeout: Arc::new(AtomicU64::new(10)), + range_info: None, + supports_snap: false, + }); // Now we must get Ready(BlockRequest) if let Poll::Ready(FetchAction::BlockRequest { peer_id, .. }) = fetcher.poll(&mut cx) { @@ -1753,14 +1880,15 @@ mod tests { let peer_old = B512::random(); let caps_old = Arc::new(Capabilities::new(vec![])); - fetcher.new_active_peer( - peer_old, - B256::random(), - 100, - caps_old, - Arc::new(AtomicU64::new(10)), - None, - ); + fetcher.new_active_peer(NewPeerInfo { + peer_id: peer_old, + best_hash: B256::random(), + best_number: 100, + capabilities: caps_old, + timeout: Arc::new(AtomicU64::new(10)), + range_info: None, + supports_snap: false, + }); let (tx, rx) = oneshot::channel(); fetcher @@ -1792,14 +1920,15 @@ mod tests { let peer_71 = B512::random(); let caps_71 = Arc::new(Capabilities::from(vec![Capability::new("eth".into(), 71)])); - fetcher.new_active_peer( - peer_71, - B256::random(), - 100, - caps_71, - Arc::new(AtomicU64::new(10)), - None, - ); + fetcher.new_active_peer(NewPeerInfo { + peer_id: peer_71, + best_hash: B256::random(), + best_number: 100, + capabilities: caps_71, + timeout: Arc::new(AtomicU64::new(10)), + range_info: None, + supports_snap: false, + }); fetcher.peers.get_mut(&peer_71).expect("peer exists").state = PeerState::GetBlockHeaders; let (tx, _rx) = oneshot::channel(); @@ -1831,25 +1960,27 @@ mod tests { let peer_old = B512::random(); let caps_old = Arc::new(Capabilities::new(vec![])); - fetcher.new_active_peer( - peer_old, - B256::random(), - 100, - caps_old, - Arc::new(AtomicU64::new(10)), - None, - ); + fetcher.new_active_peer(NewPeerInfo { + peer_id: peer_old, + best_hash: B256::random(), + best_number: 100, + capabilities: caps_old, + timeout: Arc::new(AtomicU64::new(10)), + range_info: None, + supports_snap: false, + }); let peer_71 = B512::random(); let caps_71 = Arc::new(Capabilities::from(vec![Capability::new("eth".into(), 71)])); - fetcher.new_active_peer( - peer_71, - B256::random(), - 100, - caps_71, - Arc::new(AtomicU64::new(10)), - None, - ); + fetcher.new_active_peer(NewPeerInfo { + peer_id: peer_71, + best_hash: B256::random(), + best_number: 100, + capabilities: caps_71, + timeout: Arc::new(AtomicU64::new(10)), + range_info: None, + supports_snap: false, + }); fetcher.peers.get_mut(&peer_71).expect("peer exists").state = PeerState::GetBlockHeaders; let (tx, rx) = oneshot::channel(); @@ -1875,4 +2006,222 @@ mod tests { assert!(fetcher.queued_requests.is_empty()); assert_eq!(rx.await.unwrap().unwrap_err(), RequestError::UnsupportedCapability); } + + #[tokio::test] + async fn test_next_best_peer_snap_no_support() { + let manager = PeersManager::new(PeersConfig::default()); + let mut fetcher = + StateFetcher::::new(manager.handle(), Default::default()); + + let peer = B512::random(); + fetcher.new_active_peer(NewPeerInfo { + peer_id: peer, + best_hash: B256::random(), + best_number: 100, + capabilities: Arc::new(Capabilities::new(vec![])), + timeout: Arc::new(AtomicU64::new(10)), + range_info: None, + supports_snap: false, + }); + + assert_eq!(fetcher.next_best_peer(BestPeerRequirements::SupportsSnap), None); + } + + #[tokio::test] + async fn test_next_best_peer_snap_supported() { + let manager = PeersManager::new(PeersConfig::default()); + let mut fetcher = + StateFetcher::::new(manager.handle(), Default::default()); + + let peer = B512::random(); + fetcher.new_active_peer(NewPeerInfo { + peer_id: peer, + best_hash: B256::random(), + best_number: 100, + capabilities: Arc::new(Capabilities::new(vec![])), + timeout: Arc::new(AtomicU64::new(10)), + range_info: None, + supports_snap: true, + }); + + assert_eq!(fetcher.next_best_peer(BestPeerRequirements::SupportsSnap), Some(peer)); + } + + #[tokio::test] + async fn test_next_best_peer_snap_filters_correctly() { + let manager = PeersManager::new(PeersConfig::default()); + let mut fetcher = + StateFetcher::::new(manager.handle(), Default::default()); + + let peer_no_snap = B512::random(); + let peer_with_snap = B512::random(); + + fetcher.new_active_peer(NewPeerInfo { + peer_id: peer_no_snap, + best_hash: B256::random(), + best_number: 100, + capabilities: Arc::new(Capabilities::new(vec![])), + timeout: Arc::new(AtomicU64::new(5)), + range_info: None, + supports_snap: false, + }); + fetcher.new_active_peer(NewPeerInfo { + peer_id: peer_with_snap, + best_hash: B256::random(), + best_number: 100, + capabilities: Arc::new(Capabilities::new(vec![])), + timeout: Arc::new(AtomicU64::new(50)), + range_info: None, + supports_snap: true, + }); + + // Even though peer_no_snap has a lower timeout, it must NOT be selected. + assert_eq!( + fetcher.next_best_peer(BestPeerRequirements::SupportsSnap), + Some(peer_with_snap) + ); + } + + #[tokio::test] + async fn test_snap_request_rejected_without_snap_peer() { + use futures::task::noop_waker; + use std::task::{Context, Poll}; + + let manager = PeersManager::new(PeersConfig::default()); + let mut fetcher = + StateFetcher::::new(manager.handle(), Default::default()); + + // Only an eth-only peer is connected. + fetcher.new_active_peer(NewPeerInfo { + peer_id: B512::random(), + best_hash: B256::random(), + best_number: 100, + capabilities: Arc::new(Capabilities::new(vec![])), + timeout: Arc::new(AtomicU64::new(10)), + range_info: None, + supports_snap: false, + }); + + let (tx, rx) = oneshot::channel(); + fetcher + .download_requests_tx + .send(DownloadRequest::GetSnap { + request: SnapProtocolMessage::GetAccountRange(GetAccountRangeMessage { + request_id: 0, + root_hash: B256::ZERO, + starting_hash: B256::ZERO, + limit_hash: B256::ZERO, + response_bytes: 0, + }), + response: tx, + priority: Priority::Normal, + }) + .unwrap(); + + let waker = noop_waker(); + let mut cx = Context::from_waker(&waker); + + assert!(matches!(fetcher.poll(&mut cx), Poll::Pending)); + assert!(fetcher.queued_requests.is_empty()); + assert_eq!(rx.await.unwrap().unwrap_err(), RequestError::UnsupportedCapability); + } + + #[tokio::test] + async fn test_snap_response_triggers_followup() { + let manager = PeersManager::new(PeersConfig::default()); + let mut fetcher = + StateFetcher::::new(manager.handle(), Default::default()); + + let peer_id = B512::random(); + fetcher.new_active_peer(NewPeerInfo { + peer_id, + best_hash: B256::random(), + best_number: 100, + capabilities: Arc::new(Capabilities::new(vec![])), + timeout: Arc::new(AtomicU64::new(10)), + range_info: None, + supports_snap: true, + }); + + // Queue a followup snap request for the same peer. + let (followup_tx, _followup_rx) = oneshot::channel(); + fetcher.queued_requests.push_back(DownloadRequest::GetSnap { + request: SnapProtocolMessage::GetAccountRange(GetAccountRangeMessage { + request_id: 0, + root_hash: B256::ZERO, + starting_hash: B256::ZERO, + limit_hash: B256::ZERO, + response_bytes: 0, + }), + response: followup_tx, + priority: Priority::Normal, + }); + + let (tx, mut rx) = oneshot::channel(); + fetcher.inflight_snap_requests.insert(peer_id, Request { request: (), response: tx }); + fetcher.peers.get_mut(&peer_id).unwrap().state = PeerState::GetSnap; + + let resp = SnapResponse::AccountRange(AccountRangeMessage { + request_id: 1, + accounts: vec![], + proof: vec![], + }); + let outcome = fetcher.on_snap_response(peer_id, Ok(resp)); + + assert!(matches!(outcome, Some(BlockResponseOutcome::Request(pid, _)) if pid == peer_id)); + assert!(rx.try_recv().is_ok()); + } + + #[tokio::test] + async fn test_queued_snap_request_rejected_after_last_peer_disconnects() { + use futures::task::noop_waker; + use std::task::{Context, Poll}; + + let manager = PeersManager::new(PeersConfig::default()); + let mut fetcher = + StateFetcher::::new(manager.handle(), Default::default()); + + // The only connected peer supports snap but is busy, so the request gets queued. + let peer = B512::random(); + fetcher.new_active_peer(NewPeerInfo { + peer_id: peer, + best_hash: B256::random(), + best_number: 100, + capabilities: Arc::new(Capabilities::new(vec![])), + timeout: Arc::new(AtomicU64::new(10)), + range_info: None, + supports_snap: true, + }); + fetcher.peers.get_mut(&peer).expect("peer exists").state = PeerState::GetBlockHeaders; + + let (tx, rx) = oneshot::channel(); + fetcher + .download_requests_tx + .send(DownloadRequest::GetSnap { + request: SnapProtocolMessage::GetAccountRange(GetAccountRangeMessage { + request_id: 0, + root_hash: B256::ZERO, + starting_hash: B256::ZERO, + limit_hash: B256::ZERO, + response_bytes: 0, + }), + response: tx, + priority: Priority::Normal, + }) + .unwrap(); + + let waker = noop_waker(); + let mut cx = Context::from_waker(&waker); + + assert!(matches!(fetcher.poll(&mut cx), Poll::Pending)); + assert_eq!(fetcher.queued_requests.len(), 1); + + // The only peer disconnects, leaving `self.peers` empty. The still-queued request must + // resolve immediately instead of waiting for a peer that can never come back. + fetcher.on_session_closed(&peer); + + assert!(matches!(fetcher.poll(&mut cx), Poll::Pending)); + assert!(fetcher.queued_requests.is_empty()); + assert_eq!(rx.await.unwrap().unwrap_err(), RequestError::UnsupportedCapability); + } } diff --git a/crates/net/network/src/manager.rs b/crates/net/network/src/manager.rs index 9eddcb4ca32..515b27d690b 100644 --- a/crates/net/network/src/manager.rs +++ b/crates/net/network/src/manager.rs @@ -462,7 +462,8 @@ impl NetworkManager { /// Returns a new [`FetchClient`] that can be cloned and shared. /// - /// The [`FetchClient`] is the entrypoint for sending requests to the network. + /// The [`FetchClient`] is the entrypoint for sending requests to the network, including + /// `snap/2` requests via its [`SnapClient`](reth_network_p2p::snap::client::SnapClient) impl. pub fn fetch_client(&self) -> FetchClient { self.swarm.state().fetch_client() } @@ -582,6 +583,8 @@ impl NetworkManager { response, }); } + PeerRequest::GetSnap { request, response } => self + .delegate_eth_request(IncomingEthRequest::GetSnap { peer_id, request, response }), } } diff --git a/crates/net/network/src/message.rs b/crates/net/network/src/message.rs index b7c2f2ff8e4..20b3d8c52f5 100644 --- a/crates/net/network/src/message.rs +++ b/crates/net/network/src/message.rs @@ -14,9 +14,12 @@ use reth_eth_wire::{ NewPooledTransactionHashes, NodeData, PooledTransactions, Receipts, SharedTransactions, Transactions, }; -use reth_eth_wire_types::RawCapabilityMessage; -use reth_network_api::PeerRequest; -use reth_network_p2p::error::{RequestError, RequestResult}; +use reth_eth_wire_types::{snap::SnapProtocolMessage, RawCapabilityMessage}; +use reth_network_api::{PeerRequest, RequestMessage}; +use reth_network_p2p::{ + error::{RequestError, RequestResult}, + snap::client::SnapResponse, +}; use reth_primitives_traits::Block; use std::{ sync::Arc, @@ -134,6 +137,10 @@ pub enum BlockRequest { /// /// The response should be sent through the channel. GetReceipts(GetReceipts), + /// Requests a `snap/2` (EIP-8189) message from the peer. + /// + /// The response should be sent through the channel. + GetSnap(SnapProtocolMessage), } /// Corresponding variant for [`PeerRequest`]. @@ -189,6 +196,11 @@ pub enum PeerResponse { /// The receiver channel for the response to a cells request. response: oneshot::Receiver>, }, + /// Represents a response to a `snap/2` (EIP-8189) request. + Snap { + /// The receiver channel for the response to a `snap/2` request. + response: oneshot::Receiver>, + }, } // === impl PeerResponse === @@ -236,6 +248,10 @@ impl PeerResponse { Ok(res) => PeerResponseResult::Cells(res), Err(err) => PeerResponseResult::Cells(Err(err.into())), }, + Self::Snap { response } => match ready!(response.poll_unpin(cx)) { + Ok(res) => PeerResponseResult::Snap(res), + Err(err) => PeerResponseResult::Snap(Err(err.into())), + }, }; Poll::Ready(res) } @@ -262,19 +278,23 @@ pub enum PeerResponseResult { BlockAccessLists(RequestResult), /// Represents a result containing cells or an error. Cells(RequestResult), + /// Represents a result containing a `snap/2` response or an error. + Snap(RequestResult), } // === impl PeerResponseResult === impl PeerResponseResult { - /// Converts this response into an [`EthMessage`] - pub fn try_into_message(self, id: u64) -> RequestResult> { + /// Converts this response into the [`RequestMessage`] to send back to the peer: an + /// [`EthMessage`] for every variant except [`Self::Snap`], which becomes a + /// [`SnapProtocolMessage`]. + pub fn try_into_message(self, id: u64) -> RequestResult> { macro_rules! to_message { ($response:ident, $item:ident, $request_id:ident) => { match $response { Ok(res) => { let request = RequestPair { request_id: $request_id, message: $item(res) }; - Ok(EthMessage::$item(request)) + Ok(RequestMessage::Eth(EthMessage::$item(request))) } Err(err) => Err(err), } @@ -302,21 +322,29 @@ impl PeerResponseResult { Self::Receipts70(resp) => match resp { Ok(res) => { let request = RequestPair { request_id: id, message: res }; - Ok(EthMessage::Receipts70(request)) + Ok(RequestMessage::Eth(EthMessage::Receipts70(request))) } Err(err) => Err(err), }, Self::BlockAccessLists(resp) => match resp { Ok(res) => { let request = RequestPair { request_id: id, message: res }; - Ok(EthMessage::BlockAccessLists(request)) + Ok(RequestMessage::Eth(EthMessage::BlockAccessLists(request))) } Err(err) => Err(err), }, Self::Cells(resp) => match resp { Ok(res) => { let request = RequestPair { request_id: id, message: res }; - Ok(EthMessage::Cells(request)) + Ok(RequestMessage::Eth(EthMessage::Cells(request))) + } + Err(err) => Err(err), + }, + Self::Snap(resp) => match resp { + Ok(res) => { + let mut message: SnapProtocolMessage = res.into(); + message.set_request_id(id); + Ok(RequestMessage::Snap(message)) } Err(err) => Err(err), }, @@ -335,6 +363,7 @@ impl PeerResponseResult { Self::Receipts70(res) => res.as_ref().err(), Self::BlockAccessLists(res) => res.as_ref().err(), Self::Cells(res) => res.as_ref().err(), + Self::Snap(res) => res.as_ref().err(), } } diff --git a/crates/net/network/src/metrics.rs b/crates/net/network/src/metrics.rs index 39ebfe89cc7..c94c9ebc544 100644 --- a/crates/net/network/src/metrics.rs +++ b/crates/net/network/src/metrics.rs @@ -560,6 +560,9 @@ pub struct EthRequestHandlerMetrics { /// Number of `GetBlockAccessLists` requests received pub(crate) eth_block_access_lists_requests_received_total: Counter, + /// Number of `snap/2` (EIP-8189) requests received + pub(crate) snap_requests_received_total: Counter, + /// Duration in seconds of call to poll /// [`EthRequestHandler`](crate::eth_requests::EthRequestHandler). pub(crate) acc_duration_poll_eth_req_handler: Gauge, diff --git a/crates/net/network/src/peers.rs b/crates/net/network/src/peers.rs index 3f340a8bb14..818f7cdf073 100644 --- a/crates/net/network/src/peers.rs +++ b/crates/net/network/src/peers.rs @@ -956,9 +956,15 @@ impl PeersManager { self.ban_list.ban_peer(peer_id); } - /// Removes the peer from the ban list. + /// Removes the peer from the ban list and resets its reputation. pub(crate) fn unban_peer_by_admin(&mut self, peer_id: PeerId) { self.ban_list.unban_peer(&peer_id); + if let Some(peer) = self.peers.get_mut(&peer_id) && + peer.is_banned() + { + peer.unban(); + self.queued_actions.push_back(PeerAction::UnBanPeer { peer_id }); + } } /// Connect to the given peer. NOTE: if the maximum number of outbound sessions is reached, @@ -1703,7 +1709,7 @@ mod tests { } #[tokio::test] - async fn test_admin_unban_only_removes_banlist_entry() { + async fn test_admin_unban_resets_reputation() { let peer = PeerId::random(); let socket_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 1, 2)), 8008); let mut peers = PeersManager::default(); @@ -1716,7 +1722,11 @@ mod tests { peers.unban_peer_by_admin(peer); assert!(!peers.ban_list.is_banned_peer(&peer)); - assert!(peers.peers.get(&peer).is_some_and(Peer::is_banned)); + assert!(!peers.peers.get(&peer).is_some_and(Peer::is_banned)); + assert!(matches!( + peers.queued_actions.pop_front(), + Some(PeerAction::UnBanPeer { peer_id }) if peer_id == peer + )); } #[tokio::test] diff --git a/crates/net/network/src/session/active.rs b/crates/net/network/src/session/active.rs index 57945a55edc..d3985f75966 100644 --- a/crates/net/network/src/session/active.rs +++ b/crates/net/network/src/session/active.rs @@ -32,10 +32,13 @@ use reth_eth_wire::{ Capabilities, DisconnectP2P, DisconnectReason, EthMessage, EthSnapMessage, NetworkPrimitives, NewBlockPayload, }; -use reth_eth_wire_types::{message::RequestPair, NewPooledTransactionHashes, RawCapabilityMessage}; +use reth_eth_wire_types::{ + message::RequestPair, snap::SnapProtocolMessage, NewPooledTransactionHashes, + RawCapabilityMessage, +}; use reth_metrics::common::mpsc::MeteredPollSender; -use reth_network_api::PeerRequest; -use reth_network_p2p::error::RequestError; +use reth_network_api::{PeerRequest, RequestMessage}; +use reth_network_p2p::{error::RequestError, snap::client::SnapResponse}; use reth_network_peers::PeerId; use reth_network_types::session::config::INITIAL_REQUEST_TIMEOUT; use reth_primitives_traits::Block; @@ -80,8 +83,18 @@ const TIMEOUT_SCALING: u32 = 3; /// before reading any more messages from the remote peer, throttling the peer. const MAX_QUEUED_OUTGOING_RESPONSES: usize = 4; -/// Minimum capacity to retain for buffered incoming requests from the remote peer. -const MIN_RECEIVED_REQUESTS_CAPACITY: usize = 1; +/// Capacity above which the drained outgoing message queue is shrunk back to its steady-state +/// size, see [`QueuedOutgoingMessages::shrink_to_fit`]. +const SHRINK_CAPACITY_THRESHOLD: usize = 64; + +/// Maximum number of messages read from the connection per session poll before the task yields +/// back to the scheduler, see the receive loop in the session's `Future` impl. +/// +/// Message decoding is CPU intensive, so the budget bounds how long a single busy session can +/// occupy the executor thread. Small tx gossip messages dominate under load and are cheap to +/// decode individually, so the budget is sized such that their per-poll fixed costs (draining +/// command channels, advancing the sink, flushing the transport) amortize over a larger batch. +const RECEIVE_MESSAGE_BUDGET: usize = 16; /// Soft limit for the total number of buffered outgoing broadcast items (e.g. transaction hashes). /// @@ -162,7 +175,8 @@ pub(crate) struct ActiveSession { pub(crate) pending_message_to_session: Option>, /// Incoming internal requests which are delegated to the remote peer. pub(crate) internal_request_rx: Fuse>>, - /// All requests sent to the remote peer we're waiting on a response + /// All requests sent to the remote peer we're waiting on a response for, including `snap/2` + /// requests ([`PeerRequest::GetSnap`]). pub(crate) inflight_requests: FxHashMap>>, /// All requests that were sent by the remote peer and we're waiting on an internal response pub(crate) received_requests_from_remote: Vec>, @@ -206,10 +220,37 @@ impl ActiveSession { id } - /// Shrinks the capacity of the internal buffers. + /// Shrinks the capacity of the outgoing message queue once it is drained. + /// + /// The buffered incoming requests need no shrinking: the receive loop stops reading from the + /// wire while more than [`MAX_QUEUED_OUTGOING_RESPONSES`] of them are pending, which keeps + /// that buffer's capacity small. pub fn shrink_to_fit(&mut self) { - self.received_requests_from_remote.shrink_to(MIN_RECEIVED_REQUESTS_CAPACITY); - self.queued_outgoing.shrink_to(MAX_QUEUED_OUTGOING_RESPONSES); + self.queued_outgoing.shrink_to_fit(); + } + + /// Drains messages queued for sending into the connection's sink as long as the connection + /// can accept more, without flushing the underlying transport. + /// + /// This always advances the sink at least once, even with nothing queued, so connection + /// keepalive (ping) timers embedded in the sink's readiness logic are polled every session + /// poll. + /// + /// Returns `true` if at least one message was handed to the connection. + fn poll_send_queued(&mut self, cx: &mut Context<'_>) -> Result { + let mut progress = false; + while self.conn.poll_ready_unpin(cx).is_ready() { + let Some(msg) = self.queued_outgoing.pop_front() else { break }; + progress = true; + let res = match msg { + OutgoingMessage::Snap(msg) => self.conn.start_send_snap(msg), + OutgoingMessage::Eth(msg) => self.conn.start_send_unpin(msg), + OutgoingMessage::Broadcast(msg) => self.conn.start_send_broadcast(msg), + OutgoingMessage::Raw(msg) => self.conn.start_send_raw(msg), + }; + res?; + } + Ok(progress) } /// Handle a message read from the connection. @@ -380,6 +421,70 @@ impl ActiveSession { } } + /// Handles an inbound `snap/2` message. + /// + /// Responses are correlated to the in-flight [`PeerRequest::GetSnap`] by `request_id` (shared + /// with eth requests in [`Self::inflight_requests`]) and type-checked against the originally + /// sent request kind; unsolicited or mismatched ones count as bad messages. Inbound requests + /// are routed upward as [`PeerRequest::GetSnap`], same as any other eth request. + fn on_incoming_snap_message( + &mut self, + mut msg: SnapProtocolMessage, + ) -> OnIncomingMessageOutcome { + let request_id = msg.request_id(); + if !msg.is_response() { + let (tx, response) = oneshot::channel(); + self.received_requests_from_remote.push(ReceivedRequest { + request_id, + rx: PeerResponse::Snap { response }, + received: Instant::now(), + }); + return self + .try_emit_request(PeerMessage::EthRequest(PeerRequest::GetSnap { + request: msg, + response: tx, + })) + .into() + } + + let Some(req) = self.inflight_requests.remove(&request_id) else { + trace!(target: "net::session", ?request_id, remote_peer_id=?self.remote_peer_id, "received snap response to unknown request"); + self.on_bad_message(); + return OnIncomingMessageOutcome::Ok + }; + + match req.request { + RequestState::Waiting(PeerRequest::GetSnap { request, response }) => { + if Some(msg.message_id()) != request.message_id().response() { + debug!(target: "net::session", ?request_id, msg_id=?msg.message_id(), remote_peer_id=?self.remote_peer_id, "received snap response of wrong type"); + self.on_bad_message(); + let _ = response.send(Err(RequestError::BadResponse)); + return OnIncomingMessageOutcome::Ok + } + // Restore the caller's original request id, not the wire-assigned one. + msg.set_request_id(request.request_id()); + match SnapResponse::try_from(msg) { + Ok(snap_response) => { + trace!(target: "net::session", ?request_id, remote_peer_id=?self.remote_peer_id, "received snap response from peer"); + let _ = response.send(Ok(snap_response)); + self.update_request_timeout(req.timestamp, Instant::now()); + } + Err(_) => { + let _ = response.send(Err(RequestError::BadResponse)); + } + } + } + RequestState::Waiting(request) => { + // A different PeerRequest kind was pending for this id. + request.send_bad_response(); + } + RequestState::TimedOut => { + self.update_request_timeout(req.timestamp, Instant::now()); + } + } + OnIncomingMessageOutcome::Ok + } + /// Handle an internal peer request that will be sent to the remote. fn on_internal_peer_request(&mut self, request: PeerRequest, deadline: Instant) { let version = self.conn.version(); @@ -395,11 +500,21 @@ impl ActiveSession { return; } + // `GetSnap` isn't covered by the eth-version check above, and a connection that never + // negotiated `snap/2` can't send one without erroring the whole session. + if matches!(request, PeerRequest::GetSnap { .. }) && !self.conn.supports_snap() { + request.send_err_response(RequestError::UnsupportedCapability); + return; + } + let request_id = self.next_id(); trace!(?request, peer_id=?self.remote_peer_id, ?request_id, "sending request to peer"); - let msg = request.create_request_message(request_id).map_versioned(version); + let msg = match request.create_request_message(request_id) { + RequestMessage::Eth(msg) => msg.map_versioned(version).into(), + RequestMessage::Snap(msg) => OutgoingMessage::Snap(msg), + }; - self.queued_outgoing.push_back(msg.into()); + self.queued_outgoing.push_back(msg); let req = InflightRequest { request: RequestState::Waiting(request), timestamp: Instant::now(), @@ -462,9 +577,12 @@ impl ActiveSession { /// This will queue the response to be sent to the peer fn handle_outgoing_response(&mut self, id: u64, resp: PeerResponseResult) { match resp.try_into_message(id) { - Ok(msg) => { + Ok(RequestMessage::Eth(msg)) => { self.queued_outgoing.push_back(msg.into()); } + Ok(RequestMessage::Snap(msg)) => { + self.queued_outgoing.push_back(OutgoingMessage::Snap(msg)); + } Err(err) => { debug!(target: "net", %err, "Failed to respond to received request"); } @@ -657,7 +775,7 @@ impl Future for ActiveSession { // If the budget is exhausted we manually yield back control to the (coop) scheduler. This // manual yield point should prevent situations where polling appears to be frozen. See also // And tokio's docs on cooperative scheduling - let mut budget = 4; + let mut budget = RECEIVE_MESSAGE_BUDGET; // The main poll loop that drives the session 'main: loop { @@ -706,7 +824,7 @@ impl Future for ActiveSession { } SessionCommand::Disconnect { reason } => { let reason = reason.unwrap_or(DisconnectReason::DisconnectRequested); - return this.try_disconnect(reason, cx) + return this.try_disconnect(reason, cx); } } } @@ -733,33 +851,15 @@ impl Future for ActiveSession { } } - // Send messages by advancing the sink and queuing in buffered messages - while this.conn.poll_ready_unpin(cx).is_ready() { - if let Some(msg) = this.queued_outgoing.pop_front() { - progress = true; - let res = match msg { - OutgoingMessage::Eth(msg) => this.conn.start_send_unpin(msg), - OutgoingMessage::Broadcast(msg) => this.conn.start_send_broadcast(msg), - OutgoingMessage::Raw(msg) => this.conn.start_send_raw(msg), - }; - if let Err(err) = res { - debug!(target: "net::session", %err, remote_peer_id=?this.remote_peer_id, "failed to send message"); - // notify the manager - return this.close_on_error(err, cx) - } - } else { - // no more messages to send over the wire - break - } - } - - // The sink only buffers sent messages; `poll_flush` performs the actual writes and - // flushes the transport once for the entire batch queued above. This also resumes a - // flush that returned pending on an earlier pass; a no-op if nothing is buffered. - match this.conn.poll_flush_unpin(cx) { - Poll::Pending | Poll::Ready(Ok(())) => {} - Poll::Ready(Err(err)) => { - debug!(target: "net::session", %err, remote_peer_id=?this.remote_peer_id, "failed to flush connection"); + // Send messages by advancing the sink and queuing in buffered messages. The sink only + // buffers sent messages; the explicit flush happens once per poll after the main + // loop, so messages queued across the loop's passes batch up (the sink still writes + // out on its own for control messages and when its write buffer runs full). + match this.poll_send_queued(cx) { + Ok(sent) => progress |= sent, + Err(err) => { + debug!(target: "net::session", %err, remote_peer_id=?this.remote_peer_id, "failed to send message"); + // notify the manager return this.close_on_error(err, cx) } } @@ -832,13 +932,7 @@ impl Future for ActiveSession { // decode and handle message this.on_incoming_message(msg) } - // TODO: snap/2 is negotiated but not consumed yet; - // request/response handling - // lands with the snap client. - EthSnapMessage::Snap(_msg) => { - trace!(target: "net::session", remote_peer_id=?this.remote_peer_id, "ignoring inbound snap/2 message"); - OnIncomingMessageOutcome::Ok - } + EthSnapMessage::Snap(msg) => this.on_incoming_snap_message(msg), }; match outcome { OnIncomingMessageOutcome::Ok => { @@ -917,6 +1011,21 @@ impl Future for ActiveSession { } } + // Send anything the interval handlers above queued, then flush the transport for + // everything buffered during this poll. This also resumes a flush that returned pending + // on an earlier poll; a no-op if nothing is buffered. + if let Err(err) = this.poll_send_queued(cx) { + debug!(target: "net::session", %err, remote_peer_id=?this.remote_peer_id, "failed to send message"); + return this.close_on_error(err, cx) + } + match this.conn.poll_flush_unpin(cx) { + Poll::Pending | Poll::Ready(Ok(())) => {} + Poll::Ready(Err(err)) => { + debug!(target: "net::session", %err, remote_peer_id=?this.remote_peer_id, "failed to flush connection"); + return this.close_on_error(err, cx) + } + } + this.shrink_to_fit(); Poll::Pending @@ -944,8 +1053,6 @@ pub(crate) struct InflightRequest { deadline: Instant, } -// === impl InflightRequest === - impl InflightRequest> { /// Returns true if the request is timedout #[inline] @@ -1007,6 +1114,8 @@ pub(crate) enum OutgoingMessage { Broadcast(EthBroadcastMessage), /// A raw capability message Raw(RawCapabilityMessage), + /// A `snap/2` message to send over the dedicated `eth`+`snap` stream. + Snap(SnapProtocolMessage), } impl OutgoingMessage { @@ -1014,7 +1123,10 @@ impl OutgoingMessage { const fn is_response(&self) -> bool { match self { Self::Eth(msg) => msg.is_response(), - _ => false, + // Served snap responses count toward response backpressure; outbound snap requests do + // not. `SnapProtocolMessage::is_response` distinguishes the two. + Self::Snap(msg) => msg.is_response(), + Self::Broadcast(_) | Self::Raw(_) => false, } } @@ -1037,7 +1149,7 @@ impl OutgoingMessage { EthBroadcastMessage::Transactions(txs) => txs.len(), EthBroadcastMessage::BroadcastPoolTransactions(txs) => txs.len(), }, - Self::Raw(_) => 0, + Self::Raw(_) | Self::Snap(_) => 0, } } @@ -1174,8 +1286,13 @@ impl QueuedOutgoingMessages { self.count.increment(1); } - pub(crate) fn shrink_to(&mut self, min_capacity: usize) { - self.messages.shrink_to(min_capacity); + /// Shrinks the queue's capacity back to its steady-state size once it is drained, if it grew + /// well beyond it. The threshold avoids a shrink/regrow reallocation cycle on every poll + /// under regular bursty traffic. + pub(crate) fn shrink_to_fit(&mut self) { + if self.messages.is_empty() && self.messages.capacity() > SHRINK_CAPACITY_THRESHOLD { + self.messages.shrink_to(MAX_QUEUED_OUTGOING_RESPONSES); + } } } @@ -1194,17 +1311,25 @@ mod tests { use super::*; use crate::session::{handle::PendingSessionEvent, start_pending_incoming_session}; use alloy_eips::eip2124::ForkFilter; + use alloy_primitives::B256; + use futures::task::noop_waker; use reth_chainspec::MAINNET; use reth_ecies::stream::ECIESStream; use reth_eth_wire::{ - handshake::EthHandshake, EthNetworkPrimitives, EthStream, GetBlockAccessLists, - GetBlockBodies, HelloMessageWithProtocols, P2PStream, StatusBuilder, UnauthedEthStream, - UnauthedP2PStream, UnifiedStatus, + handshake::EthHandshake, protocol::Protocol, EthNetworkPrimitives, EthStream, + GetBlockAccessLists, GetBlockBodies, HelloMessageWithProtocols, P2PStream, StatusBuilder, + UnauthedEthStream, UnauthedP2PStream, UnifiedStatus, }; use reth_eth_wire_types::{ - message::MAX_MESSAGE_SIZE, EthMessageID, NewPooledTransactionHashes72, RawCapabilityMessage, + message::MAX_MESSAGE_SIZE, + snap::{ + AccountRangeMessage, BlockAccessListsMessage, GetAccountRangeMessage, + GetBlockAccessListsMessage, + }, + BlockAccessLists, EthMessageID, NewPooledTransactionHashes72, }; use reth_ethereum_forks::EthereumHardfork; + use reth_network_p2p::error::RequestResult; use reth_network_peers::pk2id; use reth_network_types::session::config::PROTOCOL_BREACH_REQUEST_TIMEOUT; use secp256k1::{SecretKey, SECP256K1}; @@ -1381,6 +1506,33 @@ mod tests { } } + /// Returns a [`SessionBuilder`] whose hello also advertises `snap/2`, so the negotiated + /// session ends up on an [`EthSnapStream`](reth_eth_wire::EthSnapStream) connection instead + /// of a plain `eth`-only one. + fn snap_session_builder() -> SessionBuilder { + let mut builder = SessionBuilder::default(); + builder.hello.try_add_protocol(Protocol::snap_2()).unwrap(); + builder + } + + /// Dispatches a `snap/2` request via [`ActiveSession::on_internal_peer_request`] and returns + /// the session-assigned request id plus the caller's response receiver. + fn dispatch_snap_request( + session: &mut ActiveSession, + caller_request_id: u64, + ) -> (u64, oneshot::Receiver>) { + let (response, rx) = oneshot::channel(); + let request = SnapProtocolMessage::GetBlockAccessLists(GetBlockAccessListsMessage { + request_id: caller_request_id, + block_hashes: Vec::new(), + response_bytes: 0, + }); + let deadline = session.request_deadline(); + session.on_internal_peer_request(PeerRequest::GetSnap { request, response }, deadline); + let id = *session.inflight_requests.keys().next().expect("snap request tracked"); + (id, rx) + } + #[tokio::test(flavor = "multi_thread")] async fn test_disconnect() { let mut builder = SessionBuilder::default(); @@ -1546,6 +1698,251 @@ mod tests { } } + #[tokio::test(flavor = "multi_thread")] + async fn snap_request_is_assigned_unique_id_and_response_correlated() { + let mut builder = snap_session_builder(); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let local_addr = listener.local_addr().unwrap(); + let fut = builder.with_client_stream(local_addr, async move |client_stream| { + let _client_stream = client_stream; + tokio::time::sleep(Duration::from_secs(60)).await; + }); + tokio::task::spawn(fut); + let (incoming, _) = listener.accept().await.unwrap(); + let mut session = builder.connect_incoming(incoming).await; + + // The session assigns its own request id (not the caller's sentinel) and tracks it. + let (id, rx) = dispatch_snap_request(&mut session, u64::MAX); + assert_ne!(id, u64::MAX, "session must assign its own request id"); + + // A response carrying that id is correlated back to the caller's future. + let outcome = session.on_incoming_snap_message(SnapProtocolMessage::BlockAccessLists( + BlockAccessListsMessage { + request_id: id, + block_access_lists: BlockAccessLists(Vec::new()), + }, + )); + assert!(matches!(outcome, OnIncomingMessageOutcome::Ok)); + assert!(!session.inflight_requests.contains_key(&id)); + + // The delivered response carries the caller's original id again, not the session's. + let response = rx.await.unwrap().unwrap(); + assert!(matches!( + response, + SnapResponse::BlockAccessLists(m) if m.request_id == u64::MAX + )); + } + + #[tokio::test(flavor = "multi_thread")] + async fn wrong_type_snap_response_is_rejected() { + let mut builder = snap_session_builder(); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let local_addr = listener.local_addr().unwrap(); + let fut = builder.with_client_stream(local_addr, async move |client_stream| { + let _client_stream = client_stream; + tokio::time::sleep(Duration::from_secs(60)).await; + }); + tokio::task::spawn(fut); + let (incoming, _) = listener.accept().await.unwrap(); + let mut session = builder.connect_incoming(incoming).await; + + let (id, rx) = dispatch_snap_request(&mut session, 0); + + // Answering a GetBlockAccessLists with an AccountRange under the same id is a bad message. + let outcome = session.on_incoming_snap_message(SnapProtocolMessage::AccountRange( + AccountRangeMessage { request_id: id, accounts: Vec::new(), proof: Vec::new() }, + )); + assert!(matches!(outcome, OnIncomingMessageOutcome::Ok)); + assert!(!session.inflight_requests.contains_key(&id)); + assert_eq!(rx.await.unwrap().unwrap_err(), RequestError::BadResponse); + assert!(matches!( + builder.active_session_rx.next().await, + Some(ActiveSessionMessage::BadMessage { .. }) + )); + } + + #[tokio::test(flavor = "multi_thread")] + async fn snap_request_times_out() { + let mut builder = snap_session_builder(); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let local_addr = listener.local_addr().unwrap(); + let fut = builder.with_client_stream(local_addr, async move |client_stream| { + let _client_stream = client_stream; + tokio::time::sleep(Duration::from_secs(60)).await; + }); + tokio::task::spawn(fut); + let (incoming, _) = listener.accept().await.unwrap(); + let mut session = builder.connect_incoming(incoming).await; + + // Tiny timeout so the deadline (computed at insert) is already in the past. + session.internal_request_timeout.store(1, Ordering::Relaxed); + let (id, rx) = dispatch_snap_request(&mut session, 0); + + // The first check resolves the caller with a timeout but keeps the entry so the session + // can escalate to a protocol breach. + tokio::time::sleep(Duration::from_millis(20)).await; + assert!(!session.check_timed_out_requests(Instant::now())); + assert!(session.inflight_requests.contains_key(&id)); + assert_eq!(rx.await.unwrap().unwrap_err(), RequestError::Timeout); + + // Once the breach timeout passes without a response, the session flags a protocol breach. + session.protocol_breach_request_timeout = Duration::from_millis(1); + assert!(session.check_timed_out_requests(Instant::now())); + } + + #[tokio::test(flavor = "multi_thread")] + async fn late_snap_response_is_consumed_without_penalty() { + let mut builder = snap_session_builder(); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let local_addr = listener.local_addr().unwrap(); + let fut = builder.with_client_stream(local_addr, async move |client_stream| { + let _client_stream = client_stream; + tokio::time::sleep(Duration::from_secs(60)).await; + }); + tokio::task::spawn(fut); + let (incoming, _) = listener.accept().await.unwrap(); + let mut session = builder.connect_incoming(incoming).await; + + session.internal_request_timeout.store(1, Ordering::Relaxed); + let (id, _rx) = dispatch_snap_request(&mut session, 0); + tokio::time::sleep(Duration::from_millis(20)).await; + assert!(!session.check_timed_out_requests(Instant::now())); + + // A response arriving after the timeout clears the entry without a bad-message report. + let outcome = session.on_incoming_snap_message(SnapProtocolMessage::BlockAccessLists( + BlockAccessListsMessage { + request_id: id, + block_access_lists: BlockAccessLists(Vec::new()), + }, + )); + assert!(matches!(outcome, OnIncomingMessageOutcome::Ok)); + assert!(!session.inflight_requests.contains_key(&id)); + assert!(futures::FutureExt::now_or_never(builder.active_session_rx.next()) + .flatten() + .is_none()); + } + + #[tokio::test(flavor = "multi_thread")] + async fn unknown_snap_response_is_penalized() { + let mut builder = snap_session_builder(); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let local_addr = listener.local_addr().unwrap(); + let fut = builder.with_client_stream(local_addr, async move |client_stream| { + let _client_stream = client_stream; + tokio::time::sleep(Duration::from_secs(60)).await; + }); + tokio::task::spawn(fut); + let (incoming, _) = listener.accept().await.unwrap(); + let mut session = builder.connect_incoming(incoming).await; + + // A response for a request we never sent is dropped and reported as a bad message. + let outcome = session.on_incoming_snap_message(SnapProtocolMessage::BlockAccessLists( + BlockAccessListsMessage { + request_id: 999, + block_access_lists: BlockAccessLists(Vec::new()), + }, + )); + assert!(matches!(outcome, OnIncomingMessageOutcome::Ok)); + assert!(session.inflight_requests.is_empty()); + assert!(session.queued_outgoing.pop_front().is_none()); + assert!(matches!( + builder.active_session_rx.next().await, + Some(ActiveSessionMessage::BadMessage { .. }) + )); + } + + #[tokio::test(flavor = "multi_thread")] + async fn get_snap_request_rejected_without_negotiated_snap() { + // A plain `eth`-only session: no `snap/2` was negotiated. + let mut builder = SessionBuilder::default(); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let local_addr = listener.local_addr().unwrap(); + let fut = builder.with_client_stream(local_addr, async move |client_stream| { + let _client_stream = client_stream; + tokio::time::sleep(Duration::from_secs(60)).await; + }); + tokio::task::spawn(fut); + let (incoming, _) = listener.accept().await.unwrap(); + let mut session = builder.connect_incoming(incoming).await; + assert!(!session.conn.supports_snap()); + + let (response, rx) = oneshot::channel(); + let request = SnapProtocolMessage::GetBlockAccessLists(GetBlockAccessListsMessage { + request_id: 0, + block_hashes: Vec::new(), + response_bytes: 0, + }); + let deadline = session.request_deadline(); + session.on_internal_peer_request(PeerRequest::GetSnap { request, response }, deadline); + + // Rejected immediately instead of being queued for a connection that can't send it. + assert!(session.inflight_requests.is_empty()); + assert!(session.queued_outgoing.pop_front().is_none()); + assert_eq!(rx.await.unwrap().unwrap_err(), RequestError::UnsupportedCapability); + } + + #[tokio::test(flavor = "multi_thread")] + async fn inbound_snap_request_round_trips_to_a_response() { + let mut builder = snap_session_builder(); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let local_addr = listener.local_addr().unwrap(); + let fut = builder.with_client_stream(local_addr, async move |client_stream| { + let _client_stream = client_stream; + tokio::time::sleep(Duration::from_secs(60)).await; + }); + tokio::task::spawn(fut); + let (incoming, _) = listener.accept().await.unwrap(); + let mut session = builder.connect_incoming(incoming).await; + + // The peer sends an inbound GetAccountRange request. + let outcome = session.on_incoming_snap_message(SnapProtocolMessage::GetAccountRange( + GetAccountRangeMessage { + request_id: 7, + root_hash: B256::ZERO, + starting_hash: B256::ZERO, + limit_hash: B256::ZERO, + response_bytes: 1024, + }, + )); + assert!(matches!(outcome, OnIncomingMessageOutcome::Ok)); + assert_eq!(session.received_requests_from_remote.len(), 1); + + // It's routed upward instead of being served inline. + let Some(ActiveSessionMessage::ValidMessage { + message: PeerMessage::EthRequest(PeerRequest::GetSnap { request, response }), + .. + }) = builder.active_session_rx.next().await + else { + panic!("expected an outbound GetSnap request") + }; + assert!(matches!(request, SnapProtocolMessage::GetAccountRange(_))); + + // The handler answers with an empty-but-valid range. + let _ = response.send(Ok(SnapResponse::AccountRange(AccountRangeMessage { + request_id: 7, + accounts: Vec::new(), + proof: Vec::new(), + }))); + + // Drive the same conversion the session's main poll loop would. + let mut req = session.received_requests_from_remote.pop().unwrap(); + let waker = noop_waker(); + let mut cx = Context::from_waker(&waker); + let Poll::Ready(resp) = req.rx.poll(&mut cx) else { panic!("response should be ready") }; + session.handle_outgoing_response(req.request_id, resp); + + // The reply goes out as a snap/2 message carrying the original request id, not an eth + // message. + let msg = session.queued_outgoing.pop_front().expect("response queued for send"); + assert!(matches!( + msg, + OutgoingMessage::Snap(SnapProtocolMessage::AccountRange(AccountRangeMessage { + request_id: 7, + .. + })) + )); + } + #[test] fn eth72_pooled_hashes_count_broadcast_items() { let hashes = diff --git a/crates/net/network/src/session/conn.rs b/crates/net/network/src/session/conn.rs index 26f6c23aa6c..4c421f5a7b5 100644 --- a/crates/net/network/src/session/conn.rs +++ b/crates/net/network/src/session/conn.rs @@ -3,9 +3,10 @@ use futures::{Sink, SinkExt, Stream, StreamExt}; use reth_ecies::stream::ECIESStream; use reth_eth_wire::{ - errors::EthStreamError, + errors::{EthStreamError, P2PStreamError}, message::EthBroadcastMessage, multiplex::{ProtocolProxy, RlpxSatelliteStream}, + snap::SnapProtocolMessage, EthMessage, EthNetworkPrimitives, EthSnapMessage, EthSnapStream, EthStream, EthVersion, NetworkPrimitives, P2PStream, }; @@ -55,6 +56,12 @@ impl EthRlpxConnection { } } + /// Returns `true` if `snap/2` was negotiated on this connection. + #[inline] + pub(crate) const fn supports_snap(&self) -> bool { + matches!(self, Self::EthSnap(_)) + } + /// Consumes this type and returns the wrapped [`P2PStream`]. #[inline] pub(crate) fn into_inner(self) -> P2PStream> { @@ -107,6 +114,19 @@ impl EthRlpxConnection { } } + /// Queues a `snap/2` message to be sent on the wire. + /// + /// Returns an error on connections that did not negotiate `snap/2`, so a caller never believes + /// a request was sent when it was discarded. + pub fn start_send_snap(&mut self, msg: SnapProtocolMessage) -> Result<(), EthStreamError> { + match self { + Self::EthSnap(conn) => conn.start_send_unpin(EthSnapMessage::Snap(msg)), + Self::EthOnly(_) | Self::Satellite(_) => { + Err(P2PStreamError::CapabilityNotShared.into()) + } + } + } + /// Sets whether to reject block announcement messages (`NewBlock`, `NewBlockHashes`) before /// RLP decoding to avoid memory amplification from deserializing blocks that will be discarded. pub fn set_reject_block_announcements(&mut self, reject: bool) { diff --git a/crates/net/network/src/session/mod.rs b/crates/net/network/src/session/mod.rs index 80e9617210d..29d22f38543 100644 --- a/crates/net/network/src/session/mod.rs +++ b/crates/net/network/src/session/mod.rs @@ -608,6 +608,7 @@ impl SessionManager { last_sent_latest_block: None, }; + let supports_snap = session.conn.supports_snap(); self.spawn(session); let client_version = client_id.into(); @@ -644,6 +645,7 @@ impl SessionManager { direction, timeout, range_info: remote_range_info, + supports_snap, }) } PendingSessionEvent::Disconnected { remote_addr, session_id, direction, error } => { @@ -779,6 +781,8 @@ pub enum SessionEvent { timeout: Arc, /// The range info for the peer. range_info: Option, + /// Whether the connection negotiated `snap/2` and can serve [`PeerRequest::GetSnap`]. + supports_snap: bool, }, /// The peer was already connected with another session. AlreadyConnected { diff --git a/crates/net/network/src/state.rs b/crates/net/network/src/state.rs index 97f4ae1df7e..48d16fc0641 100644 --- a/crates/net/network/src/state.rs +++ b/crates/net/network/src/state.rs @@ -3,7 +3,7 @@ use crate::{ cache::LruCache, discovery::Discovery, - fetch::{BlockResponseOutcome, FetchAction, StateFetcher}, + fetch::{BlockResponseOutcome, FetchAction, NewPeerInfo, StateFetcher}, message::{BlockRequest, NewBlockMessage, PeerResponse, PeerResponseResult}, peers::{PeerAction, PeersManager}, session::BlockRangeInfo, @@ -147,15 +147,17 @@ impl NetworkState { /// /// Returns `Ok` if the session is valid, returns an `Err` if the session is not accepted and /// should be rejected. - pub(crate) fn on_session_activated( - &mut self, - peer: PeerId, - capabilities: Arc, - status: Arc, - request_tx: PeerRequestSender>, - timeout: Arc, - range_info: Option, - ) { + pub(crate) fn on_session_activated(&mut self, activation: SessionActivation) { + let SessionActivation { + peer, + capabilities, + status, + request_tx, + timeout, + range_info, + supports_snap, + } = activation; + debug_assert!(!self.active_peers.contains_key(&peer), "Already connected; not possible"); // Use the block number from the peer's status (eth/69+) if available, @@ -163,14 +165,15 @@ impl NetworkState { let block_number = status.latest_block.unwrap_or_else(|| { self.client.block_number(status.blockhash).ok().flatten().unwrap_or_default() }); - self.state_fetcher.new_active_peer( - peer, - status.blockhash, - block_number, - Arc::clone(&capabilities), + self.state_fetcher.new_active_peer(NewPeerInfo { + peer_id: peer, + best_hash: status.blockhash, + best_number: block_number, + capabilities: Arc::clone(&capabilities), timeout, range_info, - ); + supports_snap, + }); self.active_peers.insert( peer, @@ -443,6 +446,12 @@ impl NetworkState { (request, response) } } + BlockRequest::GetSnap(request) => { + let (response, rx) = oneshot::channel(); + let request = PeerRequest::GetSnap { request, response }; + let response = PeerResponse::Snap { response: rx }; + (request, response) + } }; let _ = peer.request_tx.to_session_tx.try_send(request); peer.pending_response = Some(response); @@ -498,6 +507,7 @@ impl NetworkState { PeerResponseResult::BlockAccessLists(res) => { self.state_fetcher.on_block_access_lists_response(peer, res) } + PeerResponseResult::Snap(res) => self.state_fetcher.on_snap_response(peer, res), _ => None, }; @@ -604,6 +614,25 @@ pub(crate) struct ActivePeer { pub(crate) blocks: LruCache, } +/// Everything [`NetworkState::on_session_activated`] needs to register a newly established +/// session. +pub(crate) struct SessionActivation { + /// The remote peer's identifier. + pub(crate) peer: PeerId, + /// The capabilities the peer announced. + pub(crate) capabilities: Arc, + /// The `Status` message the peer sent during the `eth` handshake. + pub(crate) status: Arc, + /// A communication channel directly to the session task. + pub(crate) request_tx: PeerRequestSender>, + /// The maximum time the session waits for a response from the peer. + pub(crate) timeout: Arc, + /// The range info for the peer. + pub(crate) range_info: Option, + /// Whether the connection negotiated `snap/2` and can serve [`PeerRequest::GetSnap`]. + pub(crate) supports_snap: bool, +} + /// Message variants triggered by the [`NetworkState`] #[derive(Debug)] pub(crate) enum StateAction { @@ -650,7 +679,7 @@ mod tests { discovery::Discovery, fetch::StateFetcher, peers::PeersManager, - state::{BlockNumReader, NetworkState}, + state::{BlockNumReader, NetworkState, SessionActivation}, PeerRequest, }; use alloy_consensus::Header; @@ -697,14 +726,15 @@ mod tests { let (tx, session_rx) = mpsc::channel(1); let peer_tx = PeerRequestSender::new(peer_id, tx); - state.on_session_activated( - peer_id, - capabilities(), - Arc::default(), - peer_tx, - Arc::new(AtomicU64::new(1)), - None, - ); + state.on_session_activated(SessionActivation { + peer: peer_id, + capabilities: capabilities(), + status: Arc::default(), + request_tx: peer_tx, + timeout: Arc::new(AtomicU64::new(1)), + range_info: None, + supports_snap: false, + }); assert!(state.active_peers.contains_key(&peer_id)); diff --git a/crates/net/network/src/swarm.rs b/crates/net/network/src/swarm.rs index bd9dca8ea36..7c0354ba6e3 100644 --- a/crates/net/network/src/swarm.rs +++ b/crates/net/network/src/swarm.rs @@ -4,7 +4,7 @@ use crate::{ peers::{InboundConnectionError, PeersManager}, protocol::IntoRlpxSubProtocol, session::{Direction, PendingSessionHandshakeError, SessionEvent, SessionId, SessionManager}, - state::{NetworkState, StateAction}, + state::{NetworkState, SessionActivation, StateAction}, }; use futures::Stream; use reth_eth_wire::{ @@ -133,15 +133,17 @@ impl Swarm { direction, timeout, range_info, + supports_snap, } => { - self.state.on_session_activated( - peer_id, - capabilities.clone(), - status.clone(), - messages.clone(), + self.state.on_session_activated(SessionActivation { + peer: peer_id, + capabilities: capabilities.clone(), + status: status.clone(), + request_tx: messages.clone(), timeout, range_info, - ); + supports_snap, + }); Some(SwarmEvent::SessionEstablished { peer_id, remote_addr, diff --git a/crates/net/network/src/test_utils/testnet.rs b/crates/net/network/src/test_utils/testnet.rs index b0983d84e5f..bc8a166e08f 100644 --- a/crates/net/network/src/test_utils/testnet.rs +++ b/crates/net/network/src/test_utils/testnet.rs @@ -29,7 +29,7 @@ use reth_network_api::{ use reth_network_peers::PeerId; use reth_storage_api::{ noop::NoopProvider, BalProvider, BlockReader, BlockReaderIdExt, HeaderProvider, - StateProviderFactory, + StateProviderFactory, StateRangeProviderFactory, }; use reth_tasks::Runtime; use reth_tokio_util::EventStream; @@ -249,6 +249,8 @@ where Header = alloy_consensus::Header, > + HeaderProvider + BalProvider + + StateProviderFactory + + StateRangeProviderFactory + Clone + Unpin + 'static, @@ -322,6 +324,8 @@ where Header = alloy_consensus::Header, > + HeaderProvider + BalProvider + + StateProviderFactory + + StateRangeProviderFactory + Unpin + 'static, Pool: TransactionPool< @@ -589,6 +593,8 @@ where Header = alloy_consensus::Header, > + HeaderProvider + BalProvider + + StateProviderFactory + + StateRangeProviderFactory + Unpin + 'static, Pool: TransactionPool< diff --git a/crates/net/p2p/src/snap/client.rs b/crates/net/p2p/src/snap/client.rs index c9199baa787..f805a4c6061 100644 --- a/crates/net/p2p/src/snap/client.rs +++ b/crates/net/p2p/src/snap/client.rs @@ -36,6 +36,17 @@ impl TryFrom for SnapResponse { } } +impl From for SnapProtocolMessage { + fn from(response: SnapResponse) -> Self { + match response { + SnapResponse::AccountRange(m) => Self::AccountRange(m), + SnapResponse::StorageRanges(m) => Self::StorageRanges(m), + SnapResponse::ByteCodes(m) => Self::ByteCodes(m), + SnapResponse::BlockAccessLists(m) => Self::BlockAccessLists(m), + } + } +} + /// The snap sync downloader client #[auto_impl::auto_impl(&, Arc, Box)] pub trait SnapClient: DownloadClient { diff --git a/crates/node/builder/src/components/mod.rs b/crates/node/builder/src/components/mod.rs index 8cf9794bd82..a90f766f348 100644 --- a/crates/node/builder/src/components/mod.rs +++ b/crates/node/builder/src/components/mod.rs @@ -5,7 +5,7 @@ //! - The network implementation. //! - The payload builder service. //! -//! Components depend on a fully type configured node: [FullNodeTypes](crate::node::FullNodeTypes). +//! Components depend on a fully type configured node: [`FullNodeTypes`]. mod builder; mod consensus; diff --git a/crates/node/builder/src/launch/invalid_block_hook.rs b/crates/node/builder/src/launch/invalid_block_hook.rs index 3c1848dceb4..7965634c55b 100644 --- a/crates/node/builder/src/launch/invalid_block_hook.rs +++ b/crates/node/builder/src/launch/invalid_block_hook.rs @@ -105,7 +105,7 @@ where healthy_node_rpc_client.clone(), )), InvalidBlockHookType::PreState | InvalidBlockHookType::Opcode => { - eyre::bail!("invalid block hook {hook:?} is not implemented yet") + eyre::bail!("invalid block hook {hook:?} is not implemented yet"); } } as Box>) }) diff --git a/crates/node/core/src/args/credible.rs b/crates/node/core/src/args/credible.rs index 35489adf07c..5b2b3098726 100644 --- a/crates/node/core/src/args/credible.rs +++ b/crates/node/core/src/args/credible.rs @@ -8,8 +8,8 @@ use reth_rpc_eth_types::CredibleRpcConfig; pub struct CredibleArgs { /// Address of the on-chain `CredibleRegistry` contract. /// - /// When set, the marker override for `eth_call` / `eth_estimateGas` is derived per-request - /// from the registry's `_credibleBlocks` mapping instead of a static override. + /// When set, the marker override for call-like RPC methods is derived per-request from the + /// registry's `_credibleBlocks` mapping instead of a static override. #[arg(long = "rpc.credible-registry-address", value_name = "ADDRESS")] pub registry_address: Option
, diff --git a/crates/node/core/src/args/rpc_server.rs b/crates/node/core/src/args/rpc_server.rs index c0e96ff5d1d..9240be67408 100644 --- a/crates/node/core/src/args/rpc_server.rs +++ b/crates/node/core/src/args/rpc_server.rs @@ -673,7 +673,7 @@ pub struct RpcServerArgs { /// /// When enabled, transactions that fail execution will be skipped, and all subsequent /// transactions from the same sender will also be skipped. - #[arg(long = "testing.skip-invalid-transactions", default_value_t = true)] + #[arg(long = "testing.skip-invalid-transactions", default_value_t = false)] pub testing_skip_invalid_transactions: bool, /// Override the gas limit used by `testing_buildBlockV1`. @@ -929,7 +929,7 @@ impl Default for RpcServerArgs { rpc_state_cache, gas_price_oracle, rpc_send_raw_transaction_sync_timeout, - testing_skip_invalid_transactions: true, + testing_skip_invalid_transactions: false, testing_gas_limit: None, rpc_force_blob_sidecar_upcasting: false, credible: CredibleArgs::default(), diff --git a/crates/node/core/src/utils.rs b/crates/node/core/src/utils.rs index 68cab7929f3..e63337c2735 100644 --- a/crates/node/core/src/utils.rs +++ b/crates/node/core/src/utils.rs @@ -41,7 +41,7 @@ where let Some(header) = response else { client.report_bad_message(peer_id); - eyre::bail!("Invalid number of headers received. Expected: 1. Received: 0") + eyre::bail!("Invalid number of headers received. Expected: 1. Received: 0"); }; let header = SealedHeader::seal_slow(header); @@ -77,7 +77,7 @@ where let Some(body) = response else { client.report_bad_message(peer_id); - eyre::bail!("Invalid number of bodies received. Expected: 1. Received: 0") + eyre::bail!("Invalid number of bodies received. Expected: 1. Received: 0"); }; let block = SealedBlock::from_sealed_parts(header, body); diff --git a/crates/payload/primitives/src/traits.rs b/crates/payload/primitives/src/traits.rs index 0ac1bc80103..4d46e6eb0fa 100644 --- a/crates/payload/primitives/src/traits.rs +++ b/crates/payload/primitives/src/traits.rs @@ -10,7 +10,7 @@ use core::fmt; use either::Either; use reth_execution_types::BlockExecutionOutput; use reth_primitives_traits::{NodePrimitives, RecoveredBlock, SealedBlock, SealedHeader}; -use reth_trie_common::{prefix_set::TriePrefixSetsMut, updates::TrieUpdates, HashedPostState}; +use reth_trie_common::{updates::TrieUpdates, HashedPostState}; /// Represents an executed block for payload building purposes. /// @@ -26,8 +26,6 @@ pub struct BuiltPayloadExecutedBlock { pub hashed_state: Arc, /// Trie updates that result from calculating the state root for the block (unsorted). pub trie_updates: Arc, - /// Changed trie node base paths, if known. - pub changed_paths: Option>, } /// Represents a successfully built execution payload (block). diff --git a/crates/prune/db/Cargo.toml b/crates/prune/db/Cargo.toml deleted file mode 100644 index 269a87bf7b6..00000000000 --- a/crates/prune/db/Cargo.toml +++ /dev/null @@ -1,15 +0,0 @@ -[package] -name = "reth-prune-db" -version.workspace = true -edition.workspace = true -rust-version.workspace = true -license.workspace = true -homepage.workspace = true -repository.workspace = true -exclude.workspace = true -description = "Database integration with prune implementation" - -[dependencies] - -[lints] -workspace = true diff --git a/crates/prune/db/src/lib.rs b/crates/prune/db/src/lib.rs deleted file mode 100644 index ef777085e54..00000000000 --- a/crates/prune/db/src/lib.rs +++ /dev/null @@ -1 +0,0 @@ -//! An integration of `reth-prune` with `reth-db`. diff --git a/crates/prune/prune/Cargo.toml b/crates/prune/prune/Cargo.toml index 09d867f5c6a..095442b78f4 100644 --- a/crates/prune/prune/Cargo.toml +++ b/crates/prune/prune/Cargo.toml @@ -37,7 +37,6 @@ alloy-primitives.workspace = true tracing.workspace = true thiserror.workspace = true itertools.workspace = true -rayon.workspace = true tokio.workspace = true rustc-hash.workspace = true diff --git a/crates/prune/prune/src/db_ext.rs b/crates/prune/prune/src/db_ext.rs index 33a687e437d..736bc48a477 100644 --- a/crates/prune/prune/src/db_ext.rs +++ b/crates/prune/prune/src/db_ext.rs @@ -1,6 +1,6 @@ use crate::PruneLimiter; use reth_db_api::{ - cursor::{DbCursorRO, DbCursorRW, RangeWalker}, + cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO, RangeWalker}, table::{DupSort, Table, TableRow}, transaction::{DbTx, DbTxMut}, DatabaseError, @@ -193,6 +193,44 @@ pub(crate) trait DbTxPruneExt: DbTxMut + DbTx { Ok((starting_entries - ending_entries, done)) } + + /// Prune duplicate entries for a single DUPSORT key. + /// + /// Returns the number of rows pruned and whether all duplicate entries for the key were + /// deleted. + #[allow(dead_code)] + fn prune_dupsort_key_entries( + &self, + key: T::Key, + limiter: &mut PruneLimiter, + ) -> Result<(usize, bool), DatabaseError> { + let mut cursor = self.cursor_dup_write::()?; + let mut entry = cursor.seek_exact(key)?; + + let mut deleted_entries = 0; + + while entry.is_some() && !limiter.is_limit_reached() { + cursor.delete_current()?; + limiter.increment_deleted_entries_count(); + deleted_entries += 1; + entry = cursor.next_dup()?; + } + + // an entry remaining means the loop stopped because a limit was reached + let done = entry.is_none(); + if !done { + debug!( + target: "providers::db", + ?limiter, + deleted_entries_limit = %limiter.is_deleted_entries_limit_reached(), + time_limit = %limiter.is_time_limit_reached(), + table = %T::NAME, + "Pruning limit reached" + ); + } + + Ok((deleted_entries, done)) + } } impl DbTxPruneExt for Tx where Tx: DbTxMut + DbTx {} @@ -201,14 +239,18 @@ impl DbTxPruneExt for Tx where Tx: DbTxMut + DbTx {} mod tests { use super::DbTxPruneExt; use crate::PruneLimiter; - use reth_db_api::tables; - use reth_primitives_traits::SignerRecoverable; + use alloy_primitives::{B256, U256}; + use reth_db_api::{tables, transaction::DbTxMut}; + use reth_primitives_traits::{SignerRecoverable, StorageEntry}; use reth_provider::{DBProvider, DatabaseProviderFactory}; use reth_stages::test_utils::{StorageKind, TestStageDB}; use reth_testing_utils::generators::{self, random_block_range, BlockRangeParams}; - use std::sync::{ - atomic::{AtomicUsize, Ordering}, - Arc, + use std::{ + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::Duration, }; struct CountingIter { @@ -244,6 +286,51 @@ mod tests { } } + fn storage_entry(slot_byte: u8) -> StorageEntry { + StorageEntry { key: B256::with_last_byte(slot_byte), value: U256::from(slot_byte) } + } + + fn insert_hashed_storages(db: &TestStageDB, rows: impl IntoIterator) { + let provider = db.factory.database_provider_rw().unwrap(); + for (address_byte, slot_byte) in rows { + provider + .tx_ref() + .put::( + B256::with_last_byte(address_byte), + storage_entry(slot_byte), + ) + .expect("insert hashed storage"); + } + provider.commit().expect("commit"); + } + + fn hashed_storage_slots(db: &TestStageDB, address_byte: u8) -> Vec { + db.table::() + .unwrap() + .into_iter() + .filter_map(|(key, entry)| { + (key == B256::with_last_byte(address_byte)).then_some(entry.key) + }) + .collect() + } + + fn prune_hashed_storage_key( + db: &TestStageDB, + address_byte: u8, + limiter: &mut PruneLimiter, + ) -> (usize, bool) { + let provider = db.factory.database_provider_rw().unwrap(); + let result = provider + .tx_ref() + .prune_dupsort_key_entries::( + B256::with_last_byte(address_byte), + limiter, + ) + .expect("prune hashed storages"); + provider.commit().expect("commit"); + result + } + #[test] fn prune_table_with_iterator_early_exit_does_not_overconsume() { let db = TestStageDB::default(); @@ -350,4 +437,97 @@ mod tests { provider.commit().expect("commit"); assert_eq!(db.table::().unwrap().len(), 0); } + + #[test] + fn prune_dupsort_key_entries_resumes_with_deleted_entries_budget() { + let db = TestStageDB::default(); + insert_hashed_storages(&db, (0..5).map(|slot| (1, slot))); + insert_hashed_storages(&db, (10..12).map(|slot| (2, slot))); + + let mut total_deleted = 0; + + let mut limiter = PruneLimiter::default().set_deleted_entries_limit(2); + let (deleted, done) = prune_hashed_storage_key(&db, 1, &mut limiter); + total_deleted += deleted; + assert_eq!((deleted, done), (2, false)); + assert_eq!( + hashed_storage_slots(&db, 1), + vec![B256::with_last_byte(2), B256::with_last_byte(3), B256::with_last_byte(4),] + ); + assert_eq!( + hashed_storage_slots(&db, 2), + vec![B256::with_last_byte(10), B256::with_last_byte(11)] + ); + + let mut limiter = PruneLimiter::default().set_deleted_entries_limit(2); + let (deleted, done) = prune_hashed_storage_key(&db, 1, &mut limiter); + total_deleted += deleted; + assert_eq!((deleted, done), (2, false)); + assert_eq!(hashed_storage_slots(&db, 1), vec![B256::with_last_byte(4)]); + assert_eq!( + hashed_storage_slots(&db, 2), + vec![B256::with_last_byte(10), B256::with_last_byte(11)] + ); + + let mut limiter = PruneLimiter::default().set_deleted_entries_limit(2); + let (deleted, done) = prune_hashed_storage_key(&db, 1, &mut limiter); + total_deleted += deleted; + assert_eq!((deleted, done), (1, true)); + assert!(hashed_storage_slots(&db, 1).is_empty()); + assert_eq!( + hashed_storage_slots(&db, 2), + vec![B256::with_last_byte(10), B256::with_last_byte(11)] + ); + assert_eq!(total_deleted, 5); + } + + #[test] + fn prune_dupsort_key_entries_stops_on_time_limit() { + let db = TestStageDB::default(); + insert_hashed_storages(&db, (0..3).map(|slot| (1, slot))); + + let mut limiter = PruneLimiter::default().set_time_limit(Duration::from_nanos(1)); + std::thread::sleep(Duration::from_millis(1)); + + let (deleted, done) = prune_hashed_storage_key(&db, 1, &mut limiter); + + assert_eq!((deleted, done), (0, false)); + assert!(limiter.is_time_limit_reached()); + assert_eq!( + hashed_storage_slots(&db, 1), + vec![B256::with_last_byte(0), B256::with_last_byte(1), B256::with_last_byte(2)] + ); + } + + #[test] + fn prune_dupsort_key_entries_missing_key_is_done() { + let db = TestStageDB::default(); + insert_hashed_storages(&db, (0..3).map(|slot| (1, slot))); + + let mut limiter = PruneLimiter::default().set_deleted_entries_limit(2); + let result = prune_hashed_storage_key(&db, 2, &mut limiter); + + assert_eq!(result, (0, true)); + assert_eq!( + hashed_storage_slots(&db, 1), + vec![B256::with_last_byte(0), B256::with_last_byte(1), B256::with_last_byte(2)] + ); + } + + #[test] + fn prune_dupsort_key_entries_leaves_other_keys_untouched() { + let db = TestStageDB::default(); + insert_hashed_storages(&db, (0..3).map(|slot| (1, slot))); + insert_hashed_storages(&db, (10..13).map(|slot| (2, slot))); + + let mut limiter = PruneLimiter::default(); + let result = prune_hashed_storage_key(&db, 1, &mut limiter); + + assert_eq!(result, (3, true)); + assert!(hashed_storage_slots(&db, 1).is_empty()); + assert_eq!( + hashed_storage_slots(&db, 2), + vec![B256::with_last_byte(10), B256::with_last_byte(11), B256::with_last_byte(12),] + ); + } } diff --git a/crates/prune/prune/src/segments/user/transaction_lookup.rs b/crates/prune/prune/src/segments/user/transaction_lookup.rs index 8ff58834744..8a85655e463 100644 --- a/crates/prune/prune/src/segments/user/transaction_lookup.rs +++ b/crates/prune/prune/src/segments/user/transaction_lookup.rs @@ -3,13 +3,12 @@ use crate::{ segments::{PruneInput, Segment, SegmentOutput}, PrunerError, }; -use alloy_consensus::transaction::TxHashRef; -use rayon::prelude::*; -use reth_db_api::{tables, transaction::DbTxMut}; -use reth_primitives_traits::SignedTransaction; +use alloy_primitives::TxNumber; +use reth_db_api::{table::Value, tables, transaction::DbTxMut}; +use reth_primitives_traits::{NodePrimitives, SignedTransaction}; use reth_provider::{ BlockReader, DBProvider, PruneCheckpointReader, RocksDBProviderFactory, - StaticFileProviderFactory, + StaticFileProviderFactory, TransactionsProviderExt, }; use reth_prune_types::{ PruneCheckpoint, PruneMode, PruneProgress, PrunePurpose, PruneSegment, SegmentOutputCheckpoint, @@ -34,9 +33,11 @@ where Provider: DBProvider + BlockReader + PruneCheckpointReader - + StaticFileProviderFactory + StorageSettingsCache - + RocksDBProviderFactory, + + RocksDBProviderFactory + + StaticFileProviderFactory< + Primitives: NodePrimitives, + >, { fn segment(&self) -> PruneSegment { PruneSegment::TransactionLookup @@ -132,11 +133,16 @@ where .unwrap(); let tx_range_end = *tx_range.end(); - // Retrieve transactions in the range and collect their hashes in parallel. let mut hashes = provider - .transactions_by_tx_range(tx_range.clone())? - .into_par_iter() - .map(|transaction| *transaction.tx_hash()) + .static_file_provider() + .transaction_hashes_by_range( + *tx_range.start().. + tx_range_end + .checked_add(1) + .ok_or(PrunerError::InconsistentData("Transaction range end overflow"))?, + )? + .into_iter() + .map(|(hash, _)| hash) .collect::>(); // Sort hashes to enable efficient cursor traversal through the TransactionHashNumbers @@ -200,14 +206,16 @@ impl TransactionLookup { &self, provider: &Provider, input: PruneInput, - start: alloy_primitives::TxNumber, - end: alloy_primitives::TxNumber, + start: TxNumber, + end: TxNumber, ) -> Result where Provider: DBProvider + BlockReader - + StaticFileProviderFactory - + RocksDBProviderFactory, + + RocksDBProviderFactory + + StaticFileProviderFactory< + Primitives: NodePrimitives, + >, { // For PruneMode::Full, clear the entire RocksDB table in one operation if self.mode.is_full() { @@ -236,12 +244,12 @@ impl TransactionLookup { .map_or(end, |limited| limited.min(end)); let tx_range = start..=tx_range_end; - // Retrieve transactions in the range and collect their hashes in parallel. - let hashes: Vec<_> = provider - .transactions_by_tx_range(tx_range.clone())? - .into_par_iter() - .map(|transaction| *transaction.tx_hash()) - .collect(); + let hashes = provider.static_file_provider().transaction_hashes_by_range( + *tx_range.start().. + tx_range_end + .checked_add(1) + .ok_or(PrunerError::InconsistentData("Transaction range end overflow"))?, + )?; // Number of transactions retrieved from the database should match the tx range count let tx_count = tx_range.count(); @@ -256,7 +264,7 @@ impl TransactionLookup { // Delete transaction hash -> number mappings from RocksDB let mut deleted = 0usize; provider.with_rocksdb_batch(|mut batch| { - for hash in &hashes { + for (hash, _) in &hashes { if limiter.is_limit_reached() { break; } diff --git a/crates/rpc/rpc-builder/src/lib.rs b/crates/rpc/rpc-builder/src/lib.rs index c3dd3392168..07051932da8 100644 --- a/crates/rpc/rpc-builder/src/lib.rs +++ b/crates/rpc/rpc-builder/src/lib.rs @@ -1005,6 +1005,7 @@ where RethRpcModule::Txpool => TxPoolApi::new( self.eth.api.pool().clone(), dyn_clone::clone(self.eth.api.converter()), + self.eth.api.credible_config(), ) .into_rpc() .into(), diff --git a/crates/rpc/rpc-eth-api/src/core.rs b/crates/rpc/rpc-eth-api/src/core.rs index bf4a7d807ff..7f55f338339 100644 --- a/crates/rpc/rpc-eth-api/src/core.rs +++ b/crates/rpc/rpc-eth-api/src/core.rs @@ -2,7 +2,7 @@ //! the `eth_` namespace. use crate::{ helpers::{EthApiSpec, EthBlocks, EthCall, EthFees, EthState, EthTransactions, FullEthApi}, - FromEthApiError, RpcBlock, RpcHeader, RpcReceipt, RpcTransaction, + RpcBlock, RpcHeader, RpcReceipt, RpcTransaction, }; use alloy_dyn_abi::TypedData; use alloy_eips::{eip2930::AccessListResult, BlockId, BlockNumberOrTag}; @@ -18,11 +18,8 @@ use alloy_serde::JsonStorageKey; use jsonrpsee::{core::RpcResult, proc_macros::rpc}; use reth_primitives_traits::TxTy; use reth_rpc_convert::RpcTxReq; -use reth_rpc_eth_types::{ - credible::credible_block_number_override, EthApiError, EthCapabilities, FillTransaction, -}; +use reth_rpc_eth_types::{EthApiError, EthCapabilities, FillTransaction}; use reth_rpc_server_types::{result::internal_rpc_err, ToRpcResult}; -use reth_storage_api::BlockIdReader; use serde_json::Value; use std::collections::HashMap; use tracing::trace; @@ -765,10 +762,13 @@ where block_overrides: Option>, ) -> RpcResult { trace!(target: "rpc::eth", ?request, ?block_number, ?state_overrides, ?block_overrides, "Serving eth_call"); - let at = block_number.unwrap_or_default(); - let overrides = - credible_call_overrides(self, at, EvmOverrides::new(state_overrides, block_overrides))?; - Ok(EthCall::call(self, request, block_number, overrides).await?) + Ok(EthCall::call( + self, + request, + block_number, + EvmOverrides::new(state_overrides, block_overrides), + ) + .await?) } /// Handler for: `eth_fillTransaction` @@ -799,11 +799,7 @@ where state_override: Option, ) -> RpcResult { trace!(target: "rpc::eth", ?request, ?block_number, ?state_override, "Serving eth_createAccessList"); - let at = block_number.unwrap_or_default(); - // createAccessList takes no block overrides, so the credible marker derives from the - // request's block tag. The override is a registry state-diff, so it rides on `state`. - let overrides = credible_call_overrides(self, at, EvmOverrides::new(state_override, None))?; - Ok(EthCall::create_access_list_at(self, request, block_number, overrides.state).await?) + Ok(EthCall::create_access_list_at(self, request, block_number, state_override).await?) } /// Handler for: `eth_estimateGas` @@ -815,10 +811,13 @@ where block_overrides: Option>, ) -> RpcResult { trace!(target: "rpc::eth", ?request, ?block_number, "Serving eth_estimateGas"); - let at = block_number.unwrap_or_default(); - let overrides = - credible_call_overrides(self, at, EvmOverrides::new(state_override, block_overrides))?; - Ok(EthCall::estimate_gas_at(self, request, at, overrides).await?) + Ok(EthCall::estimate_gas_at( + self, + request, + block_number.unwrap_or_default(), + EvmOverrides::new(state_override, block_overrides), + ) + .await?) } /// Handler for: `eth_gasPrice` @@ -1008,47 +1007,3 @@ where Ok(self.get_raw_block_access_list(block).await?) } } - -/// Applies the credible block override to `overrides` for a call simulated against `at`. -/// -/// A no-op if no credible registry is configured, skipping block number resolution entirely. -fn credible_call_overrides( - eth_api: &T, - at: BlockId, - overrides: EvmOverrides, -) -> Result { - let credible_config = eth_api.credible_config(); - if credible_config.registry_address.is_none() { - return Ok(overrides); - } - - let credible_block_number = - resolve_credible_block_number(eth_api, at, overrides.block.as_deref())?; - Ok(credible_config.apply_credible_block_override(credible_block_number, overrides)) -} - -/// Resolves the block number the EVM will actually see, for deriving the credible block -/// override. Prefers `block_overrides.number` when set; otherwise resolves `at`, erroring -/// instead of silently defaulting to block `0` if it can't be resolved. -/// -/// The provider only resolves committed blocks, so `at.is_pending()` can't be looked up -/// directly; a pending call/estimate executes as if mined on top of the current chain tip, -/// so that case resolves the latest block instead and adds one. -fn resolve_credible_block_number( - eth_api: &T, - at: BlockId, - block_overrides: Option<&BlockOverrides>, -) -> Result { - if let Some(number) = credible_block_number_override(block_overrides) { - return Ok(number); - } - - let is_pending = at.is_pending(); - let number = eth_api - .provider() - .block_number_for_id(if is_pending { BlockId::latest() } else { at }) - .map_err(T::Error::from_eth_err)? - .ok_or_else(|| T::Error::from_eth_err(EthApiError::HeaderNotFound(at)))?; - - Ok(if is_pending { number.saturating_add(1) } else { number }) -} diff --git a/crates/rpc/rpc-eth-api/src/helpers/call.rs b/crates/rpc/rpc-eth-api/src/helpers/call.rs index b6d7c85b04c..56703f37e64 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/call.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/call.rs @@ -136,14 +136,9 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA // (block 0) slot. let state_overrides = match credible_block_number_override(block_overrides.as_ref()) { - Some(target_number) => { - this.credible_config() - .apply_credible_block_override( - target_number, - EvmOverrides::state(state_overrides), - ) - .state - } + Some(target_number) => this + .credible_config() + .apply_credible_block_override(target_number, state_overrides), None => state_overrides, }; @@ -376,10 +371,6 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA } } - // Block number each bundle executes at (its own `block_override.number`, or the - // target block) — this drives the credible marker slot, just like a single call. - let base_block_number = evm_env.block_env.number().saturating_to::(); - // transact all bundles for (bundle_index, bundle) in bundles.into_iter().enumerate() { let Bundle { transactions, block_override } = bundle; @@ -390,26 +381,13 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA let mut bundle_results = Vec::with_capacity(transactions.len()); let block_overrides = block_override.map(Box::new); - let bundle_block_number = - credible_block_number_override(block_overrides.as_deref()) - .unwrap_or(base_block_number); // transact all transactions in the bundle for (tx_index, tx) in transactions.into_iter().enumerate() { - // State overrides apply only on the first tx of each bundle; inject the - // credible marker per bundle for that bundle's block (no-op with no - // registry). - let state = if tx_index == 0 { - this.credible_config() - .apply_credible_block_override( - bundle_block_number, - EvmOverrides::state(state_override.take()), - ) - .state - } else { - None - }; - let overrides = EvmOverrides::new(state, block_overrides.clone()); + // Apply overrides, state overrides are only applied for the first tx in the + // request + let overrides = + EvmOverrides::new(state_override.take(), block_overrides.clone()); let (current_evm_env, prepared_tx) = this .prepare_call_env(evm_env.clone(), tx, &mut db, overrides) @@ -495,6 +473,11 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA let state = this.state_at_block_id(at).await?; let mut db = State::builder().with_database(StateProviderDatabase::new(state)).build(); + // Inject the credible Layer marker for the block the EVM will execute at. No-op unless + // a registry is configured. + let state_override = this + .credible_config() + .apply_credible_block_override(evm_env.block_env.number(), state_override); if let Some(state_overrides) = state_override { apply_state_overrides(state_overrides, &mut db) .map_err(Self::Error::from_eth_err)?; @@ -928,7 +911,13 @@ pub trait Call: if let Some(block_overrides) = overrides.block { apply_block_overrides(*block_overrides, db, evm_env.block_env.inner_mut()); } - if let Some(state_overrides) = overrides.state { + // Inject the credible Layer marker for the block the EVM will execute at (after any block + // override is applied), so the marker slot and the execution resolve the same block. No-op + // unless a registry is configured. + let state_overrides = self + .credible_config() + .apply_credible_block_override(evm_env.block_env.number(), overrides.state); + if let Some(state_overrides) = state_overrides { apply_state_overrides(state_overrides, db) .map_err(EthApiError::from_state_overrides_err)?; } diff --git a/crates/rpc/rpc-eth-api/src/helpers/estimate.rs b/crates/rpc/rpc-eth-api/src/helpers/estimate.rs index 0990ab2c52a..4c572ea2217 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/estimate.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/estimate.rs @@ -88,8 +88,13 @@ pub trait EstimateCall: Call { apply_block_overrides(*block_overrides, &mut db, evm_env.block_env.inner_mut()); } - // Apply any state overrides if specified. - if let Some(state_override) = overrides.state { + // Inject the Credible Layer marker for the block the EVM will execute at (after any block + // override is applied), so the marker slot and the execution resolve the same block. No-op + // unless a registry is configured. + let state_override = self + .credible_config() + .apply_credible_block_override(evm_env.block_env.number(), overrides.state); + if let Some(state_override) = state_override { apply_state_overrides(state_override, &mut db).map_err(Self::Error::from_eth_err)?; } diff --git a/crates/rpc/rpc-eth-api/src/helpers/mod.rs b/crates/rpc/rpc-eth-api/src/helpers/mod.rs index e04e5ec5bc5..7641715535e 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/mod.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/mod.rs @@ -26,6 +26,7 @@ pub mod receipt; pub mod signer; pub mod spec; pub mod state; +pub mod subscriptions; pub mod trace; pub mod transaction; @@ -39,6 +40,7 @@ pub use receipt::LoadReceipt; pub use signer::EthSigner; pub use spec::EthApiSpec; pub use state::{EthState, LoadState}; +pub use subscriptions::EthSubscriptions; pub use trace::Trace; pub use transaction::{EthTransactions, LoadTransaction}; @@ -63,6 +65,7 @@ pub trait FullEthApi: + EthState + EthCall + EthFees + + EthSubscriptions + Trace + LoadReceipt + GetBlockAccessList @@ -77,6 +80,7 @@ impl FullEthApi for T where + EthState + EthCall + EthFees + + EthSubscriptions + Trace + LoadReceipt + GetBlockAccessList diff --git a/crates/rpc/rpc-eth-api/src/helpers/subscriptions.rs b/crates/rpc/rpc-eth-api/src/helpers/subscriptions.rs new file mode 100644 index 00000000000..97cf9ecb486 --- /dev/null +++ b/crates/rpc/rpc-eth-api/src/helpers/subscriptions.rs @@ -0,0 +1,133 @@ +//! Streams subscriptions providers for `eth_subscribe`. + +use crate::{EthApiTypes, RpcConvert, RpcNodeCore, RpcReceipt}; +use alloy_consensus::{transaction::TxHashRef, BlockHeader, TxReceipt}; +use alloy_rpc_types_eth::{pubsub::TransactionReceiptsParams, Filter, Log}; +use futures::StreamExt; +use reth_chain_state::CanonStateSubscriptions; +use reth_primitives_traits::TransactionMeta; +use reth_rpc_convert::{transaction::ConvertReceiptInput, RpcHeader}; +use reth_rpc_eth_types::logs_utils; +use tracing::error; + +/// Provides streams subscriptions for `eth_subscribe`. +/// +/// Override the default methods to inject additional data sources (e.g. flashblocks). +pub trait EthSubscriptions: + RpcNodeCore + EthApiTypes> +{ + /// Returns a stream that yields matching logs from canonical chain updates. + fn log_stream(&self, filter: Filter) -> impl futures::Stream + Send + Unpin { + self.provider() + .canonical_state_stream() + .map(move |canon_state| canon_state.block_receipts()) + .flat_map(futures::stream::iter) + .flat_map(move |(block_receipts, removed)| { + let all_logs = logs_utils::matching_block_logs_with_tx_hashes( + &filter, + block_receipts.block, + block_receipts.timestamp, + block_receipts.tx_receipts.iter().map(|(tx, receipt)| (*tx, receipt)), + removed, + ); + futures::stream::iter(all_logs) + }) + } + + /// Returns a stream that yields new block headers from canonical chain updates. + fn header_stream( + &self, + ) -> impl futures::Stream> + Send + Unpin { + let converter = self.converter(); + self.provider().canonical_state_stream().flat_map(move |new_chain| { + let headers = new_chain + .committed() + .blocks_iter() + .filter_map(|block| { + match converter.convert_header(block.clone_sealed_header(), block.rlp_length()) + { + Ok(header) => Some(header), + Err(err) => { + error!(target = "rpc", %err, "Failed to convert header"); + None + } + } + }) + .collect::>(); + futures::stream::iter(headers) + }) + } + + /// Returns a stream that yields matching transaction receipts from canonical chain updates. + fn transaction_receipts_stream( + &self, + filter: TransactionReceiptsParams, + ) -> impl futures::Stream>> + Send + Unpin { + let converter = self.converter(); + self.provider().canonical_state_stream().flat_map(move |new_chain| { + let results: Vec<_> = new_chain + .committed() + .blocks_and_receipts() + .filter_map(|(block, receipts)| { + let block_hash = block.hash(); + let block_number = block.number(); + let base_fee = block.base_fee_per_gas(); + let excess_blob_gas = block.excess_blob_gas(); + let timestamp = block.timestamp(); + + let mut gas_used: u64 = 0; + let mut next_log_index: usize = 0; + + let inputs: Vec<_> = block + .transactions_recovered() + .zip(receipts.iter()) + .enumerate() + .filter_map(|(idx, (tx, receipt))| { + let gas_used_before = gas_used; + let next_log_index_before = next_log_index; + let cumulative_gas_used = receipt.cumulative_gas_used(); + + gas_used = cumulative_gas_used; + next_log_index += receipt.logs().len(); + + let matches = match &filter.transaction_hashes { + Some(hashes) if !hashes.is_empty() => hashes.contains(tx.tx_hash()), + _ => true, + }; + + matches.then(|| ConvertReceiptInput { + tx, + gas_used: cumulative_gas_used - gas_used_before, + next_log_index: next_log_index_before, + meta: TransactionMeta { + tx_hash: *tx.tx_hash(), + index: idx as u64, + block_hash, + block_number, + base_fee, + excess_blob_gas, + timestamp, + }, + receipt: receipt.clone(), + }) + }) + .collect(); + + if inputs.is_empty() { + return None; + } + + match converter.convert_receipts_with_block(inputs, block.sealed_block()) { + Ok(rpc_receipts) => Some(rpc_receipts), + Err(err) => { + error!(target = "rpc", %err, "Failed to convert receipts"); + None + } + } + }) + .collect(); + + futures::stream::iter(results) + }) + } +} diff --git a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs index afe4dbdff80..3e757d5c1cb 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs @@ -229,18 +229,19 @@ pub trait EthTransactions: LoadTransaction { hash: B256, ) -> impl Future, Self::Error>> + Send { async move { - // Note: this is mostly used to fetch pooled transactions so we check the pool first. - // Suppress the pooled copy of a retained-private tx so its raw bytes don't leak before - // inclusion; once mined, the provider lookup below still returns it. - let private_in_pool = self.credible_config().hide_private_pool_txs() && - self.pool().get(&hash).is_some_and(|tx| tx.origin.is_private()); - if let Some(tx) = self - .pool() - .get_pooled_transaction_element(hash) - .filter(|_| !private_in_pool) - .map(|tx| tx.encoded_2718().into()) - { - return Ok(Some(tx)) + // Check the pool first. Gate the raw-byte fetch on the entry's own origin so a + // retained-private tx can't leak its bytes before inclusion. + if let Some(pool_tx) = self.pool().get(&hash) { + let hide_private = + self.credible_config().hide_private_pool_txs() && pool_tx.origin.is_private(); + if !hide_private && + let Some(tx) = self + .pool() + .get_pooled_transaction_element(hash) + .map(|tx| tx.encoded_2718().into()) + { + return Ok(Some(tx)) + } } self.spawn_blocking_io(move |ref this| { @@ -746,15 +747,9 @@ pub trait LoadTransaction: SpawnBlocking + FullEthApiTypes + RpcNodeCoreExt { } // tx not found on disk, check pool - if let Some(tx) = self.pool().get(&hash) { - // Suppress the pooled copy of a retained-private tx so it doesn't leak before - // inclusion; once mined, the on-disk lookup above returns it. - let hide_private = - self.credible_config().hide_private_pool_txs() && tx.origin.is_private(); - if !hide_private { - let tx = tx.transaction.clone_into_consensus(); - return Ok(Some(TransactionSource::Pool(tx.into()))); - } + if let Some(tx) = self.pool().get(&hash).map(|tx| tx.transaction.clone_into_consensus()) + { + return Ok(Some(TransactionSource::Pool(tx.into()))); } Ok(None) diff --git a/crates/rpc/rpc-eth-types/src/credible.rs b/crates/rpc/rpc-eth-types/src/credible.rs index 945608f9ea5..f622446ba7a 100644 --- a/crates/rpc/rpc-eth-types/src/credible.rs +++ b/crates/rpc/rpc-eth-types/src/credible.rs @@ -1,14 +1,14 @@ //! Credible Layer RPC integration hooks. use alloy_primitives::{keccak256, Address, B256, U256}; -use alloy_rpc_types_eth::{state::EvmOverrides, BlockOverrides}; +use alloy_rpc_types_eth::{state::StateOverride, BlockOverrides}; use alloy_sol_types::SolValue; use reth_transaction_pool::TransactionOrigin; use serde::{Deserialize, Serialize}; /// Base storage slot of `CredibleRegistry`'s `_credibleBlocks` mapping, per its current /// storage layout (`forge inspect CredibleRegistry storage-layout`). -const DEFAULT_CREDIBLE_BLOCKS_BASE_SLOT: u64 = 1; +const CREDIBLE_BLOCKS_BASE_SLOT: u64 = 1; /// Credible Layer behavior toggles for the `eth` RPC namespace. #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] @@ -23,24 +23,25 @@ pub struct CredibleRpcConfig { } impl CredibleRpcConfig { - /// Applies the credible block override for a call simulated against `block_number` to - /// call-like EVM overrides. + /// Merges the credible block marker for a call simulated against `block_number` into the + /// call's state overrides. /// - /// A no-op if no registry is configured. Registry-backed resolution is a pure hash - /// computation — no call into the registry contract, no EVM execution, no async lookup. + /// A no-op returning `state` unchanged if no registry is configured. Registry-backed + /// resolution is a pure hash computation — no call into the registry contract, no EVM + /// execution, no async lookup. pub fn apply_credible_block_override( &self, - block_number: u64, - overrides: EvmOverrides, - ) -> EvmOverrides { - let Some(registry) = self.registry_address else { return overrides }; + block_number: U256, + state: Option, + ) -> Option { + let Some(registry) = self.registry_address else { return state }; let block_override = CredibleBlockOverride { address: registry, - slot: credible_block_slot(block_number, U256::from(DEFAULT_CREDIBLE_BLOCKS_BASE_SLOT)), + slot: credible_block_slot(block_number, U256::from(CREDIBLE_BLOCKS_BASE_SLOT)), value: B256::with_last_byte(1), }; - block_override.apply_to(overrides) + block_override.apply_to(state) } /// Returns the origin a successfully forwarded transaction should be retained under. @@ -57,7 +58,10 @@ impl CredibleRpcConfig { /// Whether retained-private transactions must be hidden from public pool-reading RPCs. /// - /// Enabled exactly when the node retains forwarded transactions as private. + /// Enabled exactly when the node retains forwarded transactions as private. Callers hide every + /// `Private`-origin pool transaction, which is exact here because + /// [`Self::resolve_forwarded_origin`] is the only path that assigns `Private` origin on + /// this node. pub const fn hide_private_pool_txs(&self) -> bool { self.retain_forwarded_txs_as_private } @@ -65,15 +69,18 @@ impl CredibleRpcConfig { /// Computes the storage slot of `_credibleBlocks[block_number]`, matching Solidity's mapping /// slot derivation: `keccak256(abi.encode(block_number, base_slot))`. -fn credible_block_slot(block_number: u64, base_slot: U256) -> B256 { - keccak256((U256::from(block_number), base_slot).abi_encode()) +fn credible_block_slot(block_number: U256, base_slot: U256) -> B256 { + keccak256((block_number, base_slot).abi_encode()) } /// Extracts the block number the EVM will use from `block_overrides.number`, if the caller set /// one. This must take priority over resolving the request's block tag, since the override is /// applied after tag resolution. -pub fn credible_block_number_override(block_overrides: Option<&BlockOverrides>) -> Option { - block_overrides.and_then(|overrides| overrides.number).map(|number| number.saturating_to()) +/// +/// Kept as `U256` (the EVM's native block-number width) so the marker slot matches the number the +/// registry lookup sees, with no truncation. +pub fn credible_block_number_override(block_overrides: Option<&BlockOverrides>) -> Option { + block_overrides.and_then(|overrides| overrides.number) } /// The storage override that makes `_credibleBlocks[blockNumber]` read as `true` for one @@ -89,25 +96,25 @@ struct CredibleBlockOverride { } impl CredibleBlockOverride { - /// Merges this override into existing EVM overrides. - fn apply_to(self, mut overrides: EvmOverrides) -> EvmOverrides { - let state = overrides.state.get_or_insert_with(Default::default); + /// Merges this override into the call's state overrides. + fn apply_to(self, state: Option) -> Option { + let mut state = state.unwrap_or_default(); let account = state.entry(self.address).or_default(); - if let Some(state) = account.state.as_mut() { - state.insert(self.slot, self.value); + if let Some(slots) = account.state.as_mut() { + slots.insert(self.slot, self.value); } else { account.state_diff.get_or_insert_with(Default::default).insert(self.slot, self.value); } - overrides + Some(state) } } #[cfg(test)] mod tests { use super::*; - use alloy_rpc_types_eth::state::{AccountOverride, StateOverride}; + use alloy_rpc_types_eth::state::AccountOverride; #[test] fn credible_block_override_adds_state_diff() { @@ -116,8 +123,7 @@ mod tests { let value = B256::repeat_byte(0x33); let block_override = CredibleBlockOverride { address, slot, value }; - let overrides = block_override.apply_to(EvmOverrides::default()); - let state = overrides.state.expect("override should add state overrides"); + let state = block_override.apply_to(None).expect("override should add state overrides"); let account = state.get(&address).expect("override account should be present"); assert_eq!(account.state, None); @@ -143,8 +149,9 @@ mod tests { let block_override = CredibleBlockOverride { address, slot: override_slot, value: override_value }; - let overrides = block_override.apply_to(EvmOverrides::state(Some(state_override))); - let state = overrides.state.expect("override should keep state overrides"); + let state = block_override + .apply_to(Some(state_override)) + .expect("override should keep state overrides"); let account = state.get(&address).expect("override account should be present"); let state = account.state.as_ref().expect("full state override should be preserved"); @@ -159,14 +166,14 @@ mod tests { // `cast index uint256 12345 1`. let expected: B256 = "0x24689f9b6ba9bad3c49d2b1293bf33fa38d0c418c093b2b4bc23f5d18e11355e".parse().unwrap(); - assert_eq!(credible_block_slot(12345, U256::from(1)), expected); + assert_eq!(credible_block_slot(U256::from(12345), U256::from(1)), expected); } #[test] fn no_override_without_registry() { let config = CredibleRpcConfig::default(); - let overrides = config.apply_credible_block_override(100, EvmOverrides::default()); - assert_eq!(overrides.state, None); + let state = config.apply_credible_block_override(U256::from(100), None); + assert_eq!(state, None); } #[test] @@ -174,8 +181,9 @@ mod tests { let registry = Address::repeat_byte(0xaa); let config = CredibleRpcConfig { registry_address: Some(registry), ..Default::default() }; - let overrides = config.apply_credible_block_override(12345, EvmOverrides::default()); - let state = overrides.state.expect("registry override should add state overrides"); + let state = config + .apply_credible_block_override(U256::from(12345), None) + .expect("registry override should add state overrides"); let account = state.get(®istry).expect("registry account should be present"); let expected_slot: B256 = "0x24689f9b6ba9bad3c49d2b1293bf33fa38d0c418c093b2b4bc23f5d18e11355e".parse().unwrap(); @@ -188,7 +196,7 @@ mod tests { #[test] fn block_overrides_number_takes_priority() { let overrides = BlockOverrides { number: Some(U256::from(42)), ..Default::default() }; - assert_eq!(credible_block_number_override(Some(&overrides)), Some(42)); + assert_eq!(credible_block_number_override(Some(&overrides)), Some(U256::from(42))); } #[test] @@ -205,16 +213,17 @@ mod tests { // target block, so the marker is derived for the block that bundle executes at. let registry = Address::repeat_byte(0xaa); let config = CredibleRpcConfig { registry_address: Some(registry), ..Default::default() }; - let base_block = 100; + let base_block = U256::from(100); let bundle = BlockOverrides { number: Some(U256::from(250)), ..Default::default() }; let number = credible_block_number_override(Some(&bundle)).unwrap_or(base_block); - assert_eq!(number, 250); + assert_eq!(number, U256::from(250)); - let overrides = config.apply_credible_block_override(number, EvmOverrides::default()); - let state = overrides.state.expect("registry override should add state overrides"); + let state = config + .apply_credible_block_override(number, None) + .expect("registry override should add state overrides"); let account = state.get(®istry).expect("registry account should be present"); - let expected_slot = credible_block_slot(250, U256::from(1)); + let expected_slot = credible_block_slot(U256::from(250), U256::from(1)); assert_eq!( account.state_diff.as_ref().and_then(|diff| diff.get(&expected_slot)), Some(&B256::with_last_byte(1)) @@ -224,7 +233,10 @@ mod tests { #[test] fn call_many_bundle_marker_falls_back_to_base_block() { // Without a bundle block override, the marker derives for the target/base block. - assert_eq!(credible_block_number_override(None).unwrap_or(100), 100); + assert_eq!( + credible_block_number_override(None).unwrap_or_else(|| U256::from(100)), + U256::from(100) + ); } #[test] diff --git a/crates/rpc/rpc-testing-util/Cargo.toml b/crates/rpc/rpc-testing-util/Cargo.toml deleted file mode 100644 index 2d074ef2368..00000000000 --- a/crates/rpc/rpc-testing-util/Cargo.toml +++ /dev/null @@ -1,38 +0,0 @@ -[package] -name = "reth-rpc-api-testing-util" -version.workspace = true -edition.workspace = true -rust-version.workspace = true -license.workspace = true -homepage.workspace = true -repository.workspace = true -description = "Reth RPC testing helpers" - -[lints] -workspace = true - -[dependencies] -# reth -reth-ethereum-primitives.workspace = true -reth-rpc-api = { workspace = true, features = ["client"] } - -# ethereum -alloy-primitives.workspace = true -alloy-rpc-types-eth.workspace = true -alloy-rpc-types-trace.workspace = true -alloy-eips.workspace = true - -# async -futures.workspace = true - -# misc -jsonrpsee = { workspace = true, features = ["client", "async-client"] } -serde_json.workspace = true - -# assertions -similar-asserts.workspace = true - -[dev-dependencies] -tokio = { workspace = true, features = ["rt-multi-thread", "macros", "rt"] } -reth-rpc-eth-api.workspace = true -jsonrpsee-http-client.workspace = true diff --git a/crates/rpc/rpc-testing-util/assets/noop-tracer.js b/crates/rpc/rpc-testing-util/assets/noop-tracer.js deleted file mode 100644 index 6906e3ee7b8..00000000000 --- a/crates/rpc/rpc-testing-util/assets/noop-tracer.js +++ /dev/null @@ -1,8 +0,0 @@ -{ - // required function that is invoked when a step fails - fault: function(log, db) { }, - // required function that returns the result of the tracer: empty object - result: function(ctx, db) { return {}; }, - // optional function that is invoked for every opcode - step: function(log, db) { } -} \ No newline at end of file diff --git a/crates/rpc/rpc-testing-util/assets/tracer-template.js b/crates/rpc/rpc-testing-util/assets/tracer-template.js deleted file mode 100644 index 29e033dc735..00000000000 --- a/crates/rpc/rpc-testing-util/assets/tracer-template.js +++ /dev/null @@ -1,24 +0,0 @@ -{ - // called once - setup: function(cfg) { - // - }, - // required function that is invoked when a step fails - fault: function(log, db) { - // - }, - // required function that returns the result of the tracer: empty object - result: function(ctx, db) { - // - }, - // optional function that is invoked for every opcode - step: function(log, db) { - // - }, - enter: function(frame) { - // - }, - exit: function(res) { - // - } -} \ No newline at end of file diff --git a/crates/rpc/rpc-testing-util/src/debug.rs b/crates/rpc/rpc-testing-util/src/debug.rs deleted file mode 100644 index 65fc3e86e02..00000000000 --- a/crates/rpc/rpc-testing-util/src/debug.rs +++ /dev/null @@ -1,429 +0,0 @@ -//! Helpers for testing debug trace calls. - -use std::{ - future::Future, - pin::Pin, - task::{Context, Poll}, -}; - -use alloy_eips::BlockId; -use alloy_primitives::{TxHash, B256}; -use alloy_rpc_types_eth::{transaction::TransactionRequest, Block, Header, Transaction}; -use alloy_rpc_types_trace::{ - common::TraceResult, - geth::{GethDebugTracerType, GethDebugTracingOptions, GethTrace}, -}; -use futures::{Stream, StreamExt}; -use jsonrpsee::core::client::Error as RpcError; -use reth_ethereum_primitives::{Receipt, TransactionSigned}; -use reth_rpc_api::{clients::DebugApiClient, EthApiClient}; - -const NOOP_TRACER: &str = include_str!("../assets/noop-tracer.js"); -const JS_TRACER_TEMPLATE: &str = include_str!("../assets/tracer-template.js"); - -/// A result type for the `debug_trace_transaction` method that also captures the requested hash. -pub type TraceTransactionResult = Result<(serde_json::Value, TxHash), (RpcError, TxHash)>; - -/// A result type for the `debug_trace_block` method that also captures the requested block. -pub type DebugTraceBlockResult = - Result<(Vec>, BlockId), (RpcError, BlockId)>; - -/// An extension trait for the Trace API. -pub trait DebugApiExt { - /// The provider type that is used to make the requests. - type Provider; - - /// Same as [`DebugApiClient::debug_trace_transaction`] but returns the result as json. - fn debug_trace_transaction_json( - &self, - hash: B256, - opts: GethDebugTracingOptions, - ) -> impl Future> + Send; - - /// Trace all transactions in a block individually with the given tracing opts. - fn debug_trace_transactions_in_block( - &self, - block: B, - opts: GethDebugTracingOptions, - ) -> impl Future, RpcError>> + Send - where - B: Into + Send; - - /// Trace all given blocks with the given tracing opts, returning a stream. - fn debug_trace_block_buffered_unordered( - &self, - params: I, - opts: Option, - n: usize, - ) -> DebugTraceBlockStream<'_> - where - I: IntoIterator, - B: Into + Send; - - /// method for `debug_traceCall` - fn debug_trace_call_json( - &self, - request: TransactionRequest, - opts: GethDebugTracingOptions, - ) -> impl Future> + Send; - - /// method for `debug_traceCall` using raw JSON strings for the request and options. - fn debug_trace_call_raw_json( - &self, - request_json: String, - opts_json: String, - ) -> impl Future> + Send; -} - -impl DebugApiExt for T -where - T: EthApiClient - + DebugApiClient - + Sync, -{ - type Provider = T; - - async fn debug_trace_transaction_json( - &self, - hash: B256, - opts: GethDebugTracingOptions, - ) -> Result { - let mut params = jsonrpsee::core::params::ArrayParams::new(); - params.insert(hash).unwrap(); - params.insert(opts).unwrap(); - self.request("debug_traceTransaction", params).await - } - - async fn debug_trace_transactions_in_block( - &self, - block: B, - opts: GethDebugTracingOptions, - ) -> Result, RpcError> - where - B: Into + Send, - { - let block = match block.into() { - BlockId::Hash(hash) => self.block_by_hash(hash.block_hash, false).await, - BlockId::Number(tag) => self.block_by_number(tag, false).await, - }? - .ok_or_else(|| RpcError::Custom("block not found".to_string()))?; - let hashes = block.transactions.hashes().map(|tx| (tx, opts.clone())).collect::>(); - let stream = futures::stream::iter(hashes.into_iter().map(move |(tx, opts)| async move { - match self.debug_trace_transaction_json(tx, opts).await { - Ok(result) => Ok((result, tx)), - Err(err) => Err((err, tx)), - } - })) - .buffered(10); - - Ok(DebugTraceTransactionsStream { stream: Box::pin(stream) }) - } - - fn debug_trace_block_buffered_unordered( - &self, - params: I, - opts: Option, - n: usize, - ) -> DebugTraceBlockStream<'_> - where - I: IntoIterator, - B: Into + Send, - { - let blocks = - params.into_iter().map(|block| (block.into(), opts.clone())).collect::>(); - let stream = - futures::stream::iter(blocks.into_iter().map(move |(block, opts)| async move { - let trace_future = match block { - BlockId::Hash(hash) => { - self.debug_trace_block_by_hash(hash.block_hash, opts).await - } - BlockId::Number(tag) => self.debug_trace_block_by_number(tag, opts).await, - }; - - match trace_future { - Ok(result) => Ok((result, block)), - Err(err) => Err((err, block)), - } - })) - .buffer_unordered(n); - DebugTraceBlockStream { stream: Box::pin(stream) } - } - - async fn debug_trace_call_json( - &self, - request: TransactionRequest, - opts: GethDebugTracingOptions, - ) -> Result { - let mut params = jsonrpsee::core::params::ArrayParams::new(); - params.insert(request).unwrap(); - params.insert(opts).unwrap(); - self.request("debug_traceCall", params).await - } - - async fn debug_trace_call_raw_json( - &self, - request_json: String, - opts_json: String, - ) -> Result { - let request = serde_json::from_str::(&request_json) - .map_err(|e| RpcError::Custom(e.to_string()))?; - let opts = serde_json::from_str::(&opts_json) - .map_err(|e| RpcError::Custom(e.to_string()))?; - - self.debug_trace_call_json(request, opts).await - } -} - -/// A helper type that can be used to build a javascript tracer. -#[derive(Debug, Clone, Default)] -pub struct JsTracerBuilder { - /// `setup_body` is invoked once at the beginning, during the construction of a given - /// transaction. - setup_body: Option, - - /// `fault_body` is invoked when an error happens during the execution of an opcode which - /// wasn't reported in step. - fault_body: Option, - - /// `result_body` returns a JSON-serializable value to the RPC caller. - result_body: Option, - - /// `enter_body` is invoked on stepping in of an internal call. - enter_body: Option, - - /// `step_body` is called for each step of the EVM, or when an error occurs, as the specified - /// transaction is traced. - step_body: Option, - - /// `exit_body` is invoked on stepping out of an internal call. - exit_body: Option, -} - -impl JsTracerBuilder { - /// Sets the body of the fault function - /// - /// The body code has access to the `log` and `db` variables. - pub fn fault_body(mut self, body: impl Into) -> Self { - self.fault_body = Some(body.into()); - self - } - - /// Sets the body of the setup function - /// - /// This body includes the `cfg` object variable - pub fn setup_body(mut self, body: impl Into) -> Self { - self.setup_body = Some(body.into()); - self - } - - /// Sets the body of the result function - /// - /// The body code has access to the `ctx` and `db` variables. - /// - /// ``` - /// use reth_rpc_api_testing_util::debug::JsTracerBuilder; - /// let code = JsTracerBuilder::default().result_body("return {};").code(); - /// ``` - pub fn result_body(mut self, body: impl Into) -> Self { - self.result_body = Some(body.into()); - self - } - - /// Sets the body of the enter function - /// - /// The body code has access to the `frame` variable. - pub fn enter_body(mut self, body: impl Into) -> Self { - self.enter_body = Some(body.into()); - self - } - - /// Sets the body of the step function - /// - /// The body code has access to the `log` and `db` variables. - pub fn step_body(mut self, body: impl Into) -> Self { - self.step_body = Some(body.into()); - self - } - - /// Sets the body of the exit function - /// - /// The body code has access to the `res` variable. - pub fn exit_body(mut self, body: impl Into) -> Self { - self.exit_body = Some(body.into()); - self - } - - /// Returns the tracers JS code - pub fn code(self) -> String { - let mut template = JS_TRACER_TEMPLATE.to_string(); - template = template.replace("//", self.setup_body.as_deref().unwrap_or_default()); - template = template.replace("//", self.fault_body.as_deref().unwrap_or_default()); - template = - template.replace("//", self.result_body.as_deref().unwrap_or("return {};")); - template = template.replace("//", self.step_body.as_deref().unwrap_or_default()); - template = template.replace("//", self.enter_body.as_deref().unwrap_or_default()); - template = template.replace("//", self.exit_body.as_deref().unwrap_or_default()); - template - } -} - -impl std::fmt::Display for JsTracerBuilder { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.clone().code()) - } -} - -impl From for GethDebugTracingOptions { - fn from(b: JsTracerBuilder) -> Self { - Self { - tracer: Some(GethDebugTracerType::JsTracer(b.code())), - tracer_config: serde_json::Value::Object(Default::default()).into(), - ..Default::default() - } - } -} -impl From for Option { - fn from(b: JsTracerBuilder) -> Self { - Some(b.into()) - } -} - -/// A stream that yields the traces for the requested blocks. -#[must_use = "streams do nothing unless polled"] -pub struct DebugTraceTransactionsStream<'a> { - stream: Pin + 'a>>, -} - -impl DebugTraceTransactionsStream<'_> { - /// Returns the next error result of the stream. - pub async fn next_err(&mut self) -> Option<(RpcError, TxHash)> { - loop { - match self.next().await? { - Ok(_) => {} - Err(err) => return Some(err), - } - } - } -} - -impl Stream for DebugTraceTransactionsStream<'_> { - type Item = TraceTransactionResult; - - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - self.stream.as_mut().poll_next(cx) - } -} - -impl std::fmt::Debug for DebugTraceTransactionsStream<'_> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("DebugTraceTransactionsStream").finish_non_exhaustive() - } -} - -/// A stream that yields the `debug_` traces for the requested blocks. -#[must_use = "streams do nothing unless polled"] -pub struct DebugTraceBlockStream<'a> { - stream: Pin + 'a>>, -} - -impl DebugTraceBlockStream<'_> { - /// Returns the next error result of the stream. - pub async fn next_err(&mut self) -> Option<(RpcError, BlockId)> { - loop { - match self.next().await? { - Ok(_) => {} - Err(err) => return Some(err), - } - } - } -} - -impl Stream for DebugTraceBlockStream<'_> { - type Item = DebugTraceBlockResult; - - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - self.stream.as_mut().poll_next(cx) - } -} - -impl std::fmt::Debug for DebugTraceBlockStream<'_> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("DebugTraceBlockStream").finish_non_exhaustive() - } -} - -/// A javascript tracer that does nothing -#[derive(Debug, Clone, Copy, Default)] -#[non_exhaustive] -pub struct NoopJsTracer; - -impl From for GethDebugTracingOptions { - fn from(_: NoopJsTracer) -> Self { - Self { - tracer: Some(GethDebugTracerType::JsTracer(NOOP_TRACER.to_string())), - tracer_config: serde_json::Value::Object(Default::default()).into(), - ..Default::default() - } - } -} -impl From for Option { - fn from(_: NoopJsTracer) -> Self { - Some(NoopJsTracer.into()) - } -} - -#[cfg(test)] -mod tests { - use crate::{ - debug::{DebugApiExt, JsTracerBuilder, NoopJsTracer}, - utils::parse_env_url, - }; - use alloy_rpc_types_trace::geth::{CallConfig, GethDebugTracingOptions}; - use futures::StreamExt; - use jsonrpsee::http_client::HttpClientBuilder; - - // random tx - const TX_1: &str = "0x5525c63a805df2b83c113ebcc8c7672a3b290673c4e81335b410cd9ebc64e085"; - - #[tokio::test] - #[ignore] - async fn can_trace_noop_sepolia() { - let tx = TX_1.parse().unwrap(); - let url = parse_env_url("RETH_RPC_TEST_NODE_URL").unwrap(); - let client = HttpClientBuilder::default().build(url).unwrap(); - let res = - client.debug_trace_transaction_json(tx, NoopJsTracer::default().into()).await.unwrap(); - assert_eq!(res, serde_json::Value::Object(Default::default())); - } - - #[tokio::test] - #[ignore] - async fn can_trace_default_template() { - let tx = TX_1.parse().unwrap(); - let url = parse_env_url("RETH_RPC_TEST_NODE_URL").unwrap(); - let client = HttpClientBuilder::default().build(url).unwrap(); - let res = client - .debug_trace_transaction_json(tx, JsTracerBuilder::default().into()) - .await - .unwrap(); - assert_eq!(res, serde_json::Value::Object(Default::default())); - } - - #[tokio::test] - #[ignore] - async fn can_debug_trace_block_transactions() { - let block = 11_117_104u64; - let url = parse_env_url("RETH_RPC_TEST_NODE_URL").unwrap(); - let client = HttpClientBuilder::default().build(url).unwrap(); - - let opts = GethDebugTracingOptions::default() - .with_call_config(CallConfig::default().only_top_call()); - - let mut stream = client.debug_trace_transactions_in_block(block, opts).await.unwrap(); - while let Some(res) = stream.next().await { - if let Err((err, tx)) = res { - println!("failed to trace {tx:?} {err}"); - } - } - } -} diff --git a/crates/rpc/rpc-testing-util/src/lib.rs b/crates/rpc/rpc-testing-util/src/lib.rs deleted file mode 100644 index 6be9f74403f..00000000000 --- a/crates/rpc/rpc-testing-util/src/lib.rs +++ /dev/null @@ -1,14 +0,0 @@ -//! Reth RPC testing utilities. - -#![doc( - html_logo_url = "https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-docs.png", - html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", - issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" -)] -#![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg))] - -pub mod debug; -pub mod trace; - -pub mod utils; diff --git a/crates/rpc/rpc-testing-util/src/trace.rs b/crates/rpc/rpc-testing-util/src/trace.rs deleted file mode 100644 index 79aceebe06e..00000000000 --- a/crates/rpc/rpc-testing-util/src/trace.rs +++ /dev/null @@ -1,789 +0,0 @@ -//! Helpers for testing trace calls. - -use alloy_eips::BlockId; -use alloy_primitives::{map::HashSet, Bytes, TxHash, B256}; -use alloy_rpc_types_eth::{transaction::TransactionRequest, Index}; -use alloy_rpc_types_trace::{ - filter::TraceFilter, - opcode::BlockOpcodeGas, - parity::{LocalizedTransactionTrace, TraceResults, TraceType}, - tracerequest::TraceCallRequest, -}; -use futures::{Stream, StreamExt}; -use jsonrpsee::core::client::Error as RpcError; -use reth_rpc_api::clients::TraceApiClient; -use std::{ - pin::Pin, - task::{Context, Poll}, -}; - -/// A type alias that represents the result of a raw transaction trace stream. -type RawTransactionTraceResult<'a> = - Pin> + 'a>>; - -/// A result type for the `trace_block` method that also captures the requested block. -pub type TraceBlockResult = Result<(Vec, BlockId), (RpcError, BlockId)>; - -/// A result type for the `trace_blockOpcodeGas` method that also captures the requested block. -pub type TraceBlockOpCodeGasResult = Result<(BlockOpcodeGas, BlockId), (RpcError, BlockId)>; - -/// Type alias representing the result of replaying a transaction. -pub type ReplayTransactionResult = Result<(TraceResults, TxHash), (RpcError, TxHash)>; - -/// A type representing the result of calling `trace_call_many` method. -pub type CallManyTraceResult = Result< - (Vec, Vec<(TransactionRequest, HashSet)>), - (RpcError, Vec<(TransactionRequest, HashSet)>), ->; - -/// Result type for the `trace_get` method that also captures the requested transaction hash and -/// index. -pub type TraceGetResult = - Result<(Option, B256, Vec), (RpcError, B256, Vec)>; - -/// Represents a result type for the `trace_filter` stream extension. -pub type TraceFilterResult = - Result<(Vec, TraceFilter), (RpcError, TraceFilter)>; - -/// Represents the result of a single trace call. -pub type TraceCallResult = Result; - -/// An extension trait for the Trace API. -pub trait TraceApiExt { - /// The provider type that is used to make the requests. - type Provider; - - /// Returns a new stream that yields the traces for the given blocks. - /// - /// See also [`StreamExt::buffered`]. - fn trace_block_buffered(&self, params: I, n: usize) -> TraceBlockStream<'_> - where - I: IntoIterator, - B: Into; - - /// Returns a new stream that yields the traces for the given blocks. - /// - /// See also [`StreamExt::buffer_unordered`]. - fn trace_block_buffered_unordered(&self, params: I, n: usize) -> TraceBlockStream<'_> - where - I: IntoIterator, - B: Into; - - /// Returns a new stream that yields the traces the opcodes for the given blocks. - /// - /// See also [`StreamExt::buffer_unordered`]. - fn trace_block_opcode_gas_unordered( - &self, - params: I, - n: usize, - ) -> TraceBlockOpcodeGasStream<'_> - where - I: IntoIterator, - B: Into; - - /// Returns a new stream that replays the transactions for the given transaction hashes. - /// - /// This returns all results in order. - fn replay_transactions( - &self, - tx_hashes: I, - trace_types: HashSet, - ) -> ReplayTransactionStream<'_> - where - I: IntoIterator; - - /// Returns a new stream that traces the provided raw transaction data. - fn trace_raw_transaction_stream( - &self, - data: Bytes, - trace_types: HashSet, - block_id: Option, - ) -> RawTransactionTraceStream<'_>; - - /// Creates a stream of results for multiple dependent transaction calls on top of the same - /// block. - fn trace_call_many_stream( - &self, - calls: I, - block_id: Option, - ) -> CallManyTraceStream<'_> - where - I: IntoIterator)>; - - /// Returns a new stream that yields the traces for the given transaction hash and indices. - fn trace_get_stream(&self, hash: B256, indices: I) -> TraceGetStream<'_> - where - I: IntoIterator; - - /// Returns a new stream that yields traces for given filters. - fn trace_filter_stream(&self, filters: I) -> TraceFilterStream<'_> - where - I: IntoIterator; - - /// Returns a new stream that yields the trace results for the given call requests. - fn trace_call_stream(&self, request: TraceCallRequest) -> TraceCallStream<'_>; -} -/// `TraceCallStream` provides an asynchronous stream of tracing results. -#[must_use = "streams do nothing unless polled"] -pub struct TraceCallStream<'a> { - stream: Pin + 'a>>, -} - -impl Stream for TraceCallStream<'_> { - type Item = TraceCallResult; - - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - self.stream.as_mut().poll_next(cx) - } -} - -impl std::fmt::Debug for TraceCallStream<'_> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("TraceCallStream").finish() - } -} - -/// Represents a stream that asynchronously yields the results of the `trace_filter` method. -#[must_use = "streams do nothing unless polled"] -pub struct TraceFilterStream<'a> { - stream: Pin + 'a>>, -} - -impl Stream for TraceFilterStream<'_> { - type Item = TraceFilterResult; - - /// Attempts to pull out the next value of the stream. - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - self.stream.as_mut().poll_next(cx) - } -} - -impl std::fmt::Debug for TraceFilterStream<'_> { - /// Provides a debug representation of the `TraceFilterStream`. - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("TraceFilterStream").finish_non_exhaustive() - } -} - -/// A stream that asynchronously yields the results of the `trace_get` method for a given -/// transaction hash and a series of indices. -#[must_use = "streams do nothing unless polled"] -pub struct TraceGetStream<'a> { - stream: Pin + 'a>>, -} - -impl Stream for TraceGetStream<'_> { - type Item = TraceGetResult; - - /// Attempts to pull out the next item of the stream - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - self.stream.as_mut().poll_next(cx) - } -} - -impl std::fmt::Debug for TraceGetStream<'_> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("TraceGetStream").finish_non_exhaustive() - } -} - -/// A stream that provides asynchronous iteration over results from the `trace_call_many` function. -/// -/// The stream yields items of type `CallManyTraceResult`. -#[must_use = "streams do nothing unless polled"] -pub struct CallManyTraceStream<'a> { - stream: Pin + 'a>>, -} - -impl Stream for CallManyTraceStream<'_> { - type Item = CallManyTraceResult; - - /// Polls for the next item from the stream. - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - self.stream.as_mut().poll_next(cx) - } -} - -impl std::fmt::Debug for CallManyTraceStream<'_> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("CallManyTraceStream").finish() - } -} - -/// A stream that traces the provided raw transaction data. -#[must_use = "streams do nothing unless polled"] -pub struct RawTransactionTraceStream<'a> { - stream: RawTransactionTraceResult<'a>, -} - -impl Stream for RawTransactionTraceStream<'_> { - type Item = Result<(TraceResults, Bytes), (RpcError, Bytes)>; - - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - self.stream.as_mut().poll_next(cx) - } -} - -impl std::fmt::Debug for RawTransactionTraceStream<'_> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("RawTransactionTraceStream").finish() - } -} - -/// A stream that replays the transactions for the requested hashes. -#[must_use = "streams do nothing unless polled"] -pub struct ReplayTransactionStream<'a> { - stream: Pin + 'a>>, -} - -impl Stream for ReplayTransactionStream<'_> { - type Item = ReplayTransactionResult; - - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - self.stream.as_mut().poll_next(cx) - } -} - -impl std::fmt::Debug for ReplayTransactionStream<'_> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("ReplayTransactionStream").finish() - } -} - -impl + Sync> TraceApiExt for T { - type Provider = T; - - fn trace_block_buffered(&self, params: I, n: usize) -> TraceBlockStream<'_> - where - I: IntoIterator, - B: Into, - { - let blocks = params.into_iter().map(|b| b.into()).collect::>(); - let stream = futures::stream::iter(blocks.into_iter().map(move |block| async move { - match self.trace_block(block).await { - Ok(result) => Ok((result.unwrap_or_default(), block)), - Err(err) => Err((err, block)), - } - })) - .buffered(n); - TraceBlockStream { stream: Box::pin(stream) } - } - - fn trace_block_buffered_unordered(&self, params: I, n: usize) -> TraceBlockStream<'_> - where - I: IntoIterator, - B: Into, - { - let blocks = params.into_iter().map(|b| b.into()).collect::>(); - let stream = futures::stream::iter(blocks.into_iter().map(move |block| async move { - match self.trace_block(block).await { - Ok(result) => Ok((result.unwrap_or_default(), block)), - Err(err) => Err((err, block)), - } - })) - .buffer_unordered(n); - TraceBlockStream { stream: Box::pin(stream) } - } - - fn trace_block_opcode_gas_unordered( - &self, - params: I, - n: usize, - ) -> TraceBlockOpcodeGasStream<'_> - where - I: IntoIterator, - B: Into, - { - let blocks = params.into_iter().map(|b| b.into()).collect::>(); - let stream = futures::stream::iter(blocks.into_iter().map(move |block| async move { - match self.trace_block_opcode_gas(block).await { - Ok(result) => Ok((result.unwrap(), block)), - Err(err) => Err((err, block)), - } - })) - .buffer_unordered(n); - TraceBlockOpcodeGasStream { stream: Box::pin(stream) } - } - - fn replay_transactions( - &self, - tx_hashes: I, - trace_types: HashSet, - ) -> ReplayTransactionStream<'_> - where - I: IntoIterator, - { - let hashes = tx_hashes.into_iter().collect::>(); - let stream = futures::stream::iter(hashes.into_iter().map(move |hash| { - let trace_types_clone = trace_types.clone(); // Clone outside of the async block - async move { - match self.replay_transaction(hash, trace_types_clone).await { - Ok(result) => Ok((result, hash)), - Err(err) => Err((err, hash)), - } - } - })) - .buffered(10); - ReplayTransactionStream { stream: Box::pin(stream) } - } - - fn trace_raw_transaction_stream( - &self, - data: Bytes, - trace_types: HashSet, - block_id: Option, - ) -> RawTransactionTraceStream<'_> { - let stream = futures::stream::once(async move { - match self.trace_raw_transaction(data.clone(), trace_types, block_id).await { - Ok(result) => Ok((result, data)), - Err(err) => Err((err, data)), - } - }); - RawTransactionTraceStream { stream: Box::pin(stream) } - } - - fn trace_call_many_stream( - &self, - calls: I, - block_id: Option, - ) -> CallManyTraceStream<'_> - where - I: IntoIterator)>, - { - let call_set = calls.into_iter().collect::>(); - let stream = futures::stream::once(async move { - match self.trace_call_many(call_set.clone(), block_id).await { - Ok(results) => Ok((results, call_set)), - Err(err) => Err((err, call_set)), - } - }); - CallManyTraceStream { stream: Box::pin(stream) } - } - - fn trace_get_stream(&self, hash: B256, indices: I) -> TraceGetStream<'_> - where - I: IntoIterator, - { - let index_list = indices.into_iter().collect::>(); - let stream = futures::stream::iter(index_list.into_iter().map(move |index| async move { - match self.trace_get(hash, vec![index]).await { - Ok(result) => Ok((result, hash, vec![index])), - Err(err) => Err((err, hash, vec![index])), - } - })) - .buffered(10); - TraceGetStream { stream: Box::pin(stream) } - } - - fn trace_filter_stream(&self, filters: I) -> TraceFilterStream<'_> - where - I: IntoIterator, - { - let filter_list = filters.into_iter().collect::>(); - let stream = futures::stream::iter(filter_list.into_iter().map(move |filter| async move { - match self.trace_filter(filter.clone()).await { - Ok(result) => Ok((result, filter)), - Err(err) => Err((err, filter)), - } - })) - .buffered(10); - TraceFilterStream { stream: Box::pin(stream) } - } - - fn trace_call_stream(&self, request: TraceCallRequest) -> TraceCallStream<'_> { - let stream = futures::stream::once(async move { - match self - .trace_call( - request.call.clone(), - request.trace_types.clone(), - request.block_id, - request.state_overrides.clone(), - request.block_overrides.clone(), - ) - .await - { - Ok(result) => Ok(result), - Err(err) => Err((err, request)), - } - }); - TraceCallStream { stream: Box::pin(stream) } - } -} - -/// A stream that yields the traces for the requested blocks. -#[must_use = "streams do nothing unless polled"] -pub struct TraceBlockStream<'a> { - stream: Pin + 'a>>, -} - -impl TraceBlockStream<'_> { - /// Returns the next error result of the stream. - pub async fn next_err(&mut self) -> Option<(RpcError, BlockId)> { - loop { - match self.next().await? { - Ok(_) => {} - Err(err) => return Some(err), - } - } - } -} - -impl Stream for TraceBlockStream<'_> { - type Item = TraceBlockResult; - - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - self.stream.as_mut().poll_next(cx) - } -} - -impl std::fmt::Debug for TraceBlockStream<'_> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("TraceBlockStream").finish_non_exhaustive() - } -} - -/// A stream that yields the opcodes for the requested blocks. -#[must_use = "streams do nothing unless polled"] -pub struct TraceBlockOpcodeGasStream<'a> { - stream: Pin + 'a>>, -} - -impl TraceBlockOpcodeGasStream<'_> { - /// Returns the next error result of the stream. - pub async fn next_err(&mut self) -> Option<(RpcError, BlockId)> { - loop { - match self.next().await? { - Ok(_) => {} - Err(err) => return Some(err), - } - } - } -} - -impl Stream for TraceBlockOpcodeGasStream<'_> { - type Item = TraceBlockOpCodeGasResult; - - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - self.stream.as_mut().poll_next(cx) - } -} - -impl std::fmt::Debug for TraceBlockOpcodeGasStream<'_> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("TraceBlockOpcodeGasStream").finish_non_exhaustive() - } -} - -/// A utility to compare RPC responses from two different clients. -/// -/// The `RpcComparer` is designed to perform comparisons between two RPC clients. -/// It is useful in scenarios where there's a need to ensure that two different RPC clients -/// return consistent responses. This can be particularly valuable in testing environments -/// where one might want to compare a test client's responses against a production client -/// or compare two different Ethereum client implementations. -#[derive(Debug)] -pub struct RpcComparer -where - C1: TraceApiExt, - C2: TraceApiExt, -{ - client1: C1, - client2: C2, -} -impl RpcComparer -where - C1: TraceApiExt, - C2: TraceApiExt, -{ - /// Constructs a new `RpcComparer`. - /// - /// Initializes the comparer with two clients that will be used for fetching - /// and comparison. - /// - /// # Arguments - /// - /// * `client1` - The first RPC client. - /// * `client2` - The second RPC client. - pub const fn new(client1: C1, client2: C2) -> Self { - Self { client1, client2 } - } - - /// Compares the `trace_block` responses from the two RPC clients. - /// - /// Fetches the `trace_block` responses for the provided block IDs from both clients - /// and compares them. If there are inconsistencies between the two responses, this - /// method will panic with a relevant message indicating the difference. - pub async fn compare_trace_block_responses(&self, block_ids: Vec) { - let stream1 = self.client1.trace_block_buffered(block_ids.clone(), 2); - let stream2 = self.client2.trace_block_buffered(block_ids, 2); - - let mut zipped_streams = stream1.zip(stream2); - - while let Some((result1, result2)) = zipped_streams.next().await { - match (result1, result2) { - (Ok((ref traces1_data, ref block1)), Ok((ref traces2_data, ref block2))) => { - similar_asserts::assert_eq!( - traces1_data, - traces2_data, - "Mismatch in traces for block: {:?}", - block1 - ); - assert_eq!(block1, block2, "Mismatch in block ids."); - } - (Err((ref err1, ref block1)), Err((ref err2, ref block2))) => { - assert_eq!( - format!("{err1:?}"), - format!("{err2:?}"), - "Different errors for block: {block1:?}" - ); - assert_eq!(block1, block2, "Mismatch in block ids."); - } - _ => panic!("One client returned Ok while the other returned Err."), - } - } - } - - /// Compares the `replay_transactions` responses from the two RPC clients. - pub async fn compare_replay_transaction_responses( - &self, - transaction_hashes: Vec, - trace_types: HashSet, - ) { - let stream1 = - self.client1.replay_transactions(transaction_hashes.clone(), trace_types.clone()); - let stream2 = self.client2.replay_transactions(transaction_hashes, trace_types); - - let mut zipped_streams = stream1.zip(stream2); - - while let Some((result1, result2)) = zipped_streams.next().await { - match (result1, result2) { - (Ok((ref trace1_data, ref tx_hash1)), Ok((ref trace2_data, ref tx_hash2))) => { - similar_asserts::assert_eq!( - trace1_data, - trace2_data, - "Mismatch in trace results for transaction: {tx_hash1:?}", - ); - assert_eq!(tx_hash1, tx_hash2, "Mismatch in transaction hashes."); - } - (Err((ref err1, ref tx_hash1)), Err((ref err2, ref tx_hash2))) => { - assert_eq!( - format!("{err1:?}"), - format!("{err2:?}"), - "Different errors for transaction: {tx_hash1:?}", - ); - assert_eq!(tx_hash1, tx_hash2, "Mismatch in transaction hashes."); - } - _ => panic!("One client returned Ok while the other returned Err."), - } - } - } -} -#[cfg(test)] -mod tests { - use super::*; - use alloy_eips::BlockNumberOrTag; - use alloy_rpc_types_trace::filter::TraceFilterMode; - use futures::future::join; - use jsonrpsee::http_client::HttpClientBuilder; - - const fn assert_is_stream(_: &St) {} - - #[tokio::test] - async fn can_create_block_stream() { - let client = HttpClientBuilder::default().build("http://localhost:8545").unwrap(); - let block = vec![BlockId::Number(5u64.into()), BlockNumberOrTag::Latest.into()]; - let stream = client.trace_block_buffered(block, 2); - assert_is_stream(&stream); - } - - #[tokio::test] - #[ignore] - async fn can_create_replay_transaction_stream() { - let client = HttpClientBuilder::default().build("http://localhost:8545").unwrap(); - - // Assuming you have some transactions you want to test, replace with actual hashes. - let transactions = vec![ - "0x4e08fe36db723a338e852f89f613e606b0c9a17e649b18b01251f86236a2cef3".parse().unwrap(), - "0xea2817f1aeeb587b82f4ab87a6dbd3560fc35ed28de1be280cb40b2a24ab48bb".parse().unwrap(), - ]; - - let trace_types = HashSet::from_iter([TraceType::StateDiff, TraceType::VmTrace]); - - let mut stream = client.replay_transactions(transactions, trace_types); - let mut successes = 0; - let mut failures = 0; - - assert_is_stream(&stream); - - while let Some(result) = stream.next().await { - match result { - Ok((trace_result, tx_hash)) => { - println!("Success for tx_hash {tx_hash:?}: {trace_result:?}"); - successes += 1; - } - Err((error, tx_hash)) => { - println!("Error for tx_hash {tx_hash:?}: {error:?}"); - failures += 1; - } - } - } - - println!("Total successes: {successes}"); - println!("Total failures: {failures}"); - } - - #[tokio::test] - #[ignore] - async fn can_create_trace_call_many_stream() { - let client = HttpClientBuilder::default().build("http://localhost:8545").unwrap(); - - let call_request_1 = TransactionRequest::default(); - let call_request_2 = TransactionRequest::default(); - let trace_types = HashSet::from_iter([TraceType::StateDiff, TraceType::VmTrace]); - let calls = vec![(call_request_1, trace_types.clone()), (call_request_2, trace_types)]; - - let mut stream = client.trace_call_many_stream(calls, None); - - assert_is_stream(&stream); - - while let Some(result) = stream.next().await { - match result { - Ok(trace_result) => { - println!("Success: {trace_result:?}"); - } - Err(error) => { - println!("Error: {error:?}"); - } - } - } - } - #[tokio::test] - #[ignore] - async fn can_create_trace_get_stream() { - let client = HttpClientBuilder::default().build("http://localhost:8545").unwrap(); - - let tx_hash: B256 = "".parse().unwrap(); - - let indices: Vec = vec![Index::from(0)]; - - let mut stream = client.trace_get_stream(tx_hash, indices); - - while let Some(result) = stream.next().await { - match result { - Ok(trace) => { - println!("Received trace: {trace:?}"); - } - Err(e) => { - println!("Error fetching trace: {e:?}"); - } - } - } - } - - #[tokio::test] - #[ignore] - async fn can_create_trace_filter() { - let client = HttpClientBuilder::default().build("http://localhost:8545").unwrap(); - - let filter = TraceFilter { - from_block: None, - to_block: None, - from_address: Vec::new(), - to_address: Vec::new(), - mode: TraceFilterMode::Union, - after: None, - count: None, - }; - - let filters = vec![filter]; - let mut stream = client.trace_filter_stream(filters); - - while let Some(result) = stream.next().await { - match result { - Ok(trace) => { - println!("Received trace: {trace:?}"); - } - Err(e) => { - println!("Error fetching trace: {e:?}"); - } - } - } - } - - #[tokio::test] - #[ignore] - async fn can_create_trace_call_stream() { - let client = HttpClientBuilder::default().build("http://localhost:8545").unwrap(); - - let trace_call_request = TraceCallRequest::default(); - - let mut stream = client.trace_call_stream(trace_call_request); - let mut successes = 0; - let mut failures = 0; - - assert_is_stream(&stream); - - while let Some(result) = stream.next().await { - match result { - Ok(trace_result) => { - println!("Success: {trace_result:?}"); - successes += 1; - } - Err((error, request)) => { - println!("Error for request {request:?}: {error:?}"); - failures += 1; - } - } - } - - println!("Total successes: {successes}"); - println!("Total failures: {failures}"); - } - - #[tokio::test] - #[ignore] - async fn block_opcode_gas_stream() { - let client = HttpClientBuilder::default().build("http://localhost:8545").unwrap(); - let block = vec![BlockNumberOrTag::Latest]; - let mut stream = client.trace_block_opcode_gas_unordered(block, 2); - assert_is_stream(&stream); - let _opcodes = stream.next().await.unwrap(); - } - - #[tokio::test(flavor = "multi_thread")] - #[ignore] - async fn compare_block_stream() { - let client_a = HttpClientBuilder::default().build("http://localhost:8545").unwrap(); - let client_b = HttpClientBuilder::default().build("http://localhost:8544").unwrap(); - let blocks = 0u64..=1681464; - let mut stream_a = client_a.trace_block_buffered(blocks.clone(), 2); - let mut stream_b = client_b.trace_block_buffered(blocks, 2); - - let mut count = 0; - loop { - let (res_a, res_b) = join(stream_a.next(), stream_b.next()).await; - - if res_a.is_none() && res_b.is_none() { - break; - } - - match (res_a, res_b) { - (Some(Ok(res_a)), Some(Ok(res_b))) => { - if res_a != res_b { - println!("Received different trace results: {res_a:?}, res_b: {res_b:?}"); - } - } - (res_a, res_b) => { - println!("Received different responses: {res_a:?}, res_b: {res_b:?}"); - } - } - - if count % 1000 == 0 { - println!("Blocks traced: {count}"); - } - - count += 1; - } - println!("Total blocks traced: {count}"); - } -} diff --git a/crates/rpc/rpc-testing-util/src/utils.rs b/crates/rpc/rpc-testing-util/src/utils.rs deleted file mode 100644 index e4b4acb4eba..00000000000 --- a/crates/rpc/rpc-testing-util/src/utils.rs +++ /dev/null @@ -1,13 +0,0 @@ -//! Utils for testing RPC. - -/// This will read the value of the given environment variable and parse it as a URL. -/// -/// If the value has no http(s) scheme, it will be appended: `http://{var}`. -pub fn parse_env_url(var: &str) -> Result { - let var = std::env::var(var)?; - if var.starts_with("http") { - Ok(var) - } else { - Ok(format!("http://{var}")) - } -} diff --git a/crates/rpc/rpc-testing-util/tests/it/main.rs b/crates/rpc/rpc-testing-util/tests/it/main.rs deleted file mode 100644 index ac9d933190f..00000000000 --- a/crates/rpc/rpc-testing-util/tests/it/main.rs +++ /dev/null @@ -1,5 +0,0 @@ -#![allow(missing_docs)] - -mod trace; - -const fn main() {} diff --git a/crates/rpc/rpc-testing-util/tests/it/trace.rs b/crates/rpc/rpc-testing-util/tests/it/trace.rs deleted file mode 100644 index 19e0b202dc6..00000000000 --- a/crates/rpc/rpc-testing-util/tests/it/trace.rs +++ /dev/null @@ -1,184 +0,0 @@ -//! Integration tests for the trace API. - -use alloy_primitives::map::HashSet; -use alloy_rpc_types_eth::{Block, Header, Transaction, TransactionRequest}; -use alloy_rpc_types_trace::{ - filter::TraceFilter, parity::TraceType, tracerequest::TraceCallRequest, -}; -use futures::StreamExt; -use jsonrpsee::http_client::HttpClientBuilder; -use jsonrpsee_http_client::HttpClient; -use reth_ethereum_primitives::{Receipt, TransactionSigned}; -use reth_rpc_api_testing_util::{debug::DebugApiExt, trace::TraceApiExt, utils::parse_env_url}; -use reth_rpc_eth_api::EthApiClient; -use std::time::Instant; - -/// This is intended to be run locally against a running node. -/// -/// This is a noop of env var `RETH_RPC_TEST_NODE_URL` is not set. -#[tokio::test(flavor = "multi_thread")] -async fn trace_many_blocks() { - let url = parse_env_url("RETH_RPC_TEST_NODE_URL"); - if url.is_err() { - return - } - let url = url.unwrap(); - - let client = HttpClientBuilder::default().build(url).unwrap(); - let mut stream = client.trace_block_buffered_unordered(15_000_000..=16_000_100, 20); - let now = Instant::now(); - while let Some((err, block)) = stream.next_err().await { - eprintln!("Error tracing block {block:?}: {err}"); - } - println!("Traced all blocks in {:?}", now.elapsed()); -} - -/// Tests the replaying of transactions on a local Ethereum node. -#[tokio::test(flavor = "multi_thread")] -#[ignore] -async fn replay_transactions() { - let url = parse_env_url("RETH_RPC_TEST_NODE_URL").unwrap(); - let client = HttpClientBuilder::default().build(url).unwrap(); - - let tx_hashes = vec![ - "0x4e08fe36db723a338e852f89f613e606b0c9a17e649b18b01251f86236a2cef3".parse().unwrap(), - "0xea2817f1aeeb587b82f4ab87a6dbd3560fc35ed28de1be280cb40b2a24ab48bb".parse().unwrap(), - ]; - - let trace_types = HashSet::from_iter([TraceType::StateDiff, TraceType::VmTrace]); - - let mut stream = client.replay_transactions(tx_hashes, trace_types); - let now = Instant::now(); - while let Some(replay_txs) = stream.next().await { - println!("Transaction: {replay_txs:?}"); - println!("Replayed transactions in {:?}", now.elapsed()); - } -} - -/// Tests the tracers filters on a local Ethereum node -#[tokio::test(flavor = "multi_thread")] -#[ignore] -async fn trace_filters() { - // Parse the node URL from environment variable and create an HTTP client. - let url = parse_env_url("RETH_RPC_TEST_NODE_URL").unwrap(); - let client = HttpClientBuilder::default().build(url).unwrap(); - - // Set up trace filters. - let filter = TraceFilter::default(); - let filters = vec![filter]; - - // Initialize a stream for the trace filters. - let mut stream = client.trace_filter_stream(filters); - let start_time = Instant::now(); - while let Some(trace) = stream.next().await { - println!("Transaction Trace: {trace:?}"); - println!("Duration since test start: {:?}", start_time.elapsed()); - } -} - -#[tokio::test(flavor = "multi_thread")] -#[ignore] -async fn trace_call() { - let url = parse_env_url("RETH_RPC_TEST_NODE_URL").unwrap(); - let client = HttpClientBuilder::default().build(url).unwrap(); - let trace_call_request = TraceCallRequest::default(); - let mut stream = client.trace_call_stream(trace_call_request); - let start_time = Instant::now(); - - while let Some(result) = stream.next().await { - match result { - Ok(trace_result) => { - println!("Trace Result: {trace_result:?}"); - } - Err((error, request)) => { - eprintln!("Error for request {request:?}: {error:?}"); - } - } - } - - println!("Completed in {:?}", start_time.elapsed()); -} - -/// This is intended to be run locally against a running node. This traces all blocks for a given -/// chain. -/// -/// This is a noop of env var `RETH_RPC_TEST_NODE_URL` is not set. -#[tokio::test(flavor = "multi_thread")] -async fn debug_trace_block_entire_chain() { - let url = parse_env_url("RETH_RPC_TEST_NODE_URL"); - if url.is_err() { - return - } - let url = url.unwrap(); - - let client = HttpClientBuilder::default().build(url).unwrap(); - let current_block: u64 = >::block_number(&client) - .await - .unwrap() - .try_into() - .unwrap(); - let range = 0..=current_block; - let mut stream = client.debug_trace_block_buffered_unordered(range, None, 20); - let now = Instant::now(); - while let Some((err, block)) = stream.next_err().await { - eprintln!("Error tracing block {block:?}: {err}"); - } - println!("Traced all blocks in {:?}", now.elapsed()); -} - -/// This is intended to be run locally against a running node. This traces all blocks for a given -/// chain. -/// -/// This is a noop of env var `RETH_RPC_TEST_NODE_URL` is not set. -#[tokio::test(flavor = "multi_thread")] -async fn debug_trace_block_opcodes_entire_chain() { - let opcodes7702 = ["EXTCODESIZE", "EXTCODECOPY", "EXTCODEHASH"]; - let url = parse_env_url("RETH_RPC_TEST_NODE_URL"); - if url.is_err() { - return - } - let url = url.unwrap(); - - let client = HttpClientBuilder::default().build(url).unwrap(); - let current_block: u64 = >::block_number(&client) - .await - .unwrap() - .try_into() - .unwrap(); - let range = 0..=current_block; - println!("Tracing blocks {range:?} for opcodes"); - let mut stream = client.trace_block_opcode_gas_unordered(range, 2).enumerate(); - let now = Instant::now(); - while let Some((num, next)) = stream.next().await { - match next { - Ok((block_opcodes, block)) => { - for opcode in opcodes7702 { - if block_opcodes.contains(opcode) { - eprintln!("Found opcode {opcode}: in {block}"); - } - } - } - Err((err, block)) => { - eprintln!("Error tracing block {block:?}: {err}"); - } - }; - if num % 10000 == 0 { - println!("Traced {num} blocks"); - } - } - println!("Traced all blocks in {:?}", now.elapsed()); -} diff --git a/crates/rpc/rpc/src/eth/core.rs b/crates/rpc/rpc/src/eth/core.rs index e08291afa06..f169f6a3c0d 100644 --- a/crates/rpc/rpc/src/eth/core.rs +++ b/crates/rpc/rpc/src/eth/core.rs @@ -583,24 +583,28 @@ mod tests { use crate::{eth::helpers::types::EthRpcConverter, EthApi, EthApiBuilder}; use alloy_consensus::{Block, BlockBody, Header}; use alloy_eips::BlockNumberOrTag; - use alloy_primitives::{Signature, B256, U64}; + use alloy_primitives::{Address, Bytes, Signature, TxKind, B256, U256, U64}; use alloy_rpc_types::FeeHistory; - use alloy_rpc_types_eth::{Bundle, TransactionRequest}; + use alloy_rpc_types_eth::{BlockOverrides, Bundle, TransactionRequest}; use jsonrpsee_types::error::INVALID_PARAMS_CODE; use rand::Rng; use reth_chain_state::CanonStateSubscriptions; - use reth_chainspec::{ChainSpec, ChainSpecProvider, EthChainSpec}; + use reth_chainspec::{ChainSpec, ChainSpecProvider, EthChainSpec, DEV}; use reth_ethereum_primitives::TransactionSigned; use reth_evm_ethereum::EthEvmConfig; use reth_network_api::noop::NoopNetwork; use reth_provider::{ - test_utils::{MockEthProvider, NoopProvider}, + test_utils::{ExtendedAccount, MockEthProvider, NoopProvider}, PruneCheckpointReader, StageCheckpointReader, }; use reth_rpc_eth_api::{node::RpcNodeCoreAdapter, EthApiServer}; + use reth_rpc_eth_types::CredibleRpcConfig; use reth_storage_api::{BalProvider, BlockReader, BlockReaderIdExt, StateProviderFactory}; use reth_testing_utils::generators; use reth_transaction_pool::test_utils::{testing_pool, TestPool}; + use revm::bytecode::opcode::{ + JUMPDEST, JUMPI, KECCAK256, MSTORE, NUMBER, PUSH1, RETURN, REVERT, SLOAD, + }; type FakeEthApi

= EthApi< RpcNodeCoreAdapter, @@ -927,4 +931,192 @@ mod tests { "all: no percentiles were requested, so there should be no rewards result" ); } + + // Returns `_credibleBlocks[block.number]` (slot = keccak256(abi.encode(block.number, 1))). + fn credible_readback_bytecode() -> Bytes { + Bytes::from(vec![ + NUMBER, PUSH1, 0x00, MSTORE, PUSH1, 0x01, PUSH1, 0x20, MSTORE, PUSH1, 0x40, PUSH1, + 0x00, KECCAK256, // slot = keccak256(mem[0x00..0x40]) + SLOAD, PUSH1, 0x00, MSTORE, PUSH1, 0x20, PUSH1, 0x00, RETURN, + ]) + } + + // Reverts unless the marker slot is set; for methods whose output isn't the slot value. + fn credible_revert_unless_marker_bytecode() -> Bytes { + Bytes::from(vec![ + NUMBER, PUSH1, 0x00, MSTORE, PUSH1, 0x01, PUSH1, 0x20, MSTORE, PUSH1, 0x40, PUSH1, + 0x00, KECCAK256, SLOAD, PUSH1, 0x17, + JUMPI, // marker set -> jump to the JUMPDEST at byte 0x17 + PUSH1, 0x00, PUSH1, 0x00, REVERT, JUMPDEST, PUSH1, 0x00, PUSH1, 0x00, RETURN, + ]) + } + + /// Provider seeded with a single block whose `registry` account runs `bytecode`. + fn credible_mock_provider(registry: Address, bytecode: Bytes) -> MockEthProvider { + // DEV enables all forks (typed txs for createAccessList); Cancun needs the blob fields. + let provider = MockEthProvider::default().with_chain_spec((**DEV).clone()); + let hash = B256::repeat_byte(0xbb); + let header = Header { + number: 1, + gas_limit: 30_000_000, + excess_blob_gas: Some(0), + blob_gas_used: Some(0), + parent_beacon_block_root: Some(B256::ZERO), + ..Default::default() + }; + provider.add_block(hash, Block { header: header.clone(), body: Default::default() }); + provider.add_header(hash, header); + provider.add_account(registry, ExtendedAccount::new(0, U256::ZERO).with_bytecode(bytecode)); + provider + } + + /// Builds an `EthApi`, enabling the credible registry override when `registry` is set. + fn build_credible_eth_api(provider: MockEthProvider, registry: Option

) -> FakeEthApi { + let builder = EthApiBuilder::new( + provider.clone(), + testing_pool(), + NoopNetwork::default(), + EthEvmConfig::new(provider.chain_spec()), + ); + match registry { + Some(registry_address) => builder + .credible_config(CredibleRpcConfig { + registry_address: Some(registry_address), + ..Default::default() + }) + .build(), + None => builder.build(), + } + } + + /// A call request targeting the registry contract. + fn credible_call_request(registry: Address) -> TransactionRequest { + TransactionRequest { to: Some(TxKind::Call(registry)), ..Default::default() } + } + + /// A 32-byte value with `byte` in the last position, as returned by the readback contract. + fn u256_bytes(byte: u8) -> Bytes { + Bytes::from(B256::with_last_byte(byte)) + } + + // eth_call: disabled-config parity, explicit number, and block-number override. + #[tokio::test] + async fn credible_marker_readback_via_eth_call() { + let registry = Address::repeat_byte(0xcc); + let provider = credible_mock_provider(registry, credible_readback_bytecode()); + let request = credible_call_request(registry); + + // Disabled config: marker absent, slot reads 0. + let eth_api = build_credible_eth_api(provider.clone(), None); + let out = as EthApiServer<_, _, _, _, _, _>>::call( + ð_api, + request.clone(), + None, + None, + None, + ) + .await + .unwrap(); + assert_eq!(out, u256_bytes(0)); + + // Registry configured: marker injected at latest and at an explicit number. + let eth_api = build_credible_eth_api(provider, Some(registry)); + for block in [None, Some(BlockNumberOrTag::Number(1).into())] { + let out = as EthApiServer<_, _, _, _, _, _>>::call( + ð_api, + request.clone(), + block, + None, + None, + ) + .await + .unwrap(); + assert_eq!(out, u256_bytes(1)); + } + + // A block-number override selects the slot for the overridden block. + let overrides = BlockOverrides { number: Some(U256::from(1)), ..Default::default() }; + let out = as EthApiServer<_, _, _, _, _, _>>::call( + ð_api, + request, + None, + None, + Some(Box::new(overrides)), + ) + .await + .unwrap(); + assert_eq!(out, u256_bytes(1)); + } + + // eth_call `pending`: marker derives from the resolved env (latest + 1). + #[tokio::test] + async fn credible_marker_readback_via_eth_call_pending() { + let registry = Address::repeat_byte(0xcc); + let provider = credible_mock_provider(registry, credible_readback_bytecode()); + let eth_api = build_credible_eth_api(provider, Some(registry)); + + let out = as EthApiServer<_, _, _, _, _, _>>::call( + ð_api, + credible_call_request(registry), + Some(BlockNumberOrTag::Pending.into()), + None, + None, + ) + .await + .unwrap(); + assert_eq!(out, u256_bytes(1)); + } + + // eth_estimateGas reaches the marker (reverting probe only succeeds when set). + #[tokio::test] + async fn credible_marker_reaches_estimate_gas() { + let registry = Address::repeat_byte(0xcc); + let provider = credible_mock_provider(registry, credible_revert_unless_marker_bytecode()); + let request = credible_call_request(registry); + + let disabled = build_credible_eth_api(provider.clone(), None); + assert!( as EthApiServer<_, _, _, _, _, _>>::estimate_gas( + &disabled, + request.clone(), + None, + None, + None, + ) + .await + .is_err()); + + let enabled = build_credible_eth_api(provider, Some(registry)); + assert!( as EthApiServer<_, _, _, _, _, _>>::estimate_gas( + &enabled, request, None, None, None, + ) + .await + .is_ok()); + } + + // eth_createAccessList reaches the marker. + #[tokio::test] + async fn credible_marker_reaches_create_access_list() { + let registry = Address::repeat_byte(0xcc); + let provider = credible_mock_provider(registry, credible_revert_unless_marker_bytecode()); + let request = credible_call_request(registry); + + let disabled = build_credible_eth_api(provider.clone(), None); + let res = as EthApiServer<_, _, _, _, _, _>>::create_access_list( + &disabled, + request.clone(), + None, + None, + ) + .await + .unwrap(); + assert!(res.error.is_some()); + + let enabled = build_credible_eth_api(provider, Some(registry)); + let res = as EthApiServer<_, _, _, _, _, _>>::create_access_list( + &enabled, request, None, None, + ) + .await + .unwrap(); + assert!(res.error.is_none()); + } } diff --git a/crates/rpc/rpc/src/eth/helpers/mod.rs b/crates/rpc/rpc/src/eth/helpers/mod.rs index 2d0861461b4..537fdeb3aa2 100644 --- a/crates/rpc/rpc/src/eth/helpers/mod.rs +++ b/crates/rpc/rpc/src/eth/helpers/mod.rs @@ -13,6 +13,7 @@ mod pending_block; mod receipt; mod spec; mod state; +mod subscriptions; mod trace; mod transaction; diff --git a/crates/rpc/rpc/src/eth/helpers/subscriptions.rs b/crates/rpc/rpc/src/eth/helpers/subscriptions.rs new file mode 100644 index 00000000000..80788d1186f --- /dev/null +++ b/crates/rpc/rpc/src/eth/helpers/subscriptions.rs @@ -0,0 +1,14 @@ +//! Contains RPC handler implementations specific to streams subscriptions. + +use reth_rpc_convert::RpcConvert; +use reth_rpc_eth_api::{helpers::EthSubscriptions, RpcNodeCore}; +use reth_rpc_eth_types::EthApiError; + +use crate::EthApi; + +impl EthSubscriptions for EthApi +where + N: RpcNodeCore, + Rpc: RpcConvert, +{ +} diff --git a/crates/rpc/rpc/src/eth/helpers/transaction.rs b/crates/rpc/rpc/src/eth/helpers/transaction.rs index 70dca40b7aa..5a6ecf43ebd 100644 --- a/crates/rpc/rpc/src/eth/helpers/transaction.rs +++ b/crates/rpc/rpc/src/eth/helpers/transaction.rs @@ -97,8 +97,12 @@ where tracing::debug!(target: "rpc::eth", %hash, "forwarding raw transaction to forwarder"); let rlp_hex = hex::encode_prefixed(&tx); - // broadcast raw transaction to subscribers if there is any. - self.broadcast_raw_transaction(tx); + let retained_origin = self.credible_config().resolve_forwarded_origin(origin); + // Skip the public raw-transaction broadcast for retained-private forwarded txs, so + // they aren't exposed to subscribers. + if !retained_origin.is_private() { + self.broadcast_raw_transaction(tx); + } // The forwarder's response isn't guaranteed to be a tx hash, so only errors are // checked; the locally-computed hash is always returned to the caller. @@ -111,7 +115,6 @@ where .map_err(EthApiError::other)?; // Retain tx in local tx pool after forwarding, for local RPC usage. - let retained_origin = self.credible_config().resolve_forwarded_origin(origin); let _ = self.inner.add_pool_transaction(retained_origin, pool_transaction).await; return Ok(hash); diff --git a/crates/rpc/rpc/src/eth/pubsub.rs b/crates/rpc/rpc/src/eth/pubsub.rs index 6317c05ff0a..3bc18055f5f 100644 --- a/crates/rpc/rpc/src/eth/pubsub.rs +++ b/crates/rpc/rpc/src/eth/pubsub.rs @@ -2,7 +2,6 @@ use std::sync::Arc; -use alloy_consensus::{transaction::TxHashRef, BlockHeader, TxReceipt}; use alloy_primitives::TxHash; use alloy_rpc_types_eth::{ pubsub::{ @@ -16,12 +15,10 @@ use jsonrpsee::{ }; use reth_chain_state::CanonStateSubscriptions; use reth_network_api::NetworkInfo; -use reth_primitives_traits::TransactionMeta; -use reth_rpc_convert::{transaction::ConvertReceiptInput, RpcHeader}; +use reth_rpc_convert::RpcHeader; use reth_rpc_eth_api::{ - pubsub::EthPubSubApiServer, EthApiTypes, RpcConvert, RpcNodeCore, RpcTransaction, + helpers::EthSubscriptions, pubsub::EthPubSubApiServer, RpcConvert, RpcNodeCore, RpcTransaction, }; -use reth_rpc_eth_types::logs_utils; use reth_rpc_server_types::result::{internal_rpc_err, invalid_params_rpc_err}; use reth_storage_api::BlockNumReader; use reth_tasks::Runtime; @@ -54,7 +51,7 @@ impl EthPubSub { impl EthPubSub where - Eth: RpcNodeCore + EthApiTypes>, + Eth: EthSubscriptions, { /// Returns the current sync status for the `syncing` subscription pub fn sync_status(&self, is_syncing: bool) -> PubSubSyncStatus { @@ -73,14 +70,14 @@ where self.inner.full_pending_transaction_stream() } - /// Returns a stream that yields all new RPC blocks. + /// Returns a stream that yields new block headers. pub fn new_headers_stream(&self) -> impl Stream> { - self.inner.new_headers_stream() + self.inner.eth_api.header_stream() } - /// Returns a stream that yields all logs that match the given filter. + /// Returns a stream that yields matching logs. pub fn log_stream(&self, filter: Filter) -> impl Stream { - self.inner.log_stream(filter) + self.inner.eth_api.log_stream(filter) } /// The actual handler for an accepted [`EthPubSub::subscribe`] call. @@ -198,86 +195,11 @@ where } }; - let converter = self.inner.eth_api.converter(); - let stream = self.inner.eth_api.provider().canonical_state_stream().flat_map( - move |new_chain| { - // for each block in the new chain, build RPC receipts - let results: Vec<_> = new_chain - .committed() - .blocks_and_receipts() - .filter_map(|(block, receipts)| { - let block_hash = block.hash(); - let block_number = block.number(); - let base_fee = block.base_fee_per_gas(); - let excess_blob_gas = block.excess_blob_gas(); - let timestamp = block.timestamp(); - - let mut gas_used: u64 = 0; - let mut next_log_index: usize = 0; - - // build ConvertReceiptInput for each tx+receipt pair - // (same logic as eth_getBlockReceipts HTTP endpoint) - let inputs: Vec<_> = block - .transactions_recovered() - .zip(receipts.iter()) - .enumerate() - .filter_map(|(idx, (tx, receipt))| { - let gas_used_before = gas_used; - let next_log_index_before = next_log_index; - let cumulative_gas_used = receipt.cumulative_gas_used(); - - gas_used = cumulative_gas_used; - next_log_index += receipt.logs().len(); - - // apply transaction hash filter if provided - let matches = match &filter.transaction_hashes { - Some(hashes) if !hashes.is_empty() => { - hashes.contains(tx.tx_hash()) - } - _ => true, - }; - - matches.then(|| ConvertReceiptInput { - tx, - gas_used: cumulative_gas_used - gas_used_before, - next_log_index: next_log_index_before, - meta: TransactionMeta { - tx_hash: *tx.tx_hash(), - index: idx as u64, - block_hash, - block_number, - base_fee, - excess_blob_gas, - timestamp, - }, - receipt: receipt.clone(), - }) - }) - .collect(); - - if inputs.is_empty() { - return None; - } - - match converter.convert_receipts(inputs) { - Ok(rpc_receipts) => Some(rpc_receipts), - Err(err) => { - error!( - target = "rpc", - %err, - "Failed to convert receipts" - ); - None - } - } - }) - .collect(); - - futures::stream::iter(results) - }, - ); - - pipe_from_stream(accepted_sink, stream).await + pipe_from_stream( + accepted_sink, + self.inner.eth_api.transaction_receipts_stream(filter), + ) + .await } _ => Err(invalid_params_rpc_err("Unsupported subscription kind")), } @@ -287,7 +209,7 @@ where #[async_trait::async_trait] impl EthPubSubApiServer> for EthPubSub where - Eth: RpcNodeCore + EthApiTypes>, + Eth: EthSubscriptions, { /// Handler for `eth_subscribe` async fn subscribe( @@ -418,49 +340,3 @@ where self.eth_api.pool().new_pending_pool_transactions_listener() } } - -impl EthPubSubInner -where - Eth: EthApiTypes> + RpcNodeCore, -{ - /// Returns a stream that yields all new RPC blocks. - fn new_headers_stream(&self) -> impl Stream> { - let converter = self.eth_api.converter(); - self.eth_api.provider().canonical_state_stream().flat_map(|new_chain| { - let headers = new_chain - .committed() - .blocks_iter() - .filter_map(|block| { - match converter.convert_header(block.clone_sealed_header(), block.rlp_length()) - { - Ok(header) => Some(header), - Err(err) => { - error!(target = "rpc", %err, "Failed to convert header"); - None - } - } - }) - .collect::>(); - futures::stream::iter(headers) - }) - } - - /// Returns a stream that yields all logs that match the given filter. - fn log_stream(&self, filter: Filter) -> impl Stream { - self.eth_api - .provider() - .canonical_state_stream() - .map(move |canon_state| canon_state.block_receipts()) - .flat_map(futures::stream::iter) - .flat_map(move |(block_receipts, removed)| { - let all_logs = logs_utils::matching_block_logs_with_tx_hashes( - &filter, - block_receipts.block, - block_receipts.timestamp, - block_receipts.tx_receipts.iter().map(|(tx, receipt)| (*tx, receipt)), - removed, - ); - futures::stream::iter(all_logs) - }) - } -} diff --git a/crates/rpc/rpc/src/txpool.rs b/crates/rpc/rpc/src/txpool.rs index 51355dc1c12..89e3004b2e6 100644 --- a/crates/rpc/rpc/src/txpool.rs +++ b/crates/rpc/rpc/src/txpool.rs @@ -12,6 +12,7 @@ use reth_primitives_traits::NodePrimitives; use reth_rpc_api::TxPoolApiServer; use reth_rpc_convert::{RpcConvert, RpcTypes}; use reth_rpc_eth_api::RpcTransaction; +use reth_rpc_eth_types::CredibleRpcConfig; use reth_transaction_pool::{ AllPoolTransactions, PoolConsensusTx, PoolTransaction, TransactionPool, }; @@ -25,12 +26,13 @@ pub struct TxPoolApi { /// An interface to interact with the pool pool: Pool, converter: Eth, + credible_config: CredibleRpcConfig, } impl TxPoolApi { /// Creates a new instance of `TxpoolApi`. - pub const fn new(pool: Pool, converter: Eth) -> Self { - Self { pool, converter } + pub const fn new(pool: Pool, converter: Eth, credible_config: CredibleRpcConfig) -> Self { + Self { pool, converter, credible_config } } } @@ -61,13 +63,21 @@ where Ok(()) } + // With Credible Layer retention, private-origin pool txs must not leak before inclusion. + let hide_private = self.credible_config.hide_private_pool_txs(); let AllPoolTransactions { pending, queued } = self.pool.all_transactions(); let mut content = TxpoolContent::default(); for pending in pending { + if hide_private && pending.origin.is_private() { + continue; + } insert::<_, Eth>(&pending.transaction, &mut content.pending, &self.converter)?; } for queued in queued { + if hide_private && queued.origin.is_private() { + continue; + } insert::<_, Eth>(&queued.transaction, &mut content.queued, &self.converter)?; } @@ -88,7 +98,17 @@ where /// Handler for `txpool_status` async fn txpool_status(&self) -> RpcResult { trace!(target: "rpc::eth", "Serving txpool_status"); - let (pending, queued) = self.pool.pending_and_queued_txn_count(); + // Read totals and private counts from a single snapshot: reading them separately lets a tx + // moving between sub-pools leave a private tx in the public count under retention. + let ((pending, queued), (private_pending, private_queued)) = + self.pool.total_and_private_txn_counts(); + // With Credible Layer retention, exclude private-origin txs from the public counts. + if self.credible_config.hide_private_pool_txs() { + return Ok(TxpoolStatus { + pending: pending.saturating_sub(private_pending) as u64, + queued: queued.saturating_sub(private_queued) as u64, + }); + } Ok(TxpoolStatus { pending: pending as u64, queued: queued as u64 }) } @@ -111,17 +131,25 @@ where entry.insert(tx.nonce().to_string(), tx.into_inner().into()); } + // With Credible Layer retention, private-origin pool txs must not leak before inclusion. + let hide_private = self.credible_config.hide_private_pool_txs(); let AllPoolTransactions { pending, queued } = self.pool.all_transactions(); Ok(TxpoolInspect { - pending: pending.iter().fold(Default::default(), |mut acc, tx| { - insert(&tx.transaction, &mut acc); - acc - }), - queued: queued.iter().fold(Default::default(), |mut acc, tx| { - insert(&tx.transaction, &mut acc); - acc - }), + pending: pending.iter().filter(|tx| !hide_private || !tx.origin.is_private()).fold( + Default::default(), + |mut acc, tx| { + insert(&tx.transaction, &mut acc); + acc + }, + ), + queued: queued.iter().filter(|tx| !hide_private || !tx.origin.is_private()).fold( + Default::default(), + |mut acc, tx| { + insert(&tx.transaction, &mut acc); + acc + }, + ), }) } @@ -154,3 +182,60 @@ impl fmt::Debug for TxPoolApi { f.debug_struct("TxpoolApi").finish_non_exhaustive() } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::eth::helpers::types::EthRpcConverter; + use reth_chainspec::MAINNET; + use reth_rpc_eth_types::receipt::EthReceiptConverter; + use reth_transaction_pool::{ + test_utils::{testing_pool, MockTransaction}, + TransactionOrigin, + }; + + #[tokio::test] + async fn txpool_hides_retained_private_txs() { + let public_sender = Address::repeat_byte(0x11); + let private_sender = Address::repeat_byte(0x22); + + let pool = testing_pool(); + pool.add_transaction( + TransactionOrigin::External, + MockTransaction::eip1559().with_nonce(0).with_sender(public_sender), + ) + .await + .unwrap(); + pool.add_transaction( + TransactionOrigin::Private, + MockTransaction::eip1559().with_nonce(0).with_sender(private_sender), + ) + .await + .unwrap(); + + let converter = EthRpcConverter::new(EthReceiptConverter::new(MAINNET.clone())); + + // Retention enabled: the private-origin tx is hidden, the public one remains. + let api = TxPoolApi::new( + pool.clone(), + converter.clone(), + CredibleRpcConfig { retain_forwarded_txs_as_private: true, ..Default::default() }, + ); + let content = api.txpool_content().await.unwrap(); + assert!( + content.pending.contains_key(&public_sender) || + content.queued.contains_key(&public_sender) + ); + assert!( + !content.pending.contains_key(&private_sender) && + !content.queued.contains_key(&private_sender) + ); + let status = api.txpool_status().await.unwrap(); + assert_eq!(status.pending + status.queued, 1); + + // Retention disabled: both transactions are visible. + let api = TxPoolApi::new(pool, converter, CredibleRpcConfig::default()); + let status = api.txpool_status().await.unwrap(); + assert_eq!(status.pending + status.queued, 2); + } +} diff --git a/crates/storage/db-common/src/db_tool/mod.rs b/crates/storage/db-common/src/db_tool/mod.rs index e9d7f81b0f6..4f7c7fe2c28 100644 --- a/crates/storage/db-common/src/db_tool/mod.rs +++ b/crates/storage/db-common/src/db_tool/mod.rs @@ -36,7 +36,7 @@ impl DbTool { pub fn list(&self, filter: &ListFilter) -> Result<(Vec>, usize)> { let bmb = Rc::new(BMByte::from(&filter.search)); if bmb.is_none() && filter.has_search() { - eyre::bail!("Invalid search.") + eyre::bail!("Invalid search."); } let mut hits = 0; diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index 68c79a720dc..1605bf416d2 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -1,7 +1,7 @@ use crate::{ providers::{ - ConsistentProvider, ProviderNodeTypes, RocksDBProvider, StaticFileProvider, - StaticFileProviderRWRefMut, + ConsistentProvider, OverlayBuilder, OverlayStateProvider, OverlayStateProviderFactory, + ProviderNodeTypes, RocksDBProvider, StaticFileProvider, StaticFileProviderRWRefMut, }, AccountReader, BalProvider, BalStoreHandle, BlockHashReader, BlockIdReader, BlockNumReader, BlockReader, BlockReaderIdExt, BlockSource, CanonChainTracker, CanonStateNotifications, @@ -11,9 +11,9 @@ use crate::{ RocksDBProviderFactory, StageCheckpointReader, StateProviderBox, StateProviderFactory, StateReader, StaticFileProviderFactory, TransactionVariant, TransactionsProvider, }; -use alloy_consensus::transaction::TransactionMeta; +use alloy_consensus::{transaction::TransactionMeta, BlockHeader}; use alloy_eips::{BlockHashOrNumber, BlockId, BlockNumHash, BlockNumberOrTag}; -use alloy_primitives::{Address, BlockHash, BlockNumber, TxHash, TxNumber, B256}; +use alloy_primitives::{Address, BlockHash, BlockNumber, Bytes, TxHash, TxNumber, B256}; use alloy_rpc_types_engine::ForkchoiceState; use reth_chain_state::{ BlockState, CanonicalInMemoryState, ForkChoiceNotifications, ForkChoiceSubscriptions, @@ -29,9 +29,19 @@ use reth_primitives_traits::{ use reth_prune_types::{PruneCheckpoint, PruneSegment}; use reth_stages_types::{StageCheckpoint, StageId}; use reth_static_file_types::StaticFileSegment; -use reth_storage_api::{BlockBodyIndicesProvider, NodePrimitivesProvider, StorageChangeSetReader}; +use reth_storage_api::{ + BlockBodyIndicesProvider, NodePrimitivesProvider, RangeEnd, RangeResponse, RangeResult, + StateRangeProvider, StateRangeProviderFactory, StateRangeView, StorageChangeSetReader, + StorageRangeResult, +}; use reth_storage_errors::provider::ProviderResult; -use reth_trie::{HashedPostState, KeccakKeyHasher}; +use reth_trie::{ + hashed_cursor::{HashedCursor, HashedCursorFactory}, + metrics::TrieRootMetrics, + proof::{Proof, StorageProof}, + HashedPostState, KeccakKeyHasher, MultiProofTargets, StorageRoot, TrieInput, TrieInputSorted, + TrieType, +}; use revm::database::BundleState; use std::{ ops::{RangeBounds, RangeInclusive}, @@ -40,6 +50,11 @@ use std::{ }; use tracing::trace; +const SNAPSHOT_STATE_RETENTION: u64 = 128; + +type StateRangeDbProvider = as DatabaseProviderFactory>::Provider; +type HistoricalStateRangeProvider = OverlayStateProvider>; + /// The main type for interacting with the blockchain. /// /// This type serves as the main entry point for interacting with the blockchain and provides data @@ -141,6 +156,72 @@ impl BlockchainProvider { let latest_historical = self.database.history_by_block_hash(anchor_hash)?; Ok(state.state_provider(latest_historical)) } + + /// Returns a cursor-backed state view for a state root still only in canonical in-memory + /// blocks, overlaying their merged trie state on the persisted anchor. + fn block_state_range_provider( + &self, + state_root: B256, + ) -> ProviderResult>> { + let Some(matched) = self + .canonical_in_memory_state + .canonical_chain() + .find(|state| state.state_root() == state_root) + else { + return Ok(None) + }; + + // Merge each in-memory block's trie delta, anchor to `matched`, oldest to newest. + let blocks: Vec<_> = matched.chain().map(|state| state.block()).collect(); + let sorted: Vec<_> = + blocks.iter().rev().map(|block| (block.hashed_state(), block.trie_updates())).collect(); + let input = TrieInput::from_blocks_sorted( + sorted.iter().map(|(state, nodes)| (state.as_ref(), nodes.as_ref())), + ); + let merged = TrieInputSorted::from_unsorted(input); + + // Anchor at the persisted block; the overlay reverts any db-tip advancement past it + // via changesets, then the merged in-memory delta applies on top. + let overlay_factory = OverlayStateProviderFactory::new( + self.database.clone(), + OverlayBuilder::::new( + matched.anchor().hash, + self.database.changeset_cache(), + ) + .with_hashed_state_overlay(Some(merged.state)) + .with_trie_updates_overlay(Some(merged.nodes)), + ); + reth_storage_api::DatabaseProviderROFactory::database_provider_ro(&overlay_factory) + .map(Some) + } + + /// Returns a cursor-backed state view for a retained canonical state root. + fn historical_state_range_provider( + &self, + state_root: B256, + ) -> ProviderResult>> { + let provider = self.database.provider()?; + let Some(finish) = provider.get_stage_checkpoint(StageId::Finish)? else { return Ok(None) }; + let oldest = finish.block_number.saturating_sub(SNAPSHOT_STATE_RETENTION - 1); + let mut block_hash = None; + + for number in (oldest..=finish.block_number).rev() { + let Some(header) = provider.sealed_header(number)? else { continue }; + if header.state_root() == state_root { + block_hash = Some(header.hash()); + break + } + } + drop(provider); + + let Some(block_hash) = block_hash else { return Ok(None) }; + let overlay_factory = OverlayStateProviderFactory::new( + self.database.clone(), + OverlayBuilder::::new(block_hash, self.database.changeset_cache()), + ); + reth_storage_api::DatabaseProviderROFactory::database_provider_ro(&overlay_factory) + .map(Some) + } } impl NodePrimitivesProvider for BlockchainProvider { @@ -153,17 +234,148 @@ impl BalProvider for BlockchainProvider { } } +/// State range view backed by one resolved historical overlay. +struct HistoricalStateRangeView { + provider: HistoricalStateRangeProvider, +} + +impl StateRangeProviderFactory for BlockchainProvider { + /// Resolves a retained canonical state root into a pinned range view, preferring a still + /// in-memory block over the persisted-history fallback. + fn state_range_provider(&self, state_root: B256) -> ProviderResult> { + let provider = match self.block_state_range_provider(state_root)? { + Some(provider) => Some(provider), + None => self.historical_state_range_provider(state_root)?, + }; + Ok(provider + .map(|provider| Box::new(HistoricalStateRangeView { provider }) as StateRangeView)) + } +} + +impl StateRangeProvider for HistoricalStateRangeView { + fn account_range( + &self, + start: B256, + limit: B256, + response_bytes: usize, + ) -> RangeResult<(B256, Account)> { + let mut cursor = self.provider.hashed_account_cursor().map_err(ProviderError::Database)?; + + let mut accounts = Vec::new(); + let mut total_bytes = 0usize; + let mut end = RangeEnd::Exhausted; + + // Append before checking `limit`, so an empty `[start, limit]` still returns the account + // right past `limit`, provable as an empty range rather than a skipped one. + let mut entry = cursor.seek(start).map_err(ProviderError::Database)?; + while let Some((hash, account)) = entry { + total_bytes += 32 + 4 * 32; // hash + rough upper bound of the RLP account body + accounts.push((hash, account)); + if hash >= limit { + end = RangeEnd::HashLimit; + break + } + if total_bytes > response_bytes { + end = RangeEnd::ByteLimit; + break + } + entry = cursor.next().map_err(ProviderError::Database)?; + } + + Ok(RangeResponse { items: accounts, end }) + } + + fn storage_root_by_hash(&self, hashed_address: B256) -> ProviderResult { + let root = StorageRoot::new_hashed( + &self.provider, + &self.provider, + hashed_address, + Default::default(), + TrieRootMetrics::new(TrieType::Storage), + ) + .root() + .map_err(|err| ProviderError::Database(err.into()))?; + Ok(root) + } + + fn storage_range( + &self, + hashed_address: B256, + start: B256, + limit: B256, + response_bytes: usize, + ) -> StorageRangeResult { + // Distinguish an absent account from one with no storage, so callers don't silently + // omit it and shift later accounts' positions. + let mut account_cursor = + self.provider.hashed_account_cursor().map_err(ProviderError::Database)?; + let found = account_cursor.seek(hashed_address).map_err(ProviderError::Database)?; + if found.map(|(hash, _)| hash) != Some(hashed_address) { + return Ok(None) + } + + let mut cursor = + self.provider.hashed_storage_cursor(hashed_address).map_err(ProviderError::Database)?; + + let mut slots = Vec::new(); + let mut total_bytes = 0usize; + let mut end = RangeEnd::Exhausted; + + // Append before checking `limit`, so an empty `[start, limit]` still returns the slot + // right past `limit`, provable as an empty range rather than a skipped one. + let mut entry = cursor.seek(start).map_err(ProviderError::Database)?; + while let Some((hash, value)) = entry { + total_bytes += 64; + slots.push((hash, value)); + if hash >= limit { + end = RangeEnd::HashLimit; + break + } + if total_bytes > response_bytes { + end = RangeEnd::ByteLimit; + break + } + entry = cursor.next().map_err(ProviderError::Database)?; + } + + Ok(Some(RangeResponse { items: slots, end })) + } + + fn account_range_proof(&self, keys: &[B256]) -> ProviderResult> { + let multiproof = Proof::new(&self.provider, &self.provider) + .multiproof(MultiProofTargets::accounts(keys.iter().copied())) + .map_err(ProviderError::from)?; + Ok(multiproof + .account_subtree + .into_nodes_sorted() + .into_iter() + .map(|(_, bytes)| bytes) + .collect()) + } + + fn storage_range_proof( + &self, + hashed_address: B256, + keys: &[B256], + ) -> ProviderResult> { + let multiproof = StorageProof::new_hashed(&self.provider, &self.provider, hashed_address) + .storage_multiproof(keys.iter().copied().collect()) + .map_err(ProviderError::from)?; + Ok(multiproof.subtree.into_nodes_sorted().into_iter().map(|(_, bytes)| bytes).collect()) + } +} + impl DatabaseProviderFactory for BlockchainProvider { type DB = N::DB; type Provider = as DatabaseProviderFactory>::Provider; type ProviderRW = as DatabaseProviderFactory>::ProviderRW; fn database_provider_ro(&self) -> ProviderResult { - self.database.database_provider_ro() + DatabaseProviderFactory::database_provider_ro(&self.database) } fn database_provider_rw(&self) -> ProviderResult { - self.database.database_provider_rw() + DatabaseProviderFactory::database_provider_rw(&self.database) } } @@ -796,6 +1008,7 @@ impl StateReader for BlockchainProvider { #[cfg(test)] mod tests { + use super::SNAPSHOT_STATE_RETENTION; use crate::{ providers::BlockchainProvider, test_utils::{ @@ -804,8 +1017,9 @@ mod tests { }, BlockWriter, CanonChainTracker, ProviderFactory, SaveBlocksMode, }; + use alloy_consensus::constants::EMPTY_ROOT_HASH; use alloy_eips::{BlockHashOrNumber, BlockNumHash, BlockNumberOrTag}; - use alloy_primitives::{BlockNumber, TxNumber, B256}; + use alloy_primitives::{keccak256, Address, BlockNumber, TxNumber, B256, U256}; use itertools::Itertools; use rand::Rng; use reth_chain_state::{ @@ -819,17 +1033,23 @@ mod tests { use reth_execution_types::{ BlockExecutionOutput, BlockExecutionResult, Chain, ExecutionOutcome, }; - use reth_primitives_traits::{RecoveredBlock, SealedBlock, SignerRecoverable}; + use reth_primitives_traits::{ + Account, Block as _, RecoveredBlock, SealedBlock, SignerRecoverable, StorageEntry, + }; + use reth_stages_types::{StageCheckpoint, StageId}; use reth_storage_api::{ BlockBodyIndicesProvider, BlockHashReader, BlockIdReader, BlockNumReader, BlockReader, BlockReaderIdExt, BlockSource, ChangeSetReader, DBProvider, DatabaseProviderFactory, - HeaderProvider, ReceiptProvider, ReceiptProviderIdExt, StateProviderFactory, - StateWriteConfig, StateWriter, TransactionVariant, TransactionsProvider, + HashingWriter, HeaderProvider, RangeEnd, ReceiptProvider, ReceiptProviderIdExt, + StageCheckpointWriter, StateProviderFactory, StateRangeProvider, StateRangeProviderFactory, + StateRootProvider, StateWriteConfig, StateWriter, StorageRootProvider, TransactionVariant, + TransactionsProvider, }; use reth_testing_utils::generators::{ self, random_block, random_block_range, random_changeset_range, random_eoa_accounts, random_receipt, BlockParams, BlockRangeParams, }; + use reth_trie::{updates::TrieUpdates, ComputedTrieData, HashedPostState, HashedStorage}; use revm::database::{BundleState, OriginalValuesKnown}; use std::{ collections::BTreeMap, @@ -2616,4 +2836,344 @@ mod tests { Ok(()) } + + fn random_account(nonce: u64) -> (Address, Account) { + (Address::random(), Account { nonce, balance: U256::from(nonce), bytecode_hash: None }) + } + + /// [`BlockchainProvider::new`] needs a genesis header to initialize its chain tracker. + fn test_provider_factory_with_genesis() -> eyre::Result> { + let factory = create_test_provider_factory(); + let provider_rw = factory.provider_rw()?; + let mut rng = generators::rng(); + let genesis = + random_block(&mut rng, 0, BlockParams { tx_count: Some(0), ..Default::default() }); + provider_rw + .insert_block(&genesis.try_recover().expect("failed to seal block with senders"))?; + provider_rw.save_stage_checkpoint(StageId::Finish, StageCheckpoint::new(0))?; + provider_rw.commit()?; + Ok(factory) + } + + #[test] + fn state_range_provider_account_range_is_sorted_and_bounded() -> eyre::Result<()> { + let factory = test_provider_factory_with_genesis()?; + let provider_rw = factory.provider_rw()?; + + let accounts: Vec<_> = (0..5u64).map(random_account).collect(); + provider_rw.insert_account_for_hashing( + accounts.iter().map(|(address, account)| (*address, Some(*account))), + )?; + provider_rw.commit()?; + + let provider = BlockchainProvider::new(factory)?; + + let mut expected: Vec<_> = + accounts.iter().map(|(address, account)| (keccak256(address), *account)).collect(); + expected.sort_by_key(|(hash, _)| *hash); + let state = provider.state_range_provider(EMPTY_ROOT_HASH)?.unwrap(); + + let all = state.account_range(B256::ZERO, B256::repeat_byte(0xff), 10_000)?; + assert_eq!(all.end, RangeEnd::Exhausted); + assert_eq!(all.items, expected); + + // The limit exactly matches the second account's hash, so the range ends there rather + // than by exhausting the trie. + let bounded = state.account_range(B256::ZERO, expected[1].0, 10_000)?; + assert_eq!(bounded.end, RangeEnd::HashLimit); + assert_eq!(bounded.items, expected[..2]); + + Ok(()) + } + + #[test] + fn state_range_provider_account_range_respects_response_bytes() -> eyre::Result<()> { + let factory = test_provider_factory_with_genesis()?; + let provider_rw = factory.provider_rw()?; + + let accounts: Vec<_> = (0..5u64).map(random_account).collect(); + provider_rw.insert_account_for_hashing( + accounts.iter().map(|(address, account)| (*address, Some(*account))), + )?; + provider_rw.commit()?; + + let provider = BlockchainProvider::new(factory)?; + let state = provider.state_range_provider(EMPTY_ROOT_HASH)?.unwrap(); + + // Budget only fits a single account. + let partial = state.account_range(B256::ZERO, B256::repeat_byte(0xff), 150)?; + assert_eq!(partial.end, RangeEnd::ByteLimit); + assert_eq!(partial.items.len(), 1); + + Ok(()) + } + + #[test] + fn state_range_provider_storage_range_and_root() -> eyre::Result<()> { + let factory = test_provider_factory_with_genesis()?; + let provider_rw = factory.provider_rw()?; + + let (address, account) = random_account(1); + let hashed_address = keccak256(address); + provider_rw.insert_account_for_hashing([(address, Some(account))])?; + let slots = [ + StorageEntry { key: B256::with_last_byte(1), value: U256::from(10) }, + StorageEntry { key: B256::with_last_byte(2), value: U256::from(20) }, + ]; + provider_rw.insert_storage_for_hashing([(address, slots)])?; + provider_rw.commit()?; + + let provider = BlockchainProvider::new(factory)?; + let state = provider.state_range_provider(EMPTY_ROOT_HASH)?.unwrap(); + + let expected_root = provider.latest()?.storage_root(address, HashedStorage::default())?; + assert_eq!(state.storage_root_by_hash(hashed_address)?, expected_root); + + let returned = state + .storage_range(hashed_address, B256::ZERO, B256::repeat_byte(0xff), 10_000)? + .unwrap(); + assert_eq!(returned.end, RangeEnd::Exhausted); + let mut expected: Vec<_> = + slots.iter().map(|entry| (keccak256(entry.key), entry.value)).collect(); + expected.sort_by_key(|(hash, _)| *hash); + assert_eq!(returned.items, expected); + + // `start == limit == ZERO` means the first real slot's hash already reaches the limit. + let empty_window = + state.storage_range(hashed_address, B256::ZERO, B256::ZERO, 10_000)?.unwrap(); + assert_eq!(empty_window.end, RangeEnd::HashLimit); + assert_eq!(empty_window.items, expected[..1]); + + // An account absent from the trie is distinguished from one with no storage. + assert!(state + .storage_range(B256::repeat_byte(0xee), B256::ZERO, B256::repeat_byte(0xff), 10_000)? + .is_none()); + + Ok(()) + } + + #[test] + fn state_range_provider_proofs_start_at_the_real_root() -> eyre::Result<()> { + let factory = test_provider_factory_with_genesis()?; + let provider_rw = factory.provider_rw()?; + + let (address, account) = random_account(1); + let hashed_address = keccak256(address); + let hashed_slot = keccak256(B256::with_last_byte(1)); + provider_rw.insert_account_for_hashing([(address, Some(account))])?; + provider_rw.insert_storage_for_hashing([( + address, + [StorageEntry { key: B256::with_last_byte(1), value: U256::from(10) }], + )])?; + provider_rw.commit()?; + + let provider = BlockchainProvider::new(factory)?; + + // The first node of a sorted boundary proof is always the trie root, so this checks the + // proof was generated against the real, current root rather than a stale or empty one. + let state_root = provider.latest()?.state_root(HashedPostState::default())?; + let state = provider.state_range_provider(EMPTY_ROOT_HASH)?.unwrap(); + let account_proof = state.account_range_proof(&[hashed_address])?; + assert!(!account_proof.is_empty()); + assert_eq!(keccak256(&account_proof[0]), state_root); + + let storage_root = state.storage_root_by_hash(hashed_address)?; + let storage_proof = state.storage_range_proof(hashed_address, &[hashed_slot])?; + assert!(!storage_proof.is_empty()); + assert_eq!(keccak256(&storage_proof[0]), storage_root); + + Ok(()) + } + + #[test] + fn state_range_provider_serves_recent_root_and_rejects_expired_root() -> eyre::Result<()> { + let mut rng = generators::rng(); + let factory = create_test_provider_factory(); + let provider_rw = factory.provider_rw()?; + let expired_root = B256::repeat_byte(0x11); + let recent_root = B256::repeat_byte(0x22); + let mut parent = B256::ZERO; + + for number in 0..=SNAPSHOT_STATE_RETENTION { + let mut block = random_block( + &mut rng, + number, + BlockParams { parent: Some(parent), tx_count: Some(0), ..Default::default() }, + ) + .unseal(); + block.header.state_root = match number { + 0 => expired_root, + 64 => recent_root, + _ => EMPTY_ROOT_HASH, + }; + let block = block.seal_slow(); + parent = block.hash(); + provider_rw + .insert_block(&block.try_recover().expect("failed to seal block with senders"))?; + } + provider_rw.save_stage_checkpoint( + StageId::Finish, + StageCheckpoint::new(SNAPSHOT_STATE_RETENTION), + )?; + provider_rw.commit()?; + + let provider = BlockchainProvider::new(factory)?; + assert!(provider.state_range_provider(recent_root)?.is_some()); + assert!(provider.state_range_provider(expired_root)?.is_none()); + + Ok(()) + } + + #[test] + fn state_range_provider_serves_persisted_root_with_in_memory_overlay() -> eyre::Result<()> { + let mut rng = generators::rng(); + let (provider, _, _, _) = provider_with_random_blocks( + &mut rng, + TEST_BLOCKS_COUNT - 1, + 1, + BlockRangeParams::default(), + )?; + assert!(provider.canonical_in_memory_state.head_state().is_some()); + let provider_rw = provider.database.provider_rw()?; + provider_rw.save_stage_checkpoint( + StageId::Finish, + StageCheckpoint::new((TEST_BLOCKS_COUNT - 2) as u64), + )?; + provider_rw.commit()?; + + assert!(provider.state_range_provider(EMPTY_ROOT_HASH)?.is_some()); + + Ok(()) + } + + #[test] + fn state_range_provider_resolves_root_from_in_memory_block() -> eyre::Result<()> { + let mut rng = generators::rng(); + let factory = test_provider_factory_with_genesis()?; + let provider = BlockchainProvider::new(factory)?; + + let (address, account) = random_account(1); + let hashed_address = keccak256(address); + let mut hashed_state = HashedPostState::default(); + hashed_state.accounts.insert(hashed_address, Some(account)); + + // A root only the in-memory block carries, so a match proves the in-memory path (not + // persisted history, which has no block with this root) resolved it. + let unique_root = B256::repeat_byte(0x77); + let parent = provider.canonical_in_memory_state.get_canonical_head(); + let mut block = random_block( + &mut rng, + parent.number + 1, + BlockParams { parent: Some(parent.hash()), tx_count: Some(0), ..Default::default() }, + ) + .unseal(); + block.header.state_root = unique_root; + let block = block.seal_slow().try_recover().expect("failed to seal block with senders"); + + let trie_data = ComputedTrieData::new( + Arc::new(hashed_state.into_sorted()), + Arc::new(TrieUpdates::default().into_sorted()), + ); + let execution_output = BlockExecutionOutput { + result: BlockExecutionResult { + receipts: Default::default(), + requests: Default::default(), + gas_used: 0, + blob_gas_used: 0, + }, + state: Default::default(), + }; + let executed = ExecutedBlock::new(Arc::new(block), Arc::new(execution_output), trie_data); + provider + .canonical_in_memory_state + .update_chain(NewCanonicalChain::Commit { new: vec![executed] }); + + let state = + provider.state_range_provider(unique_root)?.expect("in-memory root must resolve"); + let range = state.account_range(B256::ZERO, B256::repeat_byte(0xff), 10_000)?; + assert_eq!(range.items, vec![(hashed_address, account)]); + + Ok(()) + } + + #[test] + fn state_range_provider_reverts_database_advancement_past_anchor() -> eyre::Result<()> { + let mut rng = generators::rng(); + let factory = test_provider_factory_with_genesis()?; + let provider = BlockchainProvider::new(factory)?; + let genesis = provider.canonical_in_memory_state.get_canonical_head(); + + // In-memory target block anchored on genesis, with a known account. + let (target_address, target_account) = random_account(1); + let target_hashed = keccak256(target_address); + let mut target_state = HashedPostState::default(); + target_state.accounts.insert(target_hashed, Some(target_account)); + + let unique_root = B256::repeat_byte(0x77); + let mut block = random_block( + &mut rng, + genesis.number + 1, + BlockParams { parent: Some(genesis.hash()), tx_count: Some(0), ..Default::default() }, + ) + .unseal(); + block.header.state_root = unique_root; + let block = block.seal_slow().try_recover().expect("failed to seal block with senders"); + let trie_data = ComputedTrieData::new( + Arc::new(target_state.into_sorted()), + Arc::new(TrieUpdates::default().into_sorted()), + ); + let execution_output = BlockExecutionOutput { + result: BlockExecutionResult { + receipts: Default::default(), + requests: Default::default(), + gas_used: 0, + blob_gas_used: 0, + }, + state: Default::default(), + }; + let executed = ExecutedBlock::new(Arc::new(block), Arc::new(execution_output), trie_data); + provider + .canonical_in_memory_state + .update_chain(NewCanonicalChain::Commit { new: vec![executed] }); + + // Persistence races ahead: a *different* block, with a *different* account, lands in + // the database on top of the same genesis anchor while the in-memory chain above still + // references genesis as its anchor. + let (noise_address, noise_account) = random_account(2); + let noise_block = random_block( + &mut rng, + genesis.number + 1, + BlockParams { parent: Some(genesis.hash()), tx_count: Some(0), ..Default::default() }, + ) + .try_recover() + .expect("failed to seal block with senders"); + let mut noise_state = HashedPostState::default(); + noise_state.accounts.insert(keccak256(noise_address), Some(noise_account)); + let provider_rw = provider.database.provider_rw()?; + provider_rw.append_blocks_with_state( + vec![noise_block], + &ExecutionOutcome { + bundle: BundleState::new( + [(noise_address, None, Some(noise_account.into()), Default::default())], + [[(noise_address, Some(None), [])]], + [], + ), + first_block: genesis.number + 1, + ..Default::default() + }, + noise_state.into_sorted(), + )?; + provider_rw + .save_stage_checkpoint(StageId::Finish, StageCheckpoint::new(genesis.number + 1))?; + provider_rw.commit()?; + + // Resolving the in-memory root must revert the database's advancement back to genesis, + // so the noise account must not leak into the result. + let state = + provider.state_range_provider(unique_root)?.expect("in-memory root must resolve"); + let range = state.account_range(B256::ZERO, B256::repeat_byte(0xff), 10_000)?; + assert_eq!(range.items, vec![(target_hashed, target_account)]); + + Ok(()) + } } diff --git a/crates/storage/provider/src/providers/database/mod.rs b/crates/storage/provider/src/providers/database/mod.rs index 40ec3834865..9f6785774af 100644 --- a/crates/storage/provider/src/providers/database/mod.rs +++ b/crates/storage/provider/src/providers/database/mod.rs @@ -204,6 +204,11 @@ impl ProviderFactory { self } + /// Returns the shared changeset cache. + pub(crate) fn changeset_cache(&self) -> ChangesetCache { + self.changeset_cache.clone() + } + /// Sets the minimum pruning distance for an existing [`ProviderFactory`]. /// /// This controls the minimum distance from tip required before pruning can occur. diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index a9a8b2ebbe5..a241ed7cff3 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -5125,7 +5125,6 @@ mod tests { }), ComputedTrieData { sorted: SortedTrieData::new(Arc::new(hashed_state), Default::default()), - ..Default::default() }, ); blocks.push(executed); diff --git a/crates/storage/provider/src/providers/rocksdb/invariants.rs b/crates/storage/provider/src/providers/rocksdb/invariants.rs index 232c629afe1..ded24e92438 100644 --- a/crates/storage/provider/src/providers/rocksdb/invariants.rs +++ b/crates/storage/provider/src/providers/rocksdb/invariants.rs @@ -6,17 +6,16 @@ use super::RocksDBProvider; use crate::StaticFileProviderFactory; -use alloy_consensus::transaction::TxHashRef; use alloy_primitives::BlockNumber; -use rayon::prelude::*; use reth_chainspec::{ChainSpecProvider, EthChainSpec}; use reth_db::models::{storage_sharded_key::StorageShardedKey, ShardedKey}; -use reth_db_api::tables; +use reth_db_api::{table::Value, tables}; +use reth_primitives_traits::NodePrimitives; use reth_stages_types::StageId; use reth_static_file_types::StaticFileSegment; use reth_storage_api::{ BlockBodyIndicesProvider, ChangeSetReader, DBProvider, StageCheckpointReader, - StorageChangeSetReader, StorageSettingsCache, TransactionsProvider, + StorageChangeSetReader, StorageSettingsCache, TransactionsProviderExt, }; use reth_storage_errors::provider::ProviderResult; use std::collections::HashSet; @@ -55,12 +54,13 @@ impl RocksDBProvider { Provider: DBProvider + StageCheckpointReader + StorageSettingsCache - + StaticFileProviderFactory + BlockBodyIndicesProvider + StorageChangeSetReader + ChangeSetReader - + TransactionsProvider - + ChainSpecProvider, + + ChainSpecProvider + + StaticFileProviderFactory< + Primitives: NodePrimitives, + >, { let mut unwind_target: Option = None; @@ -101,9 +101,10 @@ impl RocksDBProvider { where Provider: DBProvider + StageCheckpointReader - + StaticFileProviderFactory + BlockBodyIndicesProvider - + TransactionsProvider, + + StaticFileProviderFactory< + Primitives: NodePrimitives, + >, { let checkpoint = provider .get_stage_checkpoint(StageId::TransactionLookup)? @@ -205,8 +206,8 @@ impl RocksDBProvider { /// Prunes `TransactionHashNumbers` entries for transactions in the given range. /// - /// This fetches transactions from the provider, reads their hashes in parallel, - /// and deletes the corresponding entries from `RocksDB` by key. This approach is more + /// This fetches transaction hashes from the provider and deletes the corresponding + /// entries from `RocksDB` by key. This approach is more /// scalable than iterating all rows because it only processes the transactions that /// need to be pruned. /// @@ -221,18 +222,17 @@ impl RocksDBProvider { tx_range: std::ops::RangeInclusive, ) -> ProviderResult<()> where - Provider: TransactionsProvider, + Provider: StaticFileProviderFactory< + Primitives: NodePrimitives, + >, { if tx_range.is_empty() { return Ok(()); } - // Fetch transactions in the range and read their hashes in parallel. - let hashes: Vec<_> = provider - .transactions_by_tx_range(tx_range.clone())? - .into_par_iter() - .map(|tx| *tx.tx_hash()) - .collect(); + let hashes = provider + .static_file_provider() + .transaction_hashes_by_range(*tx_range.start()..tx_range.end().saturating_add(1))?; if !hashes.is_empty() { tracing::info!( @@ -244,7 +244,7 @@ impl RocksDBProvider { ); let mut batch = self.batch(); - for hash in hashes { + for (hash, _) in hashes { batch.delete::(hash)?; } batch.commit()?; diff --git a/crates/storage/provider/src/providers/state/overlay.rs b/crates/storage/provider/src/providers/state/overlay.rs index d9373603e5b..f8894ca2589 100644 --- a/crates/storage/provider/src/providers/state/overlay.rs +++ b/crates/storage/provider/src/providers/state/overlay.rs @@ -52,6 +52,9 @@ pub(crate) struct OverlayStateProviderMetrics { database_provider_ro_duration: Histogram, /// Number of cache misses when fetching [`Overlay`]s from the overlay cache. overlay_cache_misses: Counter, + /// Number of managed overlay creations skipped because the reused sparse trie already covers + /// the DB tip to parent range. + sparse_trie_overlay_skips: Counter, } /// Contains all fields required to initialize an [`OverlayStateProvider`]. @@ -61,6 +64,15 @@ pub(super) struct Overlay { pub(super) hashed_post_state: Arc, } +impl Overlay { + fn empty() -> Self { + Self { + trie_updates: Arc::new(TrieUpdatesSorted::default()), + hashed_post_state: Arc::new(HashedPostStateSorted::default()), + } + } +} + /// Source of overlay data for [`OverlayStateProviderFactory`]. #[derive(Debug, Clone)] pub(super) enum OverlaySource { @@ -97,6 +109,8 @@ pub struct OverlayBuilder { overlay_source: Option>, /// Changeset cache handle for retrieving trie changesets changeset_cache: ChangesetCache, + /// Anchor hash of the reused sparse trie, if this task reused one. + reused_sparse_trie_anchor_hash: Option, /// Metrics for tracking provider operations metrics: OverlayStateProviderMetrics, } @@ -108,6 +122,7 @@ impl OverlayBuilder { parent_hash, overlay_source: None, changeset_cache, + reused_sparse_trie_anchor_hash: None, metrics: OverlayStateProviderMetrics::default(), } } @@ -120,6 +135,13 @@ impl OverlayBuilder { self } + /// Skips managed overlay construction when the sparse trie was reused and the DB tip is + /// already covered by its anchor-to-parent range. + pub const fn with_skip_overlay_for_reused_sparse_trie(mut self, anchor_hash: B256) -> Self { + self.reused_sparse_trie_anchor_hash = Some(anchor_hash); + self + } + /// Set the state trie overlay manager used to resolve in-memory parent state. pub fn with_state_trie_overlay_manager( mut self, @@ -137,15 +159,38 @@ impl OverlayBuilder { mut self, hashed_state_overlay: Option>, ) -> Self { - if let Some(state) = hashed_state_overlay { + if let Some(new_state) = hashed_state_overlay { match &mut self.overlay_source { - Some(OverlaySource::Managed { state: managed_state, .. }) => { - *managed_state = state; - } - _ => { + Some( + OverlaySource::Managed { state, .. } | OverlaySource::Immediate { state, .. }, + ) => *state = new_state, + None => { self.overlay_source = Some(OverlaySource::Immediate { trie: Arc::new(TrieUpdatesSorted::default()), - state, + state: new_state, + }); + } + } + } + self + } + + /// Set the trie updates overlay. + /// + /// Only applies to an immediate overlay: a managed overlay's trie updates are resolved from + /// its manager instead. + pub fn with_trie_updates_overlay( + mut self, + trie_updates_overlay: Option>, + ) -> Self { + if let Some(trie) = trie_updates_overlay { + match &mut self.overlay_source { + Some(OverlaySource::Immediate { trie: existing, .. }) => *existing = trie, + Some(OverlaySource::Managed { .. }) => {} + None => { + self.overlay_source = Some(OverlaySource::Immediate { + trie, + state: Arc::new(HashedPostStateSorted::default()), }); } } @@ -213,6 +258,18 @@ impl OverlayBuilder { } } + /// Returns true if managed overlay resolution can be skipped for this builder. + fn should_skip_overlay_for_reused_sparse_trie(&self, db_tip_hash: B256) -> bool { + let Some(anchor_hash) = self.reused_sparse_trie_anchor_hash else { return false }; + + match &self.overlay_source { + Some(OverlaySource::Managed { manager, state }) if state.is_empty() => { + manager.contains_hash(self.parent_hash, anchor_hash, db_tip_hash) + } + _ => false, + } + } + /// Returns the block which is at the tip of the DB, i.e. the block which the state tables of /// the DB are currently synced to. fn get_db_tip_block(&self, provider: &Provider) -> ProviderResult @@ -395,14 +452,28 @@ impl OverlayBuilder { (trie_updates, hashed_state_updates) } else { // If no reverts are needed then the db tip is the anchor hash. Use overlays directly. - let (trie_updates, hashed_state) = self.resolve_overlays(db_tip_block.hash)?; + if self.should_skip_overlay_for_reused_sparse_trie(db_tip_block.hash) { + debug!( + target: "providers::state::overlay", + parent_hash = %self.parent_hash, + db_tip_hash = %db_tip_block.hash, + sparse_trie_anchor_hash = ?self.reused_sparse_trie_anchor_hash, + "Skipping overlay construction because reused sparse trie covers DB tip to parent" + ); + + self.metrics.sparse_trie_overlay_skips.increment(1); + + return Ok(Overlay::empty()) + } + + let (trie_updates, hashed_post_state) = self.resolve_overlays(db_tip_block.hash)?; retrieve_trie_reverts_duration = Duration::ZERO; retrieve_hashed_state_reverts_duration = Duration::ZERO; trie_updates_total_len = trie_updates.total_len(); - hashed_state_updates_total_len = hashed_state.total_len(); + hashed_state_updates_total_len = hashed_post_state.total_len(); - (trie_updates, hashed_state) + (trie_updates, hashed_post_state) }; // Record metrics @@ -466,6 +537,15 @@ impl OverlayStateProviderFactory { self } + /// Skips managed overlay construction when this factory is used by a task that reused a sparse + /// trie covering the DB tip to parent range. + pub fn with_skip_overlay_for_reused_sparse_trie(mut self, anchor_hash: B256) -> Self { + self.overlay_builder = + self.overlay_builder.with_skip_overlay_for_reused_sparse_trie(anchor_hash); + self.overlay_cache = Default::default(); + self + } + /// Extends the existing hashed state overlay with the given [`HashedPostStateSorted`]. pub fn with_extended_hashed_state_overlay(mut self, other: HashedPostStateSorted) -> Self { self.overlay_builder = self.overlay_builder.with_extended_hashed_state_overlay(other); @@ -699,4 +779,22 @@ mod tests { }; assert_eq!(state.total_len(), 1); } + + #[test] + fn managed_overlay_skip_requires_reused_sparse_trie_and_no_immediate_state() { + let parent_hash = B256::with_last_byte(1); + let builder = OverlayBuilder::::new(parent_hash, ChangesetCache::default()) + .with_state_trie_overlay_manager(StateTrieOverlayManager::default()); + assert!(!builder.should_skip_overlay_for_reused_sparse_trie(parent_hash)); + + let builder = builder.with_skip_overlay_for_reused_sparse_trie(parent_hash); + assert!(builder.should_skip_overlay_for_reused_sparse_trie(parent_hash)); + assert!(!builder.should_skip_overlay_for_reused_sparse_trie(B256::with_last_byte(3))); + + let hashed_state = HashedPostState::default() + .with_accounts([(B256::with_last_byte(2), Some(Account::default()))]) + .into_sorted(); + let builder = builder.with_extended_hashed_state_overlay(hashed_state); + assert!(!builder.should_skip_overlay_for_reused_sparse_trie(parent_hash)); + } } diff --git a/crates/storage/provider/src/test_utils/mock.rs b/crates/storage/provider/src/test_utils/mock.rs index 59b90e02643..68244202e1d 100644 --- a/crates/storage/provider/src/test_utils/mock.rs +++ b/crates/storage/provider/src/test_utils/mock.rs @@ -2,8 +2,10 @@ use crate::{ traits::{BlockSource, ReceiptProvider}, AccountReader, BalProvider, BalStoreHandle, BlockHashReader, BlockIdReader, BlockNumReader, BlockReader, BlockReaderIdExt, ChainSpecProvider, ChangeSetReader, HeaderProvider, - PruneCheckpointReader, ReceiptProviderIdExt, StateProvider, StateProviderBox, - StateProviderFactory, StateReader, StateRootProvider, TransactionVariant, TransactionsProvider, + PruneCheckpointReader, RangeEnd, RangeResponse, RangeResult, ReceiptProviderIdExt, + StateProvider, StateProviderBox, StateProviderFactory, StateRangeProvider, + StateRangeProviderFactory, StateRangeView, StateReader, StateRootProvider, StorageRangeResult, + TransactionVariant, TransactionsProvider, }; use alloy_consensus::{ constants::EMPTY_ROOT_HASH, @@ -43,10 +45,13 @@ use reth_trie::{ MultiProofTargets, StorageMultiProof, StorageProof, TrieInput, }; use std::{ - collections::BTreeMap, + collections::{BTreeMap, VecDeque}, fmt::Debug, ops::{RangeBounds, RangeInclusive}, - sync::Arc, + sync::{ + atomic::{AtomicBool, AtomicUsize, Ordering}, + Arc, + }, }; use tokio::sync::broadcast; @@ -70,10 +75,43 @@ pub struct MockEthProvider>>, /// Local BAL store handle pub bal_store: BalStoreHandle, + /// Whether snap state reads should fail for handler error-path tests. + snap_state_reads_fail: Arc, + /// Whether a snap state range view is available. + snap_state_range_available: Arc, + /// Number of snap state range view resolutions. + snap_state_range_resolutions: Arc, + /// Account range returned to snap handler tests. + snap_account_range: Arc>, + /// Storage roots returned to snap handler tests, keyed by hashed address. + snap_storage_roots: Arc>>, + /// Storage ranges returned to snap handler tests. + snap_storage_ranges: Arc>>, + /// Storage range requests observed by snap handler tests. + snap_storage_range_requests: Arc>>, + /// Account proof returned to snap handler tests. + snap_account_proof: Arc>>>, + /// Storage proof returned to snap handler tests. + snap_storage_proof: Arc>>>, tx: TxMock, prune_modes: Arc, } +/// Optional mock account entries paired with why the range ended. +type MockAccountRange = Option<(Vec<(B256, Account)>, RangeEnd)>; +/// Outcome of a queued mock `storage_range` call. +#[derive(Debug, Clone)] +enum MockStorageRangeOutcome { + /// The provider fails this call (e.g. simulating a database error). + Error, + /// The requested account isn't present in the pinned state. + AccountMissing, + /// The account is present; these are its slots and why the range ended. + Found(Vec<(B256, U256)>, RangeEnd), +} +/// Hashed address, origin, limit, and byte budget of a mock storage range request. +type MockStorageRangeRequest = (B256, B256, B256, usize); + impl Clone for MockEthProvider where T::Block: Clone, @@ -88,6 +126,15 @@ where state_roots: self.state_roots.clone(), block_body_indices: self.block_body_indices.clone(), bal_store: self.bal_store.clone(), + snap_state_reads_fail: self.snap_state_reads_fail.clone(), + snap_state_range_available: self.snap_state_range_available.clone(), + snap_state_range_resolutions: self.snap_state_range_resolutions.clone(), + snap_account_range: self.snap_account_range.clone(), + snap_storage_roots: self.snap_storage_roots.clone(), + snap_storage_ranges: self.snap_storage_ranges.clone(), + snap_storage_range_requests: self.snap_storage_range_requests.clone(), + snap_account_proof: self.snap_account_proof.clone(), + snap_storage_proof: self.snap_storage_proof.clone(), tx: self.tx.clone(), prune_modes: self.prune_modes.clone(), } @@ -106,6 +153,15 @@ impl MockEthProvider { state_roots: Default::default(), block_body_indices: Default::default(), bal_store: Default::default(), + snap_state_reads_fail: Default::default(), + snap_state_range_available: Default::default(), + snap_state_range_resolutions: Default::default(), + snap_account_range: Default::default(), + snap_storage_roots: Default::default(), + snap_storage_ranges: Default::default(), + snap_storage_range_requests: Default::default(), + snap_account_proof: Default::default(), + snap_storage_proof: Default::default(), tx: Default::default(), prune_modes: Default::default(), } @@ -113,6 +169,72 @@ impl MockEthProvider { } impl MockEthProvider { + /// Makes snap state reads return provider errors when `fail` is true. + pub fn set_snap_state_reads_fail(&self, fail: bool) { + self.snap_state_reads_fail.store(fail, Ordering::Relaxed); + } + + /// Sets the available account range returned to snap handler tests. + pub fn set_snap_account_range(&self, accounts: Vec<(B256, Account)>, end: RangeEnd) { + self.snap_state_range_available.store(true, Ordering::Relaxed); + *self.snap_account_range.lock() = Some((accounts, end)); + } + + /// Sets an account's storage root for snap handler tests. + pub fn set_snap_storage_root(&self, hashed_address: B256, storage_root: B256) { + self.snap_storage_roots.lock().insert(hashed_address, storage_root); + } + + /// Adds an available storage range for the next snap handler call. + pub fn push_snap_storage_range(&self, slots: Vec<(B256, U256)>, end: RangeEnd) { + self.snap_state_range_available.store(true, Ordering::Relaxed); + self.snap_storage_ranges.lock().push_back(MockStorageRangeOutcome::Found(slots, end)); + } + + /// Marks the account for the next snap handler call as absent from the pinned state. + pub fn push_missing_snap_storage_account(&self) { + self.snap_state_range_available.store(true, Ordering::Relaxed); + self.snap_storage_ranges.lock().push_back(MockStorageRangeOutcome::AccountMissing); + } + + /// Adds an unavailable storage range for the next snap handler call. + pub fn push_unavailable_snap_storage_range(&self) { + self.snap_state_range_available.store(true, Ordering::Relaxed); + self.snap_storage_ranges.lock().push_back(MockStorageRangeOutcome::Error); + } + + /// Returns the number of queued storage ranges for snap handler tests. + pub fn snap_storage_ranges_remaining(&self) -> usize { + self.snap_storage_ranges.lock().len() + } + + /// Returns the storage range requests observed by snap handler tests. + pub fn snap_storage_range_requests(&self) -> Vec<(B256, B256, B256, usize)> { + self.snap_storage_range_requests.lock().clone() + } + + /// Returns the number of snap state range view resolutions. + pub fn snap_state_range_resolutions(&self) -> usize { + self.snap_state_range_resolutions.load(Ordering::Relaxed) + } + + /// Sets the account proof returned to snap handler tests. + pub fn set_snap_account_proof(&self, proof: Option>) { + *self.snap_account_proof.lock() = proof; + } + + /// Sets the storage proof returned to snap handler tests. + pub fn set_snap_storage_proof(&self, proof: Option>) { + *self.snap_storage_proof.lock() = proof; + } + + fn ensure_snap_state_reads_succeed(&self) -> ProviderResult<()> { + if self.snap_state_reads_fail.load(Ordering::Relaxed) { + return Err(ProviderError::BestBlockNotFound) + } + Ok(()) + } + /// Add block to local block store pub fn add_block(&self, hash: B256, block: T::Block) { self.add_header(hash, block.header().clone()); @@ -190,6 +312,15 @@ impl MockEthProvider { state_roots: self.state_roots, block_body_indices: self.block_body_indices, bal_store: self.bal_store, + snap_state_reads_fail: self.snap_state_reads_fail, + snap_state_range_available: self.snap_state_range_available, + snap_state_range_resolutions: self.snap_state_range_resolutions, + snap_account_range: self.snap_account_range, + snap_storage_roots: self.snap_storage_roots, + snap_storage_ranges: self.snap_storage_ranges, + snap_storage_range_requests: self.snap_storage_range_requests, + snap_account_proof: self.snap_account_proof, + snap_storage_proof: self.snap_storage_proof, tx: self.tx, prune_modes: self.prune_modes, } @@ -223,6 +354,82 @@ impl BalProvider for MockEthProvider } } +impl StateRangeProviderFactory for MockEthProvider +where + T: NodePrimitives, + T::Block: Clone, + ChainSpec: Send + Sync + 'static, +{ + fn state_range_provider(&self, _state_root: B256) -> ProviderResult> { + self.snap_state_range_resolutions.fetch_add(1, Ordering::Relaxed); + self.ensure_snap_state_reads_succeed()?; + if !self.snap_state_range_available.load(Ordering::Relaxed) { + return Ok(None) + } + Ok(Some(Box::new(self.clone()))) + } +} + +impl StateRangeProvider for MockEthProvider { + fn account_range( + &self, + _start: B256, + _limit: B256, + _response_bytes: usize, + ) -> RangeResult<(B256, Account)> { + self.ensure_snap_state_reads_succeed()?; + let (items, end) = + self.snap_account_range.lock().clone().ok_or(ProviderError::BestBlockNotFound)?; + Ok(RangeResponse { items, end }) + } + + fn storage_root_by_hash(&self, hashed_address: B256) -> ProviderResult { + self.ensure_snap_state_reads_succeed()?; + self.snap_storage_roots + .lock() + .get(&hashed_address) + .copied() + .ok_or(ProviderError::BestBlockNotFound) + } + + fn storage_range( + &self, + hashed_address: B256, + start: B256, + limit: B256, + response_bytes: usize, + ) -> StorageRangeResult { + self.ensure_snap_state_reads_succeed()?; + self.snap_storage_range_requests.lock().push(( + hashed_address, + start, + limit, + response_bytes, + )); + let outcome = + self.snap_storage_ranges.lock().pop_front().ok_or(ProviderError::BestBlockNotFound)?; + match outcome { + MockStorageRangeOutcome::Error => Err(ProviderError::BestBlockNotFound), + MockStorageRangeOutcome::AccountMissing => Ok(None), + MockStorageRangeOutcome::Found(items, end) => Ok(Some(RangeResponse { items, end })), + } + } + + fn account_range_proof(&self, _keys: &[B256]) -> ProviderResult> { + self.ensure_snap_state_reads_succeed()?; + self.snap_account_proof.lock().clone().ok_or(ProviderError::BestBlockNotFound) + } + + fn storage_range_proof( + &self, + _hashed_address: B256, + _keys: &[B256], + ) -> ProviderResult> { + self.ensure_snap_state_reads_succeed()?; + self.snap_storage_proof.lock().clone().ok_or(ProviderError::BestBlockNotFound) + } +} + /// An extended account for local store #[derive(Debug, Clone)] pub struct ExtendedAccount { @@ -933,6 +1140,7 @@ impl StatePr for MockEthProvider { fn latest(&self) -> ProviderResult { + self.ensure_snap_state_reads_succeed()?; Ok(Box::new(self.clone())) } diff --git a/crates/storage/provider/src/traits/full.rs b/crates/storage/provider/src/traits/full.rs index 070d8bfefd5..d3e6f36be45 100644 --- a/crates/storage/provider/src/traits/full.rs +++ b/crates/storage/provider/src/traits/full.rs @@ -3,8 +3,8 @@ use crate::{ AccountReader, BalProvider, BlockReader, BlockReaderIdExt, ChainSpecProvider, ChangeSetReader, DatabaseProviderFactory, HashedPostStateProvider, PruneCheckpointReader, - RocksDBProviderFactory, StageCheckpointReader, StateProviderFactory, StateReader, - StaticFileProviderFactory, + RocksDBProviderFactory, StageCheckpointReader, StateProviderFactory, StateRangeProviderFactory, + StateReader, StaticFileProviderFactory, }; use reth_chain_state::{ CanonStateSubscriptions, ForkChoiceSubscriptions, PersistedBlockSubscriptions, @@ -34,6 +34,7 @@ pub trait FullProvider: > + AccountReader + BalProvider + StateProviderFactory + + StateRangeProviderFactory + StateReader + HashedPostStateProvider + ChainSpecProvider @@ -71,6 +72,7 @@ impl FullProvider for T where > + AccountReader + BalProvider + StateProviderFactory + + StateRangeProviderFactory + StateReader + HashedPostStateProvider + ChainSpecProvider diff --git a/crates/storage/storage-api/src/noop.rs b/crates/storage/storage-api/src/noop.rs index 80726e17c27..49a81d4ff26 100644 --- a/crates/storage/storage-api/src/noop.rs +++ b/crates/storage/storage-api/src/noop.rs @@ -7,8 +7,9 @@ use crate::{ BlockIdReader, BlockNumReader, BlockReader, BlockReaderIdExt, BlockSource, BytecodeReader, ChangeSetReader, HashedPostStateProvider, HeaderProvider, NodePrimitivesProvider, PruneCheckpointReader, ReceiptProvider, ReceiptProviderIdExt, StageCheckpointReader, - StateProofProvider, StateProvider, StateProviderBox, StateProviderFactory, StateReader, - StateRootProvider, StorageRootProvider, TransactionVariant, TransactionsProvider, + StateProofProvider, StateProvider, StateProviderBox, StateProviderFactory, + StateRangeProviderFactory, StateRangeView, StateReader, StateRootProvider, StorageRootProvider, + TransactionVariant, TransactionsProvider, }; #[cfg(feature = "db-api")] @@ -117,6 +118,12 @@ impl BalProvider for NoopProvider { } } +impl StateRangeProviderFactory for NoopProvider { + fn state_range_provider(&self, _state_root: B256) -> ProviderResult> { + Ok(None) + } +} + /// Noop implementation for testing purposes impl BlockHashReader for NoopProvider { fn block_hash(&self, _number: u64) -> ProviderResult> { diff --git a/crates/storage/storage-api/src/trie.rs b/crates/storage/storage-api/src/trie.rs index b5c7de7b652..dc30bbf52ad 100644 --- a/crates/storage/storage-api/src/trie.rs +++ b/crates/storage/storage-api/src/trie.rs @@ -1,5 +1,6 @@ -use alloc::vec::Vec; -use alloy_primitives::{Address, Bytes, B256}; +use alloc::{boxed::Box, vec::Vec}; +use alloy_primitives::{Address, Bytes, B256, U256}; +use reth_primitives_traits::Account; use reth_storage_errors::provider::ProviderResult; use reth_trie_common::{ updates::{StorageTrieUpdatesSorted, TrieUpdates, TrieUpdatesSorted}, @@ -65,6 +66,82 @@ pub trait StorageRootProvider { ) -> ProviderResult; } +/// A type that can iterate over consecutive hashed accounts and storage slots, and generate +/// boundary proofs for them, for serving `snap/2` (EIP-8189) `GetAccountRange`/`GetStorageRanges` +/// requests. Hash-native throughout, unlike [`StorageRootProvider`]. +#[auto_impl::auto_impl(&, Box, Arc)] +pub trait StateRangeProvider { + /// Returns accounts (hash, account) in `[start, limit]`, bounded by `response_bytes`. + fn account_range( + &self, + start: B256, + limit: B256, + response_bytes: usize, + ) -> RangeResult<(B256, Account)>; + + /// Returns the storage root for `hashed_address` without needing its address preimage. + fn storage_root_by_hash(&self, hashed_address: B256) -> ProviderResult; + + /// Same as [`Self::account_range`], but for the storage slots of `hashed_address`. + /// + /// Returns `None` if `hashed_address` isn't present in the account trie at the pinned state + /// root, distinct from an account that is present but has no storage. + fn storage_range( + &self, + hashed_address: B256, + start: B256, + limit: B256, + response_bytes: usize, + ) -> StorageRangeResult; + + /// Returns an account-trie boundary proof for the already-hashed `keys`. + fn account_range_proof(&self, keys: &[B256]) -> ProviderResult>; + + /// Same as [`Self::account_range_proof`], but for the storage trie of `hashed_address`. + fn storage_range_proof( + &self, + hashed_address: B256, + keys: &[B256], + ) -> ProviderResult>; +} + +/// A type that resolves retained state roots into reusable state range views. +#[auto_impl::auto_impl(&, Arc)] +pub trait StateRangeProviderFactory { + /// Returns a view pinned to `state_root`, or `None` if that root is not retained. + fn state_range_provider(&self, state_root: B256) -> ProviderResult>; +} + +/// A range query's items and why the range ended where it did. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RangeResponse { + /// The items found within the requested range, in ascending key order. + pub items: Vec, + /// Why `items` doesn't necessarily continue past its last entry. + pub end: RangeEnd, +} + +/// Why a range query stopped before the caller-requested `limit`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RangeEnd { + /// The cursor ran out of entries: `items` covers everything from `start` onward. + Exhausted, + /// The last returned item's key reached or passed the requested `limit`. + HashLimit, + /// `response_bytes` was exceeded before `limit` was reached. + ByteLimit, +} + +/// Result of a [`StateRangeProvider`] range query. +pub type RangeResult = ProviderResult>; + +/// Result of a [`StateRangeProvider::storage_range`] query: `None` if the account itself isn't +/// present in the trie at the pinned state root. +pub type StorageRangeResult = ProviderResult>>; + +/// A reusable state range view resolved for a specific state root. +pub type StateRangeView = Box; + /// A type that can generate state proof on top of a given post state. #[auto_impl::auto_impl(&, Box, Arc)] pub trait StateProofProvider { diff --git a/crates/transaction-pool/src/lib.rs b/crates/transaction-pool/src/lib.rs index fe4c9a22ccf..b43e666f90f 100644 --- a/crates/transaction-pool/src/lib.rs +++ b/crates/transaction-pool/src/lib.rs @@ -510,11 +510,8 @@ where if transactions.is_empty() { return Vec::new() } - let validated = self - .pool - .validator() - .validate_transactions(transactions.into_iter().map(|tx| (origin, tx))) - .await; + let validated = + self.pool.validator().validate_transactions_with_origin(origin, transactions).await; self.pool.add_transactions(origin, validated) } @@ -640,6 +637,14 @@ where (pending, queued) } + fn private_pending_and_queued_txn_count(&self) -> (usize, usize) { + self.pool.get_pool_data().private_pending_and_queued_txn_count() + } + + fn total_and_private_txn_counts(&self) -> ((usize, usize), (usize, usize)) { + self.pool.get_pool_data().total_and_private_txn_counts() + } + fn all_transactions(&self) -> AllPoolTransactions { self.pool.all_transactions() } diff --git a/crates/transaction-pool/src/pool/mod.rs b/crates/transaction-pool/src/pool/mod.rs index 5143343d51a..d098d3e89a4 100644 --- a/crates/transaction-pool/src/pool/mod.rs +++ b/crates/transaction-pool/src/pool/mod.rs @@ -715,14 +715,20 @@ where self.delete_discarded_blobs(discarded.iter()); self.with_event_listener(|listener| listener.discarded_many(&discarded)); - let discarded_hashes = - discarded.into_iter().map(|tx| *tx.hash()).collect::>(); + // Linear search avoids allocating a hash set for small eviction batches. + const MAX_LINEAR_SEARCH_DISCARDS: usize = 4; + let discarded_hashes = (discarded.len() > MAX_LINEAR_SEARCH_DISCARDS) + .then(|| discarded.iter().map(|tx| *tx.hash()).collect::>()); + let is_discarded = |hash: &TxHash| match &discarded_hashes { + Some(hashes) => hashes.contains(hash), + None => discarded.iter().any(|tx| tx.hash() == hash), + }; // A newly added transaction may be immediately discarded, so we need to // adjust the result here for res in &mut results { if let Ok(AddedTransactionOutcome { hash, .. }) = res && - discarded_hashes.contains(hash) + is_discarded(hash) { *res = Err(PoolError::new(*hash, PoolErrorKind::DiscardedOnInsert)) } diff --git a/crates/transaction-pool/src/pool/parked.rs b/crates/transaction-pool/src/pool/parked.rs index e3efd30967f..219a6dbe145 100644 --- a/crates/transaction-pool/src/pool/parked.rs +++ b/crates/transaction-pool/src/pool/parked.rs @@ -38,6 +38,11 @@ pub struct ParkedPool { /// /// See also [`reth_primitives_traits::InMemorySize::size`]. size_of: SizeTracker, + /// Running count of [`crate::TransactionOrigin::Private`] transactions in the pool. + /// + /// Maintained alongside `size_of` so callers can read the private count without walking the + /// pool. See [`Self::private_pool_count()`]. + private_pool_count: usize, } // === impl ParkedPool === @@ -55,6 +60,9 @@ impl ParkedPool { // keep track of size self.size_of += tx.size(); + if tx.origin.is_private() { + self.private_pool_count += 1; + } // update or create sender entry self.add_sender_count(tx.sender_id(), submission_id); @@ -134,6 +142,10 @@ impl ParkedPool { // keep track of size self.size_of -= tx.transaction.size(); + if tx.transaction.origin.is_private() { + debug_assert!(self.private_pool_count > 0, "private_pool_count underflow"); + self.private_pool_count = self.private_pool_count.saturating_sub(1); + } Some(tx.transaction.into()) } @@ -231,6 +243,11 @@ impl ParkedPool { self.by_id.len() } + /// Number of [`crate::TransactionOrigin::Private`] transactions in the pool. + pub(crate) const fn private_pool_count(&self) -> usize { + self.private_pool_count + } + /// Returns true if the pool exceeds the given limit #[inline] pub(crate) fn exceeds(&self, limit: &SubPoolLimit) -> bool { @@ -353,6 +370,7 @@ impl Default for ParkedPool { last_sender_submission: Default::default(), sender_transaction_count: Default::default(), size_of: Default::default(), + private_pool_count: 0, } } } diff --git a/crates/transaction-pool/src/pool/pending.rs b/crates/transaction-pool/src/pool/pending.rs index 57a70c90c91..e19fa8b1647 100644 --- a/crates/transaction-pool/src/pool/pending.rs +++ b/crates/transaction-pool/src/pool/pending.rs @@ -43,6 +43,11 @@ pub struct PendingPool { /// /// See also [`reth_primitives_traits::InMemorySize::size`]. size_of: SizeTracker, + /// Running count of [`crate::TransactionOrigin::Private`] transactions in the pool. + /// + /// Maintained alongside `size_of` so callers can read the private count without walking the + /// pool. See [`Self::private_pool_count()`]. + private_pool_count: usize, /// Used to broadcast new transactions that have been added to the `PendingPool` to existing /// `static_files` of this pool. new_transaction_notifier: broadcast::Sender>, @@ -66,6 +71,7 @@ impl PendingPool { independent_transactions: Default::default(), highest_nonces: Default::default(), size_of: Default::default(), + private_pool_count: 0, new_transaction_notifier, } } @@ -80,6 +86,7 @@ impl PendingPool { self.independent_transactions.clear(); self.highest_nonces.clear(); self.size_of.reset(); + self.private_pool_count = 0; std::mem::take(&mut self.by_id) } @@ -194,6 +201,9 @@ impl PendingPool { } } else { self.size_of += tx.transaction.size(); + if tx.transaction.origin.is_private() { + self.private_pool_count += 1; + } self.update_independents_and_highest_nonces(&tx); self.by_id.insert(id, tx); } @@ -239,6 +249,9 @@ impl PendingPool { tx.priority = self.ordering.priority(&tx.transaction.transaction, base_fee); self.size_of += tx.transaction.size(); + if tx.transaction.origin.is_private() { + self.private_pool_count += 1; + } self.update_independents_and_highest_nonces(&tx); self.by_id.insert(id, tx); } @@ -290,6 +303,9 @@ impl PendingPool { // keep track of size self.size_of += tx.size(); + if tx.origin.is_private() { + self.private_pool_count += 1; + } let tx_id = *tx.id(); @@ -327,6 +343,10 @@ impl PendingPool { let tx = self.by_id.remove(id)?; self.size_of -= tx.transaction.size(); + if tx.transaction.origin.is_private() { + debug_assert!(self.private_pool_count > 0, "private_pool_count underflow"); + self.private_pool_count = self.private_pool_count.saturating_sub(1); + } match self.highest_nonces.entry(id.sender) { Entry::Occupied(mut entry) => { @@ -540,6 +560,11 @@ impl PendingPool { self.by_id.len() } + /// Number of [`crate::TransactionOrigin::Private`] transactions in the pool. + pub(crate) const fn private_pool_count(&self) -> usize { + self.private_pool_count + } + /// All transactions grouped by id pub const fn by_id(&self) -> &OrdMap> { &self.by_id diff --git a/crates/transaction-pool/src/pool/txpool.rs b/crates/transaction-pool/src/pool/txpool.rs index a0e6782e5c8..af9eae1a850 100644 --- a/crates/transaction-pool/src/pool/txpool.rs +++ b/crates/transaction-pool/src/pool/txpool.rs @@ -507,6 +507,23 @@ impl TxPool { self.basefee_pool.len() + self.queued_pool.len() } + /// Returns the number of [`crate::TransactionOrigin::Private`] transactions in the pending and + /// queued sub-pools, using the same sub-pool grouping as + /// [`Self::pending_transactions_count`]/[`Self::queued_transactions_count`]. + pub(crate) const fn private_pending_and_queued_txn_count(&self) -> (usize, usize) { + let pending = self.pending_pool.private_pool_count(); + let queued = self.basefee_pool.private_pool_count() + self.queued_pool.private_pool_count(); + (pending, queued) + } + + /// Returns `((pending, queued), (private_pending, private_queued))` from this single pool view, + /// so callers can subtract the private counts without a cross-snapshot race. + pub(crate) fn total_and_private_txn_counts(&self) -> ((usize, usize), (usize, usize)) { + let pending = self.pending_transactions_count(); + let queued = self.queued_transactions_count(); + ((pending, queued), self.private_pending_and_queued_txn_count()) + } + /// Returns queued and pending transactions for the specified sender pub fn queued_and_pending_txs_by_sender( &self, @@ -2475,6 +2492,41 @@ mod tests { assert!(pool.pending_pool.is_empty()); } + #[test] + fn private_pool_count_tracks_pending_and_queued() { + let on_chain_balance = U256::MAX; + let on_chain_nonce = 0; + let mut f = MockTransactionFactory::default(); + let mut pool = TxPool::new(MockOrdering::default(), Default::default()); + + // Private tx at the on-chain nonce lands in the pending sub-pool. + let private_pending = + f.validated_with_origin(TransactionOrigin::Private, MockTransaction::eip1559()); + let private_pending_id = *private_pending.id(); + pool.add_transaction(private_pending, on_chain_balance, on_chain_nonce, None).unwrap(); + + // Public tx at the on-chain nonce is also pending but must not be counted. + let public_pending = + f.validated_with_origin(TransactionOrigin::External, MockTransaction::eip1559()); + pool.add_transaction(public_pending, on_chain_balance, on_chain_nonce, None).unwrap(); + + // Private tx with a nonce gap parks in the queued sub-pool. + let private_queued = f.validated_with_origin( + TransactionOrigin::Private, + MockTransaction::eip1559().with_nonce(1), + ); + pool.add_transaction(private_queued, on_chain_balance, on_chain_nonce, None).unwrap(); + + assert_eq!(pool.private_pending_and_queued_txn_count(), (1, 1)); + // The combined accessor reports totals (2 pending, 1 queued) and private counts together. + assert_eq!(pool.total_and_private_txn_counts(), ((2, 1), (1, 1))); + + // Removing the pending private tx decrements only the pending count. + pool.remove_transaction(&private_pending_id); + assert_eq!(pool.private_pending_and_queued_txn_count(), (0, 1)); + assert_eq!(pool.total_and_private_txn_counts(), ((1, 1), (0, 1))); + } + #[test] fn test_promote_valid_tx_with_decreasing_blob_fee() { let on_chain_balance = U256::MAX; diff --git a/crates/transaction-pool/src/test_utils/mod.rs b/crates/transaction-pool/src/test_utils/mod.rs index 7c0ce6ae762..6b33aa01e6f 100644 --- a/crates/transaction-pool/src/test_utils/mod.rs +++ b/crates/transaction-pool/src/test_utils/mod.rs @@ -9,8 +9,6 @@ pub use tx_gen::*; mod mock; pub use mock::*; -mod pool; - mod okvalidator; pub use okvalidator::*; diff --git a/crates/transaction-pool/src/test_utils/pool.rs b/crates/transaction-pool/src/test_utils/pool.rs deleted file mode 100644 index 3b345d5bcd1..00000000000 --- a/crates/transaction-pool/src/test_utils/pool.rs +++ /dev/null @@ -1,604 +0,0 @@ -//! Test helpers for mocking an entire pool. - -#![allow(dead_code)] - -use crate::{ - error::PoolErrorKind, - pool::{state::SubPool, txpool::TxPool, AddedTransaction}, - test_utils::{MockOrdering, MockTransactionDistribution, MockTransactionFactory}, - TransactionOrdering, -}; -use alloy_primitives::{map::AddressMap, Address, U256}; -use rand::Rng; -use std::ops::{Deref, DerefMut}; - -/// A wrapped `TxPool` with additional helpers for testing -pub(crate) struct MockPool { - // The wrapped pool. - pool: TxPool, -} - -impl MockPool { - /// The total size of all subpools - fn total_subpool_size(&self) -> usize { - self.pool.pending().len() + self.pool.base_fee().len() + self.pool.queued().len() - } - - /// Checks that all pool invariants hold. - fn enforce_invariants(&self) { - assert_eq!( - self.pool.len(), - self.total_subpool_size(), - "Tx in AllTransactions and sum(subpools) must match" - ); - } -} - -impl Default for MockPool { - fn default() -> Self { - Self { pool: TxPool::new(MockOrdering::default(), Default::default()) } - } -} - -impl Deref for MockPool { - type Target = TxPool; - - fn deref(&self) -> &Self::Target { - &self.pool - } -} - -impl DerefMut for MockPool { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.pool - } -} - -/// Simulates transaction execution. -pub(crate) struct MockTransactionSimulator { - /// The pending base fee - base_fee: u128, - /// Generator for transactions - tx_generator: MockTransactionDistribution, - /// represents the on chain balance of a sender. - balances: AddressMap, - /// represents the on chain nonce of a sender. - nonces: AddressMap, - /// A set of addresses to use as senders. - senders: Vec
, - /// What scenarios to execute. - scenarios: Vec, - /// All previous scenarios executed by a sender. - executed: AddressMap, - /// "Validates" generated transactions. - validator: MockTransactionFactory, - /// Represents the gaps in nonces for each sender. - nonce_gaps: AddressMap, - /// The rng instance used to select senders and scenarios. - rng: R, -} - -impl MockTransactionSimulator { - /// Returns a new mock instance - pub(crate) fn new(mut rng: R, config: MockSimulatorConfig) -> Self { - let senders = config.addresses(&mut rng); - Self { - base_fee: config.base_fee, - balances: senders.iter().copied().map(|a| (a, rng.random())).collect(), - nonces: senders.iter().copied().map(|a| (a, 0)).collect(), - senders, - scenarios: config.scenarios, - tx_generator: config.tx_generator, - executed: Default::default(), - validator: Default::default(), - nonce_gaps: Default::default(), - rng, - } - } - - /// Creates a pool configured for this simulator - /// - /// This is needed because `MockPool::default()` sets `pending_basefee` to 7, but we might want - /// to use different values - pub(crate) fn create_pool(&self) -> MockPool { - let mut pool = MockPool::default(); - let mut info = pool.block_info(); - info.pending_basefee = self.base_fee as u64; - pool.set_block_info(info); - pool - } - - /// Returns a random address from the senders set - fn rng_address(&mut self) -> Address { - let idx = self.rng.random_range(0..self.senders.len()); - self.senders[idx] - } - - /// Returns a random scenario from the scenario set - fn rng_scenario(&mut self) -> ScenarioType { - let idx = self.rng.random_range(0..self.scenarios.len()); - self.scenarios[idx].clone() - } - - /// Executes the next scenario and applies it to the pool - pub(crate) fn next(&mut self, pool: &mut MockPool) { - let sender = self.rng_address(); - let scenario = self.rng_scenario(); - let on_chain_nonce = self.nonces[&sender]; - let on_chain_balance = self.balances[&sender]; - - match scenario { - ScenarioType::OnchainNonce => { - // uses fee from fee_ranges - let tx = self.tx_generator.tx(on_chain_nonce, &mut self.rng).with_sender(sender); - let valid_tx = self.validator.validated(tx); - - let res = - match pool.add_transaction(valid_tx, on_chain_balance, on_chain_nonce, None) { - Ok(res) => res, - Err(e) => match e.kind { - // skip pool capacity/replacement errors (not relevant) - PoolErrorKind::SpammerExceededCapacity(_) | - PoolErrorKind::ReplacementUnderpriced => return, - _ => panic!("unexpected error: {e:?}"), - }, - }; - - match res { - AddedTransaction::Pending(_) => {} - AddedTransaction::Parked { .. } => { - panic!("expected pending") - } - } - - self.executed - .entry(sender) - .or_insert_with(|| ExecutedScenarios { sender, scenarios: vec![] }) // in the case of a new sender - .scenarios - .push(ExecutedScenario { - balance: on_chain_balance, - nonce: on_chain_nonce, - scenario: Scenario::OnchainNonce { nonce: on_chain_nonce }, - }); - - self.nonces.insert(sender, on_chain_nonce + 1); - } - - ScenarioType::HigherNonce { skip } => { - // if this sender already has a nonce gap, skip - if self.nonce_gaps.contains_key(&sender) { - return; - } - - let higher_nonce = on_chain_nonce + skip; - - // uses fee from fee_ranges - let tx = self.tx_generator.tx(higher_nonce, &mut self.rng).with_sender(sender); - let valid_tx = self.validator.validated(tx); - - let res = - match pool.add_transaction(valid_tx, on_chain_balance, on_chain_nonce, None) { - Ok(res) => res, - Err(e) => match e.kind { - // skip pool capacity/replacement errors (not relevant) - PoolErrorKind::SpammerExceededCapacity(_) | - PoolErrorKind::ReplacementUnderpriced => return, - _ => panic!("unexpected error: {e:?}"), - }, - }; - - match res { - AddedTransaction::Pending(_) => { - panic!("expected parked") - } - AddedTransaction::Parked { subpool, .. } => { - assert_eq!( - subpool, - SubPool::Queued, - "expected to be moved to queued subpool" - ); - } - } - - self.executed - .entry(sender) - .or_insert_with(|| ExecutedScenarios { sender, scenarios: vec![] }) // in the case of a new sender - .scenarios - .push(ExecutedScenario { - balance: on_chain_balance, - nonce: on_chain_nonce, - scenario: Scenario::HigherNonce { - onchain: on_chain_nonce, - nonce: higher_nonce, - }, - }); - self.nonce_gaps.insert(sender, higher_nonce); - } - - ScenarioType::BelowBaseFee { fee } => { - // fee should be in [MIN_PROTOCOL_BASE_FEE, base_fee) - let tx = self - .tx_generator - .tx(on_chain_nonce, &mut self.rng) - .with_sender(sender) - .with_gas_price(fee); - let valid_tx = self.validator.validated(tx); - - let res = - match pool.add_transaction(valid_tx, on_chain_balance, on_chain_nonce, None) { - Ok(res) => res, - Err(e) => match e.kind { - // skip pool capacity/replacement errors (not relevant) - PoolErrorKind::SpammerExceededCapacity(_) | - PoolErrorKind::ReplacementUnderpriced => return, - _ => panic!("unexpected error: {e:?}"), - }, - }; - - match res { - AddedTransaction::Pending(_) => panic!("expected parked"), - AddedTransaction::Parked { subpool, .. } => { - assert_eq!( - subpool, - SubPool::BaseFee, - "expected to be moved to base fee subpool" - ); - } - } - self.executed - .entry(sender) - .or_insert_with(|| ExecutedScenarios { sender, scenarios: vec![] }) // in the case of a new sender - .scenarios - .push(ExecutedScenario { - balance: on_chain_balance, - nonce: on_chain_nonce, - scenario: Scenario::BelowBaseFee { fee }, - }); - } - - ScenarioType::FillNonceGap => { - if self.nonce_gaps.is_empty() { - return; - } - - let gap_senders: Vec
= self.nonce_gaps.keys().copied().collect(); - let idx = self.rng.random_range(0..gap_senders.len()); - let gap_sender = gap_senders[idx]; - let queued_nonce = self.nonce_gaps[&gap_sender]; - - let sender_onchain_nonce = self.nonces[&gap_sender]; - let sender_balance = self.balances[&gap_sender]; - - for fill_nonce in sender_onchain_nonce..queued_nonce { - let tx = - self.tx_generator.tx(fill_nonce, &mut self.rng).with_sender(gap_sender); - let valid_tx = self.validator.validated(tx); - - let res = match pool.add_transaction( - valid_tx, - sender_balance, - sender_onchain_nonce, - None, - ) { - Ok(res) => res, - Err(e) => match e.kind { - // skip pool capacity/replacement errors (not relevant) - PoolErrorKind::SpammerExceededCapacity(_) | - PoolErrorKind::ReplacementUnderpriced => return, - _ => panic!("unexpected error: {e:?}"), - }, - }; - - match res { - AddedTransaction::Pending(_) => {} - AddedTransaction::Parked { .. } => { - panic!("expected pending when filling gap") - } - } - - self.executed - .entry(gap_sender) - .or_insert_with(|| ExecutedScenarios { - sender: gap_sender, - scenarios: vec![], - }) - .scenarios - .push(ExecutedScenario { - balance: sender_balance, - nonce: fill_nonce, - scenario: Scenario::FillNonceGap { - filled_nonce: fill_nonce, - promoted_nonce: queued_nonce, - }, - }); - } - self.nonces.insert(gap_sender, queued_nonce + 1); - self.nonce_gaps.remove(&gap_sender); - } - } - // make sure everything is set - pool.enforce_invariants(); - } -} - -/// How to configure a new mock transaction stream -pub(crate) struct MockSimulatorConfig { - /// How many senders to generate. - pub(crate) num_senders: usize, - /// Scenarios to test - pub(crate) scenarios: Vec, - /// The start base fee - pub(crate) base_fee: u128, - /// generator for transactions - pub(crate) tx_generator: MockTransactionDistribution, -} - -impl MockSimulatorConfig { - /// Generates a set of random addresses - pub(crate) fn addresses(&self, rng: &mut impl rand::Rng) -> Vec
{ - std::iter::repeat_with(|| Address::random_with(rng)).take(self.num_senders).collect() - } -} - -/// Represents the different types of test scenarios. -#[derive(Debug, Clone)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -pub(crate) enum ScenarioType { - OnchainNonce, - HigherNonce { skip: u64 }, - BelowBaseFee { fee: u128 }, - FillNonceGap, -} - -/// The actual scenario, ready to be executed -/// -/// A scenario produces one or more transactions and expects a certain Outcome. -/// -/// An executed scenario can affect previous executed transactions -#[derive(Debug, Clone)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -pub(crate) enum Scenario { - /// Send a tx with the same nonce as on chain. - OnchainNonce { nonce: u64 }, - /// Send a tx with a higher nonce that what the sender has on chain - HigherNonce { onchain: u64, nonce: u64 }, - /// Send a tx with a base fee below the base fee of the pool - BelowBaseFee { fee: u128 }, - /// Fill a nonce gap to promote queued transactions - FillNonceGap { filled_nonce: u64, promoted_nonce: u64 }, - /// Execute multiple test scenarios - Multi { scenario: Vec }, -} - -/// Represents an executed scenario -#[derive(Debug, Clone)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -pub(crate) struct ExecutedScenario { - /// balance at the time of execution - balance: U256, - /// nonce at the time of execution - nonce: u64, - /// The executed scenario - scenario: Scenario, -} - -/// All executed scenarios by a sender -#[derive(Debug, Clone)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -pub(crate) struct ExecutedScenarios { - sender: Address, - scenarios: Vec, -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::test_utils::{MockFeeRange, MockTransactionRatio}; - - #[test] - fn test_on_chain_nonce_scenario() { - let transaction_ratio = MockTransactionRatio { - legacy_pct: 30, - dynamic_fee_pct: 70, - access_list_pct: 0, - blob_pct: 0, - }; - - let base_fee = 10u128; - let fee_ranges = MockFeeRange { - gas_price: (base_fee..100).try_into().unwrap(), - priority_fee: (1u128..10).try_into().unwrap(), - max_fee: (base_fee..110).try_into().unwrap(), - max_fee_blob: (1u128..100).try_into().unwrap(), - }; - - let config = MockSimulatorConfig { - num_senders: 10, - scenarios: vec![ScenarioType::OnchainNonce], - base_fee, - tx_generator: MockTransactionDistribution::new( - transaction_ratio, - fee_ranges, - 10..100, - 10..100, - ), - }; - let mut simulator = MockTransactionSimulator::new(rand::rng(), config); - let mut pool = simulator.create_pool(); - - simulator.next(&mut pool); - assert_eq!(pool.pending().len(), 1); - assert_eq!(pool.queued().len(), 0); - assert_eq!(pool.base_fee().len(), 0); - } - - #[test] - fn test_higher_nonce_scenario() { - let transaction_ratio = MockTransactionRatio { - legacy_pct: 30, - dynamic_fee_pct: 70, - access_list_pct: 0, - blob_pct: 0, - }; - - let base_fee = 10u128; - let fee_ranges = MockFeeRange { - gas_price: (base_fee..100).try_into().unwrap(), - priority_fee: (1u128..10).try_into().unwrap(), - max_fee: (base_fee..110).try_into().unwrap(), - max_fee_blob: (1u128..100).try_into().unwrap(), - }; - - let config = MockSimulatorConfig { - num_senders: 10, - scenarios: vec![ScenarioType::HigherNonce { skip: 1 }], - base_fee, - tx_generator: MockTransactionDistribution::new( - transaction_ratio, - fee_ranges, - 10..100, - 10..100, - ), - }; - let mut simulator = MockTransactionSimulator::new(rand::rng(), config); - let mut pool = simulator.create_pool(); - - simulator.next(&mut pool); - assert_eq!(pool.pending().len(), 0); - assert_eq!(pool.queued().len(), 1); - assert_eq!(pool.base_fee().len(), 0); - } - - #[test] - fn test_below_base_fee_scenario() { - let transaction_ratio = MockTransactionRatio { - legacy_pct: 30, - dynamic_fee_pct: 70, - access_list_pct: 0, - blob_pct: 0, - }; - - let base_fee = 10u128; - let fee_ranges = MockFeeRange { - gas_price: (base_fee..100).try_into().unwrap(), - priority_fee: (1u128..10).try_into().unwrap(), - max_fee: (base_fee..110).try_into().unwrap(), - max_fee_blob: (1u128..100).try_into().unwrap(), - }; - - let config = MockSimulatorConfig { - num_senders: 10, - scenarios: vec![ScenarioType::BelowBaseFee { fee: 8 }], /* fee should be in - * [MIN_PROTOCOL_BASE_FEE, - * base_fee) */ - base_fee, - tx_generator: MockTransactionDistribution::new( - transaction_ratio, - fee_ranges, - 10..100, - 10..100, - ), - }; - let mut simulator = MockTransactionSimulator::new(rand::rng(), config); - let mut pool = simulator.create_pool(); - - simulator.next(&mut pool); - assert_eq!(pool.pending().len(), 0); - assert_eq!(pool.queued().len(), 0); - assert_eq!(pool.base_fee().len(), 1); - } - - #[test] - fn test_fill_nonce_gap_scenario() { - let transaction_ratio = MockTransactionRatio { - legacy_pct: 30, - dynamic_fee_pct: 70, - access_list_pct: 0, - blob_pct: 0, - }; - - let base_fee = 10u128; - let fee_ranges = MockFeeRange { - gas_price: (base_fee..100).try_into().unwrap(), - priority_fee: (1u128..10).try_into().unwrap(), - max_fee: (base_fee..110).try_into().unwrap(), - max_fee_blob: (1u128..100).try_into().unwrap(), - }; - - let config = MockSimulatorConfig { - num_senders: 5, - scenarios: vec![ScenarioType::HigherNonce { skip: 5 }], - base_fee, - tx_generator: MockTransactionDistribution::new( - transaction_ratio, - fee_ranges, - 10..100, - 10..100, - ), - }; - let mut simulator = MockTransactionSimulator::new(rand::rng(), config); - let mut pool = simulator.create_pool(); - - // create some nonce gaps - for _ in 0..10 { - simulator.next(&mut pool); - } - - let num_gaps = simulator.nonce_gaps.len(); - - assert_eq!(pool.pending().len(), 0); - assert_eq!(pool.queued().len(), num_gaps); - assert_eq!(pool.base_fee().len(), 0); - - simulator.scenarios = vec![ScenarioType::FillNonceGap]; - for _ in 0..num_gaps { - simulator.next(&mut pool); - } - - let expected_pending = num_gaps * 6; - assert_eq!(pool.pending().len(), expected_pending); - assert_eq!(pool.queued().len(), 0); - assert_eq!(pool.base_fee().len(), 0); - } - - #[test] - fn test_random_scenarios() { - let transaction_ratio = MockTransactionRatio { - legacy_pct: 30, - dynamic_fee_pct: 70, - access_list_pct: 0, - blob_pct: 0, - }; - - let base_fee = 10u128; - let fee_ranges = MockFeeRange { - gas_price: (base_fee..100).try_into().unwrap(), - priority_fee: (1u128..10).try_into().unwrap(), - max_fee: (base_fee..110).try_into().unwrap(), - max_fee_blob: (1u128..100).try_into().unwrap(), - }; - - let config = MockSimulatorConfig { - num_senders: 10, - scenarios: vec![ - ScenarioType::OnchainNonce, - ScenarioType::HigherNonce { skip: 2 }, - ScenarioType::BelowBaseFee { fee: 8 }, - ScenarioType::FillNonceGap, - ], - base_fee, - tx_generator: MockTransactionDistribution::new( - transaction_ratio, - fee_ranges, - 10..100, - 10..100, - ), - }; - let mut simulator = MockTransactionSimulator::new(rand::rng(), config); - let mut pool = simulator.create_pool(); - - for _ in 0..1000 { - simulator.next(&mut pool); - } - } -} diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index 654ef7e5a87..ac00efb1b96 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -450,6 +450,22 @@ pub trait TransactionPool: Clone + Debug + Send + Sync { /// number of transactions that are ready for inclusion in future blocks: `(pending, queued)`. fn pending_and_queued_txn_count(&self) -> (usize, usize); + /// Returns the number of [`TransactionOrigin::Private`] transactions. + /// + /// Lets callers exclude private transactions from the pool counts without materializing the + /// pool. Defaults to `(0, 0)`; pool implementations that hold private transactions should + /// override this. + fn private_pending_and_queued_txn_count(&self) -> (usize, usize) { + (0, 0) + } + + /// Returns `((pending, queued), (private_pending, private_queued))` from a single snapshot, so + /// callers can subtract the private counts consistently. The default delegates to the two + /// separate accessors; pools holding private transactions should override it. + fn total_and_private_txn_counts(&self) -> ((usize, usize), (usize, usize)) { + (self.pending_and_queued_txn_count(), self.private_pending_and_queued_txn_count()) + } + /// Returns all transactions that are currently in the pool grouped by whether they are ready /// for inclusion in the next block or not. /// diff --git a/crates/transaction-pool/src/validate/eth.rs b/crates/transaction-pool/src/validate/eth.rs index bea45af5c96..c8e178ab56f 100644 --- a/crates/transaction-pool/src/validate/eth.rs +++ b/crates/transaction-pool/src/validate/eth.rs @@ -33,7 +33,10 @@ use reth_primitives_traits::{ transaction::error::InvalidTransactionError, Account, BlockTy, GotExpected, HeaderTy, SealedBlock, }; -use reth_storage_api::{AccountInfoReader, BlockReaderIdExt, BytecodeReader, StateProviderFactory}; +use reth_storage_api::{ + errors::ProviderError, AccountInfoReader, BlockReaderIdExt, BytecodeReader, StateProviderBox, + StateProviderFactory, +}; use reth_tasks::Runtime; use revm::context_interface::Cfg; use std::{ @@ -80,6 +83,8 @@ pub type StatefulValidationFn = Arc< pub struct EthTransactionValidator { /// This type fetches account info from the db client: Client, + /// The chain ID transactions must use. + chain_id: u64, /// Blobstore used for fetching re-injected blob transactions. blob_store: Box, /// tracks activated forks relevant for transaction validation @@ -165,11 +170,8 @@ impl EthTransactionValidator { } /// Returns the configured chain id - pub fn chain_id(&self) -> u64 - where - Client: ChainSpecProvider, - { - self.client().chain_spec().chain().id() + pub const fn chain_id(&self) -> u64 { + self.chain_id } /// Returns the configured client @@ -372,7 +374,8 @@ where origin: TransactionOrigin, transaction: Tx, ) -> TransactionValidationOutcome { - self.validate_one_with_provider(origin, transaction, &mut None) + let mut state: Option = None; + self.validate_one_with_provider(origin, transaction, &mut state, || self.client.latest()) } /// Validates a single transaction with the provided state provider. @@ -387,26 +390,33 @@ where transaction: Tx, state: &mut Option>, ) -> TransactionValidationOutcome { - self.validate_one_with_provider(origin, transaction, state) + self.validate_one_with_provider(origin, transaction, state, || { + self.client.latest().map(|state| Box::new(state) as Box) + }) } /// Validates a single transaction using an optional cached state provider. /// If no provider is passed, a new one will be created. This allows reusing /// the same provider across multiple txs. - fn validate_one_with_provider( + fn validate_one_with_provider( &self, origin: TransactionOrigin, transaction: Tx, - maybe_state: &mut Option>, - ) -> TransactionValidationOutcome { + maybe_state: &mut Option

, + state_provider: F, + ) -> TransactionValidationOutcome + where + P: AccountInfoReader, + F: FnOnce() -> Result, + { match self.validate_stateless(origin, &transaction) { Ok(()) => { // stateless checks passed, pass transaction down stateful validation pipeline // If we don't have a state provider yet, fetch the latest state if maybe_state.is_none() { - match self.client.latest() { + match state_provider() { Ok(new_state) => { - *maybe_state = Some(Box::new(new_state)); + *maybe_state = Some(new_state); } Err(err) => { return TransactionValidationOutcome::Error( @@ -417,7 +427,7 @@ where } } - let state = maybe_state.as_deref().expect("provider is set"); + let state = maybe_state.as_ref().expect("provider is set"); self.validate_stateful(origin, transaction, state) } @@ -863,10 +873,12 @@ where &self, transactions: impl IntoIterator, ) -> Vec> { - let mut provider = None; + let mut provider: Option = None; transactions .into_iter() - .map(|(origin, tx)| self.validate_one_with_provider(origin, tx, &mut provider)) + .map(|(origin, tx)| { + self.validate_one_with_provider(origin, tx, &mut provider, || self.client.latest()) + }) .collect() } @@ -876,10 +888,12 @@ where origin: TransactionOrigin, transactions: impl IntoIterator + Send, ) -> Vec> { - let mut provider = None; + let mut provider: Option = None; transactions .into_iter() - .map(|tx| self.validate_one_with_provider(origin, tx, &mut provider)) + .map(|tx| { + self.validate_one_with_provider(origin, tx, &mut provider, || self.client.latest()) + }) .collect() } @@ -1002,6 +1016,8 @@ where #[derive(Debug)] pub struct EthTransactionValidatorBuilder { client: Client, + /// The chain ID transactions must use. + chain_id: u64, /// The EVM configuration to use for validation. evm_config: Evm, /// Fork indicator whether we are in the Shanghai stage. @@ -1084,6 +1100,7 @@ impl EthTransactionValidatorBuilder { Self { block_gas_limit: ETHEREUM_BLOCK_GAS_LIMIT_30M.into(), client, + chain_id: chain_spec.chain().id(), evm_config, minimum_priority_fee: None, additional_tasks: 1, @@ -1310,6 +1327,7 @@ impl EthTransactionValidatorBuilder { { let Self { client, + chain_id, evm_config, shanghai, cancun, @@ -1349,6 +1367,7 @@ impl EthTransactionValidatorBuilder { EthTransactionValidator { client, + chain_id, eip2718, eip1559, fork_tracker, @@ -1558,6 +1577,36 @@ mod tests { assert!(tx.is_some()); } + #[test] + fn validates_configured_chain_id() { + let provider = MockEthProvider::default().with_genesis_block(); + let validator = EthTransactionValidatorBuilder::new(provider, test_evm_config()) + .build(InMemoryBlobStore::default()); + let transaction = |chain_id| { + EthPooledTransaction::try_from_consensus( + TransactionBuilder::default() + .chain_id(chain_id) + .gas_limit(21_000) + .to(Address::ZERO) + .into_eip1559() + .try_into_recovered() + .unwrap(), + ) + .unwrap() + }; + + assert!(validator + .validate_stateless(TransactionOrigin::External, &transaction(validator.chain_id())) + .is_ok()); + assert!(matches!( + validator.validate_stateless( + TransactionOrigin::External, + &transaction(validator.chain_id() + 1) + ), + Err(InvalidPoolTransactionError::Consensus(InvalidTransactionError::ChainIdMismatch)) + )); + } + // #[tokio::test] async fn invalid_on_gas_limit_too_high() { diff --git a/crates/transaction-pool/src/validate/mod.rs b/crates/transaction-pool/src/validate/mod.rs index 379c0985f78..dce498ccf4b 100644 --- a/crates/transaction-pool/src/validate/mod.rs +++ b/crates/transaction-pool/src/validate/mod.rs @@ -269,6 +269,17 @@ where } } + async fn validate_transactions_with_origin( + &self, + origin: TransactionOrigin, + transactions: impl IntoIterator + Send, + ) -> Vec> { + match self { + Self::Left(v) => v.validate_transactions_with_origin(origin, transactions).await, + Self::Right(v) => v.validate_transactions_with_origin(origin, transactions).await, + } + } + fn on_new_head_block(&self, new_tip_block: &SealedBlock) { match self { Self::Left(v) => v.on_new_head_block(new_tip_block), diff --git a/crates/transaction-pool/src/validate/task.rs b/crates/transaction-pool/src/validate/task.rs index d03b1363708..bfd5b29bbf8 100644 --- a/crates/transaction-pool/src/validate/task.rs +++ b/crates/transaction-pool/src/validate/task.rs @@ -245,14 +245,12 @@ where let (tx, rx) = oneshot::channel(); { let res = { - let to_validation_task = self.to_validation_task.clone(); let validator = self.validator.clone(); let fut = Box::pin(async move { let res = validator.validate_transaction(origin, transaction).await; let _ = tx.send(res); }); - let to_validation_task = to_validation_task.lock().await; - to_validation_task.send(fut).await + self.to_validation_task.lock().await.send(fut).await }; if res.is_err() { return TransactionValidationOutcome::Error( @@ -281,38 +279,44 @@ where let (tx, rx) = oneshot::channel(); { let res = { - let to_validation_task = self.to_validation_task.clone(); let validator = self.validator.clone(); let fut = Box::pin(async move { let res = validator.validate_transactions(transactions).await; let _ = tx.send(res); }); - let to_validation_task = to_validation_task.lock().await; - to_validation_task.send(fut).await + self.to_validation_task.lock().await.send(fut).await }; if res.is_err() { - return hashes - .into_iter() - .map(|hash| { - TransactionValidationOutcome::Error( - hash, - Box::new(TransactionValidatorError::ValidationServiceUnreachable), - ) - }) - .collect(); + return validation_service_error_outcomes(hashes) } } match rx.await { Ok(res) => res, - Err(_) => hashes - .into_iter() - .map(|hash| { - TransactionValidationOutcome::Error( - hash, - Box::new(TransactionValidatorError::ValidationServiceUnreachable), - ) - }) - .collect(), + Err(_) => validation_service_error_outcomes(hashes), + } + } + + async fn validate_transactions_with_origin( + &self, + origin: TransactionOrigin, + transactions: impl IntoIterator + Send, + ) -> Vec> { + let transactions: Vec<_> = transactions.into_iter().collect(); + let hashes: Vec<_> = transactions.iter().map(|tx| *tx.hash()).collect(); + let (tx, rx) = oneshot::channel(); + let validator = self.validator.clone(); + let fut = Box::pin(async move { + let res = validator.validate_transactions_with_origin(origin, transactions).await; + let _ = tx.send(res); + }); + + if self.to_validation_task.lock().await.send(fut).await.is_err() { + return validation_service_error_outcomes(hashes) + } + + match rx.await { + Ok(res) => res, + Err(_) => validation_service_error_outcomes(hashes), } } @@ -321,6 +325,21 @@ where } } +#[inline] +fn validation_service_error_outcomes( + hashes: Vec, +) -> Vec> { + hashes + .into_iter() + .map(|hash| { + TransactionValidationOutcome::Error( + hash, + Box::new(TransactionValidatorError::ValidationServiceUnreachable), + ) + }) + .collect() +} + #[cfg(test)] mod tests { use super::*; @@ -377,4 +396,59 @@ mod tests { assert_eq!(out.len(), 2); assert!(out.iter().all(|o| matches!(o, TransactionValidationOutcome::Valid { .. }))); } + + #[derive(Debug)] + struct SameOriginBatchValidator; + + impl TransactionValidator for SameOriginBatchValidator { + type Transaction = MockTransaction; + type Block = reth_ethereum_primitives::Block; + + async fn validate_transaction( + &self, + _origin: TransactionOrigin, + _transaction: Self::Transaction, + ) -> TransactionValidationOutcome { + panic!("same-origin batches must use the batch validator") + } + + async fn validate_transactions_with_origin( + &self, + origin: TransactionOrigin, + transactions: impl IntoIterator + Send, + ) -> Vec> { + transactions + .into_iter() + .map(|transaction| TransactionValidationOutcome::Valid { + balance: U256::ZERO, + state_nonce: 0, + bytecode_hash: None, + transaction: ValidTransaction::Valid(transaction), + propagate: matches!(origin, TransactionOrigin::Local), + authorities: None, + }) + .collect() + } + } + + #[tokio::test] + async fn executor_forwards_same_origin_batches() { + let (executor, task) = TransactionValidationTaskExecutor::new(SameOriginBatchValidator); + tokio::spawn(task.run()); + + let transactions = vec![MockTransaction::legacy(), MockTransaction::eip1559()]; + let expected_hashes = transactions.iter().map(|tx| *tx.hash()).collect::>(); + let outcomes = executor + .validate_transactions_with_origin(TransactionOrigin::Local, transactions) + .await; + + assert_eq!(outcomes.len(), expected_hashes.len()); + assert!(outcomes.into_iter().zip(expected_hashes).all(|(outcome, expected_hash)| { + matches!( + outcome, + TransactionValidationOutcome::Valid { transaction, propagate: true, .. } + if transaction.hash() == &expected_hash + ) + })); + } } diff --git a/crates/trie/common/src/trie_data.rs b/crates/trie/common/src/trie_data.rs index 865cd2a0d9c..62ca96be60d 100644 --- a/crates/trie/common/src/trie_data.rs +++ b/crates/trie/common/src/trie_data.rs @@ -4,7 +4,6 @@ //! trie-related data containing sorted hashed state and trie updates. use crate::{ - prefix_set::TriePrefixSetsMut, updates::{TrieUpdates, TrieUpdatesSorted}, HashedPostState, HashedPostStateSorted, }; @@ -35,13 +34,11 @@ impl SortedTrieData { } } -/// Container for sorted trie data that also includes `changed_paths`. +/// Container for sorted trie data. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct ComputedTrieData { /// Sorted trie data: hashed state and trie updates. pub sorted: SortedTrieData, - /// Changed trie node base paths produced by state root computation. - pub changed_paths: Option>, } impl ComputedTrieData { @@ -50,16 +47,7 @@ impl ComputedTrieData { hashed_state: Arc, trie_updates: Arc, ) -> Self { - Self::new_with_changed_paths(hashed_state, trie_updates, None) - } - - /// Construct sorted trie data with changed trie node base paths for one block. - pub const fn new_with_changed_paths( - hashed_state: Arc, - trie_updates: Arc, - changed_paths: Option>, - ) -> Self { - Self { sorted: SortedTrieData::new(hashed_state, trie_updates), changed_paths } + Self { sorted: SortedTrieData::new(hashed_state, trie_updates) } } } @@ -128,15 +116,11 @@ impl LazyTrieData { pub fn pending( hashed_state: Arc, trie_updates: Arc, - changed_paths: Option>, ) -> (Self, LazyTrieDataProducer) { let value = Arc::new(OnceLock::new()); ( Self { data: Arc::clone(&value), mode: LazyTrieDataMode::Pending }, - LazyTrieDataProducer { - value, - inputs: PendingInputs { hashed_state, trie_updates, changed_paths }, - }, + LazyTrieDataProducer { value, inputs: PendingInputs { hashed_state, trie_updates } }, ) } @@ -235,7 +219,7 @@ impl LazyTrieDataProducer { /// Computes sorted trie data, publishes it to waiters, and returns it to the task owner. pub fn compute_and_publish(self) -> ComputedTrieData { let Self { value, inputs } = self; - let computed = Self::sort(inputs.hashed_state, inputs.trie_updates, inputs.changed_paths); + let computed = Self::sort(inputs.hashed_state, inputs.trie_updates); let _ = value.set(computed.clone()); computed } @@ -244,7 +228,6 @@ impl LazyTrieDataProducer { pub fn sort( hashed_state: Arc, trie_updates: Arc, - changed_paths: Option>, ) -> ComputedTrieData { #[cfg(feature = "rayon")] let (sorted_hashed_state, sorted_trie_updates) = rayon::join( @@ -270,11 +253,7 @@ impl LazyTrieDataProducer { }, ); - ComputedTrieData::new_with_changed_paths( - Arc::new(sorted_hashed_state), - Arc::new(sorted_trie_updates), - changed_paths, - ) + ComputedTrieData::new(Arc::new(sorted_hashed_state), Arc::new(sorted_trie_updates)) } } @@ -285,8 +264,6 @@ struct PendingInputs { hashed_state: Arc, /// Unsorted trie updates from state root computation. trie_updates: Arc, - /// Changed trie node base paths from state root computation. - changed_paths: Option>, } #[cfg(test)] @@ -306,21 +283,9 @@ mod tests { LazyTrieData::pending( Arc::new(HashedPostState::default()), Arc::new(TrieUpdates::default()), - None, ) } - fn assert_changed_paths_ptr_eq( - left: &Option>, - right: &Option>, - ) { - match (left, right) { - (Some(left), Some(right)) => assert!(Arc::ptr_eq(left, right)), - (None, None) => {} - _ => panic!("changed paths presence mismatch"), - } - } - #[test] fn test_lazy_ready_is_initialized() { let lazy = LazyTrieData::ready(ComputedTrieData::default()); @@ -366,10 +331,8 @@ mod tests { assert!(Arc::ptr_eq(&published.sorted.hashed_state, &first.sorted.hashed_state)); assert!(Arc::ptr_eq(&published.sorted.trie_updates, &first.sorted.trie_updates)); - assert_changed_paths_ptr_eq(&published.changed_paths, &first.changed_paths); assert!(Arc::ptr_eq(&first.sorted.hashed_state, &second.sorted.hashed_state)); assert!(Arc::ptr_eq(&first.sorted.trie_updates, &second.sorted.trie_updates)); - assert_changed_paths_ptr_eq(&first.changed_paths, &second.changed_paths); } #[test] @@ -385,7 +348,6 @@ mod tests { assert!(Arc::ptr_eq(&published.sorted.hashed_state, &result.sorted.hashed_state)); assert!(Arc::ptr_eq(&published.sorted.trie_updates, &result.sorted.trie_updates)); - assert_changed_paths_ptr_eq(&published.changed_paths, &result.changed_paths); } #[test] @@ -400,10 +362,8 @@ mod tests { assert!(Arc::ptr_eq(&published.sorted.hashed_state, &result1.sorted.hashed_state)); assert!(Arc::ptr_eq(&published.sorted.trie_updates, &result1.sorted.trie_updates)); - assert_changed_paths_ptr_eq(&published.changed_paths, &result1.changed_paths); assert!(Arc::ptr_eq(&result1.sorted.hashed_state, &result2.sorted.hashed_state)); assert!(Arc::ptr_eq(&result1.sorted.trie_updates, &result2.sorted.trie_updates)); - assert_changed_paths_ptr_eq(&result1.changed_paths, &result2.changed_paths); } #[test] @@ -418,7 +378,7 @@ mod tests { )]); let (deferred, task) = - LazyTrieData::pending(Arc::new(hashed_state), Arc::new(TrieUpdates::default()), None); + LazyTrieData::pending(Arc::new(hashed_state), Arc::new(TrieUpdates::default())); let _ = task.compute_and_publish(); let result = deferred.get().clone(); @@ -435,7 +395,6 @@ mod tests { let (deferred, task) = LazyTrieData::pending( Arc::new(HashedPostState { accounts, storages: Default::default() }), Arc::new(TrieUpdates::default()), - None, ); let _ = task.compute_and_publish(); diff --git a/crates/trie/parallel/src/error.rs b/crates/trie/parallel/src/error.rs index 86a7a07ac6f..2eb349f7700 100644 --- a/crates/trie/parallel/src/error.rs +++ b/crates/trie/parallel/src/error.rs @@ -8,9 +8,18 @@ pub enum StateRootTaskError { /// Provider error. #[error(transparent)] Provider(#[from] ProviderError), + /// Proof dispatch error. + #[error("proof dispatch failed: {_0}")] + ProofDispatch(ProviderError), /// Sparse trie error. #[error(transparent)] SparseTrie(#[from] SparseTrieError), + /// Sparse trie task stalled. + #[error("sparse trie task stalled")] + Stalled, + /// The consumer dropped its cancel guard without waiting for the result. + #[error("state root task canceled: consumer dropped the handle")] + Canceled, /// Other unspecified error. #[error("{_0}")] Other(String), diff --git a/crates/trie/parallel/src/proof_task.rs b/crates/trie/parallel/src/proof_task.rs index f788447f119..1936e828e73 100644 --- a/crates/trie/parallel/src/proof_task.rs +++ b/crates/trie/parallel/src/proof_task.rs @@ -362,7 +362,7 @@ impl ProofWorkerHandle { input.into_proof_result_sender(); let _ = result_tx.send(ProofResultMessage { - result: Err(StateRootTaskError::Provider(error.clone())), + result: Err(StateRootTaskError::ProofDispatch(error.clone())), elapsed: start.elapsed(), state, }); @@ -434,7 +434,7 @@ where let span = debug_span!( target: "trie::proof_task", - "V2 Storage proof calculation", + "Storage proof calculation", n = %targets.len(), ); let _span_guard = span.enter(); @@ -971,7 +971,7 @@ where let span = debug_span!( target: "trie::proof_task", - "Account V2 multiproof calculation", + "Account multiproof calculation", account_targets = account_targets.len(), storage_targets = storage_targets.values().map(|t| t.len()).sum::(), ); diff --git a/crates/trie/parallel/src/state_root_task.rs b/crates/trie/parallel/src/state_root_task.rs index 7b816ea6a47..3bd7b2e99e2 100644 --- a/crates/trie/parallel/src/state_root_task.rs +++ b/crates/trie/parallel/src/state_root_task.rs @@ -11,8 +11,7 @@ use crate::error::StateRootTaskError; use alloy_evm::block::OnStateHook; use alloy_primitives::{keccak256, map::B256Map, B256}; use reth_trie::{ - prefix_set::TriePrefixSetsMut, updates::TrieUpdates, HashedPostState, HashedStorage, - MultiProofTargetsV2, ProofV2Target, + updates::TrieUpdates, HashedPostState, HashedStorage, MultiProofTargetsV2, ProofV2Target, }; use revm::state::EvmState; use std::{fmt, sync::Arc}; @@ -42,8 +41,8 @@ pub struct StateRootComputeOutcome { pub state_root: B256, /// The trie updates. pub trie_updates: Arc, - /// Changed trie node base paths retained while computing the root. - pub changed_paths: Option>, + /// Hashed post state produced while computing the state root. + pub hashed_state: Arc, /// Debug recorders taken from the sparse tries, keyed by `None` for account trie /// and `Some(address)` for storage tries. #[cfg(feature = "trie-debug")] @@ -56,33 +55,46 @@ pub struct StateRootComputeOutcome { /// block building). Provides channels for streaming state updates into the pipeline and receiving /// the final computed state root. /// -/// Created by `PayloadProcessor::spawn_state_root`. +/// Created by the engine's state-root strategy. #[derive(Debug)] pub struct StateRootHandle { /// The state root that the cached sparse trie is anchored at (parent block's state root). cached_trie_state_root: B256, - /// Channel for streaming state updates and proof targets into the sparse trie pipeline. - updates_tx: crossbeam_channel::Sender, + /// Best-effort hint capability, taken once by prewarm wiring. + hint: Option, + /// The single authoritative update capability. + /// + /// Taken exactly once, either as an execution hook (serial execution) or as a hashed + /// update stream (parallel BAL streaming), so per block exactly one producer can finish + /// the update stream. Only producers hold update senders: once the taken capabilities are + /// dropped or finished, the update channel closes and the task knows producers are done. + authoritative: Option, + /// Guard whose drop cancels the state-root task if it is still running. + cancel_guard: StateRootTaskCancelGuard, /// Receiver for the final state root result. state_root_rx: Option>>, /// Receiver for the hashed post state. - hashed_state_rx: Option>, + hashed_state_rx: Option>>, } impl StateRootHandle { /// Creates a new [`StateRootHandle`]. - pub const fn new( + pub fn new( cached_trie_state_root: B256, updates_tx: crossbeam_channel::Sender, + cancel_guard: StateRootTaskCancelGuard, state_root_rx: std::sync::mpsc::Receiver< Result, >, - hashed_state_rx: std::sync::mpsc::Receiver, + hashed_state_rx: std::sync::mpsc::Receiver>, ) -> Self { + let sink: Arc = Arc::new(SparseTrieStateRootSink::new(updates_tx)); Self { cached_trie_state_root, - updates_tx, + hint: Some(StateRootHintStream::new(Arc::clone(&sink))), + authoritative: Some(StateRootUpdateStream::new(sink)), + cancel_guard, state_root_rx: Some(state_root_rx), hashed_state_rx: Some(hashed_state_rx), } @@ -93,12 +105,38 @@ impl StateRootHandle { self.cached_trie_state_root } - /// Returns semantic stream views backed by this sparse trie task. - pub fn streams(&self, install_execution_hook: bool) -> StateRootStreams { - StateRootStreams::from_sink( - Arc::new(SparseTrieStateRootSink::new(self.updates_tx.clone())), - install_execution_hook, - ) + /// Takes the best-effort hint capability used by transaction prewarming. + /// + /// # Panics + /// + /// If called more than once. + pub const fn take_hint_stream(&mut self) -> StateRootHintStream { + self.hint.take().expect("hint stream already taken") + } + + /// Takes the authoritative update capability as an EVM state hook. + /// + /// The hook finishes the update stream when dropped. It shares one slot with + /// [`Self::take_hashed_update_stream`], so only one of the two can exist per block. + /// + /// # Panics + /// + /// If the authoritative capability was already taken in either form. + pub fn take_execution_hook(&mut self) -> StateRootUpdateHook { + self.take_hashed_update_stream().into_state_hook() + } + + /// Takes the authoritative update capability as a pre-hashed update stream. + /// + /// The stream is finished explicitly with [`StateRootUpdateStream::finish`]. It shares + /// one slot with [`Self::take_execution_hook`], so only one of the two can exist per + /// block. + /// + /// # Panics + /// + /// If the authoritative capability was already taken in either form. + pub const fn take_hashed_update_stream(&mut self) -> StateRootUpdateStream { + self.authoritative.take().expect("authoritative update capability already taken") } /// Awaits the state root computation result. @@ -130,36 +168,62 @@ impl StateRootHandle { /// # Panics /// /// If called more than once. - pub const fn take_hashed_state_rx(&mut self) -> std::sync::mpsc::Receiver { + pub const fn take_hashed_state_rx( + &mut self, + ) -> std::sync::mpsc::Receiver> { self.hashed_state_rx.take().expect("hashed_state already taken") } /// Converts this sparse-trie handle into the opaque handle passed to payload builders. + /// + /// The payload builder only executes transactions, so the handle carries the execution + /// hook; the hint capability is dropped here. pub fn into_payload_state_root_handle(mut self) -> PayloadStateRootHandle { - let streams = self.streams(true); + let hook = self.take_execution_hook(); PayloadStateRootHandle { name: "sparse-trie", - streams, + hook: Some(hook), + cancel_guard: Some(self.cancel_guard), state_root_rx: self.state_root_rx.take(), hashed_state_rx: self.hashed_state_rx.take(), } } } +/// Guard that cancels a state-root task when dropped. +/// +/// The task watches the paired receiver in its event loop. No message is ever sent: the guard +/// dropping disconnects the channel, which the task treats as the consumer abandoning the +/// computation (for example on a timeout fallback or when a payload job is dropped unused). +#[derive(Debug)] +pub struct StateRootTaskCancelGuard(#[allow(dead_code)] crossbeam_channel::Sender<()>); + +impl StateRootTaskCancelGuard { + /// Creates a guard and the receiver a task watches for cancellation. + pub fn channel() -> (Self, crossbeam_channel::Receiver<()>) { + let (tx, rx) = crossbeam_channel::bounded(0); + (Self(tx), rx) + } +} + /// Opaque state-root task handle passed to payload builders. pub struct PayloadStateRootHandle { name: &'static str, - streams: StateRootStreams, + /// Execution hook that streams per-transaction updates; taken once when building starts. + hook: Option, + /// Cancels the backing task when the handle is dropped without consuming the result. + cancel_guard: Option, state_root_rx: Option>>, - hashed_state_rx: Option>, + hashed_state_rx: Option>>, } impl fmt::Debug for PayloadStateRootHandle { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("PayloadStateRootHandle") .field("name", &self.name) - .field("streams", &self.streams) + .field("has_hook", &self.hook.is_some()) + .field("has_cancel_guard", &self.cancel_guard.is_some()) .field("has_state_root_rx", &self.state_root_rx.is_some()) .field("has_hashed_state_rx", &self.hashed_state_rx.is_some()) .finish() @@ -168,15 +232,18 @@ impl fmt::Debug for PayloadStateRootHandle { impl PayloadStateRootHandle { /// Creates an opaque payload state-root handle. + /// + /// Tasks with a drop-to-cancel guard should attach it via the `StateRootHandle` + /// conversion; handles created here rely on their own task lifecycle. pub const fn new( name: &'static str, - streams: StateRootStreams, + hook: Option, state_root_rx: std::sync::mpsc::Receiver< Result, >, - hashed_state_rx: Option>, + hashed_state_rx: Option>>, ) -> Self { - Self { name, streams, state_root_rx: Some(state_root_rx), hashed_state_rx } + Self { name, hook, cancel_guard: None, state_root_rx: Some(state_root_rx), hashed_state_rx } } /// Returns the task name used in logs. @@ -184,17 +251,13 @@ impl PayloadStateRootHandle { self.name } - /// Returns a state hook that streams execution updates and finishes on drop. + /// Takes the state hook that streams execution updates and finishes the stream on drop. /// /// # Panics /// - /// If the task was created without an execution stream. - pub fn state_hook(&self) -> impl OnStateHook { - self.streams - .execution - .as_ref() - .expect("payload state root task missing execution stream") - .state_hook() + /// If the handle was created without an execution hook, or the hook was already taken. + pub const fn take_state_hook(&mut self) -> StateRootUpdateHook { + self.hook.take().expect("payload state root task missing execution hook") } /// Awaits the state root computation result. @@ -214,7 +277,7 @@ impl PayloadStateRootHandle { /// yet. pub const fn try_take_hashed_state_rx( &mut self, - ) -> Option> { + ) -> Option>> { self.hashed_state_rx.take() } } @@ -301,20 +364,29 @@ impl StateRootHintStream { } } -/// Pre-hashed authoritative update view of a state-root stream. -#[derive(Clone)] -pub struct StateRootHashedUpdateStream { +/// Authoritative update capability of a state-root stream. +/// +/// Exactly one of these exists per state-root task, so exactly one producer can end the +/// update stream: either the EVM state hook made with [`Self::into_state_hook`] (finishes on +/// drop) or a pre-hashed update producer such as BAL streaming (calls [`Self::finish`]). The +/// type is deliberately not `Clone` and finishing consumes it, so a second end-of-stream +/// signal cannot be produced. +/// +/// Dropping the stream without calling [`Self::finish`] (for example when a producer dies) +/// deliberately does not finish it: an unfinished stream means the updates are incomplete, +/// and the task must not compute a root from them. +pub struct StateRootUpdateStream { inner: Arc, } -impl fmt::Debug for StateRootHashedUpdateStream { +impl fmt::Debug for StateRootUpdateStream { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("StateRootHashedUpdateStream").finish_non_exhaustive() + f.debug_struct("StateRootUpdateStream").finish_non_exhaustive() } } -impl StateRootHashedUpdateStream { - /// Creates a new hashed update stream view. +impl StateRootUpdateStream { + /// Creates a new authoritative update stream backed by the given sink. pub fn new(inner: Arc) -> Self { Self { inner } } @@ -325,86 +397,16 @@ impl StateRootHashedUpdateStream { } /// Finishes the authoritative update stream. - pub fn on_updates_finished(&self) { + pub fn finish(self) { self.inner.on_updates_finished(); } -} - -/// Normal execution view of a state-root stream. -#[derive(Clone)] -pub struct StateRootExecutionStream { - inner: Arc, -} - -impl fmt::Debug for StateRootExecutionStream { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("StateRootExecutionStream").finish_non_exhaustive() - } -} - -impl StateRootExecutionStream { - /// Creates a new execution stream view. - pub fn new(inner: Arc) -> Self { - Self { inner } - } - - /// Returns an EVM state hook that finishes the stream when dropped. - pub fn state_hook(&self) -> StateRootUpdateHook { - StateRootUpdateHook { inner: Arc::clone(&self.inner) } - } -} - -/// State-root streams exposed to execution and prewarm code. -#[derive(Clone, Default)] -pub struct StateRootStreams { - hint: Option, - hashed_updates: Option, - execution: Option, -} - -impl fmt::Debug for StateRootStreams { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("StateRootStreams") - .field("has_hint_stream", &self.hint.is_some()) - .field("has_hashed_update_stream", &self.hashed_updates.is_some()) - .field("has_execution_stream", &self.execution.is_some()) - .finish() - } -} - -impl StateRootStreams { - /// Creates stream views backed by one sink. - pub fn from_sink(inner: Arc, install_execution_hook: bool) -> Self { - Self { - hint: Some(StateRootHintStream::new(Arc::clone(&inner))), - hashed_updates: Some(StateRootHashedUpdateStream::new(Arc::clone(&inner))), - execution: install_execution_hook.then(|| StateRootExecutionStream::new(inner)), - } - } - - /// Creates a stream set with no state-root task attached. - pub const fn empty() -> Self { - Self { hint: None, hashed_updates: None, execution: None } - } - /// Returns the hint-only stream. - pub fn hint_stream(&self) -> Option { - self.hint.clone() - } - - /// Returns the pre-hashed update stream. - pub fn hashed_update_stream(&self) -> Option { - self.hashed_updates.clone() - } - - /// Returns true if no stream views are installed. - pub const fn is_empty(&self) -> bool { - self.hint.is_none() && self.hashed_updates.is_none() && self.execution.is_none() - } - - /// Takes the execution stream. - pub const fn take_execution_stream(&mut self) -> Option { - self.execution.take() + /// Converts this capability into an EVM state hook that finishes the stream on drop. + /// + /// See [`StateRootUpdateHook`] for why the hook finishes on drop while the bare stream + /// does not, and how a panic during execution is excluded from that. + pub fn into_state_hook(self) -> StateRootUpdateHook { + StateRootUpdateHook { inner: self.inner } } } @@ -412,6 +414,14 @@ impl StateRootStreams { /// /// Dropping the hook signals the end of the update stream, so the hook is deliberately not /// `Clone`: a second copy would fire a spurious end-of-stream signal. +/// +/// Unlike [`StateRootUpdateStream::finish`], the end of the stream is signaled by drop and +/// not by an explicit call, because the EVM owns the hook until it is dropped and gives it no +/// other end-of-execution signal. A drop during a panic unwind is excluded: execution died +/// mid-block, so the stream stays unfinished and the task reports an error instead of +/// computing a root from incomplete updates. Execution that fails by returning an error still +/// drops the hook normally and finishes the stream; the caller abandons the result in that +/// case, and the stored trie is rejected by the anchor check on the next block. pub struct StateRootUpdateHook { inner: Arc, } @@ -430,6 +440,11 @@ impl OnStateHook for StateRootUpdateHook { impl Drop for StateRootUpdateHook { fn drop(&mut self) { + // A drop during a panic unwind means execution died mid-block. Leave the stream + // unfinished so the task fails instead of computing a root from partial updates. + if std::thread::panicking() { + return; + } self.inner.on_updates_finished(); } } @@ -561,25 +576,21 @@ mod tests { } #[test] - fn state_root_streams_forward_to_sink() { + fn state_root_capabilities_forward_to_sink() { let sink = Arc::new(CountingSink::default()); - let mut streams = StateRootStreams::from_sink(sink.clone(), true); + let hint_stream = StateRootHintStream::new(sink.clone()); let mut storages = B256Map::default(); storages.insert(B256::repeat_byte(0x02), vec![B256::repeat_byte(0x03)]); - streams - .hint_stream() - .expect("hint stream") + hint_stream .on_access_hint(StateAccessHint { accounts: vec![B256::repeat_byte(0x01)], storages }); - let hashed_updates = streams.hashed_update_stream().expect("hashed update stream"); - hashed_updates.on_hashed_state_update(HashedPostState::default()); - hashed_updates.on_updates_finished(); + let updates = StateRootUpdateStream::new(sink.clone()); + updates.on_hashed_state_update(HashedPostState::default()); + updates.finish(); - let execution_stream = streams.take_execution_stream().expect("execution stream"); - assert!(streams.take_execution_stream().is_none()); { - let mut hook = execution_stream.state_hook(); + let mut hook = StateRootUpdateStream::new(sink.clone()).into_state_hook(); hook.on_state(EvmState::default()); } @@ -589,6 +600,44 @@ mod tests { assert_eq!(sink.finished_updates.load(Ordering::Relaxed), 2); } + /// A hook dropped by a panic unwind must not finish the stream: the updates are + /// incomplete, and a finish marker would make the task compute a root from them. + #[test] + fn hook_dropped_during_panic_does_not_finish_stream() { + let sink = Arc::new(CountingSink::default()); + let hook = StateRootUpdateStream::new(sink.clone()).into_state_hook(); + + let result = std::thread::spawn(move || { + let _hook = hook; + panic!("execution died mid-block"); + }) + .join(); + + assert!(result.is_err()); + assert_eq!(sink.finished_updates.load(Ordering::Relaxed), 0); + } + + /// The authoritative capability is a single slot: taking it as a hook and then again as + /// a hashed update stream (or in any other combination) must panic. + #[test] + #[should_panic(expected = "authoritative update capability already taken")] + fn authoritative_capability_can_only_be_taken_once() { + let (updates_tx, _updates_rx) = crossbeam_channel::unbounded(); + let (cancel_guard, _cancel_rx) = StateRootTaskCancelGuard::channel(); + let (_state_root_tx, state_root_rx) = std::sync::mpsc::channel(); + let (_hashed_state_tx, hashed_state_rx) = std::sync::mpsc::channel(); + let mut handle = StateRootHandle::new( + B256::ZERO, + updates_tx, + cancel_guard, + state_root_rx, + hashed_state_rx, + ); + + let _hook = handle.take_execution_hook(); + let _ = handle.take_hashed_update_stream(); + } + /// Lifecycle of the opaque handle a strategy hands to the payload builder: the execution /// hook streams updates into the sink and signals completion on drop, the hashed-state /// receiver can be taken exactly once, and the outcome arrives through the state-root @@ -596,23 +645,23 @@ mod tests { #[test] fn payload_state_root_handle_lifecycle() { let sink = Arc::new(CountingSink::default()); - let streams = StateRootStreams::from_sink(sink.clone(), true); + let hook = StateRootUpdateStream::new(sink.clone()).into_state_hook(); let (state_root_tx, state_root_rx) = std::sync::mpsc::channel(); let (hashed_state_tx, hashed_state_rx) = std::sync::mpsc::channel(); let mut handle = - PayloadStateRootHandle::new("test", streams, state_root_rx, Some(hashed_state_rx)); + PayloadStateRootHandle::new("test", Some(hook), state_root_rx, Some(hashed_state_rx)); assert_eq!(handle.name(), "test"); { - let mut hook = handle.state_hook(); + let mut hook = handle.take_state_hook(); hook.on_state(EvmState::default()); } assert_eq!(sink.state_updates.load(Ordering::Relaxed), 1); assert_eq!(sink.finished_updates.load(Ordering::Relaxed), 1); - hashed_state_tx.send(HashedPostState::default()).unwrap(); + hashed_state_tx.send(Arc::new(HashedPostState::default())).unwrap(); let rx = handle.try_take_hashed_state_rx().expect("first take returns the receiver"); assert!(rx.recv().is_ok()); assert!(handle.try_take_hashed_state_rx().is_none(), "second take returns None"); @@ -621,7 +670,7 @@ mod tests { .send(Ok(StateRootComputeOutcome { state_root: B256::repeat_byte(0x42), trie_updates: Arc::new(TrieUpdates::default()), - changed_paths: None, + hashed_state: Arc::new(HashedPostState::default()), #[cfg(feature = "trie-debug")] debug_recorders: Vec::new(), })) diff --git a/crates/trie/sparse/Cargo.toml b/crates/trie/sparse/Cargo.toml index b09caf64141..59f2e1b13bf 100644 --- a/crates/trie/sparse/Cargo.toml +++ b/crates/trie/sparse/Cargo.toml @@ -29,6 +29,7 @@ serde = { workspace = true, features = ["derive"], optional = true } serde_json = { workspace = true, optional = true } smallvec = { workspace = true, optional = true } slotmap = { workspace = true, optional = true } +strum = { workspace = true, features = ["derive"] } # metrics reth-metrics = { workspace = true, optional = true } @@ -68,6 +69,7 @@ std = [ "tracing/std", "serde?/std", "serde_json?/std", + "strum/std", "reth-tracing/std", "either/std", ] diff --git a/crates/trie/sparse/src/arena/mod.rs b/crates/trie/sparse/src/arena/mod.rs index 3c200afaff2..cd4bacb4d2e 100644 --- a/crates/trie/sparse/src/arena/mod.rs +++ b/crates/trie/sparse/src/arena/mod.rs @@ -15,8 +15,8 @@ use alloy_trie::TrieMask; use core::{cmp::Reverse, mem}; use reth_execution_errors::SparseTrieResult; use reth_trie_common::{ - prefix_set::PrefixSetMut, BranchNodeMasks, BranchNodeRef, ExtensionNodeRef, LeafNodeRef, - Nibbles, ProofTrieNodeV2, RlpNode, TrieNodeV2, EMPTY_ROOT_HASH, + BranchNodeMasks, BranchNodeRef, ExtensionNodeRef, LeafNodeRef, Nibbles, ProofTrieNodeV2, + RlpNode, TrieNodeV2, EMPTY_ROOT_HASH, }; use slotmap::{DefaultKey, SlotMap}; use smallvec::SmallVec; @@ -125,8 +125,6 @@ struct ArenaTrieBuffers { /// Trie updates built up directly during hashing and structural changes. `Some` when /// tracking updates, `None` otherwise. Initialized alongside `updates` in `set_updates`. updates: Option, - /// Changed node base paths accumulated during hashing. - changed_paths: Option, /// Reusable buffer for RLP encoding. rlp_buf: Vec, /// Reusable buffer for child `RlpNode`s during hashing. @@ -138,9 +136,6 @@ impl ArenaTrieBuffers { if let Some(updates) = self.updates.as_mut() { updates.clear(); } - if let Some(changed_paths) = self.changed_paths.as_mut() { - changed_paths.clear(); - } self.rlp_buf.clear(); self.rlp_node_buf.clear(); } @@ -176,12 +171,11 @@ impl ArenaSparseSubtrie { /// Creates a new subtrie with a pre-allocated root slot containing /// [`ArenaSparseNode::EmptyRoot`]. The caller must overwrite `subtrie.arena[subtrie.root]` /// before use. - fn new(record_updates: bool, record_changed_paths: bool) -> Box { + fn new(record_updates: bool) -> Box { let mut arena = SlotMap::new(); let root = arena.insert(ArenaSparseNode::EmptyRoot); let buffers = ArenaTrieBuffers { updates: record_updates.then(SparseTrieUpdates::default), - changed_paths: record_changed_paths.then(PrefixSetMut::default), ..Default::default() }; Box::new(Self { @@ -223,9 +217,8 @@ impl ArenaSparseSubtrie { /// in lexicographic order. /// /// `retained_leaves` must yield leaves in sorted order and be scoped to this subtrie's key - /// range. Builds a fresh arena by copying only retained nodes from the root, blinding - /// non-retained children at the boundary. Non-retained subtrees are never visited — they - /// are dropped with the old arena. + /// range. Builds a fresh arena by copying retained nodes from the root, blinding non-retained + /// children at the boundary unless their parent branch is retained. /// /// Expects that all nodes have computed hashes (i.e. `prune` is called after hashing). fn prune<'a>(&mut self, retained_leaves: impl IntoIterator) -> usize { @@ -252,10 +245,12 @@ impl ArenaSparseSubtrie { let root_node = self.arena.remove(self.root).expect("root exists"); let new_root = new_arena.insert(root_node); let mut stack = Vec::new(); + let root_is_retained = retained_leaves.peek().is_some(); if let Some(frame) = prepare_retained_node( &new_arena, new_root, self.path, + root_is_retained, &mut new_num_leaves, &mut new_nodes_heap_size, ) { @@ -278,31 +273,42 @@ impl ArenaSparseSubtrie { retained_leaves.next(); } - if retained_leaves.peek().is_some_and(|retained| retained.starts_with(&child_path)) { - // Retained — move child to new arena. + let child_is_retained = + retained_leaves.peek().is_some_and(|retained| retained.starts_with(&child_path)); + if child_is_retained || frame.branch_is_retained { + // Retained or protected by a retained parent branch. let child_node = self.arena.remove(old_child_idx).expect("child exists"); let new_child_idx = new_arena.insert(child_node); - let ArenaSparseNode::Branch(b) = &mut new_arena[parent_new_idx] else { - unreachable!() - }; - b.children[child_pos] = ArenaSparseNodeBranchChild::Revealed(new_child_idx); if let Some(frame) = prepare_retained_node( &new_arena, new_child_idx, child_path, + child_is_retained, &mut new_num_leaves, &mut new_nodes_heap_size, ) { stack.push(frame); } + let ArenaSparseNode::Branch(b) = &mut new_arena[parent_new_idx] else { + unreachable!() + }; + b.children[child_pos] = ArenaSparseNodeBranchChild::Revealed(new_child_idx); } else { // Not retained — blind the child slot in the new arena. - let rlp_node = self.arena[old_child_idx] + let node = &self.arena[old_child_idx]; + let rlp_node = node .state_ref() .expect("child must have state") .cached_rlp_node() .cloned() .expect("pruned child must have cached RLP (prune runs after hashing)"); + trace!( + target: TRACE_TARGET, + path = ?child_path, + variant = %AsRef::::as_ref(node), + cached_rlp_node = ?rlp_node, + "pruning node", + ); let ArenaSparseNode::Branch(b) = &mut new_arena[parent_new_idx] else { unreachable!() }; @@ -327,6 +333,7 @@ impl ArenaSparseSubtrie { branch_logical_path: Nibbles, state_mask: TrieMask, remaining_child_mask: TrieMask, + branch_is_retained: bool, } impl CopyFrame { @@ -352,6 +359,7 @@ impl ArenaSparseSubtrie { new_arena: &NodeArena, new_idx: Index, node_path: Nibbles, + branch_is_retained: bool, new_num_leaves: &mut u64, new_nodes_heap_size: &mut usize, ) -> Option { @@ -373,6 +381,7 @@ impl ArenaSparseSubtrie { branch_logical_path, state_mask: b.state_mask, remaining_child_mask: b.state_mask, + branch_is_retained, }) } } @@ -680,43 +689,6 @@ impl ArenaParallelSparseTrie { self } - /// Set whether changed node base paths should be retained during hashing. - pub fn set_changed_paths(&mut self, retain_changed_paths: bool) { - if retain_changed_paths { - self.buffers.changed_paths.get_or_insert_with(PrefixSetMut::default).clear(); - } else { - self.buffers.changed_paths = None; - } - - for (_, node) in &mut self.upper_arena { - let ArenaSparseNode::Subtrie(subtrie) = node else { - continue; - }; - if retain_changed_paths { - subtrie.buffers.changed_paths.get_or_insert_with(PrefixSetMut::default).clear(); - } else { - subtrie.buffers.changed_paths = None; - } - } - } - - /// Set whether changed node base paths should be retained during hashing. - pub fn with_changed_paths(mut self, retain_changed_paths: bool) -> Self { - self.set_changed_paths(retain_changed_paths); - self - } - - /// Takes all retained changed node base paths, preserving allocation capacity for reuse. - pub fn take_changed_paths(&mut self) -> PrefixSetMut { - match self.buffers.changed_paths.take() { - Some(changed_paths) => { - self.buffers.changed_paths = Some(PrefixSetMut::with_capacity(changed_paths.len())); - changed_paths - } - None => PrefixSetMut::default(), - } - } - /// Resets the debug recorder and records the current trie state as `SetRoot` + `RevealNodes` /// ops, representing the initial state at the beginning of a block (after pruning). /// @@ -889,10 +861,7 @@ impl ArenaParallelSparseTrie { } trace!(target: TRACE_TARGET, ?child_path, "Wrapping child into subtrie"); - let mut subtrie = ArenaSparseSubtrie::new( - self.buffers.updates.is_some(), - self.buffers.changed_paths.is_some(), - ); + let mut subtrie = ArenaSparseSubtrie::new(self.buffers.updates.is_some()); subtrie.path = *child_path; let mut root_node = mem::replace(&mut self.upper_arena[child_idx], ArenaSparseNode::TakenSubtrie); @@ -1005,10 +974,6 @@ impl ArenaParallelSparseTrie { unreachable!("recycle_subtrie called on non-Subtrie node") }; Self::merge_subtrie_updates(&mut self.buffers.updates, &mut subtrie.buffers.updates); - Self::merge_subtrie_changed_paths( - &mut self.buffers.changed_paths, - &mut subtrie.buffers.changed_paths, - ); } /// Removes a [`ArenaSparseNode::Subtrie`] from the upper arena at `idx` and recycles it. @@ -1131,10 +1096,6 @@ impl ArenaParallelSparseTrie { &mut self.buffers.updates, &mut subtrie.buffers.updates, ); - Self::merge_subtrie_changed_paths( - &mut self.buffers.changed_paths, - &mut subtrie.buffers.changed_paths, - ); // The migrated subtrie root may be a branch whose children now live in // the upper arena at or beyond the subtrie boundary depth. Re-wrap any @@ -1173,15 +1134,6 @@ impl ArenaParallelSparseTrie { } } - /// Merges changed node base paths from a subtrie's buffer into the parent's buffer. - /// Both `dst` and `src` must be `Some` when changed path tracking is enabled. - fn merge_subtrie_changed_paths(dst: &mut Option, src: &mut Option) { - if let Some(dst_changed_paths) = dst.as_mut() { - let src_changed_paths = src.as_mut().expect("changed path tracking is enabled"); - dst_changed_paths.append(src_changed_paths); - } - } - /// Right-pads a nibble path with zeros and packs it into a [`B256`]. fn nibbles_to_padded_b256(path: &Nibbles) -> B256 { let mut bytes = [0u8; 32]; @@ -1234,7 +1186,6 @@ impl ArenaParallelSparseTrie { let rlp_buf = &mut buffers.rlp_buf; let rlp_node_buf = &mut buffers.rlp_node_buf; let updates = &mut buffers.updates; - let changed_paths = &mut buffers.changed_paths; rlp_node_buf.clear(); @@ -1245,16 +1196,11 @@ impl ArenaParallelSparseTrie { ArenaSparseNode::EmptyRoot => return RlpNode::word_rlp(&EMPTY_ROOT_HASH), ArenaSparseNode::Leaf { .. } => { Self::encode_leaf(arena, root, rlp_buf, rlp_node_buf); - let was_dirty = arena[root].state_mut().take_cached_was_dirty(); - if was_dirty && let Some(changed_paths) = changed_paths.as_mut() { - changed_paths.insert(base_path); - } return rlp_node_buf.pop().expect("encode_leaf must push an RlpNode"); } ArenaSparseNode::Branch(b) => { if let ArenaSparseNodeState::Cached { rlp_node, .. } = &b.state { let rlp_node = rlp_node.clone(); - arena[root].state_mut().take_cached_was_dirty(); return rlp_node; } } @@ -1301,14 +1247,7 @@ impl ArenaParallelSparseTrie { rlp_node_buf.clear(); let state_mask = arena[head_idx].branch_ref().state_mask; - let branch_logical_path = { - let branch = arena[head_idx].branch_ref(); - let mut path = head_path; - path.extend(&branch.short_key); - path - }; - let mut child_subtree_emitted_changed_path = false; - for (child_idx, nibble) in BranchChildIter::new(state_mask) { + for (child_idx, _nibble) in BranchChildIter::new(state_mask) { match &arena[head_idx].branch_ref().children[child_idx] { ArenaSparseNodeBranchChild::Blinded(rlp_node) => { rlp_node_buf.push(rlp_node.clone()); @@ -1318,14 +1257,6 @@ impl ArenaParallelSparseTrie { match &arena[child_idx] { ArenaSparseNode::Leaf { .. } => { Self::encode_leaf(arena, child_idx, rlp_buf, rlp_node_buf); - if arena[child_idx].state_mut().take_cached_was_dirty() { - child_subtree_emitted_changed_path = true; - if let Some(changed_paths) = changed_paths.as_mut() { - let mut child_path = branch_logical_path; - child_path.push(nibble); - changed_paths.insert(child_path); - } - } } ArenaSparseNode::Branch(child_b) => { let ArenaSparseNodeState::Cached { rlp_node, .. } = &child_b.state @@ -1333,9 +1264,6 @@ impl ArenaParallelSparseTrie { panic!("child branch must be cached after DFS"); }; let rlp_node = rlp_node.clone(); - if arena[child_idx].state_mut().take_cached_was_dirty() { - child_subtree_emitted_changed_path = true; - } rlp_node_buf.push(rlp_node); } ArenaSparseNode::Subtrie(subtrie) => { @@ -1390,16 +1318,9 @@ impl ArenaParallelSparseTrie { ); let branch = arena[head_idx].branch_mut(); - branch.state = ArenaSparseNodeState::Cached { rlp_node: rlp_node.clone(), was_dirty }; + branch.state = ArenaSparseNodeState::Cached { rlp_node: rlp_node.clone() }; branch.branch_masks = new_branch_masks; - if was_dirty && - !child_subtree_emitted_changed_path && - let Some(changed_paths) = changed_paths.as_mut() - { - changed_paths.insert(head_path); - } - // Record trie updates for dirty branches only. // Skip the root node (empty logical path) as PST does. if let Some(trie_updates) = updates.as_mut().filter(|_| was_dirty) { @@ -1422,10 +1343,7 @@ impl ArenaParallelSparseTrie { let ArenaSparseNodeState::Cached { rlp_node, .. } = &arena[root].branch_ref().state else { panic!("root must be cached after update_cached_rlp"); }; - let rlp_node = rlp_node.clone(); - // The root has no parent to consume this one-shot marker, so clear it before returning. - arena[root].state_mut().take_cached_was_dirty(); - rlp_node + rlp_node.clone() } /// Immutable traversal to find a leaf value at `full_path` starting from `root` in `arena`. @@ -1569,13 +1487,10 @@ impl ArenaParallelSparseTrie { return; } - let was_dirty = matches!(state, ArenaSparseNodeState::Dirty); - rlp_buf.clear(); let rlp_node = LeafNodeRef { key, value }.rlp(rlp_buf); - *arena[idx].state_mut() = - ArenaSparseNodeState::Cached { rlp_node: rlp_node.clone(), was_dirty }; + *arena[idx].state_mut() = ArenaSparseNodeState::Cached { rlp_node: rlp_node.clone() }; rlp_node_buf.push(rlp_node); } @@ -2192,9 +2107,16 @@ impl ArenaParallelSparseTrie { idx: Index, nibble: Option, ) -> ArenaSparseNode { - trace!(target: TRACE_TARGET, path = ?cursor.head().unwrap().path, "pruning node"); + let path = cursor.head().expect("cursor is non-empty").path; let node = arena.remove(idx).expect("node must exist to be pruned"); let rlp_node = node.state_ref().and_then(|s| s.cached_rlp_node()).cloned(); + trace!( + target: TRACE_TARGET, + ?path, + variant = %AsRef::::as_ref(&node), + cached_rlp_node = ?rlp_node, + "pruning node", + ); if let Some(rlp_node) = rlp_node { let parent_idx = cursor.parent().expect("pruned child has parent").index; @@ -2262,7 +2184,7 @@ impl ArenaParallelSparseTrie { let mut arena_node = ArenaSparseNode::from_proof_node(proof_node); let state = arena_node.state_mut(); - *state = ArenaSparseNodeState::Cached { rlp_node: cached_rlp, was_dirty: false }; + *state = ArenaSparseNodeState::Cached { rlp_node: cached_rlp }; let child_idx = arena.insert(arena_node); arena[head_idx].branch_mut().children[dense_child_idx] = @@ -2348,10 +2270,6 @@ impl ArenaParallelSparseTrie { } Self::merge_subtrie_updates(&mut self.buffers.updates, &mut subtrie.buffers.updates); - Self::merge_subtrie_changed_paths( - &mut self.buffers.changed_paths, - &mut subtrie.buffers.changed_paths, - ); } } @@ -2423,10 +2341,6 @@ impl SparseTrie for ArenaParallelSparseTrie { } } - fn set_changed_paths(&mut self, retain_changed_paths: bool) { - Self::set_changed_paths(self, retain_changed_paths); - } - #[instrument(level = "trace", target = TRACE_TARGET, skip_all, fields(num_nodes = nodes.len()))] fn reveal_nodes(&mut self, nodes: &mut [ProofTrieNodeV2]) -> SparseTrieResult<()> { if nodes.is_empty() { @@ -2542,9 +2456,13 @@ impl SparseTrie for ArenaParallelSparseTrie { } else { use rayon::iter::{IntoParallelRefMutIterator, ParallelIterator}; + let parent_span = tracing::Span::current(); let results: Vec> = taken .par_iter_mut() - .map(|(_, subtrie, node_vec)| subtrie.reveal_nodes(node_vec)) + .map(|(_, subtrie, node_vec)| { + let _guard = parent_span.enter(); + subtrie.reveal_nodes(node_vec) + }) .collect(); if let Some(err) = results.into_iter().find(|r| r.is_err()) { @@ -2627,9 +2545,11 @@ impl SparseTrie for ArenaParallelSparseTrie { } else { use rayon::iter::{IntoParallelIterator, ParallelIterator}; + let parent_span = tracing::Span::current(); taken = taken .into_par_iter() .map(|(idx, mut subtrie)| { + let _guard = parent_span.enter(); subtrie.update_cached_rlp(); (idx, subtrie) }) @@ -2712,10 +2632,6 @@ impl SparseTrie for ArenaParallelSparseTrie { } } - fn take_changed_paths(&mut self) -> PrefixSetMut { - Self::take_changed_paths(self) - } - #[instrument(level = "trace", target = TRACE_TARGET, skip_all)] fn wipe(&mut self) { trace!(target: TRACE_TARGET, "Wiping arena trie"); @@ -2813,6 +2729,12 @@ impl SparseTrie for ArenaParallelSparseTrie { let head = cursor.head().expect("cursor is non-empty"); let head_idx = head.index; let head_path = head.path; + let protected_by_retained_parent = if cursor.depth() == 0 { + false + } else { + let parent_path = cursor.parent().expect("cursor must have a parent").path; + !prefix_range(retained_leaves, 0, &parent_path).is_empty() + }; match &self.upper_arena[head_idx] { ArenaSparseNode::Branch(_) | ArenaSparseNode::Leaf { .. } => { @@ -2826,6 +2748,10 @@ impl SparseTrie for ArenaParallelSparseTrie { continue; } + if protected_by_retained_parent { + continue; + } + Self::remove_pruned_node( &mut self.upper_arena, &cursor, @@ -2839,6 +2765,15 @@ impl SparseTrie for ArenaParallelSparseTrie { retained_idx = subtrie_range.end; if subtrie_range.is_empty() { + if protected_by_retained_parent { + let ArenaSparseNode::Subtrie(subtrie) = &mut self.upper_arena[head_idx] + else { + unreachable!() + }; + pruned += subtrie.prune(&[]); + continue; + } + let removed = Self::remove_pruned_node( &mut self.upper_arena, &cursor, @@ -2888,9 +2823,9 @@ impl SparseTrie for ArenaParallelSparseTrie { pruned += taken .par_iter_mut() .map(|(_, subtrie, range)| { + let _guard = parent_span.enter(); let _span = tracing::trace_span!( target: TRACE_TARGET, - parent: &parent_span, "subtrie_prune", subtrie = ?subtrie.path, ) @@ -3162,7 +3097,9 @@ impl SparseTrie for ArenaParallelSparseTrie { } else { use rayon::iter::{IntoParallelRefMutIterator, ParallelIterator}; + let parent_span = tracing::Span::current(); taken.par_iter_mut().for_each(|(_, subtrie, range)| { + let _guard = parent_span.enter(); subtrie.update_leaves(&sorted[range.clone()]); }); } diff --git a/crates/trie/sparse/src/arena/nodes.rs b/crates/trie/sparse/src/arena/nodes.rs index f30fb9ab4b4..70f0e0e3205 100644 --- a/crates/trie/sparse/src/arena/nodes.rs +++ b/crates/trie/sparse/src/arena/nodes.rs @@ -5,9 +5,9 @@ use super::{ use alloc::{boxed::Box, vec::Vec}; use alloy_primitives::{keccak256, B256}; use alloy_trie::{BranchNodeCompact, TrieMask}; -use core::mem; use reth_trie_common::{BranchNodeMasks, Nibbles, ProofTrieNodeV2, RlpNode, TrieNodeV2}; use smallvec::SmallVec; +use strum::AsRefStr; /// Tracks whether a node's RLP encoding is cached or needs recomputation. #[derive(Debug, Clone, PartialEq, Eq)] @@ -18,9 +18,6 @@ pub(super) enum ArenaSparseNodeState { Cached { /// The cached RLP-encoded representation of the node. rlp_node: RlpNode, - /// Whether this node was dirty when its RLP was cached. This is a one-shot marker - /// consumed while retaining changed paths during parent branch encoding. - was_dirty: bool, }, /// The node has been modified and its RLP encoding needs recomputation. Dirty, @@ -39,14 +36,6 @@ impl ArenaSparseNodeState { _ => None, } } - - /// Returns and clears whether this node was dirty when its RLP was cached. - pub(super) fn take_cached_was_dirty(&mut self) -> bool { - match self { - Self::Cached { was_dirty, .. } => mem::take(was_dirty), - _ => false, - } - } } /// Represents a reference from a branch node to one of its children. @@ -163,7 +152,7 @@ impl ArenaSparseNodeBranch { } /// A node in the arena-based sparse trie. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, AsRefStr)] pub(super) enum ArenaSparseNode { /// Indicates a trie with no nodes. EmptyRoot, diff --git a/crates/trie/sparse/src/state.rs b/crates/trie/sparse/src/state.rs index 58b286b1e25..21075c405b4 100644 --- a/crates/trie/sparse/src/state.rs +++ b/crates/trie/sparse/src/state.rs @@ -9,7 +9,7 @@ use alloy_primitives::{map::B256Map, B256}; use either::Either; use reth_execution_errors::{SparseStateTrieResult, SparseTrieErrorKind}; use reth_trie_common::{ - prefix_set::{PrefixSet, PrefixSetMut, TriePrefixSets, TriePrefixSetsMut}, + prefix_set::{PrefixSet, TriePrefixSets, TriePrefixSetsMut}, updates::{StorageTrieUpdates, TrieUpdates}, DecodedMultiProof, MultiProof, Nibbles, ProofTrieNodeV2, }; @@ -39,8 +39,6 @@ pub struct SparseStateTrie< storage: StorageTries, /// Flag indicating whether trie updates should be retained. retain_updates: bool, - /// Flag indicating whether changed node base paths should be retained. - retain_changed_paths: bool, /// Holds data that should be dropped after final state root is calculated. deferred_drops: DeferredDrops, /// Global LFU tracker for hot `(address, slot)` storage entries. @@ -62,7 +60,6 @@ where state: Default::default(), storage: Default::default(), retain_updates: false, - retain_changed_paths: false, deferred_drops: DeferredDrops::default(), hot_slots_lfu: BucketedLfu::default(), hot_accounts_lfu: BucketedLfu::default(), @@ -152,49 +149,6 @@ impl SparseStateTrie { } impl SparseStateTrie { - /// Set the retention of changed node base paths. - pub fn set_changed_paths(&mut self, retain_changed_paths: bool) { - self.retain_changed_paths = retain_changed_paths; - self.state.set_changed_paths(retain_changed_paths); - for trie in self.storage.tries.values_mut() { - trie.set_changed_paths(retain_changed_paths); - } - for trie in &mut self.storage.cleared_tries { - trie.set_changed_paths(retain_changed_paths); - } - self.storage.default_trie.set_changed_paths(retain_changed_paths); - } - - /// Set the retention of changed node base paths. - pub fn with_changed_paths(mut self, retain_changed_paths: bool) -> Self { - self.set_changed_paths(retain_changed_paths); - self - } - - /// Returns storage trie changed paths for tries that have been revealed. - fn storage_trie_changed_paths(&mut self) -> B256Map { - self.storage - .tries - .iter_mut() - .filter_map(|(address, trie)| { - let changed_paths = trie.take_changed_paths()?; - (!changed_paths.is_empty()).then_some((*address, changed_paths)) - }) - .collect() - } - - /// Returns changed paths by taking them from the revealed sparse tries. - /// - /// Returns `None` if the accounts trie is not revealed. - pub fn take_changed_paths(&mut self) -> Option { - let storage_prefix_sets = self.storage_trie_changed_paths(); - self.state.take_changed_paths().map(|account_prefix_set| TriePrefixSetsMut { - account_prefix_set, - storage_prefix_sets, - destroyed_accounts: Default::default(), - }) - } - /// Takes all debug recorders from the account trie and all revealed storage tries. /// /// Returns a vec of `(Option, TrieDebugRecorder)` where `None` is the account trie @@ -369,19 +323,14 @@ where } let retain_updates = self.retain_updates; - let retain_changed_paths = self.retain_changed_paths; #[cfg(not(feature = "std"))] let results: Vec<_> = targets .into_iter() .map(|(_, target, mut nodes)| { let result = match target { - Either::Left(trie) => { - trie.reveal_v2_proof_nodes(&mut nodes, retain_updates, retain_changed_paths) - } - Either::Right(trie) => { - trie.reveal_v2_proof_nodes(&mut nodes, retain_updates, retain_changed_paths) - } + Either::Left(trie) => trie.reveal_v2_proof_nodes(&mut nodes, retain_updates), + Either::Right(trie) => trie.reveal_v2_proof_nodes(&mut nodes, retain_updates), }; (result, nodes) }) @@ -406,16 +355,12 @@ where .entered(); let result = match target { - Either::Left(trie) => trie.reveal_v2_proof_nodes( - &mut nodes, - retain_updates, - retain_changed_paths, - ), - Either::Right(trie) => trie.reveal_v2_proof_nodes( - &mut nodes, - retain_updates, - retain_changed_paths, - ), + Either::Left(trie) => { + trie.reveal_v2_proof_nodes(&mut nodes, retain_updates) + } + Either::Right(trie) => { + trie.reveal_v2_proof_nodes(&mut nodes, retain_updates) + } }; (result, nodes) }) @@ -955,37 +900,6 @@ mod tests { ); } - #[test] - fn take_changed_paths_from_sparse_state_trie() { - let account = B256::with_last_byte(0x01); - let slot = B256::with_last_byte(0x02); - let mut sparse = SparseStateTrie::::default(); - sparse.set_accounts_trie(RevealableSparseTrie::revealed_empty()); - sparse.insert_storage_trie(account, RevealableSparseTrie::revealed_empty()); - sparse.set_changed_paths(true); - - let mut account_updates = - B256Map::from_iter([(account, LeafUpdate::Changed(vec![0x01; 32]))]); - sparse.trie_mut().update_leaves(&mut account_updates, |_, _| {}).unwrap(); - assert!(account_updates.is_empty()); - let _ = sparse.root().unwrap(); - - let mut storage_updates = B256Map::from_iter([(slot, LeafUpdate::Changed(vec![0x02; 32]))]); - sparse - .storage_trie_mut(&account) - .unwrap() - .update_leaves(&mut storage_updates, |_, _| {}) - .unwrap(); - assert!(storage_updates.is_empty()); - let _ = sparse.storage_root(&account).unwrap(); - - let changed_paths = sparse.take_changed_paths().unwrap(); - assert!(changed_paths.account_prefix_set.iter().any(|path| *path == Nibbles::default())); - assert!(changed_paths.storage_prefix_sets[&account] - .iter() - .any(|path| *path == Nibbles::default())); - } - #[test] fn prune_keeps_retained_paths_overlay_account_and_storage() { let mut sparse = SparseStateTrie::::default(); diff --git a/crates/trie/sparse/src/traits.rs b/crates/trie/sparse/src/traits.rs index c096c3ca86f..dbd36c1b2c7 100644 --- a/crates/trie/sparse/src/traits.rs +++ b/crates/trie/sparse/src/traits.rs @@ -9,9 +9,7 @@ use alloy_primitives::{ }; use alloy_trie::BranchNodeCompact; use reth_execution_errors::SparseTrieResult; -use reth_trie_common::{ - prefix_set::PrefixSetMut, BranchNodeMasks, Nibbles, ProofTrieNodeV2, TrieNodeV2, -}; +use reth_trie_common::{BranchNodeMasks, Nibbles, ProofTrieNodeV2, TrieNodeV2}; #[cfg(feature = "trie-debug")] use crate::debug_recorder::TrieDebugRecorder; @@ -77,9 +75,6 @@ pub trait SparseTrie: Sized + Debug + Send + Sync { /// * `retain_updates` - Whether to track updates fn set_updates(&mut self, retain_updates: bool); - /// Configures the trie to retain changed node base paths during hashing. - fn set_changed_paths(&mut self, retain_changed_paths: bool); - /// Reveals one or more trie nodes if they have not been revealed before. /// /// This function decodes trie nodes and inserts them into the trie structure. It handles @@ -179,13 +174,6 @@ pub trait SparseTrie: Sized + Debug + Send + Sync { /// The accumulated updates, or an empty set if updates weren't being tracked. fn take_updates(&mut self) -> SparseTrieUpdates; - /// Consumes and returns the currently accumulated changed node base paths. - /// - /// Ancestor paths may be excluded when a descendant path is already present. - /// - /// Returns an empty set if changed paths weren't being tracked. - fn take_changed_paths(&mut self) -> PrefixSetMut; - /// Removes all nodes and values from the trie, resetting it to a blank state /// with only an empty root node. This is used when a storage root is deleted. /// diff --git a/crates/trie/sparse/src/trie.rs b/crates/trie/sparse/src/trie.rs index 657950fd46f..0a5e4791900 100644 --- a/crates/trie/sparse/src/trie.rs +++ b/crates/trie/sparse/src/trie.rs @@ -4,10 +4,7 @@ use crate::{ use alloc::{borrow::Cow, boxed::Box}; use alloy_primitives::{map::B256Map, B256}; use reth_execution_errors::{SparseTrieErrorKind, SparseTrieResult}; -use reth_trie_common::{ - prefix_set::PrefixSetMut, BranchNodeMasks, Nibbles, ProofTrieNodeV2, RlpNode, TrieMask, - TrieNodeV2, -}; +use reth_trie_common::{BranchNodeMasks, Nibbles, ProofTrieNodeV2, RlpNode, TrieMask, TrieNodeV2}; /// A sparse trie that is either in a "blind" state (no nodes are revealed, root node hash is /// unknown) or in a "revealed" state (root node has been revealed and the trie can be updated). @@ -56,8 +53,7 @@ impl RevealableSparseTrie { /// If the trie is blinded, its root node is replaced with `root`. /// /// The `masks` are used to determine how the node's children are stored. - /// The retention flags control whether trie updates and changed node base paths - /// should be tracked. + /// The retention flag controls whether trie updates should be tracked. /// /// # Returns /// @@ -67,7 +63,6 @@ impl RevealableSparseTrie { root: TrieNodeV2, masks: Option, retain_updates: bool, - retain_changed_paths: bool, ) -> SparseTrieResult<&mut T> { // if `Blind`, we initialize the revealed trie with the given root node, using a // pre-allocated trie if available. @@ -79,7 +74,6 @@ impl RevealableSparseTrie { }; revealed_trie.set_root(root, masks, retain_updates)?; - revealed_trie.set_changed_paths(retain_changed_paths); *self = Self::Revealed(revealed_trie); } @@ -94,15 +88,9 @@ impl RevealableSparseTrie { &mut self, nodes: &mut [ProofTrieNodeV2], retain_updates: bool, - retain_changed_paths: bool, ) -> SparseTrieResult<()> { let trie = if let Some(root_node) = nodes.iter().find(|n| n.path.is_empty()) { - self.reveal_root( - root_node.node.clone(), - root_node.masks, - retain_updates, - retain_changed_paths, - )? + self.reveal_root(root_node.node.clone(), root_node.masks, retain_updates)? } else { self.as_revealed_mut().ok_or(SparseTrieErrorKind::Blind)? }; @@ -214,23 +202,6 @@ impl RevealableSparseTrie { Some((revealed.root(), revealed.take_updates())) } - /// Configures a revealed or retained cleared trie to collect changed node base paths. - pub fn set_changed_paths(&mut self, retain_changed_paths: bool) { - match self { - Self::Revealed(trie) | Self::Blind(Some(trie)) => { - trie.set_changed_paths(retain_changed_paths); - } - Self::Blind(None) => {} - } - } - - /// Takes changed node base paths from the revealed trie. - /// - /// Returns `None` if the trie is still blind. - pub fn take_changed_paths(&mut self) -> Option { - Some(self.as_revealed_mut()?.take_changed_paths()) - } - /// Clears this trie, setting it to a blind state. /// /// If this instance was revealed, or was itself a `Blind` with a pre-allocated diff --git a/crates/trie/sparse/tests/suite/changed_paths.rs b/crates/trie/sparse/tests/suite/changed_paths.rs deleted file mode 100644 index 31595951313..00000000000 --- a/crates/trie/sparse/tests/suite/changed_paths.rs +++ /dev/null @@ -1,127 +0,0 @@ -use super::*; - -fn key_with_prefix(bytes: &[u8]) -> B256 { - let mut key = B256::ZERO; - key.0[..bytes.len()].copy_from_slice(bytes); - key -} - -fn contains_changed_path( - changed_paths: &reth_trie_common::prefix_set::PrefixSetMut, - path: Nibbles, -) -> bool { - changed_paths.iter().any(|changed_path| *changed_path == path) -} - -pub(super) fn test_changed_paths_record_base_paths_for_branches_and_leaves( - new_trie: fn() -> T, -) { - let mut trie = new_trie(); - trie.set_changed_paths(true); - - let key_a = key_with_prefix(&[0x12, 0x34]); - let key_b = key_with_prefix(&[0x12, 0x35]); - - let mut updates = B256Map::default(); - updates.insert(key_a, LeafUpdate::Changed(vec![0x01; 64])); - updates.insert(key_b, LeafUpdate::Changed(vec![0x02; 64])); - trie.update_leaves(&mut updates, |_, _| {}).expect("insertion should succeed"); - assert!(updates.is_empty()); - - let _ = trie.root(); - - let changed_paths = trie.take_changed_paths(); - assert!(!contains_changed_path(&changed_paths, Nibbles::default())); - assert!(contains_changed_path(&changed_paths, Nibbles::from_nibbles([0x01, 0x02, 0x03, 0x04]))); - assert!(contains_changed_path(&changed_paths, Nibbles::from_nibbles([0x01, 0x02, 0x03, 0x05]))); - assert!(!contains_changed_path(&changed_paths, Nibbles::from_nibbles([0x01, 0x02, 0x03]))); - assert!(!contains_changed_path(&changed_paths, Nibbles::unpack(key_a))); - assert!(!contains_changed_path(&changed_paths, Nibbles::unpack(key_b))); - - let _ = trie.root(); - assert!(trie.take_changed_paths().is_empty()); -} - -pub(super) fn test_changed_paths_skip_dirty_ancestor_branch_when_descendant_changed< - T: SparseTrie, ->( - new_trie: fn() -> T, -) { - let mut trie = new_trie(); - trie.set_changed_paths(true); - - let key_a = key_with_prefix(&[0xff, 0x10]); - let key_b = key_with_prefix(&[0xff, 0x20]); - let sibling_key = key_with_prefix(&[0xe0]); - - let mut updates = B256Map::default(); - updates.insert(key_a, LeafUpdate::Changed(vec![0x01; 64])); - updates.insert(key_b, LeafUpdate::Changed(vec![0x02; 64])); - updates.insert(sibling_key, LeafUpdate::Changed(vec![0x03; 64])); - trie.update_leaves(&mut updates, |_, _| {}).expect("insertion should succeed"); - assert!(updates.is_empty()); - - let _ = trie.root(); - let _ = trie.take_changed_paths(); - - let mut updates = B256Map::from_iter([(key_a, LeafUpdate::Changed(vec![0x04; 64]))]); - trie.update_leaves(&mut updates, |_, _| {}).expect("update should succeed"); - assert!(updates.is_empty()); - - let _ = trie.root(); - - let changed_paths = trie.take_changed_paths(); - assert!(contains_changed_path(&changed_paths, Nibbles::from_nibbles([0x0f, 0x0f, 0x01]))); - assert!(!contains_changed_path(&changed_paths, Nibbles::from_nibbles([0x0f]))); - assert!(!contains_changed_path(&changed_paths, Nibbles::default())); - - let mut updates = B256Map::from_iter([(key_b, LeafUpdate::Changed(vec![0x05; 64]))]); - trie.update_leaves(&mut updates, |_, _| {}).expect("update should succeed"); - assert!(updates.is_empty()); - - let _ = trie.root(); - - let changed_paths = trie.take_changed_paths(); - assert!(contains_changed_path(&changed_paths, Nibbles::from_nibbles([0x0f, 0x0f, 0x02]))); - assert!(!contains_changed_path(&changed_paths, Nibbles::from_nibbles([0x0f, 0x0f, 0x01]))); - assert!(!contains_changed_path(&changed_paths, Nibbles::from_nibbles([0x0f]))); - assert!(!contains_changed_path(&changed_paths, Nibbles::default())); -} - -pub(super) fn test_changed_paths_record_branch_after_leaf_removal( - new_trie: fn() -> T, -) { - let mut trie = new_trie(); - trie.set_changed_paths(true); - - let removed_key = key_with_prefix(&[0x12]); - let retained_key = key_with_prefix(&[0x13]); - let retained_key_b = key_with_prefix(&[0x14]); - let sibling_key = key_with_prefix(&[0xe0]); - - let mut updates = B256Map::default(); - updates.insert(removed_key, LeafUpdate::Changed(vec![0x01])); - updates.insert(retained_key, LeafUpdate::Changed(vec![0x02])); - updates.insert(retained_key_b, LeafUpdate::Changed(vec![0x03])); - updates.insert(sibling_key, LeafUpdate::Changed(vec![0x04])); - trie.update_leaves(&mut updates, |_, _| {}).expect("insertion should succeed"); - assert!(updates.is_empty()); - - let _ = trie.root(); - let _ = trie.take_changed_paths(); - - let mut removals = B256Map::default(); - removals.insert(removed_key, LeafUpdate::Changed(Vec::new())); - trie.update_leaves(&mut removals, |_, _| {}).expect("removal should succeed"); - assert!(removals.is_empty()); - - let _ = trie.root(); - - let changed_paths = trie.take_changed_paths(); - assert!(contains_changed_path(&changed_paths, Nibbles::from_nibbles([0x01]))); - assert!(!contains_changed_path(&changed_paths, Nibbles::from_nibbles([0x01, 0x02]))); - assert!(!contains_changed_path(&changed_paths, Nibbles::default())); - - let _ = trie.root(); - assert!(trie.take_changed_paths().is_empty()); -} diff --git a/crates/trie/sparse/tests/suite/lifecycle.rs b/crates/trie/sparse/tests/suite/lifecycle.rs index 40a0bf76ed1..3f032cee66e 100644 --- a/crates/trie/sparse/tests/suite/lifecycle.rs +++ b/crates/trie/sparse/tests/suite/lifecycle.rs @@ -64,14 +64,18 @@ pub(super) fn test_full_lifecycle_update_root_take_updates(new_tr /// Multiple rounds of (update → root → `take_updates`), followed by a prune, simulating block /// processing. pub(super) fn test_multi_round_update_take_updates_prune_cycle(new_trie: fn() -> T) { - // Build a trie with 10 leaves. + // Build a trie with 10 primary leaves, each with a sibling under the same root child. let mut storage: BTreeMap = BTreeMap::new(); let mut keys = Vec::new(); for i in 0u8..10 { let mut key = B256::ZERO; - key.0[0] = i * 16; // nibble prefixes: 0x0, 0x1, 0x2, ... 0x9 + key.0[0] = i << 4; // nibble prefixes: 0x0, 0x1, 0x2, ... 0x9 storage.insert(key, U256::from(i as u64 + 1)); keys.push(key); + + let mut sibling = B256::ZERO; + sibling.0[0] = (i << 4) | 1; + storage.insert(sibling, U256::from(i as u64 + 100)); } let mut harness = SuiteTestHarness::new(storage.clone()); diff --git a/crates/trie/sparse/tests/suite/main.rs b/crates/trie/sparse/tests/suite/main.rs index 2c062242f0e..a2cbc5d1d41 100644 --- a/crates/trie/sparse/tests/suite/main.rs +++ b/crates/trie/sparse/tests/suite/main.rs @@ -11,7 +11,6 @@ //! - [`update_leaves`]: Tests for `update_leaves`, including insert, modify, and remove //! - [`root`]: Tests for `root()` hash computation //! - [`take_updates`]: Tests for `take_updates` -//! - [`changed_paths`]: Tests for retained changed node base paths //! - [`prune`]: Tests for `prune` //! - [`wipe_clear`]: Tests for `wipe` and `clear` //! - [`get_leaf_value`]: Tests for `get_leaf_value` @@ -27,7 +26,6 @@ use reth_trie_common::{Nibbles, ProofV2Target, TrieNodeV2}; use reth_trie_sparse::{LeafLookup, LeafLookupError, LeafUpdate, SparseTrie}; use std::{collections::BTreeMap, iter::once}; -mod changed_paths; mod find_leaf; mod get_leaf_value; mod lifecycle; @@ -191,7 +189,6 @@ macro_rules! sparse_trie_tests { // Re-export test functions from submodules for the macro // --------------------------------------------------------------------------- -use changed_paths::*; use find_leaf::*; use get_leaf_value::*; use lifecycle::*; @@ -275,13 +272,12 @@ sparse_trie_tests! { test_take_updates_no_duplicate_updated_and_removed_nodes, test_take_updates_cross_cancellation_across_root_calls, - // changed_paths - test_changed_paths_record_base_paths_for_branches_and_leaves, - test_changed_paths_skip_dirty_ancestor_branch_when_descendant_changed, - test_changed_paths_record_branch_after_leaf_removal, // prune test_prune_retains_specified_leaves, + test_prune_keeps_upper_children_of_retained_branch, + test_prune_keeps_lower_children_of_retained_branch, + test_prune_protects_children_by_parent_base_path, test_prune_reduces_node_count, test_prune_empty_retained_set, test_prune_requires_computed_hashes, diff --git a/crates/trie/sparse/tests/suite/prune.rs b/crates/trie/sparse/tests/suite/prune.rs index 7866a184978..14cb70933b8 100644 --- a/crates/trie/sparse/tests/suite/prune.rs +++ b/crates/trie/sparse/tests/suite/prune.rs @@ -1,4 +1,42 @@ use super::*; +use alloy_trie::{nodes::BranchNode, TrieMask}; +use reth_trie_common::{BranchNodeV2, LeafNode, ProofTrieNodeV2, RlpNode}; + +fn key_with_prefix(prefix: &[u8]) -> B256 { + let mut key = B256::ZERO; + for (idx, &nibble) in prefix.iter().enumerate() { + let byte = &mut key.0[idx / 2]; + if idx % 2 == 0 { + *byte |= nibble << 4; + } else { + *byte |= nibble; + } + } + key +} + +fn changed_update(value: u64) -> LeafUpdate { + LeafUpdate::Changed(encode_fixed_size(&U256::from(value)).to_vec()) +} + +fn assert_update_requests_min_len(trie: &mut T, key: B256, min_len: u8, value: u64) { + let mut leaf_updates = B256Map::from_iter([(key, changed_update(value))]); + let mut targets = Vec::new(); + trie.update_leaves(&mut leaf_updates, |key, min_len| { + targets.push((key, min_len)); + }) + .expect("update_leaves should succeed"); + + assert_eq!(targets, vec![(key, min_len)]); + assert!( + leaf_updates.contains_key(&key), + "update should remain pending until proof is revealed" + ); +} + +fn rlp_node(node: TrieNodeV2) -> RlpNode { + RlpNode::from_rlp(&alloy_rlp::encode(node)) +} pub(super) fn test_prune_retains_specified_leaves(new_trie: fn() -> T) { let mut key_a = B256::ZERO; @@ -42,18 +80,118 @@ pub(super) fn test_prune_retains_specified_leaves(new_trie: fn() assert!(val_b.is_some(), "retained leaf B should be accessible after prune"); } +pub(super) fn test_prune_keeps_upper_children_of_retained_branch( + new_trie: fn() -> T, +) { + let retained_key = key_with_prefix(&[0x0]); + let protected_key_a = key_with_prefix(&[0x1, 0x2]); + let protected_key_b = key_with_prefix(&[0x1, 0x3]); + let storage = BTreeMap::from([ + (retained_key, U256::from(1)), + (protected_key_a, U256::from(2)), + (protected_key_b, U256::from(3)), + ]); + + let harness = SuiteTestHarness::new(storage); + let mut trie: T = harness.init_trie_fully_revealed(false, new_trie); + let root_before = trie.root(); + + let retained = [Nibbles::unpack(retained_key)]; + let pruned = trie.prune(&retained); + + assert_eq!(trie.root(), root_before, "root must not change after prune"); + assert_eq!(pruned, 2, "protected branch children should be blinded, not removed"); + assert_update_requests_min_len(&mut trie, protected_key_a, 2, 20); +} + +pub(super) fn test_prune_keeps_lower_children_of_retained_branch( + new_trie: fn() -> T, +) { + let retained_key = key_with_prefix(&[0x0, 0x0, 0x0]); + let protected_key_a = key_with_prefix(&[0x0, 0x0, 0x1, 0x2]); + let protected_key_b = key_with_prefix(&[0x0, 0x0, 0x1, 0x3]); + let storage = BTreeMap::from([ + (retained_key, U256::from(1)), + (protected_key_a, U256::from(2)), + (protected_key_b, U256::from(3)), + ]); + + let harness = SuiteTestHarness::new(storage); + let mut trie: T = harness.init_trie_fully_revealed(false, new_trie); + let root_before = trie.root(); + + let retained = [Nibbles::unpack(retained_key)]; + let pruned = trie.prune(&retained); + + assert_eq!(trie.root(), root_before, "root must not change after prune"); + assert_eq!(pruned, 2, "protected lower branch children should be blinded, not removed"); + assert_update_requests_min_len(&mut trie, protected_key_a, 4, 20); +} + +pub(super) fn test_prune_protects_children_by_parent_base_path(new_trie: fn() -> T) { + let protected_leaf_a = TrieNodeV2::Leaf(LeafNode::new(Nibbles::default(), vec![0x32])); + let protected_leaf_b = TrieNodeV2::Leaf(LeafNode::new(Nibbles::default(), vec![0x33])); + + let protected_state_mask = TrieMask::new(0b1100); + let protected_stack = + vec![rlp_node(protected_leaf_a.clone()), rlp_node(protected_leaf_b.clone())]; + let protected_branch_rlp = RlpNode::from_rlp(&alloy_rlp::encode(BranchNode::new( + protected_stack.clone(), + protected_state_mask, + ))); + let retained_parent = TrieNodeV2::Branch(BranchNodeV2::new( + Nibbles::from_nibbles([0xa]), + protected_stack, + protected_state_mask, + Some(protected_branch_rlp), + )); + let root = TrieNodeV2::Branch(BranchNodeV2::new( + Nibbles::default(), + vec![rlp_node(retained_parent.clone())], + TrieMask::new(0b0001), + None, + )); + + let mut trie = (new_trie)(); + trie.set_root(root, None, false).expect("set_root should succeed"); + trie.reveal_nodes(&mut [ + ProofTrieNodeV2 { path: Nibbles::from_nibbles([0x0]), node: retained_parent, masks: None }, + ProofTrieNodeV2 { + path: Nibbles::from_nibbles([0x0, 0xa, 0x2]), + node: protected_leaf_a, + masks: None, + }, + ProofTrieNodeV2 { + path: Nibbles::from_nibbles([0x0, 0xa, 0x3]), + node: protected_leaf_b, + masks: None, + }, + ]) + .expect("reveal_nodes should succeed"); + + let root_before = trie.root(); + let retained = [Nibbles::from_nibbles([0x0, 0x0])]; + let pruned = trie.prune(&retained); + + assert_eq!(pruned, 0); + assert_eq!(trie.root(), root_before, "root must not change after prune"); +} + /// Pruning should reduce the node count. /// -/// Build a trie with 10+ leaves spread across multiple subtries, fully reveal +/// Build a trie with several root children that each contain grandchildren, fully reveal /// and compute root. Then prune retaining only 1 leaf. `size_hint()` must /// decrease and `prune` must return > 0. pub(super) fn test_prune_reduces_node_count(new_trie: fn() -> T) { - // Create 16 keys with different first nibbles to spread across subtries. + // Create 16 pairs with different first nibbles. Pruning keeps direct + // children of the retained root branch, so each child needs grandchildren that can be pruned. let keys: Vec = (0u8..16) - .map(|i| { - let mut k = B256::ZERO; - k.0[0] = (i + 1) << 4; // 0x10, 0x20, ..., 0x00 (wraps, but all distinct) - k + .flat_map(|i| { + [0u8, 1].map(move |child| { + let mut k = B256::ZERO; + k.0[0] = (i << 4) | child; + k + }) }) .collect(); diff --git a/crates/trie/trie/src/proof_v2/mod.rs b/crates/trie/trie/src/proof_v2/mod.rs index 7a8cd904a89..5068d805fc6 100644 --- a/crates/trie/trie/src/proof_v2/mod.rs +++ b/crates/trie/trie/src/proof_v2/mod.rs @@ -205,15 +205,6 @@ where let (mut lower, mut upper) = targets.current(); - debug_assert!(self.retained_proofs.last().is_none_or( - |ProofTrieNodeV2 { path: last_retained_path, .. }| { - depth_first::cmp(path, last_retained_path) == Ordering::Greater - } - ), - "should_retain called with path {path:?} which is not after previously retained node {:?} in depth-first order", - self.retained_proofs.last().map(|n| n.path), - ); - loop { // If the node in question is a prefix of the target then we do not iterate targets // further. @@ -687,7 +678,7 @@ where &mut self, value_encoder: &mut VE, targets: &mut Option>, - hashed_cursor_current: &mut Option<(Nibbles, VE::DeferredEncoder)>, + hashed_cursor_state: &mut HashedCursorState, lower_bound: Nibbles, upper_bound: Option, ) -> Result<(), StateProofError> { @@ -701,28 +692,33 @@ where (key, val) }; - // If the cursor hasn't been used, or the last iterated key is prior to this range's - // key range, then seek forward to at least the first key. - if hashed_cursor_current.as_ref().is_none_or(|(key, _)| key < &lower_bound) { + // If the cursor hasn't been used, or the last iterated key is prior to this range's key + // range, then seek forward to at least the first key. + if hashed_cursor_state.needs_seek_to(&lower_bound) { trace!( target: TRACE_TARGET, - current=?hashed_cursor_current.as_ref().map(|(k, _)| k), + current=?hashed_cursor_state.path(), "Seeking hashed cursor to meet lower bound", ); let lower_key = B256::right_padding_from(&lower_bound.pack()); - *hashed_cursor_current = - self.hashed_cursor.seek(lower_key)?.map(&mut map_hashed_cursor_entry); + *hashed_cursor_state = HashedCursorState::seeked( + lower_bound, + self.hashed_cursor.seek(lower_key)?.map(&mut map_hashed_cursor_entry), + ); } // Loop over all keys in the range, calling `push_leaf` on each. - while let Some((key, _)) = hashed_cursor_current.as_ref() && - upper_bound.is_none_or(|upper_bound| key < &upper_bound) + while hashed_cursor_state + .path() + .is_some_and(|key| upper_bound.is_none_or(|upper_bound| key < &upper_bound)) { - let (key, val) = - core::mem::take(hashed_cursor_current).expect("while-let checks for Some"); + let (key, val) = hashed_cursor_state.take(); self.push_leaf(targets, key, val)?; - *hashed_cursor_current = self.hashed_cursor.next()?.map(&mut map_hashed_cursor_entry); + *hashed_cursor_state = HashedCursorState::seeked( + key, + self.hashed_cursor.next()?.map(&mut map_hashed_cursor_entry), + ); } trace!(target: TRACE_TARGET, "No further keys within range"); @@ -927,8 +923,10 @@ where // If the trie cursor is seeked to a branch whose leaves have already been processed // then we can't use it, instead we seek forward and try again. if trie_cursor_path < uncalculated_lower_bound { - *trie_cursor_state = - TrieCursorState::seeked(self.trie_cursor_seek(*uncalculated_lower_bound)?); + *trie_cursor_state = TrieCursorState::seeked( + *uncalculated_lower_bound, + self.trie_cursor_seek(*uncalculated_lower_bound)?, + ); // Having just seeked forward we need to check if the cursor is now exhausted, // extracting the new path at the same time. @@ -1244,7 +1242,8 @@ where // trie cursor to the next cached node at-or-after `child_path`. if trie_cursor_state.path().is_some_and(|path| path < &child_path) { trace!(target: TRACE_TARGET, ?child_path, "Seeking trie cursor to child path"); - *trie_cursor_state = TrieCursorState::seeked(self.trie_cursor_seek(child_path)?); + *trie_cursor_state = + TrieCursorState::seeked(child_path, self.trie_cursor_seek(child_path)?); } // If the next cached branch node is a child of `child_path` then we can assume it is @@ -1309,7 +1308,7 @@ where &mut self, value_encoder: &mut VE, trie_cursor_state: &mut TrieCursorState, - hashed_cursor_current: &mut Option<(Nibbles, VE::DeferredEncoder)>, + hashed_cursor_state: &mut HashedCursorState, sub_trie_targets: SubTrieTargets<'a>, ) -> Result<(), StateProofError> { let sub_trie_upper_bound = sub_trie_targets.upper_bound(); @@ -1330,12 +1329,24 @@ where debug_assert!(self.child_stack.is_empty()); // `next_uncached_key_range`, which will be called in the loop below, expects the trie - // cursor to have already been seeked. If it's not yet seeked, or seeked to a prior node, - // then we seek it to the prefix (the first possible node) to initialize it. - if trie_cursor_state.before(&sub_trie_targets.prefix) { - trace!(target: TRACE_TARGET, "Doing initial seek of trie cursor"); - *trie_cursor_state = - TrieCursorState::seeked(self.trie_cursor_seek(sub_trie_targets.prefix)?); + // cursor to have already been seeked. The trie cursor is forward-only, but exact sub-trie + // chunks can overlap previous chunks, so reset it if this seek needs to move backwards. + if trie_cursor_state.needs_reset_before_seek(&sub_trie_targets.prefix) { + trace!(target: TRACE_TARGET, "Resetting trie cursor before sub-trie"); + self.trie_cursor.reset(); + *trie_cursor_state = TrieCursorState::unseeked(); + } + + trace!(target: TRACE_TARGET, "Doing initial seek of trie cursor"); + *trie_cursor_state = TrieCursorState::seeked( + sub_trie_targets.prefix, + self.trie_cursor_seek(sub_trie_targets.prefix)?, + ); + + if hashed_cursor_state.needs_reset_before_seek(&sub_trie_targets.prefix) { + trace!(target: TRACE_TARGET, "Resetting hashed cursor before sub-trie"); + self.hashed_cursor.reset(); + *hashed_cursor_state = HashedCursorState::unseeked(); } // `uncalculated_lower_bound` tracks the lower bound of node paths which have yet to be @@ -1385,20 +1396,20 @@ where self.calculate_key_range( value_encoder, &mut targets, - hashed_cursor_current, + hashed_cursor_state, calc_lower_bound, calc_upper_bound, )?; - // Once outside `calculate_key_range`, `hashed_cursor_current` will be at the first key - // after the range. + // Once outside `calculate_key_range`, `hashed_cursor_state` will be at the first key + // after the range, or exhausted. // - // If the `hashed_cursor_current` is None (exhausted), or not within the range of the + // If the hashed cursor is exhausted, or not within the range of the // sub-trie, then there are no more keys at all, meaning the trie couldn't possibly have // more data and we should complete computation. - if hashed_cursor_current - .as_ref() - .is_none_or(|(key, _)| !key.starts_with(&sub_trie_targets.prefix)) + if hashed_cursor_state + .path() + .is_none_or(|key| !key.starts_with(&sub_trie_targets.prefix)) { break; } @@ -1484,7 +1495,7 @@ where // Initialize the variables which track the state of the two cursors. Both indicate the // cursors are unseeked. let mut trie_cursor_state = TrieCursorState::unseeked(); - let mut hashed_cursor_current: Option<(Nibbles, VE::DeferredEncoder)> = None; + let mut hashed_cursor_state = HashedCursorState::unseeked(); // Divide targets into chunks, each chunk corresponding to a different sub-trie within the // overall trie, and handle all proofs within that sub-trie. @@ -1492,7 +1503,7 @@ where if let Err(err) = self.proof_subtrie( value_encoder, &mut trie_cursor_state, - &mut hashed_cursor_current, + &mut hashed_cursor_state, sub_trie_targets, ) { self.clear_computation_state(); @@ -1505,6 +1516,8 @@ where retained_proofs_len = ?self.retained_proofs.len(), "proof_inner: returning", ); + self.retained_proofs.sort_unstable_by(|a, b| depth_first::cmp(&a.path, &b.path)); + self.retained_proofs.dedup_by(|a, b| a.path == b.path); Ok(core::mem::take(&mut self.retained_proofs)) } @@ -1564,7 +1577,7 @@ where // Initialize the variables which track the state of the two cursors. Both indicate the // cursors are unseeked. let mut trie_cursor_state = TrieCursorState::unseeked(); - let mut hashed_cursor_current: Option<(Nibbles, VE::DeferredEncoder)> = None; + let mut hashed_cursor_state = HashedCursorState::unseeked(); static EMPTY_TARGETS: [ProofV2Target; 0] = []; let sub_trie_targets = @@ -1573,7 +1586,7 @@ where if let Err(err) = self.proof_subtrie( value_encoder, &mut trie_cursor_state, - &mut hashed_cursor_current, + &mut hashed_cursor_state, sub_trie_targets, ) { self.clear_computation_state(); @@ -1739,8 +1752,8 @@ enum TrieCursorState { Available(Nibbles, BranchNodeCompact), /// Cursor is seeked to this path, but the node has been used. Taken(Nibbles), - /// Cursor has been exhausted. - Exhausted, + /// Cursor has been exhausted after seeking from the given lower bound. + Exhausted(Nibbles), } impl TrieCursorState { @@ -1750,8 +1763,8 @@ impl TrieCursorState { } /// Creates a [`Self`] based on an entry returned from the cursor itself. - fn seeked(entry: Option<(Nibbles, BranchNodeCompact)>) -> Self { - entry.map_or(Self::Exhausted, |(path, node)| Self::Available(path, node)) + fn seeked(key: Nibbles, entry: Option<(Nibbles, BranchNodeCompact)>) -> Self { + entry.map_or(Self::Exhausted(key), |(path, node)| Self::Available(path, node)) } /// Returns the path the cursor is seeked to, or None if it's exhausted. @@ -1763,16 +1776,16 @@ impl TrieCursorState { match self { Self::Unseeked => panic!("cursor is unseeked"), Self::Available(path, _) | Self::Taken(path) => Some(path), - Self::Exhausted => None, + Self::Exhausted(_) => None, } } - /// Returns true if the cursor is unseeked, or is seeked to a node prior to the given one. - fn before(&self, path: &Nibbles) -> bool { + /// Returns true if seeking to `key` requires resetting the forward-only cursor. + fn needs_reset_before_seek(&self, key: &Nibbles) -> bool { match self { - Self::Unseeked => true, - Self::Available(seeked_to, _) | Self::Taken(seeked_to) => path < seeked_to, - Self::Exhausted => false, + Self::Unseeked => false, + Self::Available(path, _) | Self::Taken(path) => path > key, + Self::Exhausted(exhausted_at) => exhausted_at > key, } } @@ -1791,6 +1804,62 @@ impl TrieCursorState { } } +/// Used to track the state of the hashed cursor, including the path that established exhaustion. +enum HashedCursorState { + /// The initial state of the cursor, indicating it's never been seeked. + Unseeked, + /// Cursor is seeked to this path and the value has not been used yet. + Available(Nibbles, V), + /// Cursor has been exhausted at or after the given path. + Exhausted(Nibbles), +} + +impl HashedCursorState { + /// Creates a [`Self::Unseeked`] state. + const fn unseeked() -> Self { + Self::Unseeked + } + + /// Creates a [`Self`] based on an entry returned from the cursor itself. + fn seeked(key: Nibbles, entry: Option<(Nibbles, V)>) -> Self { + entry.map_or(Self::Exhausted(key), |(path, value)| Self::Available(path, value)) + } + + /// Returns the path the cursor is seeked to, or None if it's unseeked or exhausted. + const fn path(&self) -> Option<&Nibbles> { + match self { + Self::Available(path, _) => Some(path), + Self::Unseeked | Self::Exhausted(_) => None, + } + } + + /// Returns true if the cursor must seek to be usable for a range starting at `key`. + fn needs_seek_to(&self, key: &Nibbles) -> bool { + match self { + Self::Unseeked => true, + Self::Available(path, _) => path < key, + Self::Exhausted(exhausted_at) => exhausted_at > key, + } + } + + /// Returns true if seeking to `key` requires resetting the forward-only cursor. + fn needs_reset_before_seek(&self, key: &Nibbles) -> bool { + match self { + Self::Unseeked => false, + Self::Available(path, _) => path > key, + Self::Exhausted(exhausted_at) => exhausted_at > key, + } + } + + /// Takes the path and value from a [`Self::Available`]. Panics if not [`Self::Available`]. + fn take(&mut self) -> (Nibbles, V) { + match core::mem::replace(self, Self::Unseeked) { + Self::Available(path, value) => (path, value), + _ => panic!("take called on non-Available hashed cursor state"), + } + } +} + /// Describes the state of the currently cached branch node (if any). enum PopCachedBranchOutcome { /// Cached branch has been popped from the `cached_branch_stack` and is ready to be used. @@ -2069,6 +2138,24 @@ mod tests { } } + #[test] + fn test_exact_subtrie_targets_with_root_target() { + reth_tracing::init_test_tracing(); + + let slot_80 = B256::right_padding_from(&[0x80]); + let slot_82 = B256::right_padding_from(&[0x82]); + let slot_f0 = B256::right_padding_from(&[0xf0]); + let storage = BTreeMap::from([ + (slot_80, U256::from(1)), + (slot_82, U256::from(2)), + (slot_f0, U256::from(3)), + ]); + let targets = [ProofV2Target::new(B256::ZERO), ProofV2Target::new(slot_80).with_min_len(2)]; + + let harness = ProofTestHarness::new(storage); + harness.assert_proof(targets).expect("Proof generation failed"); + } + #[test] fn test_big_trie() { use rand::prelude::*; diff --git a/crates/trie/trie/src/proof_v2/target.rs b/crates/trie/trie/src/proof_v2/target.rs index 14be86c7ac9..2077545a2a4 100644 --- a/crates/trie/trie/src/proof_v2/target.rs +++ b/crates/trie/trie/src/proof_v2/target.rs @@ -59,51 +59,23 @@ impl<'a> SubTrieTargets<'a> { pub(crate) fn iter_sub_trie_targets( targets: &mut [ProofV2Target], ) -> impl Iterator> { - // First sort by the sub-trie prefix of each target, falling back to the `min_len` in cases - // where the sub-trie prefixes are equal (to differentiate targets which match the root node and - // those which don't). + // Sort globally by sub-trie prefix, then by target key. This makes equal-prefix targets + // contiguous and already ordered for the `ProofCalculator`. targets.sort_unstable_by(|a, b| { - sub_trie_prefix(a).cmp(&sub_trie_prefix(b)).then_with(|| a.min_len.cmp(&b.min_len)) + sub_trie_prefix(a).cmp(&sub_trie_prefix(b)).then_with(|| a.key_nibbles.cmp(&b.key_nibbles)) }); - // We now chunk targets, such that each chunk contains all targets belonging to the same - // sub-trie. We are taking advantage of the following properties: - // - // - The first target in the chunk has the shortest sub-trie prefix (see previous sorting step). - // - // - The upper bound of the first target in the chunk's sub-trie will therefore be the upper - // bound of the whole chunk. - // - For example, given a chunk with sub-trie prefixes [0x2, 0x2f, 0x2fa], the upper bounds - // will be [0x3, 0x3, 0x2fb]. Note that no target could match a trie node with path equal - // to or greater than 0x3. - // - // - If a target's sub-trie's prefix does not lie within the bounds of the current chunk, then - // that target must be the first target of the next chunk, lying in a separate sub-trie. - // - Example: given sub-trie prefixes of [0x2, 0x2fa, 0x4c, 0x4ce, 0x4e], we would end up - // with the following chunks: - // - [0x2, 0x2fa] w/ upper bound 0x3 - // - [0x4c 0x4ce] w/ upper bound 0x4d - // - [0x4e] w/ upper bound 0x4f - let mut upper_bound = targets.first().and_then(|t| sub_trie_upper_bound(&sub_trie_prefix(t))); - let target_chunks = targets.chunk_by_mut(move |_, next| { - if let Some(some_upper_bound) = upper_bound { - let prefix = sub_trie_prefix(next); - let same_chunk = prefix < some_upper_bound; - if !same_chunk { - upper_bound = sub_trie_upper_bound(&prefix); - } - same_chunk - } else { - true - } - }); + // Chunk targets by exact sub-trie prefix. Prefixes nested below a previous prefix are still + // processed as their own sub-trie. + let target_chunks = + targets.chunk_by_mut(|current, next| sub_trie_prefix(current) == sub_trie_prefix(next)); - // Map the chunks to the return type. Within each chunk we want targets to be sorted by their - // key, as that will be the order they are checked by the `ProofCalculator`. + // Map the chunks to the return type. target_chunks.map(move |targets| { let prefix = sub_trie_prefix(&targets[0]); - let retain_root = targets[0].min_len == 0; - targets.sort_unstable_by_key(|target| target.key_nibbles); + // Targets with `min_len` 0 and 1 share the empty prefix, so key ordering cannot indicate + // whether the root should be retained. + let retain_root = targets.iter().any(|target| target.min_len == 0); SubTrieTargets { prefix, targets, retain_root } }) } @@ -179,21 +151,21 @@ mod tests { ("4", vec!["4040404040404040404040404040404040404040404040404040404040404040"]), ], ), - // Case 6: Targets with different min_len values in same sub-trie + // Case 6: Targets with different min_len values in different sub-tries ( vec![ ProofV2Target::new(B256::repeat_byte(0x20)).with_min_len(2), ProofV2Target::new(B256::repeat_byte(0x2f)).with_min_len(3), ], - vec![( - "2", - vec![ - "2020202020202020202020202020202020202020202020202020202020202020", - "2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f", - ], - )], + vec![ + ("2", vec!["2020202020202020202020202020202020202020202020202020202020202020"]), + ( + "2f", + vec!["2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f"], + ), + ], ), - // Case 7: More complex chunking with multiple sub-tries + // Case 7: More complex chunking with nested sub-trie prefixes ( vec![ ProofV2Target::new(B256::repeat_byte(0x20)).with_min_len(2), @@ -203,19 +175,18 @@ mod tests { ProofV2Target::new(B256::repeat_byte(0x4e)).with_min_len(3), ], vec![ + ("2", vec!["2020202020202020202020202020202020202020202020202020202020202020"]), ( - "2", - vec![ - "2020202020202020202020202020202020202020202020202020202020202020", - "2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f", - ], + "2f2", + vec!["2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f"], ), ( "4c", - vec![ - "4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c", - "4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c", - ], + vec!["4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c"], + ), + ( + "4c4", + vec!["4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c"], ), ( "4e", @@ -243,13 +214,10 @@ mod tests { ProofV2Target::new(B256::repeat_byte(0x20)).with_min_len(2), ProofV2Target::new(B256::repeat_byte(0x40)).with_min_len(1), ], - vec![( - "", - vec![ - "2020202020202020202020202020202020202020202020202020202020202020", - "4040404040404040404040404040404040404040404040404040404040404040", - ], - )], + vec![ + ("", vec!["4040404040404040404040404040404040404040404040404040404040404040"]), + ("2", vec!["2020202020202020202020202020202020202020202020202020202020202020"]), + ], ), ]; @@ -299,4 +267,18 @@ mod tests { } } } + + #[test] + fn test_iter_sub_trie_targets_retain_root_after_key_sort() { + let mut targets = [ + ProofV2Target::new(B256::repeat_byte(0x40)), + ProofV2Target::new(B256::repeat_byte(0x20)).with_min_len(1), + ]; + + let sub_tries = iter_sub_trie_targets(&mut targets).collect::>(); + + assert_eq!(sub_tries.len(), 1); + assert_eq!(sub_tries[0].targets[0].key(), B256::repeat_byte(0x20)); + assert!(sub_tries[0].retain_root); + } } diff --git a/docs/repo/layout.md b/docs/repo/layout.md index 057e5da6d31..6ecc21e1076 100644 --- a/docs/repo/layout.md +++ b/docs/repo/layout.md @@ -155,7 +155,6 @@ The IPC transport lives in [`rpc/ipc`](../../crates/rpc/ipc). - [`rpc/rpc-convert`](../../crates/rpc/rpc-convert): This crate provides various helper functions to convert between reth primitive types and rpc types. - [`rpc/layer`](../../crates/rpc/rpc-layer/): Some RPC middleware layers (e.g. `AuthValidator`, `JwtAuthValidator`) -- [`rpc/rpc-testing-util`](../../crates/rpc/rpc-testing-util/): Reth RPC testing helpers ### Payloads diff --git a/docs/vocs/docs/pages/cli/reth/node.mdx b/docs/vocs/docs/pages/cli/reth/node.mdx index b43a1c7d027..50d482270c1 100644 --- a/docs/vocs/docs/pages/cli/reth/node.mdx +++ b/docs/vocs/docs/pages/cli/reth/node.mdx @@ -584,7 +584,7 @@ Credible Layer: --rpc.credible-registry-address

Address of the on-chain `CredibleRegistry` contract. - When set, the marker override for `eth_call` / `eth_estimateGas` is derived per-request from the registry's `_credibleBlocks` mapping instead of a static override. + When set, the marker override for call-like RPC methods is derived per-request from the registry's `_credibleBlocks` mapping instead of a static override. --rpc.credible-retain-forwarded-private Retains transactions accepted by `--rpc.forwarder` as private pool transactions diff --git a/docs/vocs/docs/pages/jsonrpc/admin.mdx b/docs/vocs/docs/pages/jsonrpc/admin.mdx index 54c13f3835c..af220e7c7f4 100644 --- a/docs/vocs/docs/pages/jsonrpc/admin.mdx +++ b/docs/vocs/docs/pages/jsonrpc/admin.mdx @@ -96,7 +96,7 @@ Returns `true` once the request has been accepted. ## `admin_unbanPeer` -Removes a remote peer from the ban list. +Removes a remote peer from the ban list and resets its reputation so it can reconnect. Returns `true` once the request has been accepted. diff --git a/docs/vocs/vocs.config.ts b/docs/vocs/vocs.config.ts index 0c905d62d3d..bb8a8fba54b 100644 --- a/docs/vocs/vocs.config.ts +++ b/docs/vocs/vocs.config.ts @@ -23,7 +23,7 @@ export default defineConfig({ { text: 'Rustdocs', link: '/docs' }, { text: 'GitHub', link: 'https://github.com/paradigmxyz/reth' }, { - text: 'v2.3.0', + text: 'v2.4.1', items: [ { text: 'Releases', diff --git a/examples/custom-state-root/src/main.rs b/examples/custom-state-root/src/main.rs index 581bd54c920..813969dc8ea 100644 --- a/examples/custom-state-root/src/main.rs +++ b/examples/custom-state-root/src/main.rs @@ -24,11 +24,10 @@ use alloy_genesis::Genesis; use alloy_primitives::B256; use reth_chain_state::StateTrieOverlayManager; use reth_engine_tree::tree::{ - payload_processor::multiproof::{PayloadStateRootHandle, StateRootStreams}, - payload_validator::LazyHashedPostState, state_root_strategy::{ - DefaultStateRootStrategy, PayloadStateRootJobContext, PreparedStateRootJob, StateRootJob, - StateRootJobContext, StateRootJobOutcome, StateRootStrategy, + DefaultStateRootStrategy, LazyHashedPostState, PayloadStateRootHandle, + PayloadStateRootJobContext, PreparedStateRootJob, StateRootJob, StateRootJobContext, + StateRootJobOutcome, StateRootStrategy, }, BasicEngineValidator, TreeConfig, }; @@ -77,12 +76,12 @@ where if timestamp < self.activation_timestamp { return self.default.prepare(ctx) } - Ok(PreparedStateRootJob::new(Box::new(ZeroStateRootJob), StateRootStreams::empty(), None)) + Ok(PreparedStateRootJob::new(Box::new(ZeroStateRootJob), None)) } fn prepare_payload_builder( &self, - ctx: PayloadStateRootJobContext<'_, N, P, Evm>, + ctx: PayloadStateRootJobContext<'_, N, P>, ) -> ProviderResult> { if ctx.timestamp() < self.activation_timestamp { return self.default.prepare_payload_builder(ctx) @@ -187,7 +186,7 @@ async fn main() -> eyre::Result<()> { // Zero roots from genesis on. Set this to a fork timestamp to keep the // default state-root machinery for earlier blocks. activation_timestamp: 0, - default: DefaultStateRootStrategy, + default: DefaultStateRootStrategy::default(), }), }, Default::default(), diff --git a/examples/network-proxy/src/main.rs b/examples/network-proxy/src/main.rs index 27fe251adc8..0b73d867216 100644 --- a/examples/network-proxy/src/main.rs +++ b/examples/network-proxy/src/main.rs @@ -94,6 +94,7 @@ async fn main() -> eyre::Result<()> { IncomingEthRequest::GetReceipts70 { .. } => {} IncomingEthRequest::GetBlockAccessLists { .. } => {} IncomingEthRequest::GetCells { .. } => {} + IncomingEthRequest::GetSnap { .. } => {} } } transaction_message = transactions_rx.recv() => { diff --git a/flake.nix b/flake.nix index a0f581c4251..9237003f6ae 100644 --- a/flake.nix +++ b/flake.nix @@ -51,6 +51,7 @@ nativeBuildInputs = [ pkgs.pkg-config pkgs.libgit2 + pkgs.m4 pkgs.perl ];