From 7fba966efe1fd21766b695ae9578e0fbb74dbc09 Mon Sep 17 00:00:00 2001 From: creditcoinprotoclaw Date: Mon, 27 Jul 2026 09:00:44 +0000 Subject: [PATCH 1/8] feat: enforce per-transaction gas limit in decode-blocks Add singleTxnGasLimit = 25_000_000n and reject any proof whose gas crosses it. Checked in three places: after gasForVerification, after gasForDecoding, and after totalGas. Mirrors the same change in creditcoin3 prover-check.ts. --- src/bin/decode-blocks.ts | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/bin/decode-blocks.ts b/src/bin/decode-blocks.ts index 097fc6a..5282f8f 100644 --- a/src/bin/decode-blocks.ts +++ b/src/bin/decode-blocks.ts @@ -78,6 +78,17 @@ async function decodeFromDisk( } console.log(` ... gasForVerification=${gasForVerification} - 0 means skipped`); + // Reject any single transaction whose individual gas cost crosses the + // per-transaction cap. A single tx must fit comfortably within a block + // on its own, so each estimate is checked against singleTxnGasLimit as + // soon as it becomes available. + const singleTxnGasLimit = 25_000_000n; + if (gasForVerification >= singleTxnGasLimit) { + throw new Error( + `gasForVerification ${gasForVerification} reaches or exceeds the single transaction gas limit (${singleTxnGasLimit}); failing run`, + ); + } + const encodedData = readFileSync(pathToTxn, { encoding: 'utf8', flag: 'r', @@ -88,6 +99,11 @@ async function decodeFromDisk( }); const gasForDecoding = decoded.gasUsed ?? BigInt(0); console.log(` decoded as type ${decoded.type}, gasForDecoding=${gasForDecoding}`); + if (gasForDecoding >= singleTxnGasLimit) { + throw new Error( + `gasForDecoding ${gasForDecoding} reaches or exceeds the single transaction gas limit (${singleTxnGasLimit}); failing run`, + ); + } // Add a 10% safety margin to the raw estimates and reject if the // combined cost crosses 70% of the 75M block gas limit. Using bigint @@ -98,6 +114,11 @@ async function decodeFromDisk( const blockGasLimit = 75_000_000n; const totalGasThreshold = (blockGasLimit * 7n) / 10n; console.log(` ... totalGas (with 10% margin)=${totalGas} (threshold=${totalGasThreshold})`); + if (totalGas >= singleTxnGasLimit) { + throw new Error( + `totalGas ${totalGas} reaches or exceeds the single transaction gas limit (${singleTxnGasLimit}); failing run`, + ); + } if (totalGas >= totalGasThreshold) { throw new Error( `totalGas ${totalGas} reaches or exceeds 70% of the ${blockGasLimit} block gas limit (${totalGasThreshold}); failing run`, From 1c02ede6c908320fcb0be02ef40b310508623950 Mon Sep 17 00:00:00 2001 From: creditcoinprotoclaw Date: Mon, 27 Jul 2026 09:20:06 +0000 Subject: [PATCH 2/8] feat: encode+decode the 8 largest mainnet txns via direct query Add a deterministic worst-case coverage path that queries the 8 largest documented Ethereum Mainnet transactions directly by hash (instead of streaming live blocks and hoping to re-encounter them), encodes them via both the ethers (TS) and alloy (Rust) encoders, and decodes them downstream. - src/bin/encode-largest-txns.ts: ethers encoder, queries the 8 fixed block/tx pairs directly, fails on any missing/oversized txn. - rust/bin/encode_largest_txns.rs: alloy counterpart, same fixtures. - rust/Cargo.toml: register the new encode-largest-txns bin. - .github/workflows/compare-encoding-largest-txns.yml: mainnet-only (Sepolia dropped), encodes + diffs alloy vs ethers, then decodes on cc3-devnet. step timeouts bumped to 5 mins so a PR run completes. cc <@U028EMRHS3S> --- .../compare-encoding-largest-txns.yml | 253 ++++++++++++++++++ rust/Cargo.toml | 4 + rust/bin/encode_largest_txns.rs | 145 ++++++++++ src/bin/encode-largest-txns.ts | 118 ++++++++ 4 files changed, 520 insertions(+) create mode 100644 .github/workflows/compare-encoding-largest-txns.yml create mode 100644 rust/bin/encode_largest_txns.rs create mode 100644 src/bin/encode-largest-txns.ts diff --git a/.github/workflows/compare-encoding-largest-txns.yml b/.github/workflows/compare-encoding-largest-txns.yml new file mode 100644 index 0000000..dfc2051 --- /dev/null +++ b/.github/workflows/compare-encoding-largest-txns.yml @@ -0,0 +1,253 @@ +--- +name: compare-encoding-largest-txns + +'on': + pull_request: + paths: + - '**.rs' + - '**.ts' + - '**Cargo**' + - '**package*.json' + - '.github/workflows/compare-encoding-*.yml' + schedule: + - cron: '42 */12 * * *' + workflow_dispatch: + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +permissions: read-all + +jobs: + alloy-encode: + runs-on: ubuntu-26.04 + steps: + - uses: actions/checkout@v7 + + - name: Configure rustc version + run: | + RUSTC_VERSION=$(grep channel rust/rust-toolchain.toml | tail -n1 | tr -d " " | cut -f2 -d'"') + echo "RUSTC_VERSION=$RUSTC_VERSION" >> "$GITHUB_ENV" + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + toolchain: ${{ env.RUSTC_VERSION }} + targets: wasm32-unknown-unknown + + - uses: Swatinem/rust-cache@v2 + with: + workspaces: rust + + - name: Build + working-directory: rust + run: | + cargo build --release --bin encode-largest-txns + + - name: DEBUG - Print runner IP address + run: | + # see https://opensource.com/article/18/5/how-find-ip-address-linux + IP_ADDRESS=$(curl https://ifconfig.me) + echo "INFO: IP_ADDRESS=$IP_ADDRESS" + + - name: Encode the 8 largest mainnet transactions + timeout-minutes: 5 + working-directory: rust + run: | + ./target/release/encode-largest-txns \ + --eth-rpc-url wss://mainnet.infura.io/ws/v3/${{ secrets.INFURA_ETHEREUM_RPC_KEY }} \ + --path-to-store-json /var/tmp/encoded-data/alloy/ + + - name: Add notices in case of failure + if: failure() && github.event_name == 'pull_request' + run: | + # note: GitHub appears to display these in reverse order + echo "::notice::SUGGESTION: Trigger this workflow manually against the same branch to verify the changes." + echo "::notice::IMPORTANT: Secret credentials may be missing for PRs from Dependabot!" + + - name: Report result to Slack + if: failure() && github.event_name == 'schedule' + uses: act10ns/slack@v2 + with: + webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }} + status: ${{ job.status }} + channel: '#protocol-internal' + message: 'Alloy encode (largest txns) failed! cc <@U028EMRHS3S>' + + - name: Upload encoded data + if: always() + uses: actions/upload-artifact@v7 + with: + name: encoded-by-alloy-largest + path: /var/tmp/encoded-data/alloy + include-hidden-files: true + + ethers-encode: + runs-on: ubuntu-26.04 + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-node@v7 + with: + node-version: '24.x' + cache: 'npm' + + - run: npm ci + - run: npm run build --if-present + + - name: DEBUG - Print runner IP address + run: | + # see https://opensource.com/article/18/5/how-find-ip-address-linux + IP_ADDRESS=$(curl https://ifconfig.me) + echo "INFO: IP_ADDRESS=$IP_ADDRESS" + + - name: Encode the 8 largest mainnet transactions + timeout-minutes: 5 + run: | + set -o pipefail + node dist/bin/encode-largest-txns.js \ + wss://blockchain.googleapis.com/v1/projects/creditcoin-test/locations/us-central1/endpoints/ethereum-mainnet/rpc?key=${{ secrets.GOOGLE_ETHEREUM_RPC_KEY }} \ + /var/tmp/encoded-data/ethers/ \ + 2>&1 | tee /var/tmp/ethers-encode.log + + - name: Fail on MAX_ENCODED_SIZE errors + if: always() + run: | + if [ ! -f /var/tmp/ethers-encode.log ]; then + echo "::error::encode log not found — cannot verify MAX_ENCODED_SIZE" + exit 1 + fi + if grep -q 'MAX_ENCODED_SIZE' /var/tmp/ethers-encode.log; then + echo "::error::One or more transactions exceeded MAX_ENCODED_SIZE and were silently skipped:" + grep 'MAX_ENCODED_SIZE' /var/tmp/ethers-encode.log + exit 1 + fi + echo "No MAX_ENCODED_SIZE errors detected." + + - name: Add notices in case of failure + if: failure() && github.event_name == 'pull_request' + run: | + # note: GitHub appears to display these in reverse order + echo "::notice::SUGGESTION: Trigger this workflow manually against the same branch to verify the changes." + echo "::notice::IMPORTANT: Secret credentials may be missing for PRs from Dependabot!" + + - name: Report result to Slack + if: failure() && github.event_name == 'schedule' + uses: act10ns/slack@v2 + with: + webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }} + status: ${{ job.status }} + channel: '#protocol-internal' + message: 'Ethers encode (largest txns) failed! cc <@U028EMRHS3S>' + + - name: Upload encoded data + if: always() + uses: actions/upload-artifact@v7 + with: + name: encoded-by-ethers-largest + path: /var/tmp/encoded-data/ethers + include-hidden-files: true + + compare-data: + needs: + - alloy-encode + - ethers-encode + runs-on: ubuntu-26.04 + steps: + - name: Setup + run: | + sudo apt-get install colordiff unzip + + - name: Download encoded data from alloy + uses: actions/download-artifact@v8 + with: + path: /var/tmp/encoded-data + pattern: encoded-by-alloy-largest + + - name: Git commit alloy data + working-directory: /var/tmp/encoded-data + run: | + git config --global init.defaultBranch main + git config --global user.email "creditcoin@gluwa.com" + git config --global user.name "gluwa-bot" + + git init + git add . + git commit -a -m "Import data encoded by alloy" + + # remove everything b/c next we'll import the data from ethers encoding + rm -rf * + + - name: Download encoded data from ethers + uses: actions/download-artifact@v8 + with: + path: /var/tmp/encoded-data + pattern: encoded-by-ethers-largest + + - name: Compare + working-directory: /var/tmp/encoded-data + timeout-minutes: 5 + run: | + git status + + set -eo pipefail + git diff | colordiff + + # will exit non-zero in case of differences + git diff --quiet + + - name: Report result to Slack + if: always() + uses: act10ns/slack@v2 + with: + webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }} + status: ${{ job.status }} + channel: '#protocol-internal' + message: 'Encoding diff (largest txns) complete! cc <@U028EMRHS3S>' + + decode-transactions: + needs: + - ethers-encode + # Run even if ethers-encode "failed" (e.g. the MAX_ENCODED_SIZE guard tripped). + # decode-transactions only consumes the uploaded encoded-by-ethers-largest + # artifact, which is uploaded with `if: always()`, so it can still run and + # decode the transactions that were produced. !cancelled() keeps + # manual/cancel behaviour intact. + if: ${{ !cancelled() }} + runs-on: ubuntu-26.04 + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-node@v7 + with: + node-version: '24.x' + cache: 'npm' + + - run: npm ci + - run: npm run build --if-present + + - name: Download encoded data from ethers + uses: actions/download-artifact@v8 + with: + path: /var/tmp/encoded-data + pattern: encoded-by-ethers-largest + + - name: Decode transactions + timeout-minutes: 5 + run: | + node dist/bin/decode-blocks.js \ + wss://rpc.cc3-devnet.creditcoin.network \ + https://prover.cc3-devnet.creditcoin.network \ + 0xf882f3a71A36E29E86615B7055f42A68D7E4cfBB \ + /var/tmp/encoded-data/ \ + 4 + + - name: Report result to Slack + if: always() + uses: act10ns/slack@v2 + with: + webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }} + status: ${{ job.status }} + channel: '#protocol-internal' + message: 'Decoding (largest txns) complete! cc <@U028EMRHS3S>' diff --git a/rust/Cargo.toml b/rust/Cargo.toml index e286fe7..9d4531c 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -11,6 +11,10 @@ version = "0.1.0" name = "encode-blocks" path = "bin/encode_blocks.rs" +[[bin]] +name = "encode-largest-txns" +path = "bin/encode_largest_txns.rs" + [dependencies] anyhow = "1.0" clap = { version = "4.6.1", features = ["derive"] } diff --git a/rust/bin/encode_largest_txns.rs b/rust/bin/encode_largest_txns.rs new file mode 100644 index 0000000..5d32ef6 --- /dev/null +++ b/rust/bin/encode_largest_txns.rs @@ -0,0 +1,145 @@ +use alloy::{ + primitives::B256, + providers::{Provider, ProviderBuilder, WsConnect}, + rpc::types::TransactionReceipt, +}; + +use anyhow::{anyhow, Result}; +use clap::Parser; + +use std::fs; +use std::str::FromStr; + +use usc_abi_encoding::abi::abi_encode; +use usc_abi_encoding::common::EncodingVersion; + +/// The 8 largest transactions ever observed successfully-encoded on Ethereum +/// Mainnet. The streaming `encode-blocks` binary only re-encountered blocks of +/// this size non-deterministically on the live head; this binary instead +/// queries these exact block/tx pairs directly so they are encoded (and decoded +/// downstream) on every CI run. Mainnet-only: Sepolia is intentionally +/// unsupported here. +const LARGEST_MAINNET_TXNS: &[(u64, &str)] = &[ + ( + 25602727, + "0x4e94d836e6e2794556e1cbb3a2cfb1945248d156c97b5d902835dbd9a4b88e60", + ), + ( + 25599245, + "0x24a6129734163346da53f056a8022f3ec37d70b8350ed9b8300620bbbdba6e1e", + ), + ( + 25551628, + "0x181611bff5f83dcf85cc45e06a453ee79a4ca1a697a1316030e655901c71bee8", + ), + ( + 25551622, + "0x296d83e8a0db263ad06422be8c6bd426c70785cc7c4f2b0b559eec5586e9da86", + ), + ( + 25238768, + "0x01ca130bf04e636d26ebdf0f6256a99894a6b474d4c016af74849c6a7572928d", + ), + ( + 25238750, + "0x343b91c47944693ed1cdf3c979bd7722ed9284320ff6069bcfd46c109d9c4199", + ), + ( + 25238749, + "0x5f60979ee18aba3f76122574e987f974fb1d7bacc372666f4b3f647236d54794", + ), + ( + 25238746, + "0xf2641f3bd13a111169c007205b3d1e7188201df3ae041991d2e1e3745ed1fb2d", + ), +]; + +#[derive(Parser, Debug)] +#[command(name = "encode-largest-txns")] +pub struct CliArguments { + #[arg(long, help = "WebSockets URL to an Ethereum Mainnet RPC", required = true)] + pub eth_rpc_url: String, + + #[arg(long, help = "Directory path to store JSON files", required = true)] + pub path_to_store_json: String, +} + +async fn encode_transaction( + provider: impl Provider, + tx_hash_str: &str, + rx_or_none: Option, +) -> Result { + let tx_hash = B256::from_str(tx_hash_str)?; + + let tx = provider + .get_transaction_by_hash(tx_hash) + .await? + .ok_or_else(|| anyhow!("transaction {tx_hash_str} not found via RPC"))?; + + let rx = match rx_or_none { + Some(rx) => rx, + None => provider + .get_transaction_receipt(tx_hash) + .await? + .ok_or_else(|| anyhow!("receipt for {tx_hash_str} not found via RPC"))?, + }; + + let encoded_data = abi_encode(tx, rx, EncodingVersion::V1) + .ok_or_else(|| anyhow!("abi_encode returned None for {tx_hash_str}"))?; + let as_str = hex::encode(encoded_data.abi()); + + Ok(format!("0x{as_str}")) +} + +async fn encode_and_write_to_disk( + path: &str, + provider: impl Provider, + block_number: u64, + tx_hash: &str, +) -> Result<()> { + let encoded_data = encode_transaction(provider, tx_hash, None).await?; + + // //.txt + fs::create_dir_all(format!("{path}/{block_number}"))?; + fs::write(format!("{path}/{block_number}/{tx_hash}.txt"), encoded_data + "\n")?; + + Ok(()) +} + +#[tokio::main] +async fn main() -> Result<()> { + let args = CliArguments::parse(); + + println!( + "=== encoding {} largest mainnet transactions ...", + LARGEST_MAINNET_TXNS.len() + ); + + fs::create_dir_all(args.path_to_store_json.clone())?; + + let provider = ProviderBuilder::new() + .on_ws(WsConnect::new(args.eth_rpc_url)) + .await?; + + // Encode each documented transaction directly by hash. We fail the whole + // run on any error: unlike the streaming encoder (which skips transient + // failures on live blocks), these are fixed historical fixtures and must + // always be retrievable/encodable. + for (block_number, tx_hash) in LARGEST_MAINNET_TXNS { + println!("--- encoding block {block_number} txn {tx_hash}"); + encode_and_write_to_disk( + &args.path_to_store_json, + provider.clone(), + *block_number, + tx_hash, + ) + .await?; + } + + println!( + "<<< done encoding {} transactions", + LARGEST_MAINNET_TXNS.len() + ); + + Ok(()) +} diff --git a/src/bin/encode-largest-txns.ts b/src/bin/encode-largest-txns.ts new file mode 100644 index 0000000..584737c --- /dev/null +++ b/src/bin/encode-largest-txns.ts @@ -0,0 +1,118 @@ +import { mkdirSync, writeFileSync } from 'fs'; +import { WebSocketProvider, TransactionReceipt } from 'ethers'; +import { abiEncode } from '../encoding/abi'; +import { getTransactionWithRaw } from '../encoding'; +import { bytesInHexString } from '../utils/hex'; + +// The 8 largest transactions ever observed successfully-encoded on Ethereum +// Mainnet. Historically these were only documented as a comment inside +// encode-blocks.ts; the streaming encoder relied on eventually re-encountering +// blocks of this size on the live head. That is slow and non-deterministic. +// +// This binary instead queries these exact block/tx pairs directly so they can +// be encoded (and decoded downstream) on every CI run, giving a deterministic +// worst-case coverage check. Sepolia is intentionally unsupported here: these +// are mainnet-only fixtures. +interface LargeTxn { + blockNumber: number; + txHash: string; +} + +const LARGEST_MAINNET_TXNS: LargeTxn[] = [ + { blockNumber: 25602727, txHash: '0x4e94d836e6e2794556e1cbb3a2cfb1945248d156c97b5d902835dbd9a4b88e60' }, + { blockNumber: 25599245, txHash: '0x24a6129734163346da53f056a8022f3ec37d70b8350ed9b8300620bbbdba6e1e' }, + { blockNumber: 25551628, txHash: '0x181611bff5f83dcf85cc45e06a453ee79a4ca1a697a1316030e655901c71bee8' }, + { blockNumber: 25551622, txHash: '0x296d83e8a0db263ad06422be8c6bd426c70785cc7c4f2b0b559eec5586e9da86' }, + { blockNumber: 25238768, txHash: '0x01ca130bf04e636d26ebdf0f6256a99894a6b474d4c016af74849c6a7572928d' }, + { blockNumber: 25238750, txHash: '0x343b91c47944693ed1cdf3c979bd7722ed9284320ff6069bcfd46c109d9c4199' }, + { blockNumber: 25238749, txHash: '0x5f60979ee18aba3f76122574e987f974fb1d7bacc372666f4b3f647236d54794' }, + { blockNumber: 25238746, txHash: '0xf2641f3bd13a111169c007205b3d1e7188201df3ae041991d2e1e3745ed1fb2d' }, +]; + +// Maximum discovered size of ABI-encoded transaction data, in bytes. +// Derived from the largest observed successfully-encoded transactions above. +const MAX_ENCODED_SIZE = 530336; + +// cost 80 or 160 credits depending on arguments +async function encodeTransaction( + provider: WebSocketProvider, + txHash: string, + receipt: TransactionReceipt | null, +): Promise { + // 80 credits + const transaction = await getTransactionWithRaw(provider, txHash); + if (transaction === null) { + throw new Error(`transaction ${txHash} not found via RPC`); + } + + if (receipt === null) { + // 80 credits + receipt = await provider.getTransactionReceipt(txHash); + } + if (receipt === null) { + throw new Error(`receipt for ${txHash} not found via RPC`); + } + + const encodedData = abiEncode(transaction, receipt); + return encodedData.abi; +} + +async function encodeAndWriteToDisk( + pathToStoreJson: string, + provider: WebSocketProvider, + blockNumber: number, + txHash: string, +): Promise { + const encodedData = await encodeTransaction(provider, txHash, null); + + const encodedSize = bytesInHexString(encodedData); + if (encodedSize > MAX_ENCODED_SIZE) { + // Do NOT abort: we still want to encode and persist oversized transactions + // so the run continues. Log in the exact `encoded data exceeds + // MAX_ENCODED_SIZE` format so a downstream CI step can grep for it and fail + // the pipeline. + console.error( + `encoded data exceeds MAX_ENCODED_SIZE: blockNumber=${blockNumber} txHash=${txHash} encodedSize=${encodedSize} bytes (max=${MAX_ENCODED_SIZE})`, + ); + } + + mkdirSync(`${pathToStoreJson}/${blockNumber}`, { recursive: true }); + writeFileSync(`${pathToStoreJson}/${blockNumber}/${txHash}.txt`, encodedData + '\n', { + flag: 'w', + }); +} + +async function encodeLargestTxns(rpcUrl: string, pathToStoreJson: string): Promise { + console.log(`=== encoding ${LARGEST_MAINNET_TXNS.length} largest mainnet transactions ...`); + + mkdirSync(pathToStoreJson, { recursive: true }); + + const provider = new WebSocketProvider(rpcUrl); + + try { + // Encode each documented transaction directly by hash. We fail the whole + // run on any error: unlike the streaming encoder (which skips transient + // failures on live blocks), these are fixed historical fixtures and must + // always be retrievable/encodable. + for (const { blockNumber, txHash } of LARGEST_MAINNET_TXNS) { + console.log(`--- encoding block ${blockNumber} txn ${txHash}`); + await encodeAndWriteToDisk(pathToStoreJson, provider, blockNumber, txHash); + } + console.log(`<<< done encoding ${LARGEST_MAINNET_TXNS.length} transactions`); + } finally { + await provider.destroy(); + } +} + +if (process.argv.length < 4) { + console.error('node dist/bin/encode-largest-txns.js '); + process.exit(1); +} + +const rpcUrl = process.argv[2] || 'ws://127.0.0.1:8545'; +const pathToStoreJson = process.argv[3]; + +encodeLargestTxns(rpcUrl, pathToStoreJson).catch((reason) => { + console.error(reason); + process.exit(1); +}); From b58f7f7f9a7ec58797e8f96c44c4ce7be8f86718 Mon Sep 17 00:00:00 2001 From: creditcoinprotoclaw Date: Mon, 27 Jul 2026 09:35:35 +0000 Subject: [PATCH 3/8] refactor: ethers-only largest-txns, log-and-grep instead of throw Amend per review: - Drop rust/bin/encode_largest_txns.rs (and its Cargo.toml bin entry); keep this path ethers-only. - Drop the alloy-encode and compare-data jobs from the workflow; it now just encodes (ethers) then decodes. - Encoder no longer throws on a failed txn. Each of the 8 fixtures is attempted and any failure is logged with an ENCODE_ERROR: prefix so we see ALL failing large txns in one run instead of bailing on the first. A single grep-based CI gate fails the pipeline afterwards. cc <@U028EMRHS3S> --- .../compare-encoding-largest-txns.yml | 153 +++--------------- rust/Cargo.toml | 4 - rust/bin/encode_largest_txns.rs | 145 ----------------- src/bin/encode-largest-txns.ts | 37 +++-- 4 files changed, 48 insertions(+), 291 deletions(-) delete mode 100644 rust/bin/encode_largest_txns.rs diff --git a/.github/workflows/compare-encoding-largest-txns.yml b/.github/workflows/compare-encoding-largest-txns.yml index dfc2051..f34f187 100644 --- a/.github/workflows/compare-encoding-largest-txns.yml +++ b/.github/workflows/compare-encoding-largest-txns.yml @@ -20,69 +20,6 @@ concurrency: permissions: read-all jobs: - alloy-encode: - runs-on: ubuntu-26.04 - steps: - - uses: actions/checkout@v7 - - - name: Configure rustc version - run: | - RUSTC_VERSION=$(grep channel rust/rust-toolchain.toml | tail -n1 | tr -d " " | cut -f2 -d'"') - echo "RUSTC_VERSION=$RUSTC_VERSION" >> "$GITHUB_ENV" - - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable - with: - toolchain: ${{ env.RUSTC_VERSION }} - targets: wasm32-unknown-unknown - - - uses: Swatinem/rust-cache@v2 - with: - workspaces: rust - - - name: Build - working-directory: rust - run: | - cargo build --release --bin encode-largest-txns - - - name: DEBUG - Print runner IP address - run: | - # see https://opensource.com/article/18/5/how-find-ip-address-linux - IP_ADDRESS=$(curl https://ifconfig.me) - echo "INFO: IP_ADDRESS=$IP_ADDRESS" - - - name: Encode the 8 largest mainnet transactions - timeout-minutes: 5 - working-directory: rust - run: | - ./target/release/encode-largest-txns \ - --eth-rpc-url wss://mainnet.infura.io/ws/v3/${{ secrets.INFURA_ETHEREUM_RPC_KEY }} \ - --path-to-store-json /var/tmp/encoded-data/alloy/ - - - name: Add notices in case of failure - if: failure() && github.event_name == 'pull_request' - run: | - # note: GitHub appears to display these in reverse order - echo "::notice::SUGGESTION: Trigger this workflow manually against the same branch to verify the changes." - echo "::notice::IMPORTANT: Secret credentials may be missing for PRs from Dependabot!" - - - name: Report result to Slack - if: failure() && github.event_name == 'schedule' - uses: act10ns/slack@v2 - with: - webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }} - status: ${{ job.status }} - channel: '#protocol-internal' - message: 'Alloy encode (largest txns) failed! cc <@U028EMRHS3S>' - - - name: Upload encoded data - if: always() - uses: actions/upload-artifact@v7 - with: - name: encoded-by-alloy-largest - path: /var/tmp/encoded-data/alloy - include-hidden-files: true - ethers-encode: runs-on: ubuntu-26.04 steps: @@ -107,23 +44,36 @@ jobs: run: | set -o pipefail node dist/bin/encode-largest-txns.js \ - wss://blockchain.googleapis.com/v1/projects/creditcoin-test/locations/us-central1/endpoints/ethereum-mainnet/rpc?key=${{ secrets.GOOGLE_ETHEREUM_RPC_KEY }} \ + wss://blockchain.googleapis.com/v1/projects/creditcoin-test/locations/us-central1/endpoints/ethereum-mainnet/rpc?key=*** secrets.GOOGLE_ETHEREUM_RPC_KEY }} \ /var/tmp/encoded-data/ethers/ \ 2>&1 | tee /var/tmp/ethers-encode.log - - name: Fail on MAX_ENCODED_SIZE errors + - name: Fail on encode errors or MAX_ENCODED_SIZE if: always() run: | if [ ! -f /var/tmp/ethers-encode.log ]; then - echo "::error::encode log not found — cannot verify MAX_ENCODED_SIZE" + echo "::error::encode log not found — cannot verify results" exit 1 fi + + # The encoder never throws: it logs every failure so all 8 fixtures + # get attempted in one run. Grep for both failure markers here and + # fail the pipeline once, after everything has been attempted. + FAILED=0 + if grep -q 'ENCODE_ERROR' /var/tmp/ethers-encode.log; then + echo "::error::One or more largest txns failed to encode:" + grep 'ENCODE_ERROR' /var/tmp/ethers-encode.log + FAILED=1 + fi if grep -q 'MAX_ENCODED_SIZE' /var/tmp/ethers-encode.log; then - echo "::error::One or more transactions exceeded MAX_ENCODED_SIZE and were silently skipped:" + echo "::error::One or more transactions exceeded MAX_ENCODED_SIZE:" grep 'MAX_ENCODED_SIZE' /var/tmp/ethers-encode.log + FAILED=1 + fi + if [ "$FAILED" -ne 0 ]; then exit 1 fi - echo "No MAX_ENCODED_SIZE errors detected." + echo "No encode errors detected." - name: Add notices in case of failure if: failure() && github.event_name == 'pull_request' @@ -149,71 +99,14 @@ jobs: path: /var/tmp/encoded-data/ethers include-hidden-files: true - compare-data: - needs: - - alloy-encode - - ethers-encode - runs-on: ubuntu-26.04 - steps: - - name: Setup - run: | - sudo apt-get install colordiff unzip - - - name: Download encoded data from alloy - uses: actions/download-artifact@v8 - with: - path: /var/tmp/encoded-data - pattern: encoded-by-alloy-largest - - - name: Git commit alloy data - working-directory: /var/tmp/encoded-data - run: | - git config --global init.defaultBranch main - git config --global user.email "creditcoin@gluwa.com" - git config --global user.name "gluwa-bot" - - git init - git add . - git commit -a -m "Import data encoded by alloy" - - # remove everything b/c next we'll import the data from ethers encoding - rm -rf * - - - name: Download encoded data from ethers - uses: actions/download-artifact@v8 - with: - path: /var/tmp/encoded-data - pattern: encoded-by-ethers-largest - - - name: Compare - working-directory: /var/tmp/encoded-data - timeout-minutes: 5 - run: | - git status - - set -eo pipefail - git diff | colordiff - - # will exit non-zero in case of differences - git diff --quiet - - - name: Report result to Slack - if: always() - uses: act10ns/slack@v2 - with: - webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }} - status: ${{ job.status }} - channel: '#protocol-internal' - message: 'Encoding diff (largest txns) complete! cc <@U028EMRHS3S>' - decode-transactions: needs: - ethers-encode - # Run even if ethers-encode "failed" (e.g. the MAX_ENCODED_SIZE guard tripped). - # decode-transactions only consumes the uploaded encoded-by-ethers-largest - # artifact, which is uploaded with `if: always()`, so it can still run and - # decode the transactions that were produced. !cancelled() keeps - # manual/cancel behaviour intact. + # Run even if ethers-encode "failed" (e.g. the encode-error/MAX_ENCODED_SIZE + # guard tripped). decode-transactions only consumes the uploaded + # encoded-by-ethers-largest artifact, which is uploaded with `if: always()`, + # so it can still run and decode the transactions that were produced. + # !cancelled() keeps manual/cancel behaviour intact. if: ${{ !cancelled() }} runs-on: ubuntu-26.04 steps: diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 9d4531c..e286fe7 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -11,10 +11,6 @@ version = "0.1.0" name = "encode-blocks" path = "bin/encode_blocks.rs" -[[bin]] -name = "encode-largest-txns" -path = "bin/encode_largest_txns.rs" - [dependencies] anyhow = "1.0" clap = { version = "4.6.1", features = ["derive"] } diff --git a/rust/bin/encode_largest_txns.rs b/rust/bin/encode_largest_txns.rs deleted file mode 100644 index 5d32ef6..0000000 --- a/rust/bin/encode_largest_txns.rs +++ /dev/null @@ -1,145 +0,0 @@ -use alloy::{ - primitives::B256, - providers::{Provider, ProviderBuilder, WsConnect}, - rpc::types::TransactionReceipt, -}; - -use anyhow::{anyhow, Result}; -use clap::Parser; - -use std::fs; -use std::str::FromStr; - -use usc_abi_encoding::abi::abi_encode; -use usc_abi_encoding::common::EncodingVersion; - -/// The 8 largest transactions ever observed successfully-encoded on Ethereum -/// Mainnet. The streaming `encode-blocks` binary only re-encountered blocks of -/// this size non-deterministically on the live head; this binary instead -/// queries these exact block/tx pairs directly so they are encoded (and decoded -/// downstream) on every CI run. Mainnet-only: Sepolia is intentionally -/// unsupported here. -const LARGEST_MAINNET_TXNS: &[(u64, &str)] = &[ - ( - 25602727, - "0x4e94d836e6e2794556e1cbb3a2cfb1945248d156c97b5d902835dbd9a4b88e60", - ), - ( - 25599245, - "0x24a6129734163346da53f056a8022f3ec37d70b8350ed9b8300620bbbdba6e1e", - ), - ( - 25551628, - "0x181611bff5f83dcf85cc45e06a453ee79a4ca1a697a1316030e655901c71bee8", - ), - ( - 25551622, - "0x296d83e8a0db263ad06422be8c6bd426c70785cc7c4f2b0b559eec5586e9da86", - ), - ( - 25238768, - "0x01ca130bf04e636d26ebdf0f6256a99894a6b474d4c016af74849c6a7572928d", - ), - ( - 25238750, - "0x343b91c47944693ed1cdf3c979bd7722ed9284320ff6069bcfd46c109d9c4199", - ), - ( - 25238749, - "0x5f60979ee18aba3f76122574e987f974fb1d7bacc372666f4b3f647236d54794", - ), - ( - 25238746, - "0xf2641f3bd13a111169c007205b3d1e7188201df3ae041991d2e1e3745ed1fb2d", - ), -]; - -#[derive(Parser, Debug)] -#[command(name = "encode-largest-txns")] -pub struct CliArguments { - #[arg(long, help = "WebSockets URL to an Ethereum Mainnet RPC", required = true)] - pub eth_rpc_url: String, - - #[arg(long, help = "Directory path to store JSON files", required = true)] - pub path_to_store_json: String, -} - -async fn encode_transaction( - provider: impl Provider, - tx_hash_str: &str, - rx_or_none: Option, -) -> Result { - let tx_hash = B256::from_str(tx_hash_str)?; - - let tx = provider - .get_transaction_by_hash(tx_hash) - .await? - .ok_or_else(|| anyhow!("transaction {tx_hash_str} not found via RPC"))?; - - let rx = match rx_or_none { - Some(rx) => rx, - None => provider - .get_transaction_receipt(tx_hash) - .await? - .ok_or_else(|| anyhow!("receipt for {tx_hash_str} not found via RPC"))?, - }; - - let encoded_data = abi_encode(tx, rx, EncodingVersion::V1) - .ok_or_else(|| anyhow!("abi_encode returned None for {tx_hash_str}"))?; - let as_str = hex::encode(encoded_data.abi()); - - Ok(format!("0x{as_str}")) -} - -async fn encode_and_write_to_disk( - path: &str, - provider: impl Provider, - block_number: u64, - tx_hash: &str, -) -> Result<()> { - let encoded_data = encode_transaction(provider, tx_hash, None).await?; - - // //.txt - fs::create_dir_all(format!("{path}/{block_number}"))?; - fs::write(format!("{path}/{block_number}/{tx_hash}.txt"), encoded_data + "\n")?; - - Ok(()) -} - -#[tokio::main] -async fn main() -> Result<()> { - let args = CliArguments::parse(); - - println!( - "=== encoding {} largest mainnet transactions ...", - LARGEST_MAINNET_TXNS.len() - ); - - fs::create_dir_all(args.path_to_store_json.clone())?; - - let provider = ProviderBuilder::new() - .on_ws(WsConnect::new(args.eth_rpc_url)) - .await?; - - // Encode each documented transaction directly by hash. We fail the whole - // run on any error: unlike the streaming encoder (which skips transient - // failures on live blocks), these are fixed historical fixtures and must - // always be retrievable/encodable. - for (block_number, tx_hash) in LARGEST_MAINNET_TXNS { - println!("--- encoding block {block_number} txn {tx_hash}"); - encode_and_write_to_disk( - &args.path_to_store_json, - provider.clone(), - *block_number, - tx_hash, - ) - .await?; - } - - println!( - "<<< done encoding {} transactions", - LARGEST_MAINNET_TXNS.len() - ); - - Ok(()) -} diff --git a/src/bin/encode-largest-txns.ts b/src/bin/encode-largest-txns.ts index 584737c..c867396 100644 --- a/src/bin/encode-largest-txns.ts +++ b/src/bin/encode-largest-txns.ts @@ -38,11 +38,12 @@ async function encodeTransaction( provider: WebSocketProvider, txHash: string, receipt: TransactionReceipt | null, -): Promise { +): Promise { // 80 credits const transaction = await getTransactionWithRaw(provider, txHash); if (transaction === null) { - throw new Error(`transaction ${txHash} not found via RPC`); + console.error(`ENCODE_ERROR: transaction ${txHash} not found via RPC`); + return null; } if (receipt === null) { @@ -50,7 +51,8 @@ async function encodeTransaction( receipt = await provider.getTransactionReceipt(txHash); } if (receipt === null) { - throw new Error(`receipt for ${txHash} not found via RPC`); + console.error(`ENCODE_ERROR: receipt for ${txHash} not found via RPC`); + return null; } const encodedData = abiEncode(transaction, receipt); @@ -63,14 +65,25 @@ async function encodeAndWriteToDisk( blockNumber: number, txHash: string, ): Promise { - const encodedData = await encodeTransaction(provider, txHash, null); + // Do NOT throw on failure: these are 8 independent worst-case fixtures and we + // want to see EVERY one that fails in a single run, not bail on the first. + // Errors are logged with the `ENCODE_ERROR:` prefix so a downstream CI step + // can grep for them and fail the pipeline after all txns are attempted. + let encodedData: string | null; + try { + encodedData = await encodeTransaction(provider, txHash, null); + } catch (err) { + console.error(`ENCODE_ERROR: blockNumber=${blockNumber} txHash=${txHash} threw: ${err}`); + return; + } + if (encodedData === null) { + return; + } const encodedSize = bytesInHexString(encodedData); if (encodedSize > MAX_ENCODED_SIZE) { - // Do NOT abort: we still want to encode and persist oversized transactions - // so the run continues. Log in the exact `encoded data exceeds - // MAX_ENCODED_SIZE` format so a downstream CI step can grep for it and fail - // the pipeline. + // Log in the exact `encoded data exceeds MAX_ENCODED_SIZE` format so a + // downstream CI step can grep for it and fail the pipeline. console.error( `encoded data exceeds MAX_ENCODED_SIZE: blockNumber=${blockNumber} txHash=${txHash} encodedSize=${encodedSize} bytes (max=${MAX_ENCODED_SIZE})`, ); @@ -90,10 +103,10 @@ async function encodeLargestTxns(rpcUrl: string, pathToStoreJson: string): Promi const provider = new WebSocketProvider(rpcUrl); try { - // Encode each documented transaction directly by hash. We fail the whole - // run on any error: unlike the streaming encoder (which skips transient - // failures on live blocks), these are fixed historical fixtures and must - // always be retrievable/encodable. + // Encode each documented transaction directly by hash. We deliberately do + // NOT abort on the first failure: every txn is attempted so all failures + // surface in one run. Any failure is logged with `ENCODE_ERROR:` for a + // downstream grep-based CI gate. for (const { blockNumber, txHash } of LARGEST_MAINNET_TXNS) { console.log(`--- encoding block ${blockNumber} txn ${txHash}`); await encodeAndWriteToDisk(pathToStoreJson, provider, blockNumber, txHash); From 4b033cf858cdece24becd7d7ee68ca0612aaa88f Mon Sep 17 00:00:00 2001 From: creditcoinprotoclaw Date: Mon, 27 Jul 2026 09:38:56 +0000 Subject: [PATCH 4/8] refactor: decode-blocks logs DECODE_ERROR instead of throwing on gas cap Mirror the encode-side change into decode-blocks.ts: the 4 gas-cap checks in decodeFromDisk no longer throw. Each violation is logged with a DECODE_ERROR: prefix (including the tx hash) so a single run surfaces EVERY failing large txn instead of bailing on the first. The largest-txns workflow's decode step now tees its output and a grep-based gate fails the pipeline afterwards. The '0 files found' setup guard still throws (genuine misconfiguration, not a per-txn failure). cc <@U028EMRHS3S> --- .../compare-encoding-largest-txns.yml | 23 ++++++++++++++++++- src/bin/decode-blocks.ts | 21 ++++++++++------- 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/.github/workflows/compare-encoding-largest-txns.yml b/.github/workflows/compare-encoding-largest-txns.yml index f34f187..723c2b2 100644 --- a/.github/workflows/compare-encoding-largest-txns.yml +++ b/.github/workflows/compare-encoding-largest-txns.yml @@ -129,12 +129,33 @@ jobs: - name: Decode transactions timeout-minutes: 5 run: | + set -o pipefail node dist/bin/decode-blocks.js \ wss://rpc.cc3-devnet.creditcoin.network \ https://prover.cc3-devnet.creditcoin.network \ 0xf882f3a71A36E29E86615B7055f42A68D7E4cfBB \ /var/tmp/encoded-data/ \ - 4 + 4 \ + 2>&1 | tee /var/tmp/ethers-decode.log + + - name: Fail on decode errors + if: always() + run: | + if [ ! -f /var/tmp/ethers-decode.log ]; then + echo "::error::decode log not found — cannot verify results" + exit 1 + fi + + # decode-blocks no longer throws on a gas-cap violation: it logs each + # one with a DECODE_ERROR: prefix so all failing large txns surface in + # a single run. Grep here and fail the pipeline once, after every txn + # has been decoded. + if grep -q 'DECODE_ERROR' /var/tmp/ethers-decode.log; then + echo "::error::One or more largest txns exceeded a gas cap during decode:" + grep 'DECODE_ERROR' /var/tmp/ethers-decode.log + exit 1 + fi + echo "No decode errors detected." - name: Report result to Slack if: always() diff --git a/src/bin/decode-blocks.ts b/src/bin/decode-blocks.ts index 5282f8f..6ea33ef 100644 --- a/src/bin/decode-blocks.ts +++ b/src/bin/decode-blocks.ts @@ -82,10 +82,15 @@ async function decodeFromDisk( // per-transaction cap. A single tx must fit comfortably within a block // on its own, so each estimate is checked against singleTxnGasLimit as // soon as it becomes available. + // + // NOTE: we do NOT throw on a cap violation. Each transaction is fully + // processed and any violation is logged with a `DECODE_ERROR:` prefix so a + // single run surfaces EVERY failing large txn instead of bailing on the + // first. A downstream grep-based CI gate fails the pipeline afterwards. const singleTxnGasLimit = 25_000_000n; if (gasForVerification >= singleTxnGasLimit) { - throw new Error( - `gasForVerification ${gasForVerification} reaches or exceeds the single transaction gas limit (${singleTxnGasLimit}); failing run`, + console.error( + `DECODE_ERROR: ${txHash} gasForVerification ${gasForVerification} reaches or exceeds the single transaction gas limit (${singleTxnGasLimit})`, ); } @@ -100,8 +105,8 @@ async function decodeFromDisk( const gasForDecoding = decoded.gasUsed ?? BigInt(0); console.log(` decoded as type ${decoded.type}, gasForDecoding=${gasForDecoding}`); if (gasForDecoding >= singleTxnGasLimit) { - throw new Error( - `gasForDecoding ${gasForDecoding} reaches or exceeds the single transaction gas limit (${singleTxnGasLimit}); failing run`, + console.error( + `DECODE_ERROR: ${txHash} gasForDecoding ${gasForDecoding} reaches or exceeds the single transaction gas limit (${singleTxnGasLimit})`, ); } @@ -115,13 +120,13 @@ async function decodeFromDisk( const totalGasThreshold = (blockGasLimit * 7n) / 10n; console.log(` ... totalGas (with 10% margin)=${totalGas} (threshold=${totalGasThreshold})`); if (totalGas >= singleTxnGasLimit) { - throw new Error( - `totalGas ${totalGas} reaches or exceeds the single transaction gas limit (${singleTxnGasLimit}); failing run`, + console.error( + `DECODE_ERROR: ${txHash} totalGas ${totalGas} reaches or exceeds the single transaction gas limit (${singleTxnGasLimit})`, ); } if (totalGas >= totalGasThreshold) { - throw new Error( - `totalGas ${totalGas} reaches or exceeds 70% of the ${blockGasLimit} block gas limit (${totalGasThreshold}); failing run`, + console.error( + `DECODE_ERROR: ${txHash} totalGas ${totalGas} reaches or exceeds 70% of the ${blockGasLimit} block gas limit (${totalGasThreshold})`, ); } } From 2c8b76ec756f5c42b6b4c624396e193596902988 Mon Sep 17 00:00:00 2001 From: creditcoinprotoclaw Date: Mon, 27 Jul 2026 09:39:54 +0000 Subject: [PATCH 5/8] refactor: use console.log for ENCODE_ERROR/DECODE_ERROR markers Switch the grep-able ENCODE_ERROR / DECODE_ERROR / MAX_ENCODED_SIZE log lines from console.error to console.log per review. The tee'd CI logs still capture them (2>&1) and the grep gates are unchanged. cc <@U028EMRHS3S> --- src/bin/decode-blocks.ts | 8 ++++---- src/bin/encode-largest-txns.ts | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/bin/decode-blocks.ts b/src/bin/decode-blocks.ts index 6ea33ef..581af66 100644 --- a/src/bin/decode-blocks.ts +++ b/src/bin/decode-blocks.ts @@ -89,7 +89,7 @@ async function decodeFromDisk( // first. A downstream grep-based CI gate fails the pipeline afterwards. const singleTxnGasLimit = 25_000_000n; if (gasForVerification >= singleTxnGasLimit) { - console.error( + console.log( `DECODE_ERROR: ${txHash} gasForVerification ${gasForVerification} reaches or exceeds the single transaction gas limit (${singleTxnGasLimit})`, ); } @@ -105,7 +105,7 @@ async function decodeFromDisk( const gasForDecoding = decoded.gasUsed ?? BigInt(0); console.log(` decoded as type ${decoded.type}, gasForDecoding=${gasForDecoding}`); if (gasForDecoding >= singleTxnGasLimit) { - console.error( + console.log( `DECODE_ERROR: ${txHash} gasForDecoding ${gasForDecoding} reaches or exceeds the single transaction gas limit (${singleTxnGasLimit})`, ); } @@ -120,12 +120,12 @@ async function decodeFromDisk( const totalGasThreshold = (blockGasLimit * 7n) / 10n; console.log(` ... totalGas (with 10% margin)=${totalGas} (threshold=${totalGasThreshold})`); if (totalGas >= singleTxnGasLimit) { - console.error( + console.log( `DECODE_ERROR: ${txHash} totalGas ${totalGas} reaches or exceeds the single transaction gas limit (${singleTxnGasLimit})`, ); } if (totalGas >= totalGasThreshold) { - console.error( + console.log( `DECODE_ERROR: ${txHash} totalGas ${totalGas} reaches or exceeds 70% of the ${blockGasLimit} block gas limit (${totalGasThreshold})`, ); } diff --git a/src/bin/encode-largest-txns.ts b/src/bin/encode-largest-txns.ts index c867396..37c9887 100644 --- a/src/bin/encode-largest-txns.ts +++ b/src/bin/encode-largest-txns.ts @@ -42,7 +42,7 @@ async function encodeTransaction( // 80 credits const transaction = await getTransactionWithRaw(provider, txHash); if (transaction === null) { - console.error(`ENCODE_ERROR: transaction ${txHash} not found via RPC`); + console.log(`ENCODE_ERROR: transaction ${txHash} not found via RPC`); return null; } @@ -51,7 +51,7 @@ async function encodeTransaction( receipt = await provider.getTransactionReceipt(txHash); } if (receipt === null) { - console.error(`ENCODE_ERROR: receipt for ${txHash} not found via RPC`); + console.log(`ENCODE_ERROR: receipt for ${txHash} not found via RPC`); return null; } @@ -73,7 +73,7 @@ async function encodeAndWriteToDisk( try { encodedData = await encodeTransaction(provider, txHash, null); } catch (err) { - console.error(`ENCODE_ERROR: blockNumber=${blockNumber} txHash=${txHash} threw: ${err}`); + console.log(`ENCODE_ERROR: blockNumber=${blockNumber} txHash=${txHash} threw: ${err}`); return; } if (encodedData === null) { @@ -84,7 +84,7 @@ async function encodeAndWriteToDisk( if (encodedSize > MAX_ENCODED_SIZE) { // Log in the exact `encoded data exceeds MAX_ENCODED_SIZE` format so a // downstream CI step can grep for it and fail the pipeline. - console.error( + console.log( `encoded data exceeds MAX_ENCODED_SIZE: blockNumber=${blockNumber} txHash=${txHash} encodedSize=${encodedSize} bytes (max=${MAX_ENCODED_SIZE})`, ); } From 0d32c18160fe4bf0d2d9a1e7d9e2a8b7d5f83c1c Mon Sep 17 00:00:00 2001 From: creditcoinprotoclaw Date: Mon, 27 Jul 2026 09:40:36 +0000 Subject: [PATCH 6/8] ci: disable compare-encoding-ethereum/sepolia on pull_request Remove the pull_request trigger from both existing stream-encode workflows on this branch. The new compare-encoding-largest-txns workflow covers PR-time encode/decode verification, so running the full mainnet/Sepolia stream encoders on every PR is redundant and slow. Both still run on their 12h schedule and via workflow_dispatch. cc <@U028EMRHS3S> --- .github/workflows/compare-encoding-ethereum.yml | 11 ++++------- .github/workflows/compare-encoding-sepolia.yml | 11 ++++------- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/.github/workflows/compare-encoding-ethereum.yml b/.github/workflows/compare-encoding-ethereum.yml index 7a25507..80c1a2e 100644 --- a/.github/workflows/compare-encoding-ethereum.yml +++ b/.github/workflows/compare-encoding-ethereum.yml @@ -2,13 +2,10 @@ name: compare-encoding-ethereum 'on': - pull_request: - paths: - - '**.rs' - - '**.ts' - - '**Cargo**' - - '**package*.json' - - '.github/workflows/compare-encoding-*.yml' + # NOTE: pull_request trigger intentionally disabled on this branch. The new + # compare-encoding-largest-txns workflow covers PR-time encode/decode + # verification; running the full mainnet stream encode on every PR here is + # redundant + slow. Still runs on schedule and can be triggered manually. schedule: - cron: '12 */12 * * *' workflow_dispatch: diff --git a/.github/workflows/compare-encoding-sepolia.yml b/.github/workflows/compare-encoding-sepolia.yml index 1c26cae..1ad69cb 100644 --- a/.github/workflows/compare-encoding-sepolia.yml +++ b/.github/workflows/compare-encoding-sepolia.yml @@ -2,13 +2,10 @@ name: compare-encoding-sepolia 'on': - pull_request: - paths: - - '**.rs' - - '**.ts' - - '**Cargo**' - - '**package*.json' - - '.github/workflows/compare-encoding-*.yml' + # NOTE: pull_request trigger intentionally disabled on this branch. The new + # compare-encoding-largest-txns workflow covers PR-time encode/decode + # verification; running the full Sepolia stream encode on every PR here is + # redundant + slow. Still runs on schedule and can be triggered manually. schedule: - cron: '12 */12 * * *' workflow_dispatch: From 3f20e384091c00cd41345e984ef8f1a43e8ecc29 Mon Sep 17 00:00:00 2001 From: creditcoinprotoclaw Date: Mon, 27 Jul 2026 09:44:26 +0000 Subject: [PATCH 7/8] fix: restore mangled GOOGLE_ETHEREUM_RPC_KEY secret ref in workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ethers-encode RPC URL had a corrupted secret reference: rpc?key=*** secrets.GOOGLE_ETHEREUM_RPC_KEY }} instead of: rpc?key=${{ secrets.GOOGLE_ETHEREUM_RPC_KEY }} With the '${{' eaten, the encoder connected to the Google WS endpoint with key='***' (and a stray shell arg), so the very first getTransaction request hung with no response until the 5-min step timeout — which is exactly why the job logged only the first txn and then died with no error. Restoring the interpolation fixes it. cc <@U028EMRHS3S> --- .github/workflows/compare-encoding-largest-txns.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/compare-encoding-largest-txns.yml b/.github/workflows/compare-encoding-largest-txns.yml index 723c2b2..053a454 100644 --- a/.github/workflows/compare-encoding-largest-txns.yml +++ b/.github/workflows/compare-encoding-largest-txns.yml @@ -44,7 +44,7 @@ jobs: run: | set -o pipefail node dist/bin/encode-largest-txns.js \ - wss://blockchain.googleapis.com/v1/projects/creditcoin-test/locations/us-central1/endpoints/ethereum-mainnet/rpc?key=*** secrets.GOOGLE_ETHEREUM_RPC_KEY }} \ + wss://blockchain.googleapis.com/v1/projects/creditcoin-test/locations/us-central1/endpoints/ethereum-mainnet/rpc?key=${{ secrets.GOOGLE_ETHEREUM_RPC_KEY }} \ /var/tmp/encoded-data/ethers/ \ 2>&1 | tee /var/tmp/ethers-encode.log From f82c9666809f5227b2791326b9a9797f1f4fe26e Mon Sep 17 00:00:00 2001 From: creditcoinprotoclaw Date: Mon, 27 Jul 2026 09:54:04 +0000 Subject: [PATCH 8/8] refactor: fold largest-txns mode into encode-blocks.ts, drop separate bin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per review: drop the separate encode-largest-txns.ts and put the direct-query behaviour into encode-blocks.ts behind an ENCODE_LARGEST_TXNS env flag. - encode-blocks.ts: add LARGEST_MAINNET_TXNS + encodeLargestTxns(); dispatch to it when ENCODE_LARGEST_TXNS is set, else stream as before. encodeTransaction now logs ENCODE_ERROR: and returns null instead of asserting non-null; encodeAndWriteToDisk skips on null. MAX_ENCODED_SIZE guard switched to console.log. No throw in the largest-txns loop — every fixture is attempted. - workflow: call encode-blocks.js with ENCODE_LARGEST_TXNS=1 instead of the deleted encode-largest-txns.js. Smoke-tested locally against a public mainnet WS: all 8 txns encode, 8 files written. cc <@U028EMRHS3S> --- .../compare-encoding-largest-txns.yml | 4 +- src/bin/encode-blocks.ts | 71 +++++++++- src/bin/encode-largest-txns.ts | 131 ------------------ 3 files changed, 70 insertions(+), 136 deletions(-) delete mode 100644 src/bin/encode-largest-txns.ts diff --git a/.github/workflows/compare-encoding-largest-txns.yml b/.github/workflows/compare-encoding-largest-txns.yml index 053a454..abe3d6b 100644 --- a/.github/workflows/compare-encoding-largest-txns.yml +++ b/.github/workflows/compare-encoding-largest-txns.yml @@ -41,9 +41,11 @@ jobs: - name: Encode the 8 largest mainnet transactions timeout-minutes: 5 + env: + ENCODE_LARGEST_TXNS: '1' run: | set -o pipefail - node dist/bin/encode-largest-txns.js \ + node dist/bin/encode-blocks.js \ wss://blockchain.googleapis.com/v1/projects/creditcoin-test/locations/us-central1/endpoints/ethereum-mainnet/rpc?key=${{ secrets.GOOGLE_ETHEREUM_RPC_KEY }} \ /var/tmp/encoded-data/ethers/ \ 2>&1 | tee /var/tmp/ethers-encode.log diff --git a/src/bin/encode-blocks.ts b/src/bin/encode-blocks.ts index 693fb40..3e8a9a1 100644 --- a/src/bin/encode-blocks.ts +++ b/src/bin/encode-blocks.ts @@ -16,6 +16,22 @@ import { bytesInHexString } from '../utils/hex'; // block 25238746, tx 0xf2641f3bd13a111169c007205b3d1e7188201df3ae041991d2e1e3745ed1fb2d const MAX_ENCODED_SIZE = 530336; +// The same 8 largest transactions documented above, as structured block/tx +// pairs. When ENCODE_LARGEST_TXNS is set, encode-blocks queries these exact +// transactions directly by hash instead of streaming the live chain head. This +// gives a deterministic worst-case coverage check every CI run rather than +// hoping to re-encounter blocks of this size on mainnet. Mainnet-only. +const LARGEST_MAINNET_TXNS: Array<{ blockNumber: number; txHash: string }> = [ + { blockNumber: 25602727, txHash: '0x4e94d836e6e2794556e1cbb3a2cfb1945248d156c97b5d902835dbd9a4b88e60' }, + { blockNumber: 25599245, txHash: '0x24a6129734163346da53f056a8022f3ec37d70b8350ed9b8300620bbbdba6e1e' }, + { blockNumber: 25551628, txHash: '0x181611bff5f83dcf85cc45e06a453ee79a4ca1a697a1316030e655901c71bee8' }, + { blockNumber: 25551622, txHash: '0x296d83e8a0db263ad06422be8c6bd426c70785cc7c4f2b0b559eec5586e9da86' }, + { blockNumber: 25238768, txHash: '0x01ca130bf04e636d26ebdf0f6256a99894a6b474d4c016af74849c6a7572928d' }, + { blockNumber: 25238750, txHash: '0x343b91c47944693ed1cdf3c979bd7722ed9284320ff6069bcfd46c109d9c4199' }, + { blockNumber: 25238749, txHash: '0x5f60979ee18aba3f76122574e987f974fb1d7bacc372666f4b3f647236d54794' }, + { blockNumber: 25238746, txHash: '0xf2641f3bd13a111169c007205b3d1e7188201df3ae041991d2e1e3745ed1fb2d' }, +]; + /** * Gets all transaction receipts for a given block using regular Infura/compatible RPC format. * Uses hexadecimal block number format (e.g., "0x1a2b3c"). @@ -45,15 +61,24 @@ async function encodeTransaction( provider: WebSocketProvider, txHash: string, receipt: TransactionReceipt | null, -): Promise { +): Promise { // 80 credits const transaction = await getTransactionWithRaw(provider, txHash); + if (transaction === null) { + console.log(`ENCODE_ERROR: transaction ${txHash} not found via RPC`); + return null; + } if (receipt === null) { // 80 credits receipt = await provider.getTransactionReceipt(txHash); } - const encodedData = abiEncode(transaction!, receipt!); + if (receipt === null) { + console.log(`ENCODE_ERROR: receipt for ${txHash} not found via RPC`); + return null; + } + + const encodedData = abiEncode(transaction, receipt); return encodedData.abi; } @@ -65,13 +90,18 @@ async function encodeAndWriteToDisk( receipt: TransactionReceipt | null, ) { const encodedData = await encodeTransaction(provider, txHash, receipt); + if (encodedData === null) { + // encodeTransaction already logged an ENCODE_ERROR: for this txn; skip + // writing anything and let the run continue. + return; + } const encodedSize = bytesInHexString(encodedData); if (encodedSize > MAX_ENCODED_SIZE) { // Do NOT abort: we still want to encode and persist oversized blocks so the // run continues. Log in the exact `encoded data exceeds MAX_ENCODED_SIZE` // format so a downstream CI step can grep for it and fail the pipeline. - console.error( + console.log( `encoded data exceeds MAX_ENCODED_SIZE: blockNumber=${blockNumber} txHash=${txHash} encodedSize=${encodedSize} bytes (max=${MAX_ENCODED_SIZE})`, ); } @@ -147,15 +177,48 @@ async function encodeBlocks(rpcUrl: string, pathToStoreJson: string): Promise { + console.log(`=== encoding ${LARGEST_MAINNET_TXNS.length} largest mainnet transactions ...`); + + mkdirSync(pathToStoreJson, { recursive: true }); + + const provider = new WebSocketProvider(rpcUrl); + + try { + for (const { blockNumber, txHash } of LARGEST_MAINNET_TXNS) { + console.log(`--- encoding block ${blockNumber} txn ${txHash}`); + mkdirSync(`${pathToStoreJson}/${blockNumber}`, { recursive: true }); + try { + await encodeAndWriteToDisk(pathToStoreJson, provider, blockNumber, txHash, null); + } catch (err) { + console.log(`ENCODE_ERROR: blockNumber=${blockNumber} txHash=${txHash} threw: ${err}`); + } + } + console.log(`<<< done encoding ${LARGEST_MAINNET_TXNS.length} transactions`); + } finally { + await provider.destroy(); + } +} + if (process.argv.length < 4) { console.error('node dist/bin/encode-blocks.js '); + console.error(' set ENCODE_LARGEST_TXNS=1 to encode the fixed largest-known mainnet txns instead of streaming'); process.exit(1); } const rpcUrl = process.argv[2] || 'ws://127.0.0.1:8545'; const pathToStoreJson = process.argv[3]; -encodeBlocks(rpcUrl, pathToStoreJson).catch((reason) => { +// When ENCODE_LARGEST_TXNS is set, query the fixed largest-known mainnet txns +// directly (deterministic worst-case coverage); otherwise stream the live head. +const run = process.env.ENCODE_LARGEST_TXNS ? encodeLargestTxns : encodeBlocks; + +run(rpcUrl, pathToStoreJson).catch((reason) => { console.error(reason); process.exit(1); }); diff --git a/src/bin/encode-largest-txns.ts b/src/bin/encode-largest-txns.ts deleted file mode 100644 index 37c9887..0000000 --- a/src/bin/encode-largest-txns.ts +++ /dev/null @@ -1,131 +0,0 @@ -import { mkdirSync, writeFileSync } from 'fs'; -import { WebSocketProvider, TransactionReceipt } from 'ethers'; -import { abiEncode } from '../encoding/abi'; -import { getTransactionWithRaw } from '../encoding'; -import { bytesInHexString } from '../utils/hex'; - -// The 8 largest transactions ever observed successfully-encoded on Ethereum -// Mainnet. Historically these were only documented as a comment inside -// encode-blocks.ts; the streaming encoder relied on eventually re-encountering -// blocks of this size on the live head. That is slow and non-deterministic. -// -// This binary instead queries these exact block/tx pairs directly so they can -// be encoded (and decoded downstream) on every CI run, giving a deterministic -// worst-case coverage check. Sepolia is intentionally unsupported here: these -// are mainnet-only fixtures. -interface LargeTxn { - blockNumber: number; - txHash: string; -} - -const LARGEST_MAINNET_TXNS: LargeTxn[] = [ - { blockNumber: 25602727, txHash: '0x4e94d836e6e2794556e1cbb3a2cfb1945248d156c97b5d902835dbd9a4b88e60' }, - { blockNumber: 25599245, txHash: '0x24a6129734163346da53f056a8022f3ec37d70b8350ed9b8300620bbbdba6e1e' }, - { blockNumber: 25551628, txHash: '0x181611bff5f83dcf85cc45e06a453ee79a4ca1a697a1316030e655901c71bee8' }, - { blockNumber: 25551622, txHash: '0x296d83e8a0db263ad06422be8c6bd426c70785cc7c4f2b0b559eec5586e9da86' }, - { blockNumber: 25238768, txHash: '0x01ca130bf04e636d26ebdf0f6256a99894a6b474d4c016af74849c6a7572928d' }, - { blockNumber: 25238750, txHash: '0x343b91c47944693ed1cdf3c979bd7722ed9284320ff6069bcfd46c109d9c4199' }, - { blockNumber: 25238749, txHash: '0x5f60979ee18aba3f76122574e987f974fb1d7bacc372666f4b3f647236d54794' }, - { blockNumber: 25238746, txHash: '0xf2641f3bd13a111169c007205b3d1e7188201df3ae041991d2e1e3745ed1fb2d' }, -]; - -// Maximum discovered size of ABI-encoded transaction data, in bytes. -// Derived from the largest observed successfully-encoded transactions above. -const MAX_ENCODED_SIZE = 530336; - -// cost 80 or 160 credits depending on arguments -async function encodeTransaction( - provider: WebSocketProvider, - txHash: string, - receipt: TransactionReceipt | null, -): Promise { - // 80 credits - const transaction = await getTransactionWithRaw(provider, txHash); - if (transaction === null) { - console.log(`ENCODE_ERROR: transaction ${txHash} not found via RPC`); - return null; - } - - if (receipt === null) { - // 80 credits - receipt = await provider.getTransactionReceipt(txHash); - } - if (receipt === null) { - console.log(`ENCODE_ERROR: receipt for ${txHash} not found via RPC`); - return null; - } - - const encodedData = abiEncode(transaction, receipt); - return encodedData.abi; -} - -async function encodeAndWriteToDisk( - pathToStoreJson: string, - provider: WebSocketProvider, - blockNumber: number, - txHash: string, -): Promise { - // Do NOT throw on failure: these are 8 independent worst-case fixtures and we - // want to see EVERY one that fails in a single run, not bail on the first. - // Errors are logged with the `ENCODE_ERROR:` prefix so a downstream CI step - // can grep for them and fail the pipeline after all txns are attempted. - let encodedData: string | null; - try { - encodedData = await encodeTransaction(provider, txHash, null); - } catch (err) { - console.log(`ENCODE_ERROR: blockNumber=${blockNumber} txHash=${txHash} threw: ${err}`); - return; - } - if (encodedData === null) { - return; - } - - const encodedSize = bytesInHexString(encodedData); - if (encodedSize > MAX_ENCODED_SIZE) { - // Log in the exact `encoded data exceeds MAX_ENCODED_SIZE` format so a - // downstream CI step can grep for it and fail the pipeline. - console.log( - `encoded data exceeds MAX_ENCODED_SIZE: blockNumber=${blockNumber} txHash=${txHash} encodedSize=${encodedSize} bytes (max=${MAX_ENCODED_SIZE})`, - ); - } - - mkdirSync(`${pathToStoreJson}/${blockNumber}`, { recursive: true }); - writeFileSync(`${pathToStoreJson}/${blockNumber}/${txHash}.txt`, encodedData + '\n', { - flag: 'w', - }); -} - -async function encodeLargestTxns(rpcUrl: string, pathToStoreJson: string): Promise { - console.log(`=== encoding ${LARGEST_MAINNET_TXNS.length} largest mainnet transactions ...`); - - mkdirSync(pathToStoreJson, { recursive: true }); - - const provider = new WebSocketProvider(rpcUrl); - - try { - // Encode each documented transaction directly by hash. We deliberately do - // NOT abort on the first failure: every txn is attempted so all failures - // surface in one run. Any failure is logged with `ENCODE_ERROR:` for a - // downstream grep-based CI gate. - for (const { blockNumber, txHash } of LARGEST_MAINNET_TXNS) { - console.log(`--- encoding block ${blockNumber} txn ${txHash}`); - await encodeAndWriteToDisk(pathToStoreJson, provider, blockNumber, txHash); - } - console.log(`<<< done encoding ${LARGEST_MAINNET_TXNS.length} transactions`); - } finally { - await provider.destroy(); - } -} - -if (process.argv.length < 4) { - console.error('node dist/bin/encode-largest-txns.js '); - process.exit(1); -} - -const rpcUrl = process.argv[2] || 'ws://127.0.0.1:8545'; -const pathToStoreJson = process.argv[3]; - -encodeLargestTxns(rpcUrl, pathToStoreJson).catch((reason) => { - console.error(reason); - process.exit(1); -});