diff --git a/.github/workflows/compare-encoding-ethereum.yml b/.github/workflows/compare-encoding-ethereum.yml index 7a25507..80c1a2e 100644 --- a/.github/workflows/compare-encoding-ethereum.yml +++ b/.github/workflows/compare-encoding-ethereum.yml @@ -2,13 +2,10 @@ name: compare-encoding-ethereum 'on': - pull_request: - paths: - - '**.rs' - - '**.ts' - - '**Cargo**' - - '**package*.json' - - '.github/workflows/compare-encoding-*.yml' + # NOTE: pull_request trigger intentionally disabled on this branch. The new + # compare-encoding-largest-txns workflow covers PR-time encode/decode + # verification; running the full mainnet stream encode on every PR here is + # redundant + slow. Still runs on schedule and can be triggered manually. schedule: - cron: '12 */12 * * *' workflow_dispatch: diff --git a/.github/workflows/compare-encoding-largest-txns.yml b/.github/workflows/compare-encoding-largest-txns.yml new file mode 100644 index 0000000..abe3d6b --- /dev/null +++ b/.github/workflows/compare-encoding-largest-txns.yml @@ -0,0 +1,169 @@ +--- +name: compare-encoding-largest-txns + +'on': + pull_request: + paths: + - '**.rs' + - '**.ts' + - '**Cargo**' + - '**package*.json' + - '.github/workflows/compare-encoding-*.yml' + schedule: + - cron: '42 */12 * * *' + workflow_dispatch: + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +permissions: read-all + +jobs: + ethers-encode: + runs-on: ubuntu-26.04 + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-node@v7 + with: + node-version: '24.x' + cache: 'npm' + + - run: npm ci + - run: npm run build --if-present + + - name: DEBUG - Print runner IP address + run: | + # see https://opensource.com/article/18/5/how-find-ip-address-linux + IP_ADDRESS=$(curl https://ifconfig.me) + echo "INFO: IP_ADDRESS=$IP_ADDRESS" + + - name: Encode the 8 largest mainnet transactions + timeout-minutes: 5 + env: + ENCODE_LARGEST_TXNS: '1' + run: | + set -o pipefail + node dist/bin/encode-blocks.js \ + wss://blockchain.googleapis.com/v1/projects/creditcoin-test/locations/us-central1/endpoints/ethereum-mainnet/rpc?key=${{ secrets.GOOGLE_ETHEREUM_RPC_KEY }} \ + /var/tmp/encoded-data/ethers/ \ + 2>&1 | tee /var/tmp/ethers-encode.log + + - name: Fail on encode errors or MAX_ENCODED_SIZE + if: always() + run: | + if [ ! -f /var/tmp/ethers-encode.log ]; then + echo "::error::encode log not found — cannot verify results" + exit 1 + fi + + # The encoder never throws: it logs every failure so all 8 fixtures + # get attempted in one run. Grep for both failure markers here and + # fail the pipeline once, after everything has been attempted. + FAILED=0 + if grep -q 'ENCODE_ERROR' /var/tmp/ethers-encode.log; then + echo "::error::One or more largest txns failed to encode:" + grep 'ENCODE_ERROR' /var/tmp/ethers-encode.log + FAILED=1 + fi + if grep -q 'MAX_ENCODED_SIZE' /var/tmp/ethers-encode.log; then + echo "::error::One or more transactions exceeded MAX_ENCODED_SIZE:" + grep 'MAX_ENCODED_SIZE' /var/tmp/ethers-encode.log + FAILED=1 + fi + if [ "$FAILED" -ne 0 ]; then + exit 1 + fi + echo "No encode errors detected." + + - name: Add notices in case of failure + if: failure() && github.event_name == 'pull_request' + run: | + # note: GitHub appears to display these in reverse order + echo "::notice::SUGGESTION: Trigger this workflow manually against the same branch to verify the changes." + echo "::notice::IMPORTANT: Secret credentials may be missing for PRs from Dependabot!" + + - name: Report result to Slack + if: failure() && github.event_name == 'schedule' + uses: act10ns/slack@v2 + with: + webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }} + status: ${{ job.status }} + channel: '#protocol-internal' + message: 'Ethers encode (largest txns) failed! cc <@U028EMRHS3S>' + + - name: Upload encoded data + if: always() + uses: actions/upload-artifact@v7 + with: + name: encoded-by-ethers-largest + path: /var/tmp/encoded-data/ethers + include-hidden-files: true + + decode-transactions: + needs: + - ethers-encode + # Run even if ethers-encode "failed" (e.g. the encode-error/MAX_ENCODED_SIZE + # guard tripped). decode-transactions only consumes the uploaded + # encoded-by-ethers-largest artifact, which is uploaded with `if: always()`, + # so it can still run and decode the transactions that were produced. + # !cancelled() keeps manual/cancel behaviour intact. + if: ${{ !cancelled() }} + runs-on: ubuntu-26.04 + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-node@v7 + with: + node-version: '24.x' + cache: 'npm' + + - run: npm ci + - run: npm run build --if-present + + - name: Download encoded data from ethers + uses: actions/download-artifact@v8 + with: + path: /var/tmp/encoded-data + pattern: encoded-by-ethers-largest + + - name: Decode transactions + timeout-minutes: 5 + run: | + set -o pipefail + node dist/bin/decode-blocks.js \ + wss://rpc.cc3-devnet.creditcoin.network \ + https://prover.cc3-devnet.creditcoin.network \ + 0xf882f3a71A36E29E86615B7055f42A68D7E4cfBB \ + /var/tmp/encoded-data/ \ + 4 \ + 2>&1 | tee /var/tmp/ethers-decode.log + + - name: Fail on decode errors + if: always() + run: | + if [ ! -f /var/tmp/ethers-decode.log ]; then + echo "::error::decode log not found — cannot verify results" + exit 1 + fi + + # decode-blocks no longer throws on a gas-cap violation: it logs each + # one with a DECODE_ERROR: prefix so all failing large txns surface in + # a single run. Grep here and fail the pipeline once, after every txn + # has been decoded. + if grep -q 'DECODE_ERROR' /var/tmp/ethers-decode.log; then + echo "::error::One or more largest txns exceeded a gas cap during decode:" + grep 'DECODE_ERROR' /var/tmp/ethers-decode.log + exit 1 + fi + echo "No decode errors detected." + + - name: Report result to Slack + if: always() + uses: act10ns/slack@v2 + with: + webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }} + status: ${{ job.status }} + channel: '#protocol-internal' + message: 'Decoding (largest txns) complete! cc <@U028EMRHS3S>' diff --git a/.github/workflows/compare-encoding-sepolia.yml b/.github/workflows/compare-encoding-sepolia.yml index 1c26cae..1ad69cb 100644 --- a/.github/workflows/compare-encoding-sepolia.yml +++ b/.github/workflows/compare-encoding-sepolia.yml @@ -2,13 +2,10 @@ name: compare-encoding-sepolia 'on': - pull_request: - paths: - - '**.rs' - - '**.ts' - - '**Cargo**' - - '**package*.json' - - '.github/workflows/compare-encoding-*.yml' + # NOTE: pull_request trigger intentionally disabled on this branch. The new + # compare-encoding-largest-txns workflow covers PR-time encode/decode + # verification; running the full Sepolia stream encode on every PR here is + # redundant + slow. Still runs on schedule and can be triggered manually. schedule: - cron: '12 */12 * * *' workflow_dispatch: diff --git a/src/bin/decode-blocks.ts b/src/bin/decode-blocks.ts index 097fc6a..581af66 100644 --- a/src/bin/decode-blocks.ts +++ b/src/bin/decode-blocks.ts @@ -78,6 +78,22 @@ async function decodeFromDisk( } console.log(` ... gasForVerification=${gasForVerification} - 0 means skipped`); + // Reject any single transaction whose individual gas cost crosses the + // per-transaction cap. A single tx must fit comfortably within a block + // on its own, so each estimate is checked against singleTxnGasLimit as + // soon as it becomes available. + // + // NOTE: we do NOT throw on a cap violation. Each transaction is fully + // processed and any violation is logged with a `DECODE_ERROR:` prefix so a + // single run surfaces EVERY failing large txn instead of bailing on the + // first. A downstream grep-based CI gate fails the pipeline afterwards. + const singleTxnGasLimit = 25_000_000n; + if (gasForVerification >= singleTxnGasLimit) { + console.log( + `DECODE_ERROR: ${txHash} gasForVerification ${gasForVerification} reaches or exceeds the single transaction gas limit (${singleTxnGasLimit})`, + ); + } + const encodedData = readFileSync(pathToTxn, { encoding: 'utf8', flag: 'r', @@ -88,6 +104,11 @@ async function decodeFromDisk( }); const gasForDecoding = decoded.gasUsed ?? BigInt(0); console.log(` decoded as type ${decoded.type}, gasForDecoding=${gasForDecoding}`); + if (gasForDecoding >= singleTxnGasLimit) { + console.log( + `DECODE_ERROR: ${txHash} gasForDecoding ${gasForDecoding} reaches or exceeds the single transaction gas limit (${singleTxnGasLimit})`, + ); + } // Add a 10% safety margin to the raw estimates and reject if the // combined cost crosses 70% of the 75M block gas limit. Using bigint @@ -98,9 +119,14 @@ async function decodeFromDisk( const blockGasLimit = 75_000_000n; const totalGasThreshold = (blockGasLimit * 7n) / 10n; console.log(` ... totalGas (with 10% margin)=${totalGas} (threshold=${totalGasThreshold})`); + if (totalGas >= singleTxnGasLimit) { + console.log( + `DECODE_ERROR: ${txHash} totalGas ${totalGas} reaches or exceeds the single transaction gas limit (${singleTxnGasLimit})`, + ); + } if (totalGas >= totalGasThreshold) { - throw new Error( - `totalGas ${totalGas} reaches or exceeds 70% of the ${blockGasLimit} block gas limit (${totalGasThreshold}); failing run`, + console.log( + `DECODE_ERROR: ${txHash} totalGas ${totalGas} reaches or exceeds 70% of the ${blockGasLimit} block gas limit (${totalGasThreshold})`, ); } } diff --git a/src/bin/encode-blocks.ts b/src/bin/encode-blocks.ts index 693fb40..3e8a9a1 100644 --- a/src/bin/encode-blocks.ts +++ b/src/bin/encode-blocks.ts @@ -16,6 +16,22 @@ import { bytesInHexString } from '../utils/hex'; // block 25238746, tx 0xf2641f3bd13a111169c007205b3d1e7188201df3ae041991d2e1e3745ed1fb2d const MAX_ENCODED_SIZE = 530336; +// The same 8 largest transactions documented above, as structured block/tx +// pairs. When ENCODE_LARGEST_TXNS is set, encode-blocks queries these exact +// transactions directly by hash instead of streaming the live chain head. This +// gives a deterministic worst-case coverage check every CI run rather than +// hoping to re-encounter blocks of this size on mainnet. Mainnet-only. +const LARGEST_MAINNET_TXNS: Array<{ blockNumber: number; txHash: string }> = [ + { blockNumber: 25602727, txHash: '0x4e94d836e6e2794556e1cbb3a2cfb1945248d156c97b5d902835dbd9a4b88e60' }, + { blockNumber: 25599245, txHash: '0x24a6129734163346da53f056a8022f3ec37d70b8350ed9b8300620bbbdba6e1e' }, + { blockNumber: 25551628, txHash: '0x181611bff5f83dcf85cc45e06a453ee79a4ca1a697a1316030e655901c71bee8' }, + { blockNumber: 25551622, txHash: '0x296d83e8a0db263ad06422be8c6bd426c70785cc7c4f2b0b559eec5586e9da86' }, + { blockNumber: 25238768, txHash: '0x01ca130bf04e636d26ebdf0f6256a99894a6b474d4c016af74849c6a7572928d' }, + { blockNumber: 25238750, txHash: '0x343b91c47944693ed1cdf3c979bd7722ed9284320ff6069bcfd46c109d9c4199' }, + { blockNumber: 25238749, txHash: '0x5f60979ee18aba3f76122574e987f974fb1d7bacc372666f4b3f647236d54794' }, + { blockNumber: 25238746, txHash: '0xf2641f3bd13a111169c007205b3d1e7188201df3ae041991d2e1e3745ed1fb2d' }, +]; + /** * Gets all transaction receipts for a given block using regular Infura/compatible RPC format. * Uses hexadecimal block number format (e.g., "0x1a2b3c"). @@ -45,15 +61,24 @@ async function encodeTransaction( provider: WebSocketProvider, txHash: string, receipt: TransactionReceipt | null, -): Promise { +): Promise { // 80 credits const transaction = await getTransactionWithRaw(provider, txHash); + if (transaction === null) { + console.log(`ENCODE_ERROR: transaction ${txHash} not found via RPC`); + return null; + } if (receipt === null) { // 80 credits receipt = await provider.getTransactionReceipt(txHash); } - const encodedData = abiEncode(transaction!, receipt!); + if (receipt === null) { + console.log(`ENCODE_ERROR: receipt for ${txHash} not found via RPC`); + return null; + } + + const encodedData = abiEncode(transaction, receipt); return encodedData.abi; } @@ -65,13 +90,18 @@ async function encodeAndWriteToDisk( receipt: TransactionReceipt | null, ) { const encodedData = await encodeTransaction(provider, txHash, receipt); + if (encodedData === null) { + // encodeTransaction already logged an ENCODE_ERROR: for this txn; skip + // writing anything and let the run continue. + return; + } const encodedSize = bytesInHexString(encodedData); if (encodedSize > MAX_ENCODED_SIZE) { // Do NOT abort: we still want to encode and persist oversized blocks so the // run continues. Log in the exact `encoded data exceeds MAX_ENCODED_SIZE` // format so a downstream CI step can grep for it and fail the pipeline. - console.error( + console.log( `encoded data exceeds MAX_ENCODED_SIZE: blockNumber=${blockNumber} txHash=${txHash} encodedSize=${encodedSize} bytes (max=${MAX_ENCODED_SIZE})`, ); } @@ -147,15 +177,48 @@ async function encodeBlocks(rpcUrl: string, pathToStoreJson: string): Promise { + console.log(`=== encoding ${LARGEST_MAINNET_TXNS.length} largest mainnet transactions ...`); + + mkdirSync(pathToStoreJson, { recursive: true }); + + const provider = new WebSocketProvider(rpcUrl); + + try { + for (const { blockNumber, txHash } of LARGEST_MAINNET_TXNS) { + console.log(`--- encoding block ${blockNumber} txn ${txHash}`); + mkdirSync(`${pathToStoreJson}/${blockNumber}`, { recursive: true }); + try { + await encodeAndWriteToDisk(pathToStoreJson, provider, blockNumber, txHash, null); + } catch (err) { + console.log(`ENCODE_ERROR: blockNumber=${blockNumber} txHash=${txHash} threw: ${err}`); + } + } + console.log(`<<< done encoding ${LARGEST_MAINNET_TXNS.length} transactions`); + } finally { + await provider.destroy(); + } +} + if (process.argv.length < 4) { console.error('node dist/bin/encode-blocks.js '); + console.error(' set ENCODE_LARGEST_TXNS=1 to encode the fixed largest-known mainnet txns instead of streaming'); process.exit(1); } const rpcUrl = process.argv[2] || 'ws://127.0.0.1:8545'; const pathToStoreJson = process.argv[3]; -encodeBlocks(rpcUrl, pathToStoreJson).catch((reason) => { +// When ENCODE_LARGEST_TXNS is set, query the fixed largest-known mainnet txns +// directly (deterministic worst-case coverage); otherwise stream the live head. +const run = process.env.ENCODE_LARGEST_TXNS ? encodeLargestTxns : encodeBlocks; + +run(rpcUrl, pathToStoreJson).catch((reason) => { console.error(reason); process.exit(1); });