diff --git a/docs/runbook.md b/docs/runbook.md index dedba82f..0770b004 100644 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -97,6 +97,7 @@ Recovery semantics to expect: | Connection-pool exhaustion errors | `PG_MAX_CONNECTIONS` × replicas > DB capacity | lower pool size or raise DB `max_connections` | | Memory growth / OOM kills | heavy result sets or a leak | lower `BLOCK_RANGE_SIZE`; inspect heap metrics; cap container memory | | Startup exits immediately | invalid config | read the startup error — config is validated fail-fast (missing `PG_CONN`, bad `PORT`, etc.) | +| After a hard fork, `inBestChain: true` / events / actions stop at the fork block while `networkState.pendingMaxBlockHeight` keeps rising | the abandoned chain's blocks sit ABOVE the new tip and the archive daemon takes a while to relabel them `orphaned`; until then they are the highest `pending` blocks | expected for the relabelling window (about 30 min on mainnet at the Mesa upgrade); once they read `orphaned` the walk anchors on the live tip by itself (`src/db/sql/best-chain.ts`). If it persists, inspect `SELECT chain_status, MAX(height) FROM blocks GROUP BY 1` | ## Deploys & rollback diff --git a/src/db/sql/best-chain.ts b/src/db/sql/best-chain.ts new file mode 100644 index 00000000..1b0d58c9 --- /dev/null +++ b/src/db/sql/best-chain.ts @@ -0,0 +1,34 @@ +import type postgres from 'postgres'; + +/** + * The height of the best chain's tip — the anchor every "walk back from the tip" + * recursive CTE starts from. + * + * This is deliberately NOT `(SELECT MAX(height) FROM blocks)`. The archive keeps every + * block it ever saw, so after a hard fork the abandoned chain's blocks stay in the table + * ABOVE the new chain's tip, marked `orphaned`. A global max then lands on a dead block: + * with the `chain_status = 'pending'` guard the anchor matches nothing and the walk + * silently degrades to canonical-only rows; without the guard the walk follows the dead + * chain. Either way the live tip and everything the network produced since the fork is + * invisible to `inBestChain`, events, actions and action-state resolution. + * + * Seen on mainnet at the Mesa upgrade (2026-09-03): the old chain reached height 548187 + * before stopping, the fork block was 548147, and for the whole first hour + * `blocks(inBestChain: true)` returned nothing above 548147 while `networkState` + * correctly reported a pending tip of 548164 — the two disagreed because only one of + * them excluded orphans. + * + * Restricting the max to non-orphaned blocks makes the anchor agree with + * `networkState.pendingMaxBlockHeight` (which is computed per `chain_status`), and it is + * a no-op in normal operation, where the highest block is always the pending tip. + */ +export const BEST_CHAIN_TIP_HEIGHT_SQL = + "(SELECT MAX(height) FROM blocks WHERE chain_status <> 'orphaned')"; + +/** + * {@link BEST_CHAIN_TIP_HEIGHT_SQL} as a fragment for tagged-template queries. The text is + * a constant with no user input, which is what makes `unsafe` safe here. + */ +export function bestChainTipHeight(db_client: postgres.Sql) { + return db_client.unsafe(BEST_CHAIN_TIP_HEIGHT_SQL); +} diff --git a/src/db/sql/events-actions/queries.ts b/src/db/sql/events-actions/queries.ts index 3d8c8f5e..5d142e28 100644 --- a/src/db/sql/events-actions/queries.ts +++ b/src/db/sql/events-actions/queries.ts @@ -2,6 +2,7 @@ import type postgres from 'postgres'; import { ArchiveNodeDatabaseRow } from './types.js'; import { BlockStatusFilter } from '../../../blockchain/types.js'; import { BLOCK_RANGE_SIZE } from '../../../server/server.js'; +import { bestChainTipHeight } from '../best-chain.js'; function fullChainCTE(db_client: postgres.Sql, from?: string, to?: string) { let toAsNum = to ? Number(to) : undefined; @@ -20,7 +21,7 @@ function fullChainCTE(db_client: postgres.Sql, from?: string, to?: string) { FROM blocks b WHERE - height = (SELECT max(height) FROM blocks) + height = ${bestChainTipHeight(db_client)} ) UNION ALL SELECT @@ -38,7 +39,9 @@ function fullChainCTE(db_client: postgres.Sql, from?: string, to?: string) { ), full_chain AS ( SELECT - DISTINCT id, state_hash, parent_id, parent_hash, height, global_slot_since_genesis, global_slot_since_hard_fork, timestamp, chain_status, ledger_hash, (SELECT max(height) FROM blocks) - height AS distance_from_max_block_height, last_vrf_output + DISTINCT id, state_hash, parent_id, parent_hash, height, global_slot_since_genesis, global_slot_since_hard_fork, timestamp, chain_status, ledger_hash, ${bestChainTipHeight( + db_client + )} - height AS distance_from_max_block_height, last_vrf_output FROM ( SELECT @@ -68,10 +71,9 @@ function fullChainCTE(db_client: postgres.Sql, from?: string, to?: string) { // If no params ar provided, then we query the last BLOCK_RANGE_SIZE blocks fromAsNum ? db_client`AND b.height >= ${fromAsNum} AND b.height < ${toAsNum!}` - : db_client`AND b.height >= ( - SELECT MAX(b2.height) - FROM blocks b2 - ) - ${BLOCK_RANGE_SIZE}` + : db_client`AND b.height >= ${bestChainTipHeight( + db_client + )} - ${BLOCK_RANGE_SIZE}` } ) AS full_chain ) @@ -494,7 +496,7 @@ export function resolveActionStateBoundary( ( SELECT id, parent_id, chain_status FROM blocks - WHERE height = (SELECT max(height) FROM blocks) + WHERE height = ${bestChainTipHeight(db_client)} ) UNION ALL SELECT b.id, b.parent_id, b.chain_status @@ -511,7 +513,9 @@ export function resolveActionStateBoundary( ? db_client`${fromAsNum}::bigint` : toAsNum !== null ? db_client`${toAsNum - BLOCK_RANGE_SIZE}::bigint` - : db_client`(SELECT max(height) FROM blocks) - ${BLOCK_RANGE_SIZE}::bigint` + : db_client`${bestChainTipHeight( + db_client + )} - ${BLOCK_RANGE_SIZE}::bigint` } AS window_start ), target AS ( @@ -577,7 +581,7 @@ export function getZkappsWithPendingEventsQuery(db_client: postgres.Sql) { -- start at the tip SELECT id, parent_id, chain_status FROM blocks - WHERE height = (SELECT MAX(height) FROM blocks) + WHERE height = ${bestChainTipHeight(db_client)} UNION ALL diff --git a/src/services/blocks-service/blocks-service.ts b/src/services/blocks-service/blocks-service.ts index e290a842..2ae94e98 100644 --- a/src/services/blocks-service/blocks-service.ts +++ b/src/services/blocks-service/blocks-service.ts @@ -10,6 +10,7 @@ import type { BlockSortByInput, } from '../../resolvers-types.js'; import { IBlocksService } from './blocks-service.interface.js'; +import { BEST_CHAIN_TIP_HEIGHT_SQL } from '../../db/sql/best-chain.js'; import { TracingState, extractTraceStateFromOptions, @@ -79,7 +80,9 @@ class BlocksService implements IBlocksService { options: unknown ): Promise { const tracingState = extractTraceStateFromOptions(options); - return (await this.getBlockData(query, limit, sortBy, { tracingState })) ?? []; + return ( + (await this.getBlockData(query, limit, sortBy, { tracingState })) ?? [] + ); } async getBlockData( @@ -128,7 +131,8 @@ class BlocksService implements IBlocksService { const dateTimeLt = query?.dateTime_lt; const canonical = query?.canonical; const inBestChain = query?.inBestChain; - const orderBy: 'ASC' | 'DESC' = sortBy === 'BLOCKHEIGHT_DESC' ? 'DESC' : 'ASC'; + const orderBy: 'ASC' | 'DESC' = + sortBy === 'BLOCKHEIGHT_DESC' ? 'DESC' : 'ASC'; const limitValue = Math.min(limit ?? 200, BLOCK_RANGE_SIZE); // Build the SQL query for blocks with transactions @@ -192,16 +196,15 @@ class BlocksService implements IBlocksService { paramIndex++; } - const best_chain_til_canonical_cte = - ` + const best_chain_til_canonical_cte = ` WITH RECURSIVE best_chain_til_canonical AS ( SELECT id, parent_id, height FROM blocks - WHERE height = (SELECT MAX(height) FROM blocks) - AND chain_status = 'pending' + WHERE chain_status = 'pending' + AND height = ${BEST_CHAIN_TIP_HEIGHT_SQL} UNION @@ -213,19 +216,18 @@ class BlocksService implements IBlocksService { JOIN best_chain_til_canonical ON potential_parent.id = best_chain_til_canonical.parent_id WHERE potential_parent.chain_status <> 'canonical' ) - ` + `; if (inBestChain === true) { sql = - best_chain_til_canonical_cte - + sql - + ` AND (b.id IN (SELECT id FROM best_chain_til_canonical) OR b.chain_status = 'canonical')`; + best_chain_til_canonical_cte + + sql + + ` AND (b.id IN (SELECT id FROM best_chain_til_canonical) OR b.chain_status = 'canonical')`; } else if (inBestChain === false) { sql = - best_chain_til_canonical_cte - + sql - + ` AND (b.id NOT IN (SELECT id FROM best_chain_til_canonical) + best_chain_til_canonical_cte + + sql + + ` AND (b.id NOT IN (SELECT id FROM best_chain_til_canonical) OR b.chain_status = 'orphaned')`; - } sql += ` ORDER BY b.height ${orderBy}`; diff --git a/tests/integration/integration.test.ts b/tests/integration/integration.test.ts index c034ed25..2837d11c 100644 --- a/tests/integration/integration.test.ts +++ b/tests/integration/integration.test.ts @@ -19,6 +19,7 @@ import { ActionsService } from '../../src/services/actions-service/actions-servi import { NetworkService } from '../../src/services/network-service/network-service.js'; import { BlocksService } from '../../src/services/blocks-service/blocks-service.js'; import { BlockStatusFilter } from '../../src/blockchain/types.js'; +import { BlockSortByInput } from '../../src/resolvers-types.js'; import { DEFAULT_TOKEN_ID } from '../../src/blockchain/constants.js'; import { TracingState } from '../../src/tracing/tracer.js'; import { @@ -32,10 +33,13 @@ const nullOptions = { tracingState: new TracingState(undefined as any) }; let client: postgres.Sql; -before(async () => { - await setupTestDatabase(); - client = createTestClient(); -}, { timeout: 30000 }); +before( + async () => { + await setupTestDatabase(); + client = createTestClient(); + }, + { timeout: 30000 } +); after(async () => { await client.end(); @@ -137,7 +141,11 @@ describe('BlocksService (integration)', () => { ); assert.ok(blocks.length > 0); // 24 canonical blocks in the dump — verify we get them all within default limit - assert.strictEqual(blocks.length, 24, 'should return all 24 canonical blocks'); + assert.strictEqual( + blocks.length, + 24, + 'should return all 24 canonical blocks' + ); }); test('filters non-canonical blocks', async () => { @@ -160,7 +168,10 @@ describe('BlocksService (integration)', () => { assert.ok(blocks.length > 0); // Should include canonical blocks plus pending best chain // At minimum 24 canonical + 1 pending = 25 blocks - assert.ok(blocks.length >= 24, 'should include at least all canonical blocks'); + assert.ok( + blocks.length >= 24, + 'should include at least all canonical blocks' + ); }); test('block data has correct shape', async () => { @@ -193,9 +204,7 @@ describe('BlocksService (integration)', () => { nullOptions ); // At least one block at height 2-3 should have a coinbase - const withCoinbase = blocks.filter( - (b) => b.transactions.coinbase !== '0' - ); + const withCoinbase = blocks.filter((b) => b.transactions.coinbase !== '0'); assert.ok( withCoinbase.length > 0, 'at least one block should have coinbase' @@ -224,7 +233,10 @@ describe('BlocksService (integration)', () => { const oneHourBefore = new Date(latestTime.getTime() - 3600000); const blocks = await blocksService.getBlocks( - { dateTime_gte: oneHourBefore.toISOString(), dateTime_lt: latestTime.toISOString() }, + { + dateTime_gte: oneHourBefore.toISOString(), + dateTime_lt: latestTime.toISOString(), + }, null, null, nullOptions @@ -263,7 +275,10 @@ describe('NetworkService (integration)', () => { test('pending height > canonical height', async () => { const state = await networkService.getNetworkState(nullOptions); - assert.ok(state.maxBlockHeight, 'fixture seeds both canonical and pending rows'); + assert.ok( + state.maxBlockHeight, + 'fixture seeds both canonical and pending rows' + ); assert.ok( state.maxBlockHeight.pendingMaxBlockHeight > state.maxBlockHeight.canonicalMaxBlockHeight @@ -271,6 +286,125 @@ describe('NetworkService (integration)', () => { }); }); +// ─── Hard-fork shape: an abandoned chain above the tip ────────────── + +/** + * After a hard fork the archive still holds the OLD chain's blocks above the new + * chain's tip, marked `orphaned` (mainnet, Mesa upgrade 2026-09-03: old chain to + * 548187, fork block 548147, new tip 548164). Every "walk back from the tip" CTE + * used to anchor on the global MAX(height) — a dead block — so `inBestChain: true` + * returned nothing above the fork block for as long as the old chain outranked the + * new one. These tests plant that shape on top of the fixture: one orphaned block + * one height ABOVE the synthetic pending tip. + */ +describe('Hard-fork shape (integration)', () => { + const ORPHAN_ABOVE_TIP = '3NKorphan_above_the_pending_tip_hard_fork_shape'; + let blocksService: BlocksService; + let networkService: NetworkService; + let tipHeight: number; + + before(async () => { + blocksService = new BlocksService(client); + networkService = new NetworkService(client); + const [tip] = await client.unsafe( + `SELECT height FROM blocks WHERE chain_status = 'pending' ORDER BY height DESC LIMIT 1` + ); + tipHeight = Number(tip.height); + await client.unsafe(` + INSERT INTO blocks ( + id, state_hash, parent_id, parent_hash, creator_id, block_winner_id, + last_vrf_output, + snarked_ledger_hash_id, staking_epoch_data_id, next_epoch_data_id, + min_window_density, sub_window_densities, total_currency, + ledger_hash, height, global_slot_since_hard_fork, global_slot_since_genesis, + protocol_version_id, proposed_protocol_version_id, + timestamp, chain_status + ) + SELECT + (SELECT max(id) + 1 FROM blocks), + '${ORPHAN_ABOVE_TIP}', + id, + state_hash, + creator_id, + block_winner_id, + last_vrf_output, + snarked_ledger_hash_id, + staking_epoch_data_id, + next_epoch_data_id, + min_window_density, + sub_window_densities, + total_currency, + ledger_hash, + height + 1, + global_slot_since_hard_fork + 1, + global_slot_since_genesis + 1, + protocol_version_id, + proposed_protocol_version_id, + (timestamp::bigint + 60000)::text, + 'orphaned' + FROM blocks + WHERE chain_status = 'pending' + ORDER BY height DESC + LIMIT 1 + `); + }); + + after(async () => { + await client.unsafe( + `DELETE FROM blocks WHERE state_hash = '${ORPHAN_ABOVE_TIP}'` + ); + }); + + test('networkState ignores the abandoned block above the tip', async () => { + const state = await networkService.getNetworkState(nullOptions); + assert.ok(state.maxBlockHeight); + assert.strictEqual(state.maxBlockHeight.pendingMaxBlockHeight, tipHeight); + }); + + test('inBestChain=true still reaches the pending tip', async () => { + const blocks = await blocksService.getBlocks( + { inBestChain: true, blockHeight_gte: tipHeight }, + null, + null, + nullOptions + ); + assert.deepStrictEqual( + blocks.map((b) => Number(b.blockHeight)), + [tipHeight], + 'the pending tip must be in the best chain even with an orphan above it' + ); + assert.notStrictEqual(blocks[0].stateHash, ORPHAN_ABOVE_TIP); + }); + + test('the best chain and networkState agree on the tip', async () => { + const state = await networkService.getNetworkState(nullOptions); + const blocks = await blocksService.getBlocks( + { inBestChain: true }, + null, + BlockSortByInput.BlockheightDesc, + nullOptions + ); + assert.ok(state.maxBlockHeight); + assert.strictEqual( + Number(blocks[0].blockHeight), + state.maxBlockHeight.pendingMaxBlockHeight + ); + }); + + test('inBestChain=false reports the abandoned block', async () => { + const blocks = await blocksService.getBlocks( + { inBestChain: false, blockHeight_gte: tipHeight }, + null, + null, + nullOptions + ); + assert.deepStrictEqual( + blocks.map((b) => b.stateHash), + [ORPHAN_ABOVE_TIP] + ); + }); +}); + // ─── Events Service ────────────────────────────────────────────────── describe('EventsService (integration)', () => { diff --git a/tests/unit/best-chain-anchor.test.ts b/tests/unit/best-chain-anchor.test.ts new file mode 100644 index 00000000..c3fce5f5 --- /dev/null +++ b/tests/unit/best-chain-anchor.test.ts @@ -0,0 +1,111 @@ +/** + * Every recursive "walk back from the tip" CTE must anchor on the best chain's tip, + * never on the table's global MAX(height). + * + * The distinction only matters after a hard fork, when the abandoned chain's blocks + * sit above the new tip marked `orphaned` — which is exactly when nothing in the + * integration fixture would notice. So this test renders each query through a + * recording stand-in for the postgres client and inspects the SQL text itself. + */ +import { describe, test } from 'node:test'; +import assert from 'node:assert'; +import type postgres from 'postgres'; +import { + BEST_CHAIN_TIP_HEIGHT_SQL, + bestChainTipHeight, +} from '../../src/db/sql/best-chain.js'; +import { + getEventsQuery, + getActionsQuery, + getZkappsWithPendingEventsQuery, + resolveActionStateBoundary, +} from '../../src/db/sql/events-actions/queries.js'; +import { BlockStatusFilter } from '../../src/blockchain/types.js'; + +type Fragment = { strings: readonly string[]; values: unknown[] }; + +function isFragment(value: unknown): value is Fragment { + return ( + typeof value === 'object' && + value !== null && + 'strings' in value && + 'values' in value + ); +} + +/** Flatten a recorded tagged-template tree back into one SQL string. */ +function render(fragment: Fragment): string { + let out = ''; + fragment.strings.forEach((chunk, i) => { + out += chunk; + if (i < fragment.values.length) { + const value = fragment.values[i]; + out += isFragment(value) ? render(value) : '$param'; + } + }); + return out; +} + +/** A postgres.Sql stand-in that records instead of executing. */ +function recordingClient(): postgres.Sql { + const tag = (strings: readonly string[], ...values: unknown[]): Fragment => ({ + strings, + values, + }); + (tag as unknown as { unsafe: (sql: string) => Fragment }).unsafe = ( + sql: string + ) => ({ strings: [sql], values: [] }); + return tag as unknown as postgres.Sql; +} + +const normalise = (sql: string) => sql.replace(/\s+/g, ' ').toLowerCase(); +const GLOBAL_MAX = normalise('(SELECT MAX(height) FROM blocks)'); +const BEST_TIP = normalise(BEST_CHAIN_TIP_HEIGHT_SQL); + +describe('best-chain tip anchor', () => { + test('the anchor excludes orphaned blocks', () => { + assert.ok(BEST_TIP.includes("chain_status <> 'orphaned'")); + const rendered = render( + bestChainTipHeight(recordingClient()) as unknown as Fragment + ); + assert.strictEqual(rendered, BEST_CHAIN_TIP_HEIGHT_SQL); + }); + + const cases: [string, (db: postgres.Sql) => unknown][] = [ + [ + 'getEventsQuery', + (db) => getEventsQuery(db, 'B62qaddr', '1', BlockStatusFilter.all), + ], + [ + 'getEventsQuery with a block range', + (db) => + getEventsQuery(db, 'B62qaddr', '1', BlockStatusFilter.all, '20', '10'), + ], + [ + 'getActionsQuery', + (db) => getActionsQuery(db, 'B62qaddr', '1', BlockStatusFilter.all), + ], + [ + 'resolveActionStateBoundary', + (db) => resolveActionStateBoundary(db, 'B62qaddr', '1', 'state'), + ], + [ + 'getZkappsWithPendingEventsQuery', + (db) => getZkappsWithPendingEventsQuery(db), + ], + ]; + + for (const [name, build] of cases) { + test(`${name} walks back from the best-chain tip, not the global max height`, () => { + const sql = normalise(render(build(recordingClient()) as Fragment)); + assert.ok( + sql.includes(BEST_TIP), + `${name} must anchor on ${BEST_CHAIN_TIP_HEIGHT_SQL}` + ); + assert.ok( + !sql.includes(GLOBAL_MAX), + `${name} still anchors on the global MAX(height), which is a dead block after a hard fork` + ); + }); + } +});