fix(sql): anchor the best-chain walk on the highest non-orphaned block

docs/runbook.md

@@ -97,6 +97,7 @@ Recovery semantics to expect:
 | Connection-pool exhaustion errors        | `PG_MAX_CONNECTIONS` × replicas > DB capacity                             | lower pool size or raise DB `max_connections`                                                                                                                                                        |
 | Memory growth / OOM kills                | heavy result sets or a leak                                               | lower `BLOCK_RANGE_SIZE`; inspect heap metrics; cap container memory                                                                                                                                 |
 | Startup exits immediately                | invalid config                                                            | read the startup error — config is validated fail-fast (missing `PG_CONN`, bad `PORT`, etc.)                                                                                                         |
+| After a hard fork, `inBestChain: true` / events / actions stop at the fork block while `networkState.pendingMaxBlockHeight` keeps rising | the abandoned chain's blocks sit ABOVE the new tip and the archive daemon takes a while to relabel them `orphaned`; until then they are the highest `pending` blocks | expected for the relabelling window (about 30 min on mainnet at the Mesa upgrade); once they read `orphaned` the walk anchors on the live tip by itself (`src/db/sql/best-chain.ts`). If it persists, inspect `SELECT chain_status, MAX(height) FROM blocks GROUP BY 1` |
 
 ## Deploys & rollback
 

src/db/sql/best-chain.ts

@@ -0,0 +1,34 @@
+import type postgres from 'postgres';
+
+/**
+ * The height of the best chain's tip — the anchor every "walk back from the tip"
+ * recursive CTE starts from.
+ *
+ * This is deliberately NOT `(SELECT MAX(height) FROM blocks)`. The archive keeps every
+ * block it ever saw, so after a hard fork the abandoned chain's blocks stay in the table
+ * ABOVE the new chain's tip, marked `orphaned`. A global max then lands on a dead block:
+ * with the `chain_status = 'pending'` guard the anchor matches nothing and the walk
+ * silently degrades to canonical-only rows; without the guard the walk follows the dead
+ * chain. Either way the live tip and everything the network produced since the fork is
+ * invisible to `inBestChain`, events, actions and action-state resolution.
+ *
+ * Seen on mainnet at the Mesa upgrade (2026-09-03): the old chain reached height 548187
+ * before stopping, the fork block was 548147, and for the whole first hour
+ * `blocks(inBestChain: true)` returned nothing above 548147 while `networkState`
+ * correctly reported a pending tip of 548164 — the two disagreed because only one of
+ * them excluded orphans.
+ *
+ * Restricting the max to non-orphaned blocks makes the anchor agree with
+ * `networkState.pendingMaxBlockHeight` (which is computed per `chain_status`), and it is
+ * a no-op in normal operation, where the highest block is always the pending tip.
+ */
+export const BEST_CHAIN_TIP_HEIGHT_SQL =
+  "(SELECT MAX(height) FROM blocks WHERE chain_status <> 'orphaned')";
+
+/**
+ * {@link BEST_CHAIN_TIP_HEIGHT_SQL} as a fragment for tagged-template queries. The text is
+ * a constant with no user input, which is what makes `unsafe` safe here.
+ */
+export function bestChainTipHeight(db_client: postgres.Sql) {
+  return db_client.unsafe(BEST_CHAIN_TIP_HEIGHT_SQL);
+}

src/db/sql/events-actions/queries.ts

@@ -2,6 +2,7 @@ import type postgres from 'postgres';
 import { ArchiveNodeDatabaseRow } from './types.js';
 import { BlockStatusFilter } from '../../../blockchain/types.js';
 import { BLOCK_RANGE_SIZE } from '../../../server/server.js';
+import { bestChainTipHeight } from '../best-chain.js';
 
 function fullChainCTE(db_client: postgres.Sql, from?: string, to?: string) {
   let toAsNum = to ? Number(to) : undefined;
@@ -20,7 +21,7 @@ function fullChainCTE(db_client: postgres.Sql, from?: string, to?: string) {
       FROM
         blocks b
       WHERE
-        height = (SELECT max(height) FROM blocks)
+        height = ${bestChainTipHeight(db_client)}
     ) 
     UNION ALL
     SELECT
@@ -38,7 +39,9 @@ function fullChainCTE(db_client: postgres.Sql, from?: string, to?: string) {
   ),
   full_chain AS (
     SELECT
-      DISTINCT id, state_hash, parent_id, parent_hash, height, global_slot_since_genesis, global_slot_since_hard_fork, timestamp, chain_status, ledger_hash, (SELECT max(height) FROM blocks) - height AS distance_from_max_block_height, last_vrf_output
+      DISTINCT id, state_hash, parent_id, parent_hash, height, global_slot_since_genesis, global_slot_since_hard_fork, timestamp, chain_status, ledger_hash, ${bestChainTipHeight(
+        db_client
+      )} - height AS distance_from_max_block_height, last_vrf_output
     FROM
       (
         SELECT
@@ -68,10 +71,9 @@ function fullChainCTE(db_client: postgres.Sql, from?: string, to?: string) {
             // If no params ar provided, then we query the last BLOCK_RANGE_SIZE blocks
             fromAsNum
               ? db_client`AND b.height >= ${fromAsNum} AND b.height < ${toAsNum!}`
-              : db_client`AND b.height >= (
-                SELECT MAX(b2.height)
-                FROM blocks b2
-            ) - ${BLOCK_RANGE_SIZE}`
+              : db_client`AND b.height >= ${bestChainTipHeight(
+                  db_client
+                )} - ${BLOCK_RANGE_SIZE}`
           }
       ) AS full_chain
   )
@@ -494,7 +496,7 @@ export function resolveActionStateBoundary(
     (
       SELECT id, parent_id, chain_status
       FROM blocks
-      WHERE height = (SELECT max(height) FROM blocks)
+      WHERE height = ${bestChainTipHeight(db_client)}
     )
     UNION ALL
     SELECT b.id, b.parent_id, b.chain_status
@@ -511,7 +513,9 @@ export function resolveActionStateBoundary(
         ? db_client`${fromAsNum}::bigint`
         : toAsNum !== null
         ? db_client`${toAsNum - BLOCK_RANGE_SIZE}::bigint`
-        : db_client`(SELECT max(height) FROM blocks) - ${BLOCK_RANGE_SIZE}::bigint`
+        : db_client`${bestChainTipHeight(
+            db_client
+          )} - ${BLOCK_RANGE_SIZE}::bigint`
     } AS window_start
   ),
   target AS (
@@ -577,7 +581,7 @@ export function getZkappsWithPendingEventsQuery(db_client: postgres.Sql) {
   -- start at the tip
   SELECT id, parent_id, chain_status
     FROM blocks
-   WHERE height = (SELECT MAX(height) FROM blocks)
+   WHERE height = ${bestChainTipHeight(db_client)}
 
   UNION ALL
 

src/services/blocks-service/blocks-service.ts

@@ -10,6 +10,7 @@ import type {
   BlockSortByInput,
 } from '../../resolvers-types.js';
 import { IBlocksService } from './blocks-service.interface.js';
+import { BEST_CHAIN_TIP_HEIGHT_SQL } from '../../db/sql/best-chain.js';
 import {
   TracingState,
   extractTraceStateFromOptions,
@@ -79,7 +80,9 @@ class BlocksService implements IBlocksService {
     options: unknown
   ): Promise<Blocks> {
     const tracingState = extractTraceStateFromOptions(options);
-    return (await this.getBlockData(query, limit, sortBy, { tracingState })) ?? [];
+    return (
+      (await this.getBlockData(query, limit, sortBy, { tracingState })) ?? []
+    );
   }
 
   async getBlockData(
@@ -128,7 +131,8 @@ class BlocksService implements IBlocksService {
     const dateTimeLt = query?.dateTime_lt;
     const canonical = query?.canonical;
     const inBestChain = query?.inBestChain;
-    const orderBy: 'ASC' | 'DESC' = sortBy === 'BLOCKHEIGHT_DESC' ? 'DESC' : 'ASC';
+    const orderBy: 'ASC' | 'DESC' =
+      sortBy === 'BLOCKHEIGHT_DESC' ? 'DESC' : 'ASC';
     const limitValue = Math.min(limit ?? 200, BLOCK_RANGE_SIZE);
 
     // Build the SQL query for blocks with transactions
@@ -192,16 +196,15 @@ class BlocksService implements IBlocksService {
       paramIndex++;
     }
 
-    const best_chain_til_canonical_cte =
-      `
+    const best_chain_til_canonical_cte = `
         WITH RECURSIVE best_chain_til_canonical AS (
             SELECT
                 id,
                 parent_id,
                 height
             FROM blocks
-            WHERE height = (SELECT MAX(height) FROM blocks)
-            AND chain_status = 'pending'
+            WHERE chain_status = 'pending'
+            AND height = ${BEST_CHAIN_TIP_HEIGHT_SQL}
 
             UNION
 
@@ -213,19 +216,18 @@ class BlocksService implements IBlocksService {
             JOIN best_chain_til_canonical ON potential_parent.id = best_chain_til_canonical.parent_id
             WHERE potential_parent.chain_status <> 'canonical'
         )
-        `
+        `;
     if (inBestChain === true) {
       sql =
-        best_chain_til_canonical_cte
-        + sql
-        + ` AND (b.id IN (SELECT id FROM best_chain_til_canonical) OR b.chain_status = 'canonical')`;
+        best_chain_til_canonical_cte +
+        sql +
+        ` AND (b.id IN (SELECT id FROM best_chain_til_canonical) OR b.chain_status = 'canonical')`;
     } else if (inBestChain === false) {
       sql =
-        best_chain_til_canonical_cte
-        + sql
-        + ` AND (b.id NOT IN (SELECT id FROM best_chain_til_canonical)
+        best_chain_til_canonical_cte +
+        sql +
+        ` AND (b.id NOT IN (SELECT id FROM best_chain_til_canonical)
                  OR b.chain_status = 'orphaned')`;
-
     }
 
     sql += ` ORDER BY b.height ${orderBy}`;