fix: checkpoint sync follow-head fixes for epbs-devnet-1

packages/beacon-node/src/network/network.ts

@@ -800,6 +800,7 @@ export class Network implements INetwork {
       client: clientAgent,
       custodyColumns,
       earliestAvailableSlot, // can be undefined pre-fulu
+      headSlot: status.headSlot,
     });
   };
 

packages/beacon-node/src/network/peers/peersData.ts

@@ -11,6 +11,7 @@ export type PeerSyncMeta = {
   client: string;
   custodyColumns: CustodyIndex[];
   earliestAvailableSlot?: Slot;
+  headSlot: Slot;
 };
 
 export enum RelevantPeerStatus {

packages/beacon-node/src/network/reqresp/handlers/beaconBlocksByRange.ts

@@ -22,8 +22,8 @@ export async function* onBeaconBlocksByRange(
 
   const finalized = db.blockArchive;
   // in the case of initializing from a non-finalized state, we don't have the finalized block so this api does not work
-  // chain.forkChoice.getFinalizeBlock().slot
-  const finalizedSlot = chain.forkChoice.getFinalizedCheckpointSlot();
+  const finalizedBlock = chain.forkChoice.getFinalizedBlock();
+  const finalizedSlot = finalizedBlock.slot;
 
   const forkName = chain.config.getForkName(startSlot);
   if (isForkPostFulu(forkName) && startSlot < chain.earliestAvailableSlot) {
@@ -36,11 +36,29 @@ export async function* onBeaconBlocksByRange(
 
   // Finalized range of blocks
   if (startSlot <= finalizedSlot) {
-    // Chain of blobs won't change
+    const finalizedEntries: Array<{slot: number; data: Uint8Array}> = [];
     for await (const {key, value} of finalized.binaryEntriesStream({gte: startSlot, lt: endSlot})) {
+      finalizedEntries.push({slot: finalized.decodeKey(key), data: value});
+    }
+
+    // The finalized boundary block may still be in hot storage during archive transitions.
+    // Ensure the canonical finalized block is included when it falls inside the requested range,
+    // otherwise the response may incorrectly start at the next block (e.g. 97..127 instead of 96..127).
+    if (finalizedBlock.slot >= startSlot && finalizedBlock.slot < endSlot) {
+      const hasFinalizedBoundary = finalizedEntries.some((entry) => entry.slot === finalizedBlock.slot);
+      if (!hasFinalizedBoundary) {
+        const finalizedBoundaryBlock = await chain.getSerializedBlockByRoot(finalizedBlock.blockRoot);
+        if (finalizedBoundaryBlock) {
+          finalizedEntries.push({slot: finalizedBoundaryBlock.slot, data: finalizedBoundaryBlock.block});
+        }
+      }
+    }
+
+    finalizedEntries.sort((a, b) => a.slot - b.slot);
+    for (const {slot, data} of finalizedEntries) {
       yield {
-        data: value,
-        boundary: chain.config.getForkBoundaryAtEpoch(computeEpochAtSlot(finalized.decodeKey(key))),
+        data,
+        boundary: chain.config.getForkBoundaryAtEpoch(computeEpochAtSlot(slot)),
       };
     }
   }

packages/beacon-node/src/sync/range/batch.ts

@@ -442,6 +442,18 @@ export class Batch {
     }
   }
 
+  /**
+   * Processing -> AwaitingDownload without counting as a processing failure.
+   * Used when the downloaded response is later recognized as malformed for retry purposes.
+   */
+  retryDownload(): void {
+    if (this.state.status !== BatchStatus.Processing) {
+      throw new BatchError(this.wrongStatusErrorType(BatchStatus.Processing));
+    }
+
+    this.state = {status: BatchStatus.AwaitingDownload, blocks: [], envelopes: null};
+  }
+
   /**
    * AwaitingValidation -> Done
    */

packages/beacon-node/src/sync/range/chain.ts

@@ -5,6 +5,7 @@ import {isBlockInputBlobs, isBlockInputColumns} from "../../chain/blocks/blockIn
 import {BlockInputErrorCode} from "../../chain/blocks/blockInput/errors.js";
 import {IBlockInput} from "../../chain/blocks/blockInput/types.js";
 import {BlobSidecarErrorCode} from "../../chain/errors/blobSidecarError.js";
+import {BlockError, BlockErrorCode} from "../../chain/errors/blockError.js";
 import {DataColumnSidecarErrorCode} from "../../chain/errors/dataColumnSidecarError.js";
 import {Metrics} from "../../metrics/metrics.js";
 import {PeerAction, prettyPrintPeerIdStr} from "../../network/index.js";
@@ -67,6 +68,8 @@ export type SyncChainFns = {
   onEnd: (err: Error | null, target: ChainTarget | null) => void;
   /** Deletes an array of BlockInputs from the BlockInputCache */
   pruneBlockInputs: (blockInputs: IBlockInput[]) => void;
+  /** Fetches a single block by root from a specific peer and imports it */
+  processBlockByRoot: (peer: PeerIdStr, blockRootHex: string, syncType: RangeSyncType) => Promise<void>;
 };
 
 /**
@@ -135,6 +138,7 @@ export class SyncChain {
   private readonly reportPeer: SyncChainFns["reportPeer"];
   private readonly getConnectedPeerSyncMeta: SyncChainFns["getConnectedPeerSyncMeta"];
   private readonly pruneBlockInputs: SyncChainFns["pruneBlockInputs"];
+  private readonly processBlockByRoot: SyncChainFns["processBlockByRoot"];
 
   /** AsyncIterable that guarantees processChainSegment is run only at once at anytime */
   private readonly batchProcessor = new ItTrigger();
@@ -167,6 +171,7 @@ export class SyncChain {
     this.reportPeer = fns.reportPeer;
     this.pruneBlockInputs = fns.pruneBlockInputs;
     this.getConnectedPeerSyncMeta = fns.getConnectedPeerSyncMeta;
+    this.processBlockByRoot = fns.processBlockByRoot;
     this.config = config;
     this.clock = clock;
     this.metrics = metrics;
@@ -681,16 +686,24 @@ export class SyncChain {
       ...(envelopesMeta ?? {}),
     });
 
+    if (this.handleMalformedFinalizedBoundaryBatch(batch, blocks)) {
+      return;
+    }
+
     // wrapError ensures to never call both batch success() and batch error()
-    const res = await wrapError(this.processChainSegment(blocks, envelopes, this.syncType));
+    let res = await wrapError(this.processChainSegment(blocks, envelopes, this.syncType));
+
+    if (res.err && (await this.tryRecoverMissingBatchBoundaryParent(batch, blocks, res.err))) {
+      res = await wrapError(this.processChainSegment(blocks, envelopes, this.syncType));
+    }
 
     if (!res.err) {
       batch.processingSuccess();
 
-      // If the processed batch is not empty, validate previous AwaitingValidation blocks.
-      if (blocks.length > 0) {
-        this.advanceChain(batch.startEpoch);
-      }
+      // Advance chain for all successfully processed batches, including empty ones.
+      // Empty epochs (0 blocks) occur during periods of poor liveness — the sync
+      // chain must still advance past them to avoid deadlock.
+      this.advanceChain(batch.startEpoch);
 
       // Potentially process next AwaitingProcessing batch
       this.triggerBatchProcessor();
@@ -720,6 +733,108 @@ export class SyncChain {
     this.triggerBatchDownloader();
   }
 
+  private handleMalformedFinalizedBoundaryBatch(batch: Batch, blocks: IBlockInput[]): boolean {
+    if (this.syncType !== RangeSyncType.Finalized) {
+      return false;
+    }
+
+    const firstBlock = blocks[0];
+    if (firstBlock == null) {
+      return false;
+    }
+
+    const attemptPeers = batch.state.status === BatchStatus.Processing ? batch.state.attempt.peers : [];
+    for (const peer of attemptPeers) {
+      const peerTarget = this.peerset.get(peer);
+      if (peerTarget == null) {
+        continue;
+      }
+
+      const peerTargetRootHex = toRootHex(peerTarget.root);
+      const peerTargetInBatchRange =
+        peerTarget.slot >= batch.startSlot && peerTarget.slot < batch.startSlot + batch.count;
+      const missingBoundaryFromThisPeer =
+        peerTargetInBatchRange && firstBlock.slot > peerTarget.slot && firstBlock.parentRootHex === peerTargetRootHex;
+
+      if (!missingBoundaryFromThisPeer) {
+        continue;
+      }
+
+      this.logger.verbose("Malformed finalized boundary batch from peer", {
+        id: this.logId,
+        peer: prettyPrintPeerIdStr(peer),
+        batchStartSlot: batch.startSlot,
+        firstBlockSlot: firstBlock.slot,
+        missingBoundarySlot: peerTarget.slot,
+        missingBoundaryRoot: peerTargetRootHex,
+      });
+
+      this.pruneBlockInputs(blocks);
+      batch.retryDownload();
+      this.quarantinePeer(peer, `missing finalized boundary block slot=${peerTarget.slot} root=${peerTargetRootHex}`);
+      this.triggerBatchDownloader();
+      return true;
+    }
+
+    return false;
+  }
+
+  private async tryRecoverMissingBatchBoundaryParent(
+    batch: Batch,
+    blocks: IBlockInput[],
+    err: Error
+  ): Promise<boolean> {
+    if (this.syncType !== RangeSyncType.Finalized) {
+      return false;
+    }
+
+    if (!(err instanceof BlockError) || err.type.code !== BlockErrorCode.PARENT_UNKNOWN) {
+      return false;
+    }
+
+    const firstBlock = blocks[0];
+    if (firstBlock == null || !firstBlock.hasBlock()) {
+      return false;
+    }
+
+    // A peer may respond to the first finalized batch after a checkpoint boundary with
+    // slot N+1.. instead of including the boundary block at slot N. If the first block
+    // is exactly one slot after the requested batch start and its parent is unknown,
+    // recover that boundary parent by root and retry the batch immediately.
+    if (firstBlock.slot !== batch.startSlot + 1 || firstBlock.parentRootHex !== err.type.parentRoot) {
+      return false;
+    }
+
+    const attemptPeers = batch.state.status === BatchStatus.Processing ? batch.state.attempt.peers : [];
+    for (const peer of attemptPeers) {
+      const recovered = await wrapError(this.processBlockByRoot(peer, err.type.parentRoot, this.syncType));
+      if (!recovered.err) {
+        this.logger.debug("Recovered missing batch boundary parent by root", {
+          id: this.logId,
+          peer: prettyPrintPeerIdStr(peer),
+          batchStartSlot: batch.startSlot,
+          recoveredParentRoot: err.type.parentRoot,
+          childSlot: firstBlock.slot,
+        });
+        return true;
+      }
+
+      this.logger.verbose(
+        "Failed to recover missing batch boundary parent by root",
+        {
+          id: this.logId,
+          peer: prettyPrintPeerIdStr(peer),
+          batchStartSlot: batch.startSlot,
+          recoveredParentRoot: err.type.parentRoot,
+          childSlot: firstBlock.slot,
+        },
+        recovered.err
+      );
+    }
+
+    return false;
+  }
+
   /**
    * Drops any batches previous to `newLatestValidatedEpoch` and updates the chain boundaries
    */
@@ -809,6 +924,8 @@ function shouldTreatAsTransientDownloadError(err: DownloadByRangeError): boolean
         reason.includes("RESPONSE_ERROR_RATE_LIMITED") ||
         reason.includes("REQUEST_ERROR_DIAL_ERROR") ||
         reason.includes("REQUEST_ERROR_INVALID_REQUEST") ||
+        reason.includes("REQUEST_ERROR_TTFB_TIMEOUT") ||
+        reason.includes("REQUEST_ERROR_BODY_TIMEOUT") ||
         reason.includes("Message was truncated")
       );
     }

packages/beacon-node/src/sync/range/range.ts

@@ -3,14 +3,15 @@ import {StrictEventEmitter} from "strict-event-emitter-types";
 import {BeaconConfig} from "@lodestar/config";
 import {computeStartSlotAtEpoch} from "@lodestar/state-transition";
 import {Epoch, Status, fulu} from "@lodestar/types";
-import {Logger, toRootHex} from "@lodestar/utils";
-import {IBlockInput} from "../../chain/blocks/blockInput/types.js";
+import {Logger, fromHex, toRootHex} from "@lodestar/utils";
+import {BlockInputSource, IBlockInput} from "../../chain/blocks/blockInput/types.js";
 import {AttestationImportOpt, ImportBlockOpts} from "../../chain/blocks/index.js";
 import {IBeaconChain} from "../../chain/index.js";
 import {Metrics} from "../../metrics/index.js";
 import {INetwork} from "../../network/index.js";
 import {PeerIdStr} from "../../util/peerId.js";
 import {cacheByRangeResponses, downloadByRange} from "../utils/downloadByRange.js";
+import {fetchAndValidateBlock} from "../utils/downloadByRoot.js";
 import {RangeSyncType, getRangeSyncTarget, rangeSyncTypes} from "../utils/remoteSyncType.js";
 import {ChainTarget, SyncChain, SyncChainDebugState, SyncChainFns} from "./chain.js";
 import {updateChains} from "./utils/index.js";
@@ -225,6 +226,33 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) {
     }
   };
 
+  private processBlockByRoot: SyncChainFns["processBlockByRoot"] = async (peerId, blockRootHex, syncType) => {
+    const block = await fetchAndValidateBlock({
+      config: this.config,
+      network: this.network,
+      peerIdStr: peerId,
+      blockRoot: fromHex(blockRootHex),
+    });
+
+    const blockInput = this.chain.seenBlockInputCache.getByBlock({
+      block,
+      blockRootHex,
+      source: BlockInputSource.byRoot,
+      peerIdStr: peerId,
+      seenTimestampSec: Date.now() / 1000,
+    });
+
+    const flags: ImportBlockOpts = {
+      importAttestations: syncType === RangeSyncType.Finalized ? AttestationImportOpt.Skip : undefined,
+      ignoreIfKnown: true,
+      ignoreIfFinalized: true,
+      fromRangeSync: true,
+      blsVerifyOnMainThread: false,
+    };
+
+    await this.chain.processBlock(blockInput, flags);
+  };
+
   /** Convenience method for `SyncChain` */
   private reportPeer: SyncChainFns["reportPeer"] = (peer, action, actionName) => {
     this.network.reportPeer(peer, action, actionName);
@@ -257,6 +285,7 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) {
           reportPeer: this.reportPeer,
           getConnectedPeerSyncMeta: this.getConnectedPeerSyncMeta,
           pruneBlockInputs: this.pruneBlockInputs,
+          processBlockByRoot: this.processBlockByRoot,
           onEnd: this.onSyncChainEnd,
         },
         {

packages/beacon-node/src/sync/range/utils/peerBalancer.ts

@@ -131,7 +131,8 @@ export class ChainPeersBalancer {
         continue;
       }
 
-      if (target.slot < batch.startSlot) {
+      const peerUpperBoundSlot = this.syncType === RangeSyncType.Finalized ? peer.headSlot : target.slot;
+      if (peerUpperBoundSlot < batch.startSlot) {
         continue;
       }
 

packages/beacon-node/src/sync/unknownBlock.ts

@@ -493,11 +493,33 @@ export class BlockInputSync {
           // case BlockErrorCode.GENESIS_BLOCK:
 
           case BlockErrorCode.PARENT_UNKNOWN:
-          case BlockErrorCode.PRESTATE_MISSING:
-            // Should not happen, mark as downloaded to try again latter
-            this.logger.debug("Attempted to process block but its parent was still unknown", errorData, res.err);
+          case BlockErrorCode.PRESTATE_MISSING: {
+            // For Gloas blocks: if the parent is missing its FULL variant (envelope not received),
+            // proactively fetch the parent's envelope via reqresp, then retry.
+            // This commonly happens after checkpoint sync + range sync where the head block's
+            // envelope was already gossipped before we connected to the network.
+            const retryCtx = this.getGloasInvalidStateRootRetryContext(pendingBlock);
+            if (retryCtx.shouldRetry && retryCtx.parentRoot) {
+              // Only fetch envelope if parent FULL variant is actually absent.
+              // getGloasInvalidStateRootRetryContext uses the default (PENDING) variant,
+              // so wantsFullParent can be true even when FULL already exists.
+              const parentFullBlock = this.chain.forkChoice.getBlockHex(retryCtx.parentRoot, PayloadStatus.FULL);
+              if (!parentFullBlock) {
+                this.logger.debug("PRESTATE_MISSING due to missing parent FULL variant, resolving envelope", {
+                  ...errorData,
+                  ...retryCtx,
+                });
+                const parentBlock = this.chain.forkChoice.getBlockHexDefaultStatus(retryCtx.parentRoot);
+                if (parentBlock) {
+                  await this.resolveEnvelopeForBlock(retryCtx.parentRoot, parentBlock.slot);
+                }
+              }
+            } else {
+              this.logger.debug("Attempted to process block but its parent was still unknown", errorData, res.err);
+            }
             pendingBlock.status = PendingBlockInputStatus.downloaded;
             break;
+          }
 
           case BlockErrorCode.EXECUTION_ENGINE_ERROR:
             // Removing the block(s) without penalizing the peers, hoping for EL to