feat(cluster): crash-rejoin boot barrier + off-path rejoin tick (Shape A checkpoint)

Closes the fail-OPEN crash-rejoin hole for the SLOW-rejoin case and the
boot-to-decision race; the FAST-rejoin-within-deadband residual needs a
formation-level signal (escalated, NOT claimed done).

Mechanism (fail-closed, off path only; online_join=on untouched):
 - New per-incarnation shmem flag ClusterGrdShared.offpath_boot_decided
   (init 0).  The phase gate (cluster_gcs_block.c) fences a self-home
   block RECOVERING while the flag is 0, so from process start every
   backend fail-closes self-home reads AND writes (both reach the gate
   via cluster_pcm_lock_acquire_buffer) — zero cold-serve window.
 - cluster_reconfig_offpath_rejoin_tick(): the online_join=off counterpart
   of the joiner self-tick.  already_running -> arm self-fence({self})
   then demote self_join_admitted to 0 (8.A: never raised to 1 before the
   fence is armed); bootstrap-at-initial or single node -> lift the barrier.

Measured (scratchpad bounce-read-p0/rejoin-state-measured.log):
 - SLOW rejoin (survivor advanced epoch first): tick detects
   already_running, arms the fence, self demotes to JOINING. WORKS.
 - FAST rejoin (node restarts within the survivor dead-deadband, survivor
   still at epoch INITIAL): already_running is false and
   bootstrap_quorum_at_initial cannot distinguish a co-boot from a
   long-running-but-not-yet-reconfigured peer -> mis-lifts -> t/404 L4/L5
   still RED.  This gap is pre-existing and affects online_join=on too
   (existing tests only do slow rejoins).  The durable prior-self-epoch
   signal is erased by qvotec_poll_once (writes self slot before reading,
   cluster_qvotec.c:578) — robust closure is a formation-level change.

Spec: spec-5.16 (extended to online_join=off crash-rejoin).

Author: SqlRush

src/backend/cluster/cluster_gcs_block.c

@@ -1450,6 +1450,28 @@ cluster_gcs_block_phase_for_tag(BufferTag tag)
 	 */
 	static_master = cluster_gcs_lookup_master_static(tag);
 
+	/*
+	 * TT lane / crash-rejoin re-declare barrier (Shape A) — off-path boot
+	 * barrier.  With cluster.online_join=off a node that boots into a running
+	 * cluster self-admits immediately (cluster_reconfig.c:206) with an EMPTY
+	 * GRD and NO re-declare episode: for a block whose STATIC home is self,
+	 * the acquire path would find master==self, read the empty local GRD, and
+	 * cold-grant from the stale/empty disk page — a silent stale READ and a
+	 * silently-diverging WRITE (the P0).  Until the off-path rejoin tick has
+	 * classified this incarnation (crash-rejoin -> self-fence armed;
+	 * bootstrap -> nothing), self cannot prove its home blocks' ownership, so
+	 * fence them RECOVERING.  Both reads and writes reach this gate via
+	 * cluster_pcm_lock_acquire_buffer, so this closes the boot-to-decision
+	 * race with ZERO cold-serve window (Rule 8.A: uncertain -> fail-closed).
+	 * Skipped for online_join=on (its admission + join fence govern) and for
+	 * a single declared node (no peer can hold a conflicting copy).
+	 */
+	if (!cluster_online_join && static_master == cluster_node_id
+		&& cluster_conf_node_count() > 1 && !cluster_grd_offpath_boot_decided()) {
+		cluster_grd_inc_join_block_failclosed();
+		return GCS_BLOCK_RECOVERING;
+	}
+
 	/*
 	 * spec-5.16 D3 (r1 P1-C) — online-join PCM block snap-back fence, placed
 	 * BEFORE the non-DEAD-static-master early NORMAL below.  When a joiner (a

src/backend/cluster/cluster_grd.c

@@ -763,6 +763,8 @@ cluster_grd_shmem_init(void)
 		for (i = 0; i < CLUSTER_MAX_NODES; i++)
 			pg_atomic_init_u64(&cluster_grd_state->join_pcm_fence_member_epoch[i], 0);
 		pg_atomic_init_u32(&cluster_grd_state->recovery_direction, (uint32)GRD_REMASTER_DIR_NONE);
+		/* Shape A: off-path boot barrier starts UNDECIDED (fail-closed). */
+		pg_atomic_init_u32(&cluster_grd_state->offpath_boot_decided, 0);
 		pg_atomic_init_u64(&cluster_grd_state->join_remaster_started_count, 0);
 		pg_atomic_init_u64(&cluster_grd_state->join_remaster_done_count, 0);
 		pg_atomic_init_u64(&cluster_grd_state->join_shards_remastered_count, 0);
@@ -1799,6 +1801,36 @@ cluster_grd_inc_join_block_failclosed(void)
 		pg_atomic_fetch_add_u64(&cluster_grd_state->join_block_recovering_failclosed_count, 1);
 }
 
+/*
+ * Shape A (crash-rejoin re-declare barrier) — off-path boot-barrier flag.
+ *
+ *	cluster_grd_offpath_boot_decided() -- false until the off-path rejoin
+ *	tick has classified this incarnation (bootstrap vs crash-rejoin).  The
+ *	phase gate fences self-home blocks RECOVERING while false, so a node
+ *	that self-admits at boot with cluster.online_join=off cannot cold-serve
+ *	its home blocks before it has proven their ownership (Rule 8.A).
+ *	Defaults DECIDED (true) when the GRD region is absent so a cluster-off
+ *	build never fences.
+ */
+bool
+cluster_grd_offpath_boot_decided(void)
+{
+	if (cluster_grd_state == NULL)
+		return true;
+	return pg_atomic_read_u32(&cluster_grd_state->offpath_boot_decided) != 0;
+}
+
+/* Mark the off-path boot decision complete (idempotent; single writer = the
+ * reconfig LMON tick).  After this the boot barrier lifts; on a crash-rejoin
+ * the caller has already armed the self-fence, which keeps home blocks
+ * RECOVERING via the existing join fence check. */
+void
+cluster_grd_set_offpath_boot_decided(void)
+{
+	if (cluster_grd_state != NULL)
+		pg_atomic_write_u32(&cluster_grd_state->offpath_boot_decided, 1);
+}
+
 /* spec-4.6 D5 — bulk counter snapshot for the dump path. */
 void
 cluster_grd_recovery_counters_snapshot(ClusterGrdRecoveryCounters *out)

src/backend/cluster/cluster_reconfig.c

@@ -2136,6 +2136,96 @@ cluster_reconfig_joiner_self_tick(void)
 }
 
 
+/*
+ * cluster_reconfig_offpath_rejoin_tick -- crash-rejoin re-declare barrier
+ * (Shape A), the cluster.online_join=off counterpart of the joiner self-tick.
+ *
+ *	With online_join=off the joiner self-tick above early-returns, so a node
+ *	that crash-restarts into a running cluster keeps its shmem-init
+ *	self_join_admitted = 1 (cluster_reconfig.c:206) and boots straight to a
+ *	writable MEMBER with an EMPTY GRD and no re-declare episode.  For a block
+ *	whose static home is self, the acquire path then cold-grants from the
+ *	stale/empty disk page (silent stale read / silently-diverging write — the
+ *	P0).  The phase-gate boot barrier (cluster_gcs_block.c) fences self-home
+ *	blocks RECOVERING from process start (the flag defaults 0) until THIS tick
+ *	classifies the incarnation, so there is zero cold-serve window.
+ *
+ *	Decision (LMON single-writer; the barrier flag and self_join_admitted are
+ *	both flipped here):
+ *	  - single declared node       -> decided, no fence (no peer can conflict)
+ *	  - crash-rejoin (already run)  -> arm the self-fence, THEN demote
+ *	                                   self_join_admitted to 0 (Rule 8.A: never
+ *	                                   raised to 1 before the fence is armed);
+ *	                                   home blocks stay RECOVERING via the join
+ *	                                   fence, writes fail-closed 53R60.  The
+ *	                                   survivor re-declare self-heal (fence
+ *	                                   lift) is a separate spec (Shape B).
+ *	  - cold bootstrap at INITIAL   -> decided, no fence (fresh cluster, no
+ *	                                   stale home blocks)
+ *	  - undecided                   -> leave the barrier up (fail-closed),
+ *	                                   retry next tick
+ *
+ *	online_join=on takes its own joiner_self_tick / note_self_admitted path
+ *	and never enters here.
+ */
+static void
+cluster_reconfig_offpath_rejoin_tick(void)
+{
+	static bool offpath_decided_local = false;
+
+	if (ReconfigShmem == NULL || cluster_online_join)
+		return; /* off path only */
+	if (cluster_node_id < 0 || cluster_node_id >= CLUSTER_MAX_NODES)
+		return;
+	if (offpath_decided_local)
+		return; /* once per incarnation (LMON-local) */
+	if (cluster_reconfig_is_removed_unlocked(cluster_node_id))
+		return; /* a removed node keeps its 53R64 self-demote gate */
+
+	if (cluster_conf_node_count() <= 1) {
+		/* Lone declared node: no peer can hold a conflicting copy, so there is
+		 * nothing to re-declare — decide at once so the boot barrier never
+		 * fences a single-node deployment. */
+		cluster_grd_set_offpath_boot_decided();
+		offpath_decided_local = true;
+		return;
+	}
+
+	if (cluster_reconfig_cluster_already_running()) {
+		uint8 self_set[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES] = { 0 };
+
+		self_set[cluster_node_id >> 3] = (uint8)(1u << (cluster_node_id & 7));
+		cluster_grd_arm_join_pcm_fence(self_set); /* fence FIRST (8.A) */
+
+		LWLockAcquire(&ReconfigShmem->lock, LW_EXCLUSIVE);
+		ReconfigShmem->self_join_admitted = 0; /* then close the write gate */
+		LWLockRelease(&ReconfigShmem->lock);
+
+		cluster_grd_set_offpath_boot_decided();
+		offpath_decided_local = true;
+
+		ereport(LOG,
+				(errmsg("cluster membership: node %d crash-rejoined a running cluster with "
+						"cluster.online_join=off — home blocks fenced and writes closed "
+						"(53R60) to avoid serving stale ownership",
+						cluster_node_id),
+				 errhint("Enable cluster.online_join for an online re-declare rejoin, or "
+						 "cold-restart the cluster. Reads of peer-mastered blocks and "
+						 "non-home work are unaffected.")));
+		return;
+	}
+
+	if (cluster_reconfig_bootstrap_quorum_at_initial()) {
+		/* Cold bootstrap: fresh cluster at INITIAL, no stale home blocks. */
+		cluster_grd_set_offpath_boot_decided();
+		offpath_decided_local = true;
+		return;
+	}
+
+	/* UNDECIDED: keep the boot barrier up (fail-closed) and retry next tick. */
+}
+
+
 /* ============================================================
  * Step 2 D2 — cluster_reconfig_lmon_tick body.
  *
@@ -2208,6 +2298,14 @@ cluster_reconfig_lmon_tick(void)
 	 */
 	cluster_reconfig_joiner_self_tick();
 
+	/*
+	 * Shape A (crash-rejoin re-declare barrier) — the online_join=off
+	 * counterpart: arm the self-fence + close the write gate if THIS node
+	 * crash-rejoined a running cluster, and lift the boot barrier once the
+	 * bootstrap-vs-rejoin classification is proven.  No-op on online_join=on.
+	 */
+	cluster_reconfig_offpath_rejoin_tick();
+
 	/*
 	 * spec-5.15 D1 (INV-J8): the membership-state table — NOT raw CSSD — is the
 	 * decision SSOT for the survivor / coordinator set.  Maintain it and build

src/include/cluster/cluster_grd.h

@@ -398,6 +398,19 @@ typedef struct ClusterGrdShared {
 	pg_atomic_uint64 join_pcm_fence_member_epoch[CLUSTER_MAX_NODES];
 	pg_atomic_uint32 recovery_direction;
 
+	/*
+	 * TT lane / crash-rejoin re-declare barrier (Shape A) — off-path boot
+	 * barrier.  0 = the online_join=off rejoin/bootstrap decision has NOT
+	 * run this incarnation, so this node cannot prove ownership of its
+	 * home blocks; the phase gate fences self-home blocks RECOVERING until
+	 * it flips (fail-closed boot-race elimination).  Set to 1 by the
+	 * off-path rejoin tick once it has DECIDED (crash-rejoin -> also armed
+	 * the self-fence; cold-bootstrap -> no fence).  Per-incarnation:
+	 * re-zeroed by the !found shmem init.  online_join=on never consults
+	 * it (that path has its own admission + join fence).
+	 */
+	pg_atomic_uint32 offpath_boot_decided;
+
 	/* spec-5.16 D5 — join-direction remaster counters (dump_grd grd_recovery
 	 * segment;  kept distinct from the failure-driven remaster_* counters so
 	 * ops can tell the two remaster kinds apart — §8 Q6-A). */
@@ -669,6 +682,9 @@ extern void cluster_grd_inc_stale_request_drop(void);
 extern void cluster_grd_inc_block_path_failclosed(void);
 /* spec-5.16 D5 — join-direction 53R9L fail-closed bump (requester + master gate). */
 extern void cluster_grd_inc_join_block_failclosed(void);
+/* Shape A (crash-rejoin re-declare barrier) — off-path boot barrier flag. */
+extern bool cluster_grd_offpath_boot_decided(void);
+extern void cluster_grd_set_offpath_boot_decided(void);
 
 /* spec-4.6 D5 — bulk snapshot of the 13 grd_recovery counters for the
  * pg_cluster_state dump (category 'grd_recovery';  one t/249 leg each). */

src/test/cluster_tap/t/404_crash_rejoin_stale_read_write_2node.pl

@@ -162,7 +162,7 @@ sub membership_state
 diag("selected vuln=" . ($vuln // 'NONE') . " safe=" . ($safe // 'none'));
 
 SKIP: {
-	skip 'no node1-home candidate surfaced this run', 3 unless defined $vuln;
+	skip 'no node1-home candidate surfaced this run', 4 unless defined $vuln;
 
 	# ========================================================
 	# L4 HARD: node1's read of its home table is COHERENT or FAIL-CLOSED,
@@ -187,6 +187,34 @@ sub membership_state
 	ok($wr != 0 || ($n1 eq $n0),
 		'L5 HARD: home-block write is fail-closed or coherent, never a silent split');
 
+	# ========================================================
+	# L5b HARD (boot-race, Shape A命门): reads issued CONCURRENTLY, the
+	# instant node1 accepts connections after a fresh bounce, must ALL be
+	# fail-closed for a node1-home table — not one silent 0 may slip
+	# through the boot-to-decision window.  The phase-gate boot barrier
+	# (flag defaults 0 at shmem init) fences self-home blocks RECOVERING
+	# from process start, so every early read errors 53R9L; RED on the
+	# pre-fix binary (a burst of silent 0s).
+	# ========================================================
+	$pair->node1->stop('fast');
+	$pair->node1->start;   # do NOT wait for settle — probe the boot window
+	my ($silent0, $failclosed, $coherent) = (0, 0, 0);
+	for my $i (1 .. 40)
+	{
+		my ($rc, $out, $err) = psql_row($pair->node1, "SELECT count(*) FROM $vuln");
+		if    ($rc != 0)               { $failclosed++; }
+		elsif ($out eq '0')            { $silent0++; }
+		else                           { $coherent++; }   # 64 = decided+served
+		last if $coherent >= 3;        # barrier lifted + coherent: window over
+		usleep(150_000);
+	}
+	diag("L5b boot-window reads: fail-closed=$failclosed silent0=$silent0 coherent=$coherent");
+	is($silent0, 0,
+		'L5b HARD: zero silent 0-row reads in the boot-to-decision window');
+	# resettle for L6
+	$pair->wait_for_peer_state(0, 1, 'connected', 60);
+	usleep(1_000_000);
+
 	# ========================================================
 	# L6 HARD (blast radius): node1-first / node0-last flush, then a full
 	# restart — node0's 64 committed rows must SURVIVE on disk.  RED