Merge pull request #58 from sqlrush/fix/s3-crash-rejoin-redeclare-barrier

fix(cluster): crash-rejoin re-declare barrier (Shape A) + cold-GRD fail-closed

Author: sqlrush

src/backend/cluster/cluster_debug.c

@@ -1371,6 +1371,10 @@ dump_grd_recovery(ReturnSetInfo *rsinfo)
 			 fmt_int64((int64)c.join_block_views_rebuilt));
 	emit_row(rsinfo, "grd_recovery", "join_block_recovering_failclosed",
 			 fmt_int64((int64)c.join_block_recovering_failclosed));
+	/* Shape A (crash-rejoin re-declare barrier): off-path crash-rejoin fence-arm
+	 * events (standalone counter, not part of the snapshot struct). */
+	emit_row(rsinfo, "grd_recovery", "offpath_crash_rejoin_fenced",
+			 fmt_int64((int64)cluster_grd_offpath_crash_rejoin_fenced_count()));
 }
 
 /*

src/backend/cluster/cluster_gcs_block.c

@@ -1275,9 +1275,43 @@ cluster_gcs_block_fallback_verify_refresh(BufferDesc *buf, BufferTag tag, SCN ex
 	GcsLostWriteVerdict verdict;
 	bool refreshed = false; /* S3 forensics — storage re-read happened */
 
-	if (buf == NULL || !SCN_VALID(expected_scn))
+	if (buf == NULL)
 		return;
 
+	/*
+	 * fix 2 (crash-rejoin re-declare barrier, defense in depth): an InvalidScn
+	 * master watermark normally SKIPs (not SCN-tracked / old-binary master /
+	 * holder re-ack).  But if THIS self-home block is under the off-path crash-
+	 * rejoin fence, the local GRD watermark was wiped by the restart, so an
+	 * Invalid watermark can mask a stale home block — fail-closed instead of
+	 * SKIP, except for a genuine extension block (never cross-node written).
+	 * This is a second line behind the phase-gate boot barrier, which already
+	 * fences the self-home block before the acquire reaches here.
+	 */
+	{
+		bool self_fenced
+			= (!cluster_online_join && cluster_gcs_lookup_master_static(tag) == cluster_node_id
+			   && cluster_conf_node_count() > 1 && !cluster_grd_offpath_boot_decided());
+		ClusterColdGrdVerdict cv = cluster_gcs_cold_grd_watermark_verdict(
+			SCN_VALID(expected_scn), self_fenced,
+			self_fenced && cluster_bufmgr_block_is_extension_for_gcs(tag));
+
+		if (cv == CLUSTER_COLD_GRD_SKIP)
+			return;
+		if (cv == CLUSTER_COLD_GRD_FAIL_CLOSED) {
+			pg_atomic_fetch_add_u64(&ClusterGcsBlock->fallback_scn_failclosed_count, 1);
+			ereport(ERROR,
+					(errcode(ERRCODE_CLUSTER_GCS_BLOCK_RESOURCE_RECOVERING),
+					 errmsg("crash-rejoin: cannot prove home block ownership after restart "
+							"(cold GRD watermark) for tag spc=%u db=%u rel=%u block=%u",
+							tag.spcOid, tag.dbOid, tag.relNumber, tag.blockNum),
+					 errhint("The block resource is recovering after an unclean restart; retry the "
+							 "transaction, or enable cluster.online_join for an online re-declare "
+							 "rejoin.")));
+		}
+		/* CLUSTER_COLD_GRD_PROVE: expected_scn valid — run the normal verdict. */
+	}
+
 	page_scn = cluster_bufmgr_read_block_scn_for_gcs(buf);
 	verdict = gcs_block_lost_write_verdict(expected_scn, page_scn);
 	if (verdict == GCS_LOST_WRITE_PASS) {
@@ -1450,6 +1484,28 @@ cluster_gcs_block_phase_for_tag(BufferTag tag)
 	 */
 	static_master = cluster_gcs_lookup_master_static(tag);
 
+	/*
+	 * TT lane / crash-rejoin re-declare barrier (Shape A) — off-path boot
+	 * barrier.  With cluster.online_join=off a node that boots into a running
+	 * cluster self-admits immediately (cluster_reconfig.c:206) with an EMPTY
+	 * GRD and NO re-declare episode: for a block whose STATIC home is self,
+	 * the acquire path would find master==self, read the empty local GRD, and
+	 * cold-grant from the stale/empty disk page — a silent stale READ and a
+	 * silently-diverging WRITE (the P0).  Until the off-path rejoin tick has
+	 * classified this incarnation (crash-rejoin -> self-fence armed;
+	 * bootstrap -> nothing), self cannot prove its home blocks' ownership, so
+	 * fence them RECOVERING.  Both reads and writes reach this gate via
+	 * cluster_pcm_lock_acquire_buffer, so this closes the boot-to-decision
+	 * race with ZERO cold-serve window (Rule 8.A: uncertain -> fail-closed).
+	 * Skipped for online_join=on (its admission + join fence govern) and for
+	 * a single declared node (no peer can hold a conflicting copy).
+	 */
+	if (!cluster_online_join && static_master == cluster_node_id && cluster_conf_node_count() > 1
+		&& !cluster_grd_offpath_boot_decided()) {
+		cluster_grd_inc_join_block_failclosed();
+		return GCS_BLOCK_RECOVERING;
+	}
+
 	/*
 	 * spec-5.16 D3 (r1 P1-C) — online-join PCM block snap-back fence, placed
 	 * BEFORE the non-DEAD-static-master early NORMAL below.  When a joiner (a

src/backend/cluster/cluster_grd.c

@@ -763,11 +763,14 @@ cluster_grd_shmem_init(void)
 		for (i = 0; i < CLUSTER_MAX_NODES; i++)
 			pg_atomic_init_u64(&cluster_grd_state->join_pcm_fence_member_epoch[i], 0);
 		pg_atomic_init_u32(&cluster_grd_state->recovery_direction, (uint32)GRD_REMASTER_DIR_NONE);
+		/* Shape A: off-path boot barrier starts UNDECIDED (fail-closed). */
+		pg_atomic_init_u32(&cluster_grd_state->offpath_boot_decided, 0);
 		pg_atomic_init_u64(&cluster_grd_state->join_remaster_started_count, 0);
 		pg_atomic_init_u64(&cluster_grd_state->join_remaster_done_count, 0);
 		pg_atomic_init_u64(&cluster_grd_state->join_shards_remastered_count, 0);
 		pg_atomic_init_u64(&cluster_grd_state->join_block_views_rebuilt_count, 0);
 		pg_atomic_init_u64(&cluster_grd_state->join_block_recovering_failclosed_count, 0);
+		pg_atomic_init_u64(&cluster_grd_state->offpath_crash_rejoin_fenced_count, 0);
 	}
 
 	/* spec-2.15 v0.4 P1.1:  entry HTAB allocation gated on GUC.  GUC=0
@@ -1799,6 +1802,53 @@ cluster_grd_inc_join_block_failclosed(void)
 		pg_atomic_fetch_add_u64(&cluster_grd_state->join_block_recovering_failclosed_count, 1);
 }
 
+/*
+ * Shape A (crash-rejoin re-declare barrier) — off-path boot-barrier flag.
+ *
+ *	cluster_grd_offpath_boot_decided() -- false until the off-path rejoin
+ *	tick has classified this incarnation (bootstrap vs crash-rejoin).  The
+ *	phase gate fences self-home blocks RECOVERING while false, so a node
+ *	that self-admits at boot with cluster.online_join=off cannot cold-serve
+ *	its home blocks before it has proven their ownership (Rule 8.A).
+ *	Defaults DECIDED (true) when the GRD region is absent so a cluster-off
+ *	build never fences.
+ */
+bool
+cluster_grd_offpath_boot_decided(void)
+{
+	if (cluster_grd_state == NULL)
+		return true;
+	return pg_atomic_read_u32(&cluster_grd_state->offpath_boot_decided) != 0;
+}
+
+/* Mark the off-path boot decision complete (idempotent; single writer = the
+ * reconfig LMON tick).  After this the boot barrier lifts; on a crash-rejoin
+ * the caller has already armed the self-fence, which keeps home blocks
+ * RECOVERING via the existing join fence check. */
+void
+cluster_grd_set_offpath_boot_decided(void)
+{
+	if (cluster_grd_state != NULL)
+		pg_atomic_write_u32(&cluster_grd_state->offpath_boot_decided, 1);
+}
+
+/* Shape A observability: count an off-path crash-rejoin fence-arm (LMON single
+ * writer; read for dump_grd + t/404). */
+void
+cluster_grd_inc_offpath_crash_rejoin_fenced(void)
+{
+	if (cluster_grd_state != NULL)
+		pg_atomic_fetch_add_u64(&cluster_grd_state->offpath_crash_rejoin_fenced_count, 1);
+}
+
+uint64
+cluster_grd_offpath_crash_rejoin_fenced_count(void)
+{
+	if (cluster_grd_state == NULL)
+		return 0;
+	return pg_atomic_read_u64(&cluster_grd_state->offpath_crash_rejoin_fenced_count);
+}
+
 /* spec-4.6 D5 — bulk counter snapshot for the dump path. */
 void
 cluster_grd_recovery_counters_snapshot(ClusterGrdRecoveryCounters *out)

src/backend/cluster/cluster_qvotec.c

@@ -140,11 +140,34 @@ typedef struct ClusterQvotecShmem {
 	pg_atomic_uint32 poll_cycle_count;
 	pg_atomic_uint32 torn_write_detect_count;
 	pg_atomic_uint32 _pad;
-	uint8 _reserved[64];
+	/*
+	 * Merge-order reservation (守门 07-15): the convert-queue lane claims
+	 * offset 64..71 for its self_incarnation (pg_atomic_uint64, commit
+	 * ee536b5bb7, StaticAssert-pinned).  Queue merges first; this lane rebases
+	 * after and drops this placeholder so self_incarnation occupies 64..71 and
+	 * prior_unclean_death stays at 72.  Keeping the byte layout identical now
+	 * makes that rebase a no-op on the wire/shmem image.
+	 */
+	uint8 _reserved_queue_self_incarnation[8]; /* offset 64..71 */
+	/*
+	 * Crash-rejoin re-declare barrier (Shape A) — set ONCE at qvotec startup
+	 * (before the READY publish), read-only thereafter: 1 iff this node's
+	 * prior-incarnation self-slot on the voting disk still had the ALIVE flag
+	 * set (a clean shutdown clears it via qvotec_clear_self_alive_on_clean_
+	 * shutdown; a crash / immediate stop does NOT), i.e. this boot follows an
+	 * UNCLEAN death.  The off-path rejoin tick fences self-home blocks +
+	 * closes the write gate on this, so a crash-rejoined node never cold-
+	 * serves stale ownership even when it restarts faster than the survivor's
+	 * dead-deadband (the epoch signal is INITIAL on both sides in that race).
+	 */
+	pg_atomic_uint32 prior_unclean_death; /* offset 72..75 */
+	uint8 _reserved[52];
 } ClusterQvotecShmem;
 
 StaticAssertDecl(sizeof(ClusterQvotecShmem) == 128,
 				 "ClusterQvotecShmem must be exactly 128 bytes (2 cache lines)");
+StaticAssertDecl(offsetof(ClusterQvotecShmem, prior_unclean_death) == 72,
+				 "prior_unclean_death must sit at offset 72 (queue lane owns 64..71)");
 
 
 static ClusterQvotecShmem *QvotecShmem = NULL;
@@ -267,6 +290,9 @@ cluster_qvotec_shmem_init(void)
 		pg_atomic_init_u32(&QvotecShmem->poll_cycle_count, 0);
 		pg_atomic_init_u32(&QvotecShmem->torn_write_detect_count, 0);
 		pg_atomic_init_u32(&QvotecShmem->_pad, 0);
+		memset(QvotecShmem->_reserved_queue_self_incarnation, 0,
+			   sizeof(QvotecShmem->_reserved_queue_self_incarnation));
+		pg_atomic_init_u32(&QvotecShmem->prior_unclean_death, 0);
 		memset(QvotecShmem->_reserved, 0, sizeof(QvotecShmem->_reserved));
 	}
 }
@@ -372,6 +398,23 @@ cluster_qvotec_get_disks_total_count(void)
 	return (int)pg_atomic_read_u32(&QvotecShmem->disks_total_count);
 }
 
+/*
+ * cluster_qvotec_prior_unclean_death -- crash-rejoin re-declare barrier
+ * (Shape A).  True iff this node's prior-incarnation self-slot on the voting
+ * disk still carried the ALIVE flag at startup (an unclean death: a crash /
+ * immediate stop that skipped the clean-shutdown ALIVE blank).  Latched once
+ * before the READY publish; stable for the incarnation.  False when qvotec is
+ * absent (no voting disks) so a diskless / single-node deployment is never
+ * fenced by this signal.
+ */
+bool
+cluster_qvotec_prior_unclean_death(void)
+{
+	if (QvotecShmem == NULL)
+		return false;
+	return pg_atomic_read_u32(&QvotecShmem->prior_unclean_death) != 0;
+}
+
 uint64
 cluster_qvotec_get_current_epoch_at_boot(void)
 {
@@ -1749,7 +1792,7 @@ ClusterQvotecMain(void)
 		bool ghost_fresh = false;
 		int d;
 
-		for (d = 0; d < qvotec_n_disks && !ghost_fresh; d++) {
+		for (d = 0; d < qvotec_n_disks; d++) {
 			ClusterVotingSlot probe;
 			ClusterVotingDiskIoState rrc;
 
@@ -1759,14 +1802,29 @@ ClusterQvotecMain(void)
 			if (probe.generation == 0)
 				continue; /* never written */
 			if (!(probe.flags & CLUSTER_VOTING_SLOT_FLAG_ALIVE))
-				continue; /* prior shutdown cleared ALIVE — ok */
+				continue; /* prior shutdown cleared ALIVE — clean death, ok */
 			if (probe.incarnation == qvotec_self_incarnation)
 				continue; /* same incarnation — impossible but defensive */
+
+			/*
+			 * Crash-rejoin re-declare barrier (Shape A) — a prior-incarnation
+			 * self-slot that still carries ALIVE means the previous postmaster
+			 * of THIS node died WITHOUT running the clean-shutdown blank
+			 * (qvotec_clear_self_alive_on_clean_shutdown), i.e. an UNCLEAN
+			 * death.  Latch it REGARDLESS of freshness: a stale ALIVE ghost is
+			 * still proof we crashed (we just crashed longer ago), and the
+			 * fence must engage on a fast rejoin where the survivor has not yet
+			 * advanced its epoch (the epoch signal is INITIAL on both sides).
+			 * Single writer, before the READY publish; read-only afterwards.
+			 */
+			if (QvotecShmem != NULL)
+				pg_atomic_write_u32(&QvotecShmem->prior_unclean_death, 1);
+
 			if (probe.heartbeat_ts_us == 0)
 				continue;
 			if (now_us > probe.heartbeat_ts_us
 				&& (now_us - probe.heartbeat_ts_us) > heartbeat_timeout_us)
-				continue; /* already stale */
+				continue; /* already stale — no fast-restart Q6 sleep needed */
 			ghost_fresh = true;
 		}