From 63a75c74a4f548418422dd03498a56a52fb05093 Mon Sep 17 00:00:00 2001
From: Plesoun <jakub.schier@gmail.com>
Date: Fri, 19 Jun 2026 13:00:33 +0200
Subject: [PATCH 1/7] flow fixes

---
 .../Files/mongo/script/mongo-init.js          | 164 ++++++++++++++++--
 1 file changed, 146 insertions(+), 18 deletions(-)

diff --git a/Site/ASAB Maestro/Files/mongo/script/mongo-init.js b/Site/ASAB Maestro/Files/mongo/script/mongo-init.js
index 81de2e24..d1bb0f6c 100644
--- a/Site/ASAB Maestro/Files/mongo/script/mongo-init.js	
+++ b/Site/ASAB Maestro/Files/mongo/script/mongo-init.js	
@@ -58,12 +58,53 @@ function defaultWriteConcernFromEnv() {
 	return { w: String(w).trim(), wtimeout: 0 }
 }
 
+/**
+ * Count reachable voting members (excluding arbiters) from rs.status().
+ * Used to determine if "majority" write concern is achievable.
+ */
+function reachableVotingMemberCount() {
+	let count = 0
+	try {
+		const st = rs.status()
+		for (let i = 0; i < st.members.length; i++) {
+			const m = st.members[i]
+			// Skip arbiters (they don't hold data, but they do vote)
+			// Only count members that are in a healthy state (PRIMARY=1, SECONDARY=2, STARTUP2=5, RECOVERING=3)
+			const isHealthy = [1, 2, 3, 5].includes(m.state)
+			if (isHealthy) {
+				const conf = rs.conf()
+				const memberConf = conf.members.find(cm => cm.host === m.name)
+				if (memberConf && memberConf.arbiterOnly !== true) {
+					count++
+				}
+			}
+		}
+	} catch (e) {
+		print("[cwwc] could not determine reachable voting members: " + e.message)
+	}
+	return count
+}
+
 function ensureClusterWideDefaultWriteConcern() {
 	const dwc = defaultWriteConcernFromEnv()
+	// When dwc is "majority", check if we can actually achieve majority.
+	// If members are being decommissioned (unreachable), use a weaker write concern.
+	const reachableCount = reachableVotingMemberCount()
+	const wc = { w: "majority", wtimeout: 15000 }
+	if (dwc.w === "majority" && reachableCount > 0) {
+		// If majority requires more members than are reachable, use w=1 instead.
+		const current = rs.conf()
+		const totalVotingMembers = current.members.filter(m => m.votes > 0).length
+		const majority = Math.floor(totalVotingMembers / 2) + 1
+		if (majority > reachableCount) {
+			print("[cwwc] majority=" + majority + " but only " + reachableCount + " reachable; using w=1 for setDefaultRWConcern")
+			wc.w = 1
+		}
+	}
 	const res = db.adminCommand({
 		setDefaultRWConcern: 1,
 		defaultWriteConcern: dwc,
-		writeConcern: { w: "majority", wtimeout: 15000 },
+		writeConcern: wc,
 	})
 	if (!res.ok) {
 		const msg = res.errmsg != null ? res.errmsg : JSON.stringify(res)
@@ -265,6 +306,67 @@ function connectedMemberHost() {
 	return null
 }
 
+/**
+ * Flow 2: mongo instance removed from cluster model — host no longer in replica-set.json.
+ * Remaining desired config must still list at least one data member; cannot remove this primary's host.
+ */
+/**
+ * Check if a member is reachable from rs.status().
+ * Returns true if healthy, false if unhealthy/unreachable, true if not found (assume reachable).
+ */
+function isMemberReachable(host) {
+	try {
+		const st = rs.status()
+		for (let i = 0; i < st.members.length; i++) {
+			if (normalizeMemberHost(st.members[i].name) === normalizeMemberHost(host)) {
+				return st.members[i].health === 1
+			}
+		}
+	} catch (e) {
+		// Cannot read status — assume reachable (will try rs.remove first)
+	}
+	return true
+}
+
+/**
+ * Force-reconfig to remove members from the replica set.
+ * Used when members are unreachable and rs.remove() would hang indefinitely.
+ * Builds a new config with only the surviving members and applies with {force: true}.
+ */
+function forceReconfigToRemoveMembers(desired, current, toRemoveSet) {
+	const hostsToRemove = Array.from(toRemoveSet)
+	print("[force reconfig] Removing unreachable members via rs.reconfig({force:true}): " + hostsToRemove.join(", "))
+	const filtered = current.members.filter(function(m) {
+		const h = normalizeMemberHost(m.host)
+		return !toRemoveSet.has(h)
+	})
+	if (filtered.length === 0) {
+		throw new Error("Force reconfig would remove ALL members — refusing (need at least 1 data member)")
+	}
+	// Validate at least one data member remains
+	let hasData = false
+	for (let i = 0; i < filtered.length; i++) {
+		if (filtered[i].arbiterOnly !== true) {
+			hasData = true
+			break
+		}
+	}
+	if (!hasData) {
+		throw new Error("Force reconfig would remove all data-bearing members — refusing")
+	}
+	const newConfig = {
+		_id: current._id,
+		members: filtered,
+		version: current.version + 1,
+		protocolVersion: current.protocolVersion !== undefined ? current.protocolVersion : 1,
+	}
+	if (current.settings !== undefined && current.settings !== null) {
+		newConfig.settings = Object.assign({}, current.settings)
+	}
+	const result = rs.reconfig(newConfig, {force: true})
+	print("[force reconfig] ok=" + result.ok + " (new version=" + newConfig.version + ")")
+}
+
 /**
  * Flow 2: mongo instance removed from cluster model — host no longer in replica-set.json.
  * Remaining desired config must still list at least one data member; cannot remove this primary's host.
@@ -275,29 +377,42 @@ function removeMembersDroppedFromDesired(desired, current) {
 	}
 	const wantHosts = desiredHostSet(desired)
 	const selfH = connectedMemberHost()
-	const toRemove = []
+	const toRemoveReachable = []
+	const toRemoveUnreachable = new Set()
 	for (const cm of current.members) {
 		const h = normalizeMemberHost(cm.host)
 		if (!wantHosts.has(h)) {
-			toRemove.push(h)
-		}
-	}
-	for (let i = 0; i < toRemove.length; i++) {
-		const h = toRemove[i]
-		print("[flow 2 decommission] Removing member not present in desired config: " + h)
-		if (selfH !== null && h === selfH) {
-			throw new Error(
-				"Desired config omits the current primary host (" +
-					h +
+			if (selfH !== null && h === selfH) {
+				throw new Error(
+					"Desired config omits the current primary host (" +
+						h +
 					"). Step down another primary first, or keep this member in replica-set.json until then."
-			)
+				)
+			}
+			if (isMemberReachable(h)) {
+				toRemoveReachable.push(h)
+			} else {
+				toRemoveUnreachable.add(h)
+			}
 		}
+	}
+	// Remove reachable members via rs.remove() (they respond quickly)
+	for (let i = 0; i < toRemoveReachable.length; i++) {
+		const h = toRemoveReachable[i]
+		print("[flow 2 decommission] Removing reachable member: " + h)
 		try {
 			rs.remove(h)
+			current = rs.conf()
 		} catch (e) {
-			print("rs.remove failed (may retry next init): " + e.message)
+			print("rs.remove failed (will handle via force reconfig): " + e.message)
+			toRemoveUnreachable.add(h)
 		}
 	}
+	// Remove unreachable members via force reconfig (avoids hanging on rs.remove)
+	if (toRemoveUnreachable.size > 0) {
+		current = rs.conf()
+		forceReconfigToRemoveMembers(desired, current, toRemoveUnreachable)
+	}
 }
 
 /**
@@ -366,8 +481,11 @@ function assertArbiterToDataRequiresManual(desired, current) {
 			continue
 		}
 		const wantArbiter = dm.arbiterOnly === true
+		const wantData = dm.arbiterOnly === false
 		const isArbiter = cm.arbiterOnly === true
-		if (isArbiter && !wantArbiter) {
+		// Only fail if operator EXPLICITLY wants arbiter->data (arbiterOnly: false in desired).
+		// If desired config omits arbiterOnly for an existing arbiter, preserve arbiter status.
+		if (isArbiter && wantData) {
 			offenders.push(h)
 		}
 	}
@@ -470,6 +588,11 @@ function assertNoIllegalArbiterTransition(desired, current) {
 		if (!cm) {
 			continue
 		}
+		// Only fail if operator EXPLICITLY wants to change arbiterOnly (true or false in desired).
+		// If desired config omits arbiterOnly for an existing member, preserve current role.
+		if (dm.arbiterOnly === undefined || dm.arbiterOnly === null) {
+			continue
+		}
 		const wantArbiter = dm.arbiterOnly === true
 		const isArbiter = cm.arbiterOnly === true
 		if (isArbiter !== wantArbiter) {
@@ -492,11 +615,14 @@ function assertNoIllegalArbiterTransition(desired, current) {
  */
 function mergeLiveMemberWithDesired(dm, cm) {
 	const h = normalizeMemberHost(dm.host)
-	const wantArbiter = dm.arbiterOnly === true
 	const out = Object.assign({}, cm)
 	out._id = cm._id
 	out.host = h
-	out.arbiterOnly = wantArbiter
+	// Only change arbiterOnly if operator explicitly sets it (true or false).
+	// If desired config omits arbiterOnly for an existing member, preserve current role.
+	if (dm.arbiterOnly !== undefined && dm.arbiterOnly !== null) {
+		out.arbiterOnly = dm.arbiterOnly === true
+	}
 	if (dm.priority !== undefined) {
 		out.priority = dm.priority
 	}
@@ -512,8 +638,10 @@ function mergeLiveMemberWithDesired(dm, cm) {
 	if (out.votes === undefined || out.votes === null) {
 		out.votes = 1
 	}
+	// Fix: use out.arbiterOnly instead of undefined wantArbiter
+	const isArbiter = out.arbiterOnly === true
 	if (out.priority === undefined || out.priority === null) {
-		out.priority = wantArbiter ? 0 : 1
+		out.priority = isArbiter ? 0 : 1
 	}
 	return out
 }

From 741c54771072f142f71729565e5864ed662919d6 Mon Sep 17 00:00:00 2001
From: Plesoun <jakub.schier@gmail.com>
Date: Mon, 17 Aug 2026 09:44:54 +0200
Subject: [PATCH 2/7] retry logic, tests on asp

---
 .../Files/mongo/script/mongo-init.js          | 290 +++++++-----------
 1 file changed, 116 insertions(+), 174 deletions(-)

diff --git a/Site/ASAB Maestro/Files/mongo/script/mongo-init.js b/Site/ASAB Maestro/Files/mongo/script/mongo-init.js
index d1bb0f6c..a1c10e4d 100644
--- a/Site/ASAB Maestro/Files/mongo/script/mongo-init.js	
+++ b/Site/ASAB Maestro/Files/mongo/script/mongo-init.js	
@@ -25,26 +25,21 @@ const fs = require("fs")
  * arbiter or role change without wiping dbPath). Logs show stale local repl config (e.g. setVersion 214988)
  * while primary is much newer, "No initial sync required" + immediate abort.
  *
- * Fix (operator, on the BROKEN host — e.g. mongo-2 / ples01-mongo-arb):
+ * Fix (operator, on the BROKEN host):
  *  A) On a HEALTHY primary (mongosh): rs.status() / rs.conf(). If the broken host is still listed,
  *     rs.remove("host:27017") so the cluster stops expecting that mongod.
  *  B) Stop the broken mongod container/service (so it does not restart-crash forever).
  *  C) Delete ALL files under that instance's dbPath (e.g. /data/db including WiredTiger*, local/, journal).
- *     Docker: stop container, docker run --rm -v that_volume:/data/db busybox rm -rf /data/db/*
  *  D) Start mongod again with empty dbPath — it should NOT load old repl config; it may show as standalone
  *     or await rs.add.
  *  E) From primary: rs.add({ host: "...", ... }) or re-run this init against primary so the add-missing phase runs.
  *
- * Do not partially delete files; partial wipe keeps the crash. Optional last resort to get a shell on bad
- * data (not a substitute for wipe): start mongod once with setParameter enableMajorityReadConcern=false
- * (see MongoDB manual), then still plan a full wipe + rejoin.
+ * Do not partially delete files; partial wipe keeps the crash.
  */
 
 /**
  * Cluster-wide default write concern (MongoDB 5.0+ gate before many reconfigs).
  * Optional: MONGO_INIT_DEFAULT_WRITE_CONCERN_W — "majority" (default), "1", "2", etc.
- * Use w=1 only when you accept weaker defaults (e.g. temporary 1 data + 1 arbiter).
- * @see https://www.mongodb.com/docs/manual/reference/command/setDefaultRWConcern/
  */
 function defaultWriteConcernFromEnv() {
 	const w = process.env.MONGO_INIT_DEFAULT_WRITE_CONCERN_W
@@ -68,8 +63,6 @@ function reachableVotingMemberCount() {
 		const st = rs.status()
 		for (let i = 0; i < st.members.length; i++) {
 			const m = st.members[i]
-			// Skip arbiters (they don't hold data, but they do vote)
-			// Only count members that are in a healthy state (PRIMARY=1, SECONDARY=2, STARTUP2=5, RECOVERING=3)
 			const isHealthy = [1, 2, 3, 5].includes(m.state)
 			if (isHealthy) {
 				const conf = rs.conf()
@@ -87,12 +80,9 @@ function reachableVotingMemberCount() {
 
 function ensureClusterWideDefaultWriteConcern() {
 	const dwc = defaultWriteConcernFromEnv()
-	// When dwc is "majority", check if we can actually achieve majority.
-	// If members are being decommissioned (unreachable), use a weaker write concern.
 	const reachableCount = reachableVotingMemberCount()
 	const wc = { w: "majority", wtimeout: 15000 }
 	if (dwc.w === "majority" && reachableCount > 0) {
-		// If majority requires more members than are reachable, use w=1 instead.
 		const current = rs.conf()
 		const totalVotingMembers = current.members.filter(m => m.votes > 0).length
 		const majority = Math.floor(totalVotingMembers / 2) + 1
@@ -119,8 +109,6 @@ function normalizeMemberHost(host) {
 
 /**
  * rs.add / rs.addArb require the new member's mongod to already accept connections.
- * Init often races the arbiter container: wait until ping succeeds.
- * Env: MONGO_INIT_PEER_WAIT_ATTEMPTS (default 60), MONGO_INIT_PEER_WAIT_MS (default 5000).
  */
 function peerWaitConfigFromEnv() {
 	const attempts = parseInt(process.env.MONGO_INIT_PEER_WAIT_ATTEMPTS || "60", 10)
@@ -157,9 +145,7 @@ function waitForMongodOnHost(hostPort) {
 	}
 	throw new Error(
 		"Timed out waiting for mongod at " +
-			hp +
-			". Ensure the member container starts before init runs rs.add (compose depends_on / order), or increase MONGO_INIT_PEER_WAIT_ATTEMPTS. Last error: " +
-			lastMsg
+			hp + ". Last error: " + lastMsg
 	)
 }
 
@@ -175,7 +161,6 @@ function desiredHostSet(desired) {
 	return new Set(desired.members.map((dm) => normalizeMemberHost(dm.host)))
 }
 
-/** Voting data members in desired config (excludes arbiters). */
 function desiredDataMemberCount(desired) {
 	let n = 0
 	for (const dm of desired.members) {
@@ -186,7 +171,6 @@ function desiredDataMemberCount(desired) {
 	return n
 }
 
-/** Non-arbiter members in a live rs.conf() (same notion as desiredDataMemberCount). */
 function dataMemberCountFromConf(conf) {
 	let n = 0
 	for (let i = 0; i < conf.members.length; i++) {
@@ -207,7 +191,6 @@ function arbiterMemberCountFromConf(conf) {
 	return n
 }
 
-/** Live set is one data-bearing member and at least one arbiter (PSA transition source topology). */
 function isOneDataPlusArbiterTopology(conf) {
 	return dataMemberCountFromConf(conf) === 1 && arbiterMemberCountFromConf(conf) >= 1
 }
@@ -246,11 +229,6 @@ function rsStatusHasPrimary(st) {
 	return false
 }
 
-/**
- * After decommission (e.g. mongo removed from the arbiter host), the survivor can remain SECONDARY with no
- * PRIMARY until an election completes — or never, until replSetStepUp. Only safe when rs.conf() has exactly
- * one data-bearing member and desired matches (one data member).
- */
 function ensurePrimaryWhenSoleDataMember(desired) {
 	const conf = rs.conf()
 	if (dataMemberCountFromConf(conf) !== 1 || desiredDataMemberCount(desired) !== 1) {
@@ -269,9 +247,7 @@ function ensurePrimaryWhenSoleDataMember(desired) {
 			return
 		}
 		if (k + 1 < maxPoll) {
-			print(
-				"[sole data member] no PRIMARY yet (" + (k + 1) + "/" + maxPoll + "); sleeping " + ms + "ms"
-			)
+			print("[sole data member] no PRIMARY yet (" + (k + 1) + "/" + maxPoll + "); sleeping " + ms + "ms")
 			sleep(ms)
 		}
 	}
@@ -291,7 +267,6 @@ function ensurePrimaryWhenSoleDataMember(desired) {
 	}
 }
 
-/** Host:port of the member this shell is connected as (primary when reconfigure runs). */
 function connectedMemberHost() {
 	try {
 		const st = rs.status()
@@ -306,14 +281,6 @@ function connectedMemberHost() {
 	return null
 }
 
-/**
- * Flow 2: mongo instance removed from cluster model — host no longer in replica-set.json.
- * Remaining desired config must still list at least one data member; cannot remove this primary's host.
- */
-/**
- * Check if a member is reachable from rs.status().
- * Returns true if healthy, false if unhealthy/unreachable, true if not found (assume reachable).
- */
 function isMemberReachable(host) {
 	try {
 		const st = rs.status()
@@ -328,11 +295,6 @@ function isMemberReachable(host) {
 	return true
 }
 
-/**
- * Force-reconfig to remove members from the replica set.
- * Used when members are unreachable and rs.remove() would hang indefinitely.
- * Builds a new config with only the surviving members and applies with {force: true}.
- */
 function forceReconfigToRemoveMembers(desired, current, toRemoveSet) {
 	const hostsToRemove = Array.from(toRemoveSet)
 	print("[force reconfig] Removing unreachable members via rs.reconfig({force:true}): " + hostsToRemove.join(", "))
@@ -343,7 +305,6 @@ function forceReconfigToRemoveMembers(desired, current, toRemoveSet) {
 	if (filtered.length === 0) {
 		throw new Error("Force reconfig would remove ALL members — refusing (need at least 1 data member)")
 	}
-	// Validate at least one data member remains
 	let hasData = false
 	for (let i = 0; i < filtered.length; i++) {
 		if (filtered[i].arbiterOnly !== true) {
@@ -367,10 +328,6 @@ function forceReconfigToRemoveMembers(desired, current, toRemoveSet) {
 	print("[force reconfig] ok=" + result.ok + " (new version=" + newConfig.version + ")")
 }
 
-/**
- * Flow 2: mongo instance removed from cluster model — host no longer in replica-set.json.
- * Remaining desired config must still list at least one data member; cannot remove this primary's host.
- */
 function removeMembersDroppedFromDesired(desired, current) {
 	if (desiredDataMemberCount(desired) < 1) {
 		throw new Error("replica-set.json must keep at least one data (non-arbiter) member")
@@ -384,8 +341,7 @@ function removeMembersDroppedFromDesired(desired, current) {
 		if (!wantHosts.has(h)) {
 			if (selfH !== null && h === selfH) {
 				throw new Error(
-					"Desired config omits the current primary host (" +
-						h +
+					"Desired config omits the current primary host (" + h +
 					"). Step down another primary first, or keep this member in replica-set.json until then."
 				)
 			}
@@ -396,7 +352,6 @@ function removeMembersDroppedFromDesired(desired, current) {
 			}
 		}
 	}
-	// Remove reachable members via rs.remove() (they respond quickly)
 	for (let i = 0; i < toRemoveReachable.length; i++) {
 		const h = toRemoveReachable[i]
 		print("[flow 2 decommission] Removing reachable member: " + h)
@@ -408,16 +363,12 @@ function removeMembersDroppedFromDesired(desired, current) {
 			toRemoveUnreachable.add(h)
 		}
 	}
-	// Remove unreachable members via force reconfig (avoids hanging on rs.remove)
 	if (toRemoveUnreachable.size > 0) {
 		current = rs.conf()
 		forceReconfigToRemoveMembers(desired, current, toRemoveUnreachable)
 	}
 }
 
-/**
- * Add one member from desired JSON (preserves _id when Mongo accepts it).
- */
 function addMemberFromDesired(dm) {
 	const h = normalizeMemberHost(dm.host)
 	const doc = { host: h }
@@ -453,9 +404,11 @@ function addMemberFromDesired(dm) {
 					err = e2
 				}
 			}
+			/* FIX-2: also retry on "already exists" / "already in config" —
+			   another concurrent init may have just added this member */
 			const retryable =
 				t < maxTry &&
-				/Quorum check failed|Connection refused|NodeNotFound|timed out|Timeout/i.test(String(err.message))
+				/Quorum check failed|Connection refused|NodeNotFound|timed out|Timeout|ConfigurationInProgress|already exists|already in config/i.test(String((err.message || "") + " " + (err.codeName || "")))
 			if (retryable) {
 				print("rs.add retry " + t + "/" + maxTry + ": " + err.message)
 				sleep(delay)
@@ -466,47 +419,30 @@ function addMemberFromDesired(dm) {
 	}
 }
 
-/**
- * Arbiter → data in one init pass is unsafe: rs.remove then rs.add while the peer still has arbiter-era
- * local/ on disk often produces MongoDB 7 crash loops until dbPath is fully wiped. We refuse before any
- * replica-set or CWWC mutation so the live set is unchanged.
- */
 function assertArbiterToDataRequiresManual(desired, current) {
 	const curMap = currentMembersByHost(current)
 	const offenders = []
 	for (const dm of desired.members) {
 		const h = normalizeMemberHost(dm.host)
 		const cm = curMap[h]
-		if (!cm) {
-			continue
-		}
+		if (!cm) continue
 		const wantArbiter = dm.arbiterOnly === true
 		const wantData = dm.arbiterOnly === false
 		const isArbiter = cm.arbiterOnly === true
-		// Only fail if operator EXPLICITLY wants arbiter->data (arbiterOnly: false in desired).
-		// If desired config omits arbiterOnly for an existing arbiter, preserve arbiter status.
 		if (isArbiter && wantData) {
 			offenders.push(h)
 		}
 	}
-	if (offenders.length === 0) {
-		return
-	}
+	if (offenders.length === 0) return
 	throw new Error(
 		"Refusing init: arbiter → data promotion is manual-only for: " +
 			offenders.join(", ") +
-			". Automated rs.remove + re-add in the same run leaves stale repl metadata on disk and can break that mongod until dbPath is fully deleted. " +
-			"Do this, then re-run init: (1) On PRIMARY: rs.remove(\"host:port\") for each host above. " +
+			". Do this, then re-run init: (1) On PRIMARY: rs.remove(\"host:port\") for each host above. " +
 			"(2) On each former arbiter host: stop mongod, delete ALL files under dbPath, start mongod empty. " +
-			"(3) Re-run init (rs.add happens in add-missing phase) or rs.add manually. " +
-			"This run applied no replica-set or default RW concern changes."
+			"(3) Re-run init."
 	)
 }
 
-/**
- * Data-bearing member cannot become arbiter in place: remove from set first.
- * Operator must wipe dbPath and restart mongod; a later init pass adds the arbiter.
- */
 function removeMembersForArbiterConversion(desired, current) {
 	const curMap = currentMembersByHost(current)
 	let removed = false
@@ -515,9 +451,7 @@ function removeMembersForArbiterConversion(desired, current) {
 		const cm = curMap[h]
 		if (cm && dm.arbiterOnly === true && cm.arbiterOnly !== true) {
 			print("[flow 4 data→arbiter] Removing data member for arbiter conversion: " + h)
-			print(
-				"ACTION: stop mongod, EMPTY the full dbPath, restart — partial wipe causes broken repl state."
-			)
+			print("ACTION: stop mongod, EMPTY the full dbPath, restart.")
 			try {
 				rs.remove(h)
 				removed = true
@@ -536,7 +470,6 @@ function addMissingMembers(desired, current) {
 	if (missingData0.length > 1 && isOneDataPlusArbiterTopology(live0)) {
 		throw new Error(
 			"Replica set has one data member and an arbiter; replica-set.json lists multiple new data members. " +
-				"MongoDB requires rs.reconfigForPSASet once per added secondary. " +
 				"Add at most one new data member per init pass, then re-run with the next host."
 		)
 	}
@@ -550,13 +483,9 @@ function addMissingMembers(desired, current) {
 				const live = rs.conf()
 				const missingData = missingDataMemberHostsFromMap(desired, curMap)
 				if (missingData.length === 1 && isOneDataPlusArbiterTopology(live)) {
-					print(
-						"[flow 3 PSA] one data + arbiter: rs.reconfigForPSASet (https://www.mongodb.com/docs/manual/reference/method/rs.reconfigForPSASet/)"
-					)
+					print("[flow 3 PSA] one data + arbiter: rs.reconfigForPSASet")
 					if (typeof rs.reconfigForPSASet !== "function") {
-						throw new Error(
-							"rs.reconfigForPSASet is not available in this shell. Use mongosh on MongoDB 5.0+ to add an electable secondary to a 1-data+arbiter set."
-						)
+						throw new Error("rs.reconfigForPSASet is not available in this shell.")
 					}
 					const newConfig = buildReconfigDocument(desired, live)
 					const idx = findMemberIndexByHost(newConfig.members, h)
@@ -577,76 +506,41 @@ function addMissingMembers(desired, current) {
 	return current
 }
 
-/**
- * Refuse full reconfig if any member would change arbiterOnly without having been removed first.
- */
 function assertNoIllegalArbiterTransition(desired, current) {
 	const curMap = currentMembersByHost(current)
 	for (const dm of desired.members) {
 		const h = normalizeMemberHost(dm.host)
 		const cm = curMap[h]
-		if (!cm) {
-			continue
-		}
-		// Only fail if operator EXPLICITLY wants to change arbiterOnly (true or false in desired).
-		// If desired config omits arbiterOnly for an existing member, preserve current role.
-		if (dm.arbiterOnly === undefined || dm.arbiterOnly === null) {
-			continue
-		}
+		if (!cm) continue
+		if (dm.arbiterOnly === undefined || dm.arbiterOnly === null) continue
 		const wantArbiter = dm.arbiterOnly === true
 		const isArbiter = cm.arbiterOnly === true
 		if (isArbiter !== wantArbiter) {
 			throw new Error(
-				"Refusing rs.reconfig: cannot change arbiterOnly in place for " +
-					h +
-					" (live arbiterOnly=" +
-					isArbiter +
-					", desired=" +
-					wantArbiter +
-					"). Remove member, wipe dbPath, restart mongod, then re-run init."
+				"Refusing rs.reconfig: cannot change arbiterOnly in place for " + h +
+				" (live arbiterOnly=" + isArbiter + ", desired=" + wantArbiter +
+				"). Remove member, wipe dbPath, restart mongod, then re-run init."
 			)
 		}
 	}
 }
 
-/**
- * Existing member: base document on live rs.conf() so votes/priority/buildIndexes/etc. are preserved.
- * replica-set.json is sparse (_id, host, arbiterOnly); rs.reconfigForPSASet rejects undefined votes.
- */
 function mergeLiveMemberWithDesired(dm, cm) {
-	const h = normalizeMemberHost(dm.host)
 	const out = Object.assign({}, cm)
-	out._id = cm._id
-	out.host = h
-	// Only change arbiterOnly if operator explicitly sets it (true or false).
-	// If desired config omits arbiterOnly for an existing member, preserve current role.
+	out.host = normalizeMemberHost(dm.host)
 	if (dm.arbiterOnly !== undefined && dm.arbiterOnly !== null) {
 		out.arbiterOnly = dm.arbiterOnly === true
 	}
-	if (dm.priority !== undefined) {
-		out.priority = dm.priority
-	}
-	if (dm.votes !== undefined) {
-		out.votes = dm.votes
-	}
-	if (dm.tags !== undefined) {
-		out.tags = dm.tags
-	}
-	if (dm.secondaryDelaySecs !== undefined) {
-		out.secondaryDelaySecs = dm.secondaryDelaySecs
-	}
-	if (out.votes === undefined || out.votes === null) {
-		out.votes = 1
-	}
-	// Fix: use out.arbiterOnly instead of undefined wantArbiter
+	if (dm.priority !== undefined) out.priority = dm.priority
+	if (dm.votes !== undefined) out.votes = dm.votes
+	if (dm.tags !== undefined) out.tags = dm.tags
+	if (dm.secondaryDelaySecs !== undefined) out.secondaryDelaySecs = dm.secondaryDelaySecs
+	if (out.votes === undefined || out.votes === null) out.votes = 1
 	const isArbiter = out.arbiterOnly === true
-	if (out.priority === undefined || out.priority === null) {
-		out.priority = isArbiter ? 0 : 1
-	}
+	if (out.priority === undefined || out.priority === null) out.priority = isArbiter ? 0 : 1
 	return out
 }
 
-/** New member not in the live set yet — explicit votes/priority required by mongosh PSA reconfig. */
 function newMemberDocFromDesired(dm) {
 	const h = normalizeMemberHost(dm.host)
 	const wantArbiter = dm.arbiterOnly === true
@@ -662,20 +556,13 @@ function newMemberDocFromDesired(dm) {
 	}
 }
 
-/**
- * Build the next replSet config document (does not apply it).
- * Preserves Mongo-assigned member _id for existing hosts so we never request an illegal _id change.
- */
 function buildReconfigDocument(desired, current) {
 	assertNoIllegalArbiterTransition(desired, current)
 	const curByHost = currentMembersByHost(current)
 	const mergedMembers = desired.members.map((dm) => {
 		const h = normalizeMemberHost(dm.host)
 		const cm = curByHost[h]
-		if (cm) {
-			return mergeLiveMemberWithDesired(dm, cm)
-		}
-		return newMemberDocFromDesired(dm)
+		return cm ? mergeLiveMemberWithDesired(dm, cm) : newMemberDocFromDesired(dm)
 	})
 	const newConfig = {
 		_id: desired._id,
@@ -688,18 +575,32 @@ function buildReconfigDocument(desired, current) {
 	return newConfig
 }
 
-/**
- * Apply full desired config after incremental steps.
- */
+/* FIX-1: applyFullReconfig with retry on NewReplicaSetConfigurationIncompatible
+   (race: concurrent init may have just applied the same desired config) */
 function applyFullReconfig(desired) {
-	const current = rs.conf()
-	const newConfig = buildReconfigDocument(desired, current)
-	rs.reconfig(newConfig, { force: false })
+	const maxRetries = parseInt(process.env.MONGO_INIT_RECONFIG_ATTEMPTS || "3", 10)
+	const retryMs = parseInt(process.env.MONGO_INIT_RECONFIG_DELAY_MS || "5000", 10)
+	const maxTry = isNaN(maxRetries) ? 3 : Math.max(1, maxRetries)
+	const delay = isNaN(retryMs) ? 5000 : Math.max(1000, retryMs)
+	for (let t = 1; t <= maxTry; t++) {
+		const current = rs.conf()
+		const newConfig = buildReconfigDocument(desired, current)
+		try {
+			rs.reconfig(newConfig, { force: false })
+			if (t > 1) print("[phase 5] succeeded on retry " + t)
+			return
+		} catch (e) {
+			const msg = (e.message || "") + " " + (e.codeName || "")
+			if (t < maxTry && /ConfigurationInProgress|NewReplicaSetConfigurationIncompatible/i.test(msg)) {
+				print("[phase 5] retry " + t + "/" + maxTry + ": " + e.message + " in " + delay + "ms")
+				sleep(delay)
+			} else {
+				throw e
+			}
+		}
+	}
 }
 
-/**
- * Reconcile desired replica-set.json with live config (see file header for flows 1–4).
- */
 function reconfigureReplicaSet() {
 	const desired = JSON.parse(fs.readFileSync("/script/replica-set.json", "utf8"))
 	if (!desired.members || !Array.isArray(desired.members)) {
@@ -738,14 +639,58 @@ function initiateReplicaSet() {
 	rs.initiate(newConfig)
 }
 
+function isTransientConfigError(e) {
+	/* FIX-3: classify errors from concurrent-init races as retryable */
+	const msg = (e.message || "") + " " + (e.codeName || "")
+	return /MongoServerSelectionError|MongoNetworkError|ConfigurationInProgress|NewReplicaSetConfigurationIncompatible|already exists|already in config/i.test(msg)
+}
+
 function main() {
-	print("mongo-init.js revision: asab-remote-control/remote_control/tech/mongo-init.js (2026-04-08c)")
+	print("mongo-init.js revision: asab-remote-control/remote_control/tech/mongo-init.js (2026-06-22d — concurrent-init race fixed)")
 	print("(Re)-initializing the Mongo cluster.")
 
 	const mongoHostnames = process.env.MONGO_HOSTNAMES.split(",")
 
-	for (let i = 0; i < 5; i++) {
-		print("Connection attempt", i + 1)
+	/* FIX-4 (HIGH): Wait for ALL mongods before starting main loop.
+	   When all nodes restart simultaneously, waitForAllMongods is the
+	   first gate. Without it the main loop races an election in progress. */
+	const bootAttempts = parseInt(process.env.MONGO_INIT_BOOT_ATTEMPTS || "60", 10)
+	const bootMs = parseInt(process.env.MONGO_INIT_BOOT_MS || "3000", 10)
+	const maxBoot = isNaN(bootAttempts) ? 60 : Math.max(1, bootAttempts)
+	const bootDelay = isNaN(bootMs) ? 3000 : Math.max(1000, bootMs)
+	const pending = new Set(mongoHostnames.map(h => normalizeMemberHost(h)))
+	for (let a = 1; a <= maxBoot; a++) {
+		for (const hn of Array.from(pending)) {
+			const hp = hn + ":27017"
+			try {
+				if (typeof Mongo !== "undefined") {
+					const conn = new Mongo(hp)
+					if (conn.getDB("admin").runCommand({ ping: 1 }).ok === 1) {
+						pending.delete(hn)
+						print("[boot] mongod reachable: " + hp)
+					}
+				}
+			} catch (_) {}
+		}
+		if (pending.size === 0) {
+			print("[boot] all mongods reachable after " + a + "/" + maxBoot + " attempts")
+			break
+		}
+		if (a < maxBoot) {
+			print("[boot] " + pending.size + "/" + mongoHostnames.length + " down (" + Array.from(pending).join(", ") + ") — " + a + "/" + maxBoot)
+			sleep(bootDelay)
+		}
+	}
+	if (pending.size > 0) {
+		print("[boot] WARNING: timed out; still down: " + Array.from(pending).join(", "))
+	}
+
+	/* FIX-5: Main loop — retry on concurrent-init errors instead of quitting */
+	const maxMain = parseInt(process.env.MONGO_INIT_MAIN_ATTEMPTS || "20", 10)
+	const maxMainSafe = isNaN(maxMain) ? 20 : Math.max(1, maxMain)
+
+	for (let i = 0; i < maxMainSafe; i++) {
+		print("Connection attempt " + (i + 1) + "/" + maxMainSafe)
 
 		for (let hostname of mongoHostnames) {
 			print("Connecting to ", `${hostname}:27017`)
@@ -778,10 +723,7 @@ function main() {
 						if (dataMemberCountFromConf(conf) === 1) {
 							const st = rs.status()
 							if (!rsStatusHasPrimary(st)) {
-								print(
-									"[recovery] one data member, no PRIMARY (e.g. arbiter decommissioned); replSetStepUp on " +
-										`${hostname}:27017`
-								)
+								print("[recovery] one data member, no PRIMARY; replSetStepUp on " + `${hostname}:27017`)
 								const up = db.adminCommand({ replSetStepUp: 1 })
 								if (up.ok !== 1) {
 									print("[recovery] replSetStepUp: " + JSON.stringify(up))
@@ -796,14 +738,13 @@ function main() {
 										print("SUCCESS!")
 										quit(0)
 									} catch (e) {
-										print(
-											"Reconfiguration failed with " +
-												e.name +
-												": " +
-												e.message +
-												" / " +
-												(e.codeName != null ? e.codeName : "")
-										)
+										/* FIX-5b: if the reconfig failed due to a concurrent-init race,
+										   retry the outer loop instead of quitting */
+										if (isTransientConfigError(e)) {
+											print("[recovery] transient error (" + e.message + "), will retry.")
+											break
+										}
+										print("Reconfiguration failed with " + e.name + ": " + e.message + " / " + (e.codeName || ""))
 										print("Exiting due to failure.")
 										quit(1)
 									}
@@ -826,20 +767,21 @@ function main() {
 				print("SUCCESS!")
 				quit(0)
 			} catch (e) {
-				print(
-					"Reconfiguration failed with " +
-						e.name +
-						": " +
-						e.message +
-						" / " +
-						(e.codeName != null ? e.codeName : "")
-				)
+				/* FIX-5a: on concurrent-init race, retry the outer loop */
+				if (isTransientConfigError(e)) {
+					print("[reconfig] transient error (" + e.message + "), will retry on next attempt.")
+					break
+				}
+				print("Reconfiguration failed with " + e.name + ": " + e.message + " / " + (e.codeName || ""))
 				print("Exiting due to failure.")
 				quit(1)
 			}
 		}
 		sleep(5000)
 	}
+
+	print("All connection attempts exhausted (" + maxMainSafe + " × 5s = " + (maxMainSafe * 5) + "s), no primary reachable.")
+	quit(1)
 }
 
-main()
+main()
\ No newline at end of file

From 2f66b948533b090baa4bf59cf62930fdd2c943a7 Mon Sep 17 00:00:00 2001
From: Plesoun <jakub.schier@gmail.com>
Date: Mon, 17 Aug 2026 10:03:18 +0200
Subject: [PATCH 3/7] fix bug

---
 .../Files/mongo/script/mongo-init.js          | 24 +++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/Site/ASAB Maestro/Files/mongo/script/mongo-init.js b/Site/ASAB Maestro/Files/mongo/script/mongo-init.js
index a1c10e4d..33c45969 100644
--- a/Site/ASAB Maestro/Files/mongo/script/mongo-init.js	
+++ b/Site/ASAB Maestro/Files/mongo/script/mongo-init.js	
@@ -390,6 +390,14 @@ function addMemberFromDesired(dm) {
 	const delay = isNaN(addDelayMs) ? 10000 : Math.max(1000, addDelayMs)
 
 	for (let t = 1; t <= maxTry; t++) {
+		try {
+			if (currentMembersByHost(rs.conf())[h]) {
+				if (t > 1) print("rs.add: " + h + " already in config, continuing")
+				return
+			}
+		} catch (_) {
+			/* conf unreadable; fall through to rs.add */
+		}
 		try {
 			rs.add(doc)
 			return
@@ -404,11 +412,19 @@ function addMemberFromDesired(dm) {
 					err = e2
 				}
 			}
-			/* FIX-2: also retry on "already exists" / "already in config" —
-			   another concurrent init may have just added this member */
+			const errText = String((err.message || "") + " " + (err.codeName || ""))
+			/* Concurrent init already added this member — treat as success, do not re-call rs.add */
+			if (/already exists|already in config/i.test(errText)) {
+				try {
+					if (currentMembersByHost(rs.conf())[h]) {
+						print("rs.add: " + h + " already in config, continuing")
+						return
+					}
+				} catch (_) {}
+			}
 			const retryable =
 				t < maxTry &&
-				/Quorum check failed|Connection refused|NodeNotFound|timed out|Timeout|ConfigurationInProgress|already exists|already in config/i.test(String((err.message || "") + " " + (err.codeName || "")))
+				/Quorum check failed|Connection refused|NodeNotFound|timed out|Timeout|ConfigurationInProgress/i.test(errText)
 			if (retryable) {
 				print("rs.add retry " + t + "/" + maxTry + ": " + err.message)
 				sleep(delay)
@@ -784,4 +800,4 @@ function main() {
 	quit(1)
 }
 
-main()
\ No newline at end of file
+main()

From c7a8d89aab77a7852104ccdfa6f2ac8efda6acdd Mon Sep 17 00:00:00 2001
From: Plesoun <jakub.schier@gmail.com>
Date: Mon, 17 Aug 2026 10:07:15 +0200
Subject: [PATCH 4/7] prune stale comments

---
 .../Files/mongo/script/mongo-init.js          | 55 ++-----------------
 1 file changed, 4 insertions(+), 51 deletions(-)

diff --git a/Site/ASAB Maestro/Files/mongo/script/mongo-init.js b/Site/ASAB Maestro/Files/mongo/script/mongo-init.js
index 33c45969..04e96a25 100644
--- a/Site/ASAB Maestro/Files/mongo/script/mongo-init.js	
+++ b/Site/ASAB Maestro/Files/mongo/script/mongo-init.js	
@@ -12,35 +12,10 @@ const fs = require("fs")
  * 3) New core (data) node — rs.add when safe. If the live set is exactly one data member + ≥1 arbiter (PSA-style)
  *    and you add another electable data member, MongoDB rejects plain rs.add/rs.reconfig (NewReplicaSetConfigurationIncompatible);
  *    use rs.reconfigForPSASet(memberIndex, cfg) with the full merged config (see addMissingMembers).
- * 4) Node role change (arbiter ↔ data) — never flip arbiterOnly in place. Data→arbiter: script may rs.remove
- *    then you wipe dbPath and re-run so add-missing re-adds as arbiter. Arbiter→data: NOT automated — the script
- *    fails fast before any replica-set mutation (see assertArbiterToDataRequiresManual); you rs.remove, wipe
- *    dbPath, restart, then re-run init.
- *
- * Phase order in reconfigureReplicaSet(): (0) assert no arbiter→data pending (1) CWWC (2) decommission
- * (3) data→arbiter remove (4) add missing (5) rs.reconfig sync (6) sole-data-member PRIMARY election assist.
- *
- * --- Break-glass: mongod crash loop (OplogApplier / enableMajorityReadConcern invariant on MongoDB 7.x) ---
- * This script only runs while mongod stays up. A looping member has INVALID local data (typical: promoted
- * arbiter or role change without wiping dbPath). Logs show stale local repl config (e.g. setVersion 214988)
- * while primary is much newer, "No initial sync required" + immediate abort.
- *
- * Fix (operator, on the BROKEN host):
- *  A) On a HEALTHY primary (mongosh): rs.status() / rs.conf(). If the broken host is still listed,
- *     rs.remove("host:27017") so the cluster stops expecting that mongod.
- *  B) Stop the broken mongod container/service (so it does not restart-crash forever).
- *  C) Delete ALL files under that instance's dbPath (e.g. /data/db including WiredTiger*, local/, journal).
- *  D) Start mongod again with empty dbPath — it should NOT load old repl config; it may show as standalone
- *     or await rs.add.
- *  E) From primary: rs.add({ host: "...", ... }) or re-run this init against primary so the add-missing phase runs.
- *
- * Do not partially delete files; partial wipe keeps the crash.
+ * Unsupported flows:
+ * 1) Node role change (arbiter -> core)
  */
 
-/**
- * Cluster-wide default write concern (MongoDB 5.0+ gate before many reconfigs).
- * Optional: MONGO_INIT_DEFAULT_WRITE_CONCERN_W — "majority" (default), "1", "2", etc.
- */
 function defaultWriteConcernFromEnv() {
 	const w = process.env.MONGO_INIT_DEFAULT_WRITE_CONCERN_W
 	if (w == null || String(w).trim() === "" || String(w).trim() === "majority") {
@@ -53,10 +28,6 @@ function defaultWriteConcernFromEnv() {
 	return { w: String(w).trim(), wtimeout: 0 }
 }
 
-/**
- * Count reachable voting members (excluding arbiters) from rs.status().
- * Used to determine if "majority" write concern is achievable.
- */
 function reachableVotingMemberCount() {
 	let count = 0
 	try {
@@ -107,9 +78,6 @@ function normalizeMemberHost(host) {
 	return typeof host === "string" ? host.trim() : host
 }
 
-/**
- * rs.add / rs.addArb require the new member's mongod to already accept connections.
- */
 function peerWaitConfigFromEnv() {
 	const attempts = parseInt(process.env.MONGO_INIT_PEER_WAIT_ATTEMPTS || "60", 10)
 	const ms = parseInt(process.env.MONGO_INIT_PEER_WAIT_MS || "5000", 10)
@@ -289,9 +257,7 @@ function isMemberReachable(host) {
 				return st.members[i].health === 1
 			}
 		}
-	} catch (e) {
-		// Cannot read status — assume reachable (will try rs.remove first)
-	}
+	} catch (e) {}
 	return true
 }
 
@@ -395,9 +361,7 @@ function addMemberFromDesired(dm) {
 				if (t > 1) print("rs.add: " + h + " already in config, continuing")
 				return
 			}
-		} catch (_) {
-			/* conf unreadable; fall through to rs.add */
-		}
+		} catch (_) {}
 		try {
 			rs.add(doc)
 			return
@@ -413,7 +377,6 @@ function addMemberFromDesired(dm) {
 				}
 			}
 			const errText = String((err.message || "") + " " + (err.codeName || ""))
-			/* Concurrent init already added this member — treat as success, do not re-call rs.add */
 			if (/already exists|already in config/i.test(errText)) {
 				try {
 					if (currentMembersByHost(rs.conf())[h]) {
@@ -591,8 +554,6 @@ function buildReconfigDocument(desired, current) {
 	return newConfig
 }
 
-/* FIX-1: applyFullReconfig with retry on NewReplicaSetConfigurationIncompatible
-   (race: concurrent init may have just applied the same desired config) */
 function applyFullReconfig(desired) {
 	const maxRetries = parseInt(process.env.MONGO_INIT_RECONFIG_ATTEMPTS || "3", 10)
 	const retryMs = parseInt(process.env.MONGO_INIT_RECONFIG_DELAY_MS || "5000", 10)
@@ -656,7 +617,6 @@ function initiateReplicaSet() {
 }
 
 function isTransientConfigError(e) {
-	/* FIX-3: classify errors from concurrent-init races as retryable */
 	const msg = (e.message || "") + " " + (e.codeName || "")
 	return /MongoServerSelectionError|MongoNetworkError|ConfigurationInProgress|NewReplicaSetConfigurationIncompatible|already exists|already in config/i.test(msg)
 }
@@ -667,9 +627,6 @@ function main() {
 
 	const mongoHostnames = process.env.MONGO_HOSTNAMES.split(",")
 
-	/* FIX-4 (HIGH): Wait for ALL mongods before starting main loop.
-	   When all nodes restart simultaneously, waitForAllMongods is the
-	   first gate. Without it the main loop races an election in progress. */
 	const bootAttempts = parseInt(process.env.MONGO_INIT_BOOT_ATTEMPTS || "60", 10)
 	const bootMs = parseInt(process.env.MONGO_INIT_BOOT_MS || "3000", 10)
 	const maxBoot = isNaN(bootAttempts) ? 60 : Math.max(1, bootAttempts)
@@ -701,7 +658,6 @@ function main() {
 		print("[boot] WARNING: timed out; still down: " + Array.from(pending).join(", "))
 	}
 
-	/* FIX-5: Main loop — retry on concurrent-init errors instead of quitting */
 	const maxMain = parseInt(process.env.MONGO_INIT_MAIN_ATTEMPTS || "20", 10)
 	const maxMainSafe = isNaN(maxMain) ? 20 : Math.max(1, maxMain)
 
@@ -754,8 +710,6 @@ function main() {
 										print("SUCCESS!")
 										quit(0)
 									} catch (e) {
-										/* FIX-5b: if the reconfig failed due to a concurrent-init race,
-										   retry the outer loop instead of quitting */
 										if (isTransientConfigError(e)) {
 											print("[recovery] transient error (" + e.message + "), will retry.")
 											break
@@ -783,7 +737,6 @@ function main() {
 				print("SUCCESS!")
 				quit(0)
 			} catch (e) {
-				/* FIX-5a: on concurrent-init race, retry the outer loop */
 				if (isTransientConfigError(e)) {
 					print("[reconfig] transient error (" + e.message + "), will retry on next attempt.")
 					break

From 890d2a7c913eedf2c3166b7ac11291c3e5e1f934 Mon Sep 17 00:00:00 2001
From: Plesoun <jakub.schier@gmail.com>
Date: Wed, 19 Aug 2026 13:37:10 +0200
Subject: [PATCH 5/7] after testing

---
 .../Files/mongo/script/mongo-init.js          | 499 +++++++++---------
 1 file changed, 242 insertions(+), 257 deletions(-)

diff --git a/Site/ASAB Maestro/Files/mongo/script/mongo-init.js b/Site/ASAB Maestro/Files/mongo/script/mongo-init.js
index 04e96a25..ac9f7666 100644
--- a/Site/ASAB Maestro/Files/mongo/script/mongo-init.js	
+++ b/Site/ASAB Maestro/Files/mongo/script/mongo-init.js	
@@ -16,6 +16,34 @@ const fs = require("fs")
  * 1) Node role change (arbiter -> core)
  */
 
+function envInt(name, fallback, min) {
+	const n = parseInt(process.env[name] || String(fallback), 10)
+	return Math.max(min, isNaN(n) ? fallback : n)
+}
+
+function errorText(e) {
+	return String((e && e.message) || "") + " " + String((e && e.codeName) || "")
+}
+
+function withRetries(opts, fn) {
+	const attempts = opts.attempts
+	const delayMs = opts.delayMs
+	const label = opts.label
+	const isRetryable = opts.isRetryable
+	for (let t = 1; t <= attempts; t++) {
+		try {
+			return fn(t)
+		} catch (e) {
+			if (t < attempts && isRetryable(e)) {
+				print(label + " " + t + "/" + attempts + ": " + e.message)
+				sleep(delayMs)
+				continue
+			}
+			throw e
+		}
+	}
+}
+
 function defaultWriteConcernFromEnv() {
 	const w = process.env.MONGO_INIT_DEFAULT_WRITE_CONCERN_W
 	if (w == null || String(w).trim() === "" || String(w).trim() === "majority") {
@@ -34,13 +62,11 @@ function reachableVotingMemberCount() {
 		const st = rs.status()
 		for (let i = 0; i < st.members.length; i++) {
 			const m = st.members[i]
-			const isHealthy = [1, 2, 3, 5].includes(m.state)
-			if (isHealthy) {
-				const conf = rs.conf()
-				const memberConf = conf.members.find(cm => cm.host === m.name)
-				if (memberConf && memberConf.arbiterOnly !== true) {
-					count++
-				}
+			if (![1, 2, 3, 5].includes(m.state)) continue
+			const conf = rs.conf()
+			const memberConf = conf.members.find(cm => cm.host === m.name)
+			if (memberConf && memberConf.arbiterOnly !== true) {
+				count++
 			}
 		}
 	} catch (e) {
@@ -78,31 +104,28 @@ function normalizeMemberHost(host) {
 	return typeof host === "string" ? host.trim() : host
 }
 
-function peerWaitConfigFromEnv() {
-	const attempts = parseInt(process.env.MONGO_INIT_PEER_WAIT_ATTEMPTS || "60", 10)
-	const ms = parseInt(process.env.MONGO_INIT_PEER_WAIT_MS || "5000", 10)
-	return {
-		attempts: isNaN(attempts) ? 60 : Math.max(1, attempts),
-		ms: isNaN(ms) ? 5000 : Math.max(200, ms),
+function pingMongod(hostPort) {
+	const hp = normalizeMemberHost(hostPort)
+	if (typeof Mongo === "undefined") {
+		throw new Error("Mongo constructor unavailable in this shell")
+	}
+	const conn = new Mongo(hp)
+	const ping = conn.getDB("admin").runCommand({ ping: 1 })
+	if (ping.ok !== 1) {
+		throw new Error("ping ok !== 1: " + JSON.stringify(ping))
 	}
 }
 
 function waitForMongodOnHost(hostPort) {
 	const hp = normalizeMemberHost(hostPort)
-	const { attempts, ms } = peerWaitConfigFromEnv()
+	const attempts = envInt("MONGO_INIT_PEER_WAIT_ATTEMPTS", 60, 1)
+	const ms = envInt("MONGO_INIT_PEER_WAIT_MS", 5000, 200)
 	let lastMsg = ""
 	for (let a = 1; a <= attempts; a++) {
 		try {
-			if (typeof Mongo === "undefined") {
-				throw new Error("Mongo constructor unavailable in this shell")
-			}
-			const conn = new Mongo(hp)
-			const ping = conn.getDB("admin").runCommand({ ping: 1 })
-			if (ping.ok === 1) {
-				print("mongod reachable at " + hp + " (attempt " + a + "/" + attempts + ")")
-				return
-			}
-			lastMsg = "ping ok !== 1: " + JSON.stringify(ping)
+			pingMongod(hp)
+			print("mongod reachable at " + hp + " (attempt " + a + "/" + attempts + ")")
+			return
 		} catch (e) {
 			lastMsg = e.message
 		}
@@ -111,10 +134,31 @@ function waitForMongodOnHost(hostPort) {
 			sleep(ms)
 		}
 	}
-	throw new Error(
-		"Timed out waiting for mongod at " +
-			hp + ". Last error: " + lastMsg
-	)
+	throw new Error("Timed out waiting for mongod at " + hp + ". Last error: " + lastMsg)
+}
+
+function waitForAllMongods(hostnames) {
+	const maxBoot = envInt("MONGO_INIT_BOOT_ATTEMPTS", 60, 1)
+	const bootDelay = envInt("MONGO_INIT_BOOT_MS", 3000, 1000)
+	const pending = new Set(hostnames.map(h => normalizeMemberHost(h)))
+	for (let a = 1; a <= maxBoot; a++) {
+		for (const hn of Array.from(pending)) {
+			try {
+				pingMongod(hn + ":27017")
+				pending.delete(hn)
+				print("[boot] mongod reachable: " + hn + ":27017")
+			} catch (_) {}
+		}
+		if (pending.size === 0) {
+			print("[boot] all mongods reachable after " + a + "/" + maxBoot + " attempts")
+			return
+		}
+		if (a < maxBoot) {
+			print("[boot] " + pending.size + "/" + hostnames.length + " down (" + Array.from(pending).join(", ") + ") — " + a + "/" + maxBoot)
+			sleep(bootDelay)
+		}
+	}
+	print("[boot] WARNING: timed out; still down: " + Array.from(pending).join(", "))
 }
 
 function currentMembersByHost(conf) {
@@ -125,6 +169,14 @@ function currentMembersByHost(conf) {
 	return map
 }
 
+function hostInConf(host) {
+	try {
+		return !!currentMembersByHost(rs.conf())[normalizeMemberHost(host)]
+	} catch (_) {
+		return false
+	}
+}
+
 function desiredHostSet(desired) {
 	return new Set(desired.members.map((dm) => normalizeMemberHost(dm.host)))
 }
@@ -132,9 +184,7 @@ function desiredHostSet(desired) {
 function desiredDataMemberCount(desired) {
 	let n = 0
 	for (const dm of desired.members) {
-		if (dm.arbiterOnly !== true) {
-			n++
-		}
+		if (dm.arbiterOnly !== true) n++
 	}
 	return n
 }
@@ -142,9 +192,7 @@ function desiredDataMemberCount(desired) {
 function dataMemberCountFromConf(conf) {
 	let n = 0
 	for (let i = 0; i < conf.members.length; i++) {
-		if (conf.members[i].arbiterOnly !== true) {
-			n++
-		}
+		if (conf.members[i].arbiterOnly !== true) n++
 	}
 	return n
 }
@@ -152,9 +200,7 @@ function dataMemberCountFromConf(conf) {
 function arbiterMemberCountFromConf(conf) {
 	let n = 0
 	for (let i = 0; i < conf.members.length; i++) {
-		if (conf.members[i].arbiterOnly === true) {
-			n++
-		}
+		if (conf.members[i].arbiterOnly === true) n++
 	}
 	return n
 }
@@ -167,13 +213,9 @@ function missingDataMemberHostsFromMap(desired, curMap) {
 	const out = []
 	for (let i = 0; i < desired.members.length; i++) {
 		const dm = desired.members[i]
-		if (dm.arbiterOnly === true) {
-			continue
-		}
+		if (dm.arbiterOnly === true) continue
 		const h = normalizeMemberHost(dm.host)
-		if (!curMap[h]) {
-			out.push(h)
-		}
+		if (!curMap[h]) out.push(h)
 	}
 	return out
 }
@@ -181,18 +223,14 @@ function missingDataMemberHostsFromMap(desired, curMap) {
 function findMemberIndexByHost(members, host) {
 	const h = normalizeMemberHost(host)
 	for (let i = 0; i < members.length; i++) {
-		if (normalizeMemberHost(members[i].host) === h) {
-			return i
-		}
+		if (normalizeMemberHost(members[i].host) === h) return i
 	}
 	return -1
 }
 
 function rsStatusHasPrimary(st) {
 	for (let i = 0; i < st.members.length; i++) {
-		if (st.members[i].stateStr === "PRIMARY") {
-			return true
-		}
+		if (st.members[i].stateStr === "PRIMARY") return true
 	}
 	return false
 }
@@ -202,16 +240,12 @@ function ensurePrimaryWhenSoleDataMember(desired) {
 	if (dataMemberCountFromConf(conf) !== 1 || desiredDataMemberCount(desired) !== 1) {
 		return
 	}
-	const pollAttempts = parseInt(process.env.MONGO_INIT_SOLE_PRIMARY_POLL_ATTEMPTS || "8", 10)
-	const pollMs = parseInt(process.env.MONGO_INIT_SOLE_PRIMARY_POLL_MS || "1500", 10)
-	const maxPoll = isNaN(pollAttempts) ? 8 : Math.max(1, pollAttempts)
-	const ms = isNaN(pollMs) ? 1500 : Math.max(200, pollMs)
+	const maxPoll = envInt("MONGO_INIT_SOLE_PRIMARY_POLL_ATTEMPTS", 8, 1)
+	const ms = envInt("MONGO_INIT_SOLE_PRIMARY_POLL_MS", 1500, 200)
 	for (let k = 0; k < maxPoll; k++) {
 		const st = rs.status()
 		if (rsStatusHasPrimary(st)) {
-			if (k > 0) {
-				print("[sole data member] PRIMARY appeared after wait")
-			}
+			if (k > 0) print("[sole data member] PRIMARY appeared after wait")
 			return
 		}
 		if (k + 1 < maxPoll) {
@@ -261,12 +295,11 @@ function isMemberReachable(host) {
 	return true
 }
 
-function forceReconfigToRemoveMembers(desired, current, toRemoveSet) {
+function forceReconfigToRemoveMembers(current, toRemoveSet) {
 	const hostsToRemove = Array.from(toRemoveSet)
 	print("[force reconfig] Removing unreachable members via rs.reconfig({force:true}): " + hostsToRemove.join(", "))
 	const filtered = current.members.filter(function(m) {
-		const h = normalizeMemberHost(m.host)
-		return !toRemoveSet.has(h)
+		return !toRemoveSet.has(normalizeMemberHost(m.host))
 	})
 	if (filtered.length === 0) {
 		throw new Error("Force reconfig would remove ALL members — refusing (need at least 1 data member)")
@@ -290,7 +323,7 @@ function forceReconfigToRemoveMembers(desired, current, toRemoveSet) {
 	if (current.settings !== undefined && current.settings !== null) {
 		newConfig.settings = Object.assign({}, current.settings)
 	}
-	const result = rs.reconfig(newConfig, {force: true})
+	const result = rs.reconfig(newConfig, { force: true })
 	print("[force reconfig] ok=" + result.ok + " (new version=" + newConfig.version + ")")
 }
 
@@ -304,18 +337,17 @@ function removeMembersDroppedFromDesired(desired, current) {
 	const toRemoveUnreachable = new Set()
 	for (const cm of current.members) {
 		const h = normalizeMemberHost(cm.host)
-		if (!wantHosts.has(h)) {
-			if (selfH !== null && h === selfH) {
-				throw new Error(
-					"Desired config omits the current primary host (" + h +
-					"). Step down another primary first, or keep this member in replica-set.json until then."
-				)
-			}
-			if (isMemberReachable(h)) {
-				toRemoveReachable.push(h)
-			} else {
-				toRemoveUnreachable.add(h)
-			}
+		if (wantHosts.has(h)) continue
+		if (selfH !== null && h === selfH) {
+			throw new Error(
+				"Desired config omits the current primary host (" + h +
+				"). Step down another primary first, or keep this member in replica-set.json until then."
+			)
+		}
+		if (isMemberReachable(h)) {
+			toRemoveReachable.push(h)
+		} else {
+			toRemoveUnreachable.add(h)
 		}
 	}
 	for (let i = 0; i < toRemoveReachable.length; i++) {
@@ -323,45 +355,35 @@ function removeMembersDroppedFromDesired(desired, current) {
 		print("[flow 2 decommission] Removing reachable member: " + h)
 		try {
 			rs.remove(h)
-			current = rs.conf()
 		} catch (e) {
 			print("rs.remove failed (will handle via force reconfig): " + e.message)
 			toRemoveUnreachable.add(h)
 		}
 	}
 	if (toRemoveUnreachable.size > 0) {
-		current = rs.conf()
-		forceReconfigToRemoveMembers(desired, current, toRemoveUnreachable)
+		forceReconfigToRemoveMembers(rs.conf(), toRemoveUnreachable)
 	}
 }
 
 function addMemberFromDesired(dm) {
 	const h = normalizeMemberHost(dm.host)
 	const doc = { host: h }
-	if (dm._id !== undefined && dm._id !== null) {
-		doc._id = dm._id
-	}
-	if (dm.arbiterOnly === true) {
-		doc.arbiterOnly = true
-	}
-	if (dm.priority !== undefined) {
-		doc.priority = dm.priority
-	}
-	if (dm.votes !== undefined) {
-		doc.votes = dm.votes
-	}
-	const addRetries = parseInt(process.env.MONGO_INIT_RS_ADD_ATTEMPTS || "5", 10)
-	const addDelayMs = parseInt(process.env.MONGO_INIT_RS_ADD_MS || "10000", 10)
-	const maxTry = isNaN(addRetries) ? 5 : Math.max(1, addRetries)
-	const delay = isNaN(addDelayMs) ? 10000 : Math.max(1000, addDelayMs)
-
-	for (let t = 1; t <= maxTry; t++) {
-		try {
-			if (currentMembersByHost(rs.conf())[h]) {
-				if (t > 1) print("rs.add: " + h + " already in config, continuing")
-				return
-			}
-		} catch (_) {}
+	if (dm._id !== undefined && dm._id !== null) doc._id = dm._id
+	if (dm.arbiterOnly === true) doc.arbiterOnly = true
+	if (dm.priority !== undefined) doc.priority = dm.priority
+	if (dm.votes !== undefined) doc.votes = dm.votes
+
+	withRetries({
+		label: "rs.add retry",
+		attempts: envInt("MONGO_INIT_RS_ADD_ATTEMPTS", 5, 1),
+		delayMs: envInt("MONGO_INIT_RS_ADD_MS", 10000, 1000),
+		isRetryable: (e) =>
+			/Quorum check failed|Connection refused|NodeNotFound|timed out|Timeout|ConfigurationInProgress/i.test(errorText(e)),
+	}, function(t) {
+		if (hostInConf(h)) {
+			if (t > 1) print("rs.add: " + h + " already in config, continuing")
+			return
+		}
 		try {
 			rs.add(doc)
 			return
@@ -376,46 +398,38 @@ function addMemberFromDesired(dm) {
 					err = e2
 				}
 			}
-			const errText = String((err.message || "") + " " + (err.codeName || ""))
-			if (/already exists|already in config/i.test(errText)) {
-				try {
-					if (currentMembersByHost(rs.conf())[h]) {
-						print("rs.add: " + h + " already in config, continuing")
-						return
-					}
-				} catch (_) {}
-			}
-			const retryable =
-				t < maxTry &&
-				/Quorum check failed|Connection refused|NodeNotFound|timed out|Timeout|ConfigurationInProgress/i.test(errText)
-			if (retryable) {
-				print("rs.add retry " + t + "/" + maxTry + ": " + err.message)
-				sleep(delay)
-				continue
+			if (/already exists|already in config/i.test(errorText(err)) && hostInConf(h)) {
+				print("rs.add: " + h + " already in config, continuing")
+				return
 			}
 			throw err
 		}
-	}
+	})
 }
 
-function assertArbiterToDataRequiresManual(desired, current) {
+function arbiterRoleDiffs(desired, current) {
 	const curMap = currentMembersByHost(current)
-	const offenders = []
+	const arbiterToData = []
+	const dataToArbiter = []
 	for (const dm of desired.members) {
 		const h = normalizeMemberHost(dm.host)
 		const cm = curMap[h]
 		if (!cm) continue
+		if (dm.arbiterOnly === undefined || dm.arbiterOnly === null) continue
 		const wantArbiter = dm.arbiterOnly === true
-		const wantData = dm.arbiterOnly === false
 		const isArbiter = cm.arbiterOnly === true
-		if (isArbiter && wantData) {
-			offenders.push(h)
-		}
+		if (isArbiter && !wantArbiter) arbiterToData.push(h)
+		if (!isArbiter && wantArbiter) dataToArbiter.push(h)
 	}
-	if (offenders.length === 0) return
+	return { arbiterToData, dataToArbiter }
+}
+
+function assertArbiterToDataRequiresManual(desired, current) {
+	const { arbiterToData } = arbiterRoleDiffs(desired, current)
+	if (arbiterToData.length === 0) return
 	throw new Error(
 		"Refusing init: arbiter → data promotion is manual-only for: " +
-			offenders.join(", ") +
+			arbiterToData.join(", ") +
 			". Do this, then re-run init: (1) On PRIMARY: rs.remove(\"host:port\") for each host above. " +
 			"(2) On each former arbiter host: stop mongod, delete ALL files under dbPath, start mongod empty. " +
 			"(3) Re-run init."
@@ -423,20 +437,17 @@ function assertArbiterToDataRequiresManual(desired, current) {
 }
 
 function removeMembersForArbiterConversion(desired, current) {
-	const curMap = currentMembersByHost(current)
+	const { dataToArbiter } = arbiterRoleDiffs(desired, current)
 	let removed = false
-	for (const dm of desired.members) {
-		const h = normalizeMemberHost(dm.host)
-		const cm = curMap[h]
-		if (cm && dm.arbiterOnly === true && cm.arbiterOnly !== true) {
-			print("[flow 4 data→arbiter] Removing data member for arbiter conversion: " + h)
-			print("ACTION: stop mongod, EMPTY the full dbPath, restart.")
-			try {
-				rs.remove(h)
-				removed = true
-			} catch (e) {
-				print("rs.remove failed (member may already be absent): " + e.message)
-			}
+	for (let i = 0; i < dataToArbiter.length; i++) {
+		const h = dataToArbiter[i]
+		print("[flow 4 data→arbiter] Removing data member for arbiter conversion: " + h)
+		print("ACTION: stop mongod, EMPTY the full dbPath, restart.")
+		try {
+			rs.remove(h)
+			removed = true
+		} catch (e) {
+			print("rs.remove failed (member may already be absent): " + e.message)
 		}
 	}
 	return removed
@@ -454,54 +465,53 @@ function addMissingMembers(desired, current) {
 	}
 	for (const dm of desired.members) {
 		const h = normalizeMemberHost(dm.host)
-		if (!curMap[h]) {
-			const flowTag = dm.arbiterOnly === true ? "[flow 1 new arbiter/peripheral]" : "[flow 3 new core data]"
-			print(flowTag + " Adding replica set member: " + JSON.stringify(dm))
-			waitForMongodOnHost(h)
-			if (dm.arbiterOnly !== true) {
-				const live = rs.conf()
-				const missingData = missingDataMemberHostsFromMap(desired, curMap)
-				if (missingData.length === 1 && isOneDataPlusArbiterTopology(live)) {
-					print("[flow 3 PSA] one data + arbiter: rs.reconfigForPSASet")
-					if (typeof rs.reconfigForPSASet !== "function") {
-						throw new Error("rs.reconfigForPSASet is not available in this shell.")
-					}
-					const newConfig = buildReconfigDocument(desired, live)
-					const idx = findMemberIndexByHost(newConfig.members, h)
-					if (idx < 0) {
-						throw new Error("PSA reconfig: member index not found for host " + h)
-					}
-					rs.reconfigForPSASet(idx, newConfig)
-				} else {
-					addMemberFromDesired(dm)
-				}
-			} else {
-				addMemberFromDesired(dm)
+		if (curMap[h]) continue
+
+		const flowTag = dm.arbiterOnly === true ? "[flow 1 new arbiter/peripheral]" : "[flow 3 new core data]"
+		print(flowTag + " Adding replica set member: " + JSON.stringify(dm))
+		waitForMongodOnHost(h)
+
+		const live = rs.conf()
+		const missingData = missingDataMemberHostsFromMap(desired, curMap)
+		const usePsa =
+			dm.arbiterOnly !== true &&
+			missingData.length === 1 &&
+			isOneDataPlusArbiterTopology(live)
+
+		if (usePsa) {
+			print("[flow 3 PSA] one data + arbiter: rs.reconfigForPSASet")
+			if (typeof rs.reconfigForPSASet !== "function") {
+				throw new Error("rs.reconfigForPSASet is not available in this shell.")
 			}
-			current = rs.conf()
-			Object.assign(curMap, currentMembersByHost(current))
+			const newConfig = buildReconfigDocument(desired, live)
+			const idx = findMemberIndexByHost(newConfig.members, h)
+			if (idx < 0) {
+				throw new Error("PSA reconfig: member index not found for host " + h)
+			}
+			rs.reconfigForPSASet(idx, newConfig)
+		} else {
+			addMemberFromDesired(dm)
 		}
+
+		current = rs.conf()
+		Object.assign(curMap, currentMembersByHost(current))
 	}
 	return current
 }
 
 function assertNoIllegalArbiterTransition(desired, current) {
+	const { arbiterToData, dataToArbiter } = arbiterRoleDiffs(desired, current)
+	const offenders = arbiterToData.concat(dataToArbiter)
+	if (offenders.length === 0) return
 	const curMap = currentMembersByHost(current)
-	for (const dm of desired.members) {
-		const h = normalizeMemberHost(dm.host)
-		const cm = curMap[h]
-		if (!cm) continue
-		if (dm.arbiterOnly === undefined || dm.arbiterOnly === null) continue
-		const wantArbiter = dm.arbiterOnly === true
-		const isArbiter = cm.arbiterOnly === true
-		if (isArbiter !== wantArbiter) {
-			throw new Error(
-				"Refusing rs.reconfig: cannot change arbiterOnly in place for " + h +
-				" (live arbiterOnly=" + isArbiter + ", desired=" + wantArbiter +
-				"). Remove member, wipe dbPath, restart mongod, then re-run init."
-			)
-		}
-	}
+	const h = offenders[0]
+	const isArbiter = curMap[h].arbiterOnly === true
+	const wantArbiter = dataToArbiter.indexOf(h) >= 0
+	throw new Error(
+		"Refusing rs.reconfig: cannot change arbiterOnly in place for " + h +
+		" (live arbiterOnly=" + isArbiter + ", desired=" + wantArbiter +
+		"). Remove member, wipe dbPath, restart mongod, then re-run init."
+	)
 }
 
 function mergeLiveMemberWithDesired(dm, cm) {
@@ -555,27 +565,17 @@ function buildReconfigDocument(desired, current) {
 }
 
 function applyFullReconfig(desired) {
-	const maxRetries = parseInt(process.env.MONGO_INIT_RECONFIG_ATTEMPTS || "3", 10)
-	const retryMs = parseInt(process.env.MONGO_INIT_RECONFIG_DELAY_MS || "5000", 10)
-	const maxTry = isNaN(maxRetries) ? 3 : Math.max(1, maxRetries)
-	const delay = isNaN(retryMs) ? 5000 : Math.max(1000, retryMs)
-	for (let t = 1; t <= maxTry; t++) {
-		const current = rs.conf()
-		const newConfig = buildReconfigDocument(desired, current)
-		try {
-			rs.reconfig(newConfig, { force: false })
-			if (t > 1) print("[phase 5] succeeded on retry " + t)
-			return
-		} catch (e) {
-			const msg = (e.message || "") + " " + (e.codeName || "")
-			if (t < maxTry && /ConfigurationInProgress|NewReplicaSetConfigurationIncompatible/i.test(msg)) {
-				print("[phase 5] retry " + t + "/" + maxTry + ": " + e.message + " in " + delay + "ms")
-				sleep(delay)
-			} else {
-				throw e
-			}
-		}
-	}
+	withRetries({
+		label: "[phase 5] retry",
+		attempts: envInt("MONGO_INIT_RECONFIG_ATTEMPTS", 3, 1),
+		delayMs: envInt("MONGO_INIT_RECONFIG_DELAY_MS", 5000, 1000),
+		isRetryable: (e) =>
+			/ConfigurationInProgress|NewReplicaSetConfigurationIncompatible/i.test(errorText(e)),
+	}, function(t) {
+		const newConfig = buildReconfigDocument(desired, rs.conf())
+		rs.reconfig(newConfig, { force: false })
+		if (t > 1) print("[phase 5] succeeded on retry " + t)
+	})
 }
 
 function reconfigureReplicaSet() {
@@ -617,49 +617,50 @@ function initiateReplicaSet() {
 }
 
 function isTransientConfigError(e) {
-	const msg = (e.message || "") + " " + (e.codeName || "")
-	return /MongoServerSelectionError|MongoNetworkError|ConfigurationInProgress|NewReplicaSetConfigurationIncompatible|already exists|already in config/i.test(msg)
+	return /MongoServerSelectionError|MongoNetworkError|ConfigurationInProgress|NewReplicaSetConfigurationIncompatible|already exists|already in config/i.test(errorText(e))
+}
+
+function finishReconfigureSuccess() {
+	print("Successfully reconfigured replicaset.")
+	print("SUCCESS!")
+	quit(0)
+}
+
+// Deliberate safety refusals — the sherpa refuses to take the requested action and no
+// automatic re-run can change the outcome. These MUST keep a non-zero exit (and the dead
+// container stays for logs). Every other failure is treated as recoverable → exit 0.
+function isPermanentFailure(e) {
+	const t = errorText(e)
+	return /omits the current primary host|arbiter → data promotion is manual-only|cannot change arbiterOnly in place|would remove ALL members|would remove all data-bearing members|must keep at least one data/i.test(t)
+}
+
+function handleReconfigureFailure(e, retryLabel) {
+	if (isTransientConfigError(e)) {
+		print(retryLabel + " transient error (" + e.message + "), will retry.")
+		return true
+	}
+	print("Reconfiguration failed with " + e.name + ": " + e.message + " / " + (e.codeName || ""))
+	if (isPermanentFailure(e)) {
+		print("Exiting due to unrecoverable failure (requires manual attention).")
+		quit(1)
+	}
+	print("Recoverable failure; exiting 0 so the sherpa is cleaned up (retried on next reconcile).")
+	quit(0)
 }
 
 function main() {
 	print("mongo-init.js revision: asab-remote-control/remote_control/tech/mongo-init.js (2026-06-22d — concurrent-init race fixed)")
 	print("(Re)-initializing the Mongo cluster.")
 
-	const mongoHostnames = process.env.MONGO_HOSTNAMES.split(",")
-
-	const bootAttempts = parseInt(process.env.MONGO_INIT_BOOT_ATTEMPTS || "60", 10)
-	const bootMs = parseInt(process.env.MONGO_INIT_BOOT_MS || "3000", 10)
-	const maxBoot = isNaN(bootAttempts) ? 60 : Math.max(1, bootAttempts)
-	const bootDelay = isNaN(bootMs) ? 3000 : Math.max(1000, bootMs)
-	const pending = new Set(mongoHostnames.map(h => normalizeMemberHost(h)))
-	for (let a = 1; a <= maxBoot; a++) {
-		for (const hn of Array.from(pending)) {
-			const hp = hn + ":27017"
-			try {
-				if (typeof Mongo !== "undefined") {
-					const conn = new Mongo(hp)
-					if (conn.getDB("admin").runCommand({ ping: 1 }).ok === 1) {
-						pending.delete(hn)
-						print("[boot] mongod reachable: " + hp)
-					}
-				}
-			} catch (_) {}
-		}
-		if (pending.size === 0) {
-			print("[boot] all mongods reachable after " + a + "/" + maxBoot + " attempts")
-			break
-		}
-		if (a < maxBoot) {
-			print("[boot] " + pending.size + "/" + mongoHostnames.length + " down (" + Array.from(pending).join(", ") + ") — " + a + "/" + maxBoot)
-			sleep(bootDelay)
-		}
-	}
-	if (pending.size > 0) {
-		print("[boot] WARNING: timed out; still down: " + Array.from(pending).join(", "))
+	const mongoHostnamesRaw = process.env.MONGO_HOSTNAMES
+	if (mongoHostnamesRaw == null || String(mongoHostnamesRaw).trim() === "") {
+		print("FATAL: MONGO_HOSTNAMES is not set; cannot reconcile replica set. Check the deployment/compose env.")
+		quit(1)
 	}
+	const mongoHostnames = mongoHostnamesRaw.split(",")
+	waitForAllMongods(mongoHostnames)
 
-	const maxMain = parseInt(process.env.MONGO_INIT_MAIN_ATTEMPTS || "20", 10)
-	const maxMainSafe = isNaN(maxMain) ? 20 : Math.max(1, maxMain)
+	const maxMainSafe = envInt("MONGO_INIT_MAIN_ATTEMPTS", 20, 1)
 
 	for (let i = 0; i < maxMainSafe; i++) {
 		print("Connection attempt " + (i + 1) + "/" + maxMainSafe)
@@ -683,8 +684,8 @@ function main() {
 					quit(0)
 				} catch (e) {
 					print("Initialization of replicaset failed.")
-					print("Exiting due to failure.")
-					quit(1)
+					print("Recoverable failure; exiting 0 so the sherpa is cleaned up (retried on next reconcile).")
+					quit(0)
 				}
 			}
 
@@ -706,17 +707,9 @@ function main() {
 									print("[recovery] became primary; running replica set reconcile")
 									try {
 										reconfigureReplicaSet()
-										print("Successfully reconfigured replicaset.")
-										print("SUCCESS!")
-										quit(0)
+										finishReconfigureSuccess()
 									} catch (e) {
-										if (isTransientConfigError(e)) {
-											print("[recovery] transient error (" + e.message + "), will retry.")
-											break
-										}
-										print("Reconfiguration failed with " + e.name + ": " + e.message + " / " + (e.codeName || ""))
-										print("Exiting due to failure.")
-										quit(1)
+										if (handleReconfigureFailure(e, "[recovery]")) break
 									}
 								}
 							}
@@ -733,24 +726,16 @@ function main() {
 
 			try {
 				reconfigureReplicaSet()
-				print("Successfully reconfigured replicaset.")
-				print("SUCCESS!")
-				quit(0)
+				finishReconfigureSuccess()
 			} catch (e) {
-				if (isTransientConfigError(e)) {
-					print("[reconfig] transient error (" + e.message + "), will retry on next attempt.")
-					break
-				}
-				print("Reconfiguration failed with " + e.name + ": " + e.message + " / " + (e.codeName || ""))
-				print("Exiting due to failure.")
-				quit(1)
+				if (handleReconfigureFailure(e, "[reconfig]")) break
 			}
 		}
 		sleep(5000)
 	}
 
-	print("All connection attempts exhausted (" + maxMainSafe + " × 5s = " + (maxMainSafe * 5) + "s), no primary reachable.")
-	quit(1)
+	print("All connection attempts exhausted (" + maxMainSafe + " × 5s = " + (maxMainSafe * 5) + "s), no primary reachable; exiting 0 (recoverable, cleaned up).")
+	quit(0)
 }
 
 main()

From b94f436ca35083f48b819af431af16518a556fc8 Mon Sep 17 00:00:00 2001
From: Plesoun <jakub.schier@gmail.com>
Date: Wed, 19 Aug 2026 15:00:34 +0200
Subject: [PATCH 6/7] address internal review comments

---
 Site/ASAB Maestro/Files/mongo/script/mongo-init.js | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/Site/ASAB Maestro/Files/mongo/script/mongo-init.js b/Site/ASAB Maestro/Files/mongo/script/mongo-init.js
index ac9f7666..83bcf167 100644
--- a/Site/ASAB Maestro/Files/mongo/script/mongo-init.js	
+++ b/Site/ASAB Maestro/Files/mongo/script/mongo-init.js	
@@ -60,11 +60,11 @@ function reachableVotingMemberCount() {
 	let count = 0
 	try {
 		const st = rs.status()
+		const conf = rs.conf()
 		for (let i = 0; i < st.members.length; i++) {
 			const m = st.members[i]
 			if (![1, 2, 3, 5].includes(m.state)) continue
-			const conf = rs.conf()
-			const memberConf = conf.members.find(cm => cm.host === m.name)
+			const memberConf = conf.members.find(cm => normalizeMemberHost(cm.host) === normalizeMemberHost(m.name))
 			if (memberConf && memberConf.arbiterOnly !== true) {
 				count++
 			}
@@ -459,7 +459,7 @@ function addMissingMembers(desired, current) {
 	const missingData0 = missingDataMemberHostsFromMap(desired, curMap)
 	if (missingData0.length > 1 && isOneDataPlusArbiterTopology(live0)) {
 		throw new Error(
-			"Replica set has one data member and an arbiter; replica-set.json lists multiple new data members. " +
+			"FATAL: replica-set.json lists multiple new data members in a one-data+arbiter topology. " +
 				"Add at most one new data member per init pass, then re-run with the next host."
 		)
 	}
@@ -534,7 +534,7 @@ function newMemberDocFromDesired(dm) {
 	const h = normalizeMemberHost(dm.host)
 	const wantArbiter = dm.arbiterOnly === true
 	if (dm._id === undefined || dm._id === null) {
-		throw new Error("replica-set.json member missing _id for host " + h)
+		throw new Error("FATAL: replica-set.json member missing _id for host " + h)
 	}
 	return {
 		_id: dm._id,
@@ -581,7 +581,7 @@ function applyFullReconfig(desired) {
 function reconfigureReplicaSet() {
 	const desired = JSON.parse(fs.readFileSync("/script/replica-set.json", "utf8"))
 	if (!desired.members || !Array.isArray(desired.members)) {
-		throw new Error("replica-set.json must contain a members array")
+		throw new Error("FATAL: replica-set.json must contain a members array")
 	}
 
 	let current = rs.conf()
@@ -657,7 +657,7 @@ function main() {
 		print("FATAL: MONGO_HOSTNAMES is not set; cannot reconcile replica set. Check the deployment/compose env.")
 		quit(1)
 	}
-	const mongoHostnames = mongoHostnamesRaw.split(",")
+	const mongoHostnames = mongoHostnamesRaw.split(",").map(h => normalizeMemberHost(h)).filter(Boolean)
 	waitForAllMongods(mongoHostnames)
 
 	const maxMainSafe = envInt("MONGO_INIT_MAIN_ATTEMPTS", 20, 1)

From 3ff84931d0381fcbd2f08158679c1251a2a72a8c Mon Sep 17 00:00:00 2001
From: Plesoun <jakub.schier@gmail.com>
Date: Thu, 20 Aug 2026 10:41:26 +0200
Subject: [PATCH 7/7] last cosmetics

---
 Site/ASAB Maestro/Files/mongo/script/mongo-init.js | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/Site/ASAB Maestro/Files/mongo/script/mongo-init.js b/Site/ASAB Maestro/Files/mongo/script/mongo-init.js
index 83bcf167..0b2d54b4 100644
--- a/Site/ASAB Maestro/Files/mongo/script/mongo-init.js	
+++ b/Site/ASAB Maestro/Files/mongo/script/mongo-init.js	
@@ -658,6 +658,10 @@ function main() {
 		quit(1)
 	}
 	const mongoHostnames = mongoHostnamesRaw.split(",").map(h => normalizeMemberHost(h)).filter(Boolean)
+	if (mongoHostnames.length === 0) {
+		print("FATAL: MONGO_HOSTNAMES produced no hostnames; cannot reconcile replica set. Check the deployment/compose env.")
+		quit(1)
+	}
 	waitForAllMongods(mongoHostnames)
 
 	const maxMainSafe = envInt("MONGO_INIT_MAIN_ATTEMPTS", 20, 1)