From b33976faf9aa92b9c2b5d9040720506c1d60c83d Mon Sep 17 00:00:00 2001 From: Artem Nikitin Date: Fri, 17 Jul 2026 15:08:16 +0200 Subject: [PATCH 1/2] Widen voting-only health window for slow 1-vCPU first boot The 20s x 3 (~60s) TCP failure budget restarted the VM before Elasticsearch opened its transport port on a cold 1-vCPU boot, so the tiebreaker never joined the AWS tenant-3 cluster. Widen to 30s x 5, matching the budget tenant-2 kibana needed. Co-Authored-By: Claude Fable 5 --- tenants/tenant-3/elasticsearch-voting-only.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tenants/tenant-3/elasticsearch-voting-only.yaml b/tenants/tenant-3/elasticsearch-voting-only.yaml index 98d1f2d..f1df228 100644 --- a/tenants/tenant-3/elasticsearch-voting-only.yaml +++ b/tenants/tenant-3/elasticsearch-voting-only.yaml @@ -25,9 +25,12 @@ port_forwards: health_check: type: "tcp" port: 9300 - interval: "20s" + # First boot on 1 vCPU takes longer than a 20s x 3 (~60s) failure budget, + # so the agent kept restarting the VM before the transport port opened. + # Match the ~150s budget that tenant-2 kibana needed for slow cold starts. + interval: "30s" timeout: "5s" - retries: 3 + retries: 5 node_host_ip_env: "transport.publish_host" cross_node_links: - service: "elasticsearch-data-1" From b076e32cec1c02579e0c1bc886f7a687b95385ab Mon Sep 17 00:00:00 2001 From: Artem Nikitin Date: Fri, 17 Jul 2026 15:13:06 +0200 Subject: [PATCH 2/2] Apply the widened health window to all three tenant-3 services Co-Authored-By: Claude Fable 5 --- tenants/tenant-3/elasticsearch-data-1.yaml | 7 +++++-- tenants/tenant-3/elasticsearch-data-2.yaml | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/tenants/tenant-3/elasticsearch-data-1.yaml b/tenants/tenant-3/elasticsearch-data-1.yaml index 27ebeb1..6bbb437 100644 --- a/tenants/tenant-3/elasticsearch-data-1.yaml +++ b/tenants/tenant-3/elasticsearch-data-1.yaml @@ -26,9 +26,12 @@ port_forwards: health_check: type: "tcp" port: 9300 - interval: "20s" + # A cold Elasticsearch boot can outlast a 20s x 3 (~60s) failure budget, + # which restarts the VM just before the transport port opens. Keep the + # ~150s+ budget consistent across all three tenant-3 services. + interval: "30s" timeout: "5s" - retries: 3 + retries: 5 node_host_ip_env: "transport.publish_host" cross_node_links: - service: "elasticsearch-data-2" diff --git a/tenants/tenant-3/elasticsearch-data-2.yaml b/tenants/tenant-3/elasticsearch-data-2.yaml index 0ac4209..797323d 100644 --- a/tenants/tenant-3/elasticsearch-data-2.yaml +++ b/tenants/tenant-3/elasticsearch-data-2.yaml @@ -26,9 +26,12 @@ port_forwards: health_check: type: "tcp" port: 9300 - interval: "20s" + # A cold Elasticsearch boot can outlast a 20s x 3 (~60s) failure budget, + # which restarts the VM just before the transport port opens. Keep the + # ~150s+ budget consistent across all three tenant-3 services. + interval: "30s" timeout: "5s" - retries: 3 + retries: 5 node_host_ip_env: "transport.publish_host" cross_node_links: - service: "elasticsearch-data-1"