fix: handle multiple nginx-proxy replicas sharing a label

.github/workflows/test.yml

@@ -131,6 +131,9 @@ jobs:
           - test-name: renew_private_keys
             setup: 2containers
             pebble-config: pebble-config.json
+          - test-name: nginx_replicas
+            setup: 2containers
+            pebble-config: pebble-config.json
     runs-on: ubuntu-latest
 
     steps:

app/entrypoint.sh

@@ -207,7 +207,7 @@ if [[ "$*" == "/bin/bash /app/start.sh" ]]; then
         echo -e "\t- Set the NGINX_PROXY_CONTAINER env var on the letsencrypt-companion container to the name of the nginx-proxy container." >&2
         echo -e "\t- Label the nginx-proxy container to use with 'com.github.nginx-proxy.nginx'." >&2
         exit 1
-    elif [[ -z "$(get_docker_gen_container)" ]] && ! is_docker_gen_container "$(get_nginx_proxy_container)"; then
+    elif [[ -z "$(get_docker_gen_container)" ]] && ! nginx_proxy_is_docker_gen; then
         echo "Error: can't get docker-gen container id !" >&2
         echo "If you are running a three containers setup, check that you are doing one of the following :" >&2
         echo -e "\t- Set the NGINX_DOCKER_GEN_CONTAINER env var on the letsencrypt-companion container to the name of the docker-gen container." >&2

app/functions.sh

@@ -32,18 +32,22 @@ function parse_true() {
  declare -r END_HEADER='## End of configuration add by letsencrypt container'
 
 function check_nginx_proxy_container_run {
-    local _nginx_proxy_container; _nginx_proxy_container=$(get_nginx_proxy_container)
-    if [[ -n "${_nginx_proxy_container}" ]]; then
-        if [[ $(docker_api "/containers/${_nginx_proxy_container}/json" | jq -r '.State.Status') = "running" ]];then
+    # get_nginx_proxy_container can return several container IDs (one per line)
+    # when the label matches more than one nginx-proxy replica.
+    local -a _nginx_proxy_containers
+    mapfile -t _nginx_proxy_containers < <(get_nginx_proxy_container)
+    if [[ ${#_nginx_proxy_containers[@]} -eq 0 ]]; then
+        echo "$(date "+%Y/%m/%d %T") Error: could not get an nginx-proxy container ID." >&2
+        return 1
+    fi
+    local _nginx_proxy_container
+    for _nginx_proxy_container in "${_nginx_proxy_containers[@]}"; do
+        if [[ $(docker_api "/containers/${_nginx_proxy_container}/json" | jq -r '.State.Status') = "running" ]]; then
             return 0
-        else
-            echo "$(date "+%Y/%m/%d %T") Error: nginx-proxy container ${_nginx_proxy_container} isn't running." >&2
-            return 1
         fi
-    else
-        echo "$(date "+%Y/%m/%d %T") Error: could not get a nginx-proxy container ID." >&2
-        return 1
-fi
+    done
+    echo "$(date "+%Y/%m/%d %T") Error: no running nginx-proxy container found (${_nginx_proxy_containers[*]})." >&2
+    return 1
 }
 
 function ascending_wildcard_locations {
@@ -325,29 +329,48 @@ function get_nginx_proxy_container {
     [[ -n "${nginx_cid}" ]] && echo "${nginx_cid}"
 }
 
+function nginx_proxy_is_docker_gen {
+    # Return 0 if at least one nginx-proxy container also runs docker-gen (all-in-one
+    # image). get_nginx_proxy_container can return several IDs when the label matches
+    # multiple replicas, so check each one instead of passing the whole string.
+    local -a _nginx_proxy_containers
+    mapfile -t _nginx_proxy_containers < <(get_nginx_proxy_container)
+    local _cid
+    for _cid in "${_nginx_proxy_containers[@]}"; do
+        is_docker_gen_container "${_cid}" && return 0
+    done
+    return 1
+}
+
 ## Nginx
 function reload_nginx {
-    local _docker_gen_container; _docker_gen_container=$(get_docker_gen_container)
-    local _nginx_proxy_container; _nginx_proxy_container=$(get_nginx_proxy_container)
-
-    if [[ -n "${_docker_gen_container:-}" ]]; then
-        # Using docker-gen and nginx in separate container
-        echo "Reloading nginx docker-gen (using separate container ${_docker_gen_container})..."
-        docker_kill "${_docker_gen_container}" SIGHUP
-
-        if [[ -n "${_nginx_proxy_container:-}" ]]; then
-            # Reloading nginx in case only certificates had been renewed
-            echo "Reloading nginx (using separate container ${_nginx_proxy_container})..."
-            docker_kill "${_nginx_proxy_container}" SIGHUP
-        fi
+    # Both getters can return several container IDs (one per line) when the
+    # label matches more than one nginx-proxy / docker-gen replica, so iterate.
+    local -a _docker_gen_containers _nginx_proxy_containers
+    mapfile -t _docker_gen_containers < <(get_docker_gen_container)
+    mapfile -t _nginx_proxy_containers < <(get_nginx_proxy_container)
+    local _cid
+
+    if [[ ${#_docker_gen_containers[@]} -gt 0 ]]; then
+        # Using docker-gen and nginx in separate containers
+        for _cid in "${_docker_gen_containers[@]}"; do
+            echo "Reloading nginx docker-gen (using separate container ${_cid})..."
+            docker_kill "${_cid}" SIGHUP
+        done
+
+        # Reloading nginx in case only certificates had been renewed
+        for _cid in "${_nginx_proxy_containers[@]}"; do
+            echo "Reloading nginx (using separate container ${_cid})..."
+            docker_kill "${_cid}" SIGHUP
+        done
     else
-        if [[ -n "${_nginx_proxy_container:-}" ]]; then
-            echo "Reloading nginx proxy (${_nginx_proxy_container})..."
-            docker_exec "${_nginx_proxy_container}" \
+        for _cid in "${_nginx_proxy_containers[@]}"; do
+            echo "Reloading nginx proxy (${_cid})..."
+            docker_exec "${_cid}" \
                 '[ "sh", "-c", "/app/docker-entrypoint.sh /usr/local/bin/docker-gen /app/nginx.tmpl /etc/nginx/conf.d/default.conf; /usr/sbin/nginx -s reload" ]' \
                 | sed -rn 's/^.*([0-9]{4}\/[0-9]{2}\/[0-9]{2}.*$)/\1/p'
             [[ ${PIPESTATUS[0]} -eq 1 ]] && echo "$(date "+%Y/%m/%d %T"), Error: can't reload nginx-proxy." >&2
-        fi
+        done
     fi
 }
 

test/config.sh

@@ -25,6 +25,7 @@ globalTests+=(
 	certs_default_renew_deprecated
 	ocsp_must_staple
 	certs_persistence
+	nginx_replicas
 )
 
 # The acme_eab test requires Pebble with a specific configuration

test/tests/nginx_replicas/run.sh

@@ -0,0 +1,83 @@
+#!/bin/bash
+
+## issue #1006: reload_nginx and check_nginx_proxy_container_run must handle
+## several nginx-proxy replicas that share the same detection label.
+
+docker_gen='replicas-docker-gen'
+nginx1='replicas-nginx-1'
+nginx2='replicas-nginx-2'
+aio1='replicas-aio-1'
+aio2='replicas-aio-2'
+companion_aio='replicas-companion-aio'
+events_file="$(mktemp)"
+
+function cleanup {
+  kill "${docker_events_pid}" 2>/dev/null && wait "${docker_events_pid}" 2>/dev/null
+  rm -f "${events_file}"
+  docker rm --force "${docker_gen}" "${nginx1}" "${nginx2}" "${aio1}" "${aio2}" "${companion_aio}" &> /dev/null
+}
+trap cleanup EXIT
+
+## Part 1: separate docker-gen + several nginx replicas (reload via SIGHUP).
+
+# A fake docker-gen so reload_nginx takes the separate-container (SIGHUP) path.
+docker run --rm -d --name "${docker_gen}" --label com.github.nginx-proxy.docker-gen nginx:alpine > /dev/null
+
+# Two nginx replicas sharing the same nginx-proxy detection label.
+docker run --rm -d --name "${nginx1}" --label com.github.nginx-proxy.nginx nginx:alpine > /dev/null
+docker run --rm -d --name "${nginx2}" --label com.github.nginx-proxy.nginx nginx:alpine > /dev/null
+
+# Record kill (SIGHUP) events.
+docker events --filter event=kill \
+  --format '{{.Actor.Attributes.name}} {{.Actor.Attributes.signal}}' > "${events_file}" &
+docker_events_pid=$!
+
+# Run reload + health-check inside the companion.
+commands='source /app/functions.sh; reload_nginx > /dev/null; check_nginx_proxy_container_run && echo CHECK_OK'
+out="$(docker run --rm \
+  -v /var/run/docker.sock:/var/run/docker.sock:ro \
+  "$1" \
+  bash -c "${commands}" 2>&1)"
+
+# Wait (up to ~10s) for both replicas' SIGHUP to be recorded.
+timeout="$(($(date +%s) + 10))"
+until grep -qE "^${nginx1} (1|SIGHUP)$" "${events_file}" && grep -qE "^${nginx2} (1|SIGHUP)$" "${events_file}"; do
+  [[ "$(date +%s)" -gt "${timeout}" ]] && break
+  sleep 0.5
+done
+
+# The health-check must succeed when several replicas share the label.
+if ! grep -q 'CHECK_OK' <<< "${out}"; then
+  echo "check_nginx_proxy_container_run did not succeed with multiple nginx replicas (issue #1006): ${out}"
+fi
+
+# Both replicas must have received SIGHUP on reload (order-independent).
+for name in "${nginx1}" "${nginx2}"; do
+  if ! grep -qE "^${name} (1|SIGHUP)$" "${events_file}"; then
+    echo "nginx replica ${name} did not receive SIGHUP on reload (issue #1006)."
+  fi
+done
+
+## Part 2: several all-in-one nginx-proxy replicas (bundled docker-gen, no separate
+## docker-gen). The companion startup check must detect the bundled docker-gen across
+## replicas instead of failing with "can't get docker-gen container id" (issue #1006).
+docker run --rm -d --name "${aio1}" --label com.github.nginx-proxy.nginx \
+  -v /var/run/docker.sock:/tmp/docker.sock:ro nginxproxy/nginx-proxy > /dev/null
+docker run --rm -d --name "${aio2}" --label com.github.nginx-proxy.nginx \
+  -v /var/run/docker.sock:/tmp/docker.sock:ro nginxproxy/nginx-proxy > /dev/null
+
+# Start the companion through its real entrypoint (anonymous writable volumes so the
+# later checks pass). With the bug it exits early on the docker-gen detection.
+docker run -d --name "${companion_aio}" \
+  -v /var/run/docker.sock:/var/run/docker.sock:ro \
+  -v /etc/nginx/certs \
+  -v /etc/acme.sh \
+  "$1" > /dev/null
+sleep 6
+
+if docker logs "${companion_aio}" 2>&1 | grep -q "can't get docker-gen container id"; then
+  echo "companion failed to start with multiple all-in-one nginx-proxy replicas (issue #1006)."
+fi
+if [[ "$(docker inspect -f '{{.State.Running}}' "${companion_aio}" 2>/dev/null)" != 'true' ]]; then
+  echo "companion is not running with multiple all-in-one nginx-proxy replicas (issue #1006)."
+fi