bounded graceful shutdown for extra connections

Author: machine-moon

docs/log-message-tags/next-number

@@ -1 +1 @@
-10619
+10623

server/mpm/event/event.c

@@ -188,6 +188,7 @@ static int num_listensocks = 0;
 static apr_int32_t conns_this_child;        /* MaxConnectionsPerChild, only access
                                                in listener thread */
 static apr_uint32_t connection_count = 0;   /* Number of open connections */
+static apr_uint32_t extra_connection_count = 0; /* Number of open connections that the MPM does not own */
 static apr_uint32_t lingering_count = 0;    /* Number of connections in lingering close */
 static apr_uint32_t suspended_count = 0;    /* Number of suspended connections */
 static apr_uint32_t clogged_count = 0;      /* Number of threads processing ssl conns */
@@ -873,16 +874,39 @@ static apr_status_t decrement_connection_count(void *cs_)
 
 static void ap_mpm_note_extra_connection_added(void)
 {
-    apr_atomic_inc32(&connection_count);
+    apr_atomic_inc32(&extra_connection_count);
 }
 
 static void ap_mpm_note_extra_connection_removed(void)
 {
-    int is_last_connection = !apr_atomic_dec32(&connection_count);
+    apr_atomic_dec32(&extra_connection_count);
+}
 
-    /* Wake a listener blocked waiting for connection_count to drain. */
-    if (listener_is_wakeable && is_last_connection && listener_may_exit) {
-        apr_pollset_wakeup(event_pollset);
+static void wait_for_extra_connections(void)
+{
+    apr_uint32_t count = apr_atomic_read32(&extra_connection_count);
+    apr_time_t graceful, timeout, deadline;
+
+    if (count == 0) {
+        return;
+    }
+
+    graceful = apr_time_from_sec(ap_graceful_shutdown_timeout);
+    timeout = (graceful > ap_server_conf->timeout) ? graceful : ap_server_conf->timeout;
+    deadline = apr_time_now() + timeout;
+
+    do {
+        apr_sleep(apr_time_from_msec(100));
+        count = apr_atomic_read32(&extra_connection_count);
+    } while (count > 0 && apr_time_now() < deadline);
+
+    if (count > 0) {
+        ap_log_error(APLOG_MARK, APLOG_WARNING, 0, ap_server_conf,
+                     APLOGNO(10619)
+                     "Child: %u connection(s) noted by modules did not "
+                     "finish within %" APR_TIME_T_FMT " seconds, "
+                     "exiting anyway",
+                     count, apr_time_sec(timeout));
     }
 }
 
@@ -3154,6 +3178,10 @@ static void child_main(int child_num_arg, int child_bucket)
                      rv == AP_MPM_PODX_GRACEFUL ? "graceful" : "ungraceful");
     }
 
+    if (terminate_mode == ST_GRACEFUL) {
+        wait_for_extra_connections();
+    }
+
     free(threads);
 
     clean_child_exit(resource_shortage ? APEXIT_CHILDSICK : 0);

server/mpm/prefork/prefork.c

@@ -219,6 +219,34 @@ static void prefork_note_child_started(int slot, pid_t pid)
     ap_run_child_status(ap_server_conf, pid, gen, slot, MPM_CHILD_STARTED);
 }
 
+static void wait_for_extra_connections(void)
+{
+    apr_uint32_t count = apr_atomic_read32(&connection_count);
+    apr_time_t graceful, timeout, deadline;
+
+    if (count == 0) {
+        return;
+    }
+
+    graceful = apr_time_from_sec(ap_graceful_shutdown_timeout);
+    timeout = (graceful > ap_server_conf->timeout) ? graceful : ap_server_conf->timeout;
+    deadline = apr_time_now() + timeout;
+
+    do {
+        apr_sleep(apr_time_from_msec(100));
+        count = apr_atomic_read32(&connection_count);
+    } while (count > 0 && apr_time_now() < deadline);
+
+    if (count > 0) {
+        ap_log_error(APLOG_MARK, APLOG_WARNING, 0, ap_server_conf,
+                     APLOGNO(10620)
+                     "Child: %u connection(s) noted by modules did not "
+                     "finish within %" APR_TIME_T_FMT " seconds, "
+                     "exiting anyway",
+                     count, apr_time_sec(timeout));
+    }
+}
+
 /* a clean exit from a child with proper cleanup */
 static void clean_child_exit_ex(int code, int from_signal) __attribute__ ((noreturn));
 static void clean_child_exit_ex(int code, int from_signal)
@@ -232,9 +260,7 @@ static void clean_child_exit_ex(int code, int from_signal)
         if (!code && !from_signal) {
             ap_run_child_stopping(pchild, !retained->mpm->is_ungraceful);
             if (!retained->mpm->is_ungraceful) {
-                while (apr_atomic_read32(&connection_count) > 0) {
-                    apr_sleep(apr_time_from_msec(100));
-                }
+                wait_for_extra_connections();
             }
             ap_run_child_stopped(pchild, !retained->mpm->is_ungraceful);
         }

server/mpm/winnt/child.c

@@ -146,6 +146,36 @@ void ap_mpm_note_extra_connection_removed(void)
     apr_atomic_dec32(&extra_connection_count);
 }
 
+static void wait_for_extra_connections(void)
+{
+    apr_uint32_t count = apr_atomic_read32(&extra_connection_count);
+    apr_time_t graceful, timeout;
+    int time_remains;
+
+    if (count == 0) {
+        return;
+    }
+
+    graceful = apr_time_from_sec(ap_graceful_shutdown_timeout);
+    timeout = (graceful > ap_server_conf->timeout) ? graceful : ap_server_conf->timeout;
+    time_remains = (int)(timeout / APR_TIME_C(1000));
+
+    do {
+        Sleep(100);
+        time_remains -= 100;
+        count = apr_atomic_read32(&extra_connection_count);
+    } while (count > 0 && time_remains > 0);
+
+    if (count > 0) {
+        ap_log_error(APLOG_MARK, APLOG_WARNING, APR_SUCCESS, ap_server_conf,
+                     APLOGNO(10622)
+                     "Child: %u connection(s) noted by modules did not "
+                     "finish within %" APR_TIME_T_FMT " seconds, "
+                     "exiting anyway",
+                     count, apr_time_sec(timeout));
+    }
+}
+
 static void mpm_recycle_completion_context(winnt_conn_ctx_t *context)
 {
     /* Recycle the completion context.
@@ -1249,12 +1279,6 @@ void child_main(apr_pool_t *pconf, DWORD parent_pid)
         }
     }
 
-    /* Drain externally accepted connections within what is left of the deadline. */
-    while (apr_atomic_read32(&extra_connection_count) > 0 && time_remains >= 0) {
-        Sleep(100);
-        time_remains -= 100;
-    }
-
     if (threads_created) {
         ap_log_error(APLOG_MARK, APLOG_NOTICE, APR_SUCCESS, ap_server_conf, APLOGNO(00363)
                      "Child: Waiting for %d threads timed out, terminating process.",
@@ -1280,6 +1304,10 @@ void child_main(apr_pool_t *pconf, DWORD parent_pid)
     ap_log_error(APLOG_MARK, APLOG_NOTICE, APR_SUCCESS, ap_server_conf, APLOGNO(00364)
                  "Child: All worker threads have exited.");
 
+    if (graceful_shutdown) {
+        wait_for_extra_connections();
+    }
+
     ap_run_child_stopped(pchild, graceful_shutdown);
 
     apr_thread_mutex_destroy(child_lock);

server/mpm/worker/worker.c

@@ -523,6 +523,34 @@ static void ap_mpm_note_extra_connection_removed(void)
     apr_atomic_dec32(&connection_count);
 }
 
+static void wait_for_extra_connections(void)
+{
+    apr_uint32_t count = apr_atomic_read32(&connection_count);
+    apr_time_t graceful, timeout, deadline;
+
+    if (count == 0) {
+        return;
+    }
+
+    graceful = apr_time_from_sec(ap_graceful_shutdown_timeout);
+    timeout = (graceful > ap_server_conf->timeout) ? graceful : ap_server_conf->timeout;
+    deadline = apr_time_now() + timeout;
+
+    do {
+        apr_sleep(apr_time_from_msec(100));
+        count = apr_atomic_read32(&connection_count);
+    } while (count > 0 && apr_time_now() < deadline);
+
+    if (count > 0) {
+        ap_log_error(APLOG_MARK, APLOG_WARNING, 0, ap_server_conf,
+                     APLOGNO(10621)
+                     "Child: %u connection(s) noted by modules did not "
+                     "finish within %" APR_TIME_T_FMT " seconds, "
+                     "exiting anyway",
+                     count, apr_time_sec(timeout));
+    }
+}
+
 static void unblock_signal(int sig)
 {
     sigset_t sig_mask;
@@ -1341,9 +1369,7 @@ static void child_main(int child_num_arg, int child_bucket)
     }
 
     if (terminate_mode == ST_GRACEFUL) {
-        while (apr_atomic_read32(&connection_count) > 0) {
-            apr_sleep(apr_time_from_msec(100));
-        }
+        wait_for_extra_connections();
     }
 
     free(threads);