[BACKPORT] allow mpm to be note connections that are not TCP based.

CHANGES

@@ -1,6 +1,14 @@
                                                          -*- coding: utf-8 -*-
 Changes with Apache 2.4.69
 
+  *) mpm_event, mpm_worker, mpm_winnt, mpm_prefork: Add the optional functions
+     ap_mpm_note_extra_connection_added() and
+     ap_mpm_note_extra_connection_removed() to account for connections managed
+     outside the MPM accept loop (e.g., UDP).  On graceful stop a child waits
+     for those connections no longer than max(Timeout,
+     GracefulShutdownTimeout), then warns and exits.
+     [Tarek Ibrahim <tareki@pulsarxtech.com> <t1br4h1m@gmail.com>]
+
   *) mod_md: OpenSSL 4 compatibility.
 
   *) pytest_suite: Port of the old PERL test framework to Python

docs/log-message-tags/next-number

@@ -0,0 +1 @@
+10623

include/ap_mmn.h

@@ -612,14 +612,17 @@
  * 20120211.140 (2.4.64-dev) Add ap_set_time_process_request() to scoreboard.h
  * 20120211.141 (2.4.64-dev) add ap_stat_check() to httpd.h
  * 20120211.142 (2.4.64-dev) Add ap_*_timingsafe() to httpd.h
+ * 20120211.143 (2.4.69-dev) Add the optional functions ap_mpm_note_extra_
+ *                           connection_added() and ap_mpm_note_extra_
+ *                           connection_removed() to mpm_common.h
  */
 
 #define MODULE_MAGIC_COOKIE 0x41503234UL /* "AP24" */
 
 #ifndef MODULE_MAGIC_NUMBER_MAJOR
 #define MODULE_MAGIC_NUMBER_MAJOR 20120211
 #endif
-#define MODULE_MAGIC_NUMBER_MINOR 142                 /* 0...n */
+#define MODULE_MAGIC_NUMBER_MINOR 143                 /* 0...n */
 
 /**
  * Determine if the server's current MODULE_MAGIC_NUMBER is at least a

include/mpm_common.h

@@ -40,6 +40,7 @@
 #include "ap_config.h"
 #include "ap_mpm.h"
 #include "scoreboard.h"
+#include "apr_optional.h"
 
 #if APR_HAVE_NETINET_TCP_H
 #include <netinet/tcp.h>    /* for TCP_NODELAY */
@@ -479,6 +480,26 @@ AP_DECLARE_HOOK(void, child_stopping,
  */
 void mpm_common_pre_config(apr_pool_t *pconf);
 
+/**
+ * Hooks for modules to report connections the MPM did not accept itself.
+ *
+ * MPMs that wait for their connection count to drain before stopping a child
+ * need this so externally accepted connections keep the child alive until
+ * they finish.
+ *
+ * Call ap_mpm_note_extra_connection_added() when such a connection starts,
+ * and ap_mpm_note_extra_connection_removed() when it ends. These functions
+ * may be NULL if the active MPM does not implement them.
+ *
+ * A module using them is expected to notice that the child is stopping (e.g.
+ * with the child_stopping hook) and to end the connections it noted in a
+ * timely manner, gracefully or not.  The MPM waits for them no longer than
+ * max(Timeout, GracefulShutdownTimeout), then logs a warning and exits
+ * anyway, possibly cutting those connections short.
+ */
+APR_DECLARE_OPTIONAL_FN(void, ap_mpm_note_extra_connection_added, (void));
+APR_DECLARE_OPTIONAL_FN(void, ap_mpm_note_extra_connection_removed, (void));
+
 #ifdef __cplusplus
 }
 #endif

server/mpm/event/event.c

@@ -182,6 +182,7 @@ static int num_listensocks = 0;
 static apr_int32_t conns_this_child;        /* MaxConnectionsPerChild, only access
                                                in listener thread */
 static apr_uint32_t connection_count = 0;   /* Number of open connections */
+static apr_uint32_t extra_connection_count = 0; /* Number of open connections that the MPM does not own */
 static apr_uint32_t lingering_count = 0;    /* Number of connections in lingering close */
 static apr_uint32_t suspended_count = 0;    /* Number of suspended connections */
 static apr_uint32_t clogged_count = 0;      /* Number of threads processing ssl conns */
@@ -828,6 +829,44 @@ static apr_status_t decrement_connection_count(void *cs_)
     return APR_SUCCESS;
 }
 
+static void ap_mpm_note_extra_connection_added(void)
+{
+    apr_atomic_inc32(&extra_connection_count);
+}
+
+static void ap_mpm_note_extra_connection_removed(void)
+{
+    apr_atomic_dec32(&extra_connection_count);
+}
+
+static void wait_for_extra_connections(void)
+{
+    apr_uint32_t count = apr_atomic_read32(&extra_connection_count);
+    apr_time_t graceful, timeout, deadline;
+
+    if (count == 0) {
+        return;
+    }
+
+    graceful = apr_time_from_sec(ap_graceful_shutdown_timeout);
+    timeout = (graceful > ap_server_conf->timeout) ? graceful : ap_server_conf->timeout;
+    deadline = apr_time_now() + timeout;
+
+    do {
+        apr_sleep(apr_time_from_msec(100));
+        count = apr_atomic_read32(&extra_connection_count);
+    } while (count > 0 && apr_time_now() < deadline);
+
+    if (count > 0) {
+        ap_log_error(APLOG_MARK, APLOG_WARNING, 0, ap_server_conf,
+                     APLOGNO(10619)
+                     "Child: %u connection(s) noted by modules did not "
+                     "finish within %" APR_TIME_T_FMT " seconds, "
+                     "exiting anyway",
+                     count, apr_time_sec(timeout));
+    }
+}
+
 static void notify_suspend(event_conn_state_t *cs)
 {
     ap_run_suspend_connection(cs->c, cs->r);
@@ -2805,6 +2844,10 @@ static void child_main(int child_num_arg, int child_bucket)
                      rv == AP_MPM_PODX_GRACEFUL ? "graceful" : "ungraceful");
     }
 
+    if (terminate_mode == ST_GRACEFUL) {
+        wait_for_extra_connections();
+    }
+
     free(threads);
 
     clean_child_exit(resource_shortage ? APEXIT_CHILDSICK : 0);
@@ -3466,6 +3509,10 @@ static void setup_slave_conn(conn_rec *c, void *csd)
     event_conn_state_t *cs;
 
     mcs = ap_get_module_config(c->master->conn_config, &mpm_event_module);
+    if (!mcs) {
+        /* Master connection is not managed by this MPM; nothing to inherit. */
+        return;
+    }
 
     cs = apr_pcalloc(c->pool, sizeof(*cs));
     cs->c = c;
@@ -3607,6 +3654,9 @@ static int event_pre_config(apr_pool_t * pconf, apr_pool_t * plog,
     const char *userdata_key = "mpm_event_module";
     int test_atomics = 0;
 
+    APR_REGISTER_OPTIONAL_FN(ap_mpm_note_extra_connection_added);
+    APR_REGISTER_OPTIONAL_FN(ap_mpm_note_extra_connection_removed);
+
     debug = ap_exists_config_define("DEBUG");
 
     if (debug) {

server/mpm/prefork/prefork.c

@@ -18,6 +18,7 @@
 #include "apr_portable.h"
 #include "apr_strings.h"
 #include "apr_thread_proc.h"
+#include "apr_atomic.h"
 #include "apr_signal.h"
 
 #define APR_WANT_STDIO
@@ -88,6 +89,7 @@
 
 /* config globals */
 
+static apr_uint32_t extra_connection_count = 0; /* Number of open connections that the MPM does not own */
 static int ap_daemons_to_start=0;
 static int ap_daemons_min_free=0;
 static int ap_daemons_max_free=0;
@@ -214,20 +216,60 @@ static void prefork_note_child_started(int slot, pid_t pid)
     ap_run_child_status(ap_server_conf, pid, gen, slot, MPM_CHILD_STARTED);
 }
 
+static void ap_mpm_note_extra_connection_added(void)
+{
+    apr_atomic_inc32(&extra_connection_count);
+}
+
+static void ap_mpm_note_extra_connection_removed(void)
+{
+    apr_atomic_dec32(&extra_connection_count);
+}
+
+static void wait_for_extra_connections(void)
+{
+    apr_uint32_t count = apr_atomic_read32(&extra_connection_count);
+    apr_time_t graceful, timeout, deadline;
+
+    if (count == 0) {
+        return;
+    }
+
+    graceful = apr_time_from_sec(ap_graceful_shutdown_timeout);
+    timeout = (graceful > ap_server_conf->timeout) ? graceful : ap_server_conf->timeout;
+    deadline = apr_time_now() + timeout;
+
+    do {
+        apr_sleep(apr_time_from_msec(100));
+        count = apr_atomic_read32(&extra_connection_count);
+    } while (count > 0 && apr_time_now() < deadline);
+
+    if (count > 0) {
+        ap_log_error(APLOG_MARK, APLOG_WARNING, 0, ap_server_conf,
+                     APLOGNO(10620)
+                     "Child: %u connection(s) noted by modules did not "
+                     "finish within %" APR_TIME_T_FMT " seconds, "
+                     "exiting anyway",
+                     count, apr_time_sec(timeout));
+    }
+}
+
 /* a clean exit from a child with proper cleanup */
-static void clean_child_exit(int code) __attribute__ ((noreturn));
-static void clean_child_exit(int code)
+static void clean_child_exit_ex(int code, int from_signal) __attribute__ ((noreturn));
+static void clean_child_exit_ex(int code, int from_signal)
 {
     retained->mpm->mpm_state = AP_MPMQ_STOPPING;
 
     apr_signal(SIGHUP, SIG_IGN);
     apr_signal(SIGTERM, SIG_IGN);
 
-    if (code == 0) {
-        ap_run_child_stopping(pchild, 0);
-    }
-
     if (pchild) {
+        if (!code && !from_signal) {
+            ap_run_child_stopping(pchild, !retained->mpm->is_ungraceful);
+            if (!retained->mpm->is_ungraceful) {
+                wait_for_extra_connections();
+            }
+        }
         apr_pool_destroy(pchild);
     }
 
@@ -240,6 +282,12 @@ static void clean_child_exit(int code)
     exit(code);
 }
 
+static void clean_child_exit(int code) __attribute__ ((noreturn));
+static void clean_child_exit(int code)
+{
+    clean_child_exit_ex(code, 0);
+}
+
 static apr_status_t accept_mutex_on(void)
 {
     apr_status_t rv = apr_proc_mutex_lock(my_bucket->mutex);
@@ -356,7 +404,7 @@ static const char *prefork_get_name(void)
 
 static void just_die(int sig)
 {
-    clean_child_exit(0);
+    clean_child_exit_ex(0, 1);
 }
 
 /* volatile because it's updated from a signal handler */
@@ -1286,6 +1334,9 @@ static int prefork_pre_config(apr_pool_t *p, apr_pool_t *plog, apr_pool_t *ptemp
     apr_status_t rv;
     const char *userdata_key = "mpm_prefork_module";
 
+    APR_REGISTER_OPTIONAL_FN(ap_mpm_note_extra_connection_added);
+    APR_REGISTER_OPTIONAL_FN(ap_mpm_note_extra_connection_removed);
+
     debug = ap_exists_config_define("DEBUG");
 
     if (debug) {

server/mpm/winnt/child.c

@@ -123,9 +123,49 @@ static winnt_conn_ctx_t *qhead = NULL;
 static winnt_conn_ctx_t *qtail = NULL;
 static apr_uint32_t num_completion_contexts = 0;
 static apr_uint32_t max_num_completion_contexts = 0;
+static apr_uint32_t extra_connection_count = 0; /* Number of open connections that the MPM does not own */
 static HANDLE ThreadDispatchIOCP = NULL;
 static HANDLE qwait_event = NULL;
 
+void ap_mpm_note_extra_connection_added(void)
+{
+    apr_atomic_inc32(&extra_connection_count);
+}
+
+void ap_mpm_note_extra_connection_removed(void)
+{
+    apr_atomic_dec32(&extra_connection_count);
+}
+
+static void wait_for_extra_connections(void)
+{
+    apr_uint32_t count = apr_atomic_read32(&extra_connection_count);
+    apr_time_t graceful, timeout, time_remains;
+
+    if (count == 0) {
+        return;
+    }
+
+    graceful = apr_time_from_sec(ap_graceful_shutdown_timeout);
+    timeout = (graceful > ap_server_conf->timeout) ? graceful : ap_server_conf->timeout;
+    time_remains = timeout / APR_TIME_C(1000);
+
+    do {
+        Sleep(100);
+        time_remains -= 100;
+        count = apr_atomic_read32(&extra_connection_count);
+    } while (count > 0 && time_remains > 0);
+
+    if (count > 0) {
+        ap_log_error(APLOG_MARK, APLOG_WARNING, APR_SUCCESS, ap_server_conf,
+                     APLOGNO(10622)
+                     "Child: %u connection(s) noted by modules did not "
+                     "finish within %" APR_TIME_T_FMT " seconds, "
+                     "exiting anyway",
+                     count, apr_time_sec(timeout));
+    }
+}
+
 static void mpm_recycle_completion_context(winnt_conn_ctx_t *context)
 {
     /* Recycle the completion context.
@@ -1272,7 +1312,6 @@ void child_main(apr_pool_t *pconf, DWORD parent_pid)
         }
     }
 
-    /* Kill remaining threads off the hard way */
     if (threads_created) {
         ap_log_error(APLOG_MARK, APLOG_NOTICE, APR_SUCCESS, ap_server_conf, APLOGNO(00363)
                      "Child: Terminating %d threads that failed to exit.",
@@ -1291,6 +1330,10 @@ void child_main(apr_pool_t *pconf, DWORD parent_pid)
     ap_log_error(APLOG_MARK, APLOG_NOTICE, APR_SUCCESS, ap_server_conf, APLOGNO(00364)
                  "Child: All worker threads have exited.");
 
+    if (graceful_shutdown) {
+        wait_for_extra_connections();
+    }
+
     apr_thread_mutex_destroy(child_lock);
     apr_thread_mutex_destroy(qlock);
     CloseHandle(qwait_event);

server/mpm/winnt/mpm_winnt.c

@@ -1369,6 +1369,9 @@ static int winnt_pre_config(apr_pool_t *pconf_, apr_pool_t *plog, apr_pool_t *pt
      *   -k runservice [WinNT errors logged from rewrite_args]
      */
 
+    APR_REGISTER_OPTIONAL_FN(ap_mpm_note_extra_connection_added);
+    APR_REGISTER_OPTIONAL_FN(ap_mpm_note_extra_connection_removed);
+
     /* Initialize shared static objects.
      * TODO: Put config related statics into an sconf structure.
      */

server/mpm/winnt/mpm_winnt.h

@@ -91,6 +91,8 @@ void hold_console_open_on_error(void);
 
 /* From child.c: */
 void child_main(apr_pool_t *pconf, DWORD parent_pid);
+void ap_mpm_note_extra_connection_added(void);
+void ap_mpm_note_extra_connection_removed(void);
 
 #endif /* APACHE_MPM_WINNT_H */
 /** @} */