diff --git a/CHANGES b/CHANGES index b545f955647..d18857bb8ae 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,14 @@ -*- coding: utf-8 -*- Changes with Apache 2.4.69 + *) mpm_event, mpm_worker, mpm_winnt, mpm_prefork: Add the optional functions + ap_mpm_note_extra_connection_added() and + ap_mpm_note_extra_connection_removed() to account for connections managed + outside the MPM accept loop (e.g., UDP). On graceful stop a child waits + for those connections no longer than max(Timeout, + GracefulShutdownTimeout), then warns and exits. + [Tarek Ibrahim ] + *) mod_md: OpenSSL 4 compatibility. *) pytest_suite: Port of the old PERL test framework to Python diff --git a/docs/log-message-tags/next-number b/docs/log-message-tags/next-number new file mode 100644 index 00000000000..29cdba2b98b --- /dev/null +++ b/docs/log-message-tags/next-number @@ -0,0 +1 @@ +10623 diff --git a/include/ap_mmn.h b/include/ap_mmn.h index d00c6d1ec9c..122c1070af1 100644 --- a/include/ap_mmn.h +++ b/include/ap_mmn.h @@ -612,6 +612,9 @@ * 20120211.140 (2.4.64-dev) Add ap_set_time_process_request() to scoreboard.h * 20120211.141 (2.4.64-dev) add ap_stat_check() to httpd.h * 20120211.142 (2.4.64-dev) Add ap_*_timingsafe() to httpd.h + * 20120211.143 (2.4.69-dev) Add the optional functions ap_mpm_note_extra_ + * connection_added() and ap_mpm_note_extra_ + * connection_removed() to mpm_common.h */ #define MODULE_MAGIC_COOKIE 0x41503234UL /* "AP24" */ @@ -619,7 +622,7 @@ #ifndef MODULE_MAGIC_NUMBER_MAJOR #define MODULE_MAGIC_NUMBER_MAJOR 20120211 #endif -#define MODULE_MAGIC_NUMBER_MINOR 142 /* 0...n */ +#define MODULE_MAGIC_NUMBER_MINOR 143 /* 0...n */ /** * Determine if the server's current MODULE_MAGIC_NUMBER is at least a diff --git a/include/mpm_common.h b/include/mpm_common.h index 6b3d1f536fb..a1fe31af74a 100644 --- a/include/mpm_common.h +++ b/include/mpm_common.h @@ -40,6 +40,7 @@ #include "ap_config.h" #include "ap_mpm.h" #include "scoreboard.h" +#include "apr_optional.h" #if APR_HAVE_NETINET_TCP_H #include /* for TCP_NODELAY */ @@ -479,6 +480,26 @@ AP_DECLARE_HOOK(void, child_stopping, */ void mpm_common_pre_config(apr_pool_t *pconf); +/** + * Hooks for modules to report connections the MPM did not accept itself. + * + * MPMs that wait for their connection count to drain before stopping a child + * need this so externally accepted connections keep the child alive until + * they finish. + * + * Call ap_mpm_note_extra_connection_added() when such a connection starts, + * and ap_mpm_note_extra_connection_removed() when it ends. These functions + * may be NULL if the active MPM does not implement them. + * + * A module using them is expected to notice that the child is stopping (e.g. + * with the child_stopping hook) and to end the connections it noted in a + * timely manner, gracefully or not. The MPM waits for them no longer than + * max(Timeout, GracefulShutdownTimeout), then logs a warning and exits + * anyway, possibly cutting those connections short. + */ +APR_DECLARE_OPTIONAL_FN(void, ap_mpm_note_extra_connection_added, (void)); +APR_DECLARE_OPTIONAL_FN(void, ap_mpm_note_extra_connection_removed, (void)); + #ifdef __cplusplus } #endif diff --git a/server/mpm/event/event.c b/server/mpm/event/event.c index 050d823809b..ac79a0757f2 100644 --- a/server/mpm/event/event.c +++ b/server/mpm/event/event.c @@ -182,6 +182,7 @@ static int num_listensocks = 0; static apr_int32_t conns_this_child; /* MaxConnectionsPerChild, only access in listener thread */ static apr_uint32_t connection_count = 0; /* Number of open connections */ +static apr_uint32_t extra_connection_count = 0; /* Number of open connections that the MPM does not own */ static apr_uint32_t lingering_count = 0; /* Number of connections in lingering close */ static apr_uint32_t suspended_count = 0; /* Number of suspended connections */ static apr_uint32_t clogged_count = 0; /* Number of threads processing ssl conns */ @@ -828,6 +829,44 @@ static apr_status_t decrement_connection_count(void *cs_) return APR_SUCCESS; } +static void ap_mpm_note_extra_connection_added(void) +{ + apr_atomic_inc32(&extra_connection_count); +} + +static void ap_mpm_note_extra_connection_removed(void) +{ + apr_atomic_dec32(&extra_connection_count); +} + +static void wait_for_extra_connections(void) +{ + apr_uint32_t count = apr_atomic_read32(&extra_connection_count); + apr_time_t graceful, timeout, deadline; + + if (count == 0) { + return; + } + + graceful = apr_time_from_sec(ap_graceful_shutdown_timeout); + timeout = (graceful > ap_server_conf->timeout) ? graceful : ap_server_conf->timeout; + deadline = apr_time_now() + timeout; + + do { + apr_sleep(apr_time_from_msec(100)); + count = apr_atomic_read32(&extra_connection_count); + } while (count > 0 && apr_time_now() < deadline); + + if (count > 0) { + ap_log_error(APLOG_MARK, APLOG_WARNING, 0, ap_server_conf, + APLOGNO(10619) + "Child: %u connection(s) noted by modules did not " + "finish within %" APR_TIME_T_FMT " seconds, " + "exiting anyway", + count, apr_time_sec(timeout)); + } +} + static void notify_suspend(event_conn_state_t *cs) { ap_run_suspend_connection(cs->c, cs->r); @@ -2805,6 +2844,10 @@ static void child_main(int child_num_arg, int child_bucket) rv == AP_MPM_PODX_GRACEFUL ? "graceful" : "ungraceful"); } + if (terminate_mode == ST_GRACEFUL) { + wait_for_extra_connections(); + } + free(threads); clean_child_exit(resource_shortage ? APEXIT_CHILDSICK : 0); @@ -3466,6 +3509,10 @@ static void setup_slave_conn(conn_rec *c, void *csd) event_conn_state_t *cs; mcs = ap_get_module_config(c->master->conn_config, &mpm_event_module); + if (!mcs) { + /* Master connection is not managed by this MPM; nothing to inherit. */ + return; + } cs = apr_pcalloc(c->pool, sizeof(*cs)); cs->c = c; @@ -3607,6 +3654,9 @@ static int event_pre_config(apr_pool_t * pconf, apr_pool_t * plog, const char *userdata_key = "mpm_event_module"; int test_atomics = 0; + APR_REGISTER_OPTIONAL_FN(ap_mpm_note_extra_connection_added); + APR_REGISTER_OPTIONAL_FN(ap_mpm_note_extra_connection_removed); + debug = ap_exists_config_define("DEBUG"); if (debug) { diff --git a/server/mpm/prefork/prefork.c b/server/mpm/prefork/prefork.c index b5adb57bea1..fd6c6a039ed 100644 --- a/server/mpm/prefork/prefork.c +++ b/server/mpm/prefork/prefork.c @@ -18,6 +18,7 @@ #include "apr_portable.h" #include "apr_strings.h" #include "apr_thread_proc.h" +#include "apr_atomic.h" #include "apr_signal.h" #define APR_WANT_STDIO @@ -88,6 +89,7 @@ /* config globals */ +static apr_uint32_t extra_connection_count = 0; /* Number of open connections that the MPM does not own */ static int ap_daemons_to_start=0; static int ap_daemons_min_free=0; static int ap_daemons_max_free=0; @@ -214,20 +216,60 @@ static void prefork_note_child_started(int slot, pid_t pid) ap_run_child_status(ap_server_conf, pid, gen, slot, MPM_CHILD_STARTED); } +static void ap_mpm_note_extra_connection_added(void) +{ + apr_atomic_inc32(&extra_connection_count); +} + +static void ap_mpm_note_extra_connection_removed(void) +{ + apr_atomic_dec32(&extra_connection_count); +} + +static void wait_for_extra_connections(void) +{ + apr_uint32_t count = apr_atomic_read32(&extra_connection_count); + apr_time_t graceful, timeout, deadline; + + if (count == 0) { + return; + } + + graceful = apr_time_from_sec(ap_graceful_shutdown_timeout); + timeout = (graceful > ap_server_conf->timeout) ? graceful : ap_server_conf->timeout; + deadline = apr_time_now() + timeout; + + do { + apr_sleep(apr_time_from_msec(100)); + count = apr_atomic_read32(&extra_connection_count); + } while (count > 0 && apr_time_now() < deadline); + + if (count > 0) { + ap_log_error(APLOG_MARK, APLOG_WARNING, 0, ap_server_conf, + APLOGNO(10620) + "Child: %u connection(s) noted by modules did not " + "finish within %" APR_TIME_T_FMT " seconds, " + "exiting anyway", + count, apr_time_sec(timeout)); + } +} + /* a clean exit from a child with proper cleanup */ -static void clean_child_exit(int code) __attribute__ ((noreturn)); -static void clean_child_exit(int code) +static void clean_child_exit_ex(int code, int from_signal) __attribute__ ((noreturn)); +static void clean_child_exit_ex(int code, int from_signal) { retained->mpm->mpm_state = AP_MPMQ_STOPPING; apr_signal(SIGHUP, SIG_IGN); apr_signal(SIGTERM, SIG_IGN); - if (code == 0) { - ap_run_child_stopping(pchild, 0); - } - if (pchild) { + if (!code && !from_signal) { + ap_run_child_stopping(pchild, !retained->mpm->is_ungraceful); + if (!retained->mpm->is_ungraceful) { + wait_for_extra_connections(); + } + } apr_pool_destroy(pchild); } @@ -240,6 +282,12 @@ static void clean_child_exit(int code) exit(code); } +static void clean_child_exit(int code) __attribute__ ((noreturn)); +static void clean_child_exit(int code) +{ + clean_child_exit_ex(code, 0); +} + static apr_status_t accept_mutex_on(void) { apr_status_t rv = apr_proc_mutex_lock(my_bucket->mutex); @@ -356,7 +404,7 @@ static const char *prefork_get_name(void) static void just_die(int sig) { - clean_child_exit(0); + clean_child_exit_ex(0, 1); } /* volatile because it's updated from a signal handler */ @@ -1286,6 +1334,9 @@ static int prefork_pre_config(apr_pool_t *p, apr_pool_t *plog, apr_pool_t *ptemp apr_status_t rv; const char *userdata_key = "mpm_prefork_module"; + APR_REGISTER_OPTIONAL_FN(ap_mpm_note_extra_connection_added); + APR_REGISTER_OPTIONAL_FN(ap_mpm_note_extra_connection_removed); + debug = ap_exists_config_define("DEBUG"); if (debug) { diff --git a/server/mpm/winnt/child.c b/server/mpm/winnt/child.c index 05151a885ea..d0ea325196b 100644 --- a/server/mpm/winnt/child.c +++ b/server/mpm/winnt/child.c @@ -123,9 +123,49 @@ static winnt_conn_ctx_t *qhead = NULL; static winnt_conn_ctx_t *qtail = NULL; static apr_uint32_t num_completion_contexts = 0; static apr_uint32_t max_num_completion_contexts = 0; +static apr_uint32_t extra_connection_count = 0; /* Number of open connections that the MPM does not own */ static HANDLE ThreadDispatchIOCP = NULL; static HANDLE qwait_event = NULL; +void ap_mpm_note_extra_connection_added(void) +{ + apr_atomic_inc32(&extra_connection_count); +} + +void ap_mpm_note_extra_connection_removed(void) +{ + apr_atomic_dec32(&extra_connection_count); +} + +static void wait_for_extra_connections(void) +{ + apr_uint32_t count = apr_atomic_read32(&extra_connection_count); + apr_time_t graceful, timeout, time_remains; + + if (count == 0) { + return; + } + + graceful = apr_time_from_sec(ap_graceful_shutdown_timeout); + timeout = (graceful > ap_server_conf->timeout) ? graceful : ap_server_conf->timeout; + time_remains = timeout / APR_TIME_C(1000); + + do { + Sleep(100); + time_remains -= 100; + count = apr_atomic_read32(&extra_connection_count); + } while (count > 0 && time_remains > 0); + + if (count > 0) { + ap_log_error(APLOG_MARK, APLOG_WARNING, APR_SUCCESS, ap_server_conf, + APLOGNO(10622) + "Child: %u connection(s) noted by modules did not " + "finish within %" APR_TIME_T_FMT " seconds, " + "exiting anyway", + count, apr_time_sec(timeout)); + } +} + static void mpm_recycle_completion_context(winnt_conn_ctx_t *context) { /* Recycle the completion context. @@ -1272,7 +1312,6 @@ void child_main(apr_pool_t *pconf, DWORD parent_pid) } } - /* Kill remaining threads off the hard way */ if (threads_created) { ap_log_error(APLOG_MARK, APLOG_NOTICE, APR_SUCCESS, ap_server_conf, APLOGNO(00363) "Child: Terminating %d threads that failed to exit.", @@ -1291,6 +1330,10 @@ void child_main(apr_pool_t *pconf, DWORD parent_pid) ap_log_error(APLOG_MARK, APLOG_NOTICE, APR_SUCCESS, ap_server_conf, APLOGNO(00364) "Child: All worker threads have exited."); + if (graceful_shutdown) { + wait_for_extra_connections(); + } + apr_thread_mutex_destroy(child_lock); apr_thread_mutex_destroy(qlock); CloseHandle(qwait_event); diff --git a/server/mpm/winnt/mpm_winnt.c b/server/mpm/winnt/mpm_winnt.c index 1b8962e7457..8c782f1a309 100644 --- a/server/mpm/winnt/mpm_winnt.c +++ b/server/mpm/winnt/mpm_winnt.c @@ -1369,6 +1369,9 @@ static int winnt_pre_config(apr_pool_t *pconf_, apr_pool_t *plog, apr_pool_t *pt * -k runservice [WinNT errors logged from rewrite_args] */ + APR_REGISTER_OPTIONAL_FN(ap_mpm_note_extra_connection_added); + APR_REGISTER_OPTIONAL_FN(ap_mpm_note_extra_connection_removed); + /* Initialize shared static objects. * TODO: Put config related statics into an sconf structure. */ diff --git a/server/mpm/winnt/mpm_winnt.h b/server/mpm/winnt/mpm_winnt.h index 22ba001407e..93d46dc2a5e 100644 --- a/server/mpm/winnt/mpm_winnt.h +++ b/server/mpm/winnt/mpm_winnt.h @@ -91,6 +91,8 @@ void hold_console_open_on_error(void); /* From child.c: */ void child_main(apr_pool_t *pconf, DWORD parent_pid); +void ap_mpm_note_extra_connection_added(void); +void ap_mpm_note_extra_connection_removed(void); #endif /* APACHE_MPM_WINNT_H */ /** @} */ diff --git a/server/mpm/worker/worker.c b/server/mpm/worker/worker.c index 315371de121..00df8221bfc 100644 --- a/server/mpm/worker/worker.c +++ b/server/mpm/worker/worker.c @@ -30,6 +30,7 @@ #include "apr_thread_mutex.h" #include "apr_proc_mutex.h" #include "apr_poll.h" +#include "apr_atomic.h" #include @@ -116,6 +117,7 @@ * Actual definitions of config globals */ +static apr_uint32_t extra_connection_count = 0; /* Number of open connections that the MPM does not own */ static int threads_per_child = 0; /* Worker threads per child */ static int ap_daemons_to_start = 0; static int min_spare_threads = 0; @@ -506,6 +508,44 @@ static void check_infinite_requests(void) } } +static void ap_mpm_note_extra_connection_added(void) +{ + apr_atomic_inc32(&extra_connection_count); +} + +static void ap_mpm_note_extra_connection_removed(void) +{ + apr_atomic_dec32(&extra_connection_count); +} + +static void wait_for_extra_connections(void) +{ + apr_uint32_t count = apr_atomic_read32(&extra_connection_count); + apr_time_t graceful, timeout, deadline; + + if (count == 0) { + return; + } + + graceful = apr_time_from_sec(ap_graceful_shutdown_timeout); + timeout = (graceful > ap_server_conf->timeout) ? graceful : ap_server_conf->timeout; + deadline = apr_time_now() + timeout; + + do { + apr_sleep(apr_time_from_msec(100)); + count = apr_atomic_read32(&extra_connection_count); + } while (count > 0 && apr_time_now() < deadline); + + if (count > 0) { + ap_log_error(APLOG_MARK, APLOG_WARNING, 0, ap_server_conf, + APLOGNO(10621) + "Child: %u connection(s) noted by modules did not " + "finish within %" APR_TIME_T_FMT " seconds, " + "exiting anyway", + count, apr_time_sec(timeout)); + } +} + static void unblock_signal(int sig) { sigset_t sig_mask; @@ -1301,6 +1341,10 @@ static void child_main(int child_num_arg, int child_bucket) rv == AP_MPM_PODX_GRACEFUL ? ST_GRACEFUL : ST_UNGRACEFUL); } + if (terminate_mode == ST_GRACEFUL) { + wait_for_extra_connections(); + } + free(threads); clean_child_exit(resource_shortage ? APEXIT_CHILDSICK : 0); @@ -2059,6 +2103,9 @@ static int worker_pre_config(apr_pool_t *pconf, apr_pool_t *plog, apr_status_t rv; const char *userdata_key = "mpm_worker_module"; + APR_REGISTER_OPTIONAL_FN(ap_mpm_note_extra_connection_added); + APR_REGISTER_OPTIONAL_FN(ap_mpm_note_extra_connection_removed); + debug = ap_exists_config_define("DEBUG"); if (debug) {