1 /* Licensed to the Apache Software Foundation (ASF) under one or more
2  * contributor license agreements.  See the NOTICE file distributed with
3  * this work for additional information regarding copyright ownership.
4  * The ASF licenses this file to You under the Apache License, Version 2.0
5  * (the "License"); you may not use this file except in compliance with
6  * the License.  You may obtain a copy of the License at
7  *
8  *     http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #ifdef WIN32
18 
19 #include "apr.h"
20 #include <process.h>
21 #include "httpd.h"
22 #include "http_main.h"
23 #include "http_log.h"
24 #include "http_config.h"  /* for read_config */
25 #include "http_core.h"    /* for get_remote_host */
26 #include "http_connection.h"
27 #include "http_vhost.h"   /* for ap_update_vhost_given_ip */
28 #include "apr_portable.h"
29 #include "apr_thread_proc.h"
30 #include "apr_getopt.h"
31 #include "apr_strings.h"
32 #include "apr_lib.h"
33 #include "apr_shm.h"
34 #include "apr_thread_mutex.h"
35 #include "ap_mpm.h"
36 #include "ap_config.h"
37 #include "ap_listen.h"
38 #include "mpm_default.h"
39 #include "mpm_winnt.h"
40 #include "mpm_common.h"
41 #include <malloc.h>
42 #include "apr_atomic.h"
43 #include "apr_buckets.h"
44 #include "scoreboard.h"
45 
46 #ifdef __MINGW32__
47 #include <mswsock.h>
48 
49 #ifndef WSAID_ACCEPTEX
50 #define WSAID_ACCEPTEX \
51   {0xb5367df1, 0xcbac, 0x11cf, {0x95, 0xca, 0x00, 0x80, 0x5f, 0x48, 0xa1, 0x92}}
52 typedef BOOL (WINAPI *LPFN_ACCEPTEX)(SOCKET, SOCKET, PVOID, DWORD, DWORD, DWORD, LPDWORD, LPOVERLAPPED);
53 #endif /* WSAID_ACCEPTEX */
54 
55 #ifndef WSAID_GETACCEPTEXSOCKADDRS
56 #define WSAID_GETACCEPTEXSOCKADDRS \
57   {0xb5367df2, 0xcbac, 0x11cf, {0x95, 0xca, 0x00, 0x80, 0x5f, 0x48, 0xa1, 0x92}}
58 typedef VOID (WINAPI *LPFN_GETACCEPTEXSOCKADDRS)(PVOID, DWORD, DWORD, DWORD,
59                                                  struct sockaddr **, LPINT,
60                                                  struct sockaddr **, LPINT);
61 #endif /* WSAID_GETACCEPTEXSOCKADDRS */
62 
63 #endif /* __MINGW32__ */
64 
65 /*
66  * The Windows MPM uses a queue of completion contexts that it passes
67  * between the accept threads and the worker threads. Declare the
68  * functions to access the queue and the structures passed on the
69  * queue in the header file to enable modules to access them
70  * if necessary. The queue resides in the MPM.
71  */
72 #ifdef CONTAINING_RECORD
73 #undef CONTAINING_RECORD
74 #endif
75 #define CONTAINING_RECORD(address, type, field) ((type *)( \
76                                                   (char *)(address) - \
77                                                   (char *)(&((type *)0)->field)))
78 #if APR_HAVE_IPV6
79 #define PADDED_ADDR_SIZE (sizeof(SOCKADDR_IN6)+16)
80 #else
81 #define PADDED_ADDR_SIZE (sizeof(SOCKADDR_IN)+16)
82 #endif
83 
84 APLOG_USE_MODULE(mpm_winnt);
85 
86 /* Queue for managing the passing of winnt_conn_ctx_t between
87  * the accept and worker threads.
88  */
89 typedef struct winnt_conn_ctx_t_s {
90     struct winnt_conn_ctx_t_s *next;
91     OVERLAPPED overlapped;
92     apr_socket_t *sock;
93     SOCKET accept_socket;
94     char buff[2*PADDED_ADDR_SIZE];
95     struct sockaddr *sa_server;
96     int sa_server_len;
97     struct sockaddr *sa_client;
98     int sa_client_len;
99     apr_pool_t *ptrans;
100     apr_bucket_alloc_t *ba;
101     apr_bucket *data;
102 #if APR_HAVE_IPV6
103     short socket_family;
104 #endif
105 } winnt_conn_ctx_t;
106 
107 typedef enum {
108     IOCP_CONNECTION_ACCEPTED = 1,
109     IOCP_WAIT_FOR_RECEIVE = 2,
110     IOCP_WAIT_FOR_TRANSMITFILE = 3,
111     IOCP_SHUTDOWN = 4
112 } io_state_e;
113 
114 static apr_pool_t *pchild;
115 static int shutdown_in_progress = 0;
116 static int workers_may_exit = 0;
117 static unsigned int g_blocked_threads = 0;
118 static HANDLE max_requests_per_child_event;
119 
120 static apr_thread_mutex_t  *child_lock;
121 static apr_thread_mutex_t  *qlock;
122 static winnt_conn_ctx_t *qhead = NULL;
123 static winnt_conn_ctx_t *qtail = NULL;
124 static apr_uint32_t num_completion_contexts = 0;
125 static apr_uint32_t max_num_completion_contexts = 0;
126 static HANDLE ThreadDispatchIOCP = NULL;
127 static HANDLE qwait_event = NULL;
128 
mpm_recycle_completion_context(winnt_conn_ctx_t * context)129 static void mpm_recycle_completion_context(winnt_conn_ctx_t *context)
130 {
131     /* Recycle the completion context.
132      * - clear the ptrans pool
133      * - put the context on the queue to be consumed by the accept thread
134      * Note:
135      * context->accept_socket may be in a disconnected but reusable
136      * state so -don't- close it.
137      */
138     if (context) {
139         HANDLE saved_event;
140 
141         apr_pool_clear(context->ptrans);
142         context->ba = apr_bucket_alloc_create(context->ptrans);
143         context->next = NULL;
144 
145         saved_event = context->overlapped.hEvent;
146         memset(&context->overlapped, 0, sizeof(context->overlapped));
147         context->overlapped.hEvent = saved_event;
148         ResetEvent(context->overlapped.hEvent);
149 
150         apr_thread_mutex_lock(qlock);
151         if (qtail) {
152             qtail->next = context;
153         } else {
154             qhead = context;
155             SetEvent(qwait_event);
156         }
157         qtail = context;
158         apr_thread_mutex_unlock(qlock);
159     }
160 }
161 
mpm_get_completion_context(int * timeout)162 static winnt_conn_ctx_t *mpm_get_completion_context(int *timeout)
163 {
164     apr_status_t rv;
165     winnt_conn_ctx_t *context = NULL;
166 
167     *timeout = 0;
168     while (1) {
169         /* Grab a context off the queue */
170         apr_thread_mutex_lock(qlock);
171         if (qhead) {
172             context = qhead;
173             qhead = qhead->next;
174             if (!qhead)
175                 qtail = NULL;
176         } else {
177             ResetEvent(qwait_event);
178         }
179         apr_thread_mutex_unlock(qlock);
180 
181         if (!context) {
182             /* We failed to grab a context off the queue, consider allocating
183              * a new one out of the child pool. There may be up to
184              * (ap_threads_per_child + num_listeners) contexts in the system
185              * at once.
186              */
187             if (num_completion_contexts >= max_num_completion_contexts) {
188                 /* All workers are busy, need to wait for one */
189                 static int reported = 0;
190                 if (!reported) {
191                     ap_log_error(APLOG_MARK, APLOG_ERR, 0, ap_server_conf, APLOGNO(00326)
192                                  "Server ran out of threads to serve "
193                                  "requests. Consider raising the "
194                                  "ThreadsPerChild setting");
195                     reported = 1;
196                 }
197 
198                 /* Wait for a worker to free a context. Once per second, give
199                  * the caller a chance to check for shutdown. If the wait
200                  * succeeds, get the context off the queue. It must be
201                  * available, since there's only one consumer.
202                  */
203                 rv = WaitForSingleObject(qwait_event, 1000);
204                 if (rv == WAIT_OBJECT_0)
205                     continue;
206                 else {
207                     if (rv == WAIT_TIMEOUT) {
208                         /* somewhat-normal condition where threads are busy */
209                         ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, ap_server_conf, APLOGNO(00327)
210                                      "mpm_get_completion_context: Failed to get a "
211                                      "free context within 1 second");
212                         *timeout = 1;
213                     }
214                     else {
215                         /* should be the unexpected, generic WAIT_FAILED */
216                         ap_log_error(APLOG_MARK, APLOG_WARNING, apr_get_os_error(),
217                                      ap_server_conf, APLOGNO(00328)
218                                      "mpm_get_completion_context: "
219                                      "WaitForSingleObject failed to get free context");
220                     }
221                     return NULL;
222                 }
223             } else {
224                 /* Allocate another context.
225                  * Note: Multiple failures in the next two steps will cause
226                  * the pchild pool to 'leak' storage. I don't think this
227                  * is worth fixing...
228                  */
229                 apr_allocator_t *allocator;
230 
231                 apr_thread_mutex_lock(child_lock);
232                 context = (winnt_conn_ctx_t *)apr_pcalloc(pchild,
233                                                      sizeof(winnt_conn_ctx_t));
234 
235 
236                 context->overlapped.hEvent = CreateEvent(NULL, TRUE,
237                                                          FALSE, NULL);
238                 if (context->overlapped.hEvent == NULL) {
239                     /* Hopefully this is a temporary condition ... */
240                     ap_log_error(APLOG_MARK, APLOG_WARNING, apr_get_os_error(),
241                                  ap_server_conf, APLOGNO(00329)
242                                  "mpm_get_completion_context: "
243                                  "CreateEvent failed.");
244 
245                     apr_thread_mutex_unlock(child_lock);
246                     return NULL;
247                 }
248 
249                 /* Create the transaction pool */
250                 apr_allocator_create(&allocator);
251                 apr_allocator_max_free_set(allocator, ap_max_mem_free);
252                 rv = apr_pool_create_ex(&context->ptrans, pchild, NULL,
253                                         allocator);
254                 if (rv != APR_SUCCESS) {
255                     ap_log_error(APLOG_MARK, APLOG_WARNING, rv, ap_server_conf, APLOGNO(00330)
256                                  "mpm_get_completion_context: Failed "
257                                  "to create the transaction pool.");
258                     CloseHandle(context->overlapped.hEvent);
259 
260                     apr_thread_mutex_unlock(child_lock);
261                     return NULL;
262                 }
263                 apr_allocator_owner_set(allocator, context->ptrans);
264                 apr_pool_tag(context->ptrans, "transaction");
265 
266                 context->accept_socket = INVALID_SOCKET;
267                 context->ba = apr_bucket_alloc_create(context->ptrans);
268                 apr_atomic_inc32(&num_completion_contexts);
269 
270                 apr_thread_mutex_unlock(child_lock);
271                 break;
272             }
273         } else {
274             /* Got a context from the queue */
275             break;
276         }
277     }
278 
279     return context;
280 }
281 
282 typedef enum {
283     ACCEPT_FILTER_NONE = 0,
284     ACCEPT_FILTER_CONNECT = 1
285 } accept_filter_e;
286 
accept_filter_to_string(accept_filter_e accf)287 static const char * accept_filter_to_string(accept_filter_e accf)
288 {
289     switch (accf) {
290     case ACCEPT_FILTER_NONE:
291         return "none";
292     case ACCEPT_FILTER_CONNECT:
293         return "connect";
294     default:
295         return "";
296     }
297 }
298 
get_accept_filter(const char * protocol)299 static accept_filter_e get_accept_filter(const char *protocol)
300 {
301     core_server_config *core_sconf;
302     const char *name;
303 
304     core_sconf = ap_get_core_module_config(ap_server_conf->module_config);
305     name = apr_table_get(core_sconf->accf_map, protocol);
306     if (!name) {
307         ap_log_error(APLOG_MARK, APLOG_WARNING, 0, ap_server_conf,
308                      APLOGNO(02531) "winnt_accept: Listen protocol '%s' has "
309                      "no known accept filter. Using 'none' instead",
310                      protocol);
311         return ACCEPT_FILTER_NONE;
312     }
313     else if (strcmp(name, "data") == 0) {
314         ap_log_error(APLOG_MARK, APLOG_INFO, 0, ap_server_conf,
315                      APLOGNO(03458) "winnt_accept: 'data' accept filter is no "
316                      "longer supported. Using 'connect' instead");
317         return ACCEPT_FILTER_CONNECT;
318     }
319     else if (strcmp(name, "connect") == 0) {
320         return ACCEPT_FILTER_CONNECT;
321     }
322     else if (strcmp(name, "none") == 0) {
323         return ACCEPT_FILTER_NONE;
324     }
325     else {
326         ap_log_error(APLOG_MARK, APLOG_WARNING, 0, ap_server_conf, APLOGNO(00331)
327                      "winnt_accept: unrecognized AcceptFilter '%s', "
328                      "only 'data', 'connect' or 'none' are valid. "
329                      "Using 'none' instead", name);
330         return ACCEPT_FILTER_NONE;
331     }
332 }
333 
334 /* Windows NT/2000 specific code...
335  * Accept processing for on Windows NT uses a producer/consumer queue
336  * model. An accept thread accepts connections off the network then issues
337  * PostQueuedCompletionStatus() to awake a thread blocked on the ThreadDispatch
338  * IOCompletionPort.
339  *
340  * winnt_accept()
341  *    One or more accept threads run in this function, each of which accepts
342  *    connections off the network and calls PostQueuedCompletionStatus() to
343  *    queue an io completion packet to the ThreadDispatch IOCompletionPort.
344  * winnt_get_connection()
345  *    Worker threads block on the ThreadDispatch IOCompletionPort awaiting
346  *    connections to service.
347  */
348 #define MAX_ACCEPTEX_ERR_COUNT 10
349 
winnt_accept(void * lr_)350 static unsigned int __stdcall winnt_accept(void *lr_)
351 {
352     ap_listen_rec *lr = (ap_listen_rec *)lr_;
353     apr_os_sock_info_t sockinfo;
354     winnt_conn_ctx_t *context = NULL;
355     DWORD BytesRead = 0;
356     SOCKET nlsd;
357     LPFN_ACCEPTEX lpfnAcceptEx = NULL;
358     LPFN_GETACCEPTEXSOCKADDRS lpfnGetAcceptExSockaddrs = NULL;
359     GUID GuidAcceptEx = WSAID_ACCEPTEX;
360     GUID GuidGetAcceptExSockaddrs = WSAID_GETACCEPTEXSOCKADDRS;
361     int rv;
362     accept_filter_e accf;
363     int err_count = 0;
364     HANDLE events[3];
365 #if APR_HAVE_IPV6
366     SOCKADDR_STORAGE ss_listen;
367     int namelen = sizeof(ss_listen);
368 #endif
369     u_long zero = 0;
370 
371     apr_os_sock_get(&nlsd, lr->sd);
372 
373 #if APR_HAVE_IPV6
374     if (getsockname(nlsd, (struct sockaddr *)&ss_listen, &namelen) == SOCKET_ERROR) {
375         ap_log_error(APLOG_MARK, APLOG_ERR, apr_get_netos_error(),
376                      ap_server_conf, APLOGNO(00332)
377                      "winnt_accept: getsockname error on listening socket, "
378                      "is IPv6 available?");
379         return 1;
380    }
381 #endif
382 
383     accf = get_accept_filter(lr->protocol);
384     if (accf == ACCEPT_FILTER_CONNECT)
385     {
386         if (WSAIoctl(nlsd, SIO_GET_EXTENSION_FUNCTION_POINTER,
387                      &GuidAcceptEx, sizeof GuidAcceptEx,
388                      &lpfnAcceptEx, sizeof lpfnAcceptEx,
389                      &BytesRead, NULL, NULL) == SOCKET_ERROR) {
390             ap_log_error(APLOG_MARK, APLOG_ERR, apr_get_netos_error(),
391                          ap_server_conf, APLOGNO(02322)
392                          "winnt_accept: failed to retrieve AcceptEx, try 'AcceptFilter none'");
393             return 1;
394         }
395         if (WSAIoctl(nlsd, SIO_GET_EXTENSION_FUNCTION_POINTER,
396                      &GuidGetAcceptExSockaddrs, sizeof GuidGetAcceptExSockaddrs,
397                      &lpfnGetAcceptExSockaddrs, sizeof lpfnGetAcceptExSockaddrs,
398                      &BytesRead, NULL, NULL) == SOCKET_ERROR) {
399             ap_log_error(APLOG_MARK, APLOG_ERR, apr_get_netos_error(),
400                          ap_server_conf, APLOGNO(02323)
401                          "winnt_accept: failed to retrieve GetAcceptExSockaddrs, try 'AcceptFilter none'");
402             return 1;
403         }
404         /* first, high priority event is an already accepted connection */
405         events[1] = exit_event;
406         events[2] = max_requests_per_child_event;
407     }
408     else /* accf == ACCEPT_FILTER_NONE */
409     {
410 reinit: /* target of connect upon too many AcceptEx failures */
411 
412         /* last, low priority event is a not yet accepted connection */
413         events[0] = exit_event;
414         events[1] = max_requests_per_child_event;
415         events[2] = CreateEvent(NULL, FALSE, FALSE, NULL);
416 
417         /* The event needs to be removed from the accepted socket,
418          * if not removed from the listen socket prior to accept(),
419          */
420         rv = WSAEventSelect(nlsd, events[2], FD_ACCEPT);
421         if (rv) {
422             ap_log_error(APLOG_MARK, APLOG_ERR,
423                          apr_get_netos_error(), ap_server_conf, APLOGNO(00333)
424                          "WSAEventSelect() failed.");
425             CloseHandle(events[2]);
426             return 1;
427         }
428     }
429 
430     ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, ap_server_conf, APLOGNO(00334)
431                  "Child: Accept thread listening on %pI using AcceptFilter %s",
432                  lr->bind_addr, accept_filter_to_string(accf));
433 
434     while (!shutdown_in_progress) {
435         if (!context) {
436             int timeout;
437 
438             context = mpm_get_completion_context(&timeout);
439             if (!context) {
440                 if (!timeout) {
441                     /* Hopefully a temporary condition in the provider? */
442                     ++err_count;
443                     if (err_count > MAX_ACCEPTEX_ERR_COUNT) {
444                         ap_log_error(APLOG_MARK, APLOG_CRIT, 0, ap_server_conf, APLOGNO(00335)
445                                      "winnt_accept: Too many failures grabbing a "
446                                      "connection ctx.  Aborting.");
447                         break;
448                     }
449                 }
450                 Sleep(100);
451                 continue;
452             }
453         }
454 
455         if (accf == ACCEPT_FILTER_CONNECT)
456         {
457             char *buf;
458 
459             /* Create and initialize the accept socket */
460 #if APR_HAVE_IPV6
461             if (context->accept_socket == INVALID_SOCKET) {
462                 context->accept_socket = socket(ss_listen.ss_family, SOCK_STREAM,
463                                                 IPPROTO_TCP);
464                 context->socket_family = ss_listen.ss_family;
465             }
466             else if (context->socket_family != ss_listen.ss_family) {
467                 closesocket(context->accept_socket);
468                 context->accept_socket = socket(ss_listen.ss_family, SOCK_STREAM,
469                                                 IPPROTO_TCP);
470                 context->socket_family = ss_listen.ss_family;
471             }
472 #else
473             if (context->accept_socket == INVALID_SOCKET)
474                 context->accept_socket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
475 #endif
476 
477             if (context->accept_socket == INVALID_SOCKET) {
478                 ap_log_error(APLOG_MARK, APLOG_WARNING, apr_get_netos_error(),
479                              ap_server_conf, APLOGNO(00336)
480                              "winnt_accept: Failed to allocate an accept socket. "
481                              "Temporary resource constraint? Try again.");
482                 Sleep(100);
483                 continue;
484             }
485 
486             buf = context->buff;
487 
488             /* AcceptEx on the completion context. The completion context will be
489              * signaled when a connection is accepted.
490              */
491             if (!lpfnAcceptEx(nlsd, context->accept_socket, buf, 0,
492                               PADDED_ADDR_SIZE, PADDED_ADDR_SIZE, &BytesRead,
493                               &context->overlapped)) {
494                 rv = apr_get_netos_error();
495                 if ((rv == APR_FROM_OS_ERROR(WSAECONNRESET)) ||
496                     (rv == APR_FROM_OS_ERROR(WSAEACCES))) {
497                     /* We can get here when:
498                      * 1) the client disconnects early
499                      * 2) handshake was incomplete
500                      */
501                     closesocket(context->accept_socket);
502                     context->accept_socket = INVALID_SOCKET;
503                     continue;
504                 }
505                 else if ((rv == APR_FROM_OS_ERROR(WSAEINVAL)) ||
506                          (rv == APR_FROM_OS_ERROR(WSAENOTSOCK))) {
507                     /* We can get here when:
508                      * 1) TransmitFile does not properly recycle the accept socket (typically
509                      *    because the client disconnected)
510                      * 2) there is VPN or Firewall software installed with
511                      *    buggy WSAAccept or WSADuplicateSocket implementation
512                      * 3) the dynamic address / adapter has changed
513                      * Give five chances, then fall back on AcceptFilter 'none'
514                      */
515                     closesocket(context->accept_socket);
516                     context->accept_socket = INVALID_SOCKET;
517                     ++err_count;
518                     if (err_count > MAX_ACCEPTEX_ERR_COUNT) {
519                         ap_log_error(APLOG_MARK, APLOG_ERR, rv, ap_server_conf, APLOGNO(00337)
520                                      "Child: Encountered too many AcceptEx "
521                                      "faults accepting client connections. "
522                                      "Possible causes: dynamic address renewal, "
523                                      "or incompatible VPN or firewall software. ");
524                         ap_log_error(APLOG_MARK, APLOG_NOTICE, rv, ap_server_conf, APLOGNO(00338)
525                                      "winnt_mpm: falling back to "
526                                      "'AcceptFilter none'.");
527                         err_count = 0;
528                         accf = ACCEPT_FILTER_NONE;
529                     }
530                     continue;
531                 }
532                 else if ((rv != APR_FROM_OS_ERROR(ERROR_IO_PENDING)) &&
533                          (rv != APR_FROM_OS_ERROR(WSA_IO_PENDING))) {
534                     closesocket(context->accept_socket);
535                     context->accept_socket = INVALID_SOCKET;
536                     ++err_count;
537                     if (err_count > MAX_ACCEPTEX_ERR_COUNT) {
538                         ap_log_error(APLOG_MARK, APLOG_ERR, rv, ap_server_conf, APLOGNO(00339)
539                                      "Child: Encountered too many AcceptEx "
540                                      "faults accepting client connections.");
541                         ap_log_error(APLOG_MARK, APLOG_NOTICE, rv, ap_server_conf, APLOGNO(00340)
542                                      "winnt_mpm: falling back to "
543                                      "'AcceptFilter none'.");
544                         err_count = 0;
545                         accf = ACCEPT_FILTER_NONE;
546                         goto reinit;
547                     }
548                     continue;
549                 }
550 
551                 err_count = 0;
552                 events[0] = context->overlapped.hEvent;
553 
554                 do {
555                     rv = WaitForMultipleObjectsEx(3, events, FALSE, INFINITE, TRUE);
556                 } while (rv == WAIT_IO_COMPLETION);
557 
558                 if (rv == WAIT_OBJECT_0) {
559                     if ((context->accept_socket != INVALID_SOCKET) &&
560                         !GetOverlappedResult((HANDLE)context->accept_socket,
561                                              &context->overlapped,
562                                              &BytesRead, FALSE)) {
563                         ap_log_error(APLOG_MARK, APLOG_WARNING,
564                                      apr_get_os_error(), ap_server_conf, APLOGNO(00341)
565                              "winnt_accept: Asynchronous AcceptEx failed.");
566                         closesocket(context->accept_socket);
567                         context->accept_socket = INVALID_SOCKET;
568                     }
569                 }
570                 else {
571                     /* exit_event triggered or event handle was closed */
572                     closesocket(context->accept_socket);
573                     context->accept_socket = INVALID_SOCKET;
574                     break;
575                 }
576 
577                 if (context->accept_socket == INVALID_SOCKET) {
578                     continue;
579                 }
580             }
581             err_count = 0;
582 
583             /* Potential optimization; consider handing off to the worker */
584 
585             /* Inherit the listen socket settings. Required for
586              * shutdown() to work
587              */
588             if (setsockopt(context->accept_socket, SOL_SOCKET,
589                            SO_UPDATE_ACCEPT_CONTEXT, (char *)&nlsd,
590                            sizeof(nlsd))) {
591                 ap_log_error(APLOG_MARK, APLOG_WARNING, apr_get_netos_error(),
592                              ap_server_conf, APLOGNO(00342)
593                              "setsockopt(SO_UPDATE_ACCEPT_CONTEXT) failed.");
594                 /* Not a failure condition. Keep running. */
595             }
596 
597             /* Get the local & remote address
598              * TODO; error check
599              */
600             lpfnGetAcceptExSockaddrs(buf, 0, PADDED_ADDR_SIZE, PADDED_ADDR_SIZE,
601                                      &context->sa_server, &context->sa_server_len,
602                                      &context->sa_client, &context->sa_client_len);
603         }
604         else /* accf == ACCEPT_FILTER_NONE */
605         {
606             /* There is no socket reuse without AcceptEx() */
607             if (context->accept_socket != INVALID_SOCKET)
608                 closesocket(context->accept_socket);
609 
610             /* This could be a persistent event per-listener rather than
611              * per-accept.  However, the event needs to be removed from
612              * the target socket if not removed from the listen socket
613              * prior to accept(), or the event select is inherited.
614              * and must be removed from the accepted socket.
615              */
616 
617             do {
618                 rv = WaitForMultipleObjectsEx(3, events, FALSE, INFINITE, TRUE);
619             } while (rv == WAIT_IO_COMPLETION);
620 
621 
622             if (rv != WAIT_OBJECT_0 + 2) {
623                 /* not FD_ACCEPT;
624                  * exit_event triggered or event handle was closed
625                  */
626                 break;
627             }
628 
629             context->sa_server = (void *) context->buff;
630             context->sa_server_len = sizeof(context->buff) / 2;
631             context->sa_client_len = context->sa_server_len;
632             context->sa_client = (void *) (context->buff
633                                          + context->sa_server_len);
634 
635             context->accept_socket = accept(nlsd, context->sa_server,
636                                             &context->sa_server_len);
637 
638             if (context->accept_socket == INVALID_SOCKET) {
639 
640                 rv = apr_get_netos_error();
641                 if (   rv == APR_FROM_OS_ERROR(WSAECONNRESET)
642                     || rv == APR_FROM_OS_ERROR(WSAEINPROGRESS)
643                     || rv == APR_FROM_OS_ERROR(WSAEWOULDBLOCK) ) {
644                     ap_log_error(APLOG_MARK, APLOG_DEBUG,
645                                  rv, ap_server_conf, APLOGNO(00343)
646                                  "accept() failed, retrying.");
647                     continue;
648                 }
649 
650                 /* A more serious error than 'retry', log it */
651                 ap_log_error(APLOG_MARK, APLOG_WARNING,
652                              rv, ap_server_conf, APLOGNO(00344)
653                              "accept() failed.");
654 
655                 if (   rv == APR_FROM_OS_ERROR(WSAEMFILE)
656                     || rv == APR_FROM_OS_ERROR(WSAENOBUFS) ) {
657                     /* Hopefully a temporary condition in the provider? */
658                     Sleep(100);
659                     ++err_count;
660                     if (err_count > MAX_ACCEPTEX_ERR_COUNT) {
661                         ap_log_error(APLOG_MARK, APLOG_ERR, rv, ap_server_conf, APLOGNO(00345)
662                                      "Child: Encountered too many accept() "
663                                      "resource faults, aborting.");
664                         break;
665                     }
666                     continue;
667                 }
668                 break;
669             }
670             /* Per MSDN, cancel the inherited association of this socket
671              * to the WSAEventSelect API, and restore the state corresponding
672              * to apr_os_sock_make's default assumptions (really, a flaw within
673              * os_sock_make and os_sock_put that it does not query).
674              */
675             WSAEventSelect(context->accept_socket, 0, 0);
676             err_count = 0;
677 
678             context->sa_server_len = sizeof(context->buff) / 2;
679             if (getsockname(context->accept_socket, context->sa_server,
680                             &context->sa_server_len) == SOCKET_ERROR) {
681                 ap_log_error(APLOG_MARK, APLOG_WARNING, apr_get_netos_error(), ap_server_conf, APLOGNO(00346)
682                              "getsockname failed");
683                 continue;
684             }
685             if ((getpeername(context->accept_socket, context->sa_client,
686                              &context->sa_client_len)) == SOCKET_ERROR) {
687                 ap_log_error(APLOG_MARK, APLOG_WARNING, apr_get_netos_error(), ap_server_conf, APLOGNO(00347)
688                              "getpeername failed");
689                 memset(&context->sa_client, '\0', sizeof(context->sa_client));
690             }
691         }
692 
693         sockinfo.os_sock  = &context->accept_socket;
694         sockinfo.local    = context->sa_server;
695         sockinfo.remote   = context->sa_client;
696         sockinfo.family   = context->sa_server->sa_family;
697         sockinfo.type     = SOCK_STREAM;
698         sockinfo.protocol = IPPROTO_TCP;
699         /* Restore the state corresponding to apr_os_sock_make's default
700          * assumption of timeout -1 (really, a flaw of os_sock_make and
701          * os_sock_put that it does not query to determine ->timeout).
702          * XXX: Upon a fix to APR, these three statements should disappear.
703          */
704         ioctlsocket(context->accept_socket, FIONBIO, &zero);
705         setsockopt(context->accept_socket, SOL_SOCKET, SO_RCVTIMEO,
706                    (char *) &zero, sizeof(zero));
707         setsockopt(context->accept_socket, SOL_SOCKET, SO_SNDTIMEO,
708                    (char *) &zero, sizeof(zero));
709         apr_os_sock_make(&context->sock, &sockinfo, context->ptrans);
710 
711         /* When a connection is received, send an io completion notification
712          * to the ThreadDispatchIOCP.
713          */
714         PostQueuedCompletionStatus(ThreadDispatchIOCP, BytesRead,
715                                    IOCP_CONNECTION_ACCEPTED,
716                                    &context->overlapped);
717         context = NULL;
718     }
719     if (accf == ACCEPT_FILTER_NONE)
720         CloseHandle(events[2]);
721 
722     if (!shutdown_in_progress) {
723         /* Yow, hit an irrecoverable error! Tell the child to die. */
724         SetEvent(exit_event);
725     }
726 
727     ap_log_error(APLOG_MARK, APLOG_DEBUG, APR_SUCCESS, ap_server_conf, APLOGNO(00348)
728                  "Child: Accept thread exiting.");
729     return 0;
730 }
731 
732 
winnt_get_connection(winnt_conn_ctx_t * context)733 static winnt_conn_ctx_t *winnt_get_connection(winnt_conn_ctx_t *context)
734 {
735     int rc;
736     DWORD BytesRead;
737     LPOVERLAPPED pol;
738 #ifdef _WIN64
739     ULONG_PTR CompKey;
740 #else
741     DWORD CompKey;
742 #endif
743 
744     mpm_recycle_completion_context(context);
745 
746     apr_atomic_inc32(&g_blocked_threads);
747     while (1) {
748         if (workers_may_exit) {
749             apr_atomic_dec32(&g_blocked_threads);
750             return NULL;
751         }
752         rc = GetQueuedCompletionStatus(ThreadDispatchIOCP, &BytesRead,
753                                        &CompKey, &pol, INFINITE);
754         if (!rc) {
755             rc = apr_get_os_error();
756             ap_log_error(APLOG_MARK, APLOG_DEBUG, rc, ap_server_conf, APLOGNO(00349)
757                          "Child: GetQueuedCompletionStatus returned %d",
758                          rc);
759             continue;
760         }
761 
762         switch (CompKey) {
763         case IOCP_CONNECTION_ACCEPTED:
764             context = CONTAINING_RECORD(pol, winnt_conn_ctx_t, overlapped);
765             break;
766         case IOCP_SHUTDOWN:
767             apr_atomic_dec32(&g_blocked_threads);
768             return NULL;
769         default:
770             apr_atomic_dec32(&g_blocked_threads);
771             return NULL;
772         }
773         break;
774     }
775     apr_atomic_dec32(&g_blocked_threads);
776 
777     return context;
778 }
779 
780 /*
781  * worker_main()
782  * Main entry point for the worker threads. Worker threads block in
783  * win*_get_connection() awaiting a connection to service.
784  */
worker_main(void * thread_num_val)785 static DWORD __stdcall worker_main(void *thread_num_val)
786 {
787     apr_thread_t *thd = NULL;
788     apr_os_thread_t osthd = NULL;
789     static int requests_this_child = 0;
790     winnt_conn_ctx_t *context = NULL;
791     int thread_num = (int)thread_num_val;
792     ap_sb_handle_t *sbh;
793     conn_rec *c;
794     apr_int32_t disconnected;
795 
796 #if AP_HAS_THREAD_LOCAL
797     if (ap_thread_current_create(&thd, NULL, pchild) != APR_SUCCESS) {
798         ap_log_error(APLOG_MARK, APLOG_WARNING, 0, ap_server_conf, APLOGNO(10376)
799                      "Couldn't initialize worker thread, thread locals won't "
800                      "be available");
801         osthd = apr_os_thread_current();
802     }
803 #else
804     osthd = apr_os_thread_current();
805 #endif
806 
807     while (1) {
808 
809         ap_update_child_status_from_indexes(0, thread_num, SERVER_READY, NULL);
810 
811         /* Grab a connection off the network */
812         context = winnt_get_connection(context);
813 
814         if (!context) {
815             /* Time for the thread to exit */
816             break;
817         }
818 
819         /* Have we hit MaxConnectionsPerChild connections? */
820         if (ap_max_requests_per_child) {
821             requests_this_child++;
822             if (requests_this_child > ap_max_requests_per_child) {
823                 SetEvent(max_requests_per_child_event);
824             }
825         }
826 
827         ap_create_sb_handle(&sbh, context->ptrans, 0, thread_num);
828         c = ap_run_create_connection(context->ptrans, ap_server_conf,
829                                      context->sock, thread_num, sbh,
830                                      context->ba);
831 
832         if (!c) {
833             /* ap_run_create_connection closes the socket on failure */
834             context->accept_socket = INVALID_SOCKET;
835             continue;
836         }
837 
838         if (osthd) {
839             thd = NULL;
840             apr_os_thread_put(&thd, &osthd, context->ptrans);
841         }
842         c->current_thread = thd;
843 
844         ap_process_connection(c, context->sock);
845 
846         ap_lingering_close(c);
847 
848         apr_socket_opt_get(context->sock, APR_SO_DISCONNECTED, &disconnected);
849         if (!disconnected) {
850             context->accept_socket = INVALID_SOCKET;
851         }
852     }
853 
854     ap_update_child_status_from_indexes(0, thread_num, SERVER_DEAD, NULL);
855 
856 #if AP_HAS_THREAD_LOCAL
857     if (!osthd) {
858         apr_pool_destroy(apr_thread_pool_get(thd));
859     }
860 #endif
861 
862     return 0;
863 }
864 
865 
cleanup_thread(HANDLE * handles,int * thread_cnt,int thread_to_clean)866 static void cleanup_thread(HANDLE *handles, int *thread_cnt,
867                            int thread_to_clean)
868 {
869     int i;
870 
871     CloseHandle(handles[thread_to_clean]);
872     for (i = thread_to_clean; i < ((*thread_cnt) - 1); i++)
873         handles[i] = handles[i + 1];
874     (*thread_cnt)--;
875 }
876 
877 
878 /*
879  * child_main()
880  * Entry point for the main control thread for the child process.
881  * This thread creates the accept thread, worker threads and
882  * monitors the child process for maintenance and shutdown
883  * events.
884  */
create_listener_thread(void)885 static void create_listener_thread(void)
886 {
887     unsigned tid;
888     int num_listeners = 0;
889     /* Start an accept thread per listener
890      * XXX: Why would we have a NULL sd in our listeners?
891      */
892     ap_listen_rec *lr;
893 
894     /* Number of completion_contexts allowed in the system is
895      * (ap_threads_per_child + num_listeners). We need the additional
896      * completion contexts to prevent server hangs when ThreadsPerChild
897      * is configured to something less than or equal to the number
898      * of listeners. This is not a usual case, but people have
899      * encountered it.
900      */
901     for (lr = ap_listeners; lr ; lr = lr->next) {
902         num_listeners++;
903     }
904     max_num_completion_contexts = ap_threads_per_child + num_listeners;
905 
906     /* Now start a thread per listener */
907     for (lr = ap_listeners; lr; lr = lr->next) {
908         if (lr->sd != NULL) {
909             /* A smaller stack is sufficient.
910              * To convert to CreateThread, the returned handle cannot be
911              * ignored, it must be closed/joined.
912              */
913             _beginthreadex(NULL, 65536, winnt_accept,
914                            (void *) lr, stack_res_flag, &tid);
915         }
916     }
917 }
918 
919 
child_main(apr_pool_t * pconf,DWORD parent_pid)920 void child_main(apr_pool_t *pconf, DWORD parent_pid)
921 {
922     apr_status_t status;
923     apr_hash_t *ht;
924     ap_listen_rec *lr;
925     HANDLE child_events[3];
926     HANDLE *child_handles;
927     int listener_started = 0;
928     int threads_created = 0;
929     int watch_thread;
930     int time_remains;
931     int cld;
932     DWORD tid;
933     int rv;
934     int i;
935     int num_events;
936 
937     /* Get a sub context for global allocations in this child, so that
938      * we can have cleanups occur when the child exits.
939      */
940     apr_pool_create(&pchild, pconf);
941     apr_pool_tag(pchild, "pchild");
942 
943     ap_run_child_init(pchild, ap_server_conf);
944     ht = apr_hash_make(pchild);
945 
946     /* Initialize the child_events */
947     max_requests_per_child_event = CreateEvent(NULL, TRUE, FALSE, NULL);
948     if (!max_requests_per_child_event) {
949         ap_log_error(APLOG_MARK, APLOG_CRIT, apr_get_os_error(), ap_server_conf, APLOGNO(00350)
950                      "Child: Failed to create a max_requests event.");
951         exit(APEXIT_CHILDINIT);
952     }
953     child_events[0] = exit_event;
954     child_events[1] = max_requests_per_child_event;
955 
956     if (parent_pid != my_pid) {
957         child_events[2] = OpenProcess(SYNCHRONIZE, FALSE, parent_pid);
958         if (child_events[2] == NULL) {
959             num_events = 2;
960             ap_log_error(APLOG_MARK, APLOG_ERR, apr_get_os_error(), ap_server_conf, APLOGNO(02643)
961                          "Child: Failed to open handle to parent process %ld; "
962                          "will not react to abrupt parent termination", parent_pid);
963         }
964         else {
965             num_events = 3;
966         }
967     }
968     else {
969         /* presumably -DONE_PROCESS */
970         child_events[2] = NULL;
971         num_events = 2;
972     }
973 
974     /*
975      * Wait until we have permission to start accepting connections.
976      * start_mutex is used to ensure that only one child ever
977      * goes into the listen/accept loop at once.
978      */
979     status = apr_proc_mutex_lock(start_mutex);
980     if (status != APR_SUCCESS) {
981         ap_log_error(APLOG_MARK, APLOG_ERR, status, ap_server_conf, APLOGNO(00351)
982                      "Child: Failed to acquire the start_mutex. "
983                      "Process will exit.");
984         exit(APEXIT_CHILDINIT);
985     }
986     ap_log_error(APLOG_MARK, APLOG_DEBUG, APR_SUCCESS, ap_server_conf, APLOGNO(00352)
987                  "Child: Acquired the start mutex.");
988 
989     /*
990      * Create the worker thread dispatch IOCompletionPort
991      */
992     /* Create the worker thread dispatch IOCP */
993     ThreadDispatchIOCP = CreateIoCompletionPort(INVALID_HANDLE_VALUE,
994                                                 NULL, 0, 0);
995     apr_thread_mutex_create(&qlock, APR_THREAD_MUTEX_DEFAULT, pchild);
996     qwait_event = CreateEvent(NULL, TRUE, FALSE, NULL);
997     if (!qwait_event) {
998         ap_log_error(APLOG_MARK, APLOG_CRIT, apr_get_os_error(),
999                      ap_server_conf, APLOGNO(00353)
1000                      "Child: Failed to create a qwait event.");
1001         exit(APEXIT_CHILDINIT);
1002     }
1003 
1004     /*
1005      * Create the pool of worker threads
1006      */
1007     ap_log_error(APLOG_MARK, APLOG_NOTICE, APR_SUCCESS, ap_server_conf, APLOGNO(00354)
1008                  "Child: Starting %d worker threads.", ap_threads_per_child);
1009     child_handles = (HANDLE) apr_pcalloc(pchild, ap_threads_per_child
1010                                                   * sizeof(HANDLE));
1011     apr_thread_mutex_create(&child_lock, APR_THREAD_MUTEX_DEFAULT, pchild);
1012 
1013     while (1) {
1014         for (i = 0; i < ap_threads_per_child; i++) {
1015             int *score_idx;
1016             int status = ap_scoreboard_image->servers[0][i].status;
1017             if (status != SERVER_GRACEFUL && status != SERVER_DEAD) {
1018                 continue;
1019             }
1020             ap_update_child_status_from_indexes(0, i, SERVER_STARTING, NULL);
1021 
1022             child_handles[i] = CreateThread(NULL, ap_thread_stacksize,
1023                                             worker_main, (void *) i,
1024                                             stack_res_flag, &tid);
1025             if (child_handles[i] == 0) {
1026                 ap_log_error(APLOG_MARK, APLOG_CRIT, apr_get_os_error(),
1027                              ap_server_conf, APLOGNO(00355)
1028                              "Child: CreateThread failed. Unable to "
1029                              "create all worker threads. Created %d of the %d "
1030                              "threads requested with the ThreadsPerChild "
1031                              "configuration directive.",
1032                              threads_created, ap_threads_per_child);
1033                 ap_signal_parent(SIGNAL_PARENT_SHUTDOWN);
1034                 goto shutdown;
1035             }
1036             threads_created++;
1037             /* Save the score board index in ht keyed to the thread handle.
1038              * We need this when cleaning up threads down below...
1039              */
1040             apr_thread_mutex_lock(child_lock);
1041             score_idx = apr_pcalloc(pchild, sizeof(int));
1042             *score_idx = i;
1043             apr_hash_set(ht, &child_handles[i], sizeof(HANDLE), score_idx);
1044             apr_thread_mutex_unlock(child_lock);
1045         }
1046         /* Start the listener only when workers are available */
1047         if (!listener_started && threads_created) {
1048             create_listener_thread();
1049             listener_started = 1;
1050             winnt_mpm_state = AP_MPMQ_RUNNING;
1051         }
1052         if (threads_created == ap_threads_per_child) {
1053             break;
1054         }
1055         /* Check to see if the child has been told to exit */
1056         if (WaitForSingleObject(exit_event, 0) != WAIT_TIMEOUT) {
1057             break;
1058         }
1059         /* wait for previous generation to clean up an entry in the scoreboard
1060          */
1061         apr_sleep(1 * APR_USEC_PER_SEC);
1062     }
1063 
1064     /* Wait for one of these events:
1065      * exit_event:
1066      *    The exit_event is signaled by the parent process to notify
1067      *    the child that it is time to exit.
1068      *
1069      * max_requests_per_child_event:
1070      *    This event is signaled by the worker threads to indicate that
1071      *    the process has handled MaxConnectionsPerChild connections.
1072      *
1073      * parent process exiting
1074      *
1075      * TIMEOUT:
1076      *    To do periodic maintenance on the server (check for thread exits,
1077      *    number of completion contexts, etc.)
1078      *
1079      * XXX: thread exits *aren't* being checked.
1080      *
1081      * XXX: other_child - we need the process handles to the other children
1082      *      in order to map them to apr_proc_other_child_read (which is not
1083      *      named well, it's more like a_p_o_c_died.)
1084      *
1085      * XXX: however - if we get a_p_o_c handle inheritance working, and
1086      *      the parent process creates other children and passes the pipes
1087      *      to our worker processes, then we have no business doing such
1088      *      things in the child_main loop, but should happen in master_main.
1089      */
1090     while (1) {
1091 #if !APR_HAS_OTHER_CHILD
1092         rv = WaitForMultipleObjects(num_events, (HANDLE *)child_events, FALSE, INFINITE);
1093         cld = rv - WAIT_OBJECT_0;
1094 #else
1095         /* THIS IS THE EXPECTED BUILD VARIATION -- APR_HAS_OTHER_CHILD */
1096         rv = WaitForMultipleObjects(num_events, (HANDLE *)child_events, FALSE, 1000);
1097         cld = rv - WAIT_OBJECT_0;
1098         if (rv == WAIT_TIMEOUT) {
1099             apr_proc_other_child_refresh_all(APR_OC_REASON_RUNNING);
1100         }
1101         else
1102 #endif
1103             if (rv == WAIT_FAILED) {
1104             /* Something serious is wrong */
1105             ap_log_error(APLOG_MARK, APLOG_CRIT, apr_get_os_error(),
1106                          ap_server_conf, APLOGNO(00356)
1107                          "Child: WAIT_FAILED -- shutting down server");
1108             /* check handle validity to identify a possible culprit */
1109             for (i = 0; i < num_events; i++) {
1110                 DWORD out_flags;
1111 
1112                 if (0 == GetHandleInformation(child_events[i], &out_flags)) {
1113                     ap_log_error(APLOG_MARK, APLOG_CRIT, apr_get_os_error(),
1114                                  ap_server_conf, APLOGNO(02644)
1115                                  "Child: Event handle #%d (%pp) is invalid",
1116                                  i, child_events[i]);
1117                 }
1118             }
1119             break;
1120         }
1121         else if (cld == 0) {
1122             /* Exit event was signaled */
1123             ap_log_error(APLOG_MARK, APLOG_DEBUG, APR_SUCCESS, ap_server_conf, APLOGNO(00357)
1124                          "Child: Exit event signaled. Child process is "
1125                          "ending.");
1126             break;
1127         }
1128         else if (cld == 2) {
1129             /* The parent is dead.  Shutdown the child process. */
1130             ap_log_error(APLOG_MARK, APLOG_CRIT, 0, ap_server_conf, APLOGNO(02538)
1131                          "Child: Parent process exited abruptly. Child process "
1132                          "is ending");
1133             break;
1134         }
1135         else {
1136             /* MaxConnectionsPerChild event set by the worker threads.
1137              * Signal the parent to restart
1138              */
1139             ap_log_error(APLOG_MARK, APLOG_NOTICE, APR_SUCCESS, ap_server_conf, APLOGNO(00358)
1140                          "Child: Process exiting because it reached "
1141                          "MaxConnectionsPerChild. Signaling the parent to "
1142                          "restart a new child process.");
1143             ap_signal_parent(SIGNAL_PARENT_RESTART);
1144             break;
1145         }
1146     }
1147 
1148     /*
1149      * Time to shutdown the child process
1150      */
1151 
1152  shutdown:
1153 
1154     winnt_mpm_state = AP_MPMQ_STOPPING;
1155 
1156     /* Close the listening sockets. Note, we must close the listeners
1157      * before closing any accept sockets pending in AcceptEx to prevent
1158      * memory leaks in the kernel.
1159      */
1160     for (lr = ap_listeners; lr ; lr = lr->next) {
1161         apr_socket_close(lr->sd);
1162     }
1163 
1164     /* Shutdown listener threads and pending AcceptEx sockets
1165      * but allow the worker threads to continue consuming from
1166      * the queue of accepted connections.
1167      */
1168     shutdown_in_progress = 1;
1169 
1170     Sleep(1000);
1171 
1172     /* Tell the worker threads to exit */
1173     workers_may_exit = 1;
1174 
1175     /* Release the start_mutex to let the new process (in the restart
1176      * scenario) a chance to begin accepting and servicing requests
1177      */
1178     rv = apr_proc_mutex_unlock(start_mutex);
1179     if (rv == APR_SUCCESS) {
1180         ap_log_error(APLOG_MARK, APLOG_DEBUG, rv, ap_server_conf, APLOGNO(00359)
1181                      "Child: Released the start mutex");
1182     }
1183     else {
1184         ap_log_error(APLOG_MARK, APLOG_ERR, rv, ap_server_conf, APLOGNO(00360)
1185                      "Child: Failure releasing the start mutex");
1186     }
1187 
1188     /* Shutdown the worker threads
1189      * Post worker threads blocked on the ThreadDispatch IOCompletion port
1190      */
1191     while (g_blocked_threads > 0) {
1192         ap_log_error(APLOG_MARK, APLOG_DEBUG, APR_SUCCESS, ap_server_conf, APLOGNO(00361)
1193                      "Child: %d threads blocked on the completion port",
1194                      g_blocked_threads);
1195         for (i=g_blocked_threads; i > 0; i--) {
1196             PostQueuedCompletionStatus(ThreadDispatchIOCP, 0,
1197                                        IOCP_SHUTDOWN, NULL);
1198         }
1199         Sleep(1000);
1200     }
1201     /* Empty the accept queue of completion contexts */
1202     apr_thread_mutex_lock(qlock);
1203     while (qhead) {
1204         CloseHandle(qhead->overlapped.hEvent);
1205         closesocket(qhead->accept_socket);
1206         qhead = qhead->next;
1207     }
1208     apr_thread_mutex_unlock(qlock);
1209 
1210     /* Give busy threads a chance to service their connections
1211      * (no more than the global server timeout period which
1212      * we track in msec remaining).
1213      */
1214     watch_thread = 0;
1215     time_remains = (int)(ap_server_conf->timeout / APR_TIME_C(1000));
1216 
1217     while (threads_created)
1218     {
1219         int nFailsafe = MAXIMUM_WAIT_OBJECTS;
1220         DWORD dwRet;
1221 
1222         /* Every time we roll over to wait on the first group
1223          * of MAXIMUM_WAIT_OBJECTS threads, take a breather,
1224          * and infrequently update the error log.
1225          */
1226         if (watch_thread >= threads_created) {
1227             if ((time_remains -= 100) < 0)
1228                 break;
1229 
1230             /* Every 30 seconds give an update */
1231             if ((time_remains % 30000) == 0) {
1232                 ap_log_error(APLOG_MARK, APLOG_NOTICE, APR_SUCCESS,
1233                              ap_server_conf, APLOGNO(00362)
1234                              "Child: Waiting %d more seconds "
1235                              "for %d worker threads to finish.",
1236                              time_remains / 1000, threads_created);
1237             }
1238             /* We'll poll from the top, 10 times per second */
1239             Sleep(100);
1240             watch_thread = 0;
1241         }
1242 
1243         /* Fairness, on each iteration we will pick up with the thread
1244          * after the one we just removed, even if it's a single thread.
1245          * We don't block here.
1246          */
1247         dwRet = WaitForMultipleObjects(min(threads_created - watch_thread,
1248                                            MAXIMUM_WAIT_OBJECTS),
1249                                        child_handles + watch_thread, 0, 0);
1250 
1251         if (dwRet == WAIT_FAILED) {
1252             break;
1253         }
1254         if (dwRet == WAIT_TIMEOUT) {
1255             /* none ready */
1256             watch_thread += MAXIMUM_WAIT_OBJECTS;
1257             continue;
1258         }
1259         else if (dwRet >= WAIT_ABANDONED_0) {
1260             /* We just got the ownership of the object, which
1261              * should happen at most MAXIMUM_WAIT_OBJECTS times.
1262              * It does NOT mean that the object is signaled.
1263              */
1264             if ((nFailsafe--) < 1)
1265                 break;
1266         }
1267         else {
1268             watch_thread += (dwRet - WAIT_OBJECT_0);
1269             if (watch_thread >= threads_created)
1270                 break;
1271             cleanup_thread(child_handles, &threads_created, watch_thread);
1272         }
1273     }
1274 
1275     /* Kill remaining threads off the hard way */
1276     if (threads_created) {
1277         ap_log_error(APLOG_MARK, APLOG_NOTICE, APR_SUCCESS, ap_server_conf, APLOGNO(00363)
1278                      "Child: Terminating %d threads that failed to exit.",
1279                      threads_created);
1280     }
1281     for (i = 0; i < threads_created; i++) {
1282         int *idx;
1283         TerminateThread(child_handles[i], 1);
1284         CloseHandle(child_handles[i]);
1285         /* Reset the scoreboard entry for the thread we just whacked */
1286         idx = apr_hash_get(ht, &child_handles[i], sizeof(HANDLE));
1287         if (idx) {
1288             ap_update_child_status_from_indexes(0, *idx, SERVER_DEAD, NULL);
1289         }
1290     }
1291     ap_log_error(APLOG_MARK, APLOG_NOTICE, APR_SUCCESS, ap_server_conf, APLOGNO(00364)
1292                  "Child: All worker threads have exited.");
1293 
1294     apr_thread_mutex_destroy(child_lock);
1295     apr_thread_mutex_destroy(qlock);
1296     CloseHandle(qwait_event);
1297     CloseHandle(ThreadDispatchIOCP);
1298 
1299     apr_pool_destroy(pchild);
1300     CloseHandle(exit_event);
1301     if (child_events[2] != NULL) {
1302         CloseHandle(child_events[2]);
1303     }
1304 }
1305 
1306 #endif /* def WIN32 */
1307