1 /* Licensed to the Apache Software Foundation (ASF) under one or more
2 * contributor license agreements. See the NOTICE file distributed with
3 * this work for additional information regarding copyright ownership.
4 * The ASF licenses this file to You under the Apache License, Version 2.0
5 * (the "License"); you may not use this file except in compliance with
6 * the License. You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #ifdef WIN32
18
19 #include "apr.h"
20 #include <process.h>
21 #include "httpd.h"
22 #include "http_main.h"
23 #include "http_log.h"
24 #include "http_config.h" /* for read_config */
25 #include "http_core.h" /* for get_remote_host */
26 #include "http_connection.h"
27 #include "http_vhost.h" /* for ap_update_vhost_given_ip */
28 #include "apr_portable.h"
29 #include "apr_thread_proc.h"
30 #include "apr_getopt.h"
31 #include "apr_strings.h"
32 #include "apr_lib.h"
33 #include "apr_shm.h"
34 #include "apr_thread_mutex.h"
35 #include "ap_mpm.h"
36 #include "ap_config.h"
37 #include "ap_listen.h"
38 #include "mpm_default.h"
39 #include "mpm_winnt.h"
40 #include "mpm_common.h"
41 #include <malloc.h>
42 #include "apr_atomic.h"
43 #include "apr_buckets.h"
44 #include "scoreboard.h"
45
46 #ifdef __MINGW32__
47 #include <mswsock.h>
48
49 #ifndef WSAID_ACCEPTEX
50 #define WSAID_ACCEPTEX \
51 {0xb5367df1, 0xcbac, 0x11cf, {0x95, 0xca, 0x00, 0x80, 0x5f, 0x48, 0xa1, 0x92}}
52 typedef BOOL (WINAPI *LPFN_ACCEPTEX)(SOCKET, SOCKET, PVOID, DWORD, DWORD, DWORD, LPDWORD, LPOVERLAPPED);
53 #endif /* WSAID_ACCEPTEX */
54
55 #ifndef WSAID_GETACCEPTEXSOCKADDRS
56 #define WSAID_GETACCEPTEXSOCKADDRS \
57 {0xb5367df2, 0xcbac, 0x11cf, {0x95, 0xca, 0x00, 0x80, 0x5f, 0x48, 0xa1, 0x92}}
58 typedef VOID (WINAPI *LPFN_GETACCEPTEXSOCKADDRS)(PVOID, DWORD, DWORD, DWORD,
59 struct sockaddr **, LPINT,
60 struct sockaddr **, LPINT);
61 #endif /* WSAID_GETACCEPTEXSOCKADDRS */
62
63 #endif /* __MINGW32__ */
64
65 /*
66 * The Windows MPM uses a queue of completion contexts that it passes
67 * between the accept threads and the worker threads. Declare the
68 * functions to access the queue and the structures passed on the
69 * queue in the header file to enable modules to access them
70 * if necessary. The queue resides in the MPM.
71 */
72 #ifdef CONTAINING_RECORD
73 #undef CONTAINING_RECORD
74 #endif
75 #define CONTAINING_RECORD(address, type, field) ((type *)( \
76 (char *)(address) - \
77 (char *)(&((type *)0)->field)))
78 #if APR_HAVE_IPV6
79 #define PADDED_ADDR_SIZE (sizeof(SOCKADDR_IN6)+16)
80 #else
81 #define PADDED_ADDR_SIZE (sizeof(SOCKADDR_IN)+16)
82 #endif
83
84 APLOG_USE_MODULE(mpm_winnt);
85
86 /* Queue for managing the passing of winnt_conn_ctx_t between
87 * the accept and worker threads.
88 */
89 typedef struct winnt_conn_ctx_t_s {
90 struct winnt_conn_ctx_t_s *next;
91 OVERLAPPED overlapped;
92 apr_socket_t *sock;
93 SOCKET accept_socket;
94 char buff[2*PADDED_ADDR_SIZE];
95 struct sockaddr *sa_server;
96 int sa_server_len;
97 struct sockaddr *sa_client;
98 int sa_client_len;
99 apr_pool_t *ptrans;
100 apr_bucket_alloc_t *ba;
101 apr_bucket *data;
102 #if APR_HAVE_IPV6
103 short socket_family;
104 #endif
105 } winnt_conn_ctx_t;
106
107 typedef enum {
108 IOCP_CONNECTION_ACCEPTED = 1,
109 IOCP_WAIT_FOR_RECEIVE = 2,
110 IOCP_WAIT_FOR_TRANSMITFILE = 3,
111 IOCP_SHUTDOWN = 4
112 } io_state_e;
113
114 static apr_pool_t *pchild;
115 static int shutdown_in_progress = 0;
116 static int workers_may_exit = 0;
117 static unsigned int g_blocked_threads = 0;
118 static HANDLE max_requests_per_child_event;
119
120 static apr_thread_mutex_t *child_lock;
121 static apr_thread_mutex_t *qlock;
122 static winnt_conn_ctx_t *qhead = NULL;
123 static winnt_conn_ctx_t *qtail = NULL;
124 static apr_uint32_t num_completion_contexts = 0;
125 static apr_uint32_t max_num_completion_contexts = 0;
126 static HANDLE ThreadDispatchIOCP = NULL;
127 static HANDLE qwait_event = NULL;
128
mpm_recycle_completion_context(winnt_conn_ctx_t * context)129 static void mpm_recycle_completion_context(winnt_conn_ctx_t *context)
130 {
131 /* Recycle the completion context.
132 * - clear the ptrans pool
133 * - put the context on the queue to be consumed by the accept thread
134 * Note:
135 * context->accept_socket may be in a disconnected but reusable
136 * state so -don't- close it.
137 */
138 if (context) {
139 HANDLE saved_event;
140
141 apr_pool_clear(context->ptrans);
142 context->ba = apr_bucket_alloc_create(context->ptrans);
143 context->next = NULL;
144
145 saved_event = context->overlapped.hEvent;
146 memset(&context->overlapped, 0, sizeof(context->overlapped));
147 context->overlapped.hEvent = saved_event;
148 ResetEvent(context->overlapped.hEvent);
149
150 apr_thread_mutex_lock(qlock);
151 if (qtail) {
152 qtail->next = context;
153 } else {
154 qhead = context;
155 SetEvent(qwait_event);
156 }
157 qtail = context;
158 apr_thread_mutex_unlock(qlock);
159 }
160 }
161
mpm_get_completion_context(int * timeout)162 static winnt_conn_ctx_t *mpm_get_completion_context(int *timeout)
163 {
164 apr_status_t rv;
165 winnt_conn_ctx_t *context = NULL;
166
167 *timeout = 0;
168 while (1) {
169 /* Grab a context off the queue */
170 apr_thread_mutex_lock(qlock);
171 if (qhead) {
172 context = qhead;
173 qhead = qhead->next;
174 if (!qhead)
175 qtail = NULL;
176 } else {
177 ResetEvent(qwait_event);
178 }
179 apr_thread_mutex_unlock(qlock);
180
181 if (!context) {
182 /* We failed to grab a context off the queue, consider allocating
183 * a new one out of the child pool. There may be up to
184 * (ap_threads_per_child + num_listeners) contexts in the system
185 * at once.
186 */
187 if (num_completion_contexts >= max_num_completion_contexts) {
188 /* All workers are busy, need to wait for one */
189 static int reported = 0;
190 if (!reported) {
191 ap_log_error(APLOG_MARK, APLOG_ERR, 0, ap_server_conf, APLOGNO(00326)
192 "Server ran out of threads to serve "
193 "requests. Consider raising the "
194 "ThreadsPerChild setting");
195 reported = 1;
196 }
197
198 /* Wait for a worker to free a context. Once per second, give
199 * the caller a chance to check for shutdown. If the wait
200 * succeeds, get the context off the queue. It must be
201 * available, since there's only one consumer.
202 */
203 rv = WaitForSingleObject(qwait_event, 1000);
204 if (rv == WAIT_OBJECT_0)
205 continue;
206 else {
207 if (rv == WAIT_TIMEOUT) {
208 /* somewhat-normal condition where threads are busy */
209 ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, ap_server_conf, APLOGNO(00327)
210 "mpm_get_completion_context: Failed to get a "
211 "free context within 1 second");
212 *timeout = 1;
213 }
214 else {
215 /* should be the unexpected, generic WAIT_FAILED */
216 ap_log_error(APLOG_MARK, APLOG_WARNING, apr_get_os_error(),
217 ap_server_conf, APLOGNO(00328)
218 "mpm_get_completion_context: "
219 "WaitForSingleObject failed to get free context");
220 }
221 return NULL;
222 }
223 } else {
224 /* Allocate another context.
225 * Note: Multiple failures in the next two steps will cause
226 * the pchild pool to 'leak' storage. I don't think this
227 * is worth fixing...
228 */
229 apr_allocator_t *allocator;
230
231 apr_thread_mutex_lock(child_lock);
232 context = (winnt_conn_ctx_t *)apr_pcalloc(pchild,
233 sizeof(winnt_conn_ctx_t));
234
235
236 context->overlapped.hEvent = CreateEvent(NULL, TRUE,
237 FALSE, NULL);
238 if (context->overlapped.hEvent == NULL) {
239 /* Hopefully this is a temporary condition ... */
240 ap_log_error(APLOG_MARK, APLOG_WARNING, apr_get_os_error(),
241 ap_server_conf, APLOGNO(00329)
242 "mpm_get_completion_context: "
243 "CreateEvent failed.");
244
245 apr_thread_mutex_unlock(child_lock);
246 return NULL;
247 }
248
249 /* Create the transaction pool */
250 apr_allocator_create(&allocator);
251 apr_allocator_max_free_set(allocator, ap_max_mem_free);
252 rv = apr_pool_create_ex(&context->ptrans, pchild, NULL,
253 allocator);
254 if (rv != APR_SUCCESS) {
255 ap_log_error(APLOG_MARK, APLOG_WARNING, rv, ap_server_conf, APLOGNO(00330)
256 "mpm_get_completion_context: Failed "
257 "to create the transaction pool.");
258 CloseHandle(context->overlapped.hEvent);
259
260 apr_thread_mutex_unlock(child_lock);
261 return NULL;
262 }
263 apr_allocator_owner_set(allocator, context->ptrans);
264 apr_pool_tag(context->ptrans, "transaction");
265
266 context->accept_socket = INVALID_SOCKET;
267 context->ba = apr_bucket_alloc_create(context->ptrans);
268 apr_atomic_inc32(&num_completion_contexts);
269
270 apr_thread_mutex_unlock(child_lock);
271 break;
272 }
273 } else {
274 /* Got a context from the queue */
275 break;
276 }
277 }
278
279 return context;
280 }
281
282 typedef enum {
283 ACCEPT_FILTER_NONE = 0,
284 ACCEPT_FILTER_CONNECT = 1
285 } accept_filter_e;
286
accept_filter_to_string(accept_filter_e accf)287 static const char * accept_filter_to_string(accept_filter_e accf)
288 {
289 switch (accf) {
290 case ACCEPT_FILTER_NONE:
291 return "none";
292 case ACCEPT_FILTER_CONNECT:
293 return "connect";
294 default:
295 return "";
296 }
297 }
298
get_accept_filter(const char * protocol)299 static accept_filter_e get_accept_filter(const char *protocol)
300 {
301 core_server_config *core_sconf;
302 const char *name;
303
304 core_sconf = ap_get_core_module_config(ap_server_conf->module_config);
305 name = apr_table_get(core_sconf->accf_map, protocol);
306 if (!name) {
307 ap_log_error(APLOG_MARK, APLOG_WARNING, 0, ap_server_conf,
308 APLOGNO(02531) "winnt_accept: Listen protocol '%s' has "
309 "no known accept filter. Using 'none' instead",
310 protocol);
311 return ACCEPT_FILTER_NONE;
312 }
313 else if (strcmp(name, "data") == 0) {
314 ap_log_error(APLOG_MARK, APLOG_INFO, 0, ap_server_conf,
315 APLOGNO(03458) "winnt_accept: 'data' accept filter is no "
316 "longer supported. Using 'connect' instead");
317 return ACCEPT_FILTER_CONNECT;
318 }
319 else if (strcmp(name, "connect") == 0) {
320 return ACCEPT_FILTER_CONNECT;
321 }
322 else if (strcmp(name, "none") == 0) {
323 return ACCEPT_FILTER_NONE;
324 }
325 else {
326 ap_log_error(APLOG_MARK, APLOG_WARNING, 0, ap_server_conf, APLOGNO(00331)
327 "winnt_accept: unrecognized AcceptFilter '%s', "
328 "only 'data', 'connect' or 'none' are valid. "
329 "Using 'none' instead", name);
330 return ACCEPT_FILTER_NONE;
331 }
332 }
333
334 /* Windows NT/2000 specific code...
335 * Accept processing for on Windows NT uses a producer/consumer queue
336 * model. An accept thread accepts connections off the network then issues
337 * PostQueuedCompletionStatus() to awake a thread blocked on the ThreadDispatch
338 * IOCompletionPort.
339 *
340 * winnt_accept()
341 * One or more accept threads run in this function, each of which accepts
342 * connections off the network and calls PostQueuedCompletionStatus() to
343 * queue an io completion packet to the ThreadDispatch IOCompletionPort.
344 * winnt_get_connection()
345 * Worker threads block on the ThreadDispatch IOCompletionPort awaiting
346 * connections to service.
347 */
348 #define MAX_ACCEPTEX_ERR_COUNT 10
349
winnt_accept(void * lr_)350 static unsigned int __stdcall winnt_accept(void *lr_)
351 {
352 ap_listen_rec *lr = (ap_listen_rec *)lr_;
353 apr_os_sock_info_t sockinfo;
354 winnt_conn_ctx_t *context = NULL;
355 DWORD BytesRead = 0;
356 SOCKET nlsd;
357 LPFN_ACCEPTEX lpfnAcceptEx = NULL;
358 LPFN_GETACCEPTEXSOCKADDRS lpfnGetAcceptExSockaddrs = NULL;
359 GUID GuidAcceptEx = WSAID_ACCEPTEX;
360 GUID GuidGetAcceptExSockaddrs = WSAID_GETACCEPTEXSOCKADDRS;
361 int rv;
362 accept_filter_e accf;
363 int err_count = 0;
364 HANDLE events[3];
365 #if APR_HAVE_IPV6
366 SOCKADDR_STORAGE ss_listen;
367 int namelen = sizeof(ss_listen);
368 #endif
369 u_long zero = 0;
370
371 apr_os_sock_get(&nlsd, lr->sd);
372
373 #if APR_HAVE_IPV6
374 if (getsockname(nlsd, (struct sockaddr *)&ss_listen, &namelen) == SOCKET_ERROR) {
375 ap_log_error(APLOG_MARK, APLOG_ERR, apr_get_netos_error(),
376 ap_server_conf, APLOGNO(00332)
377 "winnt_accept: getsockname error on listening socket, "
378 "is IPv6 available?");
379 return 1;
380 }
381 #endif
382
383 accf = get_accept_filter(lr->protocol);
384 if (accf == ACCEPT_FILTER_CONNECT)
385 {
386 if (WSAIoctl(nlsd, SIO_GET_EXTENSION_FUNCTION_POINTER,
387 &GuidAcceptEx, sizeof GuidAcceptEx,
388 &lpfnAcceptEx, sizeof lpfnAcceptEx,
389 &BytesRead, NULL, NULL) == SOCKET_ERROR) {
390 ap_log_error(APLOG_MARK, APLOG_ERR, apr_get_netos_error(),
391 ap_server_conf, APLOGNO(02322)
392 "winnt_accept: failed to retrieve AcceptEx, try 'AcceptFilter none'");
393 return 1;
394 }
395 if (WSAIoctl(nlsd, SIO_GET_EXTENSION_FUNCTION_POINTER,
396 &GuidGetAcceptExSockaddrs, sizeof GuidGetAcceptExSockaddrs,
397 &lpfnGetAcceptExSockaddrs, sizeof lpfnGetAcceptExSockaddrs,
398 &BytesRead, NULL, NULL) == SOCKET_ERROR) {
399 ap_log_error(APLOG_MARK, APLOG_ERR, apr_get_netos_error(),
400 ap_server_conf, APLOGNO(02323)
401 "winnt_accept: failed to retrieve GetAcceptExSockaddrs, try 'AcceptFilter none'");
402 return 1;
403 }
404 /* first, high priority event is an already accepted connection */
405 events[1] = exit_event;
406 events[2] = max_requests_per_child_event;
407 }
408 else /* accf == ACCEPT_FILTER_NONE */
409 {
410 reinit: /* target of connect upon too many AcceptEx failures */
411
412 /* last, low priority event is a not yet accepted connection */
413 events[0] = exit_event;
414 events[1] = max_requests_per_child_event;
415 events[2] = CreateEvent(NULL, FALSE, FALSE, NULL);
416
417 /* The event needs to be removed from the accepted socket,
418 * if not removed from the listen socket prior to accept(),
419 */
420 rv = WSAEventSelect(nlsd, events[2], FD_ACCEPT);
421 if (rv) {
422 ap_log_error(APLOG_MARK, APLOG_ERR,
423 apr_get_netos_error(), ap_server_conf, APLOGNO(00333)
424 "WSAEventSelect() failed.");
425 CloseHandle(events[2]);
426 return 1;
427 }
428 }
429
430 ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, ap_server_conf, APLOGNO(00334)
431 "Child: Accept thread listening on %pI using AcceptFilter %s",
432 lr->bind_addr, accept_filter_to_string(accf));
433
434 while (!shutdown_in_progress) {
435 if (!context) {
436 int timeout;
437
438 context = mpm_get_completion_context(&timeout);
439 if (!context) {
440 if (!timeout) {
441 /* Hopefully a temporary condition in the provider? */
442 ++err_count;
443 if (err_count > MAX_ACCEPTEX_ERR_COUNT) {
444 ap_log_error(APLOG_MARK, APLOG_CRIT, 0, ap_server_conf, APLOGNO(00335)
445 "winnt_accept: Too many failures grabbing a "
446 "connection ctx. Aborting.");
447 break;
448 }
449 }
450 Sleep(100);
451 continue;
452 }
453 }
454
455 if (accf == ACCEPT_FILTER_CONNECT)
456 {
457 char *buf;
458
459 /* Create and initialize the accept socket */
460 #if APR_HAVE_IPV6
461 if (context->accept_socket == INVALID_SOCKET) {
462 context->accept_socket = socket(ss_listen.ss_family, SOCK_STREAM,
463 IPPROTO_TCP);
464 context->socket_family = ss_listen.ss_family;
465 }
466 else if (context->socket_family != ss_listen.ss_family) {
467 closesocket(context->accept_socket);
468 context->accept_socket = socket(ss_listen.ss_family, SOCK_STREAM,
469 IPPROTO_TCP);
470 context->socket_family = ss_listen.ss_family;
471 }
472 #else
473 if (context->accept_socket == INVALID_SOCKET)
474 context->accept_socket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
475 #endif
476
477 if (context->accept_socket == INVALID_SOCKET) {
478 ap_log_error(APLOG_MARK, APLOG_WARNING, apr_get_netos_error(),
479 ap_server_conf, APLOGNO(00336)
480 "winnt_accept: Failed to allocate an accept socket. "
481 "Temporary resource constraint? Try again.");
482 Sleep(100);
483 continue;
484 }
485
486 buf = context->buff;
487
488 /* AcceptEx on the completion context. The completion context will be
489 * signaled when a connection is accepted.
490 */
491 if (!lpfnAcceptEx(nlsd, context->accept_socket, buf, 0,
492 PADDED_ADDR_SIZE, PADDED_ADDR_SIZE, &BytesRead,
493 &context->overlapped)) {
494 rv = apr_get_netos_error();
495 if ((rv == APR_FROM_OS_ERROR(WSAECONNRESET)) ||
496 (rv == APR_FROM_OS_ERROR(WSAEACCES))) {
497 /* We can get here when:
498 * 1) the client disconnects early
499 * 2) handshake was incomplete
500 */
501 closesocket(context->accept_socket);
502 context->accept_socket = INVALID_SOCKET;
503 continue;
504 }
505 else if ((rv == APR_FROM_OS_ERROR(WSAEINVAL)) ||
506 (rv == APR_FROM_OS_ERROR(WSAENOTSOCK))) {
507 /* We can get here when:
508 * 1) TransmitFile does not properly recycle the accept socket (typically
509 * because the client disconnected)
510 * 2) there is VPN or Firewall software installed with
511 * buggy WSAAccept or WSADuplicateSocket implementation
512 * 3) the dynamic address / adapter has changed
513 * Give five chances, then fall back on AcceptFilter 'none'
514 */
515 closesocket(context->accept_socket);
516 context->accept_socket = INVALID_SOCKET;
517 ++err_count;
518 if (err_count > MAX_ACCEPTEX_ERR_COUNT) {
519 ap_log_error(APLOG_MARK, APLOG_ERR, rv, ap_server_conf, APLOGNO(00337)
520 "Child: Encountered too many AcceptEx "
521 "faults accepting client connections. "
522 "Possible causes: dynamic address renewal, "
523 "or incompatible VPN or firewall software. ");
524 ap_log_error(APLOG_MARK, APLOG_NOTICE, rv, ap_server_conf, APLOGNO(00338)
525 "winnt_mpm: falling back to "
526 "'AcceptFilter none'.");
527 err_count = 0;
528 accf = ACCEPT_FILTER_NONE;
529 }
530 continue;
531 }
532 else if ((rv != APR_FROM_OS_ERROR(ERROR_IO_PENDING)) &&
533 (rv != APR_FROM_OS_ERROR(WSA_IO_PENDING))) {
534 closesocket(context->accept_socket);
535 context->accept_socket = INVALID_SOCKET;
536 ++err_count;
537 if (err_count > MAX_ACCEPTEX_ERR_COUNT) {
538 ap_log_error(APLOG_MARK, APLOG_ERR, rv, ap_server_conf, APLOGNO(00339)
539 "Child: Encountered too many AcceptEx "
540 "faults accepting client connections.");
541 ap_log_error(APLOG_MARK, APLOG_NOTICE, rv, ap_server_conf, APLOGNO(00340)
542 "winnt_mpm: falling back to "
543 "'AcceptFilter none'.");
544 err_count = 0;
545 accf = ACCEPT_FILTER_NONE;
546 goto reinit;
547 }
548 continue;
549 }
550
551 err_count = 0;
552 events[0] = context->overlapped.hEvent;
553
554 do {
555 rv = WaitForMultipleObjectsEx(3, events, FALSE, INFINITE, TRUE);
556 } while (rv == WAIT_IO_COMPLETION);
557
558 if (rv == WAIT_OBJECT_0) {
559 if ((context->accept_socket != INVALID_SOCKET) &&
560 !GetOverlappedResult((HANDLE)context->accept_socket,
561 &context->overlapped,
562 &BytesRead, FALSE)) {
563 ap_log_error(APLOG_MARK, APLOG_WARNING,
564 apr_get_os_error(), ap_server_conf, APLOGNO(00341)
565 "winnt_accept: Asynchronous AcceptEx failed.");
566 closesocket(context->accept_socket);
567 context->accept_socket = INVALID_SOCKET;
568 }
569 }
570 else {
571 /* exit_event triggered or event handle was closed */
572 closesocket(context->accept_socket);
573 context->accept_socket = INVALID_SOCKET;
574 break;
575 }
576
577 if (context->accept_socket == INVALID_SOCKET) {
578 continue;
579 }
580 }
581 err_count = 0;
582
583 /* Potential optimization; consider handing off to the worker */
584
585 /* Inherit the listen socket settings. Required for
586 * shutdown() to work
587 */
588 if (setsockopt(context->accept_socket, SOL_SOCKET,
589 SO_UPDATE_ACCEPT_CONTEXT, (char *)&nlsd,
590 sizeof(nlsd))) {
591 ap_log_error(APLOG_MARK, APLOG_WARNING, apr_get_netos_error(),
592 ap_server_conf, APLOGNO(00342)
593 "setsockopt(SO_UPDATE_ACCEPT_CONTEXT) failed.");
594 /* Not a failure condition. Keep running. */
595 }
596
597 /* Get the local & remote address
598 * TODO; error check
599 */
600 lpfnGetAcceptExSockaddrs(buf, 0, PADDED_ADDR_SIZE, PADDED_ADDR_SIZE,
601 &context->sa_server, &context->sa_server_len,
602 &context->sa_client, &context->sa_client_len);
603 }
604 else /* accf == ACCEPT_FILTER_NONE */
605 {
606 /* There is no socket reuse without AcceptEx() */
607 if (context->accept_socket != INVALID_SOCKET)
608 closesocket(context->accept_socket);
609
610 /* This could be a persistent event per-listener rather than
611 * per-accept. However, the event needs to be removed from
612 * the target socket if not removed from the listen socket
613 * prior to accept(), or the event select is inherited.
614 * and must be removed from the accepted socket.
615 */
616
617 do {
618 rv = WaitForMultipleObjectsEx(3, events, FALSE, INFINITE, TRUE);
619 } while (rv == WAIT_IO_COMPLETION);
620
621
622 if (rv != WAIT_OBJECT_0 + 2) {
623 /* not FD_ACCEPT;
624 * exit_event triggered or event handle was closed
625 */
626 break;
627 }
628
629 context->sa_server = (void *) context->buff;
630 context->sa_server_len = sizeof(context->buff) / 2;
631 context->sa_client_len = context->sa_server_len;
632 context->sa_client = (void *) (context->buff
633 + context->sa_server_len);
634
635 context->accept_socket = accept(nlsd, context->sa_server,
636 &context->sa_server_len);
637
638 if (context->accept_socket == INVALID_SOCKET) {
639
640 rv = apr_get_netos_error();
641 if ( rv == APR_FROM_OS_ERROR(WSAECONNRESET)
642 || rv == APR_FROM_OS_ERROR(WSAEINPROGRESS)
643 || rv == APR_FROM_OS_ERROR(WSAEWOULDBLOCK) ) {
644 ap_log_error(APLOG_MARK, APLOG_DEBUG,
645 rv, ap_server_conf, APLOGNO(00343)
646 "accept() failed, retrying.");
647 continue;
648 }
649
650 /* A more serious error than 'retry', log it */
651 ap_log_error(APLOG_MARK, APLOG_WARNING,
652 rv, ap_server_conf, APLOGNO(00344)
653 "accept() failed.");
654
655 if ( rv == APR_FROM_OS_ERROR(WSAEMFILE)
656 || rv == APR_FROM_OS_ERROR(WSAENOBUFS) ) {
657 /* Hopefully a temporary condition in the provider? */
658 Sleep(100);
659 ++err_count;
660 if (err_count > MAX_ACCEPTEX_ERR_COUNT) {
661 ap_log_error(APLOG_MARK, APLOG_ERR, rv, ap_server_conf, APLOGNO(00345)
662 "Child: Encountered too many accept() "
663 "resource faults, aborting.");
664 break;
665 }
666 continue;
667 }
668 break;
669 }
670 /* Per MSDN, cancel the inherited association of this socket
671 * to the WSAEventSelect API, and restore the state corresponding
672 * to apr_os_sock_make's default assumptions (really, a flaw within
673 * os_sock_make and os_sock_put that it does not query).
674 */
675 WSAEventSelect(context->accept_socket, 0, 0);
676 err_count = 0;
677
678 context->sa_server_len = sizeof(context->buff) / 2;
679 if (getsockname(context->accept_socket, context->sa_server,
680 &context->sa_server_len) == SOCKET_ERROR) {
681 ap_log_error(APLOG_MARK, APLOG_WARNING, apr_get_netos_error(), ap_server_conf, APLOGNO(00346)
682 "getsockname failed");
683 continue;
684 }
685 if ((getpeername(context->accept_socket, context->sa_client,
686 &context->sa_client_len)) == SOCKET_ERROR) {
687 ap_log_error(APLOG_MARK, APLOG_WARNING, apr_get_netos_error(), ap_server_conf, APLOGNO(00347)
688 "getpeername failed");
689 memset(&context->sa_client, '\0', sizeof(context->sa_client));
690 }
691 }
692
693 sockinfo.os_sock = &context->accept_socket;
694 sockinfo.local = context->sa_server;
695 sockinfo.remote = context->sa_client;
696 sockinfo.family = context->sa_server->sa_family;
697 sockinfo.type = SOCK_STREAM;
698 sockinfo.protocol = IPPROTO_TCP;
699 /* Restore the state corresponding to apr_os_sock_make's default
700 * assumption of timeout -1 (really, a flaw of os_sock_make and
701 * os_sock_put that it does not query to determine ->timeout).
702 * XXX: Upon a fix to APR, these three statements should disappear.
703 */
704 ioctlsocket(context->accept_socket, FIONBIO, &zero);
705 setsockopt(context->accept_socket, SOL_SOCKET, SO_RCVTIMEO,
706 (char *) &zero, sizeof(zero));
707 setsockopt(context->accept_socket, SOL_SOCKET, SO_SNDTIMEO,
708 (char *) &zero, sizeof(zero));
709 apr_os_sock_make(&context->sock, &sockinfo, context->ptrans);
710
711 /* When a connection is received, send an io completion notification
712 * to the ThreadDispatchIOCP.
713 */
714 PostQueuedCompletionStatus(ThreadDispatchIOCP, BytesRead,
715 IOCP_CONNECTION_ACCEPTED,
716 &context->overlapped);
717 context = NULL;
718 }
719 if (accf == ACCEPT_FILTER_NONE)
720 CloseHandle(events[2]);
721
722 if (!shutdown_in_progress) {
723 /* Yow, hit an irrecoverable error! Tell the child to die. */
724 SetEvent(exit_event);
725 }
726
727 ap_log_error(APLOG_MARK, APLOG_DEBUG, APR_SUCCESS, ap_server_conf, APLOGNO(00348)
728 "Child: Accept thread exiting.");
729 return 0;
730 }
731
732
winnt_get_connection(winnt_conn_ctx_t * context)733 static winnt_conn_ctx_t *winnt_get_connection(winnt_conn_ctx_t *context)
734 {
735 int rc;
736 DWORD BytesRead;
737 LPOVERLAPPED pol;
738 #ifdef _WIN64
739 ULONG_PTR CompKey;
740 #else
741 DWORD CompKey;
742 #endif
743
744 mpm_recycle_completion_context(context);
745
746 apr_atomic_inc32(&g_blocked_threads);
747 while (1) {
748 if (workers_may_exit) {
749 apr_atomic_dec32(&g_blocked_threads);
750 return NULL;
751 }
752 rc = GetQueuedCompletionStatus(ThreadDispatchIOCP, &BytesRead,
753 &CompKey, &pol, INFINITE);
754 if (!rc) {
755 rc = apr_get_os_error();
756 ap_log_error(APLOG_MARK, APLOG_DEBUG, rc, ap_server_conf, APLOGNO(00349)
757 "Child: GetQueuedCompletionStatus returned %d",
758 rc);
759 continue;
760 }
761
762 switch (CompKey) {
763 case IOCP_CONNECTION_ACCEPTED:
764 context = CONTAINING_RECORD(pol, winnt_conn_ctx_t, overlapped);
765 break;
766 case IOCP_SHUTDOWN:
767 apr_atomic_dec32(&g_blocked_threads);
768 return NULL;
769 default:
770 apr_atomic_dec32(&g_blocked_threads);
771 return NULL;
772 }
773 break;
774 }
775 apr_atomic_dec32(&g_blocked_threads);
776
777 return context;
778 }
779
780 /*
781 * worker_main()
782 * Main entry point for the worker threads. Worker threads block in
783 * win*_get_connection() awaiting a connection to service.
784 */
worker_main(void * thread_num_val)785 static DWORD __stdcall worker_main(void *thread_num_val)
786 {
787 apr_thread_t *thd = NULL;
788 apr_os_thread_t osthd = NULL;
789 static int requests_this_child = 0;
790 winnt_conn_ctx_t *context = NULL;
791 int thread_num = (int)thread_num_val;
792 ap_sb_handle_t *sbh;
793 conn_rec *c;
794 apr_int32_t disconnected;
795
796 #if AP_HAS_THREAD_LOCAL
797 if (ap_thread_current_create(&thd, NULL, pchild) != APR_SUCCESS) {
798 ap_log_error(APLOG_MARK, APLOG_WARNING, 0, ap_server_conf, APLOGNO(10376)
799 "Couldn't initialize worker thread, thread locals won't "
800 "be available");
801 osthd = apr_os_thread_current();
802 }
803 #else
804 osthd = apr_os_thread_current();
805 #endif
806
807 while (1) {
808
809 ap_update_child_status_from_indexes(0, thread_num, SERVER_READY, NULL);
810
811 /* Grab a connection off the network */
812 context = winnt_get_connection(context);
813
814 if (!context) {
815 /* Time for the thread to exit */
816 break;
817 }
818
819 /* Have we hit MaxConnectionsPerChild connections? */
820 if (ap_max_requests_per_child) {
821 requests_this_child++;
822 if (requests_this_child > ap_max_requests_per_child) {
823 SetEvent(max_requests_per_child_event);
824 }
825 }
826
827 ap_create_sb_handle(&sbh, context->ptrans, 0, thread_num);
828 c = ap_run_create_connection(context->ptrans, ap_server_conf,
829 context->sock, thread_num, sbh,
830 context->ba);
831
832 if (!c) {
833 /* ap_run_create_connection closes the socket on failure */
834 context->accept_socket = INVALID_SOCKET;
835 continue;
836 }
837
838 if (osthd) {
839 thd = NULL;
840 apr_os_thread_put(&thd, &osthd, context->ptrans);
841 }
842 c->current_thread = thd;
843
844 ap_process_connection(c, context->sock);
845
846 ap_lingering_close(c);
847
848 apr_socket_opt_get(context->sock, APR_SO_DISCONNECTED, &disconnected);
849 if (!disconnected) {
850 context->accept_socket = INVALID_SOCKET;
851 }
852 }
853
854 ap_update_child_status_from_indexes(0, thread_num, SERVER_DEAD, NULL);
855
856 #if AP_HAS_THREAD_LOCAL
857 if (!osthd) {
858 apr_pool_destroy(apr_thread_pool_get(thd));
859 }
860 #endif
861
862 return 0;
863 }
864
865
cleanup_thread(HANDLE * handles,int * thread_cnt,int thread_to_clean)866 static void cleanup_thread(HANDLE *handles, int *thread_cnt,
867 int thread_to_clean)
868 {
869 int i;
870
871 CloseHandle(handles[thread_to_clean]);
872 for (i = thread_to_clean; i < ((*thread_cnt) - 1); i++)
873 handles[i] = handles[i + 1];
874 (*thread_cnt)--;
875 }
876
877
878 /*
879 * child_main()
880 * Entry point for the main control thread for the child process.
881 * This thread creates the accept thread, worker threads and
882 * monitors the child process for maintenance and shutdown
883 * events.
884 */
create_listener_thread(void)885 static void create_listener_thread(void)
886 {
887 unsigned tid;
888 int num_listeners = 0;
889 /* Start an accept thread per listener
890 * XXX: Why would we have a NULL sd in our listeners?
891 */
892 ap_listen_rec *lr;
893
894 /* Number of completion_contexts allowed in the system is
895 * (ap_threads_per_child + num_listeners). We need the additional
896 * completion contexts to prevent server hangs when ThreadsPerChild
897 * is configured to something less than or equal to the number
898 * of listeners. This is not a usual case, but people have
899 * encountered it.
900 */
901 for (lr = ap_listeners; lr ; lr = lr->next) {
902 num_listeners++;
903 }
904 max_num_completion_contexts = ap_threads_per_child + num_listeners;
905
906 /* Now start a thread per listener */
907 for (lr = ap_listeners; lr; lr = lr->next) {
908 if (lr->sd != NULL) {
909 /* A smaller stack is sufficient.
910 * To convert to CreateThread, the returned handle cannot be
911 * ignored, it must be closed/joined.
912 */
913 _beginthreadex(NULL, 65536, winnt_accept,
914 (void *) lr, stack_res_flag, &tid);
915 }
916 }
917 }
918
919
child_main(apr_pool_t * pconf,DWORD parent_pid)920 void child_main(apr_pool_t *pconf, DWORD parent_pid)
921 {
922 apr_status_t status;
923 apr_hash_t *ht;
924 ap_listen_rec *lr;
925 HANDLE child_events[3];
926 HANDLE *child_handles;
927 int listener_started = 0;
928 int threads_created = 0;
929 int watch_thread;
930 int time_remains;
931 int cld;
932 DWORD tid;
933 int rv;
934 int i;
935 int num_events;
936
937 /* Get a sub context for global allocations in this child, so that
938 * we can have cleanups occur when the child exits.
939 */
940 apr_pool_create(&pchild, pconf);
941 apr_pool_tag(pchild, "pchild");
942
943 ap_run_child_init(pchild, ap_server_conf);
944 ht = apr_hash_make(pchild);
945
946 /* Initialize the child_events */
947 max_requests_per_child_event = CreateEvent(NULL, TRUE, FALSE, NULL);
948 if (!max_requests_per_child_event) {
949 ap_log_error(APLOG_MARK, APLOG_CRIT, apr_get_os_error(), ap_server_conf, APLOGNO(00350)
950 "Child: Failed to create a max_requests event.");
951 exit(APEXIT_CHILDINIT);
952 }
953 child_events[0] = exit_event;
954 child_events[1] = max_requests_per_child_event;
955
956 if (parent_pid != my_pid) {
957 child_events[2] = OpenProcess(SYNCHRONIZE, FALSE, parent_pid);
958 if (child_events[2] == NULL) {
959 num_events = 2;
960 ap_log_error(APLOG_MARK, APLOG_ERR, apr_get_os_error(), ap_server_conf, APLOGNO(02643)
961 "Child: Failed to open handle to parent process %ld; "
962 "will not react to abrupt parent termination", parent_pid);
963 }
964 else {
965 num_events = 3;
966 }
967 }
968 else {
969 /* presumably -DONE_PROCESS */
970 child_events[2] = NULL;
971 num_events = 2;
972 }
973
974 /*
975 * Wait until we have permission to start accepting connections.
976 * start_mutex is used to ensure that only one child ever
977 * goes into the listen/accept loop at once.
978 */
979 status = apr_proc_mutex_lock(start_mutex);
980 if (status != APR_SUCCESS) {
981 ap_log_error(APLOG_MARK, APLOG_ERR, status, ap_server_conf, APLOGNO(00351)
982 "Child: Failed to acquire the start_mutex. "
983 "Process will exit.");
984 exit(APEXIT_CHILDINIT);
985 }
986 ap_log_error(APLOG_MARK, APLOG_DEBUG, APR_SUCCESS, ap_server_conf, APLOGNO(00352)
987 "Child: Acquired the start mutex.");
988
989 /*
990 * Create the worker thread dispatch IOCompletionPort
991 */
992 /* Create the worker thread dispatch IOCP */
993 ThreadDispatchIOCP = CreateIoCompletionPort(INVALID_HANDLE_VALUE,
994 NULL, 0, 0);
995 apr_thread_mutex_create(&qlock, APR_THREAD_MUTEX_DEFAULT, pchild);
996 qwait_event = CreateEvent(NULL, TRUE, FALSE, NULL);
997 if (!qwait_event) {
998 ap_log_error(APLOG_MARK, APLOG_CRIT, apr_get_os_error(),
999 ap_server_conf, APLOGNO(00353)
1000 "Child: Failed to create a qwait event.");
1001 exit(APEXIT_CHILDINIT);
1002 }
1003
1004 /*
1005 * Create the pool of worker threads
1006 */
1007 ap_log_error(APLOG_MARK, APLOG_NOTICE, APR_SUCCESS, ap_server_conf, APLOGNO(00354)
1008 "Child: Starting %d worker threads.", ap_threads_per_child);
1009 child_handles = (HANDLE) apr_pcalloc(pchild, ap_threads_per_child
1010 * sizeof(HANDLE));
1011 apr_thread_mutex_create(&child_lock, APR_THREAD_MUTEX_DEFAULT, pchild);
1012
1013 while (1) {
1014 for (i = 0; i < ap_threads_per_child; i++) {
1015 int *score_idx;
1016 int status = ap_scoreboard_image->servers[0][i].status;
1017 if (status != SERVER_GRACEFUL && status != SERVER_DEAD) {
1018 continue;
1019 }
1020 ap_update_child_status_from_indexes(0, i, SERVER_STARTING, NULL);
1021
1022 child_handles[i] = CreateThread(NULL, ap_thread_stacksize,
1023 worker_main, (void *) i,
1024 stack_res_flag, &tid);
1025 if (child_handles[i] == 0) {
1026 ap_log_error(APLOG_MARK, APLOG_CRIT, apr_get_os_error(),
1027 ap_server_conf, APLOGNO(00355)
1028 "Child: CreateThread failed. Unable to "
1029 "create all worker threads. Created %d of the %d "
1030 "threads requested with the ThreadsPerChild "
1031 "configuration directive.",
1032 threads_created, ap_threads_per_child);
1033 ap_signal_parent(SIGNAL_PARENT_SHUTDOWN);
1034 goto shutdown;
1035 }
1036 threads_created++;
1037 /* Save the score board index in ht keyed to the thread handle.
1038 * We need this when cleaning up threads down below...
1039 */
1040 apr_thread_mutex_lock(child_lock);
1041 score_idx = apr_pcalloc(pchild, sizeof(int));
1042 *score_idx = i;
1043 apr_hash_set(ht, &child_handles[i], sizeof(HANDLE), score_idx);
1044 apr_thread_mutex_unlock(child_lock);
1045 }
1046 /* Start the listener only when workers are available */
1047 if (!listener_started && threads_created) {
1048 create_listener_thread();
1049 listener_started = 1;
1050 winnt_mpm_state = AP_MPMQ_RUNNING;
1051 }
1052 if (threads_created == ap_threads_per_child) {
1053 break;
1054 }
1055 /* Check to see if the child has been told to exit */
1056 if (WaitForSingleObject(exit_event, 0) != WAIT_TIMEOUT) {
1057 break;
1058 }
1059 /* wait for previous generation to clean up an entry in the scoreboard
1060 */
1061 apr_sleep(1 * APR_USEC_PER_SEC);
1062 }
1063
1064 /* Wait for one of these events:
1065 * exit_event:
1066 * The exit_event is signaled by the parent process to notify
1067 * the child that it is time to exit.
1068 *
1069 * max_requests_per_child_event:
1070 * This event is signaled by the worker threads to indicate that
1071 * the process has handled MaxConnectionsPerChild connections.
1072 *
1073 * parent process exiting
1074 *
1075 * TIMEOUT:
1076 * To do periodic maintenance on the server (check for thread exits,
1077 * number of completion contexts, etc.)
1078 *
1079 * XXX: thread exits *aren't* being checked.
1080 *
1081 * XXX: other_child - we need the process handles to the other children
1082 * in order to map them to apr_proc_other_child_read (which is not
1083 * named well, it's more like a_p_o_c_died.)
1084 *
1085 * XXX: however - if we get a_p_o_c handle inheritance working, and
1086 * the parent process creates other children and passes the pipes
1087 * to our worker processes, then we have no business doing such
1088 * things in the child_main loop, but should happen in master_main.
1089 */
1090 while (1) {
1091 #if !APR_HAS_OTHER_CHILD
1092 rv = WaitForMultipleObjects(num_events, (HANDLE *)child_events, FALSE, INFINITE);
1093 cld = rv - WAIT_OBJECT_0;
1094 #else
1095 /* THIS IS THE EXPECTED BUILD VARIATION -- APR_HAS_OTHER_CHILD */
1096 rv = WaitForMultipleObjects(num_events, (HANDLE *)child_events, FALSE, 1000);
1097 cld = rv - WAIT_OBJECT_0;
1098 if (rv == WAIT_TIMEOUT) {
1099 apr_proc_other_child_refresh_all(APR_OC_REASON_RUNNING);
1100 }
1101 else
1102 #endif
1103 if (rv == WAIT_FAILED) {
1104 /* Something serious is wrong */
1105 ap_log_error(APLOG_MARK, APLOG_CRIT, apr_get_os_error(),
1106 ap_server_conf, APLOGNO(00356)
1107 "Child: WAIT_FAILED -- shutting down server");
1108 /* check handle validity to identify a possible culprit */
1109 for (i = 0; i < num_events; i++) {
1110 DWORD out_flags;
1111
1112 if (0 == GetHandleInformation(child_events[i], &out_flags)) {
1113 ap_log_error(APLOG_MARK, APLOG_CRIT, apr_get_os_error(),
1114 ap_server_conf, APLOGNO(02644)
1115 "Child: Event handle #%d (%pp) is invalid",
1116 i, child_events[i]);
1117 }
1118 }
1119 break;
1120 }
1121 else if (cld == 0) {
1122 /* Exit event was signaled */
1123 ap_log_error(APLOG_MARK, APLOG_DEBUG, APR_SUCCESS, ap_server_conf, APLOGNO(00357)
1124 "Child: Exit event signaled. Child process is "
1125 "ending.");
1126 break;
1127 }
1128 else if (cld == 2) {
1129 /* The parent is dead. Shutdown the child process. */
1130 ap_log_error(APLOG_MARK, APLOG_CRIT, 0, ap_server_conf, APLOGNO(02538)
1131 "Child: Parent process exited abruptly. Child process "
1132 "is ending");
1133 break;
1134 }
1135 else {
1136 /* MaxConnectionsPerChild event set by the worker threads.
1137 * Signal the parent to restart
1138 */
1139 ap_log_error(APLOG_MARK, APLOG_NOTICE, APR_SUCCESS, ap_server_conf, APLOGNO(00358)
1140 "Child: Process exiting because it reached "
1141 "MaxConnectionsPerChild. Signaling the parent to "
1142 "restart a new child process.");
1143 ap_signal_parent(SIGNAL_PARENT_RESTART);
1144 break;
1145 }
1146 }
1147
1148 /*
1149 * Time to shutdown the child process
1150 */
1151
1152 shutdown:
1153
1154 winnt_mpm_state = AP_MPMQ_STOPPING;
1155
1156 /* Close the listening sockets. Note, we must close the listeners
1157 * before closing any accept sockets pending in AcceptEx to prevent
1158 * memory leaks in the kernel.
1159 */
1160 for (lr = ap_listeners; lr ; lr = lr->next) {
1161 apr_socket_close(lr->sd);
1162 }
1163
1164 /* Shutdown listener threads and pending AcceptEx sockets
1165 * but allow the worker threads to continue consuming from
1166 * the queue of accepted connections.
1167 */
1168 shutdown_in_progress = 1;
1169
1170 Sleep(1000);
1171
1172 /* Tell the worker threads to exit */
1173 workers_may_exit = 1;
1174
1175 /* Release the start_mutex to let the new process (in the restart
1176 * scenario) a chance to begin accepting and servicing requests
1177 */
1178 rv = apr_proc_mutex_unlock(start_mutex);
1179 if (rv == APR_SUCCESS) {
1180 ap_log_error(APLOG_MARK, APLOG_DEBUG, rv, ap_server_conf, APLOGNO(00359)
1181 "Child: Released the start mutex");
1182 }
1183 else {
1184 ap_log_error(APLOG_MARK, APLOG_ERR, rv, ap_server_conf, APLOGNO(00360)
1185 "Child: Failure releasing the start mutex");
1186 }
1187
1188 /* Shutdown the worker threads
1189 * Post worker threads blocked on the ThreadDispatch IOCompletion port
1190 */
1191 while (g_blocked_threads > 0) {
1192 ap_log_error(APLOG_MARK, APLOG_DEBUG, APR_SUCCESS, ap_server_conf, APLOGNO(00361)
1193 "Child: %d threads blocked on the completion port",
1194 g_blocked_threads);
1195 for (i=g_blocked_threads; i > 0; i--) {
1196 PostQueuedCompletionStatus(ThreadDispatchIOCP, 0,
1197 IOCP_SHUTDOWN, NULL);
1198 }
1199 Sleep(1000);
1200 }
1201 /* Empty the accept queue of completion contexts */
1202 apr_thread_mutex_lock(qlock);
1203 while (qhead) {
1204 CloseHandle(qhead->overlapped.hEvent);
1205 closesocket(qhead->accept_socket);
1206 qhead = qhead->next;
1207 }
1208 apr_thread_mutex_unlock(qlock);
1209
1210 /* Give busy threads a chance to service their connections
1211 * (no more than the global server timeout period which
1212 * we track in msec remaining).
1213 */
1214 watch_thread = 0;
1215 time_remains = (int)(ap_server_conf->timeout / APR_TIME_C(1000));
1216
1217 while (threads_created)
1218 {
1219 int nFailsafe = MAXIMUM_WAIT_OBJECTS;
1220 DWORD dwRet;
1221
1222 /* Every time we roll over to wait on the first group
1223 * of MAXIMUM_WAIT_OBJECTS threads, take a breather,
1224 * and infrequently update the error log.
1225 */
1226 if (watch_thread >= threads_created) {
1227 if ((time_remains -= 100) < 0)
1228 break;
1229
1230 /* Every 30 seconds give an update */
1231 if ((time_remains % 30000) == 0) {
1232 ap_log_error(APLOG_MARK, APLOG_NOTICE, APR_SUCCESS,
1233 ap_server_conf, APLOGNO(00362)
1234 "Child: Waiting %d more seconds "
1235 "for %d worker threads to finish.",
1236 time_remains / 1000, threads_created);
1237 }
1238 /* We'll poll from the top, 10 times per second */
1239 Sleep(100);
1240 watch_thread = 0;
1241 }
1242
1243 /* Fairness, on each iteration we will pick up with the thread
1244 * after the one we just removed, even if it's a single thread.
1245 * We don't block here.
1246 */
1247 dwRet = WaitForMultipleObjects(min(threads_created - watch_thread,
1248 MAXIMUM_WAIT_OBJECTS),
1249 child_handles + watch_thread, 0, 0);
1250
1251 if (dwRet == WAIT_FAILED) {
1252 break;
1253 }
1254 if (dwRet == WAIT_TIMEOUT) {
1255 /* none ready */
1256 watch_thread += MAXIMUM_WAIT_OBJECTS;
1257 continue;
1258 }
1259 else if (dwRet >= WAIT_ABANDONED_0) {
1260 /* We just got the ownership of the object, which
1261 * should happen at most MAXIMUM_WAIT_OBJECTS times.
1262 * It does NOT mean that the object is signaled.
1263 */
1264 if ((nFailsafe--) < 1)
1265 break;
1266 }
1267 else {
1268 watch_thread += (dwRet - WAIT_OBJECT_0);
1269 if (watch_thread >= threads_created)
1270 break;
1271 cleanup_thread(child_handles, &threads_created, watch_thread);
1272 }
1273 }
1274
1275 /* Kill remaining threads off the hard way */
1276 if (threads_created) {
1277 ap_log_error(APLOG_MARK, APLOG_NOTICE, APR_SUCCESS, ap_server_conf, APLOGNO(00363)
1278 "Child: Terminating %d threads that failed to exit.",
1279 threads_created);
1280 }
1281 for (i = 0; i < threads_created; i++) {
1282 int *idx;
1283 TerminateThread(child_handles[i], 1);
1284 CloseHandle(child_handles[i]);
1285 /* Reset the scoreboard entry for the thread we just whacked */
1286 idx = apr_hash_get(ht, &child_handles[i], sizeof(HANDLE));
1287 if (idx) {
1288 ap_update_child_status_from_indexes(0, *idx, SERVER_DEAD, NULL);
1289 }
1290 }
1291 ap_log_error(APLOG_MARK, APLOG_NOTICE, APR_SUCCESS, ap_server_conf, APLOGNO(00364)
1292 "Child: All worker threads have exited.");
1293
1294 apr_thread_mutex_destroy(child_lock);
1295 apr_thread_mutex_destroy(qlock);
1296 CloseHandle(qwait_event);
1297 CloseHandle(ThreadDispatchIOCP);
1298
1299 apr_pool_destroy(pchild);
1300 CloseHandle(exit_event);
1301 if (child_events[2] != NULL) {
1302 CloseHandle(child_events[2]);
1303 }
1304 }
1305
1306 #endif /* def WIN32 */
1307