1
2 /*
3 * Copyright (C) Igor Sysoev
4 * Copyright (C) Nginx, Inc.
5 */
6
7
8 #include <ngx_config.h>
9 #include <ngx_core.h>
10 #include <ngx_event.h>
11
12
13 #if (NGX_TEST_BUILD_EPOLL)
14
15 /* epoll declarations */
16
17 #define EPOLLIN 0x001
18 #define EPOLLPRI 0x002
19 #define EPOLLOUT 0x004
20 #define EPOLLERR 0x008
21 #define EPOLLHUP 0x010
22 #define EPOLLRDNORM 0x040
23 #define EPOLLRDBAND 0x080
24 #define EPOLLWRNORM 0x100
25 #define EPOLLWRBAND 0x200
26 #define EPOLLMSG 0x400
27
28 #define EPOLLRDHUP 0x2000
29
30 #define EPOLLEXCLUSIVE 0x10000000
31 #define EPOLLONESHOT 0x40000000
32 #define EPOLLET 0x80000000
33
34 #define EPOLL_CTL_ADD 1
35 #define EPOLL_CTL_DEL 2
36 #define EPOLL_CTL_MOD 3
37
38 typedef union epoll_data {
39 void *ptr;
40 int fd;
41 uint32_t u32;
42 uint64_t u64;
43 } epoll_data_t;
44
45 struct epoll_event {
46 uint32_t events;
47 epoll_data_t data;
48 };
49
50
51 int epoll_create(int size);
52
epoll_create(int size)53 int epoll_create(int size)
54 {
55 return -1;
56 }
57
58
59 int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event);
60
epoll_ctl(int epfd,int op,int fd,struct epoll_event * event)61 int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event)
62 {
63 return -1;
64 }
65
66
67 int epoll_wait(int epfd, struct epoll_event *events, int nevents, int timeout);
68
epoll_wait(int epfd,struct epoll_event * events,int nevents,int timeout)69 int epoll_wait(int epfd, struct epoll_event *events, int nevents, int timeout)
70 {
71 return -1;
72 }
73
74 #if (NGX_HAVE_EVENTFD)
75 #define SYS_eventfd 323
76 #endif
77
78 #if (NGX_HAVE_FILE_AIO)
79
80 #define SYS_io_setup 245
81 #define SYS_io_destroy 246
82 #define SYS_io_getevents 247
83
84 typedef u_int aio_context_t;
85
86 struct io_event {
87 uint64_t data; /* the data field from the iocb */
88 uint64_t obj; /* what iocb this event came from */
89 int64_t res; /* result code for this event */
90 int64_t res2; /* secondary result */
91 };
92
93
94 #endif
95 #endif /* NGX_TEST_BUILD_EPOLL */
96
97
98 typedef struct {
99 ngx_uint_t events;
100 ngx_uint_t aio_requests;
101 } ngx_epoll_conf_t;
102
103
104 static ngx_int_t ngx_epoll_init(ngx_cycle_t *cycle, ngx_msec_t timer);
105 #if (NGX_HAVE_EVENTFD)
106 static ngx_int_t ngx_epoll_notify_init(ngx_log_t *log);
107 static void ngx_epoll_notify_handler(ngx_event_t *ev);
108 #endif
109 #if (NGX_HAVE_EPOLLRDHUP)
110 static void ngx_epoll_test_rdhup(ngx_cycle_t *cycle);
111 #endif
112 static void ngx_epoll_done(ngx_cycle_t *cycle);
113 static ngx_int_t ngx_epoll_add_event(ngx_event_t *ev, ngx_int_t event,
114 ngx_uint_t flags);
115 static ngx_int_t ngx_epoll_del_event(ngx_event_t *ev, ngx_int_t event,
116 ngx_uint_t flags);
117 static ngx_int_t ngx_epoll_add_connection(ngx_connection_t *c);
118 static ngx_int_t ngx_epoll_del_connection(ngx_connection_t *c,
119 ngx_uint_t flags);
120 #if (NGX_HAVE_EVENTFD)
121 static ngx_int_t ngx_epoll_notify(ngx_event_handler_pt handler);
122 #endif
123 static ngx_int_t ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer,
124 ngx_uint_t flags);
125
126 #if (NGX_HAVE_FILE_AIO)
127 static void ngx_epoll_eventfd_handler(ngx_event_t *ev);
128 #endif
129
130 static void *ngx_epoll_create_conf(ngx_cycle_t *cycle);
131 static char *ngx_epoll_init_conf(ngx_cycle_t *cycle, void *conf);
132
133 static int ep = -1;
134 static struct epoll_event *event_list;
135 static ngx_uint_t nevents;
136
137 #if (NGX_HAVE_EVENTFD)
138 static int notify_fd = -1;
139 static ngx_event_t notify_event;
140 static ngx_connection_t notify_conn;
141 #endif
142
143 #if (NGX_HAVE_FILE_AIO)
144
145 int ngx_eventfd = -1;
146 aio_context_t ngx_aio_ctx = 0;
147
148 static ngx_event_t ngx_eventfd_event;
149 static ngx_connection_t ngx_eventfd_conn;
150
151 #endif
152
153 #if (NGX_HAVE_EPOLLRDHUP)
154 ngx_uint_t ngx_use_epoll_rdhup;
155 #endif
156
157 static ngx_str_t epoll_name = ngx_string("epoll");
158
159 static ngx_command_t ngx_epoll_commands[] = {
160
161 { ngx_string("epoll_events"),
162 NGX_EVENT_CONF|NGX_CONF_TAKE1,
163 ngx_conf_set_num_slot,
164 0,
165 offsetof(ngx_epoll_conf_t, events),
166 NULL },
167
168 { ngx_string("worker_aio_requests"),
169 NGX_EVENT_CONF|NGX_CONF_TAKE1,
170 ngx_conf_set_num_slot,
171 0,
172 offsetof(ngx_epoll_conf_t, aio_requests),
173 NULL },
174
175 ngx_null_command
176 };
177
178
179 static ngx_event_module_t ngx_epoll_module_ctx = {
180 &epoll_name,
181 ngx_epoll_create_conf, /* create configuration */
182 ngx_epoll_init_conf, /* init configuration */
183
184 {
185 ngx_epoll_add_event, /* add an event */
186 ngx_epoll_del_event, /* delete an event */
187 ngx_epoll_add_event, /* enable an event */
188 ngx_epoll_del_event, /* disable an event */
189 ngx_epoll_add_connection, /* add an connection */
190 ngx_epoll_del_connection, /* delete an connection */
191 #if (NGX_HAVE_EVENTFD)
192 ngx_epoll_notify, /* trigger a notify */
193 #else
194 NULL, /* trigger a notify */
195 #endif
196 ngx_epoll_process_events, /* process the events */
197 ngx_epoll_init, /* init the events */
198 ngx_epoll_done, /* done the events */
199 }
200 };
201
202 ngx_module_t ngx_epoll_module = {
203 NGX_MODULE_V1,
204 &ngx_epoll_module_ctx, /* module context */
205 ngx_epoll_commands, /* module directives */
206 NGX_EVENT_MODULE, /* module type */
207 NULL, /* init master */
208 NULL, /* init module */
209 NULL, /* init process */
210 NULL, /* init thread */
211 NULL, /* exit thread */
212 NULL, /* exit process */
213 NULL, /* exit master */
214 NGX_MODULE_V1_PADDING
215 };
216
217
218 #if (NGX_HAVE_FILE_AIO)
219
220 /*
221 * We call io_setup(), io_destroy() io_submit(), and io_getevents() directly
222 * as syscalls instead of libaio usage, because the library header file
223 * supports eventfd() since 0.3.107 version only.
224 */
225
226 static int
io_setup(u_int nr_reqs,aio_context_t * ctx)227 io_setup(u_int nr_reqs, aio_context_t *ctx)
228 {
229 return syscall(SYS_io_setup, nr_reqs, ctx);
230 }
231
232
233 static int
io_destroy(aio_context_t ctx)234 io_destroy(aio_context_t ctx)
235 {
236 return syscall(SYS_io_destroy, ctx);
237 }
238
239
240 static int
io_getevents(aio_context_t ctx,long min_nr,long nr,struct io_event * events,struct timespec * tmo)241 io_getevents(aio_context_t ctx, long min_nr, long nr, struct io_event *events,
242 struct timespec *tmo)
243 {
244 return syscall(SYS_io_getevents, ctx, min_nr, nr, events, tmo);
245 }
246
247
248 static void
ngx_epoll_aio_init(ngx_cycle_t * cycle,ngx_epoll_conf_t * epcf)249 ngx_epoll_aio_init(ngx_cycle_t *cycle, ngx_epoll_conf_t *epcf)
250 {
251 int n;
252 struct epoll_event ee;
253
254 #if (NGX_HAVE_SYS_EVENTFD_H)
255 ngx_eventfd = eventfd(0, 0);
256 #else
257 ngx_eventfd = syscall(SYS_eventfd, 0);
258 #endif
259
260 if (ngx_eventfd == -1) {
261 ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno,
262 "eventfd() failed");
263 ngx_file_aio = 0;
264 return;
265 }
266
267 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
268 "eventfd: %d", ngx_eventfd);
269
270 n = 1;
271
272 if (ioctl(ngx_eventfd, FIONBIO, &n) == -1) {
273 ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno,
274 "ioctl(eventfd, FIONBIO) failed");
275 goto failed;
276 }
277
278 if (io_setup(epcf->aio_requests, &ngx_aio_ctx) == -1) {
279 ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno,
280 "io_setup() failed");
281 goto failed;
282 }
283
284 ngx_eventfd_event.data = &ngx_eventfd_conn;
285 ngx_eventfd_event.handler = ngx_epoll_eventfd_handler;
286 ngx_eventfd_event.log = cycle->log;
287 ngx_eventfd_event.active = 1;
288 ngx_eventfd_conn.fd = ngx_eventfd;
289 ngx_eventfd_conn.read = &ngx_eventfd_event;
290 ngx_eventfd_conn.log = cycle->log;
291
292 ee.events = EPOLLIN|EPOLLET;
293 ee.data.ptr = &ngx_eventfd_conn;
294
295 if (epoll_ctl(ep, EPOLL_CTL_ADD, ngx_eventfd, &ee) != -1) {
296 return;
297 }
298
299 ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno,
300 "epoll_ctl(EPOLL_CTL_ADD, eventfd) failed");
301
302 if (io_destroy(ngx_aio_ctx) == -1) {
303 ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
304 "io_destroy() failed");
305 }
306
307 failed:
308
309 if (close(ngx_eventfd) == -1) {
310 ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
311 "eventfd close() failed");
312 }
313
314 ngx_eventfd = -1;
315 ngx_aio_ctx = 0;
316 ngx_file_aio = 0;
317 }
318
319 #endif
320
321
322 static ngx_int_t
ngx_epoll_init(ngx_cycle_t * cycle,ngx_msec_t timer)323 ngx_epoll_init(ngx_cycle_t *cycle, ngx_msec_t timer)
324 {
325 ngx_epoll_conf_t *epcf;
326
327 epcf = ngx_event_get_conf(cycle->conf_ctx, ngx_epoll_module);
328
329 if (ep == -1) {
330 ep = epoll_create(cycle->connection_n / 2);
331
332 if (ep == -1) {
333 ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno,
334 "epoll_create() failed");
335 return NGX_ERROR;
336 }
337
338 #if (NGX_HAVE_EVENTFD)
339 if (ngx_epoll_notify_init(cycle->log) != NGX_OK) {
340 ngx_epoll_module_ctx.actions.notify = NULL;
341 }
342 #endif
343
344 #if (NGX_HAVE_FILE_AIO)
345 ngx_epoll_aio_init(cycle, epcf);
346 #endif
347
348 #if (NGX_HAVE_EPOLLRDHUP)
349 ngx_epoll_test_rdhup(cycle);
350 #endif
351 }
352
353 if (nevents < epcf->events) {
354 if (event_list) {
355 ngx_free(event_list);
356 }
357
358 event_list = ngx_alloc(sizeof(struct epoll_event) * epcf->events,
359 cycle->log);
360 if (event_list == NULL) {
361 return NGX_ERROR;
362 }
363 }
364
365 nevents = epcf->events;
366
367 ngx_io = ngx_os_io;
368
369 ngx_event_actions = ngx_epoll_module_ctx.actions;
370
371 #if (NGX_HAVE_CLEAR_EVENT)
372 ngx_event_flags = NGX_USE_CLEAR_EVENT
373 #else
374 ngx_event_flags = NGX_USE_LEVEL_EVENT
375 #endif
376 |NGX_USE_GREEDY_EVENT
377 |NGX_USE_EPOLL_EVENT;
378
379 return NGX_OK;
380 }
381
382
383 #if (NGX_HAVE_EVENTFD)
384
385 static ngx_int_t
ngx_epoll_notify_init(ngx_log_t * log)386 ngx_epoll_notify_init(ngx_log_t *log)
387 {
388 struct epoll_event ee;
389
390 #if (NGX_HAVE_SYS_EVENTFD_H)
391 notify_fd = eventfd(0, 0);
392 #else
393 notify_fd = syscall(SYS_eventfd, 0);
394 #endif
395
396 if (notify_fd == -1) {
397 ngx_log_error(NGX_LOG_EMERG, log, ngx_errno, "eventfd() failed");
398 return NGX_ERROR;
399 }
400
401 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, log, 0,
402 "notify eventfd: %d", notify_fd);
403
404 notify_event.handler = ngx_epoll_notify_handler;
405 notify_event.log = log;
406 notify_event.active = 1;
407
408 notify_conn.fd = notify_fd;
409 notify_conn.read = ¬ify_event;
410 notify_conn.log = log;
411
412 ee.events = EPOLLIN|EPOLLET;
413 ee.data.ptr = ¬ify_conn;
414
415 if (epoll_ctl(ep, EPOLL_CTL_ADD, notify_fd, &ee) == -1) {
416 ngx_log_error(NGX_LOG_EMERG, log, ngx_errno,
417 "epoll_ctl(EPOLL_CTL_ADD, eventfd) failed");
418
419 if (close(notify_fd) == -1) {
420 ngx_log_error(NGX_LOG_ALERT, log, ngx_errno,
421 "eventfd close() failed");
422 }
423
424 return NGX_ERROR;
425 }
426
427 return NGX_OK;
428 }
429
430
431 static void
ngx_epoll_notify_handler(ngx_event_t * ev)432 ngx_epoll_notify_handler(ngx_event_t *ev)
433 {
434 ssize_t n;
435 uint64_t count;
436 ngx_err_t err;
437 ngx_event_handler_pt handler;
438
439 if (++ev->index == NGX_MAX_UINT32_VALUE) {
440 ev->index = 0;
441
442 n = read(notify_fd, &count, sizeof(uint64_t));
443
444 err = ngx_errno;
445
446 ngx_log_debug3(NGX_LOG_DEBUG_EVENT, ev->log, 0,
447 "read() eventfd %d: %z count:%uL", notify_fd, n, count);
448
449 if ((size_t) n != sizeof(uint64_t)) {
450 ngx_log_error(NGX_LOG_ALERT, ev->log, err,
451 "read() eventfd %d failed", notify_fd);
452 }
453 }
454
455 handler = ev->data;
456 handler(ev);
457 }
458
459 #endif
460
461
462 #if (NGX_HAVE_EPOLLRDHUP)
463
464 static void
ngx_epoll_test_rdhup(ngx_cycle_t * cycle)465 ngx_epoll_test_rdhup(ngx_cycle_t *cycle)
466 {
467 int s[2], events;
468 struct epoll_event ee;
469
470 if (socketpair(AF_UNIX, SOCK_STREAM, 0, s) == -1) {
471 ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
472 "socketpair() failed");
473 return;
474 }
475
476 ee.events = EPOLLET|EPOLLIN|EPOLLRDHUP;
477
478 if (epoll_ctl(ep, EPOLL_CTL_ADD, s[0], &ee) == -1) {
479 ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
480 "epoll_ctl() failed");
481 goto failed;
482 }
483
484 if (close(s[1]) == -1) {
485 ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
486 "close() failed");
487 s[1] = -1;
488 goto failed;
489 }
490
491 s[1] = -1;
492
493 events = epoll_wait(ep, &ee, 1, 5000);
494
495 if (events == -1) {
496 ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
497 "epoll_wait() failed");
498 goto failed;
499 }
500
501 if (events) {
502 ngx_use_epoll_rdhup = ee.events & EPOLLRDHUP;
503
504 } else {
505 ngx_log_error(NGX_LOG_ALERT, cycle->log, NGX_ETIMEDOUT,
506 "epoll_wait() timed out");
507 }
508
509 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
510 "testing the EPOLLRDHUP flag: %s",
511 ngx_use_epoll_rdhup ? "success" : "fail");
512
513 failed:
514
515 if (s[1] != -1 && close(s[1]) == -1) {
516 ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
517 "close() failed");
518 }
519
520 if (close(s[0]) == -1) {
521 ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
522 "close() failed");
523 }
524 }
525
526 #endif
527
528
529 static void
ngx_epoll_done(ngx_cycle_t * cycle)530 ngx_epoll_done(ngx_cycle_t *cycle)
531 {
532 if (close(ep) == -1) {
533 ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
534 "epoll close() failed");
535 }
536
537 ep = -1;
538
539 #if (NGX_HAVE_EVENTFD)
540
541 if (close(notify_fd) == -1) {
542 ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
543 "eventfd close() failed");
544 }
545
546 notify_fd = -1;
547
548 #endif
549
550 #if (NGX_HAVE_FILE_AIO)
551
552 if (ngx_eventfd != -1) {
553
554 if (io_destroy(ngx_aio_ctx) == -1) {
555 ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
556 "io_destroy() failed");
557 }
558
559 if (close(ngx_eventfd) == -1) {
560 ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
561 "eventfd close() failed");
562 }
563
564 ngx_eventfd = -1;
565 }
566
567 ngx_aio_ctx = 0;
568
569 #endif
570
571 ngx_free(event_list);
572
573 event_list = NULL;
574 nevents = 0;
575 }
576
577
578 static ngx_int_t
ngx_epoll_add_event(ngx_event_t * ev,ngx_int_t event,ngx_uint_t flags)579 ngx_epoll_add_event(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags)
580 {
581 int op;
582 uint32_t events, prev;
583 ngx_event_t *e;
584 ngx_connection_t *c;
585 struct epoll_event ee;
586
587 c = ev->data;
588
589 events = (uint32_t) event;
590
591 if (event == NGX_READ_EVENT) {
592 e = c->write;
593 prev = EPOLLOUT;
594 #if (NGX_READ_EVENT != EPOLLIN|EPOLLRDHUP)
595 events = EPOLLIN|EPOLLRDHUP;
596 #endif
597
598 } else {
599 e = c->read;
600 prev = EPOLLIN|EPOLLRDHUP;
601 #if (NGX_WRITE_EVENT != EPOLLOUT)
602 events = EPOLLOUT;
603 #endif
604 }
605
606 if (e->active) {
607 op = EPOLL_CTL_MOD;
608 events |= prev;
609
610 } else {
611 op = EPOLL_CTL_ADD;
612 }
613
614 #if (NGX_HAVE_EPOLLEXCLUSIVE && NGX_HAVE_EPOLLRDHUP)
615 if (flags & NGX_EXCLUSIVE_EVENT) {
616 events &= ~EPOLLRDHUP;
617 }
618 #endif
619
620 ee.events = events | (uint32_t) flags;
621 ee.data.ptr = (void *) ((uintptr_t) c | ev->instance);
622
623 ngx_log_debug3(NGX_LOG_DEBUG_EVENT, ev->log, 0,
624 "epoll add event: fd:%d op:%d ev:%08XD",
625 c->fd, op, ee.events);
626
627 if (epoll_ctl(ep, op, c->fd, &ee) == -1) {
628 ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_errno,
629 "epoll_ctl(%d, %d) failed", op, c->fd);
630 return NGX_ERROR;
631 }
632
633 ev->active = 1;
634 #if 0
635 ev->oneshot = (flags & NGX_ONESHOT_EVENT) ? 1 : 0;
636 #endif
637
638 return NGX_OK;
639 }
640
641
642 static ngx_int_t
ngx_epoll_del_event(ngx_event_t * ev,ngx_int_t event,ngx_uint_t flags)643 ngx_epoll_del_event(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags)
644 {
645 int op;
646 uint32_t prev;
647 ngx_event_t *e;
648 ngx_connection_t *c;
649 struct epoll_event ee;
650
651 /*
652 * when the file descriptor is closed, the epoll automatically deletes
653 * it from its queue, so we do not need to delete explicitly the event
654 * before the closing the file descriptor
655 */
656
657 if (flags & NGX_CLOSE_EVENT) {
658 ev->active = 0;
659 return NGX_OK;
660 }
661
662 c = ev->data;
663
664 if (event == NGX_READ_EVENT) {
665 e = c->write;
666 prev = EPOLLOUT;
667
668 } else {
669 e = c->read;
670 prev = EPOLLIN|EPOLLRDHUP;
671 }
672
673 if (e->active) {
674 op = EPOLL_CTL_MOD;
675 ee.events = prev | (uint32_t) flags;
676 ee.data.ptr = (void *) ((uintptr_t) c | ev->instance);
677
678 } else {
679 op = EPOLL_CTL_DEL;
680 ee.events = 0;
681 ee.data.ptr = NULL;
682 }
683
684 ngx_log_debug3(NGX_LOG_DEBUG_EVENT, ev->log, 0,
685 "epoll del event: fd:%d op:%d ev:%08XD",
686 c->fd, op, ee.events);
687
688 if (epoll_ctl(ep, op, c->fd, &ee) == -1) {
689 ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_errno,
690 "epoll_ctl(%d, %d) failed", op, c->fd);
691 return NGX_ERROR;
692 }
693
694 ev->active = 0;
695
696 return NGX_OK;
697 }
698
699
700 static ngx_int_t
ngx_epoll_add_connection(ngx_connection_t * c)701 ngx_epoll_add_connection(ngx_connection_t *c)
702 {
703 struct epoll_event ee;
704
705 ee.events = EPOLLIN|EPOLLOUT|EPOLLET|EPOLLRDHUP;
706 ee.data.ptr = (void *) ((uintptr_t) c | c->read->instance);
707
708 ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
709 "epoll add connection: fd:%d ev:%08XD", c->fd, ee.events);
710
711 if (epoll_ctl(ep, EPOLL_CTL_ADD, c->fd, &ee) == -1) {
712 ngx_log_error(NGX_LOG_ALERT, c->log, ngx_errno,
713 "epoll_ctl(EPOLL_CTL_ADD, %d) failed", c->fd);
714 return NGX_ERROR;
715 }
716
717 c->read->active = 1;
718 c->write->active = 1;
719
720 return NGX_OK;
721 }
722
723
724 static ngx_int_t
ngx_epoll_del_connection(ngx_connection_t * c,ngx_uint_t flags)725 ngx_epoll_del_connection(ngx_connection_t *c, ngx_uint_t flags)
726 {
727 int op;
728 struct epoll_event ee;
729
730 /*
731 * when the file descriptor is closed the epoll automatically deletes
732 * it from its queue so we do not need to delete explicitly the event
733 * before the closing the file descriptor
734 */
735
736 if (flags & NGX_CLOSE_EVENT) {
737 c->read->active = 0;
738 c->write->active = 0;
739 return NGX_OK;
740 }
741
742 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
743 "epoll del connection: fd:%d", c->fd);
744
745 op = EPOLL_CTL_DEL;
746 ee.events = 0;
747 ee.data.ptr = NULL;
748
749 if (epoll_ctl(ep, op, c->fd, &ee) == -1) {
750 ngx_log_error(NGX_LOG_ALERT, c->log, ngx_errno,
751 "epoll_ctl(%d, %d) failed", op, c->fd);
752 return NGX_ERROR;
753 }
754
755 c->read->active = 0;
756 c->write->active = 0;
757
758 return NGX_OK;
759 }
760
761
762 #if (NGX_HAVE_EVENTFD)
763
764 static ngx_int_t
ngx_epoll_notify(ngx_event_handler_pt handler)765 ngx_epoll_notify(ngx_event_handler_pt handler)
766 {
767 static uint64_t inc = 1;
768
769 notify_event.data = handler;
770
771 if ((size_t) write(notify_fd, &inc, sizeof(uint64_t)) != sizeof(uint64_t)) {
772 ngx_log_error(NGX_LOG_ALERT, notify_event.log, ngx_errno,
773 "write() to eventfd %d failed", notify_fd);
774 return NGX_ERROR;
775 }
776
777 return NGX_OK;
778 }
779
780 #endif
781
782
783 static ngx_int_t
ngx_epoll_process_events(ngx_cycle_t * cycle,ngx_msec_t timer,ngx_uint_t flags)784 ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags)
785 {
786 int events;
787 uint32_t revents;
788 ngx_int_t instance, i;
789 ngx_uint_t level;
790 ngx_err_t err;
791 ngx_event_t *rev, *wev;
792 ngx_queue_t *queue;
793 ngx_connection_t *c;
794
795 /* NGX_TIMER_INFINITE == INFTIM */
796
797 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
798 "epoll timer: %M", timer);
799
800 events = epoll_wait(ep, event_list, (int) nevents, timer);
801
802 err = (events == -1) ? ngx_errno : 0;
803
804 if (flags & NGX_UPDATE_TIME || ngx_event_timer_alarm) {
805 ngx_time_update();
806 }
807
808 if (err) {
809 if (err == NGX_EINTR) {
810
811 if (ngx_event_timer_alarm) {
812 ngx_event_timer_alarm = 0;
813 return NGX_OK;
814 }
815
816 level = NGX_LOG_INFO;
817
818 } else {
819 level = NGX_LOG_ALERT;
820 }
821
822 ngx_log_error(level, cycle->log, err, "epoll_wait() failed");
823 return NGX_ERROR;
824 }
825
826 if (events == 0) {
827 if (timer != NGX_TIMER_INFINITE) {
828 return NGX_OK;
829 }
830
831 ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
832 "epoll_wait() returned no events without timeout");
833 return NGX_ERROR;
834 }
835
836 for (i = 0; i < events; i++) {
837 c = event_list[i].data.ptr;
838
839 instance = (uintptr_t) c & 1;
840 c = (ngx_connection_t *) ((uintptr_t) c & (uintptr_t) ~1);
841
842 rev = c->read;
843
844 if (c->fd == -1 || rev->instance != instance) {
845
846 /*
847 * the stale event from a file descriptor
848 * that was just closed in this iteration
849 */
850
851 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
852 "epoll: stale event %p", c);
853 continue;
854 }
855
856 revents = event_list[i].events;
857
858 ngx_log_debug3(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
859 "epoll: fd:%d ev:%04XD d:%p",
860 c->fd, revents, event_list[i].data.ptr);
861
862 if (revents & (EPOLLERR|EPOLLHUP)) {
863 ngx_log_debug2(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
864 "epoll_wait() error on fd:%d ev:%04XD",
865 c->fd, revents);
866
867 /*
868 * if the error events were returned, add EPOLLIN and EPOLLOUT
869 * to handle the events at least in one active handler
870 */
871
872 revents |= EPOLLIN|EPOLLOUT;
873 }
874
875 #if 0
876 if (revents & ~(EPOLLIN|EPOLLOUT|EPOLLERR|EPOLLHUP)) {
877 ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
878 "strange epoll_wait() events fd:%d ev:%04XD",
879 c->fd, revents);
880 }
881 #endif
882
883 if ((revents & EPOLLIN) && rev->active) {
884
885 #if (NGX_HAVE_EPOLLRDHUP)
886 if (revents & EPOLLRDHUP) {
887 rev->pending_eof = 1;
888 }
889 #endif
890
891 rev->ready = 1;
892 rev->available = -1;
893
894 if (flags & NGX_POST_EVENTS) {
895 queue = rev->accept ? &ngx_posted_accept_events
896 : &ngx_posted_events;
897
898 ngx_post_event(rev, queue);
899
900 } else {
901 rev->handler(rev);
902 }
903 }
904
905 wev = c->write;
906
907 if ((revents & EPOLLOUT) && wev->active) {
908
909 if (c->fd == -1 || wev->instance != instance) {
910
911 /*
912 * the stale event from a file descriptor
913 * that was just closed in this iteration
914 */
915
916 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
917 "epoll: stale event %p", c);
918 continue;
919 }
920
921 wev->ready = 1;
922 #if (NGX_THREADS)
923 wev->complete = 1;
924 #endif
925
926 if (flags & NGX_POST_EVENTS) {
927 ngx_post_event(wev, &ngx_posted_events);
928
929 } else {
930 wev->handler(wev);
931 }
932 }
933 }
934
935 return NGX_OK;
936 }
937
938
939 #if (NGX_HAVE_FILE_AIO)
940
941 static void
ngx_epoll_eventfd_handler(ngx_event_t * ev)942 ngx_epoll_eventfd_handler(ngx_event_t *ev)
943 {
944 int n, events;
945 long i;
946 uint64_t ready;
947 ngx_err_t err;
948 ngx_event_t *e;
949 ngx_event_aio_t *aio;
950 struct io_event event[64];
951 struct timespec ts;
952
953 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ev->log, 0, "eventfd handler");
954
955 n = read(ngx_eventfd, &ready, 8);
956
957 err = ngx_errno;
958
959 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, ev->log, 0, "eventfd: %d", n);
960
961 if (n != 8) {
962 if (n == -1) {
963 if (err == NGX_EAGAIN) {
964 return;
965 }
966
967 ngx_log_error(NGX_LOG_ALERT, ev->log, err, "read(eventfd) failed");
968 return;
969 }
970
971 ngx_log_error(NGX_LOG_ALERT, ev->log, 0,
972 "read(eventfd) returned only %d bytes", n);
973 return;
974 }
975
976 ts.tv_sec = 0;
977 ts.tv_nsec = 0;
978
979 while (ready) {
980
981 events = io_getevents(ngx_aio_ctx, 1, 64, event, &ts);
982
983 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, ev->log, 0,
984 "io_getevents: %d", events);
985
986 if (events > 0) {
987 ready -= events;
988
989 for (i = 0; i < events; i++) {
990
991 ngx_log_debug4(NGX_LOG_DEBUG_EVENT, ev->log, 0,
992 "io_event: %XL %XL %L %L",
993 event[i].data, event[i].obj,
994 event[i].res, event[i].res2);
995
996 e = (ngx_event_t *) (uintptr_t) event[i].data;
997
998 e->complete = 1;
999 e->active = 0;
1000 e->ready = 1;
1001
1002 aio = e->data;
1003 aio->res = event[i].res;
1004
1005 ngx_post_event(e, &ngx_posted_events);
1006 }
1007
1008 continue;
1009 }
1010
1011 if (events == 0) {
1012 return;
1013 }
1014
1015 /* events == -1 */
1016 ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_errno,
1017 "io_getevents() failed");
1018 return;
1019 }
1020 }
1021
1022 #endif
1023
1024
1025 static void *
ngx_epoll_create_conf(ngx_cycle_t * cycle)1026 ngx_epoll_create_conf(ngx_cycle_t *cycle)
1027 {
1028 ngx_epoll_conf_t *epcf;
1029
1030 epcf = ngx_palloc(cycle->pool, sizeof(ngx_epoll_conf_t));
1031 if (epcf == NULL) {
1032 return NULL;
1033 }
1034
1035 epcf->events = NGX_CONF_UNSET;
1036 epcf->aio_requests = NGX_CONF_UNSET;
1037
1038 return epcf;
1039 }
1040
1041
1042 static char *
ngx_epoll_init_conf(ngx_cycle_t * cycle,void * conf)1043 ngx_epoll_init_conf(ngx_cycle_t *cycle, void *conf)
1044 {
1045 ngx_epoll_conf_t *epcf = conf;
1046
1047 ngx_conf_init_uint_value(epcf->events, 512);
1048 ngx_conf_init_uint_value(epcf->aio_requests, 32);
1049
1050 return NGX_CONF_OK;
1051 }
1052