1
2 /*
3 * Copyright (C) Igor Sysoev
4 * Copyright (C) Nginx, Inc.
5 */
6
7
8 #include <ngx_config.h>
9 #include <ngx_core.h>
10 #include <ngx_event.h>
11
12
13 static ssize_t ngx_linux_sendfile(ngx_connection_t *c, ngx_buf_t *file,
14 size_t size);
15
16 #if (NGX_THREADS)
17 #include <ngx_thread_pool.h>
18
19 #if !(NGX_HAVE_SENDFILE64)
20 #error sendfile64() is required!
21 #endif
22
23 static ssize_t ngx_linux_sendfile_thread(ngx_connection_t *c, ngx_buf_t *file,
24 size_t size);
25 static void ngx_linux_sendfile_thread_handler(void *data, ngx_log_t *log);
26 #endif
27
28
29 /*
30 * On Linux up to 2.4.21 sendfile() (syscall #187) works with 32-bit
31 * offsets only, and the including <sys/sendfile.h> breaks the compiling,
32 * if off_t is 64 bit wide. So we use own sendfile() definition, where offset
33 * parameter is int32_t, and use sendfile() for the file parts below 2G only,
34 * see src/os/unix/ngx_linux_config.h
35 *
36 * Linux 2.4.21 has the new sendfile64() syscall #239.
37 *
38 * On Linux up to 2.6.16 sendfile() does not allow to pass the count parameter
39 * more than 2G-1 bytes even on 64-bit platforms: it returns EINVAL,
40 * so we limit it to 2G-1 bytes.
41 */
42
43 #define NGX_SENDFILE_MAXSIZE 2147483647L
44
45
46 ngx_chain_t *
ngx_linux_sendfile_chain(ngx_connection_t * c,ngx_chain_t * in,off_t limit)47 ngx_linux_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
48 {
49 int tcp_nodelay;
50 off_t send, prev_send;
51 size_t file_size, sent;
52 ssize_t n;
53 ngx_err_t err;
54 ngx_buf_t *file;
55 ngx_event_t *wev;
56 ngx_chain_t *cl;
57 ngx_iovec_t header;
58 struct iovec headers[NGX_IOVS_PREALLOCATE];
59
60 wev = c->write;
61
62 if (!wev->ready) {
63 return in;
64 }
65
66
67 /* the maximum limit size is 2G-1 - the page size */
68
69 if (limit == 0 || limit > (off_t) (NGX_SENDFILE_MAXSIZE - ngx_pagesize)) {
70 limit = NGX_SENDFILE_MAXSIZE - ngx_pagesize;
71 }
72
73
74 send = 0;
75
76 header.iovs = headers;
77 header.nalloc = NGX_IOVS_PREALLOCATE;
78
79 for ( ;; ) {
80 prev_send = send;
81
82 /* create the iovec and coalesce the neighbouring bufs */
83
84 cl = ngx_output_chain_to_iovec(&header, in, limit - send, c->log);
85
86 if (cl == NGX_CHAIN_ERROR) {
87 return NGX_CHAIN_ERROR;
88 }
89
90 send += header.size;
91
92 /* set TCP_CORK if there is a header before a file */
93
94 if (c->tcp_nopush == NGX_TCP_NOPUSH_UNSET
95 && header.count != 0
96 && cl
97 && cl->buf->in_file)
98 {
99 /* the TCP_CORK and TCP_NODELAY are mutually exclusive */
100
101 if (c->tcp_nodelay == NGX_TCP_NODELAY_SET) {
102
103 tcp_nodelay = 0;
104
105 if (setsockopt(c->fd, IPPROTO_TCP, TCP_NODELAY,
106 (const void *) &tcp_nodelay, sizeof(int)) == -1)
107 {
108 err = ngx_socket_errno;
109
110 /*
111 * there is a tiny chance to be interrupted, however,
112 * we continue a processing with the TCP_NODELAY
113 * and without the TCP_CORK
114 */
115
116 if (err != NGX_EINTR) {
117 wev->error = 1;
118 ngx_connection_error(c, err,
119 "setsockopt(TCP_NODELAY) failed");
120 return NGX_CHAIN_ERROR;
121 }
122
123 } else {
124 c->tcp_nodelay = NGX_TCP_NODELAY_UNSET;
125
126 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
127 "no tcp_nodelay");
128 }
129 }
130
131 if (c->tcp_nodelay == NGX_TCP_NODELAY_UNSET) {
132
133 if (ngx_tcp_nopush(c->fd) == -1) {
134 err = ngx_socket_errno;
135
136 /*
137 * there is a tiny chance to be interrupted, however,
138 * we continue a processing without the TCP_CORK
139 */
140
141 if (err != NGX_EINTR) {
142 wev->error = 1;
143 ngx_connection_error(c, err,
144 ngx_tcp_nopush_n " failed");
145 return NGX_CHAIN_ERROR;
146 }
147
148 } else {
149 c->tcp_nopush = NGX_TCP_NOPUSH_SET;
150
151 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
152 "tcp_nopush");
153 }
154 }
155 }
156
157 /* get the file buf */
158
159 if (header.count == 0 && cl && cl->buf->in_file && send < limit) {
160 file = cl->buf;
161
162 /* coalesce the neighbouring file bufs */
163
164 file_size = (size_t) ngx_chain_coalesce_file(&cl, limit - send);
165
166 send += file_size;
167 #if 1
168 if (file_size == 0) {
169 ngx_debug_point();
170 return NGX_CHAIN_ERROR;
171 }
172 #endif
173
174 n = ngx_linux_sendfile(c, file, file_size);
175
176 if (n == NGX_ERROR) {
177 return NGX_CHAIN_ERROR;
178 }
179
180 if (n == NGX_DONE) {
181 /* thread task posted */
182 return in;
183 }
184
185 sent = (n == NGX_AGAIN) ? 0 : n;
186
187 } else {
188 n = ngx_writev(c, &header);
189
190 if (n == NGX_ERROR) {
191 return NGX_CHAIN_ERROR;
192 }
193
194 sent = (n == NGX_AGAIN) ? 0 : n;
195 }
196
197 c->sent += sent;
198
199 in = ngx_chain_update_sent(in, sent);
200
201 if (n == NGX_AGAIN) {
202 wev->ready = 0;
203 return in;
204 }
205
206 if ((size_t) (send - prev_send) != sent) {
207
208 /*
209 * sendfile() on Linux 4.3+ might be interrupted at any time,
210 * and provides no indication if it was interrupted or not,
211 * so we have to retry till an explicit EAGAIN
212 *
213 * sendfile() in threads can also report less bytes written
214 * than we are prepared to send now, since it was started in
215 * some point in the past, so we again have to retry
216 */
217
218 send = prev_send + sent;
219 continue;
220 }
221
222 if (send >= limit || in == NULL) {
223 return in;
224 }
225 }
226 }
227
228
229 static ssize_t
ngx_linux_sendfile(ngx_connection_t * c,ngx_buf_t * file,size_t size)230 ngx_linux_sendfile(ngx_connection_t *c, ngx_buf_t *file, size_t size)
231 {
232 #if (NGX_HAVE_SENDFILE64)
233 off_t offset;
234 #else
235 int32_t offset;
236 #endif
237 ssize_t n;
238 ngx_err_t err;
239
240 #if (NGX_THREADS)
241
242 if (file->file->thread_handler) {
243 return ngx_linux_sendfile_thread(c, file, size);
244 }
245
246 #endif
247
248 #if (NGX_HAVE_SENDFILE64)
249 offset = file->file_pos;
250 #else
251 offset = (int32_t) file->file_pos;
252 #endif
253
254 eintr:
255
256 ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
257 "sendfile: @%O %uz", file->file_pos, size);
258
259 n = sendfile(c->fd, file->file->fd, &offset, size);
260
261 if (n == -1) {
262 err = ngx_errno;
263
264 switch (err) {
265 case NGX_EAGAIN:
266 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, err,
267 "sendfile() is not ready");
268 return NGX_AGAIN;
269
270 case NGX_EINTR:
271 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, err,
272 "sendfile() was interrupted");
273 goto eintr;
274
275 default:
276 c->write->error = 1;
277 ngx_connection_error(c, err, "sendfile() failed");
278 return NGX_ERROR;
279 }
280 }
281
282 if (n == 0) {
283 /*
284 * if sendfile returns zero, then someone has truncated the file,
285 * so the offset became beyond the end of the file
286 */
287
288 ngx_log_error(NGX_LOG_ALERT, c->log, 0,
289 "sendfile() reported that \"%s\" was truncated at %O",
290 file->file->name.data, file->file_pos);
291
292 return NGX_ERROR;
293 }
294
295 ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0, "sendfile: %z of %uz @%O",
296 n, size, file->file_pos);
297
298 return n;
299 }
300
301
302 #if (NGX_THREADS)
303
304 typedef struct {
305 ngx_buf_t *file;
306 ngx_socket_t socket;
307 size_t size;
308
309 size_t sent;
310 ngx_err_t err;
311 } ngx_linux_sendfile_ctx_t;
312
313
314 static ssize_t
ngx_linux_sendfile_thread(ngx_connection_t * c,ngx_buf_t * file,size_t size)315 ngx_linux_sendfile_thread(ngx_connection_t *c, ngx_buf_t *file, size_t size)
316 {
317 ngx_event_t *wev;
318 ngx_thread_task_t *task;
319 ngx_linux_sendfile_ctx_t *ctx;
320
321 ngx_log_debug3(NGX_LOG_DEBUG_CORE, c->log, 0,
322 "linux sendfile thread: %d, %uz, %O",
323 file->file->fd, size, file->file_pos);
324
325 task = c->sendfile_task;
326
327 if (task == NULL) {
328 task = ngx_thread_task_alloc(c->pool, sizeof(ngx_linux_sendfile_ctx_t));
329 if (task == NULL) {
330 return NGX_ERROR;
331 }
332
333 task->handler = ngx_linux_sendfile_thread_handler;
334
335 c->sendfile_task = task;
336 }
337
338 ctx = task->ctx;
339 wev = c->write;
340
341 if (task->event.complete) {
342 task->event.complete = 0;
343
344 if (ctx->err == NGX_EAGAIN) {
345 /*
346 * if wev->complete is set, this means that a write event
347 * happened while we were waiting for the thread task, so
348 * we have to retry sending even on EAGAIN
349 */
350
351 if (wev->complete) {
352 return 0;
353 }
354
355 return NGX_AGAIN;
356 }
357
358 if (ctx->err) {
359 wev->error = 1;
360 ngx_connection_error(c, ctx->err, "sendfile() failed");
361 return NGX_ERROR;
362 }
363
364 if (ctx->sent == 0) {
365 /*
366 * if sendfile returns zero, then someone has truncated the file,
367 * so the offset became beyond the end of the file
368 */
369
370 ngx_log_error(NGX_LOG_ALERT, c->log, 0,
371 "sendfile() reported that \"%s\" was truncated at %O",
372 file->file->name.data, file->file_pos);
373
374 return NGX_ERROR;
375 }
376
377 return ctx->sent;
378 }
379
380 if (task->event.active && ctx->file == file) {
381 /*
382 * tolerate duplicate calls; they can happen due to subrequests
383 * or multiple calls of the next body filter from a filter
384 */
385
386 return NGX_DONE;
387 }
388
389 ctx->file = file;
390 ctx->socket = c->fd;
391 ctx->size = size;
392
393 wev->complete = 0;
394
395 if (file->file->thread_handler(task, file->file) != NGX_OK) {
396 return NGX_ERROR;
397 }
398
399 return NGX_DONE;
400 }
401
402
403 static void
ngx_linux_sendfile_thread_handler(void * data,ngx_log_t * log)404 ngx_linux_sendfile_thread_handler(void *data, ngx_log_t *log)
405 {
406 ngx_linux_sendfile_ctx_t *ctx = data;
407
408 off_t offset;
409 ssize_t n;
410 ngx_buf_t *file;
411
412 ngx_log_debug0(NGX_LOG_DEBUG_CORE, log, 0, "linux sendfile thread handler");
413
414 file = ctx->file;
415 offset = file->file_pos;
416
417 again:
418
419 n = sendfile(ctx->socket, file->file->fd, &offset, ctx->size);
420
421 if (n == -1) {
422 ctx->err = ngx_errno;
423
424 } else {
425 ctx->sent = n;
426 ctx->err = 0;
427 }
428
429 #if 0
430 ngx_time_update();
431 #endif
432
433 ngx_log_debug4(NGX_LOG_DEBUG_EVENT, log, 0,
434 "sendfile: %z (err: %d) of %uz @%O",
435 n, ctx->err, ctx->size, file->file_pos);
436
437 if (ctx->err == NGX_EINTR) {
438 goto again;
439 }
440 }
441
442 #endif /* NGX_THREADS */
443