1 /*
2 HTTP request/response handling
3 Copyright (C) 1999-2010, Joe Orton <joe@manyfish.co.uk>
4
5 This library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Library General Public
7 License as published by the Free Software Foundation; either
8 version 2 of the License, or (at your option) any later version.
9
10 This library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Library General Public License for more details.
14
15 You should have received a copy of the GNU Library General Public
16 License along with this library; if not, write to the Free
17 Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
18 MA 02111-1307, USA
19
20 */
21
22 /* This is the HTTP client request/response implementation.
23 * The goal of this code is to be modular and simple.
24 */
25
26 #include "config.h"
27
28 #include <time.h>
29 #include <sys/types.h>
30
31 #include <errno.h>
32 #include <fcntl.h>
33
34
35 #ifdef HAVE_STRING_H
36 #include <string.h>
37 #endif
38 #ifdef HAVE_STRINGS_H
39 #include <strings.h>
40 #endif
41 #ifdef HAVE_STDLIB_H
42 #include <stdlib.h>
43 #endif
44 #ifdef HAVE_UNISTD_H
45 #include <unistd.h>
46 #endif
47
48 #include "ne_internal.h"
49
50 #include "ne_alloc.h"
51 #include "ne_request.h"
52 #include "ne_string.h" /* for ne_buffer */
53 #include "ne_utils.h"
54 #include "ne_socket.h"
55 #include "ne_uri.h"
56
57
58 #include "ne_private.h"
59
60 #define SOCK_ERR(req, op, msg) do { ssize_t sret = (op); \
61 if (sret < 0) return aborted(req, msg, sret); } while (0)
62
63 #define EOL "\r\n"
64
65 struct body_reader {
66 ne_block_reader handler;
67 ne_accept_response accept_response;
68 unsigned int use;
69 void *userdata;
70 struct body_reader *next;
71 };
72
73 struct field {
74 char *name, *value;
75 size_t vlen;
76 struct field *next;
77 };
78
79 /* Maximum number of header fields per response: */
80 #define MAX_HEADER_FIELDS (100)
81 /* Size of hash table; 43 is the smallest prime for which the common
82 * header names hash uniquely using the *33 hash function. */
83 #define HH_HASHSIZE (43)
84 /* Hash iteration step: *33 known to be a good hash for ASCII, see RSE. */
85 #define HH_ITERATE(hash, ch) (((hash)*33 + (unsigned char)(ch)) % HH_HASHSIZE)
86
87 /* pre-calculated hash values for given header names: */
88 #define HH_HV_CONNECTION (0x14)
89 #define HH_HV_PROXY_CONNECTION (0x1A)
90 #define HH_HV_CONTENT_LENGTH (0x13)
91 #define HH_HV_TRANSFER_ENCODING (0x07)
92
93 struct ne_request_s {
94 char *method, *uri; /* method and Request-URI */
95
96 ne_buffer *headers; /* request headers */
97
98 /* Request body. */
99 ne_provide_body body_cb;
100 void *body_ud;
101
102 /* Request body source: file or buffer (if not callback). */
103 union {
104 struct {
105 int fd;
106 ne_off_t offset, length;
107 ne_off_t remain; /* remaining bytes to send. */
108 } file;
109 struct {
110 /* length bytes @ buffer = whole body.
111 * remain bytes @ pnt = remaining bytes to send */
112 const char *buffer, *pnt;
113 size_t length, remain;
114 } buf;
115 } body;
116
117 ne_off_t body_length; /* length of request body */
118
119 /* temporary store for response lines. */
120 char respbuf[NE_BUFSIZ];
121
122 /**** Response ***/
123
124 /* The transfer encoding types */
125 struct ne_response {
126 enum {
127 R_TILLEOF = 0, /* read till eof */
128 R_NO_BODY, /* implicitly no body (HEAD, 204, 304) */
129 R_CHUNKED, /* using chunked transfer-encoding */
130 R_CLENGTH /* using given content-length */
131 } mode;
132 union {
133 /* clen: used if mode == R_CLENGTH; total and bytes
134 * remaining to be read of response body. */
135 struct {
136 ne_off_t total, remain;
137 } clen;
138 /* chunk: used if mode == R_CHUNKED; total and bytes
139 * remaining to be read of current chunk */
140 struct {
141 size_t total, remain;
142 } chunk;
143 } body;
144 ne_off_t progress; /* number of bytes read of response */
145 } resp;
146
147 struct hook *private;
148
149 /* response header fields */
150 struct field *response_headers[HH_HASHSIZE];
151
152 unsigned int current_index; /* response_headers cursor for iterator */
153
154 /* List of callbacks which are passed response body blocks */
155 struct body_reader *body_readers;
156
157 /*** Miscellaneous ***/
158 unsigned int method_is_head;
159 unsigned int can_persist;
160
161 struct timespec expiration_time;
162
163 int flags[NE_REQFLAG_LAST];
164
165 ne_session *session;
166 ne_status status;
167 };
168
169 static int open_connection(ne_session *sess);
170
171 /* Returns hash value for header 'name', converting it to lower-case
172 * in-place. */
hash_and_lower(char * name)173 static inline unsigned int hash_and_lower(char *name)
174 {
175 char *pnt;
176 unsigned int hash = 0;
177
178 for (pnt = name; *pnt != '\0'; pnt++) {
179 *pnt = ne_tolower(*pnt);
180 hash = HH_ITERATE(hash,*pnt);
181 }
182
183 return hash;
184 }
185
186 /* Abort a request due to an non-recoverable HTTP protocol error,
187 * whilst doing 'doing'. 'code', if non-zero, is the socket error
188 * code, NE_SOCK_*, or if zero, is ignored. */
aborted(ne_request * req,const char * doing,ssize_t code)189 static int aborted(ne_request *req, const char *doing, ssize_t code)
190 {
191 ne_session *sess = req->session;
192 int ret = NE_ERROR;
193
194 NE_DEBUG(NE_DBG_HTTP, "Aborted request (%" NE_FMT_SSIZE_T "): %s",
195 code, doing);
196
197 switch(code) {
198 case NE_SOCK_CLOSED:
199 if (sess->nexthop->proxy != PROXY_NONE) {
200 ne_set_error(sess, _("%s: connection was closed by proxy server"),
201 doing);
202 } else {
203 ne_set_error(sess, _("%s: connection was closed by server"),
204 doing);
205 }
206 break;
207 case NE_SOCK_TIMEOUT:
208 ne_set_error(sess, _("%s: connection timed out"), doing);
209 ret = NE_TIMEOUT;
210 break;
211 case NE_SOCK_ERROR:
212 case NE_SOCK_RESET:
213 case NE_SOCK_TRUNC:
214 ne_set_error(sess, "%s: %s", doing, ne_sock_error(sess->socket));
215 break;
216 case 0:
217 ne_set_error(sess, "%s", doing);
218 break;
219 }
220
221 ne_close_connection(sess);
222 return ret;
223 }
224
notify_status(ne_session * sess,ne_session_status status)225 static void notify_status(ne_session *sess, ne_session_status status)
226 {
227 if (sess->notify_cb) {
228 sess->notify_cb(sess->notify_ud, status, &sess->status);
229 }
230 }
231
get_private(const struct hook * hk,const char * id)232 static void *get_private(const struct hook *hk, const char *id)
233 {
234 for (; hk != NULL; hk = hk->next)
235 if (strcmp(hk->id, id) == 0)
236 return hk->userdata;
237 return NULL;
238 }
239
ne_get_request_private(ne_request * req,const char * id)240 void *ne_get_request_private(ne_request *req, const char *id)
241 {
242 return get_private(req->private, id);
243 }
244
ne_get_session_private(ne_session * sess,const char * id)245 void *ne_get_session_private(ne_session *sess, const char *id)
246 {
247 return get_private(sess->private, id);
248 }
249
ne_set_request_private(ne_request * req,const char * id,void * userdata)250 void ne_set_request_private(ne_request *req, const char *id, void *userdata)
251 {
252 struct hook *hk = ne_malloc(sizeof (struct hook)), *pos;
253
254 if (req->private != NULL) {
255 for (pos = req->private; pos->next != NULL; pos = pos->next)
256 /* nullop */;
257 pos->next = hk;
258 } else {
259 req->private = hk;
260 }
261
262 hk->id = id;
263 hk->fn = NULL;
264 hk->userdata = userdata;
265 hk->next = NULL;
266 }
267
body_string_send(void * userdata,char * buffer,size_t count)268 static ssize_t body_string_send(void *userdata, char *buffer, size_t count)
269 {
270 ne_request *req = userdata;
271
272 if (count == 0) {
273 req->body.buf.remain = req->body.buf.length;
274 req->body.buf.pnt = req->body.buf.buffer;
275 } else {
276 /* if body_left == 0 we fall through and return 0. */
277 if (req->body.buf.remain < count)
278 count = req->body.buf.remain;
279
280 memcpy(buffer, req->body.buf.pnt, count);
281 req->body.buf.pnt += count;
282 req->body.buf.remain -= count;
283 }
284
285 return count;
286 }
287
body_fd_send(void * userdata,char * buffer,size_t count)288 static ssize_t body_fd_send(void *userdata, char *buffer, size_t count)
289 {
290 ne_request *req = userdata;
291
292 if (count) {
293 ssize_t ret;
294
295 if (req->body.file.remain == 0)
296 return 0;
297
298 /* Casts here are necessary for LFS platforms for safe and
299 * warning-free assignment/comparison between 32-bit size_t
300 * and 64-bit off64_t: */
301 if ((ne_off_t)count > req->body.file.remain)
302 count = (size_t)req->body.file.remain;
303
304 ret = read(req->body.file.fd, buffer, count);
305 if (ret > 0) {
306 req->body.file.remain -= ret;
307 return ret;
308 }
309 else if (ret == 0) {
310 ne_set_error(req->session,
311 _("Premature EOF in request body file"));
312 }
313 else if (ret < 0) {
314 char err[200];
315 int errnum = errno;
316
317 ne_set_error(req->session,
318 _("Failed reading request body file: %s"),
319 ne_strerror(errnum, err, sizeof err));
320 }
321
322 return -1;
323 } else {
324 ne_off_t newoff;
325
326 /* rewind for next send. */
327 newoff = ne_lseek(req->body.file.fd, req->body.file.offset, SEEK_SET);
328 if (newoff == req->body.file.offset) {
329 req->body.file.remain = req->body.file.length;
330 return 0;
331 } else {
332 char err[200], offstr[20];
333
334 if (newoff == -1) {
335 /* errno was set */
336 ne_strerror(errno, err, sizeof err);
337 } else {
338 strcpy(err, _("offset invalid"));
339 }
340 ne_snprintf(offstr, sizeof offstr, "%" FMT_NE_OFF_T,
341 req->body.file.offset);
342 ne_set_error(req->session,
343 _("Could not seek to offset %s"
344 " of request body file: %s"),
345 offstr, err);
346 return -1;
347 }
348 }
349 }
350
351 /* For accurate persistent connection handling, for any write() or
352 * read() operation for a new request on an already-open connection,
353 * an EOF or RST error MUST be treated as a persistent connection
354 * timeout, and the request retried on a new connection. Once a
355 * read() operation has succeeded, any subsequent error MUST be
356 * treated as fatal. A 'retry' flag is used; retry=1 represents the
357 * first case, retry=0 the latter. */
358
359 /* RETRY_RET() crafts a function return value given the 'retry' flag,
360 * the socket error 'code', and the return value 'acode' from the
361 * aborted() function. */
362 #define RETRY_RET(retry, code, acode) \
363 ((((code) == NE_SOCK_CLOSED || (code) == NE_SOCK_RESET || \
364 (code) == NE_SOCK_TRUNC) && retry) ? NE_RETRY : (acode))
365
366 /* Sends the request body; returns 0 on success or an NE_* error code.
367 * If retry is non-zero; will return NE_RETRY on persistent connection
368 * timeout. On error, the session error string is set and the
369 * connection is closed. */
send_request_body(ne_request * req,int retry)370 static int send_request_body(ne_request *req, int retry)
371 {
372 ne_session *const sess = req->session;
373 char buffer[NE_BUFSIZ_LARGE];
374 ssize_t bytes;
375
376 NE_DEBUG(NE_DBG_CORE, "Sending request body:");
377
378 req->session->status.sr.progress = 0;
379 req->session->status.sr.total = req->body_length;
380 notify_status(sess, ne_status_sending);
381
382 /* tell the source to start again from the beginning. */
383 if (req->body_cb(req->body_ud, NULL, 0) != 0) {
384 ne_close_connection(sess);
385 return NE_ERROR;
386 }
387
388 while ((bytes = req->body_cb(req->body_ud, buffer, sizeof buffer)) > 0) {
389 int ret = ne_sock_fullwrite(sess->socket, buffer, bytes);
390 if (ret < 0) {
391 int aret = aborted(req, _("Could not send request body"), ret);
392 return RETRY_RET(retry, ret, aret);
393 }
394
395 NE_DEBUG(NE_DBG_HTTPBODY,
396 "Body block (%" NE_FMT_SSIZE_T " bytes):\n[%.*s]\n",
397 bytes, (int)bytes, buffer);
398
399 /* invoke progress callback */
400 req->session->status.sr.progress += bytes;
401 notify_status(sess, ne_status_sending);
402 }
403
404 if (bytes == 0) {
405 NE_DEBUG(NE_DBG_CORE, "Request body sent successfully");
406 return NE_OK;
407 } else {
408 NE_DEBUG(NE_DBG_HTTP, "Request body provider failed with "
409 "%" NE_FMT_SSIZE_T "\n", bytes);
410 ne_close_connection(sess);
411 return NE_ERROR;
412 }
413 }
414
415 /* Lob the User-Agent, connection and host headers in to the request
416 * headers */
add_fixed_headers(ne_request * req)417 static void add_fixed_headers(ne_request *req)
418 {
419 ne_session *const sess = req->session;
420
421 if (sess->user_agent) {
422 ne_buffer_zappend(req->headers, sess->user_agent);
423 }
424
425 /* If persistent connections are disabled, just send Connection:
426 * close; otherwise, send Connection: Keep-Alive to pre-1.1 origin
427 * servers to try harder to get a persistent connection, except if
428 * using a proxy as per 2068§19.7.1. Always add TE: trailers. */
429 if (!sess->flags[NE_SESSFLAG_PERSIST]) {
430 ne_buffer_czappend(req->headers, "Connection: close" EOL);
431 }
432 else if (!sess->is_http11 && !sess->any_proxy_http) {
433 ne_buffer_czappend(req->headers,
434 "Keep-Alive: " EOL
435 "Connection: Keep-Alive" EOL);
436 }
437 else if (!req->session->is_http11 && !sess->any_proxy_http) {
438 ne_buffer_czappend(req->headers,
439 "Keep-Alive: " EOL
440 "Proxy-Connection: Keep-Alive" EOL);
441 }
442
443 ne_buffer_concat(req->headers, "TE: trailers" EOL "Host: ",
444 req->session->server.hostport, EOL, NULL);
445 }
446
ne_accept_always(void * userdata,ne_request * req,const ne_status * st)447 int ne_accept_always(void *userdata, ne_request *req, const ne_status *st)
448 {
449 return 1;
450 }
451
ne_accept_2xx(void * userdata,ne_request * req,const ne_status * st)452 int ne_accept_2xx(void *userdata, ne_request *req, const ne_status *st)
453 {
454 return (st->klass == 2);
455 }
456
ne_request_create(ne_session * sess,const char * method,const char * path)457 ne_request *ne_request_create(ne_session *sess,
458 const char *method, const char *path)
459 {
460 ne_request *req = ne_calloc(sizeof *req);
461
462 req->session = sess;
463 req->headers = ne_buffer_create();
464
465 /* Presume the method is idempotent by default. */
466 req->flags[NE_REQFLAG_IDEMPOTENT] = 1;
467 /* Expect-100 default follows the corresponding session flag. */
468 req->flags[NE_REQFLAG_EXPECT100] = sess->flags[NE_SESSFLAG_EXPECT100];
469
470 /* Add in the fixed headers */
471 add_fixed_headers(req);
472
473 /* Set the standard stuff */
474 req->method = ne_strdup(method);
475 req->method_is_head = (strcmp(method, "HEAD") == 0);
476
477 /* Only use an absoluteURI here when we might be using an HTTP
478 * proxy, and SSL is in use: some servers can't parse them. */
479 if (sess->any_proxy_http && !req->session->use_ssl && path[0] == '/')
480 req->uri = ne_concat(req->session->scheme, "://",
481 req->session->server.hostport, path, NULL);
482 else
483 req->uri = ne_strdup(path);
484
485 {
486 struct hook *hk;
487
488 for (hk = sess->create_req_hooks; hk != NULL; hk = hk->next) {
489 ne_create_request_fn fn = (ne_create_request_fn)hk->fn;
490 fn(req, hk->userdata, req->method, req->uri);
491 }
492 }
493
494 req->expiration_time.tv_sec = 0;
495 req->expiration_time.tv_nsec= 0;
496
497 return req;
498 }
499
500 /* Set the request body length to 'length' */
set_body_length(ne_request * req,ne_off_t length)501 static void set_body_length(ne_request *req, ne_off_t length)
502 {
503 req->body_length = length;
504 ne_print_request_header(req, "Content-Length", "%" FMT_NE_OFF_T, length);
505 }
506
ne_set_request_body_buffer(ne_request * req,const char * buffer,size_t size)507 void ne_set_request_body_buffer(ne_request *req, const char *buffer,
508 size_t size)
509 {
510 req->body.buf.buffer = buffer;
511 req->body.buf.length = size;
512 req->body_cb = body_string_send;
513 req->body_ud = req;
514 set_body_length(req, size);
515 }
516
ne_set_request_body_provider(ne_request * req,ne_off_t bodysize,ne_provide_body provider,void * ud)517 void ne_set_request_body_provider(ne_request *req, ne_off_t bodysize,
518 ne_provide_body provider, void *ud)
519 {
520 req->body_cb = provider;
521 req->body_ud = ud;
522 set_body_length(req, bodysize);
523 }
524
ne_set_request_body_fd(ne_request * req,int fd,ne_off_t offset,ne_off_t length)525 void ne_set_request_body_fd(ne_request *req, int fd,
526 ne_off_t offset, ne_off_t length)
527 {
528 req->body.file.fd = fd;
529 req->body.file.offset = offset;
530 req->body.file.length = length;
531 req->body_cb = body_fd_send;
532 req->body_ud = req;
533 set_body_length(req, length);
534 }
535
ne_set_request_flag(ne_request * req,ne_request_flag flag,int value)536 void ne_set_request_flag(ne_request *req, ne_request_flag flag, int value)
537 {
538 if (flag < NE_REQFLAG_LAST) {
539 req->flags[flag] = value;
540 }
541 }
542
ne_get_request_flag(ne_request * req,ne_request_flag flag)543 int ne_get_request_flag(ne_request *req, ne_request_flag flag)
544 {
545 if (flag < NE_REQFLAG_LAST) {
546 return req->flags[flag];
547 }
548 return -1;
549 }
550
551
ne_add_request_header(ne_request * req,const char * name,const char * value)552 void ne_add_request_header(ne_request *req, const char *name,
553 const char *value)
554 {
555 ne_buffer_concat(req->headers, name, ": ", value, EOL, NULL);
556 }
557
ne_print_request_header(ne_request * req,const char * name,const char * format,...)558 void ne_print_request_header(ne_request *req, const char *name,
559 const char *format, ...)
560 {
561 va_list params;
562 char buf[NE_BUFSIZ];
563
564 va_start(params, format);
565 ne_vsnprintf(buf, sizeof buf, format, params);
566 va_end(params);
567
568 ne_buffer_concat(req->headers, name, ": ", buf, EOL, NULL);
569 }
570
571 /* Returns the value of the response header 'name', for which the hash
572 * value is 'h', or NULL if the header is not found. */
get_response_header_hv(ne_request * req,unsigned int h,const char * name)573 static inline char *get_response_header_hv(ne_request *req, unsigned int h,
574 const char *name)
575 {
576 struct field *f;
577
578 for (f = req->response_headers[h]; f; f = f->next)
579 if (strcmp(f->name, name) == 0)
580 return f->value;
581
582 return NULL;
583 }
584
ne_get_response_header(ne_request * req,const char * name)585 const char *ne_get_response_header(ne_request *req, const char *name)
586 {
587 char *lcname = ne_strdup(name);
588 unsigned int hash = hash_and_lower(lcname);
589 char *value = get_response_header_hv(req, hash, lcname);
590 ne_free(lcname);
591 return value;
592 }
593
594 /* The return value of the iterator function is a pointer to the
595 * struct field of the previously returned header. */
ne_response_header_iterate(ne_request * req,void * iterator,const char ** name,const char ** value)596 void *ne_response_header_iterate(ne_request *req, void *iterator,
597 const char **name, const char **value)
598 {
599 struct field *f = iterator;
600 unsigned int n;
601
602 if (f == NULL) {
603 n = 0;
604 } else if ((f = f->next) == NULL) {
605 n = req->current_index + 1;
606 }
607
608 if (f == NULL) {
609 while (n < HH_HASHSIZE && req->response_headers[n] == NULL)
610 n++;
611 if (n == HH_HASHSIZE)
612 return NULL; /* no more headers */
613 f = req->response_headers[n];
614 req->current_index = n;
615 }
616
617 *name = f->name;
618 *value = f->value;
619 return f;
620 }
621
622 /* Removes the response header 'name', which has hash value 'hash'. */
remove_response_header(ne_request * req,const char * name,unsigned int hash)623 static void remove_response_header(ne_request *req, const char *name,
624 unsigned int hash)
625 {
626 struct field **ptr = req->response_headers + hash;
627
628 while (*ptr) {
629 struct field *const f = *ptr;
630
631 if (strcmp(f->name, name) == 0) {
632 *ptr = f->next;
633 ne_free(f->name);
634 ne_free(f->value);
635 ne_free(f);
636 return;
637 }
638
639 ptr = &f->next;
640 }
641 }
642
643 /* Free all stored response headers. */
free_response_headers(ne_request * req)644 static void free_response_headers(ne_request *req)
645 {
646 int n;
647
648 for (n = 0; n < HH_HASHSIZE; n++) {
649 struct field **ptr = req->response_headers + n;
650
651 while (*ptr) {
652 struct field *const f = *ptr;
653 *ptr = f->next;
654 ne_free(f->name);
655 ne_free(f->value);
656 ne_free(f);
657 }
658 }
659 }
660
ne_add_response_body_reader(ne_request * req,ne_accept_response acpt,ne_block_reader rdr,void * userdata)661 void ne_add_response_body_reader(ne_request *req, ne_accept_response acpt,
662 ne_block_reader rdr, void *userdata)
663 {
664 struct body_reader *new = ne_malloc(sizeof *new);
665 new->accept_response = acpt;
666 new->handler = rdr;
667 new->userdata = userdata;
668 new->next = req->body_readers;
669 req->body_readers = new;
670 }
671
ne_request_destroy(ne_request * req)672 void ne_request_destroy(ne_request *req)
673 {
674 struct body_reader *rdr, *next_rdr;
675 struct hook *hk, *next_hk;
676
677 ne_free(req->uri);
678 ne_free(req->method);
679
680 for (rdr = req->body_readers; rdr != NULL; rdr = next_rdr) {
681 next_rdr = rdr->next;
682 ne_free(rdr);
683 }
684
685 free_response_headers(req);
686
687 ne_buffer_destroy(req->headers);
688
689 NE_DEBUG(NE_DBG_CORE, "Running destroy hooks.");
690 for (hk = req->session->destroy_req_hooks; hk; hk = next_hk) {
691 ne_destroy_req_fn fn = (ne_destroy_req_fn)hk->fn;
692 next_hk = hk->next;
693 fn(req, hk->userdata);
694 }
695
696 for (hk = req->private; hk; hk = next_hk) {
697 next_hk = hk->next;
698 ne_free(hk);
699 }
700
701 if (req->status.reason_phrase)
702 ne_free(req->status.reason_phrase);
703
704 NE_DEBUG(NE_DBG_CORE, "Request ends.");
705 ne_free(req);
706 }
707
708
709 /* Reads a block of the response into BUFFER, which is of size
710 * *BUFLEN. Returns zero on success or non-zero on error. On
711 * success, *BUFLEN is updated to be the number of bytes read into
712 * BUFFER (which will be 0 to indicate the end of the repsonse). On
713 * error, the connection is closed and the session error string is
714 * set. */
read_response_block(ne_request * req,struct ne_response * resp,char * buffer,size_t * buflen)715 static int read_response_block(ne_request *req, struct ne_response *resp,
716 char *buffer, size_t *buflen)
717 {
718 ne_socket *const sock = req->session->socket;
719 size_t willread;
720 ssize_t readlen;
721
722 switch (resp->mode) {
723 case R_CHUNKED:
724 /* Chunked transfer-encoding: chunk syntax is "SIZE CRLF CHUNK
725 * CRLF SIZE CRLF CHUNK CRLF ..." followed by zero-length
726 * chunk: "CHUNK CRLF 0 CRLF". resp.chunk.remain contains the
727 * number of bytes left to read in the current chunk. */
728 if (resp->body.chunk.remain == 0) {
729 unsigned long chunk_len;
730 char *ptr;
731
732 /* Read the chunk size line into a temporary buffer. */
733 SOCK_ERR(req,
734 ne_sock_readline(sock, req->respbuf, sizeof req->respbuf),
735 _("Could not read chunk size"));
736 NE_DEBUG(NE_DBG_HTTP, "[chunk] < %s", req->respbuf);
737 chunk_len = strtoul(req->respbuf, &ptr, 16);
738 /* limit chunk size to <= UINT_MAX, so it will probably
739 * fit in a size_t. */
740 if (ptr == req->respbuf ||
741 chunk_len == ULONG_MAX || chunk_len > UINT_MAX) {
742 return aborted(req, _("Could not parse chunk size"), 0);
743 }
744 NE_DEBUG(NE_DBG_HTTP, "Got chunk size: %lu", chunk_len);
745 resp->body.chunk.remain = chunk_len;
746 }
747 willread = resp->body.chunk.remain > *buflen
748 ? *buflen : resp->body.chunk.remain;
749 break;
750 case R_CLENGTH:
751 willread = resp->body.clen.remain > (off_t)*buflen
752 ? *buflen : (size_t)resp->body.clen.remain;
753 break;
754 case R_TILLEOF:
755 willread = *buflen;
756 break;
757 case R_NO_BODY:
758 default:
759 willread = 0;
760 break;
761 }
762 if (willread == 0) {
763 *buflen = 0;
764 return 0;
765 }
766 NE_DEBUG(NE_DBG_CORE,
767 "Reading %" NE_FMT_SIZE_T " bytes of response body.\n", willread);
768 readlen = ne_sock_read(sock, buffer, willread);
769
770 /* EOF is only valid when response body is delimited by it.
771 * Strictly, an SSL truncation should not be treated as an EOF in
772 * any case, but SSL servers are just too buggy. */
773 if (resp->mode == R_TILLEOF &&
774 (readlen == NE_SOCK_CLOSED || readlen == NE_SOCK_TRUNC)) {
775 NE_DEBUG(NE_DBG_HTTP, "Got EOF.");
776 req->can_persist = 0;
777 readlen = 0;
778 } else if (readlen < 0) {
779 return aborted(req, _("Could not read response body"), readlen);
780 } else {
781 NE_DEBUG(NE_DBG_CORE, "Got %" NE_FMT_SSIZE_T " bytes.", readlen);
782 }
783 /* safe to cast: readlen guaranteed to be >= 0 above */
784 *buflen = (size_t)readlen;
785 NE_DEBUG(NE_DBG_HTTPBODY,
786 "Read block (%" NE_FMT_SSIZE_T " bytes):\n[%.*s]\n",
787 readlen, (int)readlen, buffer);
788 if (resp->mode == R_CHUNKED) {
789 resp->body.chunk.remain -= readlen;
790 if (resp->body.chunk.remain == 0) {
791 char crlfbuf[2];
792 /* If we've read a whole chunk, read a CRLF */
793 readlen = ne_sock_fullread(sock, crlfbuf, 2);
794 if (readlen < 0)
795 return aborted(req, _("Could not read chunk delimiter"),
796 readlen);
797 else if (crlfbuf[0] != '\r' || crlfbuf[1] != '\n')
798 return aborted(req, _("Chunk delimiter was invalid"), 0);
799 }
800 } else if (resp->mode == R_CLENGTH) {
801 resp->body.clen.remain -= readlen;
802 }
803 resp->progress += readlen;
804 return NE_OK;
805 }
806
ne_read_response_block(ne_request * req,char * buffer,size_t buflen)807 ssize_t ne_read_response_block(ne_request *req, char *buffer, size_t buflen)
808 {
809 struct body_reader *rdr;
810 size_t readlen = buflen;
811 struct ne_response *const resp = &req->resp;
812
813 if (read_response_block(req, resp, buffer, &readlen))
814 return -1;
815
816 if (readlen) {
817 req->session->status.sr.progress += readlen;
818 notify_status(req->session, ne_status_recving);
819 }
820
821 for (rdr = req->body_readers; rdr!=NULL; rdr=rdr->next) {
822 if (rdr->use && rdr->handler(rdr->userdata, buffer, readlen) != 0) {
823 ne_close_connection(req->session);
824 return -1;
825 }
826 }
827
828 return readlen;
829 }
830
831 /* Build the request string, returning the buffer. */
build_request(ne_request * req)832 static ne_buffer *build_request(ne_request *req)
833 {
834 struct hook *hk;
835 ne_buffer *buf = ne_buffer_create();
836
837 /* Add Request-Line and headers: */
838 ne_buffer_concat(buf, req->method, " ", req->uri, " HTTP/1.1" EOL, NULL);
839
840 /* Add custom headers: */
841 ne_buffer_append(buf, req->headers->data, ne_buffer_size(req->headers));
842
843 if (req->body_length && req->flags[NE_REQFLAG_EXPECT100]) {
844 ne_buffer_czappend(buf, "Expect: 100-continue\r\n");
845 }
846
847 NE_DEBUG(NE_DBG_CORE, "Running pre_send hooks");
848 for (hk = req->session->pre_send_hooks; hk!=NULL; hk = hk->next) {
849 ne_pre_send_fn fn = (ne_pre_send_fn)hk->fn;
850 fn(req, hk->userdata, buf);
851 }
852
853 ne_buffer_czappend(buf, "\r\n");
854 return buf;
855 }
856
857 #define MAX_HEADER_LEN (8192)
858
859 #ifdef NE_DEBUGGING
860 #define DEBUG_DUMP_REQUEST(x) dump_request(x)
861
dump_request(const char * request)862 static void dump_request(const char *request)
863 {
864 char hdr_tmp[MAX_HEADER_LEN];
865 char hdr_debug[MAX_HEADER_LEN];
866
867 memset(hdr_tmp, '\0', sizeof(hdr_tmp));
868 memset(hdr_debug, '\0', sizeof(hdr_debug));
869
870 strcat(hdr_tmp, "> ");
871 strcat(hdr_tmp, request);
872 char* token = strtok(hdr_tmp, "\n");
873 while (token) {
874 strcat(hdr_debug, token);
875 strcat(hdr_debug, "\n> ");
876 token = strtok(NULL, "\n");
877 }
878
879 hdr_debug[strlen(hdr_debug)-2] = '\0';
880
881 if (davix_get_log_scope() & NE_DBG_HTTPPLAIN) {
882 /* Display everything mode */
883 NE_DEBUG(NE_DBG_HTTP, "%s", hdr_debug);
884 } else if (davix_get_log_scope() & NE_DBG_HTTP) {
885 /* Blank out the Authorization paramaters */
886 char *reqdebug = ne_strdup(hdr_debug), *pnt = reqdebug;
887 while ((pnt = strstr(pnt, "Authorization: ")) != NULL) {
888 for (pnt += 15; *pnt != '\r' && *pnt != '\0'; pnt++) {
889 *pnt = 'x';
890 }
891 }
892 NE_DEBUG(NE_DBG_HTTP, "%s",reqdebug);
893
894 ne_free(reqdebug);
895 }
896 }
897
898 #else
899 #define DEBUG_DUMP_REQUEST(x)
900 #endif /* DEBUGGING */
901
902 /* remove trailing EOL from 'buf', where strlen(buf) == *len. *len is
903 * adjusted in accordance with any changes made to the string to
904 * remain equal to strlen(buf). */
strip_eol(char * buf,ssize_t * len)905 static inline void strip_eol(char *buf, ssize_t *len)
906 {
907 char *pnt = buf + *len - 1;
908 while (pnt >= buf && (*pnt == '\r' || *pnt == '\n')) {
909 *pnt-- = '\0';
910 (*len)--;
911 }
912 }
913
914 /* Read and parse response status-line into 'status'. 'retry' is non-zero
915 * if an NE_RETRY should be returned if an EOF is received. */
read_status_line(ne_request * req,ne_status * status,int retry)916 static int read_status_line(ne_request *req, ne_status *status, int retry)
917 {
918 char *buffer = req->respbuf;
919 ssize_t ret;
920
921 ret = ne_sock_readline(req->session->socket, buffer, sizeof req->respbuf);
922 if (ret <= 0) {
923 int aret = aborted(req, _("Could not read status line"), ret);
924 return RETRY_RET(retry, ret, aret);
925 }
926
927 strip_eol(buffer, &ret);
928 NE_DEBUG(NE_DBG_HTTP, "< %s", buffer);
929
930 if (status->reason_phrase) ne_free(status->reason_phrase);
931 memset(status, 0, sizeof *status);
932
933 /* Hack to allow ShoutCast-style servers, if requested. */
934 if (req->session->flags[NE_SESSFLAG_ICYPROTO]
935 && strncmp(buffer, "ICY ", 4) == 0 && strlen(buffer) > 8
936 && buffer[7] == ' ') {
937 status->code = atoi(buffer + 4);
938 status->major_version = 1;
939 status->minor_version = 0;
940 status->reason_phrase = ne_strclean(ne_strdup(buffer + 8));
941 status->klass = buffer[4] - '0';
942 NE_DEBUG(NE_DBG_HTTP, "[status-line] ICY protocol; code %d",
943 status->code);
944 } else if (ne_parse_statusline(buffer, status)) {
945 return aborted(req, _("Could not parse response status line"), 0);
946 }
947
948 return 0;
949 }
950
951 /* Discard a set of message headers. */
discard_headers(ne_request * req)952 static int discard_headers(ne_request *req)
953 {
954 do {
955 SOCK_ERR(req, ne_sock_readline(req->session->socket, req->respbuf,
956 sizeof req->respbuf),
957 _("Could not read interim response headers"));
958 NE_DEBUG(NE_DBG_HTTP, "[discard] < %s", req->respbuf);
959 } while (strcmp(req->respbuf, EOL) != 0);
960 return NE_OK;
961 }
962
963 /* Send the request, and read the response Status-Line. Returns:
964 * NE_RETRY connection closed by server; persistent connection
965 * timeout
966 * NE_OK success
967 * NE_* error
968 * On NE_RETRY and NE_* responses, the connection will have been
969 * closed already.
970 */
send_request(ne_request * req,const ne_buffer * request)971 static int send_request(ne_request *req, const ne_buffer *request)
972 {
973 ne_session *const sess = req->session;
974 ne_status *const status = &req->status;
975 int sentbody = 0; /* zero until body has been sent. */
976 int ret, retry; /* retry non-zero whilst the request should be retried */
977 ssize_t sret;
978
979 /* Send the Request-Line and headers */
980 NE_DEBUG(NE_DBG_CORE, "Sending request-line and headers:");
981 /* Open the connection if necessary */
982 ret = open_connection(sess);
983 if (ret) return ret;
984
985 /* Allow retry if a persistent connection has been used. */
986 retry = sess->persisted;
987
988 sret = ne_sock_fullwrite(req->session->socket, request->data,
989 ne_buffer_size(request));
990 if (sret < 0) {
991 int aret = aborted(req, _("Could not send request"), sret);
992 return RETRY_RET(retry, sret, aret);
993 }
994
995 if (!req->flags[NE_REQFLAG_EXPECT100] && req->body_length > 0) {
996 /* Send request body, if not using 100-continue. */
997 ret = send_request_body(req, retry);
998 if (ret) {
999 return ret;
1000 }
1001 }
1002
1003 NE_DEBUG(NE_DBG_CORE, "Request sent; retry is %d.", retry);
1004
1005 /* Loop eating interim 1xx responses (RFC2616 says these MAY be
1006 * sent by the server, even if 100-continue is not used). */
1007 while ((ret = read_status_line(req, status, retry)) == NE_OK
1008 && status->klass == 1) {
1009 NE_DEBUG(NE_DBG_HTTP, "Interim %d response.", status->code);
1010 retry = 0; /* successful read() => never retry now. */
1011 /* Discard headers with the interim response. */
1012 if ((ret = discard_headers(req)) != NE_OK) break;
1013
1014 if (req->flags[NE_REQFLAG_EXPECT100] && (status->code == 100)
1015 && req->body_length > 0 && !sentbody) {
1016 /* Send the body after receiving the first 100 Continue */
1017 if ((ret = send_request_body(req, 0)) != NE_OK) break;
1018 sentbody = 1;
1019 }
1020 }
1021
1022 return ret;
1023 }
1024
1025 /* Read a message header from sock into buf, which has size 'buflen'.
1026 *
1027 * Returns:
1028 * NE_RETRY: Success, read a header into buf.
1029 * NE_OK: End of headers reached.
1030 * NE_ERROR: Error (session error is set, connection closed).
1031 */
read_message_header(ne_request * req,char * buf,size_t buflen)1032 static int read_message_header(ne_request *req, char *buf, size_t buflen)
1033 {
1034 ssize_t n;
1035 ne_socket *sock = req->session->socket;
1036
1037 n = ne_sock_readline(sock, buf, buflen);
1038 if (n <= 0)
1039 return aborted(req, _("Error reading response headers"), n);
1040
1041 strip_eol(buf, &n);
1042 NE_DEBUG(NE_DBG_HTTP, "< %s", buf);
1043
1044 if (n == 0) {
1045 NE_DEBUG(NE_DBG_CORE, "End of headers.");
1046 return NE_OK;
1047 }
1048
1049 buf += n;
1050 buflen -= n;
1051
1052 while (buflen > 0) {
1053 char ch;
1054
1055 /* Collect any extra lines into buffer */
1056 SOCK_ERR(req, ne_sock_peek(sock, &ch, 1),
1057 _("Error reading response headers"));
1058
1059 if (ch != ' ' && ch != '\t') {
1060 /* No continuation of this header: stop reading. */
1061 return NE_RETRY;
1062 }
1063
1064 /* Otherwise, read the next line onto the end of 'buf'. */
1065 n = ne_sock_readline(sock, buf, buflen);
1066 if (n <= 0) {
1067 return aborted(req, _("Error reading response headers"), n);
1068 }
1069
1070 NE_DEBUG(NE_DBG_HTTP, "[cont] %s", buf);
1071
1072 strip_eol(buf, &n);
1073
1074 /* assert(buf[0] == ch), which implies len(buf) > 0.
1075 * Otherwise the TCP stack is lying, but we'll be paranoid.
1076 * This might be a \t, so replace it with a space to be
1077 * friendly to applications (2616 says we MAY do this). */
1078 if (n) buf[0] = ' ';
1079
1080 /* ready for the next header. */
1081 buf += n;
1082 buflen -= n;
1083 }
1084
1085 ne_set_error(req->session, _("Response header too long"));
1086 return NE_ERROR;
1087 }
1088
1089
1090
1091 /* Add a respnose header field for the given request, using
1092 * precalculated hash value. */
add_response_header(ne_request * req,unsigned int hash,char * name,char * value)1093 static void add_response_header(ne_request *req, unsigned int hash,
1094 char *name, char *value)
1095 {
1096 struct field **nextf = &req->response_headers[hash];
1097 size_t vlen = strlen(value);
1098
1099 while (*nextf) {
1100 struct field *const f = *nextf;
1101 if (strcmp(f->name, name) == 0) {
1102 if (vlen + f->vlen < MAX_HEADER_LEN) {
1103 /* merge the header field */
1104 f->value = ne_realloc(f->value, f->vlen + vlen + 3);
1105 memcpy(f->value + f->vlen, ", ", 2);
1106 memcpy(f->value + f->vlen + 2, value, vlen + 1);
1107 f->vlen += vlen + 2;
1108 }
1109 return;
1110 }
1111 nextf = &f->next;
1112 }
1113
1114 (*nextf) = ne_malloc(sizeof **nextf);
1115 (*nextf)->name = ne_strdup(name);
1116 (*nextf)->value = ne_strdup(value);
1117 (*nextf)->vlen = vlen;
1118 (*nextf)->next = NULL;
1119 }
1120
1121 /* Read response headers. Returns NE_* code, sets session error and
1122 * closes connection on error. */
read_response_headers(ne_request * req)1123 static int read_response_headers(ne_request *req)
1124 {
1125 char hdr[MAX_HEADER_LEN];
1126 int ret, count = 0;
1127
1128 while ((ret = read_message_header(req, hdr, sizeof hdr)) == NE_RETRY
1129 && ++count < MAX_HEADER_FIELDS) {
1130 char *pnt;
1131 unsigned int hash = 0;
1132
1133 /* Strip any trailing whitespace */
1134 pnt = hdr + strlen(hdr) - 1;
1135 while (pnt > hdr && (*pnt == ' ' || *pnt == '\t'))
1136 *pnt-- = '\0';
1137
1138 /* Convert the header name to lower case and hash it. */
1139 for (pnt = hdr; (*pnt != '\0' && *pnt != ':' &&
1140 *pnt != ' ' && *pnt != '\t'); pnt++) {
1141 *pnt = ne_tolower(*pnt);
1142 hash = HH_ITERATE(hash,*pnt);
1143 }
1144
1145 /* Skip over any whitespace before the colon. */
1146 while (*pnt == ' ' || *pnt == '\t')
1147 *pnt++ = '\0';
1148
1149 /* ignore header lines which lack a ':'. */
1150 if (*pnt != ':')
1151 continue;
1152
1153 /* NUL-terminate at the colon (when no whitespace before) */
1154 *pnt++ = '\0';
1155
1156 /* Skip any whitespace after the colon... */
1157 while (*pnt == ' ' || *pnt == '\t')
1158 pnt++;
1159
1160 /* pnt now points to the header value. */
1161 //NE_DEBUG(NE_DBG_HTTP, "Header Name: [%s], Value: [%s]", hdr, pnt);
1162 add_response_header(req, hash, hdr, pnt);
1163 }
1164
1165 if (count == MAX_HEADER_FIELDS)
1166 ret = aborted(
1167 req, _("Response exceeded maximum number of header fields"), 0);
1168
1169 return ret;
1170 }
1171
1172 /* Perform any necessary DNS lookup for the host given by *info;
1173 * returns NE_ code with error string set on error. */
lookup_host(ne_session * sess,struct host_info * info)1174 static int lookup_host(ne_session *sess, struct host_info *info)
1175 {
1176 NE_DEBUG(NE_DBG_CORE, "Doing DNS lookup on %s...", info->hostname);
1177 sess->status.lu.hostname = info->hostname;
1178 notify_status(sess, ne_status_lookup);
1179 info->address = ne_addr_resolve(info->hostname, 0);
1180 if (ne_addr_result(info->address)) {
1181 char buf[256];
1182 ne_set_error(sess, _("Could not resolve hostname `%s': %s"),
1183 info->hostname,
1184 ne_addr_error(info->address, buf, sizeof buf));
1185 ne_addr_destroy(info->address);
1186 info->address = NULL;
1187 return NE_LOOKUP;
1188 } else {
1189 return NE_OK;
1190 }
1191 }
1192
ne_begin_request(ne_request * req)1193 int ne_begin_request(ne_request *req)
1194 {
1195 struct body_reader *rdr;
1196 ne_buffer *data;
1197 const ne_status *const st = &req->status;
1198 const char *value;
1199 struct hook *hk;
1200 int ret, forced_closure = 0;
1201
1202 /* If a non-idempotent request is sent on a persisted connection,
1203 * then it is impossible to distinguish between a server failure
1204 * and a connection timeout if an EOF/RST is received. So don't
1205 * do that. */
1206
1207 /*
1208 if (!req->flags[NE_REQFLAG_IDEMPOTENT] && req->session->persisted
1209 && !req->session->flags[NE_SESSFLAG_CONNAUTH]) {
1210 NE_DEBUG(NE_DBG_HTTP, "req: Closing connection for non-idempotent "
1211 "request.\n");
1212 ne_close_connection(req->session);
1213 }
1214 */
1215
1216 /* Build the request string, and send it */
1217 data = build_request(req);
1218 if(davix_get_log_scope() & NE_DBG_HTTP){
1219 dump_request(data->data);
1220 }
1221 ret = send_request(req, data);
1222 /* Retry this once after a persistent connection timeout. */
1223 if (ret == NE_RETRY) {
1224 NE_DEBUG(NE_DBG_HTTP, "Persistent connection timed out, retrying.");
1225 ret = send_request(req, data);
1226 }
1227 ne_buffer_destroy(data);
1228 if (ret != NE_OK) return ret == NE_RETRY ? NE_ERROR : ret;
1229
1230 /* Determine whether server claims HTTP/1.1 compliance. */
1231 req->session->is_http11 = (st->major_version == 1 &&
1232 st->minor_version > 0) || st->major_version > 1;
1233
1234 /* Persistent connections supported implicitly in HTTP/1.1 */
1235 if (req->session->is_http11) req->can_persist = 1;
1236
1237 ne_set_error(req->session, "%d %s", st->code, st->reason_phrase);
1238
1239 /* Empty the response header hash, in case this request was
1240 * retried: */
1241 free_response_headers(req);
1242
1243 /* Read the headers */
1244 ret = read_response_headers(req);
1245 if (ret) return ret;
1246
1247 /* check the Connection header */
1248 value = get_response_header_hv(req, HH_HV_CONNECTION, "connection");
1249 if (value) {
1250 char *vcopy = ne_strdup(value), *ptr = vcopy;
1251
1252 do {
1253 char *token = ne_shave(ne_token(&ptr, ','), " \t");
1254 unsigned int hash = hash_and_lower(token);
1255
1256 if (strcmp(token, "close") == 0) {
1257 req->can_persist = 0;
1258 forced_closure = 1;
1259 } else if (strcmp(token, "keep-alive") == 0) {
1260 req->can_persist = 1;
1261 } else if (!req->session->is_http11
1262 && strcmp(token, "connection")) {
1263 /* Strip the header per 2616§14.10, last para. Avoid
1264 * danger from "Connection: connection". */
1265 remove_response_header(req, token, hash);
1266 }
1267 } while (ptr);
1268
1269 ne_free(vcopy);
1270 }
1271
1272 /* Support "Proxy-Connection: keep-alive" for compatibility with
1273 * some HTTP/1.0 proxies; it is risky to do this, because an
1274 * intermediary proxy may not support this HTTP/1.0 extension, but
1275 * will not strip the header either. Persistent connection
1276 * support is enabled based on the presence of this header if:
1277 * a) it is *necessary* to do so due to the use of a connection-auth
1278 * scheme, and
1279 * b) connection closure was not forced via "Connection: close". */
1280 if (req->session->nexthop->proxy == PROXY_HTTP && !req->session->is_http11
1281 && !forced_closure && req->session->flags[NE_SESSFLAG_CONNAUTH]) {
1282 value = get_response_header_hv(req, HH_HV_PROXY_CONNECTION,
1283 "proxy-connection");
1284 if (value && ne_strcasecmp(value, "keep-alive") == 0) {
1285 NE_DEBUG(NE_DBG_HTTP, "req: Using persistent connection "
1286 "for HTTP/1.0 proxy requiring conn-auth hack.\n");
1287 req->can_persist = 1;
1288 }
1289 }
1290
1291 /* Decide which method determines the response message-length per
1292 * 2616§4.4 (multipart/byteranges is not supported): */
1293
1294 #ifdef NE_HAVE_SSL
1295 /* Special case for CONNECT handling: the response has no body,
1296 * and the connection can persist. */
1297 if (req->session->in_connect && st->klass == 2) {
1298 req->resp.mode = R_NO_BODY;
1299 req->can_persist = 1;
1300 } else
1301 #endif
1302 /* HEAD requests and 204, 304 responses have no response body,
1303 * regardless of what headers are present. */
1304 if (req->method_is_head || st->code == 204 || st->code == 304) {
1305 req->resp.mode = R_NO_BODY;
1306 }
1307 /* Broken intermediaries exist which use "transfer-encoding: identity"
1308 * to mean "no transfer-coding". So that case must be ignored. */
1309 else if ((value = get_response_header_hv(req, HH_HV_TRANSFER_ENCODING,
1310 "transfer-encoding")) != NULL
1311 && ne_strcasecmp(value, "identity") != 0) {
1312 /* Otherwise, fail iff an unknown transfer-coding is used. */
1313 if (ne_strcasecmp(value, "chunked") == 0) {
1314 req->resp.mode = R_CHUNKED;
1315 req->resp.body.chunk.remain = 0;
1316 }
1317 else {
1318 return aborted(req, _("Unknown transfer-coding in response"), 0);
1319 }
1320 }
1321 else if ((value = get_response_header_hv(req, HH_HV_CONTENT_LENGTH,
1322 "content-length")) != NULL) {
1323 char *endptr = NULL;
1324 ne_off_t len = ne_strtoff(value, &endptr, 10);
1325
1326 if (*value && len != NE_OFFT_MAX && len >= 0 && endptr && *endptr == '\0') {
1327 req->resp.mode = R_CLENGTH;
1328 req->resp.body.clen.total = req->resp.body.clen.remain = len;
1329 } else {
1330 /* fail for an invalid content-length header. */
1331 return aborted(req, _("Invalid Content-Length in response"), 0);
1332 }
1333 } else {
1334 req->resp.mode = R_TILLEOF; /* otherwise: read-till-eof mode */
1335 }
1336
1337 NE_DEBUG(NE_DBG_CORE, "Running post_headers hooks");
1338 for (hk = req->session->post_headers_hooks; hk != NULL; hk = hk->next) {
1339 ne_post_headers_fn fn = (ne_post_headers_fn)hk->fn;
1340 fn(req, hk->userdata, &req->status);
1341 }
1342
1343 /* Prepare for reading the response entity-body. Call each of the
1344 * body readers and ask them whether they want to accept this
1345 * response or not. */
1346 for (rdr = req->body_readers; rdr != NULL; rdr=rdr->next) {
1347 rdr->use = rdr->accept_response(rdr->userdata, req, st);
1348 }
1349
1350 req->session->status.sr.progress = 0;
1351 req->session->status.sr.total =
1352 req->resp.mode == R_CLENGTH ? req->resp.body.clen.total : -1;
1353 notify_status(req->session, ne_status_recving);
1354
1355 return NE_OK;
1356 }
1357
ne_end_request(ne_request * req)1358 int ne_end_request(ne_request *req)
1359 {
1360 struct hook *hk;
1361 int ret;
1362
1363 /* Read headers in chunked trailers */
1364 if (req->resp.mode == R_CHUNKED) {
1365 ret = read_response_headers(req);
1366 if (ret) return ret;
1367 } else {
1368 ret = NE_OK;
1369 }
1370
1371 NE_DEBUG(NE_DBG_CORE, "Running post_send hooks");
1372 for (hk = req->session->post_send_hooks;
1373 ret == NE_OK && hk != NULL; hk = hk->next) {
1374 ne_post_send_fn fn = (ne_post_send_fn)hk->fn;
1375 ret = fn(req, hk->userdata, &req->status);
1376 }
1377
1378 /* Close the connection if persistent connections are disabled or
1379 * not supported by the server. */
1380 if (!req->session->flags[NE_SESSFLAG_PERSIST] || !req->can_persist)
1381 ne_close_connection(req->session);
1382 else
1383 req->session->persisted = 1;
1384
1385 return ret;
1386 }
1387
1388
ne_abort_request(ne_request * req)1389 int ne_abort_request(ne_request* req){
1390 if(req->session)
1391 ne_close_connection(req->session);
1392 return 0;
1393 }
1394
ne_read_response_to_fd(ne_request * req,int fd)1395 int ne_read_response_to_fd(ne_request *req, int fd)
1396 {
1397 ssize_t len;
1398
1399 while ((len = ne_read_response_block(req, req->respbuf,
1400 sizeof req->respbuf)) > 0) {
1401 const char *block = req->respbuf;
1402
1403 do {
1404 ssize_t ret = write(fd, block, len);
1405 if (ret == -1 && errno == EINTR) {
1406 continue;
1407 } else if (ret < 0) {
1408 char err[200];
1409 ne_strerror(errno, err, sizeof err);
1410 ne_set_error(ne_get_session(req),
1411 _("Could not write to file: %s"), err);
1412 return NE_ERROR;
1413 } else {
1414 len -= ret;
1415 block += ret;
1416 }
1417 } while (len > 0);
1418 }
1419
1420 return len == 0 ? NE_OK : NE_ERROR;
1421 }
1422
ne_discard_response(ne_request * req)1423 int ne_discard_response(ne_request *req)
1424 {
1425 ssize_t len;
1426
1427 do {
1428 len = ne_read_response_block(req, req->respbuf, sizeof req->respbuf);
1429 } while (len > 0);
1430
1431 return len == 0 ? NE_OK : NE_ERROR;
1432 }
1433
ne_request_dispatch(ne_request * req)1434 int ne_request_dispatch(ne_request *req)
1435 {
1436 int ret;
1437
1438 do {
1439 ret = ne_begin_request(req);
1440 if (ret == NE_OK) ret = ne_discard_response(req);
1441 if (ret == NE_OK) ret = ne_end_request(req);
1442 } while (ret == NE_RETRY);
1443
1444 NE_DEBUG(NE_DBG_HTTP | NE_DBG_FLUSH,
1445 "Request ends, status %d class %dxx, error line:\n%s\n",
1446 req->status.code, req->status.klass, req->session->error);
1447
1448 return ret;
1449 }
1450
ne_get_status(const ne_request * req)1451 const ne_status *ne_get_status(const ne_request *req)
1452 {
1453 return &req->status;
1454 }
1455
ne_get_session(const ne_request * req)1456 ne_session *ne_get_session(const ne_request *req)
1457 {
1458 return req->session;
1459 }
1460
1461 #ifdef NE_HAVE_SSL
1462 /* Create a CONNECT tunnel through the proxy server.
1463 * Returns HTTP_* */
proxy_tunnel(ne_session * sess)1464 static int proxy_tunnel(ne_session *sess)
1465 {
1466 /* Hack up an HTTP CONNECT request... */
1467 ne_request *req;
1468 int ret = NE_OK;
1469 char ruri[200];
1470
1471 /* Can't use server.hostport here; Request-URI must include `:port' */
1472 ne_snprintf(ruri, sizeof ruri, "%s:%u", sess->server.hostname,
1473 sess->server.port);
1474 req = ne_request_create(sess, "CONNECT", ruri);
1475
1476 sess->in_connect = 1;
1477 ret = ne_request_dispatch(req);
1478 sess->in_connect = 0;
1479
1480 sess->persisted = 0; /* don't treat this is a persistent connection. */
1481
1482 if (ret != NE_OK || !sess->connected || req->status.klass != 2) {
1483 char *err = ne_strdup(sess->error);
1484 ne_set_error(sess, _("Could not create SSL connection "
1485 "through proxy server: %s"), err);
1486 ne_free(err);
1487 if (ret == NE_OK) ret = NE_ERROR;
1488 }
1489
1490 ne_request_destroy(req);
1491 return ret;
1492 }
1493 #endif
1494
1495 /* Return the first resolved address for the given host. */
resolve_first(struct host_info * host)1496 static const ne_inet_addr *resolve_first(struct host_info *host)
1497 {
1498 return host->network ? host->network : ne_addr_first(host->address);
1499 }
1500
1501 /* Return the next resolved address for the given host or NULL if
1502 * there are no more addresses. */
resolve_next(struct host_info * host)1503 static const ne_inet_addr *resolve_next(struct host_info *host)
1504 {
1505 return host->network ? NULL : ne_addr_next(host->address);
1506 }
1507
1508 /* Make new TCP connection to server at 'host' of type 'name'. Note
1509 * that once a connection to a particular network address has
1510 * succeeded, that address will be used first for the next attempt to
1511 * connect. */
do_connect(ne_session * sess,struct host_info * host)1512 static int do_connect(ne_session *sess, struct host_info *host)
1513 {
1514 int ret;
1515
1516 /* Resolve hostname if necessary. */
1517 if (host->address == NULL && host->network == NULL) {
1518 ret = lookup_host(sess, host);
1519 if (ret) return ret;
1520 }
1521
1522 if ((sess->socket = ne_sock_create()) == NULL) {
1523 ne_set_error(sess, _("Could not create socket"));
1524 return NE_ERROR;
1525 }
1526
1527 if (sess->cotimeout)
1528 ne_sock_connect_timeout(sess->socket, sess->cotimeout);
1529
1530 if (sess->local_addr)
1531 ne_sock_prebind(sess->socket, sess->local_addr, 0);
1532
1533 if (host->current == NULL)
1534 host->current = resolve_first(host);
1535
1536 sess->status.ci.hostname = host->hostname;
1537
1538 struct timespec deadline_timeout;
1539 if(sess->rdtimeout > 0){
1540 davix_get_monotonic_time(&deadline_timeout);
1541 deadline_timeout.tv_sec += sess->rdtimeout;
1542 }
1543
1544 do {
1545
1546 sess->status.ci.address = host->current;
1547 notify_status(sess, ne_status_connecting);
1548 #ifdef NE_DEBUGGING
1549 if (ne_debug_mask & NE_DBG_HTTP) {
1550 char buf[150];
1551 NE_DEBUG(NE_DBG_HTTP, "req: Connecting to %s:%u",
1552 ne_iaddr_print(host->current, buf, sizeof buf),
1553 host->port);
1554 }
1555 #endif
1556 ret = ne_sock_connect(sess->socket, host->current, host->port);
1557
1558 if(ret && sess->rdtimeout > 0){
1559 struct timespec current_time;
1560 davix_get_monotonic_time(¤t_time);
1561 if(current_time.tv_sec > deadline_timeout.tv_sec){
1562 ret= NE_SOCK_TIMEOUT;
1563 break;
1564 }
1565 }
1566
1567 } while (ret && /* try the next address... */
1568 (host->current = resolve_next(host)) != NULL);
1569
1570 if (ret) {
1571 const char *msg;
1572
1573 if (host->proxy == PROXY_NONE)
1574 msg = _("Could not connect to server");
1575 else
1576 msg = _("Could not connect to proxy server");
1577
1578 ne_set_error(sess, "%s: %s", msg, ne_sock_error(sess->socket));
1579 ne_sock_close(sess->socket);
1580 return ret == NE_SOCK_TIMEOUT ? NE_TIMEOUT : NE_CONNECT;
1581 }
1582
1583 if (sess->rdtimeout)
1584 ne_sock_read_timeout(sess->socket, sess->rdtimeout);
1585
1586 notify_status(sess, ne_status_connected);
1587 sess->nexthop = host;
1588
1589 sess->connected = 1;
1590 /* clear persistent connection flag. */
1591 sess->persisted = 0;
1592 return NE_OK;
1593 }
1594
1595 /* For a SOCKSv4 proxy only, the IP address of the origin server (in
1596 * addition to the proxy) must be known, and must be an IPv4 address.
1597 * Returns NE_*; connection closed and error string set on error. */
socks_origin_lookup(ne_session * sess)1598 static int socks_origin_lookup(ne_session *sess)
1599 {
1600 const ne_inet_addr *ia;
1601 int ret;
1602
1603 ret = lookup_host(sess, &sess->server);
1604 if (ret) {
1605 /* lookup_host already set the error string. */
1606 ne_close_connection(sess);
1607 return ret;
1608 }
1609
1610 /* Find the first IPv4 address available for the server. */
1611 for (ia = ne_addr_first(sess->server.address);
1612 ia && ne_iaddr_typeof(ia) == ne_iaddr_ipv6;
1613 ia = ne_addr_next(sess->server.address)) {
1614 /* noop */
1615 }
1616
1617 /* ... if any */
1618 if (ia == NULL) {
1619 ne_set_error(sess, _("Could not find IPv4 address of "
1620 "hostname %s for SOCKS v4 proxy"),
1621 sess->server.hostname);
1622 ne_close_connection(sess);
1623 return NE_LOOKUP;
1624 }
1625
1626 sess->server.current = ia;
1627
1628 return ret;
1629 }
1630
open_connection(ne_session * sess)1631 static int open_connection(ne_session *sess)
1632 {
1633 int ret;
1634
1635 if (sess->connected) return NE_OK;
1636
1637 if (!sess->proxies) {
1638 ret = do_connect(sess, &sess->server);
1639 if (ret) {
1640 sess->nexthop = NULL;
1641 return ret;
1642 }
1643 }
1644 else {
1645 struct host_info *hi;
1646
1647 /* Attempt to re-use proxy to avoid iterating through
1648 * unnecessarily. */
1649 if (sess->prev_proxy)
1650 ret = do_connect(sess, sess->prev_proxy);
1651 else
1652 ret = NE_ERROR;
1653
1654 /* Otherwise, try everything - but omitting prev_proxy if that
1655 * has already been tried. */
1656 for (hi = sess->proxies; hi && ret; hi = hi->next) {
1657 if (hi != sess->prev_proxy)
1658 ret = do_connect(sess, hi);
1659 }
1660
1661 if (ret == NE_OK && sess->nexthop->proxy == PROXY_SOCKS) {
1662 /* Special-case for SOCKS v4 proxies, which require the
1663 * client to resolve the origin server IP address. */
1664 if (sess->socks_ver == NE_SOCK_SOCKSV4) {
1665 ret = socks_origin_lookup(sess);
1666 }
1667
1668 if (ret == NE_OK) {
1669 /* Perform the SOCKS handshake, instructing the proxy
1670 * to set up the connection to the origin server. */
1671 ret = ne_sock_proxy(sess->socket, sess->socks_ver,
1672 sess->server.current,
1673 sess->server.hostname, sess->server.port,
1674 sess->socks_user, sess->socks_password);
1675 if (ret) {
1676 ne_set_error(sess,
1677 _("Could not establish connection from "
1678 "SOCKS proxy (%s:%u): %s"),
1679 sess->nexthop->hostname,
1680 sess->nexthop->port,
1681 ne_sock_error(sess->socket));
1682 ne_close_connection(sess);
1683 ret = NE_ERROR;
1684 }
1685 }
1686 }
1687
1688 if (ret != NE_OK) {
1689 sess->nexthop = NULL;
1690 sess->prev_proxy = NULL;
1691 return ret;
1692 }
1693
1694 /* Success - make this proxy stick. */
1695 sess->prev_proxy = hi;
1696 }
1697
1698 #ifdef NE_HAVE_SSL
1699 /* Negotiate SSL layer if required. */
1700 if (sess->use_ssl && !sess->in_connect) {
1701 /* Set up CONNECT tunnel if using an HTTP proxy. */
1702 if (sess->nexthop->proxy == PROXY_HTTP)
1703 ret = proxy_tunnel(sess);
1704
1705 if (ret == NE_OK) {
1706 ret = ne__negotiate_ssl(sess);
1707 if (ret != NE_OK)
1708 ne_close_connection(sess);
1709 }
1710 }
1711 #endif
1712
1713 return ret;
1714 }
1715