1 /* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
2
3 #include "lib.h"
4 #include "net.h"
5 #include "str.h"
6 #include "hash.h"
7 #include "llist.h"
8 #include "array.h"
9 #include "ioloop.h"
10 #include "istream.h"
11 #include "istream-timeout.h"
12 #include "ostream.h"
13 #include "time-util.h"
14 #include "file-lock.h"
15 #include "iostream-rawlog.h"
16 #include "iostream-ssl.h"
17 #include "http-response-parser.h"
18
19 #include "http-client-private.h"
20
21 /*
22 * Connection
23 */
24
25 static void http_client_connection_ready(struct http_client_connection *conn);
26 static void http_client_connection_input(struct connection *_conn);
27 static void
28 http_client_connection_disconnect(struct http_client_connection *conn);
29
30 static inline const struct http_client_settings *
http_client_connection_get_settings(struct http_client_connection * conn)31 http_client_connection_get_settings(struct http_client_connection *conn)
32 {
33 if (conn->peer != NULL)
34 return &conn->peer->client->set;
35 return &conn->ppool->peer->cctx->set;
36 }
37
38 static inline void
http_client_connection_ref_request(struct http_client_connection * conn,struct http_client_request * req)39 http_client_connection_ref_request(struct http_client_connection *conn,
40 struct http_client_request *req)
41 {
42 i_assert(req->conn == NULL);
43 req->conn = conn;
44 http_client_request_ref(req);
45 }
46
47 static inline bool
http_client_connection_unref_request(struct http_client_connection * conn,struct http_client_request ** _req)48 http_client_connection_unref_request(struct http_client_connection *conn,
49 struct http_client_request **_req)
50 {
51 struct http_client_request *req = *_req;
52
53 i_assert(req->conn == conn);
54 req->conn = NULL;
55 return http_client_request_unref(_req);
56 }
57
58 static void
http_client_connection_unlist_pending(struct http_client_connection * conn)59 http_client_connection_unlist_pending(struct http_client_connection *conn)
60 {
61 struct http_client_peer *peer = conn->peer;
62 struct http_client_peer_pool *ppool = conn->ppool;
63 ARRAY_TYPE(http_client_connection) *conn_arr;
64 struct http_client_connection *const *conn_idx;
65
66 /* Remove from pending lists */
67
68 conn_arr = &ppool->pending_conns;
69 array_foreach(conn_arr, conn_idx) {
70 if (*conn_idx == conn) {
71 array_delete(conn_arr,
72 array_foreach_idx(conn_arr, conn_idx), 1);
73 break;
74 }
75 }
76
77 if (peer == NULL)
78 return;
79
80 conn_arr = &peer->pending_conns;
81 array_foreach(conn_arr, conn_idx) {
82 if (*conn_idx == conn) {
83 array_delete(conn_arr,
84 array_foreach_idx(conn_arr, conn_idx), 1);
85 break;
86 }
87 }
88 }
89
90 static inline void
http_client_connection_failure(struct http_client_connection * conn,const char * reason)91 http_client_connection_failure(struct http_client_connection *conn,
92 const char *reason)
93 {
94 struct http_client_peer *peer = conn->peer;
95
96 conn->connect_failed = TRUE;
97 http_client_connection_unlist_pending(conn);
98 http_client_peer_connection_failure(peer, reason);
99 }
100
101 unsigned int
http_client_connection_count_pending(struct http_client_connection * conn)102 http_client_connection_count_pending(struct http_client_connection *conn)
103 {
104 unsigned int pending_count = array_count(&conn->request_wait_list);
105
106 if (conn->in_req_callback || conn->pending_request != NULL)
107 pending_count++;
108 return pending_count;
109 }
110
http_client_connection_is_idle(struct http_client_connection * conn)111 bool http_client_connection_is_idle(struct http_client_connection *conn)
112 {
113 return conn->idle;
114 }
115
http_client_connection_is_active(struct http_client_connection * conn)116 bool http_client_connection_is_active(struct http_client_connection *conn)
117 {
118 if (!conn->connected)
119 return FALSE;
120
121 if (conn->in_req_callback || conn->pending_request != NULL)
122 return TRUE;
123
124 return (array_is_created(&conn->request_wait_list) &&
125 array_count(&conn->request_wait_list) > 0);
126 }
127
128 static void
http_client_connection_retry_requests(struct http_client_connection * conn,unsigned int status,const char * error)129 http_client_connection_retry_requests(struct http_client_connection *conn,
130 unsigned int status, const char *error)
131 {
132 struct http_client_request *req, **req_idx;
133
134 if (!array_is_created(&conn->request_wait_list))
135 return;
136
137 e_debug(conn->event, "Retrying pending requests");
138
139 array_foreach_modifiable(&conn->request_wait_list, req_idx) {
140 req = *req_idx;
141 /* Drop reference from connection */
142 if (!http_client_connection_unref_request(conn, req_idx))
143 continue;
144 /* Retry the request, which may drop it */
145 if (req->state < HTTP_REQUEST_STATE_FINISHED)
146 http_client_request_retry(req, status, error);
147 }
148 array_clear(&conn->request_wait_list);
149 }
150
151 static void
http_client_connection_server_close(struct http_client_connection ** _conn)152 http_client_connection_server_close(struct http_client_connection **_conn)
153 {
154 struct http_client_connection *conn = *_conn;
155 struct http_client_peer *peer = conn->peer;
156 struct http_client *client = peer->client;
157 struct http_client_request *req, **req_idx;
158
159 e_debug(conn->event, "Server explicitly closed connection");
160
161 array_foreach_modifiable(&conn->request_wait_list, req_idx) {
162 req = *req_idx;
163 /* Drop reference from connection */
164 if (!http_client_connection_unref_request(conn, req_idx))
165 continue;
166 /* Resubmit the request, which may drop it */
167 if (req->state < HTTP_REQUEST_STATE_FINISHED)
168 http_client_request_resubmit(req);
169 }
170 array_clear(&conn->request_wait_list);
171
172 if (client != NULL && client->waiting)
173 io_loop_stop(client->ioloop);
174
175 http_client_connection_close(_conn);
176 }
177
178 static void
http_client_connection_abort_error(struct http_client_connection ** _conn,unsigned int status,const char * error)179 http_client_connection_abort_error(struct http_client_connection **_conn,
180 unsigned int status, const char *error)
181 {
182 struct http_client_connection *conn = *_conn;
183 struct http_client_request *req, **req_idx;
184
185 e_debug(conn->event, "Aborting connection: %s", error);
186
187 array_foreach_modifiable(&conn->request_wait_list, req_idx) {
188 req = *req_idx;
189 i_assert(req->submitted);
190 /* Drop reference from connection */
191 if (!http_client_connection_unref_request(conn, req_idx))
192 continue;
193 /* Drop request if not already aborted */
194 http_client_request_error(&req, status, error);
195 }
196 array_clear(&conn->request_wait_list);
197 http_client_connection_close(_conn);
198 }
199
200 static void
http_client_connection_abort_any_requests(struct http_client_connection * conn)201 http_client_connection_abort_any_requests(struct http_client_connection *conn)
202 {
203 struct http_client_request *req, **req_idx;
204
205 if (array_is_created(&conn->request_wait_list)) {
206 array_foreach_modifiable(&conn->request_wait_list, req_idx) {
207 req = *req_idx;
208 i_assert(req->submitted);
209 /* Drop reference from connection */
210 if (!http_client_connection_unref_request(conn, req_idx))
211 continue;
212 /* Drop request if not already aborted */
213 http_client_request_error(
214 &req, HTTP_CLIENT_REQUEST_ERROR_ABORTED,
215 "Aborting");
216 }
217 array_clear(&conn->request_wait_list);
218 }
219 if (conn->pending_request != NULL) {
220 req = conn->pending_request;
221 /* Drop reference from connection */
222 if (http_client_connection_unref_request(
223 conn, &conn->pending_request)) {
224 /* Drop request if not already aborted */
225 http_client_request_error(
226 &req, HTTP_CLIENT_REQUEST_ERROR_ABORTED,
227 "Aborting");
228 }
229 }
230 }
231
232 static const char *
http_client_connection_get_timing_info(struct http_client_connection * conn)233 http_client_connection_get_timing_info(struct http_client_connection *conn)
234 {
235 struct http_client_request *const *requestp;
236 unsigned int connected_msecs;
237 string_t *str = t_str_new(64);
238
239 if (array_count(&conn->request_wait_list) > 0) {
240 requestp = array_front(&conn->request_wait_list);
241
242 str_append(str, "Request ");
243 http_client_request_append_stats_text(*requestp, str);
244 } else {
245 str_append(str, "No requests");
246 if (conn->conn.last_input != 0) {
247 str_printfa(str, ", last input %d secs ago",
248 (int)(ioloop_time - conn->conn.last_input));
249 }
250 }
251 connected_msecs = timeval_diff_msecs(&ioloop_timeval,
252 &conn->connected_timestamp);
253 str_printfa(str, ", connected %u.%03u secs ago",
254 connected_msecs/1000, connected_msecs%1000);
255 return str_c(str);
256 }
257
258 static void
http_client_connection_abort_temp_error(struct http_client_connection ** _conn,unsigned int status,const char * error)259 http_client_connection_abort_temp_error(struct http_client_connection **_conn,
260 unsigned int status, const char *error)
261 {
262 struct http_client_connection *conn = *_conn;
263
264 error = t_strdup_printf("%s (%s)", error,
265 http_client_connection_get_timing_info(conn));
266
267 e_debug(conn->event,
268 "Aborting connection with temporary error: %s", error);
269
270 http_client_connection_disconnect(conn);
271 http_client_connection_retry_requests(conn, status, error);
272 http_client_connection_close(_conn);
273 }
274
http_client_connection_lost(struct http_client_connection ** _conn,const char * error)275 void http_client_connection_lost(struct http_client_connection **_conn,
276 const char *error)
277 {
278 struct http_client_connection *conn = *_conn;
279 const char *sslerr;
280
281 if (error == NULL)
282 error = "Connection lost";
283 else
284 error = t_strdup_printf("Connection lost: %s", error);
285
286 if (conn->ssl_iostream != NULL) {
287 sslerr = ssl_iostream_get_last_error(conn->ssl_iostream);
288 if (sslerr != NULL) {
289 error = t_strdup_printf("%s (last SSL error: %s)",
290 error, sslerr);
291 }
292 if (ssl_iostream_has_handshake_failed(conn->ssl_iostream)) {
293 /* This isn't really a "connection lost", but that we
294 don't trust the remote's SSL certificate. don't
295 retry. */
296 http_client_connection_abort_error(
297 _conn,
298 HTTP_CLIENT_REQUEST_ERROR_BAD_RESPONSE, error);
299 return;
300 }
301 }
302
303 conn->lost_prematurely =
304 (conn->conn.input != NULL &&
305 conn->conn.input->v_offset == 0 &&
306 i_stream_get_data_size(conn->conn.input) == 0);
307 http_client_connection_abort_temp_error(
308 _conn, HTTP_CLIENT_REQUEST_ERROR_CONNECTION_LOST, error);
309 }
310
http_client_connection_handle_output_error(struct http_client_connection * conn)311 void http_client_connection_handle_output_error(
312 struct http_client_connection *conn)
313 {
314 struct ostream *output = conn->conn.output;
315
316 if (output->stream_errno != EPIPE &&
317 output->stream_errno != ECONNRESET) {
318 http_client_connection_lost(
319 &conn,
320 t_strdup_printf("write(%s) failed: %s",
321 o_stream_get_name(output),
322 o_stream_get_error(output)));
323 } else {
324 http_client_connection_lost(&conn, "Remote disconnected");
325 }
326 }
327
http_client_connection_check_ready(struct http_client_connection * conn)328 int http_client_connection_check_ready(struct http_client_connection *conn)
329 {
330 const struct http_client_settings *set =
331 http_client_connection_get_settings(conn);
332
333 if (conn->in_req_callback) {
334 /* This can happen when a nested ioloop is created inside
335 request callback. we currently don't reuse connections that
336 are occupied this way, but theoretically we could, although
337 that would add quite a bit of complexity.
338 */
339 return 0;
340 }
341
342 if (!conn->connected || conn->output_locked || conn->output_broken ||
343 conn->close_indicated || conn->tunneling ||
344 (http_client_connection_count_pending(conn) >=
345 set->max_pipelined_requests))
346 return 0;
347
348 if (conn->last_ioloop != NULL && conn->last_ioloop != current_ioloop) {
349 conn->last_ioloop = current_ioloop;
350 /* Active ioloop is different from what we saw earlier;
351 we may have missed a disconnection event on this connection.
352 Verify status by reading from connection. */
353 if (i_stream_read(conn->conn.input) == -1) {
354 int stream_errno = conn->conn.input->stream_errno;
355
356 i_assert(conn->conn.input->stream_errno != 0 ||
357 conn->conn.input->eof);
358 http_client_connection_lost(&conn,
359 t_strdup_printf("read(%s) failed: %s",
360 i_stream_get_name(conn->conn.input),
361 stream_errno != 0 ?
362 i_stream_get_error(conn->conn.input) :
363 "EOF"));
364 return -1;
365 }
366
367 /* We may have read some data */
368 if (i_stream_get_data_size(conn->conn.input) > 0)
369 i_stream_set_input_pending(conn->conn.input, TRUE);
370 }
371 return 1;
372 }
373
374 static void
http_client_connection_detach_peer(struct http_client_connection * conn)375 http_client_connection_detach_peer(struct http_client_connection *conn)
376 {
377 struct http_client_peer *peer = conn->peer;
378 struct http_client_connection *const *conn_idx;
379 ARRAY_TYPE(http_client_connection) *conn_arr;
380 bool found = FALSE;
381
382 if (peer == NULL)
383 return;
384
385 http_client_peer_ref(peer);
386 conn_arr = &peer->conns;
387 array_foreach(conn_arr, conn_idx) {
388 if (*conn_idx == conn) {
389 array_delete(conn_arr,
390 array_foreach_idx(conn_arr, conn_idx), 1);
391 found = TRUE;
392 break;
393 }
394 }
395 i_assert(found);
396
397 conn_arr = &peer->pending_conns;
398 array_foreach(conn_arr, conn_idx) {
399 if (*conn_idx == conn) {
400 array_delete(conn_arr,
401 array_foreach_idx(conn_arr, conn_idx), 1);
402 break;
403 }
404 }
405
406 conn->peer = NULL;
407 e_debug(conn->event, "Detached peer");
408
409 if (conn->connect_succeeded)
410 http_client_peer_connection_lost(peer, conn->lost_prematurely);
411 http_client_peer_unref(&peer);
412 }
413
414 static void
http_client_connection_idle_timeout(struct http_client_connection * conn)415 http_client_connection_idle_timeout(struct http_client_connection *conn)
416 {
417 e_debug(conn->event, "Idle connection timed out");
418
419 /* Cannot get here unless connection was established at some point */
420 i_assert(conn->connect_succeeded);
421
422 http_client_connection_close(&conn);
423 }
424
425 static unsigned int
http_client_connection_start_idle_timeout(struct http_client_connection * conn)426 http_client_connection_start_idle_timeout(struct http_client_connection *conn)
427 {
428 const struct http_client_settings *set =
429 http_client_connection_get_settings(conn);
430 struct http_client_peer_pool *ppool = conn->ppool;
431 struct http_client_peer_shared *pshared = ppool->peer;
432 unsigned int timeout, count, idle_count, max;
433
434 i_assert(conn->to_idle == NULL);
435
436 if (set->max_idle_time_msecs == 0)
437 return UINT_MAX;
438
439 count = array_count(&ppool->conns);
440 idle_count = array_count(&ppool->idle_conns);
441 max = http_client_peer_shared_max_connections(pshared);
442 i_assert(count > 0);
443 i_assert(count >= idle_count + 1);
444 i_assert(max > 0);
445
446 /* Set timeout for this connection */
447 if (idle_count == 0 || max == UINT_MAX) {
448 /* No idle connections yet or infinite connections allowed;
449 use the maximum idle time. */
450 timeout = set->max_idle_time_msecs;
451 } else if (count > max || idle_count >= max) {
452 /* Instant death for (urgent) connections above limit */
453 timeout = 0;
454 } else {
455 unsigned int idle_slots_avail;
456 double idle_time_per_slot;
457
458 /* Kill duplicate connections quicker;
459 linearly based on the number of connections */
460 idle_slots_avail = max - idle_count;
461 idle_time_per_slot = (double)set->max_idle_time_msecs / max;
462 timeout = (unsigned int)(idle_time_per_slot * idle_slots_avail);
463 if (timeout < HTTP_CLIENT_MIN_IDLE_TIMEOUT_MSECS)
464 timeout = HTTP_CLIENT_MIN_IDLE_TIMEOUT_MSECS;
465 }
466
467 conn->to_idle = timeout_add_short_to(
468 conn->conn.ioloop, timeout,
469 http_client_connection_idle_timeout, conn);
470 return timeout;
471 }
472
473 static void
http_client_connection_start_idle(struct http_client_connection * conn,const char * reason)474 http_client_connection_start_idle(struct http_client_connection *conn,
475 const char *reason)
476 {
477 struct http_client_peer_pool *ppool = conn->ppool;
478 unsigned int timeout;
479
480 if (conn->idle) {
481 e_debug(conn->event, "%s; already idle", reason);
482 return;
483 }
484
485 timeout = http_client_connection_start_idle_timeout(conn);
486 if (timeout == UINT_MAX)
487 e_debug(conn->event, "%s; going idle", reason);
488 else {
489 e_debug(conn->event, "%s; going idle (timeout = %u msecs)",
490 reason, timeout);
491 }
492
493 conn->idle = TRUE;
494 array_push_back(&ppool->idle_conns, &conn);
495 }
496
http_client_connection_lost_peer(struct http_client_connection * conn)497 void http_client_connection_lost_peer(struct http_client_connection *conn)
498 {
499 if (!conn->connected) {
500 http_client_connection_unref(&conn);
501 return;
502 }
503
504 i_assert(!conn->in_req_callback);
505
506 http_client_connection_start_idle(conn, "Lost peer");
507 http_client_connection_detach_peer(conn);
508 }
509
http_client_connection_check_idle(struct http_client_connection * conn)510 void http_client_connection_check_idle(struct http_client_connection *conn)
511 {
512 struct http_client_peer *peer;
513
514 peer = conn->peer;
515 if (peer == NULL) {
516 i_assert(conn->idle);
517 return;
518 }
519
520 if (conn->idle) {
521 /* Already idle */
522 return;
523 }
524
525 if (conn->connected && !http_client_connection_is_active(conn)) {
526 struct http_client *client = peer->client;
527
528 i_assert(conn->to_requests == NULL);
529
530 if (client->waiting)
531 io_loop_stop(client->ioloop);
532
533 http_client_connection_start_idle(
534 conn, "No more requests queued");
535 }
536 }
537
538 static void
http_client_connection_stop_idle(struct http_client_connection * conn)539 http_client_connection_stop_idle(struct http_client_connection *conn)
540 {
541 struct http_client_connection *const *conn_idx;
542 ARRAY_TYPE(http_client_connection) *conn_arr;
543
544 timeout_remove(&conn->to_idle);
545 conn->idle = FALSE;
546
547 conn_arr = &conn->ppool->idle_conns;
548 array_foreach(conn_arr, conn_idx) {
549 if (*conn_idx == conn) {
550 array_delete(conn_arr,
551 array_foreach_idx(conn_arr, conn_idx), 1);
552 break;
553 }
554 }
555 }
556
http_client_connection_claim_idle(struct http_client_connection * conn,struct http_client_peer * peer)557 void http_client_connection_claim_idle(struct http_client_connection *conn,
558 struct http_client_peer *peer)
559 {
560 e_debug(conn->event, "Claimed as idle");
561
562 i_assert(peer->ppool == conn->ppool);
563 http_client_connection_stop_idle(conn);
564
565 if (conn->peer == NULL || conn->peer != peer) {
566 http_client_connection_detach_peer(conn);
567
568 conn->peer = peer;
569 conn->debug = peer->client->set.debug;
570 array_push_back(&peer->conns, &conn);
571 }
572 }
573
574 static void
http_client_connection_request_timeout(struct http_client_connection * conn)575 http_client_connection_request_timeout(struct http_client_connection *conn)
576 {
577 conn->conn.input->stream_errno = ETIMEDOUT;
578 http_client_connection_abort_temp_error(
579 &conn, HTTP_CLIENT_REQUEST_ERROR_TIMED_OUT,
580 "Request timed out");
581 }
582
http_client_connection_start_request_timeout(struct http_client_connection * conn)583 void http_client_connection_start_request_timeout(
584 struct http_client_connection *conn)
585 {
586 struct http_client_request *const *requestp;
587 unsigned int timeout_msecs;
588
589 if (conn->pending_request != NULL)
590 return;
591
592 i_assert(array_is_created(&conn->request_wait_list));
593 i_assert(array_count(&conn->request_wait_list) > 0);
594 requestp = array_front(&conn->request_wait_list);
595 timeout_msecs = (*requestp)->attempt_timeout_msecs;
596
597 if (timeout_msecs == 0)
598 ;
599 else if (conn->to_requests != NULL)
600 timeout_reset(conn->to_requests);
601 else {
602 conn->to_requests = timeout_add_to(
603 conn->conn.ioloop, timeout_msecs,
604 http_client_connection_request_timeout, conn);
605 }
606 }
607
http_client_connection_reset_request_timeout(struct http_client_connection * conn)608 void http_client_connection_reset_request_timeout(
609 struct http_client_connection *conn)
610 {
611 if (conn->to_requests != NULL)
612 timeout_reset(conn->to_requests);
613 }
614
http_client_connection_stop_request_timeout(struct http_client_connection * conn)615 void http_client_connection_stop_request_timeout(
616 struct http_client_connection *conn)
617 {
618 timeout_remove(&conn->to_requests);
619 }
620
621 static void
http_client_connection_continue_timeout(struct http_client_connection * conn)622 http_client_connection_continue_timeout(struct http_client_connection *conn)
623 {
624 struct http_client_peer_shared *pshared = conn->ppool->peer;
625 struct http_client_request *const *wait_reqs;
626 struct http_client_request *req;
627 unsigned int wait_count;
628
629 i_assert(conn->pending_request == NULL);
630
631 timeout_remove(&conn->to_response);
632 pshared->no_payload_sync = TRUE;
633
634 e_debug(conn->event,
635 "Expected 100-continue response timed out; "
636 "sending payload anyway");
637
638 wait_reqs = array_get(&conn->request_wait_list, &wait_count);
639 i_assert(wait_count == 1);
640 req = wait_reqs[wait_count-1];
641
642 req->payload_sync_continue = TRUE;
643 if (conn->conn.output != NULL)
644 o_stream_set_flush_pending(conn->conn.output, TRUE);
645 }
646
http_client_connection_next_request(struct http_client_connection * conn)647 int http_client_connection_next_request(struct http_client_connection *conn)
648 {
649 struct http_client_connection *tmp_conn;
650 struct http_client_peer *peer = conn->peer;
651 struct http_client_peer_shared *pshared = conn->ppool->peer;
652 struct http_client_request *req = NULL;
653 bool pipelined;
654 int ret;
655
656 if ((ret = http_client_connection_check_ready(conn)) <= 0) {
657 if (ret == 0)
658 e_debug(conn->event, "Not ready for next request");
659 return ret;
660 }
661
662 /* Claim request, but no urgent request can be second in line */
663 pipelined = (array_count(&conn->request_wait_list) > 0 ||
664 conn->pending_request != NULL);
665 req = http_client_peer_claim_request(peer, pipelined);
666 if (req == NULL)
667 return 0;
668
669 i_assert(req->state == HTTP_REQUEST_STATE_QUEUED);
670
671 http_client_connection_stop_idle(conn);
672
673 req->payload_sync_continue = FALSE;
674 if (pshared->no_payload_sync)
675 req->payload_sync = FALSE;
676
677 /* Add request to wait list and add a reference */
678 array_push_back(&conn->request_wait_list, &req);
679 http_client_connection_ref_request(conn, req);
680
681 e_debug(conn->event, "Claimed request %s",
682 http_client_request_label(req));
683
684 tmp_conn = conn;
685 http_client_connection_ref(tmp_conn);
686 ret = http_client_request_send(req, pipelined);
687 if (ret == 0 && conn->conn.output != NULL)
688 o_stream_set_flush_pending(conn->conn.output, TRUE);
689 if (!http_client_connection_unref(&tmp_conn) || ret < 0)
690 return -1;
691
692 if (req->connect_tunnel)
693 conn->tunneling = TRUE;
694
695 /* RFC 7231, Section 5.1.1: Expect
696
697 o A client that sends a 100-continue expectation is not required to
698 wait for any specific length of time; such a client MAY proceed
699 to send the message body even if it has not yet received a
700 response. Furthermore, since 100 (Continue) responses cannot be
701 sent through an HTTP/1.0 intermediary, such a client SHOULD NOT
702 wait for an indefinite period before sending the message body.
703 */
704 if (req->payload_sync && !pshared->seen_100_response) {
705 i_assert(!pipelined);
706 i_assert(req->payload_chunked || req->payload_size > 0);
707 i_assert(conn->to_response == NULL);
708 conn->to_response = timeout_add_to(
709 conn->conn.ioloop, HTTP_CLIENT_CONTINUE_TIMEOUT_MSECS,
710 http_client_connection_continue_timeout, conn);
711 }
712
713 return 1;
714 }
715
http_client_connection_destroy(struct connection * _conn)716 static void http_client_connection_destroy(struct connection *_conn)
717 {
718 struct http_client_connection *conn =
719 (struct http_client_connection *)_conn;
720 const char *error;
721 unsigned int msecs;
722
723 switch (_conn->disconnect_reason) {
724 case CONNECTION_DISCONNECT_CONNECT_TIMEOUT:
725 if (conn->connected_timestamp.tv_sec == 0) {
726 msecs = timeval_diff_msecs(
727 &ioloop_timeval,
728 &conn->connect_start_timestamp);
729 error = t_strdup_printf(
730 "connect(%s) failed: "
731 "Connection timed out in %u.%03u secs",
732 _conn->name, msecs/1000, msecs%1000);
733 } else {
734 msecs = timeval_diff_msecs(&ioloop_timeval,
735 &conn->connected_timestamp);
736 error = t_strdup_printf(
737 "SSL handshaking with %s failed: "
738 "Connection timed out in %u.%03u secs",
739 _conn->name, msecs/1000, msecs%1000);
740 }
741 e_debug(conn->event, "%s", error);
742 http_client_connection_failure(conn, error);
743 break;
744 case CONNECTION_DISCONNECT_CONN_CLOSED:
745 if (conn->connect_failed) {
746 i_assert(!array_is_created(&conn->request_wait_list) ||
747 array_count(&conn->request_wait_list) == 0);
748 break;
749 }
750 http_client_connection_lost(
751 &conn, (_conn->input == NULL ?
752 NULL : i_stream_get_error(_conn->input)));
753 return;
754 default:
755 break;
756 }
757
758 http_client_connection_close(&conn);
759 }
760
http_client_payload_finished(struct http_client_connection * conn)761 static void http_client_payload_finished(struct http_client_connection *conn)
762 {
763 timeout_remove(&conn->to_input);
764 connection_input_resume(&conn->conn);
765 if (array_count(&conn->request_wait_list) > 0)
766 http_client_connection_start_request_timeout(conn);
767 else
768 http_client_connection_stop_request_timeout(conn);
769 }
770
771 static void
http_client_payload_destroyed_timeout(struct http_client_connection * conn)772 http_client_payload_destroyed_timeout(struct http_client_connection *conn)
773 {
774 if (conn->close_indicated) {
775 http_client_connection_server_close(&conn);
776 return;
777 }
778 http_client_connection_input(&conn->conn);
779 }
780
http_client_payload_destroyed(struct http_client_request * req)781 static void http_client_payload_destroyed(struct http_client_request *req)
782 {
783 struct http_client_connection *conn = req->conn;
784
785 i_assert(conn != NULL);
786 i_assert(conn->pending_request == req);
787 i_assert(conn->incoming_payload != NULL);
788 i_assert(conn->conn.io == NULL);
789
790 e_debug(conn->event,
791 "Response payload stream destroyed "
792 "(%u ms after initial response)",
793 timeval_diff_msecs(&ioloop_timeval, &req->response_time));
794
795 /* Caller is allowed to change the socket fd to blocking while reading
796 the payload. make sure here that it's switched back. */
797 net_set_nonblock(conn->conn.fd_in, TRUE);
798
799 i_assert(req->response_offset < conn->conn.input->v_offset);
800 req->bytes_in = conn->conn.input->v_offset - req->response_offset;
801
802 /* Drop reference from connection */
803 if (http_client_connection_unref_request(
804 conn, &conn->pending_request)) {
805 /* Finish request if not already aborted */
806 http_client_request_finish(req);
807 }
808
809 conn->incoming_payload = NULL;
810
811 /* Input stream may have pending input. make sure input handler
812 gets called (but don't do it directly, since we get get here
813 somewhere from the API user's code, which we can't really know what
814 state it is in). this call also triggers sending a new request if
815 necessary. */
816 if (!conn->disconnected) {
817 conn->to_input = timeout_add_short_to(
818 conn->conn.ioloop, 0,
819 http_client_payload_destroyed_timeout, conn);
820 }
821
822 /* Room for new requests */
823 if (http_client_connection_check_ready(conn) > 0)
824 http_client_peer_trigger_request_handler(conn->peer);
825 }
826
http_client_connection_request_destroyed(struct http_client_connection * conn,struct http_client_request * req)827 void http_client_connection_request_destroyed(
828 struct http_client_connection *conn, struct http_client_request *req)
829 {
830 struct istream *payload;
831
832 i_assert(req->conn == conn);
833 if (conn->pending_request != req)
834 return;
835
836 e_debug(conn->event, "Pending request destroyed prematurely");
837
838 payload = conn->incoming_payload;
839 if (payload == NULL) {
840 /* Payload already gone */
841 return;
842 }
843
844 /* Destroy the payload, so that the timeout istream is closed */
845 i_stream_ref(payload);
846 i_stream_destroy(&payload);
847
848 payload = conn->incoming_payload;
849 if (payload == NULL) {
850 /* Not going to happen, but check for it anyway */
851 return;
852 }
853
854 /* The application still holds a reference to the payload stream, but it
855 is closed and we don't care about it anymore, so act as though it is
856 destroyed. */
857 i_stream_remove_destroy_callback(payload,
858 http_client_payload_destroyed);
859 http_client_payload_destroyed(req);
860 }
861
862 static bool
http_client_connection_return_response(struct http_client_connection * conn,struct http_client_request * req,struct http_response * response)863 http_client_connection_return_response(struct http_client_connection *conn,
864 struct http_client_request *req,
865 struct http_response *response)
866 {
867 struct http_client_peer_shared *pshared = conn->ppool->peer;
868 struct istream *payload;
869 bool retrying;
870
871 i_assert(!conn->in_req_callback);
872 i_assert(conn->incoming_payload == NULL);
873 i_assert(conn->pending_request == NULL);
874
875 http_client_connection_ref(conn);
876 http_client_connection_ref_request(conn, req);
877 req->state = HTTP_REQUEST_STATE_GOT_RESPONSE;
878
879 if (response->payload != NULL) {
880 /* Wrap the stream to capture the destroy event without
881 destroying the actual payload stream. we are already expected
882 to be on the correct ioloop, so there should be no need to
883 switch the stream's ioloop here. */
884 conn->incoming_payload = response->payload =
885 i_stream_create_timeout(response->payload,
886 req->attempt_timeout_msecs);
887 i_stream_add_destroy_callback(response->payload,
888 http_client_payload_destroyed,
889 req);
890 /* The callback may add its own I/O, so we need to remove
891 our one before calling it */
892 connection_input_halt(&conn->conn);
893 /* We've received the request itself, and we can't reset the
894 timeout during the payload reading. */
895 http_client_connection_stop_request_timeout(conn);
896 }
897
898 conn->in_req_callback = TRUE;
899 retrying = !http_client_request_callback(req, response);
900 if (conn->disconnected) {
901 /* The callback managed to get this connection disconnected */
902 if (!retrying)
903 http_client_request_finish(req);
904 http_client_connection_unref_request(conn, &req);
905 http_client_connection_unref(&conn);
906 return FALSE;
907 }
908 conn->in_req_callback = FALSE;
909
910 if (retrying) {
911 /* Retrying, don't destroy the request */
912 if (response->payload != NULL) {
913 i_stream_remove_destroy_callback(conn->incoming_payload,
914 http_client_payload_destroyed);
915 i_stream_unref(&conn->incoming_payload);
916 connection_input_resume(&conn->conn);
917 }
918 http_client_connection_unref_request(conn, &req);
919 return http_client_connection_unref(&conn);
920 }
921
922 if (response->payload != NULL) {
923 req->state = HTTP_REQUEST_STATE_PAYLOAD_IN;
924 payload = response->payload;
925 response->payload = NULL;
926
927 /* Maintain request reference while payload is pending */
928 conn->pending_request = req;
929
930 /* Request is dereferenced in payload destroy callback */
931 i_stream_unref(&payload);
932
933 if (conn->to_input != NULL && conn->conn.input != NULL) {
934 /* Already finished reading the payload */
935 http_client_payload_finished(conn);
936 }
937 } else {
938 http_client_request_finish(req);
939 http_client_connection_unref_request(conn, &req);
940 }
941
942 if (conn->incoming_payload == NULL && conn->conn.input != NULL) {
943 i_assert(conn->conn.io != NULL ||
944 pshared->addr.type == HTTP_CLIENT_PEER_ADDR_RAW);
945 return http_client_connection_unref(&conn);
946 }
947 http_client_connection_unref(&conn);
948 return FALSE;
949 }
950
951 static const char *
http_client_request_add_event_headers(struct http_client_request * req,const struct http_response * response)952 http_client_request_add_event_headers(struct http_client_request *req,
953 const struct http_response *response)
954 {
955 if (req->event_headers == NULL)
956 return "";
957
958 string_t *str = t_str_new(128);
959 for (unsigned int i = 0; req->event_headers[i] != NULL; i++) {
960 const char *hdr_name = req->event_headers[i];
961 const char *value = http_response_header_get(response, hdr_name);
962
963 if (value == NULL)
964 continue;
965
966 str_append(str, str_len(str) == 0 ? " (" : ", ");
967 event_add_str(req->event,
968 t_strconcat("http_hdr_", hdr_name, NULL), value);
969 str_printfa(str, "%s:%s", hdr_name, value);
970 }
971 if (str_len(str) > 0)
972 str_append_c(str, ')');
973 return str_c(str);
974 }
975
http_client_connection_input(struct connection * _conn)976 static void http_client_connection_input(struct connection *_conn)
977 {
978 struct http_client_connection *conn =
979 (struct http_client_connection *)_conn;
980 struct http_client_peer *peer = conn->peer;
981 struct http_client_peer_shared *pshared = conn->ppool->peer;
982 struct http_response response;
983 struct http_client_request *const *reqs;
984 struct http_client_request *req = NULL, *req_ref;
985 enum http_response_payload_type payload_type;
986 unsigned int count;
987 int finished = 0, ret;
988 const char *error;
989
990 i_assert(conn->incoming_payload == NULL);
991
992 _conn->last_input = ioloop_time;
993
994 if (conn->ssl_iostream != NULL &&
995 !ssl_iostream_is_handshaked(conn->ssl_iostream)) {
996 /* Finish SSL negotiation by reading from input stream */
997 while ((ret = i_stream_read(conn->conn.input)) > 0 ||
998 ret == -2) {
999 if (ssl_iostream_is_handshaked(conn->ssl_iostream))
1000 break;
1001 }
1002 if (ret < 0) {
1003 int stream_errno = conn->conn.input->stream_errno;
1004
1005 /* Failed somehow */
1006 i_assert(ret != -2);
1007 error = t_strdup_printf(
1008 "SSL handshaking with %s failed: "
1009 "read(%s) failed: %s",
1010 _conn->name,
1011 i_stream_get_name(conn->conn.input),
1012 (stream_errno != 0 ?
1013 i_stream_get_error(conn->conn.input) : "EOF"));
1014 http_client_connection_failure(conn, error);
1015 e_debug(conn->event, "%s", error);
1016 http_client_connection_close(&conn);
1017 return;
1018 }
1019
1020 if (!ssl_iostream_is_handshaked(conn->ssl_iostream)) {
1021 /* Not finished */
1022 i_assert(ret == 0);
1023 return;
1024 }
1025 }
1026
1027 if (!conn->connect_succeeded) {
1028 /* Just got ready for first request */
1029 http_client_connection_ready(conn);
1030 }
1031
1032 if (conn->to_input != NULL) {
1033 /* We came here from a timeout added by
1034 http_client_payload_destroyed(). The IO couldn't be added
1035 back immediately in there, because the HTTP API user may
1036 still have had its own IO pointed to the same fd. It should
1037 be removed by now, so we can add it back. */
1038 http_client_payload_finished(conn);
1039 finished++;
1040 }
1041
1042 /* We've seen activity from the server; reset request timeout */
1043 http_client_connection_reset_request_timeout(conn);
1044
1045 /* Get first waiting request */
1046 reqs = array_get(&conn->request_wait_list, &count);
1047 if (count > 0) {
1048 req = reqs[0];
1049
1050 /* Determine whether to expect a response payload */
1051 payload_type = http_client_request_get_payload_type(req);
1052 } else {
1053 req = NULL;
1054 payload_type = HTTP_RESPONSE_PAYLOAD_TYPE_ALLOWED;
1055 i_assert(conn->to_requests == NULL);
1056 }
1057
1058 /* Drop connection with broken output if last possible input was
1059 received */
1060 if (conn->output_broken && (count == 0 ||
1061 (count == 1 && req->state == HTTP_REQUEST_STATE_ABORTED))) {
1062 http_client_connection_server_close(&conn);
1063 return;
1064 }
1065
1066 while ((ret = http_response_parse_next(conn->http_parser, payload_type,
1067 &response, &error)) > 0) {
1068 bool aborted, early = FALSE;
1069
1070 if (req == NULL) {
1071 /* Server sent response without any requests in the wait
1072 list */
1073 if (response.status == 408) {
1074 e_debug(conn->event,
1075 "Server explicitly closed connection: "
1076 "408 %s", response.reason);
1077 } else {
1078 e_debug(conn->event,
1079 "Got unexpected input from server: "
1080 "%u %s", response.status,
1081 response.reason);
1082 }
1083 http_client_connection_close(&conn);
1084 return;
1085 }
1086
1087 req->response_time = ioloop_timeval;
1088 req->response_offset =
1089 http_response_parser_get_last_offset(conn->http_parser);
1090 i_assert(req->response_offset != UOFF_T_MAX);
1091 i_assert(req->response_offset < conn->conn.input->v_offset);
1092 req->bytes_in = conn->conn.input->v_offset - req->response_offset;
1093
1094 /* Got some response; cancel response timeout */
1095 timeout_remove(&conn->to_response);
1096
1097 /* RFC 7231, Section 6.2:
1098
1099 A client MUST be able to parse one or more 1xx responses
1100 received prior to a final response, even if the client does
1101 not expect one. A user agent MAY ignore unexpected 1xx
1102 responses.
1103 */
1104 if (req->payload_sync && response.status == 100) {
1105 if (req->payload_sync_continue) {
1106 e_debug(conn->event,
1107 "Got 100-continue response after timeout");
1108 continue;
1109 }
1110
1111 pshared->no_payload_sync = FALSE;
1112 pshared->seen_100_response = TRUE;
1113 req->payload_sync_continue = TRUE;
1114
1115 e_debug(conn->event,
1116 "Got expected 100-continue response");
1117
1118 if (req->state == HTTP_REQUEST_STATE_ABORTED) {
1119 e_debug(conn->event,
1120 "Request aborted before sending payload was complete.");
1121 http_client_connection_close(&conn);
1122 return;
1123 }
1124
1125 if (conn->conn.output != NULL)
1126 o_stream_set_flush_pending(conn->conn.output, TRUE);
1127 return;
1128 } else if (response.status / 100 == 1) {
1129 /* Ignore other 1xx for now */
1130 e_debug(conn->event,
1131 "Got unexpected %u response; ignoring",
1132 response.status);
1133 continue;
1134 } else if (!req->payload_sync && !req->payload_finished &&
1135 req->state == HTTP_REQUEST_STATE_PAYLOAD_OUT) {
1136 /* Got early response from server while we're still
1137 sending request payload. we cannot recover from this
1138 reliably, so we stop sending payload and close the
1139 connection once the response is processed */
1140 e_debug(conn->event,
1141 "Got early input from server; "
1142 "request payload not completely sent "
1143 "(will close connection)");
1144 o_stream_unset_flush_callback(conn->conn.output);
1145 conn->output_broken = early = TRUE;
1146 }
1147
1148 const char *suffix =
1149 http_client_request_add_event_headers(req, &response);
1150 e_debug(conn->event,
1151 "Got %u response for request %s: %s%s "
1152 "(took %u ms + %u ms in queue)",
1153 response.status, http_client_request_label(req),
1154 response.reason, suffix,
1155 timeval_diff_msecs(&req->response_time, &req->sent_time),
1156 timeval_diff_msecs(&req->sent_time, &req->submit_time));
1157
1158 /* Make sure connection output is unlocked if 100-continue
1159 failed */
1160 if (req->payload_sync && !req->payload_sync_continue) {
1161 e_debug(conn->event, "Unlocked output");
1162 conn->output_locked = FALSE;
1163 }
1164
1165 /* Remove request from queue */
1166 array_pop_front(&conn->request_wait_list);
1167 aborted = (req->state == HTTP_REQUEST_STATE_ABORTED);
1168 req_ref = req;
1169 if (!http_client_connection_unref_request(conn, &req_ref)) {
1170 i_assert(aborted);
1171 req = NULL;
1172 }
1173
1174 conn->close_indicated = response.connection_close;
1175
1176 if (!aborted) {
1177 bool handled = FALSE;
1178
1179 /* Response cannot be 2xx if request payload was not
1180 completely sent */
1181 if (early && response.status / 100 == 2) {
1182 http_client_request_error(
1183 &req, HTTP_CLIENT_REQUEST_ERROR_BAD_RESPONSE,
1184 "Server responded with success response "
1185 "before all payload was sent");
1186 http_client_connection_close(&conn);
1187 return;
1188 }
1189
1190 /* Don't redirect/retry if we're sending data in small
1191 blocks via http_client_request_send_payload()
1192 and we're not waiting for 100 continue */
1193 if (!req->payload_wait ||
1194 (req->payload_sync && !req->payload_sync_continue)) {
1195 /* Failed Expect: */
1196 if (response.status == 417 && req->payload_sync) {
1197 /* Drop Expect: continue */
1198 req->payload_sync = FALSE;
1199 conn->output_locked = FALSE;
1200 pshared->no_payload_sync = TRUE;
1201 if (http_client_request_try_retry(req))
1202 handled = TRUE;
1203 /* Redirection */
1204 } else if (!req->client->set.no_auto_redirect &&
1205 response.status / 100 == 3 &&
1206 response.status != 304 &&
1207 response.location != NULL) {
1208 /* Redirect (possibly after delay) */
1209 if (http_client_request_delay_from_response(
1210 req, &response) >= 0) {
1211 http_client_request_redirect(
1212 req, response.status,
1213 response.location);
1214 handled = TRUE;
1215 }
1216 /* Service unavailable */
1217 } else if (response.status == 503) {
1218 /* Automatically retry after delay if
1219 indicated */
1220 if (response.retry_after != (time_t)-1 &&
1221 http_client_request_delay_from_response(
1222 req, &response) > 0 &&
1223 http_client_request_try_retry(req))
1224 handled = TRUE;
1225 /* Request timeout (by server) */
1226 } else if (response.status == 408) {
1227 /* Automatically retry */
1228 if (http_client_request_try_retry(req))
1229 handled = TRUE;
1230 /* Connection close is implicit,
1231 although server should indicate that
1232 explicitly */
1233 conn->close_indicated = TRUE;
1234 }
1235 }
1236
1237 if (!handled) {
1238 /* Response for application */
1239 if (!http_client_connection_return_response(
1240 conn, req, &response))
1241 return;
1242 }
1243 }
1244
1245 finished++;
1246
1247 /* Server closing connection? */
1248 if (conn->close_indicated) {
1249 http_client_connection_server_close(&conn);
1250 return;
1251 }
1252
1253 /* Get next waiting request */
1254 reqs = array_get(&conn->request_wait_list, &count);
1255 if (count > 0) {
1256 req = reqs[0];
1257
1258 /* Determine whether to expect a response payload */
1259 payload_type = http_client_request_get_payload_type(req);
1260 } else {
1261 /* No more requests waiting for the connection */
1262 req = NULL;
1263 payload_type = HTTP_RESPONSE_PAYLOAD_TYPE_ALLOWED;
1264 http_client_connection_stop_request_timeout(conn);
1265 }
1266
1267 /* Drop connection with broken output if last possible input was
1268 received */
1269 if (conn->output_broken && (count == 0 ||
1270 (count == 1 && req->state == HTTP_REQUEST_STATE_ABORTED))) {
1271 http_client_connection_server_close(&conn);
1272 return;
1273 }
1274 }
1275
1276 if (ret <= 0 &&
1277 (conn->conn.input->eof || conn->conn.input->stream_errno != 0)) {
1278 int stream_errno = conn->conn.input->stream_errno;
1279 http_client_connection_lost(
1280 &conn,
1281 t_strdup_printf("read(%s) failed: %s",
1282 i_stream_get_name(conn->conn.input),
1283 (stream_errno != 0 ?
1284 i_stream_get_error(conn->conn.input) :
1285 "EOF")));
1286 return;
1287 }
1288
1289 if (ret < 0) {
1290 http_client_connection_abort_error(
1291 &conn, HTTP_CLIENT_REQUEST_ERROR_BAD_RESPONSE, error);
1292 return;
1293 }
1294
1295 if (finished > 0) {
1296 /* Connection still alive after (at least one) request;
1297 we can pipeline -> mark for subsequent connections */
1298 pshared->allows_pipelining = TRUE;
1299
1300 /* Room for new requests */
1301 if (peer != NULL &&
1302 http_client_connection_check_ready(conn) > 0)
1303 http_client_peer_trigger_request_handler(peer);
1304 }
1305 }
1306
1307 static int
http_client_connection_continue_request(struct http_client_connection * conn)1308 http_client_connection_continue_request(struct http_client_connection *conn)
1309 {
1310 struct http_client_connection *tmp_conn;
1311 struct http_client_request *const *reqs;
1312 unsigned int count;
1313 struct http_client_request *req;
1314 bool pipelined;
1315 int ret;
1316
1317 reqs = array_get(&conn->request_wait_list, &count);
1318 i_assert(count > 0 || conn->to_requests == NULL);
1319 if (count == 0 || !conn->output_locked)
1320 return 1;
1321
1322 req = reqs[count-1];
1323 pipelined = (count > 1 || conn->pending_request != NULL);
1324
1325 if (req->state == HTTP_REQUEST_STATE_ABORTED) {
1326 e_debug(conn->event,
1327 "Request aborted before sending payload was complete.");
1328 if (count == 1) {
1329 http_client_connection_close(&conn);
1330 return -1;
1331 }
1332 o_stream_unset_flush_callback(conn->conn.output);
1333 conn->output_broken = TRUE;
1334 return -1;
1335 }
1336
1337 if (req->payload_sync && !req->payload_sync_continue)
1338 return 1;
1339
1340 tmp_conn = conn;
1341 http_client_connection_ref(tmp_conn);
1342 ret = http_client_request_send_more(req, pipelined);
1343 if (!http_client_connection_unref(&tmp_conn) || ret < 0)
1344 return -1;
1345
1346 if (!conn->output_locked) {
1347 /* Room for new requests */
1348 if (http_client_connection_check_ready(conn) > 0)
1349 http_client_peer_trigger_request_handler(conn->peer);
1350 }
1351 return ret;
1352 }
1353
http_client_connection_output(struct http_client_connection * conn)1354 int http_client_connection_output(struct http_client_connection *conn)
1355 {
1356 struct ostream *output = conn->conn.output;
1357 int ret;
1358
1359 /* We've seen activity from the server; reset request timeout */
1360 http_client_connection_reset_request_timeout(conn);
1361
1362 if ((ret = o_stream_flush(output)) <= 0) {
1363 if (ret < 0)
1364 http_client_connection_handle_output_error(conn);
1365 return ret;
1366 }
1367
1368 i_assert(!conn->output_broken);
1369
1370 if (conn->ssl_iostream != NULL &&
1371 !ssl_iostream_is_handshaked(conn->ssl_iostream))
1372 return 1;
1373
1374 return http_client_connection_continue_request(conn);
1375 }
1376
http_client_connection_start_tunnel(struct http_client_connection ** _conn,struct http_client_tunnel * tunnel)1377 void http_client_connection_start_tunnel(struct http_client_connection **_conn,
1378 struct http_client_tunnel *tunnel)
1379 {
1380 struct http_client_connection *conn = *_conn;
1381
1382 i_assert(conn->tunneling);
1383
1384 /* Claim connection streams */
1385 i_zero(tunnel);
1386 tunnel->input = conn->conn.input;
1387 tunnel->output = conn->conn.output;
1388 tunnel->fd_in = conn->conn.fd_in;
1389 tunnel->fd_out = conn->conn.fd_out;
1390
1391 /* Detach from connection */
1392 conn->conn.input = NULL;
1393 conn->conn.output = NULL;
1394 conn->conn.fd_in = -1;
1395 conn->conn.fd_out = -1;
1396 conn->closing = TRUE;
1397 conn->connected = FALSE;
1398 connection_disconnect(&conn->conn);
1399
1400 http_client_connection_unref(_conn);
1401 }
1402
http_client_connection_ready(struct http_client_connection * conn)1403 static void http_client_connection_ready(struct http_client_connection *conn)
1404 {
1405 struct http_client_peer *peer = conn->peer;
1406 struct http_client_peer_pool *ppool = conn->ppool;
1407 struct http_client_peer_shared *pshared = ppool->peer;
1408 const struct http_client_settings *set =
1409 http_client_connection_get_settings(conn);
1410
1411 e_debug(conn->event, "Ready for requests");
1412 i_assert(!conn->connect_succeeded);
1413
1414 /* Connected */
1415 conn->connected = TRUE;
1416 conn->last_ioloop = current_ioloop;
1417 timeout_remove(&conn->to_connect);
1418
1419 /* Indicate connection success */
1420 conn->connect_succeeded = TRUE;
1421 http_client_connection_unlist_pending(conn);
1422 http_client_peer_connection_success(peer);
1423
1424 /* Start raw log */
1425 if (ppool->rawlog_dir != NULL) {
1426 iostream_rawlog_create(ppool->rawlog_dir,
1427 &conn->conn.input, &conn->conn.output);
1428 }
1429
1430 /* Direct tunneling connections handle connect requests just by
1431 providing a raw connection */
1432 if (pshared->addr.type == HTTP_CLIENT_PEER_ADDR_RAW) {
1433 struct http_client_request *req;
1434
1435 req = http_client_peer_claim_request(conn->peer, FALSE);
1436 if (req != NULL) {
1437 struct http_response response;
1438
1439 conn->tunneling = TRUE;
1440
1441 i_zero(&response);
1442 response.status = 200;
1443 response.reason = "OK";
1444
1445 (void)http_client_connection_return_response(conn, req,
1446 &response);
1447 return;
1448 }
1449
1450 e_debug(conn->event,
1451 "No raw connect requests pending; "
1452 "closing useless connection");
1453 http_client_connection_close(&conn);
1454 return;
1455 }
1456
1457 /* Start protocol I/O */
1458 conn->http_parser = http_response_parser_init(
1459 conn->conn.input, &set->response_hdr_limits, 0);
1460 o_stream_set_finish_via_child(conn->conn.output, FALSE);
1461 o_stream_set_flush_callback(conn->conn.output,
1462 http_client_connection_output, conn);
1463 }
1464
1465 static int
http_client_connection_ssl_handshaked(const char ** error_r,void * context)1466 http_client_connection_ssl_handshaked(const char **error_r, void *context)
1467 {
1468 struct http_client_connection *conn = context;
1469 struct http_client_peer_shared *pshared = conn->ppool->peer;
1470 const struct http_client_settings *set =
1471 http_client_connection_get_settings(conn);
1472 const char *error, *host = pshared->addr.a.tcp.https_name;
1473
1474 if (ssl_iostream_check_cert_validity(conn->ssl_iostream,
1475 host, &error) == 0)
1476 e_debug(conn->event, "SSL handshake successful");
1477 else if (set->ssl->allow_invalid_cert) {
1478 e_debug(conn->event, "SSL handshake successful, "
1479 "ignoring invalid certificate: %s", error);
1480 } else {
1481 *error_r = error;
1482 return -1;
1483 }
1484 return 0;
1485 }
1486
1487 static int
http_client_connection_ssl_init(struct http_client_connection * conn,const char ** error_r)1488 http_client_connection_ssl_init(struct http_client_connection *conn,
1489 const char **error_r)
1490 {
1491 struct http_client_peer_pool *ppool = conn->ppool;
1492 struct http_client_peer_shared *pshared = ppool->peer;
1493 const struct http_client_settings *set =
1494 http_client_connection_get_settings(conn);
1495 struct ssl_iostream_settings ssl_set;
1496 struct ssl_iostream_context *ssl_ctx = ppool->ssl_ctx;
1497 const char *error;
1498
1499 i_assert(ssl_ctx != NULL);
1500
1501 ssl_set = *set->ssl;
1502 if (!set->ssl->allow_invalid_cert) {
1503 ssl_set.verbose_invalid_cert = TRUE;
1504 }
1505
1506 e_debug(conn->event, "Starting SSL handshake");
1507
1508 connection_input_halt(&conn->conn);
1509 if (io_stream_create_ssl_client(ssl_ctx, pshared->addr.a.tcp.https_name,
1510 &ssl_set,
1511 &conn->conn.input, &conn->conn.output,
1512 &conn->ssl_iostream, &error) < 0) {
1513 *error_r = t_strdup_printf(
1514 "Couldn't initialize SSL client for %s: %s",
1515 conn->conn.name, error);
1516 return -1;
1517 }
1518 connection_input_resume(&conn->conn);
1519 ssl_iostream_set_handshake_callback(
1520 conn->ssl_iostream,
1521 http_client_connection_ssl_handshaked, conn);
1522 if (ssl_iostream_handshake(conn->ssl_iostream) < 0) {
1523 *error_r = t_strdup_printf(
1524 "SSL handshake to %s failed: %s", conn->conn.name,
1525 ssl_iostream_get_last_error(conn->ssl_iostream));
1526 return -1;
1527 }
1528
1529 if (ssl_iostream_is_handshaked(conn->ssl_iostream)) {
1530 http_client_connection_ready(conn);
1531 } else {
1532 /* Wait for handshake to complete; connection input handler does
1533 the rest by reading from the input stream */
1534 o_stream_set_flush_callback(
1535 conn->conn.output, http_client_connection_output, conn);
1536 }
1537 return 0;
1538 }
1539
1540 static void
http_client_connection_connected(struct connection * _conn,bool success)1541 http_client_connection_connected(struct connection *_conn, bool success)
1542 {
1543 struct http_client_connection *conn =
1544 (struct http_client_connection *)_conn;
1545 struct http_client_peer_shared *pshared = conn->ppool->peer;
1546 const struct http_client_settings *set =
1547 http_client_connection_get_settings(conn);
1548 const char *error;
1549
1550 if (!success) {
1551 http_client_connection_failure(
1552 conn, t_strdup_printf("connect(%s) failed: %m",
1553 _conn->name));
1554 } else {
1555 conn->connected_timestamp = ioloop_timeval;
1556 e_debug(conn->event, "Connected");
1557
1558 (void)net_set_tcp_nodelay(_conn->fd_out, TRUE);
1559 if (set->socket_send_buffer_size > 0 &&
1560 net_set_send_buffer_size(
1561 _conn->fd_out, set->socket_send_buffer_size) < 0) {
1562 i_error("net_set_send_buffer_size(%zu) failed: %m",
1563 set->socket_send_buffer_size);
1564 }
1565 if (set->socket_recv_buffer_size > 0 &&
1566 net_set_recv_buffer_size(
1567 _conn->fd_in, set->socket_recv_buffer_size) < 0) {
1568 i_error("net_set_recv_buffer_size(%zu) failed: %m",
1569 set->socket_recv_buffer_size);
1570 }
1571
1572 if (http_client_peer_addr_is_https(&pshared->addr)) {
1573 if (http_client_connection_ssl_init(conn, &error) < 0) {
1574 e_debug(conn->event, "%s", error);
1575 http_client_connection_failure(conn, error);
1576 http_client_connection_close(&conn);
1577 }
1578 return;
1579 }
1580 http_client_connection_ready(conn);
1581 }
1582 }
1583
1584 static const struct connection_settings http_client_connection_set = {
1585 .input_max_size = SIZE_MAX,
1586 .output_max_size = SIZE_MAX,
1587 .client = TRUE,
1588 .delayed_unix_client_connected_callback = TRUE,
1589 .log_connection_id = TRUE,
1590 };
1591
1592 static const struct connection_vfuncs http_client_connection_vfuncs = {
1593 .destroy = http_client_connection_destroy,
1594 .input = http_client_connection_input,
1595 .client_connected = http_client_connection_connected,
1596 };
1597
http_client_connection_list_init(void)1598 struct connection_list *http_client_connection_list_init(void)
1599 {
1600 return connection_list_init(&http_client_connection_set,
1601 &http_client_connection_vfuncs);
1602 }
1603
1604 static void
http_client_connection_delayed_connect_error(struct http_client_connection * conn)1605 http_client_connection_delayed_connect_error(
1606 struct http_client_connection *conn)
1607 {
1608 timeout_remove(&conn->to_input);
1609 errno = conn->connect_errno;
1610 http_client_connection_connected(&conn->conn, FALSE);
1611 http_client_connection_close(&conn);
1612 }
1613
http_client_connect_timeout(struct http_client_connection * conn)1614 static void http_client_connect_timeout(struct http_client_connection *conn)
1615 {
1616 conn->conn.disconnect_reason = CONNECTION_DISCONNECT_CONNECT_TIMEOUT;
1617 http_client_connection_destroy(&conn->conn);
1618 }
1619
1620 static void
http_client_connection_connect(struct http_client_connection * conn,unsigned int timeout_msecs)1621 http_client_connection_connect(struct http_client_connection *conn,
1622 unsigned int timeout_msecs)
1623 {
1624 struct http_client_context *cctx = conn->ppool->peer->cctx;
1625
1626 conn->connect_start_timestamp = ioloop_timeval;
1627 if (connection_client_connect(&conn->conn) < 0) {
1628 conn->connect_errno = errno;
1629 e_debug(conn->event, "Connect failed: %m");
1630 conn->to_input = timeout_add_short_to(
1631 conn->conn.ioloop, 0,
1632 http_client_connection_delayed_connect_error, conn);
1633 return;
1634 }
1635
1636 /* Don't use connection.h timeout because we want this timeout
1637 to include also the SSL handshake */
1638 if (timeout_msecs > 0) {
1639 conn->to_connect = timeout_add_to(
1640 cctx->ioloop, timeout_msecs,
1641 http_client_connect_timeout, conn);
1642 }
1643 }
1644
1645 static void
http_client_connect_tunnel_timeout(struct http_client_connection * conn)1646 http_client_connect_tunnel_timeout(struct http_client_connection *conn)
1647 {
1648 struct http_client_peer_shared *pshared = conn->ppool->peer;
1649 const char *error, *name = http_client_peer_addr2str(&pshared->addr);
1650 unsigned int msecs;
1651
1652 msecs = timeval_diff_msecs(&ioloop_timeval,
1653 &conn->connect_start_timestamp);
1654 error = t_strdup_printf("Tunnel connect(%s) failed: "
1655 "Connection timed out in %u.%03u secs",
1656 name, msecs/1000, msecs%1000);
1657
1658 e_debug(conn->event, "%s", error);
1659 http_client_connection_failure(conn, error);
1660 http_client_connection_close(&conn);
1661 }
1662
1663 static void
http_client_connection_tunnel_response(const struct http_response * response,struct http_client_connection * conn)1664 http_client_connection_tunnel_response(const struct http_response *response,
1665 struct http_client_connection *conn)
1666 {
1667 struct http_client_peer_shared *pshared = conn->ppool->peer;
1668 struct http_client_context *cctx = pshared->cctx;
1669 struct http_client_tunnel tunnel;
1670 const char *name = http_client_peer_addr2str(&pshared->addr);
1671 struct http_client_request *req = conn->connect_request;
1672
1673 conn->connect_request = NULL;
1674
1675 if (response->status != 200) {
1676 http_client_connection_failure(
1677 conn,
1678 t_strdup_printf("Tunnel connect(%s) failed: %s", name,
1679 http_response_get_message(response)));
1680 return;
1681 }
1682
1683 http_client_request_start_tunnel(req, &tunnel);
1684
1685 conn->conn.event_parent = conn->event;
1686 connection_init_from_streams(cctx->conn_list, &conn->conn,
1687 name, tunnel.input, tunnel.output);
1688 connection_switch_ioloop_to(&conn->conn, cctx->ioloop);
1689 i_stream_unref(&tunnel.input);
1690 o_stream_unref(&tunnel.output);
1691 }
1692
1693 static void
http_client_connection_connect_tunnel(struct http_client_connection * conn,const struct ip_addr * ip,in_port_t port,unsigned int timeout_msecs)1694 http_client_connection_connect_tunnel(struct http_client_connection *conn,
1695 const struct ip_addr *ip, in_port_t port,
1696 unsigned int timeout_msecs)
1697 {
1698 struct http_client_context *cctx = conn->ppool->peer->cctx;
1699 struct http_client *client = conn->peer->client;
1700
1701 conn->connect_start_timestamp = ioloop_timeval;
1702
1703 conn->connect_request = http_client_request_connect_ip(
1704 client, ip, port, http_client_connection_tunnel_response, conn);
1705 http_client_request_set_urgent(conn->connect_request);
1706 http_client_request_submit(conn->connect_request);
1707
1708 /* Don't use connection.h timeout because we want this timeout
1709 to include also the SSL handshake */
1710 if (timeout_msecs > 0) {
1711 conn->to_connect = timeout_add_to(
1712 cctx->ioloop, timeout_msecs,
1713 http_client_connect_tunnel_timeout, conn);
1714 }
1715 }
1716
1717 struct http_client_connection *
http_client_connection_create(struct http_client_peer * peer)1718 http_client_connection_create(struct http_client_peer *peer)
1719 {
1720 struct http_client_peer_shared *pshared = peer->shared;
1721 struct http_client_peer_pool *ppool = peer->ppool;
1722 struct http_client_context *cctx = pshared->cctx;
1723 struct http_client *client = peer->client;
1724 const struct http_client_settings *set = &client->set;
1725 struct http_client_connection *conn;
1726 const struct http_client_peer_addr *addr = &pshared->addr;
1727 const char *conn_type = "UNKNOWN";
1728 unsigned int timeout_msecs;
1729
1730 switch (pshared->addr.type) {
1731 case HTTP_CLIENT_PEER_ADDR_HTTP:
1732 conn_type = "HTTP";
1733 break;
1734 case HTTP_CLIENT_PEER_ADDR_HTTPS:
1735 conn_type = "HTTPS";
1736 break;
1737 case HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL:
1738 conn_type = "Tunneled HTTPS";
1739 break;
1740 case HTTP_CLIENT_PEER_ADDR_RAW:
1741 conn_type = "Raw";
1742 break;
1743 case HTTP_CLIENT_PEER_ADDR_UNIX:
1744 conn_type = "Unix";
1745 break;
1746 }
1747
1748 timeout_msecs = set->connect_timeout_msecs;
1749 if (timeout_msecs == 0)
1750 timeout_msecs = set->request_timeout_msecs;
1751
1752 conn = i_new(struct http_client_connection, 1);
1753 conn->refcount = 1;
1754 conn->ppool = ppool;
1755 conn->peer = peer;
1756 conn->debug = client->set.debug;
1757 if (pshared->addr.type != HTTP_CLIENT_PEER_ADDR_RAW)
1758 i_array_init(&conn->request_wait_list, 16);
1759 conn->io_wait_timer = io_wait_timer_add_to(cctx->ioloop);
1760
1761 conn->conn.event_parent = ppool->peer->cctx->event;
1762 connection_init(cctx->conn_list, &conn->conn,
1763 http_client_peer_shared_label(pshared));
1764 conn->event = conn->conn.event;
1765
1766 switch (pshared->addr.type) {
1767 case HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL:
1768 http_client_connection_connect_tunnel(
1769 conn, &addr->a.tcp.ip, addr->a.tcp.port, timeout_msecs);
1770 break;
1771 case HTTP_CLIENT_PEER_ADDR_UNIX:
1772 connection_init_client_unix(cctx->conn_list, &conn->conn,
1773 addr->a.un.path);
1774 connection_switch_ioloop_to(&conn->conn, cctx->ioloop);
1775 http_client_connection_connect(conn, timeout_msecs);
1776 break;
1777 default:
1778 connection_init_client_ip(cctx->conn_list, &conn->conn, NULL,
1779 &addr->a.tcp.ip, addr->a.tcp.port);
1780 connection_switch_ioloop_to(&conn->conn, cctx->ioloop);
1781 http_client_connection_connect(conn, timeout_msecs);
1782 }
1783
1784 array_push_back(&ppool->pending_conns, &conn);
1785 array_push_back(&ppool->conns, &conn);
1786 array_push_back(&peer->pending_conns, &conn);
1787 array_push_back(&peer->conns, &conn);
1788
1789 http_client_peer_pool_ref(ppool);
1790
1791 e_debug(conn->event,
1792 "%s connection created (%d parallel connections exist)%s",
1793 conn_type, array_count(&ppool->conns),
1794 (conn->to_input == NULL ? "" : " [broken]"));
1795 return conn;
1796 }
1797
http_client_connection_ref(struct http_client_connection * conn)1798 void http_client_connection_ref(struct http_client_connection *conn)
1799 {
1800 i_assert(conn->refcount > 0);
1801 conn->refcount++;
1802 }
1803
1804 static void
http_client_connection_disconnect(struct http_client_connection * conn)1805 http_client_connection_disconnect(struct http_client_connection *conn)
1806 {
1807 struct http_client_peer_pool *ppool = conn->ppool;
1808 ARRAY_TYPE(http_client_connection) *conn_arr;
1809 struct http_client_connection *const *conn_idx;
1810
1811 if (conn->disconnected)
1812 return;
1813 conn->disconnected = TRUE;
1814
1815 e_debug(conn->event, "Connection disconnect");
1816
1817 conn->closing = TRUE;
1818 conn->connected = FALSE;
1819
1820 http_client_request_abort(&conn->connect_request);
1821
1822 if (conn->incoming_payload != NULL) {
1823 /* The stream is still accessed by lib-http caller. */
1824 i_stream_remove_destroy_callback(conn->incoming_payload,
1825 http_client_payload_destroyed);
1826 conn->incoming_payload = NULL;
1827 }
1828
1829 if (conn->http_parser != NULL)
1830 http_response_parser_deinit(&conn->http_parser);
1831
1832 connection_disconnect(&conn->conn);
1833
1834 io_remove(&conn->io_req_payload);
1835 timeout_remove(&conn->to_requests);
1836 timeout_remove(&conn->to_connect);
1837 timeout_remove(&conn->to_input);
1838 timeout_remove(&conn->to_response);
1839
1840 /* Remove this connection from the lists */
1841 conn_arr = &ppool->conns;
1842 array_foreach(conn_arr, conn_idx) {
1843 if (*conn_idx == conn) {
1844 array_delete(conn_arr,
1845 array_foreach_idx(conn_arr, conn_idx), 1);
1846 break;
1847 }
1848 }
1849 conn_arr = &ppool->pending_conns;
1850 array_foreach(conn_arr, conn_idx) {
1851 if (*conn_idx == conn) {
1852 array_delete(conn_arr,
1853 array_foreach_idx(conn_arr, conn_idx), 1);
1854 break;
1855 }
1856 }
1857
1858 http_client_connection_detach_peer(conn);
1859
1860 http_client_connection_stop_idle(conn); // FIXME: needed?
1861 }
1862
http_client_connection_unref(struct http_client_connection ** _conn)1863 bool http_client_connection_unref(struct http_client_connection **_conn)
1864 {
1865 struct http_client_connection *conn = *_conn;
1866 struct http_client_peer_pool *ppool = conn->ppool;
1867
1868 i_assert(conn->refcount > 0);
1869
1870 *_conn = NULL;
1871
1872 if (--conn->refcount > 0)
1873 return TRUE;
1874
1875 e_debug(conn->event, "Connection destroy");
1876
1877 http_client_connection_disconnect(conn);
1878 http_client_connection_abort_any_requests(conn);
1879
1880 i_assert(conn->io_req_payload == NULL);
1881 i_assert(conn->to_requests == NULL);
1882 i_assert(conn->to_connect == NULL);
1883 i_assert(conn->to_input == NULL);
1884 i_assert(conn->to_idle == NULL);
1885 i_assert(conn->to_response == NULL);
1886
1887 if (array_is_created(&conn->request_wait_list))
1888 array_free(&conn->request_wait_list);
1889
1890 ssl_iostream_destroy(&conn->ssl_iostream);
1891 connection_deinit(&conn->conn);
1892 io_wait_timer_remove(&conn->io_wait_timer);
1893
1894 i_free(conn);
1895
1896 http_client_peer_pool_unref(&ppool);
1897 return FALSE;
1898 }
1899
http_client_connection_close(struct http_client_connection ** _conn)1900 void http_client_connection_close(struct http_client_connection **_conn)
1901 {
1902 struct http_client_connection *conn = *_conn;
1903
1904 e_debug(conn->event, "Connection close");
1905
1906 http_client_connection_disconnect(conn);
1907 http_client_connection_abort_any_requests(conn);
1908 http_client_connection_unref(_conn);
1909 }
1910
http_client_connection_switch_ioloop(struct http_client_connection * conn)1911 void http_client_connection_switch_ioloop(struct http_client_connection *conn)
1912 {
1913 struct http_client_peer_shared *pshared = conn->ppool->peer;
1914 struct http_client_context *cctx = pshared->cctx;
1915 struct ioloop *ioloop = cctx->ioloop;
1916
1917 connection_switch_ioloop_to(&conn->conn, ioloop);
1918 if (conn->io_req_payload != NULL) {
1919 conn->io_req_payload =
1920 io_loop_move_io_to(ioloop, &conn->io_req_payload);
1921 }
1922 if (conn->to_requests != NULL) {
1923 conn->to_requests =
1924 io_loop_move_timeout_to(ioloop, &conn->to_requests);
1925 }
1926 if (conn->to_connect != NULL) {
1927 conn->to_connect =
1928 io_loop_move_timeout_to(ioloop, &conn->to_connect);
1929 }
1930 if (conn->to_input != NULL) {
1931 conn->to_input =
1932 io_loop_move_timeout_to(ioloop, &conn->to_input);
1933 }
1934 if (conn->to_idle != NULL) {
1935 conn->to_idle =
1936 io_loop_move_timeout_to(ioloop, &conn->to_idle);
1937 }
1938 if (conn->to_response != NULL) {
1939 conn->to_response =
1940 io_loop_move_timeout_to(ioloop, &conn->to_response);
1941 }
1942 if (conn->incoming_payload != NULL)
1943 i_stream_switch_ioloop_to(conn->incoming_payload, ioloop);
1944 conn->io_wait_timer =
1945 io_wait_timer_move_to(&conn->io_wait_timer, ioloop);
1946 }
1947