1 #ifndef HTTP_CLIENT_H 2 #define HTTP_CLIENT_H 3 4 #include "net.h" 5 6 #include "http-common.h" 7 #include "http-response.h" 8 9 struct timeval; 10 struct http_response; 11 12 struct http_client_request; 13 struct http_client; 14 struct http_client_context; 15 16 struct ssl_iostream_settings; 17 18 /* 19 * Client settings 20 */ 21 22 struct http_client_settings { 23 /* a) If dns_client is set, all lookups are done via it. 24 b) If dns_client_socket_path is set, each DNS lookup does its own 25 dns-lookup UNIX socket connection. 26 c) Otherwise, blocking gethostbyname() lookups are used. */ 27 struct dns_client *dns_client; 28 const char *dns_client_socket_path; 29 /* How long to cache DNS records internally 30 (default = HTTP_CLIENT_DEFAULT_DNS_TTL_MSECS) */ 31 unsigned int dns_ttl_msecs; 32 33 const struct ssl_iostream_settings *ssl; 34 35 /* User-Agent: header (default: none) */ 36 const char *user_agent; 37 38 /* proxy on unix socket */ 39 const char *proxy_socket_path; 40 /* URL for normal proxy (ignored if proxy_socket_path is set) */ 41 const struct http_url *proxy_url; 42 /* credentials for proxy */ 43 const char *proxy_username; 44 const char *proxy_password; 45 46 /* directory for writing raw log data for debugging purposes */ 47 const char *rawlog_dir; 48 49 /* maximum time a connection will idle. if parallel connections are idle, 50 the duplicates will end earlier based on how many idle connections exist 51 to that same service */ 52 unsigned int max_idle_time_msecs; 53 54 /* maximum number of parallel connections per peer (default = 1) */ 55 unsigned int max_parallel_connections; 56 57 /* maximum number of pipelined requests per connection (default = 1) */ 58 unsigned int max_pipelined_requests; 59 60 /* don't automatically act upon redirect responses */ 61 bool no_auto_redirect; 62 63 /* never automatically retry requests */ 64 bool no_auto_retry; 65 66 /* if we use a proxy, delegate SSL negotiation to proxy, rather than 67 creating a CONNECT tunnel through the proxy for the SSL link */ 68 bool no_ssl_tunnel; 69 70 /* maximum number of redirects for a request 71 (default = 0; redirects refused) 72 */ 73 unsigned int max_redirects; 74 75 /* maximum number of attempts for a request */ 76 unsigned int max_attempts; 77 78 /* maximum number of connection attempts to a host before all associated 79 requests fail. 80 81 if > 1, the maximum will be enforced across all IPs for that host, 82 meaning that IPs may be tried more than once eventually if the number 83 of IPs is smaller than the specified maximum attempts. If the number of IPs 84 is higher than the maximum attempts, not all IPs are tried. If <= 1, all 85 IPs are tried at most once. 86 */ 87 unsigned int max_connect_attempts; 88 89 /* Initial backoff time; doubled at each connection failure 90 (default = HTTP_CLIENT_DEFAULT_BACKOFF_TIME_MSECS) */ 91 unsigned int connect_backoff_time_msecs; 92 /* Maximum backoff time 93 (default = HTTP_CLIENT_DEFAULT_BACKOFF_MAX_TIME_MSECS) */ 94 unsigned int connect_backoff_max_time_msecs; 95 96 /* response header limits */ 97 struct http_header_limits response_hdr_limits; 98 99 /* max total time to wait for HTTP request to finish 100 this can be overridden/reset for individual requests using 101 http_client_request_set_timeout() and friends. 102 (default is no timeout) 103 */ 104 unsigned int request_absolute_timeout_msecs; 105 /* max time to wait for HTTP request to finish before retrying 106 (default = HTTP_CLIENT_DEFAULT_REQUEST_TIMEOUT_MSECS) */ 107 unsigned int request_timeout_msecs; 108 /* max time to wait for connect() (and SSL handshake) to finish before 109 retrying (default = request_timeout_msecs) */ 110 unsigned int connect_timeout_msecs; 111 /* time to wait for connect() (and SSL handshake) to finish for the first 112 connection before trying the next IP in parallel 113 (default = 0; wait until current connection attempt finishes) */ 114 unsigned int soft_connect_timeout_msecs; 115 116 /* maximum acceptable delay in seconds for automatically 117 retrying/redirecting requests. if a server sends a response with a 118 Retry-After header that causes a delay longer than this, the request 119 is not automatically retried and the response is returned */ 120 unsigned int max_auto_retry_delay; 121 122 /* the kernel send/receive buffer sizes used for the connection sockets. 123 Configuring this is mainly useful for the test suite. The kernel 124 defaults are used when these settings are 0. */ 125 size_t socket_send_buffer_size; 126 size_t socket_recv_buffer_size; 127 128 /* Event to use as parent for the http client event. For specific 129 requests this can be overridden with http_client_request_set_event(). 130 */ 131 struct event *event_parent; 132 133 /* enable logging debug messages */ 134 bool debug; 135 }; 136 137 /* 138 * Request 139 */ 140 141 enum http_client_request_error { 142 /* The request was aborted */ 143 HTTP_CLIENT_REQUEST_ERROR_ABORTED = HTTP_RESPONSE_STATUS_INTERNAL, 144 /* Failed to parse HTTP target url */ 145 HTTP_CLIENT_REQUEST_ERROR_INVALID_URL, 146 /* Failed to perform DNS lookup for the host */ 147 HTTP_CLIENT_REQUEST_ERROR_HOST_LOOKUP_FAILED, 148 /* Failed to setup any connection for the host and client settings allowed 149 no more attempts */ 150 HTTP_CLIENT_REQUEST_ERROR_CONNECT_FAILED, 151 /* Service returned an invalid redirect response for this request */ 152 HTTP_CLIENT_REQUEST_ERROR_INVALID_REDIRECT, 153 /* The connection was lost unexpectedly while handling the request and 154 client settings allowed no more attempts */ 155 HTTP_CLIENT_REQUEST_ERROR_CONNECTION_LOST, 156 /* The input stream passed to the request using 157 http_client_request_set_payload() returned an error while sending the 158 request. */ 159 HTTP_CLIENT_REQUEST_ERROR_BROKEN_PAYLOAD, 160 /* The service returned a bad response */ 161 HTTP_CLIENT_REQUEST_ERROR_BAD_RESPONSE, 162 /* The request timed out (either this was the last attempt or the 163 absolute timeout was hit) */ 164 HTTP_CLIENT_REQUEST_ERROR_TIMED_OUT, 165 }; 166 167 enum http_request_state { 168 /* New request; not yet submitted */ 169 HTTP_REQUEST_STATE_NEW = 0, 170 /* Request is queued; waiting for a connection */ 171 HTTP_REQUEST_STATE_QUEUED, 172 /* Request header is sent; still sending request payload to server */ 173 HTTP_REQUEST_STATE_PAYLOAD_OUT, 174 /* Request is fully sent; waiting for response */ 175 HTTP_REQUEST_STATE_WAITING, 176 /* Response header is received for the request */ 177 HTTP_REQUEST_STATE_GOT_RESPONSE, 178 /* Reading response payload; response handler still needs to read more 179 payload. */ 180 HTTP_REQUEST_STATE_PAYLOAD_IN, 181 /* Request is finished; still lingering due to references */ 182 HTTP_REQUEST_STATE_FINISHED, 183 /* Request is aborted; still lingering due to references */ 184 HTTP_REQUEST_STATE_ABORTED 185 }; 186 extern const char *http_request_state_names[]; 187 188 struct http_client_tunnel { 189 int fd_in, fd_out; 190 struct istream *input; 191 struct ostream *output; 192 }; 193 194 struct http_client_request_stats { 195 /* Total elapsed time since message was submitted */ 196 unsigned int total_msecs; 197 /* Elapsed time since message was first sent */ 198 unsigned int first_sent_msecs; 199 /* Elapsed time since message was last sent */ 200 unsigned int last_sent_msecs; 201 202 /* Time spent in other ioloops */ 203 unsigned int other_ioloop_msecs; 204 /* Time spent in the http-client's own ioloop */ 205 unsigned int http_ioloop_msecs; 206 /* Total time spent on waiting for file locks */ 207 unsigned int lock_msecs; 208 209 /* Number of times this request was retried */ 210 unsigned int attempts; 211 /* Number of times the client attempted to actually send the request 212 to a server */ 213 unsigned int send_attempts; 214 }; 215 216 typedef void 217 http_client_request_callback_t(const struct http_response *response, 218 void *context); 219 220 /* create new HTTP request */ 221 struct http_client_request * 222 http_client_request(struct http_client *client, 223 const char *method, const char *host, const char *target, 224 http_client_request_callback_t *callback, void *context); 225 #define http_client_request(client, method, host, target, callback, context) \ 226 http_client_request(client, method, host, target - \ 227 CALLBACK_TYPECHECK(callback, void (*)( \ 228 const struct http_response *response, typeof(context))), \ 229 (http_client_request_callback_t *)callback, context) 230 231 /* create new HTTP request using provided URL. This implicitly sets 232 port, ssl, and username:password if provided. */ 233 struct http_client_request * 234 http_client_request_url(struct http_client *client, 235 const char *method, const struct http_url *target_url, 236 http_client_request_callback_t *callback, void *context); 237 #define http_client_request_url(client, method, target_url, callback, context) \ 238 http_client_request_url(client, method, target_url - \ 239 CALLBACK_TYPECHECK(callback, void (*)( \ 240 const struct http_response *response, typeof(context))), \ 241 (http_client_request_callback_t *)callback, context) 242 struct http_client_request * 243 http_client_request_url_str(struct http_client *client, 244 const char *method, const char *url_str, 245 http_client_request_callback_t *callback, void *context); 246 #define http_client_request_url_str(client, method, url_str, callback, context) \ 247 http_client_request_url_str(client, method, url_str - \ 248 CALLBACK_TYPECHECK(callback, void (*)( \ 249 const struct http_response *response, typeof(context))), \ 250 (http_client_request_callback_t *)callback, context) 251 252 /* create new HTTP CONNECT request. If this HTTP is configured to use a proxy, 253 a CONNECT request will be submitted at that proxy, otherwise the connection 254 is created directly. Call http_client_request_start_tunnel() to 255 to take over the connection. 256 */ 257 struct http_client_request * 258 http_client_request_connect(struct http_client *client, 259 const char *host, in_port_t port, 260 http_client_request_callback_t *callback, 261 void *context); 262 #define http_client_request_connect(client, host, port, callback, context) \ 263 http_client_request_connect(client, host, port - \ 264 CALLBACK_TYPECHECK(callback, void (*)( \ 265 const struct http_response *response, typeof(context))), \ 266 (http_client_request_callback_t *)callback, context) 267 268 /* same as http_client_request_connect, but uses an IP rather than a host 269 name. */ 270 struct http_client_request * 271 http_client_request_connect_ip(struct http_client *client, 272 const struct ip_addr *ip, in_port_t port, 273 http_client_request_callback_t *callback, 274 void *context); 275 #define http_client_request_connect_ip(client, ip, port, callback, context) \ 276 http_client_request_connect_ip(client, ip, port - \ 277 CALLBACK_TYPECHECK(callback, void (*)( \ 278 const struct http_response *response, typeof(context))), \ 279 (http_client_request_callback_t *)callback, context) 280 281 void http_client_request_set_event(struct http_client_request *req, 282 struct event *event); 283 /* set the port for the service the request is directed at */ 284 void http_client_request_set_port(struct http_client_request *req, 285 in_port_t port); 286 /* indicate whether service the request is directed at uses ssl */ 287 void http_client_request_set_ssl(struct http_client_request *req, 288 bool ssl); 289 /* set the urgent flag: this means that this request will get priority over 290 non-urgent request. Also, if no idle connection is available, a new 291 connection is created. Urgent requests are never pipelined. */ 292 void http_client_request_set_urgent(struct http_client_request *req); 293 void http_client_request_set_preserve_exact_reason(struct http_client_request *req); 294 295 /* add a custom header to the request. This can override headers that are 296 otherwise created implicitly. If the same header key was already added, 297 the value is replaced. */ 298 void http_client_request_add_header(struct http_client_request *req, 299 const char *key, const char *value); 300 /* add a custom header to the request. Do nothing if it was already added. */ 301 void http_client_request_add_missing_header(struct http_client_request *req, 302 const char *key, const char *value); 303 /* remove a header added earlier. This has no influence on implicitly created 304 headers. */ 305 void http_client_request_remove_header(struct http_client_request *req, 306 const char *key); 307 /* lookup the value for a header added earlier. Returns NULL if not found. */ 308 const char *http_client_request_lookup_header(struct http_client_request *req, 309 const char *key); 310 311 /* set the value of the "Date" header for the request using a time_t value. 312 Use this instead of setting it directly using 313 http_client_request_add_header() */ 314 void http_client_request_set_date(struct http_client_request *req, 315 time_t date); 316 317 /* assign an input stream for the outgoing payload of this request. The input 318 stream is read asynchronously while the request is sent to the server. 319 320 when sync=TRUE a "100 Continue" response is requested from the service. The 321 client will then postpone sending the payload until a provisional response 322 with code 100 is received. This way, an error response can be sent by the 323 service before any potentially big payload is transmitted. Use this only for 324 payload that can be large. */ 325 void http_client_request_set_payload(struct http_client_request *req, 326 struct istream *input, bool sync); 327 /* assign payload data to the request. The data is copied to the request pool. 328 If your data is already durably allocated during the existence of the 329 request, you should consider using http_client_request_set_payload() with 330 a data input stream instead. This will avoid copying the data unnecessarily. 331 */ 332 void http_client_request_set_payload_data(struct http_client_request *req, 333 const unsigned char *data, size_t size); 334 /* send an empty payload for this request. This means that a Content-Length 335 header is generated with zero size. Calling this function is not necessary 336 for the standard POST and PUT methods, for which this is done implicitly if 337 there is no payload set. */ 338 void http_client_request_set_payload_empty(struct http_client_request *req); 339 340 /* set an absolute timeout for this request specifically, overriding the 341 default client-wide absolute request timeout */ 342 void http_client_request_set_timeout_msecs(struct http_client_request *req, 343 unsigned int msecs); 344 void http_client_request_set_timeout(struct http_client_request *req, 345 const struct timeval *time); 346 347 /* Override http_client_settings.request_timeout_msecs */ 348 void http_client_request_set_attempt_timeout_msecs(struct http_client_request *req, 349 unsigned int msecs); 350 /* Override http_client_settings.max_attempts */ 351 void http_client_request_set_max_attempts(struct http_client_request *req, 352 unsigned int max_attempts); 353 354 /* Include the specified HTTP response headers in the http_request_finished 355 event parameters with "http_hdr_" prefix. */ 356 void http_client_request_set_event_headers(struct http_client_request *req, 357 const char *const *headers); 358 359 /* set the username:password credentials for this request for simple 360 authentication. This function is meant for simple schemes that use a 361 password. More complex schemes will need to be handled manually. 362 363 This currently only supports the "basic" authentication scheme. */ 364 void http_client_request_set_auth_simple(struct http_client_request *req, 365 const char *username, const char *password); 366 367 /* Assign a proxy to use for this particular request. This overrides any 368 proxy defined in the client settings. */ 369 void http_client_request_set_proxy_url(struct http_client_request *req, 370 const struct http_url *proxy_url); 371 /* Like http_client_request_set_proxy_url(), but the proxy is behind a unix 372 socket. */ 373 void http_client_request_set_proxy_socket(struct http_client_request *req, 374 const char *proxy_socket); 375 376 /* delay handling of this request to a later time. This way, a request can be 377 submitted that is held for some time until a certain time period has passed. 378 */ 379 void http_client_request_delay_until(struct http_client_request *req, 380 time_t time); 381 void http_client_request_delay(struct http_client_request *req, 382 time_t seconds); 383 void http_client_request_delay_msecs(struct http_client_request *req, 384 unsigned int msecs); 385 386 /* Try to set request delay based on the Retry-After header. Returns 1 if 387 successful, 0 if it doesn't exist or is already expired, -1 if the delay 388 would be too long. */ 389 int http_client_request_delay_from_response(struct http_client_request *req, 390 const struct http_response *response); 391 392 /* return the HTTP method for the request */ 393 const char * 394 http_client_request_get_method(const struct http_client_request *req) 395 ATTR_PURE; 396 /* return the HTTP target for the request */ 397 const char * 398 http_client_request_get_target(const struct http_client_request *req) 399 ATTR_PURE; 400 /* return the request state */ 401 enum http_request_state 402 http_client_request_get_state(const struct http_client_request *req) 403 ATTR_PURE; 404 /* return number of retry attempts */ 405 unsigned int 406 http_client_request_get_attempts(const struct http_client_request *req) 407 ATTR_PURE; 408 /* return origin_url */ 409 const struct http_url * 410 http_client_request_get_origin_url(const struct http_client_request *req) 411 ATTR_PURE; 412 413 /* get statistics for the request */ 414 void http_client_request_get_stats(struct http_client_request *req, 415 struct http_client_request_stats *stats); 416 /* append text with request statistics to provided string buffer */ 417 void http_client_request_append_stats_text(struct http_client_request *req, 418 string_t *str); 419 420 /* submit the request. It is queued for transmission to the service */ 421 void http_client_request_submit(struct http_client_request *req); 422 423 /* attempt to retry the request. This function is called within the request 424 callback. It returns false if the request cannot be retried */ 425 bool http_client_request_try_retry(struct http_client_request *req); 426 427 /* abort the request immediately. It may still linger for a while when it is 428 already sent to the service, but the callback will not be called anymore. */ 429 void http_client_request_abort(struct http_client_request **req); 430 431 /* call the specified callback when HTTP request is destroyed. */ 432 void http_client_request_set_destroy_callback(struct http_client_request *req, 433 void (*callback)(void *), 434 void *context); 435 #define http_client_request_set_destroy_callback(req, callback, context) \ 436 http_client_request_set_destroy_callback(req, (void(*)(void*))callback, \ 437 TRUE ? context : \ 438 CALLBACK_TYPECHECK(callback, void (*)(typeof(context)))) 439 440 /* submits request and blocks until the provided payload is sent. Multiple 441 calls are allowed; payload transmission is ended with 442 http_client_request_finish_payload(). If the sending fails, returns -1 443 and sets req=NULL to indicate that the request was freed, otherwise 444 returns 0 and req is unchanged. */ 445 int http_client_request_send_payload(struct http_client_request **req, 446 const unsigned char *data, size_t size); 447 /* finish sending the payload. Always frees req and sets it to NULL. 448 Returns 0 on success, -1 on error. */ 449 int http_client_request_finish_payload(struct http_client_request **req); 450 451 /* take over the connection this request was sent over for use as a HTTP 452 CONNECT tunnel. This only applies to requests that were created using 453 http_client_request_connect() or http_client_request_connect_ip(). */ 454 void http_client_request_start_tunnel(struct http_client_request *req, 455 struct http_client_tunnel *tunnel); 456 457 /* 458 * Client 459 */ 460 461 /* Create a client using the global shared client context. */ 462 struct http_client * 463 http_client_init(const struct http_client_settings *set); 464 /* Create a client without a shared context. */ 465 struct http_client * 466 http_client_init_private(const struct http_client_settings *set); 467 struct http_client * 468 http_client_init_shared(struct http_client_context *cctx, 469 const struct http_client_settings *set) ATTR_NULL(1); 470 void http_client_deinit(struct http_client **_client); 471 472 /* switch this client to the current ioloop */ 473 struct ioloop *http_client_switch_ioloop(struct http_client *client); 474 475 /* blocks until all currently submitted requests are handled */ 476 void http_client_wait(struct http_client *client); 477 478 /* Returns the total number of pending HTTP requests. */ 479 unsigned int 480 http_client_get_pending_request_count(struct http_client *client); 481 482 /* 483 * Client shared context 484 */ 485 486 struct http_client_context * 487 http_client_context_create(const struct http_client_settings *set); 488 void http_client_context_ref(struct http_client_context *cctx); 489 void http_client_context_unref(struct http_client_context **_cctx); 490 491 /* Return the default global shared client context, creating it if necessary. 492 The context is freed automatically at exit. Don't unreference the 493 returned context. */ 494 struct http_client_context *http_client_get_global_context(void); 495 496 #endif 497