1 /* 2 * HTTP GET/PUT IO engine 3 * 4 * IO engine to perform HTTP(S) GET/PUT requests via libcurl-easy. 5 * 6 * Copyright (C) 2018 SUSE LLC 7 * 8 * This program is free software; you can redistribute it and/or 9 * modify it under the terms of the GNU General Public License, 10 * version 2 as published by the Free Software Foundation.. 11 * 12 * This program is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 * GNU General Public License for more details. 16 * 17 * You should have received a copy of the GNU General Public 18 * License along with this program; if not, write to the Free 19 * Software Foundation, Inc., 51 Franklin Street, Fifth Floor, 20 * Boston, MA 02110-1301, USA. 21 */ 22 23 #include <pthread.h> 24 #include <time.h> 25 #include <curl/curl.h> 26 #include <openssl/hmac.h> 27 #include <openssl/sha.h> 28 #include <openssl/md5.h> 29 #include "fio.h" 30 #include "../optgroup.h" 31 32 33 enum { 34 FIO_HTTP_WEBDAV = 0, 35 FIO_HTTP_S3 = 1, 36 FIO_HTTP_SWIFT = 2, 37 38 FIO_HTTPS_OFF = 0, 39 FIO_HTTPS_ON = 1, 40 FIO_HTTPS_INSECURE = 2, 41 }; 42 43 struct http_data { 44 CURL *curl; 45 }; 46 47 struct http_options { 48 void *pad; 49 unsigned int https; 50 char *host; 51 char *user; 52 char *pass; 53 char *s3_key; 54 char *s3_keyid; 55 char *s3_region; 56 char *swift_auth_token; 57 int verbose; 58 unsigned int mode; 59 }; 60 61 struct http_curl_stream { 62 char *buf; 63 size_t pos; 64 size_t max; 65 }; 66 67 static struct fio_option options[] = { 68 { 69 .name = "https", 70 .lname = "https", 71 .type = FIO_OPT_STR, 72 .help = "Enable https", 73 .off1 = offsetof(struct http_options, https), 74 .def = "off", 75 .posval = { 76 { .ival = "off", 77 .oval = FIO_HTTPS_OFF, 78 .help = "No HTTPS", 79 }, 80 { .ival = "on", 81 .oval = FIO_HTTPS_ON, 82 .help = "Enable HTTPS", 83 }, 84 { .ival = "insecure", 85 .oval = FIO_HTTPS_INSECURE, 86 .help = "Enable HTTPS, disable peer verification", 87 }, 88 }, 89 .category = FIO_OPT_C_ENGINE, 90 .group = FIO_OPT_G_HTTP, 91 }, 92 { 93 .name = "http_host", 94 .lname = "http_host", 95 .type = FIO_OPT_STR_STORE, 96 .help = "Hostname (S3 bucket)", 97 .off1 = offsetof(struct http_options, host), 98 .def = "localhost", 99 .category = FIO_OPT_C_ENGINE, 100 .group = FIO_OPT_G_HTTP, 101 }, 102 { 103 .name = "http_user", 104 .lname = "http_user", 105 .type = FIO_OPT_STR_STORE, 106 .help = "HTTP user name", 107 .off1 = offsetof(struct http_options, user), 108 .category = FIO_OPT_C_ENGINE, 109 .group = FIO_OPT_G_HTTP, 110 }, 111 { 112 .name = "http_pass", 113 .lname = "http_pass", 114 .type = FIO_OPT_STR_STORE, 115 .help = "HTTP password", 116 .off1 = offsetof(struct http_options, pass), 117 .category = FIO_OPT_C_ENGINE, 118 .group = FIO_OPT_G_HTTP, 119 }, 120 { 121 .name = "http_s3_key", 122 .lname = "S3 secret key", 123 .type = FIO_OPT_STR_STORE, 124 .help = "S3 secret key", 125 .off1 = offsetof(struct http_options, s3_key), 126 .def = "", 127 .category = FIO_OPT_C_ENGINE, 128 .group = FIO_OPT_G_HTTP, 129 }, 130 { 131 .name = "http_s3_keyid", 132 .lname = "S3 key id", 133 .type = FIO_OPT_STR_STORE, 134 .help = "S3 key id", 135 .off1 = offsetof(struct http_options, s3_keyid), 136 .def = "", 137 .category = FIO_OPT_C_ENGINE, 138 .group = FIO_OPT_G_HTTP, 139 }, 140 { 141 .name = "http_swift_auth_token", 142 .lname = "Swift auth token", 143 .type = FIO_OPT_STR_STORE, 144 .help = "OpenStack Swift auth token", 145 .off1 = offsetof(struct http_options, swift_auth_token), 146 .def = "", 147 .category = FIO_OPT_C_ENGINE, 148 .group = FIO_OPT_G_HTTP, 149 }, 150 { 151 .name = "http_s3_region", 152 .lname = "S3 region", 153 .type = FIO_OPT_STR_STORE, 154 .help = "S3 region", 155 .off1 = offsetof(struct http_options, s3_region), 156 .def = "us-east-1", 157 .category = FIO_OPT_C_ENGINE, 158 .group = FIO_OPT_G_HTTP, 159 }, 160 { 161 .name = "http_mode", 162 .lname = "Request mode to use", 163 .type = FIO_OPT_STR, 164 .help = "Whether to use WebDAV, Swift, or S3", 165 .off1 = offsetof(struct http_options, mode), 166 .def = "webdav", 167 .posval = { 168 { .ival = "webdav", 169 .oval = FIO_HTTP_WEBDAV, 170 .help = "WebDAV server", 171 }, 172 { .ival = "s3", 173 .oval = FIO_HTTP_S3, 174 .help = "S3 storage backend", 175 }, 176 { .ival = "swift", 177 .oval = FIO_HTTP_SWIFT, 178 .help = "OpenStack Swift storage", 179 }, 180 }, 181 .category = FIO_OPT_C_ENGINE, 182 .group = FIO_OPT_G_HTTP, 183 }, 184 { 185 .name = "http_verbose", 186 .lname = "HTTP verbosity level", 187 .type = FIO_OPT_INT, 188 .help = "increase http engine verbosity", 189 .off1 = offsetof(struct http_options, verbose), 190 .def = "0", 191 .category = FIO_OPT_C_ENGINE, 192 .group = FIO_OPT_G_HTTP, 193 }, 194 { 195 .name = NULL, 196 }, 197 }; 198 199 static char *_aws_uriencode(const char *uri) 200 { 201 size_t bufsize = 1024; 202 char *r = malloc(bufsize); 203 char c; 204 int i, n; 205 const char *hex = "0123456789ABCDEF"; 206 207 if (!r) { 208 log_err("malloc failed\n"); 209 return NULL; 210 } 211 212 n = 0; 213 for (i = 0; (c = uri[i]); i++) { 214 if (n > bufsize-5) { 215 log_err("encoding the URL failed\n"); 216 return NULL; 217 } 218 219 if ( (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') 220 || (c >= '0' && c <= '9') || c == '_' || c == '-' 221 || c == '~' || c == '.' || c == '/') 222 r[n++] = c; 223 else { 224 r[n++] = '%'; 225 r[n++] = hex[(c >> 4 ) & 0xF]; 226 r[n++] = hex[c & 0xF]; 227 } 228 } 229 r[n++] = 0; 230 return r; 231 } 232 233 static char *_conv_hex(const unsigned char *p, size_t len) 234 { 235 char *r; 236 int i,n; 237 const char *hex = "0123456789abcdef"; 238 r = malloc(len * 2 + 1); 239 n = 0; 240 for (i = 0; i < len; i++) { 241 r[n++] = hex[(p[i] >> 4 ) & 0xF]; 242 r[n++] = hex[p[i] & 0xF]; 243 } 244 r[n] = 0; 245 246 return r; 247 } 248 249 static char *_gen_hex_sha256(const char *p, size_t len) 250 { 251 unsigned char hash[SHA256_DIGEST_LENGTH]; 252 253 SHA256((unsigned char*)p, len, hash); 254 return _conv_hex(hash, SHA256_DIGEST_LENGTH); 255 } 256 257 static char *_gen_hex_md5(const char *p, size_t len) 258 { 259 unsigned char hash[MD5_DIGEST_LENGTH]; 260 261 MD5((unsigned char*)p, len, hash); 262 return _conv_hex(hash, MD5_DIGEST_LENGTH); 263 } 264 265 static void _hmac(unsigned char *md, void *key, int key_len, char *data) { 266 #ifndef CONFIG_HAVE_OPAQUE_HMAC_CTX 267 HMAC_CTX _ctx; 268 #endif 269 HMAC_CTX *ctx; 270 unsigned int hmac_len; 271 272 #ifdef CONFIG_HAVE_OPAQUE_HMAC_CTX 273 ctx = HMAC_CTX_new(); 274 #else 275 ctx = &_ctx; 276 /* work-around crash in certain versions of libssl */ 277 HMAC_CTX_init(ctx); 278 #endif 279 HMAC_Init_ex(ctx, key, key_len, EVP_sha256(), NULL); 280 HMAC_Update(ctx, (unsigned char*)data, strlen(data)); 281 HMAC_Final(ctx, md, &hmac_len); 282 #ifdef CONFIG_HAVE_OPAQUE_HMAC_CTX 283 HMAC_CTX_free(ctx); 284 #else 285 HMAC_CTX_cleanup(ctx); 286 #endif 287 } 288 289 static int _curl_trace(CURL *handle, curl_infotype type, 290 char *data, size_t size, 291 void *userp) 292 { 293 const char *text; 294 (void)handle; /* prevent compiler warning */ 295 (void)userp; 296 297 switch (type) { 298 case CURLINFO_TEXT: 299 fprintf(stderr, "== Info: %s", data); 300 fallthrough; 301 default: 302 case CURLINFO_SSL_DATA_OUT: 303 case CURLINFO_SSL_DATA_IN: 304 return 0; 305 306 case CURLINFO_HEADER_OUT: 307 text = "=> Send header"; 308 break; 309 case CURLINFO_DATA_OUT: 310 text = "=> Send data"; 311 break; 312 case CURLINFO_HEADER_IN: 313 text = "<= Recv header"; 314 break; 315 case CURLINFO_DATA_IN: 316 text = "<= Recv data"; 317 break; 318 } 319 320 log_info("%s: %s", text, data); 321 return 0; 322 } 323 324 /* https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html 325 * https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html#signing-request-intro 326 */ 327 static void _add_aws_auth_header(CURL *curl, struct curl_slist *slist, struct http_options *o, 328 int op, const char *uri, char *buf, size_t len) 329 { 330 char date_short[16]; 331 char date_iso[32]; 332 char method[8]; 333 char dkey[128]; 334 char creq[512]; 335 char sts[256]; 336 char s[512]; 337 char *uri_encoded = NULL; 338 char *dsha = NULL; 339 char *csha = NULL; 340 char *signature = NULL; 341 const char *service = "s3"; 342 const char *aws = "aws4_request"; 343 unsigned char md[SHA256_DIGEST_LENGTH]; 344 345 time_t t = time(NULL); 346 struct tm *gtm = gmtime(&t); 347 348 strftime (date_short, sizeof(date_short), "%Y%m%d", gtm); 349 strftime (date_iso, sizeof(date_iso), "%Y%m%dT%H%M%SZ", gtm); 350 uri_encoded = _aws_uriencode(uri); 351 352 if (op == DDIR_WRITE) { 353 dsha = _gen_hex_sha256(buf, len); 354 sprintf(method, "PUT"); 355 } else { 356 /* DDIR_READ && DDIR_TRIM supply an empty body */ 357 if (op == DDIR_READ) 358 sprintf(method, "GET"); 359 else 360 sprintf(method, "DELETE"); 361 dsha = _gen_hex_sha256("", 0); 362 } 363 364 /* Create the canonical request first */ 365 snprintf(creq, sizeof(creq), 366 "%s\n" 367 "%s\n" 368 "\n" 369 "host:%s\n" 370 "x-amz-content-sha256:%s\n" 371 "x-amz-date:%s\n" 372 "\n" 373 "host;x-amz-content-sha256;x-amz-date\n" 374 "%s" 375 , method 376 , uri_encoded, o->host, dsha, date_iso, dsha); 377 378 csha = _gen_hex_sha256(creq, strlen(creq)); 379 snprintf(sts, sizeof(sts), "AWS4-HMAC-SHA256\n%s\n%s/%s/%s/%s\n%s", 380 date_iso, date_short, o->s3_region, service, aws, csha); 381 382 snprintf((char *)dkey, sizeof(dkey), "AWS4%s", o->s3_key); 383 _hmac(md, dkey, strlen(dkey), date_short); 384 _hmac(md, md, SHA256_DIGEST_LENGTH, o->s3_region); 385 _hmac(md, md, SHA256_DIGEST_LENGTH, (char*) service); 386 _hmac(md, md, SHA256_DIGEST_LENGTH, (char*) aws); 387 _hmac(md, md, SHA256_DIGEST_LENGTH, sts); 388 389 signature = _conv_hex(md, SHA256_DIGEST_LENGTH); 390 391 /* Surpress automatic Accept: header */ 392 slist = curl_slist_append(slist, "Accept:"); 393 394 snprintf(s, sizeof(s), "x-amz-content-sha256: %s", dsha); 395 slist = curl_slist_append(slist, s); 396 397 snprintf(s, sizeof(s), "x-amz-date: %s", date_iso); 398 slist = curl_slist_append(slist, s); 399 400 snprintf(s, sizeof(s), "Authorization: AWS4-HMAC-SHA256 Credential=%s/%s/%s/s3/aws4_request," 401 "SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=%s", 402 o->s3_keyid, date_short, o->s3_region, signature); 403 slist = curl_slist_append(slist, s); 404 405 curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist); 406 407 free(uri_encoded); 408 free(csha); 409 free(dsha); 410 free(signature); 411 } 412 413 static void _add_swift_header(CURL *curl, struct curl_slist *slist, struct http_options *o, 414 int op, const char *uri, char *buf, size_t len) 415 { 416 char *dsha = NULL; 417 char s[512]; 418 419 if (op == DDIR_WRITE) { 420 dsha = _gen_hex_md5(buf, len); 421 } 422 /* Surpress automatic Accept: header */ 423 slist = curl_slist_append(slist, "Accept:"); 424 425 snprintf(s, sizeof(s), "etag: %s", dsha); 426 slist = curl_slist_append(slist, s); 427 428 snprintf(s, sizeof(s), "x-auth-token: %s", o->swift_auth_token); 429 slist = curl_slist_append(slist, s); 430 431 curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist); 432 433 free(dsha); 434 } 435 436 static void fio_http_cleanup(struct thread_data *td) 437 { 438 struct http_data *http = td->io_ops_data; 439 440 if (http) { 441 curl_easy_cleanup(http->curl); 442 free(http); 443 } 444 } 445 446 static size_t _http_read(void *ptr, size_t size, size_t nmemb, void *stream) 447 { 448 struct http_curl_stream *state = stream; 449 size_t len = size * nmemb; 450 /* We're retrieving; nothing is supposed to be read locally */ 451 if (!stream) 452 return 0; 453 if (len+state->pos > state->max) 454 len = state->max - state->pos; 455 memcpy(ptr, &state->buf[state->pos], len); 456 state->pos += len; 457 return len; 458 } 459 460 static size_t _http_write(void *ptr, size_t size, size_t nmemb, void *stream) 461 { 462 struct http_curl_stream *state = stream; 463 /* We're just discarding the returned body after a PUT */ 464 if (!stream) 465 return nmemb; 466 if (size != 1) 467 return CURLE_WRITE_ERROR; 468 if (nmemb + state->pos > state->max) 469 return CURLE_WRITE_ERROR; 470 memcpy(&state->buf[state->pos], ptr, nmemb); 471 state->pos += nmemb; 472 return nmemb; 473 } 474 475 static int _http_seek(void *stream, curl_off_t offset, int origin) 476 { 477 struct http_curl_stream *state = stream; 478 if (offset < state->max && origin == SEEK_SET) { 479 state->pos = offset; 480 return CURL_SEEKFUNC_OK; 481 } else 482 return CURL_SEEKFUNC_FAIL; 483 } 484 485 static enum fio_q_status fio_http_queue(struct thread_data *td, 486 struct io_u *io_u) 487 { 488 struct http_data *http = td->io_ops_data; 489 struct http_options *o = td->eo; 490 struct http_curl_stream _curl_stream; 491 struct curl_slist *slist = NULL; 492 char object[512]; 493 char url[1024]; 494 long status; 495 CURLcode res; 496 int r = -1; 497 498 fio_ro_check(td, io_u); 499 memset(&_curl_stream, 0, sizeof(_curl_stream)); 500 snprintf(object, sizeof(object), "%s_%llu_%llu", td->files[0]->file_name, 501 io_u->offset, io_u->xfer_buflen); 502 if (o->https == FIO_HTTPS_OFF) 503 snprintf(url, sizeof(url), "http://%s%s", o->host, object); 504 else 505 snprintf(url, sizeof(url), "https://%s%s", o->host, object); 506 curl_easy_setopt(http->curl, CURLOPT_URL, url); 507 _curl_stream.buf = io_u->xfer_buf; 508 _curl_stream.max = io_u->xfer_buflen; 509 curl_easy_setopt(http->curl, CURLOPT_SEEKDATA, &_curl_stream); 510 curl_easy_setopt(http->curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)io_u->xfer_buflen); 511 512 if (o->mode == FIO_HTTP_S3) 513 _add_aws_auth_header(http->curl, slist, o, io_u->ddir, object, 514 io_u->xfer_buf, io_u->xfer_buflen); 515 else if (o->mode == FIO_HTTP_SWIFT) 516 _add_swift_header(http->curl, slist, o, io_u->ddir, object, 517 io_u->xfer_buf, io_u->xfer_buflen); 518 519 if (io_u->ddir == DDIR_WRITE) { 520 curl_easy_setopt(http->curl, CURLOPT_READDATA, &_curl_stream); 521 curl_easy_setopt(http->curl, CURLOPT_WRITEDATA, NULL); 522 curl_easy_setopt(http->curl, CURLOPT_UPLOAD, 1L); 523 res = curl_easy_perform(http->curl); 524 if (res == CURLE_OK) { 525 curl_easy_getinfo(http->curl, CURLINFO_RESPONSE_CODE, &status); 526 if (status == 100 || (status >= 200 && status <= 204)) 527 goto out; 528 log_err("DDIR_WRITE failed with HTTP status code %ld\n", status); 529 goto err; 530 } 531 } else if (io_u->ddir == DDIR_READ) { 532 curl_easy_setopt(http->curl, CURLOPT_READDATA, NULL); 533 curl_easy_setopt(http->curl, CURLOPT_WRITEDATA, &_curl_stream); 534 curl_easy_setopt(http->curl, CURLOPT_HTTPGET, 1L); 535 res = curl_easy_perform(http->curl); 536 if (res == CURLE_OK) { 537 curl_easy_getinfo(http->curl, CURLINFO_RESPONSE_CODE, &status); 538 if (status == 200) 539 goto out; 540 else if (status == 404) { 541 /* Object doesn't exist. Pretend we read 542 * zeroes */ 543 memset(io_u->xfer_buf, 0, io_u->xfer_buflen); 544 goto out; 545 } 546 log_err("DDIR_READ failed with HTTP status code %ld\n", status); 547 } 548 goto err; 549 } else if (io_u->ddir == DDIR_TRIM) { 550 curl_easy_setopt(http->curl, CURLOPT_HTTPGET, 1L); 551 curl_easy_setopt(http->curl, CURLOPT_CUSTOMREQUEST, "DELETE"); 552 curl_easy_setopt(http->curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)0); 553 curl_easy_setopt(http->curl, CURLOPT_READDATA, NULL); 554 curl_easy_setopt(http->curl, CURLOPT_WRITEDATA, NULL); 555 res = curl_easy_perform(http->curl); 556 if (res == CURLE_OK) { 557 curl_easy_getinfo(http->curl, CURLINFO_RESPONSE_CODE, &status); 558 if (status == 200 || status == 202 || status == 204 || status == 404) 559 goto out; 560 log_err("DDIR_TRIM failed with HTTP status code %ld\n", status); 561 } 562 goto err; 563 } 564 565 log_err("WARNING: Only DDIR_READ/DDIR_WRITE/DDIR_TRIM are supported!\n"); 566 567 err: 568 io_u->error = r; 569 td_verror(td, io_u->error, "transfer"); 570 out: 571 curl_slist_free_all(slist); 572 return FIO_Q_COMPLETED; 573 } 574 575 static struct io_u *fio_http_event(struct thread_data *td, int event) 576 { 577 /* sync IO engine - never any outstanding events */ 578 return NULL; 579 } 580 581 int fio_http_getevents(struct thread_data *td, unsigned int min, 582 unsigned int max, const struct timespec *t) 583 { 584 /* sync IO engine - never any outstanding events */ 585 return 0; 586 } 587 588 static int fio_http_setup(struct thread_data *td) 589 { 590 struct http_data *http = NULL; 591 struct http_options *o = td->eo; 592 593 /* allocate engine specific structure to deal with libhttp. */ 594 http = calloc(1, sizeof(*http)); 595 if (!http) { 596 log_err("calloc failed.\n"); 597 goto cleanup; 598 } 599 600 http->curl = curl_easy_init(); 601 if (o->verbose) 602 curl_easy_setopt(http->curl, CURLOPT_VERBOSE, 1L); 603 if (o->verbose > 1) 604 curl_easy_setopt(http->curl, CURLOPT_DEBUGFUNCTION, &_curl_trace); 605 curl_easy_setopt(http->curl, CURLOPT_NOPROGRESS, 1L); 606 curl_easy_setopt(http->curl, CURLOPT_FOLLOWLOCATION, 1L); 607 curl_easy_setopt(http->curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP|CURLPROTO_HTTPS); 608 if (o->https == FIO_HTTPS_INSECURE) { 609 curl_easy_setopt(http->curl, CURLOPT_SSL_VERIFYPEER, 0L); 610 curl_easy_setopt(http->curl, CURLOPT_SSL_VERIFYHOST, 0L); 611 } 612 curl_easy_setopt(http->curl, CURLOPT_READFUNCTION, _http_read); 613 curl_easy_setopt(http->curl, CURLOPT_WRITEFUNCTION, _http_write); 614 curl_easy_setopt(http->curl, CURLOPT_SEEKFUNCTION, &_http_seek); 615 if (o->user && o->pass) { 616 curl_easy_setopt(http->curl, CURLOPT_USERNAME, o->user); 617 curl_easy_setopt(http->curl, CURLOPT_PASSWORD, o->pass); 618 curl_easy_setopt(http->curl, CURLOPT_HTTPAUTH, CURLAUTH_ANY); 619 } 620 621 td->io_ops_data = http; 622 623 /* Force single process mode. */ 624 td->o.use_thread = 1; 625 626 return 0; 627 cleanup: 628 fio_http_cleanup(td); 629 return 1; 630 } 631 632 static int fio_http_open(struct thread_data *td, struct fio_file *f) 633 { 634 return 0; 635 } 636 static int fio_http_invalidate(struct thread_data *td, struct fio_file *f) 637 { 638 return 0; 639 } 640 641 FIO_STATIC struct ioengine_ops ioengine = { 642 .name = "http", 643 .version = FIO_IOOPS_VERSION, 644 .flags = FIO_DISKLESSIO | FIO_SYNCIO, 645 .setup = fio_http_setup, 646 .queue = fio_http_queue, 647 .getevents = fio_http_getevents, 648 .event = fio_http_event, 649 .cleanup = fio_http_cleanup, 650 .open_file = fio_http_open, 651 .invalidate = fio_http_invalidate, 652 .options = options, 653 .option_struct_size = sizeof(struct http_options), 654 }; 655 656 static void fio_init fio_http_register(void) 657 { 658 register_ioengine(&ioengine); 659 } 660 661 static void fio_exit fio_http_unregister(void) 662 { 663 unregister_ioengine(&ioengine); 664 } 665