1 2 /* 3 * s3backer - FUSE-based single file backing store via Amazon S3 4 * 5 * Copyright 2008-2011 Archie L. Cobbs <archie@dellroad.org> 6 * 7 * This program is free software; you can redistribute it and/or 8 * modify it under the terms of the GNU General Public License 9 * as published by the Free Software Foundation; either version 2 10 * of the License, or (at your option) any later version. 11 * 12 * This program is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 * GNU General Public License for more details. 16 * 17 * You should have received a copy of the GNU General Public License 18 * along with this program; if not, write to the Free Software 19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 20 * 02110-1301, USA. 21 * 22 * In addition, as a special exception, the copyright holders give 23 * permission to link the code of portions of this program with the 24 * OpenSSL library under certain conditions as described in each 25 * individual source file, and distribute linked combinations including 26 * the two. 27 * 28 * You must obey the GNU General Public License in all respects for all 29 * of the code used other than OpenSSL. If you modify file(s) with this 30 * exception, you may extend this exception to your version of the 31 * file(s), but you are not obligated to do so. If you do not wish to do 32 * so, delete this exception statement from your version. If you delete 33 * this exception statement from all source files in the program, then 34 * also delete it here. 35 */ 36 37 #include "s3backer.h" 38 #include "block_cache.h" 39 #include "ec_protect.h" 40 #include "fuse_ops.h" 41 #include "http_io.h" 42 #include "test_io.h" 43 #include "s3b_config.h" 44 #include "dcache.h" 45 46 /**************************************************************************** 47 * DEFINITIONS * 48 ****************************************************************************/ 49 50 /* S3 URL */ 51 #define S3_DOMAIN "amazonaws.com" 52 53 /* S3 access permission strings */ 54 #define S3_ACCESS_PRIVATE "private" 55 #define S3_ACCESS_PUBLIC_READ "public-read" 56 #define S3_ACCESS_PUBLIC_READ_WRITE "public-read-write" 57 #define S3_ACCESS_AUTHENTICATED_READ "authenticated-read" 58 59 /* Default values for some configuration parameters */ 60 #define S3BACKER_DEFAULT_ACCESS_TYPE S3_ACCESS_PRIVATE 61 #define S3BACKER_DEFAULT_AUTH_VERSION AUTH_VERSION_AWS4 62 #define S3BACKER_DEFAULT_REGION "us-east-1" 63 #define S3BACKER_DEFAULT_PWD_FILE ".s3backer_passwd" 64 #define S3BACKER_DEFAULT_PREFIX "" 65 #define S3BACKER_DEFAULT_FILENAME "file" 66 #define S3BACKER_DEFAULT_STATS_FILENAME "stats" 67 #define S3BACKER_DEFAULT_BLOCKSIZE 4096 68 #define S3BACKER_DEFAULT_TIMEOUT 30 // 30s 69 #define S3BACKER_DEFAULT_FILE_MODE 0600 70 #define S3BACKER_DEFAULT_FILE_MODE_READ_ONLY 0400 71 #define S3BACKER_DEFAULT_INITIAL_RETRY_PAUSE 200 // 200ms 72 #define S3BACKER_DEFAULT_MAX_RETRY_PAUSE 30000 // 30s 73 #define S3BACKER_DEFAULT_MIN_WRITE_DELAY 500 // 500ms 74 #define S3BACKER_DEFAULT_MD5_CACHE_TIME 10000 // 10s 75 #define S3BACKER_DEFAULT_MD5_CACHE_SIZE 10000 76 #define S3BACKER_DEFAULT_BLOCK_CACHE_SIZE 1000 77 #define S3BACKER_DEFAULT_BLOCK_CACHE_NUM_THREADS 20 78 #define S3BACKER_DEFAULT_BLOCK_CACHE_WRITE_DELAY 250 // 250ms 79 #define S3BACKER_DEFAULT_BLOCK_CACHE_TIMEOUT 0 80 #define S3BACKER_DEFAULT_BLOCK_CACHE_MAX_DIRTY 0 81 #define S3BACKER_DEFAULT_READ_AHEAD 4 82 #define S3BACKER_DEFAULT_READ_AHEAD_TRIGGER 2 83 #define S3BACKER_DEFAULT_COMPRESSION Z_NO_COMPRESSION 84 #define S3BACKER_DEFAULT_ENCRYPTION "AES-128-CBC" 85 86 /* MacFUSE setting for kernel daemon timeout */ 87 #ifdef __APPLE__ 88 #ifndef FUSE_MAX_DAEMON_TIMEOUT 89 #define FUSE_MAX_DAEMON_TIMEOUT 600 90 #endif 91 #define s3bquote0(x) #x 92 #define s3bquote(x) s3bquote0(x) 93 #define FUSE_MAX_DAEMON_TIMEOUT_STRING s3bquote(FUSE_MAX_DAEMON_TIMEOUT) 94 #endif /* __APPLE__ */ 95 96 /* Block counting info */ 97 struct list_blocks { 98 u_int *bitmap; 99 int print_dots; 100 uintmax_t count; 101 }; 102 #define BLOCKS_PER_DOT 0x100 103 104 /**************************************************************************** 105 * FUNCTION DECLARATIONS * 106 ****************************************************************************/ 107 108 static print_stats_t s3b_config_print_stats; 109 static clear_stats_t s3b_config_clear_stats; 110 111 static int parse_size_string(const char *s, uintmax_t *valp); 112 static void unparse_size_string(char *buf, size_t bmax, uintmax_t value); 113 static int search_access_for(const char *file, const char *accessId, char **idptr, char **pwptr); 114 static int handle_unknown_option(void *data, const char *arg, int key, struct fuse_args *outargs); 115 static void syslog_logger(int level, const char *fmt, ...) __attribute__ ((__format__ (__printf__, 2, 3))); 116 static void stderr_logger(int level, const char *fmt, ...) __attribute__ ((__format__ (__printf__, 2, 3))); 117 static int validate_config(void); 118 static void list_blocks_callback(void *arg, s3b_block_t block_num); 119 static void dump_config(void); 120 static void usage(void); 121 122 /**************************************************************************** 123 * VARIABLE DEFINITIONS * 124 ****************************************************************************/ 125 126 /* Upload/download strings */ 127 static const char *const upload_download_names[] = { "download", "upload" }; 128 129 /* Valid S3 access values */ 130 static const char *const s3_acls[] = { 131 S3_ACCESS_PRIVATE, 132 S3_ACCESS_PUBLIC_READ, 133 S3_ACCESS_PUBLIC_READ_WRITE, 134 S3_ACCESS_AUTHENTICATED_READ 135 }; 136 137 /* Valid S3 authentication types */ 138 static const char *const s3_auth_types[] = { 139 AUTH_VERSION_AWS2, 140 AUTH_VERSION_AWS4, 141 }; 142 143 /* Configuration structure */ 144 static char user_agent_buf[64]; 145 static struct s3b_config config = { 146 147 /* HTTP config */ 148 .http_io= { 149 .accessId= NULL, 150 .accessKey= NULL, 151 .baseURL= NULL, 152 .region= NULL, 153 .bucket= NULL, 154 .sse= NULL, 155 .blockHashPrefix= 0, 156 .prefix= S3BACKER_DEFAULT_PREFIX, 157 .accessType= S3BACKER_DEFAULT_ACCESS_TYPE, 158 .authVersion= S3BACKER_DEFAULT_AUTH_VERSION, 159 .user_agent= user_agent_buf, 160 .compress= S3BACKER_DEFAULT_COMPRESSION, 161 .timeout= S3BACKER_DEFAULT_TIMEOUT, 162 .initial_retry_pause= S3BACKER_DEFAULT_INITIAL_RETRY_PAUSE, 163 .max_retry_pause= S3BACKER_DEFAULT_MAX_RETRY_PAUSE, 164 }, 165 166 /* "Eventual consistency" protection config */ 167 .ec_protect= { 168 .min_write_delay= S3BACKER_DEFAULT_MIN_WRITE_DELAY, 169 .cache_time= S3BACKER_DEFAULT_MD5_CACHE_TIME, 170 .cache_size= S3BACKER_DEFAULT_MD5_CACHE_SIZE, 171 }, 172 173 /* Block cache config */ 174 .block_cache= { 175 .cache_size= S3BACKER_DEFAULT_BLOCK_CACHE_SIZE, 176 .num_threads= S3BACKER_DEFAULT_BLOCK_CACHE_NUM_THREADS, 177 .write_delay= S3BACKER_DEFAULT_BLOCK_CACHE_WRITE_DELAY, 178 .max_dirty= S3BACKER_DEFAULT_BLOCK_CACHE_MAX_DIRTY, 179 .timeout= S3BACKER_DEFAULT_BLOCK_CACHE_TIMEOUT, 180 .read_ahead= S3BACKER_DEFAULT_READ_AHEAD, 181 .read_ahead_trigger= S3BACKER_DEFAULT_READ_AHEAD_TRIGGER, 182 }, 183 184 /* FUSE operations config */ 185 .fuse_ops= { 186 .filename= S3BACKER_DEFAULT_FILENAME, 187 .stats_filename= S3BACKER_DEFAULT_STATS_FILENAME, 188 .file_mode= -1, /* default depends on 'read_only' */ 189 }, 190 191 /* Common stuff */ 192 .block_size= 0, 193 .file_size= 0, 194 .quiet= 0, 195 .erase= 0, 196 .no_auto_detect= 0, 197 .reset= 0, 198 .log= syslog_logger 199 }; 200 201 /* 202 * Command line flags 203 * 204 * Note: each entry here is listed twice, so both version "--foo=X" and "-o foo=X" work. 205 * See http://code.google.com/p/s3backer/issues/detail?id=7 206 */ 207 static const struct fuse_opt option_list[] = { 208 { 209 .templ= "--accessFile=%s", 210 .offset= offsetof(struct s3b_config, accessFile), 211 }, 212 { 213 .templ= "--accessId=%s", 214 .offset= offsetof(struct s3b_config, http_io.accessId), 215 }, 216 { 217 .templ= "--accessKey=%s", 218 .offset= offsetof(struct s3b_config, http_io.accessKey), 219 }, 220 { 221 .templ= "--accessType=%s", 222 .offset= offsetof(struct s3b_config, http_io.accessType), 223 }, 224 { 225 .templ= "--accessEC2IAM=%s", 226 .offset= offsetof(struct s3b_config, http_io.ec2iam_role), 227 }, 228 { 229 .templ= "--authVersion=%s", 230 .offset= offsetof(struct s3b_config, http_io.authVersion), 231 }, 232 { 233 .templ= "--listBlocks", 234 .offset= offsetof(struct s3b_config, list_blocks), 235 .value= 1 236 }, 237 { 238 .templ= "--baseURL=%s", 239 .offset= offsetof(struct s3b_config, http_io.baseURL), 240 }, 241 { 242 .templ= "--region=%s", 243 .offset= offsetof(struct s3b_config, http_io.region), 244 }, 245 { 246 .templ= "--sse=%s", 247 .offset= offsetof(struct s3b_config, http_io.sse), 248 }, 249 { 250 .templ= "--blockCacheSize=%u", 251 .offset= offsetof(struct s3b_config, block_cache.cache_size), 252 }, 253 { 254 .templ= "--blockCacheSync", 255 .offset= offsetof(struct s3b_config, block_cache.synchronous), 256 .value= 1 257 }, 258 { 259 .templ= "--blockCacheThreads=%u", 260 .offset= offsetof(struct s3b_config, block_cache.num_threads), 261 }, 262 { 263 .templ= "--blockCacheTimeout=%u", 264 .offset= offsetof(struct s3b_config, block_cache.timeout), 265 }, 266 { 267 .templ= "--blockCacheWriteDelay=%u", 268 .offset= offsetof(struct s3b_config, block_cache.write_delay), 269 }, 270 { 271 .templ= "--blockCacheMaxDirty=%u", 272 .offset= offsetof(struct s3b_config, block_cache.max_dirty), 273 }, 274 { 275 .templ= "--blockCacheRecoverDirtyBlocks", 276 .offset= offsetof(struct s3b_config, block_cache.recover_dirty_blocks), 277 .value= 1 278 }, 279 { 280 .templ= "--readAhead=%u", 281 .offset= offsetof(struct s3b_config, block_cache.read_ahead), 282 }, 283 { 284 .templ= "--readAheadTrigger=%u", 285 .offset= offsetof(struct s3b_config, block_cache.read_ahead_trigger), 286 }, 287 { 288 .templ= "--blockCacheFile=%s", 289 .offset= offsetof(struct s3b_config, block_cache.cache_file), 290 }, 291 { 292 .templ= "--blockCacheNoVerify", 293 .offset= offsetof(struct s3b_config, block_cache.no_verify), 294 .value= 1 295 }, 296 { 297 .templ= "--blockSize=%s", 298 .offset= offsetof(struct s3b_config, block_size_str), 299 }, 300 { 301 .templ= "--maxUploadSpeed=%s", 302 .offset= offsetof(struct s3b_config, max_speed_str[HTTP_UPLOAD]), 303 }, 304 { 305 .templ= "--maxDownloadSpeed=%s", 306 .offset= offsetof(struct s3b_config, max_speed_str[HTTP_DOWNLOAD]), 307 }, 308 { 309 .templ= "--md5CacheSize=%u", 310 .offset= offsetof(struct s3b_config, ec_protect.cache_size), 311 }, 312 { 313 .templ= "--md5CacheTime=%u", 314 .offset= offsetof(struct s3b_config, ec_protect.cache_time), 315 }, 316 { 317 .templ= "--debug", 318 .offset= offsetof(struct s3b_config, debug), 319 .value= 1 320 }, 321 { 322 .templ= "--debug-http", 323 .offset= offsetof(struct s3b_config, http_io.debug_http), 324 .value= 1 325 }, 326 { 327 .templ= "--quiet", 328 .offset= offsetof(struct s3b_config, quiet), 329 .value= 1 330 }, 331 { 332 .templ= "--erase", 333 .offset= offsetof(struct s3b_config, erase), 334 .value= 1 335 }, 336 { 337 .templ= "--reset-mounted-flag", 338 .offset= offsetof(struct s3b_config, reset), 339 .value= 1 340 }, 341 { 342 .templ= "--vhost", 343 .offset= offsetof(struct s3b_config, http_io.vhost), 344 .value= 1 345 }, 346 { 347 .templ= "--fileMode=%o", 348 .offset= offsetof(struct s3b_config, fuse_ops.file_mode), 349 }, 350 { 351 .templ= "--filename=%s", 352 .offset= offsetof(struct s3b_config, fuse_ops.filename), 353 }, 354 { 355 .templ= "--force", 356 .offset= offsetof(struct s3b_config, force), 357 .value= 1 358 }, 359 { 360 .templ= "--noAutoDetect", 361 .offset= offsetof(struct s3b_config, no_auto_detect), 362 .value= 1 363 }, 364 { 365 .templ= "--initialRetryPause=%u", 366 .offset= offsetof(struct s3b_config, http_io.initial_retry_pause), 367 }, 368 { 369 .templ= "--maxRetryPause=%u", 370 .offset= offsetof(struct s3b_config, http_io.max_retry_pause), 371 }, 372 { 373 .templ= "--minWriteDelay=%u", 374 .offset= offsetof(struct s3b_config, ec_protect.min_write_delay), 375 }, 376 { 377 .templ= "--blockHashPrefix", 378 .offset= offsetof(struct s3b_config, http_io.blockHashPrefix), 379 .value= 1 380 }, 381 { 382 .templ= "--prefix=%s", 383 .offset= offsetof(struct s3b_config, http_io.prefix), 384 }, 385 { 386 .templ= "--defaultContentEncoding=%s", 387 .offset= offsetof(struct s3b_config, http_io.default_ce), 388 }, 389 { 390 .templ= "--readOnly", 391 .offset= offsetof(struct s3b_config, fuse_ops.read_only), 392 .value= 1 393 }, 394 { 395 .templ= "--size=%s", 396 .offset= offsetof(struct s3b_config, file_size_str), 397 }, 398 { 399 .templ= "--statsFilename=%s", 400 .offset= offsetof(struct s3b_config, fuse_ops.stats_filename), 401 }, 402 { 403 .templ= "--rrs", 404 .offset= offsetof(struct s3b_config, http_io.rrs), 405 .value= 1 406 }, 407 { 408 .templ= "--storageClass=%s", 409 .offset= offsetof(struct s3b_config, http_io.storage_class), 410 }, 411 { 412 .templ= "--ssl", 413 .offset= offsetof(struct s3b_config, ssl), 414 .value= 1 415 }, 416 { 417 .templ= "--cacert=%s", 418 .offset= offsetof(struct s3b_config, http_io.cacert), 419 }, 420 { 421 .templ= "--insecure", 422 .offset= offsetof(struct s3b_config, http_io.insecure), 423 .value= 1 424 }, 425 { 426 .templ= "--compress", 427 .offset= offsetof(struct s3b_config, http_io.compress), 428 .value= Z_DEFAULT_COMPRESSION 429 }, 430 { 431 .templ= "--compress=%d", 432 .offset= offsetof(struct s3b_config, http_io.compress), 433 }, 434 { 435 .templ= "--encrypt", 436 .offset= offsetof(struct s3b_config, encrypt), 437 .value= 1 438 }, 439 { 440 .templ= "--encrypt=%s", 441 .offset= offsetof(struct s3b_config, http_io.encryption), 442 }, 443 { 444 .templ= "--keyLength=%u", 445 .offset= offsetof(struct s3b_config, http_io.key_length), 446 }, 447 { 448 .templ= "--password=%s", 449 .offset= offsetof(struct s3b_config, http_io.password), 450 }, 451 { 452 .templ= "--passwordFile=%s", 453 .offset= offsetof(struct s3b_config, password_file), 454 }, 455 { 456 .templ= "--test", 457 .offset= offsetof(struct s3b_config, test), 458 .value= 1 459 }, 460 { 461 .templ= "--timeout=%u", 462 .offset= offsetof(struct s3b_config, http_io.timeout), 463 }, 464 { 465 .templ= "--directIO", 466 .offset= offsetof(struct s3b_config, fuse_ops.direct_io), 467 .value= 1 468 }, 469 }; 470 471 /* Default flags we send to FUSE */ 472 static const char *const s3backer_fuse_defaults[] = { 473 "-okernel_cache", 474 "-oallow_other", 475 "-ouse_ino", 476 "-omax_readahead=0", 477 "-osubtype=s3backer", 478 "-oentry_timeout=31536000", 479 "-onegative_timeout=31536000", 480 "-oattr_timeout=0", // because statistics file length changes 481 "-odefault_permissions", 482 #ifndef __FreeBSD__ 483 "-onodev", 484 #endif 485 "-onosuid", 486 #ifdef __APPLE__ 487 "-odaemon_timeout=" FUSE_MAX_DAEMON_TIMEOUT_STRING, 488 #endif 489 /* "-ointr", */ 490 }; 491 492 /* Size suffixes */ 493 struct size_suffix { 494 const char *suffix; 495 int bits; 496 }; 497 static const struct size_suffix size_suffixes[] = { 498 { 499 .suffix= "k", 500 .bits= 10 501 }, 502 { 503 .suffix= "m", 504 .bits= 20 505 }, 506 { 507 .suffix= "g", 508 .bits= 30 509 }, 510 { 511 .suffix= "t", 512 .bits= 40 513 }, 514 { 515 .suffix= "p", 516 .bits= 50 517 }, 518 { 519 .suffix= "e", 520 .bits= 60 521 }, 522 { 523 .suffix= "z", 524 .bits= 70 525 }, 526 { 527 .suffix= "y", 528 .bits= 80 529 }, 530 }; 531 532 /* s3backer_store layers */ 533 struct s3backer_store *block_cache_store; 534 struct s3backer_store *ec_protect_store; 535 struct s3backer_store *http_io_store; 536 struct s3backer_store *test_io_store; 537 538 /**************************************************************************** 539 * PUBLIC FUNCTION DEFINITIONS * 540 ****************************************************************************/ 541 542 struct s3b_config * 543 s3backer_get_config(int argc, char **argv) 544 { 545 const int num_options = sizeof(option_list) / sizeof(*option_list); 546 struct fuse_opt dup_option_list[2 * sizeof(option_list) + 1]; 547 char buf[1024]; 548 int i; 549 550 /* Remember user creds */ 551 config.fuse_ops.uid = getuid(); 552 config.fuse_ops.gid = getgid(); 553 554 /* Set user-agent */ 555 snprintf(user_agent_buf, sizeof(user_agent_buf), "%s/%s/%s", PACKAGE, VERSION, s3backer_version); 556 557 /* Copy passed args */ 558 memset(&config.fuse_args, 0, sizeof(config.fuse_args)); 559 for (i = 0; i < argc; i++) { 560 if (fuse_opt_insert_arg(&config.fuse_args, i, argv[i]) != 0) 561 err(1, "fuse_opt_insert_arg"); 562 } 563 564 /* Insert our default FUSE options */ 565 for (i = 0; i < sizeof(s3backer_fuse_defaults) / sizeof(*s3backer_fuse_defaults); i++) { 566 if (fuse_opt_insert_arg(&config.fuse_args, i + 1, s3backer_fuse_defaults[i]) != 0) 567 err(1, "fuse_opt_insert_arg"); 568 } 569 570 /* Create the equivalent fstab options (without the "--") for each option in the option list */ 571 memcpy(dup_option_list, option_list, sizeof(option_list)); 572 memcpy(dup_option_list + num_options, option_list, sizeof(option_list)); 573 for (i = num_options; i < 2 * num_options; i++) 574 dup_option_list[i].templ += 2; 575 dup_option_list[2 * num_options].templ = NULL; 576 577 /* Parse command line flags */ 578 if (fuse_opt_parse(&config.fuse_args, &config, dup_option_list, handle_unknown_option) != 0) 579 return NULL; 580 581 /* Validate configuration */ 582 if (validate_config() != 0) 583 return NULL; 584 585 /* Set fsname based on configuration */ 586 snprintf(buf, sizeof(buf), "-ofsname=%s", config.description); 587 if (fuse_opt_insert_arg(&config.fuse_args, 1, buf) != 0) 588 err(1, "fuse_opt_insert_arg"); 589 590 /* Set up fuse_ops callbacks */ 591 config.fuse_ops.print_stats = s3b_config_print_stats; 592 config.fuse_ops.clear_stats = s3b_config_clear_stats; 593 config.fuse_ops.s3bconf = &config; 594 595 /* Debug */ 596 if (config.debug) 597 dump_config(); 598 599 /* Done */ 600 return &config; 601 } 602 603 /* 604 * Create the s3backer_store used at runtime. 605 */ 606 struct s3backer_store * 607 s3backer_create_store(struct s3b_config *conf) 608 { 609 struct s3backer_store *store; 610 int32_t old_mount_token; 611 int32_t new_mount_token; 612 int r; 613 614 /* Sanity check */ 615 if (http_io_store != NULL || test_io_store != NULL) { 616 errno = EINVAL; 617 return NULL; 618 } 619 620 /* Create HTTP (or test) layer */ 621 if (conf->test) { 622 if ((test_io_store = test_io_create(&conf->http_io)) == NULL) 623 return NULL; 624 store = test_io_store; 625 } else { 626 if ((http_io_store = http_io_create(&conf->http_io)) == NULL) 627 return NULL; 628 store = http_io_store; 629 } 630 631 /* Create eventual consistency protection layer (if desired) */ 632 if (conf->ec_protect.cache_size > 0) { 633 if ((ec_protect_store = ec_protect_create(&conf->ec_protect, store)) == NULL) 634 goto fail_with_errno; 635 store = ec_protect_store; 636 } 637 638 /* Create block cache layer (if desired) */ 639 if (conf->block_cache.cache_size > 0) { 640 if ((block_cache_store = block_cache_create(&conf->block_cache, store)) == NULL) 641 goto fail_with_errno; 642 store = block_cache_store; 643 } 644 645 /* Set mount token and check previous value one last time */ 646 new_mount_token = -1; 647 if (!conf->fuse_ops.read_only) { 648 srandom((long)time(NULL) ^ (long)&old_mount_token); 649 do 650 new_mount_token = random(); 651 while (new_mount_token <= 0); 652 } 653 if ((r = (*store->set_mount_token)(store, &old_mount_token, new_mount_token)) != 0) { 654 (*conf->log)(LOG_ERR, "error reading mount token on %s: %s", conf->description, strerror(r)); 655 goto fail; 656 } 657 if (old_mount_token != 0) { 658 if (!conf->force && !conf->block_cache.perform_flush) { 659 (*conf->log)(LOG_ERR, "%s appears to be mounted by another s3backer process (using mount token 0x%08x)", 660 config.description, (int)old_mount_token); 661 r = EBUSY; 662 goto fail; 663 } 664 } 665 if (new_mount_token != -1) 666 (*conf->log)(LOG_INFO, "established new mount token 0x%08x", (int)new_mount_token); 667 668 /* Done */ 669 return store; 670 671 fail_with_errno: 672 r = errno; 673 fail: 674 if (store != NULL) 675 (*store->destroy)(store); 676 block_cache_store = NULL; 677 ec_protect_store = NULL; 678 http_io_store = NULL; 679 test_io_store = NULL; 680 errno = r; 681 return NULL; 682 } 683 684 /**************************************************************************** 685 * INTERNAL FUNCTION DEFINITIONS * 686 ****************************************************************************/ 687 688 static void 689 s3b_config_print_stats(void *prarg, printer_t *printer) 690 { 691 struct http_io_stats http_io_stats; 692 struct ec_protect_stats ec_protect_stats; 693 struct block_cache_stats block_cache_stats; 694 double curl_reuse_ratio = 0.0; 695 u_int total_oom = 0; 696 u_int total_curls; 697 698 /* Get HTTP stats */ 699 if (http_io_store != NULL) 700 http_io_get_stats(http_io_store, &http_io_stats); 701 702 /* Get EC protection stats */ 703 if (ec_protect_store != NULL) 704 ec_protect_get_stats(ec_protect_store, &ec_protect_stats); 705 706 /* Get block cache stats */ 707 if (block_cache_store != NULL) 708 block_cache_get_stats(block_cache_store, &block_cache_stats); 709 710 /* Print stats in human-readable form */ 711 if (http_io_store != NULL) { 712 (*printer)(prarg, "%-28s %u\n", "http_normal_blocks_read", http_io_stats.normal_blocks_read); 713 (*printer)(prarg, "%-28s %u\n", "http_normal_blocks_written", http_io_stats.normal_blocks_written); 714 (*printer)(prarg, "%-28s %u\n", "http_zero_blocks_read", http_io_stats.zero_blocks_read); 715 (*printer)(prarg, "%-28s %u\n", "http_zero_blocks_written", http_io_stats.zero_blocks_written); 716 if (config.list_blocks) { 717 (*printer)(prarg, "%-28s %u\n", "http_empty_blocks_read", http_io_stats.empty_blocks_read); 718 (*printer)(prarg, "%-28s %u\n", "http_empty_blocks_written", http_io_stats.empty_blocks_written); 719 } 720 (*printer)(prarg, "%-28s %u\n", "http_gets", http_io_stats.http_gets.count); 721 (*printer)(prarg, "%-28s %u\n", "http_puts", http_io_stats.http_puts.count); 722 (*printer)(prarg, "%-28s %u\n", "http_deletes", http_io_stats.http_deletes.count); 723 (*printer)(prarg, "%-28s %.3f sec\n", "http_avg_get_time", http_io_stats.http_gets.count > 0 ? 724 http_io_stats.http_gets.time / http_io_stats.http_gets.count : 0.0); 725 (*printer)(prarg, "%-28s %.3f sec\n", "http_avg_put_time", http_io_stats.http_puts.count > 0 ? 726 http_io_stats.http_puts.time / http_io_stats.http_puts.count : 0.0); 727 (*printer)(prarg, "%-28s %.3f sec\n", "http_avg_delete_time", http_io_stats.http_deletes.count > 0 ? 728 http_io_stats.http_deletes.time / http_io_stats.http_deletes.count : 0.0); 729 (*printer)(prarg, "%-28s %u\n", "http_unauthorized", http_io_stats.http_unauthorized); 730 (*printer)(prarg, "%-28s %u\n", "http_forbidden", http_io_stats.http_forbidden); 731 (*printer)(prarg, "%-28s %u\n", "http_stale", http_io_stats.http_stale); 732 (*printer)(prarg, "%-28s %u\n", "http_verified", http_io_stats.http_verified); 733 (*printer)(prarg, "%-28s %u\n", "http_mismatch", http_io_stats.http_mismatch); 734 (*printer)(prarg, "%-28s %u\n", "http_5xx_error", http_io_stats.http_5xx_error); 735 (*printer)(prarg, "%-28s %u\n", "http_4xx_error", http_io_stats.http_4xx_error); 736 (*printer)(prarg, "%-28s %u\n", "http_other_error", http_io_stats.http_other_error); 737 (*printer)(prarg, "%-28s %u\n", "http_canceled_writes", http_io_stats.http_canceled_writes); 738 (*printer)(prarg, "%-28s %u\n", "http_num_retries", http_io_stats.num_retries); 739 (*printer)(prarg, "%-28s %ju.%03u sec\n", "http_total_retry_delay", 740 (uintmax_t)(http_io_stats.retry_delay / 1000), (u_int)(http_io_stats.retry_delay % 1000)); 741 total_curls = http_io_stats.curl_handles_created + http_io_stats.curl_handles_reused; 742 if (total_curls > 0) 743 curl_reuse_ratio = (double)http_io_stats.curl_handles_reused / (double)total_curls; 744 (*printer)(prarg, "%-28s %.4f\n", "curl_handle_reuse_ratio", curl_reuse_ratio); 745 (*printer)(prarg, "%-28s %u\n", "curl_timeouts", http_io_stats.curl_timeouts); 746 (*printer)(prarg, "%-28s %u\n", "curl_connect_failed", http_io_stats.curl_connect_failed); 747 (*printer)(prarg, "%-28s %u\n", "curl_host_unknown", http_io_stats.curl_host_unknown); 748 (*printer)(prarg, "%-28s %u\n", "curl_out_of_memory", http_io_stats.curl_out_of_memory); 749 (*printer)(prarg, "%-28s %u\n", "curl_other_error", http_io_stats.curl_other_error); 750 total_oom += http_io_stats.out_of_memory_errors; 751 } 752 if (block_cache_store != NULL) { 753 double read_hit_ratio = 0.0; 754 double write_hit_ratio = 0.0; 755 u_int total_reads; 756 u_int total_writes; 757 758 total_reads = block_cache_stats.read_hits + block_cache_stats.read_misses; 759 if (total_reads != 0) 760 read_hit_ratio = (double)block_cache_stats.read_hits / (double)total_reads; 761 total_writes = block_cache_stats.write_hits + block_cache_stats.write_misses; 762 if (total_writes != 0) 763 write_hit_ratio = (double)block_cache_stats.write_hits / (double)total_writes; 764 (*printer)(prarg, "%-28s %u blocks\n", "block_cache_current_size", block_cache_stats.current_size); 765 (*printer)(prarg, "%-28s %u blocks\n", "block_cache_initial_size", block_cache_stats.initial_size); 766 (*printer)(prarg, "%-28s %.4f\n", "block_cache_dirty_ratio", block_cache_stats.dirty_ratio); 767 (*printer)(prarg, "%-28s %u\n", "block_cache_read_hits", block_cache_stats.read_hits); 768 (*printer)(prarg, "%-28s %u\n", "block_cache_read_misses", block_cache_stats.read_misses); 769 (*printer)(prarg, "%-28s %.4f\n", "block_cache_read_hit_ratio", read_hit_ratio); 770 (*printer)(prarg, "%-28s %u\n", "block_cache_write_hits", block_cache_stats.write_hits); 771 (*printer)(prarg, "%-28s %u\n", "block_cache_write_misses", block_cache_stats.write_misses); 772 (*printer)(prarg, "%-28s %.4f\n", "block_cache_write_hit_ratio", write_hit_ratio); 773 (*printer)(prarg, "%-28s %u\n", "block_cache_verified", block_cache_stats.verified); 774 (*printer)(prarg, "%-28s %u\n", "block_cache_mismatch", block_cache_stats.mismatch); 775 total_oom += block_cache_stats.out_of_memory_errors; 776 } 777 if (ec_protect_store != NULL) { 778 (*printer)(prarg, "%-28s %u blocks\n", "md5_cache_current_size", ec_protect_stats.current_cache_size); 779 (*printer)(prarg, "%-28s %u\n", "md5_cache_data_hits", ec_protect_stats.cache_data_hits); 780 (*printer)(prarg, "%-28s %ju.%03u sec\n", "md5_cache_full_delays", 781 (uintmax_t)(ec_protect_stats.cache_full_delay / 1000), (u_int)(ec_protect_stats.cache_full_delay % 1000)); 782 (*printer)(prarg, "%-28s %ju.%03u sec\n", "md5_cache_write_delays", 783 (uintmax_t)(ec_protect_stats.repeated_write_delay / 1000), (u_int)(ec_protect_stats.repeated_write_delay % 1000)); 784 total_oom += ec_protect_stats.out_of_memory_errors; 785 } 786 (*printer)(prarg, "%-28s %u\n", "out_of_memory_errors", total_oom); 787 } 788 789 static void 790 s3b_config_clear_stats(void) 791 { 792 /* Clear HTTP stats */ 793 if (http_io_store != NULL) 794 http_io_clear_stats(http_io_store); 795 796 /* Clear EC protection stats */ 797 if (ec_protect_store != NULL) 798 ec_protect_clear_stats(ec_protect_store); 799 800 /* Clear block cache stats */ 801 if (block_cache_store != NULL) 802 block_cache_clear_stats(block_cache_store); 803 } 804 805 static int 806 parse_size_string(const char *s, uintmax_t *valp) 807 { 808 char suffix[3] = { '\0' }; 809 int nconv; 810 811 nconv = sscanf(s, "%ju%2s", valp, suffix); 812 if (nconv < 1) 813 return -1; 814 if (nconv >= 2) { 815 int found = 0; 816 int i; 817 818 for (i = 0; i < sizeof(size_suffixes) / sizeof(*size_suffixes); i++) { 819 const struct size_suffix *const ss = &size_suffixes[i]; 820 821 if (ss->bits >= sizeof(off_t) * 8) 822 break; 823 if (strcasecmp(suffix, ss->suffix) == 0) { 824 *valp <<= ss->bits; 825 found = 1; 826 break; 827 } 828 } 829 if (!found) 830 return -1; 831 } 832 return 0; 833 } 834 835 static void 836 unparse_size_string(char *buf, size_t bmax, uintmax_t value) 837 { 838 uintmax_t unit; 839 int i; 840 841 if (value == 0) { 842 snprintf(buf, bmax, "0"); 843 return; 844 } 845 for (i = sizeof(size_suffixes) / sizeof(*size_suffixes); i-- > 0; ) { 846 const struct size_suffix *const ss = &size_suffixes[i]; 847 848 if (ss->bits >= sizeof(off_t) * 8) 849 continue; 850 unit = (uintmax_t)1 << ss->bits; 851 if (value % unit == 0) { 852 snprintf(buf, bmax, "%ju%s", value / unit, ss->suffix); 853 return; 854 } 855 } 856 snprintf(buf, bmax, "%ju", value); 857 } 858 859 /** 860 * Handle command-line flag. 861 */ 862 static int 863 handle_unknown_option(void *data, const char *arg, int key, struct fuse_args *outargs) 864 { 865 /* Check options */ 866 if (key == FUSE_OPT_KEY_OPT) { 867 868 /* Debug flags */ 869 if (strcmp(arg, "-d") == 0) 870 config.debug = 1; 871 if (strcmp(arg, "-d") == 0 || strcmp(arg, "-f") == 0) 872 config.log = stderr_logger; 873 874 /* Version */ 875 if (strcmp(arg, "--version") == 0 || strcmp(arg, "-v") == 0) { 876 fprintf(stderr, "%s version %s (%s)\n", PACKAGE, VERSION, s3backer_version); 877 fprintf(stderr, "Copyright (C) 2008-2011 Archie L. Cobbs.\n"); 878 fprintf(stderr, "This is free software; see the source for copying conditions. There is NO\n"); 879 fprintf(stderr, "warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.\n"); 880 exit(0); 881 } 882 883 /* Help */ 884 if (strcmp(arg, "--help") == 0 || strcmp(arg, "-h") == 0 || strcmp(arg, "-?") == 0) { 885 usage(); 886 exit(0); 887 } 888 889 /* Unknown; pass it through to fuse_main() */ 890 return 1; 891 } 892 893 /* Get bucket parameter */ 894 if (config.http_io.bucket == NULL) { 895 if ((config.http_io.bucket = strdup(arg)) == NULL) 896 err(1, "strdup"); 897 return 0; 898 } 899 900 /* Copy mount point */ 901 if (config.mount == NULL) { 902 if ((config.mount = strdup(arg)) == NULL) 903 err(1, "strdup"); 904 return 1; 905 } 906 907 /* Pass subsequent paramters on to fuse_main() */ 908 return 1; 909 } 910 911 static int 912 search_access_for(const char *file, const char *accessId, char **idptr, char **pwptr) 913 { 914 char buf[1024]; 915 FILE *fp; 916 917 if (idptr != NULL) 918 *idptr = NULL; 919 if (pwptr != NULL) 920 *pwptr = NULL; 921 if ((fp = fopen(file, "r")) == NULL) 922 return 0; 923 while (fgets(buf, sizeof(buf), fp) != NULL) { 924 char *colon; 925 926 if (*buf == '#' || *buf == '\0' || isspace(*buf) || (colon = strchr(buf, ':')) == NULL) 927 continue; 928 while (*buf != '\0' && isspace(buf[strlen(buf) - 1])) 929 buf[strlen(buf) - 1] = '\0'; 930 *colon = '\0'; 931 if (accessId != NULL && strcmp(buf, accessId) != 0) 932 continue; 933 if (idptr != NULL && (*idptr = strdup(buf)) == NULL) 934 err(1, "strdup"); 935 if (pwptr != NULL && (*pwptr = strdup(colon + 1)) == NULL) 936 err(1, "strdup"); 937 fclose(fp); 938 return 1; 939 } 940 fclose(fp); 941 return 0; 942 } 943 944 static int 945 validate_config(void) 946 { 947 struct s3backer_store *s3b; 948 const int customBaseURL = config.http_io.baseURL != NULL; 949 const int customRegion = config.http_io.region != NULL; 950 off_t auto_file_size; 951 u_int auto_block_size; 952 uintmax_t value; 953 const char *s; 954 char blockSizeBuf[64]; 955 char fileSizeBuf[64]; 956 struct stat sb; 957 char urlbuf[512]; 958 int i; 959 int r; 960 961 /* Default to $HOME/.s3backer for accessFile */ 962 if (config.http_io.ec2iam_role == NULL && config.accessFile == NULL) { 963 const char *home = getenv("HOME"); 964 char buf[PATH_MAX]; 965 966 if (home != NULL) { 967 snprintf(buf, sizeof(buf), "%s/%s", home, S3BACKER_DEFAULT_PWD_FILE); 968 if ((config.accessFile = strdup(buf)) == NULL) 969 err(1, "strdup"); 970 } 971 } 972 973 /* Auto-set file mode in read_only if not explicitly set */ 974 if (config.fuse_ops.file_mode == -1) { 975 config.fuse_ops.file_mode = config.fuse_ops.read_only ? 976 S3BACKER_DEFAULT_FILE_MODE_READ_ONLY : S3BACKER_DEFAULT_FILE_MODE; 977 } 978 979 /* If no accessId specified, default to first in accessFile */ 980 if (config.http_io.accessId == NULL && config.accessFile != NULL) 981 search_access_for(config.accessFile, NULL, &config.http_io.accessId, NULL); 982 if (config.http_io.accessId != NULL && *config.http_io.accessId == '\0') 983 config.http_io.accessId = NULL; 984 985 /* If no accessId, only read operations will succeed */ 986 if (!config.test && config.http_io.accessId == NULL 987 && !config.fuse_ops.read_only && !customBaseURL && config.http_io.ec2iam_role == NULL) { 988 warnx("warning: no `accessId' specified; only read operations will succeed"); 989 warnx("you can eliminate this warning by providing the `--readOnly' flag"); 990 } 991 992 /* Find key in file if not specified explicitly */ 993 if (config.http_io.accessId == NULL && config.http_io.accessKey != NULL) { 994 warnx("an `accessKey' was specified but no `accessId' was specified"); 995 return -1; 996 } 997 if (config.http_io.accessId != NULL) { 998 if (config.http_io.accessKey == NULL && config.accessFile != NULL) 999 search_access_for(config.accessFile, config.http_io.accessId, NULL, &config.http_io.accessKey); 1000 if (config.http_io.accessKey == NULL) { 1001 warnx("no `accessKey' specified"); 1002 return -1; 1003 } 1004 } 1005 1006 /* Check for conflict between explicit accessId and EC2 IAM role */ 1007 if (config.http_io.accessId != NULL && config.http_io.ec2iam_role != NULL) { 1008 warnx("an `accessKey' must not be specified when an `accessEC2IAM' role is specified"); 1009 return -1; 1010 } 1011 1012 /* Check auth version */ 1013 for (i = 0; i < sizeof(s3_auth_types) / sizeof(*s3_auth_types); i++) { 1014 if (strcmp(config.http_io.authVersion, s3_auth_types[i]) == 0) 1015 break; 1016 } 1017 if (i == sizeof(s3_auth_types) / sizeof(*s3_auth_types)) { 1018 warnx("illegal authentication version `%s'", config.http_io.authVersion); 1019 return -1; 1020 } 1021 1022 /* Check bucket/testdir */ 1023 if (!config.test) { 1024 if (config.http_io.bucket == NULL) { 1025 warnx("no S3 bucket specified"); 1026 return -1; 1027 } 1028 if (*config.http_io.bucket == '\0' || *config.http_io.bucket == '/' || strchr(config.http_io.bucket, '/') != 0) { 1029 warnx("invalid S3 bucket `%s'", config.http_io.bucket); 1030 return -1; 1031 } 1032 } else { 1033 if (config.http_io.bucket == NULL) { 1034 warnx("no test directory specified"); 1035 return -1; 1036 } 1037 if (stat(config.http_io.bucket, &sb) == -1) { 1038 warn("%s", config.http_io.bucket); 1039 return -1; 1040 } 1041 if (!S_ISDIR(sb.st_mode)) { 1042 errno = ENOTDIR; 1043 warn("%s", config.http_io.bucket); 1044 return -1; 1045 } 1046 } 1047 1048 /* Check storage class */ 1049 if (config.http_io.storage_class != NULL 1050 && strcmp(config.http_io.storage_class, STORAGE_CLASS_STANDARD) != 0 1051 && strcmp(config.http_io.storage_class, STORAGE_CLASS_STANDARD_IA) != 0 1052 && strcmp(config.http_io.storage_class, STORAGE_CLASS_REDUCED_REDUNDANCY) != 0) { 1053 warnx("invalid storage class `%s'", config.http_io.storage_class); 1054 return -1; 1055 } 1056 1057 /* Check server side encryption type */ 1058 if (config.http_io.sse != NULL && strcmp(config.http_io.sse, REQUIRED_SSE_VALUE) != 0) { 1059 warnx("invalid sse type `%s' (only `%s' is supported)", config.http_io.sse, REQUIRED_SSE_VALUE); 1060 return -1; 1061 } 1062 1063 /* Set default or custom region */ 1064 if (config.http_io.region == NULL) 1065 config.http_io.region = S3BACKER_DEFAULT_REGION; 1066 if (customRegion) 1067 config.http_io.vhost = 1; 1068 1069 /* Set default base URL */ 1070 if (config.http_io.baseURL == NULL) { 1071 if (customRegion && strcmp(config.http_io.region, S3BACKER_DEFAULT_REGION) != 0) 1072 snprintf(urlbuf, sizeof(urlbuf), "http%s://s3-%s.%s/", config.ssl ? "s" : "", config.http_io.region, S3_DOMAIN); 1073 else 1074 snprintf(urlbuf, sizeof(urlbuf), "http%s://s3.%s/", config.ssl ? "s" : "", S3_DOMAIN); 1075 if ((config.http_io.baseURL = strdup(urlbuf)) == NULL) { 1076 warn("malloc"); 1077 return -1; 1078 } 1079 } 1080 1081 /* Check base URL */ 1082 s = NULL; 1083 if (strncmp(config.http_io.baseURL, "http://", 7) == 0) 1084 s = config.http_io.baseURL + 7; 1085 else if (strncmp(config.http_io.baseURL, "https://", 8) == 0) 1086 s = config.http_io.baseURL + 8; 1087 if (s != NULL && (*s == '/' || *s == '\0')) 1088 s = NULL; 1089 if (s != NULL && (s = strrchr(s, '/')) == NULL) { 1090 warnx("base URL must end with a '/'"); 1091 s = NULL; 1092 } 1093 if (s != NULL && s[1] != '\0') { 1094 warnx("base URL must end with a '/' not '%c'", s[1]); 1095 s = NULL; 1096 } 1097 if (s == NULL) { 1098 warnx("invalid base URL `%s'", config.http_io.baseURL); 1099 return -1; 1100 } 1101 if (config.ssl && customBaseURL && strncmp(config.http_io.baseURL, "https", 5) != 0) { 1102 warnx("non-SSL `--baseURL' conflicts with `--ssl'"); 1103 return -1; 1104 } 1105 1106 /* Handle virtual host style URL (prefix hostname with bucket name) */ 1107 if (config.http_io.vhost) { 1108 size_t buflen; 1109 int schemelen; 1110 char *buf; 1111 1112 schemelen = strchr(config.http_io.baseURL, ':') - config.http_io.baseURL + 3; 1113 buflen = strlen(config.http_io.bucket) + 1 + strlen(config.http_io.baseURL) + 1; 1114 if ((buf = malloc(buflen)) == NULL) 1115 err(1, "malloc(%u)", (u_int)buflen); 1116 snprintf(buf, buflen, "%.*s%s.%s", schemelen, config.http_io.baseURL, 1117 config.http_io.bucket, config.http_io.baseURL + schemelen); 1118 config.http_io.baseURL = buf; 1119 } 1120 1121 /* Check S3 access privilege */ 1122 for (i = 0; i < sizeof(s3_acls) / sizeof(*s3_acls); i++) { 1123 if (strcmp(config.http_io.accessType, s3_acls[i]) == 0) 1124 break; 1125 } 1126 if (i == sizeof(s3_acls) / sizeof(*s3_acls)) { 1127 warnx("illegal access type `%s'", config.http_io.accessType); 1128 return -1; 1129 } 1130 1131 /* Check filenames */ 1132 if (strchr(config.fuse_ops.filename, '/') != NULL || *config.fuse_ops.filename == '\0') { 1133 warnx("illegal filename `%s'", config.fuse_ops.filename); 1134 return -1; 1135 } 1136 if (strchr(config.fuse_ops.stats_filename, '/') != NULL) { 1137 warnx("illegal stats filename `%s'", config.fuse_ops.stats_filename); 1138 return -1; 1139 } 1140 1141 /* Apply default encryption */ 1142 if (config.http_io.encryption == NULL && config.encrypt) 1143 config.http_io.encryption = strdup(S3BACKER_DEFAULT_ENCRYPTION); 1144 1145 /* Uppercase encryption name for consistency */ 1146 if (config.http_io.encryption != NULL) { 1147 char *t; 1148 1149 if ((t = strdup(config.http_io.encryption)) == NULL) 1150 err(1, "strdup()"); 1151 for (i = 0; t[i] != '\0'; i++) 1152 t[i] = toupper(t[i]); 1153 config.http_io.encryption = t; 1154 } 1155 1156 /* Check encryption and get key */ 1157 if (config.http_io.encryption != NULL) { 1158 char pwbuf[1024]; 1159 FILE *fp; 1160 1161 if (config.password_file != NULL && config.http_io.password != NULL) { 1162 warnx("specify only one of `--password' or `--passwordFile'"); 1163 return -1; 1164 } 1165 if (config.password_file == NULL && config.http_io.password == NULL) { 1166 if ((s = getpass("Password: ")) == NULL) 1167 err(1, "getpass()"); 1168 } 1169 if (config.password_file != NULL) { 1170 assert(config.http_io.password == NULL); 1171 if ((fp = fopen(config.password_file, "r")) == NULL) { 1172 warn("can't open encryption key file `%s'", config.password_file); 1173 return -1; 1174 } 1175 if (fgets(pwbuf, sizeof(pwbuf), fp) == NULL || *pwbuf == '\0') { 1176 warnx("can't read encryption key from file `%s'", config.password_file); 1177 fclose(fp); 1178 return -1; 1179 } 1180 if (pwbuf[strlen(pwbuf) - 1] == '\n') 1181 pwbuf[strlen(pwbuf) - 1] = '\0'; 1182 fclose(fp); 1183 s = pwbuf; 1184 } 1185 if (config.http_io.password == NULL && (config.http_io.password = strdup(s)) == NULL) 1186 err(1, "strdup()"); 1187 if (config.http_io.key_length > EVP_MAX_KEY_LENGTH) { 1188 warnx("`--keyLength' value must be positive and at most %u", EVP_MAX_KEY_LENGTH); 1189 return -1; 1190 } 1191 } else { 1192 if (config.http_io.password != NULL) 1193 warnx("unexpected flag `%s' (`--encrypt' was not specified)", "--password"); 1194 else if (config.password_file != NULL) 1195 warnx("unexpected flag `%s' (`--encrypt' was not specified)", "--passwordFile"); 1196 if (config.http_io.key_length != 0) 1197 warnx("unexpected flag `%s' (`--encrypt' was not specified)", "--keyLength"); 1198 } 1199 1200 /* We always want to compress if we are encrypting */ 1201 if (config.http_io.encryption != NULL && config.http_io.compress == Z_NO_COMPRESSION) 1202 config.http_io.compress = Z_DEFAULT_COMPRESSION; 1203 1204 /* Check compression level */ 1205 switch (config.http_io.compress) { 1206 case Z_DEFAULT_COMPRESSION: 1207 case Z_NO_COMPRESSION: 1208 break; 1209 default: 1210 if (config.http_io.compress < Z_BEST_SPEED || config.http_io.compress > Z_BEST_COMPRESSION) { 1211 warnx("illegal compression level `%d'", config.http_io.compress); 1212 return -1; 1213 } 1214 break; 1215 } 1216 1217 /* Disable md5 cache when in read only mode */ 1218 if (config.fuse_ops.read_only) { 1219 config.ec_protect.cache_size = 0; 1220 config.ec_protect.cache_time = 0; 1221 config.ec_protect.min_write_delay = 0; 1222 } 1223 1224 /* Check time/cache values */ 1225 if (config.ec_protect.cache_size == 0 && config.ec_protect.cache_time > 0) { 1226 warnx("`md5CacheTime' must zero when MD5 cache is disabled"); 1227 return -1; 1228 } 1229 if (config.ec_protect.cache_size == 0 && config.ec_protect.min_write_delay > 0) { 1230 warnx("`minWriteDelay' must zero when MD5 cache is disabled"); 1231 return -1; 1232 } 1233 if (config.ec_protect.cache_time > 0 1234 && config.ec_protect.cache_time < config.ec_protect.min_write_delay) { 1235 warnx("`md5CacheTime' must be at least `minWriteDelay'"); 1236 return -1; 1237 } 1238 if (config.http_io.initial_retry_pause > config.http_io.max_retry_pause) { 1239 warnx("`maxRetryPause' must be at least `initialRetryPause'"); 1240 return -1; 1241 } 1242 1243 /* Parse block and file sizes */ 1244 if (config.block_size_str != NULL) { 1245 if (parse_size_string(config.block_size_str, &value) == -1 || value == 0) { 1246 warnx("invalid block size `%s'", config.block_size_str); 1247 return -1; 1248 } 1249 if ((u_int)value != value) { 1250 warnx("block size `%s' is too big", config.block_size_str); 1251 return -1; 1252 } 1253 config.block_size = value; 1254 } 1255 if (config.file_size_str != NULL) { 1256 if (parse_size_string(config.file_size_str, &value) == -1 || value == 0) { 1257 warnx("invalid file size `%s'", config.file_size_str); 1258 return -1; 1259 } 1260 config.file_size = value; 1261 } 1262 1263 /* Parse upload/download speeds */ 1264 for (i = 0; i < 2; i++) { 1265 if (config.max_speed_str[i] != NULL) { 1266 if (parse_size_string(config.max_speed_str[i], &value) == -1 || value == 0) { 1267 warnx("invalid max %s speed `%s'", upload_download_names[i], config.max_speed_str[i]); 1268 return -1; 1269 } 1270 if ((curl_off_t)(value / 8) != (value / 8)) { 1271 warnx("max %s speed `%s' is too big", upload_download_names[i], config.max_speed_str[i]); 1272 return -1; 1273 } 1274 config.http_io.max_speed[i] = value; 1275 } 1276 if (config.http_io.max_speed[i] != 0 && config.block_size / (config.http_io.max_speed[i] / 8) >= config.http_io.timeout) { 1277 warnx("configured timeout of %us is too short for block size of %u bytes and max %s speed %s bps", 1278 config.http_io.timeout, config.block_size, upload_download_names[i], config.max_speed_str[i]); 1279 return -1; 1280 } 1281 } 1282 1283 /* Check block cache config */ 1284 if (config.block_cache.cache_size > 0 && config.block_cache.num_threads <= 0) { 1285 warnx("invalid block cache thread pool size %u", config.block_cache.num_threads); 1286 return -1; 1287 } 1288 if (config.block_cache.write_delay > 0 && config.block_cache.synchronous) { 1289 warnx("`--blockCacheSync' requires setting `--blockCacheWriteDelay=0'"); 1290 return -1; 1291 } 1292 if (config.block_cache.cache_size > 0 && config.block_cache.cache_file != NULL) { 1293 int bs_bits = ffs(config.block_size) - 1; 1294 int cs_bits = ffs(config.block_cache.cache_size); 1295 1296 if (bs_bits + cs_bits >= sizeof(off_t) * 8 - 1) { 1297 warnx("the block cache is too big to fit within a single file (%u blocks x %u bytes)", 1298 config.block_cache.cache_size, config.block_size); 1299 return -1; 1300 } 1301 } 1302 if (config.block_cache.cache_file == NULL && config.block_cache.recover_dirty_blocks) { 1303 warnx("`--blockCacheRecoverDirtyBlocks' requires specifying `--blockCacheFile'"); 1304 return -1; 1305 } 1306 1307 /* Check mount point */ 1308 if (config.erase || config.reset) { 1309 if (config.mount != NULL) { 1310 warnx("no mount point should be specified with `--erase' or `--reset-mounted-flag'"); 1311 return -1; 1312 } 1313 } else { 1314 if (config.mount == NULL) { 1315 warnx("no mount point specified"); 1316 return -1; 1317 } 1318 } 1319 1320 /* Format descriptive string of what we're mounting */ 1321 if (config.test) { 1322 snprintf(config.description, sizeof(config.description), "%s%s/%s", 1323 "file://", config.http_io.bucket, config.http_io.prefix); 1324 } else if (config.http_io.vhost) 1325 snprintf(config.description, sizeof(config.description), "%s%s", config.http_io.baseURL, config.http_io.prefix); 1326 else { 1327 snprintf(config.description, sizeof(config.description), "%s%s/%s", 1328 config.http_io.baseURL, config.http_io.bucket, config.http_io.prefix); 1329 } 1330 1331 /* 1332 * Read the first block (if any) to determine existing file and block size, 1333 * and compare with configured sizes (if given). 1334 */ 1335 if (config.test) 1336 config.no_auto_detect = 1; 1337 if (config.no_auto_detect) 1338 r = ENOENT; 1339 else { 1340 config.http_io.debug = config.debug; 1341 config.http_io.quiet = config.quiet; 1342 config.http_io.log = config.log; 1343 if ((s3b = http_io_create(&config.http_io)) == NULL) 1344 err(1, "http_io_create"); 1345 if (!config.quiet) 1346 warnx("auto-detecting block size and total file size..."); 1347 r = (*s3b->meta_data)(s3b, &auto_file_size, &auto_block_size); 1348 (*s3b->destroy)(s3b); 1349 } 1350 1351 /* Check result */ 1352 switch (r) { 1353 case 0: 1354 unparse_size_string(blockSizeBuf, sizeof(blockSizeBuf), (uintmax_t)auto_block_size); 1355 unparse_size_string(fileSizeBuf, sizeof(fileSizeBuf), (uintmax_t)auto_file_size); 1356 if (!config.quiet) 1357 warnx("auto-detected block size=%s and total size=%s", blockSizeBuf, fileSizeBuf); 1358 if (config.block_size == 0) 1359 config.block_size = auto_block_size; 1360 else if (auto_block_size != config.block_size) { 1361 char buf[64]; 1362 1363 unparse_size_string(buf, sizeof(buf), (uintmax_t)config.block_size); 1364 if (config.force) { 1365 if (!config.quiet) { 1366 warnx("warning: configured block size %s != filesystem block size %s,\n" 1367 "but you said `--force' so I'll proceed anyway even though your data will\n" 1368 "probably not read back correctly.", buf, blockSizeBuf); 1369 } 1370 } else 1371 errx(1, "error: configured block size %s != filesystem block size %s", buf, blockSizeBuf); 1372 } 1373 if (config.file_size == 0) 1374 config.file_size = auto_file_size; 1375 else if (auto_file_size != config.file_size) { 1376 char buf[64]; 1377 1378 unparse_size_string(buf, sizeof(buf), (uintmax_t)config.file_size); 1379 if (config.force) { 1380 if (!config.quiet) { 1381 warnx("warning: configured file size %s != filesystem file size %s,\n" 1382 "but you said `--force' so I'll proceed anyway even though your data will\n" 1383 "probably not read back correctly.", buf, fileSizeBuf); 1384 } 1385 } else 1386 errx(1, "error: configured file size %s != filesystem file size %s", buf, fileSizeBuf); 1387 } 1388 break; 1389 case ENOENT: 1390 { 1391 const char *why = config.no_auto_detect ? "disabled" : "failed"; 1392 int config_block_size = config.block_size; 1393 1394 if (config.file_size == 0) 1395 errx(1, "error: auto-detection of filesystem size %s; please specify `--size'", why); 1396 if (config.block_size == 0) 1397 config.block_size = S3BACKER_DEFAULT_BLOCKSIZE; 1398 unparse_size_string(blockSizeBuf, sizeof(blockSizeBuf), (uintmax_t)config.block_size); 1399 unparse_size_string(fileSizeBuf, sizeof(fileSizeBuf), (uintmax_t)config.file_size); 1400 if (!config.quiet) { 1401 warnx("auto-detection %s; using %s block size %s and file size %s", why, 1402 config_block_size == 0 ? "default" : "configured", blockSizeBuf, fileSizeBuf); 1403 } 1404 break; 1405 } 1406 default: 1407 errno = r; 1408 err(1, "can't read data store meta-data"); 1409 break; 1410 } 1411 1412 /* Check computed block and file sizes */ 1413 if (config.block_size != (1 << (ffs(config.block_size) - 1))) { 1414 warnx("block size must be a power of 2"); 1415 return -1; 1416 } 1417 if (config.file_size % config.block_size != 0) { 1418 warnx("file size must be a multiple of block size"); 1419 return -1; 1420 } 1421 config.num_blocks = config.file_size / config.block_size; 1422 if (sizeof(s3b_block_t) < sizeof(config.num_blocks) 1423 && config.num_blocks > ((off_t)1 << (sizeof(s3b_block_t) * 8))) { 1424 warnx("more than 2^%d blocks: decrease file size or increase block size", (int)(sizeof(s3b_block_t) * 8)); 1425 return -1; 1426 } 1427 1428 /* Check block size vs. encryption block size */ 1429 if (config.http_io.encryption != NULL && config.block_size % EVP_MAX_IV_LENGTH != 0) { 1430 warnx("block size must be at least %u when encryption is enabled", EVP_MAX_IV_LENGTH); 1431 return -1; 1432 } 1433 1434 /* Check that MD5 cache won't eventually deadlock */ 1435 if (config.ec_protect.cache_size > 0 1436 && config.ec_protect.cache_time == 0 1437 && config.ec_protect.cache_size < config.num_blocks) { 1438 warnx("`md5CacheTime' is infinite but `md5CacheSize' is less than the number of blocks, so eventual deadlock will result"); 1439 return -1; 1440 } 1441 1442 /* No point in the caches being bigger than necessary */ 1443 if (config.ec_protect.cache_size > config.num_blocks) { 1444 warnx("MD5 cache size (%ju) is greater that the total number of blocks (%ju); automatically reducing", 1445 (uintmax_t)config.ec_protect.cache_size, (uintmax_t)config.num_blocks); 1446 config.ec_protect.cache_size = config.num_blocks; 1447 } 1448 if (config.block_cache.cache_size > config.num_blocks) { 1449 warnx("block cache size (%ju) is greater that the total number of blocks (%ju); automatically reducing", 1450 (uintmax_t)config.block_cache.cache_size, (uintmax_t)config.num_blocks); 1451 config.block_cache.cache_size = config.num_blocks; 1452 } 1453 1454 #ifdef __APPLE__ 1455 /* On MacOS, warn if kernel timeouts can happen prior to our own timeout */ 1456 { 1457 u_int total_time = 0; 1458 u_int retry_pause = 0; 1459 u_int total_pause; 1460 1461 /* 1462 * Determine how much total time an operation can take including retries. 1463 * We have to use the same exponential backoff algorithm. 1464 */ 1465 for (total_pause = 0; 1; total_pause += retry_pause) { 1466 total_time += config.http_io.timeout * 1000; 1467 if (total_pause >= config.http_io.max_retry_pause) 1468 break; 1469 retry_pause = retry_pause > 0 ? retry_pause * 2 : config.http_io.initial_retry_pause; 1470 if (total_pause + retry_pause > config.http_io.max_retry_pause) 1471 retry_pause = config.http_io.max_retry_pause - total_pause; 1472 total_time += retry_pause; 1473 } 1474 1475 /* Convert from milliseconds to seconds */ 1476 total_time = (total_time + 999) / 1000; 1477 1478 /* Warn if exceeding MacFUSE limit */ 1479 if (total_time >= FUSE_MAX_DAEMON_TIMEOUT && !config.quiet) { 1480 warnx("warning: maximum possible I/O delay (%us) >= MacFUSE limit (%us);", total_time, FUSE_MAX_DAEMON_TIMEOUT); 1481 warnx("consider lower settings for `--maxRetryPause' and/or `--timeout'."); 1482 } 1483 } 1484 #endif /* __APPLE__ */ 1485 1486 /* Copy common stuff into sub-module configs */ 1487 config.block_cache.block_size = config.block_size; 1488 config.block_cache.log = config.log; 1489 config.http_io.debug = config.debug; 1490 config.http_io.quiet = config.quiet; 1491 config.http_io.block_size = config.block_size; 1492 config.http_io.num_blocks = config.num_blocks; 1493 config.http_io.log = config.log; 1494 config.ec_protect.block_size = config.block_size; 1495 config.ec_protect.log = config.log; 1496 config.fuse_ops.block_size = config.block_size; 1497 config.fuse_ops.num_blocks = config.num_blocks; 1498 config.fuse_ops.log = config.log; 1499 1500 /* Check whether already mounted, and if so, compare mount token against on-disk cache (if any) */ 1501 if (!config.test && !config.erase && !config.reset) { 1502 int32_t mount_token; 1503 int conflict; 1504 1505 /* Read s3 mount token */ 1506 config.http_io.debug = config.debug; 1507 config.http_io.quiet = config.quiet; 1508 config.http_io.log = config.log; 1509 if ((s3b = http_io_create(&config.http_io)) == NULL) 1510 err(1, "http_io_create"); 1511 r = (*s3b->set_mount_token)(s3b, &mount_token, -1); 1512 (*s3b->destroy)(s3b); 1513 if (r != 0) { 1514 errno = r; 1515 err(1, "error reading mount token"); 1516 } 1517 conflict = mount_token != 0; 1518 1519 /* 1520 * The disk cache also has a mount token, so we need to do some extra checking. 1521 * Either token can be 0 (i.e., not present -> not mounted) or != 0 (mounted). 1522 * 1523 * If neither token is present, proceed with mount. Note: there should not be 1524 * any dirty blocks in the disk cache in this case, because this represents a 1525 * clean unmount situation. 1526 * 1527 * If the cache has a token, but S3 has none, that means someone must have used 1528 * `--reset-mounted-flag' to clear it from S3 since the last time the disk cache was 1529 * used. In that case, `--force' is required to continue using the disk cache, 1530 * or `--reset-mounted-flag' must be used to clear the disk cache flag as well. 1531 * 1532 * If --blockCacheRecoverDirtyBlocks is specified and the tokens match, we 1533 * have the corresponding cache file for the last mount. Proceed with mount and, 1534 * if configured, enable cache writeback of dirty blocks. 1535 */ 1536 if (config.block_cache.cache_file != NULL) { 1537 int32_t cache_mount_token = -1; 1538 struct stat cache_file_stat; 1539 struct s3b_dcache *dcache; 1540 1541 /* Open disk cache file, if any, and read the mount token therein, if any */ 1542 if (stat(config.block_cache.cache_file, &cache_file_stat) == -1) { 1543 if (errno != ENOENT) 1544 err(1, "can't open cache file `%s'", config.block_cache.cache_file); 1545 } else { 1546 if ((r = s3b_dcache_open(&dcache, config.log, config.block_cache.cache_file, 1547 config.block_cache.block_size, config.block_cache.cache_size, NULL, NULL, 0)) != 0) 1548 errx(1, "error opening cache file `%s': %s", config.block_cache.cache_file, strerror(r)); 1549 if (s3b_dcache_has_mount_token(dcache) && (r = s3b_dcache_set_mount_token(dcache, &cache_mount_token, -1)) != 0) 1550 errx(1, "error reading mount token from `%s': %s", config.block_cache.cache_file, strerror(r)); 1551 s3b_dcache_close(dcache); 1552 } 1553 1554 /* If cache file is older format, then cache_mount_token will be -1, otherwise >= 0 */ 1555 if (cache_mount_token > 0) { 1556 1557 /* If tokens do not agree, bail out, otherwise enable write-back of dirty blocks if tokens are non-zero */ 1558 if (cache_mount_token != mount_token) { 1559 warnx("cache file `%s' mount token mismatch (disk:0x%08x != s3:0x%08x)", 1560 config.block_cache.cache_file, cache_mount_token, mount_token); 1561 } else if (config.block_cache.recover_dirty_blocks) { 1562 if (!config.quiet) 1563 warnx("recovering from unclean shutdown: dirty blocks in cache file will be written back to S3"); 1564 config.block_cache.perform_flush = 1; 1565 conflict = 0; 1566 } 1567 } 1568 } 1569 1570 /* If there is a conflicting mount, additional `--force' is required */ 1571 if (conflict) { 1572 if (!config.force) { 1573 warnx("%s appears to be already mounted (using mount token 0x%08x)", config.description, (int)mount_token); 1574 errx(1, "reset mount token with `--reset-mounted-flag', or use `--force' to override"); 1575 } 1576 if (!config.quiet) { 1577 warnx("warning: filesystem appears already mounted but you said `--force'\n" 1578 " so I'll proceed anyway even though your data may get corrupted.\n"); 1579 } 1580 } 1581 } 1582 1583 /* If `--listBlocks' was given, build non-empty block bitmap */ 1584 if (config.erase || config.reset) 1585 config.list_blocks = 0; 1586 if (config.list_blocks) { 1587 struct s3backer_store *temp_store; 1588 struct list_blocks lb; 1589 size_t nwords; 1590 1591 /* Logging */ 1592 if (!config.quiet) { 1593 fprintf(stderr, "s3backer: listing non-zero blocks..."); 1594 fflush(stderr); 1595 } 1596 1597 /* Create temporary lower layer */ 1598 if ((temp_store = config.test ? test_io_create(&config.http_io) : http_io_create(&config.http_io)) == NULL) 1599 err(1, config.test ? "test_io_create" : "http_io_create"); 1600 1601 /* Initialize bitmap */ 1602 nwords = (config.num_blocks + (sizeof(*lb.bitmap) * 8) - 1) / (sizeof(*lb.bitmap) * 8); 1603 if ((lb.bitmap = calloc(nwords, sizeof(*lb.bitmap))) == NULL) 1604 err(1, "calloc"); 1605 lb.print_dots = !config.quiet; 1606 lb.count = 0; 1607 1608 /* Generate non-zero block bitmap */ 1609 assert(config.http_io.nonzero_bitmap == NULL); 1610 if ((r = (*temp_store->list_blocks)(temp_store, list_blocks_callback, &lb)) != 0) 1611 errx(1, "can't list blocks: %s", strerror(r)); 1612 1613 /* Close temporary store */ 1614 (*temp_store->destroy)(temp_store); 1615 1616 /* Save generated bitmap */ 1617 config.http_io.nonzero_bitmap = lb.bitmap; 1618 1619 /* Logging */ 1620 if (!config.quiet) { 1621 fprintf(stderr, "done\n"); 1622 warnx("found %ju non-zero blocks", lb.count); 1623 } 1624 } 1625 1626 /* Done */ 1627 return 0; 1628 } 1629 1630 static void 1631 list_blocks_callback(void *arg, s3b_block_t block_num) 1632 { 1633 struct list_blocks *const lb = arg; 1634 const int bits_per_word = sizeof(*lb->bitmap) * 8; 1635 1636 lb->bitmap[block_num / bits_per_word] |= 1 << (block_num % bits_per_word); 1637 lb->count++; 1638 if (lb->print_dots && (lb->count % BLOCKS_PER_DOT) == 0) { 1639 fprintf(stderr, "."); 1640 fflush(stderr); 1641 } 1642 } 1643 1644 static void 1645 dump_config(void) 1646 { 1647 int i; 1648 1649 (*config.log)(LOG_DEBUG, "s3backer config:"); 1650 (*config.log)(LOG_DEBUG, "%24s: %s", "test mode", config.test ? "true" : "false"); 1651 (*config.log)(LOG_DEBUG, "%24s: %s", "directIO", config.fuse_ops.direct_io ? "true" : "false"); 1652 (*config.log)(LOG_DEBUG, "%24s: \"%s\"", "accessId", config.http_io.accessId != NULL ? config.http_io.accessId : ""); 1653 (*config.log)(LOG_DEBUG, "%24s: \"%s\"", "accessKey", config.http_io.accessKey != NULL ? "****" : ""); 1654 (*config.log)(LOG_DEBUG, "%24s: \"%s\"", "accessFile", config.accessFile); 1655 (*config.log)(LOG_DEBUG, "%24s: %s", "accessType", config.http_io.accessType); 1656 (*config.log)(LOG_DEBUG, "%24s: \"%s\"", "ec2iam_role", config.http_io.ec2iam_role != NULL ? config.http_io.ec2iam_role : ""); 1657 (*config.log)(LOG_DEBUG, "%24s: %s", "authVersion", config.http_io.authVersion); 1658 (*config.log)(LOG_DEBUG, "%24s: \"%s\"", "baseURL", config.http_io.baseURL); 1659 (*config.log)(LOG_DEBUG, "%24s: \"%s\"", "region", config.http_io.region); 1660 (*config.log)(LOG_DEBUG, "%24s: \"%s\"", config.test ? "testdir" : "bucket", config.http_io.bucket); 1661 (*config.log)(LOG_DEBUG, "%24s: \"%s\"", "prefix", config.http_io.prefix); 1662 (*config.log)(LOG_DEBUG, "%24s: %s", "blockHashPrefix", config.http_io.blockHashPrefix ? "true" : "false"); 1663 (*config.log)(LOG_DEBUG, "%24s: \"%s\"", "defaultContentEncoding", 1664 config.http_io.default_ce != NULL ? config.http_io.default_ce : "(none)"); 1665 (*config.log)(LOG_DEBUG, "%24s: %s", "list_blocks", config.list_blocks ? "true" : "false"); 1666 (*config.log)(LOG_DEBUG, "%24s: \"%s\"", "mount", config.mount); 1667 (*config.log)(LOG_DEBUG, "%24s: \"%s\"", "filename", config.fuse_ops.filename); 1668 (*config.log)(LOG_DEBUG, "%24s: \"%s\"", "stats_filename", config.fuse_ops.stats_filename); 1669 (*config.log)(LOG_DEBUG, "%24s: %s (%u)", "block_size", 1670 config.block_size_str != NULL ? config.block_size_str : "-", config.block_size); 1671 (*config.log)(LOG_DEBUG, "%24s: %s (%jd)", "file_size", 1672 config.file_size_str != NULL ? config.file_size_str : "-", (intmax_t)config.file_size); 1673 (*config.log)(LOG_DEBUG, "%24s: %jd", "num_blocks", (intmax_t)config.num_blocks); 1674 (*config.log)(LOG_DEBUG, "%24s: 0%o", "file_mode", config.fuse_ops.file_mode); 1675 (*config.log)(LOG_DEBUG, "%24s: %s", "read_only", config.fuse_ops.read_only ? "true" : "false"); 1676 (*config.log)(LOG_DEBUG, "%24s: %d", "compress", config.http_io.compress); 1677 (*config.log)(LOG_DEBUG, "%24s: %s", "encryption", config.http_io.encryption != NULL ? config.http_io.encryption : "(none)"); 1678 (*config.log)(LOG_DEBUG, "%24s: %u", "key_length", config.http_io.key_length); 1679 (*config.log)(LOG_DEBUG, "%24s: \"%s\"", "password", config.http_io.password != NULL ? "****" : ""); 1680 (*config.log)(LOG_DEBUG, "%24s: %s bps (%ju)", "max_upload", 1681 config.max_speed_str[HTTP_UPLOAD] != NULL ? config.max_speed_str[HTTP_UPLOAD] : "-", 1682 config.http_io.max_speed[HTTP_UPLOAD]); 1683 (*config.log)(LOG_DEBUG, "%24s: %s bps (%ju)", "max_download", 1684 config.max_speed_str[HTTP_DOWNLOAD] != NULL ? config.max_speed_str[HTTP_DOWNLOAD] : "-", 1685 config.http_io.max_speed[HTTP_DOWNLOAD]); 1686 (*config.log)(LOG_DEBUG, "%24s: %us", "timeout", config.http_io.timeout); 1687 (*config.log)(LOG_DEBUG, "%24s: \"%s\"", "sse", config.http_io.sse); 1688 (*config.log)(LOG_DEBUG, "%24s: %ums", "initial_retry_pause", config.http_io.initial_retry_pause); 1689 (*config.log)(LOG_DEBUG, "%24s: %ums", "max_retry_pause", config.http_io.max_retry_pause); 1690 (*config.log)(LOG_DEBUG, "%24s: %ums", "min_write_delay", config.ec_protect.min_write_delay); 1691 (*config.log)(LOG_DEBUG, "%24s: %ums", "md5_cache_time", config.ec_protect.cache_time); 1692 (*config.log)(LOG_DEBUG, "%24s: %u entries", "md5_cache_size", config.ec_protect.cache_size); 1693 (*config.log)(LOG_DEBUG, "%24s: %u entries", "block_cache_size", config.block_cache.cache_size); 1694 (*config.log)(LOG_DEBUG, "%24s: %u threads", "block_cache_threads", config.block_cache.num_threads); 1695 (*config.log)(LOG_DEBUG, "%24s: %ums", "block_cache_timeout", config.block_cache.timeout); 1696 (*config.log)(LOG_DEBUG, "%24s: %ums", "block_cache_write_delay", config.block_cache.write_delay); 1697 (*config.log)(LOG_DEBUG, "%24s: %u blocks", "block_cache_max_dirty", config.block_cache.max_dirty); 1698 (*config.log)(LOG_DEBUG, "%24s: %s", "block_cache_sync", config.block_cache.synchronous ? "true" : "false"); 1699 (*config.log)(LOG_DEBUG, "%24s: %s", "recover_dirty_blocks", config.block_cache.recover_dirty_blocks ? "true" : "false"); 1700 (*config.log)(LOG_DEBUG, "%24s: %u blocks", "read_ahead", config.block_cache.read_ahead); 1701 (*config.log)(LOG_DEBUG, "%24s: %u blocks", "read_ahead_trigger", config.block_cache.read_ahead_trigger); 1702 (*config.log)(LOG_DEBUG, "%24s: \"%s\"", "block_cache_cache_file", 1703 config.block_cache.cache_file != NULL ? config.block_cache.cache_file : ""); 1704 (*config.log)(LOG_DEBUG, "%24s: %s", "block_cache_no_verify", config.block_cache.no_verify ? "true" : "false"); 1705 (*config.log)(LOG_DEBUG, "fuse_main arguments:"); 1706 for (i = 0; i < config.fuse_args.argc; i++) 1707 (*config.log)(LOG_DEBUG, " [%d] = \"%s\"", i, config.fuse_args.argv[i]); 1708 } 1709 1710 static void 1711 syslog_logger(int level, const char *fmt, ...) 1712 { 1713 va_list args; 1714 1715 /* Filter debug messages */ 1716 if (!config.debug && level == LOG_DEBUG) 1717 return; 1718 1719 /* Send message to syslog */ 1720 va_start(args, fmt); 1721 vsyslog(level, fmt, args); 1722 va_end(args); 1723 } 1724 1725 static void 1726 stderr_logger(int level, const char *fmt, ...) 1727 { 1728 const char *levelstr; 1729 char timebuf[32]; 1730 va_list args; 1731 struct tm tm; 1732 time_t now; 1733 1734 /* Filter debug messages */ 1735 if (!config.debug && level == LOG_DEBUG) 1736 return; 1737 1738 /* Get level descriptor */ 1739 switch (level) { 1740 case LOG_ERR: 1741 levelstr = "ERROR"; 1742 break; 1743 case LOG_WARNING: 1744 levelstr = "WARNING"; 1745 break; 1746 case LOG_NOTICE: 1747 levelstr = "NOTICE"; 1748 break; 1749 case LOG_INFO: 1750 levelstr = "INFO"; 1751 break; 1752 case LOG_DEBUG: 1753 levelstr = "DEBUG"; 1754 break; 1755 default: 1756 levelstr = "<?>"; 1757 break; 1758 } 1759 1760 /* Format and print log message */ 1761 time(&now); 1762 strftime(timebuf, sizeof(timebuf), "%F %T", localtime_r(&now, &tm)); 1763 va_start(args, fmt); 1764 fprintf(stderr, "%s %s: ", timebuf, levelstr); 1765 vfprintf(stderr, fmt, args); 1766 fprintf(stderr, "\n"); 1767 va_end(args); 1768 } 1769 1770 static void 1771 usage(void) 1772 { 1773 int i; 1774 1775 fprintf(stderr, "Usage:\n"); 1776 fprintf(stderr, "\ts3backer [options] bucket /mount/point\n"); 1777 fprintf(stderr, "\ts3backer --test [options] directory /mount/point\n"); 1778 fprintf(stderr, "\ts3backer --erase [options] bucket\n"); 1779 fprintf(stderr, "\ts3backer --reset-mounted-flag [options] bucket\n"); 1780 fprintf(stderr, "Options:\n"); 1781 fprintf(stderr, "\t--%-27s %s\n", "accessFile=FILE", "File containing `accessID:accessKey' pairs"); 1782 fprintf(stderr, "\t--%-27s %s\n", "accessId=ID", "S3 access key ID"); 1783 fprintf(stderr, "\t--%-27s %s\n", "accessKey=KEY", "S3 secret access key"); 1784 fprintf(stderr, "\t--%-27s %s\n", "accessType=TYPE", "S3 ACL used when creating new items; one of:"); 1785 fprintf(stderr, "\t %-27s ", ""); 1786 for (i = 0; i < sizeof(s3_acls) / sizeof(*s3_acls); i++) 1787 fprintf(stderr, "%s%s", i > 0 ? ", " : " ", s3_acls[i]); 1788 fprintf(stderr, "\n"); 1789 fprintf(stderr, "\t--%-27s %s\n", "authVersion=TYPE", "Specify S3 authentication style; one of:"); 1790 fprintf(stderr, "\t %-27s ", ""); 1791 for (i = 0; i < sizeof(s3_auth_types) / sizeof(*s3_auth_types); i++) 1792 fprintf(stderr, "%s%s", i > 0 ? ", " : " ", s3_auth_types[i]); 1793 fprintf(stderr, "\n"); 1794 fprintf(stderr, "\t--%-27s %s\n", "accessEC2IAM=ROLE", "Acquire S3 credentials from EC2 machine via IAM role"); 1795 fprintf(stderr, "\t--%-27s %s\n", "baseURL=URL", "Base URL for all requests"); 1796 fprintf(stderr, "\t--%-27s %s\n", "blockCacheFile=FILE", "Block cache persistent file"); 1797 fprintf(stderr, "\t--%-27s %s\n", "blockCacheMaxDirty=NUM", "Block cache maximum number of dirty blocks"); 1798 fprintf(stderr, "\t--%-27s %s\n", "blockCacheNoVerify", "Disable verification of data loaded from cache file"); 1799 fprintf(stderr, "\t--%-27s %s\n", "blockCacheSize=NUM", "Block cache size (in number of blocks)"); 1800 fprintf(stderr, "\t--%-27s %s\n", "blockCacheSync", "Block cache performs all writes synchronously"); 1801 fprintf(stderr, "\t--%-27s %s\n", "blockCacheRecoverDirtyBlocks", "Recover dirty cache file blocks on startup"); 1802 fprintf(stderr, "\t--%-27s %s\n", "blockCacheThreads=NUM", "Block cache write-back thread pool size"); 1803 fprintf(stderr, "\t--%-27s %s\n", "blockCacheTimeout=MILLIS", "Block cache entry timeout (zero = infinite)"); 1804 fprintf(stderr, "\t--%-27s %s\n", "blockCacheWriteDelay=MILLIS", "Block cache maximum write-back delay"); 1805 fprintf(stderr, "\t--%-27s %s\n", "blockSize=SIZE", "Block size (with optional suffix 'K', 'M', 'G', etc.)"); 1806 fprintf(stderr, "\t--%-27s %s\n", "blockHashPrefix", "Prepend hash to block names for even distribution"); 1807 fprintf(stderr, "\t--%-27s %s\n", "cacert=FILE", "Specify SSL certificate authority file"); 1808 fprintf(stderr, "\t--%-27s %s\n", "compress[=LEVEL]", "Enable block compression, with 1=fast up to 9=small"); 1809 fprintf(stderr, "\t--%-27s %s\n", "debug", "Enable logging of debug messages"); 1810 fprintf(stderr, "\t--%-27s %s\n", "debug-http", "Print HTTP headers to standard output"); 1811 fprintf(stderr, "\t--%-27s %s\n", "directIO", "Disable kernel caching of the backed file"); 1812 fprintf(stderr, "\t--%-27s %s\n", "encrypt[=CIPHER]", "Enable encryption (implies `--compress')"); 1813 fprintf(stderr, "\t--%-27s %s\n", "erase", "Erase all blocks in the filesystem"); 1814 fprintf(stderr, "\t--%-27s %s\n", "fileMode=MODE", "Permissions of backed file in filesystem"); 1815 fprintf(stderr, "\t--%-27s %s\n", "filename=NAME", "Name of backed file in filesystem"); 1816 fprintf(stderr, "\t--%-27s %s\n", "force", "Ignore different auto-detected block and file sizes"); 1817 fprintf(stderr, "\t--%-27s %s\n", "help", "Show this information and exit"); 1818 fprintf(stderr, "\t--%-27s %s\n", "initialRetryPause=MILLIS", "Initial retry pause after stale data or server error"); 1819 fprintf(stderr, "\t--%-27s %s\n", "insecure", "Don't verify SSL server identity"); 1820 fprintf(stderr, "\t--%-27s %s\n", "keyLength", "Override generated cipher key length"); 1821 fprintf(stderr, "\t--%-27s %s\n", "listBlocks", "Auto-detect non-empty blocks at startup"); 1822 fprintf(stderr, "\t--%-27s %s\n", "maxDownloadSpeed=BITSPERSEC", "Max download bandwidth for a single read"); 1823 fprintf(stderr, "\t--%-27s %s\n", "maxRetryPause=MILLIS", "Max total pause after stale data or server error"); 1824 fprintf(stderr, "\t--%-27s %s\n", "maxUploadSpeed=BITSPERSEC", "Max upload bandwidth for a single write"); 1825 fprintf(stderr, "\t--%-27s %s\n", "md5CacheSize=NUM", "Max size of MD5 cache (zero = disabled)"); 1826 fprintf(stderr, "\t--%-27s %s\n", "md5CacheTime=MILLIS", "Expire time for MD5 cache (zero = infinite)"); 1827 fprintf(stderr, "\t--%-27s %s\n", "minWriteDelay=MILLIS", "Minimum time between same block writes"); 1828 fprintf(stderr, "\t--%-27s %s\n", "password=PASSWORD", "Encrypt using PASSWORD"); 1829 fprintf(stderr, "\t--%-27s %s\n", "passwordFile=FILE", "Encrypt using password read from FILE"); 1830 fprintf(stderr, "\t--%-27s %s\n", "prefix=STRING", "Prefix for resource names within bucket"); 1831 fprintf(stderr, "\t--%-27s %s\n", "defaultContentEncoding=STRING", "Default HTTP Content-Encoding if none given"); 1832 fprintf(stderr, "\t--%-27s %s\n", "quiet", "Omit progress output at startup"); 1833 fprintf(stderr, "\t--%-27s %s\n", "readAhead=NUM", "Number of blocks to read-ahead"); 1834 fprintf(stderr, "\t--%-27s %s\n", "readAheadTrigger=NUM", "# of sequentially read blocks to trigger read-ahead"); 1835 fprintf(stderr, "\t--%-27s %s\n", "readOnly", "Return `Read-only file system' error for write attempts"); 1836 fprintf(stderr, "\t--%-27s %s\n", "region=region", "Specify AWS region"); 1837 fprintf(stderr, "\t--%-27s %s\n", "reset-mounted-flag", "Reset `already mounted' flag in the filesystem"); 1838 fprintf(stderr, "\t--%-27s %s\n", "rrs", "Target written blocks for Reduced Redundancy Storage (deprecated)"); 1839 fprintf(stderr, "\t--%-27s %s\n", "size=SIZE", "File size (with optional suffix 'K', 'M', 'G', etc.)"); 1840 fprintf(stderr, "\t--%-27s %s\n", "sse=" REQUIRED_SSE_VALUE, "Specify server side encryption"); 1841 fprintf(stderr, "\t--%-27s %s\n", "ssl", "Enable SSL"); 1842 fprintf(stderr, "\t--%-27s %s\n", "statsFilename=NAME", "Name of statistics file in filesystem"); 1843 fprintf(stderr, "\t--%-27s %s\n", "storageClass=TYPE", "Specify storage class for written blocks"); 1844 fprintf(stderr, "\t--%-27s %s\n", "test", "Run in local test mode (bucket is a directory)"); 1845 fprintf(stderr, "\t--%-27s %s\n", "timeout=SECONDS", "Max time allowed for one HTTP operation"); 1846 fprintf(stderr, "\t--%-27s %s\n", "timeout=SECONDS", "Specify HTTP operation timeout"); 1847 fprintf(stderr, "\t--%-27s %s\n", "version", "Show version information and exit"); 1848 fprintf(stderr, "\t--%-27s %s\n", "vhost", "Use virtual host bucket style URL for all requests"); 1849 fprintf(stderr, "Default values:\n"); 1850 fprintf(stderr, "\t--%-27s \"%s\"\n", "accessFile", "$HOME/" S3BACKER_DEFAULT_PWD_FILE); 1851 fprintf(stderr, "\t--%-27s %s\n", "accessId", "The first one listed in `accessFile'"); 1852 fprintf(stderr, "\t--%-27s \"%s\"\n", "accessType", S3BACKER_DEFAULT_ACCESS_TYPE); 1853 fprintf(stderr, "\t--%-27s \"%s\"\n", "authVersion", S3BACKER_DEFAULT_AUTH_VERSION); 1854 fprintf(stderr, "\t--%-27s \"%s\"\n", "baseURL", "http://s3." S3_DOMAIN "/"); 1855 fprintf(stderr, "\t--%-27s %u\n", "blockCacheSize", S3BACKER_DEFAULT_BLOCK_CACHE_SIZE); 1856 fprintf(stderr, "\t--%-27s %u\n", "blockCacheThreads", S3BACKER_DEFAULT_BLOCK_CACHE_NUM_THREADS); 1857 fprintf(stderr, "\t--%-27s %u\n", "blockCacheTimeout", S3BACKER_DEFAULT_BLOCK_CACHE_TIMEOUT); 1858 fprintf(stderr, "\t--%-27s %u\n", "blockCacheWriteDelay", S3BACKER_DEFAULT_BLOCK_CACHE_WRITE_DELAY); 1859 fprintf(stderr, "\t--%-27s %d\n", "blockSize", S3BACKER_DEFAULT_BLOCKSIZE); 1860 fprintf(stderr, "\t--%-27s \"%s\"\n", "filename", S3BACKER_DEFAULT_FILENAME); 1861 fprintf(stderr, "\t--%-27s %u\n", "initialRetryPause", S3BACKER_DEFAULT_INITIAL_RETRY_PAUSE); 1862 fprintf(stderr, "\t--%-27s %u\n", "md5CacheSize", S3BACKER_DEFAULT_MD5_CACHE_SIZE); 1863 fprintf(stderr, "\t--%-27s %u\n", "md5CacheTime", S3BACKER_DEFAULT_MD5_CACHE_TIME); 1864 fprintf(stderr, "\t--%-27s 0%03o (0%03o if `--readOnly')\n", "fileMode", 1865 S3BACKER_DEFAULT_FILE_MODE, S3BACKER_DEFAULT_FILE_MODE_READ_ONLY); 1866 fprintf(stderr, "\t--%-27s %u\n", "maxRetryPause", S3BACKER_DEFAULT_MAX_RETRY_PAUSE); 1867 fprintf(stderr, "\t--%-27s %u\n", "minWriteDelay", S3BACKER_DEFAULT_MIN_WRITE_DELAY); 1868 fprintf(stderr, "\t--%-27s \"%s\"\n", "prefix", S3BACKER_DEFAULT_PREFIX); 1869 fprintf(stderr, "\t--%-27s %u\n", "readAhead", S3BACKER_DEFAULT_READ_AHEAD); 1870 fprintf(stderr, "\t--%-27s %u\n", "readAheadTrigger", S3BACKER_DEFAULT_READ_AHEAD_TRIGGER); 1871 fprintf(stderr, "\t--%-27s \"%s\"\n", "region", S3BACKER_DEFAULT_REGION); 1872 fprintf(stderr, "\t--%-27s \"%s\"\n", "statsFilename", S3BACKER_DEFAULT_STATS_FILENAME); 1873 fprintf(stderr, "\t--%-27s %u\n", "timeout", S3BACKER_DEFAULT_TIMEOUT); 1874 fprintf(stderr, "FUSE options (partial list):\n"); 1875 fprintf(stderr, "\t%-29s %s\n", "-o nonempty", "Allows mount over a non-empty directory"); 1876 fprintf(stderr, "\t%-29s %s\n", "-o uid=UID", "Set user ID"); 1877 fprintf(stderr, "\t%-29s %s\n", "-o gid=GID", "Set group ID"); 1878 fprintf(stderr, "\t%-29s %s\n", "-o sync_read", "Do synchronous reads"); 1879 fprintf(stderr, "\t%-29s %s\n", "-o max_readahead=NUM", "Set maximum read-ahead (bytes)"); 1880 fprintf(stderr, "\t%-29s %s\n", "-f", "Run in the foreground (do not fork)"); 1881 fprintf(stderr, "\t%-29s %s\n", "-d", "Debug mode (implies -f)"); 1882 fprintf(stderr, "\t%-29s %s\n", "-s", "Run in single-threaded mode"); 1883 } 1884 1885