1 /* $OpenBSD: repo.c,v 1.51 2023/07/20 05:18:31 claudio Exp $ */ 2 /* 3 * Copyright (c) 2021 Claudio Jeker <claudio@openbsd.org> 4 * Copyright (c) 2019 Kristaps Dzonsons <kristaps@bsd.lv> 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 #include <sys/queue.h> 20 #include <sys/tree.h> 21 #include <sys/types.h> 22 #include <sys/stat.h> 23 24 #include <assert.h> 25 #include <err.h> 26 #include <errno.h> 27 #include <fcntl.h> 28 #include <fts.h> 29 #include <limits.h> 30 #include <poll.h> 31 #include <stdio.h> 32 #include <stdlib.h> 33 #include <string.h> 34 #include <unistd.h> 35 36 #include <imsg.h> 37 38 #include "extern.h" 39 40 extern struct stats stats; 41 extern int noop; 42 extern int rrdpon; 43 extern int repo_timeout; 44 extern time_t deadline; 45 int nofetch; 46 47 enum repo_state { 48 REPO_LOADING = 0, 49 REPO_DONE = 1, 50 REPO_FAILED = -1, 51 }; 52 53 /* 54 * A ta, rsync or rrdp repository. 55 * Depending on what is needed the generic repository is backed by 56 * a ta, rsync or rrdp repository. Multiple repositories can use the 57 * same backend. 58 */ 59 struct rrdprepo { 60 SLIST_ENTRY(rrdprepo) entry; 61 char *notifyuri; 62 char *basedir; 63 struct filepath_tree deleted; 64 unsigned int id; 65 enum repo_state state; 66 }; 67 static SLIST_HEAD(, rrdprepo) rrdprepos = SLIST_HEAD_INITIALIZER(rrdprepos); 68 69 struct rsyncrepo { 70 SLIST_ENTRY(rsyncrepo) entry; 71 char *repouri; 72 char *basedir; 73 unsigned int id; 74 enum repo_state state; 75 }; 76 static SLIST_HEAD(, rsyncrepo) rsyncrepos = SLIST_HEAD_INITIALIZER(rsyncrepos); 77 78 struct tarepo { 79 SLIST_ENTRY(tarepo) entry; 80 char *descr; 81 char *basedir; 82 char *temp; 83 char **uri; 84 size_t urisz; 85 size_t uriidx; 86 unsigned int id; 87 enum repo_state state; 88 }; 89 static SLIST_HEAD(, tarepo) tarepos = SLIST_HEAD_INITIALIZER(tarepos); 90 91 struct repo { 92 SLIST_ENTRY(repo) entry; 93 char *repouri; 94 char *notifyuri; 95 char *basedir; 96 const struct rrdprepo *rrdp; 97 const struct rsyncrepo *rsync; 98 const struct tarepo *ta; 99 struct entityq queue; /* files waiting for repo */ 100 struct repotalstats stats[TALSZ_MAX]; 101 struct repostats repostats; 102 struct timespec start_time; 103 time_t alarm; /* sync timeout */ 104 int talid; 105 int stats_used[TALSZ_MAX]; 106 unsigned int id; /* identifier */ 107 }; 108 static SLIST_HEAD(, repo) repos = SLIST_HEAD_INITIALIZER(repos); 109 110 /* counter for unique repo id */ 111 unsigned int repoid; 112 113 static struct rsyncrepo *rsync_get(const char *, const char *); 114 static void remove_contents(char *); 115 116 /* 117 * Database of all file path accessed during a run. 118 */ 119 struct filepath { 120 RB_ENTRY(filepath) entry; 121 char *file; 122 time_t mtime; 123 }; 124 125 static inline int 126 filepathcmp(struct filepath *a, struct filepath *b) 127 { 128 return strcmp(a->file, b->file); 129 } 130 131 RB_PROTOTYPE(filepath_tree, filepath, entry, filepathcmp); 132 133 /* 134 * Functions to lookup which files have been accessed during computation. 135 */ 136 int 137 filepath_add(struct filepath_tree *tree, char *file, time_t mtime) 138 { 139 struct filepath *fp; 140 141 if ((fp = malloc(sizeof(*fp))) == NULL) 142 err(1, NULL); 143 fp->mtime = mtime; 144 if ((fp->file = strdup(file)) == NULL) 145 err(1, NULL); 146 147 if (RB_INSERT(filepath_tree, tree, fp) != NULL) { 148 /* already in the tree */ 149 free(fp->file); 150 free(fp); 151 return 0; 152 } 153 154 return 1; 155 } 156 157 /* 158 * Lookup a file path in the tree and return the object if found or NULL. 159 */ 160 static struct filepath * 161 filepath_find(struct filepath_tree *tree, char *file) 162 { 163 struct filepath needle = { .file = file }; 164 165 return RB_FIND(filepath_tree, tree, &needle); 166 } 167 168 /* 169 * Returns true if file exists in the tree. 170 */ 171 static int 172 filepath_exists(struct filepath_tree *tree, char *file) 173 { 174 return filepath_find(tree, file) != NULL; 175 } 176 177 /* 178 * Remove entry from tree and free it. 179 */ 180 static void 181 filepath_put(struct filepath_tree *tree, struct filepath *fp) 182 { 183 RB_REMOVE(filepath_tree, tree, fp); 184 free((void *)fp->file); 185 free(fp); 186 } 187 188 /* 189 * Free all elements of a filepath tree. 190 */ 191 static void 192 filepath_free(struct filepath_tree *tree) 193 { 194 struct filepath *fp, *nfp; 195 196 RB_FOREACH_SAFE(fp, filepath_tree, tree, nfp) 197 filepath_put(tree, fp); 198 } 199 200 RB_GENERATE(filepath_tree, filepath, entry, filepathcmp); 201 202 /* 203 * Function to hash a string into a unique directory name. 204 * Returned hash needs to be freed. 205 */ 206 static char * 207 hash_dir(const char *uri) 208 { 209 unsigned char m[SHA256_DIGEST_LENGTH]; 210 211 SHA256(uri, strlen(uri), m); 212 return hex_encode(m, sizeof(m)); 213 } 214 215 /* 216 * Function to build the directory name based on URI and a directory 217 * as prefix. Skip the proto:// in URI but keep everything else. 218 */ 219 static char * 220 repo_dir(const char *uri, const char *dir, int hash) 221 { 222 const char *local; 223 char *out, *hdir = NULL; 224 225 if (hash) { 226 local = hdir = hash_dir(uri); 227 } else { 228 local = strchr(uri, ':'); 229 if (local != NULL) 230 local += strlen("://"); 231 else 232 local = uri; 233 } 234 235 if (dir == NULL) { 236 if ((out = strdup(local)) == NULL) 237 err(1, NULL); 238 } else { 239 if (asprintf(&out, "%s/%s", dir, local) == -1) 240 err(1, NULL); 241 } 242 243 free(hdir); 244 return out; 245 } 246 247 /* 248 * Function to create all missing directories to a path. 249 * This functions alters the path temporarily. 250 */ 251 static int 252 repo_mkpath(int fd, char *file) 253 { 254 char *slash; 255 256 /* build directory hierarchy */ 257 slash = strrchr(file, '/'); 258 assert(slash != NULL); 259 *slash = '\0'; 260 if (mkpathat(fd, file) == -1) { 261 warn("mkpath %s", file); 262 return -1; 263 } 264 *slash = '/'; 265 return 0; 266 } 267 268 /* 269 * Return the state of a repository. 270 */ 271 static enum repo_state 272 repo_state(const struct repo *rp) 273 { 274 if (rp->ta) 275 return rp->ta->state; 276 if (rp->rsync) 277 return rp->rsync->state; 278 if (rp->rrdp) 279 return rp->rrdp->state; 280 /* No backend so sync is by definition done. */ 281 return REPO_DONE; 282 } 283 284 /* 285 * Function called once a repository is done with the sync. Either 286 * successfully or after failure. 287 */ 288 static void 289 repo_done(const void *vp, int ok) 290 { 291 struct repo *rp; 292 struct timespec flush_time; 293 294 SLIST_FOREACH(rp, &repos, entry) { 295 if (vp != rp->ta && vp != rp->rsync && vp != rp->rrdp) 296 continue; 297 298 /* for rrdp try to fall back to rsync */ 299 if (vp == rp->rrdp && !ok && !nofetch) { 300 rp->rrdp = NULL; 301 rp->rsync = rsync_get(rp->repouri, rp->basedir); 302 /* need to check if it was already loaded */ 303 if (repo_state(rp) == REPO_LOADING) 304 continue; 305 } 306 307 entityq_flush(&rp->queue, rp); 308 clock_gettime(CLOCK_MONOTONIC, &flush_time); 309 timespecsub(&flush_time, &rp->start_time, 310 &rp->repostats.sync_time); 311 } 312 } 313 314 /* 315 * Build TA file name based on the repo info. 316 * If temp is set add Xs for mkostemp. 317 */ 318 static char * 319 ta_filename(const struct tarepo *tr, int temp) 320 { 321 const char *file; 322 char *nfile; 323 324 /* does not matter which URI, all end with same filename */ 325 file = strrchr(tr->uri[0], '/'); 326 assert(file); 327 328 if (asprintf(&nfile, "%s%s%s", tr->basedir, file, 329 temp ? ".XXXXXXXX" : "") == -1) 330 err(1, NULL); 331 332 return nfile; 333 } 334 335 static void 336 ta_fetch(struct tarepo *tr) 337 { 338 if (!rrdpon) { 339 for (; tr->uriidx < tr->urisz; tr->uriidx++) { 340 if (strncasecmp(tr->uri[tr->uriidx], 341 "rsync://", 8) == 0) 342 break; 343 } 344 } 345 346 if (tr->uriidx >= tr->urisz) { 347 tr->state = REPO_FAILED; 348 logx("ta/%s: fallback to cache", tr->descr); 349 350 repo_done(tr, 0); 351 return; 352 } 353 354 logx("ta/%s: pulling from %s", tr->descr, tr->uri[tr->uriidx]); 355 356 if (strncasecmp(tr->uri[tr->uriidx], "rsync://", 8) == 0) { 357 /* 358 * Create destination location. 359 * Build up the tree to this point. 360 */ 361 rsync_fetch(tr->id, tr->uri[tr->uriidx], tr->basedir, NULL); 362 } else { 363 int fd; 364 365 tr->temp = ta_filename(tr, 1); 366 fd = mkostemp(tr->temp, O_CLOEXEC); 367 if (fd == -1) { 368 warn("mkostemp: %s", tr->temp); 369 http_finish(tr->id, HTTP_FAILED, NULL); 370 return; 371 } 372 if (fchmod(fd, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH) == -1) 373 warn("fchmod: %s", tr->temp); 374 375 http_fetch(tr->id, tr->uri[tr->uriidx], NULL, fd); 376 } 377 } 378 379 static struct tarepo * 380 ta_get(struct tal *tal) 381 { 382 struct tarepo *tr; 383 384 /* no need to look for possible other repo */ 385 386 if ((tr = calloc(1, sizeof(*tr))) == NULL) 387 err(1, NULL); 388 tr->id = ++repoid; 389 SLIST_INSERT_HEAD(&tarepos, tr, entry); 390 391 if ((tr->descr = strdup(tal->descr)) == NULL) 392 err(1, NULL); 393 tr->basedir = repo_dir(tal->descr, "ta", 0); 394 395 /* steal URI information from TAL */ 396 tr->urisz = tal->urisz; 397 tr->uri = tal->uri; 398 tal->urisz = 0; 399 tal->uri = NULL; 400 401 ta_fetch(tr); 402 403 return tr; 404 } 405 406 static struct tarepo * 407 ta_find(unsigned int id) 408 { 409 struct tarepo *tr; 410 411 SLIST_FOREACH(tr, &tarepos, entry) 412 if (id == tr->id) 413 break; 414 return tr; 415 } 416 417 static void 418 ta_free(void) 419 { 420 struct tarepo *tr; 421 422 while ((tr = SLIST_FIRST(&tarepos)) != NULL) { 423 SLIST_REMOVE_HEAD(&tarepos, entry); 424 free(tr->descr); 425 free(tr->basedir); 426 free(tr->temp); 427 free(tr->uri); 428 free(tr); 429 } 430 } 431 432 static struct rsyncrepo * 433 rsync_get(const char *uri, const char *validdir) 434 { 435 struct rsyncrepo *rr; 436 char *repo; 437 438 if ((repo = rsync_base_uri(uri)) == NULL) 439 errx(1, "bad caRepository URI: %s", uri); 440 441 SLIST_FOREACH(rr, &rsyncrepos, entry) 442 if (strcmp(rr->repouri, repo) == 0) { 443 free(repo); 444 return rr; 445 } 446 447 if ((rr = calloc(1, sizeof(*rr))) == NULL) 448 err(1, NULL); 449 450 rr->id = ++repoid; 451 SLIST_INSERT_HEAD(&rsyncrepos, rr, entry); 452 453 rr->repouri = repo; 454 rr->basedir = repo_dir(repo, ".rsync", 0); 455 456 /* create base directory */ 457 if (mkpath(rr->basedir) == -1) { 458 warn("mkpath %s", rr->basedir); 459 rsync_finish(rr->id, 0); 460 return rr; 461 } 462 463 logx("%s: pulling from %s", rr->basedir, rr->repouri); 464 rsync_fetch(rr->id, rr->repouri, rr->basedir, validdir); 465 466 return rr; 467 } 468 469 static struct rsyncrepo * 470 rsync_find(unsigned int id) 471 { 472 struct rsyncrepo *rr; 473 474 SLIST_FOREACH(rr, &rsyncrepos, entry) 475 if (id == rr->id) 476 break; 477 return rr; 478 } 479 480 static void 481 rsync_free(void) 482 { 483 struct rsyncrepo *rr; 484 485 while ((rr = SLIST_FIRST(&rsyncrepos)) != NULL) { 486 SLIST_REMOVE_HEAD(&rsyncrepos, entry); 487 free(rr->repouri); 488 free(rr->basedir); 489 free(rr); 490 } 491 } 492 493 /* 494 * Build local file name base on the URI and the rrdprepo info. 495 */ 496 static char * 497 rrdp_filename(const struct rrdprepo *rr, const char *uri, int valid) 498 { 499 char *nfile; 500 const char *dir = rr->basedir; 501 502 if (!valid_uri(uri, strlen(uri), "rsync://")) 503 errx(1, "%s: bad URI %s", rr->basedir, uri); 504 uri += strlen("rsync://"); /* skip proto */ 505 if (valid) { 506 if ((nfile = strdup(uri)) == NULL) 507 err(1, NULL); 508 } else { 509 if (asprintf(&nfile, "%s/%s", dir, uri) == -1) 510 err(1, NULL); 511 } 512 return nfile; 513 } 514 515 /* 516 * Build RRDP state file name based on the repo info. 517 * If temp is set add Xs for mkostemp. 518 */ 519 static char * 520 rrdp_state_filename(const struct rrdprepo *rr, int temp) 521 { 522 char *nfile; 523 524 if (asprintf(&nfile, "%s/.state%s", rr->basedir, 525 temp ? ".XXXXXXXX" : "") == -1) 526 err(1, NULL); 527 528 return nfile; 529 } 530 531 static struct rrdprepo * 532 rrdp_find(unsigned int id) 533 { 534 struct rrdprepo *rr; 535 536 SLIST_FOREACH(rr, &rrdprepos, entry) 537 if (id == rr->id) 538 break; 539 return rr; 540 } 541 542 static void 543 rrdp_free(void) 544 { 545 struct rrdprepo *rr; 546 547 while ((rr = SLIST_FIRST(&rrdprepos)) != NULL) { 548 SLIST_REMOVE_HEAD(&rrdprepos, entry); 549 550 free(rr->notifyuri); 551 free(rr->basedir); 552 553 filepath_free(&rr->deleted); 554 555 free(rr); 556 } 557 } 558 559 /* 560 * Check if a directory is an active rrdp repository. 561 * Returns 1 if found else 0. 562 */ 563 static struct repo * 564 repo_rrdp_bypath(const char *dir) 565 { 566 struct repo *rp; 567 568 SLIST_FOREACH(rp, &repos, entry) { 569 if (rp->rrdp == NULL) 570 continue; 571 if (strcmp(dir, rp->rrdp->basedir) == 0) 572 return rp; 573 } 574 return NULL; 575 } 576 577 /* 578 * Check if the URI is actually covered by one of the repositories 579 * that depend on this RRDP repository. 580 * Returns 1 if the URI is valid, 0 if no repouri matches the URI. 581 */ 582 static int 583 rrdp_uri_valid(struct rrdprepo *rr, const char *uri) 584 { 585 struct repo *rp; 586 587 SLIST_FOREACH(rp, &repos, entry) { 588 if (rp->rrdp != rr) 589 continue; 590 if (strncmp(uri, rp->repouri, strlen(rp->repouri)) == 0) 591 return 1; 592 } 593 return 0; 594 } 595 596 /* 597 * Allocate and insert a new repository. 598 */ 599 static struct repo * 600 repo_alloc(int talid) 601 { 602 struct repo *rp; 603 604 if ((rp = calloc(1, sizeof(*rp))) == NULL) 605 err(1, NULL); 606 607 rp->id = ++repoid; 608 rp->talid = talid; 609 rp->alarm = getmonotime() + repo_timeout; 610 TAILQ_INIT(&rp->queue); 611 SLIST_INSERT_HEAD(&repos, rp, entry); 612 clock_gettime(CLOCK_MONOTONIC, &rp->start_time); 613 614 stats.repos++; 615 return rp; 616 } 617 618 /* 619 * Parse the RRDP state file if it exists and set the session struct 620 * based on that information. 621 */ 622 static struct rrdp_session * 623 rrdp_session_parse(const struct rrdprepo *rr) 624 { 625 FILE *f; 626 struct rrdp_session *state; 627 int fd, ln = 0, deltacnt = 0; 628 const char *errstr; 629 char *line = NULL, *file; 630 size_t len = 0; 631 ssize_t n; 632 633 if ((state = calloc(1, sizeof(*state))) == NULL) 634 err(1, NULL); 635 636 file = rrdp_state_filename(rr, 0); 637 if ((fd = open(file, O_RDONLY)) == -1) { 638 if (errno != ENOENT) 639 warn("%s: open state file", rr->basedir); 640 free(file); 641 return state; 642 } 643 free(file); 644 f = fdopen(fd, "r"); 645 if (f == NULL) 646 err(1, "fdopen"); 647 648 while ((n = getline(&line, &len, f)) != -1) { 649 if (line[n - 1] == '\n') 650 line[n - 1] = '\0'; 651 switch (ln) { 652 case 0: 653 if ((state->session_id = strdup(line)) == NULL) 654 err(1, NULL); 655 break; 656 case 1: 657 state->serial = strtonum(line, 1, LLONG_MAX, &errstr); 658 if (errstr) 659 goto fail; 660 break; 661 case 2: 662 if (strcmp(line, "-") == 0) 663 break; 664 if ((state->last_mod = strdup(line)) == NULL) 665 err(1, NULL); 666 break; 667 default: 668 if (deltacnt >= MAX_RRDP_DELTAS) 669 goto fail; 670 if ((state->deltas[deltacnt++] = strdup(line)) == NULL) 671 err(1, NULL); 672 break; 673 } 674 ln++; 675 } 676 677 if (ferror(f)) 678 goto fail; 679 fclose(f); 680 free(line); 681 return state; 682 683 fail: 684 warnx("%s: troubles reading state file", rr->basedir); 685 fclose(f); 686 free(line); 687 free(state->session_id); 688 free(state->last_mod); 689 memset(state, 0, sizeof(*state)); 690 return state; 691 } 692 693 /* 694 * Carefully write the RRDP session state file back. 695 */ 696 void 697 rrdp_session_save(unsigned int id, struct rrdp_session *state) 698 { 699 struct rrdprepo *rr; 700 char *temp, *file; 701 FILE *f = NULL; 702 int fd, i; 703 704 rr = rrdp_find(id); 705 if (rr == NULL) 706 errx(1, "non-existent rrdp repo %u", id); 707 708 file = rrdp_state_filename(rr, 0); 709 temp = rrdp_state_filename(rr, 1); 710 711 if ((fd = mkostemp(temp, O_CLOEXEC)) == -1) 712 goto fail; 713 (void)fchmod(fd, 0644); 714 f = fdopen(fd, "w"); 715 if (f == NULL) 716 err(1, "fdopen"); 717 718 /* write session state file out */ 719 if (fprintf(f, "%s\n%lld\n", state->session_id, 720 state->serial) < 0) 721 goto fail; 722 723 if (state->last_mod != NULL) { 724 if (fprintf(f, "%s\n", state->last_mod) < 0) 725 goto fail; 726 } else { 727 if (fprintf(f, "-\n") < 0) 728 goto fail; 729 } 730 for (i = 0; i < MAX_RRDP_DELTAS && state->deltas[i] != NULL; i++) { 731 if (fprintf(f, "%s\n", state->deltas[i]) < 0) 732 goto fail; 733 } 734 if (fclose(f) != 0) { 735 f = NULL; 736 goto fail; 737 } 738 739 if (rename(temp, file) == -1) { 740 warn("%s: rename %s to %s", rr->basedir, temp, file); 741 unlink(temp); 742 } 743 744 free(temp); 745 free(file); 746 return; 747 748 fail: 749 warn("%s: save state to %s", rr->basedir, temp); 750 if (f != NULL) 751 fclose(f); 752 unlink(temp); 753 free(temp); 754 free(file); 755 } 756 757 /* 758 * Free an rrdp_session pointer. Safe to call with NULL. 759 */ 760 void 761 rrdp_session_free(struct rrdp_session *s) 762 { 763 size_t i; 764 765 if (s == NULL) 766 return; 767 free(s->session_id); 768 free(s->last_mod); 769 for (i = 0; i < sizeof(s->deltas) / sizeof(s->deltas[0]); i++) 770 free(s->deltas[i]); 771 free(s); 772 } 773 774 void 775 rrdp_session_buffer(struct ibuf *b, const struct rrdp_session *s) 776 { 777 size_t i; 778 779 io_str_buffer(b, s->session_id); 780 io_simple_buffer(b, &s->serial, sizeof(s->serial)); 781 io_str_buffer(b, s->last_mod); 782 for (i = 0; i < sizeof(s->deltas) / sizeof(s->deltas[0]); i++) 783 io_str_buffer(b, s->deltas[i]); 784 } 785 786 struct rrdp_session * 787 rrdp_session_read(struct ibuf *b) 788 { 789 struct rrdp_session *s; 790 size_t i; 791 792 if ((s = calloc(1, sizeof(*s))) == NULL) 793 err(1, NULL); 794 795 io_read_str(b, &s->session_id); 796 io_read_buf(b, &s->serial, sizeof(s->serial)); 797 io_read_str(b, &s->last_mod); 798 for (i = 0; i < sizeof(s->deltas) / sizeof(s->deltas[0]); i++) 799 io_read_str(b, &s->deltas[i]); 800 801 return s; 802 } 803 804 static struct rrdprepo * 805 rrdp_get(const char *uri) 806 { 807 struct rrdp_session *state; 808 struct rrdprepo *rr; 809 810 SLIST_FOREACH(rr, &rrdprepos, entry) 811 if (strcmp(rr->notifyuri, uri) == 0) { 812 if (rr->state == REPO_FAILED) 813 return NULL; 814 return rr; 815 } 816 817 if ((rr = calloc(1, sizeof(*rr))) == NULL) 818 err(1, NULL); 819 820 rr->id = ++repoid; 821 SLIST_INSERT_HEAD(&rrdprepos, rr, entry); 822 823 if ((rr->notifyuri = strdup(uri)) == NULL) 824 err(1, NULL); 825 rr->basedir = repo_dir(uri, ".rrdp", 1); 826 827 RB_INIT(&rr->deleted); 828 829 /* create base directory */ 830 if (mkpath(rr->basedir) == -1) { 831 warn("mkpath %s", rr->basedir); 832 rrdp_finish(rr->id, 0); 833 return rr; 834 } 835 836 /* parse state and start the sync */ 837 state = rrdp_session_parse(rr); 838 rrdp_fetch(rr->id, rr->notifyuri, rr->notifyuri, state); 839 rrdp_session_free(state); 840 841 logx("%s: pulling from %s", rr->notifyuri, "network"); 842 843 return rr; 844 } 845 846 /* 847 * Remove RRDP repo and start over. 848 */ 849 void 850 rrdp_clear(unsigned int id) 851 { 852 struct rrdprepo *rr; 853 854 rr = rrdp_find(id); 855 if (rr == NULL) 856 errx(1, "non-existent rrdp repo %u", id); 857 858 /* remove rrdp repository contents */ 859 remove_contents(rr->basedir); 860 } 861 862 /* 863 * Write a file into the temporary RRDP dir but only after checking 864 * its hash (if required). The function also makes sure that the file 865 * tracking is properly adjusted. 866 * Returns 1 on success, 0 if the repo is corrupt, -1 on IO error 867 */ 868 int 869 rrdp_handle_file(unsigned int id, enum publish_type pt, char *uri, 870 char *hash, size_t hlen, char *data, size_t dlen) 871 { 872 struct rrdprepo *rr; 873 struct filepath *fp; 874 ssize_t s; 875 char *fn = NULL; 876 int fd = -1, try = 0, deleted = 0; 877 int flags; 878 879 rr = rrdp_find(id); 880 if (rr == NULL) 881 errx(1, "non-existent rrdp repo %u", id); 882 if (rr->state == REPO_FAILED) 883 return -1; 884 885 /* check hash of original file for updates and deletes */ 886 if (pt == PUB_UPD || pt == PUB_DEL) { 887 if (filepath_exists(&rr->deleted, uri)) { 888 warnx("%s: already deleted", uri); 889 return 0; 890 } 891 /* try to open file first in rrdp then in valid repo */ 892 do { 893 free(fn); 894 if ((fn = rrdp_filename(rr, uri, try++)) == NULL) 895 return 0; 896 fd = open(fn, O_RDONLY); 897 } while (fd == -1 && try < 2); 898 899 if (!valid_filehash(fd, hash, hlen)) { 900 warnx("%s: bad file digest for %s", rr->notifyuri, fn); 901 free(fn); 902 return 0; 903 } 904 free(fn); 905 } 906 907 /* write new content or mark uri as deleted. */ 908 if (pt == PUB_DEL) { 909 filepath_add(&rr->deleted, uri, 0); 910 } else { 911 fp = filepath_find(&rr->deleted, uri); 912 if (fp != NULL) { 913 filepath_put(&rr->deleted, fp); 914 deleted = 1; 915 } 916 917 /* add new file to rrdp dir */ 918 if ((fn = rrdp_filename(rr, uri, 0)) == NULL) 919 return 0; 920 921 if (repo_mkpath(AT_FDCWD, fn) == -1) 922 goto fail; 923 924 flags = O_WRONLY|O_CREAT|O_TRUNC; 925 if (pt == PUB_ADD && !deleted) 926 flags |= O_EXCL; 927 fd = open(fn, flags, 0644); 928 if (fd == -1) { 929 if (errno == EEXIST) { 930 warnx("%s: duplicate publish element for %s", 931 rr->notifyuri, fn); 932 free(fn); 933 return 0; 934 } 935 warn("open %s", fn); 936 goto fail; 937 } 938 939 if ((s = write(fd, data, dlen)) == -1) { 940 warn("write %s", fn); 941 goto fail; 942 } 943 close(fd); 944 if ((size_t)s != dlen) /* impossible */ 945 errx(1, "short write %s", fn); 946 free(fn); 947 } 948 949 return 1; 950 951 fail: 952 rr->state = REPO_FAILED; 953 if (fd != -1) 954 close(fd); 955 free(fn); 956 return -1; 957 } 958 959 /* 960 * RSYNC sync finished, either with or without success. 961 */ 962 void 963 rsync_finish(unsigned int id, int ok) 964 { 965 struct rsyncrepo *rr; 966 struct tarepo *tr; 967 968 tr = ta_find(id); 969 if (tr != NULL) { 970 /* repository changed state already, ignore request */ 971 if (tr->state != REPO_LOADING) 972 return; 973 if (ok) { 974 logx("ta/%s: loaded from network", tr->descr); 975 stats.rsync_repos++; 976 tr->state = REPO_DONE; 977 repo_done(tr, 1); 978 } else { 979 warnx("ta/%s: load from network failed", tr->descr); 980 stats.rsync_fails++; 981 tr->uriidx++; 982 ta_fetch(tr); 983 } 984 return; 985 } 986 987 rr = rsync_find(id); 988 if (rr == NULL) 989 errx(1, "unknown rsync repo %u", id); 990 /* repository changed state already, ignore request */ 991 if (rr->state != REPO_LOADING) 992 return; 993 994 if (ok) { 995 logx("%s: loaded from network", rr->basedir); 996 stats.rsync_repos++; 997 rr->state = REPO_DONE; 998 } else { 999 warnx("%s: load from network failed, fallback to cache", 1000 rr->basedir); 1001 stats.rsync_fails++; 1002 rr->state = REPO_FAILED; 1003 /* clear rsync repo since it failed */ 1004 remove_contents(rr->basedir); 1005 } 1006 1007 repo_done(rr, ok); 1008 } 1009 1010 /* 1011 * RRDP sync finished, either with or without success. 1012 */ 1013 void 1014 rrdp_finish(unsigned int id, int ok) 1015 { 1016 struct rrdprepo *rr; 1017 1018 rr = rrdp_find(id); 1019 if (rr == NULL) 1020 errx(1, "unknown RRDP repo %u", id); 1021 /* repository changed state already, ignore request */ 1022 if (rr->state != REPO_LOADING) 1023 return; 1024 1025 if (ok) { 1026 logx("%s: loaded from network", rr->notifyuri); 1027 stats.rrdp_repos++; 1028 rr->state = REPO_DONE; 1029 } else { 1030 warnx("%s: load from network failed, fallback to %s", 1031 rr->notifyuri, nofetch ? "cache" : "rsync"); 1032 stats.rrdp_fails++; 1033 rr->state = REPO_FAILED; 1034 /* clear the RRDP repo since it failed */ 1035 remove_contents(rr->basedir); 1036 /* also clear the list of deleted files */ 1037 filepath_free(&rr->deleted); 1038 } 1039 1040 repo_done(rr, ok); 1041 } 1042 1043 /* 1044 * Handle responses from the http process. For TA file, either rename 1045 * or delete the temporary file. For RRDP requests relay the request 1046 * over to the rrdp process. 1047 */ 1048 void 1049 http_finish(unsigned int id, enum http_result res, const char *last_mod) 1050 { 1051 struct tarepo *tr; 1052 1053 tr = ta_find(id); 1054 if (tr == NULL) { 1055 /* not a TA fetch therefore RRDP */ 1056 rrdp_http_done(id, res, last_mod); 1057 return; 1058 } 1059 1060 /* repository changed state already, ignore request */ 1061 if (tr->state != REPO_LOADING) 1062 return; 1063 1064 /* Move downloaded TA file into place, or unlink on failure. */ 1065 if (res == HTTP_OK) { 1066 char *file; 1067 1068 file = ta_filename(tr, 0); 1069 if (rename(tr->temp, file) == -1) 1070 warn("rename to %s", file); 1071 free(file); 1072 1073 logx("ta/%s: loaded from network", tr->descr); 1074 tr->state = REPO_DONE; 1075 stats.http_repos++; 1076 repo_done(tr, 1); 1077 } else { 1078 if (unlink(tr->temp) == -1 && errno != ENOENT) 1079 warn("unlink %s", tr->temp); 1080 1081 tr->uriidx++; 1082 warnx("ta/%s: load from network failed", tr->descr); 1083 ta_fetch(tr); 1084 } 1085 } 1086 1087 /* 1088 * Look up a trust anchor, queueing it for download if not found. 1089 */ 1090 struct repo * 1091 ta_lookup(int id, struct tal *tal) 1092 { 1093 struct repo *rp; 1094 1095 if (tal->urisz == 0) 1096 errx(1, "TAL %s has no URI", tal->descr); 1097 1098 /* Look up in repository table. (Lookup should actually fail here) */ 1099 SLIST_FOREACH(rp, &repos, entry) { 1100 if (strcmp(rp->repouri, tal->uri[0]) == 0) 1101 return rp; 1102 } 1103 1104 rp = repo_alloc(id); 1105 rp->basedir = repo_dir(tal->descr, "ta", 0); 1106 if ((rp->repouri = strdup(tal->uri[0])) == NULL) 1107 err(1, NULL); 1108 1109 /* check if sync disabled ... */ 1110 if (noop) { 1111 logx("%s: using cache", rp->basedir); 1112 entityq_flush(&rp->queue, rp); 1113 return rp; 1114 } 1115 1116 /* try to create base directory */ 1117 if (mkpath(rp->basedir) == -1) 1118 warn("mkpath %s", rp->basedir); 1119 1120 rp->ta = ta_get(tal); 1121 1122 /* need to check if it was already loaded */ 1123 if (repo_state(rp) != REPO_LOADING) 1124 entityq_flush(&rp->queue, rp); 1125 1126 return rp; 1127 } 1128 1129 /* 1130 * Look up a repository, queueing it for discovery if not found. 1131 */ 1132 struct repo * 1133 repo_lookup(int talid, const char *uri, const char *notify) 1134 { 1135 struct repo *rp; 1136 char *repouri; 1137 1138 if ((repouri = rsync_base_uri(uri)) == NULL) 1139 errx(1, "bad caRepository URI: %s", uri); 1140 1141 /* Look up in repository table. */ 1142 SLIST_FOREACH(rp, &repos, entry) { 1143 if (strcmp(rp->repouri, repouri) != 0) 1144 continue; 1145 if (rp->notifyuri != NULL) { 1146 if (notify == NULL) 1147 continue; 1148 if (strcmp(rp->notifyuri, notify) != 0) 1149 continue; 1150 } else if (notify != NULL) 1151 continue; 1152 /* found matching repo */ 1153 free(repouri); 1154 return rp; 1155 } 1156 1157 rp = repo_alloc(talid); 1158 rp->basedir = repo_dir(repouri, NULL, 0); 1159 rp->repouri = repouri; 1160 if (notify != NULL) 1161 if ((rp->notifyuri = strdup(notify)) == NULL) 1162 err(1, NULL); 1163 1164 if (++talrepocnt[talid] >= MAX_REPO_PER_TAL) { 1165 if (talrepocnt[talid] == MAX_REPO_PER_TAL) 1166 warnx("too many repositories under %s", tals[talid]); 1167 nofetch = 1; 1168 } 1169 1170 /* check if sync disabled ... */ 1171 if (noop || nofetch) { 1172 logx("%s: using cache", rp->basedir); 1173 entityq_flush(&rp->queue, rp); 1174 return rp; 1175 } 1176 1177 /* try to create base directory */ 1178 if (mkpath(rp->basedir) == -1) 1179 warn("mkpath %s", rp->basedir); 1180 1181 /* ... else try RRDP first if available then rsync */ 1182 if (notify != NULL) 1183 rp->rrdp = rrdp_get(notify); 1184 if (rp->rrdp == NULL) 1185 rp->rsync = rsync_get(uri, rp->basedir); 1186 1187 /* need to check if it was already loaded */ 1188 if (repo_state(rp) != REPO_LOADING) 1189 entityq_flush(&rp->queue, rp); 1190 1191 return rp; 1192 } 1193 1194 /* 1195 * Find repository by identifier. 1196 */ 1197 struct repo * 1198 repo_byid(unsigned int id) 1199 { 1200 struct repo *rp; 1201 1202 SLIST_FOREACH(rp, &repos, entry) { 1203 if (rp->id == id) 1204 return rp; 1205 } 1206 return NULL; 1207 } 1208 1209 /* 1210 * Find repository by base path. 1211 */ 1212 static struct repo * 1213 repo_bypath(const char *path) 1214 { 1215 struct repo *rp; 1216 1217 SLIST_FOREACH(rp, &repos, entry) { 1218 if (strcmp(rp->basedir, path) == 0) 1219 return rp; 1220 } 1221 return NULL; 1222 } 1223 1224 /* 1225 * Return the repository base or alternate directory. 1226 * Returned string must be freed by caller. 1227 */ 1228 char * 1229 repo_basedir(const struct repo *rp, int wantvalid) 1230 { 1231 char *path = NULL; 1232 1233 if (!wantvalid) { 1234 if (rp->ta) { 1235 if ((path = strdup(rp->ta->basedir)) == NULL) 1236 err(1, NULL); 1237 } else if (rp->rsync) { 1238 if ((path = strdup(rp->rsync->basedir)) == NULL) 1239 err(1, NULL); 1240 } else if (rp->rrdp) { 1241 path = rrdp_filename(rp->rrdp, rp->repouri, 0); 1242 } else 1243 path = NULL; /* only valid repo available */ 1244 } else if (rp->basedir != NULL) { 1245 if ((path = strdup(rp->basedir)) == NULL) 1246 err(1, NULL); 1247 } 1248 1249 return path; 1250 } 1251 1252 /* 1253 * Return the repository identifier. 1254 */ 1255 unsigned int 1256 repo_id(const struct repo *rp) 1257 { 1258 return rp->id; 1259 } 1260 1261 /* 1262 * Return the repository URI. 1263 */ 1264 const char * 1265 repo_uri(const struct repo *rp) 1266 { 1267 return rp->repouri; 1268 } 1269 1270 /* 1271 * Return the repository URI. 1272 */ 1273 void 1274 repo_fetch_uris(const struct repo *rp, const char **carepo, 1275 const char **notifyuri) 1276 { 1277 *carepo = rp->repouri; 1278 *notifyuri = rp->notifyuri; 1279 } 1280 1281 /* 1282 * Return 1 if repository is synced else 0. 1283 */ 1284 int 1285 repo_synced(const struct repo *rp) 1286 { 1287 if (repo_state(rp) == REPO_DONE && 1288 !(rp->rrdp == NULL && rp->rsync == NULL && rp->ta == NULL)) 1289 return 1; 1290 return 0; 1291 } 1292 1293 /* 1294 * Return the protocol string "rrdp", "rsync", "https" which was used to sync. 1295 * Result is only correct if repository was properly synced. 1296 */ 1297 const char * 1298 repo_proto(const struct repo *rp) 1299 { 1300 1301 if (rp->ta != NULL) { 1302 const struct tarepo *tr = rp->ta; 1303 if (tr->uriidx < tr->urisz && 1304 strncasecmp(tr->uri[tr->uriidx], "rsync://", 8) == 0) 1305 return "rsync"; 1306 else 1307 return "https"; 1308 } 1309 if (rp->rrdp != NULL) 1310 return "rrdp"; 1311 return "rsync"; 1312 } 1313 1314 /* 1315 * Return the repository tal ID. 1316 */ 1317 int 1318 repo_talid(const struct repo *rp) 1319 { 1320 return rp->talid; 1321 } 1322 1323 int 1324 repo_queued(struct repo *rp, struct entity *p) 1325 { 1326 if (repo_state(rp) == REPO_LOADING) { 1327 TAILQ_INSERT_TAIL(&rp->queue, p, entries); 1328 return 1; 1329 } 1330 return 0; 1331 } 1332 1333 static void 1334 repo_fail(struct repo *rp) 1335 { 1336 /* reset the alarm since code may fallback to rsync */ 1337 rp->alarm = getmonotime() + repo_timeout; 1338 1339 if (rp->ta) 1340 http_finish(rp->ta->id, HTTP_FAILED, NULL); 1341 else if (rp->rsync) 1342 rsync_finish(rp->rsync->id, 0); 1343 else if (rp->rrdp) 1344 rrdp_finish(rp->rrdp->id, 0); 1345 else 1346 errx(1, "%s: bad repo", rp->repouri); 1347 } 1348 1349 static void 1350 repo_abort(struct repo *rp) 1351 { 1352 /* reset the alarm */ 1353 rp->alarm = getmonotime() + repo_timeout; 1354 1355 if (rp->rsync) 1356 rsync_abort(rp->rsync->id); 1357 else if (rp->rrdp) 1358 rrdp_abort(rp->rrdp->id); 1359 else 1360 repo_fail(rp); 1361 } 1362 1363 int 1364 repo_check_timeout(int timeout) 1365 { 1366 struct repo *rp; 1367 time_t now; 1368 int diff; 1369 1370 now = getmonotime(); 1371 1372 /* check against our runtime deadline first */ 1373 if (deadline != 0) { 1374 if (deadline <= now) { 1375 warnx("deadline reached, giving up on repository sync"); 1376 nofetch = 1; 1377 /* clear deadline since nofetch is set */ 1378 deadline = 0; 1379 /* increase now enough so that all pending repos fail */ 1380 now += repo_timeout; 1381 } else { 1382 diff = deadline - now; 1383 diff *= 1000; 1384 if (timeout == INFTIM || diff < timeout) 1385 timeout = diff; 1386 } 1387 } 1388 /* Look up in repository table. (Lookup should actually fail here) */ 1389 SLIST_FOREACH(rp, &repos, entry) { 1390 if (repo_state(rp) == REPO_LOADING) { 1391 if (rp->alarm <= now) { 1392 warnx("%s: synchronisation timeout", 1393 rp->repouri); 1394 repo_abort(rp); 1395 } else { 1396 diff = rp->alarm - now; 1397 diff *= 1000; 1398 if (timeout == INFTIM || diff < timeout) 1399 timeout = diff; 1400 } 1401 } 1402 } 1403 return timeout; 1404 } 1405 1406 /* 1407 * Update stats object of repository depending on rtype and subtype. 1408 */ 1409 void 1410 repo_stat_inc(struct repo *rp, int talid, enum rtype type, enum stype subtype) 1411 { 1412 if (rp == NULL) 1413 return; 1414 rp->stats_used[talid] = 1; 1415 switch (type) { 1416 case RTYPE_CER: 1417 if (subtype == STYPE_OK) 1418 rp->stats[talid].certs++; 1419 if (subtype == STYPE_FAIL) 1420 rp->stats[talid].certs_fail++; 1421 if (subtype == STYPE_BGPSEC) { 1422 rp->stats[talid].certs--; 1423 rp->stats[talid].brks++; 1424 } 1425 break; 1426 case RTYPE_MFT: 1427 if (subtype == STYPE_OK) 1428 rp->stats[talid].mfts++; 1429 if (subtype == STYPE_FAIL) 1430 rp->stats[talid].mfts_fail++; 1431 if (subtype == STYPE_STALE) 1432 rp->stats[talid].mfts_stale++; 1433 break; 1434 case RTYPE_ROA: 1435 switch (subtype) { 1436 case STYPE_OK: 1437 rp->stats[talid].roas++; 1438 break; 1439 case STYPE_FAIL: 1440 rp->stats[talid].roas_fail++; 1441 break; 1442 case STYPE_INVALID: 1443 rp->stats[talid].roas_invalid++; 1444 break; 1445 case STYPE_TOTAL: 1446 rp->stats[talid].vrps++; 1447 break; 1448 case STYPE_UNIQUE: 1449 rp->stats[talid].vrps_uniqs++; 1450 break; 1451 case STYPE_DEC_UNIQUE: 1452 rp->stats[talid].vrps_uniqs--; 1453 break; 1454 default: 1455 break; 1456 } 1457 break; 1458 case RTYPE_ASPA: 1459 switch (subtype) { 1460 case STYPE_OK: 1461 rp->stats[talid].aspas++; 1462 break; 1463 case STYPE_FAIL: 1464 rp->stats[talid].aspas_fail++; 1465 break; 1466 case STYPE_INVALID: 1467 rp->stats[talid].aspas_invalid++; 1468 break; 1469 case STYPE_TOTAL: 1470 rp->stats[talid].vaps++; 1471 break; 1472 case STYPE_UNIQUE: 1473 rp->stats[talid].vaps_uniqs++; 1474 break; 1475 case STYPE_DEC_UNIQUE: 1476 rp->stats[talid].vaps_uniqs--; 1477 break; 1478 case STYPE_PROVIDERS: 1479 rp->stats[talid].vaps_pas++; 1480 break; 1481 default: 1482 break; 1483 } 1484 break; 1485 case RTYPE_CRL: 1486 rp->stats[talid].crls++; 1487 break; 1488 case RTYPE_GBR: 1489 rp->stats[talid].gbrs++; 1490 break; 1491 case RTYPE_TAK: 1492 rp->stats[talid].taks++; 1493 break; 1494 default: 1495 break; 1496 } 1497 } 1498 1499 void 1500 repo_tal_stats_collect(void (*cb)(const struct repo *, 1501 const struct repotalstats *, void *), int talid, void *arg) 1502 { 1503 struct repo *rp; 1504 1505 SLIST_FOREACH(rp, &repos, entry) { 1506 if (rp->stats_used[talid]) 1507 cb(rp, &rp->stats[talid], arg); 1508 } 1509 } 1510 1511 void 1512 repo_stats_collect(void (*cb)(const struct repo *, const struct repostats *, 1513 void *), void *arg) 1514 { 1515 struct repo *rp; 1516 1517 SLIST_FOREACH(rp, &repos, entry) 1518 cb(rp, &rp->repostats, arg); 1519 } 1520 1521 /* 1522 * Delayed delete of files from RRDP. Since RRDP has no security built-in 1523 * this code needs to check if this RRDP repository is actually allowed to 1524 * remove the file referenced by the URI. 1525 */ 1526 static void 1527 repo_cleanup_rrdp(struct filepath_tree *tree) 1528 { 1529 struct repo *rp; 1530 struct rrdprepo *rr; 1531 struct filepath *fp, *nfp; 1532 char *fn; 1533 1534 SLIST_FOREACH(rp, &repos, entry) { 1535 if (rp->rrdp == NULL) 1536 continue; 1537 rr = (struct rrdprepo *)rp->rrdp; 1538 RB_FOREACH_SAFE(fp, filepath_tree, &rr->deleted, nfp) { 1539 if (!rrdp_uri_valid(rr, fp->file)) { 1540 warnx("%s: external URI %s", rr->notifyuri, 1541 fp->file); 1542 filepath_put(&rr->deleted, fp); 1543 continue; 1544 } 1545 /* try to remove file from rrdp repo ... */ 1546 fn = rrdp_filename(rr, fp->file, 0); 1547 1548 if (unlink(fn) == -1) { 1549 if (errno != ENOENT) 1550 warn("unlink %s", fn); 1551 } else { 1552 if (verbose > 1) 1553 logx("deleted %s", fn); 1554 rp->repostats.del_files++; 1555 } 1556 free(fn); 1557 1558 /* ... and from the valid repository if unused. */ 1559 fn = rrdp_filename(rr, fp->file, 1); 1560 if (!filepath_exists(tree, fn)) { 1561 if (unlink(fn) == -1) { 1562 if (errno != ENOENT) 1563 warn("unlink %s", fn); 1564 } else { 1565 if (verbose > 1) 1566 logx("deleted %s", fn); 1567 rp->repostats.del_files++; 1568 } 1569 } else 1570 warnx("%s: referenced file supposed to be " 1571 "deleted", fn); 1572 1573 free(fn); 1574 filepath_put(&rr->deleted, fp); 1575 } 1576 } 1577 } 1578 1579 /* 1580 * All files in tree are valid and should be moved to the valid repository 1581 * if not already there. Rename the files to the new path and readd the 1582 * filepath entry with the new path if successful. 1583 */ 1584 static void 1585 repo_move_valid(struct filepath_tree *tree) 1586 { 1587 struct filepath *fp, *nfp; 1588 size_t rsyncsz = strlen(".rsync/"); 1589 size_t rrdpsz = strlen(".rrdp/"); 1590 char *fn, *base; 1591 1592 RB_FOREACH_SAFE(fp, filepath_tree, tree, nfp) { 1593 if (strncmp(fp->file, ".rsync/", rsyncsz) != 0 && 1594 strncmp(fp->file, ".rrdp/", rrdpsz) != 0) 1595 continue; /* not a temporary file path */ 1596 1597 if (strncmp(fp->file, ".rsync/", rsyncsz) == 0) { 1598 fn = fp->file + rsyncsz; 1599 } else { 1600 base = strchr(fp->file + rrdpsz, '/'); 1601 assert(base != NULL); 1602 fn = base + 1; 1603 1604 /* 1605 * Adjust file last modification time in order to 1606 * minimize RSYNC synchronization load after transport 1607 * failover. 1608 * While serializing RRDP datastructures to disk, set 1609 * the last modified timestamp to the CMS signing-time, 1610 * the X.509 notBefore, or CRL lastUpdate timestamp. 1611 */ 1612 if (fp->mtime != 0) { 1613 int ret; 1614 struct timespec ts[2]; 1615 1616 ts[0].tv_nsec = UTIME_OMIT; 1617 ts[1].tv_sec = fp->mtime; 1618 ts[1].tv_nsec = 0; 1619 ret = utimensat(AT_FDCWD, fp->file, ts, 0); 1620 if (ret == -1) { 1621 warn("utimensat %s", fp->file); 1622 continue; 1623 } 1624 } 1625 } 1626 1627 if (repo_mkpath(AT_FDCWD, fn) == -1) 1628 continue; 1629 1630 if (rename(fp->file, fn) == -1) { 1631 warn("rename %s", fp->file); 1632 continue; 1633 } 1634 1635 /* switch filepath node to new path */ 1636 RB_REMOVE(filepath_tree, tree, fp); 1637 base = fp->file; 1638 if ((fp->file = strdup(fn)) == NULL) 1639 err(1, NULL); 1640 free(base); 1641 if (RB_INSERT(filepath_tree, tree, fp) != NULL) 1642 errx(1, "%s: both possibilities of file present", 1643 fp->file); 1644 } 1645 } 1646 1647 struct fts_state { 1648 enum { BASE_DIR, RSYNC_DIR, RRDP_DIR } type; 1649 struct repo *rp; 1650 } fts_state; 1651 1652 static const struct rrdprepo * 1653 repo_is_rrdp(struct repo *rp) 1654 { 1655 /* check for special pointers first these are not a repository */ 1656 if (rp != NULL && rp->rrdp != NULL) 1657 return rp->rrdp->state == REPO_DONE ? rp->rrdp : NULL; 1658 return NULL; 1659 } 1660 1661 static inline char * 1662 skip_dotslash(char *in) 1663 { 1664 if (memcmp(in, "./", 2) == 0) 1665 return in + 2; 1666 return in; 1667 } 1668 1669 static void 1670 repo_cleanup_entry(FTSENT *e, struct filepath_tree *tree, int cachefd) 1671 { 1672 const struct rrdprepo *rr; 1673 char *path; 1674 1675 path = skip_dotslash(e->fts_path); 1676 switch (e->fts_info) { 1677 case FTS_NSOK: 1678 if (filepath_exists(tree, path)) { 1679 e->fts_parent->fts_number++; 1680 break; 1681 } 1682 if (fts_state.type == RRDP_DIR && fts_state.rp != NULL) { 1683 e->fts_parent->fts_number++; 1684 /* handle rrdp .state files explicitly */ 1685 if (e->fts_level == 3 && 1686 strcmp(e->fts_name, ".state") == 0) 1687 break; 1688 /* can't delete these extra files */ 1689 fts_state.rp->repostats.extra_files++; 1690 if (verbose > 1) 1691 logx("superfluous %s", path); 1692 break; 1693 } 1694 rr = repo_is_rrdp(fts_state.rp); 1695 if (rr != NULL) { 1696 struct stat st; 1697 char *fn; 1698 1699 if (asprintf(&fn, "%s/%s", rr->basedir, path) == -1) 1700 err(1, NULL); 1701 1702 /* 1703 * If the file exists in the rrdp dir 1704 * that file is newer and needs to be kept 1705 * so unlink this file instead of moving 1706 * it over the file in the rrdp dir. 1707 */ 1708 if (fstatat(cachefd, fn, &st, 0) == 0 && 1709 S_ISREG(st.st_mode)) { 1710 free(fn); 1711 goto unlink; 1712 } 1713 if (repo_mkpath(cachefd, fn) == 0) { 1714 if (renameat(AT_FDCWD, e->fts_accpath, 1715 cachefd, fn) == -1) 1716 warn("rename %s to %s", path, fn); 1717 else if (verbose > 1) 1718 logx("moved %s", path); 1719 fts_state.rp->repostats.extra_files++; 1720 } 1721 free(fn); 1722 } else { 1723 unlink: 1724 if (unlink(e->fts_accpath) == -1) { 1725 warn("unlink %s", path); 1726 } else if (fts_state.type == RSYNC_DIR) { 1727 /* no need to keep rsync files */ 1728 if (verbose > 1) 1729 logx("deleted superfluous %s", path); 1730 if (fts_state.rp != NULL) 1731 fts_state.rp->repostats.del_extra_files++; 1732 else 1733 stats.repo_stats.del_extra_files++; 1734 } else { 1735 if (verbose > 1) 1736 logx("deleted %s", path); 1737 if (fts_state.rp != NULL) 1738 fts_state.rp->repostats.del_files++; 1739 else 1740 stats.repo_stats.del_files++; 1741 } 1742 } 1743 break; 1744 case FTS_D: 1745 if (e->fts_level == FTS_ROOTLEVEL) 1746 fts_state.type = BASE_DIR; 1747 if (e->fts_level == 1) { 1748 /* rpki.example.org or .rrdp / .rsync */ 1749 if (strcmp(".rsync", e->fts_name) == 0) { 1750 fts_state.type = RSYNC_DIR; 1751 fts_state.rp = NULL; 1752 } else if (strcmp(".rrdp", e->fts_name) == 0) { 1753 fts_state.type = RRDP_DIR; 1754 fts_state.rp = NULL; 1755 } 1756 } 1757 if (e->fts_level == 2) { 1758 /* rpki.example.org/repository or .rrdp/hashdir */ 1759 if (fts_state.type == BASE_DIR) 1760 fts_state.rp = repo_bypath(path); 1761 /* 1762 * special handling for rrdp directories, 1763 * clear them if they are not used anymore but 1764 * only if rrdp is active. 1765 * Look them up just using the hash. 1766 */ 1767 if (fts_state.type == RRDP_DIR) 1768 fts_state.rp = repo_rrdp_bypath(path); 1769 } 1770 if (e->fts_level == 3 && fts_state.type == RSYNC_DIR) { 1771 /* .rsync/rpki.example.org/repository */ 1772 fts_state.rp = repo_bypath(path + strlen(".rsync/")); 1773 } 1774 break; 1775 case FTS_DP: 1776 if (e->fts_level == FTS_ROOTLEVEL) 1777 break; 1778 if (e->fts_level == 1) { 1779 /* do not remove .rsync and .rrdp */ 1780 fts_state.rp = NULL; 1781 if (fts_state.type == RRDP_DIR || 1782 fts_state.type == RSYNC_DIR) 1783 break; 1784 } 1785 1786 e->fts_parent->fts_number += e->fts_number; 1787 1788 if (e->fts_number == 0) { 1789 if (rmdir(e->fts_accpath) == -1) 1790 warn("rmdir %s", path); 1791 if (fts_state.rp != NULL) 1792 fts_state.rp->repostats.del_dirs++; 1793 else 1794 stats.repo_stats.del_dirs++; 1795 } 1796 break; 1797 case FTS_SL: 1798 case FTS_SLNONE: 1799 warnx("symlink %s", path); 1800 if (unlink(e->fts_accpath) == -1) 1801 warn("unlink %s", path); 1802 stats.repo_stats.del_extra_files++; 1803 break; 1804 case FTS_NS: 1805 case FTS_ERR: 1806 if (e->fts_errno == ENOENT && e->fts_level == FTS_ROOTLEVEL) 1807 break; 1808 warnx("fts_read %s: %s", path, strerror(e->fts_errno)); 1809 break; 1810 default: 1811 warnx("fts_read %s: unhandled[%x]", path, e->fts_info); 1812 break; 1813 } 1814 } 1815 1816 void 1817 repo_cleanup(struct filepath_tree *tree, int cachefd) 1818 { 1819 char *argv[2] = { ".", NULL }; 1820 FTS *fts; 1821 FTSENT *e; 1822 1823 /* first move temp files which have been used to valid dir */ 1824 repo_move_valid(tree); 1825 /* then delete files requested by rrdp */ 1826 repo_cleanup_rrdp(tree); 1827 1828 if ((fts = fts_open(argv, FTS_PHYSICAL | FTS_NOSTAT, NULL)) == NULL) 1829 err(1, "fts_open"); 1830 errno = 0; 1831 while ((e = fts_read(fts)) != NULL) { 1832 repo_cleanup_entry(e, tree, cachefd); 1833 errno = 0; 1834 } 1835 if (errno) 1836 err(1, "fts_read"); 1837 if (fts_close(fts) == -1) 1838 err(1, "fts_close"); 1839 } 1840 1841 void 1842 repo_free(void) 1843 { 1844 struct repo *rp; 1845 1846 while ((rp = SLIST_FIRST(&repos)) != NULL) { 1847 SLIST_REMOVE_HEAD(&repos, entry); 1848 free(rp->repouri); 1849 free(rp->notifyuri); 1850 free(rp->basedir); 1851 free(rp); 1852 } 1853 1854 ta_free(); 1855 rrdp_free(); 1856 rsync_free(); 1857 } 1858 1859 /* 1860 * Remove all files and directories under base. 1861 * Do not remove base directory itself and the .state file. 1862 */ 1863 static void 1864 remove_contents(char *base) 1865 { 1866 char *argv[2] = { base, NULL }; 1867 FTS *fts; 1868 FTSENT *e; 1869 1870 if ((fts = fts_open(argv, FTS_PHYSICAL | FTS_NOSTAT, NULL)) == NULL) 1871 err(1, "fts_open"); 1872 errno = 0; 1873 while ((e = fts_read(fts)) != NULL) { 1874 switch (e->fts_info) { 1875 case FTS_NSOK: 1876 case FTS_SL: 1877 case FTS_SLNONE: 1878 if (e->fts_level == 1 && 1879 strcmp(e->fts_name, ".state") == 0) 1880 break; 1881 if (unlink(e->fts_accpath) == -1) 1882 warn("unlink %s", e->fts_path); 1883 break; 1884 case FTS_D: 1885 break; 1886 case FTS_DP: 1887 /* keep root directory */ 1888 if (e->fts_level == FTS_ROOTLEVEL) 1889 break; 1890 if (rmdir(e->fts_accpath) == -1) 1891 warn("rmdir %s", e->fts_path); 1892 break; 1893 case FTS_NS: 1894 case FTS_ERR: 1895 warnx("fts_read %s: %s", e->fts_path, 1896 strerror(e->fts_errno)); 1897 break; 1898 default: 1899 warnx("unhandled[%x] %s", e->fts_info, 1900 e->fts_path); 1901 break; 1902 } 1903 errno = 0; 1904 } 1905 if (errno) 1906 err(1, "fts_read"); 1907 if (fts_close(fts) == -1) 1908 err(1, "fts_close"); 1909 } 1910