1 /* $OpenBSD: repo.c,v 1.15 2021/12/07 12:46:47 claudio Exp $ */ 2 /* 3 * Copyright (c) 2021 Claudio Jeker <claudio@openbsd.org> 4 * Copyright (c) 2019 Kristaps Dzonsons <kristaps@bsd.lv> 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 #include <sys/queue.h> 20 #include <sys/tree.h> 21 #include <sys/types.h> 22 #include <sys/stat.h> 23 24 #include <assert.h> 25 #include <err.h> 26 #include <errno.h> 27 #include <fcntl.h> 28 #include <fts.h> 29 #include <limits.h> 30 #include <poll.h> 31 #include <stdio.h> 32 #include <stdlib.h> 33 #include <string.h> 34 #include <unistd.h> 35 36 #include <imsg.h> 37 38 #include "extern.h" 39 40 extern struct stats stats; 41 extern int noop; 42 extern int rrdpon; 43 extern int repo_timeout; 44 45 enum repo_state { 46 REPO_LOADING = 0, 47 REPO_DONE = 1, 48 REPO_FAILED = -1, 49 }; 50 51 /* 52 * A ta, rsync or rrdp repository. 53 * Depending on what is needed the generic repository is backed by 54 * a ta, rsync or rrdp repository. Multiple repositories can use the 55 * same backend. 56 */ 57 struct rrdprepo { 58 SLIST_ENTRY(rrdprepo) entry; 59 char *notifyuri; 60 char *basedir; 61 char *temp; 62 struct filepath_tree added; 63 struct filepath_tree deleted; 64 size_t id; 65 enum repo_state state; 66 }; 67 SLIST_HEAD(, rrdprepo) rrdprepos = SLIST_HEAD_INITIALIZER(rrdprepos); 68 69 struct rsyncrepo { 70 SLIST_ENTRY(rsyncrepo) entry; 71 char *repouri; 72 char *basedir; 73 size_t id; 74 enum repo_state state; 75 }; 76 SLIST_HEAD(, rsyncrepo) rsyncrepos = SLIST_HEAD_INITIALIZER(rsyncrepos); 77 78 struct tarepo { 79 SLIST_ENTRY(tarepo) entry; 80 char *descr; 81 char *basedir; 82 char *temp; 83 char **uri; 84 size_t urisz; 85 size_t uriidx; 86 size_t id; 87 enum repo_state state; 88 }; 89 SLIST_HEAD(, tarepo) tarepos = SLIST_HEAD_INITIALIZER(tarepos); 90 91 struct repo { 92 SLIST_ENTRY(repo) entry; 93 char *repouri; 94 char *notifyuri; 95 const struct rrdprepo *rrdp; 96 const struct rsyncrepo *rsync; 97 const struct tarepo *ta; 98 struct entityq queue; /* files waiting for repo */ 99 time_t alarm; /* sync timeout */ 100 int talid; 101 size_t id; /* identifier */ 102 }; 103 SLIST_HEAD(, repo) repos = SLIST_HEAD_INITIALIZER(repos); 104 105 /* counter for unique repo id */ 106 size_t repoid; 107 108 /* 109 * Database of all file path accessed during a run. 110 */ 111 struct filepath { 112 RB_ENTRY(filepath) entry; 113 char *file; 114 }; 115 116 static inline int 117 filepathcmp(struct filepath *a, struct filepath *b) 118 { 119 return strcmp(a->file, b->file); 120 } 121 122 RB_PROTOTYPE(filepath_tree, filepath, entry, filepathcmp); 123 124 /* 125 * Functions to lookup which files have been accessed during computation. 126 */ 127 int 128 filepath_add(struct filepath_tree *tree, char *file) 129 { 130 struct filepath *fp; 131 132 if ((fp = malloc(sizeof(*fp))) == NULL) 133 err(1, NULL); 134 if ((fp->file = strdup(file)) == NULL) 135 err(1, NULL); 136 137 if (RB_INSERT(filepath_tree, tree, fp) != NULL) { 138 /* already in the tree */ 139 free(fp->file); 140 free(fp); 141 return 0; 142 } 143 144 return 1; 145 } 146 147 /* 148 * Lookup a file path in the tree and return the object if found or NULL. 149 */ 150 static struct filepath * 151 filepath_find(struct filepath_tree *tree, char *file) 152 { 153 struct filepath needle; 154 155 needle.file = file; 156 return RB_FIND(filepath_tree, tree, &needle); 157 } 158 159 /* 160 * Returns true if file exists in the tree. 161 */ 162 static int 163 filepath_exists(struct filepath_tree *tree, char *file) 164 { 165 return filepath_find(tree, file) != NULL; 166 } 167 168 /* 169 * Return true if a filepath entry exists that starts with path. 170 */ 171 static int 172 filepath_dir_exists(struct filepath_tree *tree, char *path) 173 { 174 struct filepath needle; 175 struct filepath *res; 176 177 needle.file = path; 178 res = RB_NFIND(filepath_tree, tree, &needle); 179 while (res != NULL && strstr(res->file, path) == res->file) { 180 /* make sure that filepath actually is in that path */ 181 if (res->file[strlen(path)] == '/') 182 return 1; 183 res = RB_NEXT(filepath_tree, tree, res); 184 } 185 return 0; 186 } 187 188 /* 189 * Remove entry from tree and free it. 190 */ 191 static void 192 filepath_put(struct filepath_tree *tree, struct filepath *fp) 193 { 194 RB_REMOVE(filepath_tree, tree, fp); 195 free((void *)fp->file); 196 free(fp); 197 } 198 199 /* 200 * Free all elements of a filepath tree. 201 */ 202 static void 203 filepath_free(struct filepath_tree *tree) 204 { 205 struct filepath *fp, *nfp; 206 207 RB_FOREACH_SAFE(fp, filepath_tree, tree, nfp) 208 filepath_put(tree, fp); 209 } 210 211 RB_GENERATE(filepath_tree, filepath, entry, filepathcmp); 212 213 /* 214 * Function to hash a string into a unique directory name. 215 * Returned hash needs to be freed. 216 */ 217 static char * 218 hash_dir(const char *uri) 219 { 220 unsigned char m[SHA256_DIGEST_LENGTH]; 221 222 SHA256(uri, strlen(uri), m); 223 return hex_encode(m, sizeof(m)); 224 } 225 226 /* 227 * Function to build the directory name based on URI and a directory 228 * as prefix. Skip the proto:// in URI but keep everything else. 229 */ 230 static char * 231 repo_dir(const char *uri, const char *dir, int hash) 232 { 233 const char *local; 234 char *out, *hdir = NULL; 235 236 if (hash) { 237 local = hdir = hash_dir(uri); 238 } else { 239 local = strchr(uri, ':'); 240 if (local != NULL) 241 local += strlen("://"); 242 else 243 local = uri; 244 } 245 246 if (asprintf(&out, "%s/%s", dir, local) == -1) 247 err(1, NULL); 248 free(hdir); 249 return out; 250 } 251 252 /* 253 * Function to create all missing directories to a path. 254 * This functions alters the path temporarily. 255 */ 256 static int 257 repo_mkpath(char *file) 258 { 259 char *slash; 260 261 /* build directory hierarchy */ 262 slash = strrchr(file, '/'); 263 assert(slash != NULL); 264 *slash = '\0'; 265 if (mkpath(file) == -1) { 266 warn("mkpath %s", file); 267 return -1; 268 } 269 *slash = '/'; 270 return 0; 271 } 272 273 /* 274 * Build TA file name based on the repo info. 275 * If temp is set add Xs for mkostemp. 276 */ 277 static char * 278 ta_filename(const struct tarepo *tr, int temp) 279 { 280 const char *file; 281 char *nfile; 282 283 /* does not matter which URI, all end with same filename */ 284 file = strrchr(tr->uri[0], '/'); 285 assert(file); 286 287 if (asprintf(&nfile, "%s%s%s", tr->basedir, file, 288 temp ? ".XXXXXXXX": "") == -1) 289 err(1, NULL); 290 291 return nfile; 292 } 293 294 /* 295 * Build local file name base on the URI and the rrdprepo info. 296 */ 297 static char * 298 rrdp_filename(const struct rrdprepo *rr, const char *uri, int temp) 299 { 300 char *nfile; 301 char *dir = rr->basedir; 302 303 if (temp) 304 dir = rr->temp; 305 306 if (!valid_uri(uri, strlen(uri), "rsync://")) { 307 warnx("%s: bad URI %s", rr->basedir, uri); 308 return NULL; 309 } 310 311 uri += strlen("rsync://"); /* skip proto */ 312 if (asprintf(&nfile, "%s/%s", dir, uri) == -1) 313 err(1, NULL); 314 return nfile; 315 } 316 317 /* 318 * Build RRDP state file name based on the repo info. 319 * If temp is set add Xs for mkostemp. 320 */ 321 static char * 322 rrdp_state_filename(const struct rrdprepo *rr, int temp) 323 { 324 char *nfile; 325 326 if (asprintf(&nfile, "%s/.state%s", rr->basedir, 327 temp ? ".XXXXXXXX": "") == -1) 328 err(1, NULL); 329 330 return nfile; 331 } 332 333 334 335 static void 336 ta_fetch(struct tarepo *tr) 337 { 338 if (!rrdpon) { 339 for (; tr->uriidx < tr->urisz; tr->uriidx++) { 340 if (strncasecmp(tr->uri[tr->uriidx], 341 "rsync://", 8) == 0) 342 break; 343 } 344 } 345 346 if (tr->uriidx >= tr->urisz) { 347 struct repo *rp; 348 349 tr->state = REPO_FAILED; 350 logx("ta/%s: fallback to cache", tr->descr); 351 352 SLIST_FOREACH(rp, &repos, entry) 353 if (rp->ta == tr) 354 entityq_flush(&rp->queue, rp); 355 return; 356 } 357 358 logx("ta/%s: pulling from %s", tr->descr, tr->uri[tr->uriidx]); 359 360 if (strncasecmp(tr->uri[tr->uriidx], "rsync://", 8) == 0) { 361 /* 362 * Create destination location. 363 * Build up the tree to this point. 364 */ 365 rsync_fetch(tr->id, tr->uri[tr->uriidx], tr->basedir); 366 } else { 367 int fd; 368 369 tr->temp = ta_filename(tr, 1); 370 fd = mkostemp(tr->temp, O_CLOEXEC); 371 if (fd == -1) { 372 warn("mkostemp: %s", tr->temp); 373 http_finish(tr->id, HTTP_FAILED, NULL); 374 return; 375 } 376 if (fchmod(fd, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH) == -1) 377 warn("fchmod: %s", tr->temp); 378 379 http_fetch(tr->id, tr->uri[tr->uriidx], NULL, fd); 380 } 381 } 382 383 static struct tarepo * 384 ta_get(struct tal *tal, int nofetch) 385 { 386 struct tarepo *tr; 387 388 /* no need to look for possible other repo */ 389 390 if (tal->urisz == 0) 391 errx(1, "TAL %s has no URI", tal->descr); 392 393 if ((tr = calloc(1, sizeof(*tr))) == NULL) 394 err(1, NULL); 395 tr->id = ++repoid; 396 SLIST_INSERT_HEAD(&tarepos, tr, entry); 397 398 if ((tr->descr = strdup(tal->descr)) == NULL) 399 err(1, NULL); 400 tr->basedir = repo_dir(tal->descr, "ta", 0); 401 402 /* steal URI infromation from TAL */ 403 tr->urisz = tal->urisz; 404 tr->uri = tal->uri; 405 tal->urisz = 0; 406 tal->uri = NULL; 407 408 if (noop || nofetch) { 409 tr->state = REPO_DONE; 410 logx("ta/%s: using cache", tr->descr); 411 /* there is nothing in the queue so no need to flush */ 412 } else { 413 /* try to create base directory */ 414 if (mkpath(tr->basedir) == -1) 415 warn("mkpath %s", tr->basedir); 416 417 ta_fetch(tr); 418 } 419 420 return tr; 421 } 422 423 static struct tarepo * 424 ta_find(size_t id) 425 { 426 struct tarepo *tr; 427 428 SLIST_FOREACH(tr, &tarepos, entry) 429 if (id == tr->id) 430 break; 431 return tr; 432 } 433 434 static void 435 ta_free(void) 436 { 437 struct tarepo *tr; 438 439 while ((tr = SLIST_FIRST(&tarepos)) != NULL) { 440 SLIST_REMOVE_HEAD(&tarepos, entry); 441 free(tr->descr); 442 free(tr->basedir); 443 free(tr->temp); 444 free(tr->uri); 445 free(tr); 446 } 447 } 448 449 static struct rsyncrepo * 450 rsync_get(const char *uri, int nofetch) 451 { 452 struct rsyncrepo *rr; 453 char *repo; 454 455 if ((repo = rsync_base_uri(uri)) == NULL) 456 errx(1, "bad caRepository URI: %s", uri); 457 458 SLIST_FOREACH(rr, &rsyncrepos, entry) 459 if (strcmp(rr->repouri, repo) == 0) { 460 free(repo); 461 return rr; 462 } 463 464 if ((rr = calloc(1, sizeof(*rr))) == NULL) 465 err(1, NULL); 466 467 rr->id = ++repoid; 468 SLIST_INSERT_HEAD(&rsyncrepos, rr, entry); 469 470 rr->repouri = repo; 471 rr->basedir = repo_dir(repo, "rsync", 0); 472 473 if (noop || nofetch) { 474 rr->state = REPO_DONE; 475 logx("%s: using cache", rr->basedir); 476 /* there is nothing in the queue so no need to flush */ 477 } else { 478 /* create base directory */ 479 if (mkpath(rr->basedir) == -1) { 480 warn("mkpath %s", rr->basedir); 481 rsync_finish(rr->id, 0); 482 return rr; 483 } 484 485 logx("%s: pulling from %s", rr->basedir, rr->repouri); 486 rsync_fetch(rr->id, rr->repouri, rr->basedir); 487 } 488 489 return rr; 490 } 491 492 static struct rsyncrepo * 493 rsync_find(size_t id) 494 { 495 struct rsyncrepo *rr; 496 497 SLIST_FOREACH(rr, &rsyncrepos, entry) 498 if (id == rr->id) 499 break; 500 return rr; 501 } 502 503 static void 504 rsync_free(void) 505 { 506 struct rsyncrepo *rr; 507 508 while ((rr = SLIST_FIRST(&rsyncrepos)) != NULL) { 509 SLIST_REMOVE_HEAD(&rsyncrepos, entry); 510 free(rr->repouri); 511 free(rr->basedir); 512 free(rr); 513 } 514 } 515 516 static int rrdprepo_fetch(struct rrdprepo *); 517 518 static struct rrdprepo * 519 rrdp_get(const char *uri, int nofetch) 520 { 521 struct rrdprepo *rr; 522 523 SLIST_FOREACH(rr, &rrdprepos, entry) 524 if (strcmp(rr->notifyuri, uri) == 0) { 525 if (rr->state == REPO_FAILED) 526 return NULL; 527 return rr; 528 } 529 530 if ((rr = calloc(1, sizeof(*rr))) == NULL) 531 err(1, NULL); 532 533 rr->id = ++repoid; 534 SLIST_INSERT_HEAD(&rrdprepos, rr, entry); 535 536 if ((rr->notifyuri = strdup(uri)) == NULL) 537 err(1, NULL); 538 rr->basedir = repo_dir(uri, "rrdp", 1); 539 540 RB_INIT(&rr->added); 541 RB_INIT(&rr->deleted); 542 543 if (noop || nofetch) { 544 rr->state = REPO_DONE; 545 logx("%s: using cache", rr->notifyuri); 546 /* there is nothing in the queue so no need to flush */ 547 } else { 548 /* create base directory */ 549 if (mkpath(rr->basedir) == -1) { 550 warn("mkpath %s", rr->basedir); 551 rrdp_finish(rr->id, 0); 552 return rr; 553 } 554 if (rrdprepo_fetch(rr) == -1) { 555 rrdp_finish(rr->id, 0); 556 return rr; 557 } 558 559 logx("%s: pulling from %s", rr->notifyuri, "network"); 560 } 561 562 return rr; 563 } 564 565 static struct rrdprepo * 566 rrdp_find(size_t id) 567 { 568 struct rrdprepo *rr; 569 570 SLIST_FOREACH(rr, &rrdprepos, entry) 571 if (id == rr->id) 572 break; 573 return rr; 574 } 575 576 static void 577 rrdp_free(void) 578 { 579 struct rrdprepo *rr; 580 581 while ((rr = SLIST_FIRST(&rrdprepos)) != NULL) { 582 SLIST_REMOVE_HEAD(&rrdprepos, entry); 583 584 free(rr->notifyuri); 585 free(rr->basedir); 586 free(rr->temp); 587 588 filepath_free(&rr->added); 589 filepath_free(&rr->deleted); 590 591 free(rr); 592 } 593 } 594 595 static struct rrdprepo * 596 rrdp_basedir(const char *dir) 597 { 598 struct rrdprepo *rr; 599 600 SLIST_FOREACH(rr, &rrdprepos, entry) 601 if (strcmp(dir, rr->basedir) == 0) { 602 if (rr->state == REPO_FAILED) 603 return NULL; 604 return rr; 605 } 606 607 return NULL; 608 } 609 610 /* 611 * Allocate and insert a new repository. 612 */ 613 static struct repo * 614 repo_alloc(int talid) 615 { 616 struct repo *rp; 617 618 if ((rp = calloc(1, sizeof(*rp))) == NULL) 619 err(1, NULL); 620 621 rp->id = ++repoid; 622 rp->talid = talid; 623 rp->alarm = getmonotime() + repo_timeout; 624 TAILQ_INIT(&rp->queue); 625 SLIST_INSERT_HEAD(&repos, rp, entry); 626 627 stats.repos++; 628 return rp; 629 } 630 631 /* 632 * Return the state of a repository. 633 */ 634 static enum repo_state 635 repo_state(struct repo *rp) 636 { 637 if (rp->ta) 638 return rp->ta->state; 639 if (rp->rrdp) 640 return rp->rrdp->state; 641 if (rp->rsync) 642 return rp->rsync->state; 643 errx(1, "%s: bad repo", rp->repouri); 644 } 645 646 /* 647 * Parse the RRDP state file if it exists and set the session struct 648 * based on that information. 649 */ 650 static void 651 rrdp_parse_state(const struct rrdprepo *rr, struct rrdp_session *state) 652 { 653 FILE *f; 654 int fd, ln = 0; 655 const char *errstr; 656 char *line = NULL, *file; 657 size_t len = 0; 658 ssize_t n; 659 660 file = rrdp_state_filename(rr, 0); 661 if ((fd = open(file, O_RDONLY)) == -1) { 662 if (errno != ENOENT) 663 warn("%s: open state file", rr->basedir); 664 free(file); 665 return; 666 } 667 free(file); 668 f = fdopen(fd, "r"); 669 if (f == NULL) 670 err(1, "fdopen"); 671 672 while ((n = getline(&line, &len, f)) != -1) { 673 if (line[n - 1] == '\n') 674 line[n - 1] = '\0'; 675 switch (ln) { 676 case 0: 677 if ((state->session_id = strdup(line)) == NULL) 678 err(1, NULL); 679 break; 680 case 1: 681 state->serial = strtonum(line, 1, LLONG_MAX, &errstr); 682 if (errstr) 683 goto fail; 684 break; 685 case 2: 686 if ((state->last_mod = strdup(line)) == NULL) 687 err(1, NULL); 688 break; 689 default: 690 goto fail; 691 } 692 ln++; 693 } 694 695 free(line); 696 if (ferror(f)) 697 goto fail; 698 fclose(f); 699 return; 700 701 fail: 702 warnx("%s: troubles reading state file", rr->basedir); 703 fclose(f); 704 free(state->session_id); 705 free(state->last_mod); 706 memset(state, 0, sizeof(*state)); 707 } 708 709 /* 710 * Carefully write the RRDP session state file back. 711 */ 712 void 713 rrdp_save_state(size_t id, struct rrdp_session *state) 714 { 715 struct rrdprepo *rr; 716 char *temp, *file; 717 FILE *f; 718 int fd; 719 720 rr = rrdp_find(id); 721 if (rr == NULL) 722 errx(1, "non-existant rrdp repo %zu", id); 723 724 file = rrdp_state_filename(rr, 0); 725 temp = rrdp_state_filename(rr, 1); 726 727 if ((fd = mkostemp(temp, O_CLOEXEC)) == -1) { 728 warn("mkostemp %s", temp); 729 goto fail; 730 } 731 (void) fchmod(fd, 0644); 732 f = fdopen(fd, "w"); 733 if (f == NULL) 734 err(1, "fdopen"); 735 736 /* write session state file out */ 737 if (fprintf(f, "%s\n%lld\n", state->session_id, 738 state->serial) < 0) { 739 fclose(f); 740 goto fail; 741 } 742 if (state->last_mod != NULL) { 743 if (fprintf(f, "%s\n", state->last_mod) < 0) { 744 fclose(f); 745 goto fail; 746 } 747 } 748 if (fclose(f) != 0) 749 goto fail; 750 751 if (rename(temp, file) == -1) 752 warn("%s: rename state file", rr->basedir); 753 754 free(temp); 755 free(file); 756 return; 757 758 fail: 759 warnx("%s: failed to save state", rr->basedir); 760 unlink(temp); 761 free(temp); 762 free(file); 763 } 764 765 /* 766 * Write a file into the temporary RRDP dir but only after checking 767 * its hash (if required). The function also makes sure that the file 768 * tracking is properly adjusted. 769 * Returns 1 on success, 0 if the repo is corrupt, -1 on IO error 770 */ 771 int 772 rrdp_handle_file(size_t id, enum publish_type pt, char *uri, 773 char *hash, size_t hlen, char *data, size_t dlen) 774 { 775 struct rrdprepo *rr; 776 struct filepath *fp; 777 ssize_t s; 778 char *fn; 779 int fd = -1; 780 781 rr = rrdp_find(id); 782 if (rr == NULL) 783 errx(1, "non-existant rrdp repo %zu", id); 784 if (rr->state == REPO_FAILED) 785 return -1; 786 787 if (pt == PUB_UPD || pt == PUB_DEL) { 788 if (filepath_exists(&rr->deleted, uri)) { 789 warnx("%s: already deleted", uri); 790 return 0; 791 } 792 fp = filepath_find(&rr->added, uri); 793 if (fp == NULL) { 794 if ((fn = rrdp_filename(rr, uri, 0)) == NULL) 795 return 0; 796 } else { 797 filepath_put(&rr->added, fp); 798 if ((fn = rrdp_filename(rr, uri, 1)) == NULL) 799 return 0; 800 } 801 if (!valid_filehash(fn, hash, hlen)) { 802 warnx("%s: bad message digest", fn); 803 free(fn); 804 return 0; 805 } 806 free(fn); 807 } 808 809 if (pt == PUB_DEL) { 810 filepath_add(&rr->deleted, uri); 811 } else { 812 fp = filepath_find(&rr->deleted, uri); 813 if (fp != NULL) 814 filepath_put(&rr->deleted, fp); 815 816 /* add new file to temp dir */ 817 if ((fn = rrdp_filename(rr, uri, 1)) == NULL) 818 return 0; 819 820 if (repo_mkpath(fn) == -1) 821 goto fail; 822 823 fd = open(fn, O_WRONLY|O_CREAT|O_TRUNC, 0644); 824 if (fd == -1) { 825 warn("open %s", fn); 826 goto fail; 827 } 828 829 if ((s = write(fd, data, dlen)) == -1) { 830 warn("write %s", fn); 831 goto fail; 832 } 833 close(fd); 834 if ((size_t)s != dlen) /* impossible */ 835 errx(1, "short write %s", fn); 836 free(fn); 837 filepath_add(&rr->added, uri); 838 } 839 840 return 1; 841 842 fail: 843 rr->state = REPO_FAILED; 844 if (fd != -1) 845 close(fd); 846 free(fn); 847 return -1; 848 } 849 850 /* 851 * Initiate a RRDP sync, create the required temporary directory and 852 * parse a possible state file before sending the request to the RRDP process. 853 */ 854 static int 855 rrdprepo_fetch(struct rrdprepo *rr) 856 { 857 struct rrdp_session state = { 0 }; 858 859 if (asprintf(&rr->temp, "%s.XXXXXXXX", rr->basedir) == -1) 860 err(1, NULL); 861 if (mkdtemp(rr->temp) == NULL) { 862 warn("mkdtemp %s", rr->temp); 863 return -1; 864 } 865 866 rrdp_parse_state(rr, &state); 867 rrdp_fetch(rr->id, rr->notifyuri, rr->notifyuri, &state); 868 869 free(state.session_id); 870 free(state.last_mod); 871 872 return 0; 873 } 874 875 static int 876 rrdp_merge_repo(struct rrdprepo *rr) 877 { 878 struct filepath *fp, *nfp; 879 char *fn, *rfn; 880 881 RB_FOREACH_SAFE(fp, filepath_tree, &rr->added, nfp) { 882 fn = rrdp_filename(rr, fp->file, 1); 883 rfn = rrdp_filename(rr, fp->file, 0); 884 885 if (fn == NULL || rfn == NULL) 886 errx(1, "bad filepath"); /* should not happen */ 887 888 if (repo_mkpath(rfn) == -1) { 889 goto fail; 890 } 891 892 if (rename(fn, rfn) == -1) { 893 warn("rename %s", rfn); 894 goto fail; 895 } 896 897 free(rfn); 898 free(fn); 899 filepath_put(&rr->added, fp); 900 } 901 902 return 1; 903 904 fail: 905 free(rfn); 906 free(fn); 907 return 0; 908 } 909 910 static void 911 rrdp_clean_temp(struct rrdprepo *rr) 912 { 913 struct filepath *fp, *nfp; 914 char *fn; 915 916 filepath_free(&rr->deleted); 917 918 RB_FOREACH_SAFE(fp, filepath_tree, &rr->added, nfp) { 919 if ((fn = rrdp_filename(rr, fp->file, 1)) != NULL) { 920 if (unlink(fn) == -1) 921 warn("unlink %s", fn); 922 free(fn); 923 } 924 filepath_put(&rr->added, fp); 925 } 926 } 927 928 /* 929 * RSYNC sync finished, either with or without success. 930 */ 931 void 932 rsync_finish(size_t id, int ok) 933 { 934 struct rsyncrepo *rr; 935 struct tarepo *tr; 936 struct repo *rp; 937 938 tr = ta_find(id); 939 if (tr != NULL) { 940 /* repository changed state already, ignore request */ 941 if (tr->state != REPO_LOADING) 942 return; 943 if (ok) { 944 logx("ta/%s: loaded from network", tr->descr); 945 stats.rsync_repos++; 946 tr->state = REPO_DONE; 947 } else { 948 logx("ta/%s: load from network failed", tr->descr); 949 stats.rsync_fails++; 950 tr->uriidx++; 951 ta_fetch(tr); 952 return; 953 } 954 SLIST_FOREACH(rp, &repos, entry) 955 if (rp->ta == tr) 956 entityq_flush(&rp->queue, rp); 957 958 return; 959 } 960 961 rr = rsync_find(id); 962 if (rr == NULL) 963 errx(1, "unknown rsync repo %zu", id); 964 965 /* repository changed state already, ignore request */ 966 if (rr->state != REPO_LOADING) 967 return; 968 if (ok) { 969 logx("%s: loaded from network", rr->basedir); 970 stats.rsync_repos++; 971 rr->state = REPO_DONE; 972 } else { 973 logx("%s: load from network failed, fallback to cache", 974 rr->basedir); 975 stats.rsync_fails++; 976 rr->state = REPO_FAILED; 977 } 978 979 SLIST_FOREACH(rp, &repos, entry) 980 if (rp->rsync == rr) 981 entityq_flush(&rp->queue, rp); 982 } 983 984 /* 985 * RRDP sync finshed, either with or without success. 986 */ 987 void 988 rrdp_finish(size_t id, int ok) 989 { 990 struct rrdprepo *rr; 991 struct repo *rp; 992 993 rr = rrdp_find(id); 994 if (rr == NULL) 995 errx(1, "unknown RRDP repo %zu", id); 996 /* repository changed state already, ignore request */ 997 if (rr->state != REPO_LOADING) 998 return; 999 1000 if (ok && rrdp_merge_repo(rr)) { 1001 logx("%s: loaded from network", rr->notifyuri); 1002 rr->state = REPO_DONE; 1003 stats.rrdp_repos++; 1004 SLIST_FOREACH(rp, &repos, entry) 1005 if (rp->rrdp == rr) 1006 entityq_flush(&rp->queue, rp); 1007 } else if (!ok) { 1008 rrdp_clean_temp(rr); 1009 stats.rrdp_fails++; 1010 rr->state = REPO_FAILED; 1011 logx("%s: load from network failed, fallback to rsync", 1012 rr->notifyuri); 1013 SLIST_FOREACH(rp, &repos, entry) 1014 if (rp->rrdp == rr) { 1015 rp->rrdp = NULL; 1016 rp->rsync = rsync_get(rp->repouri, 0); 1017 /* need to check if it was already loaded */ 1018 if (repo_state(rp) != REPO_LOADING) 1019 entityq_flush(&rp->queue, rp); 1020 } 1021 } else { 1022 rrdp_clean_temp(rr); 1023 stats.rrdp_fails++; 1024 rr->state = REPO_FAILED; 1025 logx("%s: load from network failed", rr->notifyuri); 1026 SLIST_FOREACH(rp, &repos, entry) 1027 if (rp->rrdp == rr) 1028 entityq_flush(&rp->queue, rp); 1029 } 1030 } 1031 1032 /* 1033 * Handle responses from the http process. For TA file, either rename 1034 * or delete the temporary file. For RRDP requests relay the request 1035 * over to the rrdp process. 1036 */ 1037 void 1038 http_finish(size_t id, enum http_result res, const char *last_mod) 1039 { 1040 struct tarepo *tr; 1041 struct repo *rp; 1042 1043 tr = ta_find(id); 1044 if (tr == NULL) { 1045 /* not a TA fetch therefor RRDP */ 1046 rrdp_http_done(id, res, last_mod); 1047 return; 1048 } 1049 1050 /* repository changed state already, ignore request */ 1051 if (tr->state != REPO_LOADING) 1052 return; 1053 1054 /* Move downloaded TA file into place, or unlink on failure. */ 1055 if (res == HTTP_OK) { 1056 char *file; 1057 1058 file = ta_filename(tr, 0); 1059 if (rename(tr->temp, file) == -1) 1060 warn("rename to %s", file); 1061 free(file); 1062 1063 logx("ta/%s: loaded from network", tr->descr); 1064 tr->state = REPO_DONE; 1065 stats.http_repos++; 1066 } else { 1067 if (unlink(tr->temp) == -1 && errno != ENOENT) 1068 warn("unlink %s", tr->temp); 1069 1070 tr->uriidx++; 1071 logx("ta/%s: load from network failed", tr->descr); 1072 ta_fetch(tr); 1073 return; 1074 } 1075 1076 SLIST_FOREACH(rp, &repos, entry) 1077 if (rp->ta == tr) 1078 entityq_flush(&rp->queue, rp); 1079 } 1080 1081 1082 1083 /* 1084 * Look up a trust anchor, queueing it for download if not found. 1085 */ 1086 struct repo * 1087 ta_lookup(int id, struct tal *tal) 1088 { 1089 struct repo *rp; 1090 int nofetch = 0; 1091 1092 /* Look up in repository table. (Lookup should actually fail here) */ 1093 SLIST_FOREACH(rp, &repos, entry) { 1094 if (strcmp(rp->repouri, tal->descr) == 0) 1095 return rp; 1096 } 1097 1098 rp = repo_alloc(id); 1099 if ((rp->repouri = strdup(tal->descr)) == NULL) 1100 err(1, NULL); 1101 1102 if (++talrepocnt[id] >= MAX_REPO_PER_TAL) { 1103 if (talrepocnt[id] == MAX_REPO_PER_TAL) 1104 warnx("too many repositories under %s", tals[id]); 1105 nofetch = 1; 1106 } 1107 1108 rp->ta = ta_get(tal, nofetch); 1109 1110 return rp; 1111 } 1112 1113 /* 1114 * Look up a repository, queueing it for discovery if not found. 1115 */ 1116 struct repo * 1117 repo_lookup(int id, const char *uri, const char *notify) 1118 { 1119 struct repo *rp; 1120 char *repouri; 1121 int nofetch = 0; 1122 1123 if ((repouri = rsync_base_uri(uri)) == NULL) 1124 errx(1, "bad caRepository URI: %s", uri); 1125 1126 /* Look up in repository table. */ 1127 SLIST_FOREACH(rp, &repos, entry) { 1128 if (strcmp(rp->repouri, repouri) != 0) 1129 continue; 1130 if (rp->notifyuri != NULL) { 1131 if (notify == NULL) 1132 continue; 1133 if (strcmp(rp->notifyuri, notify) != 0) 1134 continue; 1135 } else if (notify != NULL) 1136 continue; 1137 /* found matching repo */ 1138 free(repouri); 1139 return rp; 1140 } 1141 1142 rp = repo_alloc(id); 1143 rp->repouri = repouri; 1144 if (notify != NULL) 1145 if ((rp->notifyuri = strdup(notify)) == NULL) 1146 err(1, NULL); 1147 1148 if (++talrepocnt[id] >= MAX_REPO_PER_TAL) { 1149 if (talrepocnt[id] == MAX_REPO_PER_TAL) 1150 warnx("too many repositories under %s", tals[id]); 1151 nofetch = 1; 1152 } 1153 1154 /* try RRDP first if available */ 1155 if (notify != NULL) 1156 rp->rrdp = rrdp_get(notify, nofetch); 1157 if (rp->rrdp == NULL) 1158 rp->rsync = rsync_get(uri, nofetch); 1159 1160 return rp; 1161 } 1162 1163 /* 1164 * Build local file name base on the URI and the repo info. 1165 */ 1166 char * 1167 repo_filename(const struct repo *rp, const char *uri) 1168 { 1169 char *nfile; 1170 char *dir, *repouri; 1171 1172 if (uri == NULL && rp->ta) 1173 return ta_filename(rp->ta, 0); 1174 1175 assert(uri != NULL); 1176 if (rp->rrdp) 1177 return rrdp_filename(rp->rrdp, uri, 0); 1178 1179 /* must be rsync */ 1180 dir = rp->rsync->basedir; 1181 repouri = rp->rsync->repouri; 1182 1183 if (strstr(uri, repouri) != uri) { 1184 warnx("%s: URI %s outside of repository", repouri, uri); 1185 return NULL; 1186 } 1187 1188 uri += strlen(repouri) + 1; /* skip base and '/' */ 1189 1190 if (asprintf(&nfile, "%s/%s", dir, uri) == -1) 1191 err(1, NULL); 1192 return nfile; 1193 } 1194 1195 int 1196 repo_queued(struct repo *rp, struct entity *p) 1197 { 1198 if (repo_state(rp) == REPO_LOADING) { 1199 TAILQ_INSERT_TAIL(&rp->queue, p, entries); 1200 return 1; 1201 } 1202 return 0; 1203 } 1204 1205 int 1206 repo_next_timeout(int timeout) 1207 { 1208 struct repo *rp; 1209 time_t now; 1210 1211 now = getmonotime(); 1212 /* Look up in repository table. (Lookup should actually fail here) */ 1213 SLIST_FOREACH(rp, &repos, entry) { 1214 if (repo_state(rp) == REPO_LOADING) { 1215 int diff = rp->alarm - now; 1216 if (diff < 0) 1217 diff = 0; 1218 diff *= 1000; 1219 if (timeout == INFTIM || diff < timeout) 1220 timeout = diff; 1221 } 1222 } 1223 return timeout; 1224 } 1225 1226 static void 1227 repo_fail(struct repo *rp) 1228 { 1229 /* reset the alarm since code may fallback to rsync */ 1230 rp->alarm = getmonotime() + repo_timeout; 1231 1232 if (rp->ta) 1233 http_finish(rp->ta->id, HTTP_FAILED, NULL); 1234 else if (rp->rrdp) 1235 rrdp_finish(rp->rrdp->id, 0); 1236 else if (rp->rsync) 1237 rsync_finish(rp->rsync->id, 0); 1238 else 1239 errx(1, "%s: bad repo", rp->repouri); 1240 } 1241 1242 void 1243 repo_check_timeout(void) 1244 { 1245 struct repo *rp; 1246 time_t now; 1247 1248 now = getmonotime(); 1249 /* Look up in repository table. (Lookup should actually fail here) */ 1250 SLIST_FOREACH(rp, &repos, entry) { 1251 if (repo_state(rp) == REPO_LOADING) { 1252 if (rp->alarm <= now) { 1253 warnx("%s: synchronisation timeout", 1254 rp->repouri); 1255 repo_fail(rp); 1256 } 1257 } 1258 } 1259 } 1260 1261 static char ** 1262 add_to_del(char **del, size_t *dsz, char *file) 1263 { 1264 size_t i = *dsz; 1265 1266 del = reallocarray(del, i + 1, sizeof(*del)); 1267 if (del == NULL) 1268 err(1, NULL); 1269 if ((del[i] = strdup(file)) == NULL) 1270 err(1, NULL); 1271 *dsz = i + 1; 1272 return del; 1273 } 1274 1275 static char ** 1276 repo_rrdp_cleanup(struct filepath_tree *tree, struct rrdprepo *rr, 1277 char **del, size_t *delsz) 1278 { 1279 struct filepath *fp, *nfp; 1280 char *fn; 1281 1282 RB_FOREACH_SAFE(fp, filepath_tree, &rr->deleted, nfp) { 1283 fn = rrdp_filename(rr, fp->file, 0); 1284 /* temp dir will be cleaned up by repo_cleanup() */ 1285 1286 if (fn == NULL) 1287 errx(1, "bad filepath"); /* should not happen */ 1288 1289 if (!filepath_exists(tree, fn)) 1290 del = add_to_del(del, delsz, fn); 1291 else 1292 warnx("%s: referenced file supposed to be deleted", fn); 1293 1294 free(fn); 1295 filepath_put(&rr->deleted, fp); 1296 } 1297 1298 return del; 1299 } 1300 1301 void 1302 repo_cleanup(struct filepath_tree *tree) 1303 { 1304 size_t i, cnt, delsz = 0, dirsz = 0; 1305 char **del = NULL, **dir = NULL; 1306 char *argv[4] = { "ta", "rsync", "rrdp", NULL }; 1307 struct rrdprepo *rr; 1308 FTS *fts; 1309 FTSENT *e; 1310 1311 if ((fts = fts_open(argv, FTS_PHYSICAL | FTS_NOSTAT, NULL)) == NULL) 1312 err(1, "fts_open"); 1313 errno = 0; 1314 while ((e = fts_read(fts)) != NULL) { 1315 switch (e->fts_info) { 1316 case FTS_NSOK: 1317 if (!filepath_exists(tree, e->fts_path)) 1318 del = add_to_del(del, &delsz, 1319 e->fts_path); 1320 break; 1321 case FTS_D: 1322 /* special cleanup for rrdp directories */ 1323 if ((rr = rrdp_basedir(e->fts_path)) != NULL) { 1324 del = repo_rrdp_cleanup(tree, rr, del, &delsz); 1325 if (fts_set(fts, e, FTS_SKIP) == -1) 1326 err(1, "fts_set"); 1327 } 1328 break; 1329 case FTS_DP: 1330 if (!filepath_dir_exists(tree, e->fts_path)) 1331 dir = add_to_del(dir, &dirsz, 1332 e->fts_path); 1333 break; 1334 case FTS_SL: 1335 case FTS_SLNONE: 1336 warnx("symlink %s", e->fts_path); 1337 del = add_to_del(del, &delsz, e->fts_path); 1338 break; 1339 case FTS_NS: 1340 case FTS_ERR: 1341 if (e->fts_errno == ENOENT && 1342 (strcmp(e->fts_path, "rsync") == 0 || 1343 strcmp(e->fts_path, "rrdp") == 0)) 1344 continue; 1345 warnx("fts_read %s: %s", e->fts_path, 1346 strerror(e->fts_errno)); 1347 break; 1348 default: 1349 warnx("unhandled[%x] %s", e->fts_info, 1350 e->fts_path); 1351 break; 1352 } 1353 1354 errno = 0; 1355 } 1356 if (errno) 1357 err(1, "fts_read"); 1358 if (fts_close(fts) == -1) 1359 err(1, "fts_close"); 1360 1361 cnt = 0; 1362 for (i = 0; i < delsz; i++) { 1363 if (unlink(del[i]) == -1) { 1364 if (errno != ENOENT) 1365 warn("unlink %s", del[i]); 1366 } else { 1367 if (verbose > 1) 1368 logx("deleted %s", del[i]); 1369 cnt++; 1370 } 1371 free(del[i]); 1372 } 1373 free(del); 1374 stats.del_files = cnt; 1375 1376 cnt = 0; 1377 for (i = 0; i < dirsz; i++) { 1378 if (rmdir(dir[i]) == -1) 1379 warn("rmdir %s", dir[i]); 1380 else 1381 cnt++; 1382 free(dir[i]); 1383 } 1384 free(dir); 1385 stats.del_dirs = cnt; 1386 } 1387 1388 void 1389 repo_free(void) 1390 { 1391 struct repo *rp; 1392 1393 while ((rp = SLIST_FIRST(&repos)) != NULL) { 1394 SLIST_REMOVE_HEAD(&repos, entry); 1395 free(rp->repouri); 1396 free(rp); 1397 } 1398 1399 ta_free(); 1400 rrdp_free(); 1401 rsync_free(); 1402 } 1403