1 /* $OpenBSD: repo.c,v 1.57 2024/04/21 19:27:44 claudio Exp $ */
2 /*
3 * Copyright (c) 2021 Claudio Jeker <claudio@openbsd.org>
4 * Copyright (c) 2019 Kristaps Dzonsons <kristaps@bsd.lv>
5 *
6 * Permission to use, copy, modify, and distribute this software for any
7 * purpose with or without fee is hereby granted, provided that the above
8 * copyright notice and this permission notice appear in all copies.
9 *
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 */
18
19 #include <sys/queue.h>
20 #include <sys/tree.h>
21 #include <sys/types.h>
22 #include <sys/stat.h>
23
24 #include <assert.h>
25 #include <err.h>
26 #include <errno.h>
27 #include <fcntl.h>
28 #include <fts.h>
29 #include <limits.h>
30 #include <poll.h>
31 #include <stdio.h>
32 #include <stdlib.h>
33 #include <string.h>
34 #include <unistd.h>
35
36 #include <imsg.h>
37
38 #include "extern.h"
39
40 extern struct stats stats;
41 extern int rrdpon;
42 extern int repo_timeout;
43 extern time_t deadline;
44 int nofetch;
45
46 enum repo_state {
47 REPO_LOADING = 0,
48 REPO_DONE = 1,
49 REPO_FAILED = -1,
50 };
51
52 /*
53 * A ta, rsync or rrdp repository.
54 * Depending on what is needed the generic repository is backed by
55 * a ta, rsync or rrdp repository. Multiple repositories can use the
56 * same backend.
57 */
58 struct rrdprepo {
59 SLIST_ENTRY(rrdprepo) entry;
60 char *notifyuri;
61 char *basedir;
62 struct filepath_tree deleted;
63 unsigned int id;
64 enum repo_state state;
65 };
66 static SLIST_HEAD(, rrdprepo) rrdprepos = SLIST_HEAD_INITIALIZER(rrdprepos);
67
68 struct rsyncrepo {
69 SLIST_ENTRY(rsyncrepo) entry;
70 char *repouri;
71 char *basedir;
72 unsigned int id;
73 enum repo_state state;
74 };
75 static SLIST_HEAD(, rsyncrepo) rsyncrepos = SLIST_HEAD_INITIALIZER(rsyncrepos);
76
77 struct tarepo {
78 SLIST_ENTRY(tarepo) entry;
79 char *descr;
80 char *basedir;
81 char *temp;
82 char **uri;
83 size_t urisz;
84 size_t uriidx;
85 unsigned int id;
86 enum repo_state state;
87 };
88 static SLIST_HEAD(, tarepo) tarepos = SLIST_HEAD_INITIALIZER(tarepos);
89
90 struct repo {
91 SLIST_ENTRY(repo) entry;
92 char *repouri;
93 char *notifyuri;
94 char *basedir;
95 const struct rrdprepo *rrdp;
96 const struct rsyncrepo *rsync;
97 const struct tarepo *ta;
98 struct entityq queue; /* files waiting for repo */
99 struct repotalstats stats[TALSZ_MAX];
100 struct repostats repostats;
101 struct timespec start_time;
102 time_t alarm; /* sync timeout */
103 int talid;
104 int stats_used[TALSZ_MAX];
105 unsigned int id; /* identifier */
106 };
107 static SLIST_HEAD(, repo) repos = SLIST_HEAD_INITIALIZER(repos);
108
109 /* counter for unique repo id */
110 unsigned int repoid;
111
112 static struct rsyncrepo *rsync_get(const char *, const char *);
113 static void remove_contents(char *);
114
115 /*
116 * Database of all file path accessed during a run.
117 */
118 struct filepath {
119 RB_ENTRY(filepath) entry;
120 char *file;
121 time_t mtime;
122 };
123
124 static inline int
filepathcmp(struct filepath * a,struct filepath * b)125 filepathcmp(struct filepath *a, struct filepath *b)
126 {
127 return strcmp(a->file, b->file);
128 }
129
130 RB_PROTOTYPE(filepath_tree, filepath, entry, filepathcmp);
131
132 /*
133 * Functions to lookup which files have been accessed during computation.
134 */
135 int
filepath_add(struct filepath_tree * tree,char * file,time_t mtime)136 filepath_add(struct filepath_tree *tree, char *file, time_t mtime)
137 {
138 struct filepath *fp;
139
140 if ((fp = malloc(sizeof(*fp))) == NULL)
141 err(1, NULL);
142 fp->mtime = mtime;
143 if ((fp->file = strdup(file)) == NULL)
144 err(1, NULL);
145
146 if (RB_INSERT(filepath_tree, tree, fp) != NULL) {
147 /* already in the tree */
148 free(fp->file);
149 free(fp);
150 return 0;
151 }
152
153 return 1;
154 }
155
156 /*
157 * Lookup a file path in the tree and return the object if found or NULL.
158 */
159 static struct filepath *
filepath_find(struct filepath_tree * tree,char * file)160 filepath_find(struct filepath_tree *tree, char *file)
161 {
162 struct filepath needle = { .file = file };
163
164 return RB_FIND(filepath_tree, tree, &needle);
165 }
166
167 /*
168 * Returns true if file exists in the tree.
169 */
170 static int
filepath_exists(struct filepath_tree * tree,char * file)171 filepath_exists(struct filepath_tree *tree, char *file)
172 {
173 return filepath_find(tree, file) != NULL;
174 }
175
176 /*
177 * Remove entry from tree and free it.
178 */
179 static void
filepath_put(struct filepath_tree * tree,struct filepath * fp)180 filepath_put(struct filepath_tree *tree, struct filepath *fp)
181 {
182 RB_REMOVE(filepath_tree, tree, fp);
183 free((void *)fp->file);
184 free(fp);
185 }
186
187 /*
188 * Free all elements of a filepath tree.
189 */
190 static void
filepath_free(struct filepath_tree * tree)191 filepath_free(struct filepath_tree *tree)
192 {
193 struct filepath *fp, *nfp;
194
195 RB_FOREACH_SAFE(fp, filepath_tree, tree, nfp)
196 filepath_put(tree, fp);
197 }
198
199 RB_GENERATE(filepath_tree, filepath, entry, filepathcmp);
200
201 /*
202 * Function to hash a string into a unique directory name.
203 * Returned hash needs to be freed.
204 */
205 static char *
hash_dir(const char * uri)206 hash_dir(const char *uri)
207 {
208 unsigned char m[SHA256_DIGEST_LENGTH];
209
210 SHA256(uri, strlen(uri), m);
211 return hex_encode(m, sizeof(m));
212 }
213
214 /*
215 * Function to build the directory name based on URI and a directory
216 * as prefix. Skip the proto:// in URI but keep everything else.
217 */
218 static char *
repo_dir(const char * uri,const char * dir,int hash)219 repo_dir(const char *uri, const char *dir, int hash)
220 {
221 const char *local;
222 char *out, *hdir = NULL;
223
224 if (hash) {
225 local = hdir = hash_dir(uri);
226 } else {
227 local = strchr(uri, ':');
228 if (local != NULL)
229 local += strlen("://");
230 else
231 local = uri;
232 }
233
234 if (dir == NULL) {
235 if ((out = strdup(local)) == NULL)
236 err(1, NULL);
237 } else {
238 if (asprintf(&out, "%s/%s", dir, local) == -1)
239 err(1, NULL);
240 }
241
242 free(hdir);
243 return out;
244 }
245
246 /*
247 * Function to create all missing directories to a path.
248 * This functions alters the path temporarily.
249 */
250 static int
repo_mkpath(int fd,char * file)251 repo_mkpath(int fd, char *file)
252 {
253 char *slash;
254
255 /* build directory hierarchy */
256 slash = strrchr(file, '/');
257 assert(slash != NULL);
258 *slash = '\0';
259 if (mkpathat(fd, file) == -1) {
260 warn("mkpath %s", file);
261 return -1;
262 }
263 *slash = '/';
264 return 0;
265 }
266
267 /*
268 * Return the state of a repository.
269 */
270 static enum repo_state
repo_state(const struct repo * rp)271 repo_state(const struct repo *rp)
272 {
273 if (rp->ta)
274 return rp->ta->state;
275 if (rp->rsync)
276 return rp->rsync->state;
277 if (rp->rrdp)
278 return rp->rrdp->state;
279 /* No backend so sync is by definition done. */
280 return REPO_DONE;
281 }
282
283 /*
284 * Function called once a repository is done with the sync. Either
285 * successfully or after failure.
286 */
287 static void
repo_done(const void * vp,int ok)288 repo_done(const void *vp, int ok)
289 {
290 struct repo *rp;
291 struct timespec flush_time;
292
293 SLIST_FOREACH(rp, &repos, entry) {
294 if (vp != rp->ta && vp != rp->rsync && vp != rp->rrdp)
295 continue;
296
297 /* for rrdp try to fall back to rsync */
298 if (vp == rp->rrdp && !ok && !nofetch) {
299 rp->rrdp = NULL;
300 rp->rsync = rsync_get(rp->repouri, rp->basedir);
301 /* need to check if it was already loaded */
302 if (repo_state(rp) == REPO_LOADING)
303 continue;
304 }
305
306 entityq_flush(&rp->queue, rp);
307 clock_gettime(CLOCK_MONOTONIC, &flush_time);
308 timespecsub(&flush_time, &rp->start_time,
309 &rp->repostats.sync_time);
310 }
311 }
312
313 /*
314 * Build TA file name based on the repo info.
315 * If temp is set add Xs for mkostemp.
316 */
317 static char *
ta_filename(const struct tarepo * tr,int temp)318 ta_filename(const struct tarepo *tr, int temp)
319 {
320 const char *file;
321 char *nfile;
322
323 /* does not matter which URI, all end with same filename */
324 file = strrchr(tr->uri[0], '/');
325 assert(file);
326
327 if (asprintf(&nfile, "%s%s%s", tr->basedir, file,
328 temp ? ".XXXXXXXX" : "") == -1)
329 err(1, NULL);
330
331 return nfile;
332 }
333
334 static void
ta_fetch(struct tarepo * tr)335 ta_fetch(struct tarepo *tr)
336 {
337 if (!rrdpon) {
338 for (; tr->uriidx < tr->urisz; tr->uriidx++) {
339 if (strncasecmp(tr->uri[tr->uriidx],
340 RSYNC_PROTO, RSYNC_PROTO_LEN) == 0)
341 break;
342 }
343 }
344
345 if (tr->uriidx >= tr->urisz) {
346 tr->state = REPO_FAILED;
347 logx("ta/%s: fallback to cache", tr->descr);
348
349 repo_done(tr, 0);
350 return;
351 }
352
353 logx("ta/%s: pulling from %s", tr->descr, tr->uri[tr->uriidx]);
354
355 if (strncasecmp(tr->uri[tr->uriidx], RSYNC_PROTO,
356 RSYNC_PROTO_LEN) == 0) {
357 /*
358 * Create destination location.
359 * Build up the tree to this point.
360 */
361 rsync_fetch(tr->id, tr->uri[tr->uriidx], tr->basedir, NULL);
362 } else {
363 int fd;
364
365 tr->temp = ta_filename(tr, 1);
366 fd = mkostemp(tr->temp, O_CLOEXEC);
367 if (fd == -1) {
368 warn("mkostemp: %s", tr->temp);
369 http_finish(tr->id, HTTP_FAILED, NULL);
370 return;
371 }
372 if (fchmod(fd, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH) == -1)
373 warn("fchmod: %s", tr->temp);
374
375 http_fetch(tr->id, tr->uri[tr->uriidx], NULL, fd);
376 }
377 }
378
379 static struct tarepo *
ta_get(struct tal * tal)380 ta_get(struct tal *tal)
381 {
382 struct tarepo *tr;
383
384 /* no need to look for possible other repo */
385
386 if ((tr = calloc(1, sizeof(*tr))) == NULL)
387 err(1, NULL);
388 tr->id = ++repoid;
389 SLIST_INSERT_HEAD(&tarepos, tr, entry);
390
391 if ((tr->descr = strdup(tal->descr)) == NULL)
392 err(1, NULL);
393 tr->basedir = repo_dir(tal->descr, "ta", 0);
394
395 /* steal URI information from TAL */
396 tr->urisz = tal->urisz;
397 tr->uri = tal->uri;
398 tal->urisz = 0;
399 tal->uri = NULL;
400
401 ta_fetch(tr);
402
403 return tr;
404 }
405
406 static struct tarepo *
ta_find(unsigned int id)407 ta_find(unsigned int id)
408 {
409 struct tarepo *tr;
410
411 SLIST_FOREACH(tr, &tarepos, entry)
412 if (id == tr->id)
413 break;
414 return tr;
415 }
416
417 static void
ta_free(void)418 ta_free(void)
419 {
420 struct tarepo *tr;
421
422 while ((tr = SLIST_FIRST(&tarepos)) != NULL) {
423 SLIST_REMOVE_HEAD(&tarepos, entry);
424 free(tr->descr);
425 free(tr->basedir);
426 free(tr->temp);
427 free(tr->uri);
428 free(tr);
429 }
430 }
431
432 static struct rsyncrepo *
rsync_get(const char * uri,const char * validdir)433 rsync_get(const char *uri, const char *validdir)
434 {
435 struct rsyncrepo *rr;
436 char *repo;
437
438 if ((repo = rsync_base_uri(uri)) == NULL)
439 errx(1, "bad caRepository URI: %s", uri);
440
441 SLIST_FOREACH(rr, &rsyncrepos, entry)
442 if (strcmp(rr->repouri, repo) == 0) {
443 free(repo);
444 return rr;
445 }
446
447 if ((rr = calloc(1, sizeof(*rr))) == NULL)
448 err(1, NULL);
449
450 rr->id = ++repoid;
451 SLIST_INSERT_HEAD(&rsyncrepos, rr, entry);
452
453 rr->repouri = repo;
454 rr->basedir = repo_dir(repo, ".rsync", 0);
455
456 /* create base directory */
457 if (mkpath(rr->basedir) == -1) {
458 warn("mkpath %s", rr->basedir);
459 rsync_finish(rr->id, 0);
460 return rr;
461 }
462
463 logx("%s: pulling from %s", rr->basedir, rr->repouri);
464 rsync_fetch(rr->id, rr->repouri, rr->basedir, validdir);
465
466 return rr;
467 }
468
469 static struct rsyncrepo *
rsync_find(unsigned int id)470 rsync_find(unsigned int id)
471 {
472 struct rsyncrepo *rr;
473
474 SLIST_FOREACH(rr, &rsyncrepos, entry)
475 if (id == rr->id)
476 break;
477 return rr;
478 }
479
480 static void
rsync_free(void)481 rsync_free(void)
482 {
483 struct rsyncrepo *rr;
484
485 while ((rr = SLIST_FIRST(&rsyncrepos)) != NULL) {
486 SLIST_REMOVE_HEAD(&rsyncrepos, entry);
487 free(rr->repouri);
488 free(rr->basedir);
489 free(rr);
490 }
491 }
492
493 /*
494 * Build local file name base on the URI and the rrdprepo info.
495 */
496 static char *
rrdp_filename(const struct rrdprepo * rr,const char * uri,int valid)497 rrdp_filename(const struct rrdprepo *rr, const char *uri, int valid)
498 {
499 char *nfile;
500 const char *dir = rr->basedir;
501
502 if (!valid_uri(uri, strlen(uri), RSYNC_PROTO))
503 errx(1, "%s: bad URI %s", rr->basedir, uri);
504 uri += RSYNC_PROTO_LEN; /* skip proto */
505 if (valid) {
506 if ((nfile = strdup(uri)) == NULL)
507 err(1, NULL);
508 } else {
509 if (asprintf(&nfile, "%s/%s", dir, uri) == -1)
510 err(1, NULL);
511 }
512 return nfile;
513 }
514
515 /*
516 * Build RRDP state file name based on the repo info.
517 * If temp is set add Xs for mkostemp.
518 */
519 static char *
rrdp_state_filename(const struct rrdprepo * rr,int temp)520 rrdp_state_filename(const struct rrdprepo *rr, int temp)
521 {
522 char *nfile;
523
524 if (asprintf(&nfile, "%s/.state%s", rr->basedir,
525 temp ? ".XXXXXXXX" : "") == -1)
526 err(1, NULL);
527
528 return nfile;
529 }
530
531 static struct rrdprepo *
rrdp_find(unsigned int id)532 rrdp_find(unsigned int id)
533 {
534 struct rrdprepo *rr;
535
536 SLIST_FOREACH(rr, &rrdprepos, entry)
537 if (id == rr->id)
538 break;
539 return rr;
540 }
541
542 static void
rrdp_free(void)543 rrdp_free(void)
544 {
545 struct rrdprepo *rr;
546
547 while ((rr = SLIST_FIRST(&rrdprepos)) != NULL) {
548 SLIST_REMOVE_HEAD(&rrdprepos, entry);
549
550 free(rr->notifyuri);
551 free(rr->basedir);
552
553 filepath_free(&rr->deleted);
554
555 free(rr);
556 }
557 }
558
559 /*
560 * Check if a directory is an active rrdp repository.
561 * Returns 1 if found else 0.
562 */
563 static struct repo *
repo_rrdp_bypath(const char * dir)564 repo_rrdp_bypath(const char *dir)
565 {
566 struct repo *rp;
567
568 SLIST_FOREACH(rp, &repos, entry) {
569 if (rp->rrdp == NULL)
570 continue;
571 if (strcmp(dir, rp->rrdp->basedir) == 0)
572 return rp;
573 }
574 return NULL;
575 }
576
577 /*
578 * Check if the URI is actually covered by one of the repositories
579 * that depend on this RRDP repository.
580 * Returns 1 if the URI is valid, 0 if no repouri matches the URI.
581 */
582 static int
rrdp_uri_valid(struct rrdprepo * rr,const char * uri)583 rrdp_uri_valid(struct rrdprepo *rr, const char *uri)
584 {
585 struct repo *rp;
586
587 SLIST_FOREACH(rp, &repos, entry) {
588 if (rp->rrdp != rr)
589 continue;
590 if (strncmp(uri, rp->repouri, strlen(rp->repouri)) == 0)
591 return 1;
592 }
593 return 0;
594 }
595
596 /*
597 * Allocate and insert a new repository.
598 */
599 static struct repo *
repo_alloc(int talid)600 repo_alloc(int talid)
601 {
602 struct repo *rp;
603
604 if ((rp = calloc(1, sizeof(*rp))) == NULL)
605 err(1, NULL);
606
607 rp->id = ++repoid;
608 rp->talid = talid;
609 rp->alarm = getmonotime() + repo_timeout;
610 TAILQ_INIT(&rp->queue);
611 SLIST_INSERT_HEAD(&repos, rp, entry);
612 clock_gettime(CLOCK_MONOTONIC, &rp->start_time);
613
614 stats.repos++;
615 return rp;
616 }
617
618 /*
619 * Parse the RRDP state file if it exists and set the session struct
620 * based on that information.
621 */
622 static struct rrdp_session *
rrdp_session_parse(const struct rrdprepo * rr)623 rrdp_session_parse(const struct rrdprepo *rr)
624 {
625 FILE *f;
626 struct rrdp_session *state;
627 int fd, ln = 0, deltacnt = 0;
628 const char *errstr;
629 char *line = NULL, *file;
630 size_t len = 0;
631 ssize_t n;
632
633 if ((state = calloc(1, sizeof(*state))) == NULL)
634 err(1, NULL);
635
636 file = rrdp_state_filename(rr, 0);
637 if ((fd = open(file, O_RDONLY)) == -1) {
638 if (errno != ENOENT)
639 warn("%s: open state file", rr->basedir);
640 free(file);
641 return state;
642 }
643 free(file);
644 f = fdopen(fd, "r");
645 if (f == NULL)
646 err(1, "fdopen");
647
648 while ((n = getline(&line, &len, f)) != -1) {
649 if (line[n - 1] == '\n')
650 line[n - 1] = '\0';
651 switch (ln) {
652 case 0:
653 if ((state->session_id = strdup(line)) == NULL)
654 err(1, NULL);
655 break;
656 case 1:
657 state->serial = strtonum(line, 1, LLONG_MAX, &errstr);
658 if (errstr)
659 goto fail;
660 break;
661 case 2:
662 if (strcmp(line, "-") == 0)
663 break;
664 if ((state->last_mod = strdup(line)) == NULL)
665 err(1, NULL);
666 break;
667 default:
668 if (deltacnt >= MAX_RRDP_DELTAS)
669 goto fail;
670 if ((state->deltas[deltacnt++] = strdup(line)) == NULL)
671 err(1, NULL);
672 break;
673 }
674 ln++;
675 }
676
677 if (ferror(f))
678 goto fail;
679 fclose(f);
680 free(line);
681 return state;
682
683 fail:
684 warnx("%s: troubles reading state file", rr->basedir);
685 fclose(f);
686 free(line);
687 free(state->session_id);
688 free(state->last_mod);
689 memset(state, 0, sizeof(*state));
690 return state;
691 }
692
693 /*
694 * Carefully write the RRDP session state file back.
695 */
696 void
rrdp_session_save(unsigned int id,struct rrdp_session * state)697 rrdp_session_save(unsigned int id, struct rrdp_session *state)
698 {
699 struct rrdprepo *rr;
700 char *temp, *file;
701 FILE *f = NULL;
702 int fd, i;
703
704 rr = rrdp_find(id);
705 if (rr == NULL)
706 errx(1, "non-existent rrdp repo %u", id);
707
708 file = rrdp_state_filename(rr, 0);
709 temp = rrdp_state_filename(rr, 1);
710
711 if ((fd = mkostemp(temp, O_CLOEXEC)) == -1)
712 goto fail;
713 (void)fchmod(fd, 0644);
714 f = fdopen(fd, "w");
715 if (f == NULL)
716 err(1, "fdopen");
717
718 /* write session state file out */
719 if (fprintf(f, "%s\n%lld\n", state->session_id,
720 state->serial) < 0)
721 goto fail;
722
723 if (state->last_mod != NULL) {
724 if (fprintf(f, "%s\n", state->last_mod) < 0)
725 goto fail;
726 } else {
727 if (fprintf(f, "-\n") < 0)
728 goto fail;
729 }
730 for (i = 0; i < MAX_RRDP_DELTAS && state->deltas[i] != NULL; i++) {
731 if (fprintf(f, "%s\n", state->deltas[i]) < 0)
732 goto fail;
733 }
734 if (fclose(f) != 0) {
735 f = NULL;
736 goto fail;
737 }
738
739 if (rename(temp, file) == -1) {
740 warn("%s: rename %s to %s", rr->basedir, temp, file);
741 unlink(temp);
742 }
743
744 free(temp);
745 free(file);
746 return;
747
748 fail:
749 warn("%s: save state to %s", rr->basedir, temp);
750 if (f != NULL)
751 fclose(f);
752 unlink(temp);
753 free(temp);
754 free(file);
755 }
756
757 /*
758 * Free an rrdp_session pointer. Safe to call with NULL.
759 */
760 void
rrdp_session_free(struct rrdp_session * s)761 rrdp_session_free(struct rrdp_session *s)
762 {
763 size_t i;
764
765 if (s == NULL)
766 return;
767 free(s->session_id);
768 free(s->last_mod);
769 for (i = 0; i < sizeof(s->deltas) / sizeof(s->deltas[0]); i++)
770 free(s->deltas[i]);
771 free(s);
772 }
773
774 void
rrdp_session_buffer(struct ibuf * b,const struct rrdp_session * s)775 rrdp_session_buffer(struct ibuf *b, const struct rrdp_session *s)
776 {
777 size_t i;
778
779 io_str_buffer(b, s->session_id);
780 io_simple_buffer(b, &s->serial, sizeof(s->serial));
781 io_str_buffer(b, s->last_mod);
782 for (i = 0; i < sizeof(s->deltas) / sizeof(s->deltas[0]); i++)
783 io_str_buffer(b, s->deltas[i]);
784 }
785
786 struct rrdp_session *
rrdp_session_read(struct ibuf * b)787 rrdp_session_read(struct ibuf *b)
788 {
789 struct rrdp_session *s;
790 size_t i;
791
792 if ((s = calloc(1, sizeof(*s))) == NULL)
793 err(1, NULL);
794
795 io_read_str(b, &s->session_id);
796 io_read_buf(b, &s->serial, sizeof(s->serial));
797 io_read_str(b, &s->last_mod);
798 for (i = 0; i < sizeof(s->deltas) / sizeof(s->deltas[0]); i++)
799 io_read_str(b, &s->deltas[i]);
800
801 return s;
802 }
803
804 static struct rrdprepo *
rrdp_get(const char * uri)805 rrdp_get(const char *uri)
806 {
807 struct rrdp_session *state;
808 struct rrdprepo *rr;
809
810 SLIST_FOREACH(rr, &rrdprepos, entry)
811 if (strcmp(rr->notifyuri, uri) == 0) {
812 if (rr->state == REPO_FAILED)
813 return NULL;
814 return rr;
815 }
816
817 if ((rr = calloc(1, sizeof(*rr))) == NULL)
818 err(1, NULL);
819
820 rr->id = ++repoid;
821 SLIST_INSERT_HEAD(&rrdprepos, rr, entry);
822
823 if ((rr->notifyuri = strdup(uri)) == NULL)
824 err(1, NULL);
825 rr->basedir = repo_dir(uri, ".rrdp", 1);
826
827 RB_INIT(&rr->deleted);
828
829 /* create base directory */
830 if (mkpath(rr->basedir) == -1) {
831 warn("mkpath %s", rr->basedir);
832 rrdp_finish(rr->id, 0);
833 return rr;
834 }
835
836 /* parse state and start the sync */
837 state = rrdp_session_parse(rr);
838 rrdp_fetch(rr->id, rr->notifyuri, rr->notifyuri, state);
839 rrdp_session_free(state);
840
841 logx("%s: pulling from %s", rr->notifyuri, "network");
842
843 return rr;
844 }
845
846 /*
847 * Remove RRDP repo and start over.
848 */
849 void
rrdp_clear(unsigned int id)850 rrdp_clear(unsigned int id)
851 {
852 struct rrdprepo *rr;
853
854 rr = rrdp_find(id);
855 if (rr == NULL)
856 errx(1, "non-existent rrdp repo %u", id);
857
858 /* remove rrdp repository contents */
859 remove_contents(rr->basedir);
860 }
861
862 /*
863 * Write a file into the temporary RRDP dir but only after checking
864 * its hash (if required). The function also makes sure that the file
865 * tracking is properly adjusted.
866 * Returns 1 on success, 0 if the repo is corrupt, -1 on IO error
867 */
868 int
rrdp_handle_file(unsigned int id,enum publish_type pt,char * uri,char * hash,size_t hlen,char * data,size_t dlen)869 rrdp_handle_file(unsigned int id, enum publish_type pt, char *uri,
870 char *hash, size_t hlen, char *data, size_t dlen)
871 {
872 struct rrdprepo *rr;
873 struct filepath *fp;
874 ssize_t s;
875 char *fn = NULL;
876 int fd = -1, try = 0, deleted = 0;
877 int flags;
878
879 rr = rrdp_find(id);
880 if (rr == NULL)
881 errx(1, "non-existent rrdp repo %u", id);
882 if (rr->state == REPO_FAILED)
883 return -1;
884
885 /* check hash of original file for updates and deletes */
886 if (pt == PUB_UPD || pt == PUB_DEL) {
887 if (filepath_exists(&rr->deleted, uri)) {
888 warnx("%s: already deleted", uri);
889 return 0;
890 }
891 /* try to open file first in rrdp then in valid repo */
892 do {
893 free(fn);
894 if ((fn = rrdp_filename(rr, uri, try++)) == NULL)
895 return 0;
896 fd = open(fn, O_RDONLY);
897 } while (fd == -1 && try < 2);
898
899 if (!valid_filehash(fd, hash, hlen)) {
900 warnx("%s: bad file digest for %s", rr->notifyuri, fn);
901 free(fn);
902 return 0;
903 }
904 free(fn);
905 }
906
907 /* write new content or mark uri as deleted. */
908 if (pt == PUB_DEL) {
909 filepath_add(&rr->deleted, uri, 0);
910 } else {
911 fp = filepath_find(&rr->deleted, uri);
912 if (fp != NULL) {
913 filepath_put(&rr->deleted, fp);
914 deleted = 1;
915 }
916
917 /* add new file to rrdp dir */
918 if ((fn = rrdp_filename(rr, uri, 0)) == NULL)
919 return 0;
920
921 if (repo_mkpath(AT_FDCWD, fn) == -1)
922 goto fail;
923
924 flags = O_WRONLY|O_CREAT|O_TRUNC;
925 if (pt == PUB_ADD && !deleted)
926 flags |= O_EXCL;
927 fd = open(fn, flags, 0644);
928 if (fd == -1) {
929 if (errno == EEXIST) {
930 warnx("%s: duplicate publish element for %s",
931 rr->notifyuri, fn);
932 free(fn);
933 return 0;
934 }
935 warn("open %s", fn);
936 goto fail;
937 }
938
939 if ((s = write(fd, data, dlen)) == -1) {
940 warn("write %s", fn);
941 goto fail;
942 }
943 close(fd);
944 if ((size_t)s != dlen) /* impossible */
945 errx(1, "short write %s", fn);
946 free(fn);
947 }
948
949 return 1;
950
951 fail:
952 rr->state = REPO_FAILED;
953 if (fd != -1)
954 close(fd);
955 free(fn);
956 return -1;
957 }
958
959 /*
960 * RSYNC sync finished, either with or without success.
961 */
962 void
rsync_finish(unsigned int id,int ok)963 rsync_finish(unsigned int id, int ok)
964 {
965 struct rsyncrepo *rr;
966 struct tarepo *tr;
967
968 tr = ta_find(id);
969 if (tr != NULL) {
970 /* repository changed state already, ignore request */
971 if (tr->state != REPO_LOADING)
972 return;
973 if (ok) {
974 logx("ta/%s: loaded from network", tr->descr);
975 stats.rsync_repos++;
976 tr->state = REPO_DONE;
977 repo_done(tr, 1);
978 } else {
979 warnx("ta/%s: load from network failed", tr->descr);
980 stats.rsync_fails++;
981 tr->uriidx++;
982 ta_fetch(tr);
983 }
984 return;
985 }
986
987 rr = rsync_find(id);
988 if (rr == NULL)
989 errx(1, "unknown rsync repo %u", id);
990 /* repository changed state already, ignore request */
991 if (rr->state != REPO_LOADING)
992 return;
993
994 if (ok) {
995 logx("%s: loaded from network", rr->basedir);
996 stats.rsync_repos++;
997 rr->state = REPO_DONE;
998 } else {
999 warnx("%s: load from network failed, fallback to cache",
1000 rr->basedir);
1001 stats.rsync_fails++;
1002 rr->state = REPO_FAILED;
1003 /* clear rsync repo since it failed */
1004 remove_contents(rr->basedir);
1005 }
1006
1007 repo_done(rr, ok);
1008 }
1009
1010 /*
1011 * RRDP sync finished, either with or without success.
1012 */
1013 void
rrdp_finish(unsigned int id,int ok)1014 rrdp_finish(unsigned int id, int ok)
1015 {
1016 struct rrdprepo *rr;
1017
1018 rr = rrdp_find(id);
1019 if (rr == NULL)
1020 errx(1, "unknown RRDP repo %u", id);
1021 /* repository changed state already, ignore request */
1022 if (rr->state != REPO_LOADING)
1023 return;
1024
1025 if (ok) {
1026 logx("%s: loaded from network", rr->notifyuri);
1027 stats.rrdp_repos++;
1028 rr->state = REPO_DONE;
1029 } else {
1030 warnx("%s: load from network failed, fallback to %s",
1031 rr->notifyuri, nofetch ? "cache" : "rsync");
1032 stats.rrdp_fails++;
1033 rr->state = REPO_FAILED;
1034 /* clear the RRDP repo since it failed */
1035 remove_contents(rr->basedir);
1036 /* also clear the list of deleted files */
1037 filepath_free(&rr->deleted);
1038 }
1039
1040 repo_done(rr, ok);
1041 }
1042
1043 /*
1044 * Handle responses from the http process. For TA file, either rename
1045 * or delete the temporary file. For RRDP requests relay the request
1046 * over to the rrdp process.
1047 */
1048 void
http_finish(unsigned int id,enum http_result res,const char * last_mod)1049 http_finish(unsigned int id, enum http_result res, const char *last_mod)
1050 {
1051 struct tarepo *tr;
1052
1053 tr = ta_find(id);
1054 if (tr == NULL) {
1055 /* not a TA fetch therefore RRDP */
1056 rrdp_http_done(id, res, last_mod);
1057 return;
1058 }
1059
1060 /* repository changed state already, ignore request */
1061 if (tr->state != REPO_LOADING)
1062 return;
1063
1064 /* Move downloaded TA file into place, or unlink on failure. */
1065 if (res == HTTP_OK) {
1066 char *file;
1067
1068 file = ta_filename(tr, 0);
1069 if (rename(tr->temp, file) == -1)
1070 warn("rename to %s", file);
1071 free(file);
1072
1073 logx("ta/%s: loaded from network", tr->descr);
1074 tr->state = REPO_DONE;
1075 stats.http_repos++;
1076 repo_done(tr, 1);
1077 } else {
1078 if (unlink(tr->temp) == -1 && errno != ENOENT)
1079 warn("unlink %s", tr->temp);
1080
1081 tr->uriidx++;
1082 warnx("ta/%s: load from network failed", tr->descr);
1083 ta_fetch(tr);
1084 }
1085 }
1086
1087 /*
1088 * Look up a trust anchor, queueing it for download if not found.
1089 */
1090 struct repo *
ta_lookup(int id,struct tal * tal)1091 ta_lookup(int id, struct tal *tal)
1092 {
1093 struct repo *rp;
1094
1095 if (tal->urisz == 0)
1096 errx(1, "TAL %s has no URI", tal->descr);
1097
1098 /* Look up in repository table. (Lookup should actually fail here) */
1099 SLIST_FOREACH(rp, &repos, entry) {
1100 if (strcmp(rp->repouri, tal->uri[0]) == 0)
1101 return rp;
1102 }
1103
1104 rp = repo_alloc(id);
1105 rp->basedir = repo_dir(tal->descr, "ta", 0);
1106 if ((rp->repouri = strdup(tal->uri[0])) == NULL)
1107 err(1, NULL);
1108
1109 /* check if sync disabled ... */
1110 if (noop) {
1111 logx("%s: using cache", rp->basedir);
1112 entityq_flush(&rp->queue, rp);
1113 return rp;
1114 }
1115
1116 /* try to create base directory */
1117 if (mkpath(rp->basedir) == -1)
1118 warn("mkpath %s", rp->basedir);
1119
1120 rp->ta = ta_get(tal);
1121
1122 /* need to check if it was already loaded */
1123 if (repo_state(rp) != REPO_LOADING)
1124 entityq_flush(&rp->queue, rp);
1125
1126 return rp;
1127 }
1128
1129 /*
1130 * Look up a repository, queueing it for discovery if not found.
1131 */
1132 struct repo *
repo_lookup(int talid,const char * uri,const char * notify)1133 repo_lookup(int talid, const char *uri, const char *notify)
1134 {
1135 struct repo *rp;
1136 char *repouri;
1137
1138 if ((repouri = rsync_base_uri(uri)) == NULL)
1139 errx(1, "bad caRepository URI: %s", uri);
1140
1141 /* Look up in repository table. */
1142 SLIST_FOREACH(rp, &repos, entry) {
1143 if (strcmp(rp->repouri, repouri) != 0)
1144 continue;
1145 if (rp->notifyuri != NULL) {
1146 if (notify == NULL)
1147 continue;
1148 if (strcmp(rp->notifyuri, notify) != 0)
1149 continue;
1150 } else if (notify != NULL)
1151 continue;
1152 /* found matching repo */
1153 free(repouri);
1154 return rp;
1155 }
1156
1157 rp = repo_alloc(talid);
1158 rp->basedir = repo_dir(repouri, NULL, 0);
1159 rp->repouri = repouri;
1160 if (notify != NULL)
1161 if ((rp->notifyuri = strdup(notify)) == NULL)
1162 err(1, NULL);
1163
1164 if (++talrepocnt[talid] >= MAX_REPO_PER_TAL) {
1165 if (talrepocnt[talid] == MAX_REPO_PER_TAL)
1166 warnx("too many repositories under %s", tals[talid]);
1167 nofetch = 1;
1168 }
1169
1170 /* check if sync disabled ... */
1171 if (noop || nofetch) {
1172 logx("%s: using cache", rp->basedir);
1173 entityq_flush(&rp->queue, rp);
1174 return rp;
1175 }
1176
1177 /* try to create base directory */
1178 if (mkpath(rp->basedir) == -1)
1179 warn("mkpath %s", rp->basedir);
1180
1181 /* ... else try RRDP first if available then rsync */
1182 if (notify != NULL)
1183 rp->rrdp = rrdp_get(notify);
1184 if (rp->rrdp == NULL)
1185 rp->rsync = rsync_get(uri, rp->basedir);
1186
1187 /* need to check if it was already loaded */
1188 if (repo_state(rp) != REPO_LOADING)
1189 entityq_flush(&rp->queue, rp);
1190
1191 return rp;
1192 }
1193
1194 /*
1195 * Find repository by identifier.
1196 */
1197 struct repo *
repo_byid(unsigned int id)1198 repo_byid(unsigned int id)
1199 {
1200 struct repo *rp;
1201
1202 SLIST_FOREACH(rp, &repos, entry) {
1203 if (rp->id == id)
1204 return rp;
1205 }
1206 return NULL;
1207 }
1208
1209 /*
1210 * Find repository by base path.
1211 */
1212 static struct repo *
repo_bypath(const char * path)1213 repo_bypath(const char *path)
1214 {
1215 struct repo *rp;
1216
1217 SLIST_FOREACH(rp, &repos, entry) {
1218 if (strcmp(rp->basedir, path) == 0)
1219 return rp;
1220 }
1221 return NULL;
1222 }
1223
1224 /*
1225 * Return the repository base or alternate directory.
1226 * Returned string must be freed by caller.
1227 */
1228 char *
repo_basedir(const struct repo * rp,int wantvalid)1229 repo_basedir(const struct repo *rp, int wantvalid)
1230 {
1231 char *path = NULL;
1232
1233 if (!wantvalid) {
1234 if (rp->ta) {
1235 if ((path = strdup(rp->ta->basedir)) == NULL)
1236 err(1, NULL);
1237 } else if (rp->rsync) {
1238 if ((path = strdup(rp->rsync->basedir)) == NULL)
1239 err(1, NULL);
1240 } else if (rp->rrdp) {
1241 path = rrdp_filename(rp->rrdp, rp->repouri, 0);
1242 } else
1243 path = NULL; /* only valid repo available */
1244 } else if (rp->basedir != NULL) {
1245 if ((path = strdup(rp->basedir)) == NULL)
1246 err(1, NULL);
1247 }
1248
1249 return path;
1250 }
1251
1252 /*
1253 * Return the repository identifier.
1254 */
1255 unsigned int
repo_id(const struct repo * rp)1256 repo_id(const struct repo *rp)
1257 {
1258 return rp->id;
1259 }
1260
1261 /*
1262 * Return the repository URI.
1263 */
1264 const char *
repo_uri(const struct repo * rp)1265 repo_uri(const struct repo *rp)
1266 {
1267 return rp->repouri;
1268 }
1269
1270 /*
1271 * Return the repository URI.
1272 */
1273 void
repo_fetch_uris(const struct repo * rp,const char ** carepo,const char ** notifyuri)1274 repo_fetch_uris(const struct repo *rp, const char **carepo,
1275 const char **notifyuri)
1276 {
1277 *carepo = rp->repouri;
1278 *notifyuri = rp->notifyuri;
1279 }
1280
1281 /*
1282 * Return 1 if repository is synced else 0.
1283 */
1284 int
repo_synced(const struct repo * rp)1285 repo_synced(const struct repo *rp)
1286 {
1287 if (repo_state(rp) == REPO_DONE &&
1288 !(rp->rrdp == NULL && rp->rsync == NULL && rp->ta == NULL))
1289 return 1;
1290 return 0;
1291 }
1292
1293 /*
1294 * Return the protocol string "rrdp", "rsync", "https" which was used to sync.
1295 * Result is only correct if repository was properly synced.
1296 */
1297 const char *
repo_proto(const struct repo * rp)1298 repo_proto(const struct repo *rp)
1299 {
1300
1301 if (rp->ta != NULL) {
1302 const struct tarepo *tr = rp->ta;
1303 if (tr->uriidx < tr->urisz &&
1304 strncasecmp(tr->uri[tr->uriidx], RSYNC_PROTO,
1305 RSYNC_PROTO_LEN) == 0)
1306 return "rsync";
1307 else
1308 return "https";
1309 }
1310 if (rp->rrdp != NULL)
1311 return "rrdp";
1312 return "rsync";
1313 }
1314
1315 /*
1316 * Return the repository tal ID.
1317 */
1318 int
repo_talid(const struct repo * rp)1319 repo_talid(const struct repo *rp)
1320 {
1321 return rp->talid;
1322 }
1323
1324 int
repo_queued(struct repo * rp,struct entity * p)1325 repo_queued(struct repo *rp, struct entity *p)
1326 {
1327 if (repo_state(rp) == REPO_LOADING) {
1328 TAILQ_INSERT_TAIL(&rp->queue, p, entries);
1329 return 1;
1330 }
1331 return 0;
1332 }
1333
1334 static void
repo_fail(struct repo * rp)1335 repo_fail(struct repo *rp)
1336 {
1337 /* reset the alarm since code may fallback to rsync */
1338 rp->alarm = getmonotime() + repo_timeout;
1339
1340 if (rp->ta)
1341 http_finish(rp->ta->id, HTTP_FAILED, NULL);
1342 else if (rp->rsync)
1343 rsync_finish(rp->rsync->id, 0);
1344 else if (rp->rrdp)
1345 rrdp_finish(rp->rrdp->id, 0);
1346 else
1347 errx(1, "%s: bad repo", rp->repouri);
1348 }
1349
1350 static void
repo_abort(struct repo * rp)1351 repo_abort(struct repo *rp)
1352 {
1353 /* reset the alarm */
1354 rp->alarm = getmonotime() + repo_timeout;
1355
1356 if (rp->rsync)
1357 rsync_abort(rp->rsync->id);
1358 else if (rp->rrdp)
1359 rrdp_abort(rp->rrdp->id);
1360 else
1361 repo_fail(rp);
1362 }
1363
1364 int
repo_check_timeout(int timeout)1365 repo_check_timeout(int timeout)
1366 {
1367 struct repo *rp;
1368 time_t now;
1369 int diff;
1370
1371 now = getmonotime();
1372
1373 /* check against our runtime deadline first */
1374 if (deadline != 0) {
1375 if (deadline <= now) {
1376 warnx("deadline reached, giving up on repository sync");
1377 nofetch = 1;
1378 /* clear deadline since nofetch is set */
1379 deadline = 0;
1380 /* increase now enough so that all pending repos fail */
1381 now += repo_timeout;
1382 } else {
1383 diff = deadline - now;
1384 diff *= 1000;
1385 if (timeout == INFTIM || diff < timeout)
1386 timeout = diff;
1387 }
1388 }
1389 /* Look up in repository table. (Lookup should actually fail here) */
1390 SLIST_FOREACH(rp, &repos, entry) {
1391 if (repo_state(rp) == REPO_LOADING) {
1392 if (rp->alarm <= now) {
1393 warnx("%s: synchronisation timeout",
1394 rp->repouri);
1395 repo_abort(rp);
1396 } else {
1397 diff = rp->alarm - now;
1398 diff *= 1000;
1399 if (timeout == INFTIM || diff < timeout)
1400 timeout = diff;
1401 }
1402 }
1403 }
1404 return timeout;
1405 }
1406
1407 /*
1408 * Update repo-specific stats when files are going to be moved
1409 * from DIR_TEMP to DIR_VALID.
1410 */
1411 void
repostats_new_files_inc(struct repo * rp,const char * file)1412 repostats_new_files_inc(struct repo *rp, const char *file)
1413 {
1414 if (strncmp(file, ".rsync/", strlen(".rsync/")) == 0 ||
1415 strncmp(file, ".rrdp/", strlen(".rrdp/")) == 0)
1416 rp->repostats.new_files++;
1417 }
1418
1419 /*
1420 * Update stats object of repository depending on rtype and subtype.
1421 */
1422 void
repo_stat_inc(struct repo * rp,int talid,enum rtype type,enum stype subtype)1423 repo_stat_inc(struct repo *rp, int talid, enum rtype type, enum stype subtype)
1424 {
1425 if (rp == NULL)
1426 return;
1427 rp->stats_used[talid] = 1;
1428 switch (type) {
1429 case RTYPE_CER:
1430 if (subtype == STYPE_OK)
1431 rp->stats[talid].certs++;
1432 if (subtype == STYPE_FAIL)
1433 rp->stats[talid].certs_fail++;
1434 if (subtype == STYPE_BGPSEC) {
1435 rp->stats[talid].certs--;
1436 rp->stats[talid].brks++;
1437 }
1438 break;
1439 case RTYPE_MFT:
1440 if (subtype == STYPE_OK)
1441 rp->stats[talid].mfts++;
1442 if (subtype == STYPE_FAIL)
1443 rp->stats[talid].mfts_fail++;
1444 break;
1445 case RTYPE_ROA:
1446 switch (subtype) {
1447 case STYPE_OK:
1448 rp->stats[talid].roas++;
1449 break;
1450 case STYPE_FAIL:
1451 rp->stats[talid].roas_fail++;
1452 break;
1453 case STYPE_INVALID:
1454 rp->stats[talid].roas_invalid++;
1455 break;
1456 case STYPE_TOTAL:
1457 rp->stats[talid].vrps++;
1458 break;
1459 case STYPE_UNIQUE:
1460 rp->stats[talid].vrps_uniqs++;
1461 break;
1462 case STYPE_DEC_UNIQUE:
1463 rp->stats[talid].vrps_uniqs--;
1464 break;
1465 default:
1466 break;
1467 }
1468 break;
1469 case RTYPE_ASPA:
1470 switch (subtype) {
1471 case STYPE_OK:
1472 rp->stats[talid].aspas++;
1473 break;
1474 case STYPE_FAIL:
1475 rp->stats[talid].aspas_fail++;
1476 break;
1477 case STYPE_INVALID:
1478 rp->stats[talid].aspas_invalid++;
1479 break;
1480 case STYPE_TOTAL:
1481 rp->stats[talid].vaps++;
1482 break;
1483 case STYPE_UNIQUE:
1484 rp->stats[talid].vaps_uniqs++;
1485 break;
1486 case STYPE_DEC_UNIQUE:
1487 rp->stats[talid].vaps_uniqs--;
1488 break;
1489 case STYPE_PROVIDERS:
1490 rp->stats[talid].vaps_pas++;
1491 break;
1492 case STYPE_OVERFLOW:
1493 rp->stats[talid].vaps_overflowed++;
1494 break;
1495 default:
1496 break;
1497 }
1498 break;
1499 case RTYPE_SPL:
1500 switch (subtype) {
1501 case STYPE_OK:
1502 rp->stats[talid].spls++;
1503 break;
1504 case STYPE_FAIL:
1505 rp->stats[talid].spls_fail++;
1506 break;
1507 case STYPE_INVALID:
1508 rp->stats[talid].spls_invalid++;
1509 break;
1510 case STYPE_TOTAL:
1511 rp->stats[talid].vsps++;
1512 break;
1513 case STYPE_UNIQUE:
1514 rp->stats[talid].vsps_uniqs++;
1515 break;
1516 case STYPE_DEC_UNIQUE:
1517 rp->stats[talid].vsps_uniqs--;
1518 break;
1519 default:
1520 break;
1521 }
1522 break;
1523 case RTYPE_CRL:
1524 rp->stats[talid].crls++;
1525 break;
1526 case RTYPE_GBR:
1527 rp->stats[talid].gbrs++;
1528 break;
1529 case RTYPE_TAK:
1530 rp->stats[talid].taks++;
1531 break;
1532 default:
1533 break;
1534 }
1535 }
1536
1537 void
repo_tal_stats_collect(void (* cb)(const struct repo *,const struct repotalstats *,void *),int talid,void * arg)1538 repo_tal_stats_collect(void (*cb)(const struct repo *,
1539 const struct repotalstats *, void *), int talid, void *arg)
1540 {
1541 struct repo *rp;
1542
1543 SLIST_FOREACH(rp, &repos, entry) {
1544 if (rp->stats_used[talid])
1545 cb(rp, &rp->stats[talid], arg);
1546 }
1547 }
1548
1549 void
repo_stats_collect(void (* cb)(const struct repo *,const struct repostats *,void *),void * arg)1550 repo_stats_collect(void (*cb)(const struct repo *, const struct repostats *,
1551 void *), void *arg)
1552 {
1553 struct repo *rp;
1554
1555 SLIST_FOREACH(rp, &repos, entry)
1556 cb(rp, &rp->repostats, arg);
1557 }
1558
1559 /*
1560 * Delayed delete of files from RRDP. Since RRDP has no security built-in
1561 * this code needs to check if this RRDP repository is actually allowed to
1562 * remove the file referenced by the URI.
1563 */
1564 static void
repo_cleanup_rrdp(struct filepath_tree * tree)1565 repo_cleanup_rrdp(struct filepath_tree *tree)
1566 {
1567 struct repo *rp;
1568 struct rrdprepo *rr;
1569 struct filepath *fp, *nfp;
1570 char *fn;
1571
1572 SLIST_FOREACH(rp, &repos, entry) {
1573 if (rp->rrdp == NULL)
1574 continue;
1575 rr = (struct rrdprepo *)rp->rrdp;
1576 RB_FOREACH_SAFE(fp, filepath_tree, &rr->deleted, nfp) {
1577 if (!rrdp_uri_valid(rr, fp->file)) {
1578 warnx("%s: external URI %s", rr->notifyuri,
1579 fp->file);
1580 filepath_put(&rr->deleted, fp);
1581 continue;
1582 }
1583 /* try to remove file from rrdp repo ... */
1584 fn = rrdp_filename(rr, fp->file, 0);
1585
1586 if (unlink(fn) == -1) {
1587 if (errno != ENOENT)
1588 warn("unlink %s", fn);
1589 } else {
1590 if (verbose > 1)
1591 logx("deleted %s", fn);
1592 rp->repostats.del_files++;
1593 }
1594 free(fn);
1595
1596 /* ... and from the valid repository if unused. */
1597 fn = rrdp_filename(rr, fp->file, 1);
1598 if (!filepath_exists(tree, fn)) {
1599 if (unlink(fn) == -1) {
1600 if (errno != ENOENT)
1601 warn("unlink %s", fn);
1602 } else {
1603 if (verbose > 1)
1604 logx("deleted %s", fn);
1605 rp->repostats.del_files++;
1606 }
1607 } else
1608 warnx("%s: referenced file supposed to be "
1609 "deleted", fn);
1610
1611 free(fn);
1612 filepath_put(&rr->deleted, fp);
1613 }
1614 }
1615 }
1616
1617 /*
1618 * All files in tree are valid and should be moved to the valid repository
1619 * if not already there. Rename the files to the new path and readd the
1620 * filepath entry with the new path if successful.
1621 */
1622 static void
repo_move_valid(struct filepath_tree * tree)1623 repo_move_valid(struct filepath_tree *tree)
1624 {
1625 struct filepath *fp, *nfp;
1626 size_t rsyncsz = strlen(".rsync/");
1627 size_t rrdpsz = strlen(".rrdp/");
1628 char *fn, *base;
1629
1630 RB_FOREACH_SAFE(fp, filepath_tree, tree, nfp) {
1631 if (strncmp(fp->file, ".rsync/", rsyncsz) != 0 &&
1632 strncmp(fp->file, ".rrdp/", rrdpsz) != 0)
1633 continue; /* not a temporary file path */
1634
1635 if (strncmp(fp->file, ".rsync/", rsyncsz) == 0) {
1636 fn = fp->file + rsyncsz;
1637 } else {
1638 base = strchr(fp->file + rrdpsz, '/');
1639 assert(base != NULL);
1640 fn = base + 1;
1641
1642 /*
1643 * Adjust file last modification time in order to
1644 * minimize RSYNC synchronization load after transport
1645 * failover.
1646 * While serializing RRDP datastructures to disk, set
1647 * the last modified timestamp to the CMS signing-time,
1648 * the X.509 notBefore, or CRL lastUpdate timestamp.
1649 */
1650 if (fp->mtime != 0) {
1651 int ret;
1652 struct timespec ts[2];
1653
1654 ts[0].tv_nsec = UTIME_OMIT;
1655 ts[1].tv_sec = fp->mtime;
1656 ts[1].tv_nsec = 0;
1657 ret = utimensat(AT_FDCWD, fp->file, ts, 0);
1658 if (ret == -1) {
1659 warn("utimensat %s", fp->file);
1660 continue;
1661 }
1662 }
1663 }
1664
1665 if (repo_mkpath(AT_FDCWD, fn) == -1)
1666 continue;
1667
1668 if (rename(fp->file, fn) == -1) {
1669 warn("rename %s", fp->file);
1670 continue;
1671 }
1672
1673 /* switch filepath node to new path */
1674 RB_REMOVE(filepath_tree, tree, fp);
1675 base = fp->file;
1676 if ((fp->file = strdup(fn)) == NULL)
1677 err(1, NULL);
1678 free(base);
1679 if (RB_INSERT(filepath_tree, tree, fp) != NULL)
1680 errx(1, "%s: both possibilities of file present",
1681 fp->file);
1682 }
1683 }
1684
1685 struct fts_state {
1686 enum { BASE_DIR, RSYNC_DIR, RRDP_DIR } type;
1687 struct repo *rp;
1688 } fts_state;
1689
1690 static const struct rrdprepo *
repo_is_rrdp(struct repo * rp)1691 repo_is_rrdp(struct repo *rp)
1692 {
1693 /* check for special pointers first these are not a repository */
1694 if (rp != NULL && rp->rrdp != NULL)
1695 return rp->rrdp->state == REPO_DONE ? rp->rrdp : NULL;
1696 return NULL;
1697 }
1698
1699 static inline char *
skip_dotslash(char * in)1700 skip_dotslash(char *in)
1701 {
1702 if (memcmp(in, "./", 2) == 0)
1703 return in + 2;
1704 return in;
1705 }
1706
1707 static void
repo_cleanup_entry(FTSENT * e,struct filepath_tree * tree,int cachefd)1708 repo_cleanup_entry(FTSENT *e, struct filepath_tree *tree, int cachefd)
1709 {
1710 const struct rrdprepo *rr;
1711 char *path;
1712
1713 path = skip_dotslash(e->fts_path);
1714 switch (e->fts_info) {
1715 case FTS_NSOK:
1716 if (filepath_exists(tree, path)) {
1717 e->fts_parent->fts_number++;
1718 break;
1719 }
1720 if (fts_state.type == RRDP_DIR && fts_state.rp != NULL) {
1721 e->fts_parent->fts_number++;
1722 /* handle rrdp .state files explicitly */
1723 if (e->fts_level == 3 &&
1724 strcmp(e->fts_name, ".state") == 0)
1725 break;
1726 /* can't delete these extra files */
1727 fts_state.rp->repostats.extra_files++;
1728 if (verbose > 1)
1729 logx("superfluous %s", path);
1730 break;
1731 }
1732 rr = repo_is_rrdp(fts_state.rp);
1733 if (rr != NULL) {
1734 struct stat st;
1735 char *fn;
1736
1737 if (asprintf(&fn, "%s/%s", rr->basedir, path) == -1)
1738 err(1, NULL);
1739
1740 /*
1741 * If the file exists in the rrdp dir
1742 * that file is newer and needs to be kept
1743 * so unlink this file instead of moving
1744 * it over the file in the rrdp dir.
1745 */
1746 if (fstatat(cachefd, fn, &st, 0) == 0 &&
1747 S_ISREG(st.st_mode)) {
1748 free(fn);
1749 goto unlink;
1750 }
1751 if (repo_mkpath(cachefd, fn) == 0) {
1752 if (renameat(AT_FDCWD, e->fts_accpath,
1753 cachefd, fn) == -1)
1754 warn("rename %s to %s", path, fn);
1755 else if (verbose > 1)
1756 logx("moved %s", path);
1757 fts_state.rp->repostats.extra_files++;
1758 }
1759 free(fn);
1760 } else {
1761 unlink:
1762 if (unlink(e->fts_accpath) == -1) {
1763 warn("unlink %s", path);
1764 } else if (fts_state.type == RSYNC_DIR) {
1765 /* no need to keep rsync files */
1766 if (verbose > 1)
1767 logx("deleted superfluous %s", path);
1768 if (fts_state.rp != NULL)
1769 fts_state.rp->repostats.del_extra_files++;
1770 else
1771 stats.repo_stats.del_extra_files++;
1772 } else {
1773 if (verbose > 1)
1774 logx("deleted %s", path);
1775 if (fts_state.rp != NULL)
1776 fts_state.rp->repostats.del_files++;
1777 else
1778 stats.repo_stats.del_files++;
1779 }
1780 }
1781 break;
1782 case FTS_D:
1783 if (e->fts_level == FTS_ROOTLEVEL)
1784 fts_state.type = BASE_DIR;
1785 if (e->fts_level == 1) {
1786 /* rpki.example.org or .rrdp / .rsync */
1787 if (strcmp(".rsync", e->fts_name) == 0) {
1788 fts_state.type = RSYNC_DIR;
1789 fts_state.rp = NULL;
1790 } else if (strcmp(".rrdp", e->fts_name) == 0) {
1791 fts_state.type = RRDP_DIR;
1792 fts_state.rp = NULL;
1793 }
1794 }
1795 if (e->fts_level == 2) {
1796 /* rpki.example.org/repository or .rrdp/hashdir */
1797 if (fts_state.type == BASE_DIR)
1798 fts_state.rp = repo_bypath(path);
1799 /*
1800 * special handling for rrdp directories,
1801 * clear them if they are not used anymore but
1802 * only if rrdp is active.
1803 * Look them up just using the hash.
1804 */
1805 if (fts_state.type == RRDP_DIR)
1806 fts_state.rp = repo_rrdp_bypath(path);
1807 }
1808 if (e->fts_level == 3 && fts_state.type == RSYNC_DIR) {
1809 /* .rsync/rpki.example.org/repository */
1810 fts_state.rp = repo_bypath(path + strlen(".rsync/"));
1811 }
1812 break;
1813 case FTS_DP:
1814 if (e->fts_level == FTS_ROOTLEVEL)
1815 break;
1816 if (e->fts_level == 1) {
1817 /* do not remove .rsync and .rrdp */
1818 fts_state.rp = NULL;
1819 if (fts_state.type == RRDP_DIR ||
1820 fts_state.type == RSYNC_DIR)
1821 break;
1822 }
1823
1824 e->fts_parent->fts_number += e->fts_number;
1825
1826 if (e->fts_number == 0) {
1827 if (rmdir(e->fts_accpath) == -1)
1828 warn("rmdir %s", path);
1829 if (fts_state.rp != NULL)
1830 fts_state.rp->repostats.del_dirs++;
1831 else
1832 stats.repo_stats.del_dirs++;
1833 }
1834 break;
1835 case FTS_SL:
1836 case FTS_SLNONE:
1837 warnx("symlink %s", path);
1838 if (unlink(e->fts_accpath) == -1)
1839 warn("unlink %s", path);
1840 stats.repo_stats.del_extra_files++;
1841 break;
1842 case FTS_NS:
1843 case FTS_ERR:
1844 if (e->fts_errno == ENOENT && e->fts_level == FTS_ROOTLEVEL)
1845 break;
1846 warnx("fts_read %s: %s", path, strerror(e->fts_errno));
1847 break;
1848 default:
1849 warnx("fts_read %s: unhandled[%x]", path, e->fts_info);
1850 break;
1851 }
1852 }
1853
1854 void
repo_cleanup(struct filepath_tree * tree,int cachefd)1855 repo_cleanup(struct filepath_tree *tree, int cachefd)
1856 {
1857 char *argv[2] = { ".", NULL };
1858 FTS *fts;
1859 FTSENT *e;
1860
1861 /* first move temp files which have been used to valid dir */
1862 repo_move_valid(tree);
1863 /* then delete files requested by rrdp */
1864 repo_cleanup_rrdp(tree);
1865
1866 if ((fts = fts_open(argv, FTS_PHYSICAL | FTS_NOSTAT, NULL)) == NULL)
1867 err(1, "fts_open");
1868 errno = 0;
1869 while ((e = fts_read(fts)) != NULL) {
1870 repo_cleanup_entry(e, tree, cachefd);
1871 errno = 0;
1872 }
1873 if (errno)
1874 err(1, "fts_read");
1875 if (fts_close(fts) == -1)
1876 err(1, "fts_close");
1877 }
1878
1879 void
repo_free(void)1880 repo_free(void)
1881 {
1882 struct repo *rp;
1883
1884 while ((rp = SLIST_FIRST(&repos)) != NULL) {
1885 SLIST_REMOVE_HEAD(&repos, entry);
1886 free(rp->repouri);
1887 free(rp->notifyuri);
1888 free(rp->basedir);
1889 free(rp);
1890 }
1891
1892 ta_free();
1893 rrdp_free();
1894 rsync_free();
1895 }
1896
1897 /*
1898 * Remove all files and directories under base.
1899 * Do not remove base directory itself and the .state file.
1900 */
1901 static void
remove_contents(char * base)1902 remove_contents(char *base)
1903 {
1904 char *argv[2] = { base, NULL };
1905 FTS *fts;
1906 FTSENT *e;
1907
1908 if ((fts = fts_open(argv, FTS_PHYSICAL | FTS_NOSTAT, NULL)) == NULL)
1909 err(1, "fts_open");
1910 errno = 0;
1911 while ((e = fts_read(fts)) != NULL) {
1912 switch (e->fts_info) {
1913 case FTS_NSOK:
1914 case FTS_SL:
1915 case FTS_SLNONE:
1916 if (e->fts_level == 1 &&
1917 strcmp(e->fts_name, ".state") == 0)
1918 break;
1919 if (unlink(e->fts_accpath) == -1)
1920 warn("unlink %s", e->fts_path);
1921 break;
1922 case FTS_D:
1923 break;
1924 case FTS_DP:
1925 /* keep root directory */
1926 if (e->fts_level == FTS_ROOTLEVEL)
1927 break;
1928 if (rmdir(e->fts_accpath) == -1)
1929 warn("rmdir %s", e->fts_path);
1930 break;
1931 case FTS_NS:
1932 case FTS_ERR:
1933 warnx("fts_read %s: %s", e->fts_path,
1934 strerror(e->fts_errno));
1935 break;
1936 default:
1937 warnx("unhandled[%x] %s", e->fts_info,
1938 e->fts_path);
1939 break;
1940 }
1941 errno = 0;
1942 }
1943 if (errno)
1944 err(1, "fts_read");
1945 if (fts_close(fts) == -1)
1946 err(1, "fts_close");
1947 }
1948