1 /* $OpenBSD: repo.c,v 1.60 2024/06/07 08:22:53 claudio Exp $ */
2 /*
3 * Copyright (c) 2021 Claudio Jeker <claudio@openbsd.org>
4 * Copyright (c) 2019 Kristaps Dzonsons <kristaps@bsd.lv>
5 *
6 * Permission to use, copy, modify, and distribute this software for any
7 * purpose with or without fee is hereby granted, provided that the above
8 * copyright notice and this permission notice appear in all copies.
9 *
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 */
18
19 #include <sys/queue.h>
20 #include <sys/tree.h>
21 #include <sys/types.h>
22 #include <sys/stat.h>
23
24 #include <assert.h>
25 #include <err.h>
26 #include <errno.h>
27 #include <fcntl.h>
28 #include <fts.h>
29 #include <limits.h>
30 #include <poll.h>
31 #include <stdio.h>
32 #include <stdlib.h>
33 #include <string.h>
34 #include <unistd.h>
35
36 #include <imsg.h>
37
38 #include "extern.h"
39
40 extern struct stats stats;
41 extern int rrdpon;
42 extern int repo_timeout;
43 extern time_t deadline;
44 int nofetch;
45
46 enum repo_state {
47 REPO_LOADING = 0,
48 REPO_DONE = 1,
49 REPO_FAILED = -1,
50 };
51
52 /*
53 * A ta, rsync or rrdp repository.
54 * Depending on what is needed the generic repository is backed by
55 * a ta, rsync or rrdp repository. Multiple repositories can use the
56 * same backend.
57 */
58 struct rrdprepo {
59 SLIST_ENTRY(rrdprepo) entry;
60 char *notifyuri;
61 char *basedir;
62 struct filepath_tree deleted;
63 unsigned int id;
64 enum repo_state state;
65 };
66 static SLIST_HEAD(, rrdprepo) rrdprepos = SLIST_HEAD_INITIALIZER(rrdprepos);
67
68 struct rsyncrepo {
69 SLIST_ENTRY(rsyncrepo) entry;
70 char *repouri;
71 char *basedir;
72 unsigned int id;
73 enum repo_state state;
74 };
75 static SLIST_HEAD(, rsyncrepo) rsyncrepos = SLIST_HEAD_INITIALIZER(rsyncrepos);
76
77 struct tarepo {
78 SLIST_ENTRY(tarepo) entry;
79 char *descr;
80 char *basedir;
81 char **uri;
82 size_t urisz;
83 size_t uriidx;
84 unsigned int id;
85 enum repo_state state;
86 };
87 static SLIST_HEAD(, tarepo) tarepos = SLIST_HEAD_INITIALIZER(tarepos);
88
89 struct repo {
90 SLIST_ENTRY(repo) entry;
91 char *repouri;
92 char *notifyuri;
93 char *basedir;
94 const struct rrdprepo *rrdp;
95 const struct rsyncrepo *rsync;
96 const struct tarepo *ta;
97 struct entityq queue; /* files waiting for repo */
98 struct repotalstats stats[TALSZ_MAX];
99 struct repostats repostats;
100 struct timespec start_time;
101 time_t alarm; /* sync timeout */
102 int talid;
103 int stats_used[TALSZ_MAX];
104 unsigned int id; /* identifier */
105 };
106 static SLIST_HEAD(, repo) repos = SLIST_HEAD_INITIALIZER(repos);
107
108 /* counter for unique repo id */
109 unsigned int repoid;
110
111 static struct rsyncrepo *rsync_get(const char *, const char *);
112 static void remove_contents(char *);
113
114 /*
115 * Database of all file path accessed during a run.
116 */
117 struct filepath {
118 RB_ENTRY(filepath) entry;
119 char *file;
120 time_t mtime;
121 unsigned int talmask;
122 };
123
124 static inline int
filepathcmp(const struct filepath * a,const struct filepath * b)125 filepathcmp(const struct filepath *a, const struct filepath *b)
126 {
127 return strcmp(a->file, b->file);
128 }
129
130 RB_PROTOTYPE(filepath_tree, filepath, entry, filepathcmp);
131
132 /*
133 * Functions to lookup which files have been accessed during computation.
134 */
135 int
filepath_add(struct filepath_tree * tree,char * file,int id,time_t mtime)136 filepath_add(struct filepath_tree *tree, char *file, int id, time_t mtime)
137 {
138 struct filepath *fp, *rfp;
139
140 CTASSERT(TALSZ_MAX < 8 * sizeof(fp->talmask));
141 assert(id >= 0 && id < 8 * (int)sizeof(fp->talmask));
142
143 if ((fp = calloc(1, sizeof(*fp))) == NULL)
144 err(1, NULL);
145 if ((fp->file = strdup(file)) == NULL)
146 err(1, NULL);
147 fp->mtime = mtime;
148
149 if ((rfp = RB_INSERT(filepath_tree, tree, fp)) != NULL) {
150 /* already in the tree */
151 free(fp->file);
152 free(fp);
153 if (rfp->talmask & (1 << id))
154 return 0;
155 fp = rfp;
156 }
157 fp->talmask |= (1 << id);
158
159 return 1;
160 }
161
162 /*
163 * Lookup a file path in the tree and return the object if found or NULL.
164 */
165 static struct filepath *
filepath_find(struct filepath_tree * tree,char * file)166 filepath_find(struct filepath_tree *tree, char *file)
167 {
168 struct filepath needle = { .file = file };
169
170 return RB_FIND(filepath_tree, tree, &needle);
171 }
172
173 /*
174 * Returns true if file exists in the tree.
175 */
176 static int
filepath_exists(struct filepath_tree * tree,char * file)177 filepath_exists(struct filepath_tree *tree, char *file)
178 {
179 return filepath_find(tree, file) != NULL;
180 }
181
182 /*
183 * Remove entry from tree and free it.
184 */
185 static void
filepath_put(struct filepath_tree * tree,struct filepath * fp)186 filepath_put(struct filepath_tree *tree, struct filepath *fp)
187 {
188 RB_REMOVE(filepath_tree, tree, fp);
189 free((void *)fp->file);
190 free(fp);
191 }
192
193 /*
194 * Free all elements of a filepath tree.
195 */
196 static void
filepath_free(struct filepath_tree * tree)197 filepath_free(struct filepath_tree *tree)
198 {
199 struct filepath *fp, *nfp;
200
201 RB_FOREACH_SAFE(fp, filepath_tree, tree, nfp)
202 filepath_put(tree, fp);
203 }
204
205 RB_GENERATE(filepath_tree, filepath, entry, filepathcmp);
206
207 /*
208 * Function to hash a string into a unique directory name.
209 * Returned hash needs to be freed.
210 */
211 static char *
hash_dir(const char * uri)212 hash_dir(const char *uri)
213 {
214 unsigned char m[SHA256_DIGEST_LENGTH];
215
216 SHA256(uri, strlen(uri), m);
217 return hex_encode(m, sizeof(m));
218 }
219
220 /*
221 * Function to build the directory name based on URI and a directory
222 * as prefix. Skip the proto:// in URI but keep everything else.
223 */
224 static char *
repo_dir(const char * uri,const char * dir,int hash)225 repo_dir(const char *uri, const char *dir, int hash)
226 {
227 const char *local;
228 char *out, *hdir = NULL;
229
230 if (hash) {
231 local = hdir = hash_dir(uri);
232 } else {
233 local = strchr(uri, ':');
234 if (local != NULL)
235 local += strlen("://");
236 else
237 local = uri;
238 }
239
240 if (dir == NULL) {
241 if ((out = strdup(local)) == NULL)
242 err(1, NULL);
243 } else {
244 if (asprintf(&out, "%s/%s", dir, local) == -1)
245 err(1, NULL);
246 }
247
248 free(hdir);
249 return out;
250 }
251
252 /*
253 * Function to create all missing directories to a path.
254 * This functions alters the path temporarily.
255 */
256 static int
repo_mkpath(int fd,char * file)257 repo_mkpath(int fd, char *file)
258 {
259 char *slash;
260
261 /* build directory hierarchy */
262 slash = strrchr(file, '/');
263 assert(slash != NULL);
264 *slash = '\0';
265 if (mkpathat(fd, file) == -1) {
266 warn("mkpath %s", file);
267 return -1;
268 }
269 *slash = '/';
270 return 0;
271 }
272
273 /*
274 * Return the state of a repository.
275 */
276 static enum repo_state
repo_state(const struct repo * rp)277 repo_state(const struct repo *rp)
278 {
279 if (rp->ta)
280 return rp->ta->state;
281 if (rp->rsync)
282 return rp->rsync->state;
283 if (rp->rrdp)
284 return rp->rrdp->state;
285 /* No backend so sync is by definition done. */
286 return REPO_DONE;
287 }
288
289 /*
290 * Function called once a repository is done with the sync. Either
291 * successfully or after failure.
292 */
293 static void
repo_done(const void * vp,int ok)294 repo_done(const void *vp, int ok)
295 {
296 struct repo *rp;
297 struct timespec flush_time;
298
299 SLIST_FOREACH(rp, &repos, entry) {
300 if (vp != rp->ta && vp != rp->rsync && vp != rp->rrdp)
301 continue;
302
303 /* for rrdp try to fall back to rsync */
304 if (vp == rp->rrdp && !ok && !nofetch) {
305 rp->rrdp = NULL;
306 rp->rsync = rsync_get(rp->repouri, rp->basedir);
307 /* need to check if it was already loaded */
308 if (repo_state(rp) == REPO_LOADING)
309 continue;
310 }
311
312 entityq_flush(&rp->queue, rp);
313 clock_gettime(CLOCK_MONOTONIC, &flush_time);
314 timespecsub(&flush_time, &rp->start_time,
315 &rp->repostats.sync_time);
316 }
317 }
318
319 /*
320 * Build TA file name based on the repo info.
321 * If temp is set add Xs for mkostemp.
322 */
323 static char *
ta_filename(const struct tarepo * tr)324 ta_filename(const struct tarepo *tr)
325 {
326 const char *file;
327 char *nfile;
328
329 /* does not matter which URI, all end with same filename */
330 file = strrchr(tr->uri[0], '/');
331 assert(file);
332
333 if (asprintf(&nfile, "%s%s", tr->basedir, file) == -1)
334 err(1, NULL);
335
336 return nfile;
337 }
338
339 static void
ta_fetch(struct tarepo * tr)340 ta_fetch(struct tarepo *tr)
341 {
342 if (!rrdpon) {
343 for (; tr->uriidx < tr->urisz; tr->uriidx++) {
344 if (strncasecmp(tr->uri[tr->uriidx],
345 RSYNC_PROTO, RSYNC_PROTO_LEN) == 0)
346 break;
347 }
348 }
349
350 if (tr->uriidx >= tr->urisz) {
351 tr->state = REPO_FAILED;
352 logx("ta/%s: fallback to cache", tr->descr);
353
354 repo_done(tr, 0);
355 return;
356 }
357
358 logx("ta/%s: pulling from %s", tr->descr, tr->uri[tr->uriidx]);
359
360 if (strncasecmp(tr->uri[tr->uriidx], RSYNC_PROTO,
361 RSYNC_PROTO_LEN) == 0) {
362 /*
363 * Create destination location.
364 * Build up the tree to this point.
365 */
366 rsync_fetch(tr->id, tr->uri[tr->uriidx], tr->basedir, NULL);
367 } else {
368 char *temp;
369 int fd;
370
371 temp = ta_filename(tr);
372 fd = open(temp,
373 O_WRONLY | O_CREAT | O_TRUNC | O_NOFOLLOW | O_CLOEXEC,
374 S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
375 if (fd == -1) {
376 warn("open: %s", temp);
377 free(temp);
378 http_finish(tr->id, HTTP_FAILED, NULL);
379 return;
380 }
381
382 free(temp);
383 http_fetch(tr->id, tr->uri[tr->uriidx], NULL, fd);
384 }
385 }
386
387 static struct tarepo *
ta_get(struct tal * tal)388 ta_get(struct tal *tal)
389 {
390 struct tarepo *tr;
391
392 /* no need to look for possible other repo */
393
394 if ((tr = calloc(1, sizeof(*tr))) == NULL)
395 err(1, NULL);
396
397 tr->id = ++repoid;
398 SLIST_INSERT_HEAD(&tarepos, tr, entry);
399
400 if ((tr->descr = strdup(tal->descr)) == NULL)
401 err(1, NULL);
402 tr->basedir = repo_dir(tal->descr, ".ta", 0);
403
404 /* create base directory */
405 if (mkpath(tr->basedir) == -1) {
406 warn("mkpath %s", tr->basedir);
407 tr->state = REPO_FAILED;
408 repo_done(tr, 0);
409 return tr;
410 }
411
412 /* steal URI information from TAL */
413 tr->urisz = tal->urisz;
414 tr->uri = tal->uri;
415 tal->urisz = 0;
416 tal->uri = NULL;
417
418 ta_fetch(tr);
419
420 return tr;
421 }
422
423 static struct tarepo *
ta_find(unsigned int id)424 ta_find(unsigned int id)
425 {
426 struct tarepo *tr;
427
428 SLIST_FOREACH(tr, &tarepos, entry)
429 if (id == tr->id)
430 break;
431 return tr;
432 }
433
434 static void
ta_free(void)435 ta_free(void)
436 {
437 struct tarepo *tr;
438
439 while ((tr = SLIST_FIRST(&tarepos)) != NULL) {
440 SLIST_REMOVE_HEAD(&tarepos, entry);
441 free(tr->descr);
442 free(tr->basedir);
443 free(tr->uri);
444 free(tr);
445 }
446 }
447
448 static struct rsyncrepo *
rsync_get(const char * uri,const char * validdir)449 rsync_get(const char *uri, const char *validdir)
450 {
451 struct rsyncrepo *rr;
452 char *repo;
453
454 if ((repo = rsync_base_uri(uri)) == NULL)
455 errx(1, "bad caRepository URI: %s", uri);
456
457 SLIST_FOREACH(rr, &rsyncrepos, entry)
458 if (strcmp(rr->repouri, repo) == 0) {
459 free(repo);
460 return rr;
461 }
462
463 if ((rr = calloc(1, sizeof(*rr))) == NULL)
464 err(1, NULL);
465
466 rr->id = ++repoid;
467 SLIST_INSERT_HEAD(&rsyncrepos, rr, entry);
468
469 rr->repouri = repo;
470 rr->basedir = repo_dir(repo, ".rsync", 0);
471
472 /* create base directory */
473 if (mkpath(rr->basedir) == -1) {
474 warn("mkpath %s", rr->basedir);
475 rsync_finish(rr->id, 0);
476 return rr;
477 }
478
479 logx("%s: pulling from %s", rr->basedir, rr->repouri);
480 rsync_fetch(rr->id, rr->repouri, rr->basedir, validdir);
481
482 return rr;
483 }
484
485 static struct rsyncrepo *
rsync_find(unsigned int id)486 rsync_find(unsigned int id)
487 {
488 struct rsyncrepo *rr;
489
490 SLIST_FOREACH(rr, &rsyncrepos, entry)
491 if (id == rr->id)
492 break;
493 return rr;
494 }
495
496 static void
rsync_free(void)497 rsync_free(void)
498 {
499 struct rsyncrepo *rr;
500
501 while ((rr = SLIST_FIRST(&rsyncrepos)) != NULL) {
502 SLIST_REMOVE_HEAD(&rsyncrepos, entry);
503 free(rr->repouri);
504 free(rr->basedir);
505 free(rr);
506 }
507 }
508
509 /*
510 * Build local file name base on the URI and the rrdprepo info.
511 */
512 static char *
rrdp_filename(const struct rrdprepo * rr,const char * uri,int valid)513 rrdp_filename(const struct rrdprepo *rr, const char *uri, int valid)
514 {
515 char *nfile;
516 const char *dir = rr->basedir;
517
518 if (!valid_uri(uri, strlen(uri), RSYNC_PROTO))
519 errx(1, "%s: bad URI %s", rr->basedir, uri);
520 uri += RSYNC_PROTO_LEN; /* skip proto */
521 if (valid) {
522 if ((nfile = strdup(uri)) == NULL)
523 err(1, NULL);
524 } else {
525 if (asprintf(&nfile, "%s/%s", dir, uri) == -1)
526 err(1, NULL);
527 }
528 return nfile;
529 }
530
531 /*
532 * Build RRDP state file name based on the repo info.
533 * If temp is set add Xs for mkostemp.
534 */
535 static char *
rrdp_state_filename(const struct rrdprepo * rr,int temp)536 rrdp_state_filename(const struct rrdprepo *rr, int temp)
537 {
538 char *nfile;
539
540 if (asprintf(&nfile, "%s/.state%s", rr->basedir,
541 temp ? ".XXXXXXXX" : "") == -1)
542 err(1, NULL);
543
544 return nfile;
545 }
546
547 static struct rrdprepo *
rrdp_find(unsigned int id)548 rrdp_find(unsigned int id)
549 {
550 struct rrdprepo *rr;
551
552 SLIST_FOREACH(rr, &rrdprepos, entry)
553 if (id == rr->id)
554 break;
555 return rr;
556 }
557
558 static void
rrdp_free(void)559 rrdp_free(void)
560 {
561 struct rrdprepo *rr;
562
563 while ((rr = SLIST_FIRST(&rrdprepos)) != NULL) {
564 SLIST_REMOVE_HEAD(&rrdprepos, entry);
565
566 free(rr->notifyuri);
567 free(rr->basedir);
568
569 filepath_free(&rr->deleted);
570
571 free(rr);
572 }
573 }
574
575 /*
576 * Check if a directory is an active rrdp repository.
577 * Returns 1 if found else 0.
578 */
579 static struct repo *
repo_rrdp_bypath(const char * dir)580 repo_rrdp_bypath(const char *dir)
581 {
582 struct repo *rp;
583
584 SLIST_FOREACH(rp, &repos, entry) {
585 if (rp->rrdp == NULL)
586 continue;
587 if (strcmp(dir, rp->rrdp->basedir) == 0)
588 return rp;
589 }
590 return NULL;
591 }
592
593 /*
594 * Check if the URI is actually covered by one of the repositories
595 * that depend on this RRDP repository.
596 * Returns 1 if the URI is valid, 0 if no repouri matches the URI.
597 */
598 static int
rrdp_uri_valid(struct rrdprepo * rr,const char * uri)599 rrdp_uri_valid(struct rrdprepo *rr, const char *uri)
600 {
601 struct repo *rp;
602
603 SLIST_FOREACH(rp, &repos, entry) {
604 if (rp->rrdp != rr)
605 continue;
606 if (strncmp(uri, rp->repouri, strlen(rp->repouri)) == 0)
607 return 1;
608 }
609 return 0;
610 }
611
612 /*
613 * Allocate and insert a new repository.
614 */
615 static struct repo *
repo_alloc(int talid)616 repo_alloc(int talid)
617 {
618 struct repo *rp;
619
620 if ((rp = calloc(1, sizeof(*rp))) == NULL)
621 err(1, NULL);
622
623 rp->id = ++repoid;
624 rp->talid = talid;
625 rp->alarm = getmonotime() + repo_timeout;
626 TAILQ_INIT(&rp->queue);
627 SLIST_INSERT_HEAD(&repos, rp, entry);
628 clock_gettime(CLOCK_MONOTONIC, &rp->start_time);
629
630 stats.repos++;
631 return rp;
632 }
633
634 /*
635 * Parse the RRDP state file if it exists and set the session struct
636 * based on that information.
637 */
638 static struct rrdp_session *
rrdp_session_parse(const struct rrdprepo * rr)639 rrdp_session_parse(const struct rrdprepo *rr)
640 {
641 FILE *f;
642 struct rrdp_session *state;
643 int fd, ln = 0, deltacnt = 0;
644 const char *errstr;
645 char *line = NULL, *file;
646 size_t len = 0;
647 ssize_t n;
648
649 if ((state = calloc(1, sizeof(*state))) == NULL)
650 err(1, NULL);
651
652 file = rrdp_state_filename(rr, 0);
653 if ((fd = open(file, O_RDONLY)) == -1) {
654 if (errno != ENOENT)
655 warn("%s: open state file", rr->basedir);
656 free(file);
657 return state;
658 }
659 free(file);
660 f = fdopen(fd, "r");
661 if (f == NULL)
662 err(1, "fdopen");
663
664 while ((n = getline(&line, &len, f)) != -1) {
665 if (line[n - 1] == '\n')
666 line[n - 1] = '\0';
667 switch (ln) {
668 case 0:
669 if ((state->session_id = strdup(line)) == NULL)
670 err(1, NULL);
671 break;
672 case 1:
673 state->serial = strtonum(line, 1, LLONG_MAX, &errstr);
674 if (errstr)
675 goto fail;
676 break;
677 case 2:
678 if (strcmp(line, "-") == 0)
679 break;
680 if ((state->last_mod = strdup(line)) == NULL)
681 err(1, NULL);
682 break;
683 default:
684 if (deltacnt >= MAX_RRDP_DELTAS)
685 goto fail;
686 if ((state->deltas[deltacnt++] = strdup(line)) == NULL)
687 err(1, NULL);
688 break;
689 }
690 ln++;
691 }
692
693 if (ferror(f))
694 goto fail;
695 fclose(f);
696 free(line);
697 return state;
698
699 fail:
700 warnx("%s: troubles reading state file", rr->basedir);
701 fclose(f);
702 free(line);
703 free(state->session_id);
704 free(state->last_mod);
705 memset(state, 0, sizeof(*state));
706 return state;
707 }
708
709 /*
710 * Carefully write the RRDP session state file back.
711 */
712 void
rrdp_session_save(unsigned int id,struct rrdp_session * state)713 rrdp_session_save(unsigned int id, struct rrdp_session *state)
714 {
715 struct rrdprepo *rr;
716 char *temp, *file;
717 FILE *f = NULL;
718 int fd, i;
719
720 rr = rrdp_find(id);
721 if (rr == NULL)
722 errx(1, "non-existent rrdp repo %u", id);
723
724 file = rrdp_state_filename(rr, 0);
725 temp = rrdp_state_filename(rr, 1);
726
727 if ((fd = mkostemp(temp, O_CLOEXEC)) == -1)
728 goto fail;
729 (void)fchmod(fd, 0644);
730 f = fdopen(fd, "w");
731 if (f == NULL)
732 err(1, "fdopen");
733
734 /* write session state file out */
735 if (fprintf(f, "%s\n%lld\n", state->session_id,
736 state->serial) < 0)
737 goto fail;
738
739 if (state->last_mod != NULL) {
740 if (fprintf(f, "%s\n", state->last_mod) < 0)
741 goto fail;
742 } else {
743 if (fprintf(f, "-\n") < 0)
744 goto fail;
745 }
746 for (i = 0; i < MAX_RRDP_DELTAS && state->deltas[i] != NULL; i++) {
747 if (fprintf(f, "%s\n", state->deltas[i]) < 0)
748 goto fail;
749 }
750 if (fclose(f) != 0) {
751 f = NULL;
752 goto fail;
753 }
754
755 if (rename(temp, file) == -1) {
756 warn("%s: rename %s to %s", rr->basedir, temp, file);
757 unlink(temp);
758 }
759
760 free(temp);
761 free(file);
762 return;
763
764 fail:
765 warn("%s: save state to %s", rr->basedir, temp);
766 if (f != NULL)
767 fclose(f);
768 unlink(temp);
769 free(temp);
770 free(file);
771 }
772
773 /*
774 * Free an rrdp_session pointer. Safe to call with NULL.
775 */
776 void
rrdp_session_free(struct rrdp_session * s)777 rrdp_session_free(struct rrdp_session *s)
778 {
779 size_t i;
780
781 if (s == NULL)
782 return;
783 free(s->session_id);
784 free(s->last_mod);
785 for (i = 0; i < sizeof(s->deltas) / sizeof(s->deltas[0]); i++)
786 free(s->deltas[i]);
787 free(s);
788 }
789
790 void
rrdp_session_buffer(struct ibuf * b,const struct rrdp_session * s)791 rrdp_session_buffer(struct ibuf *b, const struct rrdp_session *s)
792 {
793 size_t i;
794
795 io_str_buffer(b, s->session_id);
796 io_simple_buffer(b, &s->serial, sizeof(s->serial));
797 io_str_buffer(b, s->last_mod);
798 for (i = 0; i < sizeof(s->deltas) / sizeof(s->deltas[0]); i++)
799 io_str_buffer(b, s->deltas[i]);
800 }
801
802 struct rrdp_session *
rrdp_session_read(struct ibuf * b)803 rrdp_session_read(struct ibuf *b)
804 {
805 struct rrdp_session *s;
806 size_t i;
807
808 if ((s = calloc(1, sizeof(*s))) == NULL)
809 err(1, NULL);
810
811 io_read_str(b, &s->session_id);
812 io_read_buf(b, &s->serial, sizeof(s->serial));
813 io_read_str(b, &s->last_mod);
814 for (i = 0; i < sizeof(s->deltas) / sizeof(s->deltas[0]); i++)
815 io_read_str(b, &s->deltas[i]);
816
817 return s;
818 }
819
820 static struct rrdprepo *
rrdp_get(const char * uri)821 rrdp_get(const char *uri)
822 {
823 struct rrdp_session *state;
824 struct rrdprepo *rr;
825
826 SLIST_FOREACH(rr, &rrdprepos, entry)
827 if (strcmp(rr->notifyuri, uri) == 0) {
828 if (rr->state == REPO_FAILED)
829 return NULL;
830 return rr;
831 }
832
833 if ((rr = calloc(1, sizeof(*rr))) == NULL)
834 err(1, NULL);
835
836 rr->id = ++repoid;
837 SLIST_INSERT_HEAD(&rrdprepos, rr, entry);
838
839 if ((rr->notifyuri = strdup(uri)) == NULL)
840 err(1, NULL);
841 rr->basedir = repo_dir(uri, ".rrdp", 1);
842
843 RB_INIT(&rr->deleted);
844
845 /* create base directory */
846 if (mkpath(rr->basedir) == -1) {
847 warn("mkpath %s", rr->basedir);
848 rrdp_finish(rr->id, 0);
849 return rr;
850 }
851
852 /* parse state and start the sync */
853 state = rrdp_session_parse(rr);
854 rrdp_fetch(rr->id, rr->notifyuri, rr->notifyuri, state);
855 rrdp_session_free(state);
856
857 logx("%s: pulling from %s", rr->notifyuri, "network");
858
859 return rr;
860 }
861
862 /*
863 * Remove RRDP repo and start over.
864 */
865 void
rrdp_clear(unsigned int id)866 rrdp_clear(unsigned int id)
867 {
868 struct rrdprepo *rr;
869
870 rr = rrdp_find(id);
871 if (rr == NULL)
872 errx(1, "non-existent rrdp repo %u", id);
873
874 /* remove rrdp repository contents */
875 remove_contents(rr->basedir);
876 }
877
878 /*
879 * Write a file into the temporary RRDP dir but only after checking
880 * its hash (if required). The function also makes sure that the file
881 * tracking is properly adjusted.
882 * Returns 1 on success, 0 if the repo is corrupt, -1 on IO error
883 */
884 int
rrdp_handle_file(unsigned int id,enum publish_type pt,char * uri,char * hash,size_t hlen,char * data,size_t dlen)885 rrdp_handle_file(unsigned int id, enum publish_type pt, char *uri,
886 char *hash, size_t hlen, char *data, size_t dlen)
887 {
888 struct rrdprepo *rr;
889 struct filepath *fp;
890 ssize_t s;
891 char *fn = NULL;
892 int fd = -1, try = 0, deleted = 0;
893 int flags;
894
895 rr = rrdp_find(id);
896 if (rr == NULL)
897 errx(1, "non-existent rrdp repo %u", id);
898 if (rr->state == REPO_FAILED)
899 return -1;
900
901 /* check hash of original file for updates and deletes */
902 if (pt == PUB_UPD || pt == PUB_DEL) {
903 if (filepath_exists(&rr->deleted, uri)) {
904 warnx("%s: already deleted", uri);
905 return 0;
906 }
907 /* try to open file first in rrdp then in valid repo */
908 do {
909 free(fn);
910 if ((fn = rrdp_filename(rr, uri, try++)) == NULL)
911 return 0;
912 fd = open(fn, O_RDONLY);
913 } while (fd == -1 && try < 2);
914
915 if (!valid_filehash(fd, hash, hlen)) {
916 warnx("%s: bad file digest for %s", rr->notifyuri, fn);
917 free(fn);
918 return 0;
919 }
920 free(fn);
921 }
922
923 /* write new content or mark uri as deleted. */
924 if (pt == PUB_DEL) {
925 filepath_add(&rr->deleted, uri, 0, 0);
926 } else {
927 fp = filepath_find(&rr->deleted, uri);
928 if (fp != NULL) {
929 filepath_put(&rr->deleted, fp);
930 deleted = 1;
931 }
932
933 /* add new file to rrdp dir */
934 if ((fn = rrdp_filename(rr, uri, 0)) == NULL)
935 return 0;
936
937 if (repo_mkpath(AT_FDCWD, fn) == -1)
938 goto fail;
939
940 flags = O_WRONLY|O_CREAT|O_TRUNC;
941 if (pt == PUB_ADD && !deleted)
942 flags |= O_EXCL;
943 fd = open(fn, flags, 0644);
944 if (fd == -1) {
945 if (errno == EEXIST) {
946 warnx("%s: duplicate publish element for %s",
947 rr->notifyuri, fn);
948 free(fn);
949 return 0;
950 }
951 warn("open %s", fn);
952 goto fail;
953 }
954
955 if ((s = write(fd, data, dlen)) == -1) {
956 warn("write %s", fn);
957 goto fail;
958 }
959 close(fd);
960 if ((size_t)s != dlen) /* impossible */
961 errx(1, "short write %s", fn);
962 free(fn);
963 }
964
965 return 1;
966
967 fail:
968 rr->state = REPO_FAILED;
969 if (fd != -1)
970 close(fd);
971 free(fn);
972 return -1;
973 }
974
975 /*
976 * RSYNC sync finished, either with or without success.
977 */
978 void
rsync_finish(unsigned int id,int ok)979 rsync_finish(unsigned int id, int ok)
980 {
981 struct rsyncrepo *rr;
982 struct tarepo *tr;
983
984 tr = ta_find(id);
985 if (tr != NULL) {
986 /* repository changed state already, ignore request */
987 if (tr->state != REPO_LOADING)
988 return;
989 if (ok) {
990 logx("ta/%s: loaded from network", tr->descr);
991 stats.rsync_repos++;
992 tr->state = REPO_DONE;
993 repo_done(tr, 1);
994 } else {
995 warnx("ta/%s: load from network failed", tr->descr);
996 stats.rsync_fails++;
997 tr->uriidx++;
998 ta_fetch(tr);
999 }
1000 return;
1001 }
1002
1003 rr = rsync_find(id);
1004 if (rr == NULL)
1005 errx(1, "unknown rsync repo %u", id);
1006 /* repository changed state already, ignore request */
1007 if (rr->state != REPO_LOADING)
1008 return;
1009
1010 if (ok) {
1011 logx("%s: loaded from network", rr->basedir);
1012 stats.rsync_repos++;
1013 rr->state = REPO_DONE;
1014 } else {
1015 warnx("%s: load from network failed, fallback to cache",
1016 rr->basedir);
1017 stats.rsync_fails++;
1018 rr->state = REPO_FAILED;
1019 /* clear rsync repo since it failed */
1020 remove_contents(rr->basedir);
1021 }
1022
1023 repo_done(rr, ok);
1024 }
1025
1026 /*
1027 * RRDP sync finished, either with or without success.
1028 */
1029 void
rrdp_finish(unsigned int id,int ok)1030 rrdp_finish(unsigned int id, int ok)
1031 {
1032 struct rrdprepo *rr;
1033
1034 rr = rrdp_find(id);
1035 if (rr == NULL)
1036 errx(1, "unknown RRDP repo %u", id);
1037 /* repository changed state already, ignore request */
1038 if (rr->state != REPO_LOADING)
1039 return;
1040
1041 if (ok) {
1042 logx("%s: loaded from network", rr->notifyuri);
1043 stats.rrdp_repos++;
1044 rr->state = REPO_DONE;
1045 } else {
1046 warnx("%s: load from network failed, fallback to %s",
1047 rr->notifyuri, nofetch ? "cache" : "rsync");
1048 stats.rrdp_fails++;
1049 rr->state = REPO_FAILED;
1050 /* clear the RRDP repo since it failed */
1051 remove_contents(rr->basedir);
1052 /* also clear the list of deleted files */
1053 filepath_free(&rr->deleted);
1054 }
1055
1056 repo_done(rr, ok);
1057 }
1058
1059 /*
1060 * Handle responses from the http process. For TA file, either rename
1061 * or delete the temporary file. For RRDP requests relay the request
1062 * over to the rrdp process.
1063 */
1064 void
http_finish(unsigned int id,enum http_result res,const char * last_mod)1065 http_finish(unsigned int id, enum http_result res, const char *last_mod)
1066 {
1067 struct tarepo *tr;
1068
1069 tr = ta_find(id);
1070 if (tr == NULL) {
1071 /* not a TA fetch therefore RRDP */
1072 rrdp_http_done(id, res, last_mod);
1073 return;
1074 }
1075
1076 /* repository changed state already, ignore request */
1077 if (tr->state != REPO_LOADING)
1078 return;
1079
1080 /* Move downloaded TA file into place, or unlink on failure. */
1081 if (res == HTTP_OK) {
1082 logx("ta/%s: loaded from network", tr->descr);
1083 tr->state = REPO_DONE;
1084 stats.http_repos++;
1085 repo_done(tr, 1);
1086 } else {
1087 remove_contents(tr->basedir);
1088
1089 tr->uriidx++;
1090 warnx("ta/%s: load from network failed", tr->descr);
1091 ta_fetch(tr);
1092 }
1093 }
1094
1095 /*
1096 * Look up a trust anchor, queueing it for download if not found.
1097 */
1098 struct repo *
ta_lookup(int id,struct tal * tal)1099 ta_lookup(int id, struct tal *tal)
1100 {
1101 struct repo *rp;
1102
1103 if (tal->urisz == 0)
1104 errx(1, "TAL %s has no URI", tal->descr);
1105
1106 /* Look up in repository table. (Lookup should actually fail here) */
1107 SLIST_FOREACH(rp, &repos, entry) {
1108 if (strcmp(rp->repouri, tal->uri[0]) == 0)
1109 return rp;
1110 }
1111
1112 rp = repo_alloc(id);
1113 rp->basedir = repo_dir(tal->descr, "ta", 0);
1114 if ((rp->repouri = strdup(tal->uri[0])) == NULL)
1115 err(1, NULL);
1116
1117 /* check if sync disabled ... */
1118 if (noop) {
1119 logx("%s: using cache", rp->basedir);
1120 entityq_flush(&rp->queue, rp);
1121 return rp;
1122 }
1123
1124 /* try to create base directory */
1125 if (mkpath(rp->basedir) == -1)
1126 warn("mkpath %s", rp->basedir);
1127
1128 rp->ta = ta_get(tal);
1129
1130 /* need to check if it was already loaded */
1131 if (repo_state(rp) != REPO_LOADING)
1132 entityq_flush(&rp->queue, rp);
1133
1134 return rp;
1135 }
1136
1137 /*
1138 * Look up a repository, queueing it for discovery if not found.
1139 */
1140 struct repo *
repo_lookup(int talid,const char * uri,const char * notify)1141 repo_lookup(int talid, const char *uri, const char *notify)
1142 {
1143 struct repo *rp;
1144 char *repouri;
1145
1146 if ((repouri = rsync_base_uri(uri)) == NULL)
1147 errx(1, "bad caRepository URI: %s", uri);
1148
1149 /* Look up in repository table. */
1150 SLIST_FOREACH(rp, &repos, entry) {
1151 if (strcmp(rp->repouri, repouri) != 0)
1152 continue;
1153 if (rp->notifyuri != NULL) {
1154 if (notify == NULL)
1155 continue;
1156 if (strcmp(rp->notifyuri, notify) != 0)
1157 continue;
1158 } else if (notify != NULL)
1159 continue;
1160 /* found matching repo */
1161 free(repouri);
1162 return rp;
1163 }
1164
1165 rp = repo_alloc(talid);
1166 rp->basedir = repo_dir(repouri, NULL, 0);
1167 rp->repouri = repouri;
1168 if (notify != NULL)
1169 if ((rp->notifyuri = strdup(notify)) == NULL)
1170 err(1, NULL);
1171
1172 if (++talrepocnt[talid] >= MAX_REPO_PER_TAL) {
1173 if (talrepocnt[talid] == MAX_REPO_PER_TAL)
1174 warnx("too many repositories under %s", tals[talid]);
1175 nofetch = 1;
1176 }
1177
1178 /* check if sync disabled ... */
1179 if (noop || nofetch) {
1180 logx("%s: using cache", rp->basedir);
1181 entityq_flush(&rp->queue, rp);
1182 return rp;
1183 }
1184
1185 /* try to create base directory */
1186 if (mkpath(rp->basedir) == -1)
1187 warn("mkpath %s", rp->basedir);
1188
1189 /* ... else try RRDP first if available then rsync */
1190 if (notify != NULL)
1191 rp->rrdp = rrdp_get(notify);
1192 if (rp->rrdp == NULL)
1193 rp->rsync = rsync_get(uri, rp->basedir);
1194
1195 /* need to check if it was already loaded */
1196 if (repo_state(rp) != REPO_LOADING)
1197 entityq_flush(&rp->queue, rp);
1198
1199 return rp;
1200 }
1201
1202 /*
1203 * Find repository by identifier.
1204 */
1205 struct repo *
repo_byid(unsigned int id)1206 repo_byid(unsigned int id)
1207 {
1208 struct repo *rp;
1209
1210 SLIST_FOREACH(rp, &repos, entry) {
1211 if (rp->id == id)
1212 return rp;
1213 }
1214 return NULL;
1215 }
1216
1217 /*
1218 * Find repository by base path.
1219 */
1220 static struct repo *
repo_bypath(const char * path)1221 repo_bypath(const char *path)
1222 {
1223 struct repo *rp;
1224
1225 SLIST_FOREACH(rp, &repos, entry) {
1226 if (strcmp(rp->basedir, path) == 0)
1227 return rp;
1228 }
1229 return NULL;
1230 }
1231
1232 /*
1233 * Return the repository base or alternate directory.
1234 * Returned string must be freed by caller.
1235 */
1236 char *
repo_basedir(const struct repo * rp,int wantvalid)1237 repo_basedir(const struct repo *rp, int wantvalid)
1238 {
1239 char *path = NULL;
1240
1241 if (!wantvalid) {
1242 if (rp->ta) {
1243 if ((path = strdup(rp->ta->basedir)) == NULL)
1244 err(1, NULL);
1245 } else if (rp->rsync) {
1246 if ((path = strdup(rp->rsync->basedir)) == NULL)
1247 err(1, NULL);
1248 } else if (rp->rrdp) {
1249 path = rrdp_filename(rp->rrdp, rp->repouri, 0);
1250 } else
1251 path = NULL; /* only valid repo available */
1252 } else if (rp->basedir != NULL) {
1253 if ((path = strdup(rp->basedir)) == NULL)
1254 err(1, NULL);
1255 }
1256
1257 return path;
1258 }
1259
1260 /*
1261 * Return the repository identifier.
1262 */
1263 unsigned int
repo_id(const struct repo * rp)1264 repo_id(const struct repo *rp)
1265 {
1266 return rp->id;
1267 }
1268
1269 /*
1270 * Return the repository URI.
1271 */
1272 const char *
repo_uri(const struct repo * rp)1273 repo_uri(const struct repo *rp)
1274 {
1275 return rp->repouri;
1276 }
1277
1278 /*
1279 * Return the repository URI.
1280 */
1281 void
repo_fetch_uris(const struct repo * rp,const char ** carepo,const char ** notifyuri)1282 repo_fetch_uris(const struct repo *rp, const char **carepo,
1283 const char **notifyuri)
1284 {
1285 *carepo = rp->repouri;
1286 *notifyuri = rp->notifyuri;
1287 }
1288
1289 /*
1290 * Return 1 if repository is synced else 0.
1291 */
1292 int
repo_synced(const struct repo * rp)1293 repo_synced(const struct repo *rp)
1294 {
1295 if (repo_state(rp) == REPO_DONE &&
1296 !(rp->rrdp == NULL && rp->rsync == NULL && rp->ta == NULL))
1297 return 1;
1298 return 0;
1299 }
1300
1301 /*
1302 * Return the protocol string "rrdp", "rsync", "https" which was used to sync.
1303 * Result is only correct if repository was properly synced.
1304 */
1305 const char *
repo_proto(const struct repo * rp)1306 repo_proto(const struct repo *rp)
1307 {
1308
1309 if (rp->ta != NULL) {
1310 const struct tarepo *tr = rp->ta;
1311 if (tr->uriidx < tr->urisz &&
1312 strncasecmp(tr->uri[tr->uriidx], RSYNC_PROTO,
1313 RSYNC_PROTO_LEN) == 0)
1314 return "rsync";
1315 else
1316 return "https";
1317 }
1318 if (rp->rrdp != NULL)
1319 return "rrdp";
1320 return "rsync";
1321 }
1322
1323 /*
1324 * Return the repository tal ID.
1325 */
1326 int
repo_talid(const struct repo * rp)1327 repo_talid(const struct repo *rp)
1328 {
1329 return rp->talid;
1330 }
1331
1332 int
repo_queued(struct repo * rp,struct entity * p)1333 repo_queued(struct repo *rp, struct entity *p)
1334 {
1335 if (repo_state(rp) == REPO_LOADING) {
1336 TAILQ_INSERT_TAIL(&rp->queue, p, entries);
1337 return 1;
1338 }
1339 return 0;
1340 }
1341
1342 static void
repo_fail(struct repo * rp)1343 repo_fail(struct repo *rp)
1344 {
1345 /* reset the alarm since code may fallback to rsync */
1346 rp->alarm = getmonotime() + repo_timeout;
1347
1348 if (rp->ta)
1349 http_finish(rp->ta->id, HTTP_FAILED, NULL);
1350 else if (rp->rsync)
1351 rsync_finish(rp->rsync->id, 0);
1352 else if (rp->rrdp)
1353 rrdp_finish(rp->rrdp->id, 0);
1354 else
1355 errx(1, "%s: bad repo", rp->repouri);
1356 }
1357
1358 static void
repo_abort(struct repo * rp)1359 repo_abort(struct repo *rp)
1360 {
1361 /* reset the alarm */
1362 rp->alarm = getmonotime() + repo_timeout;
1363
1364 if (rp->rsync)
1365 rsync_abort(rp->rsync->id);
1366 else if (rp->rrdp)
1367 rrdp_abort(rp->rrdp->id);
1368 else
1369 repo_fail(rp);
1370 }
1371
1372 int
repo_check_timeout(int timeout)1373 repo_check_timeout(int timeout)
1374 {
1375 struct repo *rp;
1376 time_t now;
1377 int diff;
1378
1379 now = getmonotime();
1380
1381 /* check against our runtime deadline first */
1382 if (deadline != 0) {
1383 if (deadline <= now) {
1384 warnx("deadline reached, giving up on repository sync");
1385 nofetch = 1;
1386 /* clear deadline since nofetch is set */
1387 deadline = 0;
1388 /* increase now enough so that all pending repos fail */
1389 now += repo_timeout;
1390 } else {
1391 diff = deadline - now;
1392 diff *= 1000;
1393 if (timeout == INFTIM || diff < timeout)
1394 timeout = diff;
1395 }
1396 }
1397 /* Look up in repository table. (Lookup should actually fail here) */
1398 SLIST_FOREACH(rp, &repos, entry) {
1399 if (repo_state(rp) == REPO_LOADING) {
1400 if (rp->alarm <= now) {
1401 warnx("%s: synchronisation timeout",
1402 rp->repouri);
1403 repo_abort(rp);
1404 } else {
1405 diff = rp->alarm - now;
1406 diff *= 1000;
1407 if (timeout == INFTIM || diff < timeout)
1408 timeout = diff;
1409 }
1410 }
1411 }
1412 return timeout;
1413 }
1414
1415 /*
1416 * Update repo-specific stats when files are going to be moved
1417 * from DIR_TEMP to DIR_VALID.
1418 */
1419 void
repostats_new_files_inc(struct repo * rp,const char * file)1420 repostats_new_files_inc(struct repo *rp, const char *file)
1421 {
1422 if (strncmp(file, ".rsync/", strlen(".rsync/")) == 0 ||
1423 strncmp(file, ".rrdp/", strlen(".rrdp/")) == 0)
1424 rp->repostats.new_files++;
1425 }
1426
1427 /*
1428 * Update stats object of repository depending on rtype and subtype.
1429 */
1430 void
repo_stat_inc(struct repo * rp,int talid,enum rtype type,enum stype subtype)1431 repo_stat_inc(struct repo *rp, int talid, enum rtype type, enum stype subtype)
1432 {
1433 if (rp == NULL)
1434 return;
1435 rp->stats_used[talid] = 1;
1436 switch (type) {
1437 case RTYPE_CER:
1438 if (subtype == STYPE_OK)
1439 rp->stats[talid].certs++;
1440 if (subtype == STYPE_FAIL)
1441 rp->stats[talid].certs_fail++;
1442 if (subtype == STYPE_BGPSEC) {
1443 rp->stats[talid].certs--;
1444 rp->stats[talid].brks++;
1445 }
1446 break;
1447 case RTYPE_MFT:
1448 if (subtype == STYPE_OK)
1449 rp->stats[talid].mfts++;
1450 if (subtype == STYPE_FAIL)
1451 rp->stats[talid].mfts_fail++;
1452 break;
1453 case RTYPE_ROA:
1454 switch (subtype) {
1455 case STYPE_OK:
1456 rp->stats[talid].roas++;
1457 break;
1458 case STYPE_FAIL:
1459 rp->stats[talid].roas_fail++;
1460 break;
1461 case STYPE_INVALID:
1462 rp->stats[talid].roas_invalid++;
1463 break;
1464 case STYPE_TOTAL:
1465 rp->stats[talid].vrps++;
1466 break;
1467 case STYPE_UNIQUE:
1468 rp->stats[talid].vrps_uniqs++;
1469 break;
1470 case STYPE_DEC_UNIQUE:
1471 rp->stats[talid].vrps_uniqs--;
1472 break;
1473 default:
1474 break;
1475 }
1476 break;
1477 case RTYPE_ASPA:
1478 switch (subtype) {
1479 case STYPE_OK:
1480 rp->stats[talid].aspas++;
1481 break;
1482 case STYPE_FAIL:
1483 rp->stats[talid].aspas_fail++;
1484 break;
1485 case STYPE_INVALID:
1486 rp->stats[talid].aspas_invalid++;
1487 break;
1488 case STYPE_TOTAL:
1489 rp->stats[talid].vaps++;
1490 break;
1491 case STYPE_UNIQUE:
1492 rp->stats[talid].vaps_uniqs++;
1493 break;
1494 case STYPE_DEC_UNIQUE:
1495 rp->stats[talid].vaps_uniqs--;
1496 break;
1497 case STYPE_PROVIDERS:
1498 rp->stats[talid].vaps_pas++;
1499 break;
1500 case STYPE_OVERFLOW:
1501 rp->stats[talid].vaps_overflowed++;
1502 break;
1503 default:
1504 break;
1505 }
1506 break;
1507 case RTYPE_SPL:
1508 switch (subtype) {
1509 case STYPE_OK:
1510 rp->stats[talid].spls++;
1511 break;
1512 case STYPE_FAIL:
1513 rp->stats[talid].spls_fail++;
1514 break;
1515 case STYPE_INVALID:
1516 rp->stats[talid].spls_invalid++;
1517 break;
1518 case STYPE_TOTAL:
1519 rp->stats[talid].vsps++;
1520 break;
1521 case STYPE_UNIQUE:
1522 rp->stats[talid].vsps_uniqs++;
1523 break;
1524 case STYPE_DEC_UNIQUE:
1525 rp->stats[talid].vsps_uniqs--;
1526 break;
1527 default:
1528 break;
1529 }
1530 break;
1531 case RTYPE_CRL:
1532 rp->stats[talid].crls++;
1533 break;
1534 case RTYPE_GBR:
1535 rp->stats[talid].gbrs++;
1536 break;
1537 case RTYPE_TAK:
1538 rp->stats[talid].taks++;
1539 break;
1540 default:
1541 break;
1542 }
1543 }
1544
1545 void
repo_tal_stats_collect(void (* cb)(const struct repo *,const struct repotalstats *,void *),int talid,void * arg)1546 repo_tal_stats_collect(void (*cb)(const struct repo *,
1547 const struct repotalstats *, void *), int talid, void *arg)
1548 {
1549 struct repo *rp;
1550
1551 SLIST_FOREACH(rp, &repos, entry) {
1552 if (rp->stats_used[talid])
1553 cb(rp, &rp->stats[talid], arg);
1554 }
1555 }
1556
1557 void
repo_stats_collect(void (* cb)(const struct repo *,const struct repostats *,void *),void * arg)1558 repo_stats_collect(void (*cb)(const struct repo *, const struct repostats *,
1559 void *), void *arg)
1560 {
1561 struct repo *rp;
1562
1563 SLIST_FOREACH(rp, &repos, entry)
1564 cb(rp, &rp->repostats, arg);
1565 }
1566
1567 /*
1568 * Delayed delete of files from RRDP. Since RRDP has no security built-in
1569 * this code needs to check if this RRDP repository is actually allowed to
1570 * remove the file referenced by the URI.
1571 */
1572 static void
repo_cleanup_rrdp(struct filepath_tree * tree)1573 repo_cleanup_rrdp(struct filepath_tree *tree)
1574 {
1575 struct repo *rp;
1576 struct rrdprepo *rr;
1577 struct filepath *fp, *nfp;
1578 char *fn;
1579
1580 SLIST_FOREACH(rp, &repos, entry) {
1581 if (rp->rrdp == NULL)
1582 continue;
1583 rr = (struct rrdprepo *)rp->rrdp;
1584 RB_FOREACH_SAFE(fp, filepath_tree, &rr->deleted, nfp) {
1585 if (!rrdp_uri_valid(rr, fp->file)) {
1586 warnx("%s: external URI %s", rr->notifyuri,
1587 fp->file);
1588 filepath_put(&rr->deleted, fp);
1589 continue;
1590 }
1591 /* try to remove file from rrdp repo ... */
1592 fn = rrdp_filename(rr, fp->file, 0);
1593
1594 if (unlink(fn) == -1) {
1595 if (errno != ENOENT)
1596 warn("unlink %s", fn);
1597 } else {
1598 if (verbose > 1)
1599 logx("deleted %s", fn);
1600 rp->repostats.del_files++;
1601 }
1602 free(fn);
1603
1604 /* ... and from the valid repository if unused. */
1605 fn = rrdp_filename(rr, fp->file, 1);
1606 if (!filepath_exists(tree, fn)) {
1607 if (unlink(fn) == -1) {
1608 if (errno != ENOENT)
1609 warn("unlink %s", fn);
1610 } else {
1611 if (verbose > 1)
1612 logx("deleted %s", fn);
1613 rp->repostats.del_files++;
1614 }
1615 } else
1616 warnx("%s: referenced file supposed to be "
1617 "deleted", fn);
1618
1619 free(fn);
1620 filepath_put(&rr->deleted, fp);
1621 }
1622 }
1623 }
1624
1625 /*
1626 * All files in tree are valid and should be moved to the valid repository
1627 * if not already there. Rename the files to the new path and readd the
1628 * filepath entry with the new path if successful.
1629 */
1630 static void
repo_move_valid(struct filepath_tree * tree)1631 repo_move_valid(struct filepath_tree *tree)
1632 {
1633 struct filepath *fp, *nfp;
1634 size_t rsyncsz = strlen(".rsync/");
1635 size_t rrdpsz = strlen(".rrdp/");
1636 size_t tasz = strlen(".ta/");
1637 char *fn, *base;
1638
1639 RB_FOREACH_SAFE(fp, filepath_tree, tree, nfp) {
1640 if (strncmp(fp->file, ".rsync/", rsyncsz) != 0 &&
1641 strncmp(fp->file, ".rrdp/", rrdpsz) != 0 &&
1642 strncmp(fp->file, ".ta/", tasz) != 0)
1643 continue; /* not a temporary file path */
1644
1645 if (strncmp(fp->file, ".rsync/", rsyncsz) == 0) {
1646 fn = fp->file + rsyncsz;
1647 } else if (strncmp(fp->file, ".ta/", tasz) == 0) {
1648 fn = fp->file + 1; /* just skip the '.' */
1649 } else {
1650 base = strchr(fp->file + rrdpsz, '/');
1651 assert(base != NULL);
1652 fn = base + 1;
1653
1654 /*
1655 * Adjust file last modification time in order to
1656 * minimize RSYNC synchronization load after transport
1657 * failover.
1658 * While serializing RRDP datastructures to disk, set
1659 * the last modified timestamp to the CMS signing-time,
1660 * the X.509 notBefore, or CRL lastUpdate timestamp.
1661 */
1662 if (fp->mtime != 0) {
1663 int ret;
1664 struct timespec ts[2];
1665
1666 ts[0].tv_nsec = UTIME_OMIT;
1667 ts[1].tv_sec = fp->mtime;
1668 ts[1].tv_nsec = 0;
1669 ret = utimensat(AT_FDCWD, fp->file, ts, 0);
1670 if (ret == -1) {
1671 warn("utimensat %s", fp->file);
1672 continue;
1673 }
1674 }
1675 }
1676
1677 if (repo_mkpath(AT_FDCWD, fn) == -1)
1678 continue;
1679
1680 if (rename(fp->file, fn) == -1) {
1681 warn("rename %s", fp->file);
1682 continue;
1683 }
1684
1685 /* switch filepath node to new path */
1686 RB_REMOVE(filepath_tree, tree, fp);
1687 base = fp->file;
1688 if ((fp->file = strdup(fn)) == NULL)
1689 err(1, NULL);
1690 free(base);
1691 if (RB_INSERT(filepath_tree, tree, fp) != NULL)
1692 errx(1, "%s: both possibilities of file present",
1693 fp->file);
1694 }
1695 }
1696
1697 struct fts_state {
1698 enum { BASE_DIR, RSYNC_DIR, TA_DIR, RRDP_DIR } type;
1699 struct repo *rp;
1700 } fts_state;
1701
1702 static const struct rrdprepo *
repo_is_rrdp(struct repo * rp)1703 repo_is_rrdp(struct repo *rp)
1704 {
1705 /* check for special pointers first these are not a repository */
1706 if (rp != NULL && rp->rrdp != NULL)
1707 return rp->rrdp->state == REPO_DONE ? rp->rrdp : NULL;
1708 return NULL;
1709 }
1710
1711 static inline char *
skip_dotslash(char * in)1712 skip_dotslash(char *in)
1713 {
1714 if (memcmp(in, "./", 2) == 0)
1715 return in + 2;
1716 return in;
1717 }
1718
1719 static void
repo_cleanup_entry(FTSENT * e,struct filepath_tree * tree,int cachefd)1720 repo_cleanup_entry(FTSENT *e, struct filepath_tree *tree, int cachefd)
1721 {
1722 const struct rrdprepo *rr;
1723 char *path;
1724
1725 path = skip_dotslash(e->fts_path);
1726 switch (e->fts_info) {
1727 case FTS_NSOK:
1728 if (filepath_exists(tree, path)) {
1729 e->fts_parent->fts_number++;
1730 break;
1731 }
1732 if (fts_state.type == RRDP_DIR && fts_state.rp != NULL) {
1733 e->fts_parent->fts_number++;
1734 /* handle rrdp .state files explicitly */
1735 if (e->fts_level == 3 &&
1736 strcmp(e->fts_name, ".state") == 0)
1737 break;
1738 /* can't delete these extra files */
1739 fts_state.rp->repostats.extra_files++;
1740 if (verbose > 1)
1741 logx("superfluous %s", path);
1742 break;
1743 }
1744 rr = repo_is_rrdp(fts_state.rp);
1745 if (rr != NULL) {
1746 struct stat st;
1747 char *fn;
1748
1749 if (asprintf(&fn, "%s/%s", rr->basedir, path) == -1)
1750 err(1, NULL);
1751
1752 /*
1753 * If the file exists in the rrdp dir
1754 * that file is newer and needs to be kept
1755 * so unlink this file instead of moving
1756 * it over the file in the rrdp dir.
1757 */
1758 if (fstatat(cachefd, fn, &st, 0) == 0 &&
1759 S_ISREG(st.st_mode)) {
1760 free(fn);
1761 goto unlink;
1762 }
1763 if (repo_mkpath(cachefd, fn) == 0) {
1764 if (renameat(AT_FDCWD, e->fts_accpath,
1765 cachefd, fn) == -1)
1766 warn("rename %s to %s", path, fn);
1767 else if (verbose > 1)
1768 logx("moved %s", path);
1769 fts_state.rp->repostats.extra_files++;
1770 }
1771 free(fn);
1772 } else {
1773 unlink:
1774 if (unlink(e->fts_accpath) == -1) {
1775 warn("unlink %s", path);
1776 } else if (fts_state.type == RSYNC_DIR ||
1777 fts_state.type == TA_DIR) {
1778 /* no need to keep rsync or ta files */
1779 if (verbose > 1)
1780 logx("deleted superfluous %s", path);
1781 if (fts_state.rp != NULL)
1782 fts_state.rp->repostats.del_extra_files++;
1783 else
1784 stats.repo_stats.del_extra_files++;
1785 } else {
1786 if (verbose > 1)
1787 logx("deleted %s", path);
1788 if (fts_state.rp != NULL)
1789 fts_state.rp->repostats.del_files++;
1790 else
1791 stats.repo_stats.del_files++;
1792 }
1793 }
1794 break;
1795 case FTS_D:
1796 if (e->fts_level == FTS_ROOTLEVEL) {
1797 fts_state.type = BASE_DIR;
1798 fts_state.rp = NULL;
1799 }
1800 if (e->fts_level == 1) {
1801 /* rpki.example.org or .rrdp / .rsync / .ta */
1802 if (strcmp(".rsync", e->fts_name) == 0)
1803 fts_state.type = RSYNC_DIR;
1804 else if (strcmp(".ta", e->fts_name) == 0)
1805 fts_state.type = TA_DIR;
1806 else if (strcmp(".rrdp", e->fts_name) == 0)
1807 fts_state.type = RRDP_DIR;
1808 else
1809 fts_state.type = BASE_DIR;
1810 fts_state.rp = NULL;
1811 }
1812 if (e->fts_level == 2) {
1813 /* rpki.example.org/repository or .rrdp/hashdir */
1814 if (fts_state.type == BASE_DIR)
1815 fts_state.rp = repo_bypath(path);
1816 if (fts_state.type == TA_DIR)
1817 fts_state.rp = repo_bypath(path + 1);
1818 /*
1819 * special handling for rrdp directories,
1820 * clear them if they are not used anymore but
1821 * only if rrdp is active.
1822 * Look them up just using the hash.
1823 */
1824 if (fts_state.type == RRDP_DIR)
1825 fts_state.rp = repo_rrdp_bypath(path);
1826 }
1827 if (e->fts_level == 3 && fts_state.type == RSYNC_DIR) {
1828 /* .rsync/rpki.example.org/repository */
1829 fts_state.rp = repo_bypath(path + strlen(".rsync/"));
1830 }
1831 break;
1832 case FTS_DP:
1833 if (e->fts_level == FTS_ROOTLEVEL)
1834 break;
1835 if (e->fts_level == 1) {
1836 /* do not remove .rsync and .rrdp */
1837 fts_state.rp = NULL;
1838 if (fts_state.type == RRDP_DIR ||
1839 fts_state.type == RSYNC_DIR ||
1840 fts_state.type == TA_DIR)
1841 break;
1842 }
1843
1844 e->fts_parent->fts_number += e->fts_number;
1845
1846 if (e->fts_number == 0) {
1847 if (rmdir(e->fts_accpath) == -1)
1848 warn("rmdir %s", path);
1849 if (fts_state.rp != NULL)
1850 fts_state.rp->repostats.del_dirs++;
1851 else
1852 stats.repo_stats.del_dirs++;
1853 }
1854 break;
1855 case FTS_SL:
1856 case FTS_SLNONE:
1857 warnx("symlink %s", path);
1858 if (unlink(e->fts_accpath) == -1)
1859 warn("unlink %s", path);
1860 stats.repo_stats.del_extra_files++;
1861 break;
1862 case FTS_NS:
1863 case FTS_ERR:
1864 if (e->fts_errno == ENOENT && e->fts_level == FTS_ROOTLEVEL)
1865 break;
1866 warnx("fts_read %s: %s", path, strerror(e->fts_errno));
1867 break;
1868 default:
1869 warnx("fts_read %s: unhandled[%x]", path, e->fts_info);
1870 break;
1871 }
1872 }
1873
1874 void
repo_cleanup(struct filepath_tree * tree,int cachefd)1875 repo_cleanup(struct filepath_tree *tree, int cachefd)
1876 {
1877 char *argv[2] = { ".", NULL };
1878 FTS *fts;
1879 FTSENT *e;
1880
1881 /* first move temp files which have been used to valid dir */
1882 repo_move_valid(tree);
1883 /* then delete files requested by rrdp */
1884 repo_cleanup_rrdp(tree);
1885
1886 if ((fts = fts_open(argv, FTS_PHYSICAL | FTS_NOSTAT, NULL)) == NULL)
1887 err(1, "fts_open");
1888 errno = 0;
1889 while ((e = fts_read(fts)) != NULL) {
1890 repo_cleanup_entry(e, tree, cachefd);
1891 errno = 0;
1892 }
1893 if (errno)
1894 err(1, "fts_read");
1895 if (fts_close(fts) == -1)
1896 err(1, "fts_close");
1897 }
1898
1899 void
repo_free(void)1900 repo_free(void)
1901 {
1902 struct repo *rp;
1903
1904 while ((rp = SLIST_FIRST(&repos)) != NULL) {
1905 SLIST_REMOVE_HEAD(&repos, entry);
1906 free(rp->repouri);
1907 free(rp->notifyuri);
1908 free(rp->basedir);
1909 free(rp);
1910 }
1911
1912 ta_free();
1913 rrdp_free();
1914 rsync_free();
1915 }
1916
1917 /*
1918 * Remove all files and directories under base.
1919 * Do not remove base directory itself and the .state file.
1920 */
1921 static void
remove_contents(char * base)1922 remove_contents(char *base)
1923 {
1924 char *argv[2] = { base, NULL };
1925 FTS *fts;
1926 FTSENT *e;
1927
1928 if ((fts = fts_open(argv, FTS_PHYSICAL | FTS_NOSTAT, NULL)) == NULL)
1929 err(1, "fts_open");
1930 errno = 0;
1931 while ((e = fts_read(fts)) != NULL) {
1932 switch (e->fts_info) {
1933 case FTS_NSOK:
1934 case FTS_SL:
1935 case FTS_SLNONE:
1936 if (e->fts_level == 1 &&
1937 strcmp(e->fts_name, ".state") == 0)
1938 break;
1939 if (unlink(e->fts_accpath) == -1)
1940 warn("unlink %s", e->fts_path);
1941 break;
1942 case FTS_D:
1943 break;
1944 case FTS_DP:
1945 /* keep root directory */
1946 if (e->fts_level == FTS_ROOTLEVEL)
1947 break;
1948 if (rmdir(e->fts_accpath) == -1)
1949 warn("rmdir %s", e->fts_path);
1950 break;
1951 case FTS_NS:
1952 case FTS_ERR:
1953 warnx("fts_read %s: %s", e->fts_path,
1954 strerror(e->fts_errno));
1955 break;
1956 default:
1957 warnx("unhandled[%x] %s", e->fts_info,
1958 e->fts_path);
1959 break;
1960 }
1961 errno = 0;
1962 }
1963 if (errno)
1964 err(1, "fts_read");
1965 if (fts_close(fts) == -1)
1966 err(1, "fts_close");
1967 }
1968