1 /* $OpenBSD: repo.c,v 1.30 2022/02/02 15:13:00 claudio Exp $ */
2 /*
3 * Copyright (c) 2021 Claudio Jeker <claudio@openbsd.org>
4 * Copyright (c) 2019 Kristaps Dzonsons <kristaps@bsd.lv>
5 *
6 * Permission to use, copy, modify, and distribute this software for any
7 * purpose with or without fee is hereby granted, provided that the above
8 * copyright notice and this permission notice appear in all copies.
9 *
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 */
18
19 #include <sys/queue.h>
20 #include <sys/tree.h>
21 #include <sys/types.h>
22 #include <sys/stat.h>
23
24 #include <assert.h>
25 #include <err.h>
26 #include <errno.h>
27 #include <fcntl.h>
28 #include <fts.h>
29 #include <limits.h>
30 #include <poll.h>
31 #include <stdio.h>
32 #include <stdlib.h>
33 #include <string.h>
34 #include <unistd.h>
35
36 #include <imsg.h>
37
38 #include "extern.h"
39
40 extern struct stats stats;
41 extern int noop;
42 extern int rrdpon;
43 extern int repo_timeout;
44
45 enum repo_state {
46 REPO_LOADING = 0,
47 REPO_DONE = 1,
48 REPO_FAILED = -1,
49 };
50
51 /*
52 * A ta, rsync or rrdp repository.
53 * Depending on what is needed the generic repository is backed by
54 * a ta, rsync or rrdp repository. Multiple repositories can use the
55 * same backend.
56 */
57 struct rrdprepo {
58 SLIST_ENTRY(rrdprepo) entry;
59 char *notifyuri;
60 char *basedir;
61 struct filepath_tree deleted;
62 unsigned int id;
63 enum repo_state state;
64 };
65 static SLIST_HEAD(, rrdprepo) rrdprepos = SLIST_HEAD_INITIALIZER(rrdprepos);
66
67 struct rsyncrepo {
68 SLIST_ENTRY(rsyncrepo) entry;
69 char *repouri;
70 char *basedir;
71 unsigned int id;
72 enum repo_state state;
73 };
74 static SLIST_HEAD(, rsyncrepo) rsyncrepos = SLIST_HEAD_INITIALIZER(rsyncrepos);
75
76 struct tarepo {
77 SLIST_ENTRY(tarepo) entry;
78 char *descr;
79 char *basedir;
80 char *temp;
81 char **uri;
82 size_t urisz;
83 size_t uriidx;
84 unsigned int id;
85 enum repo_state state;
86 };
87 static SLIST_HEAD(, tarepo) tarepos = SLIST_HEAD_INITIALIZER(tarepos);
88
89 struct repo {
90 SLIST_ENTRY(repo) entry;
91 char *repouri;
92 char *notifyuri;
93 char *basedir;
94 const struct rrdprepo *rrdp;
95 const struct rsyncrepo *rsync;
96 const struct tarepo *ta;
97 struct entityq queue; /* files waiting for repo */
98 time_t alarm; /* sync timeout */
99 int talid;
100 unsigned int id; /* identifier */
101 };
102 static SLIST_HEAD(, repo) repos = SLIST_HEAD_INITIALIZER(repos);
103
104 /* counter for unique repo id */
105 unsigned int repoid;
106
107 static struct rsyncrepo *rsync_get(const char *, const char *);
108 static void remove_contents(char *);
109
110 /*
111 * Database of all file path accessed during a run.
112 */
113 struct filepath {
114 RB_ENTRY(filepath) entry;
115 char *file;
116 };
117
118 static inline int
filepathcmp(struct filepath * a,struct filepath * b)119 filepathcmp(struct filepath *a, struct filepath *b)
120 {
121 return strcmp(a->file, b->file);
122 }
123
124 RB_PROTOTYPE(filepath_tree, filepath, entry, filepathcmp);
125
126 /*
127 * Functions to lookup which files have been accessed during computation.
128 */
129 int
filepath_add(struct filepath_tree * tree,char * file)130 filepath_add(struct filepath_tree *tree, char *file)
131 {
132 struct filepath *fp;
133
134 if ((fp = malloc(sizeof(*fp))) == NULL)
135 err(1, NULL);
136 if ((fp->file = strdup(file)) == NULL)
137 err(1, NULL);
138
139 if (RB_INSERT(filepath_tree, tree, fp) != NULL) {
140 /* already in the tree */
141 free(fp->file);
142 free(fp);
143 return 0;
144 }
145
146 return 1;
147 }
148
149 /*
150 * Lookup a file path in the tree and return the object if found or NULL.
151 */
152 static struct filepath *
filepath_find(struct filepath_tree * tree,char * file)153 filepath_find(struct filepath_tree *tree, char *file)
154 {
155 struct filepath needle = { .file = file };
156
157 return RB_FIND(filepath_tree, tree, &needle);
158 }
159
160 /*
161 * Returns true if file exists in the tree.
162 */
163 static int
filepath_exists(struct filepath_tree * tree,char * file)164 filepath_exists(struct filepath_tree *tree, char *file)
165 {
166 return filepath_find(tree, file) != NULL;
167 }
168
169 /*
170 * Remove entry from tree and free it.
171 */
172 static void
filepath_put(struct filepath_tree * tree,struct filepath * fp)173 filepath_put(struct filepath_tree *tree, struct filepath *fp)
174 {
175 RB_REMOVE(filepath_tree, tree, fp);
176 free((void *)fp->file);
177 free(fp);
178 }
179
180 /*
181 * Free all elements of a filepath tree.
182 */
183 static void
filepath_free(struct filepath_tree * tree)184 filepath_free(struct filepath_tree *tree)
185 {
186 struct filepath *fp, *nfp;
187
188 RB_FOREACH_SAFE(fp, filepath_tree, tree, nfp)
189 filepath_put(tree, fp);
190 }
191
192 RB_GENERATE(filepath_tree, filepath, entry, filepathcmp);
193
194 /*
195 * Function to hash a string into a unique directory name.
196 * Returned hash needs to be freed.
197 */
198 static char *
hash_dir(const char * uri)199 hash_dir(const char *uri)
200 {
201 unsigned char m[SHA256_DIGEST_LENGTH];
202
203 SHA256(uri, strlen(uri), m);
204 return hex_encode(m, sizeof(m));
205 }
206
207 /*
208 * Function to build the directory name based on URI and a directory
209 * as prefix. Skip the proto:// in URI but keep everything else.
210 */
211 static char *
repo_dir(const char * uri,const char * dir,int hash)212 repo_dir(const char *uri, const char *dir, int hash)
213 {
214 const char *local;
215 char *out, *hdir = NULL;
216
217 if (hash) {
218 local = hdir = hash_dir(uri);
219 } else {
220 local = strchr(uri, ':');
221 if (local != NULL)
222 local += strlen("://");
223 else
224 local = uri;
225 }
226
227 if (dir == NULL) {
228 if ((out = strdup(local)) == NULL)
229 err(1, NULL);
230 } else {
231 if (asprintf(&out, "%s/%s", dir, local) == -1)
232 err(1, NULL);
233 }
234
235 free(hdir);
236 return out;
237 }
238
239 /*
240 * Function to create all missing directories to a path.
241 * This functions alters the path temporarily.
242 */
243 static int
repo_mkpath(char * file)244 repo_mkpath(char *file)
245 {
246 char *slash;
247
248 /* build directory hierarchy */
249 slash = strrchr(file, '/');
250 assert(slash != NULL);
251 *slash = '\0';
252 if (mkpath(file) == -1) {
253 warn("mkpath %s", file);
254 return -1;
255 }
256 *slash = '/';
257 return 0;
258 }
259
260 /*
261 * Return the state of a repository.
262 */
263 static enum repo_state
repo_state(struct repo * rp)264 repo_state(struct repo *rp)
265 {
266 if (rp->ta)
267 return rp->ta->state;
268 if (rp->rsync)
269 return rp->rsync->state;
270 if (rp->rrdp)
271 return rp->rrdp->state;
272 /* No backend so sync is by definition done. */
273 return REPO_DONE;
274 }
275
276 /*
277 * Function called once a repository is done with the sync. Either
278 * successfully or after failure.
279 */
280 static void
repo_done(const void * vp,int ok)281 repo_done(const void *vp, int ok)
282 {
283 struct repo *rp;
284
285 SLIST_FOREACH(rp, &repos, entry) {
286 if (vp == rp->ta)
287 entityq_flush(&rp->queue, rp);
288 if (vp == rp->rsync)
289 entityq_flush(&rp->queue, rp);
290 if (vp == rp->rrdp) {
291 if (!ok) {
292 /* try to fall back to rsync */
293 rp->rrdp = NULL;
294 rp->rsync = rsync_get(rp->repouri,
295 rp->basedir);
296 /* need to check if it was already loaded */
297 if (repo_state(rp) != REPO_LOADING)
298 entityq_flush(&rp->queue, rp);
299 } else
300 entityq_flush(&rp->queue, rp);
301 }
302 }
303 }
304
305 /*
306 * Build TA file name based on the repo info.
307 * If temp is set add Xs for mkostemp.
308 */
309 static char *
ta_filename(const struct tarepo * tr,int temp)310 ta_filename(const struct tarepo *tr, int temp)
311 {
312 const char *file;
313 char *nfile;
314
315 /* does not matter which URI, all end with same filename */
316 file = strrchr(tr->uri[0], '/');
317 assert(file);
318
319 if (asprintf(&nfile, "%s%s%s", tr->basedir, file,
320 temp ? ".XXXXXXXX": "") == -1)
321 err(1, NULL);
322
323 return nfile;
324 }
325
326 static void
ta_fetch(struct tarepo * tr)327 ta_fetch(struct tarepo *tr)
328 {
329 if (!rrdpon) {
330 for (; tr->uriidx < tr->urisz; tr->uriidx++) {
331 if (strncasecmp(tr->uri[tr->uriidx],
332 "rsync://", 8) == 0)
333 break;
334 }
335 }
336
337 if (tr->uriidx >= tr->urisz) {
338 tr->state = REPO_FAILED;
339 logx("ta/%s: fallback to cache", tr->descr);
340
341 repo_done(tr, 0);
342 return;
343 }
344
345 logx("ta/%s: pulling from %s", tr->descr, tr->uri[tr->uriidx]);
346
347 if (strncasecmp(tr->uri[tr->uriidx], "rsync://", 8) == 0) {
348 /*
349 * Create destination location.
350 * Build up the tree to this point.
351 */
352 rsync_fetch(tr->id, tr->uri[tr->uriidx], tr->basedir, NULL);
353 } else {
354 int fd;
355
356 tr->temp = ta_filename(tr, 1);
357 fd = mkostemp(tr->temp, O_CLOEXEC);
358 if (fd == -1) {
359 warn("mkostemp: %s", tr->temp);
360 http_finish(tr->id, HTTP_FAILED, NULL);
361 return;
362 }
363 if (fchmod(fd, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH) == -1)
364 warn("fchmod: %s", tr->temp);
365
366 http_fetch(tr->id, tr->uri[tr->uriidx], NULL, fd);
367 }
368 }
369
370 static struct tarepo *
ta_get(struct tal * tal)371 ta_get(struct tal *tal)
372 {
373 struct tarepo *tr;
374
375 /* no need to look for possible other repo */
376
377 if ((tr = calloc(1, sizeof(*tr))) == NULL)
378 err(1, NULL);
379 tr->id = ++repoid;
380 SLIST_INSERT_HEAD(&tarepos, tr, entry);
381
382 if ((tr->descr = strdup(tal->descr)) == NULL)
383 err(1, NULL);
384 tr->basedir = repo_dir(tal->descr, "ta", 0);
385
386 /* steal URI infromation from TAL */
387 tr->urisz = tal->urisz;
388 tr->uri = tal->uri;
389 tal->urisz = 0;
390 tal->uri = NULL;
391
392 ta_fetch(tr);
393
394 return tr;
395 }
396
397 static struct tarepo *
ta_find(unsigned int id)398 ta_find(unsigned int id)
399 {
400 struct tarepo *tr;
401
402 SLIST_FOREACH(tr, &tarepos, entry)
403 if (id == tr->id)
404 break;
405 return tr;
406 }
407
408 static void
ta_free(void)409 ta_free(void)
410 {
411 struct tarepo *tr;
412
413 while ((tr = SLIST_FIRST(&tarepos)) != NULL) {
414 SLIST_REMOVE_HEAD(&tarepos, entry);
415 free(tr->descr);
416 free(tr->basedir);
417 free(tr->temp);
418 free(tr->uri);
419 free(tr);
420 }
421 }
422
423 static struct rsyncrepo *
rsync_get(const char * uri,const char * validdir)424 rsync_get(const char *uri, const char *validdir)
425 {
426 struct rsyncrepo *rr;
427 char *repo;
428
429 if ((repo = rsync_base_uri(uri)) == NULL)
430 errx(1, "bad caRepository URI: %s", uri);
431
432 SLIST_FOREACH(rr, &rsyncrepos, entry)
433 if (strcmp(rr->repouri, repo) == 0) {
434 free(repo);
435 return rr;
436 }
437
438 if ((rr = calloc(1, sizeof(*rr))) == NULL)
439 err(1, NULL);
440
441 rr->id = ++repoid;
442 SLIST_INSERT_HEAD(&rsyncrepos, rr, entry);
443
444 rr->repouri = repo;
445 rr->basedir = repo_dir(repo, ".rsync", 0);
446
447 /* create base directory */
448 if (mkpath(rr->basedir) == -1) {
449 warn("mkpath %s", rr->basedir);
450 rsync_finish(rr->id, 0);
451 return rr;
452 }
453
454 logx("%s: pulling from %s", rr->basedir, rr->repouri);
455 rsync_fetch(rr->id, rr->repouri, rr->basedir, validdir);
456
457 return rr;
458 }
459
460 static struct rsyncrepo *
rsync_find(unsigned int id)461 rsync_find(unsigned int id)
462 {
463 struct rsyncrepo *rr;
464
465 SLIST_FOREACH(rr, &rsyncrepos, entry)
466 if (id == rr->id)
467 break;
468 return rr;
469 }
470
471 static void
rsync_free(void)472 rsync_free(void)
473 {
474 struct rsyncrepo *rr;
475
476 while ((rr = SLIST_FIRST(&rsyncrepos)) != NULL) {
477 SLIST_REMOVE_HEAD(&rsyncrepos, entry);
478 free(rr->repouri);
479 free(rr->basedir);
480 free(rr);
481 }
482 }
483
484 /*
485 * Build local file name base on the URI and the rrdprepo info.
486 */
487 static char *
rrdp_filename(const struct rrdprepo * rr,const char * uri,int valid)488 rrdp_filename(const struct rrdprepo *rr, const char *uri, int valid)
489 {
490 char *nfile;
491 const char *dir = rr->basedir;
492
493 if (!valid_uri(uri, strlen(uri), "rsync://"))
494 errx(1, "%s: bad URI %s", rr->basedir, uri);
495 uri += strlen("rsync://"); /* skip proto */
496 if (valid) {
497 if ((nfile = strdup(uri)) == NULL)
498 err(1, NULL);
499 } else {
500 if (asprintf(&nfile, "%s/%s", dir, uri) == -1)
501 err(1, NULL);
502 }
503 return nfile;
504 }
505
506 /*
507 * Build RRDP state file name based on the repo info.
508 * If temp is set add Xs for mkostemp.
509 */
510 static char *
rrdp_state_filename(const struct rrdprepo * rr,int temp)511 rrdp_state_filename(const struct rrdprepo *rr, int temp)
512 {
513 char *nfile;
514
515 if (asprintf(&nfile, "%s/.state%s", rr->basedir,
516 temp ? ".XXXXXXXX": "") == -1)
517 err(1, NULL);
518
519 return nfile;
520 }
521
522 static struct rrdprepo *
rrdp_find(unsigned int id)523 rrdp_find(unsigned int id)
524 {
525 struct rrdprepo *rr;
526
527 SLIST_FOREACH(rr, &rrdprepos, entry)
528 if (id == rr->id)
529 break;
530 return rr;
531 }
532
533 static void
rrdp_free(void)534 rrdp_free(void)
535 {
536 struct rrdprepo *rr;
537
538 while ((rr = SLIST_FIRST(&rrdprepos)) != NULL) {
539 SLIST_REMOVE_HEAD(&rrdprepos, entry);
540
541 free(rr->notifyuri);
542 free(rr->basedir);
543
544 filepath_free(&rr->deleted);
545
546 free(rr);
547 }
548 }
549
550 /*
551 * Check if a directory is an active rrdp repository.
552 * Returns 1 if found else 0.
553 */
554 static int
rrdp_is_active(const char * dir)555 rrdp_is_active(const char *dir)
556 {
557 struct rrdprepo *rr;
558
559 SLIST_FOREACH(rr, &rrdprepos, entry)
560 if (strcmp(dir, rr->basedir) == 0)
561 return rr->state != REPO_FAILED;
562
563 return 0;
564 }
565
566 /*
567 * Check if the URI is actually covered by one of the repositories
568 * that depend on this RRDP repository.
569 * Returns 1 if the URI is valid, 0 if no repouri matches the URI.
570 */
571 static int
rrdp_uri_valid(struct rrdprepo * rr,const char * uri)572 rrdp_uri_valid(struct rrdprepo *rr, const char *uri)
573 {
574 struct repo *rp;
575
576 SLIST_FOREACH(rp, &repos, entry) {
577 if (rp->rrdp != rr)
578 continue;
579 if (strncmp(uri, rp->repouri, strlen(rp->repouri)) == 0)
580 return 1;
581 }
582 return 0;
583 }
584
585 /*
586 * Allocate and insert a new repository.
587 */
588 static struct repo *
repo_alloc(int talid)589 repo_alloc(int talid)
590 {
591 struct repo *rp;
592
593 if ((rp = calloc(1, sizeof(*rp))) == NULL)
594 err(1, NULL);
595
596 rp->id = ++repoid;
597 rp->talid = talid;
598 rp->alarm = getmonotime() + repo_timeout;
599 TAILQ_INIT(&rp->queue);
600 SLIST_INSERT_HEAD(&repos, rp, entry);
601
602 stats.repos++;
603 return rp;
604 }
605
606 /*
607 * Parse the RRDP state file if it exists and set the session struct
608 * based on that information.
609 */
610 static void
rrdp_parse_state(const struct rrdprepo * rr,struct rrdp_session * state)611 rrdp_parse_state(const struct rrdprepo *rr, struct rrdp_session *state)
612 {
613 FILE *f;
614 int fd, ln = 0;
615 const char *errstr;
616 char *line = NULL, *file;
617 size_t len = 0;
618 ssize_t n;
619
620 file = rrdp_state_filename(rr, 0);
621 if ((fd = open(file, O_RDONLY)) == -1) {
622 if (errno != ENOENT)
623 warn("%s: open state file", rr->basedir);
624 free(file);
625 return;
626 }
627 free(file);
628 f = fdopen(fd, "r");
629 if (f == NULL)
630 err(1, "fdopen");
631
632 while ((n = getline(&line, &len, f)) != -1) {
633 if (line[n - 1] == '\n')
634 line[n - 1] = '\0';
635 switch (ln) {
636 case 0:
637 if ((state->session_id = strdup(line)) == NULL)
638 err(1, NULL);
639 break;
640 case 1:
641 state->serial = strtonum(line, 1, LLONG_MAX, &errstr);
642 if (errstr)
643 goto fail;
644 break;
645 case 2:
646 if ((state->last_mod = strdup(line)) == NULL)
647 err(1, NULL);
648 break;
649 default:
650 goto fail;
651 }
652 ln++;
653 }
654
655 free(line);
656 if (ferror(f))
657 goto fail;
658 fclose(f);
659 return;
660
661 fail:
662 warnx("%s: troubles reading state file", rr->basedir);
663 fclose(f);
664 free(state->session_id);
665 free(state->last_mod);
666 memset(state, 0, sizeof(*state));
667 }
668
669 /*
670 * Carefully write the RRDP session state file back.
671 */
672 void
rrdp_save_state(unsigned int id,struct rrdp_session * state)673 rrdp_save_state(unsigned int id, struct rrdp_session *state)
674 {
675 struct rrdprepo *rr;
676 char *temp, *file;
677 FILE *f;
678 int fd;
679
680 rr = rrdp_find(id);
681 if (rr == NULL)
682 errx(1, "non-existant rrdp repo %u", id);
683
684 file = rrdp_state_filename(rr, 0);
685 temp = rrdp_state_filename(rr, 1);
686
687 if ((fd = mkostemp(temp, O_CLOEXEC)) == -1) {
688 warn("mkostemp %s", temp);
689 goto fail;
690 }
691 (void) fchmod(fd, 0644);
692 f = fdopen(fd, "w");
693 if (f == NULL)
694 err(1, "fdopen");
695
696 /* write session state file out */
697 if (fprintf(f, "%s\n%lld\n", state->session_id,
698 state->serial) < 0) {
699 fclose(f);
700 goto fail;
701 }
702 if (state->last_mod != NULL) {
703 if (fprintf(f, "%s\n", state->last_mod) < 0) {
704 fclose(f);
705 goto fail;
706 }
707 }
708 if (fclose(f) != 0)
709 goto fail;
710
711 if (rename(temp, file) == -1)
712 warn("%s: rename state file", rr->basedir);
713
714 free(temp);
715 free(file);
716 return;
717
718 fail:
719 warnx("%s: failed to save state", rr->basedir);
720 unlink(temp);
721 free(temp);
722 free(file);
723 }
724
725 static struct rrdprepo *
rrdp_get(const char * uri)726 rrdp_get(const char *uri)
727 {
728 struct rrdp_session state = { 0 };
729 struct rrdprepo *rr;
730
731 SLIST_FOREACH(rr, &rrdprepos, entry)
732 if (strcmp(rr->notifyuri, uri) == 0) {
733 if (rr->state == REPO_FAILED)
734 return NULL;
735 return rr;
736 }
737
738 if ((rr = calloc(1, sizeof(*rr))) == NULL)
739 err(1, NULL);
740
741 rr->id = ++repoid;
742 SLIST_INSERT_HEAD(&rrdprepos, rr, entry);
743
744 if ((rr->notifyuri = strdup(uri)) == NULL)
745 err(1, NULL);
746 rr->basedir = repo_dir(uri, ".rrdp", 1);
747
748 RB_INIT(&rr->deleted);
749
750
751 /* create base directory */
752 if (mkpath(rr->basedir) == -1) {
753 warn("mkpath %s", rr->basedir);
754 rrdp_finish(rr->id, 0);
755 return rr;
756 }
757
758 /* parse state and start the sync */
759 rrdp_parse_state(rr, &state);
760 rrdp_fetch(rr->id, rr->notifyuri, rr->notifyuri, &state);
761 free(state.session_id);
762 free(state.last_mod);
763
764 logx("%s: pulling from %s", rr->notifyuri, "network");
765
766 return rr;
767 }
768
769 /*
770 * Remove RRDP repo and start over.
771 */
772 void
rrdp_clear(unsigned int id)773 rrdp_clear(unsigned int id)
774 {
775 struct rrdprepo *rr;
776
777 rr = rrdp_find(id);
778 if (rr == NULL)
779 errx(1, "non-existant rrdp repo %u", id);
780
781 /* remove rrdp repository contents */
782 remove_contents(rr->basedir);
783 }
784
785 /*
786 * Write a file into the temporary RRDP dir but only after checking
787 * its hash (if required). The function also makes sure that the file
788 * tracking is properly adjusted.
789 * Returns 1 on success, 0 if the repo is corrupt, -1 on IO error
790 */
791 int
rrdp_handle_file(unsigned int id,enum publish_type pt,char * uri,char * hash,size_t hlen,char * data,size_t dlen)792 rrdp_handle_file(unsigned int id, enum publish_type pt, char *uri,
793 char *hash, size_t hlen, char *data, size_t dlen)
794 {
795 struct rrdprepo *rr;
796 struct filepath *fp;
797 ssize_t s;
798 char *fn = NULL;
799 int fd = -1, try = 0;
800
801 rr = rrdp_find(id);
802 if (rr == NULL)
803 errx(1, "non-existant rrdp repo %u", id);
804 if (rr->state == REPO_FAILED)
805 return -1;
806
807 /* check hash of original file for updates and deletes */
808 if (pt == PUB_UPD || pt == PUB_DEL) {
809 if (filepath_exists(&rr->deleted, uri)) {
810 warnx("%s: already deleted", uri);
811 return 0;
812 }
813 /* try to open file first in rrdp then in valid repo */
814 do {
815 free(fn);
816 if ((fn = rrdp_filename(rr, uri, try++)) == NULL)
817 return 0;
818 fd = open(fn, O_RDONLY);
819 } while (fd == -1 && try < 2);
820
821 if (!valid_filehash(fd, hash, hlen)) {
822 warnx("%s: bad file digest for %s", rr->notifyuri, fn);
823 free(fn);
824 return 0;
825 }
826 free(fn);
827 }
828
829 /* write new content or mark uri as deleted. */
830 if (pt == PUB_DEL) {
831 filepath_add(&rr->deleted, uri);
832 } else {
833 fp = filepath_find(&rr->deleted, uri);
834 if (fp != NULL)
835 filepath_put(&rr->deleted, fp);
836
837 /* add new file to rrdp dir */
838 if ((fn = rrdp_filename(rr, uri, 0)) == NULL)
839 return 0;
840
841 if (repo_mkpath(fn) == -1)
842 goto fail;
843
844 fd = open(fn, O_WRONLY|O_CREAT|O_TRUNC, 0644);
845 if (fd == -1) {
846 warn("open %s", fn);
847 goto fail;
848 }
849
850 if ((s = write(fd, data, dlen)) == -1) {
851 warn("write %s", fn);
852 goto fail;
853 }
854 close(fd);
855 if ((size_t)s != dlen) /* impossible */
856 errx(1, "short write %s", fn);
857 free(fn);
858 }
859
860 return 1;
861
862 fail:
863 rr->state = REPO_FAILED;
864 if (fd != -1)
865 close(fd);
866 free(fn);
867 return -1;
868 }
869
870 /*
871 * RSYNC sync finished, either with or without success.
872 */
873 void
rsync_finish(unsigned int id,int ok)874 rsync_finish(unsigned int id, int ok)
875 {
876 struct rsyncrepo *rr;
877 struct tarepo *tr;
878
879 tr = ta_find(id);
880 if (tr != NULL) {
881 /* repository changed state already, ignore request */
882 if (tr->state != REPO_LOADING)
883 return;
884 if (ok) {
885 logx("ta/%s: loaded from network", tr->descr);
886 stats.rsync_repos++;
887 tr->state = REPO_DONE;
888 repo_done(tr, 1);
889 } else {
890 logx("ta/%s: load from network failed", tr->descr);
891 stats.rsync_fails++;
892 tr->uriidx++;
893 ta_fetch(tr);
894 }
895 return;
896 }
897
898 rr = rsync_find(id);
899 if (rr == NULL)
900 errx(1, "unknown rsync repo %u", id);
901 /* repository changed state already, ignore request */
902 if (rr->state != REPO_LOADING)
903 return;
904
905 if (ok) {
906 logx("%s: loaded from network", rr->basedir);
907 stats.rsync_repos++;
908 rr->state = REPO_DONE;
909 } else {
910 logx("%s: load from network failed, fallback to cache",
911 rr->basedir);
912 stats.rsync_fails++;
913 rr->state = REPO_FAILED;
914 /* clear rsync repo since it failed */
915 remove_contents(rr->basedir);
916 }
917
918 repo_done(rr, ok);
919 }
920
921 /*
922 * RRDP sync finshed, either with or without success.
923 */
924 void
rrdp_finish(unsigned int id,int ok)925 rrdp_finish(unsigned int id, int ok)
926 {
927 struct rrdprepo *rr;
928
929 rr = rrdp_find(id);
930 if (rr == NULL)
931 errx(1, "unknown RRDP repo %u", id);
932 /* repository changed state already, ignore request */
933 if (rr->state != REPO_LOADING)
934 return;
935
936 if (ok) {
937 logx("%s: loaded from network", rr->notifyuri);
938 stats.rrdp_repos++;
939 rr->state = REPO_DONE;
940 } else {
941 logx("%s: load from network failed, fallback to rsync",
942 rr->notifyuri);
943 stats.rrdp_fails++;
944 rr->state = REPO_FAILED;
945 /* clear the RRDP repo since it failed */
946 remove_contents(rr->basedir);
947 /* also clear the list of deleted files */
948 filepath_free(&rr->deleted);
949 }
950
951 repo_done(rr, ok);
952 }
953
954 /*
955 * Handle responses from the http process. For TA file, either rename
956 * or delete the temporary file. For RRDP requests relay the request
957 * over to the rrdp process.
958 */
959 void
http_finish(unsigned int id,enum http_result res,const char * last_mod)960 http_finish(unsigned int id, enum http_result res, const char *last_mod)
961 {
962 struct tarepo *tr;
963
964 tr = ta_find(id);
965 if (tr == NULL) {
966 /* not a TA fetch therefor RRDP */
967 rrdp_http_done(id, res, last_mod);
968 return;
969 }
970
971 /* repository changed state already, ignore request */
972 if (tr->state != REPO_LOADING)
973 return;
974
975 /* Move downloaded TA file into place, or unlink on failure. */
976 if (res == HTTP_OK) {
977 char *file;
978
979 file = ta_filename(tr, 0);
980 if (rename(tr->temp, file) == -1)
981 warn("rename to %s", file);
982 free(file);
983
984 logx("ta/%s: loaded from network", tr->descr);
985 tr->state = REPO_DONE;
986 stats.http_repos++;
987 repo_done(tr, 1);
988 } else {
989 if (unlink(tr->temp) == -1 && errno != ENOENT)
990 warn("unlink %s", tr->temp);
991
992 tr->uriidx++;
993 logx("ta/%s: load from network failed", tr->descr);
994 ta_fetch(tr);
995 }
996 }
997
998
999
1000 /*
1001 * Look up a trust anchor, queueing it for download if not found.
1002 */
1003 struct repo *
ta_lookup(int id,struct tal * tal)1004 ta_lookup(int id, struct tal *tal)
1005 {
1006 struct repo *rp;
1007
1008 if (tal->urisz == 0)
1009 errx(1, "TAL %s has no URI", tal->descr);
1010
1011 /* Look up in repository table. (Lookup should actually fail here) */
1012 SLIST_FOREACH(rp, &repos, entry) {
1013 if (strcmp(rp->repouri, tal->descr) == 0)
1014 return rp;
1015 }
1016
1017 rp = repo_alloc(id);
1018 rp->basedir = repo_dir(tal->descr, "ta", 0);
1019 if ((rp->repouri = strdup(tal->descr)) == NULL)
1020 err(1, NULL);
1021
1022 /* try to create base directory */
1023 if (mkpath(rp->basedir) == -1)
1024 warn("mkpath %s", rp->basedir);
1025
1026 /* check if sync disabled ... */
1027 if (noop) {
1028 logx("ta/%s: using cache", rp->repouri);
1029 entityq_flush(&rp->queue, rp);
1030 return rp;
1031 }
1032
1033 rp->ta = ta_get(tal);
1034
1035 /* need to check if it was already loaded */
1036 if (repo_state(rp) != REPO_LOADING)
1037 entityq_flush(&rp->queue, rp);
1038
1039 return rp;
1040 }
1041
1042 /*
1043 * Look up a repository, queueing it for discovery if not found.
1044 */
1045 struct repo *
repo_lookup(int talid,const char * uri,const char * notify)1046 repo_lookup(int talid, const char *uri, const char *notify)
1047 {
1048 struct repo *rp;
1049 char *repouri;
1050 int nofetch = 0;
1051
1052 if ((repouri = rsync_base_uri(uri)) == NULL)
1053 errx(1, "bad caRepository URI: %s", uri);
1054
1055 /* Look up in repository table. */
1056 SLIST_FOREACH(rp, &repos, entry) {
1057 if (strcmp(rp->repouri, repouri) != 0)
1058 continue;
1059 if (rp->notifyuri != NULL) {
1060 if (notify == NULL)
1061 continue;
1062 if (strcmp(rp->notifyuri, notify) != 0)
1063 continue;
1064 } else if (notify != NULL)
1065 continue;
1066 /* found matching repo */
1067 free(repouri);
1068 return rp;
1069 }
1070
1071 rp = repo_alloc(talid);
1072 rp->basedir = repo_dir(repouri, NULL, 0);
1073 rp->repouri = repouri;
1074 if (notify != NULL)
1075 if ((rp->notifyuri = strdup(notify)) == NULL)
1076 err(1, NULL);
1077
1078 if (++talrepocnt[talid] >= MAX_REPO_PER_TAL) {
1079 if (talrepocnt[talid] == MAX_REPO_PER_TAL)
1080 warnx("too many repositories under %s", tals[talid]);
1081 nofetch = 1;
1082 }
1083
1084 /* try to create base directory */
1085 if (mkpath(rp->basedir) == -1)
1086 warn("mkpath %s", rp->basedir);
1087
1088 /* check if sync disabled ... */
1089 if (noop || nofetch) {
1090 logx("%s: using cache", rp->basedir);
1091 entityq_flush(&rp->queue, rp);
1092 return rp;
1093 }
1094
1095 /* ... else try RRDP first if available then rsync */
1096 if (notify != NULL)
1097 rp->rrdp = rrdp_get(notify);
1098 if (rp->rrdp == NULL)
1099 rp->rsync = rsync_get(uri, rp->basedir);
1100
1101 /* need to check if it was already loaded */
1102 if (repo_state(rp) != REPO_LOADING)
1103 entityq_flush(&rp->queue, rp);
1104
1105 return rp;
1106 }
1107
1108 /*
1109 * Find repository by identifier.
1110 */
1111 struct repo *
repo_byid(unsigned int id)1112 repo_byid(unsigned int id)
1113 {
1114 struct repo *rp;
1115
1116 SLIST_FOREACH(rp, &repos, entry) {
1117 if (rp->id == id)
1118 return rp;
1119 }
1120 return NULL;
1121 }
1122
1123 /*
1124 * Return the repository base or alternate directory.
1125 * Returned string must be freed by caller.
1126 */
1127 char *
repo_basedir(const struct repo * rp,int wantvalid)1128 repo_basedir(const struct repo *rp, int wantvalid)
1129 {
1130 char *path = NULL;
1131
1132 if (!wantvalid) {
1133 if (rp->ta) {
1134 if ((path = strdup(rp->ta->basedir)) == NULL)
1135 err(1, NULL);
1136 } else if (rp->rsync) {
1137 if ((path = strdup(rp->rsync->basedir)) == NULL)
1138 err(1, NULL);
1139 } else if (rp->rrdp) {
1140 path = rrdp_filename(rp->rrdp, rp->repouri, 0);
1141 } else
1142 path = NULL; /* only valid repo available */
1143 } else if (rp->basedir != NULL) {
1144 if ((path = strdup(rp->basedir)) == NULL)
1145 err(1, NULL);
1146 }
1147
1148 return path;
1149 }
1150
1151 /*
1152 * Return the repository identifier.
1153 */
1154 unsigned int
repo_id(const struct repo * rp)1155 repo_id(const struct repo *rp)
1156 {
1157 return rp->id;
1158 }
1159
1160 /*
1161 * Return the repository URI.
1162 */
1163 const char *
repo_uri(const struct repo * rp)1164 repo_uri(const struct repo *rp)
1165 {
1166 return rp->repouri;
1167 }
1168
1169 int
repo_queued(struct repo * rp,struct entity * p)1170 repo_queued(struct repo *rp, struct entity *p)
1171 {
1172 if (repo_state(rp) == REPO_LOADING) {
1173 TAILQ_INSERT_TAIL(&rp->queue, p, entries);
1174 return 1;
1175 }
1176 return 0;
1177 }
1178
1179 static void
repo_fail(struct repo * rp)1180 repo_fail(struct repo *rp)
1181 {
1182 /* reset the alarm since code may fallback to rsync */
1183 rp->alarm = getmonotime() + repo_timeout;
1184
1185 if (rp->ta)
1186 http_finish(rp->ta->id, HTTP_FAILED, NULL);
1187 else if (rp->rsync)
1188 rsync_finish(rp->rsync->id, 0);
1189 else if (rp->rrdp)
1190 rrdp_finish(rp->rrdp->id, 0);
1191 else
1192 errx(1, "%s: bad repo", rp->repouri);
1193 }
1194
1195 int
repo_check_timeout(int timeout)1196 repo_check_timeout(int timeout)
1197 {
1198 struct repo *rp;
1199 time_t now;
1200
1201 now = getmonotime();
1202 /* Look up in repository table. (Lookup should actually fail here) */
1203 SLIST_FOREACH(rp, &repos, entry) {
1204 if (repo_state(rp) == REPO_LOADING) {
1205 if (rp->alarm <= now) {
1206 warnx("%s: synchronisation timeout",
1207 rp->repouri);
1208 repo_fail(rp);
1209 } else {
1210 int diff = rp->alarm - now;
1211 diff *= 1000;
1212 if (timeout == INFTIM || diff < timeout)
1213 timeout = diff;
1214 }
1215 }
1216 }
1217 return timeout;
1218 }
1219
1220 static char **
add_to_del(char ** del,size_t * dsz,char * file)1221 add_to_del(char **del, size_t *dsz, char *file)
1222 {
1223 size_t i = *dsz;
1224
1225 del = reallocarray(del, i + 1, sizeof(*del));
1226 if (del == NULL)
1227 err(1, NULL);
1228 if ((del[i] = strdup(file)) == NULL)
1229 err(1, NULL);
1230 *dsz = i + 1;
1231 return del;
1232 }
1233
1234 /*
1235 * Delayed delete of files from RRDP. Since RRDP has no security built-in
1236 * this code needs to check if this RRDP repository is actually allowed to
1237 * remove the file referenced by the URI.
1238 */
1239 static char **
repo_cleanup_rrdp(struct filepath_tree * tree,char ** del,size_t * delsz)1240 repo_cleanup_rrdp(struct filepath_tree *tree, char **del, size_t *delsz)
1241 {
1242 struct rrdprepo *rr;
1243 struct filepath *fp, *nfp;
1244 char *fn;
1245
1246 SLIST_FOREACH(rr, &rrdprepos, entry) {
1247 RB_FOREACH_SAFE(fp, filepath_tree, &rr->deleted, nfp) {
1248 if (!rrdp_uri_valid(rr, fp->file)) {
1249 warnx("%s: external URI %s", rr->notifyuri,
1250 fp->file);
1251 filepath_put(&rr->deleted, fp);
1252 continue;
1253 }
1254 /* try to remove file from rrdp repo ... */
1255 fn = rrdp_filename(rr, fp->file, 0);
1256 del = add_to_del(del, delsz, fn);
1257 free(fn);
1258
1259 /* ... and from the valid repository if unused. */
1260 fn = rrdp_filename(rr, fp->file, 1);
1261 if (!filepath_exists(tree, fn))
1262 del = add_to_del(del, delsz, fn);
1263 else
1264 warnx("%s: referenced file supposed to be "
1265 "deleted", fn);
1266
1267 free(fn);
1268 filepath_put(&rr->deleted, fp);
1269 }
1270 }
1271
1272 return del;
1273 }
1274
1275 /*
1276 * All files in tree are valid and should be moved to the valid repository
1277 * if not already there. Rename the files to the new path and readd the
1278 * filepath entry with the new path if successful.
1279 */
1280 static void
repo_move_valid(struct filepath_tree * tree)1281 repo_move_valid(struct filepath_tree *tree)
1282 {
1283 struct filepath *fp, *nfp;
1284 size_t rsyncsz = strlen(".rsync/");
1285 size_t rrdpsz = strlen(".rrdp/");
1286 char *fn, *base;
1287
1288 RB_FOREACH_SAFE(fp, filepath_tree, tree, nfp) {
1289 if (strncmp(fp->file, ".rsync/", rsyncsz) != 0 &&
1290 strncmp(fp->file, ".rrdp/", rrdpsz) != 0)
1291 continue; /* not a temporary file path */
1292
1293 if (strncmp(fp->file, ".rsync/", rsyncsz) == 0) {
1294 fn = fp->file + rsyncsz;
1295 } else {
1296 base = strchr(fp->file + rrdpsz, '/');
1297 assert(base != NULL);
1298 fn = base + 1;
1299 }
1300
1301 if (repo_mkpath(fn) == -1)
1302 continue;
1303
1304 if (rename(fp->file, fn) == -1) {
1305 warn("rename %s", fp->file);
1306 continue;
1307 }
1308
1309 /* switch filepath node to new path */
1310 RB_REMOVE(filepath_tree, tree, fp);
1311 base = fp->file;
1312 if ((fp->file = strdup(fn)) == NULL)
1313 err(1, NULL);
1314 free(base);
1315 if (RB_INSERT(filepath_tree, tree, fp) != NULL)
1316 errx(1, "%s: both possibilities of file present",
1317 fp->file);
1318 }
1319 }
1320
1321 #define BASE_DIR (void *)0x62617365
1322 #define RSYNC_DIR (void *)0x73796e63
1323 #define RRDP_DIR (void *)0x52524450
1324
1325 static inline char *
skip_dotslash(char * in)1326 skip_dotslash(char *in)
1327 {
1328 if (memcmp(in, "./", 2) == 0)
1329 return in + 2;
1330 return in;
1331 }
1332
1333 void
repo_cleanup(struct filepath_tree * tree)1334 repo_cleanup(struct filepath_tree *tree)
1335 {
1336 size_t i, cnt, delsz = 0, dirsz = 0;
1337 char **del = NULL, **dir = NULL;
1338 char *argv[2] = { ".", NULL };
1339 FTS *fts;
1340 FTSENT *e;
1341
1342 /* first move temp files which have been used to valid dir */
1343 repo_move_valid(tree);
1344 /* then delete files requested by rrdp */
1345 del = repo_cleanup_rrdp(tree, del, &delsz);
1346
1347 if ((fts = fts_open(argv, FTS_PHYSICAL | FTS_NOSTAT, NULL)) == NULL)
1348 err(1, "fts_open");
1349 errno = 0;
1350 while ((e = fts_read(fts)) != NULL) {
1351 char *path = skip_dotslash(e->fts_path);
1352 switch (e->fts_info) {
1353 case FTS_NSOK:
1354 if (filepath_exists(tree, path)) {
1355 e->fts_parent->fts_number++;
1356 break;
1357 }
1358 if (e->fts_parent->fts_pointer == RRDP_DIR) {
1359 e->fts_parent->fts_number++;
1360 /* handle rrdp .state files explicitly */
1361 if (e->fts_level == 3 &&
1362 strcmp(e->fts_name, ".state") == 0)
1363 break;
1364 /* can't delete these extra files */
1365 stats.extra_files++;
1366 if (verbose > 1)
1367 logx("superfluous %s", path);
1368 break;
1369 }
1370 if (e->fts_parent->fts_pointer == RSYNC_DIR) {
1371 /* no need to keep rsync files */
1372 stats.extra_files++;
1373 if (verbose > 1)
1374 logx("superfluous %s", path);
1375 }
1376 del = add_to_del(del, &delsz, path);
1377 break;
1378 case FTS_D:
1379 if (e->fts_level == 1) {
1380 if (strcmp(".rsync", e->fts_name) == 0)
1381 e->fts_pointer = RSYNC_DIR;
1382 else if (strcmp(".rrdp", e->fts_name) == 0)
1383 e->fts_pointer = RRDP_DIR;
1384 else
1385 e->fts_pointer = BASE_DIR;
1386 } else
1387 e->fts_pointer = e->fts_parent->fts_pointer;
1388
1389 /*
1390 * special handling for rrdp directories,
1391 * clear them if they are not used anymore but
1392 * only if rrdp is active.
1393 */
1394 if (e->fts_pointer == RRDP_DIR && !noop &&
1395 e->fts_level == 2) {
1396 if (!rrdp_is_active(path))
1397 e->fts_pointer = NULL;
1398 }
1399 break;
1400 case FTS_DP:
1401 if (e->fts_level == FTS_ROOTLEVEL)
1402 break;
1403 if (e->fts_level == 1)
1404 /* do not remove .rsync and .rrdp */
1405 if (e->fts_pointer == RRDP_DIR ||
1406 e->fts_pointer == RSYNC_DIR)
1407 break;
1408 if (e->fts_number == 0)
1409 dir = add_to_del(dir, &dirsz, path);
1410
1411 e->fts_parent->fts_number += e->fts_number;
1412 break;
1413 case FTS_SL:
1414 case FTS_SLNONE:
1415 warnx("symlink %s", path);
1416 del = add_to_del(del, &delsz, path);
1417 break;
1418 case FTS_NS:
1419 case FTS_ERR:
1420 if (e->fts_errno == ENOENT &&
1421 e->fts_level == FTS_ROOTLEVEL)
1422 continue;
1423 warnx("fts_read %s: %s", path,
1424 strerror(e->fts_errno));
1425 break;
1426 default:
1427 warnx("fts_read %s: unhandled[%x]", path,
1428 e->fts_info);
1429 break;
1430 }
1431
1432 errno = 0;
1433 }
1434 if (errno)
1435 err(1, "fts_read");
1436 if (fts_close(fts) == -1)
1437 err(1, "fts_close");
1438
1439 cnt = 0;
1440 for (i = 0; i < delsz; i++) {
1441 if (unlink(del[i]) == -1) {
1442 if (errno != ENOENT)
1443 warn("unlink %s", del[i]);
1444 } else {
1445 if (verbose > 1)
1446 logx("deleted %s", del[i]);
1447 cnt++;
1448 }
1449 free(del[i]);
1450 }
1451 free(del);
1452 stats.del_files += cnt;
1453
1454 cnt = 0;
1455 for (i = 0; i < dirsz; i++) {
1456 if (rmdir(dir[i]) == -1)
1457 warn("rmdir %s", dir[i]);
1458 else
1459 cnt++;
1460 free(dir[i]);
1461 }
1462 free(dir);
1463 stats.del_dirs += cnt;
1464 }
1465
1466 void
repo_free(void)1467 repo_free(void)
1468 {
1469 struct repo *rp;
1470
1471 while ((rp = SLIST_FIRST(&repos)) != NULL) {
1472 SLIST_REMOVE_HEAD(&repos, entry);
1473 free(rp->repouri);
1474 free(rp->notifyuri);
1475 free(rp->basedir);
1476 free(rp);
1477 }
1478
1479 ta_free();
1480 rrdp_free();
1481 rsync_free();
1482 }
1483
1484 /*
1485 * Remove all files and directories under base but do not remove base itself.
1486 */
1487 static void
remove_contents(char * base)1488 remove_contents(char *base)
1489 {
1490 char *argv[2] = { base, NULL };
1491 FTS *fts;
1492 FTSENT *e;
1493
1494 if ((fts = fts_open(argv, FTS_PHYSICAL | FTS_NOSTAT, NULL)) == NULL)
1495 err(1, "fts_open");
1496 errno = 0;
1497 while ((e = fts_read(fts)) != NULL) {
1498 switch (e->fts_info) {
1499 case FTS_NSOK:
1500 case FTS_SL:
1501 case FTS_SLNONE:
1502 if (unlink(e->fts_accpath) == -1)
1503 warn("unlink %s", e->fts_path);
1504 break;
1505 case FTS_D:
1506 break;
1507 case FTS_DP:
1508 /* keep root directory */
1509 if (e->fts_level == FTS_ROOTLEVEL)
1510 break;
1511 if (rmdir(e->fts_accpath) == -1)
1512 warn("rmdir %s", e->fts_path);
1513 break;
1514 case FTS_NS:
1515 case FTS_ERR:
1516 warnx("fts_read %s: %s", e->fts_path,
1517 strerror(e->fts_errno));
1518 break;
1519 default:
1520 warnx("unhandled[%x] %s", e->fts_info,
1521 e->fts_path);
1522 break;
1523 }
1524 errno = 0;
1525 }
1526 if (errno)
1527 err(1, "fts_read");
1528 if (fts_close(fts) == -1)
1529 err(1, "fts_close");
1530 }
1531