1 /* 2 * Copyright (c) 2019-2020 The DragonFly Project. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Matthew Dillon <dillon@backplane.com> 6 * 7 * This code uses concepts and configuration based on 'synth', by 8 * John R. Marino <draco@marino.st>, which was written in ada. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in 18 * the documentation and/or other materials provided with the 19 * distribution. 20 * 3. Neither the name of The DragonFly Project nor the names of its 21 * contributors may be used to endorse or promote products derived 22 * from this software without specific, prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 25 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 26 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 27 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 28 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 29 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 30 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 31 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 32 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 33 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 34 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 35 * SUCH DAMAGE. 36 */ 37 #include "dsynth.h" 38 #include <openssl/md5.h> 39 40 typedef struct pinfo { 41 struct pinfo *next; 42 char *spath; 43 int foundit; 44 int inlocks; 45 } pinfo_t; 46 47 static void removePackagesMetaRecurse(pkg_t *pkg); 48 static int pinfocmp(const void *s1, const void *s2); 49 static void scanit(const char *path, const char *subpath, 50 int *countp, pinfo_t ***list_tailp, 51 int inlocks); 52 pinfo_t *pinfofind(pinfo_t **ary, int count, char *spath); 53 static void childRebuildRepo(bulk_t *bulk); 54 static void scandeletenew(const char *path); 55 56 static void rebuildTerminateSignal(int signo); 57 static char *md5lkfile(char *rpath, int which); 58 static int lkdircount(char *buf); 59 60 static char *RebuildRemovePath; 61 62 void 63 DoRebuildRepo(int ask) 64 { 65 bulk_t *bulk; 66 FILE *fp; 67 int fd; 68 char tpath[256]; 69 const char *sufx; 70 71 if (ask) { 72 if (askyn("Rebuild the repository? ") == 0) 73 return; 74 } 75 76 /* 77 * Scan the repository for temporary .new files and delete them. 78 */ 79 scandeletenew(RepositoryPath); 80 81 /* 82 * Generate temporary file 83 */ 84 snprintf(tpath, sizeof(tpath), "/tmp/meta.XXXXXXXX.conf"); 85 86 signal(SIGTERM, rebuildTerminateSignal); 87 signal(SIGINT, rebuildTerminateSignal); 88 signal(SIGHUP, rebuildTerminateSignal); 89 90 RebuildRemovePath = tpath; 91 92 sufx = UsePkgSufx; 93 fd = mkostemps(tpath, 5, 0); 94 if (fd < 0) 95 dfatal_errno("Cannot create %s", tpath); 96 fp = fdopen(fd, "w"); 97 fprintf(fp, "version = 1;\n"); 98 fprintf(fp, "packing_format = \"%s\";\n", sufx + 1); 99 fclose(fp); 100 101 /* 102 * Run the operation under our bulk infrastructure to 103 * get the correct environment. 104 */ 105 initbulk(childRebuildRepo, 1); 106 queuebulk(tpath, NULL, NULL, NULL); 107 bulk = getbulk(); 108 109 if (bulk->r1) 110 printf("Rebuild succeeded\n"); 111 else 112 printf("Rebuild failed\n"); 113 donebulk(); 114 115 remove(tpath); 116 } 117 118 static void 119 repackage(const char *basepath, const char *basefile, 120 const char *decomp_suffix, const char *comp_suffix, 121 const char *decomp, const char *comp); 122 123 static void 124 childRebuildRepo(bulk_t *bulk) 125 { 126 FILE *fp; 127 char *ptr; 128 size_t len; 129 pid_t pid; 130 const char *cav[MAXCAC]; 131 int cac; 132 int repackage_mode = 0; 133 134 cac = 0; 135 cav[cac++] = PKG_BINARY; 136 cav[cac++] = "repo"; 137 cav[cac++] = "-m"; 138 cav[cac++] = bulk->s1; 139 cav[cac++] = "-o"; 140 cav[cac++] = PackagesPath; 141 142 /* 143 * The yaml needs to generate paths relative to PackagePath 144 */ 145 if (strncmp(PackagesPath, RepositoryPath, strlen(PackagesPath)) == 0) 146 cav[cac++] = PackagesPath; 147 else 148 cav[cac++] = RepositoryPath; 149 150 printf("pkg repo -m %s -o %s %s\n", bulk->s1, cav[cac-2], cav[cac-1]); 151 152 fp = dexec_open(NULL, cav, cac, &pid, NULL, 1, 0); 153 while ((ptr = fgetln(fp, &len)) != NULL) 154 fwrite(ptr, 1, len, stdout); 155 if (dexec_close(fp, pid) == 0) 156 bulk->r1 = strdup(""); 157 158 /* 159 * Check package version. Pkg version 1.12 and later generates 160 * the proper repo compression format. Prior to that version 161 * the repo directive always generated .txz files. 162 */ 163 cac = 0; 164 cav[cac++] = PKG_BINARY; 165 cav[cac++] = "-v"; 166 fp = dexec_open(NULL, cav, cac, &pid, NULL, 1, 0); 167 if ((ptr = fgetln(fp, &len)) != NULL && len > 0) { 168 int v1; 169 int v2; 170 171 ptr[len-1] = 0; 172 if (sscanf(ptr, "%d.%d", &v1, &v2) == 2) { 173 if (v1 > 1 || (v1 == 1 && v2 >= 12)) 174 repackage_mode = 1; 175 } 176 } 177 dexec_close(fp, pid); 178 179 /* 180 * Repackage the .txz files created by pkg repo if necessary 181 */ 182 if (repackage_mode == 0 && strcmp(UsePkgSufx, ".txz") != 0) { 183 const char *comp; 184 const char *decomp; 185 186 printf("pkg repo - recompressing digests and packagesite\n"); 187 188 if (strcmp(UsePkgSufx, ".tar") == 0) { 189 decomp = "unxz"; 190 comp = "cat"; 191 } else if (strcmp(UsePkgSufx, ".tgz") == 0) { 192 decomp = "unxz"; 193 comp = "gzip"; 194 } else if (strcmp(UsePkgSufx, ".tbz") == 0) { 195 decomp = "unxz"; 196 comp = "bzip"; 197 } else if (strcmp(UsePkgSufx, ".tzst") == 0) { 198 decomp = "unxz"; 199 comp = "zstd"; 200 } else { 201 dfatal("recompressing as %s not supported", 202 UsePkgSufx); 203 decomp = "unxz"; 204 comp = "cat"; 205 } 206 repackage(PackagesPath, "digests", 207 ".txz", UsePkgSufx, 208 decomp, comp); 209 repackage(PackagesPath, "packagesite", 210 ".txz", UsePkgSufx, 211 decomp, comp); 212 } else if (repackage_mode == 1 && strcmp(UsePkgSufx, ".txz") != 0) { 213 const char *comp; 214 const char *decomp; 215 216 printf("pkg repo - recompressing meta\n"); 217 218 if (strcmp(UsePkgSufx, ".tar") == 0) { 219 decomp = "cat"; 220 comp = "xz"; 221 } else if (strcmp(UsePkgSufx, ".tgz") == 0) { 222 decomp = "gunzip"; 223 comp = "xz"; 224 } else if (strcmp(UsePkgSufx, ".tbz") == 0) { 225 decomp = "bunzip2"; 226 comp = "xz"; 227 } else if (strcmp(UsePkgSufx, ".tzst") == 0) { 228 decomp = "unzstd"; 229 comp = "xz"; 230 } else { 231 dfatal("recompressing from %s not supported", 232 UsePkgSufx); 233 decomp = "cat"; 234 comp = "cat"; 235 } 236 repackage(PackagesPath, "meta", 237 UsePkgSufx, ".txz", 238 decomp, comp); 239 } 240 } 241 242 static 243 void 244 repackage(const char *basepath, const char *basefile, 245 const char *decomp_suffix, const char *comp_suffix, 246 const char *decomp, const char *comp) 247 { 248 char *buf; 249 250 asprintf(&buf, "%s < %s/%s%s | %s > %s/%s%s", 251 decomp, basepath, basefile, decomp_suffix, 252 comp, basepath, basefile, comp_suffix); 253 if (system(buf) != 0) { 254 dfatal("command failed: %s", buf); 255 } 256 free(buf); 257 } 258 259 void 260 DoUpgradePkgs(pkg_t *pkgs __unused, int ask __unused) 261 { 262 dfatal("Not Implemented"); 263 } 264 265 void 266 PurgeDistfiles(pkg_t *pkgs) 267 { 268 pinfo_t *list; 269 pinfo_t *item; 270 pinfo_t **list_tail; 271 pinfo_t **ary; 272 char *dstr; 273 char *buf; 274 int count; 275 int delcount; 276 int i; 277 278 printf("Scanning distfiles... "); 279 fflush(stdout); 280 count = 0; 281 list = NULL; 282 list_tail = &list; 283 scanit(DistFilesPath, NULL, &count, &list_tail, 0); 284 printf("Checking %d distfiles\n", count); 285 fflush(stdout); 286 287 ary = calloc(count, sizeof(pinfo_t *)); 288 for (i = 0; i < count; ++i) { 289 ary[i] = list; 290 list = list->next; 291 } 292 ddassert(list == NULL); 293 qsort(ary, count, sizeof(pinfo_t *), pinfocmp); 294 295 for (; pkgs; pkgs = pkgs->bnext) { 296 if (pkgs->distfiles == NULL || pkgs->distfiles[0] == 0) 297 continue; 298 ddprintf(0, "distfiles %s\n", pkgs->distfiles); 299 dstr = strtok(pkgs->distfiles, " \t"); 300 while (dstr) { 301 for (;;) { 302 /* 303 * Look for distfile 304 */ 305 if (pkgs->distsubdir) { 306 asprintf(&buf, "%s/%s", 307 pkgs->distsubdir, dstr); 308 } else { 309 buf = dstr; 310 } 311 item = pinfofind(ary, count, buf); 312 if (item) 313 item->foundit = 1; 314 if (item && item->inlocks == 0) { 315 /* 316 * Look for the lock file 317 */ 318 int scount; 319 320 scount = lkdircount(buf); 321 322 for (i = 0; i <= scount; ++i) { 323 item = pinfofind(ary, count, 324 md5lkfile(buf, i)); 325 if (item) 326 item->foundit = 1; 327 } 328 } 329 330 /* 331 * Cleanup and iterate 332 */ 333 if (buf != dstr) { 334 free(buf); 335 buf = NULL; 336 } 337 if (strrchr(dstr, ':') == NULL) 338 break; 339 *strrchr(dstr, ':') = 0; 340 } 341 dstr = strtok(NULL, " \t"); 342 } 343 } 344 345 delcount = 0; 346 for (i = 0; i < count; ++i) { 347 item = ary[i]; 348 if (item->foundit == 0) { 349 ++delcount; 350 } 351 } 352 if (delcount == 0) { 353 printf("No obsolete source files out of %d found\n", count); 354 } else if (askyn("Delete %d of %d items? ", delcount, count)) { 355 printf("Deleting %d/%d obsolete source distfiles\n", 356 delcount, count); 357 for (i = 0; i < count; ++i) { 358 item = ary[i]; 359 if (item->foundit == 0) { 360 asprintf(&buf, "%s/%s", 361 DistFilesPath, item->spath); 362 if (remove(buf) < 0) 363 printf("Cannot delete %s\n", buf); 364 else 365 printf("Deleted %s\n", item->spath); 366 free(buf); 367 } 368 } 369 } 370 371 372 free(ary); 373 } 374 375 void 376 RemovePackages(pkg_t *list) 377 { 378 pkg_t *scan; 379 char *path; 380 381 for (scan = list; scan; scan = scan->bnext) { 382 if ((scan->flags & PKGF_MANUALSEL) == 0) 383 continue; 384 if (scan->pkgfile) { 385 scan->flags &= ~PKGF_PACKAGED; 386 scan->pkgfile_size = 0; 387 asprintf(&path, "%s/%s", RepositoryPath, scan->pkgfile); 388 if (remove(path) == 0) 389 printf("Removed: %s\n", path); 390 free(path); 391 } 392 if (scan->pkgfile == NULL || 393 (scan->flags & (PKGF_DUMMY | PKGF_META))) { 394 removePackagesMetaRecurse(scan); 395 } 396 } 397 } 398 399 static void 400 removePackagesMetaRecurse(pkg_t *pkg) 401 { 402 pkglink_t *link; 403 pkg_t *scan; 404 char *path; 405 406 PKGLIST_FOREACH(link, &pkg->idepon_list) { 407 scan = link->pkg; 408 if (scan == NULL) 409 continue; 410 if (scan->pkgfile == NULL || 411 (scan->flags & (PKGF_DUMMY | PKGF_META))) { 412 removePackagesMetaRecurse(scan); 413 continue; 414 } 415 scan->flags &= ~PKGF_PACKAGED; 416 scan->pkgfile_size = 0; 417 418 asprintf(&path, "%s/%s", RepositoryPath, scan->pkgfile); 419 if (remove(path) == 0) 420 printf("Removed: %s\n", path); 421 free(path); 422 } 423 } 424 425 static int 426 pinfocmp(const void *s1, const void *s2) 427 { 428 const pinfo_t *item1 = *(const pinfo_t *const*)s1; 429 const pinfo_t *item2 = *(const pinfo_t *const*)s2; 430 431 return (strcmp(item1->spath, item2->spath)); 432 } 433 434 pinfo_t * 435 pinfofind(pinfo_t **ary, int count, char *spath) 436 { 437 pinfo_t *item; 438 int res; 439 int b; 440 int e; 441 int m; 442 443 b = 0; 444 e = count; 445 while (b != e) { 446 m = b + (e - b) / 2; 447 item = ary[m]; 448 res = strcmp(spath, item->spath); 449 if (res == 0) 450 return item; 451 if (res < 0) { 452 e = m; 453 } else { 454 b = m + 1; 455 } 456 } 457 return NULL; 458 } 459 460 void 461 scanit(const char *path, const char *subpath, 462 int *countp, pinfo_t ***list_tailp, 463 int inlocks) 464 { 465 struct dirent *den; 466 pinfo_t *item; 467 char *npath; 468 char *spath; 469 DIR *dir; 470 struct stat st; 471 472 if ((dir = opendir(path)) != NULL) { 473 while ((den = readdir(dir)) != NULL) { 474 if (den->d_namlen == 1 && den->d_name[0] == '.') 475 continue; 476 if (den->d_namlen == 2 && den->d_name[0] == '.' && 477 den->d_name[1] == '.') 478 continue; 479 asprintf(&npath, "%s/%s", path, den->d_name); 480 if (lstat(npath, &st) < 0) { 481 free(npath); 482 continue; 483 } 484 if (S_ISDIR(st.st_mode)) { 485 int sublocks; 486 487 sublocks = 488 (strcmp(den->d_name, ".locks") == 0); 489 490 if (subpath) { 491 asprintf(&spath, "%s/%s", 492 subpath, den->d_name); 493 scanit(npath, spath, countp, 494 list_tailp, sublocks); 495 free(spath); 496 } else { 497 scanit(npath, den->d_name, countp, 498 list_tailp, sublocks); 499 } 500 } else if (S_ISREG(st.st_mode)) { 501 item = calloc(1, sizeof(*item)); 502 if (subpath) { 503 asprintf(&item->spath, "%s/%s", 504 subpath, den->d_name); 505 } else { 506 item->spath = strdup(den->d_name); 507 } 508 item->inlocks = inlocks; 509 510 **list_tailp = item; 511 *list_tailp = &item->next; 512 ++*countp; 513 ddprintf(0, "scan %s\n", item->spath); 514 } 515 free(npath); 516 } 517 closedir(dir); 518 } 519 } 520 521 /* 522 * This removes any .new files left over in the repo. These can wind 523 * being left around when dsynth is killed. 524 */ 525 static void 526 scandeletenew(const char *path) 527 { 528 struct dirent *den; 529 const char *ptr; 530 DIR *dir; 531 char *buf; 532 533 if ((dir = opendir(path)) == NULL) 534 dfatal_errno("Cannot scan directory %s", path); 535 while ((den = readdir(dir)) != NULL) { 536 if ((ptr = strrchr(den->d_name, '.')) != NULL && 537 strcmp(ptr, ".new") == 0) { 538 asprintf(&buf, "%s/%s", path, den->d_name); 539 if (remove(buf) < 0) 540 dfatal_errno("remove: Garbage %s\n", buf); 541 printf("Deleted Garbage %s\n", buf); 542 free(buf); 543 } 544 } 545 closedir(dir); 546 } 547 548 static void 549 rebuildTerminateSignal(int signo __unused) 550 { 551 if (RebuildRemovePath) 552 remove(RebuildRemovePath); 553 exit(1); 554 555 } 556 557 /* 558 * There will be a .locks sub-directory in /usr/distfiles and also 559 * in each sub-directory underneath it containing the MD5 sums for 560 * the files in that subdirectory. 561 * 562 * This is a bit of a mess. Sometimes the .locks/ for a subdirectory 563 * are in parentdir/.locks and not parentdir/subdir/.locks. The invocation 564 * of do-fetch can be a bit messy so we look for a .locks subdir everywhere. 565 * 566 * The /usr/dports/Mk/Scripts/do-fetch.sh script uses 'echo blah | md5', 567 * so we have to add a newline to the buffer being md5'd. 568 * 569 * The pass-in rpath is relative to the distfiles base. 570 */ 571 static char * 572 md5lkfile(char *rpath, int which_slash) 573 { 574 static char mstr[128]; 575 static char lkfile[128]; 576 uint8_t digest[MD5_DIGEST_LENGTH]; 577 int bplen; 578 int i; 579 580 bplen = 0; 581 for (i = 0; i < which_slash; ++i) { 582 while (rpath[bplen] && rpath[bplen] != '/') 583 ++bplen; 584 if (rpath[bplen]) 585 ++bplen; 586 } 587 snprintf(mstr, sizeof(mstr), "%s\n", rpath + bplen); 588 MD5(mstr, strlen(mstr), digest); 589 590 snprintf(lkfile, sizeof(lkfile), 591 "%*.*s.locks/" 592 "%02x%02x%02x%02x%02x%02x%02x%02x" 593 "%02x%02x%02x%02x%02x%02x%02x%02x" 594 ".lk", 595 bplen, bplen, rpath, 596 digest[0], digest[1], digest[2], digest[3], 597 digest[4], digest[5], digest[6], digest[7], 598 digest[8], digest[9], digest[10], digest[11], 599 digest[12], digest[13], digest[14], digest[15]); 600 601 return lkfile; 602 } 603 604 static int 605 lkdircount(char *buf) 606 { 607 int i; 608 int n; 609 610 n = 0; 611 for (i = 0; buf[i]; ++i) { 612 if (buf[i] == '/') 613 ++n; 614 } 615 return n; 616 } 617