1 /* 2 * Copyright (c) 2019 The DragonFly Project. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Matthew Dillon <dillon@backplane.com> 6 * 7 * This code uses concepts and configuration based on 'synth', by 8 * John R. Marino <draco@marino.st>, which was written in ada. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in 18 * the documentation and/or other materials provided with the 19 * distribution. 20 * 3. Neither the name of The DragonFly Project nor the names of its 21 * contributors may be used to endorse or promote products derived 22 * from this software without specific, prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 25 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 26 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 27 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 28 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 29 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 30 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 31 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 32 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 33 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 34 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 35 * SUCH DAMAGE. 36 */ 37 38 #include "dsynth.h" 39 40 #define PKG_HSIZE 32768 41 #define PKG_HMASK 32767 42 43 static int parsepkglist_file(const char *path, int debugstop); 44 static void childGetPackageInfo(bulk_t *bulk); 45 static void childGetBinaryDistInfo(bulk_t *bulk); 46 static void childOptimizeEnv(bulk_t *bulk); 47 static pkg_t *resolveDeps(pkg_t *dep_list, pkg_t ***list_tailp, int gentopo); 48 static void resolveFlavors(pkg_t *pkg, char *flavors, int gentopo); 49 static void resolveDepString(pkg_t *pkg, char *depstr, 50 int gentopo, int dep_type); 51 static pkg_t *processPackageListBulk(int total); 52 static int scan_and_queue_dir(const char *path, const char *level1, int level); 53 static int scan_binary_repo(const char *path); 54 #if 0 55 static void pkgfree(pkg_t *pkg); 56 #endif 57 58 static int PrepareSystemFlag; 59 60 static pkg_t *PkgHash1[PKG_HSIZE]; /* by portdir */ 61 static pkg_t *PkgHash2[PKG_HSIZE]; /* by pkgfile */ 62 63 /* 64 * Allocate a new pkg structure plus basic initialization. 65 */ 66 static __inline pkg_t * 67 allocpkg(void) 68 { 69 pkg_t *pkg; 70 71 pkg = calloc(1, sizeof(*pkg)); 72 pkg->idepon_list.next = &pkg->idepon_list; 73 pkg->idepon_list.prev = &pkg->idepon_list; 74 pkg->deponi_list.next = &pkg->deponi_list; 75 pkg->deponi_list.prev = &pkg->deponi_list; 76 77 return pkg; 78 } 79 80 /* 81 * Simple hash for lookups 82 */ 83 static __inline int 84 pkghash(const char *str) 85 { 86 int hv = 0xABC32923; 87 while (*str) { 88 hv = (hv << 5) ^ *str; 89 ++str; 90 } 91 hv = hv ^ (hv / PKG_HSIZE) ^ (hv / PKG_HSIZE / PKG_HSIZE); 92 return (hv & PKG_HMASK); 93 } 94 95 static __inline const char * 96 deptype2str(int dep_type) 97 { 98 switch (dep_type) { 99 case DEP_TYPE_FETCH: 100 return("FETCH_DEPENDS"); 101 case DEP_TYPE_EXT: 102 return("EXTRACT_DEPENDS"); 103 case DEP_TYPE_PATCH: 104 return("PATCH_DEPENDS"); 105 case DEP_TYPE_BUILD: 106 return("BUILD_DEPENDS"); 107 case DEP_TYPE_LIB: 108 return("LIB_DEPENDS"); 109 case DEP_TYPE_RUN: 110 return("RUN_DEPENDS"); 111 default: 112 return("UNKNOWN"); 113 } 114 } 115 116 static void 117 pkg_enter(pkg_t *pkg) 118 { 119 pkg_t **pkgp; 120 pkg_t *scan; 121 122 if (pkg->portdir) { 123 pkgp = &PkgHash1[pkghash(pkg->portdir)]; 124 while ((scan = *pkgp) != NULL) { 125 if (strcmp(pkg->portdir, scan->portdir) == 0) 126 break; 127 pkgp = &scan->hnext1; 128 } 129 ddassert(scan == NULL || (scan->flags & PKGF_PLACEHOLD)); 130 if (scan && (scan->flags & PKGF_PLACEHOLD)) { 131 ddassert(scan->idepon_list.next == &scan->idepon_list); 132 ddassert(scan->deponi_list.next == &scan->deponi_list); 133 *pkgp = pkg; 134 pkg->hnext1 = scan->hnext1; 135 free(scan->portdir); 136 free(scan); 137 scan = NULL; 138 } 139 if (scan == NULL) 140 *pkgp = pkg; 141 } 142 143 if (pkg->pkgfile) { 144 pkgp = &PkgHash2[pkghash(pkg->pkgfile)]; 145 while ((scan = *pkgp) != NULL) { 146 if (strcmp(pkg->pkgfile, scan->pkgfile) == 0) 147 break; 148 pkgp = &scan->hnext2; 149 } 150 if (scan == NULL) 151 *pkgp = pkg; 152 } 153 } 154 155 static pkg_t * 156 pkg_find(const char *match) 157 { 158 pkg_t **pkgp; 159 pkg_t *pkg; 160 161 pkgp = &PkgHash1[pkghash(match)]; 162 for (pkg = *pkgp; pkg; pkg = pkg->hnext1) { 163 if (strcmp(pkg->portdir, match) == 0) 164 return pkg; 165 } 166 pkgp = &PkgHash2[pkghash(match)]; 167 for (pkg = *pkgp; pkg; pkg = pkg->hnext2) { 168 if (strcmp(pkg->pkgfile, match) == 0) 169 return pkg; 170 } 171 return NULL; 172 } 173 174 /* 175 * Parse a specific list of ports via origin name (portdir/subdir) 176 */ 177 pkg_t * 178 ParsePackageList(int n, char **ary, int debugstop) 179 { 180 pkg_t *list; 181 int total; 182 int fail; 183 int i; 184 185 total = 0; 186 fail = 0; 187 initbulk(childGetPackageInfo, MaxBulk); 188 189 /* 190 * Always include ports-mgmt/pkg. s4 is "x" meaning not a manual 191 * selection, "d" meaning DEBUGSTOP mode, or NULL. 192 */ 193 queuebulk("ports-mgmt", "pkg", NULL, "x"); 194 195 for (i = 0; i < n; ++i) { 196 char *l1; 197 char *l2; 198 char *l3; 199 struct stat st; 200 201 l1 = strdup(ary[i]); 202 if (stat(l1, &st) == 0 && S_ISREG(st.st_mode)) { 203 total += parsepkglist_file(l1, debugstop); 204 continue; 205 } 206 207 l2 = strchr(l1, '/'); 208 if (l2 == NULL) { 209 printf("Bad portdir specification: %s\n", l1); 210 free(l1); 211 fail = 1; 212 continue; 213 } 214 *l2++ = 0; 215 l3 = strchr(l2, '@'); 216 if (l3) 217 *l3++ = 0; 218 219 /* 220 * Silently ignore any manually specified ports-mgmt/pkg, 221 * which we already auto-added. 222 */ 223 if (strcmp(l1, "ports-mgmt") != 0 || 224 strcmp(l2, "pkg") != 0) 225 { 226 queuebulk(l1, l2, l3, (debugstop ? "d" : NULL)); 227 } 228 ++total; 229 free(l1); 230 } 231 printf("Processing %d ports\n", total); 232 233 list = processPackageListBulk(total); 234 if (fail) { 235 dfatal("Bad specifications, exiting"); 236 exit(1); 237 } 238 239 return list; 240 } 241 242 static 243 int 244 parsepkglist_file(const char *path, int debugstop) 245 { 246 FILE *fp; 247 char *base; 248 char *l1; 249 char *l2; 250 char *l3; 251 size_t len; 252 int total; 253 254 if ((fp = fopen(path, "r")) == NULL) { 255 dpanic_errno("Cannot read %s\n", path); 256 /* NOT REACHED */ 257 return 0; 258 } 259 260 total = 0; 261 262 while ((base = fgetln(fp, &len)) != NULL) { 263 if (len == 0 || base[len-1] != '\n') 264 continue; 265 base[--len] = 0; 266 l1 = strtok(base, " \t\r\n"); 267 if (l1 == NULL) { 268 printf("Badly formatted pkg info line: %s\n", base); 269 continue; 270 } 271 l2 = strchr(l1, '/'); 272 if (l2 == NULL) { 273 printf("Badly formatted specification: %s\n", l1); 274 continue; 275 } 276 *l2++ = 0; 277 l3 = strchr(l2, '@'); 278 if (l3) 279 *l3++ = 0; 280 queuebulk(l1, l2, l3, (debugstop ? "d" : NULL)); 281 ++total; 282 } 283 fclose(fp); 284 285 return total; 286 } 287 288 /* 289 * Parse packages from the list installed on the system. 290 */ 291 pkg_t * 292 GetLocalPackageList(void) 293 { 294 pkg_t *list; 295 FILE *fp; 296 char *base; 297 char *data; 298 char *l1; 299 char *l2; 300 char *l3; 301 int total; 302 int state; 303 size_t len; 304 305 PrepareSystemFlag = 1; 306 initbulk(childGetPackageInfo, MaxBulk); 307 total = 0; 308 state = 0; 309 l1 = NULL; 310 l2 = NULL; 311 l3 = NULL; 312 313 fp = popen("pkg info -a -o -A", "r"); 314 315 /* 316 * Always include ports-mgmt/pkg. s4 is "x" meaning not a manual 317 * selection, "d" meaning DEBUGSTOP mode, or NULL. 318 */ 319 queuebulk("ports-mgmt", "pkg", NULL, "x"); 320 321 while ((base = fgetln(fp, &len)) != NULL) { 322 if (len == 0 || base[len-1] != '\n') 323 continue; 324 base[--len] = 0; 325 326 data = strchr(base, ':'); 327 if (data == NULL) 328 continue; 329 *data++ = 0; 330 331 base = strtok(base, " \t\r"); 332 data = strtok(data, " \t\r"); 333 334 if (base == NULL || data == NULL) 335 continue; 336 337 if (strcmp(base, "Origin") == 0) { 338 if (state == 1) { 339 queuebulk(l1, l2, NULL, NULL); 340 state = 0; 341 ++total; 342 } 343 344 if (strchr(data, '/') == NULL) { 345 printf("Badly formatted origin: %s\n", l1); 346 } 347 if (l1) 348 free(l1); 349 if (l3) 350 free(l3); 351 l1 = strdup(data); 352 l2 = strchr(l1, '/'); 353 *l2++ = 0; 354 l3 = strchr(l2, '@'); /* typically NULL */ 355 if (l3) { 356 *l3++ = 0; 357 l3 = strdup(l3); 358 } 359 360 /* 361 * Don't queue ports-mgmt/pkg twice, we already 362 * queued it manually. 363 */ 364 if (strcmp(l1, "ports-mgmt") != 0 || 365 strcmp(l2, "pkg") != 0) { 366 state = 1; 367 } 368 continue; 369 } 370 if (state == 1 && strcmp(base, "flavor") == 0) { 371 queuebulk(l1, l2, data, NULL); 372 state = 0; 373 ++total; 374 } 375 } 376 if (state == 1) { 377 queuebulk(l1, l2, NULL, NULL); 378 /*state = 0; not needed */ 379 } 380 if (l1) 381 free(l1); 382 if (l3) 383 free(l3); 384 385 pclose(fp); 386 387 printf("Processing %d ports\n", total); 388 389 list = processPackageListBulk(total); 390 391 return list; 392 } 393 394 pkg_t * 395 GetFullPackageList(void) 396 { 397 int total; 398 399 initbulk(childGetPackageInfo, MaxBulk); 400 total = scan_and_queue_dir(DPortsPath, NULL, 1); 401 printf("Scanning %d ports\n", total); 402 403 return processPackageListBulk(total); 404 } 405 406 /* 407 * Caller has queued the process list for bulk operation. We retrieve 408 * the results and clean up the bulk operation (we may have to do a second 409 * bulk operation so we have to be the ones to clean it up). 410 */ 411 static pkg_t * 412 processPackageListBulk(int total) 413 { 414 bulk_t *bulk; 415 pkg_t *scan; 416 pkg_t *list; 417 pkg_t *dep_list; 418 pkg_t **list_tail; 419 int count; 420 int stop_fail; 421 int stop_base_list; 422 int remove_corrupt; 423 424 list = NULL; 425 list_tail = &list; 426 count = 0; 427 remove_corrupt = 0; 428 429 while ((bulk = getbulk()) != NULL) { 430 ++count; 431 if ((count & 255) == 0) { 432 printf("%6.2f%%\r", 433 (double)count * 100.0 / (double)total + 0.001); 434 fflush(stdout); 435 } 436 if (bulk->list) { 437 *list_tail = bulk->list; 438 bulk->list = NULL; 439 while ((scan = *list_tail) != NULL) { 440 if (bulk->s4 == NULL || bulk->s4[0] != 'x') 441 scan->flags |= PKGF_MANUALSEL; 442 pkg_enter(scan); 443 list_tail = &scan->bnext; 444 } 445 } 446 freebulk(bulk); 447 } 448 printf("100.00%%\n"); 449 printf("\nTotal %d\n", count); 450 fflush(stdout); 451 452 /* 453 * Resolve all dependencies for the related packages, potentially 454 * adding anything that could not be found to the list. This will 455 * continue to issue bulk operations and process the result until 456 * no dependencies are left. 457 */ 458 printf("Resolving dependencies..."); 459 fflush(stdout); 460 dep_list = list; 461 while (dep_list) { 462 dep_list = resolveDeps(dep_list, &list_tail, 0); 463 } 464 printf("done\n"); 465 466 donebulk(); 467 468 /* 469 * Generate the topology 470 */ 471 resolveDeps(list, NULL, 1); 472 473 /* 474 * Do a final count, ignore place holders. 475 * 476 * Also set stop_fail if appropriate. Check for direct specifications 477 * which fail to probe and any direct dependencies of those 478 * specifications, but don't recurse (for now)... don't check indirect 479 * dependencies (i.e. A -> B -> C where A is directly specified, B 480 * is adirect dependency, and C fails to probe). 481 */ 482 count = 0; 483 stop_fail = 0; 484 stop_base_list = 0; 485 for (scan = list; scan; scan = scan->bnext) { 486 if ((scan->flags & PKGF_ERROR) == 0) { 487 ++count; 488 } 489 if ((scan->flags & PKGF_MANUALSEL) && MaskProbeAbort == 0) { 490 pkglink_t *link; 491 492 /* 493 * Directly specified package failed to probe 494 */ 495 if (scan->flags & PKGF_CORRUPT) { 496 ++stop_fail; 497 ++stop_base_list; 498 } 499 500 /* 501 * Directly specified package had a direct dependency 502 * that failed to probe (don't go further). 503 */ 504 PKGLIST_FOREACH(link, &scan->idepon_list) { 505 if (link->pkg && 506 (link->pkg->flags & PKGF_CORRUPT)) { 507 ++stop_fail; 508 } 509 } 510 } 511 } 512 printf("Total Returned %d\n", count); 513 514 /* 515 * Check to see if any PKGF_MANUALSEL packages 516 */ 517 if (stop_fail) { 518 printf("%d packages failed to probe\n", stop_fail); 519 if (PrepareSystemFlag) { 520 if (stop_fail == stop_base_list) { 521 printf( 522 "prepare-system: Some of your installed packages no longer exist in\n" 523 "dports, do you wish to continue rebuilding what does exist?\n"); 524 if (askyn("Continue anyway? ")) 525 remove_corrupt = 1; 526 } else { 527 printf( 528 "prepare-system: Some of your installed packages have dependencies\n" 529 "which could not be found in dports, cannot continue, aborting\n"); 530 } 531 } else { 532 printf("unable to continue, aborting\n"); 533 } 534 if (remove_corrupt == 0) 535 exit(1); 536 } 537 538 /* 539 * Remove corrupt packages before continuing 540 */ 541 if (remove_corrupt) { 542 list_tail = &list; 543 while ((scan = *list_tail) != NULL) { 544 if (scan->flags & PKGF_CORRUPT) 545 *list_tail = scan->bnext; 546 else 547 list_tail = &scan->bnext; 548 } 549 } 550 551 /* 552 * Scan our binary distributions and related dependencies looking 553 * for any packages that have already been built. 554 */ 555 initbulk(childGetBinaryDistInfo, MaxBulk); 556 total = scan_binary_repo(RepositoryPath); 557 count = 0; 558 printf("Scanning %d packages\n", total); 559 560 while ((bulk = getbulk()) != NULL) { 561 ++count; 562 if ((count & 255) == 0) { 563 printf("%6.2f%%\r", 564 (double)count * 100.0 / (double)total + 0.001); 565 fflush(stdout); 566 } 567 freebulk(bulk); 568 } 569 printf("100.00%%\n"); 570 printf("\nTotal %d\n", count); 571 fflush(stdout); 572 donebulk(); 573 574 printf("all done\n"); 575 576 return list; 577 } 578 579 pkg_t * 580 GetPkgPkg(pkg_t **listp) 581 { 582 bulk_t *bulk; 583 pkg_t *scan; 584 pkg_t *s2; 585 586 for (scan = *listp; scan; scan = scan->bnext) { 587 if (strcmp(scan->portdir, "ports-mgmt/pkg") == 0) 588 return scan; 589 } 590 591 /* 592 * This will force pkg to be built, but generally this code 593 * is not reached because the package list processing code 594 * adds ports-mgmt/pkg unconditionally. 595 */ 596 initbulk(childGetPackageInfo, MaxBulk); 597 queuebulk("ports-mgmt", "pkg", NULL, "x"); 598 bulk = getbulk(); 599 dassert(bulk, "Cannot find ports-mgmt/pkg"); 600 scan = bulk->list; 601 bulk->list = NULL; 602 freebulk(bulk); 603 donebulk(); 604 605 606 /* 607 * Include added packages to the total and add the initial bulk 608 * built packages to the list so they get counted. 609 */ 610 for (s2 = scan; s2->bnext; s2 = s2->bnext) 611 ++BuildTotal; 612 for (s2 = scan; s2->bnext; s2 = s2->bnext) 613 ; 614 s2->bnext = *listp; 615 *listp = scan; 616 ++BuildTotal; 617 618 return scan; 619 } 620 621 /* 622 * Try to optimize the environment by supplying information that 623 * the ports system would generally have to run stuff to get on 624 * every package. 625 * 626 * See childOptimizeEnv() for the actual handling. We execute 627 * a single make -V... -V... for ports-mgmt/pkg from within the 628 * bulk system (which handles the environment and disables 629 * /etc/make.conf), and we then call addbuildenv() as appropriate. 630 * 631 * _PERL5_FROM_BIN 632 * add others... 633 */ 634 void 635 OptimizeEnv(void) 636 { 637 bulk_t *bulk; 638 639 initbulk(childOptimizeEnv, MaxBulk); 640 queuebulk("ports-mgmt", "pkg", NULL, NULL); 641 bulk = getbulk(); 642 freebulk(bulk); 643 donebulk(); 644 } 645 646 /* 647 * Run through the list resolving dependencies and constructing the topology 648 * linkages. This may append packages to the list. Dependencies to dummy 649 * nodes which do not specify a flavor do not need special handling, the 650 * search code in build.c will properly follow the first flavor. 651 */ 652 static pkg_t * 653 resolveDeps(pkg_t *list, pkg_t ***list_tailp, int gentopo) 654 { 655 pkg_t *ret_list = NULL; 656 pkg_t *scan; 657 pkg_t *use; 658 bulk_t *bulk; 659 660 for (scan = list; scan; scan = scan->bnext) { 661 use = pkg_find(scan->portdir); 662 resolveFlavors(use, scan->flavors, gentopo); 663 resolveDepString(use, scan->fetch_deps, 664 gentopo, DEP_TYPE_FETCH); 665 resolveDepString(use, scan->ext_deps, 666 gentopo, DEP_TYPE_EXT); 667 resolveDepString(use, scan->patch_deps, 668 gentopo, DEP_TYPE_PATCH); 669 resolveDepString(use, scan->build_deps, 670 gentopo, DEP_TYPE_BUILD); 671 resolveDepString(use, scan->lib_deps, 672 gentopo, DEP_TYPE_LIB); 673 resolveDepString(use, scan->run_deps, 674 gentopo, DEP_TYPE_RUN); 675 } 676 677 /* 678 * No bulk ops are queued when doing the final topology 679 * generation. 680 * 681 * Avoid entering duplicate results from the bulk ops. Duplicate 682 * results are mostly filtered out, but not always. A dummy node 683 * representing multiple flavors will parse-out the flavors 684 */ 685 if (gentopo) 686 return NULL; 687 while ((bulk = getbulk()) != NULL) { 688 if (bulk->list) { 689 if (ret_list == NULL) 690 ret_list = bulk->list; 691 **list_tailp = bulk->list; 692 bulk->list = NULL; 693 while ((scan = **list_tailp) != NULL) { 694 pkg_enter(scan); 695 *list_tailp = &scan->bnext; 696 } 697 } 698 freebulk(bulk); 699 } 700 return (ret_list); 701 } 702 703 /* 704 * Resolve a generic node that has flavors, queue to retrieve info for 705 * each flavor and setup linkages as appropriate. 706 */ 707 static void 708 resolveFlavors(pkg_t *pkg, char *flavors, int gentopo) 709 { 710 char *flavor_base; 711 char *flavor_scan; 712 char *flavor; 713 char *portdir; 714 char *s1; 715 char *s2; 716 pkg_t *dpkg; 717 pkglink_t *link; 718 719 if ((pkg->flags & PKGF_DUMMY) == 0) 720 return; 721 if (pkg->flavors == NULL || pkg->flavors[0] == 0) 722 return; 723 flavor_base = strdup(flavors); 724 flavor_scan = flavor_base; 725 726 for (;;) { 727 do { 728 flavor = strsep(&flavor_scan, " \t"); 729 } while (flavor && *flavor == 0); 730 if (flavor == NULL) 731 break; 732 733 /* 734 * Iterate each flavor generating "s1/s2@flavor". 735 * 736 * queuebulk() info for each flavor, and set-up the 737 * linkages in the topology generation pass. 738 */ 739 asprintf(&portdir, "%s@%s", pkg->portdir, flavor); 740 s1 = strdup(pkg->portdir); 741 s2 = strchr(s1, '/'); 742 *s2++ = 0; 743 744 dpkg = pkg_find(portdir); 745 if (dpkg && gentopo) { 746 /* 747 * Setup linkages 748 */ 749 free(portdir); 750 751 link = calloc(1, sizeof(*link)); 752 link->pkg = dpkg; 753 link->next = &pkg->idepon_list; 754 link->prev = pkg->idepon_list.prev; 755 link->next->prev = link; 756 link->prev->next = link; 757 link->dep_type = DEP_TYPE_BUILD; 758 759 link = calloc(1, sizeof(*link)); 760 link->pkg = pkg; 761 link->next = &dpkg->deponi_list; 762 link->prev = dpkg->deponi_list.prev; 763 link->next->prev = link; 764 link->prev->next = link; 765 link->dep_type = DEP_TYPE_BUILD; 766 ++dpkg->depi_count; 767 } else if (gentopo == 0 && dpkg == NULL) { 768 /* 769 * Use a place-holder to prevent duplicate 770 * dependencies from being processed. The placeholder 771 * will be replaced by the actual dependency. 772 */ 773 dpkg = allocpkg(); 774 dpkg->portdir = portdir; 775 dpkg->flags = PKGF_PLACEHOLD; 776 pkg_enter(dpkg); 777 queuebulk(s1, s2, flavor, NULL); 778 } 779 free(s1); 780 } 781 free(flavor_base); 782 } 783 784 static void 785 resolveDepString(pkg_t *pkg, char *depstr, int gentopo, int dep_type) 786 { 787 char *copy_base; 788 char *copy; 789 char *dep; 790 char *log_component; 791 char *sep; 792 char *tag; 793 char *flavor; 794 pkg_t *dpkg; 795 796 if (depstr == NULL || depstr[0] == 0) 797 return; 798 799 copy_base = strdup(depstr); 800 copy = copy_base; 801 log_component = copy; 802 803 for (;;) { 804 do { 805 dep = strsep(©, " \t"); 806 } while (dep && *dep == 0); 807 if (dep == NULL) 808 break; 809 810 /* 811 * Ignore dependencies prefixed with ${NONEXISTENT} 812 */ 813 if (strncmp(dep, "/nonexistent:", 13) == 0) 814 continue; 815 816 log_component = dep; 817 dep = strchr(dep, ':'); 818 if (dep == NULL || *dep != ':') { 819 printf("Error parsing %s dependency for " 820 "%s: '%s' at index %zd '%s' " 821 "(looking for ':')\n", 822 deptype2str(dep_type), 823 pkg->portdir, depstr, 824 log_component - copy_base, 825 log_component); 826 continue; 827 } 828 ++dep; 829 830 /* 831 * Strip-off any DPortsPath prefix. EXTRACT_DEPENDS 832 * often (always?) generates this prefix. 833 */ 834 if (strncmp(dep, DPortsPath, strlen(DPortsPath)) == 0) { 835 dep += strlen(DPortsPath); 836 if (*dep == '/') 837 ++dep; 838 } 839 840 /* 841 * Strip-off any tag (such as :patch). We don't try to 842 * organize dependencies at this fine a grain (for now). 843 */ 844 tag = strchr(dep, ':'); 845 if (tag) 846 *tag++ = 0; 847 log_component = dep; 848 849 /* 850 * Locate the dependency 851 */ 852 if ((dpkg = pkg_find(dep)) != NULL) { 853 if (gentopo) { 854 pkglink_t *link; 855 856 /* 857 * NOTE: idep_count is calculated recursively 858 * at build-time 859 */ 860 ddprintf(0, "Add Dependency %s -> %s\n", 861 pkg->portdir, dpkg->portdir); 862 link = calloc(1, sizeof(*link)); 863 link->pkg = dpkg; 864 link->next = &pkg->idepon_list; 865 link->prev = pkg->idepon_list.prev; 866 link->next->prev = link; 867 link->prev->next = link; 868 link->dep_type = dep_type; 869 870 link = calloc(1, sizeof(*link)); 871 link->pkg = pkg; 872 link->next = &dpkg->deponi_list; 873 link->prev = dpkg->deponi_list.prev; 874 link->next->prev = link; 875 link->prev->next = link; 876 link->dep_type = dep_type; 877 ++dpkg->depi_count; 878 } 879 continue; 880 } 881 882 /* 883 * This shouldn't happen because we already took a first 884 * pass and should have generated the pkgs. 885 */ 886 if (gentopo) { 887 printf("Topology Generate failed for %s: %s\n", 888 pkg->portdir, copy_base); 889 continue; 890 } 891 892 /* 893 * Separate out the two dports directory components and 894 * extract the optional '@flavor' specification. 895 */ 896 sep = strchr(dep, '/'); 897 if (sep == NULL) { 898 printf("Error parsing %s dependency for " 899 "%s: '%s' at index %zd '%s' " 900 "(looking for '/')\n", 901 deptype2str(dep_type), 902 pkg->portdir, depstr, 903 log_component - copy_base, 904 log_component); 905 continue; 906 } 907 *sep++ = 0; 908 909 /* 910 * The flavor hangs off the separator, not the tag 911 */ 912 flavor = strrchr(sep, '@'); 913 #if 0 914 if (tag) 915 flavor = strrchr(tag, '@'); 916 else 917 flavor = strrchr(sep, '@'); 918 #endif 919 if (flavor) 920 *flavor++ = 0; 921 922 if (flavor) 923 ddprintf(0, "QUEUE DEPENDENCY FROM PKG %s: %s/%s@%s\n", 924 pkg->portdir, dep, sep, flavor); 925 else 926 ddprintf(0, "QUEUE DEPENDENCY FROM PKG %s: %s/%s\n", 927 pkg->portdir, dep, sep); 928 929 /* 930 * Use a place-holder to prevent duplicate dependencies from 931 * being processed. The placeholder will be replaced by 932 * the actual dependency. 933 */ 934 dpkg = allocpkg(); 935 if (flavor) 936 asprintf(&dpkg->portdir, "%s/%s@%s", dep, sep, flavor); 937 else 938 asprintf(&dpkg->portdir, "%s/%s", dep, sep); 939 dpkg->flags = PKGF_PLACEHOLD; 940 pkg_enter(dpkg); 941 942 queuebulk(dep, sep, flavor, NULL); 943 } 944 free(copy_base); 945 } 946 947 void 948 FreePackageList(pkg_t *pkgs __unused) 949 { 950 dfatal("not implemented"); 951 } 952 953 /* 954 * Scan some or all dports to allocate the related pkg structure. Dependencies 955 * are stored but not processed. 956 * 957 * Threaded function 958 */ 959 static void 960 childGetPackageInfo(bulk_t *bulk) 961 { 962 pkg_t *pkg; 963 char *flavor; 964 char *ptr; 965 FILE *fp; 966 int line; 967 size_t len; 968 char *portpath; 969 char *flavarg; 970 char *localbase; 971 const char *cav[MAXCAC]; 972 pid_t pid; 973 int cac; 974 975 /* 976 * If the package has flavors we will loop on each one. If a flavor 977 * is not passed in s3 we will loop on all flavors, otherwise we will 978 * only process the passed-in flavor. 979 */ 980 flavor = bulk->s3; /* usually NULL */ 981 982 bulk->list = NULL; 983 984 asprintf(&portpath, "%s/%s/%s", DPortsPath, bulk->s1, bulk->s2); 985 if (flavor) 986 asprintf(&flavarg, "FLAVOR=%s", flavor); 987 else 988 flavarg = NULL; 989 990 cac = 0; 991 cav[cac++] = MAKE_BINARY; 992 cav[cac++] = "-C"; 993 cav[cac++] = portpath; 994 if (flavarg) 995 cav[cac++] = flavarg; 996 997 /* 998 * Prevent postgresql, mysql, and other package Makefile tests 999 * from accessing the host system's /usr/local by setting LOCALBASE 1000 * to an empty directory. 1001 */ 1002 asprintf(&localbase, "LOCALBASE=%s/empty", BuildBase); 1003 cav[cac++] = localbase; 1004 1005 /* 1006 * Variables we need to retrieve (order is specific to the switch 1007 * below) 1008 */ 1009 cav[cac++] = "-VPKGVERSION"; 1010 cav[cac++] = "-VPKGFILE:T"; 1011 cav[cac++] = "-VALLFILES"; 1012 cav[cac++] = "-VDIST_SUBDIR"; 1013 cav[cac++] = "-VMAKE_JOBS_NUMBER"; 1014 cav[cac++] = "-VIGNORE"; 1015 cav[cac++] = "-VFETCH_DEPENDS"; 1016 cav[cac++] = "-VEXTRACT_DEPENDS"; 1017 cav[cac++] = "-VPATCH_DEPENDS"; 1018 cav[cac++] = "-VBUILD_DEPENDS"; 1019 cav[cac++] = "-VLIB_DEPENDS"; 1020 cav[cac++] = "-VRUN_DEPENDS"; 1021 cav[cac++] = "-VSELECTED_OPTIONS"; 1022 cav[cac++] = "-VDESELECTED_OPTIONS"; 1023 cav[cac++] = "-VUSE_LINUX"; 1024 cav[cac++] = "-VFLAVORS"; 1025 cav[cac++] = "-VUSES"; 1026 1027 fp = dexec_open(portpath + strlen(DPortsPath) + 1, cav, cac, 1028 &pid, NULL, 1, 1); 1029 freestrp(&flavarg); 1030 freestrp(&localbase); 1031 1032 pkg = allocpkg(); 1033 if (flavor) 1034 asprintf(&pkg->portdir, "%s/%s@%s", bulk->s1, bulk->s2, flavor); 1035 else 1036 asprintf(&pkg->portdir, "%s/%s", bulk->s1, bulk->s2); 1037 1038 line = 1; 1039 while ((ptr = fgetln(fp, &len)) != NULL) { 1040 if (len == 0 || ptr[len-1] != '\n') { 1041 dfatal("Bad package info for %s/%s response line %d", 1042 bulk->s1, bulk->s2, line); 1043 } 1044 ptr[--len] = 0; 1045 1046 switch(line) { 1047 case 1: /* PKGVERSION */ 1048 asprintf(&pkg->version, "%s", ptr); 1049 break; 1050 case 2: /* PKGFILE */ 1051 asprintf(&pkg->pkgfile, "%s", ptr); 1052 break; 1053 case 3: /* ALLFILES (aka DISTFILES + patch files) */ 1054 asprintf(&pkg->distfiles, "%s", ptr); 1055 break; 1056 case 4: /* DIST_SUBDIR */ 1057 pkg->distsubdir = strdup_or_null(ptr); 1058 break; 1059 case 5: /* MAKE_JOBS_NUMBER */ 1060 pkg->make_jobs_number = strtol(ptr, NULL, 0); 1061 break; 1062 case 6: /* IGNORE */ 1063 pkg->ignore = strdup_or_null(ptr); 1064 break; 1065 case 7: /* FETCH_DEPENDS */ 1066 pkg->fetch_deps = strdup_or_null(ptr); 1067 break; 1068 case 8: /* EXTRACT_DEPENDS */ 1069 pkg->ext_deps = strdup_or_null(ptr); 1070 break; 1071 case 9: /* PATCH_DEPENDS */ 1072 pkg->patch_deps = strdup_or_null(ptr); 1073 break; 1074 case 10: /* BUILD_DEPENDS */ 1075 pkg->build_deps = strdup_or_null(ptr); 1076 break; 1077 case 11: /* LIB_DEPENDS */ 1078 pkg->lib_deps = strdup_or_null(ptr); 1079 break; 1080 case 12: /* RUN_DEPENDS */ 1081 pkg->run_deps = strdup_or_null(ptr); 1082 break; 1083 case 13: /* SELECTED_OPTIONS */ 1084 pkg->pos_options = strdup_or_null(ptr); 1085 break; 1086 case 14: /* DESELECTED_OPTIONS */ 1087 pkg->neg_options = strdup_or_null(ptr); 1088 break; 1089 case 15: /* USE_LINUX */ 1090 if (ptr[0]) 1091 pkg->use_linux = 1; 1092 break; 1093 case 16: /* FLAVORS */ 1094 asprintf(&pkg->flavors, "%s", ptr); 1095 break; 1096 case 17: /* USES */ 1097 asprintf(&pkg->uses, "%s", ptr); 1098 if (strstr(pkg->uses, "metaport")) 1099 pkg->flags |= PKGF_META; 1100 break; 1101 default: 1102 printf("EXTRA LINE: %s\n", ptr); 1103 break; 1104 } 1105 ++line; 1106 } 1107 if (line == 1) { 1108 printf("DPort not found: %s/%s\n", bulk->s1, bulk->s2); 1109 pkg->flags |= PKGF_NOTFOUND; 1110 } else if (line != 17 + 1) { 1111 printf("DPort corrupt: %s/%s\n", bulk->s1, bulk->s2); 1112 pkg->flags |= PKGF_CORRUPT; 1113 } 1114 if (dexec_close(fp, pid)) { 1115 printf("make -V* command for %s/%s failed\n", 1116 bulk->s1, bulk->s2); 1117 pkg->flags |= PKGF_CORRUPT; 1118 } 1119 ddassert(bulk->s1); 1120 1121 /* 1122 * DEBUGSTOP mode 1123 */ 1124 if (bulk->s4 && bulk->s4[0] == 'd') 1125 pkg->flags |= PKGF_DEBUGSTOP; 1126 1127 /* 1128 * Mark as a dummy node, the front-end will iterate the flavors 1129 * and create sub-nodes for us. 1130 * 1131 * Get rid of elements returned that are for the first flavor. 1132 * We are creating a dummy node here, not the node for the first 1133 * flavor. 1134 */ 1135 if (flavor == NULL && pkg->flavors && pkg->flavors[0]) { 1136 pkg->flags |= PKGF_DUMMY; 1137 freestrp(&pkg->fetch_deps); 1138 freestrp(&pkg->ext_deps); 1139 freestrp(&pkg->patch_deps); 1140 freestrp(&pkg->build_deps); 1141 freestrp(&pkg->lib_deps); 1142 freestrp(&pkg->run_deps); 1143 freestrp(&pkg->pkgfile); 1144 } 1145 1146 /* 1147 * Checksum the port directory tree. This just rollsup crcs of the 1148 * path names and a few stat fields (mtime, size) in order to detect 1149 * if any modification has been made to the port. 1150 */ 1151 pkg->crc32 = crcDirTree(portpath); 1152 1153 /* 1154 * Only one pkg is put on the return list now. This code no 1155 * longer creates pseudo-nodes for flavors (the frontend requests 1156 * each flavor instead). 1157 */ 1158 bulk->list = pkg; 1159 free(portpath); 1160 } 1161 1162 /* 1163 * Query the package (at least to make sure it hasn't been truncated) 1164 * and mark it as PACKAGED if found. 1165 * 1166 * This is a pre-req prior to doing builds, so we cannot assume that 1167 * the template has its pkg-static binary yet. 1168 * 1169 * Threaded function 1170 */ 1171 static void 1172 childGetBinaryDistInfo(bulk_t *bulk) 1173 { 1174 char *ptr; 1175 FILE *fp; 1176 size_t len; 1177 pkg_t *pkg; 1178 const char *cav[MAXCAC]; 1179 char *repopath; 1180 char buf[1024]; 1181 pid_t pid; 1182 int cac; 1183 int deleteme; 1184 1185 asprintf(&repopath, "%s/%s", RepositoryPath, bulk->s1); 1186 1187 cac = 0; 1188 cav[cac++] = PKG_BINARY; 1189 cav[cac++] = "query"; 1190 cav[cac++] = "-F"; 1191 cav[cac++] = repopath; 1192 cav[cac++] = "%n-%v"; 1193 1194 fp = dexec_open(NULL, cav, cac, &pid, NULL, 1, 0); 1195 deleteme = DeleteObsoletePkgs; 1196 1197 while ((ptr = fgetln(fp, &len)) != NULL) { 1198 if (len == 0 || ptr[len-1] != '\n') 1199 continue; 1200 ptr[len-1] = 0; 1201 1202 /* 1203 * As of pkg 1.17 the binary dist files use a .pkg suffix 1204 * regardless of the compression format, so always check 1205 * those. 1206 */ 1207 snprintf(buf, sizeof(buf), "%s%s", ptr, ".pkg"); 1208 pkg = pkg_find(buf); 1209 if (pkg == NULL) { 1210 snprintf(buf, sizeof(buf), "%s%s", ptr, UsePkgSufx); 1211 pkg = pkg_find(buf); 1212 } 1213 if (pkg) { 1214 if (FetchOnlyOpt) { 1215 ddprintf(0, "Note: Package is already packaged, ignore for -f: %s\n", ptr); 1216 deleteme = 0; 1217 } else { 1218 pkg->flags |= PKGF_PACKAGED; 1219 deleteme = 0; 1220 } 1221 } else { 1222 ddprintf(0, "Note: package scan, not in list, " 1223 "skipping %s{.%s,.pkg}\n", ptr, UsePkgSufx); 1224 } 1225 } 1226 if (dexec_close(fp, pid)) { 1227 printf("pkg query command failed for %s\n", repopath); 1228 } 1229 if (deleteme) { 1230 dlog(DLOG_ALL | DLOG_STDOUT, 1231 "Deleting obsolete package %s\n", repopath); 1232 remove(repopath); 1233 } 1234 free(repopath); 1235 } 1236 1237 static void 1238 childOptimizeEnv(bulk_t *bulk) 1239 { 1240 char *portpath; 1241 char *localbase; 1242 char *ptr; 1243 FILE *fp; 1244 int line; 1245 size_t len; 1246 const char *cav[MAXCAC]; 1247 pid_t pid; 1248 int cac; 1249 1250 asprintf(&portpath, "%s/%s/%s", DPortsPath, bulk->s1, bulk->s2); 1251 1252 /* 1253 * Prevent postgresql, mysql, and other package Makefile tests 1254 * from accessing the host system's /usr/local by setting LOCALBASE 1255 * to an empty directory. 1256 */ 1257 asprintf(&localbase, "LOCALBASE=%s/empty", BuildBase); 1258 1259 cac = 0; 1260 cav[cac++] = MAKE_BINARY; 1261 cav[cac++] = "-C"; 1262 cav[cac++] = portpath; 1263 cav[cac++] = localbase; 1264 cav[cac++] = "-V_PERL5_FROM_BIN"; 1265 1266 fp = dexec_open(portpath + strlen(DPortsPath) + 1, cav, cac, 1267 &pid, NULL, 1, 1); 1268 free(localbase); 1269 free(portpath); 1270 1271 line = 1; 1272 while ((ptr = fgetln(fp, &len)) != NULL) { 1273 if (len == 0 || ptr[len-1] != '\n') { 1274 dfatal("Bad package info for %s/%s response line %d", 1275 bulk->s1, bulk->s2, line); 1276 } 1277 ptr[--len] = 0; 1278 1279 switch(line) { 1280 case 1: /* _PERL5_FROM_BIN */ 1281 addbuildenv("_PERL5_FROM_BIN", ptr, BENV_ENVIRONMENT); 1282 break; 1283 default: 1284 printf("childOptimizeEnv: EXTRA LINE: %s\n", ptr); 1285 break; 1286 } 1287 ++line; 1288 } 1289 if (line == 1) { 1290 printf("DPort not found: %s/%s\n", bulk->s1, bulk->s2); 1291 } else if (line != 1 + 1) { 1292 printf("DPort corrupt: %s/%s\n", bulk->s1, bulk->s2); 1293 } 1294 if (dexec_close(fp, pid)) { 1295 printf("childOptimizeEnv() failed\n"); 1296 } 1297 } 1298 1299 static int 1300 scan_and_queue_dir(const char *path, const char *level1, int level) 1301 { 1302 DIR *dir; 1303 char *s1; 1304 char *s2; 1305 struct dirent *den; 1306 struct stat st; 1307 int count = 0; 1308 1309 dir = opendir(path); 1310 dassert(dir, "Cannot open dports path \"%s\"", path); 1311 1312 while ((den = readdir(dir)) != NULL) { 1313 if (den->d_namlen == 1 && den->d_name[0] == '.') 1314 continue; 1315 if (den->d_namlen == 2 && 1316 den->d_name[0] == '.' && den->d_name[1] == '.') 1317 continue; 1318 asprintf(&s1, "%s/%s", path, den->d_name); 1319 if (lstat(s1, &st) < 0 || !S_ISDIR(st.st_mode)) { 1320 free(s1); 1321 continue; 1322 } 1323 if (level == 1) { 1324 count += scan_and_queue_dir(s1, den->d_name, 2); 1325 free(s1); 1326 continue; 1327 } 1328 asprintf(&s2, "%s/Makefile", s1); 1329 if (lstat(s2, &st) == 0) { 1330 queuebulk(level1, den->d_name, NULL, NULL); 1331 ++count; 1332 } 1333 free(s1); 1334 free(s2); 1335 } 1336 closedir(dir); 1337 1338 return count; 1339 } 1340 1341 static int 1342 scan_binary_repo(const char *path) 1343 { 1344 DIR *dir; 1345 struct dirent *den; 1346 int count; 1347 1348 count = 0; 1349 dir = opendir(path); 1350 dassert(dir, "Cannot open repository path \"%s\"", path); 1351 1352 /* 1353 * NOTE: Test includes the '.' in the suffix. 1354 */ 1355 while ((den = readdir(dir)) != NULL) { 1356 const char *suffix; 1357 1358 suffix = strrchr(den->d_name, '.'); 1359 if (suffix && suffix != den->d_name && 1360 strcmp(suffix, UsePkgSufx) == 0) 1361 { 1362 queuebulk(den->d_name, NULL, NULL, NULL); 1363 ++count; 1364 } 1365 1366 /* 1367 * As of 1.17, pkg generates .pkg files. 1368 */ 1369 if (suffix && suffix != den->d_name && 1370 strcmp(suffix, ".pkg") == 0) 1371 { 1372 queuebulk(den->d_name, NULL, NULL, NULL); 1373 ++count; 1374 } 1375 } 1376 closedir(dir); 1377 1378 return count; 1379 } 1380 1381 #if 0 1382 static void 1383 pkgfree(pkg_t *pkg) 1384 { 1385 freestrp(&pkg->portdir); 1386 freestrp(&pkg->version); 1387 freestrp(&pkg->pkgfile); 1388 freestrp(&pkg->ignore); 1389 freestrp(&pkg->fetch_deps); 1390 freestrp(&pkg->ext_deps); 1391 freestrp(&pkg->patch_deps); 1392 freestrp(&pkg->build_deps); 1393 freestrp(&pkg->lib_deps); 1394 freestrp(&pkg->run_deps); 1395 freestrp(&pkg->pos_options); 1396 freestrp(&pkg->neg_options); 1397 freestrp(&pkg->flavors); 1398 free(pkg); 1399 } 1400 #endif 1401