1 /* 2 * Copyright (c) 2019 The DragonFly Project. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Matthew Dillon <dillon@backplane.com> 6 * 7 * This code uses concepts and configuration based on 'synth', by 8 * John R. Marino <draco@marino.st>, which was written in ada. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in 18 * the documentation and/or other materials provided with the 19 * distribution. 20 * 3. Neither the name of The DragonFly Project nor the names of its 21 * contributors may be used to endorse or promote products derived 22 * from this software without specific, prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 25 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 26 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 27 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 28 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 29 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 30 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 31 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 32 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 33 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 34 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 35 * SUCH DAMAGE. 36 */ 37 38 #include "dsynth.h" 39 40 #define PKG_HSIZE 32768 41 #define PKG_HMASK 32767 42 43 static int parsepkglist_file(const char *path, int debugstop); 44 static void childGetPackageInfo(bulk_t *bulk); 45 static void childGetBinaryDistInfo(bulk_t *bulk); 46 static void childOptimizeEnv(bulk_t *bulk); 47 static pkg_t *resolveDeps(pkg_t *dep_list, pkg_t ***list_tailp, int gentopo); 48 static void resolveFlavors(pkg_t *pkg, char *flavors, int gentopo); 49 static void resolveDepString(pkg_t *pkg, char *depstr, 50 int gentopo, int dep_type); 51 static pkg_t *processPackageListBulk(int total); 52 static int scan_and_queue_dir(const char *path, const char *level1, int level); 53 static int scan_binary_repo(const char *path); 54 #if 0 55 static void pkgfree(pkg_t *pkg); 56 #endif 57 58 static int PrepareSystemFlag; 59 60 static pkg_t *PkgHash1[PKG_HSIZE]; /* by portdir */ 61 static pkg_t *PkgHash2[PKG_HSIZE]; /* by pkgfile */ 62 63 /* 64 * Allocate a new pkg structure plus basic initialization. 65 */ 66 static __inline pkg_t * 67 allocpkg(void) 68 { 69 pkg_t *pkg; 70 71 pkg = calloc(1, sizeof(*pkg)); 72 pkg->idepon_list.next = &pkg->idepon_list; 73 pkg->idepon_list.prev = &pkg->idepon_list; 74 pkg->deponi_list.next = &pkg->deponi_list; 75 pkg->deponi_list.prev = &pkg->deponi_list; 76 77 return pkg; 78 } 79 80 /* 81 * Simple hash for lookups 82 */ 83 static __inline int 84 pkghash(const char *str) 85 { 86 int hv = 0xABC32923; 87 while (*str) { 88 hv = (hv << 5) ^ *str; 89 ++str; 90 } 91 hv = hv ^ (hv / PKG_HSIZE) ^ (hv / PKG_HSIZE / PKG_HSIZE); 92 return (hv & PKG_HMASK); 93 } 94 95 static __inline const char * 96 deptype2str(int dep_type) 97 { 98 switch (dep_type) { 99 case DEP_TYPE_FETCH: 100 return("FETCH_DEPENDS"); 101 case DEP_TYPE_EXT: 102 return("EXTRACT_DEPENDS"); 103 case DEP_TYPE_PATCH: 104 return("PATCH_DEPENDS"); 105 case DEP_TYPE_BUILD: 106 return("BUILD_DEPENDS"); 107 case DEP_TYPE_LIB: 108 return("LIB_DEPENDS"); 109 case DEP_TYPE_RUN: 110 return("RUN_DEPENDS"); 111 default: 112 return("UNKNOWN"); 113 } 114 } 115 116 static void 117 pkg_enter(pkg_t *pkg) 118 { 119 pkg_t **pkgp; 120 pkg_t *scan; 121 122 if (pkg->portdir) { 123 pkgp = &PkgHash1[pkghash(pkg->portdir)]; 124 while ((scan = *pkgp) != NULL) { 125 if (strcmp(pkg->portdir, scan->portdir) == 0) 126 break; 127 pkgp = &scan->hnext1; 128 } 129 ddassert(scan == NULL || (scan->flags & PKGF_PLACEHOLD)); 130 if (scan && (scan->flags & PKGF_PLACEHOLD)) { 131 ddassert(scan->idepon_list.next == &scan->idepon_list); 132 ddassert(scan->deponi_list.next == &scan->deponi_list); 133 *pkgp = pkg; 134 pkg->hnext1 = scan->hnext1; 135 free(scan->portdir); 136 free(scan); 137 scan = NULL; 138 } 139 if (scan == NULL) 140 *pkgp = pkg; 141 } 142 143 if (pkg->pkgfile) { 144 pkgp = &PkgHash2[pkghash(pkg->pkgfile)]; 145 while ((scan = *pkgp) != NULL) { 146 if (strcmp(pkg->pkgfile, scan->pkgfile) == 0) 147 break; 148 pkgp = &scan->hnext2; 149 } 150 if (scan == NULL) 151 *pkgp = pkg; 152 } 153 } 154 155 static pkg_t * 156 pkg_find(const char *match) 157 { 158 pkg_t **pkgp; 159 pkg_t *pkg; 160 161 pkgp = &PkgHash1[pkghash(match)]; 162 for (pkg = *pkgp; pkg; pkg = pkg->hnext1) { 163 if (strcmp(pkg->portdir, match) == 0) 164 return pkg; 165 } 166 pkgp = &PkgHash2[pkghash(match)]; 167 for (pkg = *pkgp; pkg; pkg = pkg->hnext2) { 168 if (strcmp(pkg->pkgfile, match) == 0) 169 return pkg; 170 } 171 return NULL; 172 } 173 174 /* 175 * Parse a specific list of ports via origin name (portdir/subdir) 176 */ 177 pkg_t * 178 ParsePackageList(int n, char **ary, int debugstop) 179 { 180 pkg_t *list; 181 int total; 182 int fail; 183 int i; 184 185 total = 0; 186 fail = 0; 187 initbulk(childGetPackageInfo, MaxBulk); 188 189 /* 190 * Always include ports-mgmt/pkg. s4 is "x" meaning not a manual 191 * selection, "d" meaning DEBUGSTOP mode, or NULL. 192 */ 193 queuebulk("ports-mgmt", "pkg", NULL, "x"); 194 195 for (i = 0; i < n; ++i) { 196 char *l1; 197 char *l2; 198 char *l3; 199 struct stat st; 200 201 l1 = strdup(ary[i]); 202 if (stat(l1, &st) == 0 && S_ISREG(st.st_mode)) { 203 total += parsepkglist_file(l1, debugstop); 204 continue; 205 } 206 207 l2 = strchr(l1, '/'); 208 if (l2 == NULL) { 209 printf("Bad portdir specification: %s\n", l1); 210 free(l1); 211 fail = 1; 212 continue; 213 } 214 *l2++ = 0; 215 l3 = strchr(l2, '@'); 216 if (l3) 217 *l3++ = 0; 218 219 /* 220 * Silently ignore any manually specified ports-mgmt/pkg, 221 * which we already auto-added. 222 */ 223 if (strcmp(l1, "ports-mgmt") != 0 || 224 strcmp(l2, "pkg") != 0) 225 { 226 queuebulk(l1, l2, l3, (debugstop ? "d" : NULL)); 227 } 228 ++total; 229 free(l1); 230 } 231 printf("Processing %d ports\n", total); 232 233 list = processPackageListBulk(total); 234 if (fail) { 235 dfatal("Bad specifications, exiting"); 236 exit(1); 237 } 238 239 return list; 240 } 241 242 static 243 int 244 parsepkglist_file(const char *path, int debugstop) 245 { 246 FILE *fp; 247 char *base; 248 char *l1; 249 char *l2; 250 char *l3; 251 size_t len; 252 int total; 253 254 if ((fp = fopen(path, "r")) == NULL) { 255 dpanic_errno("Cannot read %s\n", path); 256 /* NOT REACHED */ 257 return 0; 258 } 259 260 total = 0; 261 262 while ((base = fgetln(fp, &len)) != NULL) { 263 if (len == 0 || base[len-1] != '\n') 264 continue; 265 base[--len] = 0; 266 l1 = strtok(base, " \t\r\n"); 267 if (l1 == NULL) { 268 printf("Badly formatted pkg info line: %s\n", base); 269 continue; 270 } 271 l2 = strchr(l1, '/'); 272 if (l2 == NULL) { 273 printf("Badly formatted specification: %s\n", l1); 274 continue; 275 } 276 *l2++ = 0; 277 l3 = strchr(l2, '@'); 278 if (l3) 279 *l3++ = 0; 280 queuebulk(l1, l2, l3, (debugstop ? "d" : NULL)); 281 ++total; 282 } 283 fclose(fp); 284 285 return total; 286 } 287 288 /* 289 * Parse packages from the list installed on the system. 290 */ 291 pkg_t * 292 GetLocalPackageList(void) 293 { 294 pkg_t *list; 295 FILE *fp; 296 char *base; 297 char *data; 298 char *l1; 299 char *l2; 300 char *l3; 301 int total; 302 int state; 303 size_t len; 304 305 PrepareSystemFlag = 1; 306 initbulk(childGetPackageInfo, MaxBulk); 307 total = 0; 308 state = 0; 309 l1 = NULL; 310 l2 = NULL; 311 l3 = NULL; 312 313 fp = popen("pkg info -a -o -A", "r"); 314 315 /* 316 * Always include ports-mgmt/pkg. s4 is "x" meaning not a manual 317 * selection, "d" meaning DEBUGSTOP mode, or NULL. 318 */ 319 queuebulk("ports-mgmt", "pkg", NULL, "x"); 320 321 while ((base = fgetln(fp, &len)) != NULL) { 322 if (len == 0 || base[len-1] != '\n') 323 continue; 324 base[--len] = 0; 325 326 data = strchr(base, ':'); 327 if (data == NULL) 328 continue; 329 *data++ = 0; 330 331 base = strtok(base, " \t\r"); 332 data = strtok(data, " \t\r"); 333 334 if (base == NULL || data == NULL) 335 continue; 336 337 if (strcmp(base, "Origin") == 0) { 338 if (state == 1) { 339 queuebulk(l1, l2, NULL, NULL); 340 state = 0; 341 ++total; 342 } 343 344 if (strchr(data, '/') == NULL) { 345 printf("Badly formatted origin: %s\n", l1); 346 } 347 if (l1) 348 free(l1); 349 if (l3) 350 free(l3); 351 l1 = strdup(data); 352 l2 = strchr(l1, '/'); 353 *l2++ = 0; 354 l3 = strchr(l2, '@'); /* typically NULL */ 355 if (l3) { 356 *l3++ = 0; 357 l3 = strdup(l3); 358 } 359 360 /* 361 * Don't queue ports-mgmt/pkg twice, we already 362 * queued it manually. 363 */ 364 if (strcmp(l1, "ports-mgmt") != 0 || 365 strcmp(l2, "pkg") != 0) { 366 state = 1; 367 } 368 continue; 369 } 370 if (state == 1 && strcmp(base, "flavor") == 0) { 371 queuebulk(l1, l2, data, NULL); 372 state = 0; 373 ++total; 374 } 375 } 376 if (state == 1) { 377 queuebulk(l1, l2, NULL, NULL); 378 /*state = 0; not needed */ 379 } 380 if (l1) 381 free(l1); 382 if (l3) 383 free(l3); 384 385 pclose(fp); 386 387 printf("Processing %d ports\n", total); 388 389 list = processPackageListBulk(total); 390 391 return list; 392 } 393 394 pkg_t * 395 GetFullPackageList(void) 396 { 397 int total; 398 399 initbulk(childGetPackageInfo, MaxBulk); 400 total = scan_and_queue_dir(DPortsPath, NULL, 1); 401 printf("Scanning %d ports\n", total); 402 403 return processPackageListBulk(total); 404 } 405 406 /* 407 * Caller has queued the process list for bulk operation. We retrieve 408 * the results and clean up the bulk operation (we may have to do a second 409 * bulk operation so we have to be the ones to clean it up). 410 */ 411 static pkg_t * 412 processPackageListBulk(int total) 413 { 414 bulk_t *bulk; 415 pkg_t *scan; 416 pkg_t *list; 417 pkg_t *dep_list; 418 pkg_t **list_tail; 419 int count; 420 int stop_fail; 421 int stop_base_list; 422 int remove_corrupt; 423 424 list = NULL; 425 list_tail = &list; 426 count = 0; 427 remove_corrupt = 0; 428 429 while ((bulk = getbulk()) != NULL) { 430 ++count; 431 if ((count & 255) == 0) { 432 printf("%6.2f%%\r", 433 (double)count * 100.0 / (double)total + 0.001); 434 fflush(stdout); 435 } 436 if (bulk->list) { 437 *list_tail = bulk->list; 438 bulk->list = NULL; 439 while ((scan = *list_tail) != NULL) { 440 if (bulk->s4 == NULL || bulk->s4[0] != 'x') 441 scan->flags |= PKGF_MANUALSEL; 442 pkg_enter(scan); 443 list_tail = &scan->bnext; 444 } 445 } 446 freebulk(bulk); 447 } 448 printf("100.00%%\n"); 449 printf("\nTotal %d\n", count); 450 fflush(stdout); 451 452 /* 453 * Resolve all dependencies for the related packages, potentially 454 * adding anything that could not be found to the list. This will 455 * continue to issue bulk operations and process the result until 456 * no dependencies are left. 457 */ 458 printf("Resolving dependencies..."); 459 fflush(stdout); 460 dep_list = list; 461 while (dep_list) { 462 dep_list = resolveDeps(dep_list, &list_tail, 0); 463 } 464 printf("done\n"); 465 466 donebulk(); 467 468 /* 469 * Generate the topology 470 */ 471 resolveDeps(list, NULL, 1); 472 473 /* 474 * Do a final count, ignore place holders. 475 * 476 * Also set stop_fail if appropriate. Check for direct specifications 477 * which fail to probe and any direct dependencies of those 478 * specifications, but don't recurse (for now)... don't check indirect 479 * dependencies (i.e. A -> B -> C where A is directly specified, B 480 * is adirect dependency, and C fails to probe). 481 */ 482 count = 0; 483 stop_fail = 0; 484 stop_base_list = 0; 485 for (scan = list; scan; scan = scan->bnext) { 486 if ((scan->flags & PKGF_ERROR) == 0) { 487 ++count; 488 } 489 if ((scan->flags & PKGF_MANUALSEL) && MaskProbeAbort == 0) { 490 pkglink_t *link; 491 492 /* 493 * Directly specified package failed to probe 494 */ 495 if (scan->flags & PKGF_CORRUPT) { 496 ++stop_fail; 497 ++stop_base_list; 498 } 499 500 /* 501 * Directly specified package had a direct dependency 502 * that failed to probe (don't go further). 503 */ 504 PKGLIST_FOREACH(link, &scan->idepon_list) { 505 if (link->pkg && 506 (link->pkg->flags & PKGF_CORRUPT)) { 507 ++stop_fail; 508 } 509 } 510 } 511 } 512 printf("Total Returned %d\n", count); 513 514 /* 515 * Check to see if any PKGF_MANUALSEL packages 516 */ 517 if (stop_fail) { 518 printf("%d packages failed to probe\n", stop_fail); 519 if (PrepareSystemFlag) { 520 if (stop_fail == stop_base_list) { 521 printf( 522 "prepare-system: Some of your installed packages no longer exist in\n" 523 "dports, do you wish to continue rebuilding what does exist?\n"); 524 if (askyn("Continue anyway? ")) 525 remove_corrupt = 1; 526 } else { 527 printf( 528 "prepare-system: Some of your installed packages have dependencies\n" 529 "which could not be found in dports, cannot continue, aborting\n"); 530 } 531 } else { 532 if (ForceOpt) { 533 remove_corrupt = 1; 534 printf("continuing despite pkglist " 535 "errors (-f)\n"); 536 } else { 537 printf("unable to continue, aborting\n"); 538 } 539 } 540 if (remove_corrupt == 0) 541 exit(1); 542 } 543 544 /* 545 * Remove corrupt packages before continuing 546 */ 547 if (remove_corrupt) { 548 list_tail = &list; 549 while ((scan = *list_tail) != NULL) { 550 if (scan->flags & PKGF_CORRUPT) 551 *list_tail = scan->bnext; 552 else 553 list_tail = &scan->bnext; 554 } 555 } 556 557 /* 558 * Scan our binary distributions and related dependencies looking 559 * for any packages that have already been built. 560 */ 561 initbulk(childGetBinaryDistInfo, MaxBulk); 562 total = scan_binary_repo(RepositoryPath); 563 count = 0; 564 printf("Scanning %d packages\n", total); 565 566 while ((bulk = getbulk()) != NULL) { 567 ++count; 568 if ((count & 255) == 0) { 569 printf("%6.2f%%\r", 570 (double)count * 100.0 / (double)total + 0.001); 571 fflush(stdout); 572 } 573 freebulk(bulk); 574 } 575 printf("100.00%%\n"); 576 printf("\nTotal %d\n", count); 577 fflush(stdout); 578 donebulk(); 579 580 printf("all done\n"); 581 582 return list; 583 } 584 585 pkg_t * 586 GetPkgPkg(pkg_t **listp) 587 { 588 bulk_t *bulk; 589 pkg_t *scan; 590 pkg_t *s2; 591 592 for (scan = *listp; scan; scan = scan->bnext) { 593 if (strcmp(scan->portdir, "ports-mgmt/pkg") == 0) 594 return scan; 595 } 596 597 /* 598 * This will force pkg to be built, but generally this code 599 * is not reached because the package list processing code 600 * adds ports-mgmt/pkg unconditionally. 601 */ 602 initbulk(childGetPackageInfo, MaxBulk); 603 queuebulk("ports-mgmt", "pkg", NULL, "x"); 604 bulk = getbulk(); 605 dassert(bulk, "Cannot find ports-mgmt/pkg"); 606 scan = bulk->list; 607 bulk->list = NULL; 608 freebulk(bulk); 609 donebulk(); 610 611 612 /* 613 * Include added packages to the total and add the initial bulk 614 * built packages to the list so they get counted. 615 */ 616 for (s2 = scan; s2->bnext; s2 = s2->bnext) 617 ++BuildTotal; 618 for (s2 = scan; s2->bnext; s2 = s2->bnext) 619 ; 620 s2->bnext = *listp; 621 *listp = scan; 622 ++BuildTotal; 623 624 return scan; 625 } 626 627 /* 628 * Try to optimize the environment by supplying information that 629 * the ports system would generally have to run stuff to get on 630 * every package. 631 * 632 * See childOptimizeEnv() for the actual handling. We execute 633 * a single make -V... -V... for ports-mgmt/pkg from within the 634 * bulk system (which handles the environment and disables 635 * /etc/make.conf), and we then call addbuildenv() as appropriate. 636 * 637 * _PERL5_FROM_BIN 638 * add others... 639 */ 640 void 641 OptimizeEnv(void) 642 { 643 bulk_t *bulk; 644 645 initbulk(childOptimizeEnv, MaxBulk); 646 queuebulk("ports-mgmt", "pkg", NULL, NULL); 647 bulk = getbulk(); 648 freebulk(bulk); 649 donebulk(); 650 } 651 652 /* 653 * Run through the list resolving dependencies and constructing the topology 654 * linkages. This may append packages to the list. Dependencies to dummy 655 * nodes which do not specify a flavor do not need special handling, the 656 * search code in build.c will properly follow the first flavor. 657 */ 658 static pkg_t * 659 resolveDeps(pkg_t *list, pkg_t ***list_tailp, int gentopo) 660 { 661 pkg_t *ret_list = NULL; 662 pkg_t *scan; 663 pkg_t *use; 664 bulk_t *bulk; 665 666 for (scan = list; scan; scan = scan->bnext) { 667 use = pkg_find(scan->portdir); 668 resolveFlavors(use, scan->flavors, gentopo); 669 resolveDepString(use, scan->fetch_deps, 670 gentopo, DEP_TYPE_FETCH); 671 resolveDepString(use, scan->ext_deps, 672 gentopo, DEP_TYPE_EXT); 673 resolveDepString(use, scan->patch_deps, 674 gentopo, DEP_TYPE_PATCH); 675 resolveDepString(use, scan->build_deps, 676 gentopo, DEP_TYPE_BUILD); 677 resolveDepString(use, scan->lib_deps, 678 gentopo, DEP_TYPE_LIB); 679 resolveDepString(use, scan->run_deps, 680 gentopo, DEP_TYPE_RUN); 681 } 682 683 /* 684 * No bulk ops are queued when doing the final topology 685 * generation. 686 * 687 * Avoid entering duplicate results from the bulk ops. Duplicate 688 * results are mostly filtered out, but not always. A dummy node 689 * representing multiple flavors will parse-out the flavors 690 */ 691 if (gentopo) 692 return NULL; 693 while ((bulk = getbulk()) != NULL) { 694 if (bulk->list) { 695 if (ret_list == NULL) 696 ret_list = bulk->list; 697 **list_tailp = bulk->list; 698 bulk->list = NULL; 699 while ((scan = **list_tailp) != NULL) { 700 pkg_enter(scan); 701 *list_tailp = &scan->bnext; 702 } 703 } 704 freebulk(bulk); 705 } 706 return (ret_list); 707 } 708 709 /* 710 * Resolve a generic node that has flavors, queue to retrieve info for 711 * each flavor and setup linkages as appropriate. 712 */ 713 static void 714 resolveFlavors(pkg_t *pkg, char *flavors, int gentopo) 715 { 716 char *flavor_base; 717 char *flavor_scan; 718 char *flavor; 719 char *portdir; 720 char *s1; 721 char *s2; 722 pkg_t *dpkg; 723 pkglink_t *link; 724 725 if ((pkg->flags & PKGF_DUMMY) == 0) 726 return; 727 if (pkg->flavors == NULL || pkg->flavors[0] == 0) 728 return; 729 flavor_base = strdup(flavors); 730 flavor_scan = flavor_base; 731 732 for (;;) { 733 do { 734 flavor = strsep(&flavor_scan, " \t"); 735 } while (flavor && *flavor == 0); 736 if (flavor == NULL) 737 break; 738 739 /* 740 * Iterate each flavor generating "s1/s2@flavor". 741 * 742 * queuebulk() info for each flavor, and set-up the 743 * linkages in the topology generation pass. 744 */ 745 asprintf(&portdir, "%s@%s", pkg->portdir, flavor); 746 s1 = strdup(pkg->portdir); 747 s2 = strchr(s1, '/'); 748 *s2++ = 0; 749 750 dpkg = pkg_find(portdir); 751 if (dpkg && gentopo) { 752 /* 753 * Setup linkages 754 */ 755 free(portdir); 756 757 link = calloc(1, sizeof(*link)); 758 link->pkg = dpkg; 759 link->next = &pkg->idepon_list; 760 link->prev = pkg->idepon_list.prev; 761 link->next->prev = link; 762 link->prev->next = link; 763 link->dep_type = DEP_TYPE_BUILD; 764 765 link = calloc(1, sizeof(*link)); 766 link->pkg = pkg; 767 link->next = &dpkg->deponi_list; 768 link->prev = dpkg->deponi_list.prev; 769 link->next->prev = link; 770 link->prev->next = link; 771 link->dep_type = DEP_TYPE_BUILD; 772 ++dpkg->depi_count; 773 } else if (gentopo == 0 && dpkg == NULL) { 774 /* 775 * Use a place-holder to prevent duplicate 776 * dependencies from being processed. The placeholder 777 * will be replaced by the actual dependency. 778 */ 779 dpkg = allocpkg(); 780 dpkg->portdir = portdir; 781 dpkg->flags = PKGF_PLACEHOLD; 782 pkg_enter(dpkg); 783 queuebulk(s1, s2, flavor, NULL); 784 } 785 free(s1); 786 } 787 free(flavor_base); 788 } 789 790 static void 791 resolveDepString(pkg_t *pkg, char *depstr, int gentopo, int dep_type) 792 { 793 char *copy_base; 794 char *copy; 795 char *dep; 796 char *log_component; 797 char *sep; 798 char *tag; 799 char *flavor; 800 pkg_t *dpkg; 801 802 if (depstr == NULL || depstr[0] == 0) 803 return; 804 805 copy_base = strdup(depstr); 806 copy = copy_base; 807 log_component = copy; 808 809 for (;;) { 810 do { 811 dep = strsep(©, " \t"); 812 } while (dep && *dep == 0); 813 if (dep == NULL) 814 break; 815 816 /* 817 * Ignore dependencies prefixed with ${NONEXISTENT} 818 */ 819 if (strncmp(dep, "/nonexistent:", 13) == 0) 820 continue; 821 822 log_component = dep; 823 dep = strchr(dep, ':'); 824 if (dep == NULL || *dep != ':') { 825 printf("Error parsing %s dependency for " 826 "%s: '%s' at index %zd '%s' " 827 "(looking for ':')\n", 828 deptype2str(dep_type), 829 pkg->portdir, depstr, 830 log_component - copy_base, 831 log_component); 832 continue; 833 } 834 ++dep; 835 836 /* 837 * Strip-off any DPortsPath prefix. EXTRACT_DEPENDS 838 * often (always?) generates this prefix. 839 */ 840 if (strncmp(dep, DPortsPath, strlen(DPortsPath)) == 0) { 841 dep += strlen(DPortsPath); 842 if (*dep == '/') 843 ++dep; 844 } 845 846 /* 847 * Strip-off any tag (such as :patch). We don't try to 848 * organize dependencies at this fine a grain (for now). 849 */ 850 tag = strchr(dep, ':'); 851 if (tag) 852 *tag++ = 0; 853 log_component = dep; 854 855 /* 856 * Locate the dependency 857 */ 858 if ((dpkg = pkg_find(dep)) != NULL) { 859 if (gentopo) { 860 pkglink_t *link; 861 862 /* 863 * NOTE: idep_count is calculated recursively 864 * at build-time 865 */ 866 ddprintf(0, "Add Dependency %s -> %s\n", 867 pkg->portdir, dpkg->portdir); 868 link = calloc(1, sizeof(*link)); 869 link->pkg = dpkg; 870 link->next = &pkg->idepon_list; 871 link->prev = pkg->idepon_list.prev; 872 link->next->prev = link; 873 link->prev->next = link; 874 link->dep_type = dep_type; 875 876 link = calloc(1, sizeof(*link)); 877 link->pkg = pkg; 878 link->next = &dpkg->deponi_list; 879 link->prev = dpkg->deponi_list.prev; 880 link->next->prev = link; 881 link->prev->next = link; 882 link->dep_type = dep_type; 883 ++dpkg->depi_count; 884 } 885 continue; 886 } 887 888 /* 889 * This shouldn't happen because we already took a first 890 * pass and should have generated the pkgs. 891 */ 892 if (gentopo) { 893 printf("Topology Generate failed for %s: %s\n", 894 pkg->portdir, copy_base); 895 continue; 896 } 897 898 /* 899 * Separate out the two dports directory components and 900 * extract the optional '@flavor' specification. 901 */ 902 sep = strchr(dep, '/'); 903 if (sep == NULL) { 904 printf("Error parsing %s dependency for " 905 "%s: '%s' at index %zd '%s' " 906 "(looking for '/')\n", 907 deptype2str(dep_type), 908 pkg->portdir, depstr, 909 log_component - copy_base, 910 log_component); 911 continue; 912 } 913 *sep++ = 0; 914 915 /* 916 * The flavor hangs off the separator, not the tag 917 */ 918 flavor = strrchr(sep, '@'); 919 #if 0 920 if (tag) 921 flavor = strrchr(tag, '@'); 922 else 923 flavor = strrchr(sep, '@'); 924 #endif 925 if (flavor) 926 *flavor++ = 0; 927 928 if (flavor) 929 ddprintf(0, "QUEUE DEPENDENCY FROM PKG %s: %s/%s@%s\n", 930 pkg->portdir, dep, sep, flavor); 931 else 932 ddprintf(0, "QUEUE DEPENDENCY FROM PKG %s: %s/%s\n", 933 pkg->portdir, dep, sep); 934 935 /* 936 * Use a place-holder to prevent duplicate dependencies from 937 * being processed. The placeholder will be replaced by 938 * the actual dependency. 939 */ 940 dpkg = allocpkg(); 941 if (flavor) 942 asprintf(&dpkg->portdir, "%s/%s@%s", dep, sep, flavor); 943 else 944 asprintf(&dpkg->portdir, "%s/%s", dep, sep); 945 dpkg->flags = PKGF_PLACEHOLD; 946 pkg_enter(dpkg); 947 948 queuebulk(dep, sep, flavor, NULL); 949 } 950 free(copy_base); 951 } 952 953 void 954 FreePackageList(pkg_t *pkgs __unused) 955 { 956 dfatal("not implemented"); 957 } 958 959 /* 960 * Scan some or all dports to allocate the related pkg structure. Dependencies 961 * are stored but not processed. 962 * 963 * Threaded function 964 */ 965 static void 966 childGetPackageInfo(bulk_t *bulk) 967 { 968 pkg_t *pkg; 969 char *flavor; 970 char *ptr; 971 FILE *fp; 972 int line; 973 size_t len; 974 char *portpath; 975 char *flavarg; 976 char *localbase; 977 const char *cav[MAXCAC]; 978 pid_t pid; 979 int cac; 980 981 /* 982 * If the package has flavors we will loop on each one. If a flavor 983 * is not passed in s3 we will loop on all flavors, otherwise we will 984 * only process the passed-in flavor. 985 */ 986 flavor = bulk->s3; /* usually NULL */ 987 988 bulk->list = NULL; 989 990 asprintf(&portpath, "%s/%s/%s", DPortsPath, bulk->s1, bulk->s2); 991 if (flavor) 992 asprintf(&flavarg, "FLAVOR=%s", flavor); 993 else 994 flavarg = NULL; 995 996 cac = 0; 997 cav[cac++] = MAKE_BINARY; 998 cav[cac++] = "-C"; 999 cav[cac++] = portpath; 1000 if (flavarg) 1001 cav[cac++] = flavarg; 1002 1003 /* 1004 * Prevent postgresql, mysql, and other package Makefile tests 1005 * from accessing the host system's /usr/local by setting LOCALBASE 1006 * to an empty directory. 1007 */ 1008 asprintf(&localbase, "LOCALBASE=%s/empty", BuildBase); 1009 cav[cac++] = localbase; 1010 1011 /* 1012 * Variables we need to retrieve (order is specific to the switch 1013 * below) 1014 */ 1015 cav[cac++] = "-VPKGVERSION"; 1016 cav[cac++] = "-VPKGFILE:T"; 1017 cav[cac++] = "-VALLFILES"; 1018 cav[cac++] = "-VDIST_SUBDIR"; 1019 cav[cac++] = "-VMAKE_JOBS_NUMBER"; 1020 cav[cac++] = "-VIGNORE"; 1021 cav[cac++] = "-VFETCH_DEPENDS"; 1022 cav[cac++] = "-VEXTRACT_DEPENDS"; 1023 cav[cac++] = "-VPATCH_DEPENDS"; 1024 cav[cac++] = "-VBUILD_DEPENDS"; 1025 cav[cac++] = "-VLIB_DEPENDS"; 1026 cav[cac++] = "-VRUN_DEPENDS"; 1027 cav[cac++] = "-VSELECTED_OPTIONS"; 1028 cav[cac++] = "-VDESELECTED_OPTIONS"; 1029 cav[cac++] = "-VUSE_LINUX"; 1030 cav[cac++] = "-VFLAVORS"; 1031 cav[cac++] = "-VUSES"; 1032 1033 fp = dexec_open(portpath + strlen(DPortsPath) + 1, cav, cac, 1034 &pid, NULL, 1, 1); 1035 freestrp(&flavarg); 1036 freestrp(&localbase); 1037 1038 pkg = allocpkg(); 1039 if (flavor) 1040 asprintf(&pkg->portdir, "%s/%s@%s", bulk->s1, bulk->s2, flavor); 1041 else 1042 asprintf(&pkg->portdir, "%s/%s", bulk->s1, bulk->s2); 1043 1044 line = 1; 1045 while ((ptr = fgetln(fp, &len)) != NULL) { 1046 if (len == 0 || ptr[len-1] != '\n') { 1047 dfatal("Bad package info for %s/%s response line %d", 1048 bulk->s1, bulk->s2, line); 1049 } 1050 ptr[--len] = 0; 1051 1052 switch(line) { 1053 case 1: /* PKGVERSION */ 1054 asprintf(&pkg->version, "%s", ptr); 1055 break; 1056 case 2: /* PKGFILE */ 1057 asprintf(&pkg->pkgfile, "%s", ptr); 1058 break; 1059 case 3: /* ALLFILES (aka DISTFILES + patch files) */ 1060 asprintf(&pkg->distfiles, "%s", ptr); 1061 break; 1062 case 4: /* DIST_SUBDIR */ 1063 pkg->distsubdir = strdup_or_null(ptr); 1064 break; 1065 case 5: /* MAKE_JOBS_NUMBER */ 1066 pkg->make_jobs_number = strtol(ptr, NULL, 0); 1067 break; 1068 case 6: /* IGNORE */ 1069 pkg->ignore = strdup_or_null(ptr); 1070 break; 1071 case 7: /* FETCH_DEPENDS */ 1072 pkg->fetch_deps = strdup_or_null(ptr); 1073 break; 1074 case 8: /* EXTRACT_DEPENDS */ 1075 pkg->ext_deps = strdup_or_null(ptr); 1076 break; 1077 case 9: /* PATCH_DEPENDS */ 1078 pkg->patch_deps = strdup_or_null(ptr); 1079 break; 1080 case 10: /* BUILD_DEPENDS */ 1081 pkg->build_deps = strdup_or_null(ptr); 1082 break; 1083 case 11: /* LIB_DEPENDS */ 1084 pkg->lib_deps = strdup_or_null(ptr); 1085 break; 1086 case 12: /* RUN_DEPENDS */ 1087 pkg->run_deps = strdup_or_null(ptr); 1088 break; 1089 case 13: /* SELECTED_OPTIONS */ 1090 pkg->pos_options = strdup_or_null(ptr); 1091 break; 1092 case 14: /* DESELECTED_OPTIONS */ 1093 pkg->neg_options = strdup_or_null(ptr); 1094 break; 1095 case 15: /* USE_LINUX */ 1096 if (ptr[0]) 1097 pkg->use_linux = 1; 1098 break; 1099 case 16: /* FLAVORS */ 1100 asprintf(&pkg->flavors, "%s", ptr); 1101 break; 1102 case 17: /* USES */ 1103 asprintf(&pkg->uses, "%s", ptr); 1104 if (strstr(pkg->uses, "metaport")) 1105 pkg->flags |= PKGF_META; 1106 break; 1107 default: 1108 printf("EXTRA LINE: %s\n", ptr); 1109 break; 1110 } 1111 ++line; 1112 } 1113 if (line == 1) { 1114 printf("DPort not found: %s/%s\n", bulk->s1, bulk->s2); 1115 pkg->flags |= PKGF_NOTFOUND; 1116 } else if (line != 17 + 1) { 1117 printf("DPort corrupt: %s/%s\n", bulk->s1, bulk->s2); 1118 pkg->flags |= PKGF_CORRUPT; 1119 } 1120 if (dexec_close(fp, pid)) { 1121 printf("make -V* command for %s/%s failed\n", 1122 bulk->s1, bulk->s2); 1123 pkg->flags |= PKGF_CORRUPT; 1124 } 1125 ddassert(bulk->s1); 1126 1127 /* 1128 * DEBUGSTOP mode 1129 */ 1130 if (bulk->s4 && bulk->s4[0] == 'd') 1131 pkg->flags |= PKGF_DEBUGSTOP; 1132 1133 /* 1134 * Mark as a dummy node, the front-end will iterate the flavors 1135 * and create sub-nodes for us. 1136 * 1137 * Get rid of elements returned that are for the first flavor. 1138 * We are creating a dummy node here, not the node for the first 1139 * flavor. 1140 */ 1141 if (flavor == NULL && pkg->flavors && pkg->flavors[0]) { 1142 pkg->flags |= PKGF_DUMMY; 1143 freestrp(&pkg->fetch_deps); 1144 freestrp(&pkg->ext_deps); 1145 freestrp(&pkg->patch_deps); 1146 freestrp(&pkg->build_deps); 1147 freestrp(&pkg->lib_deps); 1148 freestrp(&pkg->run_deps); 1149 freestrp(&pkg->pkgfile); 1150 } 1151 1152 /* 1153 * Checksum the port directory tree. This just rollsup crcs of the 1154 * path names and a few stat fields (mtime, size) in order to detect 1155 * if any modification has been made to the port. 1156 */ 1157 pkg->crc32 = crcDirTree(portpath); 1158 1159 /* 1160 * Only one pkg is put on the return list now. This code no 1161 * longer creates pseudo-nodes for flavors (the frontend requests 1162 * each flavor instead). 1163 */ 1164 bulk->list = pkg; 1165 free(portpath); 1166 } 1167 1168 /* 1169 * Query the package (at least to make sure it hasn't been truncated) 1170 * and mark it as PACKAGED if found. 1171 * 1172 * This is a pre-req prior to doing builds, so we cannot assume that 1173 * the template has its pkg-static binary yet. 1174 * 1175 * Threaded function 1176 */ 1177 static void 1178 childGetBinaryDistInfo(bulk_t *bulk) 1179 { 1180 char *ptr; 1181 FILE *fp; 1182 size_t len; 1183 pkg_t *pkg; 1184 const char *cav[MAXCAC]; 1185 char *repopath; 1186 char buf[1024]; 1187 pid_t pid; 1188 int cac; 1189 int deleteme; 1190 1191 asprintf(&repopath, "%s/%s", RepositoryPath, bulk->s1); 1192 1193 cac = 0; 1194 cav[cac++] = PKG_BINARY; 1195 cav[cac++] = "query"; 1196 cav[cac++] = "-F"; 1197 cav[cac++] = repopath; 1198 cav[cac++] = "%n-%v"; 1199 1200 fp = dexec_open(NULL, cav, cac, &pid, NULL, 1, 0); 1201 deleteme = DeleteObsoletePkgs; 1202 1203 while ((ptr = fgetln(fp, &len)) != NULL) { 1204 if (len == 0 || ptr[len-1] != '\n') 1205 continue; 1206 ptr[len-1] = 0; 1207 1208 /* 1209 * As of pkg 1.17 the binary dist files use a .pkg suffix 1210 * regardless of the compression format, so always check 1211 * those. 1212 */ 1213 snprintf(buf, sizeof(buf), "%s%s", ptr, ".pkg"); 1214 pkg = pkg_find(buf); 1215 if (pkg == NULL) { 1216 snprintf(buf, sizeof(buf), "%s%s", ptr, UsePkgSufx); 1217 pkg = pkg_find(buf); 1218 } 1219 if (pkg) { 1220 if (FetchOnlyOpt) { 1221 ddprintf(0, "Note: Package is already packaged, ignore for -f: %s\n", ptr); 1222 deleteme = 0; 1223 } else { 1224 pkg->flags |= PKGF_PACKAGED; 1225 deleteme = 0; 1226 } 1227 } else { 1228 ddprintf(0, "Note: package scan, not in list, " 1229 "skipping %s{.%s,.pkg}\n", ptr, UsePkgSufx); 1230 } 1231 } 1232 if (dexec_close(fp, pid)) { 1233 printf("pkg query command failed for %s\n", repopath); 1234 } 1235 if (deleteme) { 1236 dlog(DLOG_ALL | DLOG_STDOUT, 1237 "Deleting obsolete package %s\n", repopath); 1238 remove(repopath); 1239 } 1240 free(repopath); 1241 } 1242 1243 static void 1244 childOptimizeEnv(bulk_t *bulk) 1245 { 1246 char *portpath; 1247 char *localbase; 1248 char *ptr; 1249 FILE *fp; 1250 int line; 1251 size_t len; 1252 const char *cav[MAXCAC]; 1253 pid_t pid; 1254 int cac; 1255 1256 asprintf(&portpath, "%s/%s/%s", DPortsPath, bulk->s1, bulk->s2); 1257 1258 /* 1259 * Prevent postgresql, mysql, and other package Makefile tests 1260 * from accessing the host system's /usr/local by setting LOCALBASE 1261 * to an empty directory. 1262 */ 1263 asprintf(&localbase, "LOCALBASE=%s/empty", BuildBase); 1264 1265 cac = 0; 1266 cav[cac++] = MAKE_BINARY; 1267 cav[cac++] = "-C"; 1268 cav[cac++] = portpath; 1269 cav[cac++] = localbase; 1270 cav[cac++] = "-V_PERL5_FROM_BIN"; 1271 1272 fp = dexec_open(portpath + strlen(DPortsPath) + 1, cav, cac, 1273 &pid, NULL, 1, 1); 1274 free(localbase); 1275 free(portpath); 1276 1277 line = 1; 1278 while ((ptr = fgetln(fp, &len)) != NULL) { 1279 if (len == 0 || ptr[len-1] != '\n') { 1280 dfatal("Bad package info for %s/%s response line %d", 1281 bulk->s1, bulk->s2, line); 1282 } 1283 ptr[--len] = 0; 1284 1285 switch(line) { 1286 case 1: /* _PERL5_FROM_BIN */ 1287 addbuildenv("_PERL5_FROM_BIN", ptr, BENV_ENVIRONMENT); 1288 break; 1289 default: 1290 printf("childOptimizeEnv: EXTRA LINE: %s\n", ptr); 1291 break; 1292 } 1293 ++line; 1294 } 1295 if (line == 1) { 1296 printf("DPort not found: %s/%s\n", bulk->s1, bulk->s2); 1297 } else if (line != 1 + 1) { 1298 printf("DPort corrupt: %s/%s\n", bulk->s1, bulk->s2); 1299 } 1300 if (dexec_close(fp, pid)) { 1301 printf("childOptimizeEnv() failed\n"); 1302 } 1303 } 1304 1305 static int 1306 scan_and_queue_dir(const char *path, const char *level1, int level) 1307 { 1308 DIR *dir; 1309 char *s1; 1310 char *s2; 1311 struct dirent *den; 1312 struct stat st; 1313 int count = 0; 1314 1315 dir = opendir(path); 1316 dassert(dir, "Cannot open dports path \"%s\"", path); 1317 1318 while ((den = readdir(dir)) != NULL) { 1319 if (den->d_namlen == 1 && den->d_name[0] == '.') 1320 continue; 1321 if (den->d_namlen == 2 && 1322 den->d_name[0] == '.' && den->d_name[1] == '.') 1323 continue; 1324 asprintf(&s1, "%s/%s", path, den->d_name); 1325 if (lstat(s1, &st) < 0 || !S_ISDIR(st.st_mode)) { 1326 free(s1); 1327 continue; 1328 } 1329 if (level == 1) { 1330 count += scan_and_queue_dir(s1, den->d_name, 2); 1331 free(s1); 1332 continue; 1333 } 1334 asprintf(&s2, "%s/Makefile", s1); 1335 if (lstat(s2, &st) == 0) { 1336 queuebulk(level1, den->d_name, NULL, NULL); 1337 ++count; 1338 } 1339 free(s1); 1340 free(s2); 1341 } 1342 closedir(dir); 1343 1344 return count; 1345 } 1346 1347 static int 1348 scan_binary_repo(const char *path) 1349 { 1350 DIR *dir; 1351 struct dirent *den; 1352 int count; 1353 1354 count = 0; 1355 dir = opendir(path); 1356 dassert(dir, "Cannot open repository path \"%s\"", path); 1357 1358 /* 1359 * NOTE: Test includes the '.' in the suffix. 1360 */ 1361 while ((den = readdir(dir)) != NULL) { 1362 const char *suffix; 1363 1364 suffix = strrchr(den->d_name, '.'); 1365 if (suffix && suffix != den->d_name && 1366 strcmp(suffix, UsePkgSufx) == 0) 1367 { 1368 queuebulk(den->d_name, NULL, NULL, NULL); 1369 ++count; 1370 } 1371 1372 /* 1373 * As of 1.17, pkg generates .pkg files. 1374 */ 1375 if (suffix && suffix != den->d_name && 1376 strcmp(suffix, ".pkg") == 0) 1377 { 1378 queuebulk(den->d_name, NULL, NULL, NULL); 1379 ++count; 1380 } 1381 } 1382 closedir(dir); 1383 1384 return count; 1385 } 1386 1387 #if 0 1388 static void 1389 pkgfree(pkg_t *pkg) 1390 { 1391 freestrp(&pkg->portdir); 1392 freestrp(&pkg->version); 1393 freestrp(&pkg->pkgfile); 1394 freestrp(&pkg->ignore); 1395 freestrp(&pkg->fetch_deps); 1396 freestrp(&pkg->ext_deps); 1397 freestrp(&pkg->patch_deps); 1398 freestrp(&pkg->build_deps); 1399 freestrp(&pkg->lib_deps); 1400 freestrp(&pkg->run_deps); 1401 freestrp(&pkg->pos_options); 1402 freestrp(&pkg->neg_options); 1403 freestrp(&pkg->flavors); 1404 free(pkg); 1405 } 1406 #endif 1407