1 /* 2 * Copyright (c) 2019 The DragonFly Project. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Matthew Dillon <dillon@backplane.com> 6 * 7 * This code uses concepts and configuration based on 'synth', by 8 * John R. Marino <draco@marino.st>, which was written in ada. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in 18 * the documentation and/or other materials provided with the 19 * distribution. 20 * 3. Neither the name of The DragonFly Project nor the names of its 21 * contributors may be used to endorse or promote products derived 22 * from this software without specific, prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 25 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 26 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 27 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 28 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 29 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 30 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 31 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 32 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 33 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 34 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 35 * SUCH DAMAGE. 36 */ 37 38 #include "dsynth.h" 39 40 #define PKG_HSIZE 32768 41 #define PKG_HMASK 32767 42 43 static int parsepkglist_file(const char *path, int debugstop); 44 static void childGetPackageInfo(bulk_t *bulk); 45 static void childGetBinaryDistInfo(bulk_t *bulk); 46 static void childOptimizeEnv(bulk_t *bulk); 47 static pkg_t *resolveDeps(pkg_t *dep_list, pkg_t ***list_tailp, int gentopo); 48 static void resolveFlavors(pkg_t *pkg, char *flavors, int gentopo); 49 static void resolveDepString(pkg_t *pkg, char *depstr, 50 int gentopo, int dep_type); 51 static pkg_t *processPackageListBulk(int total); 52 static int scan_and_queue_dir(const char *path, const char *level1, int level); 53 static int scan_binary_repo(const char *path); 54 #if 0 55 static void pkgfree(pkg_t *pkg); 56 #endif 57 58 static int PrepareSystemFlag; 59 60 static pkg_t *PkgHash1[PKG_HSIZE]; /* by portdir */ 61 static pkg_t *PkgHash2[PKG_HSIZE]; /* by pkgfile */ 62 63 /* 64 * Allocate a new pkg structure plus basic initialization. 65 */ 66 static __inline pkg_t * 67 allocpkg(void) 68 { 69 pkg_t *pkg; 70 71 pkg = calloc(1, sizeof(*pkg)); 72 pkg->idepon_list.next = &pkg->idepon_list; 73 pkg->idepon_list.prev = &pkg->idepon_list; 74 pkg->deponi_list.next = &pkg->deponi_list; 75 pkg->deponi_list.prev = &pkg->deponi_list; 76 77 return pkg; 78 } 79 80 /* 81 * Simple hash for lookups 82 */ 83 static __inline int 84 pkghash(const char *str) 85 { 86 int hv = 0xABC32923; 87 while (*str) { 88 hv = (hv << 5) ^ *str; 89 ++str; 90 } 91 hv = hv ^ (hv / PKG_HSIZE) ^ (hv / PKG_HSIZE / PKG_HSIZE); 92 return (hv & PKG_HMASK); 93 } 94 95 static void 96 pkg_enter(pkg_t *pkg) 97 { 98 pkg_t **pkgp; 99 pkg_t *scan; 100 101 if (pkg->portdir) { 102 pkgp = &PkgHash1[pkghash(pkg->portdir)]; 103 while ((scan = *pkgp) != NULL) { 104 if (strcmp(pkg->portdir, scan->portdir) == 0) 105 break; 106 pkgp = &scan->hnext1; 107 } 108 ddassert(scan == NULL || (scan->flags & PKGF_PLACEHOLD)); 109 if (scan && (scan->flags & PKGF_PLACEHOLD)) { 110 ddassert(scan->idepon_list.next == &scan->idepon_list); 111 ddassert(scan->deponi_list.next == &scan->deponi_list); 112 *pkgp = pkg; 113 pkg->hnext1 = scan->hnext1; 114 free(scan->portdir); 115 free(scan); 116 scan = NULL; 117 } 118 if (scan == NULL) 119 *pkgp = pkg; 120 } 121 122 if (pkg->pkgfile) { 123 pkgp = &PkgHash2[pkghash(pkg->pkgfile)]; 124 while ((scan = *pkgp) != NULL) { 125 if (strcmp(pkg->pkgfile, scan->pkgfile) == 0) 126 break; 127 pkgp = &scan->hnext2; 128 } 129 if (scan == NULL) 130 *pkgp = pkg; 131 } 132 } 133 134 static pkg_t * 135 pkg_find(const char *match) 136 { 137 pkg_t **pkgp; 138 pkg_t *pkg; 139 140 pkgp = &PkgHash1[pkghash(match)]; 141 for (pkg = *pkgp; pkg; pkg = pkg->hnext1) { 142 if (strcmp(pkg->portdir, match) == 0) 143 return pkg; 144 } 145 pkgp = &PkgHash2[pkghash(match)]; 146 for (pkg = *pkgp; pkg; pkg = pkg->hnext2) { 147 if (strcmp(pkg->pkgfile, match) == 0) 148 return pkg; 149 } 150 return NULL; 151 } 152 153 /* 154 * Parse a specific list of ports via origin name (portdir/subdir) 155 */ 156 pkg_t * 157 ParsePackageList(int n, char **ary, int debugstop) 158 { 159 pkg_t *list; 160 int total; 161 int fail; 162 int i; 163 164 total = 0; 165 fail = 0; 166 initbulk(childGetPackageInfo, MaxBulk); 167 168 /* 169 * Always include ports-mgmt/pkg. s4 is "x" meaning not a manual 170 * selection, "d" meaning DEBUGSTOP mode, or NULL. 171 */ 172 queuebulk("ports-mgmt", "pkg", NULL, "x"); 173 174 for (i = 0; i < n; ++i) { 175 char *l1; 176 char *l2; 177 char *l3; 178 struct stat st; 179 180 l1 = strdup(ary[i]); 181 if (stat(l1, &st) == 0 && S_ISREG(st.st_mode)) { 182 total += parsepkglist_file(l1, debugstop); 183 continue; 184 } 185 186 l2 = strchr(l1, '/'); 187 if (l2 == NULL) { 188 printf("Bad portdir specification: %s\n", l1); 189 free(l1); 190 fail = 1; 191 continue; 192 } 193 *l2++ = 0; 194 l3 = strchr(l2, '@'); 195 if (l3) 196 *l3++ = 0; 197 198 /* 199 * Silently ignore any manually specified ports-mgmt/pkg, 200 * which we already auto-added. 201 */ 202 if (strcmp(l1, "ports-mgmt") != 0 || 203 strcmp(l2, "pkg") != 0) 204 { 205 queuebulk(l1, l2, l3, (debugstop ? "d" : NULL)); 206 } 207 ++total; 208 free(l1); 209 } 210 printf("Processing %d ports\n", total); 211 212 list = processPackageListBulk(total); 213 if (fail) { 214 dfatal("Bad specifications, exiting"); 215 exit(1); 216 } 217 218 return list; 219 } 220 221 static 222 int 223 parsepkglist_file(const char *path, int debugstop) 224 { 225 FILE *fp; 226 char *base; 227 char *l1; 228 char *l2; 229 char *l3; 230 size_t len; 231 int total; 232 233 if ((fp = fopen(path, "r")) == NULL) { 234 dpanic_errno("Cannot read %s\n", path); 235 /* NOT REACHED */ 236 return 0; 237 } 238 239 total = 0; 240 241 while ((base = fgetln(fp, &len)) != NULL) { 242 if (len == 0 || base[len-1] != '\n') 243 continue; 244 base[--len] = 0; 245 l1 = strtok(base, " \t\r\n"); 246 if (l1 == NULL) { 247 printf("Badly formatted pkg info line: %s\n", base); 248 continue; 249 } 250 l2 = strchr(l1, '/'); 251 if (l2 == NULL) { 252 printf("Badly formatted specification: %s\n", l1); 253 continue; 254 } 255 *l2++ = 0; 256 l3 = strchr(l2, '@'); 257 if (l3) 258 *l3++ = 0; 259 queuebulk(l1, l2, l3, (debugstop ? "d" : NULL)); 260 ++total; 261 } 262 fclose(fp); 263 264 return total; 265 } 266 267 /* 268 * Parse packages from the list installed on the system. 269 */ 270 pkg_t * 271 GetLocalPackageList(void) 272 { 273 pkg_t *list; 274 FILE *fp; 275 char *base; 276 char *data; 277 char *l1; 278 char *l2; 279 char *l3; 280 int total; 281 int state; 282 size_t len; 283 284 PrepareSystemFlag = 1; 285 initbulk(childGetPackageInfo, MaxBulk); 286 total = 0; 287 state = 0; 288 l1 = NULL; 289 l2 = NULL; 290 l3 = NULL; 291 292 fp = popen("pkg info -a -o -A", "r"); 293 294 /* 295 * Always include ports-mgmt/pkg. s4 is "x" meaning not a manual 296 * selection, "d" meaning DEBUGSTOP mode, or NULL. 297 */ 298 queuebulk("ports-mgmt", "pkg", NULL, "x"); 299 300 while ((base = fgetln(fp, &len)) != NULL) { 301 if (len == 0 || base[len-1] != '\n') 302 continue; 303 base[--len] = 0; 304 305 data = strchr(base, ':'); 306 if (data == NULL) 307 continue; 308 *data++ = 0; 309 310 base = strtok(base, " \t\r"); 311 data = strtok(data, " \t\r"); 312 313 if (base == NULL || data == NULL) 314 continue; 315 316 if (strcmp(base, "Origin") == 0) { 317 if (state == 1) { 318 queuebulk(l1, l2, NULL, NULL); 319 state = 0; 320 ++total; 321 } 322 323 if (strchr(data, '/') == NULL) { 324 printf("Badly formatted origin: %s\n", l1); 325 } 326 if (l1) 327 free(l1); 328 if (l3) 329 free(l3); 330 l1 = strdup(data); 331 l2 = strchr(l1, '/'); 332 *l2++ = 0; 333 l3 = strchr(l2, '@'); /* typically NULL */ 334 if (l3) { 335 *l3++ = 0; 336 l3 = strdup(l3); 337 } 338 339 /* 340 * Don't queue ports-mgmt/pkg twice, we already 341 * queued it manually. 342 */ 343 if (strcmp(l1, "ports-mgmt") != 0 || 344 strcmp(l2, "pkg") != 0) { 345 state = 1; 346 } 347 continue; 348 } 349 if (state == 1 && strcmp(base, "flavor") == 0) { 350 queuebulk(l1, l2, data, NULL); 351 state = 0; 352 ++total; 353 } 354 } 355 if (state == 1) { 356 queuebulk(l1, l2, NULL, NULL); 357 /*state = 0; not needed */ 358 } 359 if (l1) 360 free(l1); 361 if (l3) 362 free(l3); 363 364 pclose(fp); 365 366 printf("Processing %d ports\n", total); 367 368 list = processPackageListBulk(total); 369 370 return list; 371 } 372 373 pkg_t * 374 GetFullPackageList(void) 375 { 376 int total; 377 378 initbulk(childGetPackageInfo, MaxBulk); 379 total = scan_and_queue_dir(DPortsPath, NULL, 1); 380 printf("Scanning %d ports\n", total); 381 382 return processPackageListBulk(total); 383 } 384 385 /* 386 * Caller has queued the process list for bulk operation. We retrieve 387 * the results and clean up the bulk operation (we may have to do a second 388 * bulk operation so we have to be the ones to clean it up). 389 */ 390 static pkg_t * 391 processPackageListBulk(int total) 392 { 393 bulk_t *bulk; 394 pkg_t *scan; 395 pkg_t *list; 396 pkg_t *dep_list; 397 pkg_t **list_tail; 398 int count; 399 int stop_fail; 400 int stop_base_list; 401 int remove_corrupt; 402 403 list = NULL; 404 list_tail = &list; 405 count = 0; 406 remove_corrupt = 0; 407 408 while ((bulk = getbulk()) != NULL) { 409 ++count; 410 if ((count & 255) == 0) { 411 printf("%6.2f%%\r", 412 (double)count * 100.0 / (double)total + 0.001); 413 fflush(stdout); 414 } 415 if (bulk->list) { 416 *list_tail = bulk->list; 417 bulk->list = NULL; 418 while ((scan = *list_tail) != NULL) { 419 if (bulk->s4 == NULL || bulk->s4[0] != 'x') 420 scan->flags |= PKGF_MANUALSEL; 421 pkg_enter(scan); 422 list_tail = &scan->bnext; 423 } 424 } 425 freebulk(bulk); 426 } 427 printf("100.00%%\n"); 428 printf("\nTotal %d\n", count); 429 fflush(stdout); 430 431 /* 432 * Resolve all dependencies for the related packages, potentially 433 * adding anything that could not be found to the list. This will 434 * continue to issue bulk operations and process the result until 435 * no dependencies are left. 436 */ 437 printf("Resolving dependencies..."); 438 fflush(stdout); 439 dep_list = list; 440 while (dep_list) { 441 dep_list = resolveDeps(dep_list, &list_tail, 0); 442 } 443 printf("done\n"); 444 445 donebulk(); 446 447 /* 448 * Generate the topology 449 */ 450 resolveDeps(list, NULL, 1); 451 452 /* 453 * Do a final count, ignore place holders. 454 * 455 * Also set stop_fail if appropriate. Check for direct specifications 456 * which fail to probe and any direct dependencies of those 457 * specifications, but don't recurse (for now)... don't check indirect 458 * dependencies (i.e. A -> B -> C where A is directly specified, B 459 * is adirect dependency, and C fails to probe). 460 */ 461 count = 0; 462 stop_fail = 0; 463 stop_base_list = 0; 464 for (scan = list; scan; scan = scan->bnext) { 465 if ((scan->flags & PKGF_ERROR) == 0) { 466 ++count; 467 } 468 if ((scan->flags & PKGF_MANUALSEL) && MaskProbeAbort == 0) { 469 pkglink_t *link; 470 471 /* 472 * Directly specified package failed to probe 473 */ 474 if (scan->flags & PKGF_CORRUPT) { 475 ++stop_fail; 476 ++stop_base_list; 477 } 478 479 /* 480 * Directly specified package had a direct dependency 481 * that failed to probe (don't go further). 482 */ 483 PKGLIST_FOREACH(link, &scan->idepon_list) { 484 if (link->pkg && 485 (link->pkg->flags & PKGF_CORRUPT)) { 486 ++stop_fail; 487 } 488 } 489 } 490 } 491 printf("Total Returned %d\n", count); 492 493 /* 494 * Check to see if any PKGF_MANUALSEL packages 495 */ 496 if (stop_fail) { 497 printf("%d packages failed to probe\n", stop_fail); 498 if (PrepareSystemFlag) { 499 if (stop_fail == stop_base_list) { 500 printf( 501 "prepare-system: Some of your installed packages no longer exist in\n" 502 "dports, do you wish to continue rebuilding what does exist?\n"); 503 if (askyn("Continue anyway? ")) 504 remove_corrupt = 1; 505 } else { 506 printf( 507 "prepare-system: Some of your installed packages have dependencies\n" 508 "which could not be found in dports, cannot continue, aborting\n"); 509 } 510 } else { 511 printf("unable to continue, aborting\n"); 512 } 513 if (remove_corrupt == 0) 514 exit(1); 515 } 516 517 /* 518 * Remove corrupt packages before continuing 519 */ 520 if (remove_corrupt) { 521 list_tail = &list; 522 while ((scan = *list_tail) != NULL) { 523 if (scan->flags & PKGF_CORRUPT) 524 *list_tail = scan->bnext; 525 else 526 list_tail = &scan->bnext; 527 } 528 } 529 530 /* 531 * Scan our binary distributions and related dependencies looking 532 * for any packages that have already been built. 533 */ 534 initbulk(childGetBinaryDistInfo, MaxBulk); 535 total = scan_binary_repo(RepositoryPath); 536 count = 0; 537 printf("Scanning %d packages\n", total); 538 539 while ((bulk = getbulk()) != NULL) { 540 ++count; 541 if ((count & 255) == 0) { 542 printf("%6.2f%%\r", 543 (double)count * 100.0 / (double)total + 0.001); 544 fflush(stdout); 545 } 546 freebulk(bulk); 547 } 548 printf("100.00%%\n"); 549 printf("\nTotal %d\n", count); 550 fflush(stdout); 551 donebulk(); 552 553 printf("all done\n"); 554 555 return list; 556 } 557 558 pkg_t * 559 GetPkgPkg(pkg_t **listp) 560 { 561 bulk_t *bulk; 562 pkg_t *scan; 563 pkg_t *s2; 564 565 for (scan = *listp; scan; scan = scan->bnext) { 566 if (strcmp(scan->portdir, "ports-mgmt/pkg") == 0) 567 return scan; 568 } 569 570 /* 571 * This will force pkg to be built, but generally this code 572 * is not reached because the package list processing code 573 * adds ports-mgmt/pkg unconditionally. 574 */ 575 initbulk(childGetPackageInfo, MaxBulk); 576 queuebulk("ports-mgmt", "pkg", NULL, "x"); 577 bulk = getbulk(); 578 dassert(bulk, "Cannot find ports-mgmt/pkg"); 579 scan = bulk->list; 580 bulk->list = NULL; 581 freebulk(bulk); 582 donebulk(); 583 584 585 /* 586 * Include added packages to the total and add the initial bulk 587 * built packages to the list so they get counted. 588 */ 589 for (s2 = scan; s2->bnext; s2 = s2->bnext) 590 ++BuildTotal; 591 for (s2 = scan; s2->bnext; s2 = s2->bnext) 592 ; 593 s2->bnext = *listp; 594 *listp = scan; 595 ++BuildTotal; 596 597 return scan; 598 } 599 600 /* 601 * Try to optimize the environment by supplying information that 602 * the ports system would generally have to run stuff to get on 603 * every package. 604 * 605 * See childOptimizeEnv() for the actual handling. We execute 606 * a single make -V... -V... for ports-mgmt/pkg from within the 607 * bulk system (which handles the environment and disables 608 * /etc/make.conf), and we then call addbuildenv() as appropriate. 609 * 610 * _PERL5_FROM_BIN 611 * add others... 612 */ 613 void 614 OptimizeEnv(void) 615 { 616 bulk_t *bulk; 617 618 initbulk(childOptimizeEnv, MaxBulk); 619 queuebulk("ports-mgmt", "pkg", NULL, NULL); 620 bulk = getbulk(); 621 freebulk(bulk); 622 donebulk(); 623 } 624 625 /* 626 * Run through the list resolving dependencies and constructing the topology 627 * linkages. This may append packages to the list. Dependencies to dummy 628 * nodes which do not specify a flavor do not need special handling, the 629 * search code in build.c will properly follow the first flavor. 630 */ 631 static pkg_t * 632 resolveDeps(pkg_t *list, pkg_t ***list_tailp, int gentopo) 633 { 634 pkg_t *ret_list = NULL; 635 pkg_t *scan; 636 pkg_t *use; 637 bulk_t *bulk; 638 639 for (scan = list; scan; scan = scan->bnext) { 640 use = pkg_find(scan->portdir); 641 resolveFlavors(use, scan->flavors, gentopo); 642 resolveDepString(use, scan->fetch_deps, 643 gentopo, DEP_TYPE_FETCH); 644 resolveDepString(use, scan->ext_deps, 645 gentopo, DEP_TYPE_EXT); 646 resolveDepString(use, scan->patch_deps, 647 gentopo, DEP_TYPE_PATCH); 648 resolveDepString(use, scan->build_deps, 649 gentopo, DEP_TYPE_BUILD); 650 resolveDepString(use, scan->lib_deps, 651 gentopo, DEP_TYPE_LIB); 652 resolveDepString(use, scan->run_deps, 653 gentopo, DEP_TYPE_RUN); 654 } 655 656 /* 657 * No bulk ops are queued when doing the final topology 658 * generation. 659 * 660 * Avoid entering duplicate results from the bulk ops. Duplicate 661 * results are mostly filtered out, but not always. A dummy node 662 * representing multiple flavors will parse-out the flavors 663 */ 664 if (gentopo) 665 return NULL; 666 while ((bulk = getbulk()) != NULL) { 667 if (bulk->list) { 668 if (ret_list == NULL) 669 ret_list = bulk->list; 670 **list_tailp = bulk->list; 671 bulk->list = NULL; 672 while ((scan = **list_tailp) != NULL) { 673 pkg_enter(scan); 674 *list_tailp = &scan->bnext; 675 } 676 } 677 freebulk(bulk); 678 } 679 return (ret_list); 680 } 681 682 /* 683 * Resolve a generic node that has flavors, queue to retrieve info for 684 * each flavor and setup linkages as appropriate. 685 */ 686 static void 687 resolveFlavors(pkg_t *pkg, char *flavors, int gentopo) 688 { 689 char *flavor_base; 690 char *flavor_scan; 691 char *flavor; 692 char *portdir; 693 char *s1; 694 char *s2; 695 pkg_t *dpkg; 696 pkglink_t *link; 697 698 if ((pkg->flags & PKGF_DUMMY) == 0) 699 return; 700 if (pkg->flavors == NULL || pkg->flavors[0] == 0) 701 return; 702 flavor_base = strdup(flavors); 703 flavor_scan = flavor_base; 704 705 for (;;) { 706 do { 707 flavor = strsep(&flavor_scan, " \t"); 708 } while (flavor && *flavor == 0); 709 if (flavor == NULL) 710 break; 711 712 /* 713 * Iterate each flavor generating "s1/s2@flavor". 714 * 715 * queuebulk() info for each flavor, and set-up the 716 * linkages in the topology generation pass. 717 */ 718 asprintf(&portdir, "%s@%s", pkg->portdir, flavor); 719 s1 = strdup(pkg->portdir); 720 s2 = strchr(s1, '/'); 721 *s2++ = 0; 722 723 dpkg = pkg_find(portdir); 724 if (dpkg && gentopo) { 725 /* 726 * Setup linkages 727 */ 728 free(portdir); 729 730 link = calloc(1, sizeof(*link)); 731 link->pkg = dpkg; 732 link->next = &pkg->idepon_list; 733 link->prev = pkg->idepon_list.prev; 734 link->next->prev = link; 735 link->prev->next = link; 736 link->dep_type = DEP_TYPE_BUILD; 737 738 link = calloc(1, sizeof(*link)); 739 link->pkg = pkg; 740 link->next = &dpkg->deponi_list; 741 link->prev = dpkg->deponi_list.prev; 742 link->next->prev = link; 743 link->prev->next = link; 744 link->dep_type = DEP_TYPE_BUILD; 745 ++dpkg->depi_count; 746 } else if (gentopo == 0 && dpkg == NULL) { 747 /* 748 * Use a place-holder to prevent duplicate 749 * dependencies from being processed. The placeholder 750 * will be replaced by the actual dependency. 751 */ 752 dpkg = allocpkg(); 753 dpkg->portdir = portdir; 754 dpkg->flags = PKGF_PLACEHOLD; 755 pkg_enter(dpkg); 756 queuebulk(s1, s2, flavor, NULL); 757 } 758 free(s1); 759 } 760 free(flavor_base); 761 } 762 763 static void 764 resolveDepString(pkg_t *pkg, char *depstr, int gentopo, int dep_type) 765 { 766 char *copy_base; 767 char *copy; 768 char *dep; 769 char *sep; 770 char *tag; 771 char *flavor; 772 pkg_t *dpkg; 773 774 if (depstr == NULL || depstr[0] == 0) 775 return; 776 777 copy_base = strdup(depstr); 778 copy = copy_base; 779 780 for (;;) { 781 do { 782 dep = strsep(©, " \t"); 783 } while (dep && *dep == 0); 784 if (dep == NULL) 785 break; 786 787 /* 788 * Ignore dependencies prefixed with ${NONEXISTENT} 789 */ 790 if (strncmp(dep, "/nonexistent:", 13) == 0) 791 continue; 792 793 dep = strchr(dep, ':'); 794 if (dep == NULL || *dep != ':') { 795 printf("Error parsing dependency for %s: %s\n", 796 pkg->portdir, copy_base); 797 continue; 798 } 799 ++dep; 800 801 /* 802 * Strip-off any DPortsPath prefix. EXTRACT_DEPENDS 803 * often (always?) generates this prefix. 804 */ 805 if (strncmp(dep, DPortsPath, strlen(DPortsPath)) == 0) { 806 dep += strlen(DPortsPath); 807 if (*dep == '/') 808 ++dep; 809 } 810 811 /* 812 * Strip-off any tag (such as :patch). We don't try to 813 * organize dependencies at this fine a grain (for now). 814 */ 815 tag = strchr(dep, ':'); 816 if (tag) 817 *tag++ = 0; 818 819 /* 820 * Locate the dependency 821 */ 822 if ((dpkg = pkg_find(dep)) != NULL) { 823 if (gentopo) { 824 pkglink_t *link; 825 826 /* 827 * NOTE: idep_count is calculated recursively 828 * at build-time 829 */ 830 ddprintf(0, "Add Dependency %s -> %s\n", 831 pkg->portdir, dpkg->portdir); 832 link = calloc(1, sizeof(*link)); 833 link->pkg = dpkg; 834 link->next = &pkg->idepon_list; 835 link->prev = pkg->idepon_list.prev; 836 link->next->prev = link; 837 link->prev->next = link; 838 link->dep_type = dep_type; 839 840 link = calloc(1, sizeof(*link)); 841 link->pkg = pkg; 842 link->next = &dpkg->deponi_list; 843 link->prev = dpkg->deponi_list.prev; 844 link->next->prev = link; 845 link->prev->next = link; 846 link->dep_type = dep_type; 847 ++dpkg->depi_count; 848 } 849 continue; 850 } 851 852 /* 853 * This shouldn't happen because we already took a first 854 * pass and should have generated the pkgs. 855 */ 856 if (gentopo) { 857 printf("Topology Generate failed for %s: %s\n", 858 pkg->portdir, copy_base); 859 continue; 860 } 861 862 /* 863 * Separate out the two dports directory components and 864 * extract the optional '@flavor' specification. 865 */ 866 sep = strchr(dep, '/'); 867 if (sep == NULL) { 868 printf("Error parsing dependency for %s: %s\n", 869 pkg->portdir, copy_base); 870 continue; 871 } 872 *sep++ = 0; 873 874 /* 875 * The flavor hangs off the separator, not the tag 876 */ 877 flavor = strrchr(sep, '@'); 878 #if 0 879 if (tag) 880 flavor = strrchr(tag, '@'); 881 else 882 flavor = strrchr(sep, '@'); 883 #endif 884 if (flavor) 885 *flavor++ = 0; 886 887 if (flavor) 888 ddprintf(0, "QUEUE DEPENDENCY FROM PKG %s: %s/%s@%s\n", 889 pkg->portdir, dep, sep, flavor); 890 else 891 ddprintf(0, "QUEUE DEPENDENCY FROM PKG %s: %s/%s\n", 892 pkg->portdir, dep, sep); 893 894 /* 895 * Use a place-holder to prevent duplicate dependencies from 896 * being processed. The placeholder will be replaced by 897 * the actual dependency. 898 */ 899 dpkg = allocpkg(); 900 if (flavor) 901 asprintf(&dpkg->portdir, "%s/%s@%s", dep, sep, flavor); 902 else 903 asprintf(&dpkg->portdir, "%s/%s", dep, sep); 904 dpkg->flags = PKGF_PLACEHOLD; 905 pkg_enter(dpkg); 906 907 queuebulk(dep, sep, flavor, NULL); 908 } 909 free(copy_base); 910 } 911 912 void 913 FreePackageList(pkg_t *pkgs __unused) 914 { 915 dfatal("not implemented"); 916 } 917 918 /* 919 * Scan some or all dports to allocate the related pkg structure. Dependencies 920 * are stored but not processed. 921 * 922 * Threaded function 923 */ 924 static void 925 childGetPackageInfo(bulk_t *bulk) 926 { 927 pkg_t *pkg; 928 char *flavor; 929 char *ptr; 930 FILE *fp; 931 int line; 932 size_t len; 933 char *portpath; 934 char *flavarg; 935 const char *cav[MAXCAC]; 936 pid_t pid; 937 int cac; 938 939 /* 940 * If the package has flavors we will loop on each one. If a flavor 941 * is not passed in s3 we will loop on all flavors, otherwise we will 942 * only process the passed-in flavor. 943 */ 944 flavor = bulk->s3; /* usually NULL */ 945 946 bulk->list = NULL; 947 948 asprintf(&portpath, "%s/%s/%s", DPortsPath, bulk->s1, bulk->s2); 949 if (flavor) 950 asprintf(&flavarg, "FLAVOR=%s", flavor); 951 else 952 flavarg = NULL; 953 954 cac = 0; 955 cav[cac++] = MAKE_BINARY; 956 cav[cac++] = "-C"; 957 cav[cac++] = portpath; 958 if (flavarg) 959 cav[cac++] = flavarg; 960 cav[cac++] = "-VPKGVERSION"; 961 cav[cac++] = "-VPKGFILE:T"; 962 cav[cac++] = "-VALLFILES"; 963 cav[cac++] = "-VDIST_SUBDIR"; 964 cav[cac++] = "-VMAKE_JOBS_NUMBER"; 965 cav[cac++] = "-VIGNORE"; 966 cav[cac++] = "-VFETCH_DEPENDS"; 967 cav[cac++] = "-VEXTRACT_DEPENDS"; 968 cav[cac++] = "-VPATCH_DEPENDS"; 969 cav[cac++] = "-VBUILD_DEPENDS"; 970 cav[cac++] = "-VLIB_DEPENDS"; 971 cav[cac++] = "-VRUN_DEPENDS"; 972 cav[cac++] = "-VSELECTED_OPTIONS"; 973 cav[cac++] = "-VDESELECTED_OPTIONS"; 974 cav[cac++] = "-VUSE_LINUX"; 975 cav[cac++] = "-VFLAVORS"; 976 cav[cac++] = "-VUSES"; 977 978 fp = dexec_open(portpath + strlen(DPortsPath) + 1, cav, cac, 979 &pid, NULL, 1, 1); 980 freestrp(&flavarg); 981 982 pkg = allocpkg(); 983 if (flavor) 984 asprintf(&pkg->portdir, "%s/%s@%s", bulk->s1, bulk->s2, flavor); 985 else 986 asprintf(&pkg->portdir, "%s/%s", bulk->s1, bulk->s2); 987 988 line = 1; 989 while ((ptr = fgetln(fp, &len)) != NULL) { 990 if (len == 0 || ptr[len-1] != '\n') { 991 dfatal("Bad package info for %s/%s response line %d", 992 bulk->s1, bulk->s2, line); 993 } 994 ptr[--len] = 0; 995 996 switch(line) { 997 case 1: /* PKGVERSION */ 998 asprintf(&pkg->version, "%s", ptr); 999 break; 1000 case 2: /* PKGFILE */ 1001 asprintf(&pkg->pkgfile, "%s", ptr); 1002 break; 1003 case 3: /* ALLFILES (aka DISTFILES + patch files) */ 1004 asprintf(&pkg->distfiles, "%s", ptr); 1005 break; 1006 case 4: /* DIST_SUBDIR */ 1007 pkg->distsubdir = strdup_or_null(ptr); 1008 break; 1009 case 5: /* MAKE_JOBS_NUMBER */ 1010 pkg->make_jobs_number = strtol(ptr, NULL, 0); 1011 break; 1012 case 6: /* IGNORE */ 1013 pkg->ignore = strdup_or_null(ptr); 1014 break; 1015 case 7: /* FETCH_DEPENDS */ 1016 pkg->fetch_deps = strdup_or_null(ptr); 1017 break; 1018 case 8: /* EXTRACT_DEPENDS */ 1019 pkg->ext_deps = strdup_or_null(ptr); 1020 break; 1021 case 9: /* PATCH_DEPENDS */ 1022 pkg->patch_deps = strdup_or_null(ptr); 1023 break; 1024 case 10: /* BUILD_DEPENDS */ 1025 pkg->build_deps = strdup_or_null(ptr); 1026 break; 1027 case 11: /* LIB_DEPENDS */ 1028 pkg->lib_deps = strdup_or_null(ptr); 1029 break; 1030 case 12: /* RUN_DEPENDS */ 1031 pkg->run_deps = strdup_or_null(ptr); 1032 break; 1033 case 13: /* SELECTED_OPTIONS */ 1034 pkg->pos_options = strdup_or_null(ptr); 1035 break; 1036 case 14: /* DESELECTED_OPTIONS */ 1037 pkg->neg_options = strdup_or_null(ptr); 1038 break; 1039 case 15: /* USE_LINUX */ 1040 if (ptr[0]) 1041 pkg->use_linux = 1; 1042 break; 1043 case 16: /* FLAVORS */ 1044 asprintf(&pkg->flavors, "%s", ptr); 1045 break; 1046 case 17: /* USES */ 1047 asprintf(&pkg->uses, "%s", ptr); 1048 if (strstr(pkg->uses, "metaport")) 1049 pkg->flags |= PKGF_META; 1050 break; 1051 default: 1052 printf("EXTRA LINE: %s\n", ptr); 1053 break; 1054 } 1055 ++line; 1056 } 1057 if (line == 1) { 1058 printf("DPort not found: %s/%s\n", bulk->s1, bulk->s2); 1059 pkg->flags |= PKGF_NOTFOUND; 1060 } else if (line != 17 + 1) { 1061 printf("DPort corrupt: %s/%s\n", bulk->s1, bulk->s2); 1062 pkg->flags |= PKGF_CORRUPT; 1063 } 1064 if (dexec_close(fp, pid)) { 1065 printf("make -V* command for %s/%s failed\n", 1066 bulk->s1, bulk->s2); 1067 pkg->flags |= PKGF_CORRUPT; 1068 } 1069 ddassert(bulk->s1); 1070 1071 /* 1072 * DEBUGSTOP mode 1073 */ 1074 if (bulk->s4 && bulk->s4[0] == 'd') 1075 pkg->flags |= PKGF_DEBUGSTOP; 1076 1077 /* 1078 * Mark as a dummy node, the front-end will iterate the flavors 1079 * and create sub-nodes for us. 1080 * 1081 * Get rid of elements returned that are for the first flavor. 1082 * We are creating a dummy node here, not the node for the first 1083 * flavor. 1084 */ 1085 if (flavor == NULL && pkg->flavors && pkg->flavors[0]) { 1086 pkg->flags |= PKGF_DUMMY; 1087 freestrp(&pkg->fetch_deps); 1088 freestrp(&pkg->ext_deps); 1089 freestrp(&pkg->patch_deps); 1090 freestrp(&pkg->build_deps); 1091 freestrp(&pkg->lib_deps); 1092 freestrp(&pkg->run_deps); 1093 freestrp(&pkg->pkgfile); 1094 } 1095 1096 /* 1097 * Checksum the port directory tree. This just rollsup crcs of the 1098 * path names and a few stat fields (mtime, size) in order to detect 1099 * if any modification has been made to the port. 1100 */ 1101 pkg->crc32 = crcDirTree(portpath); 1102 1103 /* 1104 * Only one pkg is put on the return list now. This code no 1105 * longer creates pseudo-nodes for flavors (the frontend requests 1106 * each flavor instead). 1107 */ 1108 bulk->list = pkg; 1109 free(portpath); 1110 } 1111 1112 /* 1113 * Query the package (at least to make sure it hasn't been truncated) 1114 * and mark it as PACKAGED if found. 1115 * 1116 * Threaded function 1117 */ 1118 static void 1119 childGetBinaryDistInfo(bulk_t *bulk) 1120 { 1121 char *ptr; 1122 FILE *fp; 1123 size_t len; 1124 pkg_t *pkg; 1125 const char *cav[MAXCAC]; 1126 char *repopath; 1127 char buf[1024]; 1128 pid_t pid; 1129 int cac; 1130 int deleteme; 1131 1132 asprintf(&repopath, "%s/%s", RepositoryPath, bulk->s1); 1133 1134 cac = 0; 1135 cav[cac++] = PKG_BINARY; 1136 cav[cac++] = "query"; 1137 cav[cac++] = "-F"; 1138 cav[cac++] = repopath; 1139 cav[cac++] = "%n-%v"; 1140 1141 fp = dexec_open(NULL, cav, cac, &pid, NULL, 1, 0); 1142 deleteme = DeleteObsoletePkgs; 1143 1144 while ((ptr = fgetln(fp, &len)) != NULL) { 1145 if (len == 0 || ptr[len-1] != '\n') 1146 continue; 1147 ptr[len-1] = 0; 1148 snprintf(buf, sizeof(buf), "%s%s", ptr, UsePkgSufx); 1149 1150 pkg = pkg_find(buf); 1151 if (pkg) { 1152 pkg->flags |= PKGF_PACKAGED; 1153 deleteme = 0; 1154 } else { 1155 ddprintf(0, "Note: package scan, not in list, " 1156 "skipping %s\n", buf); 1157 } 1158 } 1159 if (dexec_close(fp, pid)) { 1160 printf("pkg query command failed for %s\n", repopath); 1161 } 1162 if (deleteme) { 1163 dlog(DLOG_ALL | DLOG_STDOUT, 1164 "Deleting obsolete package %s\n", repopath); 1165 remove(repopath); 1166 } 1167 free(repopath); 1168 } 1169 1170 static void 1171 childOptimizeEnv(bulk_t *bulk) 1172 { 1173 char *portpath; 1174 char *ptr; 1175 FILE *fp; 1176 int line; 1177 size_t len; 1178 const char *cav[MAXCAC]; 1179 pid_t pid; 1180 int cac; 1181 1182 asprintf(&portpath, "%s/%s/%s", DPortsPath, bulk->s1, bulk->s2); 1183 1184 cac = 0; 1185 cav[cac++] = MAKE_BINARY; 1186 cav[cac++] = "-C"; 1187 cav[cac++] = portpath; 1188 cav[cac++] = "-V_PERL5_FROM_BIN"; 1189 1190 fp = dexec_open(portpath + strlen(DPortsPath) + 1, cav, cac, 1191 &pid, NULL, 1, 1); 1192 free(portpath); 1193 1194 line = 1; 1195 while ((ptr = fgetln(fp, &len)) != NULL) { 1196 if (len == 0 || ptr[len-1] != '\n') { 1197 dfatal("Bad package info for %s/%s response line %d", 1198 bulk->s1, bulk->s2, line); 1199 } 1200 ptr[--len] = 0; 1201 1202 switch(line) { 1203 case 1: /* _PERL5_FROM_BIN */ 1204 addbuildenv("_PERL5_FROM_BIN", ptr, BENV_ENVIRONMENT); 1205 break; 1206 default: 1207 printf("childOptimizeEnv: EXTRA LINE: %s\n", ptr); 1208 break; 1209 } 1210 ++line; 1211 } 1212 if (line == 1) { 1213 printf("DPort not found: %s/%s\n", bulk->s1, bulk->s2); 1214 } else if (line != 1 + 1) { 1215 printf("DPort corrupt: %s/%s\n", bulk->s1, bulk->s2); 1216 } 1217 if (dexec_close(fp, pid)) { 1218 printf("childOptimizeEnv() failed\n"); 1219 } 1220 } 1221 1222 static int 1223 scan_and_queue_dir(const char *path, const char *level1, int level) 1224 { 1225 DIR *dir; 1226 char *s1; 1227 char *s2; 1228 struct dirent *den; 1229 struct stat st; 1230 int count = 0; 1231 1232 dir = opendir(path); 1233 dassert(dir, "Cannot open dports path \"%s\"", path); 1234 1235 while ((den = readdir(dir)) != NULL) { 1236 if (den->d_namlen == 1 && den->d_name[0] == '.') 1237 continue; 1238 if (den->d_namlen == 2 && 1239 den->d_name[0] == '.' && den->d_name[1] == '.') 1240 continue; 1241 asprintf(&s1, "%s/%s", path, den->d_name); 1242 if (lstat(s1, &st) < 0 || !S_ISDIR(st.st_mode)) { 1243 free(s1); 1244 continue; 1245 } 1246 if (level == 1) { 1247 count += scan_and_queue_dir(s1, den->d_name, 2); 1248 free(s1); 1249 continue; 1250 } 1251 asprintf(&s2, "%s/Makefile", s1); 1252 if (lstat(s2, &st) == 0) { 1253 queuebulk(level1, den->d_name, NULL, NULL); 1254 ++count; 1255 } 1256 free(s1); 1257 free(s2); 1258 } 1259 closedir(dir); 1260 1261 return count; 1262 } 1263 1264 static int 1265 scan_binary_repo(const char *path) 1266 { 1267 DIR *dir; 1268 struct dirent *den; 1269 int count; 1270 1271 count = 0; 1272 dir = opendir(path); 1273 dassert(dir, "Cannot open repository path \"%s\"", path); 1274 1275 /* 1276 * NOTE: Test includes the '.' in the suffix. 1277 */ 1278 while ((den = readdir(dir)) != NULL) { 1279 const char *suffix; 1280 1281 suffix = strrchr(den->d_name, '.'); 1282 if (suffix && suffix != den->d_name && 1283 strcmp(suffix, UsePkgSufx) == 0) { 1284 queuebulk(den->d_name, NULL, NULL, NULL); 1285 ++count; 1286 } 1287 } 1288 closedir(dir); 1289 1290 return count; 1291 } 1292 1293 #if 0 1294 static void 1295 pkgfree(pkg_t *pkg) 1296 { 1297 freestrp(&pkg->portdir); 1298 freestrp(&pkg->version); 1299 freestrp(&pkg->pkgfile); 1300 freestrp(&pkg->ignore); 1301 freestrp(&pkg->fetch_deps); 1302 freestrp(&pkg->ext_deps); 1303 freestrp(&pkg->patch_deps); 1304 freestrp(&pkg->build_deps); 1305 freestrp(&pkg->lib_deps); 1306 freestrp(&pkg->run_deps); 1307 freestrp(&pkg->pos_options); 1308 freestrp(&pkg->neg_options); 1309 freestrp(&pkg->flavors); 1310 free(pkg); 1311 } 1312 #endif 1313