1 /* 2 * Copyright (c) 2019 The DragonFly Project. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Matthew Dillon <dillon@backplane.com> 6 * 7 * This code uses concepts and configuration based on 'synth', by 8 * John R. Marino <draco@marino.st>, which was written in ada. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in 18 * the documentation and/or other materials provided with the 19 * distribution. 20 * 3. Neither the name of The DragonFly Project nor the names of its 21 * contributors may be used to endorse or promote products derived 22 * from this software without specific, prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 25 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 26 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 27 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 28 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 29 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 30 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 31 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 32 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 33 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 34 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 35 * SUCH DAMAGE. 36 */ 37 38 #include "dsynth.h" 39 40 #define PKG_HSIZE 32768 41 #define PKG_HMASK 32767 42 43 static int parsepkglist_file(const char *path, int debugstop); 44 static void childGetPackageInfo(bulk_t *bulk); 45 static void childGetBinaryDistInfo(bulk_t *bulk); 46 static void childOptimizeEnv(bulk_t *bulk); 47 static pkg_t *resolveDeps(pkg_t *dep_list, pkg_t ***list_tailp, int gentopo); 48 static void resolveFlavors(pkg_t *pkg, char *flavors, int gentopo); 49 static void resolveDepString(pkg_t *pkg, char *depstr, 50 int gentopo, int dep_type); 51 static pkg_t *processPackageListBulk(int total); 52 static int scan_and_queue_dir(const char *path, const char *level1, int level); 53 static int scan_binary_repo(const char *path); 54 #if 0 55 static void pkgfree(pkg_t *pkg); 56 #endif 57 58 static int PrepareSystemFlag; 59 60 static pkg_t *PkgHash1[PKG_HSIZE]; /* by portdir */ 61 static pkg_t *PkgHash2[PKG_HSIZE]; /* by pkgfile */ 62 63 /* 64 * Allocate a new pkg structure plus basic initialization. 65 */ 66 static __inline pkg_t * 67 allocpkg(void) 68 { 69 pkg_t *pkg; 70 71 pkg = calloc(1, sizeof(*pkg)); 72 pkg->idepon_list.next = &pkg->idepon_list; 73 pkg->idepon_list.prev = &pkg->idepon_list; 74 pkg->deponi_list.next = &pkg->deponi_list; 75 pkg->deponi_list.prev = &pkg->deponi_list; 76 77 return pkg; 78 } 79 80 /* 81 * Simple hash for lookups 82 */ 83 static __inline int 84 pkghash(const char *str) 85 { 86 int hv = 0xABC32923; 87 while (*str) { 88 hv = (hv << 5) ^ *str; 89 ++str; 90 } 91 hv = hv ^ (hv / PKG_HSIZE) ^ (hv / PKG_HSIZE / PKG_HSIZE); 92 return (hv & PKG_HMASK); 93 } 94 95 static void 96 pkg_enter(pkg_t *pkg) 97 { 98 pkg_t **pkgp; 99 pkg_t *scan; 100 101 if (pkg->portdir) { 102 pkgp = &PkgHash1[pkghash(pkg->portdir)]; 103 while ((scan = *pkgp) != NULL) { 104 if (strcmp(pkg->portdir, scan->portdir) == 0) 105 break; 106 pkgp = &scan->hnext1; 107 } 108 ddassert(scan == NULL || (scan->flags & PKGF_PLACEHOLD)); 109 if (scan && (scan->flags & PKGF_PLACEHOLD)) { 110 ddassert(scan->idepon_list.next == &scan->idepon_list); 111 ddassert(scan->deponi_list.next == &scan->deponi_list); 112 *pkgp = pkg; 113 pkg->hnext1 = scan->hnext1; 114 free(scan->portdir); 115 free(scan); 116 scan = NULL; 117 } 118 if (scan == NULL) 119 *pkgp = pkg; 120 } 121 122 if (pkg->pkgfile) { 123 pkgp = &PkgHash2[pkghash(pkg->pkgfile)]; 124 while ((scan = *pkgp) != NULL) { 125 if (strcmp(pkg->pkgfile, scan->pkgfile) == 0) 126 break; 127 pkgp = &scan->hnext2; 128 } 129 if (scan == NULL) 130 *pkgp = pkg; 131 } 132 } 133 134 static pkg_t * 135 pkg_find(const char *match) 136 { 137 pkg_t **pkgp; 138 pkg_t *pkg; 139 140 pkgp = &PkgHash1[pkghash(match)]; 141 for (pkg = *pkgp; pkg; pkg = pkg->hnext1) { 142 if (strcmp(pkg->portdir, match) == 0) 143 return pkg; 144 } 145 pkgp = &PkgHash2[pkghash(match)]; 146 for (pkg = *pkgp; pkg; pkg = pkg->hnext2) { 147 if (strcmp(pkg->pkgfile, match) == 0) 148 return pkg; 149 } 150 return NULL; 151 } 152 153 /* 154 * Parse a specific list of ports via origin name (portdir/subdir) 155 */ 156 pkg_t * 157 ParsePackageList(int n, char **ary, int debugstop) 158 { 159 pkg_t *list; 160 int total; 161 int fail; 162 int i; 163 164 total = 0; 165 fail = 0; 166 initbulk(childGetPackageInfo, MaxBulk); 167 168 /* 169 * Always include ports-mgmt/pkg. s4 is "x" meaning not a manual 170 * selection, "d" meaning DEBUGSTOP mode, or NULL. 171 */ 172 queuebulk("ports-mgmt", "pkg", NULL, "x"); 173 174 for (i = 0; i < n; ++i) { 175 char *l1; 176 char *l2; 177 char *l3; 178 struct stat st; 179 180 l1 = strdup(ary[i]); 181 if (stat(l1, &st) == 0 && S_ISREG(st.st_mode)) { 182 total += parsepkglist_file(l1, debugstop); 183 continue; 184 } 185 186 l2 = strchr(l1, '/'); 187 if (l2 == NULL) { 188 printf("Bad portdir specification: %s\n", l1); 189 free(l1); 190 fail = 1; 191 continue; 192 } 193 *l2++ = 0; 194 l3 = strchr(l2, '@'); 195 if (l3) 196 *l3++ = 0; 197 queuebulk(l1, l2, l3, (debugstop ? "d" : NULL)); 198 ++total; 199 free(l1); 200 } 201 printf("Processing %d ports\n", total); 202 203 list = processPackageListBulk(total); 204 if (fail) { 205 dfatal("Bad specifications, exiting"); 206 exit(1); 207 } 208 209 return list; 210 } 211 212 static 213 int 214 parsepkglist_file(const char *path, int debugstop) 215 { 216 FILE *fp; 217 char *base; 218 char *l1; 219 char *l2; 220 char *l3; 221 size_t len; 222 int total; 223 224 if ((fp = fopen(path, "r")) == NULL) { 225 dpanic_errno("Cannot read %s\n", path); 226 /* NOT REACHED */ 227 return 0; 228 } 229 230 total = 0; 231 232 while ((base = fgetln(fp, &len)) != NULL) { 233 if (len == 0 || base[len-1] != '\n') 234 continue; 235 base[--len] = 0; 236 l1 = strtok(base, " \t\r\n"); 237 if (l1 == NULL) { 238 printf("Badly formatted pkg info line: %s\n", base); 239 continue; 240 } 241 l2 = strchr(l1, '/'); 242 if (l2 == NULL) { 243 printf("Badly formatted specification: %s\n", l1); 244 continue; 245 } 246 *l2++ = 0; 247 l3 = strchr(l2, '@'); 248 if (l3) 249 *l3++ = 0; 250 queuebulk(l1, l2, l3, (debugstop ? "d" : NULL)); 251 ++total; 252 } 253 fclose(fp); 254 255 return total; 256 } 257 258 /* 259 * Parse packages from the list installed on the system. 260 */ 261 pkg_t * 262 GetLocalPackageList(void) 263 { 264 pkg_t *list; 265 FILE *fp; 266 char *base; 267 char *data; 268 char *l1; 269 char *l2; 270 char *l3; 271 int total; 272 int state; 273 size_t len; 274 275 PrepareSystemFlag = 1; 276 initbulk(childGetPackageInfo, MaxBulk); 277 total = 0; 278 state = 0; 279 l1 = NULL; 280 l2 = NULL; 281 l3 = NULL; 282 283 fp = popen("pkg info -a -o -A", "r"); 284 285 /* 286 * Always include ports-mgmt/pkg. s4 is "x" meaning not a manual 287 * selection, "d" meaning DEBUGSTOP mode, or NULL. 288 */ 289 queuebulk("ports-mgmt", "pkg", NULL, "x"); 290 291 while ((base = fgetln(fp, &len)) != NULL) { 292 if (len == 0 || base[len-1] != '\n') 293 continue; 294 base[--len] = 0; 295 296 data = strchr(base, ':'); 297 if (data == NULL) 298 continue; 299 *data++ = 0; 300 301 base = strtok(base, " \t\r"); 302 data = strtok(data, " \t\r"); 303 304 if (base == NULL || data == NULL) 305 continue; 306 307 if (strcmp(base, "Origin") == 0) { 308 if (state == 1) { 309 queuebulk(l1, l2, NULL, NULL); 310 state = 0; 311 ++total; 312 } 313 314 if (strchr(data, '/') == NULL) { 315 printf("Badly formatted origin: %s\n", l1); 316 } 317 if (l1) 318 free(l1); 319 if (l3) 320 free(l3); 321 l1 = strdup(data); 322 l2 = strchr(l1, '/'); 323 *l2++ = 0; 324 l3 = strchr(l2, '@'); /* typically NULL */ 325 if (l3) { 326 *l3++ = 0; 327 l3 = strdup(l3); 328 } 329 330 /* 331 * Don't queue ports-mgmt/pkg twice, we already 332 * queued it manually. 333 */ 334 if (strcmp(l1, "ports-mgmt") != 0 || 335 strcmp(l2, "pkg") != 0) { 336 state = 1; 337 } 338 continue; 339 } 340 if (state == 1 && strcmp(base, "flavor") == 0) { 341 queuebulk(l1, l2, data, NULL); 342 state = 0; 343 ++total; 344 } 345 } 346 if (state == 1) { 347 queuebulk(l1, l2, NULL, NULL); 348 /*state = 0; not needed */ 349 } 350 if (l1) 351 free(l1); 352 if (l3) 353 free(l3); 354 355 pclose(fp); 356 357 printf("Processing %d ports\n", total); 358 359 list = processPackageListBulk(total); 360 361 return list; 362 } 363 364 pkg_t * 365 GetFullPackageList(void) 366 { 367 int total; 368 369 initbulk(childGetPackageInfo, MaxBulk); 370 total = scan_and_queue_dir(DPortsPath, NULL, 1); 371 printf("Scanning %d ports\n", total); 372 373 return processPackageListBulk(total); 374 } 375 376 /* 377 * Caller has queued the process list for bulk operation. We retrieve 378 * the results and clean up the bulk operation (we may have to do a second 379 * bulk operation so we have to be the ones to clean it up). 380 */ 381 static pkg_t * 382 processPackageListBulk(int total) 383 { 384 bulk_t *bulk; 385 pkg_t *scan; 386 pkg_t *list; 387 pkg_t *dep_list; 388 pkg_t **list_tail; 389 int count; 390 int stop_fail; 391 int stop_base_list; 392 int remove_corrupt; 393 394 list = NULL; 395 list_tail = &list; 396 count = 0; 397 remove_corrupt = 0; 398 399 while ((bulk = getbulk()) != NULL) { 400 ++count; 401 if ((count & 255) == 0) { 402 printf("%6.2f%%\r", 403 (double)count * 100.0 / (double)total + 0.001); 404 fflush(stdout); 405 } 406 if (bulk->list) { 407 *list_tail = bulk->list; 408 bulk->list = NULL; 409 while ((scan = *list_tail) != NULL) { 410 if (bulk->s4 == NULL || bulk->s4[0] != 'x') 411 scan->flags |= PKGF_MANUALSEL; 412 pkg_enter(scan); 413 list_tail = &scan->bnext; 414 } 415 } 416 freebulk(bulk); 417 } 418 printf("100.00%%\n"); 419 printf("\nTotal %d\n", count); 420 fflush(stdout); 421 422 /* 423 * Resolve all dependencies for the related packages, potentially 424 * adding anything that could not be found to the list. This will 425 * continue to issue bulk operations and process the result until 426 * no dependencies are left. 427 */ 428 printf("Resolving dependencies..."); 429 fflush(stdout); 430 dep_list = list; 431 while (dep_list) { 432 dep_list = resolveDeps(dep_list, &list_tail, 0); 433 } 434 printf("done\n"); 435 436 donebulk(); 437 438 /* 439 * Generate the topology 440 */ 441 resolveDeps(list, NULL, 1); 442 443 /* 444 * Do a final count, ignore place holders. 445 * 446 * Also set stop_fail if appropriate. Check for direct specifications 447 * which fail to probe and any direct dependencies of those 448 * specifications, but don't recurse (for now)... don't check indirect 449 * dependencies (i.e. A -> B -> C where A is directly specified, B 450 * is adirect dependency, and C fails to probe). 451 */ 452 count = 0; 453 stop_fail = 0; 454 stop_base_list = 0; 455 for (scan = list; scan; scan = scan->bnext) { 456 if ((scan->flags & PKGF_ERROR) == 0) { 457 ++count; 458 } 459 if ((scan->flags & PKGF_MANUALSEL) && MaskProbeAbort == 0) { 460 pkglink_t *link; 461 462 /* 463 * Directly specified package failed to probe 464 */ 465 if (scan->flags & PKGF_CORRUPT) { 466 ++stop_fail; 467 ++stop_base_list; 468 } 469 470 /* 471 * Directly specified package had a direct dependency 472 * that failed to probe (don't go further). 473 */ 474 PKGLIST_FOREACH(link, &scan->idepon_list) { 475 if (link->pkg && 476 (link->pkg->flags & PKGF_CORRUPT)) { 477 ++stop_fail; 478 } 479 } 480 } 481 } 482 printf("Total Returned %d\n", count); 483 484 /* 485 * Check to see if any PKGF_MANUALSEL packages 486 */ 487 if (stop_fail) { 488 printf("%d packages failed to probe\n", stop_fail); 489 if (PrepareSystemFlag) { 490 if (stop_fail == stop_base_list) { 491 printf( 492 "prepare-system: Some of your installed packages no longer exist in\n" 493 "dports, do you wish to continue rebuilding what does exist?\n"); 494 if (askyn("Continue anyway? ")) 495 remove_corrupt = 1; 496 } else { 497 printf( 498 "prepare-system: Some of your installed packages have dependencies\n" 499 "which could not be found in dports, cannot continue, aborting\n"); 500 } 501 } else { 502 printf("unable to continue, aborting\n"); 503 } 504 if (remove_corrupt == 0) 505 exit(1); 506 } 507 508 /* 509 * Remove corrupt packages before continuing 510 */ 511 if (remove_corrupt) { 512 list_tail = &list; 513 while ((scan = *list_tail) != NULL) { 514 if (scan->flags & PKGF_CORRUPT) 515 *list_tail = scan->bnext; 516 else 517 list_tail = &scan->bnext; 518 } 519 } 520 521 /* 522 * Scan our binary distributions and related dependencies looking 523 * for any packages that have already been built. 524 */ 525 initbulk(childGetBinaryDistInfo, MaxBulk); 526 total = scan_binary_repo(RepositoryPath); 527 count = 0; 528 printf("Scanning %d packages\n", total); 529 530 while ((bulk = getbulk()) != NULL) { 531 ++count; 532 if ((count & 255) == 0) { 533 printf("%6.2f%%\r", 534 (double)count * 100.0 / (double)total + 0.001); 535 fflush(stdout); 536 } 537 freebulk(bulk); 538 } 539 printf("100.00%%\n"); 540 printf("\nTotal %d\n", count); 541 fflush(stdout); 542 donebulk(); 543 544 printf("all done\n"); 545 546 return list; 547 } 548 549 pkg_t * 550 GetPkgPkg(pkg_t **listp) 551 { 552 bulk_t *bulk; 553 pkg_t *scan; 554 pkg_t *s2; 555 556 for (scan = *listp; scan; scan = scan->bnext) { 557 if (strcmp(scan->portdir, "ports-mgmt/pkg") == 0) 558 return scan; 559 } 560 561 /* 562 * This will force pkg to be built, but generally this code 563 * is not reached because the package list processing code 564 * adds ports-mgmt/pkg unconditionally. 565 */ 566 initbulk(childGetPackageInfo, MaxBulk); 567 queuebulk("ports-mgmt", "pkg", NULL, "x"); 568 bulk = getbulk(); 569 dassert(bulk, "Cannot find ports-mgmt/pkg"); 570 scan = bulk->list; 571 bulk->list = NULL; 572 freebulk(bulk); 573 donebulk(); 574 575 576 /* 577 * Include added packages to the total and add the initial bulk 578 * built packages to the list so they get counted. 579 */ 580 for (s2 = scan; s2->bnext; s2 = s2->bnext) 581 ++BuildTotal; 582 for (s2 = scan; s2->bnext; s2 = s2->bnext) 583 ; 584 s2->bnext = *listp; 585 *listp = scan; 586 ++BuildTotal; 587 588 return scan; 589 } 590 591 /* 592 * Try to optimize the environment by supplying information that 593 * the ports system would generally have to run stuff to get on 594 * every package. 595 * 596 * See childOptimizeEnv() for the actual handling. We execute 597 * a single make -V... -V... for ports-mgmt/pkg from within the 598 * bulk system (which handles the environment and disables 599 * /etc/make.conf), and we then call addbuildenv() as appropriate. 600 * 601 * _PERL5_FROM_BIN 602 * add others... 603 */ 604 void 605 OptimizeEnv(void) 606 { 607 bulk_t *bulk; 608 609 initbulk(childOptimizeEnv, MaxBulk); 610 queuebulk("ports-mgmt", "pkg", NULL, NULL); 611 bulk = getbulk(); 612 freebulk(bulk); 613 donebulk(); 614 } 615 616 /* 617 * Run through the list resolving dependencies and constructing the topology 618 * linkages. This may append packages to the list. Dependencies to dummy 619 * nodes which do not specify a flavor do not need special handling, the 620 * search code in build.c will properly follow the first flavor. 621 */ 622 static pkg_t * 623 resolveDeps(pkg_t *list, pkg_t ***list_tailp, int gentopo) 624 { 625 pkg_t *ret_list = NULL; 626 pkg_t *scan; 627 pkg_t *use; 628 bulk_t *bulk; 629 630 for (scan = list; scan; scan = scan->bnext) { 631 use = pkg_find(scan->portdir); 632 resolveFlavors(use, scan->flavors, gentopo); 633 resolveDepString(use, scan->fetch_deps, 634 gentopo, DEP_TYPE_FETCH); 635 resolveDepString(use, scan->ext_deps, 636 gentopo, DEP_TYPE_EXT); 637 resolveDepString(use, scan->patch_deps, 638 gentopo, DEP_TYPE_PATCH); 639 resolveDepString(use, scan->build_deps, 640 gentopo, DEP_TYPE_BUILD); 641 resolveDepString(use, scan->lib_deps, 642 gentopo, DEP_TYPE_LIB); 643 resolveDepString(use, scan->run_deps, 644 gentopo, DEP_TYPE_RUN); 645 } 646 647 /* 648 * No bulk ops are queued when doing the final topology 649 * generation. 650 * 651 * Avoid entering duplicate results from the bulk ops. Duplicate 652 * results are mostly filtered out, but not always. A dummy node 653 * representing multiple flavors will parse-out the flavors 654 */ 655 if (gentopo) 656 return NULL; 657 while ((bulk = getbulk()) != NULL) { 658 if (bulk->list) { 659 if (ret_list == NULL) 660 ret_list = bulk->list; 661 **list_tailp = bulk->list; 662 bulk->list = NULL; 663 while ((scan = **list_tailp) != NULL) { 664 pkg_enter(scan); 665 *list_tailp = &scan->bnext; 666 } 667 } 668 freebulk(bulk); 669 } 670 return (ret_list); 671 } 672 673 /* 674 * Resolve a generic node that has flavors, queue to retrieve info for 675 * each flavor and setup linkages as appropriate. 676 */ 677 static void 678 resolveFlavors(pkg_t *pkg, char *flavors, int gentopo) 679 { 680 char *flavor_base; 681 char *flavor_scan; 682 char *flavor; 683 char *portdir; 684 char *s1; 685 char *s2; 686 pkg_t *dpkg; 687 pkglink_t *link; 688 689 if ((pkg->flags & PKGF_DUMMY) == 0) 690 return; 691 if (pkg->flavors == NULL || pkg->flavors[0] == 0) 692 return; 693 flavor_base = strdup(flavors); 694 flavor_scan = flavor_base; 695 696 for (;;) { 697 do { 698 flavor = strsep(&flavor_scan, " \t"); 699 } while (flavor && *flavor == 0); 700 if (flavor == NULL) 701 break; 702 703 /* 704 * Iterate each flavor generating "s1/s2@flavor". 705 * 706 * queuebulk() info for each flavor, and set-up the 707 * linkages in the topology generation pass. 708 */ 709 asprintf(&portdir, "%s@%s", pkg->portdir, flavor); 710 s1 = strdup(pkg->portdir); 711 s2 = strchr(s1, '/'); 712 *s2++ = 0; 713 714 dpkg = pkg_find(portdir); 715 if (dpkg && gentopo) { 716 /* 717 * Setup linkages 718 */ 719 free(portdir); 720 721 link = calloc(1, sizeof(*link)); 722 link->pkg = dpkg; 723 link->next = &pkg->idepon_list; 724 link->prev = pkg->idepon_list.prev; 725 link->next->prev = link; 726 link->prev->next = link; 727 link->dep_type = DEP_TYPE_BUILD; 728 729 link = calloc(1, sizeof(*link)); 730 link->pkg = pkg; 731 link->next = &dpkg->deponi_list; 732 link->prev = dpkg->deponi_list.prev; 733 link->next->prev = link; 734 link->prev->next = link; 735 link->dep_type = DEP_TYPE_BUILD; 736 ++dpkg->depi_count; 737 } else if (gentopo == 0 && dpkg == NULL) { 738 /* 739 * Use a place-holder to prevent duplicate 740 * dependencies from being processed. The placeholder 741 * will be replaced by the actual dependency. 742 */ 743 dpkg = allocpkg(); 744 dpkg->portdir = portdir; 745 dpkg->flags = PKGF_PLACEHOLD; 746 pkg_enter(dpkg); 747 queuebulk(s1, s2, flavor, NULL); 748 } 749 free(s1); 750 } 751 free(flavor_base); 752 } 753 754 static void 755 resolveDepString(pkg_t *pkg, char *depstr, int gentopo, int dep_type) 756 { 757 char *copy_base; 758 char *copy; 759 char *dep; 760 char *sep; 761 char *tag; 762 char *flavor; 763 pkg_t *dpkg; 764 765 if (depstr == NULL || depstr[0] == 0) 766 return; 767 768 copy_base = strdup(depstr); 769 copy = copy_base; 770 771 for (;;) { 772 do { 773 dep = strsep(©, " \t"); 774 } while (dep && *dep == 0); 775 if (dep == NULL) 776 break; 777 778 /* 779 * Ignore dependencies prefixed with ${NONEXISTENT} 780 */ 781 if (strncmp(dep, "/nonexistent:", 13) == 0) 782 continue; 783 784 dep = strchr(dep, ':'); 785 if (dep == NULL || *dep != ':') { 786 printf("Error parsing dependency for %s: %s\n", 787 pkg->portdir, copy_base); 788 continue; 789 } 790 ++dep; 791 792 /* 793 * Strip-off any DPortsPath prefix. EXTRACT_DEPENDS 794 * often (always?) generates this prefix. 795 */ 796 if (strncmp(dep, DPortsPath, strlen(DPortsPath)) == 0) { 797 dep += strlen(DPortsPath); 798 if (*dep == '/') 799 ++dep; 800 } 801 802 /* 803 * Strip-off any tag (such as :patch). We don't try to 804 * organize dependencies at this fine a grain (for now). 805 */ 806 tag = strchr(dep, ':'); 807 if (tag) 808 *tag++ = 0; 809 810 /* 811 * Locate the dependency 812 */ 813 if ((dpkg = pkg_find(dep)) != NULL) { 814 if (gentopo) { 815 pkglink_t *link; 816 817 /* 818 * NOTE: idep_count is calculated recursively 819 * at build-time 820 */ 821 ddprintf(0, "Add Dependency %s -> %s\n", 822 pkg->portdir, dpkg->portdir); 823 link = calloc(1, sizeof(*link)); 824 link->pkg = dpkg; 825 link->next = &pkg->idepon_list; 826 link->prev = pkg->idepon_list.prev; 827 link->next->prev = link; 828 link->prev->next = link; 829 link->dep_type = dep_type; 830 831 link = calloc(1, sizeof(*link)); 832 link->pkg = pkg; 833 link->next = &dpkg->deponi_list; 834 link->prev = dpkg->deponi_list.prev; 835 link->next->prev = link; 836 link->prev->next = link; 837 link->dep_type = dep_type; 838 ++dpkg->depi_count; 839 } 840 continue; 841 } 842 843 /* 844 * This shouldn't happen because we already took a first 845 * pass and should have generated the pkgs. 846 */ 847 if (gentopo) { 848 printf("Topology Generate failed for %s: %s\n", 849 pkg->portdir, copy_base); 850 continue; 851 } 852 853 /* 854 * Separate out the two dports directory components and 855 * extract the optional '@flavor' specification. 856 */ 857 sep = strchr(dep, '/'); 858 if (sep == NULL) { 859 printf("Error parsing dependency for %s: %s\n", 860 pkg->portdir, copy_base); 861 continue; 862 } 863 *sep++ = 0; 864 865 /* 866 * The flavor hangs off the separator, not the tag 867 */ 868 flavor = strrchr(sep, '@'); 869 #if 0 870 if (tag) 871 flavor = strrchr(tag, '@'); 872 else 873 flavor = strrchr(sep, '@'); 874 #endif 875 if (flavor) 876 *flavor++ = 0; 877 878 if (flavor) 879 ddprintf(0, "QUEUE DEPENDENCY FROM PKG %s: %s/%s@%s\n", 880 pkg->portdir, dep, sep, flavor); 881 else 882 ddprintf(0, "QUEUE DEPENDENCY FROM PKG %s: %s/%s\n", 883 pkg->portdir, dep, sep); 884 885 /* 886 * Use a place-holder to prevent duplicate dependencies from 887 * being processed. The placeholder will be replaced by 888 * the actual dependency. 889 */ 890 dpkg = allocpkg(); 891 if (flavor) 892 asprintf(&dpkg->portdir, "%s/%s@%s", dep, sep, flavor); 893 else 894 asprintf(&dpkg->portdir, "%s/%s", dep, sep); 895 dpkg->flags = PKGF_PLACEHOLD; 896 pkg_enter(dpkg); 897 898 queuebulk(dep, sep, flavor, NULL); 899 } 900 free(copy_base); 901 } 902 903 void 904 FreePackageList(pkg_t *pkgs __unused) 905 { 906 dfatal("not implemented"); 907 } 908 909 /* 910 * Scan some or all dports to allocate the related pkg structure. Dependencies 911 * are stored but not processed. 912 * 913 * Threaded function 914 */ 915 static void 916 childGetPackageInfo(bulk_t *bulk) 917 { 918 pkg_t *pkg; 919 char *flavor; 920 char *ptr; 921 FILE *fp; 922 int line; 923 size_t len; 924 char *portpath; 925 char *flavarg; 926 const char *cav[MAXCAC]; 927 pid_t pid; 928 int cac; 929 930 /* 931 * If the package has flavors we will loop on each one. If a flavor 932 * is not passed in s3 we will loop on all flavors, otherwise we will 933 * only process the passed-in flavor. 934 */ 935 flavor = bulk->s3; /* usually NULL */ 936 937 bulk->list = NULL; 938 939 asprintf(&portpath, "%s/%s/%s", DPortsPath, bulk->s1, bulk->s2); 940 if (flavor) 941 asprintf(&flavarg, "FLAVOR=%s", flavor); 942 else 943 flavarg = NULL; 944 945 cac = 0; 946 cav[cac++] = MAKE_BINARY; 947 cav[cac++] = "-C"; 948 cav[cac++] = portpath; 949 if (flavarg) 950 cav[cac++] = flavarg; 951 cav[cac++] = "-VPKGVERSION"; 952 cav[cac++] = "-VPKGFILE:T"; 953 cav[cac++] = "-VALLFILES"; 954 cav[cac++] = "-VDIST_SUBDIR"; 955 cav[cac++] = "-VMAKE_JOBS_NUMBER"; 956 cav[cac++] = "-VIGNORE"; 957 cav[cac++] = "-VFETCH_DEPENDS"; 958 cav[cac++] = "-VEXTRACT_DEPENDS"; 959 cav[cac++] = "-VPATCH_DEPENDS"; 960 cav[cac++] = "-VBUILD_DEPENDS"; 961 cav[cac++] = "-VLIB_DEPENDS"; 962 cav[cac++] = "-VRUN_DEPENDS"; 963 cav[cac++] = "-VSELECTED_OPTIONS"; 964 cav[cac++] = "-VDESELECTED_OPTIONS"; 965 cav[cac++] = "-VUSE_LINUX"; 966 cav[cac++] = "-VFLAVORS"; 967 cav[cac++] = "-VUSES"; 968 969 fp = dexec_open(portpath + strlen(DPortsPath) + 1, cav, cac, 970 &pid, NULL, 1, 1); 971 freestrp(&flavarg); 972 973 pkg = allocpkg(); 974 if (flavor) 975 asprintf(&pkg->portdir, "%s/%s@%s", bulk->s1, bulk->s2, flavor); 976 else 977 asprintf(&pkg->portdir, "%s/%s", bulk->s1, bulk->s2); 978 979 line = 1; 980 while ((ptr = fgetln(fp, &len)) != NULL) { 981 if (len == 0 || ptr[len-1] != '\n') { 982 dfatal("Bad package info for %s/%s response line %d", 983 bulk->s1, bulk->s2, line); 984 } 985 ptr[--len] = 0; 986 987 switch(line) { 988 case 1: /* PKGVERSION */ 989 asprintf(&pkg->version, "%s", ptr); 990 break; 991 case 2: /* PKGFILE */ 992 asprintf(&pkg->pkgfile, "%s", ptr); 993 break; 994 case 3: /* ALLFILES (aka DISTFILES + patch files) */ 995 asprintf(&pkg->distfiles, "%s", ptr); 996 break; 997 case 4: /* DIST_SUBDIR */ 998 pkg->distsubdir = strdup_or_null(ptr); 999 break; 1000 case 5: /* MAKE_JOBS_NUMBER */ 1001 pkg->make_jobs_number = strtol(ptr, NULL, 0); 1002 break; 1003 case 6: /* IGNORE */ 1004 pkg->ignore = strdup_or_null(ptr); 1005 break; 1006 case 7: /* FETCH_DEPENDS */ 1007 pkg->fetch_deps = strdup_or_null(ptr); 1008 break; 1009 case 8: /* EXTRACT_DEPENDS */ 1010 pkg->ext_deps = strdup_or_null(ptr); 1011 break; 1012 case 9: /* PATCH_DEPENDS */ 1013 pkg->patch_deps = strdup_or_null(ptr); 1014 break; 1015 case 10: /* BUILD_DEPENDS */ 1016 pkg->build_deps = strdup_or_null(ptr); 1017 break; 1018 case 11: /* LIB_DEPENDS */ 1019 pkg->lib_deps = strdup_or_null(ptr); 1020 break; 1021 case 12: /* RUN_DEPENDS */ 1022 pkg->run_deps = strdup_or_null(ptr); 1023 break; 1024 case 13: /* SELECTED_OPTIONS */ 1025 pkg->pos_options = strdup_or_null(ptr); 1026 break; 1027 case 14: /* DESELECTED_OPTIONS */ 1028 pkg->neg_options = strdup_or_null(ptr); 1029 break; 1030 case 15: /* USE_LINUX */ 1031 if (ptr[0]) 1032 pkg->use_linux = 1; 1033 break; 1034 case 16: /* FLAVORS */ 1035 asprintf(&pkg->flavors, "%s", ptr); 1036 break; 1037 case 17: /* USES */ 1038 asprintf(&pkg->uses, "%s", ptr); 1039 if (strstr(pkg->uses, "metaport")) 1040 pkg->flags |= PKGF_META; 1041 break; 1042 default: 1043 printf("EXTRA LINE: %s\n", ptr); 1044 break; 1045 } 1046 ++line; 1047 } 1048 if (line == 1) { 1049 printf("DPort not found: %s/%s\n", bulk->s1, bulk->s2); 1050 pkg->flags |= PKGF_NOTFOUND; 1051 } else if (line != 17 + 1) { 1052 printf("DPort corrupt: %s/%s\n", bulk->s1, bulk->s2); 1053 pkg->flags |= PKGF_CORRUPT; 1054 } 1055 if (dexec_close(fp, pid)) { 1056 printf("make -V* command for %s/%s failed\n", 1057 bulk->s1, bulk->s2); 1058 pkg->flags |= PKGF_CORRUPT; 1059 } 1060 ddassert(bulk->s1); 1061 1062 /* 1063 * DEBUGSTOP mode 1064 */ 1065 if (bulk->s4 && bulk->s4[0] == 'd') 1066 pkg->flags |= PKGF_DEBUGSTOP; 1067 1068 /* 1069 * Mark as a dummy node, the front-end will iterate the flavors 1070 * and create sub-nodes for us. 1071 * 1072 * Get rid of elements returned that are for the first flavor. 1073 * We are creating a dummy node here, not the node for the first 1074 * flavor. 1075 */ 1076 if (flavor == NULL && pkg->flavors && pkg->flavors[0]) { 1077 pkg->flags |= PKGF_DUMMY; 1078 freestrp(&pkg->fetch_deps); 1079 freestrp(&pkg->ext_deps); 1080 freestrp(&pkg->patch_deps); 1081 freestrp(&pkg->build_deps); 1082 freestrp(&pkg->lib_deps); 1083 freestrp(&pkg->run_deps); 1084 freestrp(&pkg->pkgfile); 1085 } 1086 1087 /* 1088 * Checksum the port directory tree. This just rollsup crcs of the 1089 * path names and a few stat fields (mtime, size) in order to detect 1090 * if any modification has been made to the port. 1091 */ 1092 pkg->crc32 = crcDirTree(portpath); 1093 1094 /* 1095 * Only one pkg is put on the return list now. This code no 1096 * longer creates pseudo-nodes for flavors (the frontend requests 1097 * each flavor instead). 1098 */ 1099 bulk->list = pkg; 1100 free(portpath); 1101 } 1102 1103 /* 1104 * Query the package (at least to make sure it hasn't been truncated) 1105 * and mark it as PACKAGED if found. 1106 * 1107 * Threaded function 1108 */ 1109 static void 1110 childGetBinaryDistInfo(bulk_t *bulk) 1111 { 1112 char *ptr; 1113 FILE *fp; 1114 size_t len; 1115 pkg_t *pkg; 1116 const char *cav[MAXCAC]; 1117 char *repopath; 1118 char buf[1024]; 1119 pid_t pid; 1120 int cac; 1121 int deleteme; 1122 1123 asprintf(&repopath, "%s/%s", RepositoryPath, bulk->s1); 1124 1125 cac = 0; 1126 cav[cac++] = PKG_BINARY; 1127 cav[cac++] = "query"; 1128 cav[cac++] = "-F"; 1129 cav[cac++] = repopath; 1130 cav[cac++] = "%n-%v"; 1131 1132 fp = dexec_open(NULL, cav, cac, &pid, NULL, 1, 0); 1133 deleteme = DeleteObsoletePkgs; 1134 1135 while ((ptr = fgetln(fp, &len)) != NULL) { 1136 if (len == 0 || ptr[len-1] != '\n') 1137 continue; 1138 ptr[len-1] = 0; 1139 snprintf(buf, sizeof(buf), "%s%s", ptr, UsePkgSufx); 1140 1141 pkg = pkg_find(buf); 1142 if (pkg) { 1143 pkg->flags |= PKGF_PACKAGED; 1144 deleteme = 0; 1145 } else { 1146 ddprintf(0, "Note: package scan, not in list, " 1147 "skipping %s\n", buf); 1148 } 1149 } 1150 if (dexec_close(fp, pid)) { 1151 printf("pkg query command failed for %s\n", repopath); 1152 } 1153 if (deleteme) { 1154 dlog(DLOG_ALL | DLOG_STDOUT, 1155 "Deleting obsolete package %s\n", repopath); 1156 remove(repopath); 1157 } 1158 free(repopath); 1159 } 1160 1161 static void 1162 childOptimizeEnv(bulk_t *bulk) 1163 { 1164 char *portpath; 1165 char *ptr; 1166 FILE *fp; 1167 int line; 1168 size_t len; 1169 const char *cav[MAXCAC]; 1170 pid_t pid; 1171 int cac; 1172 1173 asprintf(&portpath, "%s/%s/%s", DPortsPath, bulk->s1, bulk->s2); 1174 1175 cac = 0; 1176 cav[cac++] = MAKE_BINARY; 1177 cav[cac++] = "-C"; 1178 cav[cac++] = portpath; 1179 cav[cac++] = "-V_PERL5_FROM_BIN"; 1180 1181 fp = dexec_open(portpath + strlen(DPortsPath) + 1, cav, cac, 1182 &pid, NULL, 1, 1); 1183 free(portpath); 1184 1185 line = 1; 1186 while ((ptr = fgetln(fp, &len)) != NULL) { 1187 if (len == 0 || ptr[len-1] != '\n') { 1188 dfatal("Bad package info for %s/%s response line %d", 1189 bulk->s1, bulk->s2, line); 1190 } 1191 ptr[--len] = 0; 1192 1193 switch(line) { 1194 case 1: /* _PERL5_FROM_BIN */ 1195 addbuildenv("_PERL5_FROM_BIN", ptr, BENV_ENVIRONMENT); 1196 break; 1197 default: 1198 printf("childOptimizeEnv: EXTRA LINE: %s\n", ptr); 1199 break; 1200 } 1201 ++line; 1202 } 1203 if (line == 1) { 1204 printf("DPort not found: %s/%s\n", bulk->s1, bulk->s2); 1205 } else if (line != 1 + 1) { 1206 printf("DPort corrupt: %s/%s\n", bulk->s1, bulk->s2); 1207 } 1208 if (dexec_close(fp, pid)) { 1209 printf("childOptimizeEnv() failed\n"); 1210 } 1211 } 1212 1213 static int 1214 scan_and_queue_dir(const char *path, const char *level1, int level) 1215 { 1216 DIR *dir; 1217 char *s1; 1218 char *s2; 1219 struct dirent *den; 1220 struct stat st; 1221 int count = 0; 1222 1223 dir = opendir(path); 1224 dassert(dir, "Cannot open dports path \"%s\"", path); 1225 1226 while ((den = readdir(dir)) != NULL) { 1227 if (den->d_namlen == 1 && den->d_name[0] == '.') 1228 continue; 1229 if (den->d_namlen == 2 && 1230 den->d_name[0] == '.' && den->d_name[1] == '.') 1231 continue; 1232 asprintf(&s1, "%s/%s", path, den->d_name); 1233 if (lstat(s1, &st) < 0 || !S_ISDIR(st.st_mode)) { 1234 free(s1); 1235 continue; 1236 } 1237 if (level == 1) { 1238 count += scan_and_queue_dir(s1, den->d_name, 2); 1239 free(s1); 1240 continue; 1241 } 1242 asprintf(&s2, "%s/Makefile", s1); 1243 if (lstat(s2, &st) == 0) { 1244 queuebulk(level1, den->d_name, NULL, NULL); 1245 ++count; 1246 } 1247 free(s1); 1248 free(s2); 1249 } 1250 closedir(dir); 1251 1252 return count; 1253 } 1254 1255 static int 1256 scan_binary_repo(const char *path) 1257 { 1258 DIR *dir; 1259 struct dirent *den; 1260 int count; 1261 1262 count = 0; 1263 dir = opendir(path); 1264 dassert(dir, "Cannot open repository path \"%s\"", path); 1265 1266 /* 1267 * NOTE: Test includes the '.' in the suffix. 1268 */ 1269 while ((den = readdir(dir)) != NULL) { 1270 const char *suffix; 1271 1272 suffix = strrchr(den->d_name, '.'); 1273 if (suffix && suffix != den->d_name && 1274 strcmp(suffix, UsePkgSufx) == 0) { 1275 queuebulk(den->d_name, NULL, NULL, NULL); 1276 ++count; 1277 } 1278 } 1279 closedir(dir); 1280 1281 return count; 1282 } 1283 1284 #if 0 1285 static void 1286 pkgfree(pkg_t *pkg) 1287 { 1288 freestrp(&pkg->portdir); 1289 freestrp(&pkg->version); 1290 freestrp(&pkg->pkgfile); 1291 freestrp(&pkg->ignore); 1292 freestrp(&pkg->fetch_deps); 1293 freestrp(&pkg->ext_deps); 1294 freestrp(&pkg->patch_deps); 1295 freestrp(&pkg->build_deps); 1296 freestrp(&pkg->lib_deps); 1297 freestrp(&pkg->run_deps); 1298 freestrp(&pkg->pos_options); 1299 freestrp(&pkg->neg_options); 1300 freestrp(&pkg->flavors); 1301 free(pkg); 1302 } 1303 #endif 1304