1 /* 2 * Copyright (c) 2019 The DragonFly Project. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Matthew Dillon <dillon@backplane.com> 6 * 7 * This code uses concepts and configuration based on 'synth', by 8 * John R. Marino <draco@marino.st>, which was written in ada. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in 18 * the documentation and/or other materials provided with the 19 * distribution. 20 * 3. Neither the name of The DragonFly Project nor the names of its 21 * contributors may be used to endorse or promote products derived 22 * from this software without specific, prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 25 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 26 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 27 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 28 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 29 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 30 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 31 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 32 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 33 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 34 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 35 * SUCH DAMAGE. 36 */ 37 38 #include "dsynth.h" 39 40 #define PKG_HSIZE 32768 41 #define PKG_HMASK 32767 42 43 static int parsepkglist_file(const char *path, int debugstop); 44 static void childGetPackageInfo(bulk_t *bulk); 45 static void childGetBinaryDistInfo(bulk_t *bulk); 46 static void childOptimizeEnv(bulk_t *bulk); 47 static pkg_t *resolveDeps(pkg_t *dep_list, pkg_t ***list_tailp, int gentopo); 48 static void resolveFlavors(pkg_t *pkg, char *flavors, int gentopo); 49 static void resolveDepString(pkg_t *pkg, char *depstr, 50 int gentopo, int dep_type); 51 static pkg_t *processPackageListBulk(int total); 52 static int scan_and_queue_dir(const char *path, const char *level1, int level); 53 static int scan_binary_repo(const char *path); 54 #if 0 55 static void pkgfree(pkg_t *pkg); 56 #endif 57 58 static int PrepareSystemFlag; 59 60 pkg_t *PkgHash1[PKG_HSIZE]; /* by portdir */ 61 pkg_t *PkgHash2[PKG_HSIZE]; /* by pkgfile */ 62 63 /* 64 * Allocate a new pkg structure plus basic initialization. 65 */ 66 static __inline pkg_t * 67 allocpkg(void) 68 { 69 pkg_t *pkg; 70 71 pkg = calloc(1, sizeof(*pkg)); 72 pkg->idepon_list.next = &pkg->idepon_list; 73 pkg->idepon_list.prev = &pkg->idepon_list; 74 pkg->deponi_list.next = &pkg->deponi_list; 75 pkg->deponi_list.prev = &pkg->deponi_list; 76 77 return pkg; 78 } 79 80 /* 81 * Simple hash for lookups 82 */ 83 static __inline int 84 pkghash(const char *str) 85 { 86 int hv = 0xABC32923; 87 while (*str) { 88 hv = (hv << 5) ^ *str; 89 ++str; 90 } 91 hv = hv ^ (hv / PKG_HSIZE) ^ (hv / PKG_HSIZE / PKG_HSIZE); 92 return (hv & PKG_HMASK); 93 } 94 95 static void 96 pkg_enter(pkg_t *pkg) 97 { 98 pkg_t **pkgp; 99 pkg_t *scan; 100 101 if (pkg->portdir) { 102 pkgp = &PkgHash1[pkghash(pkg->portdir)]; 103 while ((scan = *pkgp) != NULL) { 104 if (strcmp(pkg->portdir, scan->portdir) == 0) 105 break; 106 pkgp = &scan->hnext1; 107 } 108 ddassert(scan == NULL || (scan->flags & PKGF_PLACEHOLD)); 109 if (scan && (scan->flags & PKGF_PLACEHOLD)) { 110 ddassert(scan->idepon_list.next == &scan->idepon_list); 111 ddassert(scan->deponi_list.next == &scan->deponi_list); 112 *pkgp = pkg; 113 pkg->hnext1 = scan->hnext1; 114 free(scan->portdir); 115 free(scan); 116 scan = NULL; 117 } 118 if (scan == NULL) 119 *pkgp = pkg; 120 } 121 122 if (pkg->pkgfile) { 123 pkgp = &PkgHash2[pkghash(pkg->pkgfile)]; 124 while ((scan = *pkgp) != NULL) { 125 if (strcmp(pkg->pkgfile, scan->pkgfile) == 0) 126 break; 127 pkgp = &scan->hnext2; 128 } 129 if (scan == NULL) 130 *pkgp = pkg; 131 } 132 } 133 134 static pkg_t * 135 pkg_find(const char *match) 136 { 137 pkg_t **pkgp; 138 pkg_t *pkg; 139 140 pkgp = &PkgHash1[pkghash(match)]; 141 for (pkg = *pkgp; pkg; pkg = pkg->hnext1) { 142 if (strcmp(pkg->portdir, match) == 0) 143 return pkg; 144 } 145 pkgp = &PkgHash2[pkghash(match)]; 146 for (pkg = *pkgp; pkg; pkg = pkg->hnext2) { 147 if (strcmp(pkg->pkgfile, match) == 0) 148 return pkg; 149 } 150 return NULL; 151 } 152 153 /* 154 * Parse a specific list of ports via origin name (portdir/subdir) 155 */ 156 pkg_t * 157 ParsePackageList(int n, char **ary, int debugstop) 158 { 159 pkg_t *list; 160 int total; 161 int fail; 162 int i; 163 164 total = 0; 165 fail = 0; 166 initbulk(childGetPackageInfo, MaxBulk); 167 168 /* 169 * Always include ports-mgmt/pkg. s4 is "x" meaning not a manual 170 * selection, "d" meaning DEBUGSTOP mode, or NULL. 171 */ 172 queuebulk("ports-mgmt", "pkg", NULL, "x"); 173 174 for (i = 0; i < n; ++i) { 175 char *l1; 176 char *l2; 177 char *l3; 178 struct stat st; 179 180 l1 = strdup(ary[i]); 181 if (stat(l1, &st) == 0 && S_ISREG(st.st_mode)) { 182 total += parsepkglist_file(l1, debugstop); 183 continue; 184 } 185 186 l2 = strchr(l1, '/'); 187 if (l2 == NULL) { 188 printf("Bad portdir specification: %s\n", l1); 189 free(l1); 190 fail = 1; 191 continue; 192 } 193 *l2++ = 0; 194 l3 = strchr(l2, '@'); 195 if (l3) 196 *l3++ = 0; 197 queuebulk(l1, l2, l3, (debugstop ? "d" : NULL)); 198 ++total; 199 free(l1); 200 } 201 printf("Processing %d ports\n", total); 202 203 list = processPackageListBulk(total); 204 if (fail) { 205 dfatal("Bad specifications, exiting"); 206 exit(1); 207 } 208 209 return list; 210 } 211 212 static 213 int 214 parsepkglist_file(const char *path, int debugstop) 215 { 216 FILE *fp; 217 char *base; 218 char *l1; 219 char *l2; 220 char *l3; 221 size_t len; 222 int total; 223 224 if ((fp = fopen(path, "r")) == NULL) { 225 dpanic_errno("Cannot read %s\n", path); 226 /* NOT REACHED */ 227 return 0; 228 } 229 230 total = 0; 231 232 while ((base = fgetln(fp, &len)) != NULL) { 233 if (len == 0 || base[len-1] != '\n') 234 continue; 235 base[--len] = 0; 236 l1 = strtok(base, " \t\r\n"); 237 if (l1 == NULL) { 238 printf("Badly formatted pkg info line: %s\n", base); 239 continue; 240 } 241 l2 = strchr(l1, '/'); 242 if (l2 == NULL) { 243 printf("Badly formatted specification: %s\n", l1); 244 continue; 245 } 246 *l2++ = 0; 247 l3 = strchr(l2, '@'); 248 if (l3) 249 *l3++ = 0; 250 queuebulk(l1, l2, l3, (debugstop ? "d" : NULL)); 251 ++total; 252 } 253 fclose(fp); 254 255 return total; 256 } 257 258 /* 259 * Parse packages from the list installed on the system. 260 */ 261 pkg_t * 262 GetLocalPackageList(void) 263 { 264 pkg_t *list; 265 FILE *fp; 266 char *base; 267 char *data; 268 char *l1; 269 char *l2; 270 char *l3; 271 int total; 272 int state; 273 size_t len; 274 275 PrepareSystemFlag = 1; 276 initbulk(childGetPackageInfo, MaxBulk); 277 total = 0; 278 state = 0; 279 l1 = NULL; 280 l2 = NULL; 281 l3 = NULL; 282 283 fp = popen("pkg info -a -o -A", "r"); 284 285 /* 286 * Always include ports-mgmt/pkg. s4 is "x" meaning not a manual 287 * selection, "d" meaning DEBUGSTOP mode, or NULL. 288 */ 289 queuebulk("ports-mgmt", "pkg", NULL, "x"); 290 291 while ((base = fgetln(fp, &len)) != NULL) { 292 if (len == 0 || base[len-1] != '\n') 293 continue; 294 base[--len] = 0; 295 296 data = strchr(base, ':'); 297 if (data == NULL) 298 continue; 299 *data++ = 0; 300 301 base = strtok(base, " \t\r"); 302 data = strtok(data, " \t\r"); 303 304 if (base == NULL || data == NULL) 305 continue; 306 307 if (strcmp(base, "Origin") == 0) { 308 if (state == 1) { 309 queuebulk(l1, l2, NULL, NULL); 310 state = 0; 311 ++total; 312 } 313 314 if (strchr(data, '/') == NULL) { 315 printf("Badly formatted origin: %s\n", l1); 316 } 317 if (l1) 318 free(l1); 319 if (l3) 320 free(l3); 321 l1 = strdup(data); 322 l2 = strchr(l1, '/'); 323 *l2++ = 0; 324 l3 = strchr(l2, '@'); /* typically NULL */ 325 if (l3) { 326 *l3++ = 0; 327 l3 = strdup(l3); 328 } 329 330 /* 331 * Don't queue ports-mgmt/pkg twice, we already 332 * queued it manually. 333 */ 334 if (strcmp(l1, "ports-mgmt") != 0 || 335 strcmp(l2, "pkg") != 0) { 336 state = 1; 337 } 338 continue; 339 } 340 if (state == 1 && strcmp(base, "flavor") == 0) { 341 queuebulk(l1, l2, data, NULL); 342 state = 0; 343 ++total; 344 } 345 } 346 if (state == 1) { 347 queuebulk(l1, l2, NULL, NULL); 348 /*state = 0; not needed */ 349 } 350 if (l1) 351 free(l1); 352 if (l3) 353 free(l3); 354 355 pclose(fp); 356 357 printf("Processing %d ports\n", total); 358 359 list = processPackageListBulk(total); 360 361 return list; 362 } 363 364 pkg_t * 365 GetFullPackageList(void) 366 { 367 int total; 368 369 initbulk(childGetPackageInfo, MaxBulk); 370 total = scan_and_queue_dir(DPortsPath, NULL, 1); 371 printf("Scanning %d ports\n", total); 372 373 return processPackageListBulk(total); 374 } 375 376 /* 377 * Caller has queued the process list for bulk operation. We retrieve 378 * the results and clean up the bulk operation (we may have to do a second 379 * bulk operation so we have to be the ones to clean it up). 380 */ 381 static pkg_t * 382 processPackageListBulk(int total) 383 { 384 bulk_t *bulk; 385 pkg_t *scan; 386 pkg_t *list; 387 pkg_t *dep_list; 388 pkg_t **list_tail; 389 int count; 390 int stop_fail; 391 int stop_base_list; 392 int remove_corrupt; 393 394 list = NULL; 395 list_tail = &list; 396 count = 0; 397 remove_corrupt = 0; 398 399 while ((bulk = getbulk()) != NULL) { 400 ++count; 401 if ((count & 255) == 0) { 402 printf("%6.2f%%\r", 403 (double)count * 100.0 / (double)total + 0.001); 404 fflush(stdout); 405 } 406 if (bulk->list) { 407 *list_tail = bulk->list; 408 bulk->list = NULL; 409 while ((scan = *list_tail) != NULL) { 410 if (bulk->s4 == NULL || bulk->s4[0] != 'x') 411 scan->flags |= PKGF_MANUALSEL; 412 pkg_enter(scan); 413 list_tail = &scan->bnext; 414 } 415 } 416 freebulk(bulk); 417 } 418 printf("100.00%%\n"); 419 printf("\nTotal %d\n", count); 420 fflush(stdout); 421 422 /* 423 * Resolve all dependencies for the related packages, potentially 424 * adding anything that could not be found to the list. This will 425 * continue to issue bulk operations and process the result until 426 * no dependencies are left. 427 */ 428 printf("Resolving dependencies..."); 429 fflush(stdout); 430 dep_list = list; 431 while (dep_list) { 432 dep_list = resolveDeps(dep_list, &list_tail, 0); 433 } 434 printf("done\n"); 435 436 donebulk(); 437 438 /* 439 * Generate the topology 440 */ 441 resolveDeps(list, NULL, 1); 442 443 /* 444 * Do a final count, ignore place holders. 445 * 446 * Also set stop_fail if appropriate. Check for direct specifications 447 * which fail to probe and any direct dependencies of those 448 * specifications, but don't recurse (for now)... don't check indirect 449 * dependencies (i.e. A -> B -> C where A is directly specified, B 450 * is adirect dependency, and C fails to probe). 451 */ 452 count = 0; 453 stop_fail = 0; 454 stop_base_list = 0; 455 for (scan = list; scan; scan = scan->bnext) { 456 if ((scan->flags & PKGF_ERROR) == 0) { 457 ++count; 458 } 459 if ((scan->flags & PKGF_MANUALSEL) && MaskProbeAbort == 0) { 460 pkglink_t *link; 461 462 /* 463 * Directly specified package failed to probe 464 */ 465 if (scan->flags & PKGF_CORRUPT) { 466 ++stop_fail; 467 ++stop_base_list; 468 } 469 470 /* 471 * Directly specified package had a direct dependency 472 * that failed to probe (don't go further). 473 */ 474 PKGLIST_FOREACH(link, &scan->idepon_list) { 475 if (link->pkg && 476 (link->pkg->flags & PKGF_CORRUPT)) { 477 ++stop_fail; 478 } 479 } 480 } 481 } 482 printf("Total Returned %d\n", count); 483 484 /* 485 * Check to see if any PKGF_MANUALSEL packages 486 */ 487 if (stop_fail) { 488 printf("%d packages failed to probe\n", stop_fail); 489 if (PrepareSystemFlag) { 490 if (stop_fail == stop_base_list) { 491 printf( 492 "prepare-system: Some of your installed packages no longer exist in\n" 493 "dports, do you wish to continue rebuilding what does exist?\n"); 494 if (askyn("Continue anyway? ")) 495 remove_corrupt = 1; 496 } else { 497 printf( 498 "prepare-system: Some of your installed packages have dependencies\n" 499 "which could not be found in dports, cannot continue, aborting\n"); 500 } 501 } else { 502 printf("unable to continue, aborting\n"); 503 } 504 if (remove_corrupt == 0) 505 exit(1); 506 } 507 508 /* 509 * Remove corrupt packages before continuing 510 */ 511 if (remove_corrupt) { 512 list_tail = &list; 513 while ((scan = *list_tail) != NULL) { 514 if (scan->flags & PKGF_CORRUPT) 515 *list_tail = scan->bnext; 516 else 517 list_tail = &scan->bnext; 518 } 519 } 520 521 /* 522 * Scan our binary distributions and related dependencies looking 523 * for any packages that have already been built. 524 */ 525 initbulk(childGetBinaryDistInfo, MaxBulk); 526 total = scan_binary_repo(RepositoryPath); 527 count = 0; 528 printf("Scanning %d packages\n", total); 529 530 while ((bulk = getbulk()) != NULL) { 531 ++count; 532 if ((count & 255) == 0) { 533 printf("%6.2f%%\r", 534 (double)count * 100.0 / (double)total + 0.001); 535 fflush(stdout); 536 } 537 freebulk(bulk); 538 } 539 printf("100.00%%\n"); 540 printf("\nTotal %d\n", count); 541 fflush(stdout); 542 donebulk(); 543 544 printf("all done\n"); 545 546 return list; 547 } 548 549 pkg_t * 550 GetPkgPkg(pkg_t *list) 551 { 552 bulk_t *bulk; 553 pkg_t *scan; 554 555 for (scan = list; scan; scan = scan->bnext) { 556 if (strcmp(scan->portdir, "ports-mgmt/pkg") == 0) 557 return scan; 558 } 559 560 /* 561 * This will force pkg to be built, but generally this code 562 * is not reached because the package list processing code 563 * adds ports-mgmt/pkg unconditionally. 564 */ 565 initbulk(childGetPackageInfo, MaxBulk); 566 queuebulk("ports-mgmt", "pkg", NULL, "x"); 567 bulk = getbulk(); 568 dassert(bulk, "Cannot find ports-mgmt/pkg"); 569 scan = bulk->list; 570 bulk->list = NULL; 571 freebulk(bulk); 572 donebulk(); 573 574 return scan; 575 } 576 577 /* 578 * Try to optimize the environment by supplying information that 579 * the ports system would generally have to run stuff to get on 580 * every package. 581 * 582 * See childOptimizeEnv() for the actual handling. We execute 583 * a single make -V... -V... for ports-mgmt/pkg from within the 584 * bulk system (which handles the environment and disables 585 * /etc/make.conf), and we then call addbuildenv() as appropriate. 586 * 587 * _PERL5_FROM_BIN 588 * add others... 589 */ 590 void 591 OptimizeEnv(void) 592 { 593 bulk_t *bulk; 594 595 initbulk(childOptimizeEnv, MaxBulk); 596 queuebulk("ports-mgmt", "pkg", NULL, NULL); 597 bulk = getbulk(); 598 freebulk(bulk); 599 donebulk(); 600 } 601 602 /* 603 * Run through the list resolving dependencies and constructing the topology 604 * linkages. This may append packages to the list. Dependencies to dummy 605 * nodes which do not specify a flavor do not need special handling, the 606 * search code in build.c will properly follow the first flavor. 607 */ 608 static pkg_t * 609 resolveDeps(pkg_t *list, pkg_t ***list_tailp, int gentopo) 610 { 611 pkg_t *ret_list = NULL; 612 pkg_t *scan; 613 pkg_t *use; 614 bulk_t *bulk; 615 616 for (scan = list; scan; scan = scan->bnext) { 617 use = pkg_find(scan->portdir); 618 resolveFlavors(use, scan->flavors, gentopo); 619 resolveDepString(use, scan->fetch_deps, 620 gentopo, DEP_TYPE_FETCH); 621 resolveDepString(use, scan->ext_deps, 622 gentopo, DEP_TYPE_EXT); 623 resolveDepString(use, scan->patch_deps, 624 gentopo, DEP_TYPE_PATCH); 625 resolveDepString(use, scan->build_deps, 626 gentopo, DEP_TYPE_BUILD); 627 resolveDepString(use, scan->lib_deps, 628 gentopo, DEP_TYPE_LIB); 629 resolveDepString(use, scan->run_deps, 630 gentopo, DEP_TYPE_RUN); 631 } 632 633 /* 634 * No bulk ops are queued when doing the final topology 635 * generation. 636 * 637 * Avoid entering duplicate results from the bulk ops. Duplicate 638 * results are mostly filtered out, but not always. A dummy node 639 * representing multiple flavors will parse-out the flavors 640 */ 641 if (gentopo) 642 return NULL; 643 while ((bulk = getbulk()) != NULL) { 644 if (bulk->list) { 645 if (ret_list == NULL) 646 ret_list = bulk->list; 647 **list_tailp = bulk->list; 648 bulk->list = NULL; 649 while ((scan = **list_tailp) != NULL) { 650 pkg_enter(scan); 651 *list_tailp = &scan->bnext; 652 } 653 } 654 freebulk(bulk); 655 } 656 return (ret_list); 657 } 658 659 /* 660 * Resolve a generic node that has flavors, queue to retrieve info for 661 * each flavor and setup linkages as appropriate. 662 */ 663 static void 664 resolveFlavors(pkg_t *pkg, char *flavors, int gentopo) 665 { 666 char *flavor_base; 667 char *flavor_scan; 668 char *flavor; 669 char *portdir; 670 char *s1; 671 char *s2; 672 pkg_t *dpkg; 673 pkglink_t *link; 674 675 if ((pkg->flags & PKGF_DUMMY) == 0) 676 return; 677 if (pkg->flavors == NULL || pkg->flavors[0] == 0) 678 return; 679 flavor_base = strdup(flavors); 680 flavor_scan = flavor_base; 681 682 for (;;) { 683 do { 684 flavor = strsep(&flavor_scan, " \t"); 685 } while (flavor && *flavor == 0); 686 if (flavor == NULL) 687 break; 688 689 /* 690 * Iterate each flavor generating "s1/s2@flavor". 691 * 692 * queuebulk() info for each flavor, and set-up the 693 * linkages in the topology generation pass. 694 */ 695 asprintf(&portdir, "%s@%s", pkg->portdir, flavor); 696 s1 = strdup(pkg->portdir); 697 s2 = strchr(s1, '/'); 698 *s2++ = 0; 699 700 dpkg = pkg_find(portdir); 701 if (dpkg && gentopo) { 702 /* 703 * Setup linkages 704 */ 705 free(portdir); 706 707 link = calloc(1, sizeof(*link)); 708 link->pkg = dpkg; 709 link->next = &pkg->idepon_list; 710 link->prev = pkg->idepon_list.prev; 711 link->next->prev = link; 712 link->prev->next = link; 713 link->dep_type = DEP_TYPE_BUILD; 714 715 link = calloc(1, sizeof(*link)); 716 link->pkg = pkg; 717 link->next = &dpkg->deponi_list; 718 link->prev = dpkg->deponi_list.prev; 719 link->next->prev = link; 720 link->prev->next = link; 721 link->dep_type = DEP_TYPE_BUILD; 722 ++dpkg->depi_count; 723 } else if (gentopo == 0 && dpkg == NULL) { 724 /* 725 * Use a place-holder to prevent duplicate 726 * dependencies from being processed. The placeholder 727 * will be replaced by the actual dependency. 728 */ 729 dpkg = allocpkg(); 730 dpkg->portdir = portdir; 731 dpkg->flags = PKGF_PLACEHOLD; 732 pkg_enter(dpkg); 733 queuebulk(s1, s2, flavor, NULL); 734 } 735 free(s1); 736 } 737 free(flavor_base); 738 } 739 740 static void 741 resolveDepString(pkg_t *pkg, char *depstr, int gentopo, int dep_type) 742 { 743 char *copy_base; 744 char *copy; 745 char *dep; 746 char *sep; 747 char *tag; 748 char *flavor; 749 pkg_t *dpkg; 750 751 if (depstr == NULL || depstr[0] == 0) 752 return; 753 754 copy_base = strdup(depstr); 755 copy = copy_base; 756 757 for (;;) { 758 do { 759 dep = strsep(©, " \t"); 760 } while (dep && *dep == 0); 761 if (dep == NULL) 762 break; 763 764 /* 765 * Ignore dependencies prefixed with ${NONEXISTENT} 766 */ 767 if (strncmp(dep, "/nonexistent:", 13) == 0) 768 continue; 769 770 dep = strchr(dep, ':'); 771 if (dep == NULL || *dep != ':') { 772 printf("Error parsing dependency for %s: %s\n", 773 pkg->portdir, copy_base); 774 continue; 775 } 776 ++dep; 777 778 /* 779 * Strip-off any DPortsPath prefix. EXTRACT_DEPENDS 780 * often (always?) generates this prefix. 781 */ 782 if (strncmp(dep, DPortsPath, strlen(DPortsPath)) == 0) { 783 dep += strlen(DPortsPath); 784 if (*dep == '/') 785 ++dep; 786 } 787 788 /* 789 * Strip-off any tag (such as :patch). We don't try to 790 * organize dependencies at this fine a grain (for now). 791 */ 792 tag = strchr(dep, ':'); 793 if (tag) 794 *tag++ = 0; 795 796 /* 797 * Locate the dependency 798 */ 799 if ((dpkg = pkg_find(dep)) != NULL) { 800 if (gentopo) { 801 pkglink_t *link; 802 803 /* 804 * NOTE: idep_count is calculated recursively 805 * at build-time 806 */ 807 ddprintf(0, "Add Dependency %s -> %s\n", 808 pkg->portdir, dpkg->portdir); 809 link = calloc(1, sizeof(*link)); 810 link->pkg = dpkg; 811 link->next = &pkg->idepon_list; 812 link->prev = pkg->idepon_list.prev; 813 link->next->prev = link; 814 link->prev->next = link; 815 link->dep_type = dep_type; 816 817 link = calloc(1, sizeof(*link)); 818 link->pkg = pkg; 819 link->next = &dpkg->deponi_list; 820 link->prev = dpkg->deponi_list.prev; 821 link->next->prev = link; 822 link->prev->next = link; 823 link->dep_type = dep_type; 824 ++dpkg->depi_count; 825 } 826 continue; 827 } 828 829 /* 830 * This shouldn't happen because we already took a first 831 * pass and should have generated the pkgs. 832 */ 833 if (gentopo) { 834 printf("Topology Generate failed for %s: %s\n", 835 pkg->portdir, copy_base); 836 continue; 837 } 838 839 /* 840 * Separate out the two dports directory components and 841 * extract the optional '@flavor' specification. 842 */ 843 sep = strchr(dep, '/'); 844 if (sep == NULL) { 845 printf("Error parsing dependency for %s: %s\n", 846 pkg->portdir, copy_base); 847 continue; 848 } 849 *sep++ = 0; 850 851 /* 852 * The flavor hangs off the separator, not the tag 853 */ 854 flavor = strrchr(sep, '@'); 855 #if 0 856 if (tag) 857 flavor = strrchr(tag, '@'); 858 else 859 flavor = strrchr(sep, '@'); 860 #endif 861 if (flavor) 862 *flavor++ = 0; 863 864 if (flavor) 865 ddprintf(0, "QUEUE DEPENDENCY FROM PKG %s: %s/%s@%s\n", 866 pkg->portdir, dep, sep, flavor); 867 else 868 ddprintf(0, "QUEUE DEPENDENCY FROM PKG %s: %s/%s\n", 869 pkg->portdir, dep, sep); 870 871 /* 872 * Use a place-holder to prevent duplicate dependencies from 873 * being processed. The placeholder will be replaced by 874 * the actual dependency. 875 */ 876 dpkg = allocpkg(); 877 if (flavor) 878 asprintf(&dpkg->portdir, "%s/%s@%s", dep, sep, flavor); 879 else 880 asprintf(&dpkg->portdir, "%s/%s", dep, sep); 881 dpkg->flags = PKGF_PLACEHOLD; 882 pkg_enter(dpkg); 883 884 queuebulk(dep, sep, flavor, NULL); 885 } 886 free(copy_base); 887 } 888 889 void 890 FreePackageList(pkg_t *pkgs __unused) 891 { 892 dfatal("not implemented"); 893 } 894 895 /* 896 * Scan some or all dports to allocate the related pkg structure. Dependencies 897 * are stored but not processed. 898 * 899 * Threaded function 900 */ 901 static void 902 childGetPackageInfo(bulk_t *bulk) 903 { 904 pkg_t *pkg; 905 char *flavor; 906 char *ptr; 907 FILE *fp; 908 int line; 909 size_t len; 910 char *portpath; 911 char *flavarg; 912 const char *cav[MAXCAC]; 913 pid_t pid; 914 int cac; 915 916 /* 917 * If the package has flavors we will loop on each one. If a flavor 918 * is not passed in s3 we will loop on all flavors, otherwise we will 919 * only process the passed-in flavor. 920 */ 921 flavor = bulk->s3; /* usually NULL */ 922 923 bulk->list = NULL; 924 925 asprintf(&portpath, "%s/%s/%s", DPortsPath, bulk->s1, bulk->s2); 926 if (flavor) 927 asprintf(&flavarg, "FLAVOR=%s", flavor); 928 else 929 flavarg = NULL; 930 931 cac = 0; 932 cav[cac++] = MAKE_BINARY; 933 cav[cac++] = "-C"; 934 cav[cac++] = portpath; 935 if (flavarg) 936 cav[cac++] = flavarg; 937 cav[cac++] = "-VPKGVERSION"; 938 cav[cac++] = "-VPKGFILE:T"; 939 cav[cac++] = "-VALLFILES"; 940 cav[cac++] = "-VDIST_SUBDIR"; 941 cav[cac++] = "-VMAKE_JOBS_NUMBER"; 942 cav[cac++] = "-VIGNORE"; 943 cav[cac++] = "-VFETCH_DEPENDS"; 944 cav[cac++] = "-VEXTRACT_DEPENDS"; 945 cav[cac++] = "-VPATCH_DEPENDS"; 946 cav[cac++] = "-VBUILD_DEPENDS"; 947 cav[cac++] = "-VLIB_DEPENDS"; 948 cav[cac++] = "-VRUN_DEPENDS"; 949 cav[cac++] = "-VSELECTED_OPTIONS"; 950 cav[cac++] = "-VDESELECTED_OPTIONS"; 951 cav[cac++] = "-VUSE_LINUX"; 952 cav[cac++] = "-VFLAVORS"; 953 cav[cac++] = "-VUSES"; 954 955 fp = dexec_open(cav, cac, &pid, NULL, 1, 1); 956 free(portpath); 957 freestrp(&flavarg); 958 959 pkg = allocpkg(); 960 if (flavor) 961 asprintf(&pkg->portdir, "%s/%s@%s", bulk->s1, bulk->s2, flavor); 962 else 963 asprintf(&pkg->portdir, "%s/%s", bulk->s1, bulk->s2); 964 965 line = 1; 966 while ((ptr = fgetln(fp, &len)) != NULL) { 967 if (len == 0 || ptr[len-1] != '\n') { 968 dfatal("Bad package info for %s/%s response line %d", 969 bulk->s1, bulk->s2, line); 970 } 971 ptr[--len] = 0; 972 973 switch(line) { 974 case 1: /* PKGVERSION */ 975 asprintf(&pkg->version, "%s", ptr); 976 break; 977 case 2: /* PKGFILE */ 978 asprintf(&pkg->pkgfile, "%s", ptr); 979 break; 980 case 3: /* ALLFILES (aka DISTFILES + patch files) */ 981 asprintf(&pkg->distfiles, "%s", ptr); 982 break; 983 case 4: /* DIST_SUBDIR */ 984 pkg->distsubdir = strdup_or_null(ptr); 985 break; 986 case 5: /* MAKE_JOBS_NUMBER */ 987 pkg->make_jobs_number = strtol(ptr, NULL, 0); 988 break; 989 case 6: /* IGNORE */ 990 pkg->ignore = strdup_or_null(ptr); 991 break; 992 case 7: /* FETCH_DEPENDS */ 993 pkg->fetch_deps = strdup_or_null(ptr); 994 break; 995 case 8: /* EXTRACT_DEPENDS */ 996 pkg->ext_deps = strdup_or_null(ptr); 997 break; 998 case 9: /* PATCH_DEPENDS */ 999 pkg->patch_deps = strdup_or_null(ptr); 1000 break; 1001 case 10: /* BUILD_DEPENDS */ 1002 pkg->build_deps = strdup_or_null(ptr); 1003 break; 1004 case 11: /* LIB_DEPENDS */ 1005 pkg->lib_deps = strdup_or_null(ptr); 1006 break; 1007 case 12: /* RUN_DEPENDS */ 1008 pkg->run_deps = strdup_or_null(ptr); 1009 break; 1010 case 13: /* SELECTED_OPTIONS */ 1011 pkg->pos_options = strdup_or_null(ptr); 1012 break; 1013 case 14: /* DESELECTED_OPTIONS */ 1014 pkg->neg_options = strdup_or_null(ptr); 1015 break; 1016 case 15: /* USE_LINUX */ 1017 if (ptr[0]) 1018 pkg->use_linux = 1; 1019 break; 1020 case 16: /* FLAVORS */ 1021 asprintf(&pkg->flavors, "%s", ptr); 1022 break; 1023 case 17: /* USES */ 1024 asprintf(&pkg->uses, "%s", ptr); 1025 if (strstr(pkg->uses, "metaport")) 1026 pkg->flags |= PKGF_META; 1027 break; 1028 default: 1029 printf("EXTRA LINE: %s\n", ptr); 1030 break; 1031 } 1032 ++line; 1033 } 1034 if (line == 1) { 1035 printf("DPort not found: %s/%s\n", bulk->s1, bulk->s2); 1036 pkg->flags |= PKGF_NOTFOUND; 1037 } else if (line != 17 + 1) { 1038 printf("DPort corrupt: %s/%s\n", bulk->s1, bulk->s2); 1039 pkg->flags |= PKGF_CORRUPT; 1040 } 1041 if (dexec_close(fp, pid)) { 1042 printf("make -V* command for %s/%s failed\n", 1043 bulk->s1, bulk->s2); 1044 pkg->flags |= PKGF_CORRUPT; 1045 } 1046 ddassert(bulk->s1); 1047 1048 /* 1049 * DEBUGSTOP mode 1050 */ 1051 if (bulk->s4 && bulk->s4[0] == 'd') 1052 pkg->flags |= PKGF_DEBUGSTOP; 1053 1054 /* 1055 * Mark as a dummy node, the front-end will iterate the flavors 1056 * and create sub-nodes for us. 1057 * 1058 * Get rid of elements returned that are for the first flavor. 1059 * We are creating a dummy node here, not the node for the first 1060 * flavor. 1061 */ 1062 if (flavor == NULL && pkg->flavors && pkg->flavors[0]) { 1063 pkg->flags |= PKGF_DUMMY; 1064 freestrp(&pkg->fetch_deps); 1065 freestrp(&pkg->ext_deps); 1066 freestrp(&pkg->patch_deps); 1067 freestrp(&pkg->build_deps); 1068 freestrp(&pkg->lib_deps); 1069 freestrp(&pkg->run_deps); 1070 freestrp(&pkg->pkgfile); 1071 } 1072 1073 /* 1074 * Only one pkg is put on the return list now. This code no 1075 * longer creates pseudo-nodes for flavors (the frontend requests 1076 * each flavor instead). 1077 */ 1078 bulk->list = pkg; 1079 } 1080 1081 /* 1082 * Query the package (at least to make sure it hasn't been truncated) 1083 * and mark it as PACKAGED if found. 1084 * 1085 * Threaded function 1086 */ 1087 static void 1088 childGetBinaryDistInfo(bulk_t *bulk) 1089 { 1090 char *ptr; 1091 FILE *fp; 1092 size_t len; 1093 pkg_t *pkg; 1094 const char *cav[MAXCAC]; 1095 char *repopath; 1096 char buf[1024]; 1097 pid_t pid; 1098 int cac; 1099 int deleteme; 1100 1101 asprintf(&repopath, "%s/%s", RepositoryPath, bulk->s1); 1102 1103 cac = 0; 1104 cav[cac++] = PKG_BINARY; 1105 cav[cac++] = "query"; 1106 cav[cac++] = "-F"; 1107 cav[cac++] = repopath; 1108 cav[cac++] = "%n-%v"; 1109 1110 fp = dexec_open(cav, cac, &pid, NULL, 1, 0); 1111 deleteme = DeleteObsoletePkgs; 1112 1113 while ((ptr = fgetln(fp, &len)) != NULL) { 1114 if (len == 0 || ptr[len-1] != '\n') 1115 continue; 1116 ptr[len-1] = 0; 1117 snprintf(buf, sizeof(buf), "%s%s", ptr, UsePkgSufx); 1118 1119 pkg = pkg_find(buf); 1120 if (pkg) { 1121 pkg->flags |= PKGF_PACKAGED; 1122 deleteme = 0; 1123 } else { 1124 ddprintf(0, "Note: package scan, not in list, " 1125 "skipping %s\n", buf); 1126 } 1127 } 1128 if (dexec_close(fp, pid)) { 1129 printf("pkg query command failed for %s\n", repopath); 1130 } 1131 if (deleteme) { 1132 dlog(DLOG_ALL | DLOG_STDOUT, 1133 "Deleting obsolete package %s\n", repopath); 1134 remove(repopath); 1135 } 1136 free(repopath); 1137 } 1138 1139 static void 1140 childOptimizeEnv(bulk_t *bulk) 1141 { 1142 char *portpath; 1143 char *ptr; 1144 FILE *fp; 1145 int line; 1146 size_t len; 1147 const char *cav[MAXCAC]; 1148 pid_t pid; 1149 int cac; 1150 1151 asprintf(&portpath, "%s/%s/%s", DPortsPath, bulk->s1, bulk->s2); 1152 1153 cac = 0; 1154 cav[cac++] = MAKE_BINARY; 1155 cav[cac++] = "-C"; 1156 cav[cac++] = portpath; 1157 cav[cac++] = "-V_PERL5_FROM_BIN"; 1158 1159 fp = dexec_open(cav, cac, &pid, NULL, 1, 1); 1160 free(portpath); 1161 1162 line = 1; 1163 while ((ptr = fgetln(fp, &len)) != NULL) { 1164 if (len == 0 || ptr[len-1] != '\n') { 1165 dfatal("Bad package info for %s/%s response line %d", 1166 bulk->s1, bulk->s2, line); 1167 } 1168 ptr[--len] = 0; 1169 1170 switch(line) { 1171 case 1: /* _PERL5_FROM_BIN */ 1172 addbuildenv("_PERL5_FROM_BIN", ptr, BENV_ENVIRONMENT); 1173 break; 1174 default: 1175 printf("childOptimizeEnv: EXTRA LINE: %s\n", ptr); 1176 break; 1177 } 1178 ++line; 1179 } 1180 if (line == 1) { 1181 printf("DPort not found: %s/%s\n", bulk->s1, bulk->s2); 1182 } else if (line != 1 + 1) { 1183 printf("DPort corrupt: %s/%s\n", bulk->s1, bulk->s2); 1184 } 1185 if (dexec_close(fp, pid)) { 1186 printf("childOptimizeEnv() failed\n"); 1187 } 1188 } 1189 1190 static int 1191 scan_and_queue_dir(const char *path, const char *level1, int level) 1192 { 1193 DIR *dir; 1194 char *s1; 1195 char *s2; 1196 struct dirent *den; 1197 struct stat st; 1198 int count = 0; 1199 1200 dir = opendir(path); 1201 dassert(dir, "Cannot open dports path \"%s\"", path); 1202 1203 while ((den = readdir(dir)) != NULL) { 1204 if (den->d_namlen == 1 && den->d_name[0] == '.') 1205 continue; 1206 if (den->d_namlen == 2 && 1207 den->d_name[0] == '.' && den->d_name[1] == '.') 1208 continue; 1209 asprintf(&s1, "%s/%s", path, den->d_name); 1210 if (lstat(s1, &st) < 0 || !S_ISDIR(st.st_mode)) { 1211 free(s1); 1212 continue; 1213 } 1214 if (level == 1) { 1215 count += scan_and_queue_dir(s1, den->d_name, 2); 1216 free(s1); 1217 continue; 1218 } 1219 asprintf(&s2, "%s/Makefile", s1); 1220 if (lstat(s2, &st) == 0) { 1221 queuebulk(level1, den->d_name, NULL, NULL); 1222 ++count; 1223 } 1224 free(s1); 1225 free(s2); 1226 } 1227 closedir(dir); 1228 1229 return count; 1230 } 1231 1232 static int 1233 scan_binary_repo(const char *path) 1234 { 1235 DIR *dir; 1236 struct dirent *den; 1237 size_t len; 1238 int count; 1239 1240 count = 0; 1241 dir = opendir(path); 1242 dassert(dir, "Cannot open repository path \"%s\"", path); 1243 1244 /* 1245 * NOTE: Test includes the '.' in the suffix. 1246 */ 1247 while ((den = readdir(dir)) != NULL) { 1248 len = strlen(den->d_name); 1249 if (len > 4 && 1250 strcmp(den->d_name + len - 4, UsePkgSufx) == 0) { 1251 queuebulk(den->d_name, NULL, NULL, NULL); 1252 ++count; 1253 } 1254 } 1255 closedir(dir); 1256 1257 return count; 1258 } 1259 1260 #if 0 1261 static void 1262 pkgfree(pkg_t *pkg) 1263 { 1264 freestrp(&pkg->portdir); 1265 freestrp(&pkg->version); 1266 freestrp(&pkg->pkgfile); 1267 freestrp(&pkg->ignore); 1268 freestrp(&pkg->fetch_deps); 1269 freestrp(&pkg->ext_deps); 1270 freestrp(&pkg->patch_deps); 1271 freestrp(&pkg->build_deps); 1272 freestrp(&pkg->lib_deps); 1273 freestrp(&pkg->run_deps); 1274 freestrp(&pkg->pos_options); 1275 freestrp(&pkg->neg_options); 1276 freestrp(&pkg->flavors); 1277 free(pkg); 1278 } 1279 #endif 1280