1 /* 2 * Copyright (c) 2019 The DragonFly Project. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Matthew Dillon <dillon@backplane.com> 6 * 7 * This code uses concepts and configuration based on 'synth', by 8 * John R. Marino <draco@marino.st>, which was written in ada. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in 18 * the documentation and/or other materials provided with the 19 * distribution. 20 * 3. Neither the name of The DragonFly Project nor the names of its 21 * contributors may be used to endorse or promote products derived 22 * from this software without specific, prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 25 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 26 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 27 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 28 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 29 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 30 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 31 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 32 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 33 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 34 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 35 * SUCH DAMAGE. 36 */ 37 #include "dsynth.h" 38 39 worker_t WorkerAry[MAXWORKERS]; 40 int BuildInitialized; 41 int RunningWorkers; 42 int DynamicMaxWorkers; 43 int FailedWorkers; 44 long RunningPkgDepSize; 45 pthread_mutex_t WorkerMutex; 46 pthread_cond_t WorkerCond; 47 48 static int build_find_leaves(pkg_t *parent, pkg_t *pkg, 49 pkg_t ***build_tailp, int *app, int *hasworkp, 50 int depth, int first, int first_one_only); 51 static int buildCalculateDepiDepth(pkg_t *pkg); 52 static void build_clear_trav(pkg_t *pkg); 53 static void startbuild(pkg_t **build_listp, pkg_t ***build_tailp); 54 static int qsort_depi(const void *pkg1, const void *pkg2); 55 static int qsort_idep(const void *pkg1, const void *pkg2); 56 static void startworker(pkg_t *pkg, worker_t *work); 57 static void cleanworker(worker_t *work); 58 static void waitbuild(int whilematch, int dynamicmax); 59 static void workercomplete(worker_t *work); 60 static void *childBuilderThread(void *arg); 61 static int childInstallPkgDeps(worker_t *work); 62 static size_t childInstallPkgDeps_recurse(FILE *fp, pkglink_t *list, 63 int undoit, int depth, int first_one_only); 64 static void dophase(worker_t *work, wmsg_t *wmsg, 65 int wdog, int phaseid, const char *phase); 66 static void phaseReapAll(void); 67 static void phaseTerminateSignal(int sig); 68 static char *buildskipreason(pkglink_t *parent, pkg_t *pkg); 69 static int buildskipcount_dueto(pkg_t *pkg, int mode); 70 static int mptylogpoll(int ptyfd, int fdlog, wmsg_t *wmsg, 71 time_t *wdog_timep); 72 static int copyfile(char *src, char *dst); 73 static void doHook(pkg_t *pkg, const char *id, const char *path, int waitfor); 74 static void childHookRun(bulk_t *bulk); 75 76 static worker_t *SigWork; 77 static int MasterPtyFd = -1; 78 static int CopyFileFd = -1; 79 static pid_t SigPid; 80 static const char *WorkerFlavorPrt = ""; /* "" or "@flavor" */ 81 82 #define MPTY_FAILED -2 83 #define MPTY_AGAIN -1 84 #define MPTY_EOF 0 85 #define MPTY_DATA 1 86 87 int BuildTotal; 88 int BuildCount; 89 int BuildSkipCount; 90 int BuildIgnoreCount; 91 int BuildFailCount; 92 int BuildSuccessCount; 93 94 /* 95 * Initialize the WorkerAry[] 96 */ 97 void 98 DoInitBuild(int slot_override) 99 { 100 worker_t *work; 101 struct stat st; 102 int i; 103 104 ddassert(slot_override < 0 || MaxWorkers == 1); 105 106 bzero(WorkerAry, MaxWorkers * sizeof(worker_t)); 107 pthread_mutex_init(&WorkerMutex, NULL); 108 109 for (i = 0; i < MaxWorkers; ++i) { 110 work = &WorkerAry[i]; 111 work->index = (slot_override >= 0) ? slot_override : i; 112 work->state = WORKER_NONE; 113 asprintf(&work->basedir, "%s/SL%02d", BuildBase, work->index); 114 pthread_cond_init(&work->cond, NULL); 115 } 116 BuildCount = 0; 117 118 /* 119 * Create required sub-directories. The base directories must already 120 * exist as a dsynth configuration safety. 121 */ 122 if (stat(RepositoryPath, &st) < 0) { 123 if (mkdir(RepositoryPath, 0755) < 0) 124 dfatal("Cannot mkdir %s\n", RepositoryPath); 125 } 126 127 BuildInitialized = 1; 128 129 /* 130 * slow-start (increases at a rate of 1 per 5 seconds) 131 */ 132 if (SlowStartOpt > MaxWorkers) 133 DynamicMaxWorkers = MaxWorkers; 134 else if (SlowStartOpt > 0) 135 DynamicMaxWorkers = SlowStartOpt; 136 else 137 DynamicMaxWorkers = MaxWorkers; 138 } 139 140 /* 141 * Called by the frontend to clean-up any hanging mounts. 142 */ 143 void 144 DoCleanBuild(int resetlogs) 145 { 146 int i; 147 148 ddassert(BuildInitialized); 149 150 if (resetlogs) 151 dlogreset(); 152 for (i = 0; i < MaxWorkers; ++i) { 153 DoWorkerUnmounts(&WorkerAry[i]); 154 } 155 } 156 157 void 158 DoBuild(pkg_t *pkgs) 159 { 160 pkg_t *build_list = NULL; 161 pkg_t **build_tail = &build_list; 162 pkg_t *scan; 163 bulk_t *bulk; 164 int haswork = 1; 165 int first = 1; 166 int newtemplate; 167 time_t startTime; 168 time_t t; 169 int h, m, s; 170 171 /* 172 * We use our bulk system to run hooks. This will be for 173 * Skipped and Ignored. Success and Failure are handled by 174 * WorkerProcess() which is a separately-exec'd dsynth. 175 */ 176 if (UsingHooks) 177 initbulk(childHookRun, MaxBulk); 178 179 /* 180 * Count up the packages, not counting dummies 181 */ 182 for (scan = pkgs; scan; scan = scan->bnext) { 183 if ((scan->flags & PKGF_DUMMY) == 0) 184 ++BuildTotal; 185 } 186 187 doHook(NULL, "hook_run_start", HookRunStart, 1); 188 189 /* 190 * The pkg and pkg-static binaries are needed. If already present 191 * then assume that the template is also valid, otherwise build 192 * both. 193 */ 194 scan = GetPkgPkg(pkgs); 195 196 /* 197 * Create our template. The template will be missing pkg 198 * and pkg-static. 199 */ 200 if ((scan->flags & (PKGF_SUCCESS | PKGF_PACKAGED)) == 0) { 201 /* force a fresh template */ 202 newtemplate = DoCreateTemplate(1); 203 } else { 204 newtemplate = DoCreateTemplate(0); 205 } 206 207 /* 208 * This will clear the screen and set-up our gui, so sleep 209 * a little first in case the user wants to see what was 210 * printed before. 211 */ 212 sleep(2); 213 pthread_mutex_lock(&WorkerMutex); 214 startTime = time(NULL); 215 RunStatsInit(); 216 RunStatsReset(); 217 218 /* 219 * Build pkg/pkg-static. 220 */ 221 if ((scan->flags & (PKGF_SUCCESS | PKGF_PACKAGED)) == 0) { 222 build_list = scan; 223 build_tail = &scan->build_next; 224 startbuild(&build_list, &build_tail); 225 while (RunningWorkers == 1) 226 waitbuild(1, 0); 227 228 if (scan->flags & PKGF_NOBUILD) 229 dfatal("Unable to build 'pkg'"); 230 if (scan->flags & PKGF_ERROR) 231 dfatal("Error building 'pkg'"); 232 if ((scan->flags & PKGF_SUCCESS) == 0) 233 dfatal("Error building 'pkg'"); 234 newtemplate = 1; 235 } 236 237 /* 238 * Install pkg/pkg-static into the template 239 */ 240 if (newtemplate) { 241 char *buf; 242 int rc; 243 244 asprintf(&buf, 245 "cd %s/Template; " 246 "tar --exclude '+*' --exclude '*/man/*' " 247 "-xvzpf %s/%s > /dev/null 2>&1", 248 BuildBase, 249 RepositoryPath, 250 scan->pkgfile); 251 rc = system(buf); 252 if (rc) 253 dfatal("Command failed: %s\n", buf); 254 freestrp(&buf); 255 } 256 257 /* 258 * Calculate depi_depth, the longest chain of dependencies 259 * for who depends on me, weighted by powers of two. 260 */ 261 for (scan = pkgs; scan; scan = scan->bnext) { 262 buildCalculateDepiDepth(scan); 263 } 264 265 /* 266 * Nominal bulk build sequence 267 */ 268 while (haswork) { 269 haswork = 0; 270 fflush(stdout); 271 for (scan = pkgs; scan; scan = scan->bnext) { 272 ddprintf(0, "SCANLEAVES %08x %s\n", 273 scan->flags, scan->portdir); 274 scan->flags |= PKGF_BUILDLOOP; 275 /* 276 * NOTE: We must still find dependencies if PACKAGED 277 * to fill in the gaps, as some of them may 278 * need to be rebuilt. 279 */ 280 if (scan->flags & (PKGF_SUCCESS | PKGF_FAILURE | 281 PKGF_ERROR)) { 282 #if 0 283 ddprintf(0, "%s: already built\n", 284 scan->portdir); 285 #endif 286 } else { 287 int ap = 0; 288 build_find_leaves(NULL, scan, &build_tail, 289 &ap, &haswork, 0, first, 0); 290 ddprintf(0, "TOPLEVEL %s %08x\n", 291 scan->portdir, ap); 292 } 293 scan->flags &= ~PKGF_BUILDLOOP; 294 build_clear_trav(scan); 295 } 296 first = 0; 297 fflush(stdout); 298 startbuild(&build_list, &build_tail); 299 300 if (haswork == 0 && RunningWorkers) { 301 waitbuild(RunningWorkers, 1); 302 haswork = 1; 303 } 304 } 305 pthread_mutex_unlock(&WorkerMutex); 306 307 RunStatsUpdateTop(); 308 RunStatsUpdateLogs(); 309 RunStatsSync(); 310 RunStatsDone(); 311 312 doHook(NULL, "hook_run_end", HookRunEnd, 1); 313 if (UsingHooks) { 314 while ((bulk = getbulk()) != NULL) 315 freebulk(bulk); 316 donebulk(); 317 } 318 319 t = time(NULL) - startTime; 320 h = t / 3600; 321 m = t / 60 % 60; 322 s = t % 60; 323 324 dlog(DLOG_ALL|DLOG_STDOUT, 325 "\n" 326 "Initial queue size: %d\n" 327 " packages built: %d\n" 328 " ignored: %d\n" 329 " skipped: %d\n" 330 " failed: %d\n" 331 "\n" 332 "Duration: %02d:%02d:%02d\n" 333 "\n", 334 BuildTotal, 335 BuildSuccessCount, 336 BuildIgnoreCount, 337 BuildSkipCount, 338 BuildFailCount, 339 h, m, s); 340 } 341 342 /* 343 * Traverse the packages (pkg) depends on recursively until we find 344 * a leaf to build or report as unbuildable. Calculates and assigns a 345 * dependency count. Returns all parallel-buildable packages. 346 * 347 * (pkg) itself is only added to the list if it is immediately buildable. 348 */ 349 static 350 int 351 build_find_leaves(pkg_t *parent, pkg_t *pkg, pkg_t ***build_tailp, 352 int *app, int *hasworkp, int depth, int first, 353 int first_one_only) 354 { 355 pkglink_t *link; 356 pkg_t *scan; 357 int idep_count = 0; 358 int apsub; 359 int dfirst_one_only; 360 int ndepth; 361 char *buf; 362 363 ndepth = depth + 1; 364 365 /* 366 * Already on build list, possibly in-progress, tell caller that 367 * it is not ready. 368 */ 369 ddprintf(ndepth, "sbuild_find_leaves %d %s %08x {\n", 370 depth, pkg->portdir, pkg->flags); 371 if (pkg->flags & PKGF_BUILDLIST) { 372 ddprintf(ndepth, "} (already on build list)\n"); 373 *app |= PKGF_NOTREADY; 374 return (pkg->idep_count); 375 } 376 377 /* 378 * Check dependencies 379 */ 380 PKGLIST_FOREACH(link, &pkg->idepon_list) { 381 scan = link->pkg; 382 383 if (scan == NULL) { 384 if (first_one_only) 385 break; 386 continue; 387 } 388 ddprintf(ndepth, "check %s %08x\t", scan->portdir, scan->flags); 389 390 /* 391 * If this dependency is to a DUMMY node it is a dependency 392 * only on the default flavor which is only the first node 393 * under this one, not all of them. 394 * 395 * NOTE: The depth is not being for complex dependency type 396 * tests like it is in childInstallPkgDeps_recurse(), 397 * so we don't have to hicup it like we do in that 398 * procedure. 399 */ 400 dfirst_one_only = (scan->flags & PKGF_DUMMY) ? 1 : 0; 401 402 /* 403 * When accounting for a successful build, just bump 404 * idep_count by one. scan->idep_count will heavily 405 * overlap packages that we count down multiple branches. 406 * 407 * We must still recurse through PACKAGED packages as 408 * some of their dependencies might be missing. 409 */ 410 if (scan->flags & PKGF_SUCCESS) { 411 ddprintf(0, "SUCCESS - OK\n"); 412 ++idep_count; 413 if (first_one_only) 414 break; 415 continue; 416 } 417 418 if (dfirst_one_only) 419 goto skip_to_flavor; 420 421 /* 422 * ERROR includes FAILURE, which is set in numerous situations 423 * including when NOBUILD state is processed. So check for 424 * NOBUILD state first. 425 * 426 * An ERROR in a sub-package causes a NOBUILD in packages 427 * that depend on it. 428 */ 429 if (scan->flags & PKGF_NOBUILD) { 430 ddprintf(0, "NOBUILD - OK " 431 "(propagate failure upward)\n"); 432 *app |= PKGF_NOBUILD_S; 433 if (first_one_only) 434 break; 435 continue; 436 } 437 if (scan->flags & PKGF_ERROR) { 438 ddprintf(0, "ERROR - OK (propagate failure upward)\n"); 439 *app |= PKGF_NOBUILD_S; 440 if (first_one_only) 441 break; 442 continue; 443 } 444 445 /* 446 * If already on build-list this dependency is not ready. 447 */ 448 if (scan->flags & PKGF_BUILDLIST) { 449 ddprintf(0, " [BUILDLIST]"); 450 *app |= PKGF_NOTREADY; 451 } 452 453 /* 454 * If not packaged this dependency is not ready for 455 * the caller. 456 */ 457 if ((scan->flags & PKGF_PACKAGED) == 0) { 458 ddprintf(0, " [NOT_PACKAGED]"); 459 *app |= PKGF_NOTREADY; 460 } 461 462 skip_to_flavor: 463 /* 464 * Reduce search complexity, if we have already processed 465 * scan in the traversal it will either already be on the 466 * build list or it will not be buildable. Either way 467 * the parent is not buildable. 468 */ 469 if (scan->flags & PKGF_BUILDTRAV) { 470 ddprintf(0, " [BUILDTRAV]\n"); 471 *app |= PKGF_NOTREADY; 472 if (first_one_only) 473 break; 474 continue; 475 } 476 477 /* 478 * Assert on dependency loop 479 */ 480 ++idep_count; 481 if (scan->flags & PKGF_BUILDLOOP) { 482 dfatal("pkg dependency loop %s -> %s", 483 parent->portdir, scan->portdir); 484 } 485 486 /* 487 * NOTE: For debug tabbing purposes we use (ndepth + 1) 488 * here (i.e. depth + 2) in our iteration. 489 */ 490 scan->flags |= PKGF_BUILDLOOP; 491 apsub = 0; 492 ddprintf(0, " SUBRECURSION {\n"); 493 idep_count += build_find_leaves(pkg, scan, build_tailp, 494 &apsub, hasworkp, 495 ndepth + 1, first, 496 dfirst_one_only); 497 scan->flags &= ~PKGF_BUILDLOOP; 498 *app |= apsub; 499 if (apsub & PKGF_NOBUILD) { 500 ddprintf(ndepth, "} (sub-nobuild)\n"); 501 } else if (apsub & PKGF_ERROR) { 502 ddprintf(ndepth, "} (sub-error)\n"); 503 } else if (apsub & PKGF_NOTREADY) { 504 ddprintf(ndepth, "} (sub-notready)\n"); 505 } else { 506 ddprintf(ndepth, "} (sub-ok)\n"); 507 } 508 if (first_one_only) 509 break; 510 } 511 pkg->idep_count = idep_count; 512 pkg->flags |= PKGF_BUILDTRAV; 513 514 /* 515 * Incorporate scan results into pkg state. 516 */ 517 if ((pkg->flags & PKGF_NOBUILD) == 0 && (*app & PKGF_NOBUILD)) { 518 *hasworkp = 1; 519 } else if ((pkg->flags & PKGF_ERROR) == 0 && (*app & PKGF_ERROR)) { 520 *hasworkp = 1; 521 } 522 pkg->flags |= *app & ~PKGF_NOTREADY; 523 524 /* 525 * Clear PACKAGED bit if sub-dependencies aren't clean 526 */ 527 if ((pkg->flags & PKGF_PACKAGED) && 528 (pkg->flags & (PKGF_NOTREADY|PKGF_ERROR|PKGF_NOBUILD))) { 529 pkg->flags &= ~PKGF_PACKAGED; 530 ddassert(pkg->pkgfile); 531 asprintf(&buf, "%s/%s", RepositoryPath, pkg->pkgfile); 532 if (remove(buf) < 0) { 533 dlog(DLOG_ALL, 534 "[XXX] %s DELETE-PACKAGE %s (failed)\n", 535 pkg->portdir, buf); 536 } else { 537 dlog(DLOG_ALL, 538 "[XXX] %s DELETE-PACKAGE %s " 539 "(due to dependencies)\n", 540 pkg->portdir, buf); 541 } 542 freestrp(&buf); 543 *hasworkp = 1; 544 } 545 546 /* 547 * Set PKGF_NOBUILD_I if there is IGNORE data 548 */ 549 if (pkg->ignore) 550 pkg->flags |= PKGF_NOBUILD_I; 551 552 /* 553 * Handle propagated flags 554 */ 555 if (pkg->flags & PKGF_ERROR) { 556 /* 557 * This can only happen if the ERROR has already been 558 * processed and accounted for. 559 */ 560 ddprintf(depth, "} (ERROR - %s)\n", pkg->portdir); 561 } else if (*app & PKGF_NOTREADY) { 562 /* 563 * We don't set PKGF_NOTREADY in the pkg, it is strictly 564 * a transient flag propagated via build_find_leaves(). 565 * 566 * Just don't add the package to the list. 567 * 568 * NOTE: Even if NOBUILD is set (meaning we could list it 569 * and let startbuild() finish it up as a skip, we 570 * don't process it to the list because we want to 571 * process all the dependencies, so someone doing a 572 * manual build can get more complete information and 573 * does not have to iterate each failed dependency one 574 * at a time. 575 */ 576 ; 577 } else if (pkg->flags & PKGF_SUCCESS) { 578 ddprintf(depth, "} (SUCCESS - %s)\n", pkg->portdir); 579 } else if (pkg->flags & PKGF_DUMMY) { 580 /* 581 * Just mark dummy packages as successful when all of their 582 * sub-depends (flavors) complete successfully. Note that 583 * dummy packages are not counted in the total, so do not 584 * decrement BuildTotal. 585 */ 586 ddprintf(depth, "} (DUMMY/META - SUCCESS)\n"); 587 pkg->flags |= PKGF_SUCCESS; 588 *hasworkp = 1; 589 if (first) { 590 dlog(DLOG_ALL | DLOG_FILTER, 591 "[XXX] %s META-ALREADY-BUILT\n", 592 pkg->portdir); 593 } else { 594 dlog(DLOG_SUCC, "[XXX] %s meta-node complete\n", 595 pkg->portdir); 596 } 597 } else if (pkg->flags & PKGF_PACKAGED) { 598 /* 599 * We can just mark the pkg successful. If this is 600 * the first pass, we count this as an initial pruning 601 * pass and reduce BuildTotal. 602 */ 603 ddprintf(depth, "} (PACKAGED - SUCCESS)\n"); 604 pkg->flags |= PKGF_SUCCESS; 605 *hasworkp = 1; 606 if (first) { 607 dlog(DLOG_ALL | DLOG_FILTER, 608 "[XXX] %s ALREADY-BUILT\n", 609 pkg->portdir); 610 --BuildTotal; 611 } 612 } else { 613 /* 614 * All dependencies are successful, queue new work 615 * and indicate not-ready to the parent (since our 616 * package has to be built). 617 * 618 * NOTE: The NOBUILD case propagates to here as well 619 * and is ultimately handled by startbuild(). 620 */ 621 *hasworkp = 1; 622 if (pkg->flags & PKGF_NOBUILD_I) 623 ddprintf(depth, "} (ADDLIST(IGNORE/BROKEN) - %s)\n", 624 pkg->portdir); 625 else if (pkg->flags & PKGF_NOBUILD) 626 ddprintf(depth, "} (ADDLIST(NOBUILD) - %s)\n", 627 pkg->portdir); 628 else 629 ddprintf(depth, "} (ADDLIST - %s)\n", pkg->portdir); 630 pkg->flags |= PKGF_BUILDLIST; 631 **build_tailp = pkg; 632 *build_tailp = &pkg->build_next; 633 *app |= PKGF_NOTREADY; 634 } 635 636 return idep_count; 637 } 638 639 static 640 void 641 build_clear_trav(pkg_t *pkg) 642 { 643 pkglink_t *link; 644 pkg_t *scan; 645 646 pkg->flags &= ~PKGF_BUILDTRAV; 647 PKGLIST_FOREACH(link, &pkg->idepon_list) { 648 scan = link->pkg; 649 if (scan && (scan->flags & PKGF_BUILDTRAV)) 650 build_clear_trav(scan); 651 } 652 } 653 654 /* 655 * Calculate the longest chain of packages that depend on me. The 656 * long the chain, the more important my package is to build earlier 657 * rather than later. 658 */ 659 static int 660 buildCalculateDepiDepth(pkg_t *pkg) 661 { 662 pkglink_t *link; 663 pkg_t *scan; 664 int best_depth = 0; 665 int res; 666 667 if (pkg->depi_depth) 668 return(pkg->depi_depth + 1); 669 pkg->flags |= PKGF_BUILDLOOP; 670 PKGLIST_FOREACH(link, &pkg->deponi_list) { 671 scan = link->pkg; 672 if (scan && (scan->flags & PKGF_BUILDLOOP) == 0) { 673 res = buildCalculateDepiDepth(scan); 674 if (best_depth < res) 675 best_depth = res; 676 } 677 } 678 pkg->flags &= ~PKGF_BUILDLOOP; 679 pkg->depi_depth = best_depth; 680 681 return (best_depth + 1); 682 } 683 684 /* 685 * Take a list of pkg ready to go, sort it, and assign it to worker 686 * slots. This routine blocks in waitbuild() until it can dispose of 687 * the entire list. 688 * 689 * WorkerMutex is held by the caller. 690 */ 691 static 692 void 693 startbuild(pkg_t **build_listp, pkg_t ***build_tailp) 694 { 695 pkg_t *pkg; 696 pkg_t **idep_ary; 697 pkg_t **depi_ary; 698 int count; 699 int idep_index; 700 int depi_index; 701 int i; 702 int n; 703 worker_t *work; 704 static int IterateWorker; 705 706 /* 707 * Nothing to do 708 */ 709 if (*build_listp == NULL) 710 return; 711 712 /* 713 * Sort 714 */ 715 count = 0; 716 for (pkg = *build_listp; pkg; pkg = pkg->build_next) 717 ++count; 718 idep_ary = calloc(count, sizeof(pkg_t *)); 719 depi_ary = calloc(count, sizeof(pkg_t *)); 720 721 count = 0; 722 for (pkg = *build_listp; pkg; pkg = pkg->build_next) { 723 idep_ary[count] = pkg; 724 depi_ary[count] = pkg; 725 ++count; 726 } 727 728 /* 729 * idep_ary - sorted by #of dependencies this pkg has. 730 * depi_ary - sorted by #of other packages that depend on this pkg. 731 */ 732 qsort(idep_ary, count, sizeof(pkg_t *), qsort_idep); 733 qsort(depi_ary, count, sizeof(pkg_t *), qsort_depi); 734 735 idep_index = 0; 736 depi_index = 0; 737 738 /* 739 * Half the workers build based on the highest depi count, 740 * the other half build based on the highest idep count. 741 * 742 * This is an attempt to get projects which many other projects 743 * depend on built first, but to also try to build large projects 744 * (which tend to have a lot of dependencies) earlier rather than 745 * later so the end of the bulk run doesn't inefficiently build 746 * the last few huge projects. 747 * 748 * Loop until we manage to assign slots to everyone. We do not 749 * wait for build completion. 750 * 751 * This is the point where we handle DUMMY packages (these are 752 * dummy unflavored packages which 'cover' all the flavors for 753 * a package). These are not real packages are marked SUCCESS 754 * at this time because their dependencies (the flavors) have all 755 * been built. 756 */ 757 while (idep_index != count || depi_index != count) { 758 pkg_t *pkgi; 759 pkg_t *ipkg; 760 761 /* 762 * Find candidate to start sorted by depi or idep. 763 */ 764 ipkg = NULL; 765 while (idep_index < count) { 766 ipkg = idep_ary[idep_index]; 767 if ((ipkg->flags & 768 (PKGF_SUCCESS | PKGF_FAILURE | 769 PKGF_ERROR | PKGF_RUNNING)) == 0) { 770 break; 771 } 772 ipkg = NULL; 773 ++idep_index; 774 } 775 776 pkgi = NULL; 777 while (depi_index < count) { 778 pkgi = depi_ary[depi_index]; 779 if ((pkgi->flags & 780 (PKGF_SUCCESS | PKGF_FAILURE | 781 PKGF_ERROR | PKGF_RUNNING)) == 0) { 782 break; 783 } 784 pkgi = NULL; 785 ++depi_index; 786 } 787 788 /* 789 * ipkg and pkgi must either both be NULL, or both 790 * be non-NULL. 791 */ 792 if (ipkg == NULL && pkgi == NULL) 793 break; 794 ddassert(ipkg && pkgi); 795 796 /* 797 * Handle the NOBUILD case right here, there's no point 798 * queueing it anywhere. 799 */ 800 if (ipkg->flags & PKGF_NOBUILD) { 801 char *reason; 802 803 ipkg->flags |= PKGF_FAILURE; 804 ipkg->flags &= ~PKGF_BUILDLIST; 805 806 reason = buildskipreason(NULL, ipkg); 807 if (ipkg->flags & PKGF_NOBUILD_I) { 808 ++BuildIgnoreCount; 809 dlog(DLOG_IGN, "[XXX] %s ignored due to %s\n", 810 ipkg->portdir, reason); 811 doHook(ipkg, "hook_pkg_ignored", 812 HookPkgIgnored, 0); 813 } else { 814 ++BuildSkipCount; 815 dlog(DLOG_SKIP, "[XXX] %s skipped due to %s\n", 816 ipkg->portdir, reason); 817 doHook(ipkg, "hook_pkg_skipped", 818 HookPkgSkipped, 0); 819 } 820 free(reason); 821 ++BuildCount; 822 continue; 823 } 824 if (pkgi->flags & PKGF_NOBUILD) { 825 char *reason; 826 827 pkgi->flags |= PKGF_FAILURE; 828 pkgi->flags &= ~PKGF_BUILDLIST; 829 830 reason = buildskipreason(NULL, pkgi); 831 if (pkgi->flags & PKGF_NOBUILD_I) { 832 ++BuildIgnoreCount; 833 dlog(DLOG_IGN, "[XXX] %s ignored due to %s\n", 834 pkgi->portdir, reason); 835 doHook(pkgi, "hook_pkg_ignored", 836 HookPkgIgnored, 0); 837 } else { 838 ++BuildSkipCount; 839 dlog(DLOG_SKIP, "[XXX] %s skipped due to %s\n", 840 pkgi->portdir, reason); 841 doHook(pkgi, "hook_pkg_skipped", 842 HookPkgSkipped, 0); 843 } 844 free(reason); 845 ++BuildCount; 846 continue; 847 } 848 849 /* 850 * Block while no slots are available. waitbuild() 851 * will clean out any DONE states. 852 */ 853 while (RunningWorkers >= DynamicMaxWorkers || 854 RunningWorkers >= MaxWorkers - FailedWorkers) { 855 waitbuild(RunningWorkers, 1); 856 } 857 858 /* 859 * Find an available worker slot, there should be at 860 * least one. 861 */ 862 for (i = 0; i < MaxWorkers; ++i) { 863 n = IterateWorker % MaxWorkers; 864 work = &WorkerAry[n]; 865 866 if (work->state == WORKER_DONE || 867 work->state == WORKER_FAILED) { 868 workercomplete(work); 869 } 870 if (work->state == WORKER_NONE || 871 work->state == WORKER_IDLE) { 872 if (n <= MaxWorkers / 2) { 873 startworker(pkgi, work); 874 } else { 875 startworker(ipkg, work); 876 } 877 /*RunStatsUpdate(work);*/ 878 break; 879 } 880 ++IterateWorker; 881 } 882 ddassert(i != MaxWorkers); 883 } 884 RunStatsSync(); 885 886 /* 887 * We disposed of the whole list 888 */ 889 free(idep_ary); 890 free(depi_ary); 891 *build_listp = NULL; 892 *build_tailp = build_listp; 893 } 894 895 typedef const pkg_t *pkg_tt; 896 897 static int 898 qsort_idep(const void *pkg1_arg, const void *pkg2_arg) 899 { 900 const pkg_t *pkg1 = *(const pkg_tt *)pkg1_arg; 901 const pkg_t *pkg2 = *(const pkg_tt *)pkg2_arg; 902 903 return (pkg2->idep_count - pkg1->idep_count); 904 } 905 906 static int 907 qsort_depi(const void *pkg1_arg, const void *pkg2_arg) 908 { 909 const pkg_t *pkg1 = *(const pkg_tt *)pkg1_arg; 910 const pkg_t *pkg2 = *(const pkg_tt *)pkg2_arg; 911 912 return ((pkg2->depi_count * pkg2->depi_depth) - 913 (pkg1->depi_count * pkg1->depi_depth)); 914 } 915 916 /* 917 * Frontend starts a pkg up on a worker 918 * 919 * WorkerMutex must be held. 920 */ 921 static void 922 startworker(pkg_t *pkg, worker_t *work) 923 { 924 switch(work->state) { 925 case WORKER_NONE: 926 pthread_create(&work->td, NULL, childBuilderThread, work); 927 work->state = WORKER_IDLE; 928 /* fall through */ 929 case WORKER_IDLE: 930 work->pkg_dep_size = 931 childInstallPkgDeps_recurse(NULL, &pkg->idepon_list, 0, 1, 0); 932 childInstallPkgDeps_recurse(NULL, &pkg->idepon_list, 1, 1, 0); 933 RunningPkgDepSize += work->pkg_dep_size; 934 935 dlog(DLOG_ALL, "[%03d] START %s " 936 "##idep=%02d depi=%02d/%02d dep=%-4.2fG\n", 937 work->index, pkg->portdir, 938 pkg->idep_count, pkg->depi_count, pkg->depi_depth, 939 (double)work->pkg_dep_size / (double)ONEGB); 940 941 cleanworker(work); 942 pkg->flags |= PKGF_RUNNING; 943 work->pkg = pkg; 944 pthread_cond_signal(&work->cond); 945 ++RunningWorkers; 946 /*RunStatsUpdate(work);*/ 947 break; 948 case WORKER_PENDING: 949 case WORKER_RUNNING: 950 case WORKER_DONE: 951 case WORKER_FAILED: 952 case WORKER_FROZEN: 953 case WORKER_EXITING: 954 default: 955 dfatal("startworker: [%03d] Unexpected state %d for worker %d", 956 work->index, work->state, work->index); 957 break; 958 } 959 } 960 961 static void 962 cleanworker(worker_t *work) 963 { 964 work->state = WORKER_PENDING; 965 work->flags = 0; 966 work->accum_error = 0; 967 work->start_time = time(NULL); 968 } 969 970 /* 971 * Frontend finishes up a completed pkg on a worker. 972 * 973 * If the worker is in a FAILED state we clean the pkg out but (for now) 974 * leave it in its failed state so we can debug. At this point 975 * workercomplete() will be called every once in a while on the state 976 * and we have to deal with the NULL pkg. 977 * 978 * WorkerMutex must be held. 979 */ 980 static void 981 workercomplete(worker_t *work) 982 { 983 pkg_t *pkg; 984 time_t t; 985 int h; 986 int m; 987 int s; 988 989 /* 990 * Steady state FAILED case. 991 */ 992 if (work->state == WORKER_FAILED) { 993 if (work->pkg == NULL) 994 return; 995 } 996 997 t = time(NULL) - work->start_time; 998 h = t / 3600; 999 m = t / 60 % 60; 1000 s = t % 60; 1001 1002 /* 1003 * Reduce total dep size 1004 */ 1005 RunningPkgDepSize -= work->pkg_dep_size; 1006 RunningPkgDepSize -= work->memuse; 1007 work->pkg_dep_size = 0; 1008 work->memuse = 0; 1009 1010 /* 1011 * Process pkg out of the worker 1012 */ 1013 pkg = work->pkg; 1014 if (pkg->flags & (PKGF_ERROR|PKGF_NOBUILD)) { 1015 pkg->flags |= PKGF_FAILURE; 1016 1017 /* 1018 * This NOBUILD condition XXX can occur if the package is 1019 * not allowed to be built. 1020 */ 1021 if (pkg->flags & PKGF_NOBUILD) { 1022 char *reason; 1023 1024 reason = buildskipreason(NULL, pkg); 1025 if (pkg->flags & PKGF_NOBUILD_I) { 1026 ++BuildIgnoreCount; 1027 dlog(DLOG_SKIP, "[%03d] IGNORD %s - %s\n", 1028 work->index, pkg->portdir, reason); 1029 doHook(pkg, "hook_pkg_ignored", 1030 HookPkgIgnored, 0); 1031 } else { 1032 ++BuildSkipCount; 1033 dlog(DLOG_SKIP, "[%03d] SKIPPD %s - %s\n", 1034 work->index, pkg->portdir, reason); 1035 doHook(pkg, "hook_pkg_skipped", 1036 HookPkgSkipped, 0); 1037 } 1038 free(reason); 1039 } else { 1040 char skipbuf[16]; 1041 int scount; 1042 1043 scount = buildskipcount_dueto(pkg, 1); 1044 buildskipcount_dueto(pkg, 0); 1045 if (scount) 1046 snprintf(skipbuf, sizeof(skipbuf), " (%d)", scount); 1047 else 1048 skipbuf[0] = 0; 1049 1050 ++BuildFailCount; 1051 dlog(DLOG_FAIL | DLOG_RED, 1052 "[%03d] FAILURE %s%s ##%16.16s %02d:%02d:%02d\n", 1053 work->index, pkg->portdir, skipbuf, 1054 getphasestr(work->phase), 1055 h, m, s); 1056 doHook(pkg, "hook_pkg_failure", HookPkgFailure, 0); 1057 } 1058 } else { 1059 pkg->flags |= PKGF_SUCCESS; 1060 ++BuildSuccessCount; 1061 dlog(DLOG_SUCC | DLOG_GRN, 1062 "[%03d] SUCCESS %s ##%02d:%02d:%02d\n", 1063 work->index, pkg->portdir, h, m, s); 1064 doHook(pkg, "hook_pkg_success", HookPkgSuccess, 0); 1065 } 1066 ++BuildCount; 1067 pkg->flags &= ~PKGF_BUILDLIST; 1068 pkg->flags &= ~PKGF_RUNNING; 1069 work->pkg = NULL; 1070 --RunningWorkers; 1071 1072 if (work->state == WORKER_FAILED) { 1073 dlog(DLOG_ALL, "[%03d] XXX/XXX WORKER IS IN A FAILED STATE\n", 1074 work->index); 1075 ++FailedWorkers; 1076 } else if (work->flags & WORKERF_FREEZE) { 1077 dlog(DLOG_ALL, "[%03d] FROZEN(DEBUG) %s\n", 1078 work->index, pkg->portdir); 1079 work->state = WORKER_FROZEN; 1080 } else { 1081 work->state = WORKER_IDLE; 1082 } 1083 } 1084 1085 /* 1086 * Wait for one or more workers to complete. 1087 * 1088 * WorkerMutex must be held. 1089 */ 1090 static void 1091 waitbuild(int whilematch, int dynamicmax) 1092 { 1093 static time_t wblast_time; 1094 static time_t dmlast_time; 1095 struct timespec ts; 1096 worker_t *work; 1097 time_t t; 1098 int i; 1099 1100 if (whilematch == 0) 1101 whilematch = 1; 1102 1103 while (RunningWorkers == whilematch) { 1104 for (i = 0; i < MaxWorkers; ++i) { 1105 work = &WorkerAry[i]; 1106 if (work->state == WORKER_DONE || 1107 work->state == WORKER_FAILED) { 1108 workercomplete(work); 1109 } else { 1110 pthread_cond_signal(&work->cond); 1111 } 1112 RunStatsUpdate(work, NULL); 1113 } 1114 RunStatsUpdateTop(); 1115 RunStatsUpdateLogs(); 1116 RunStatsSync(); 1117 if (RunningWorkers == whilematch) { 1118 clock_gettime(CLOCK_REALTIME, &ts); 1119 ts.tv_sec += 1; 1120 ts.tv_nsec = 0; 1121 pthread_cond_timedwait(&WorkerCond, &WorkerMutex, &ts); 1122 } 1123 1124 /* 1125 * Dynamically reduce MaxWorkers based on the load. When 1126 * the load exceeds 2 x ncpus we reduce the number of workers 1127 * up to 75% of MaxWorkers @ (5 x ncpus) load. 1128 * 1129 * Dynamically reduce MaxWorkers based on swap use, starting 1130 * at 10% swap and up to 75% of MaxWorkers at 40% swap. 1131 * 1132 * NOTE! Generally speaking this allows more workers to be 1133 * configured which helps in two ways. First it allows 1134 * a higher build rate for smaller packages. Second 1135 * it allows dsynth to ratchet-down the number of slots 1136 * when large packages are forcing the load up. 1137 * 1138 * A high load doesn't hurt efficiency, but swap usage 1139 * due to loading the tmpfs in lots of worker slots up 1140 * with tons of pkg install's (pre-reqs for a build) 1141 * does. Reducing the number of worker slots has a 1142 * huge beneficial effect on reducing swap use / paging. 1143 */ 1144 t = time(NULL); 1145 if (dynamicmax && (wblast_time == 0 || 1146 (unsigned)(t - wblast_time) >= 5)) { 1147 double min_load = 1.5 * NumCores; 1148 double max_load = 5.0 * NumCores; 1149 double min_swap = 0.10; 1150 double max_swap = 0.40; 1151 double dload[3]; 1152 double dswap; 1153 int max1; 1154 int max2; 1155 int max3; 1156 int max_sel; 1157 int noswap; 1158 1159 wblast_time = t; 1160 1161 /* 1162 * Cap based on load. This is back-loaded. 1163 */ 1164 getloadavg(dload, 3); 1165 if (dload[0] < min_load) { 1166 max1 = MaxWorkers; 1167 } else if (dload[0] <= max_load) { 1168 max1 = MaxWorkers - 1169 MaxWorkers * 0.75 * 1170 (dload[0] - min_load) / 1171 (max_load - min_load); 1172 } else { 1173 max1 = MaxWorkers * 25 / 100; 1174 } 1175 1176 /* 1177 * Cap based on swap use. This is back-loaded. 1178 */ 1179 dswap = getswappct(&noswap); 1180 if (dswap < min_swap) { 1181 max2 = MaxWorkers; 1182 } else if (dswap <= max_swap) { 1183 max2 = MaxWorkers - 1184 MaxWorkers * 0.75 * 1185 (dswap - min_swap) / 1186 (max_swap - min_swap); 1187 } else { 1188 max2 = MaxWorkers * 25 / 100; 1189 } 1190 1191 /* 1192 * Cap based on aggregate pkg-dependency memory 1193 * use installed in worker slots. This is 1194 * front-loaded. 1195 * 1196 * Since it can take a while for workers to retire 1197 * (to reduce RunningPkgDepSize), just set our 1198 * target 1 below the current run count to allow 1199 * jobs to retire without being replaced with new 1200 * jobs. 1201 * 1202 * In addition, in order to avoid a paging 'shock', 1203 * We enforce a 30 second-per-increment slow-start 1204 * once RunningPkgDepSize exceeds 1/2 the target. 1205 */ 1206 if (RunningPkgDepSize > PkgDepMemoryTarget) { 1207 max3 = RunningWorkers - 1; 1208 } else if (RunningPkgDepSize > PkgDepMemoryTarget / 2) { 1209 if (dmlast_time == 0 || 1210 (unsigned)(t - dmlast_time) >= 30) { 1211 dmlast_time = t; 1212 max3 = RunningWorkers + 1; 1213 } else { 1214 max3 = RunningWorkers; 1215 } 1216 } else { 1217 max3 = MaxWorkers; 1218 } 1219 1220 /* 1221 * Priority reduction, convert to DynamicMaxWorkers 1222 */ 1223 max_sel = max1; 1224 if (max_sel > max2) 1225 max_sel = max2; 1226 if (max_sel > max3) 1227 max_sel = max3; 1228 1229 /* 1230 * Restrict to allowed range, and also handle 1231 * slow-start. 1232 */ 1233 if (max_sel < 1) 1234 max_sel = 1; 1235 if (max_sel > DynamicMaxWorkers + 1) 1236 max_sel = DynamicMaxWorkers + 1; 1237 if (max_sel > MaxWorkers) 1238 max_sel = MaxWorkers; 1239 1240 /* 1241 * Stop waiting if DynamicMaxWorkers is going to 1242 * increase. 1243 */ 1244 if (DynamicMaxWorkers < max1) 1245 whilematch = -1; 1246 1247 /* 1248 * And adjust 1249 */ 1250 if (DynamicMaxWorkers != max1) { 1251 dlog(DLOG_ALL | DLOG_FILTER, 1252 "[XXX] Load=%-6.2f(%2d) " 1253 "Swap=%-3.2f%%(%2d) " 1254 "Mem=%3.2fG(%2d) " 1255 "Adjust Workers %d->%d\n", 1256 dload[0], max1, 1257 dswap * 100.0, max2, 1258 RunningPkgDepSize / (double)ONEGB, max3, 1259 DynamicMaxWorkers, max_sel); 1260 DynamicMaxWorkers = max_sel; 1261 } 1262 } 1263 } 1264 } 1265 1266 1267 /* 1268 * Worker pthread (WorkerAry) 1269 * 1270 * This thread belongs to the dsynth master process and handled a worker slot. 1271 * (this_thread) -> WORKER fork/exec (WorkerPocess) -> (pty) -> sub-processes. 1272 */ 1273 static void * 1274 childBuilderThread(void *arg) 1275 { 1276 char *envary[1] = { NULL }; 1277 worker_t *work = arg; 1278 wmsg_t wmsg; 1279 pkg_t *pkg; 1280 pid_t pid; 1281 int status; 1282 int flags; 1283 volatile int dowait; 1284 char slotbuf[8]; 1285 char fdbuf[8]; 1286 char flagsbuf[16]; 1287 1288 pthread_mutex_lock(&WorkerMutex); 1289 while (work->terminate == 0) { 1290 dowait = 1; 1291 1292 switch(work->state) { 1293 case WORKER_IDLE: 1294 break; 1295 case WORKER_PENDING: 1296 /* 1297 * Fork the management process for the pkg operation 1298 * on this worker slot. 1299 * 1300 * This process will set up the environment, do the 1301 * mounts, will become the reaper, and will process 1302 * pipe commands and handle chroot operations. 1303 * 1304 * NOTE: If SOCK_CLOEXEC is not supported WorkerMutex 1305 * is sufficient to interlock F_SETFD FD_CLOEXEC 1306 * operations. 1307 */ 1308 ddassert(work->pkg); 1309 if (socketpair(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 1310 PF_UNSPEC, work->fds)) { 1311 dfatal_errno("socketpair() during worker fork"); 1312 } 1313 snprintf(slotbuf, sizeof(slotbuf), "%d", work->index); 1314 snprintf(fdbuf, sizeof(fdbuf), "3"); 1315 1316 /* 1317 * Pass global flags and add-in the DEBUGSTOP if 1318 * the package is flagged for debugging. 1319 */ 1320 flags = WorkerProcFlags; 1321 if (work->pkg->flags & PKGF_DEBUGSTOP) { 1322 flags |= WORKER_PROC_DEBUGSTOP; 1323 } else { 1324 flags &= ~WORKER_PROC_DEBUGSTOP; 1325 } 1326 snprintf(flagsbuf, sizeof(flagsbuf), "%d", flags); 1327 1328 /* 1329 * fds[0] - master 1330 * fds[1] - slave 1331 * 1332 * We pass the salve descriptor in fd 3 and close all 1333 * other descriptors for security. 1334 */ 1335 pthread_mutex_unlock(&WorkerMutex); 1336 pid = vfork(); 1337 if (pid == 0) { 1338 close(work->fds[0]); 1339 dup2(work->fds[1], 3); 1340 closefrom(4); 1341 fcntl(3, F_SETFD, 0); 1342 execle(DSynthExecPath, DSynthExecPath, 1343 "-p", Profile, 1344 "WORKER", slotbuf, fdbuf, 1345 work->pkg->portdir, work->pkg->pkgfile, 1346 flagsbuf, 1347 NULL, envary); 1348 write(2, "EXECLE FAILURE\n", 15); 1349 _exit(1); 1350 } 1351 pthread_mutex_lock(&WorkerMutex); 1352 close(work->fds[1]); 1353 work->phase = PHASE_PENDING; 1354 work->lines = 0; 1355 work->memuse = 0; 1356 work->pid = pid; 1357 work->state = WORKER_RUNNING; 1358 /* fall through */ 1359 case WORKER_RUNNING: 1360 /* 1361 * Poll for status updates, if NULL is returned 1362 * and status is non-zero, the communications link 1363 * failed unexpectedly. 1364 */ 1365 pkg = work->pkg; 1366 pthread_mutex_unlock(&WorkerMutex); 1367 status = ipcreadmsg(work->fds[0], &wmsg); 1368 pthread_mutex_lock(&WorkerMutex); 1369 1370 if (status == 0) { 1371 /* 1372 * Normal message, can include normal 1373 * termination which changes us over 1374 * to another state. 1375 */ 1376 dowait = 0; 1377 switch(wmsg.cmd) { 1378 case WMSG_CMD_INSTALL_PKGS: 1379 wmsg.cmd = WMSG_RES_INSTALL_PKGS; 1380 wmsg.status = childInstallPkgDeps(work); 1381 pthread_mutex_unlock(&WorkerMutex); 1382 ipcwritemsg(work->fds[0], &wmsg); 1383 pthread_mutex_lock(&WorkerMutex); 1384 break; 1385 case WMSG_CMD_STATUS_UPDATE: 1386 work->phase = wmsg.phase; 1387 work->lines = wmsg.lines; 1388 if (work->memuse != wmsg.memuse) { 1389 RunningPkgDepSize += 1390 wmsg.memuse - work->memuse; 1391 work->memuse = wmsg.memuse; 1392 } 1393 break; 1394 case WMSG_CMD_SUCCESS: 1395 work->flags |= WORKERF_SUCCESS; 1396 break; 1397 case WMSG_CMD_FAILURE: 1398 work->flags |= WORKERF_FAILURE; 1399 break; 1400 case WMSG_CMD_FREEZEWORKER: 1401 work->flags |= WORKERF_FREEZE; 1402 break; 1403 default: 1404 break; 1405 } 1406 RunStatsUpdate(work, NULL); 1407 RunStatsSync(); 1408 } else { 1409 close(work->fds[0]); 1410 pthread_mutex_unlock(&WorkerMutex); 1411 while (waitpid(work->pid, &status, 0) < 0 && 1412 errno == EINTR) { 1413 ; 1414 } 1415 pthread_mutex_lock(&WorkerMutex); 1416 1417 if (work->flags & WORKERF_SUCCESS) { 1418 pkg->flags |= PKGF_SUCCESS; 1419 work->state = WORKER_DONE; 1420 } else if (work->flags & WORKERF_FAILURE) { 1421 pkg->flags |= PKGF_FAILURE; 1422 work->state = WORKER_DONE; 1423 } else { 1424 pkg->flags |= PKGF_FAILURE; 1425 work->state = WORKER_FAILED; 1426 } 1427 work->flags |= WORKERF_STATUS_UPDATE; 1428 pthread_cond_signal(&WorkerCond); 1429 } 1430 break; 1431 case WORKER_DONE: 1432 /* 1433 * pkg remains attached until frontend processes the 1434 * completion. The frontend will then set the state 1435 * back to idle. 1436 */ 1437 break; 1438 case WORKER_FAILED: 1439 /* 1440 * A worker failure means that the worker did not 1441 * send us a WMSG_CMD_SUCCESS or WMSG_CMD_FAILURE 1442 * ipc before terminating. 1443 * 1444 * We just sit in this state until the front-end 1445 * does something about it. 1446 */ 1447 break; 1448 case WORKER_FROZEN: 1449 /* 1450 * A worker getting frozen is debug-related. We 1451 * just sit in this state (likely forever). 1452 */ 1453 break; 1454 default: 1455 dfatal("worker: [%03d] Unexpected state %d " 1456 "for worker %d", 1457 work->index, work->state, work->index); 1458 /* NOT REACHED */ 1459 break; 1460 } 1461 1462 /* 1463 * The dsynth frontend will poll us approximately once 1464 * a second (its variable). 1465 */ 1466 if (dowait) 1467 pthread_cond_wait(&work->cond, &WorkerMutex); 1468 } 1469 1470 /* 1471 * Scrap the comm socket if running, this should cause the worker 1472 * process to kill its sub-programs and cleanup. 1473 */ 1474 if (work->state == WORKER_RUNNING) { 1475 pthread_mutex_unlock(&WorkerMutex); 1476 close(work->fds[0]); 1477 while (waitpid(work->pid, &status, 0) < 0 && 1478 errno == EINTR); 1479 pthread_mutex_lock(&WorkerMutex); 1480 } 1481 1482 /* 1483 * Final handshake 1484 */ 1485 work->state = WORKER_EXITING; 1486 pthread_cond_signal(&WorkerCond); 1487 pthread_mutex_unlock(&WorkerMutex); 1488 1489 return NULL; 1490 } 1491 1492 /* 1493 * Install all the binary packages (we have already built them) that 1494 * the current work package depends on, without duplicates, in a script 1495 * which will be run from within the specified work jail. 1496 * 1497 * Locked by WorkerMutex (global) 1498 */ 1499 static int 1500 childInstallPkgDeps(worker_t *work) 1501 { 1502 char *buf; 1503 FILE *fp; 1504 1505 if (PKGLIST_EMPTY(&work->pkg->idepon_list)) 1506 return 0; 1507 1508 asprintf(&buf, "%s/tmp/dsynth_install_pkgs", work->basedir); 1509 fp = fopen(buf, "w"); 1510 ddassert(fp != NULL); 1511 fprintf(fp, "#!/bin/sh\n"); 1512 fprintf(fp, "#\n"); 1513 fchmod(fileno(fp), 0755); 1514 1515 childInstallPkgDeps_recurse(fp, &work->pkg->idepon_list, 0, 1, 0); 1516 childInstallPkgDeps_recurse(fp, &work->pkg->idepon_list, 1, 1, 0); 1517 fprintf(fp, "\nexit 0\n"); 1518 fclose(fp); 1519 freestrp(&buf); 1520 1521 return 1; 1522 } 1523 1524 /* 1525 * Recursive child install dependencies. 1526 * 1527 * first_one_only is only specified if the pkg the list comes from 1528 * is a generic unflavored package that has flavors, telling us to 1529 * dive the first flavor only. 1530 * 1531 * However, in nearly all cases this flag will now be zero because 1532 * this code now dives the first flavor when encountering a dummy node 1533 * and clears nfirst on success. Hence if you are asking why 'nfirst' 1534 * is set to 1, and then zero, instead of just being removed entirely, 1535 * it is because there might still be an edge case here. 1536 */ 1537 static size_t 1538 childInstallPkgDeps_recurse(FILE *fp, pkglink_t *list, int undoit, 1539 int depth, int first_one_only) 1540 { 1541 pkglink_t *link; 1542 pkg_t *pkg; 1543 size_t tot = 0; 1544 int ndepth; 1545 int nfirst; 1546 1547 PKGLIST_FOREACH(link, list) { 1548 pkg = link->pkg; 1549 1550 /* 1551 * We don't want to mess up our depth test just below if 1552 * a DUMMY node had to be inserted. The nodes under the 1553 * dummy node. 1554 * 1555 * The elements under a dummy node represent all the flabor, 1556 * a dependency that directly references a dummy node only 1557 * uses the first flavor (first_one_only / nfirst). 1558 */ 1559 ndepth = (pkg->flags & PKGF_DUMMY) ? depth : depth + 1; 1560 nfirst = (pkg->flags & PKGF_DUMMY) ? 1 : 0; 1561 1562 /* 1563 * We only need all packages for the top-level dependencies. 1564 * The deeper ones only need DEP_TYPE_LIB and DEP_TYPE_RUN 1565 * (types greater than DEP_TYPE_BUILD) since they are already 1566 * built. 1567 */ 1568 if (depth > 1 && link->dep_type <= DEP_TYPE_BUILD) { 1569 if (first_one_only) 1570 break; 1571 continue; 1572 } 1573 1574 /* 1575 * If this is a dummy node with no package, the originator 1576 * is requesting a flavored package. We select the default 1577 * flavor which we presume is the first one. 1578 */ 1579 if (pkg->pkgfile == NULL && (pkg->flags & PKGF_DUMMY)) { 1580 pkg_t *spkg = pkg->idepon_list.next->pkg; 1581 1582 if (spkg) { 1583 if (fp) { 1584 fprintf(fp, 1585 "echo 'UNFLAVORED %s -> use " 1586 "%s'\n", 1587 pkg->portdir, 1588 spkg->portdir); 1589 } 1590 pkg = spkg; 1591 nfirst = 0; 1592 } else { 1593 if (fp) { 1594 fprintf(fp, 1595 "echo 'CANNOT FIND DEFAULT " 1596 "FLAVOR FOR %s'\n", 1597 pkg->portdir); 1598 } 1599 } 1600 } 1601 1602 if (undoit) { 1603 if (pkg->dsynth_install_flg == 1) { 1604 pkg->dsynth_install_flg = 0; 1605 tot += childInstallPkgDeps_recurse(fp, 1606 &pkg->idepon_list, 1607 undoit, 1608 ndepth, nfirst); 1609 } 1610 if (first_one_only) 1611 break; 1612 continue; 1613 } 1614 1615 if (pkg->dsynth_install_flg) { 1616 if (DebugOpt >= 2 && pkg->pkgfile && fp) { 1617 fprintf(fp, "echo 'AlreadyHave %s'\n", 1618 pkg->pkgfile); 1619 } 1620 if (first_one_only) 1621 break; 1622 continue; 1623 } 1624 1625 tot += childInstallPkgDeps_recurse(fp, &pkg->idepon_list, 1626 undoit, ndepth, nfirst); 1627 if (pkg->dsynth_install_flg) { 1628 if (first_one_only) 1629 break; 1630 continue; 1631 } 1632 pkg->dsynth_install_flg = 1; 1633 1634 /* 1635 * Generate package installation command 1636 */ 1637 if (fp && pkg->pkgfile) { 1638 fprintf(fp, "echo 'Installing /packages/All/%s'\n", 1639 pkg->pkgfile); 1640 fprintf(fp, "pkg install -q -y /packages/All/%s " 1641 "|| exit 1\n", 1642 pkg->pkgfile); 1643 } else if (fp) { 1644 fprintf(fp, "echo 'CANNOT FIND PKG FOR %s'\n", 1645 pkg->portdir); 1646 } 1647 1648 if (pkg->pkgfile) { 1649 struct stat st; 1650 char *path; 1651 char *ptr; 1652 1653 asprintf(&path, "%s/%s", RepositoryPath, pkg->pkgfile); 1654 ptr = strrchr(pkg->pkgfile, '.'); 1655 if (stat(path, &st) == 0) { 1656 if (strcmp(ptr, ".tar") == 0) 1657 tot += st.st_size; 1658 else if (strcmp(ptr, ".tgz") == 0) 1659 tot += st.st_size * 3; 1660 else if (strcmp(ptr, ".txz") == 0) 1661 tot += st.st_size * 5; 1662 else if (strcmp(ptr, ".tbz") == 0) 1663 tot += st.st_size * 3; 1664 else 1665 tot += st.st_size * 2; 1666 } 1667 free(path); 1668 } 1669 if (first_one_only) 1670 break; 1671 } 1672 return tot; 1673 } 1674 1675 /* 1676 * Worker process interactions. 1677 * 1678 * The worker process is responsible for managing the build of a single 1679 * package. It is exec'd by the master dsynth and only loads the 1680 * configuration. 1681 * 1682 * This process does not run in the chroot. It will become the reaper for 1683 * all sub-processes and it will enter the chroot to execute various phases. 1684 * It catches SIGINTR, SIGHUP, and SIGPIPE and will iterate, terminate, and 1685 * reap all sub-process upon kill or exit. 1686 * 1687 * The command line forwarded to this function is: 1688 * 1689 * WORKER slot# socketfd portdir/subdir 1690 * 1691 * TERM=dumb 1692 * USER=root 1693 * HOME=/root 1694 * LANG=C 1695 * SSL_NO_VERIFY_PEER=1 1696 * USE_PACKAGE_DEPENDS_ONLY=1 1697 * PORTSDIR=/xports 1698 * PORT_DBDIR=/options For ports options 1699 * PACKAGE_BUILDING=yes Don't build packages that aren't legally 1700 * buildable for a binary repo. 1701 * PKG_DBDIR=/var/db/pkg 1702 * PKG_CACHEDIR=/var/cache/pkg 1703 * PKG_CREATE_VERBOSE=yes Ensure periodic output during packaging 1704 * (custom environment) 1705 * PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin 1706 * UNAME_s=DragonFly (example) 1707 * UNAME_v=DragonFly 5.7-SYNTH (example) 1708 * UNAME_p=x86_64 (example) 1709 * UNAME_m=x86_64 (example) 1710 * UNAME_r=5.7-SYNTH (example) 1711 * NO_DEPENDS=yes (conditional based on phase) 1712 * DISTDIR=/distfiles 1713 * WRKDIRPREFIX=/construction 1714 * BATCH=yes 1715 * MAKE_JOBS_NUMBER=n 1716 * 1717 * SETUP: 1718 * ldconfig -R 1719 * /usr/local/sbin/pkg-static install /packages/All/<the pkg pkg> 1720 * /usr/local/sbin/pkg-static install /packages/All/<pkg> 1721 * (for all dependencies) 1722 * 1723 * PHASES: make -C path FLAVOR=flavor <phase> 1724 * check-sanity 1725 * pkg-depends 1726 * fetch-depends 1727 * fetch 1728 * checksum 1729 * extract-depends 1730 * extract 1731 * patch-depends 1732 * patch 1733 * build-depends 1734 * lib-depends 1735 * configure 1736 * build 1737 * run-depends 1738 * stage 1739 * test (skipped) 1740 * check-plist ('dsynth test blahblah' or 'dsynth -D everything' only) 1741 * package e.g. /construction/lang/perl5.28/pkg/perl5-5.28.2.txz 1742 * install-mtree (skipped) 1743 * install (skipped) 1744 * deinstall (skipped) 1745 */ 1746 void 1747 WorkerProcess(int ac, char **av) 1748 { 1749 wmsg_t wmsg; 1750 int fd; 1751 int slot; 1752 int tmpfd; 1753 int pkgpkg = 0; 1754 int status; 1755 int len; 1756 int do_install_phase; 1757 char *portdir; 1758 char *pkgfile; 1759 char *flavor; 1760 char *buf; 1761 worker_t *work; 1762 bulk_t *bulk; 1763 pkg_t pkg; 1764 buildenv_t *benv; 1765 FILE *fp; 1766 1767 /* 1768 * Parse arguments 1769 */ 1770 if (ac != 6) { 1771 dlog(DLOG_ALL, "WORKER PROCESS %d- bad arguments\n", getpid()); 1772 exit(1); 1773 } 1774 slot = strtol(av[1], NULL, 0); 1775 fd = strtol(av[2], NULL, 0); /* master<->slave messaging */ 1776 portdir = av[3]; 1777 pkgfile = av[4]; 1778 flavor = strchr(portdir, '@'); 1779 if (flavor) { 1780 *flavor++ = 0; 1781 asprintf(&buf, "@%s", flavor); 1782 WorkerFlavorPrt = buf; 1783 buf = NULL; /* safety */ 1784 } 1785 WorkerProcFlags = strtol(av[5], NULL, 0); 1786 1787 bzero(&wmsg, sizeof(wmsg)); 1788 1789 setproctitle("[%02d] WORKER STARTUP %s%s", 1790 slot, portdir, WorkerFlavorPrt); 1791 1792 if (strcmp(portdir, "ports-mgmt/pkg") == 0) 1793 pkgpkg = 1; 1794 1795 signal(SIGTERM, phaseTerminateSignal); 1796 signal(SIGINT, phaseTerminateSignal); 1797 signal(SIGHUP, phaseTerminateSignal); 1798 1799 /* 1800 * Set up the environment 1801 */ 1802 setenv("TERM", "dumb", 1); 1803 setenv("USER", "root", 1); 1804 setenv("HOME", "/root", 1); 1805 setenv("LANG", "C", 1); 1806 setenv("SSL_NO_VERIFY_PEER", "1", 1); 1807 1808 addbuildenv("USE_PACKAGE_DEPENDS_ONLY", "yes", BENV_MAKECONF); 1809 addbuildenv("PORTSDIR", "/xports", BENV_MAKECONF); 1810 addbuildenv("PORT_DBDIR", "/options", BENV_MAKECONF); 1811 addbuildenv("PKG_DBDIR", "/var/db/pkg", BENV_MAKECONF); 1812 addbuildenv("PKG_CACHEDIR", "/var/cache/pkg", BENV_MAKECONF); 1813 addbuildenv("PKG_SUFX", UsePkgSufx, BENV_MAKECONF); 1814 if (WorkerProcFlags & WORKER_PROC_DEVELOPER) 1815 addbuildenv("DEVELOPER", "1", BENV_MAKECONF); 1816 1817 /* 1818 * CCache is a horrible unreliable hack but... leave the 1819 * mechanism in-place in case someone has a death wish. 1820 */ 1821 if (UseCCache) { 1822 addbuildenv("WITH_CCACHE_BUILD", "yes", BENV_MAKECONF); 1823 addbuildenv("CCACHE_DIR", "/ccache", BENV_MAKECONF); 1824 } 1825 1826 addbuildenv("UID", "0", BENV_MAKECONF); 1827 addbuildenv("ARCH", ArchitectureName, BENV_MAKECONF); 1828 1829 #ifdef __DragonFly__ 1830 addbuildenv("OPSYS", "DragonFly", BENV_MAKECONF); 1831 addbuildenv("DFLYVERSION", VersionFromParamHeader, BENV_MAKECONF); 1832 addbuildenv("OSVERSION", "9999999", BENV_MAKECONF); 1833 #else 1834 #error "Need OS-specific data to generate make.conf" 1835 #endif 1836 1837 addbuildenv("OSREL", ReleaseName, BENV_MAKECONF); 1838 addbuildenv("_OSRELEASE", VersionOnlyName, BENV_MAKECONF); 1839 1840 setenv("PATH", 1841 "/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin", 1842 1); 1843 1844 setenv("UNAME_s", OperatingSystemName, 1); 1845 setenv("UNAME_v", VersionName, 1); 1846 setenv("UNAME_p", ArchitectureName, 1); 1847 setenv("UNAME_m", MachineName, 1); 1848 setenv("UNAME_r", ReleaseName, 1); 1849 1850 addbuildenv("DISTDIR", "/distfiles", BENV_MAKECONF); 1851 addbuildenv("WRKDIRPREFIX", "/construction", BENV_MAKECONF); 1852 addbuildenv("BATCH", "yes", BENV_MAKECONF); 1853 1854 /* 1855 * Special consideration 1856 * 1857 * PACKAGE_BUILDING - Disallow packaging ports which do not allow 1858 * for binary distribution. 1859 * 1860 * PKG_CREATE_VERBOSE - Ensure periodic output during the packaging 1861 * process to avoid a watchdog timeout. 1862 * 1863 */ 1864 addbuildenv("PACKAGE_BUILDING", "yes", BENV_MAKECONF); 1865 addbuildenv("PKG_CREATE_VERBOSE", "yes", BENV_MAKECONF); 1866 asprintf(&buf, "%d", MaxJobs); 1867 addbuildenv("MAKE_JOBS_NUMBER", buf, BENV_MAKECONF); 1868 freestrp(&buf); 1869 1870 if (flavor) 1871 setenv("FLAVOR", flavor, 1); 1872 1873 /* 1874 * Become the reaper 1875 */ 1876 if (procctl(P_PID, getpid(), PROC_REAP_ACQUIRE, NULL) < 0) 1877 dfatal_errno("procctl() - Cannot become reaper"); 1878 1879 /* 1880 * Initialize a worker structure 1881 */ 1882 DoInitBuild(slot); 1883 1884 bzero(&pkg, sizeof(pkg)); 1885 pkg.portdir = portdir; /* sans flavor */ 1886 pkg.pkgfile = pkgfile; 1887 if (strchr(portdir, '/')) 1888 len = strchr(portdir, '/') - portdir; 1889 else 1890 len = 0; 1891 1892 /* 1893 * Setup the logfile 1894 */ 1895 asprintf(&pkg.logfile, 1896 "%s/%*.*s___%s%s%s.log", 1897 LogsPath, len, len, portdir, 1898 ((portdir[len] == '/') ? portdir + len + 1 : portdir + len), 1899 (flavor ? "@" : ""), 1900 (flavor ? flavor : "")); 1901 tmpfd = open(pkg.logfile, O_RDWR|O_CREAT|O_TRUNC, 0666); 1902 if (tmpfd >= 0) { 1903 if (DebugOpt >= 2) { 1904 dlog(DLOG_ALL, "[%03d] %s LOGFILE %s\n", 1905 slot, pkg.portdir, pkg.logfile); 1906 } 1907 close(tmpfd); 1908 } else { 1909 dlog(DLOG_ALL, "[%03d] LOGFILE %s (create failed)\n", 1910 slot, pkg.logfile); 1911 } 1912 1913 /* 1914 * Setup the work structure. Because this is an exec'd sub-process, 1915 * there is only one work structure. 1916 */ 1917 work = &WorkerAry[0]; 1918 work->flavor = flavor; 1919 work->fds[0] = fd; 1920 work->pkg = &pkg; 1921 work->start_time = time(NULL); 1922 1923 /* 1924 * Do mounts 1925 */ 1926 SigWork = work; 1927 setproctitle("[%02d] WORKER MOUNTS %s%s", 1928 slot, portdir, WorkerFlavorPrt); 1929 DoWorkerMounts(work); 1930 1931 /* 1932 * Generate an /etc/make.conf in the build base 1933 */ 1934 asprintf(&buf, "%s/etc/make.conf", work->basedir); 1935 fp = fopen(buf, "w"); 1936 dassert_errno(fp, "Unable to create %s\n", buf); 1937 for (benv = BuildEnv; benv; benv = benv->next) { 1938 if (benv->type & BENV_PKGLIST) 1939 continue; 1940 if ((benv->type & BENV_CMDMASK) == BENV_MAKECONF) { 1941 if (DebugOpt >= 2) { 1942 dlog(DLOG_ALL, "[%03d] ENV %s=%s\n", 1943 slot, benv->label, benv->data); 1944 } 1945 fprintf(fp, "%s=%s\n", benv->label, benv->data); 1946 } 1947 } 1948 fclose(fp); 1949 freestrp(&buf); 1950 1951 /* 1952 * Set up our hooks 1953 */ 1954 if (UsingHooks) 1955 initbulk(childHookRun, MaxBulk); 1956 1957 /* 1958 * Start phases 1959 */ 1960 wmsg.cmd = WMSG_CMD_INSTALL_PKGS; 1961 ipcwritemsg(fd, &wmsg); 1962 status = ipcreadmsg(fd, &wmsg); 1963 if (status < 0 || wmsg.cmd != WMSG_RES_INSTALL_PKGS) 1964 dfatal("pkg installation handshake failed"); 1965 do_install_phase = wmsg.status; 1966 1967 wmsg.cmd = WMSG_CMD_STATUS_UPDATE; 1968 wmsg.phase = PHASE_INSTALL_PKGS; 1969 wmsg.lines = 0; 1970 1971 status = ipcwritemsg(fd, &wmsg); 1972 1973 if (pkgpkg) { 1974 dophase(work, &wmsg, 1975 WDOG5, PHASE_PACKAGE, "package"); 1976 } else { 1977 if (do_install_phase) { 1978 dophase(work, &wmsg, 1979 WDOG4, PHASE_INSTALL_PKGS, "setup"); 1980 } 1981 dophase(work, &wmsg, 1982 WDOG2, PHASE_CHECK_SANITY, "check-sanity"); 1983 dophase(work, &wmsg, 1984 WDOG2, PHASE_PKG_DEPENDS, "pkg-depends"); 1985 dophase(work, &wmsg, 1986 WDOG7, PHASE_FETCH_DEPENDS, "fetch-depends"); 1987 dophase(work, &wmsg, 1988 WDOG7, PHASE_FETCH, "fetch"); 1989 dophase(work, &wmsg, 1990 WDOG2, PHASE_CHECKSUM, "checksum"); 1991 dophase(work, &wmsg, 1992 WDOG3, PHASE_EXTRACT_DEPENDS, "extract-depends"); 1993 dophase(work, &wmsg, 1994 WDOG3, PHASE_EXTRACT, "extract"); 1995 dophase(work, &wmsg, 1996 WDOG2, PHASE_PATCH_DEPENDS, "patch-depends"); 1997 dophase(work, &wmsg, 1998 WDOG2, PHASE_PATCH, "patch"); 1999 dophase(work, &wmsg, 2000 WDOG5, PHASE_BUILD_DEPENDS, "build-depends"); 2001 dophase(work, &wmsg, 2002 WDOG5, PHASE_LIB_DEPENDS, "lib-depends"); 2003 dophase(work, &wmsg, 2004 WDOG3, PHASE_CONFIGURE, "configure"); 2005 dophase(work, &wmsg, 2006 WDOG9, PHASE_BUILD, "build"); 2007 dophase(work, &wmsg, 2008 WDOG5, PHASE_RUN_DEPENDS, "run-depends"); 2009 dophase(work, &wmsg, 2010 WDOG5, PHASE_STAGE, "stage"); 2011 #if 0 2012 dophase(work, &wmsg, 2013 WDOG5, PHASE_TEST, "test"); 2014 #endif 2015 if (WorkerProcFlags & WORKER_PROC_CHECK_PLIST) { 2016 dophase(work, &wmsg, 2017 WDOG1, PHASE_CHECK_PLIST, "check-plist"); 2018 } 2019 dophase(work, &wmsg, 2020 WDOG5, PHASE_PACKAGE, "package"); 2021 #if 0 2022 dophase(work, &wmsg, 2023 WDOG5, PHASE_INSTALL_MTREE, "install-mtree"); 2024 dophase(work, &wmsg, 2025 WDOG5, PHASE_INSTALL, "install"); 2026 dophase(work, &wmsg, 2027 WDOG5, PHASE_DEINSTALL, "deinstall"); 2028 #endif 2029 } 2030 2031 if (MasterPtyFd >= 0) { 2032 close(MasterPtyFd); 2033 MasterPtyFd = -1; 2034 } 2035 2036 setproctitle("[%02d] WORKER CLEANUP %s%s", 2037 slot, portdir, WorkerFlavorPrt); 2038 2039 /* 2040 * Copy the package to the repo. 2041 */ 2042 if (work->accum_error == 0) { 2043 char *b1; 2044 char *b2; 2045 2046 asprintf(&b1, "%s/construction/%s/pkg/%s", 2047 work->basedir, pkg.portdir, pkg.pkgfile); 2048 asprintf(&b2, "%s/%s", RepositoryPath, pkg.pkgfile); 2049 if (copyfile(b1, b2)) { 2050 ++work->accum_error; 2051 dlog(DLOG_ALL, "[%03d] %s Unable to copy %s to %s\n", 2052 work->index, pkg.portdir, b1, b2); 2053 } 2054 free(b1); 2055 free(b2); 2056 } 2057 2058 /* 2059 * Unmount, unless we are in DebugStopMode. 2060 */ 2061 if ((WorkerProcFlags & WORKER_PROC_DEBUGSTOP) == 0) 2062 DoWorkerUnmounts(work); 2063 2064 /* 2065 * Send completion status to master dsynth worker thread. 2066 */ 2067 if (work->accum_error) { 2068 wmsg.cmd = WMSG_CMD_FAILURE; 2069 } else { 2070 wmsg.cmd = WMSG_CMD_SUCCESS; 2071 } 2072 ipcwritemsg(fd, &wmsg); 2073 if (WorkerProcFlags & WORKER_PROC_DEBUGSTOP) { 2074 wmsg.cmd = WMSG_CMD_FREEZEWORKER; 2075 ipcwritemsg(fd, &wmsg); 2076 } 2077 if (UsingHooks) { 2078 while ((bulk = getbulk()) != NULL) 2079 freebulk(bulk); 2080 donebulk(); 2081 } 2082 } 2083 2084 static void 2085 dophase(worker_t *work, wmsg_t *wmsg, int wdog, int phaseid, const char *phase) 2086 { 2087 pkg_t *pkg = work->pkg; 2088 char buf[1024]; 2089 pid_t pid; 2090 int status; 2091 int ms; 2092 pid_t wpid; 2093 int wpid_reaped; 2094 int fdlog; 2095 time_t start_time; 2096 time_t last_time; 2097 time_t next_time; 2098 time_t wdog_time; 2099 FILE *fp; 2100 2101 if (work->accum_error) 2102 return; 2103 setproctitle("[%02d] WORKER %-8.8s %s%s", 2104 work->index, phase, pkg->portdir, WorkerFlavorPrt); 2105 wmsg->phase = phaseid; 2106 if (ipcwritemsg(work->fds[0], wmsg) < 0) { 2107 dlog(DLOG_ALL, "[%03d] %s Lost Communication with dsynth, " 2108 "aborting worker\n", 2109 work->index, pkg->portdir); 2110 ++work->accum_error; 2111 return; 2112 } 2113 2114 /* 2115 * Execute the port make command in chroot on a pty. 2116 */ 2117 fflush(stdout); 2118 fflush(stderr); 2119 if (MasterPtyFd >= 0) { 2120 int slavefd; 2121 2122 /* 2123 * NOTE: We can't open the slave in the child because the 2124 * master may race a disconnection test. If we open 2125 * it in the parent our close() will flush any pending 2126 * output not read by the master (which is the same 2127 * parent process) and deadlock. 2128 * 2129 * Solve this by hand-shaking the slave tty to give 2130 * the master time to close its slavefd (after this 2131 * section). 2132 * 2133 * Leave the tty defaults intact, which also likely 2134 * means it will be in line-buffered mode, so handshake 2135 * with a full line. 2136 * 2137 * TODO: Our handshake probably echos back to the master pty 2138 * due to tty echo, and ends up in the log, so just 2139 * pass through a newline. 2140 */ 2141 slavefd = open(ptsname(MasterPtyFd), O_RDWR); 2142 dassert_errno(slavefd >= 0, "Cannot open slave pty"); 2143 2144 /* 2145 * Now do the fork. 2146 */ 2147 pid = fork(); 2148 if (pid == 0) { 2149 login_tty(slavefd); 2150 /* login_tty() closes slavefd */ 2151 } else { 2152 close(slavefd); 2153 } 2154 } else { 2155 /* 2156 * Initial MasterPtyFd for the slot, just use forkpty(). 2157 */ 2158 pid = forkpty(&MasterPtyFd, NULL, NULL, NULL); 2159 } 2160 2161 /* 2162 * The slave must make sure the master has time to close slavefd 2163 * in the re-use case before going its merry way. The master needs 2164 * to set terminal modes and the window as well. 2165 */ 2166 if (pid == 0) { 2167 /* 2168 * Slave waits for handshake 2169 */ 2170 char ttybuf[2]; 2171 2172 read(0, ttybuf, 1); 2173 } else { 2174 /* 2175 * We are going through a pty, so set the tty modes to 2176 * Set tty modes so we do not get ^M's in the log files. 2177 * 2178 * This isn't fatal if it doesn't work. Remember that 2179 * our output goes through the pty to the management 2180 * process which will log it. 2181 */ 2182 struct termios tio; 2183 struct winsize win; 2184 2185 if (tcgetattr(MasterPtyFd, &tio) == 0) { 2186 tio.c_oflag |= OPOST | ONOCR; 2187 tio.c_oflag &= ~(OCRNL | ONLCR); 2188 tio.c_iflag |= ICRNL; 2189 tio.c_iflag &= ~(INLCR | IGNCR); 2190 if (tcsetattr(MasterPtyFd, TCSANOW, &tio)) { 2191 printf("tcsetattr failed: %s\n", 2192 strerror(errno)); 2193 } 2194 2195 /* 2196 * Give the tty a non-zero columns field. 2197 * This fixes at least one port (textproc/po4a) 2198 */ 2199 if (ioctl(MasterPtyFd, TIOCGWINSZ, &win) == 0) { 2200 win.ws_col = 80; 2201 ioctl(MasterPtyFd, TIOCSWINSZ, &win); 2202 } else { 2203 printf("TIOCGWINSZ failed: %s\n", 2204 strerror(errno)); 2205 } 2206 2207 } else { 2208 printf("tcgetattr failed: %s\n", strerror(errno)); 2209 } 2210 2211 /* 2212 * Master issues handshake 2213 */ 2214 write(MasterPtyFd, "\n", 1); 2215 } 2216 2217 if (pid == 0) { 2218 /* 2219 * Additional phase-specific environment variables 2220 * 2221 * - Do not try to process missing depends outside of the 2222 * depends phases. Also relies on USE_PACKAGE_DEPENDS_ONLY 2223 * in the make.conf. 2224 */ 2225 switch(phaseid) { 2226 case PHASE_CHECK_SANITY: 2227 case PHASE_FETCH: 2228 case PHASE_CHECKSUM: 2229 case PHASE_EXTRACT: 2230 case PHASE_PATCH: 2231 case PHASE_CONFIGURE: 2232 case PHASE_STAGE: 2233 case PHASE_TEST: 2234 case PHASE_CHECK_PLIST: 2235 case PHASE_INSTALL_MTREE: 2236 case PHASE_INSTALL: 2237 case PHASE_DEINSTALL: 2238 break; 2239 case PHASE_PKG_DEPENDS: 2240 case PHASE_FETCH_DEPENDS: 2241 case PHASE_EXTRACT_DEPENDS: 2242 case PHASE_PATCH_DEPENDS: 2243 case PHASE_BUILD_DEPENDS: 2244 case PHASE_LIB_DEPENDS: 2245 case PHASE_RUN_DEPENDS: 2246 break; 2247 default: 2248 setenv("NO_DEPENDS", "1", 1); 2249 break; 2250 } 2251 2252 /* 2253 * Clean-up, chdir, and chroot. 2254 */ 2255 closefrom(3); 2256 if (chdir(work->basedir) < 0) 2257 dfatal_errno("chdir in phase initialization"); 2258 if (chroot(work->basedir) < 0) 2259 dfatal_errno("chroot in phase initialization"); 2260 2261 /* 2262 * We have a choice here on how to handle stdin (fd 0). 2263 * We can leave it connected to the pty in which case 2264 * the build will just block if it tries to ask a 2265 * question (and the watchdog will kill it, eventually), 2266 * or we can try to EOF the pty, or we can attach /dev/null 2267 * to descriptor 0. 2268 */ 2269 if (NullStdinOpt) { 2270 int fd; 2271 2272 fd = open("/dev/null", O_RDWR); 2273 dassert_errno(fd >= 0, "cannot open /dev/null"); 2274 if (fd != 0) { 2275 dup2(fd, 0); 2276 close(fd); 2277 } 2278 } 2279 2280 /* 2281 * Execute the appropriate command. 2282 */ 2283 switch(phaseid) { 2284 case PHASE_INSTALL_PKGS: 2285 snprintf(buf, sizeof(buf), "/tmp/dsynth_install_pkgs"); 2286 execl(buf, buf, NULL); 2287 break; 2288 default: 2289 snprintf(buf, sizeof(buf), "/xports/%s", pkg->portdir); 2290 execl(MAKE_BINARY, MAKE_BINARY, "-C", buf, phase, NULL); 2291 break; 2292 } 2293 _exit(1); 2294 } 2295 fcntl(MasterPtyFd, F_SETFL, O_NONBLOCK); 2296 2297 if (pid < 0) { 2298 dlog(DLOG_ALL, "[%03d] %s Fork Failed: %s\n", 2299 work->index, pkg->logfile, strerror(errno)); 2300 ++work->accum_error; 2301 return; 2302 } 2303 2304 SigPid = pid; 2305 2306 fdlog = open(pkg->logfile, O_RDWR|O_CREAT|O_APPEND, 0644); 2307 if (fdlog < 0) { 2308 dlog(DLOG_ALL, "[%03d] %s Cannot open logfile '%s': %s\n", 2309 work->index, pkg->portdir, 2310 pkg->logfile, strerror(errno)); 2311 } 2312 2313 snprintf(buf, sizeof(buf), 2314 "----------------------------------------" 2315 "---------------------------------------\n" 2316 "-- Phase: %s\n" 2317 "----------------------------------------" 2318 "---------------------------------------\n", 2319 phase); 2320 write(fdlog, buf, strlen(buf)); 2321 2322 start_time = time(NULL); 2323 last_time = start_time; 2324 wdog_time = start_time; 2325 wpid_reaped = 0; 2326 2327 status = 0; 2328 for (;;) { 2329 ms = mptylogpoll(MasterPtyFd, fdlog, wmsg, &wdog_time); 2330 if (ms == MPTY_FAILED) { 2331 dlog(DLOG_ALL, 2332 "[%03d] %s lost pty in phase %s, terminating\n", 2333 work->index, pkg->portdir, phase); 2334 break; 2335 } 2336 if (ms == MPTY_EOF) 2337 break; 2338 2339 /* 2340 * Generally speaking update status once a second. 2341 * This also allows us to detect if the management 2342 * dsynth process has gone away. 2343 */ 2344 next_time = time(NULL); 2345 if (next_time != last_time) { 2346 double dload[3]; 2347 double dv; 2348 int wdog_scaled; 2349 2350 /* 2351 * Send status update to the worker management thread 2352 * in the master dsynth process. Remember, *WE* are 2353 * the worker management process sub-fork. 2354 */ 2355 if (ipcwritemsg(work->fds[0], wmsg) < 0) 2356 break; 2357 last_time = next_time; 2358 2359 /* 2360 * Watchdog scaling 2361 */ 2362 getloadavg(dload, 3); 2363 dv = dload[2] / NumCores; 2364 if (dv < (double)NumCores) { 2365 wdog_scaled = wdog; 2366 } else { 2367 if (dv > 4.0 * NumCores) 2368 dv = 4.0 * NumCores; 2369 wdog_scaled = wdog * dv / NumCores; 2370 } 2371 2372 /* 2373 * Watchdog 2374 */ 2375 if (next_time - wdog_time >= wdog_scaled * 60) { 2376 snprintf(buf, sizeof(buf), 2377 "\n--------\n" 2378 "WATCHDOG TIMEOUT FOR %s in %s " 2379 "after %d minutes\n" 2380 "Killing pid %d\n" 2381 "--------\n", 2382 pkg->portdir, phase, wdog_scaled, pid); 2383 if (fdlog >= 0) 2384 write(fdlog, buf, strlen(buf)); 2385 dlog(DLOG_ALL, 2386 "[%03d] %s WATCHDOG TIMEOUT in %s " 2387 "after %d minutes (%d min scaled)\n", 2388 work->index, pkg->portdir, phase, 2389 wdog, wdog_scaled); 2390 kill(pid, SIGKILL); 2391 ++work->accum_error; 2392 break; 2393 } 2394 } 2395 2396 /* 2397 * Check process exit. Normally the pty will EOF 2398 * but if background processes remain we need to 2399 * check here to see if our primary exec is done, 2400 * so we can break out and reap those processes. 2401 * 2402 * Generally reap any other processes we have inherited 2403 * while we are here. 2404 */ 2405 do { 2406 wpid = wait3(&status, WNOHANG, NULL); 2407 } while (wpid > 0 && wpid != pid); 2408 if (wpid == pid && WIFEXITED(status)) { 2409 wpid_reaped = 1; 2410 break; 2411 } 2412 } 2413 2414 next_time = time(NULL); 2415 2416 setproctitle("[%02d] WORKER EXITREAP %s%s", 2417 work->index, pkg->portdir, WorkerFlavorPrt); 2418 2419 /* 2420 * We usually get here due to a mpty EOF, but not always as there 2421 * could be persistent processes still holding the slave. Finish 2422 * up getting the exit status for the main process we are waiting 2423 * on and clean out any data left on the MasterPtyFd (as it could 2424 * be blocking the exit). 2425 */ 2426 while (wpid_reaped == 0) { 2427 (void)mptylogpoll(MasterPtyFd, fdlog, wmsg, &wdog_time); 2428 wpid = waitpid(pid, &status, WNOHANG); 2429 if (wpid == pid && WIFEXITED(status)) { 2430 wpid_reaped = 1; 2431 break; 2432 } 2433 if (wpid < 0 && errno != EINTR) { 2434 break; 2435 } 2436 2437 /* 2438 * Safety. The normal phase waits until the fork/exec'd 2439 * pid finishes, causing a pty EOF on exit (the slave 2440 * descriptor is closed by the kernel on exit so the 2441 * process should already have exited). 2442 * 2443 * However, it is also possible to get here if the pty fails 2444 * for some reason. In this case, make sure that the process 2445 * is killed. 2446 */ 2447 kill(pid, SIGKILL); 2448 } 2449 2450 /* 2451 * Clean out anything left on the pty but don't wait around 2452 * because there could be background processes preventing the 2453 * slave side from closing. 2454 */ 2455 while (mptylogpoll(MasterPtyFd, fdlog, wmsg, &wdog_time) == MPTY_DATA) 2456 ; 2457 2458 /* 2459 * Report on the exit condition. If the pid was somehow lost 2460 * (probably due to someone gdb'ing the process), assume an error. 2461 */ 2462 if (wpid_reaped) { 2463 if (WEXITSTATUS(status)) { 2464 dlog(DLOG_ALL | DLOG_FILTER, 2465 "[%03d] %s Build phase '%s' failed exit %d\n", 2466 work->index, pkg->portdir, phase, 2467 WEXITSTATUS(status)); 2468 ++work->accum_error; 2469 } 2470 } else { 2471 dlog(DLOG_ALL, "[%03d] %s Build phase '%s' failed - lost pid\n", 2472 work->index, pkg->portdir, phase); 2473 ++work->accum_error; 2474 } 2475 2476 /* 2477 * Kill any processes still running (sometimes processes end up in 2478 * the background during a dports build), and clean up any other 2479 * children that we have inherited. 2480 */ 2481 phaseReapAll(); 2482 2483 /* 2484 * After the extraction phase add the space used by /construction 2485 * to the memory use. This helps us reduce the amount of paging 2486 * we do due to extremely large package extractions (languages, 2487 * chromium, etc). 2488 * 2489 * (dsynth already estimated the space used by the package deps 2490 * up front, but this will help us further). 2491 */ 2492 if (work->accum_error == 0 && phaseid == PHASE_EXTRACT) { 2493 struct statfs sfs; 2494 char *b1; 2495 2496 asprintf(&b1, "%s/construction", work->basedir); 2497 if (statfs(b1, &sfs) == 0) { 2498 wmsg->memuse = (sfs.f_blocks - sfs.f_bfree) * 2499 sfs.f_bsize; 2500 ipcwritemsg(work->fds[0], wmsg); 2501 } 2502 } 2503 2504 /* 2505 * Update log 2506 */ 2507 if (fdlog >= 0) { 2508 struct stat st; 2509 int h; 2510 int m; 2511 int s; 2512 2513 last_time = next_time - start_time; 2514 s = last_time % 60; 2515 m = last_time / 60 % 60; 2516 h = last_time / 3600; 2517 2518 fp = fdopen(fdlog, "a"); 2519 if (fp == NULL) { 2520 dlog(DLOG_ALL, "[%03d] %s Cannot fdopen fdlog: %s %d\n", 2521 work->index, pkg->portdir, 2522 strerror(errno), fstat(fdlog, &st)); 2523 close(fdlog); 2524 goto skip; 2525 } 2526 2527 fprintf(fp, "\n"); 2528 if (work->accum_error) { 2529 fprintf(fp, "FAILED %02d:%02d:%02d\n", h, m, s); 2530 } else { 2531 if (phaseid == PHASE_EXTRACT && wmsg->memuse) { 2532 fprintf(fp, "Extracted Memory Use: %6.2fM\n", 2533 wmsg->memuse / (1024.0 * 1024.0)); 2534 } 2535 fprintf(fp, "SUCCEEDED %02d:%02d:%02d\n", h, m, s); 2536 } 2537 last_time = next_time - work->start_time; 2538 s = last_time % 60; 2539 m = last_time / 60 % 60; 2540 h = last_time / 3600; 2541 if (phaseid == PHASE_PACKAGE) { 2542 fprintf(fp, "TOTAL TIME %02d:%02d:%02d\n", h, m, s); 2543 } 2544 fprintf(fp, "\n"); 2545 fclose(fp); 2546 skip: 2547 ; 2548 } 2549 2550 } 2551 2552 static void 2553 phaseReapAll(void) 2554 { 2555 struct reaper_status rs; 2556 int status; 2557 2558 while (procctl(P_PID, getpid(), PROC_REAP_STATUS, &rs) == 0) { 2559 if ((rs.flags & PROC_REAP_ACQUIRE) == 0) 2560 break; 2561 if (rs.pid_head < 0) 2562 break; 2563 if (kill(rs.pid_head, SIGKILL) == 0) { 2564 while (waitpid(rs.pid_head, &status, 0) < 0) 2565 ; 2566 } 2567 } 2568 while (wait3(&status, 0, NULL) > 0) 2569 ; 2570 } 2571 2572 static void 2573 phaseTerminateSignal(int sig __unused) 2574 { 2575 if (CopyFileFd >= 0) 2576 close(CopyFileFd); 2577 if (MasterPtyFd >= 0) 2578 close(MasterPtyFd); 2579 if (SigPid > 1) 2580 kill(SigPid, SIGKILL); 2581 phaseReapAll(); 2582 if (SigWork) 2583 DoWorkerUnmounts(SigWork); 2584 exit(1); 2585 } 2586 2587 static 2588 char * 2589 buildskipreason(pkglink_t *parent, pkg_t *pkg) 2590 { 2591 pkglink_t *link; 2592 pkg_t *scan; 2593 char *reason = NULL; 2594 char *ptr; 2595 size_t tot; 2596 size_t len; 2597 pkglink_t stack; 2598 2599 if ((pkg->flags & PKGF_NOBUILD_I) && pkg->ignore) 2600 asprintf(&reason, "%s ", pkg->ignore); 2601 2602 tot = 0; 2603 PKGLIST_FOREACH(link, &pkg->idepon_list) { 2604 #if 0 2605 if (link->dep_type > DEP_TYPE_BUILD) 2606 continue; 2607 #endif 2608 scan = link->pkg; 2609 if (scan == NULL) 2610 continue; 2611 if ((scan->flags & (PKGF_ERROR | PKGF_NOBUILD)) == 0) 2612 continue; 2613 if (scan->flags & PKGF_NOBUILD) { 2614 stack.pkg = scan; 2615 stack.next = parent; 2616 ptr = buildskipreason(&stack, scan); 2617 len = strlen(scan->portdir) + strlen(ptr) + 8; 2618 reason = realloc(reason, tot + len); 2619 snprintf(reason + tot, len, "%s->%s", 2620 scan->portdir, ptr); 2621 free(ptr); 2622 } else { 2623 len = strlen(scan->portdir) + 8; 2624 reason = realloc(reason, tot + len); 2625 snprintf(reason + tot, len, "%s", scan->portdir); 2626 } 2627 2628 /* 2629 * Don't try to print the entire graph 2630 */ 2631 if (parent) 2632 break; 2633 tot += strlen(reason + tot); 2634 reason[tot++] = ' '; 2635 reason[tot] = 0; 2636 } 2637 return (reason); 2638 } 2639 2640 /* 2641 * Count number of packages that would be skipped due to the 2642 * specified package having failed. 2643 * 2644 * Call with mode 1 to count, and mode 0 to clear the 2645 * cumulative rscan flag (used to de-duplicate the count). 2646 * 2647 * Must be serialized. 2648 */ 2649 static int 2650 buildskipcount_dueto(pkg_t *pkg, int mode) 2651 { 2652 pkglink_t *link; 2653 pkg_t *scan; 2654 int total; 2655 2656 total = 0; 2657 PKGLIST_FOREACH(link, &pkg->deponi_list) { 2658 scan = link->pkg; 2659 if (scan == NULL || scan->rscan == mode) 2660 continue; 2661 scan->rscan = mode; 2662 ++total; 2663 total += buildskipcount_dueto(scan, mode); 2664 } 2665 return total; 2666 } 2667 2668 /* 2669 * The master ptyfd is in non-blocking mode. Drain up to 1024 bytes 2670 * and update wmsg->lines and *wdog_timep as appropriate. 2671 * 2672 * This function will poll, stalling up to 1 second. 2673 */ 2674 static int 2675 mptylogpoll(int ptyfd, int fdlog, wmsg_t *wmsg, time_t *wdog_timep) 2676 { 2677 struct pollfd pfd; 2678 char buf[1024]; 2679 ssize_t r; 2680 2681 pfd.fd = ptyfd; 2682 pfd.events = POLLIN; 2683 pfd.revents = 0; 2684 2685 poll(&pfd, 1, 1000); 2686 if (pfd.revents) { 2687 r = read(ptyfd, buf, sizeof(buf)); 2688 if (r > 0) { 2689 *wdog_timep = time(NULL); 2690 if (r > 0 && fdlog >= 0) 2691 write(fdlog, buf, r); 2692 while (--r >= 0) { 2693 if (buf[r] == '\n') 2694 ++wmsg->lines; 2695 } 2696 return MPTY_DATA; 2697 } else if (r < 0) { 2698 if (errno != EINTR && errno != EAGAIN) 2699 return MPTY_FAILED; 2700 return MPTY_AGAIN; 2701 } else if (r == 0) { 2702 return MPTY_EOF; 2703 } 2704 } 2705 return MPTY_AGAIN; 2706 } 2707 2708 /* 2709 * Copy a (package) file from (src) to (dst), use an intermediate file and 2710 * rename to ensure that interruption does not leave us with a corrupt 2711 * package file. 2712 * 2713 * This is called by the WORKER process. 2714 * 2715 * (dsynth management thread -> WORKER process -> sub-processes) 2716 */ 2717 #define COPYBLKSIZE 32768 2718 2719 static int 2720 copyfile(char *src, char *dst) 2721 { 2722 char *tmp; 2723 char *buf; 2724 int fd1; 2725 int fd2; 2726 int error = 0; 2727 int mask; 2728 ssize_t r; 2729 2730 asprintf(&tmp, "%s.new", dst); 2731 buf = malloc(COPYBLKSIZE); 2732 2733 mask = sigsetmask(sigmask(SIGTERM)|sigmask(SIGINT)|sigmask(SIGHUP)); 2734 fd1 = open(src, O_RDONLY|O_CLOEXEC); 2735 fd2 = open(tmp, O_RDWR|O_CREAT|O_TRUNC|O_CLOEXEC, 0644); 2736 CopyFileFd = fd1; 2737 sigsetmask(mask); 2738 while ((r = read(fd1, buf, COPYBLKSIZE)) > 0) { 2739 if (write(fd2, buf, r) != r) 2740 error = 1; 2741 } 2742 if (r < 0) 2743 error = 1; 2744 mask = sigsetmask(sigmask(SIGTERM)|sigmask(SIGINT)|sigmask(SIGHUP)); 2745 CopyFileFd = -1; 2746 close(fd1); 2747 close(fd2); 2748 sigsetmask(mask); 2749 if (error) { 2750 remove(tmp); 2751 } else { 2752 if (rename(tmp, dst)) { 2753 error = 1; 2754 remove(tmp); 2755 } 2756 } 2757 2758 freestrp(&buf); 2759 freestrp(&tmp); 2760 2761 return error; 2762 } 2763 2764 /* 2765 * doHook() 2766 * 2767 * primary process (threaded) - run_start, run_end, pkg_ignored, pkg_skipped 2768 * worker process (threaded) - pkg_sucess, pkg_failure 2769 * 2770 * If waitfor is non-zero this hook will be serialized. 2771 */ 2772 static void 2773 doHook(pkg_t *pkg, const char *id, const char *path, int waitfor) 2774 { 2775 if (path == NULL) 2776 return; 2777 while (waitfor && getbulk() != NULL) 2778 ; 2779 if (pkg) 2780 queuebulk(pkg->portdir, id, path, pkg->pkgfile); 2781 else 2782 queuebulk(NULL, id, path, NULL); 2783 while (waitfor && getbulk() != NULL) 2784 ; 2785 } 2786 2787 /* 2788 * Execute hook (backend) 2789 * 2790 * s1 - portdir 2791 * s2 - id 2792 * s3 - script path 2793 * s4 - pkgfile (if applicable) 2794 */ 2795 static void 2796 childHookRun(bulk_t *bulk) 2797 { 2798 const char *cav[MAXCAC]; 2799 buildenv_t benv[MAXCAC]; 2800 char buf1[128]; 2801 char buf2[128]; 2802 char buf3[128]; 2803 char buf4[128]; 2804 FILE *fp; 2805 char *ptr; 2806 size_t len; 2807 pid_t pid; 2808 int cac; 2809 int bi; 2810 2811 cac = 0; 2812 bi = 0; 2813 bzero(benv, sizeof(benv)); 2814 2815 cav[cac++] = bulk->s3; 2816 2817 benv[bi].label = "PROFILE"; 2818 benv[bi].data = Profile; 2819 ++bi; 2820 2821 benv[bi].label = "DIR_PACKAGES"; 2822 benv[bi].data = PackagesPath; 2823 ++bi; 2824 2825 benv[bi].label = "DIR_REPOSITORY"; 2826 benv[bi].data = RepositoryPath; 2827 ++bi; 2828 2829 benv[bi].label = "DIR_PORTS"; 2830 benv[bi].data = DPortsPath; 2831 ++bi; 2832 2833 benv[bi].label = "DIR_OPTIONS"; 2834 benv[bi].data = OptionsPath; 2835 ++bi; 2836 2837 benv[bi].label = "DIR_DISTFILES"; 2838 benv[bi].data = DistFilesPath; 2839 ++bi; 2840 2841 benv[bi].label = "DIR_LOGS"; 2842 benv[bi].data = LogsPath; 2843 ++bi; 2844 2845 benv[bi].label = "DIR_BUILDBASE"; 2846 benv[bi].data = BuildBase; 2847 ++bi; 2848 2849 if (strcmp(bulk->s2, "hook_run_start") == 0) { 2850 snprintf(buf1, sizeof(buf1), "%d", BuildTotal); 2851 benv[bi].label = "PORTS_QUEUED"; 2852 benv[bi].data = buf1; 2853 ++bi; 2854 } else if (strcmp(bulk->s2, "hook_run_end") == 0) { 2855 snprintf(buf1, sizeof(buf1), "%d", BuildSuccessCount); 2856 benv[bi].label = "PORTS_BUILT"; 2857 benv[bi].data = buf1; 2858 ++bi; 2859 snprintf(buf2, sizeof(buf2), "%d", BuildFailCount); 2860 benv[bi].label = "PORTS_FAILED"; 2861 benv[bi].data = buf2; 2862 ++bi; 2863 snprintf(buf3, sizeof(buf3), "%d", BuildIgnoreCount); 2864 benv[bi].label = "PORTS_IGNORED"; 2865 benv[bi].data = buf3; 2866 ++bi; 2867 snprintf(buf4, sizeof(buf4), "%d", BuildSkipCount); 2868 benv[bi].label = "PORTS_SKIPPED"; 2869 benv[bi].data = buf4; 2870 ++bi; 2871 } else { 2872 /* 2873 * success, failure, ignored, skipped 2874 */ 2875 benv[bi].label = "RESULT"; 2876 if (strcmp(bulk->s2, "hook_pkg_success") == 0) { 2877 benv[bi].data = "success"; 2878 } else if (strcmp(bulk->s2, "hook_pkg_failure") == 0) { 2879 benv[bi].data = "failure"; 2880 } else if (strcmp(bulk->s2, "hook_pkg_ignored") == 0) { 2881 benv[bi].data = "ignored"; 2882 } else if (strcmp(bulk->s2, "hook_pkg_skipped") == 0) { 2883 benv[bi].data = "skipped"; 2884 } else { 2885 dfatal("Unknown hook id: %s", bulk->s2); 2886 /* NOT REACHED */ 2887 } 2888 ++bi; 2889 2890 /* 2891 * For compatibility with synth: 2892 * 2893 * ORIGIN does not include any @flavor, thus it is suitable 2894 * for finding the actual port directory/subdirectory. 2895 * 2896 * FLAVOR is set to ORIGIN if there is no flavor, otherwise 2897 * it is set to only the flavor sans the '@'. 2898 */ 2899 if ((ptr = strchr(bulk->s1, '@')) != NULL) { 2900 snprintf(buf1, sizeof(buf1), "%*.*s", 2901 (int)(ptr - bulk->s1), 2902 (int)(ptr - bulk->s1), 2903 bulk->s1); 2904 benv[bi].label = "ORIGIN"; 2905 benv[bi].data = buf1; 2906 ++bi; 2907 benv[bi].label = "FLAVOR"; 2908 benv[bi].data = ptr + 1; 2909 ++bi; 2910 } else { 2911 benv[bi].label = "ORIGIN"; 2912 benv[bi].data = bulk->s1; 2913 ++bi; 2914 benv[bi].label = "FLAVOR"; 2915 benv[bi].data = bulk->s1; 2916 ++bi; 2917 } 2918 benv[bi].label = "PKGNAME"; 2919 benv[bi].data = bulk->s4; 2920 ++bi; 2921 } 2922 2923 benv[bi].label = NULL; 2924 benv[bi].data = NULL; 2925 2926 fp = dexec_open(cav, cac, &pid, benv, 0, 0); 2927 while ((ptr = fgetln(fp, &len)) != NULL) 2928 ; 2929 2930 if (dexec_close(fp, pid)) { 2931 dlog(DLOG_ALL, 2932 "[XXX] %s SCRIPT %s (%s)\n", 2933 bulk->s1, bulk->s2, bulk->s3); 2934 } 2935 } 2936