xref: /dragonfly/usr.bin/dsynth/build.c (revision a361ab31)
1 /*
2  * Copyright (c) 2019-2022 The DragonFly Project.  All rights reserved.
3  *
4  * This code is derived from software contributed to The DragonFly Project
5  * by Matthew Dillon <dillon@backplane.com>
6  *
7  * This code uses concepts and configuration based on 'synth', by
8  * John R. Marino <draco@marino.st>, which was written in ada.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  *
14  * 1. Redistributions of source code must retain the above copyright
15  *    notice, this list of conditions and the following disclaimer.
16  * 2. Redistributions in binary form must reproduce the above copyright
17  *    notice, this list of conditions and the following disclaimer in
18  *    the documentation and/or other materials provided with the
19  *    distribution.
20  * 3. Neither the name of The DragonFly Project nor the names of its
21  *    contributors may be used to endorse or promote products derived
22  *    from this software without specific, prior written permission.
23  *
24  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
25  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
26  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
27  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
28  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
29  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
30  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
31  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
32  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
33  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
34  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
35  * SUCH DAMAGE.
36  */
37 
38 #include <netdb.h>
39 
40 #include "dsynth.h"
41 
42 static worker_t WorkerAry[MAXWORKERS];
43 static int BuildInitialized;
44 static int RunningWorkers;
45 int DynamicMaxWorkers;
46 static int FailedWorkers;
47 static long RunningPkgDepSize;
48 static pthread_mutex_t DbmMutex;
49 static pthread_mutex_t WorkerMutex;
50 static pthread_cond_t WorkerCond;
51 
52 static int build_find_leaves(pkg_t *parent, pkg_t *pkg,
53 			pkg_t ***build_tailp, int *app, int *hasworkp,
54 			int depth, int first, int first_one_only);
55 static int buildCalculateDepiDepth(pkg_t *pkg);
56 static void build_clear_trav(pkg_t *pkg);
57 static void startbuild(pkg_t **build_listp, pkg_t ***build_tailp);
58 static int qsort_depi(const void *pkg1, const void *pkg2);
59 static int qsort_idep(const void *pkg1, const void *pkg2);
60 static void startworker(pkg_t *pkg, worker_t *work);
61 static void cleanworker(worker_t *work);
62 static void waitbuild(int whilematch, int dynamicmax);
63 static void workercomplete(worker_t *work);
64 static void *childBuilderThread(void *arg);
65 static int childInstallPkgDeps(worker_t *work);
66 static size_t childInstallPkgDeps_recurse(FILE *fp, pkglink_t *list,
67 			int undoit, int depth, int first_one_only);
68 static void dophase(worker_t *work, wmsg_t *wmsg,
69 			int wdog, int phaseid, const char *phase);
70 static void phaseReapAll(void);
71 static void phaseTerminateSignal(int sig);
72 static char *buildskipreason(pkglink_t *parent, pkg_t *pkg);
73 static int buildskipcount_dueto(pkg_t *pkg, int mode);
74 static int mptylogpoll(int ptyfd, int fdlog, wmsg_t *wmsg,
75 			time_t *wdog_timep);
76 static void doHook(pkg_t *pkg, const char *id, const char *path, int waitfor);
77 static void childHookRun(bulk_t *bulk);
78 static void adjloadavg(double *dload);
79 static void check_packaged(const char *dbmpath, pkg_t *pkgs);
80 
81 static worker_t *SigWork;
82 static int MasterPtyFd = -1;
83 static int CopyFileFd = -1;
84 static pid_t SigPid;
85 static const char *WorkerFlavorPrt = "";	/* "" or "@flavor" */
86 static DBM *CheckDBM;
87 
88 #define MPTY_FAILED	-2
89 #define MPTY_AGAIN	-1
90 #define MPTY_EOF	0
91 #define MPTY_DATA	1
92 
93 int BuildTotal;
94 int BuildCount;
95 int BuildSkipCount;
96 int BuildIgnoreCount;
97 int BuildFailCount;
98 int BuildSuccessCount;
99 int BuildMissingCount;
100 int BuildMetaCount;
101 int PkgVersionPkgSuffix;
102 
103 /*
104  * Initialize the WorkerAry[]
105  */
106 void
DoInitBuild(int slot_override)107 DoInitBuild(int slot_override)
108 {
109 	worker_t *work;
110 	struct stat st;
111 	char *path;
112 	int i;
113 
114 	ddassert(slot_override < 0 || MaxWorkers == 1);
115 
116 	bzero(WorkerAry, MaxWorkers * sizeof(worker_t));
117 	pthread_mutex_init(&WorkerMutex, NULL);
118 	pthread_mutex_init(&DbmMutex, NULL);
119 
120 	for (i = 0; i < MaxWorkers; ++i) {
121 		work = &WorkerAry[i];
122 		work->index = (slot_override >= 0) ? slot_override : i;
123 		work->state = WORKER_NONE;
124 		asprintf(&work->basedir, "%s/SL%02d", BuildBase, work->index);
125 		pthread_cond_init(&work->cond, NULL);
126 	}
127 	BuildCount = 0;
128 
129 	/*
130 	 * Create required sub-directories. The base directories must already
131 	 * exist as a dsynth configuration safety.
132 	 */
133 	if (stat(RepositoryPath, &st) < 0) {
134 		if (mkdir(RepositoryPath, 0755) < 0)
135 			dfatal("Cannot mkdir %s\n", RepositoryPath);
136 	}
137 
138 	/*
139 	 * An empty directory for LOCALBASE= in the pkg scan.
140 	 */
141 	asprintf(&path, "%s/empty", BuildBase);
142 	if (stat(path, &st) < 0) {
143 		if (mkdir(path, 0755) < 0)
144 			dfatal("Cannot mkdir %s\n", path);
145 	}
146 	free(path);
147 
148 	BuildInitialized = 1;
149 
150 	/*
151 	 * slow-start (increases at a rate of 1 per 5 seconds)
152 	 */
153 	if (SlowStartOpt > MaxWorkers)
154 		DynamicMaxWorkers = MaxWorkers;
155 	else if (SlowStartOpt > 0)
156 		DynamicMaxWorkers = SlowStartOpt;
157 	else
158 		DynamicMaxWorkers = MaxWorkers;
159 }
160 
161 /*
162  * Called by the frontend to clean-up any hanging mounts.
163  */
164 void
DoCleanBuild(int resetlogs)165 DoCleanBuild(int resetlogs)
166 {
167 	int i;
168 
169 	ddassert(BuildInitialized);
170 
171 	if (resetlogs)
172 		dlogreset();
173 	for (i = 0; i < MaxWorkers; ++i) {
174 		DoWorkerUnmounts(&WorkerAry[i]);
175 	}
176 }
177 
178 void
DoBuild(pkg_t * pkgs)179 DoBuild(pkg_t *pkgs)
180 {
181 	pkg_t *build_list = NULL;
182 	pkg_t **build_tail = &build_list;
183 	pkg_t *scan;
184 	bulk_t *bulk;
185 	int haswork = 1;
186 	int first = 1;
187 	int newtemplate;
188 	time_t startTime;
189 	time_t t;
190 	int h, m, s;
191 	char *dbmpath;
192 
193 	/*
194 	 * We use our bulk system to run hooks.  This will be for
195 	 * Skipped and Ignored.  Success and Failure are handled by
196 	 * WorkerProcess() which is a separately-exec'd dsynth.
197 	 */
198 	if (UsingHooks)
199 		initbulk(childHookRun, MaxBulk);
200 
201 	/*
202 	 * Count up the packages, not counting dummies
203 	 */
204 	for (scan = pkgs; scan; scan = scan->bnext) {
205 		if ((scan->flags & PKGF_DUMMY) == 0)
206 			++BuildTotal;
207 	}
208 
209 	/*
210 	 * Remove binary package files for dports whos directory
211 	 * has changed.
212 	 */
213 	asprintf(&dbmpath, "%s/ports_crc", BuildBase);
214 	CheckDBM = dbm_open(dbmpath, O_CREAT|O_RDWR, 0644);
215 	check_packaged(dbmpath, pkgs);
216 
217 	doHook(NULL, "hook_run_start", HookRunStart, 1);
218 
219 	/*
220 	 * The pkg and pkg-static binaries are needed.  If already present
221 	 * then assume that the template is also valid, otherwise add to
222 	 * the list and build both.
223 	 */
224 	scan = GetPkgPkg(&pkgs);
225 
226 	/*
227 	 * Create our template.  The template will be missing pkg
228 	 * and pkg-static.
229 	 */
230 	if (FetchOnlyOpt) {
231 		newtemplate = DoCreateTemplate(0);
232 	} else if ((scan->flags & (PKGF_SUCCESS | PKGF_PACKAGED)) == 0) {
233 		/* force a fresh template */
234 		newtemplate = DoCreateTemplate(1);
235 	} else {
236 		newtemplate = DoCreateTemplate(0);
237 	}
238 
239 	/*
240 	 * This will clear the screen and set-up our gui, so sleep
241 	 * a little first in case the user wants to see what was
242 	 * printed before.
243 	 */
244 	sleep(2);
245 	pthread_mutex_lock(&WorkerMutex);
246 	startTime = time(NULL);
247 	RunStatsInit();
248 	RunStatsReset();
249 
250 	/*
251 	 * Build pkg/pkg-static
252 	 */
253 	if ((scan->flags & (PKGF_SUCCESS | PKGF_PACKAGED)) == 0 && FetchOnlyOpt == 0) {
254 		build_list = scan;
255 		build_tail = &scan->build_next;
256 		startbuild(&build_list, &build_tail);
257 		while (RunningWorkers == 1)
258 			waitbuild(1, 0);
259 
260 		if (scan->flags & PKGF_NOBUILD)
261 			dfatal("Unable to build 'pkg'");
262 		if (scan->flags & PKGF_ERROR)
263 			dfatal("Error building 'pkg'");
264 		if ((scan->flags & PKGF_SUCCESS) == 0)
265 			dfatal("Error building 'pkg'");
266 		newtemplate = 1;
267 	}
268 
269 	/*
270 	 * Install pkg/pkg-static into the template
271 	 */
272 	if (newtemplate && FetchOnlyOpt == 0) {
273 		char *buf;
274 		int rc;
275 
276 		asprintf(&buf,
277 			 "cd %s/Template; "
278 			 "tar --exclude '+*' --exclude '*/man/*' "
279 			 "-xvzpf %s/%s > /dev/null 2>&1",
280 			 BuildBase,
281 			 RepositoryPath,
282 			 scan->pkgfile);
283 		rc = system(buf);
284 		if (rc)
285 			dfatal("Command failed: %s\n", buf);
286 		freestrp(&buf);
287 	}
288 
289 	/*
290 	 * Figure out pkg version
291 	 */
292 	if (scan->version) {
293 		int v1;
294 		int v2;
295 
296 		dlog(DLOG_ALL, "[XXX] pkg version %s\n", scan->version);
297 		if (sscanf(scan->version, "%d.%d", &v1, &v2) == 2) {
298 			if ((v1 == 1 && v2 >= 17) || v1 > 1)
299 			    PkgVersionPkgSuffix = 1;
300 		}
301 	}
302 
303 	/*
304 	 * Calculate depi_depth, the longest chain of dependencies
305 	 * for who depends on me, weighted by powers of two.
306 	 */
307 	for (scan = pkgs; scan; scan = scan->bnext) {
308 		buildCalculateDepiDepth(scan);
309 	}
310 
311 	/*
312 	 * Nominal bulk build sequence
313 	 */
314 	while (haswork) {
315 		haswork = 0;
316 		fflush(stdout);
317 		for (scan = pkgs; scan; scan = scan->bnext) {
318 			ddprintf(0, "SCANLEAVES %08x %s\n",
319 				 scan->flags, scan->portdir);
320 			scan->flags |= PKGF_BUILDLOOP;
321 			/*
322 			 * NOTE: We must still find dependencies if PACKAGED
323 			 *	 to fill in the gaps, as some of them may
324 			 *	 need to be rebuilt.
325 			 */
326 			if (scan->flags & (PKGF_SUCCESS | PKGF_FAILURE |
327 					   PKGF_ERROR)) {
328 #if 0
329 				ddprintf(0, "%s: already built\n",
330 					 scan->portdir);
331 #endif
332 			} else {
333 				int ap = 0;
334 				build_find_leaves(NULL, scan, &build_tail,
335 						  &ap, &haswork, 0, first, 0);
336 				ddprintf(0, "TOPLEVEL %s %08x\n",
337 					 scan->portdir, ap);
338 			}
339 			scan->flags &= ~PKGF_BUILDLOOP;
340 			build_clear_trav(scan);
341 		}
342 		first = 0;
343 		fflush(stdout);
344 		startbuild(&build_list, &build_tail);
345 
346 		if (haswork == 0 && RunningWorkers) {
347 			waitbuild(RunningWorkers, 1);
348 			haswork = 1;
349 		}
350 	}
351 	pthread_mutex_unlock(&WorkerMutex);
352 
353 	/*
354 	 * What is left that cannot be built?  The list really ought to be
355 	 * empty at this point, report anything that remains.
356 	 */
357 	for (scan = pkgs; scan; scan = scan->bnext) {
358 		if (scan->flags & (PKGF_SUCCESS | PKGF_FAILURE))
359 			continue;
360 		dlog(DLOG_ABN, "[XXX] %s lost in the ether [flags=%08x]\n",
361 		     scan->portdir, scan->flags);
362 		++BuildMissingCount;
363 	}
364 
365 	/*
366 	 * Final updates
367 	 */
368 	RunStatsUpdateTop(0);
369 	RunStatsUpdateLogs();
370 	RunStatsSync();
371 	RunStatsDone();
372 
373 	doHook(NULL, "hook_run_end", HookRunEnd, 1);
374 	if (UsingHooks) {
375 		while ((bulk = getbulk()) != NULL)
376 			freebulk(bulk);
377 		donebulk();
378 	}
379 
380 	t = time(NULL) - startTime;
381 	h = t / 3600;
382 	m = t / 60 % 60;
383 	s = t % 60;
384 
385 	if (CheckDBM) {
386 		dbm_close(CheckDBM);
387 		CheckDBM = NULL;
388 	}
389 
390 	dlog(DLOG_ALL|DLOG_STDOUT,
391 		"\n"
392 		"Initial queue size: %d\n"
393 		"    packages built: %d\n"
394 		"           ignored: %d\n"
395 		"           skipped: %d\n"
396 		"            failed: %d\n"
397 		"           missing: %d\n"
398 		"        meta-nodes: %d\n"
399 		"\n"
400 		"Duration: %02d:%02d:%02d\n"
401 		"\n",
402 		BuildTotal,
403 		BuildSuccessCount,
404 		BuildIgnoreCount,
405 		BuildSkipCount,
406 		BuildFailCount,
407 		BuildMissingCount,
408 		BuildMetaCount,
409 		h, m, s);
410 }
411 
412 /*
413  * Traverse the packages (pkg) depends on recursively until we find
414  * a leaf to build or report as unbuildable.  Calculates and assigns a
415  * dependency count.  Returns all parallel-buildable packages.
416  *
417  * (pkg) itself is only added to the list if it is immediately buildable.
418  */
419 static
420 int
build_find_leaves(pkg_t * parent,pkg_t * pkg,pkg_t *** build_tailp,int * app,int * hasworkp,int depth,int first,int first_one_only)421 build_find_leaves(pkg_t *parent, pkg_t *pkg, pkg_t ***build_tailp,
422 		  int *app, int *hasworkp, int depth, int first,
423 		  int first_one_only)
424 {
425 	pkglink_t *link;
426 	pkg_t *scan;
427 	int idep_count = 0;
428 	int apsub;
429 	int dfirst_one_only;
430 	int ndepth;
431 	char *buf;
432 
433 	ndepth = depth + 1;
434 
435 	/*
436 	 * Already on build list, possibly in-progress, tell caller that
437 	 * it is not ready.
438 	 */
439 	ddprintf(ndepth, "sbuild_find_leaves %d %s %08x {\n",
440 		 depth, pkg->portdir, pkg->flags);
441 	if (pkg->flags & PKGF_BUILDLIST) {
442 		ddprintf(ndepth, "} (already on build list)\n");
443 		*app |= PKGF_NOTREADY;
444 		return (pkg->idep_count);
445 	}
446 
447 	/*
448 	 * Check dependencies
449 	 */
450 	PKGLIST_FOREACH(link, &pkg->idepon_list) {
451 		scan = link->pkg;
452 
453 		if (scan == NULL) {
454 			if (first_one_only)
455 				break;
456 			continue;
457 		}
458 		ddprintf(ndepth, "check %s %08x\t", scan->portdir, scan->flags);
459 
460 		/*
461 		 * If this dependency is to a DUMMY node it is a dependency
462 		 * only on the default flavor which is only the first node
463 		 * under this one, not all of them.
464 		 *
465 		 * DUMMY nodes can be marked SUCCESS so the build skips past
466 		 * them, but this doesn't mean that their sub-nodes succeeded.
467 		 * We have to check, so recurse even if it is marked
468 		 * successful.
469 		 *
470 		 * NOTE: The depth is not being for complex dependency type
471 		 *	 tests like it is in childInstallPkgDeps_recurse(),
472 		 *	 so we don't have to hicup it like we do in that
473 		 *	 procedure.
474 		 */
475 		dfirst_one_only = (scan->flags & PKGF_DUMMY) ? 1 : 0;
476 		if (dfirst_one_only)
477 			goto skip_to_flavor;
478 
479 		/*
480 		 * When accounting for a successful build, just bump
481 		 * idep_count by one.  scan->idep_count will heavily
482 		 * overlap packages that we count down multiple branches.
483 		 *
484 		 * We must still recurse through PACKAGED packages as
485 		 * some of their dependencies might be missing.
486 		 */
487 		if (scan->flags & PKGF_SUCCESS) {
488 			ddprintf(0, "SUCCESS - OK\n");
489 			++idep_count;
490 			if (first_one_only)
491 				break;
492 			continue;
493 		}
494 
495 		/*
496 		 * ERROR includes FAILURE, which is set in numerous situations
497 		 * including when NOBUILD state is finally processed.  So
498 		 * check for NOBUILD state first.
499 		 *
500 		 * An ERROR in a sub-package causes a NOBUILD in packages
501 		 * that depend on it.
502 		 */
503 		if (scan->flags & PKGF_NOBUILD) {
504 			ddprintf(0, "NOBUILD - OK "
505 				    "(propagate failure upward)\n");
506 			*app |= PKGF_NOBUILD_S;
507 			if (first_one_only)
508 				break;
509 			continue;
510 		}
511 		if (scan->flags & PKGF_ERROR) {
512 			ddprintf(0, "ERROR - OK (propagate failure upward)\n");
513 			*app |= PKGF_NOBUILD_S;
514 			if (first_one_only)
515 				break;
516 			continue;
517 		}
518 
519 		/*
520 		 * If already on build-list this dependency is not ready.
521 		 */
522 		if (scan->flags & PKGF_BUILDLIST) {
523 			ddprintf(0, " [BUILDLIST]");
524 			*app |= PKGF_NOTREADY;
525 		}
526 
527 		/*
528 		 * If not packaged this dependency is not ready for
529 		 * the caller.
530 		 */
531 		if ((scan->flags & PKGF_PACKAGED) == 0) {
532 			ddprintf(0, " [NOT_PACKAGED]");
533 			*app |= PKGF_NOTREADY;
534 		}
535 
536 		/*
537 		 * Reduce search complexity, if we have already processed
538 		 * scan in the traversal it will either already be on the
539 		 * build list or it will not be buildable.  Either way
540 		 * the parent is not buildable.
541 		 */
542 		if (scan->flags & PKGF_BUILDTRAV) {
543 			ddprintf(0, " [BUILDTRAV]\n");
544 			*app |= PKGF_NOTREADY;
545 			if (first_one_only)
546 				break;
547 			continue;
548 		}
549 skip_to_flavor:
550 
551 		/*
552 		 * Assert on dependency loop
553 		 */
554 		++idep_count;
555 		if (scan->flags & PKGF_BUILDLOOP) {
556 			dfatal("pkg dependency loop %s -> %s",
557 				parent->portdir, scan->portdir);
558 		}
559 
560 		/*
561 		 * NOTE: For debug tabbing purposes we use (ndepth + 1)
562 		 *	 here (i.e. depth + 2) in our iteration.
563 		 */
564 		scan->flags |= PKGF_BUILDLOOP;
565 		apsub = 0;
566 		ddprintf(0, " SUBRECURSION {\n");
567 		idep_count += build_find_leaves(pkg, scan, build_tailp,
568 						&apsub, hasworkp,
569 						ndepth + 1, first,
570 						dfirst_one_only);
571 		scan->flags &= ~PKGF_BUILDLOOP;
572 		*app |= apsub;
573 		if (apsub & PKGF_NOBUILD) {
574 			ddprintf(ndepth, "} (sub-nobuild)\n");
575 		} else if (apsub & PKGF_ERROR) {
576 			ddprintf(ndepth, "} (sub-error)\n");
577 		} else if (apsub & PKGF_NOTREADY) {
578 			ddprintf(ndepth, "} (sub-notready)\n");
579 		} else {
580 			ddprintf(ndepth, "} (sub-ok)\n");
581 		}
582 		if (first_one_only)
583 			break;
584 	}
585 	pkg->idep_count = idep_count;
586 	pkg->flags |= PKGF_BUILDTRAV;
587 
588 	/*
589 	 * Incorporate scan results into pkg state.
590 	 */
591 	if ((pkg->flags & PKGF_NOBUILD) == 0 && (*app & PKGF_NOBUILD)) {
592 		*hasworkp = 1;
593 	} else if ((pkg->flags & PKGF_ERROR) == 0 && (*app & PKGF_ERROR)) {
594 		*hasworkp = 1;
595 	}
596 	pkg->flags |= *app & ~PKGF_NOTREADY;
597 
598 	/*
599 	 * Clear the PACKAGED bit if sub-dependencies aren't clean.
600 	 *
601 	 * NOTE: PKGF_NOTREADY is not stored in pkg->flags, only in *app,
602 	 *	 so incorporate *app to test for it.
603 	 */
604 	if ((pkg->flags & PKGF_PACKAGED) &&
605 	    ((pkg->flags | *app) & (PKGF_NOTREADY|PKGF_ERROR|PKGF_NOBUILD))) {
606 		pkg->flags &= ~PKGF_PACKAGED;
607 		ddassert(pkg->pkgfile);
608 		asprintf(&buf, "%s/%s", RepositoryPath, pkg->pkgfile);
609 		if (OverridePkgDeleteOpt >= 1) {
610 			pkg->flags |= PKGF_PACKAGED;
611 			dlog(DLOG_ALL,
612 			     "[XXX] %s DELETE-PACKAGE %s "
613 			     "(OVERRIDE, NOT DELETED)\n",
614 			     pkg->portdir, buf);
615 		} else if (remove(buf) < 0) {
616 			dlog(DLOG_ALL,
617 			     "[XXX] %s DELETE-PACKAGE %s (failed)\n",
618 			     pkg->portdir, buf);
619 		} else {
620 			dlog(DLOG_ALL,
621 			     "[XXX] %s DELETE-PACKAGE %s "
622 			     "(due to dependencies)\n",
623 			     pkg->portdir, buf);
624 		}
625 		freestrp(&buf);
626 		*hasworkp = 1;
627 	}
628 
629 	/*
630 	 * Set PKGF_NOBUILD_I if there is IGNORE data
631 	 */
632 	if (pkg->ignore) {
633 		pkg->flags |= PKGF_NOBUILD_I;
634 	}
635 
636 	/*
637 	 * Handle propagated flags
638 	 */
639 	if (pkg->flags & PKGF_ERROR) {
640 		/*
641 		 * This can only happen if the ERROR has already been
642 		 * processed and accounted for.
643 		 */
644 		ddprintf(depth, "} (ERROR - %s)\n", pkg->portdir);
645 	} else if (*app & PKGF_NOTREADY) {
646 		/*
647 		 * We don't set PKGF_NOTREADY in the pkg, it is strictly
648 		 * a transient flag propagated via build_find_leaves().
649 		 *
650 		 * Just don't add the package to the list.
651 		 *
652 		 * NOTE: Even if NOBUILD is set (meaning we could list it
653 		 *	 and let startbuild() finish it up as a skip, we
654 		 *	 don't process it to the list because we want to
655 		 *	 process all the dependencies, so someone doing a
656 		 *	 manual build can get more complete information and
657 		 *	 does not have to iterate each failed dependency one
658 		 *	 at a time.
659 		 */
660 		;
661 	} else if (pkg->flags & PKGF_SUCCESS) {
662 		ddprintf(depth, "} (SUCCESS - %s)\n", pkg->portdir);
663 	} else if (pkg->flags & PKGF_DUMMY) {
664 		/*
665 		 * Just mark dummy packages as successful when all of their
666 		 * sub-depends (flavors) complete successfully.  Note that
667 		 * dummy packages are not counted in the total, so do not
668 		 * decrement BuildTotal.
669 		 *
670 		 * Do not propagate *app up for the dummy node or add it to
671 		 * the build list.  The dummy node itself is not an actual
672 		 * dependency.  Packages which depend on the default flavor
673 		 * (aka this dummy node) actually depend on the first flavor
674 		 * under this node.
675 		 *
676 		 * So if there is a generic dependency (i.e. no flavor
677 		 * specified), the upper recursion detects PKGF_DUMMY and
678 		 * traverses through the dummy node to the default flavor
679 		 * without checking the error/nobuild flags on this dummy
680 		 * node.
681 		 */
682 		if (pkg->flags & PKGF_NOBUILD) {
683 			ddprintf(depth, "} (DUMMY/META - IGNORED "
684 				 "- MARK SUCCESS ANYWAY)\n");
685 		} else {
686 			ddprintf(depth, "} (DUMMY/META - SUCCESS)\n");
687 		}
688 		pkg->flags |= PKGF_SUCCESS;
689 		*hasworkp = 1;
690 		if (first) {
691 			dlog(DLOG_ALL | DLOG_FILTER,
692 			     "[XXX] %s META-ALREADY-BUILT\n",
693 			     pkg->portdir);
694 		} else {
695 			dlog(DLOG_SUCC, "[XXX] %s meta-node complete\n",
696 			     pkg->portdir);
697 			RunStatsUpdateCompletion(NULL, DLOG_SUCC, pkg, "", "");
698 			++BuildMetaCount;   /* Only for not built meta nodes */
699 		}
700 	} else if (pkg->flags & PKGF_PACKAGED) {
701 		/*
702 		 * We can just mark the pkg successful.  If this is
703 		 * the first pass, we count this as an initial pruning
704 		 * pass and reduce BuildTotal.
705 		 */
706 		ddprintf(depth, "} (PACKAGED - SUCCESS)\n");
707 		pkg->flags |= PKGF_SUCCESS;
708 		*hasworkp = 1;
709 		if (first) {
710 			dlog(DLOG_ALL | DLOG_FILTER,
711 			     "[XXX] %s Already-Built\n",
712 			     pkg->portdir);
713 			--BuildTotal;
714 		} else {
715 			dlog(DLOG_ABN | DLOG_FILTER,
716 			     "[XXX] %s flags=%08x Packaged Unexpectedly\n",
717 			     pkg->portdir, pkg->flags);
718 			/* ++BuildSuccessTotal; XXX not sure */
719 			goto addlist;
720 		}
721 	} else {
722 		/*
723 		 * All dependencies are successful, queue new work
724 		 * and indicate not-ready to the parent (since our
725 		 * package has to be built).
726 		 *
727 		 * NOTE: The NOBUILD case propagates to here as well
728 		 *	 and is ultimately handled by startbuild().
729 		 */
730 addlist:
731 		*hasworkp = 1;
732 		if (pkg->flags & PKGF_NOBUILD_I)
733 			ddprintf(depth, "} (ADDLIST(IGNORE/BROKEN) - %s)\n",
734 				 pkg->portdir);
735 		else if (pkg->flags & PKGF_NOBUILD)
736 			ddprintf(depth, "} (ADDLIST(NOBUILD) - %s)\n",
737 				 pkg->portdir);
738 		else
739 			ddprintf(depth, "} (ADDLIST - %s)\n", pkg->portdir);
740 		pkg->flags |= PKGF_BUILDLIST;
741 		**build_tailp = pkg;
742 		*build_tailp = &pkg->build_next;
743 		pkg->build_next = NULL;
744 		*app |= PKGF_NOTREADY;
745 	}
746 
747 	return idep_count;
748 }
749 
750 static
751 void
build_clear_trav(pkg_t * pkg)752 build_clear_trav(pkg_t *pkg)
753 {
754 	pkglink_t *link;
755 	pkg_t *scan;
756 
757 	pkg->flags &= ~PKGF_BUILDTRAV;
758 	PKGLIST_FOREACH(link, &pkg->idepon_list) {
759 		scan = link->pkg;
760 		if (scan && (scan->flags & PKGF_BUILDTRAV))
761 			build_clear_trav(scan);
762 	}
763 }
764 
765 /*
766  * Calculate the longest chain of packages that depend on me.  The
767  * long the chain, the more important my package is to build earlier
768  * rather than later.
769  */
770 static int
buildCalculateDepiDepth(pkg_t * pkg)771 buildCalculateDepiDepth(pkg_t *pkg)
772 {
773 	pkglink_t *link;
774 	pkg_t *scan;
775 	int best_depth = 0;
776 	int res;
777 
778 	if (pkg->depi_depth)
779 		return(pkg->depi_depth + 1);
780 	pkg->flags |= PKGF_BUILDLOOP;
781 	PKGLIST_FOREACH(link, &pkg->deponi_list) {
782 		scan = link->pkg;
783 		if (scan && (scan->flags & PKGF_BUILDLOOP) == 0) {
784 			res = buildCalculateDepiDepth(scan);
785 			if (best_depth < res)
786 				best_depth = res;
787 		}
788 	}
789 	pkg->flags &= ~PKGF_BUILDLOOP;
790 	pkg->depi_depth = best_depth;
791 
792 	return (best_depth + 1);
793 }
794 
795 /*
796  * Take a list of pkg ready to go, sort it, and assign it to worker
797  * slots.  This routine blocks in waitbuild() until it can dispose of
798  * the entire list.
799  *
800  * WorkerMutex is held by the caller.
801  */
802 static
803 void
startbuild(pkg_t ** build_listp,pkg_t *** build_tailp)804 startbuild(pkg_t **build_listp, pkg_t ***build_tailp)
805 {
806 	pkg_t *pkg;
807 	pkg_t **idep_ary;
808 	pkg_t **depi_ary;
809 	int count;
810 	int idep_index;
811 	int depi_index;
812 	int i;
813 	int n;
814 	worker_t *work;
815 	static int IterateWorker;
816 
817 	/*
818 	 * Nothing to do
819 	 */
820 	if (*build_listp == NULL)
821 		return;
822 
823 	/*
824 	 * Sort
825 	 */
826 	count = 0;
827 	for (pkg = *build_listp; pkg; pkg = pkg->build_next)
828 		++count;
829 	idep_ary = calloc(count, sizeof(pkg_t *));
830 	depi_ary = calloc(count, sizeof(pkg_t *));
831 
832 	count = 0;
833 	for (pkg = *build_listp; pkg; pkg = pkg->build_next) {
834 		idep_ary[count] = pkg;
835 		depi_ary[count] = pkg;
836 		++count;
837 	}
838 
839 	/*
840 	 * idep_ary - sorted by #of dependencies this pkg has.
841 	 * depi_ary - sorted by #of other packages that depend on this pkg.
842 	 */
843 	qsort(idep_ary, count, sizeof(pkg_t *), qsort_idep);
844 	qsort(depi_ary, count, sizeof(pkg_t *), qsort_depi);
845 
846 	idep_index = 0;
847 	depi_index = 0;
848 
849 	/*
850 	 * Half the workers build based on the highest depi count,
851 	 * the other half build based on the highest idep count.
852 	 *
853 	 * This is an attempt to get projects which many other projects
854 	 * depend on built first, but to also try to build large projects
855 	 * (which tend to have a lot of dependencies) earlier rather than
856 	 * later so the end of the bulk run doesn't inefficiently build
857 	 * the last few huge projects.
858 	 *
859 	 * Loop until we manage to assign slots to everyone.  We do not
860 	 * wait for build completion.
861 	 *
862 	 * This is the point where we handle DUMMY packages (these are
863 	 * dummy unflavored packages which 'cover' all the flavors for
864 	 * a package).  These are not real packages are marked SUCCESS
865 	 * at this time because their dependencies (the flavors) have all
866 	 * been built.
867 	 */
868 	while (idep_index != count || depi_index != count) {
869 		pkg_t *pkgi;
870 		pkg_t *ipkg;
871 
872 		/*
873 		 * Find candidate to start sorted by depi or idep.
874 		 */
875 		ipkg = NULL;
876 		while (idep_index < count) {
877 			ipkg = idep_ary[idep_index];
878 			if ((ipkg->flags &
879 			     (PKGF_SUCCESS | PKGF_FAILURE |
880 			      PKGF_ERROR | PKGF_RUNNING)) == 0) {
881 				break;
882 			}
883 			ipkg = NULL;
884 			++idep_index;
885 		}
886 
887 		pkgi = NULL;
888 		while (depi_index < count) {
889 			pkgi = depi_ary[depi_index];
890 			if ((pkgi->flags &
891 			     (PKGF_SUCCESS | PKGF_FAILURE |
892 			      PKGF_ERROR | PKGF_RUNNING)) == 0) {
893 				break;
894 			}
895 			pkgi = NULL;
896 			++depi_index;
897 		}
898 
899 		/*
900 		 * ipkg and pkgi must either both be NULL, or both
901 		 * be non-NULL.
902 		 */
903 		if (ipkg == NULL && pkgi == NULL)
904 			break;
905 		ddassert(ipkg && pkgi);
906 
907 		/*
908 		 * Handle the NOBUILD case right here, there's no point
909 		 * queueing it anywhere.
910 		 */
911 		if (ipkg->flags & PKGF_NOBUILD) {
912 			char *reason;
913 			char skipbuf[16];
914 			int scount;
915 
916 			scount = buildskipcount_dueto(ipkg, 1);
917 			buildskipcount_dueto(ipkg, 0);
918 			if (scount) {
919 				snprintf(skipbuf, sizeof(skipbuf), " %d",
920 					 scount);
921 			} else {
922 				skipbuf[0] = 0;
923 			}
924 
925 			ipkg->flags |= PKGF_FAILURE;
926 			ipkg->flags &= ~PKGF_BUILDLIST;
927 
928 			reason = buildskipreason(NULL, ipkg);
929 			if (ipkg->flags & PKGF_NOBUILD_I) {
930 				++BuildIgnoreCount;
931 				dlog(DLOG_IGN,
932 				     "[XXX] %s%s ignored due to %s\n",
933 				     ipkg->portdir, skipbuf, reason);
934 				RunStatsUpdateCompletion(NULL, DLOG_IGN, ipkg,
935 							 reason, skipbuf);
936 				doHook(ipkg, "hook_pkg_ignored",
937 				       HookPkgIgnored, 0);
938 			} else {
939 				++BuildSkipCount;
940 				dlog(DLOG_SKIP,
941 				     "[XXX] %s%s skipped due to %s\n",
942 				     ipkg->portdir, skipbuf, reason);
943 				RunStatsUpdateCompletion(NULL, DLOG_SKIP, ipkg,
944 							 reason, skipbuf);
945 				doHook(ipkg, "hook_pkg_skipped",
946 				       HookPkgSkipped, 0);
947 			}
948 			free(reason);
949 			++BuildCount;
950 			continue;
951 		}
952 		if (pkgi->flags & PKGF_NOBUILD) {
953 			char *reason;
954 			char skipbuf[16];
955 			int scount;
956 
957 			scount = buildskipcount_dueto(pkgi, 1);
958 			buildskipcount_dueto(pkgi, 0);
959 			if (scount) {
960 				snprintf(skipbuf, sizeof(skipbuf), " %d",
961 					 scount);
962 			} else {
963 				skipbuf[0] = 0;
964 			}
965 
966 			pkgi->flags |= PKGF_FAILURE;
967 			pkgi->flags &= ~PKGF_BUILDLIST;
968 
969 			reason = buildskipreason(NULL, pkgi);
970 			if (pkgi->flags & PKGF_NOBUILD_I) {
971 				++BuildIgnoreCount;
972 				dlog(DLOG_IGN, "[XXX] %s%s ignored due to %s\n",
973 				     pkgi->portdir, skipbuf, reason);
974 				RunStatsUpdateCompletion(NULL, DLOG_IGN, pkgi,
975 							 reason, skipbuf);
976 				doHook(pkgi, "hook_pkg_ignored",
977 				       HookPkgIgnored, 0);
978 			} else {
979 				++BuildSkipCount;
980 				dlog(DLOG_SKIP,
981 				     "[XXX] %s%s skipped due to %s\n",
982 				     pkgi->portdir, skipbuf, reason);
983 				RunStatsUpdateCompletion(NULL, DLOG_SKIP, pkgi,
984 							 reason, skipbuf);
985 				doHook(pkgi, "hook_pkg_skipped",
986 				       HookPkgSkipped, 0);
987 			}
988 			free(reason);
989 			++BuildCount;
990 			continue;
991 		}
992 
993 		/*
994 		 * Block while no slots are available.  waitbuild()
995 		 * will clean out any DONE states.
996 		 */
997 		while (RunningWorkers >= DynamicMaxWorkers ||
998 		       RunningWorkers >= MaxWorkers - FailedWorkers) {
999 			waitbuild(RunningWorkers, 1);
1000 		}
1001 
1002 		/*
1003 		 * Find an available worker slot, there should be at
1004 		 * least one.
1005 		 */
1006 		for (i = 0; i < MaxWorkers; ++i) {
1007 			n = IterateWorker % MaxWorkers;
1008 			work = &WorkerAry[n];
1009 
1010 			if (work->state == WORKER_DONE ||
1011 			    work->state == WORKER_FAILED) {
1012 				workercomplete(work);
1013 			}
1014 			if (work->state == WORKER_NONE ||
1015 			    work->state == WORKER_IDLE) {
1016 				if (n <= MaxWorkers / 2) {
1017 					startworker(pkgi, work);
1018 				} else {
1019 					startworker(ipkg, work);
1020 				}
1021 				/*RunStatsUpdate(work);*/
1022 				break;
1023 			}
1024 			++IterateWorker;
1025 		}
1026 		ddassert(i != MaxWorkers);
1027 	}
1028 	RunStatsSync();
1029 
1030 	/*
1031 	 * We disposed of the whole list
1032 	 */
1033 	free(idep_ary);
1034 	free(depi_ary);
1035 	*build_listp = NULL;
1036 	*build_tailp = build_listp;
1037 }
1038 
1039 typedef const pkg_t *pkg_tt;
1040 
1041 static int
qsort_idep(const void * pkg1_arg,const void * pkg2_arg)1042 qsort_idep(const void *pkg1_arg, const void *pkg2_arg)
1043 {
1044 	const pkg_t *pkg1 = *(const pkg_tt *)pkg1_arg;
1045 	const pkg_t *pkg2 = *(const pkg_tt *)pkg2_arg;
1046 
1047 	return (pkg2->idep_count - pkg1->idep_count);
1048 }
1049 
1050 static int
qsort_depi(const void * pkg1_arg,const void * pkg2_arg)1051 qsort_depi(const void *pkg1_arg, const void *pkg2_arg)
1052 {
1053 	const pkg_t *pkg1 = *(const pkg_tt *)pkg1_arg;
1054 	const pkg_t *pkg2 = *(const pkg_tt *)pkg2_arg;
1055 
1056 	return ((pkg2->depi_count * pkg2->depi_depth) -
1057 		(pkg1->depi_count * pkg1->depi_depth));
1058 }
1059 
1060 /*
1061  * Frontend starts a pkg up on a worker
1062  *
1063  * WorkerMutex must be held.
1064  */
1065 static void
startworker(pkg_t * pkg,worker_t * work)1066 startworker(pkg_t *pkg, worker_t *work)
1067 {
1068 	switch(work->state) {
1069 	case WORKER_NONE:
1070 		pthread_create(&work->td, NULL, childBuilderThread, work);
1071 		work->state = WORKER_IDLE;
1072 		/* fall through */
1073 	case WORKER_IDLE:
1074 		work->pkg_dep_size =
1075 		childInstallPkgDeps_recurse(NULL, &pkg->idepon_list, 0, 1, 0);
1076 		childInstallPkgDeps_recurse(NULL, &pkg->idepon_list, 1, 1, 0);
1077 		RunningPkgDepSize += work->pkg_dep_size;
1078 
1079 		dlog(DLOG_ALL, "[%03d] START   %s "
1080 			       "##idep=%02d depi=%02d/%02d dep=%-4.2fG\n",
1081 		     work->index, pkg->portdir,
1082 		     pkg->idep_count, pkg->depi_count, pkg->depi_depth,
1083 		     (double)work->pkg_dep_size / (double)ONEGB);
1084 
1085 		cleanworker(work);
1086 		pkg->flags |= PKGF_RUNNING;
1087 		work->pkg = pkg;
1088 		pthread_cond_signal(&work->cond);
1089 		++RunningWorkers;
1090 		/*RunStatsUpdate(work);*/
1091 		break;
1092 	case WORKER_PENDING:
1093 	case WORKER_RUNNING:
1094 	case WORKER_DONE:
1095 	case WORKER_FAILED:
1096 	case WORKER_FROZEN:
1097 	case WORKER_EXITING:
1098 	default:
1099 		dfatal("startworker: [%03d] Unexpected state %d for worker %d",
1100 		       work->index, work->state, work->index);
1101 		break;
1102 	}
1103 }
1104 
1105 static void
cleanworker(worker_t * work)1106 cleanworker(worker_t *work)
1107 {
1108 	work->state = WORKER_PENDING;
1109 	work->flags = 0;
1110 	work->accum_error = 0;
1111 	work->start_time = time(NULL);
1112 }
1113 
1114 /*
1115  * Frontend finishes up a completed pkg on a worker.
1116  *
1117  * If the worker is in a FAILED state we clean the pkg out but (for now)
1118  * leave it in its failed state so we can debug.  At this point
1119  * workercomplete() will be called every once in a while on the state
1120  * and we have to deal with the NULL pkg.
1121  *
1122  * WorkerMutex must be held.
1123  */
1124 static void
workercomplete(worker_t * work)1125 workercomplete(worker_t *work)
1126 {
1127 	pkg_t *pkg;
1128 	time_t t;
1129 	int h;
1130 	int m;
1131 	int s;
1132 
1133 	/*
1134 	 * Steady state FAILED case.
1135 	 */
1136 	if (work->state == WORKER_FAILED) {
1137 		if (work->pkg == NULL)
1138 			return;
1139 	}
1140 
1141 	t = time(NULL) - work->start_time;
1142 	h = t / 3600;
1143 	m = t / 60 % 60;
1144 	s = t % 60;
1145 
1146 	/*
1147 	 * Reduce total dep size
1148 	 */
1149 	RunningPkgDepSize -= work->pkg_dep_size;
1150 	RunningPkgDepSize -= work->memuse;
1151 	work->pkg_dep_size = 0;
1152 	work->memuse = 0;
1153 
1154 	/*
1155 	 * Process pkg out of the worker
1156 	 */
1157 	pkg = work->pkg;
1158 	if (pkg->flags & (PKGF_ERROR|PKGF_NOBUILD)) {
1159 		char skipbuf[16];
1160 		int scount;
1161 
1162 		/*
1163 		 * Normally mark the package as failed, but if we are doing
1164 		 * a fetch-only, mark it as successful so dependant ports
1165 		 * still get fetched.
1166 		 */
1167 		if (FetchOnlyOpt)
1168 			pkg->flags |= PKGF_SUCCESS;
1169 		else
1170 			pkg->flags |= PKGF_FAILURE;
1171 
1172 		scount = buildskipcount_dueto(pkg, 1);
1173 		buildskipcount_dueto(pkg, 0);
1174 		if (scount) {
1175 			snprintf(skipbuf, sizeof(skipbuf), " %d",
1176 				 scount);
1177 		} else {
1178 			skipbuf[0] = 0;
1179 		}
1180 
1181 		/*
1182 		 * This NOBUILD condition XXX can occur if the package is
1183 		 * not allowed to be built.
1184 		 */
1185 		if (pkg->flags & PKGF_NOBUILD) {
1186 			char *reason;
1187 
1188 			reason = buildskipreason(NULL, pkg);
1189 			if (pkg->flags & PKGF_NOBUILD_I) {
1190 				++BuildIgnoreCount;
1191 				dlog(DLOG_SKIP, "[%03d] IGNORD %s%s - %s\n",
1192 				     work->index, pkg->portdir,
1193 				     skipbuf, reason);
1194 				RunStatsUpdateCompletion(work, DLOG_SKIP, pkg,
1195 							 reason, skipbuf);
1196 				doHook(pkg, "hook_pkg_ignored",
1197 				       HookPkgIgnored, 0);
1198 			} else {
1199 				++BuildSkipCount;
1200 				dlog(DLOG_SKIP, "[%03d] SKIPPD %s%s - %s\n",
1201 				     work->index, pkg->portdir,
1202 				     skipbuf, reason);
1203 				RunStatsUpdateCompletion(work, DLOG_SKIP, pkg,
1204 							 reason, skipbuf);
1205 				doHook(pkg, "hook_pkg_skipped",
1206 				       HookPkgSkipped, 0);
1207 			}
1208 			free(reason);
1209 		} else {
1210 			++BuildFailCount;
1211 			dlog(DLOG_FAIL | DLOG_RED,
1212 			     "[%03d] FAILURE %s%s ##%16.16s %02d:%02d:%02d\n",
1213 			     work->index, pkg->portdir, skipbuf,
1214 			     getphasestr(work->phase),
1215 			     h, m, s);
1216 			RunStatsUpdateCompletion(work, DLOG_FAIL, pkg,
1217 						 skipbuf, "");
1218 			doHook(pkg, "hook_pkg_failure", HookPkgFailure, 0);
1219 		}
1220 	} else {
1221 		if (CheckDBM) {
1222 			datum key;
1223 			datum data;
1224 
1225 			key.dptr = pkg->portdir;
1226 			key.dsize = strlen(pkg->portdir);
1227 			data.dptr = &pkg->crc32;
1228 			data.dsize = sizeof(pkg->crc32);
1229 #ifndef __DB_IS_THREADSAFE
1230 			pthread_mutex_lock(&DbmMutex);
1231 #endif
1232 			dbm_store(CheckDBM, key, data, DBM_REPLACE);
1233 #ifndef __DB_IS_THREADSAFE
1234 			pthread_mutex_unlock(&DbmMutex);
1235 #endif
1236 		}
1237 		pkg->flags |= PKGF_SUCCESS;
1238 		++BuildSuccessCount;
1239 		dlog(DLOG_SUCC | DLOG_GRN,
1240 		     "[%03d] SUCCESS %s ##%02d:%02d:%02d\n",
1241 		     work->index, pkg->portdir, h, m, s);
1242 		RunStatsUpdateCompletion(work, DLOG_SUCC, pkg, "", "");
1243 		doHook(pkg, "hook_pkg_success", HookPkgSuccess, 0);
1244 	}
1245 	++BuildCount;
1246 	pkg->flags &= ~PKGF_BUILDLIST;
1247 	pkg->flags &= ~PKGF_RUNNING;
1248 	work->pkg = NULL;
1249 	--RunningWorkers;
1250 
1251 	if (work->state == WORKER_FAILED) {
1252 		dlog(DLOG_ALL, "[%03d] XXX/XXX WORKER IS IN A FAILED STATE\n",
1253 		     work->index);
1254 		++FailedWorkers;
1255 	} else if (work->flags & WORKERF_FREEZE) {
1256 		dlog(DLOG_ALL, "[%03d] FROZEN(DEBUG) %s\n",
1257 		     work->index, pkg->portdir);
1258 		work->state = WORKER_FROZEN;
1259 	} else {
1260 		work->state = WORKER_IDLE;
1261 	}
1262 }
1263 
1264 /*
1265  * Wait for one or more workers to complete.
1266  *
1267  * WorkerMutex must be held.
1268  */
1269 static void
waitbuild(int whilematch,int dynamicmax)1270 waitbuild(int whilematch, int dynamicmax)
1271 {
1272 	static time_t wblast_time;
1273 	static time_t dmlast_time;
1274 	struct timespec ts;
1275 	worker_t *work;
1276 	time_t t;
1277 	int i;
1278 
1279 	if (whilematch == 0)
1280 		whilematch = 1;
1281 
1282 	while (RunningWorkers == whilematch) {
1283 		for (i = 0; i < MaxWorkers; ++i) {
1284 			work = &WorkerAry[i];
1285 			if (work->state == WORKER_DONE ||
1286 			    work->state == WORKER_FAILED) {
1287 				workercomplete(work);
1288 			} else {
1289 				pthread_cond_signal(&work->cond);
1290 			}
1291 			RunStatsUpdate(work, NULL);
1292 		}
1293 		RunStatsUpdateTop(1);
1294 		RunStatsUpdateLogs();
1295 		RunStatsSync();
1296 		if (RunningWorkers == whilematch) {
1297 			clock_gettime(CLOCK_REALTIME, &ts);
1298 			ts.tv_sec += 1;
1299 			ts.tv_nsec = 0;
1300 			pthread_cond_timedwait(&WorkerCond, &WorkerMutex, &ts);
1301 		}
1302 
1303 		/*
1304 		 * Dynamically reduce MaxWorkers based on the load.  When
1305 		 * the load exceeds 2 x ncpus we reduce the number of workers
1306 		 * up to 75% of MaxWorkers @ (5 x ncpus) load.
1307 		 *
1308 		 * Dynamically reduce MaxWorkers based on swap use, starting
1309 		 * at 10% swap and up to 75% of MaxWorkers at 40% swap.
1310 		 *
1311 		 * NOTE! Generally speaking this allows more workers to be
1312 		 *	 configured which helps in two ways.  First it allows
1313 		 *	 a higher build rate for smaller packages.  Second
1314 		 *	 it allows dsynth to ratchet-down the number of slots
1315 		 *	 when large packages are forcing the load up.
1316 		 *
1317 		 *	 A high load doesn't hurt efficiency, but swap usage
1318 		 *	 due to loading the tmpfs in lots of worker slots up
1319 		 *	 with tons of pkg install's (pre-reqs for a build)
1320 		 *	 does.  Reducing the number of worker slots has a
1321 		 *	 huge beneficial effect on reducing swap use / paging.
1322 		 */
1323 		t = time(NULL);
1324 		if (dynamicmax && (wblast_time == 0 ||
1325 				   (unsigned)(t - wblast_time) >= 5)) {
1326 			double min_load = 1.5 * NumCores;
1327 			double max_load = 5.0 * NumCores;
1328 			double min_swap = 0.10;
1329 			double max_swap = 0.40;
1330 			double dload[3];
1331 			double dswap;
1332 			int max1;
1333 			int max2;
1334 			int max3;
1335 			int max_sel;
1336 			int noswap;
1337 
1338 			wblast_time = t;
1339 
1340 			/*
1341 			 * Cap based on load.  This is back-loaded.
1342 			 */
1343 			getloadavg(dload, 3);
1344 			adjloadavg(dload);
1345 			if (dload[0] < min_load) {
1346 				max1 = MaxWorkers;
1347 			} else if (dload[0] <= max_load) {
1348 				max1 = MaxWorkers -
1349 				       MaxWorkers * 0.75 *
1350 				       (dload[0] - min_load) /
1351 				       (max_load - min_load);
1352 			} else {
1353 				max1 = MaxWorkers * 25 / 100;
1354 			}
1355 
1356 			/*
1357 			 * Cap based on swap use.  This is back-loaded.
1358 			 */
1359 			dswap = getswappct(&noswap);
1360 			if (dswap < min_swap) {
1361 				max2 = MaxWorkers;
1362 			} else if (dswap <= max_swap) {
1363 				max2 = MaxWorkers -
1364 				       MaxWorkers * 0.75 *
1365 				       (dswap - min_swap) /
1366 				       (max_swap - min_swap);
1367 			} else {
1368 				max2 = MaxWorkers * 25 / 100;
1369 			}
1370 
1371 			/*
1372 			 * Cap based on aggregate pkg-dependency memory
1373 			 * use installed in worker slots.  This is
1374 			 * front-loaded.
1375 			 *
1376 			 * Since it can take a while for workers to retire
1377 			 * (to reduce RunningPkgDepSize), just set our
1378 			 * target 1 below the current run count to allow
1379 			 * jobs to retire without being replaced with new
1380 			 * jobs.
1381 			 *
1382 			 * In addition, in order to avoid a paging 'shock',
1383 			 * We enforce a 30 second-per-increment slow-start
1384 			 * once RunningPkgDepSize exceeds 1/2 the target.
1385 			 */
1386 			if (RunningPkgDepSize > PkgDepMemoryTarget) {
1387 				max3 = RunningWorkers - 1;
1388 			} else if (RunningPkgDepSize > PkgDepMemoryTarget / 2) {
1389 				if (dmlast_time == 0 ||
1390 				    (unsigned)(t - dmlast_time) >= 30) {
1391 					dmlast_time = t;
1392 					max3 = RunningWorkers + 1;
1393 				} else {
1394 					max3 = RunningWorkers;
1395 				}
1396 			} else {
1397 				max3 = MaxWorkers;
1398 			}
1399 
1400 			/*
1401 			 * Dynamic scale adjustment
1402 			 */
1403 
1404 			if (PkgDepScaleTarget != 100) {
1405 				max1 = max1 * (PkgDepScaleTarget + 50) / 100;
1406 				max2 = max2 * (PkgDepScaleTarget + 50) / 100;
1407 				max3 = max3 * (PkgDepScaleTarget + 50) / 100;
1408 			}
1409 
1410 			/*
1411 			 * Priority reduction, convert to DynamicMaxWorkers
1412 			 */
1413 			max_sel = max1;
1414 			if (max_sel > max2)
1415 				max_sel = max2;
1416 			if (max_sel > max3)
1417 				max_sel = max3;
1418 
1419 			/*
1420 			 * Restrict to allowed range, and also handle
1421 			 * slow-start.
1422 			 */
1423 			if (max_sel < 1)
1424 				max_sel = 1;
1425 			if (max_sel > DynamicMaxWorkers + 1)
1426 				max_sel = DynamicMaxWorkers + 1;
1427 			if (max_sel > MaxWorkers)
1428 				max_sel = MaxWorkers;
1429 
1430 			/*
1431 			 * Stop waiting if DynamicMaxWorkers is going to
1432 			 * increase.
1433 			 */
1434 			if (DynamicMaxWorkers < max1)
1435 				whilematch = -1;
1436 
1437 			/*
1438 			 * And adjust
1439 			 */
1440 			if (DynamicMaxWorkers != max1) {
1441 				dlog(DLOG_ALL | DLOG_FILTER,
1442 				     "[XXX] Load=%-6.2f(%2d) "
1443 				     "Swap=%-3.2f%%(%2d) "
1444 				     "Mem=%3.2fG(%2d) "
1445 				     "Adjust Workers %d->%d\n",
1446 				     dload[0], max1,
1447 				     dswap * 100.0, max2,
1448 				     RunningPkgDepSize / (double)ONEGB, max3,
1449 				     DynamicMaxWorkers, max_sel);
1450 				DynamicMaxWorkers = max_sel;
1451 			}
1452 		}
1453 	}
1454 }
1455 
1456 
1457 /*
1458  * Worker pthread (WorkerAry)
1459  *
1460  * This thread belongs to the dsynth master process and handles a worker slot.
1461  * (this_thread) -> WORKER fork/exec (WorkerPocess) -> (pty) -> sub-processes.
1462  */
1463 static void *
childBuilderThread(void * arg)1464 childBuilderThread(void *arg)
1465 {
1466 	char *envary[1] = { NULL };
1467 	worker_t *work = arg;
1468 	wmsg_t wmsg;
1469 	pkg_t *pkg;
1470 	pid_t pid;
1471 	int status;
1472 	int flags;
1473 	volatile int dowait;
1474 	char slotbuf[8];
1475 	char fdbuf[8];
1476 	char flagsbuf[16];
1477 
1478 	setNumaDomain(work->index);
1479 	pthread_mutex_lock(&WorkerMutex);
1480 
1481 	while (work->terminate == 0) {
1482 		dowait = 1;
1483 
1484 		switch(work->state) {
1485 		case WORKER_IDLE:
1486 			break;
1487 		case WORKER_PENDING:
1488 			/*
1489 			 * Fork the management process for the pkg operation
1490 			 * on this worker slot.
1491 			 *
1492 			 * This process will set up the environment, do the
1493 			 * mounts, will become the reaper, and will process
1494 			 * pipe commands and handle chroot operations.
1495 			 *
1496 			 * NOTE: If SOCK_CLOEXEC is not supported WorkerMutex
1497 			 *	 is sufficient to interlock F_SETFD FD_CLOEXEC
1498 			 *	 operations.
1499 			 */
1500 			ddassert(work->pkg);
1501 			if (socketpair(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC,
1502 				       PF_UNSPEC, work->fds)) {
1503 				dfatal_errno("socketpair() during worker fork");
1504 			}
1505 			snprintf(slotbuf, sizeof(slotbuf), "%d", work->index);
1506 			snprintf(fdbuf, sizeof(fdbuf), "3");
1507 
1508 			/*
1509 			 * Pass global flags and add-in the DEBUGSTOP if
1510 			 * the package is flagged for debugging.
1511 			 */
1512 			flags = WorkerProcFlags;
1513 
1514 			if (work->pkg->flags & PKGF_DEBUGSTOP)
1515 				flags |= WORKER_PROC_DEBUGSTOP;
1516 			else
1517 				flags &= ~WORKER_PROC_DEBUGSTOP;
1518 
1519 			if (PkgVersionPkgSuffix)
1520 				flags |= WORKER_PROC_PKGV17;
1521 			else
1522 				flags &= ~WORKER_PROC_PKGV17;
1523 
1524 			snprintf(flagsbuf, sizeof(flagsbuf), "%d", flags);
1525 
1526 			/*
1527 			 * fds[0] - master
1528 			 * fds[1] - slave
1529 			 *
1530 			 * We pass the salve descriptor in fd 3 and close all
1531 			 * other descriptors for security.
1532 			 */
1533 			pthread_mutex_unlock(&WorkerMutex);
1534 			pid = vfork();
1535 			if (pid == 0) {
1536 				close(work->fds[0]);
1537 				dup2(work->fds[1], 3);
1538 				closefrom(4);
1539 				fcntl(3, F_SETFD, 0);
1540 				execle(DSynthExecPath, DSynthExecPath,
1541 				       "-C", ConfigBase,
1542 				       "-p", Profile,
1543 				       "WORKER", slotbuf, fdbuf,
1544 				       work->pkg->portdir, work->pkg->pkgfile,
1545 				       flagsbuf,
1546 				       NULL, envary);
1547 				write(2, "EXECLE FAILURE\n", 15);
1548 				_exit(1);
1549 			}
1550 			pthread_mutex_lock(&WorkerMutex);
1551 			close(work->fds[1]);
1552 			work->phase = PHASE_PENDING;
1553 			work->lines = 0;
1554 			work->memuse = 0;
1555 			work->pid = pid;
1556 			work->state = WORKER_RUNNING;
1557 			/* fall through */
1558 		case WORKER_RUNNING:
1559 			/*
1560 			 * Poll for status updates, if NULL is returned
1561 			 * and status is non-zero, the communications link
1562 			 * failed unexpectedly.
1563 			 */
1564 			pkg = work->pkg;
1565 			pthread_mutex_unlock(&WorkerMutex);
1566 			status = ipcreadmsg(work->fds[0], &wmsg);
1567 			pthread_mutex_lock(&WorkerMutex);
1568 
1569 			if (status == 0) {
1570 				/*
1571 				 * Normal message, can include normal
1572 				 * termination which changes us over
1573 				 * to another state.
1574 				 */
1575 				dowait = 0;
1576 				switch(wmsg.cmd) {
1577 				case WMSG_CMD_INSTALL_PKGS:
1578 					wmsg.cmd = WMSG_RES_INSTALL_PKGS;
1579 					wmsg.status = childInstallPkgDeps(work);
1580 					pthread_mutex_unlock(&WorkerMutex);
1581 					ipcwritemsg(work->fds[0], &wmsg);
1582 					pthread_mutex_lock(&WorkerMutex);
1583 					break;
1584 				case WMSG_CMD_STATUS_UPDATE:
1585 					work->phase = wmsg.phase;
1586 					work->lines = wmsg.lines;
1587 					if (work->memuse != wmsg.memuse) {
1588 						RunningPkgDepSize +=
1589 						wmsg.memuse - work->memuse;
1590 						work->memuse = wmsg.memuse;
1591 					}
1592 					break;
1593 				case WMSG_CMD_SUCCESS:
1594 					work->flags |= WORKERF_SUCCESS;
1595 					break;
1596 				case WMSG_CMD_FAILURE:
1597 					work->flags |= WORKERF_FAILURE;
1598 					break;
1599 				case WMSG_CMD_FREEZEWORKER:
1600 					work->flags |= WORKERF_FREEZE;
1601 					break;
1602 				default:
1603 					break;
1604 				}
1605 				RunStatsUpdate(work, NULL);
1606 				RunStatsSync();
1607 			} else {
1608 				close(work->fds[0]);
1609 				pthread_mutex_unlock(&WorkerMutex);
1610 				while (waitpid(work->pid, &status, 0) < 0 &&
1611 				       errno == EINTR) {
1612 					;
1613 				}
1614 				pthread_mutex_lock(&WorkerMutex);
1615 
1616 				if (work->flags & WORKERF_SUCCESS) {
1617 					pkg->flags |= PKGF_SUCCESS;
1618 					work->state = WORKER_DONE;
1619 				} else if (work->flags & WORKERF_FAILURE) {
1620 					pkg->flags |= PKGF_FAILURE;
1621 					work->state = WORKER_DONE;
1622 				} else {
1623 					pkg->flags |= PKGF_FAILURE;
1624 					work->state = WORKER_FAILED;
1625 				}
1626 				work->flags |= WORKERF_STATUS_UPDATE;
1627 				pthread_cond_signal(&WorkerCond);
1628 			}
1629 			break;
1630 		case WORKER_DONE:
1631 			/*
1632 			 * pkg remains attached until frontend processes the
1633 			 * completion.  The frontend will then set the state
1634 			 * back to idle.
1635 			 */
1636 			break;
1637 		case WORKER_FAILED:
1638 			/*
1639 			 * A worker failure means that the worker did not
1640 			 * send us a WMSG_CMD_SUCCESS or WMSG_CMD_FAILURE
1641 			 * ipc before terminating.
1642 			 *
1643 			 * We just sit in this state until the front-end
1644 			 * does something about it.
1645 			 */
1646 			break;
1647 		case WORKER_FROZEN:
1648 			/*
1649 			 * A worker getting frozen is debug-related.  We
1650 			 * just sit in this state (likely forever).
1651 			 */
1652 			break;
1653 		default:
1654 			dfatal("worker: [%03d] Unexpected state %d "
1655 			       "for worker %d",
1656 			       work->index, work->state, work->index);
1657 			/* NOT REACHED */
1658 			break;
1659 		}
1660 
1661 		/*
1662 		 * The dsynth frontend will poll us approximately once
1663 		 * a second (its variable).
1664 		 */
1665 		if (dowait)
1666 			pthread_cond_wait(&work->cond, &WorkerMutex);
1667 	}
1668 
1669 	/*
1670 	 * Scrap the comm socket if running, this should cause the worker
1671 	 * process to kill its sub-programs and cleanup.
1672 	 */
1673 	if (work->state == WORKER_RUNNING) {
1674 		pthread_mutex_unlock(&WorkerMutex);
1675 		close(work->fds[0]);
1676 		while (waitpid(work->pid, &status, 0) < 0 &&
1677 		       errno == EINTR);
1678 		pthread_mutex_lock(&WorkerMutex);
1679 	}
1680 
1681 	/*
1682 	 * Final handshake
1683 	 */
1684 	work->state = WORKER_EXITING;
1685 	pthread_cond_signal(&WorkerCond);
1686 	pthread_mutex_unlock(&WorkerMutex);
1687 
1688 	return NULL;
1689 }
1690 
1691 /*
1692  * Install all the binary packages (we have already built them) that
1693  * the current work package depends on, without duplicates, in a script
1694  * which will be run from within the specified work jail.
1695  *
1696  * Locked by WorkerMutex (global)
1697  */
1698 static int
childInstallPkgDeps(worker_t * work)1699 childInstallPkgDeps(worker_t *work)
1700 {
1701 	char *buf;
1702 	FILE *fp;
1703 
1704 	if (PKGLIST_EMPTY(&work->pkg->idepon_list))
1705 		return 0;
1706 
1707 	asprintf(&buf, "%s/tmp/dsynth_install_pkgs", work->basedir);
1708 	fp = fopen(buf, "w");
1709 	ddassert(fp != NULL);
1710 	fprintf(fp, "#!/bin/sh\n");
1711 	fprintf(fp, "#\n");
1712 	fchmod(fileno(fp), 0755);
1713 
1714 	childInstallPkgDeps_recurse(fp, &work->pkg->idepon_list, 0, 1, 0);
1715 	childInstallPkgDeps_recurse(fp, &work->pkg->idepon_list, 1, 1, 0);
1716 	fprintf(fp, "\nexit 0\n");
1717 	fclose(fp);
1718 	freestrp(&buf);
1719 
1720 	return 1;
1721 }
1722 
1723 /*
1724  * Recursive child install dependencies.
1725  *
1726  * first_one_only is only specified if the pkg the list comes from
1727  * is a generic unflavored package that has flavors, telling us to
1728  * dive the first flavor only.
1729  *
1730  * However, in nearly all cases this flag will now be zero because
1731  * this code now dives the first flavor when encountering a dummy node
1732  * and clears nfirst on success.  Hence if you are asking why 'nfirst'
1733  * is set to 1, and then zero, instead of just being removed entirely,
1734  * it is because there might still be an edge case here.
1735  */
1736 static size_t
childInstallPkgDeps_recurse(FILE * fp,pkglink_t * list,int undoit,int depth,int first_one_only)1737 childInstallPkgDeps_recurse(FILE *fp, pkglink_t *list, int undoit,
1738 			    int depth, int first_one_only)
1739 {
1740 	pkglink_t *link;
1741 	pkg_t *pkg;
1742 	size_t tot = 0;
1743 	int ndepth;
1744 	int nfirst;
1745 
1746 	PKGLIST_FOREACH(link, list) {
1747 		pkg = link->pkg;
1748 
1749 		/*
1750 		 * We don't want to mess up our depth test just below if
1751 		 * a DUMMY node had to be inserted.  The nodes under the
1752 		 * dummy node.
1753 		 *
1754 		 * The elements under a dummy node represent all the flabor,
1755 		 * a dependency that directly references a dummy node only
1756 		 * uses the first flavor (first_one_only / nfirst).
1757 		 */
1758 		ndepth = (pkg->flags & PKGF_DUMMY) ? depth : depth + 1;
1759 		nfirst = (pkg->flags & PKGF_DUMMY) ? 1 : 0;
1760 
1761 		/*
1762 		 * We only need all packages for the top-level dependencies.
1763 		 * The deeper ones only need DEP_TYPE_LIB and DEP_TYPE_RUN
1764 		 * (types greater than DEP_TYPE_BUILD) since they are already
1765 		 * built.
1766 		 */
1767 		if (depth > 1 && link->dep_type <= DEP_TYPE_BUILD) {
1768 			if (first_one_only)
1769 				break;
1770 			continue;
1771 		}
1772 
1773 		/*
1774 		 * If this is a dummy node with no package, the originator
1775 		 * is requesting a flavored package.  We select the default
1776 		 * flavor which we presume is the first one.
1777 		 */
1778 		if (pkg->pkgfile == NULL && (pkg->flags & PKGF_DUMMY)) {
1779 			pkg_t *spkg = pkg->idepon_list.next->pkg;
1780 
1781 			if (spkg) {
1782 				if (fp) {
1783 					fprintf(fp,
1784 						"echo 'UNFLAVORED %s -> use "
1785 						"%s'\n",
1786 						pkg->portdir,
1787 						spkg->portdir);
1788 				}
1789 				pkg = spkg;
1790 				nfirst = 0;
1791 			} else {
1792 				if (fp) {
1793 					fprintf(fp,
1794 						"echo 'CANNOT FIND DEFAULT "
1795 						"FLAVOR FOR %s'\n",
1796 						pkg->portdir);
1797 				}
1798 			}
1799 		}
1800 
1801 		if (undoit) {
1802 			if (pkg->dsynth_install_flg == 1) {
1803 				pkg->dsynth_install_flg = 0;
1804 				tot += childInstallPkgDeps_recurse(fp,
1805 							    &pkg->idepon_list,
1806 							    undoit,
1807 							    ndepth, nfirst);
1808 			}
1809 			if (first_one_only)
1810 				break;
1811 			continue;
1812 		}
1813 
1814 		if (pkg->dsynth_install_flg) {
1815 			if (DebugOpt >= 2 && pkg->pkgfile && fp) {
1816 				fprintf(fp, "echo 'AlreadyHave %s'\n",
1817 					pkg->pkgfile);
1818 			}
1819 			if (first_one_only)
1820 				break;
1821 			continue;
1822 		}
1823 
1824 		tot += childInstallPkgDeps_recurse(fp, &pkg->idepon_list,
1825 						   undoit, ndepth, nfirst);
1826 		if (pkg->dsynth_install_flg) {
1827 			if (first_one_only)
1828 				break;
1829 			continue;
1830 		}
1831 		pkg->dsynth_install_flg = 1;
1832 
1833 		/*
1834 		 * Generate package installation command
1835 		 */
1836 		if (fp && pkg->pkgfile) {
1837 			fprintf(fp, "echo 'Installing /packages/All/%s'\n",
1838 				pkg->pkgfile);
1839 			fprintf(fp, "pkg install -q -U -y /packages/All/%s "
1840 				"|| exit 1\n",
1841 				pkg->pkgfile);
1842 		} else if (fp) {
1843 			fprintf(fp, "echo 'CANNOT FIND PKG FOR %s'\n",
1844 				pkg->portdir);
1845 		}
1846 
1847 		if (pkg->pkgfile) {
1848 			struct stat st;
1849 			char *path;
1850 			char *ptr;
1851 
1852 			asprintf(&path, "%s/%s", RepositoryPath, pkg->pkgfile);
1853 			ptr = strrchr(pkg->pkgfile, '.');
1854 			if (stat(path, &st) == 0) {
1855 				if (strcmp(ptr, ".tar") == 0)
1856 					tot += st.st_size;
1857 				else if (strcmp(ptr, ".tgz") == 0)
1858 					tot += st.st_size * 3;
1859 				else if (strcmp(ptr, ".txz") == 0)
1860 					tot += st.st_size * 5;
1861 				else if (strcmp(ptr, ".tzst") == 0)
1862 					tot += st.st_size * 5;
1863 				else if (strcmp(ptr, ".tbz") == 0)
1864 					tot += st.st_size * 3;
1865 				else
1866 					tot += st.st_size * 2;
1867 			}
1868 			free(path);
1869 		}
1870 		if (first_one_only)
1871 			break;
1872 	}
1873 	return tot;
1874 }
1875 
1876 /*
1877  * Worker process interactions.
1878  *
1879  * The worker process is responsible for managing the build of a single
1880  * package.  It is exec'd by the master dsynth and only loads the
1881  * configuration.
1882  *
1883  * This process does not run in the chroot.  It will become the reaper for
1884  * all sub-processes and it will enter the chroot to execute various phases.
1885  * It catches SIGINTR, SIGHUP, and SIGPIPE and will iterate, terminate, and
1886  * reap all sub-process upon kill or exit.
1887  *
1888  * The command line forwarded to this function is:
1889  *
1890  *	WORKER slot# socketfd portdir/subdir
1891  *
1892  * TERM=dumb
1893  * USER=root
1894  * HOME=/root
1895  * LANG=C
1896  * SSL_NO_VERIFY_PEER=1
1897  * USE_PACKAGE_DEPENDS_ONLY=1
1898  * PORTSDIR=/xports
1899  * PORT_DBDIR=/options		For ports options
1900  * PACKAGE_BUILDING=yes		Don't build packages that aren't legally
1901  *				buildable for a binary repo.
1902  * PKG_DBDIR=/var/db/pkg
1903  * PKG_CACHEDIR=/var/cache/pkg
1904  * PKG_CREATE_VERBOSE=yes	Ensure periodic output during packaging
1905  * (custom environment)
1906  * PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin
1907  * UNAME_s=DragonFly		(example)
1908  * UNAME_v=DragonFly 5.7-SYNTH	(example)
1909  * UNAME_p=x86_64		(example)
1910  * UNAME_m=x86_64		(example)
1911  * UNAME_r=5.7-SYNTH		(example)
1912  * NO_DEPENDS=yes		(conditional based on phase)
1913  * DISTDIR=/distfiles
1914  * WRKDIRPREFIX=/construction
1915  * BATCH=yes
1916  * MAKE_JOBS_NUMBER=n
1917  *
1918  * SETUP:
1919  *	ldconfig -R
1920  *	/usr/local/sbin/pkg-static install /packages/All/<the pkg pkg>
1921  *	/usr/local/sbin/pkg-static install /packages/All/<pkg>
1922  *			(for all dependencies)
1923  *
1924  * PHASES: 		make -C path FLAVOR=flavor <phase>
1925  *	check-sanity
1926  *	pkg-depends
1927  *	fetch-depends
1928  *	fetch
1929  *	checksum
1930  *	extract-depends
1931  *	extract
1932  *	patch-depends
1933  *	patch
1934  *	build-depends
1935  *	lib-depends
1936  *	configure
1937  *	build
1938  *	run-depends
1939  *	stage
1940  *	test		(skipped)
1941  *	check-plist	('dsynth test blahblah' or 'dsynth -D everything' only)
1942  *	package		 e.g. /construction/lang/perl5.28/pkg/perl5-5.28.2.txz
1943  *	install		(skipped)
1944  *	deinstall	(skipped)
1945  */
1946 void
WorkerProcess(int ac,char ** av)1947 WorkerProcess(int ac, char **av)
1948 {
1949 	wmsg_t wmsg;
1950 	int fd;
1951 	int slot;
1952 	int tmpfd;
1953 	int pkgpkg = 0;
1954 	int status;
1955 	int len;
1956 	int do_install_phase;
1957 	char *portdir;
1958 	char *pkgfile;
1959 	char *flavor;
1960 	char *buf;
1961 	worker_t *work;
1962 	bulk_t *bulk;
1963 	pkg_t pkg;
1964 	buildenv_t *benv;
1965 	FILE *fp;
1966 
1967 	/*
1968 	 * Parse arguments
1969 	 */
1970 	if (ac != 6) {
1971 		dlog(DLOG_ALL, "WORKER PROCESS %d- bad arguments\n", getpid());
1972 		exit(1);
1973 	}
1974 	slot = strtol(av[1], NULL, 0);
1975 	setNumaDomain(slot);
1976 
1977 	fd = strtol(av[2], NULL, 0);	/* master<->slave messaging */
1978 	portdir = av[3];
1979 	pkgfile = av[4];
1980 
1981 	flavor = strchr(portdir, '@');
1982 	if (flavor) {
1983 		*flavor++ = 0;
1984 		asprintf(&buf, "@%s", flavor);
1985 		WorkerFlavorPrt = buf;
1986 		buf = NULL;	/* safety */
1987 	}
1988 	WorkerProcFlags = strtol(av[5], NULL, 0);
1989 
1990 	if (WorkerProcFlags & WORKER_PROC_FETCHONLY)
1991 		FetchOnlyOpt = 1;
1992 	if (WorkerProcFlags & WORKER_PROC_PKGV17)
1993 		PkgVersionPkgSuffix = 1;
1994 
1995 	bzero(&wmsg, sizeof(wmsg));
1996 
1997 	setproctitle("[%02d] WORKER STARTUP  %s%s",
1998 		     slot, portdir, WorkerFlavorPrt);
1999 
2000 	if (strcmp(portdir, "ports-mgmt/pkg") == 0)
2001 		pkgpkg = 1;
2002 
2003 	signal(SIGTERM, phaseTerminateSignal);
2004 	signal(SIGINT, phaseTerminateSignal);
2005 	signal(SIGHUP, phaseTerminateSignal);
2006 
2007 	/*
2008 	 * Set up the environment
2009 	 */
2010 	setenv("TERM", "dumb", 1);
2011 	setenv("USER", "root", 1);
2012 	setenv("HOME", "/root", 1);
2013 	setenv("LANG", "C", 1);
2014 	setenv("SSL_NO_VERIFY_PEER", "1", 1);
2015 
2016 	/*
2017 	 * NOTE: PKG_SUFX - pkg versions older than 1.17
2018 	 *	 PKG_COMPRESSION_FORMAT - pkg versions >= 1.17
2019 	 *
2020 	 *	 Avoid WARNING messages in the logs by omitting
2021 	 *	 PKG_SUFX when we know the pkg version is >= 1.17.
2022 	 */
2023 	addbuildenv("USE_PACKAGE_DEPENDS_ONLY", "yes", BENV_MAKECONF);
2024 	addbuildenv("PORTSDIR", "/xports", BENV_MAKECONF);
2025 	addbuildenv("PORT_DBDIR", "/options", BENV_MAKECONF);
2026 	addbuildenv("PKG_DBDIR", "/var/db/pkg", BENV_MAKECONF);
2027 	addbuildenv("PKG_CACHEDIR", "/var/cache/pkg", BENV_MAKECONF);
2028 	addbuildenv("PKG_COMPRESSION_FORMAT", UsePkgSufx, BENV_MAKECONF);
2029 	if (PkgVersionPkgSuffix == 0)
2030 		addbuildenv("PKG_SUFX", UsePkgSufx, BENV_MAKECONF);
2031 
2032 	/*
2033 	 * We are exec'ing the worker process so various bits of global
2034 	 * state that we want to inherit have to be passed in.
2035 	 */
2036 	if (WorkerProcFlags & WORKER_PROC_DEVELOPER)
2037 		addbuildenv("DEVELOPER", "1", BENV_MAKECONF);
2038 
2039 	/*
2040 	 * CCache is a horrible unreliable hack but... leave the
2041 	 * mechanism in-place in case someone has a death wish.
2042 	 */
2043 	if (UseCCache) {
2044 		addbuildenv("WITH_CCACHE_BUILD", "yes", BENV_MAKECONF);
2045 		addbuildenv("CCACHE_DIR", "/ccache", BENV_MAKECONF);
2046 	}
2047 
2048 	addbuildenv("UID", "0", BENV_MAKECONF);
2049 	addbuildenv("ARCH", ArchitectureName, BENV_MAKECONF);
2050 
2051 	/*
2052 	 * Always honor either the operating system detection or the
2053 	 * operating system selection in the config file.
2054 	 */
2055 	addbuildenv("OPSYS", OperatingSystemName, BENV_MAKECONF);
2056 
2057 #ifdef __DragonFly__
2058 	addbuildenv("DFLYVERSION", VersionFromParamHeader, BENV_MAKECONF);
2059 	addbuildenv("OSVERSION", "9999999", BENV_MAKECONF);
2060 #else
2061 #error "Need OS-specific data to generate make.conf"
2062 #endif
2063 
2064 	addbuildenv("OSREL", ReleaseName, BENV_MAKECONF);
2065 	addbuildenv("_OSRELEASE", VersionOnlyName, BENV_MAKECONF);
2066 
2067 	setenv("PATH",
2068 	       "/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin",
2069 	       1);
2070 
2071 	setenv("UNAME_s", OperatingSystemName, 1);
2072 	setenv("UNAME_v", VersionName, 1);
2073 	setenv("UNAME_p", ArchitectureName, 1);
2074 	setenv("UNAME_m", MachineName, 1);
2075 	setenv("UNAME_r", ReleaseName, 1);
2076 
2077 	addbuildenv("DISTDIR", "/distfiles", BENV_MAKECONF);
2078 	addbuildenv("WRKDIRPREFIX", "/construction", BENV_MAKECONF);
2079 	addbuildenv("BATCH", "yes", BENV_MAKECONF);
2080 
2081 	/*
2082 	 * Special consideration
2083 	 *
2084 	 * PACKAGE_BUILDING	- Disallow packaging ports which do not allow
2085 	 *			  for binary distribution.
2086 	 *
2087 	 * PKG_CREATE_VERBOSE	- Ensure periodic output during the packaging
2088 	 *			  process to avoid a watchdog timeout.
2089 	 *
2090 	 */
2091 	addbuildenv("PACKAGE_BUILDING", "yes", BENV_MAKECONF);
2092 	addbuildenv("PKG_CREATE_VERBOSE", "yes", BENV_MAKECONF);
2093 	asprintf(&buf, "%d", MaxJobs);
2094 	addbuildenv("MAKE_JOBS_NUMBER", buf, BENV_MAKECONF);
2095 	freestrp(&buf);
2096 
2097 	if (flavor)
2098 		setenv("FLAVOR", flavor, 1);
2099 
2100 	/*
2101 	 * Become the reaper
2102 	 */
2103 	if (procctl(P_PID, getpid(), PROC_REAP_ACQUIRE, NULL) < 0)
2104 		dfatal_errno("procctl() - Cannot become reaper");
2105 
2106 	/*
2107 	 * Initialize a worker structure
2108 	 */
2109 	DoInitBuild(slot);
2110 
2111 	bzero(&pkg, sizeof(pkg));
2112 	pkg.portdir = portdir;		/* sans flavor */
2113 	pkg.pkgfile = pkgfile;
2114 	if (strchr(portdir, '/'))
2115 		len = strchr(portdir, '/') - portdir;
2116 	else
2117 		len = 0;
2118 
2119 	/*
2120 	 * Setup the logfile
2121 	 */
2122 	asprintf(&pkg.logfile,
2123 		 "%s/%*.*s___%s%s%s.log",
2124 		 LogsPath, len, len, portdir,
2125 		 ((portdir[len] == '/') ? portdir + len + 1 : portdir + len),
2126 		 (flavor ? "@" : ""),
2127 		 (flavor ? flavor : ""));
2128 	tmpfd = open(pkg.logfile, O_RDWR|O_CREAT|O_TRUNC, 0666);
2129 	if (tmpfd >= 0) {
2130 		if (DebugOpt >= 2) {
2131 			dlog(DLOG_ALL, "[%03d] %s LOGFILE %s\n",
2132 			     slot, pkg.portdir, pkg.logfile);
2133 		}
2134 		close(tmpfd);
2135 	} else {
2136 		dlog(DLOG_ALL, "[%03d] LOGFILE %s (create failed)\n",
2137 		     slot, pkg.logfile);
2138 	}
2139 
2140 	/*
2141 	 * Setup the work structure.  Because this is an exec'd sub-process,
2142 	 * there is only one work structure.
2143 	 */
2144 	work = &WorkerAry[0];
2145 	work->flavor = flavor;
2146 	work->fds[0] = fd;
2147 	work->pkg = &pkg;
2148 	work->start_time = time(NULL);
2149 
2150 	/*
2151 	 * Do mounts
2152 	 */
2153 	SigWork = work;
2154 	setproctitle("[%02d] WORKER MOUNTS   %s%s",
2155 		     slot, portdir, WorkerFlavorPrt);
2156 	DoWorkerMounts(work);
2157 
2158 	/*
2159 	 * Generate an /etc/make.conf in the build base
2160 	 */
2161 	asprintf(&buf, "%s/etc/make.conf", work->basedir);
2162 	fp = fopen(buf, "w");
2163 	dassert_errno(fp, "Unable to create %s\n", buf);
2164 	for (benv = BuildEnv; benv; benv = benv->next) {
2165 		if (benv->type & BENV_PKGLIST)
2166 			continue;
2167 		if ((benv->type & BENV_CMDMASK) == BENV_MAKECONF) {
2168 			if (DebugOpt >= 2) {
2169 				dlog(DLOG_ALL, "[%03d] ENV %s=%s\n",
2170 				     slot, benv->label, benv->data);
2171 			}
2172 			fprintf(fp, "%s=%s\n", benv->label, benv->data);
2173 		}
2174 	}
2175 	fclose(fp);
2176 	freestrp(&buf);
2177 
2178 	/*
2179 	 * Set up our hooks
2180 	 */
2181 	if (UsingHooks)
2182 		initbulk(childHookRun, MaxBulk);
2183 
2184 	/*
2185 	 * Start phases
2186 	 */
2187 	wmsg.cmd = WMSG_CMD_INSTALL_PKGS;
2188 	ipcwritemsg(fd, &wmsg);
2189 	status = ipcreadmsg(fd, &wmsg);
2190 	if (status < 0 || wmsg.cmd != WMSG_RES_INSTALL_PKGS)
2191 		dfatal("pkg installation handshake failed");
2192 	do_install_phase = wmsg.status;
2193 	if (FetchOnlyOpt)
2194 		do_install_phase = 0;
2195 
2196 	wmsg.cmd = WMSG_CMD_STATUS_UPDATE;
2197 	wmsg.phase = PHASE_INSTALL_PKGS;
2198 	wmsg.lines = 0;
2199 
2200 	status = ipcwritemsg(fd, &wmsg);
2201 
2202 	if (pkgpkg && FetchOnlyOpt == 0) {
2203 		dophase(work, &wmsg,
2204 			WDOG5, PHASE_PACKAGE, "package");
2205 	} else {
2206 		/*
2207 		 * Dump as much information of the build process as possible.
2208 		 * Will help troubleshooting port build breakages.
2209 		 * Only enabled when DEVELOPER is set.
2210 		 *
2211 		 * This sort of mimics what synth did.
2212 		 */
2213 		if (WorkerProcFlags & WORKER_PROC_DEVELOPER) {
2214 			dophase(work, &wmsg,
2215 			    WDOG2, PHASE_DUMP_ENV, "Environment");
2216 			dophase(work, &wmsg,
2217 			    WDOG2, PHASE_SHOW_CONFIG, "showconfig");
2218 			dophase(work, &wmsg,
2219 			    WDOG2, PHASE_DUMP_VAR, "CONFIGURE_ENV");
2220 			dophase(work, &wmsg,
2221 			    WDOG2, PHASE_DUMP_VAR, "CONFIGURE_ARGS");
2222 			dophase(work, &wmsg,
2223 			    WDOG2, PHASE_DUMP_VAR, "MAKE_ENV");
2224 			dophase(work, &wmsg,
2225 			    WDOG2, PHASE_DUMP_VAR, "MAKE_ARGS");
2226 			dophase(work, &wmsg,
2227 			    WDOG2, PHASE_DUMP_VAR, "PLIST_SUB");
2228 			dophase(work, &wmsg,
2229 			    WDOG2, PHASE_DUMP_VAR, "SUB_LIST");
2230 			dophase(work, &wmsg,
2231 			    WDOG2, PHASE_DUMP_MAKECONF, "/etc/make.conf");
2232 		}
2233 
2234 		if (do_install_phase) {
2235 			dophase(work, &wmsg,
2236 				WDOG4, PHASE_INSTALL_PKGS, "setup");
2237 		}
2238 		dophase(work, &wmsg,
2239 			WDOG2, PHASE_CHECK_SANITY, "check-sanity");
2240 		if (FetchOnlyOpt == 0) {
2241 			dophase(work, &wmsg,
2242 				WDOG2, PHASE_PKG_DEPENDS, "pkg-depends");
2243 		}
2244 		dophase(work, &wmsg,
2245 			WDOG7, PHASE_FETCH_DEPENDS, "fetch-depends");
2246 		dophase(work, &wmsg,
2247 			WDOG7, PHASE_FETCH, "fetch");
2248 		dophase(work, &wmsg,
2249 			WDOG2, PHASE_CHECKSUM, "checksum");
2250 		if (FetchOnlyOpt == 0) {
2251 			dophase(work, &wmsg,
2252 				WDOG3, PHASE_EXTRACT_DEPENDS, "extract-depends");
2253 			dophase(work, &wmsg,
2254 				WDOG3, PHASE_EXTRACT, "extract");
2255 			dophase(work, &wmsg,
2256 				WDOG2, PHASE_PATCH_DEPENDS, "patch-depends");
2257 			dophase(work, &wmsg,
2258 				WDOG2, PHASE_PATCH, "patch");
2259 			dophase(work, &wmsg,
2260 				WDOG5, PHASE_BUILD_DEPENDS, "build-depends");
2261 			dophase(work, &wmsg,
2262 				WDOG5, PHASE_LIB_DEPENDS, "lib-depends");
2263 			dophase(work, &wmsg,
2264 				WDOG3, PHASE_CONFIGURE, "configure");
2265 			dophase(work, &wmsg,
2266 				WDOG9, PHASE_BUILD, "build");
2267 			dophase(work, &wmsg,
2268 				WDOG5, PHASE_RUN_DEPENDS, "run-depends");
2269 			dophase(work, &wmsg,
2270 				WDOG5, PHASE_STAGE, "stage");
2271 #if 0
2272 			dophase(work, &wmsg,
2273 				WDOG5, PHASE_TEST, "test");
2274 #endif
2275 			if (WorkerProcFlags & WORKER_PROC_CHECK_PLIST) {
2276 				dophase(work, &wmsg,
2277 					WDOG1, PHASE_CHECK_PLIST, "check-plist");
2278 			}
2279 			dophase(work, &wmsg,
2280 				WDOG5, PHASE_PACKAGE, "package");
2281 
2282 			if (WorkerProcFlags & WORKER_PROC_INSTALL) {
2283 				dophase(work, &wmsg,
2284 				    WDOG5, PHASE_INSTALL, "install");
2285 			}
2286 
2287 			if (WorkerProcFlags & WORKER_PROC_DEINSTALL) {
2288 				dophase(work, &wmsg,
2289 				    WDOG5, PHASE_DEINSTALL, "deinstall");
2290 			}
2291 		}
2292 	}
2293 
2294 	if (MasterPtyFd >= 0) {
2295 		close(MasterPtyFd);
2296 		MasterPtyFd = -1;
2297 	}
2298 
2299 	setproctitle("[%02d] WORKER CLEANUP  %s%s",
2300 		     slot, portdir, WorkerFlavorPrt);
2301 
2302 	/*
2303 	 * Copy the package to the repo.
2304 	 */
2305 	if (work->accum_error == 0 && FetchOnlyOpt == 0) {
2306 		char *b1;
2307 		char *b2;
2308 
2309 		asprintf(&b1, "%s/construction/%s/pkg/%s",
2310 			 work->basedir, pkg.portdir, pkg.pkgfile);
2311 		asprintf(&b2, "%s/%s", RepositoryPath, pkg.pkgfile);
2312 		if (copyfile(b1, b2)) {
2313 			++work->accum_error;
2314 			dlog(DLOG_ALL, "[%03d] %s Unable to copy %s to %s\n",
2315 			     work->index, pkg.portdir, b1, b2);
2316 		}
2317 		free(b1);
2318 		free(b2);
2319 	}
2320 
2321 	/*
2322 	 * Unmount, unless we are in DebugStopMode.
2323 	 */
2324 	if ((WorkerProcFlags & WORKER_PROC_DEBUGSTOP) == 0)
2325 		DoWorkerUnmounts(work);
2326 
2327 	/*
2328 	 * Send completion status to master dsynth worker thread.
2329 	 */
2330 	if (work->accum_error) {
2331 		wmsg.cmd = WMSG_CMD_FAILURE;
2332 	} else {
2333 		wmsg.cmd = WMSG_CMD_SUCCESS;
2334 	}
2335 	ipcwritemsg(fd, &wmsg);
2336 	if (WorkerProcFlags & WORKER_PROC_DEBUGSTOP) {
2337 		wmsg.cmd = WMSG_CMD_FREEZEWORKER;
2338 		ipcwritemsg(fd, &wmsg);
2339 	}
2340 	if (UsingHooks) {
2341 		while ((bulk = getbulk()) != NULL)
2342 			freebulk(bulk);
2343 		donebulk();
2344 	}
2345 }
2346 
2347 static int
check_dns(void)2348 check_dns(void)
2349 {
2350 	char check_domains[4][24] = {
2351 		"www.dragonflybsd.org",
2352 		"www.freebsd.org",
2353 		"www.openbsd.org",
2354 		"www.netbsd.org",
2355 	};
2356 	int failures = 0;
2357 
2358 	for (int i = 0; i < 4; i++)
2359 		if (gethostbyname (check_domains[i]) == NULL)
2360 			failures++;
2361 	if (failures > 1)
2362 		return -1;
2363 
2364 	return 0;
2365 }
2366 
2367 static void
dophase(worker_t * work,wmsg_t * wmsg,int wdog,int phaseid,const char * phase)2368 dophase(worker_t *work, wmsg_t *wmsg, int wdog, int phaseid, const char *phase)
2369 {
2370 	pkg_t *pkg = work->pkg;
2371 	char buf[1024];
2372 	pid_t pid;
2373 	int status;
2374 	int ms;
2375 	pid_t wpid;
2376 	int wpid_reaped;
2377 	int fdlog;
2378 	time_t start_time;
2379 	time_t last_time;
2380 	time_t next_time;
2381 	time_t wdog_time;
2382 	FILE *fp;
2383 
2384 	if (work->accum_error)
2385 		return;
2386 	setproctitle("[%02d] WORKER %-8.8s %s%s",
2387 		     work->index, phase, pkg->portdir, WorkerFlavorPrt);
2388 	wmsg->phase = phaseid;
2389 	if (ipcwritemsg(work->fds[0], wmsg) < 0) {
2390 		dlog(DLOG_ALL, "[%03d] %s Lost Communication with dsynth, "
2391 		     "aborting worker\n",
2392 		     work->index, pkg->portdir);
2393 		++work->accum_error;
2394 		return;
2395 	}
2396 
2397 	/*
2398 	 * Execute the port make command in chroot on a pty.
2399 	 */
2400 	fflush(stdout);
2401 	fflush(stderr);
2402 	if (MasterPtyFd >= 0) {
2403 		int slavefd;
2404 
2405 		/*
2406 		 * NOTE: We can't open the slave in the child because the
2407 		 *	 master may race a disconnection test.  If we open
2408 		 *	 it in the parent our close() will flush any pending
2409 		 *	 output not read by the master (which is the same
2410 		 *	 parent process) and deadlock.
2411 		 *
2412 		 *	 Solve this by hand-shaking the slave tty to give
2413 		 *	 the master time to close its slavefd (after this
2414 		 *	 section).
2415 		 *
2416 		 *	 Leave the tty defaults intact, which also likely
2417 		 *	 means it will be in line-buffered mode, so handshake
2418 		 *	 with a full line.
2419 		 *
2420 		 * TODO: Our handshake probably echos back to the master pty
2421 		 *	 due to tty echo, and ends up in the log, so just
2422 		 *	 pass through a newline.
2423 		 */
2424 		slavefd = open(ptsname(MasterPtyFd), O_RDWR);
2425 		dassert_errno(slavefd >= 0, "Cannot open slave pty");
2426 
2427 		/*
2428 		 * Now do the fork.
2429 		 */
2430 		pid = fork();
2431 		if (pid == 0) {
2432 			login_tty(slavefd);
2433 			/* login_tty() closes slavefd */
2434 		} else {
2435 			close(slavefd);
2436 		}
2437 	} else {
2438 		/*
2439 		 * Initial MasterPtyFd for the slot, just use forkpty().
2440 		 */
2441 		pid = forkpty(&MasterPtyFd, NULL, NULL, NULL);
2442 	}
2443 
2444 	/*
2445 	 * The slave must make sure the master has time to close slavefd
2446 	 * in the re-use case before going its merry way.  The master needs
2447 	 * to set terminal modes and the window as well.
2448 	 */
2449 	if (pid == 0) {
2450 		/*
2451 		 * Slave nices itself and waits for handshake
2452 		 */
2453 		char ttybuf[2];
2454 
2455 		/*
2456 		 * Self-nice to be nice (ignore any error)
2457 		 */
2458 		if (NiceOpt)
2459 			setpriority(PRIO_PROCESS, 0, NiceOpt);
2460 
2461 		read(0, ttybuf, 1);
2462 	} else {
2463 		/*
2464 		 * We are going through a pty, so set the tty modes to
2465 		 * Set tty modes so we do not get ^M's in the log files.
2466 		 *
2467 		 * This isn't fatal if it doesn't work.  Remember that
2468 		 * our output goes through the pty to the management
2469 		 * process which will log it.
2470 		 */
2471 		struct termios tio;
2472 		struct winsize win;
2473 
2474 		if (tcgetattr(MasterPtyFd, &tio) == 0) {
2475 			tio.c_oflag |= OPOST | ONOCR;
2476 			tio.c_oflag &= ~(OCRNL | ONLCR);
2477 			tio.c_iflag |= ICRNL;
2478 			tio.c_iflag &= ~(INLCR | IGNCR);
2479 			if (tcsetattr(MasterPtyFd, TCSANOW, &tio)) {
2480 				printf("tcsetattr failed: %s\n",
2481 				       strerror(errno));
2482 			}
2483 
2484 			/*
2485 			 * Give the tty a non-zero columns field.
2486 			 * This fixes at least one port (textproc/po4a)
2487 			 */
2488 			if (ioctl(MasterPtyFd, TIOCGWINSZ, &win) == 0) {
2489 				win.ws_col = 80;
2490 				ioctl(MasterPtyFd, TIOCSWINSZ, &win);
2491 			} else {
2492 				printf("TIOCGWINSZ failed: %s\n",
2493 				       strerror(errno));
2494 			}
2495 
2496 		} else {
2497 			printf("tcgetattr failed: %s\n", strerror(errno));
2498 		}
2499 
2500 		/*
2501 		 * Master issues handshake
2502 		 */
2503 		write(MasterPtyFd, "\n", 1);
2504 	}
2505 
2506 	if (pid == 0) {
2507 		/*
2508 		 * Additional phase-specific environment variables
2509 		 *
2510 		 * - Do not try to process missing depends outside of the
2511 		 *   depends phases.  Also relies on USE_PACKAGE_DEPENDS_ONLY
2512 		 *   in the make.conf.
2513 		 */
2514 		switch(phaseid) {
2515 		case PHASE_CHECK_SANITY:
2516 		case PHASE_FETCH:
2517 		case PHASE_CHECKSUM:
2518 		case PHASE_EXTRACT:
2519 		case PHASE_PATCH:
2520 		case PHASE_CONFIGURE:
2521 		case PHASE_STAGE:
2522 		case PHASE_TEST:
2523 		case PHASE_CHECK_PLIST:
2524 		case PHASE_INSTALL:
2525 		case PHASE_DEINSTALL:
2526 			break;
2527 		case PHASE_PKG_DEPENDS:
2528 		case PHASE_FETCH_DEPENDS:
2529 		case PHASE_EXTRACT_DEPENDS:
2530 		case PHASE_PATCH_DEPENDS:
2531 		case PHASE_BUILD_DEPENDS:
2532 		case PHASE_LIB_DEPENDS:
2533 		case PHASE_RUN_DEPENDS:
2534 			break;
2535 		default:
2536 			setenv("NO_DEPENDS", "1", 1);
2537 			break;
2538 		}
2539 
2540 		/*
2541 		 * Clean-up, chdir, and chroot.
2542 		 */
2543 		closefrom(3);
2544 		if (chdir(work->basedir) < 0)
2545 			dfatal_errno("chdir in phase initialization");
2546 		if (chroot(work->basedir) < 0)
2547 			dfatal_errno("chroot in phase initialization");
2548 
2549 		/* Explicitly fail when DNS is not working */
2550 		if (check_dns() != 0)
2551 			dfatal("DNS resolution not working");
2552 
2553 		/*
2554 		 * We have a choice here on how to handle stdin (fd 0).
2555 		 * We can leave it connected to the pty in which case
2556 		 * the build will just block if it tries to ask a
2557 		 * question (and the watchdog will kill it, eventually),
2558 		 * or we can try to EOF the pty, or we can attach /dev/null
2559 		 * to descriptor 0.
2560 		 */
2561 		if (NullStdinOpt) {
2562 			int fd;
2563 
2564 			fd = open("/dev/null", O_RDWR);
2565 			dassert_errno(fd >= 0, "cannot open /dev/null");
2566 			if (fd != 0) {
2567 				dup2(fd, 0);
2568 				close(fd);
2569 			}
2570 		}
2571 
2572 		/*
2573 		 * Capability restrictions to make root safer.
2574 		 */
2575 		set_capability_restrictions();
2576 
2577 		/*
2578 		 * Execute the appropriate command.
2579 		 */
2580 		switch(phaseid) {
2581 		case PHASE_INSTALL_PKGS:
2582 			snprintf(buf, sizeof(buf), "/tmp/dsynth_install_pkgs");
2583 			execl(buf, buf, NULL);
2584 			break;
2585 		case PHASE_DUMP_ENV:
2586 			snprintf(buf, sizeof(buf), "/usr/bin/env");
2587 			execl(buf, buf, NULL);
2588 			break;
2589 		case PHASE_DUMP_VAR:
2590 			snprintf(buf, sizeof(buf), "/xports/%s", pkg->portdir);
2591 			execl(MAKE_BINARY, MAKE_BINARY, "-C", buf, "-V", phase,
2592 			    NULL);
2593 			break;
2594 		case PHASE_DUMP_MAKECONF:
2595 			snprintf(buf, sizeof(buf), "/bin/cat");
2596 			execl(buf, buf, "/etc/make.conf", NULL);
2597 			break;
2598 		case PHASE_SHOW_CONFIG:
2599 			/* fall-through */
2600 		default:
2601 			snprintf(buf, sizeof(buf), "/xports/%s", pkg->portdir);
2602 			execl(MAKE_BINARY, MAKE_BINARY, "-C", buf, phase, NULL);
2603 			break;
2604 		}
2605 		_exit(1);
2606 	}
2607 	fcntl(MasterPtyFd, F_SETFL, O_NONBLOCK);
2608 
2609 	if (pid < 0) {
2610 		dlog(DLOG_ALL, "[%03d] %s Fork Failed: %s\n",
2611 		     work->index, pkg->logfile, strerror(errno));
2612 		++work->accum_error;
2613 		return;
2614 	}
2615 
2616 	SigPid = pid;
2617 
2618 	fdlog = open(pkg->logfile, O_RDWR|O_CREAT|O_APPEND, 0644);
2619 	if (fdlog < 0) {
2620 		dlog(DLOG_ALL, "[%03d] %s Cannot open logfile '%s': %s\n",
2621 		     work->index, pkg->portdir,
2622 		     pkg->logfile, strerror(errno));
2623 	}
2624 
2625 	snprintf(buf, sizeof(buf),
2626 		 "----------------------------------------"
2627 		 "---------------------------------------\n"
2628 		 "-- Phase: %s\n"
2629 		 "----------------------------------------"
2630 		 "---------------------------------------\n",
2631 		 phase);
2632 	write(fdlog, buf, strlen(buf));
2633 
2634 	start_time = time(NULL);
2635 	last_time = start_time;
2636 	wdog_time = start_time;
2637 	wpid_reaped = 0;
2638 
2639 	status = 0;
2640 	for (;;) {
2641 		ms = mptylogpoll(MasterPtyFd, fdlog, wmsg, &wdog_time);
2642 		if (ms == MPTY_FAILED) {
2643 			dlog(DLOG_ALL,
2644 			     "[%03d] %s lost pty in phase %s, terminating\n",
2645 			     work->index, pkg->portdir, phase);
2646 			break;
2647 		}
2648 		if (ms == MPTY_EOF)
2649 			break;
2650 
2651 		/*
2652 		 * Generally speaking update status once a second.
2653 		 * This also allows us to detect if the management
2654 		 * dsynth process has gone away.
2655 		 */
2656 		next_time = time(NULL);
2657 		if (next_time != last_time) {
2658 			double dload[3];
2659 			double dv;
2660 			int wdog_scaled;
2661 
2662 			/*
2663 			 * Send status update to the worker management thread
2664 			 * in the master dsynth process.  Remember, *WE* are
2665 			 * the worker management process sub-fork.
2666 			 */
2667 			if (ipcwritemsg(work->fds[0], wmsg) < 0)
2668 				break;
2669 			last_time = next_time;
2670 
2671 			/*
2672 			 * Watchdog scaling
2673 			 */
2674 			getloadavg(dload, 3);
2675 			adjloadavg(dload);
2676 			dv = dload[2] / NumCores;
2677 			if (dv < (double)NumCores) {
2678 				wdog_scaled = wdog;
2679 			} else {
2680 				if (dv > 4.0 * NumCores)
2681 					dv = 4.0 * NumCores;
2682 				wdog_scaled = wdog * dv / NumCores;
2683 			}
2684 
2685 			/*
2686 			 * Watchdog
2687 			 */
2688 			if (next_time - wdog_time >= wdog_scaled * 60) {
2689 				snprintf(buf, sizeof(buf),
2690 					 "\n--------\n"
2691 					 "WATCHDOG TIMEOUT FOR %s in %s "
2692 					 "after %d minutes\n"
2693 					 "Killing pid %d\n"
2694 					 "--------\n",
2695 					 pkg->portdir, phase, wdog_scaled, pid);
2696 				if (fdlog >= 0)
2697 					write(fdlog, buf, strlen(buf));
2698 				dlog(DLOG_ALL,
2699 				     "[%03d] %s WATCHDOG TIMEOUT in %s "
2700 				     "after %d minutes (%d min scaled)\n",
2701 				     work->index, pkg->portdir, phase,
2702 				     wdog, wdog_scaled);
2703 				kill(pid, SIGKILL);
2704 				++work->accum_error;
2705 				break;
2706 			}
2707 		}
2708 
2709 		/*
2710 		 * Check process exit.  Normally the pty will EOF
2711 		 * but if background processes remain we need to
2712 		 * check here to see if our primary exec is done,
2713 		 * so we can break out and reap those processes.
2714 		 *
2715 		 * Generally reap any other processes we have inherited
2716 		 * while we are here.
2717 		 */
2718 		do {
2719 			wpid = wait3(&status, WNOHANG, NULL);
2720 		} while (wpid > 0 && wpid != pid);
2721 		if (wpid == pid && WIFEXITED(status)) {
2722 			wpid_reaped = 1;
2723 			break;
2724 		}
2725 	}
2726 
2727 	next_time = time(NULL);
2728 
2729 	setproctitle("[%02d] WORKER EXITREAP %s%s",
2730 		     work->index, pkg->portdir, WorkerFlavorPrt);
2731 
2732 	/*
2733 	 * We usually get here due to a mpty EOF, but not always as there
2734 	 * could be persistent processes still holding the slave.  Finish
2735 	 * up getting the exit status for the main process we are waiting
2736 	 * on and clean out any data left on the MasterPtyFd (as it could
2737 	 * be blocking the exit).
2738 	 */
2739 	while (wpid_reaped == 0) {
2740 		(void)mptylogpoll(MasterPtyFd, fdlog, wmsg, &wdog_time);
2741 		wpid = waitpid(pid, &status, WNOHANG);
2742 		if (wpid == pid && WIFEXITED(status)) {
2743 			wpid_reaped = 1;
2744 			break;
2745 		}
2746 		if (wpid < 0 && errno != EINTR) {
2747 			break;
2748 		}
2749 
2750 		/*
2751 		 * Safety.  The normal phase waits until the fork/exec'd
2752 		 * pid finishes, causing a pty EOF on exit (the slave
2753 		 * descriptor is closed by the kernel on exit so the
2754 		 * process should already have exited).
2755 		 *
2756 		 * However, it is also possible to get here if the pty fails
2757 		 * for some reason.  In this case, make sure that the process
2758 		 * is killed.
2759 		 */
2760 		kill(pid, SIGKILL);
2761 	}
2762 
2763 	/*
2764 	 * Clean out anything left on the pty but don't wait around
2765 	 * because there could be background processes preventing the
2766 	 * slave side from closing.
2767 	 */
2768 	while (mptylogpoll(MasterPtyFd, fdlog, wmsg, &wdog_time) == MPTY_DATA)
2769 		;
2770 
2771 	/*
2772 	 * Report on the exit condition.  If the pid was somehow lost
2773 	 * (probably due to someone gdb'ing the process), assume an error.
2774 	 */
2775 	if (wpid_reaped) {
2776 		if (WEXITSTATUS(status)) {
2777 			dlog(DLOG_ALL | DLOG_FILTER,
2778 			     "[%03d] %s Build phase '%s' failed exit %d\n",
2779 			     work->index, pkg->portdir, phase,
2780 			     WEXITSTATUS(status));
2781 			++work->accum_error;
2782 		}
2783 	} else {
2784 		dlog(DLOG_ALL, "[%03d] %s Build phase '%s' failed - lost pid\n",
2785 		     work->index, pkg->portdir, phase);
2786 		++work->accum_error;
2787 	}
2788 
2789 	/*
2790 	 * Kill any processes still running (sometimes processes end up in
2791 	 * the background during a dports build), and clean up any other
2792 	 * children that we have inherited.
2793 	 */
2794 	phaseReapAll();
2795 
2796 	/*
2797 	 * After the extraction phase add the space used by /construction
2798 	 * to the memory use.  This helps us reduce the amount of paging
2799 	 * we do due to extremely large package extractions (languages,
2800 	 * chromium, etc).
2801 	 *
2802 	 * (dsynth already estimated the space used by the package deps
2803 	 * up front, but this will help us further).
2804 	 */
2805 	if (work->accum_error == 0 && phaseid == PHASE_EXTRACT) {
2806 		struct statfs sfs;
2807 		char *b1;
2808 
2809 		asprintf(&b1, "%s/construction", work->basedir);
2810 		if (statfs(b1, &sfs) == 0) {
2811 			wmsg->memuse = (sfs.f_blocks - sfs.f_bfree) *
2812 				       sfs.f_bsize;
2813 			ipcwritemsg(work->fds[0], wmsg);
2814 		}
2815 	}
2816 
2817 	/*
2818 	 * Update log
2819 	 */
2820 	if (fdlog >= 0) {
2821 		struct stat st;
2822 		int h;
2823 		int m;
2824 		int s;
2825 
2826 		last_time = next_time - start_time;
2827 		s = last_time % 60;
2828 		m = last_time / 60 % 60;
2829 		h = last_time / 3600;
2830 
2831 		fp = fdopen(fdlog, "a");
2832 		if (fp == NULL) {
2833 			dlog(DLOG_ALL, "[%03d] %s Cannot fdopen fdlog: %s %d\n",
2834 			     work->index, pkg->portdir,
2835 			     strerror(errno), fstat(fdlog, &st));
2836 			close(fdlog);
2837 			goto skip;
2838 		}
2839 
2840 		fprintf(fp, "\n");
2841 		if (work->accum_error) {
2842 			fprintf(fp, "FAILED %02d:%02d:%02d\n", h, m, s);
2843 		} else {
2844 			if (phaseid == PHASE_EXTRACT && wmsg->memuse) {
2845 				fprintf(fp, "Extracted Memory Use: %6.2fM\n",
2846 					wmsg->memuse / (1024.0 * 1024.0));
2847 			}
2848 			fprintf(fp, "SUCCEEDED %02d:%02d:%02d\n", h, m, s);
2849 		}
2850 		last_time = next_time - work->start_time;
2851 		s = last_time % 60;
2852 		m = last_time / 60 % 60;
2853 		h = last_time / 3600;
2854 		if (phaseid == PHASE_PACKAGE) {
2855 			fprintf(fp, "TOTAL TIME %02d:%02d:%02d\n", h, m, s);
2856 		}
2857 		fprintf(fp, "\n");
2858 		fclose(fp);
2859 skip:
2860 		;
2861 	}
2862 
2863 }
2864 
2865 static void
phaseReapAll(void)2866 phaseReapAll(void)
2867 {
2868 	struct reaper_status rs;
2869 	int status;
2870 
2871 	while (procctl(P_PID, getpid(), PROC_REAP_STATUS, &rs) == 0) {
2872 		if ((rs.flags & PROC_REAP_ACQUIRE) == 0)
2873 			break;
2874 		if (rs.pid_head < 0)
2875 			break;
2876 		if (kill(rs.pid_head, SIGKILL) == 0) {
2877 			while (waitpid(rs.pid_head, &status, 0) < 0)
2878 				;
2879 		}
2880 	}
2881 	while (wait3(&status, 0, NULL) > 0)
2882 		;
2883 }
2884 
2885 static void
phaseTerminateSignal(int sig __unused)2886 phaseTerminateSignal(int sig __unused)
2887 {
2888 	if (CopyFileFd >= 0)
2889 		close(CopyFileFd);
2890 	if (MasterPtyFd >= 0)
2891 		close(MasterPtyFd);
2892 	if (SigPid > 1)
2893 		kill(SigPid, SIGKILL);
2894 	phaseReapAll();
2895 	if (SigWork)
2896 		DoWorkerUnmounts(SigWork);
2897 	exit(1);
2898 }
2899 
2900 static
2901 char *
buildskipreason(pkglink_t * parent,pkg_t * pkg)2902 buildskipreason(pkglink_t *parent, pkg_t *pkg)
2903 {
2904 	pkglink_t *link;
2905 	pkg_t *scan;
2906 	char *reason = NULL;
2907 	char *ptr;
2908 	size_t tot;
2909 	size_t len;
2910 	pkglink_t stack;
2911 
2912 	if ((pkg->flags & PKGF_NOBUILD_I) && pkg->ignore)
2913 		asprintf(&reason, "%s ", pkg->ignore);
2914 
2915 	tot = 0;
2916 	PKGLIST_FOREACH(link, &pkg->idepon_list) {
2917 #if 0
2918 		if (link->dep_type > DEP_TYPE_BUILD)
2919 			continue;
2920 #endif
2921 		scan = link->pkg;
2922 		if (scan == NULL)
2923 			continue;
2924 		if ((scan->flags & (PKGF_ERROR | PKGF_NOBUILD)) == 0)
2925 			continue;
2926 		if (scan->flags & PKGF_NOBUILD) {
2927 			stack.pkg = scan;
2928 			stack.next = parent;
2929 			ptr = buildskipreason(&stack, scan);
2930 			len = strlen(scan->portdir) + strlen(ptr) + 8;
2931 			reason = realloc(reason, tot + len);
2932 			snprintf(reason + tot, len, "%s->%s",
2933 				 scan->portdir, ptr);
2934 			free(ptr);
2935 		} else {
2936 			len = strlen(scan->portdir) + 8;
2937 			reason = realloc(reason, tot + len);
2938 			snprintf(reason + tot, len, "%s", scan->portdir);
2939 		}
2940 
2941 		/*
2942 		 * Don't try to print the entire graph
2943 		 */
2944 		if (parent)
2945 			break;
2946 		tot += strlen(reason + tot);
2947 		reason[tot++] = ' ';
2948 		reason[tot] = 0;
2949 	}
2950 	return (reason);
2951 }
2952 
2953 /*
2954  * Count number of packages that would be skipped due to the
2955  * specified package having failed.
2956  *
2957  * Call with mode 1 to count, and mode 0 to clear the
2958  * cumulative rscan flag (used to de-duplicate the count).
2959  *
2960  * Must be serialized.
2961  */
2962 static int
buildskipcount_dueto(pkg_t * pkg,int mode)2963 buildskipcount_dueto(pkg_t *pkg, int mode)
2964 {
2965 	pkglink_t *link;
2966 	pkg_t *scan;
2967 	int total;
2968 
2969 	total = 0;
2970 	PKGLIST_FOREACH(link, &pkg->deponi_list) {
2971 		scan = link->pkg;
2972 		if (scan == NULL || scan->rscan == mode)
2973 			continue;
2974 		scan->rscan = mode;
2975 		++total;
2976 		total += buildskipcount_dueto(scan, mode);
2977 	}
2978 	return total;
2979 }
2980 
2981 /*
2982  * The master ptyfd is in non-blocking mode.  Drain up to 1024 bytes
2983  * and update wmsg->lines and *wdog_timep as appropriate.
2984  *
2985  * This function will poll, stalling up to 1 second.
2986  */
2987 static int
mptylogpoll(int ptyfd,int fdlog,wmsg_t * wmsg,time_t * wdog_timep)2988 mptylogpoll(int ptyfd, int fdlog, wmsg_t *wmsg, time_t *wdog_timep)
2989 {
2990 	struct pollfd pfd;
2991 	char buf[1024];
2992 	ssize_t r;
2993 
2994 	pfd.fd = ptyfd;
2995 	pfd.events = POLLIN;
2996 	pfd.revents = 0;
2997 
2998 	poll(&pfd, 1, 1000);
2999 	if (pfd.revents) {
3000 		r = read(ptyfd, buf, sizeof(buf));
3001 		if (r > 0) {
3002 			*wdog_timep = time(NULL);
3003 			if (r > 0 && fdlog >= 0)
3004 				write(fdlog, buf, r);
3005 			while (--r >= 0) {
3006 				if (buf[r] == '\n')
3007 					++wmsg->lines;
3008 			}
3009 			return MPTY_DATA;
3010 		} else if (r < 0) {
3011 			if (errno != EINTR && errno != EAGAIN)
3012 				return MPTY_FAILED;
3013 			return MPTY_AGAIN;
3014 		} else if (r == 0) {
3015 			return MPTY_EOF;
3016 		}
3017 	}
3018 	return MPTY_AGAIN;
3019 }
3020 
3021 /*
3022  * Copy a (package) file from (src) to (dst), use an intermediate file and
3023  * rename to ensure that interruption does not leave us with a corrupt
3024  * package file.
3025  *
3026  * This is called by the WORKER process.
3027  *
3028  * (dsynth management thread -> WORKER process -> sub-processes)
3029  */
3030 #define COPYBLKSIZE	32768
3031 
3032 int
copyfile(char * src,char * dst)3033 copyfile(char *src, char *dst)
3034 {
3035 	char *tmp;
3036 	char *buf;
3037 	int fd1;
3038 	int fd2;
3039 	int error = 0;
3040 	int mask;
3041 	ssize_t r;
3042 
3043 	asprintf(&tmp, "%s.new", dst);
3044 	buf = malloc(COPYBLKSIZE);
3045 
3046 	mask = sigsetmask(sigmask(SIGTERM)|sigmask(SIGINT)|sigmask(SIGHUP));
3047 	fd1 = open(src, O_RDONLY|O_CLOEXEC);
3048 	fd2 = open(tmp, O_RDWR|O_CREAT|O_TRUNC|O_CLOEXEC, 0644);
3049 	CopyFileFd = fd1;
3050 	sigsetmask(mask);
3051 	while ((r = read(fd1, buf, COPYBLKSIZE)) > 0) {
3052 		if (write(fd2, buf, r) != r)
3053 			error = 1;
3054 	}
3055 	if (r < 0)
3056 		error = 1;
3057 	mask = sigsetmask(sigmask(SIGTERM)|sigmask(SIGINT)|sigmask(SIGHUP));
3058 	CopyFileFd = -1;
3059 	close(fd1);
3060 	close(fd2);
3061 	sigsetmask(mask);
3062 	if (error) {
3063 		remove(tmp);
3064 	} else {
3065 		if (rename(tmp, dst)) {
3066 			error = 1;
3067 			remove(tmp);
3068 		}
3069 	}
3070 
3071 	freestrp(&buf);
3072 	freestrp(&tmp);
3073 
3074 	return error;
3075 }
3076 
3077 /*
3078  * doHook()
3079  *
3080  * primary process (threaded) - run_start, run_end, pkg_ignored, pkg_skipped
3081  * worker process  (threaded) - pkg_sucess, pkg_failure
3082  *
3083  * If waitfor is non-zero this hook will be serialized.
3084  */
3085 static void
doHook(pkg_t * pkg,const char * id,const char * path,int waitfor)3086 doHook(pkg_t *pkg, const char *id, const char *path, int waitfor)
3087 {
3088 	if (path == NULL)
3089 		return;
3090 	while (waitfor && getbulk() != NULL)
3091 		;
3092 	if (pkg)
3093 		queuebulk(pkg->portdir, id, path, pkg->pkgfile);
3094 	else
3095 		queuebulk(NULL, id, path, NULL);
3096 	while (waitfor && getbulk() != NULL)
3097 		;
3098 }
3099 
3100 /*
3101  * Execute hook (backend)
3102  *
3103  * s1 - portdir
3104  * s2 - id
3105  * s3 - script path
3106  * s4 - pkgfile		(if applicable)
3107  */
3108 static void
childHookRun(bulk_t * bulk)3109 childHookRun(bulk_t *bulk)
3110 {
3111 	const char *cav[MAXCAC];
3112 	buildenv_t benv[MAXCAC];
3113 	char buf1[128];
3114 	char buf2[128];
3115 	char buf3[128];
3116 	char buf4[128];
3117 	FILE *fp;
3118 	char *ptr;
3119 	size_t len;
3120 	pid_t pid;
3121 	int cac;
3122 	int bi;
3123 
3124 	cac = 0;
3125 	bi = 0;
3126 	bzero(benv, sizeof(benv));
3127 
3128 	cav[cac++] = bulk->s3;
3129 
3130 	benv[bi].label = "PROFILE";
3131 	benv[bi].data = Profile;
3132 	++bi;
3133 
3134 	benv[bi].label = "DIR_PACKAGES";
3135 	benv[bi].data = PackagesPath;
3136 	++bi;
3137 
3138 	benv[bi].label = "DIR_REPOSITORY";
3139 	benv[bi].data = RepositoryPath;
3140 	++bi;
3141 
3142 	benv[bi].label = "DIR_PORTS";
3143 	benv[bi].data = DPortsPath;
3144 	++bi;
3145 
3146 	benv[bi].label = "DIR_OPTIONS";
3147 	benv[bi].data = OptionsPath;
3148 	++bi;
3149 
3150 	benv[bi].label = "DIR_DISTFILES";
3151 	benv[bi].data = DistFilesPath;
3152 	++bi;
3153 
3154 	benv[bi].label = "DIR_LOGS";
3155 	benv[bi].data = LogsPath;
3156 	++bi;
3157 
3158 	benv[bi].label = "DIR_BUILDBASE";
3159 	benv[bi].data = BuildBase;
3160 	++bi;
3161 
3162 	if (strcmp(bulk->s2, "hook_run_start") == 0) {
3163 		snprintf(buf1, sizeof(buf1), "%d", BuildTotal);
3164 		benv[bi].label = "PORTS_QUEUED";
3165 		benv[bi].data = buf1;
3166 		++bi;
3167 	} else if (strcmp(bulk->s2, "hook_run_end") == 0) {
3168 		snprintf(buf1, sizeof(buf1), "%d", BuildSuccessCount);
3169 		benv[bi].label = "PORTS_BUILT";
3170 		benv[bi].data = buf1;
3171 		++bi;
3172 		snprintf(buf2, sizeof(buf2), "%d", BuildFailCount);
3173 		benv[bi].label = "PORTS_FAILED";
3174 		benv[bi].data = buf2;
3175 		++bi;
3176 		snprintf(buf3, sizeof(buf3), "%d", BuildIgnoreCount);
3177 		benv[bi].label = "PORTS_IGNORED";
3178 		benv[bi].data = buf3;
3179 		++bi;
3180 		snprintf(buf4, sizeof(buf4), "%d", BuildSkipCount);
3181 		benv[bi].label = "PORTS_SKIPPED";
3182 		benv[bi].data = buf4;
3183 		++bi;
3184 	} else {
3185 		/*
3186 		 * success, failure, ignored, skipped
3187 		 */
3188 		benv[bi].label = "RESULT";
3189 		if (strcmp(bulk->s2, "hook_pkg_success") == 0) {
3190 			benv[bi].data = "success";
3191 		} else if (strcmp(bulk->s2, "hook_pkg_failure") == 0) {
3192 			benv[bi].data = "failure";
3193 		} else if (strcmp(bulk->s2, "hook_pkg_ignored") == 0) {
3194 			benv[bi].data = "ignored";
3195 		} else if (strcmp(bulk->s2, "hook_pkg_skipped") == 0) {
3196 			benv[bi].data = "skipped";
3197 		} else {
3198 			dfatal("Unknown hook id: %s", bulk->s2);
3199 			/* NOT REACHED */
3200 		}
3201 		++bi;
3202 
3203 		/*
3204 		 * For compatibility with synth:
3205 		 *
3206 		 * ORIGIN does not include any @flavor, thus it is suitable
3207 		 * for finding the actual port directory/subdirectory.
3208 		 *
3209 		 * FLAVOR is set to ORIGIN if there is no flavor, otherwise
3210 		 * it is set to only the flavor sans the '@'.
3211 		 */
3212 		if ((ptr = strchr(bulk->s1, '@')) != NULL) {
3213 			snprintf(buf1, sizeof(buf1), "%*.*s",
3214 				 (int)(ptr - bulk->s1),
3215 				 (int)(ptr - bulk->s1),
3216 				 bulk->s1);
3217 			benv[bi].label = "ORIGIN";
3218 			benv[bi].data = buf1;
3219 			++bi;
3220 			benv[bi].label = "FLAVOR";
3221 			benv[bi].data = ptr + 1;
3222 			++bi;
3223 		} else {
3224 			benv[bi].label = "ORIGIN";
3225 			benv[bi].data = bulk->s1;
3226 			++bi;
3227 			benv[bi].label = "FLAVOR";
3228 			benv[bi].data = bulk->s1;
3229 			++bi;
3230 		}
3231 		benv[bi].label = "PKGNAME";
3232 		benv[bi].data = bulk->s4;
3233 		++bi;
3234 	}
3235 
3236 	benv[bi].label = NULL;
3237 	benv[bi].data = NULL;
3238 
3239 	fp = dexec_open(bulk->s1, cav, cac, &pid, benv, 0, 0);
3240 	while ((ptr = fgetln(fp, &len)) != NULL)
3241 		;
3242 
3243 	if (dexec_close(fp, pid)) {
3244 		dlog(DLOG_ALL,
3245 		     "[XXX] %s SCRIPT %s (%s)\n",
3246 		     bulk->s1, bulk->s2, bulk->s3);
3247 	}
3248 }
3249 
3250 /*
3251  * Adjusts dload[0] by adding in t_pw (processes waiting on page-fault).
3252  * We don't want load reductions due to e.g. thrashing to cause dsynth
3253  * to increase the dynamic limit because it thinks the load is low.
3254  *
3255  * This has a desirable property.  If the system pager cannot keep up
3256  * with process demand t_pw will spike while loadavg will only drop
3257  * slowly, resulting in a high adjusted load calculation that causes
3258  * dsynth to quickly clamp-down the limit.  If the condition alleviates,
3259  * the limit will then rise slowly again, possibly even before existing
3260  * jobs are retired to meet the clamp-down from the original spike.
3261  */
3262 static void
adjloadavg(double * dload)3263 adjloadavg(double *dload)
3264 {
3265 #if defined(__DragonFly__)
3266 	struct vmtotal total;
3267 	size_t size;
3268 
3269 	size = sizeof(total);
3270 	if (sysctlbyname("vm.vmtotal", &total, &size, NULL, 0) == 0) {
3271 		dload[0] += (double)total.t_pw;
3272 	}
3273 #else
3274 	dload[0] += 0.0;	/* just avoid compiler 'unused' warnings */
3275 #endif
3276 }
3277 
3278 /*
3279  * The list of pkgs has already been flagged PKGF_PACKAGED if a pkg
3280  * file exists.  Check if the ports directory contents for such packages
3281  * has changed by comparing against a small DBM database that we maintain.
3282  *
3283  * Force-clear PKGF_PACKAGED if the ports directory content has changed.
3284  *
3285  * If no DBM database entry is present, update the entry and assume that
3286  * the package does not need to be rebuilt (allows the .dbm file to be
3287  * manually deleted without forcing a complete rebuild).
3288  */
3289 static
3290 void
check_packaged(const char * dbmpath,pkg_t * pkgs)3291 check_packaged(const char *dbmpath, pkg_t *pkgs)
3292 {
3293 	pkg_t *scan;
3294 	datum key;
3295 	datum data;
3296 	char *buf;
3297 
3298 	if (CheckDBM == NULL) {
3299 		dlog(DLOG_ABN, "[XXX] Unable to open/create dbm %s\n", dbmpath);
3300 		return;
3301 	}
3302 	for (scan = pkgs; scan; scan = scan->bnext) {
3303 		if ((scan->flags & PKGF_PACKAGED) == 0)
3304 			continue;
3305 		key.dptr = scan->portdir;
3306 		key.dsize = strlen(scan->portdir);
3307 		data = dbm_fetch(CheckDBM, key);
3308 		if (data.dptr && data.dsize == sizeof(uint32_t) &&
3309 		    *(uint32_t *)data.dptr != scan->crc32) {
3310 			scan->flags &= ~PKGF_PACKAGED;
3311 			asprintf(&buf, "%s/%s", RepositoryPath, scan->pkgfile);
3312 			if (OverridePkgDeleteOpt >= 2) {
3313 				scan->flags |= PKGF_PACKAGED;
3314 				dlog(DLOG_ALL,
3315 				     "[XXX] %s DELETE-PACKAGE %s "
3316 				     "(port content changed CRC %08x/%08x "
3317 				     "OVERRIDE, NOT DELETED)\n",
3318 				     scan->portdir, buf,
3319 				     *(uint32_t *)data.dptr, scan->crc32);
3320 			} else if (remove(buf) < 0) {
3321 				dlog(DLOG_ALL,
3322 				     "[XXX] %s DELETE-PACKAGE %s (failed)\n",
3323 				     scan->portdir, buf);
3324 			} else {
3325 				dlog(DLOG_ALL,
3326 				     "[XXX] %s DELETE-PACKAGE %s "
3327 				     "(port content changed CRC %08x/%08x)\n",
3328 				     scan->portdir, buf,
3329 				     *(uint32_t *)data.dptr, scan->crc32);
3330 			}
3331 			freestrp(&buf);
3332 		} else if (data.dptr == NULL) {
3333 			data.dptr = &scan->crc32;
3334 			data.dsize = sizeof(scan->crc32);
3335 			dbm_store(CheckDBM, key, data, DBM_REPLACE);
3336 		}
3337 	}
3338 }
3339