1 /* 2 * Copyright (c) 2019 The DragonFly Project. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Matthew Dillon <dillon@backplane.com> 6 * 7 * This code uses concepts and configuration based on 'synth', by 8 * John R. Marino <draco@marino.st>, which was written in ada. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in 18 * the documentation and/or other materials provided with the 19 * distribution. 20 * 3. Neither the name of The DragonFly Project nor the names of its 21 * contributors may be used to endorse or promote products derived 22 * from this software without specific, prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 25 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 26 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 27 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 28 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 29 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 30 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 31 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 32 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 33 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 34 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 35 * SUCH DAMAGE. 36 */ 37 38 #include <sys/types.h> 39 #include <sys/wait.h> 40 #include <sys/stat.h> 41 #include <sys/sysctl.h> 42 #include <sys/socket.h> 43 #include <sys/mount.h> 44 #include <sys/procctl.h> 45 #include <stdio.h> 46 #include <stdlib.h> 47 #include <stddef.h> 48 #include <stdarg.h> 49 #include <unistd.h> 50 #include <string.h> 51 #include <fcntl.h> 52 #include <signal.h> 53 #include <poll.h> 54 #include <assert.h> 55 #include <errno.h> 56 #include <pthread.h> 57 #include <dirent.h> 58 #include <termios.h> 59 #include <ctype.h> 60 61 /* 62 * More esoteric headers 63 */ 64 #include <libutil.h> /* forkpty() */ 65 #include <arpa/inet.h> /* ntohl() */ 66 #include <elf.h> /* try to get elf info */ 67 68 struct pkglink; 69 70 #define DSYNTH_VERSION "1.01" 71 #define MAXWORKERS 1024 72 #define MAXJOBS 8192 /* just used for -j sanity */ 73 #define MAXBULK MAXWORKERS 74 75 #define MAKE_BINARY "/usr/bin/make" 76 #define PKG_BINARY "/usr/local/sbin/pkg" 77 #define MOUNT_BINARY "/sbin/mount" 78 #define MOUNT_NULLFS_BINARY "/sbin/mount_null" 79 #define MOUNT_TMPFS_BINARY "/sbin/mount_tmpfs" 80 #define MOUNT_DEVFS_BINARY "/sbin/mount_devfs" 81 #define MOUNT_PROCFS_BINARY "/sbin/mount_procfs" 82 #define UMOUNT_BINARY "/sbin/umount" 83 84 #define ONEGB (1024L * 1024 * 1024) 85 #define DISABLED_STR "disabled" 86 87 /* 88 * This can be ".tar", ".tgz", ".txz", or ".tbz". 89 * 90 * .tar - very fast but you'll need 1TB+ of storage just for the package files. 91 * .txz - very compact but decompression speed is horrible. 92 * .tgz - reasonable compression, extremely fast decompression. Roughly 93 * 1.1x to 2.0x the size of a .txz, but decompresses 10x faster. 94 * .tbz - worse than .tgz generally 95 */ 96 #define USE_PKG_SUFX ".tgz" 97 98 /* 99 * Topology linkages 100 */ 101 typedef struct pkglink { 102 struct pkglink *next; 103 struct pkglink *prev; 104 struct pkg *pkg; 105 int dep_type; 106 } pkglink_t; 107 108 #define DEP_TYPE_FETCH 1 109 #define DEP_TYPE_EXT 2 110 #define DEP_TYPE_PATCH 3 111 #define DEP_TYPE_BUILD 4 112 #define DEP_TYPE_LIB 5 113 #define DEP_TYPE_RUN 6 114 115 /* 116 * Describes a [flavored] package 117 */ 118 typedef struct pkg { 119 struct pkg *build_next; /* topology inversion build list */ 120 struct pkg *bnext; /* linked list from bulk return */ 121 struct pkg *hnext1; /* hash based on portdir */ 122 struct pkg *hnext2; /* hash based on pkgfile */ 123 pkglink_t idepon_list; /* I need these pkgs */ 124 pkglink_t deponi_list; /* pkgs which depend on me */ 125 char *portdir; /* origin name e.g. www/chromium[@flavor] */ 126 char *logfile; /* relative logfile path */ 127 char *version; /* PKGVERSION - e.g. 3.5.0_1 */ 128 char *pkgfile; /* PKGFILE - e.g. flav-blah-3.5.0_1.txz */ 129 char *distfiles; /* DISTFILES - e.g. blah-68.0.source.tar.xz */ 130 char *distsubdir; /* DIST_SUBDIR- e.g. cabal */ 131 char *ignore; /* IGNORE (also covers BROKEN) */ 132 char *fetch_deps; /* FETCH_DEPENDS */ 133 char *ext_deps; /* EXTRACT_DEPENDS */ 134 char *patch_deps; /* PATCH_DEPENDS */ 135 char *build_deps; /* BUILD_DEPENDS */ 136 char *lib_deps; /* LIB_DEPENDS */ 137 char *run_deps; /* RUN_DEPENDS */ 138 char *pos_options; /* SELECTED_OPTIONS */ 139 char *neg_options; /* DESELECTED_OPTIONS */ 140 char *flavors; /* FLAVORS - e.g. py36 py27 */ 141 char *uses; /* USES (metaport test) */ 142 int make_jobs_number; /* MAKE_JOBS_NUMBER */ 143 int use_linux; /* USE_LINUX */ 144 int idep_count; /* count recursive idepon build deps */ 145 int depi_count; /* count recursive deponi build deps */ 146 int depi_depth; /* tree depth who depends on me */ 147 int dsynth_install_flg; /* locked with WorkerMutex */ 148 int flags; 149 size_t pkgfile_size; /* size of pkgfile */ 150 } pkg_t; 151 152 #define PKGF_PACKAGED 0x00000001 /* has a repo package */ 153 #define PKGF_DUMMY 0x00000002 /* generic root for flavors */ 154 #define PKGF_NOTFOUND 0x00000004 /* dport not found */ 155 #define PKGF_CORRUPT 0x00000008 /* dport corrupt */ 156 #define PKGF_PLACEHOLD 0x00000010 /* pre-entered */ 157 #define PKGF_BUILDLIST 0x00000020 /* on build_list */ 158 #define PKGF_BUILDLOOP 0x00000040 /* traversal loop test */ 159 #define PKGF_BUILDTRAV 0x00000080 /* traversal optimization */ 160 #define PKGF_NOBUILD_D 0x00000100 /* can't build - dependency problem */ 161 #define PKGF_NOBUILD_S 0x00000200 /* can't build - skipped */ 162 #define PKGF_NOBUILD_F 0x00000400 /* can't build - failed */ 163 #define PKGF_NOBUILD_I 0x00000800 /* can't build - ignored or broken */ 164 #define PKGF_SUCCESS 0x00001000 /* build complete */ 165 #define PKGF_FAILURE 0x00002000 /* build complete */ 166 #define PKGF_RUNNING 0x00004000 /* build complete */ 167 #define PKGF_PKGPKG 0x00008000 /* pkg/pkg-static special */ 168 #define PKGF_NOTREADY 0x00010000 /* build_find_leaves() only */ 169 #define PKGF_MANUALSEL 0x00020000 /* manually specified */ 170 #define PKGF_META 0x00040000 /* USES contains 'metaport' */ 171 #define PKGF_ERROR (PKGF_PLACEHOLD | PKGF_CORRUPT | PKGF_NOTFOUND | \ 172 PKGF_FAILURE) 173 #define PKGF_NOBUILD (PKGF_NOBUILD_D | PKGF_NOBUILD_S | PKGF_NOBUILD_F | \ 174 PKGF_NOBUILD_I) 175 176 #define PKGLIST_EMPTY(pkglink) ((pkglink)->next == (pkglink)) 177 #define PKGLIST_FOREACH(var, head) \ 178 for (var = (head)->next; var != (head); var = (var)->next) 179 180 typedef struct bulk { 181 struct bulk *next; 182 pthread_t td; 183 int debug; 184 int flags; 185 enum { UNLISTED, ONSUBMIT, ONRUN, ISRUNNING, ONRESPONSE } state; 186 char *s1; 187 char *s2; 188 char *s3; 189 char *s4; 190 char *r1; 191 char *r2; 192 char *r3; 193 char *r4; 194 pkg_t *list; /* pkgs linked by bnext */ 195 } bulk_t; 196 197 /* 198 * Worker state (up to MAXWORKERS). Each worker operates within a 199 * chroot or jail. A system mirror is setup and the template 200 * is copied in. 201 * 202 * basedir - tmpfs 203 * /bin - nullfs (ro) 204 * /sbin - nullfs (ro) 205 * /lib - nullfs (ro) 206 * /libexec - nullfs (ro) 207 * /usr/bin - nullfs (ro) 208 * /usr/include - nullfs (ro) 209 * /usr/lib - nullfs (ro) 210 * /usr/libdata - nullfs (ro) 211 * /usr/libexec - nullfs (ro) 212 * /usr/sbin - nullfs (ro) 213 * /usr/share - nullfs (ro) 214 * /xports - nullfs (ro) 215 * /options - nullfs (ro) 216 * /packages - nullfs (ro) 217 * /distfiles - nullfs (ro) 218 * construction - tmpfs 219 * /usr/local - tmpfs 220 * /boot - nullfs (ro) 221 * /boot/modules.local - tmpfs 222 * /usr/games - nullfs (ro) 223 * /usr/src - nullfs (ro) 224 * /dev - devfs 225 */ 226 enum worker_state { WORKER_NONE, WORKER_IDLE, WORKER_PENDING, 227 WORKER_RUNNING, WORKER_DONE, WORKER_FAILED, 228 WORKER_FROZEN, WORKER_EXITING }; 229 typedef enum worker_state worker_state_t; 230 231 enum worker_phase { PHASE_PENDING, 232 PHASE_INSTALL_PKGS, 233 PHASE_CHECK_SANITY, 234 PHASE_PKG_DEPENDS, 235 PHASE_FETCH_DEPENDS, 236 PHASE_FETCH, 237 PHASE_CHECKSUM, 238 PHASE_EXTRACT_DEPENDS, 239 PHASE_EXTRACT, 240 PHASE_PATCH_DEPENDS, 241 PHASE_PATCH, 242 PHASE_BUILD_DEPENDS, 243 PHASE_LIB_DEPENDS, 244 PHASE_CONFIGURE, 245 PHASE_BUILD, 246 PHASE_RUN_DEPENDS, 247 PHASE_STAGE, 248 PHASE_TEST, 249 PHASE_CHECK_PLIST, 250 PHASE_PACKAGE, 251 PHASE_INSTALL_MTREE, 252 PHASE_INSTALL, 253 PHASE_DEINSTALL 254 }; 255 256 typedef enum worker_phase worker_phase_t; 257 258 /* 259 * Watchdog timeouts, in minutes, baseline, scales up with load/ncpus but 260 * does not scale down. 261 */ 262 #define WDOG1 (5) 263 #define WDOG2 (10) 264 #define WDOG3 (15) 265 #define WDOG4 (30) 266 #define WDOG5 (60) 267 #define WDOG6 (60 + 30) 268 #define WDOG7 (60 * 2) 269 #define WDOG8 (60 * 2 + 30) 270 #define WDOG9 (60 * 3) 271 272 typedef struct worker { 273 int index; /* worker number 0..N-1 */ 274 int flags; 275 int accum_error; /* cumulative error */ 276 int mount_error; /* mount and unmount error */ 277 int terminate : 1; /* request sub-thread to terminate */ 278 char *basedir; /* base directory including id */ 279 char *flavor; 280 pthread_t td; /* pthread */ 281 pthread_cond_t cond; /* interlock cond (w/ WorkerMutex) */ 282 pkg_t *pkg; 283 worker_state_t state; /* general worker state */ 284 worker_phase_t phase; /* phase control in childBuilderThread */ 285 time_t start_time; 286 long lines; 287 long memuse; 288 pid_t pid; 289 int fds[2]; /* forked environment process */ 290 char status[64]; 291 size_t pkg_dep_size; /* pkg dependency size(s) */ 292 } worker_t; 293 294 #define WORKERF_STATUS_UPDATE 0x0001 /* display update */ 295 #define WORKERF_SUCCESS 0x0002 /* completion flag */ 296 #define WORKERF_FAILURE 0x0004 /* completion flag */ 297 #define WORKERF_FREEZE 0x0008 /* freeze the worker */ 298 299 #define MOUNT_TYPE_MASK 0x000F 300 #define MOUNT_TYPE_TMPFS 0x0001 301 #define MOUNT_TYPE_NULLFS 0x0002 302 #define MOUNT_TYPE_DEVFS 0x0003 303 #define MOUNT_TYPE_PROCFS 0x0004 304 #define MOUNT_TYPE_RW 0x0010 305 #define MOUNT_TYPE_BIG 0x0020 306 #define MOUNT_TYPE_TMP 0x0040 307 308 #define NULLFS_RO (MOUNT_TYPE_NULLFS) 309 #define NULLFS_RW (MOUNT_TYPE_NULLFS | MOUNT_TYPE_RW) 310 #define PROCFS_RO (MOUNT_TYPE_PROCFS) 311 #define TMPFS_RW (MOUNT_TYPE_TMPFS | MOUNT_TYPE_RW) 312 #define TMPFS_RW_BIG (MOUNT_TYPE_TMPFS | MOUNT_TYPE_RW | \ 313 MOUNT_TYPE_BIG) 314 #define DEVFS_RW (MOUNT_TYPE_DEVFS | MOUNT_TYPE_RW) 315 316 /* 317 * IPC messages between the worker support thread and the worker process. 318 */ 319 typedef struct wmsg { 320 int cmd; 321 int status; 322 long lines; 323 long memuse; 324 worker_phase_t phase; 325 } wmsg_t; 326 327 #define WMSG_CMD_STATUS_UPDATE 0x0001 328 #define WMSG_CMD_SUCCESS 0x0002 329 #define WMSG_CMD_FAILURE 0x0003 330 #define WMSG_CMD_INSTALL_PKGS 0x0004 331 #define WMSG_RES_INSTALL_PKGS 0x0005 332 #define WMSG_CMD_FREEZEWORKER 0x0006 333 334 /* 335 * Make variables and build environment 336 */ 337 typedef struct buildenv { 338 struct buildenv *next; 339 const char *label; 340 const char *data; 341 char *a1; /* allocations */ 342 char *a2; /* allocations */ 343 int type; 344 } buildenv_t; 345 346 /* 347 * Operating systems recognized by dsynth 348 */ 349 enum os_id { 350 OS_UNKNOWN, OS_DRAGONFLY, OS_FREEBSD, OS_NETBSD, OS_LINUX 351 }; 352 353 typedef enum os_id os_id_t; 354 355 /* 356 * DLOG 357 */ 358 #define DLOG_ALL 0 /* Usually stdout when curses disabled */ 359 #define DLOG_SUCC 1 /* success_list.log */ 360 #define DLOG_FAIL 2 /* failure_list.log */ 361 #define DLOG_IGN 3 /* ignored_list.log */ 362 #define DLOG_SKIP 4 /* skipped_list.log */ 363 #define DLOG_ABN 5 /* abnormal_command_output */ 364 #define DLOG_OBS 6 /* obsolete_packages.log */ 365 #define DLOG_COUNT 7 /* total number of DLOGs */ 366 #define DLOG_MASK 0x0FF 367 368 #define DLOG_FILTER 0x100 /* Filter out of stdout in non-curses mode */ 369 #define DLOG_RED 0x200 /* Print in color */ 370 #define DLOG_GRN 0x400 /* Print in color */ 371 372 #define dassert(exp, fmt, ...) \ 373 if (!(exp)) dpanic(fmt, ## __VA_ARGS__) 374 375 #define ddassert(exp) \ 376 dassert((exp), "\"%s\" line %d", __FILE__, __LINE__) 377 378 #define dassert_errno(exp, fmt, ...) \ 379 if (!(exp)) dpanic_errno(fmt, ## __VA_ARGS__) 380 381 #define dlog(which, fmt, ...) \ 382 _dlog(which, fmt, ## __VA_ARGS__) 383 384 #define dlog_tsnl(which, fmt, ...) \ 385 _dlog(which, fmt, ## __VA_ARGS__) 386 387 #define dfatal(fmt, ...) \ 388 _dfatal(__FILE__, __LINE__, __func__, 0, fmt, ## __VA_ARGS__) 389 390 #define dpanic(fmt, ...) \ 391 _dfatal(__FILE__, __LINE__, __func__, 2, fmt, ## __VA_ARGS__) 392 393 #define dfatal_errno(fmt, ...) \ 394 _dfatal(__FILE__, __LINE__, __func__, 1, fmt, ## __VA_ARGS__) 395 396 #define dpanic_errno(fmt, ...) \ 397 _dfatal(__FILE__, __LINE__, __func__, 3, fmt, ## __VA_ARGS__) 398 399 #define ddprintf(tab, fmt, ...) \ 400 do { if (DebugOpt >= 2) _ddprintf(tab, fmt, ## __VA_ARGS__); } while(0) 401 402 /* 403 * addbuildenv() types 404 */ 405 #define BENV_ENVIRONMENT 1 406 #define BENV_MAKECONF 2 407 #define BENV_CMDMASK 0x000F 408 409 #define BENV_PKGLIST 0x0010 410 411 /* 412 * WORKER process flags 413 */ 414 #define WORKER_PROC_DEBUGSTOP 0x0001 415 #define WORKER_PROC_DEVELOPER 0x0002 416 417 /* 418 * Misc 419 */ 420 #define DOSTRING(label) #label 421 #define SCRIPTPATH(x) DOSTRING(x) 422 #define MAXCAC 256 423 424 /* 425 * RunStats satellite modules 426 */ 427 typedef struct topinfo { 428 int pkgimpulse; 429 int pkgrate; 430 int noswap; 431 int h; 432 int m; 433 int s; 434 int total; 435 int successful; 436 int ignored; 437 int remaining; 438 int failed; 439 int skipped; 440 double dswap; 441 double dload[3]; 442 } topinfo_t; 443 444 typedef struct runstats { 445 struct runstats *next; 446 void (*init)(void); 447 void (*done)(void); 448 void (*reset)(void); 449 void (*update)(worker_t *work); 450 void (*updateTop)(topinfo_t *info); 451 void (*updateLogs)(void); 452 void (*sync)(void); 453 } runstats_t; 454 455 extern runstats_t NCursesRunStats; 456 extern runstats_t MonitorRunStats; 457 extern runstats_t HtmlRunStats; 458 459 extern int BuildCount; 460 extern int BuildTotal; 461 extern int BuildFailCount; 462 extern int BuildSkipCount; 463 extern int BuildIgnoreCount; 464 extern int BuildSuccessCount; 465 extern int DynamicMaxWorkers; 466 467 extern buildenv_t *BuildEnv; 468 extern int WorkerProcFlags; 469 extern int DebugOpt; 470 extern int ColorOpt; 471 extern int SlowStartOpt; 472 extern int YesOpt; 473 extern int NullStdinOpt; 474 extern int UseCCache; 475 extern int UseUsrSrc; 476 extern int UseTmpfs; 477 extern int NumCores; 478 extern long PhysMem; 479 extern long PkgDepMemoryTarget; 480 extern int MaxBulk; 481 extern int MaxWorkers; 482 extern int MaxJobs; 483 extern int UseTmpfsWork; 484 extern int UseTmpfsBase; 485 extern int UseNCurses; 486 extern int LeveragePrebuilt; 487 extern char *DSynthExecPath; 488 489 490 extern const char *OperatingSystemName; 491 extern const char *ArchitectureName; 492 extern const char *MachineName; 493 extern const char *ReleaseName; 494 extern const char *VersionName; 495 496 extern const char *ConfigBase1; 497 extern const char *ConfigBase2; 498 extern const char *ConfigBase; 499 extern const char *DPortsPath; 500 extern const char *CCachePath; 501 extern const char *SynthConfig; 502 extern const char *PackagesPath; 503 extern const char *RepositoryPath; 504 extern const char *OptionsPath; 505 extern const char *DistFilesPath; 506 extern const char *BuildBase; 507 extern const char *LogsPath; 508 extern const char *SystemPath; 509 extern const char *Profile; 510 511 extern int UsingHooks; 512 extern const char *HookRunStart; 513 extern const char *HookRunEnd; 514 extern const char *HookPkgSuccess; 515 extern const char *HookPkgFailure; 516 extern const char *HookPkgIgnored; 517 extern const char *HookPkgSkipped; 518 519 void _dfatal(const char *file, int line, const char *func, int do_errno, 520 const char *fmt, ...); 521 void _ddprintf(int tab, const char *fmt, ...); 522 void _dlog(int which, const char *fmt, ...); 523 char *strdup_or_null(char *str); 524 void dlogreset(void); 525 int dlog00_fd(void); 526 void addbuildenv(const char *label, const char *data, int type); 527 void delbuildenv(const char *label); 528 529 void initbulk(void (*func)(bulk_t *bulk), int jobs); 530 void queuebulk(const char *s1, const char *s2, const char *s3, 531 const char *s4); 532 bulk_t *getbulk(void); 533 void donebulk(void); 534 void freebulk(bulk_t *bulk); 535 void freestrp(char **strp); 536 void dupstrp(char **strp); 537 int askyn(const char *ctl, ...); 538 double getswappct(int *noswapp); 539 FILE *dexec_open(const char **cav, int cac, pid_t *pidp, buildenv_t *xenv, 540 int with_env, int with_mvars); 541 int dexec_close(FILE *fp, pid_t pid); 542 const char *getphasestr(worker_phase_t phase); 543 544 void ParseConfiguration(int isworker); 545 pkg_t *ParsePackageList(int ac, char **av); 546 void FreePackageList(pkg_t *pkgs); 547 pkg_t *GetLocalPackageList(void); 548 pkg_t *GetFullPackageList(void); 549 pkg_t *GetPkgPkg(pkg_t *list); 550 551 void DoConfigure(void); 552 void DoStatus(pkg_t *pkgs); 553 void DoBuild(pkg_t *pkgs); 554 void DoInitBuild(int slot_override); 555 void DoCleanBuild(int resetlogs); 556 void OptimizeEnv(void); 557 void WorkerProcess(int ac, char **av); 558 559 int DoCreateTemplate(int force); 560 void DoDestroyTemplate(void); 561 void DoWorkerMounts(worker_t *work); 562 void DoWorkerUnmounts(worker_t *work); 563 void DoRebuildRepo(int ask); 564 void DoUpgradePkgs(pkg_t *pkgs, int ask); 565 void RemovePackages(pkg_t *pkgs); 566 void PurgeDistfiles(pkg_t *pkgs); 567 568 void RunStatsInit(void); 569 void RunStatsDone(void); 570 void RunStatsReset(void); 571 void RunStatsUpdate(worker_t *work); 572 void RunStatsUpdateTop(void); 573 void RunStatsUpdateLogs(void); 574 void RunStatsSync(void); 575 576 int ipcreadmsg(int fd, wmsg_t *msg); 577 int ipcwritemsg(int fd, wmsg_t *msg); 578