1 /* 2 * Copyright (c) 2019 The DragonFly Project. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Matthew Dillon <dillon@backplane.com> 6 * 7 * This code uses concepts and configuration based on 'synth', by 8 * John R. Marino <draco@marino.st>, which was written in ada. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in 18 * the documentation and/or other materials provided with the 19 * distribution. 20 * 3. Neither the name of The DragonFly Project nor the names of its 21 * contributors may be used to endorse or promote products derived 22 * from this software without specific, prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 25 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 26 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 27 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 28 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 29 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 30 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 31 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 32 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 33 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 34 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 35 * SUCH DAMAGE. 36 */ 37 38 #include "dsynth.h" 39 40 static void DoInit(void); 41 static void usage(int ecode) __dead2; 42 43 int OverridePkgDeleteOpt; 44 int YesOpt; 45 int DebugOpt; 46 int MaskProbeAbort; 47 int ColorOpt = 1; 48 int NullStdinOpt = 1; 49 int SlowStartOpt = -1; 50 long PkgDepMemoryTarget; 51 char *DSynthExecPath; 52 char *ProfileOverrideOpt; 53 int NiceOpt = 10; 54 55 int 56 main(int ac, char **av) 57 { 58 pkg_t *pkgs; 59 int isworker; 60 int c; 61 int sopt; 62 63 /* 64 * Get our exec path so we can self-exec clean WORKER 65 * processes. 66 */ 67 { 68 size_t len; 69 const int name[] = { 70 CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1, 71 }; 72 if (sysctl(name, 4, NULL, &len, NULL, 0) < 0) 73 dfatal_errno("Cannot get binary path"); 74 DSynthExecPath = malloc(len + 1); 75 if (sysctl(name, 4, DSynthExecPath, &len, NULL, 0) < 0) 76 dfatal_errno("Cannot get binary path"); 77 DSynthExecPath[len] = 0; 78 } 79 80 /* 81 * Override profile in dsynth.ini (can be further overridden 82 * with the -p profile option). 83 */ 84 ProfileOverrideOpt = getenv("DSYNTH_PROFILE"); 85 86 /* 87 * Process options and make sure the directive is present 88 */ 89 sopt = 0; 90 while ((c = getopt(ac, av, "dhm:p:vxys:DPSN")) != -1) { 91 switch(c) { 92 case 'x': 93 ++OverridePkgDeleteOpt; 94 break; 95 case 'y': 96 ++YesOpt; 97 break; 98 case 'D': 99 WorkerProcFlags |= WORKER_PROC_DEVELOPER; 100 break; 101 case 'P': 102 WorkerProcFlags |= WORKER_PROC_CHECK_PLIST; 103 break; 104 case 'S': 105 UseNCurses = 0; 106 if (++sopt == 2) 107 ColorOpt = 0; 108 break; 109 case 'N': 110 NiceOpt = 0; 111 break; 112 case 'd': 113 ++DebugOpt; 114 if (DebugOpt >= 2) 115 UseNCurses = 0; 116 break; 117 case 'h': 118 usage(0); 119 /* NOT REACHED */ 120 exit(0); 121 case 'v': 122 printf("dsynth %s\n", DSYNTH_VERSION); 123 exit(0); 124 case 's': 125 /* 126 * Start with N jobs, increasing to the configured 127 * maximum slowly. 0 to disable (starts with the 128 * full count). 129 */ 130 SlowStartOpt = strtol(optarg, NULL, 0); 131 break; 132 case 'm': 133 PkgDepMemoryTarget = strtoul(optarg, NULL, 0); 134 PkgDepMemoryTarget *= ONEGB; 135 break; 136 case 'p': 137 ProfileOverrideOpt = optarg; 138 break; 139 default: 140 fprintf(stderr, "Unknown option: %c\n", c); 141 usage(2); 142 /* NOT REACHED */ 143 break; 144 } 145 } 146 ac -= optind; 147 av += optind; 148 pkgs = NULL; 149 if (ac < 1) { 150 fprintf(stderr, "Missing directive\n"); 151 usage(2); 152 /* NOT REACHED */ 153 } 154 155 /* 156 * Directives which do not require a working configuration 157 */ 158 if (strcmp(av[0], "init") == 0) { 159 DoInit(); 160 exit(0); 161 /* NOT REACHED */ 162 } 163 if (strcmp(av[0], "help") == 0) { 164 usage(0); 165 exit(0); 166 /* NOT REACHED */ 167 } 168 if (strcmp(av[0], "version") == 0) { 169 printf("dsynth %s\n", DSYNTH_VERSION); 170 exit(0); 171 /* NOT REACHED */ 172 } 173 174 /* 175 * Preconfiguration. 176 */ 177 if (strcmp(av[0], "WORKER") == 0) { 178 isworker = 1; 179 } else { 180 isworker = 0; 181 } 182 183 signal(SIGPIPE, SIG_IGN); 184 ParseConfiguration(isworker); 185 186 /* 187 * Setup some environment for bulk operations (pkglist scan). 188 * These are not used by the builder (the builder will replicate 189 * all of these). 190 */ 191 addbuildenv("PORTSDIR", DPortsPath, 192 BENV_ENVIRONMENT | BENV_PKGLIST); 193 addbuildenv("BATCH", "yes", 194 BENV_ENVIRONMENT | BENV_PKGLIST); 195 addbuildenv("PKG_SUFX", UsePkgSufx, 196 BENV_ENVIRONMENT | BENV_PKGLIST); 197 addbuildenv("PACKAGE_BUILDING", "yes", 198 BENV_ENVIRONMENT | BENV_PKGLIST); 199 addbuildenv("ARCH", ArchitectureName, 200 BENV_ENVIRONMENT | BENV_PKGLIST); 201 202 #if 0 203 /* 204 * 205 */ 206 addbuildenv("OSTYPE", OperatingSystemName, 207 BENV_ENVIRONMENT | BENV_PKGLIST); 208 addbuildenv("MACHTYPE", MachineName, 209 BENV_ENVIRONMENT | BENV_PKGLIST); 210 #endif 211 /* 212 * SlowStart auto adjust. We nominally start with 1 job and increase 213 * it to the maximum every 5 seconds to give various dynamic management 214 * parameters time to stabilize. 215 * 216 * This can take a while on a many-core box with a high jobs setting, 217 * so increase the initial jobs in such cases. 218 */ 219 if (SlowStartOpt > MaxWorkers) 220 SlowStartOpt = MaxWorkers; 221 if (SlowStartOpt < 0) { 222 if (MaxWorkers < 16) 223 SlowStartOpt = 1; 224 else 225 SlowStartOpt = MaxWorkers / 4; 226 } 227 228 /* 229 * Special directive for when dsynth execs itself to manage 230 * a worker chroot. 231 */ 232 if (isworker) { 233 WorkerProcess(ac, av); 234 exit(0); 235 } 236 237 /* 238 * Build initialization and directive handling 239 */ 240 DoInitBuild(-1); 241 242 /* 243 * Directives that use the configuration but are not interlocked 244 * against a running dsynth. 245 */ 246 if (strcmp(av[0], "monitor") == 0) { 247 char *spath; 248 char *lpath; 249 250 if (ac == 1) { 251 asprintf(&spath, "%s/%s", StatsBase, STATS_FILE); 252 asprintf(&lpath, "%s/%s", StatsBase, STATS_LOCKFILE); 253 MonitorDirective(spath, lpath); 254 free(spath); 255 free(lpath); 256 } else { 257 MonitorDirective(av[1], NULL); 258 } 259 exit(0); 260 /* NOT REACHED */ 261 } 262 263 /* 264 * Front-end exec (not a WORKER exec), normal startup. We have 265 * the configuration so the first thing we need to do is check 266 * the lock file. 267 */ 268 { 269 char *lkpath; 270 int fd; 271 272 asprintf(&lkpath, "%s/.lock", BuildBase); 273 fd = open(lkpath, O_RDWR | O_CREAT | O_CLOEXEC, 0644); 274 if (fd < 0) 275 dfatal_errno("Unable to create %s", lkpath); 276 if (flock(fd, LOCK_EX | LOCK_NB) < 0) { 277 dfatal("Another dsynth is using %s, exiting", 278 BuildBase); 279 } 280 /* leave descriptor open */ 281 } 282 283 if (strcmp(av[0], "debug") == 0) { 284 DoCleanBuild(1); 285 OptimizeEnv(); 286 pkgs = ParsePackageList(ac - 1, av + 1, 1); 287 RemovePackages(pkgs); 288 DoBuild(pkgs); 289 } else if (strcmp(av[0], "status") == 0) { 290 OptimizeEnv(); 291 if (ac - 1) 292 pkgs = ParsePackageList(ac - 1, av + 1, 0); 293 else 294 pkgs = GetLocalPackageList(); 295 DoStatus(pkgs); 296 } else if (strcmp(av[0], "cleanup") == 0) { 297 DoCleanBuild(0); 298 } else if (strcmp(av[0], "configure") == 0) { 299 DoCleanBuild(0); 300 DoConfigure(); 301 } else if (strcmp(av[0], "upgrade-system") == 0) { 302 DoCleanBuild(1); 303 OptimizeEnv(); 304 pkgs = GetLocalPackageList(); 305 DoBuild(pkgs); 306 DoRebuildRepo(0); 307 DoUpgradePkgs(pkgs, 0); 308 } else if (strcmp(av[0], "prepare-system") == 0) { 309 DeleteObsoletePkgs = 1; 310 DoCleanBuild(1); 311 OptimizeEnv(); 312 pkgs = GetLocalPackageList(); 313 DoBuild(pkgs); 314 DoRebuildRepo(0); 315 } else if (strcmp(av[0], "rebuild-repository") == 0) { 316 OptimizeEnv(); 317 DoRebuildRepo(0); 318 } else if (strcmp(av[0], "purge-distfiles") == 0) { 319 OptimizeEnv(); 320 pkgs = GetFullPackageList(); 321 PurgeDistfiles(pkgs); 322 } else if (strcmp(av[0], "reset-db") == 0) { 323 char *dbmpath; 324 325 asprintf(&dbmpath, "%s/ports_crc.db", BuildBase); 326 remove(dbmpath); 327 printf("%s reset, will be regenerated on next build\n", 328 dbmpath); 329 free(dbmpath); 330 } else if (strcmp(av[0], "status-everything") == 0) { 331 OptimizeEnv(); 332 pkgs = GetFullPackageList(); 333 DoStatus(pkgs); 334 } else if (strcmp(av[0], "everything") == 0) { 335 if (WorkerProcFlags & WORKER_PROC_DEVELOPER) 336 WorkerProcFlags |= WORKER_PROC_CHECK_PLIST; 337 MaskProbeAbort = 1; 338 DeleteObsoletePkgs = 1; 339 DoCleanBuild(1); 340 OptimizeEnv(); 341 pkgs = GetFullPackageList(); 342 DoBuild(pkgs); 343 DoRebuildRepo(1); 344 } else if (strcmp(av[0], "build") == 0) { 345 DoCleanBuild(1); 346 OptimizeEnv(); 347 pkgs = ParsePackageList(ac - 1, av + 1, 0); 348 DoBuild(pkgs); 349 DoRebuildRepo(1); 350 DoUpgradePkgs(pkgs, 1); 351 } else if (strcmp(av[0], "just-build") == 0) { 352 DoCleanBuild(1); 353 OptimizeEnv(); 354 pkgs = ParsePackageList(ac - 1, av + 1, 0); 355 DoBuild(pkgs); 356 } else if (strcmp(av[0], "install") == 0) { 357 DoCleanBuild(1); 358 OptimizeEnv(); 359 pkgs = ParsePackageList(ac - 1, av + 1, 0); 360 DoBuild(pkgs); 361 DoRebuildRepo(0); 362 DoUpgradePkgs(pkgs, 0); 363 } else if (strcmp(av[0], "force") == 0) { 364 DoCleanBuild(1); 365 OptimizeEnv(); 366 pkgs = ParsePackageList(ac - 1, av + 1, 0); 367 RemovePackages(pkgs); 368 DoBuild(pkgs); 369 DoRebuildRepo(1); 370 DoUpgradePkgs(pkgs, 1); 371 } else if (strcmp(av[0], "test") == 0) { 372 WorkerProcFlags |= WORKER_PROC_CHECK_PLIST | WORKER_PROC_INSTALL | WORKER_PROC_DEINSTALL; 373 DoCleanBuild(1); 374 OptimizeEnv(); 375 pkgs = ParsePackageList(ac - 1, av + 1, 0); 376 RemovePackages(pkgs); 377 WorkerProcFlags |= WORKER_PROC_DEVELOPER; 378 DoBuild(pkgs); 379 } else { 380 fprintf(stderr, "Unknown directive '%s'\n", av[0]); 381 usage(2); 382 } 383 return 0; 384 } 385 386 static void 387 DoInit(void) 388 { 389 struct stat st; 390 char *path; 391 FILE *fp; 392 393 if (stat(ConfigBase1, &st) == 0) { 394 dfatal("init will not overwrite %s", ConfigBase1); 395 } 396 if (stat(ConfigBase2, &st) == 0) { 397 dfatal("init will not create %s if %s exists", 398 ConfigBase2, ConfigBase1); 399 } 400 if (mkdir(ConfigBase1, 0755) < 0) 401 dfatal_errno("Unable to mkdir %s", ConfigBase1); 402 403 asprintf(&path, "%s/dsynth.ini", ConfigBase1); 404 fp = fopen(path, "w"); 405 dassert_errno(fp, "Unable to create %s", path); 406 fprintf(fp, "%s", 407 "; This Synth configuration file is automatically generated\n" 408 "; Take care when hand editing!\n" 409 "\n" 410 "[Global Configuration]\n" 411 "profile_selected= LiveSystem\n" 412 "\n" 413 "[LiveSystem]\n" 414 "Operating_system= DragonFly\n" 415 "Directory_packages= /build/synth/live_packages\n" 416 "Directory_repository= /build/synth/live_packages/All\n" 417 "Directory_portsdir= /build/synth/dports\n" 418 "Directory_options= /build/synth/options\n" 419 "Directory_distfiles= /build/synth/distfiles\n" 420 "Directory_buildbase= /build/synth/build\n" 421 "Directory_logs= /build/synth/logs\n" 422 "Directory_ccache= disabled\n" 423 "Directory_system= /\n" 424 "Package_suffix= .txz\n" 425 "Number_of_builders= 0\n" 426 "Max_jobs_per_builder= 0\n" 427 "Tmpfs_workdir= true\n" 428 "Tmpfs_localbase= true\n" 429 "Display_with_ncurses= true\n" 430 "leverage_prebuilt= false\n" 431 "\n"); 432 if (fclose(fp)) 433 dfatal_errno("Unable to write to %s\n", ConfigBase1); 434 free(path); 435 436 asprintf(&path, "%s/LiveSystem-make.conf", ConfigBase1); 437 fp = fopen(path, "w"); 438 dassert_errno(fp, "Unable to create %s", path); 439 fprintf(fp, "%s", 440 "#\n" 441 "# Various dports options that might be of interest\n" 442 "#\n" 443 "#LICENSES_ACCEPTED= NONE\n" 444 "#DISABLE_LICENSES= yes\n" 445 "#DEFAULT_VERSIONS= ssl=openssl\n" 446 "#FORCE_PACKAGE= yes\n" 447 "#DPORTS_BUILDER= yes\n" 448 "#\n" 449 "# Turn these on to generate debug binaries. However, these\n" 450 "# options will seriously bloat memory use and storage use,\n" 451 "# do not use lightly\n" 452 "#\n" 453 "#STRIP=\n" 454 "#WITH_DEBUG=yes\n" 455 ); 456 if (fclose(fp)) 457 dfatal_errno("Unable to write to %s\n", ConfigBase1); 458 free(path); 459 } 460 461 __dead2 static void 462 usage(int ecode) 463 { 464 if (ecode == 2) { 465 fprintf(stderr, "Run 'dsynth help' for usage\n"); 466 exit(1); 467 } 468 469 fprintf(stderr, 470 "dsynth [options] directive\n" 471 " -d - Debug verbosity (-dd disables ncurses)\n" 472 " -h - Display this screen and exit\n" 473 " -m gb - Load management based on pkgdep memory\n" 474 " -p profile - Override profile selected in dsynth.ini\n" 475 " -s n - Set initial DynamicMaxWorkers\n" 476 " -v - Print version info and exit\n" 477 " -x - Do not rebuild packages with dependencies\n" 478 " which require rebuilding\n" 479 " -xx - Do not rebuild packages whos dports trees\n" 480 " change\n" 481 " -y - Automatically answer yes to dsynth questions\n" 482 " -D - Enable DEVELOPER mode\n" 483 " -P - Include the check-plist stage\n" 484 " -S - Disable ncurses\n" 485 " -N - Do not nice-up sub-processes (else nice +10)\n" 486 "\n" 487 " init - Initialize /etc/dsynth\n" 488 " status - Dry-run of 'upgrade-system'\n" 489 " cleanup - Clean-up mounts\n" 490 " configure - Bring up configuration menu\n" 491 " upgrade-system - Incremental build and upgrade using pkg list\n" 492 " from local system, then upgrade the local\n" 493 " system.\n" 494 " prepare-system - 'upgrade-system' but stops after building\n" 495 " rebuild-repository - Rebuild database files for current repository\n" 496 " purge-distfiles - Delete obsolete source distribution files\n" 497 " reset-db - Delete ports_crc.db, regenerate next build\n" 498 " status-everything - Dry-run of 'everything'\n" 499 " everything - Build entire dports tree and repo database\n" 500 " (-D everything infers -P)\n" 501 " version - Print version info and exit\n" 502 " help - Display this screen and exit\n" 503 " status [ports] - Dry-run of 'build' with given list\n" 504 " build [ports] - Incrementally build dports based on the given\n" 505 " list, but asks before updating the repo\n" 506 " database and system\n" 507 " just-build [ports] - 'build' but skips post-build steps\n" 508 " install [ports] - 'build' but upgrades system without asking\n" 509 " force [ports] - 'build' but deletes existing packages first\n" 510 " test [ports] - 'build' w/DEVELOPER=yes and pre-deletes pkgs\n" 511 " (also infers -P)\n" 512 " debug [ports] - like 'test' but leaves mounts intact\n" 513 " monitor [datfile] - Monitor a running dsynth\n" 514 "\n" 515 " [ports] is a space-delimited list of origins, e.g. editors/joe. It\n" 516 " may also be a path to a file containing one origin per line.\n" 517 ); 518 519 exit(ecode); 520 } 521