1 /* 2 * Copyright (c) 2019 The DragonFly Project. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Matthew Dillon <dillon@backplane.com> 6 * 7 * This code uses concepts and configuration based on 'synth', by 8 * John R. Marino <draco@marino.st>, which was written in ada. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in 18 * the documentation and/or other materials provided with the 19 * distribution. 20 * 3. Neither the name of The DragonFly Project nor the names of its 21 * contributors may be used to endorse or promote products derived 22 * from this software without specific, prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 25 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 26 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 27 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 28 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 29 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 30 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 31 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 32 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 33 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 34 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 35 * SUCH DAMAGE. 36 */ 37 38 #include "dsynth.h" 39 40 static void DoInit(void); 41 static void usage(int ecode) __dead2; 42 43 int OverridePkgDeleteOpt; 44 int YesOpt; 45 int DebugOpt; 46 int MaskProbeAbort; 47 int ColorOpt = 1; 48 int NullStdinOpt = 1; 49 int SlowStartOpt = -1; 50 long PkgDepMemoryTarget; 51 long PkgDepScaleTarget = 100; /* 1.00 */ 52 char *DSynthExecPath; 53 char *ProfileOverrideOpt; 54 int NiceOpt = 10; 55 56 int 57 main(int ac, char **av) 58 { 59 pkg_t *pkgs; 60 int isworker; 61 int c; 62 int sopt; 63 64 /* 65 * Get our exec path so we can self-exec clean WORKER 66 * processes. 67 */ 68 { 69 size_t len; 70 const int name[] = { 71 CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1, 72 }; 73 if (sysctl(name, 4, NULL, &len, NULL, 0) < 0) 74 dfatal_errno("Cannot get binary path"); 75 DSynthExecPath = malloc(len + 1); 76 if (sysctl(name, 4, DSynthExecPath, &len, NULL, 0) < 0) 77 dfatal_errno("Cannot get binary path"); 78 DSynthExecPath[len] = 0; 79 } 80 81 /* 82 * Override profile in dsynth.ini (can be further overridden 83 * with the -p profile option). 84 */ 85 ProfileOverrideOpt = getenv("DSYNTH_PROFILE"); 86 87 /* 88 * Process options and make sure the directive is present 89 */ 90 sopt = 0; 91 while ((c = getopt(ac, av, "dhm:p:vxys:DPM:NS")) != -1) { 92 switch(c) { 93 case 'x': 94 ++OverridePkgDeleteOpt; 95 break; 96 case 'y': 97 ++YesOpt; 98 break; 99 case 'D': 100 WorkerProcFlags |= WORKER_PROC_DEVELOPER; 101 break; 102 case 'P': 103 WorkerProcFlags |= WORKER_PROC_CHECK_PLIST; 104 break; 105 case 'S': 106 UseNCurses = 0; 107 if (++sopt == 2) 108 ColorOpt = 0; 109 break; 110 case 'N': 111 NiceOpt = 0; 112 break; 113 case 'd': 114 ++DebugOpt; 115 if (DebugOpt >= 2) 116 UseNCurses = 0; 117 break; 118 case 'h': 119 usage(0); 120 /* NOT REACHED */ 121 exit(0); 122 case 'v': 123 printf("dsynth %s\n", DSYNTH_VERSION); 124 exit(0); 125 case 's': 126 /* 127 * Start with N jobs, increasing to the configured 128 * maximum slowly. 0 to disable (starts with the 129 * full count). 130 */ 131 SlowStartOpt = strtol(optarg, NULL, 0); 132 break; 133 case 'm': 134 PkgDepMemoryTarget = strtoul(optarg, NULL, 0); 135 PkgDepMemoryTarget *= ONEGB; 136 break; 137 case 'M': 138 PkgDepScaleTarget = strtod(optarg, NULL) * 100; 139 if (PkgDepScaleTarget < 1) 140 PkgDepScaleTarget = 1; 141 if (PkgDepScaleTarget > 9900) 142 PkgDepScaleTarget = 9900; 143 break; 144 case 'p': 145 ProfileOverrideOpt = optarg; 146 break; 147 default: 148 fprintf(stderr, "Unknown option: %c\n", c); 149 usage(2); 150 /* NOT REACHED */ 151 break; 152 } 153 } 154 ac -= optind; 155 av += optind; 156 pkgs = NULL; 157 if (ac < 1) { 158 fprintf(stderr, "Missing directive\n"); 159 usage(2); 160 /* NOT REACHED */ 161 } 162 163 /* 164 * Directives which do not require a working configuration 165 */ 166 if (strcmp(av[0], "init") == 0) { 167 DoInit(); 168 exit(0); 169 /* NOT REACHED */ 170 } 171 if (strcmp(av[0], "help") == 0) { 172 usage(0); 173 exit(0); 174 /* NOT REACHED */ 175 } 176 if (strcmp(av[0], "version") == 0) { 177 printf("dsynth %s\n", DSYNTH_VERSION); 178 exit(0); 179 /* NOT REACHED */ 180 } 181 182 /* 183 * Preconfiguration. 184 */ 185 if (strcmp(av[0], "WORKER") == 0) { 186 isworker = 1; 187 } else { 188 isworker = 0; 189 } 190 191 signal(SIGPIPE, SIG_IGN); 192 ParseConfiguration(isworker); 193 194 /* 195 * Setup some environment for bulk operations (pkglist scan). 196 * These are not used by the builder (the builder will replicate 197 * all of these). 198 */ 199 addbuildenv("PORTSDIR", DPortsPath, 200 BENV_ENVIRONMENT | BENV_PKGLIST); 201 addbuildenv("BATCH", "yes", 202 BENV_ENVIRONMENT | BENV_PKGLIST); 203 addbuildenv("PKG_SUFX", UsePkgSufx, 204 BENV_ENVIRONMENT | BENV_PKGLIST); 205 addbuildenv("PACKAGE_BUILDING", "yes", 206 BENV_ENVIRONMENT | BENV_PKGLIST); 207 addbuildenv("ARCH", ArchitectureName, 208 BENV_ENVIRONMENT | BENV_PKGLIST); 209 210 #if 0 211 /* 212 * 213 */ 214 addbuildenv("OSTYPE", OperatingSystemName, 215 BENV_ENVIRONMENT | BENV_PKGLIST); 216 addbuildenv("MACHTYPE", MachineName, 217 BENV_ENVIRONMENT | BENV_PKGLIST); 218 #endif 219 /* 220 * SlowStart auto adjust. We nominally start with 1 job and increase 221 * it to the maximum every 5 seconds to give various dynamic management 222 * parameters time to stabilize. 223 * 224 * This can take a while on a many-core box with a high jobs setting, 225 * so increase the initial jobs in such cases. 226 */ 227 if (SlowStartOpt > MaxWorkers) 228 SlowStartOpt = MaxWorkers; 229 if (SlowStartOpt < 0) { 230 if (MaxWorkers < 16) 231 SlowStartOpt = 1; 232 else 233 SlowStartOpt = MaxWorkers / 4; 234 } 235 236 /* 237 * Special directive for when dsynth execs itself to manage 238 * a worker chroot. 239 */ 240 if (isworker) { 241 WorkerProcess(ac, av); 242 exit(0); 243 } 244 245 /* 246 * Build initialization and directive handling 247 */ 248 DoInitBuild(-1); 249 250 /* 251 * Directives that use the configuration but are not interlocked 252 * against a running dsynth. 253 */ 254 if (strcmp(av[0], "monitor") == 0) { 255 char *spath; 256 char *lpath; 257 258 if (ac == 1) { 259 asprintf(&spath, "%s/%s", StatsBase, STATS_FILE); 260 asprintf(&lpath, "%s/%s", StatsBase, STATS_LOCKFILE); 261 MonitorDirective(spath, lpath); 262 free(spath); 263 free(lpath); 264 } else { 265 MonitorDirective(av[1], NULL); 266 } 267 exit(0); 268 /* NOT REACHED */ 269 } 270 271 /* 272 * Front-end exec (not a WORKER exec), normal startup. We have 273 * the configuration so the first thing we need to do is check 274 * the lock file. 275 */ 276 { 277 char *lkpath; 278 int fd; 279 280 asprintf(&lkpath, "%s/.lock", BuildBase); 281 fd = open(lkpath, O_RDWR | O_CREAT | O_CLOEXEC, 0644); 282 if (fd < 0) 283 dfatal_errno("Unable to create %s", lkpath); 284 if (flock(fd, LOCK_EX | LOCK_NB) < 0) { 285 dfatal("Another dsynth is using %s, exiting", 286 BuildBase); 287 } 288 /* leave descriptor open */ 289 } 290 291 if (strcmp(av[0], "debug") == 0) { 292 DoCleanBuild(1); 293 OptimizeEnv(); 294 pkgs = ParsePackageList(ac - 1, av + 1, 1); 295 RemovePackages(pkgs); 296 DoBuild(pkgs); 297 } else if (strcmp(av[0], "status") == 0) { 298 OptimizeEnv(); 299 if (ac - 1) 300 pkgs = ParsePackageList(ac - 1, av + 1, 0); 301 else 302 pkgs = GetLocalPackageList(); 303 DoStatus(pkgs); 304 } else if (strcmp(av[0], "cleanup") == 0) { 305 DoCleanBuild(0); 306 } else if (strcmp(av[0], "configure") == 0) { 307 DoCleanBuild(0); 308 DoConfigure(); 309 } else if (strcmp(av[0], "upgrade-system") == 0) { 310 DoCleanBuild(1); 311 OptimizeEnv(); 312 pkgs = GetLocalPackageList(); 313 DoBuild(pkgs); 314 DoRebuildRepo(0); 315 DoUpgradePkgs(pkgs, 0); 316 } else if (strcmp(av[0], "prepare-system") == 0) { 317 DeleteObsoletePkgs = 1; 318 DoCleanBuild(1); 319 OptimizeEnv(); 320 pkgs = GetLocalPackageList(); 321 DoBuild(pkgs); 322 DoRebuildRepo(0); 323 } else if (strcmp(av[0], "rebuild-repository") == 0) { 324 OptimizeEnv(); 325 DoRebuildRepo(0); 326 } else if (strcmp(av[0], "purge-distfiles") == 0) { 327 OptimizeEnv(); 328 pkgs = GetFullPackageList(); 329 PurgeDistfiles(pkgs); 330 } else if (strcmp(av[0], "reset-db") == 0) { 331 char *dbmpath; 332 333 asprintf(&dbmpath, "%s/ports_crc.db", BuildBase); 334 remove(dbmpath); 335 printf("%s reset, will be regenerated on next build\n", 336 dbmpath); 337 free(dbmpath); 338 } else if (strcmp(av[0], "status-everything") == 0) { 339 OptimizeEnv(); 340 pkgs = GetFullPackageList(); 341 DoStatus(pkgs); 342 } else if (strcmp(av[0], "everything") == 0) { 343 if (WorkerProcFlags & WORKER_PROC_DEVELOPER) 344 WorkerProcFlags |= WORKER_PROC_CHECK_PLIST; 345 MaskProbeAbort = 1; 346 DeleteObsoletePkgs = 1; 347 DoCleanBuild(1); 348 OptimizeEnv(); 349 pkgs = GetFullPackageList(); 350 DoBuild(pkgs); 351 DoRebuildRepo(1); 352 } else if (strcmp(av[0], "build") == 0) { 353 DoCleanBuild(1); 354 OptimizeEnv(); 355 pkgs = ParsePackageList(ac - 1, av + 1, 0); 356 DoBuild(pkgs); 357 DoRebuildRepo(1); 358 DoUpgradePkgs(pkgs, 1); 359 } else if (strcmp(av[0], "just-build") == 0) { 360 DoCleanBuild(1); 361 OptimizeEnv(); 362 pkgs = ParsePackageList(ac - 1, av + 1, 0); 363 DoBuild(pkgs); 364 } else if (strcmp(av[0], "install") == 0) { 365 DoCleanBuild(1); 366 OptimizeEnv(); 367 pkgs = ParsePackageList(ac - 1, av + 1, 0); 368 DoBuild(pkgs); 369 DoRebuildRepo(0); 370 DoUpgradePkgs(pkgs, 0); 371 } else if (strcmp(av[0], "force") == 0) { 372 DoCleanBuild(1); 373 OptimizeEnv(); 374 pkgs = ParsePackageList(ac - 1, av + 1, 0); 375 RemovePackages(pkgs); 376 DoBuild(pkgs); 377 DoRebuildRepo(1); 378 DoUpgradePkgs(pkgs, 1); 379 } else if (strcmp(av[0], "test") == 0) { 380 WorkerProcFlags |= WORKER_PROC_CHECK_PLIST | WORKER_PROC_INSTALL | WORKER_PROC_DEINSTALL; 381 DoCleanBuild(1); 382 OptimizeEnv(); 383 pkgs = ParsePackageList(ac - 1, av + 1, 0); 384 RemovePackages(pkgs); 385 WorkerProcFlags |= WORKER_PROC_DEVELOPER; 386 DoBuild(pkgs); 387 } else { 388 fprintf(stderr, "Unknown directive '%s'\n", av[0]); 389 usage(2); 390 } 391 return 0; 392 } 393 394 static void 395 DoInit(void) 396 { 397 struct stat st; 398 char *path; 399 FILE *fp; 400 401 if (stat(ConfigBase1, &st) == 0) { 402 dfatal("init will not overwrite %s", ConfigBase1); 403 } 404 if (stat(ConfigBase2, &st) == 0) { 405 dfatal("init will not create %s if %s exists", 406 ConfigBase2, ConfigBase1); 407 } 408 if (mkdir(ConfigBase1, 0755) < 0) 409 dfatal_errno("Unable to mkdir %s", ConfigBase1); 410 411 asprintf(&path, "%s/dsynth.ini", ConfigBase1); 412 fp = fopen(path, "w"); 413 dassert_errno(fp, "Unable to create %s", path); 414 fprintf(fp, "%s", 415 "; This Synth configuration file is automatically generated\n" 416 "; Take care when hand editing!\n" 417 "\n" 418 "[Global Configuration]\n" 419 "profile_selected= LiveSystem\n" 420 "\n" 421 "[LiveSystem]\n" 422 "Operating_system= DragonFly\n" 423 "Directory_packages= /build/synth/live_packages\n" 424 "Directory_repository= /build/synth/live_packages/All\n" 425 "Directory_portsdir= /build/synth/dports\n" 426 "Directory_options= /build/synth/options\n" 427 "Directory_distfiles= /build/synth/distfiles\n" 428 "Directory_buildbase= /build/synth/build\n" 429 "Directory_logs= /build/synth/logs\n" 430 "Directory_ccache= disabled\n" 431 "Directory_system= /\n" 432 "Package_suffix= .txz\n" 433 "Number_of_builders= 0\n" 434 "Max_jobs_per_builder= 0\n" 435 "Tmpfs_workdir= true\n" 436 "Tmpfs_localbase= true\n" 437 "Display_with_ncurses= true\n" 438 "leverage_prebuilt= false\n" 439 "\n"); 440 if (fclose(fp)) 441 dfatal_errno("Unable to write to %s\n", ConfigBase1); 442 free(path); 443 444 asprintf(&path, "%s/LiveSystem-make.conf", ConfigBase1); 445 fp = fopen(path, "w"); 446 dassert_errno(fp, "Unable to create %s", path); 447 fprintf(fp, "%s", 448 "#\n" 449 "# Various dports options that might be of interest\n" 450 "#\n" 451 "#LICENSES_ACCEPTED= NONE\n" 452 "#DISABLE_LICENSES= yes\n" 453 "#DEFAULT_VERSIONS= ssl=openssl\n" 454 "#FORCE_PACKAGE= yes\n" 455 "#DPORTS_BUILDER= yes\n" 456 "#\n" 457 "# Turn these on to generate debug binaries. However, these\n" 458 "# options will seriously bloat memory use and storage use,\n" 459 "# do not use lightly\n" 460 "#\n" 461 "#STRIP=\n" 462 "#WITH_DEBUG=yes\n" 463 ); 464 if (fclose(fp)) 465 dfatal_errno("Unable to write to %s\n", ConfigBase1); 466 free(path); 467 } 468 469 __dead2 static void 470 usage(int ecode) 471 { 472 if (ecode == 2) { 473 fprintf(stderr, "Run 'dsynth help' for usage\n"); 474 exit(1); 475 } 476 477 fprintf(stderr, 478 "dsynth [options] directive\n" 479 " -d - Debug verbosity (-dd disables ncurses)\n" 480 " -h - Display this screen and exit\n" 481 " -m gb - Load management based on pkgdep memory\n" 482 " -p profile - Override profile selected in dsynth.ini\n" 483 " -s n - Set initial DynamicMaxWorkers\n" 484 " -v - Print version info and exit\n" 485 " -x - Do not rebuild packages with dependencies\n" 486 " which require rebuilding\n" 487 " -xx - Do not rebuild packages whos dports trees\n" 488 " change\n" 489 " -y - Automatically answer yes to dsynth questions\n" 490 " -D - Enable DEVELOPER mode\n" 491 " -P - Include the check-plist stage\n" 492 " -S - Disable ncurses\n" 493 " -N - Do not nice-up sub-processes (else nice +10)\n" 494 "\n" 495 " init - Initialize /etc/dsynth\n" 496 " status - Dry-run of 'upgrade-system'\n" 497 " cleanup - Clean-up mounts\n" 498 " configure - Bring up configuration menu\n" 499 " upgrade-system - Incremental build and upgrade using pkg list\n" 500 " from local system, then upgrade the local\n" 501 " system.\n" 502 " prepare-system - 'upgrade-system' but stops after building\n" 503 " rebuild-repository - Rebuild database files for current repository\n" 504 " purge-distfiles - Delete obsolete source distribution files\n" 505 " reset-db - Delete ports_crc.db, regenerate next build\n" 506 " status-everything - Dry-run of 'everything'\n" 507 " everything - Build entire dports tree and repo database\n" 508 " (-D everything infers -P)\n" 509 " version - Print version info and exit\n" 510 " help - Display this screen and exit\n" 511 " status [ports] - Dry-run of 'build' with given list\n" 512 " build [ports] - Incrementally build dports based on the given\n" 513 " list, but asks before updating the repo\n" 514 " database and system\n" 515 " just-build [ports] - 'build' but skips post-build steps\n" 516 " install [ports] - 'build' but upgrades system without asking\n" 517 " force [ports] - 'build' but deletes existing packages first\n" 518 " test [ports] - 'build' w/DEVELOPER=yes and pre-deletes pkgs\n" 519 " (also infers -P)\n" 520 " debug [ports] - like 'test' but leaves mounts intact\n" 521 " monitor [datfile] - Monitor a running dsynth\n" 522 "\n" 523 " [ports] is a space-delimited list of origins, e.g. editors/joe. It\n" 524 " may also be a path to a file containing one origin per line.\n" 525 ); 526 527 exit(ecode); 528 } 529