1 /* 2 * Copyright (c) 2019 The DragonFly Project. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Matthew Dillon <dillon@backplane.com> 6 * 7 * This code uses concepts and configuration based on 'synth', by 8 * John R. Marino <draco@marino.st>, which was written in ada. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in 18 * the documentation and/or other materials provided with the 19 * distribution. 20 * 3. Neither the name of The DragonFly Project nor the names of its 21 * contributors may be used to endorse or promote products derived 22 * from this software without specific, prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 25 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 26 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 27 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 28 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 29 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 30 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 31 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 32 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 33 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 34 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 35 * SUCH DAMAGE. 36 */ 37 38 #include "dsynth.h" 39 40 static void DoInit(void); 41 static void usage(int ecode) __dead2; 42 43 int YesOpt; 44 int DebugOpt; 45 int MaskProbeAbort; 46 int ColorOpt = 1; 47 int NullStdinOpt = 1; 48 int SlowStartOpt = -1; 49 long PkgDepMemoryTarget; 50 char *DSynthExecPath; 51 char *ProfileOverrideOpt; 52 53 int 54 main(int ac, char **av) 55 { 56 pkg_t *pkgs; 57 int isworker; 58 int c; 59 int sopt; 60 61 /* 62 * Get our exec path so we can self-exec clean WORKER 63 * processes. 64 */ 65 { 66 size_t len; 67 const int name[] = { 68 CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1, 69 }; 70 if (sysctl(name, 4, NULL, &len, NULL, 0) < 0) 71 dfatal_errno("Cannot get binary path"); 72 DSynthExecPath = malloc(len + 1); 73 if (sysctl(name, 4, DSynthExecPath, &len, NULL, 0) < 0) 74 dfatal_errno("Cannot get binary path"); 75 DSynthExecPath[len] = 0; 76 } 77 78 /* 79 * Override profile in dsynth.ini (can be further overridden 80 * with the -p profile option). 81 */ 82 ProfileOverrideOpt = getenv("DSYNTH_PROFILE"); 83 84 /* 85 * Process options and make sure the directive is present 86 */ 87 sopt = 0; 88 while ((c = getopt(ac, av, "dhm:p:vys:DPS")) != -1) { 89 switch(c) { 90 case 'y': 91 ++YesOpt; 92 break; 93 case 'D': 94 WorkerProcFlags |= WORKER_PROC_DEVELOPER; 95 break; 96 case 'P': 97 WorkerProcFlags |= WORKER_PROC_CHECK_PLIST; 98 break; 99 case 'S': 100 UseNCurses = 0; 101 if (++sopt == 2) 102 ColorOpt = 0; 103 break; 104 case 'd': 105 ++DebugOpt; 106 if (DebugOpt >= 2) 107 UseNCurses = 0; 108 break; 109 case 'h': 110 usage(0); 111 /* NOT REACHED */ 112 exit(0); 113 case 'v': 114 printf("dsynth %s\n", DSYNTH_VERSION); 115 exit(0); 116 case 's': 117 /* 118 * Start with N jobs, increasing to the configured 119 * maximum slowly. 0 to disable (starts with the 120 * full count). 121 */ 122 SlowStartOpt = strtol(optarg, NULL, 0); 123 break; 124 case 'm': 125 PkgDepMemoryTarget = strtoul(optarg, NULL, 0); 126 PkgDepMemoryTarget *= ONEGB; 127 break; 128 case 'p': 129 ProfileOverrideOpt = optarg; 130 break; 131 default: 132 fprintf(stderr, "Unknown option: %c\n", c); 133 usage(2); 134 /* NOT REACHED */ 135 break; 136 } 137 } 138 ac -= optind; 139 av += optind; 140 pkgs = NULL; 141 if (ac < 1) { 142 fprintf(stderr, "Missing directive\n"); 143 usage(2); 144 /* NOT REACHED */ 145 } 146 147 /* 148 * Directives which do not require a working configuration 149 */ 150 if (strcmp(av[0], "init") == 0) { 151 DoInit(); 152 exit(0); 153 /* NOT REACHED */ 154 } 155 if (strcmp(av[0], "help") == 0) { 156 usage(0); 157 exit(0); 158 /* NOT REACHED */ 159 } 160 if (strcmp(av[0], "version") == 0) { 161 printf("dsynth %s\n", DSYNTH_VERSION); 162 exit(0); 163 /* NOT REACHED */ 164 } 165 166 /* 167 * Preconfiguration. 168 */ 169 if (strcmp(av[0], "WORKER") == 0) { 170 isworker = 1; 171 } else { 172 isworker = 0; 173 } 174 175 signal(SIGPIPE, SIG_IGN); 176 ParseConfiguration(isworker); 177 178 /* 179 * Setup some environment for bulk operations (pkglist scan). 180 * These are not used by the builder (the builder will replicate 181 * all of these). 182 */ 183 addbuildenv("PORTSDIR", DPortsPath, 184 BENV_ENVIRONMENT | BENV_PKGLIST); 185 addbuildenv("BATCH", "yes", 186 BENV_ENVIRONMENT | BENV_PKGLIST); 187 addbuildenv("PKG_SUFX", UsePkgSufx, 188 BENV_ENVIRONMENT | BENV_PKGLIST); 189 addbuildenv("PACKAGE_BUILDING", "yes", 190 BENV_ENVIRONMENT | BENV_PKGLIST); 191 addbuildenv("ARCH", ArchitectureName, 192 BENV_ENVIRONMENT | BENV_PKGLIST); 193 194 #if 0 195 /* 196 * 197 */ 198 addbuildenv("OSTYPE", OperatingSystemName, 199 BENV_ENVIRONMENT | BENV_PKGLIST); 200 addbuildenv("MACHTYPE", MachineName, 201 BENV_ENVIRONMENT | BENV_PKGLIST); 202 #endif 203 /* 204 * SlowStart auto adjust. We nominally start with 1 job and increase 205 * it to the maximum every 5 seconds to give various dynamic management 206 * parameters time to stabilize. 207 * 208 * This can take a while on a many-core box with a high jobs setting, 209 * so increase the initial jobs in such cases. 210 */ 211 if (SlowStartOpt > MaxWorkers) 212 SlowStartOpt = MaxWorkers; 213 if (SlowStartOpt < 0) { 214 if (MaxWorkers < 16) 215 SlowStartOpt = 1; 216 else 217 SlowStartOpt = MaxWorkers / 4; 218 } 219 220 /* 221 * Special directive for when dsynth execs itself to manage 222 * a worker chroot. 223 */ 224 if (isworker) { 225 WorkerProcess(ac, av); 226 exit(0); 227 } 228 229 /* 230 * Build initialization and directive handling 231 */ 232 DoInitBuild(-1); 233 234 /* 235 * Directives that use the configuration but are not interlocked 236 * against a running dsynth. 237 */ 238 if (strcmp(av[0], "monitor") == 0) { 239 char *spath; 240 char *lpath; 241 242 if (ac == 1) { 243 asprintf(&spath, "%s/%s", StatsBase, STATS_FILE); 244 asprintf(&lpath, "%s/%s", StatsBase, STATS_LOCKFILE); 245 MonitorDirective(spath, lpath); 246 free(spath); 247 free(lpath); 248 } else { 249 MonitorDirective(av[1], NULL); 250 } 251 exit(0); 252 /* NOT REACHED */ 253 } 254 255 /* 256 * Front-end exec (not a WORKER exec), normal startup. We have 257 * the configuration so the first thing we need to do is check 258 * the lock file. 259 */ 260 { 261 char *lkpath; 262 int fd; 263 264 asprintf(&lkpath, "%s/.lock", BuildBase); 265 fd = open(lkpath, O_RDWR | O_CREAT | O_CLOEXEC, 0644); 266 if (fd < 0) 267 dfatal_errno("Unable to create %s", lkpath); 268 if (flock(fd, LOCK_EX | LOCK_NB) < 0) { 269 dfatal("Another dsynth is using %s, exiting", 270 BuildBase); 271 } 272 /* leave descriptor open */ 273 } 274 275 if (strcmp(av[0], "debug") == 0) { 276 DoCleanBuild(1); 277 OptimizeEnv(); 278 pkgs = ParsePackageList(ac - 1, av + 1, 1); 279 RemovePackages(pkgs); 280 DoBuild(pkgs); 281 } else if (strcmp(av[0], "status") == 0) { 282 OptimizeEnv(); 283 if (ac - 1) 284 pkgs = ParsePackageList(ac - 1, av + 1, 0); 285 else 286 pkgs = GetLocalPackageList(); 287 DoStatus(pkgs); 288 } else if (strcmp(av[0], "cleanup") == 0) { 289 DoCleanBuild(0); 290 } else if (strcmp(av[0], "configure") == 0) { 291 DoCleanBuild(0); 292 DoConfigure(); 293 } else if (strcmp(av[0], "upgrade-system") == 0) { 294 DoCleanBuild(1); 295 OptimizeEnv(); 296 pkgs = GetLocalPackageList(); 297 DoBuild(pkgs); 298 DoRebuildRepo(0); 299 DoUpgradePkgs(pkgs, 0); 300 } else if (strcmp(av[0], "prepare-system") == 0) { 301 DoCleanBuild(1); 302 OptimizeEnv(); 303 pkgs = GetLocalPackageList(); 304 DoBuild(pkgs); 305 DoRebuildRepo(0); 306 } else if (strcmp(av[0], "rebuild-repository") == 0) { 307 OptimizeEnv(); 308 DoRebuildRepo(0); 309 } else if (strcmp(av[0], "purge-distfiles") == 0) { 310 OptimizeEnv(); 311 pkgs = GetFullPackageList(); 312 PurgeDistfiles(pkgs); 313 } else if (strcmp(av[0], "status-everything") == 0) { 314 OptimizeEnv(); 315 pkgs = GetFullPackageList(); 316 DoStatus(pkgs); 317 } else if (strcmp(av[0], "everything") == 0) { 318 if (WorkerProcFlags & WORKER_PROC_DEVELOPER) 319 WorkerProcFlags |= WORKER_PROC_CHECK_PLIST; 320 MaskProbeAbort = 1; 321 DeleteObsoletePkgs = 1; 322 DoCleanBuild(1); 323 OptimizeEnv(); 324 pkgs = GetFullPackageList(); 325 DoBuild(pkgs); 326 DoRebuildRepo(1); 327 } else if (strcmp(av[0], "build") == 0) { 328 DoCleanBuild(1); 329 OptimizeEnv(); 330 pkgs = ParsePackageList(ac - 1, av + 1, 0); 331 DoBuild(pkgs); 332 DoRebuildRepo(1); 333 DoUpgradePkgs(pkgs, 1); 334 } else if (strcmp(av[0], "just-build") == 0) { 335 DoCleanBuild(1); 336 OptimizeEnv(); 337 pkgs = ParsePackageList(ac - 1, av + 1, 0); 338 DoBuild(pkgs); 339 } else if (strcmp(av[0], "install") == 0) { 340 DoCleanBuild(1); 341 OptimizeEnv(); 342 pkgs = ParsePackageList(ac - 1, av + 1, 0); 343 DoBuild(pkgs); 344 DoRebuildRepo(0); 345 DoUpgradePkgs(pkgs, 0); 346 } else if (strcmp(av[0], "force") == 0) { 347 DoCleanBuild(1); 348 OptimizeEnv(); 349 pkgs = ParsePackageList(ac - 1, av + 1, 0); 350 RemovePackages(pkgs); 351 DoBuild(pkgs); 352 DoRebuildRepo(1); 353 DoUpgradePkgs(pkgs, 1); 354 } else if (strcmp(av[0], "test") == 0) { 355 WorkerProcFlags |= WORKER_PROC_CHECK_PLIST; 356 DoCleanBuild(1); 357 OptimizeEnv(); 358 pkgs = ParsePackageList(ac - 1, av + 1, 0); 359 RemovePackages(pkgs); 360 WorkerProcFlags |= WORKER_PROC_DEVELOPER; 361 DoBuild(pkgs); 362 } else { 363 fprintf(stderr, "Unknown directive '%s'\n", av[0]); 364 usage(2); 365 } 366 return 0; 367 } 368 369 static void 370 DoInit(void) 371 { 372 struct stat st; 373 char *path; 374 FILE *fp; 375 376 if (stat(ConfigBase1, &st) == 0) { 377 dfatal("init will not overwrite %s", ConfigBase1); 378 } 379 if (stat(ConfigBase2, &st) == 0) { 380 dfatal("init will not create %s if %s exists", 381 ConfigBase2, ConfigBase1); 382 } 383 if (mkdir(ConfigBase1, 0755) < 0) 384 dfatal_errno("Unable to mkdir %s", ConfigBase1); 385 386 asprintf(&path, "%s/dsynth.ini", ConfigBase1); 387 fp = fopen(path, "w"); 388 dassert_errno(fp, "Unable to create %s", path); 389 fprintf(fp, "%s", 390 "; This Synth configuration file is automatically generated\n" 391 "; Take care when hand editing!\n" 392 "\n" 393 "[Global Configuration]\n" 394 "profile_selected= LiveSystem\n" 395 "\n" 396 "[LiveSystem]\n" 397 "Operating_system= DragonFly\n" 398 "Directory_packages= /build/synth/live_packages\n" 399 "Directory_repository= /build/synth/live_packages/All\n" 400 "Directory_portsdir= /build/synth/dports\n" 401 "Directory_options= /build/synth/options\n" 402 "Directory_distfiles= /build/synth/distfiles\n" 403 "Directory_buildbase= /build/synth/build\n" 404 "Directory_logs= /build/synth/logs\n" 405 "Directory_ccache= disabled\n" 406 "Directory_system= /\n" 407 "Package_suffix= .txz\n" 408 "Number_of_builders= 0\n" 409 "Max_jobs_per_builder= 0\n" 410 "Tmpfs_workdir= true\n" 411 "Tmpfs_localbase= true\n" 412 "Display_with_ncurses= true\n" 413 "leverage_prebuilt= false\n" 414 "\n"); 415 if (fclose(fp)) 416 dfatal_errno("Unable to write to %s\n", ConfigBase1); 417 free(path); 418 419 asprintf(&path, "%s/LiveSystem-make.conf", ConfigBase1); 420 fp = fopen(path, "w"); 421 dassert_errno(fp, "Unable to create %s", path); 422 fprintf(fp, "%s", 423 "#\n" 424 "# Various dports options that might be of interest\n" 425 "#\n" 426 "#LICENSES_ACCEPTED= NONE\n" 427 "#DISABLE_LICENSES= yes\n" 428 "#DEFAULT_VERSIONS= ssl=openssl\n" 429 "#FORCE_PACKAGE= yes\n" 430 "#DPORTS_BUILDER= yes\n" 431 "#\n" 432 "# Turn these on to generate debug binaries. However, these\n" 433 "# options will seriously bloat memory use and storage use,\n" 434 "# do not use lightly\n" 435 "#\n" 436 "#STRIP=\n" 437 "#WITH_DEBUG=yes\n" 438 ); 439 if (fclose(fp)) 440 dfatal_errno("Unable to write to %s\n", ConfigBase1); 441 free(path); 442 } 443 444 __dead2 static void 445 usage(int ecode) 446 { 447 if (ecode == 2) { 448 fprintf(stderr, "Run 'dsynth help' for usage\n"); 449 exit(1); 450 } 451 452 fprintf(stderr, 453 "dsynth [options] directive\n" 454 " -d - Debug verbosity (-dd disables ncurses)\n" 455 " -h - Display this screen and exit\n" 456 " -m gb - Load management based on pkgdep memory\n" 457 " -p profile - Override profile selected in dsynth.ini\n" 458 " -s n - Set initial DynamicMaxWorkers\n" 459 " -v - Print version info and exit\n" 460 " -y - Automatically answer yes to dsynth questions\n" 461 " -D - Enable DEVELOPER mode\n" 462 " -P - Include the check-plist stage\n" 463 " -S - Disable ncurses\n" 464 "\n" 465 " init - Initialize /etc/dsynth\n" 466 " status - Dry-run of 'upgrade-system'\n" 467 " cleanup - Clean-up mounts\n" 468 " configure - Bring up configuration menu\n" 469 " upgrade-system - Incremental build and upgrade using pkg list\n" 470 " from local system, then upgrade the local\n" 471 " system.\n" 472 " prepare-system - 'upgrade-system' but stops after building\n" 473 " rebuild-repository - Rebuild database files for current repository\n" 474 " purge-distfiles - Delete obsolete source distribution files\n" 475 " status-everything - Dry-run of 'everything'\n" 476 " everything - Build entire dports tree and repo database\n" 477 " (-D everything infers -P)\n" 478 " version - Print version info and exit\n" 479 " help - Display this screen and exit\n" 480 " status [ports] - Dry-run of 'build' with given list\n" 481 " build [ports] - Incrementally build dports based on the given\n" 482 " list, but asks before updating the repo\n" 483 " database and system\n" 484 " just-build [ports] - 'build' but skips post-build steps\n" 485 " install [ports] - 'build' but upgrades system without asking\n" 486 " force [ports] - 'build' but deletes existing packages first\n" 487 " test [ports] - 'build' w/DEVELOPER=yes and pre-deletes pkgs\n" 488 " (also infers -P)\n" 489 " debug [ports] - like 'test' but leaves mounts intact\n" 490 " monitor [datfile] - Monitor a running dsynth\n" 491 "\n" 492 " [ports] is a space-delimited list of origins, e.g. editors/joe. It\n" 493 " may also be a path to a file containing one origin per line.\n" 494 ); 495 496 exit(ecode); 497 } 498