1 /* 2 * Copyright (c) 2019 The DragonFly Project. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Matthew Dillon <dillon@backplane.com> 6 * 7 * This code uses concepts and configuration based on 'synth', by 8 * John R. Marino <draco@marino.st>, which was written in ada. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in 18 * the documentation and/or other materials provided with the 19 * distribution. 20 * 3. Neither the name of The DragonFly Project nor the names of its 21 * contributors may be used to endorse or promote products derived 22 * from this software without specific, prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 25 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 26 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 27 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 28 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 29 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 30 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 31 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 32 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 33 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 34 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 35 * SUCH DAMAGE. 36 */ 37 38 #include "dsynth.h" 39 40 static void DoInit(void); 41 static void usage(int ecode) __dead2; 42 43 int YesOpt; 44 int DebugOpt; 45 int MaskProbeAbort; 46 int ColorOpt = 1; 47 int NullStdinOpt = 1; 48 int SlowStartOpt = -1; 49 long PkgDepMemoryTarget; 50 char *DSynthExecPath; 51 char *ProfileOverrideOpt; 52 int NiceOpt = 10; 53 54 int 55 main(int ac, char **av) 56 { 57 pkg_t *pkgs; 58 int isworker; 59 int c; 60 int sopt; 61 62 /* 63 * Get our exec path so we can self-exec clean WORKER 64 * processes. 65 */ 66 { 67 size_t len; 68 const int name[] = { 69 CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1, 70 }; 71 if (sysctl(name, 4, NULL, &len, NULL, 0) < 0) 72 dfatal_errno("Cannot get binary path"); 73 DSynthExecPath = malloc(len + 1); 74 if (sysctl(name, 4, DSynthExecPath, &len, NULL, 0) < 0) 75 dfatal_errno("Cannot get binary path"); 76 DSynthExecPath[len] = 0; 77 } 78 79 /* 80 * Override profile in dsynth.ini (can be further overridden 81 * with the -p profile option). 82 */ 83 ProfileOverrideOpt = getenv("DSYNTH_PROFILE"); 84 85 /* 86 * Process options and make sure the directive is present 87 */ 88 sopt = 0; 89 while ((c = getopt(ac, av, "dhm:p:vys:DPSN")) != -1) { 90 switch(c) { 91 case 'y': 92 ++YesOpt; 93 break; 94 case 'D': 95 WorkerProcFlags |= WORKER_PROC_DEVELOPER; 96 break; 97 case 'P': 98 WorkerProcFlags |= WORKER_PROC_CHECK_PLIST; 99 break; 100 case 'S': 101 UseNCurses = 0; 102 if (++sopt == 2) 103 ColorOpt = 0; 104 break; 105 case 'N': 106 NiceOpt = 0; 107 break; 108 case 'd': 109 ++DebugOpt; 110 if (DebugOpt >= 2) 111 UseNCurses = 0; 112 break; 113 case 'h': 114 usage(0); 115 /* NOT REACHED */ 116 exit(0); 117 case 'v': 118 printf("dsynth %s\n", DSYNTH_VERSION); 119 exit(0); 120 case 's': 121 /* 122 * Start with N jobs, increasing to the configured 123 * maximum slowly. 0 to disable (starts with the 124 * full count). 125 */ 126 SlowStartOpt = strtol(optarg, NULL, 0); 127 break; 128 case 'm': 129 PkgDepMemoryTarget = strtoul(optarg, NULL, 0); 130 PkgDepMemoryTarget *= ONEGB; 131 break; 132 case 'p': 133 ProfileOverrideOpt = optarg; 134 break; 135 default: 136 fprintf(stderr, "Unknown option: %c\n", c); 137 usage(2); 138 /* NOT REACHED */ 139 break; 140 } 141 } 142 ac -= optind; 143 av += optind; 144 pkgs = NULL; 145 if (ac < 1) { 146 fprintf(stderr, "Missing directive\n"); 147 usage(2); 148 /* NOT REACHED */ 149 } 150 151 /* 152 * Directives which do not require a working configuration 153 */ 154 if (strcmp(av[0], "init") == 0) { 155 DoInit(); 156 exit(0); 157 /* NOT REACHED */ 158 } 159 if (strcmp(av[0], "help") == 0) { 160 usage(0); 161 exit(0); 162 /* NOT REACHED */ 163 } 164 if (strcmp(av[0], "version") == 0) { 165 printf("dsynth %s\n", DSYNTH_VERSION); 166 exit(0); 167 /* NOT REACHED */ 168 } 169 170 /* 171 * Preconfiguration. 172 */ 173 if (strcmp(av[0], "WORKER") == 0) { 174 isworker = 1; 175 } else { 176 isworker = 0; 177 } 178 179 signal(SIGPIPE, SIG_IGN); 180 ParseConfiguration(isworker); 181 182 /* 183 * Setup some environment for bulk operations (pkglist scan). 184 * These are not used by the builder (the builder will replicate 185 * all of these). 186 */ 187 addbuildenv("PORTSDIR", DPortsPath, 188 BENV_ENVIRONMENT | BENV_PKGLIST); 189 addbuildenv("BATCH", "yes", 190 BENV_ENVIRONMENT | BENV_PKGLIST); 191 addbuildenv("PKG_SUFX", UsePkgSufx, 192 BENV_ENVIRONMENT | BENV_PKGLIST); 193 addbuildenv("PACKAGE_BUILDING", "yes", 194 BENV_ENVIRONMENT | BENV_PKGLIST); 195 addbuildenv("ARCH", ArchitectureName, 196 BENV_ENVIRONMENT | BENV_PKGLIST); 197 198 #if 0 199 /* 200 * 201 */ 202 addbuildenv("OSTYPE", OperatingSystemName, 203 BENV_ENVIRONMENT | BENV_PKGLIST); 204 addbuildenv("MACHTYPE", MachineName, 205 BENV_ENVIRONMENT | BENV_PKGLIST); 206 #endif 207 /* 208 * SlowStart auto adjust. We nominally start with 1 job and increase 209 * it to the maximum every 5 seconds to give various dynamic management 210 * parameters time to stabilize. 211 * 212 * This can take a while on a many-core box with a high jobs setting, 213 * so increase the initial jobs in such cases. 214 */ 215 if (SlowStartOpt > MaxWorkers) 216 SlowStartOpt = MaxWorkers; 217 if (SlowStartOpt < 0) { 218 if (MaxWorkers < 16) 219 SlowStartOpt = 1; 220 else 221 SlowStartOpt = MaxWorkers / 4; 222 } 223 224 /* 225 * Special directive for when dsynth execs itself to manage 226 * a worker chroot. 227 */ 228 if (isworker) { 229 WorkerProcess(ac, av); 230 exit(0); 231 } 232 233 /* 234 * Build initialization and directive handling 235 */ 236 DoInitBuild(-1); 237 238 /* 239 * Directives that use the configuration but are not interlocked 240 * against a running dsynth. 241 */ 242 if (strcmp(av[0], "monitor") == 0) { 243 char *spath; 244 char *lpath; 245 246 if (ac == 1) { 247 asprintf(&spath, "%s/%s", StatsBase, STATS_FILE); 248 asprintf(&lpath, "%s/%s", StatsBase, STATS_LOCKFILE); 249 MonitorDirective(spath, lpath); 250 free(spath); 251 free(lpath); 252 } else { 253 MonitorDirective(av[1], NULL); 254 } 255 exit(0); 256 /* NOT REACHED */ 257 } 258 259 /* 260 * Front-end exec (not a WORKER exec), normal startup. We have 261 * the configuration so the first thing we need to do is check 262 * the lock file. 263 */ 264 { 265 char *lkpath; 266 int fd; 267 268 asprintf(&lkpath, "%s/.lock", BuildBase); 269 fd = open(lkpath, O_RDWR | O_CREAT | O_CLOEXEC, 0644); 270 if (fd < 0) 271 dfatal_errno("Unable to create %s", lkpath); 272 if (flock(fd, LOCK_EX | LOCK_NB) < 0) { 273 dfatal("Another dsynth is using %s, exiting", 274 BuildBase); 275 } 276 /* leave descriptor open */ 277 } 278 279 if (strcmp(av[0], "debug") == 0) { 280 DoCleanBuild(1); 281 OptimizeEnv(); 282 pkgs = ParsePackageList(ac - 1, av + 1, 1); 283 RemovePackages(pkgs); 284 DoBuild(pkgs); 285 } else if (strcmp(av[0], "status") == 0) { 286 OptimizeEnv(); 287 if (ac - 1) 288 pkgs = ParsePackageList(ac - 1, av + 1, 0); 289 else 290 pkgs = GetLocalPackageList(); 291 DoStatus(pkgs); 292 } else if (strcmp(av[0], "cleanup") == 0) { 293 DoCleanBuild(0); 294 } else if (strcmp(av[0], "configure") == 0) { 295 DoCleanBuild(0); 296 DoConfigure(); 297 } else if (strcmp(av[0], "upgrade-system") == 0) { 298 DoCleanBuild(1); 299 OptimizeEnv(); 300 pkgs = GetLocalPackageList(); 301 DoBuild(pkgs); 302 DoRebuildRepo(0); 303 DoUpgradePkgs(pkgs, 0); 304 } else if (strcmp(av[0], "prepare-system") == 0) { 305 DoCleanBuild(1); 306 OptimizeEnv(); 307 pkgs = GetLocalPackageList(); 308 DoBuild(pkgs); 309 DoRebuildRepo(0); 310 } else if (strcmp(av[0], "rebuild-repository") == 0) { 311 OptimizeEnv(); 312 DoRebuildRepo(0); 313 } else if (strcmp(av[0], "purge-distfiles") == 0) { 314 OptimizeEnv(); 315 pkgs = GetFullPackageList(); 316 PurgeDistfiles(pkgs); 317 } else if (strcmp(av[0], "status-everything") == 0) { 318 OptimizeEnv(); 319 pkgs = GetFullPackageList(); 320 DoStatus(pkgs); 321 } else if (strcmp(av[0], "everything") == 0) { 322 if (WorkerProcFlags & WORKER_PROC_DEVELOPER) 323 WorkerProcFlags |= WORKER_PROC_CHECK_PLIST; 324 MaskProbeAbort = 1; 325 DeleteObsoletePkgs = 1; 326 DoCleanBuild(1); 327 OptimizeEnv(); 328 pkgs = GetFullPackageList(); 329 DoBuild(pkgs); 330 DoRebuildRepo(1); 331 } else if (strcmp(av[0], "build") == 0) { 332 DoCleanBuild(1); 333 OptimizeEnv(); 334 pkgs = ParsePackageList(ac - 1, av + 1, 0); 335 DoBuild(pkgs); 336 DoRebuildRepo(1); 337 DoUpgradePkgs(pkgs, 1); 338 } else if (strcmp(av[0], "just-build") == 0) { 339 DoCleanBuild(1); 340 OptimizeEnv(); 341 pkgs = ParsePackageList(ac - 1, av + 1, 0); 342 DoBuild(pkgs); 343 } else if (strcmp(av[0], "install") == 0) { 344 DoCleanBuild(1); 345 OptimizeEnv(); 346 pkgs = ParsePackageList(ac - 1, av + 1, 0); 347 DoBuild(pkgs); 348 DoRebuildRepo(0); 349 DoUpgradePkgs(pkgs, 0); 350 } else if (strcmp(av[0], "force") == 0) { 351 DoCleanBuild(1); 352 OptimizeEnv(); 353 pkgs = ParsePackageList(ac - 1, av + 1, 0); 354 RemovePackages(pkgs); 355 DoBuild(pkgs); 356 DoRebuildRepo(1); 357 DoUpgradePkgs(pkgs, 1); 358 } else if (strcmp(av[0], "test") == 0) { 359 WorkerProcFlags |= WORKER_PROC_CHECK_PLIST; 360 DoCleanBuild(1); 361 OptimizeEnv(); 362 pkgs = ParsePackageList(ac - 1, av + 1, 0); 363 RemovePackages(pkgs); 364 WorkerProcFlags |= WORKER_PROC_DEVELOPER; 365 DoBuild(pkgs); 366 } else { 367 fprintf(stderr, "Unknown directive '%s'\n", av[0]); 368 usage(2); 369 } 370 return 0; 371 } 372 373 static void 374 DoInit(void) 375 { 376 struct stat st; 377 char *path; 378 FILE *fp; 379 380 if (stat(ConfigBase1, &st) == 0) { 381 dfatal("init will not overwrite %s", ConfigBase1); 382 } 383 if (stat(ConfigBase2, &st) == 0) { 384 dfatal("init will not create %s if %s exists", 385 ConfigBase2, ConfigBase1); 386 } 387 if (mkdir(ConfigBase1, 0755) < 0) 388 dfatal_errno("Unable to mkdir %s", ConfigBase1); 389 390 asprintf(&path, "%s/dsynth.ini", ConfigBase1); 391 fp = fopen(path, "w"); 392 dassert_errno(fp, "Unable to create %s", path); 393 fprintf(fp, "%s", 394 "; This Synth configuration file is automatically generated\n" 395 "; Take care when hand editing!\n" 396 "\n" 397 "[Global Configuration]\n" 398 "profile_selected= LiveSystem\n" 399 "\n" 400 "[LiveSystem]\n" 401 "Operating_system= DragonFly\n" 402 "Directory_packages= /build/synth/live_packages\n" 403 "Directory_repository= /build/synth/live_packages/All\n" 404 "Directory_portsdir= /build/synth/dports\n" 405 "Directory_options= /build/synth/options\n" 406 "Directory_distfiles= /build/synth/distfiles\n" 407 "Directory_buildbase= /build/synth/build\n" 408 "Directory_logs= /build/synth/logs\n" 409 "Directory_ccache= disabled\n" 410 "Directory_system= /\n" 411 "Package_suffix= .txz\n" 412 "Number_of_builders= 0\n" 413 "Max_jobs_per_builder= 0\n" 414 "Tmpfs_workdir= true\n" 415 "Tmpfs_localbase= true\n" 416 "Display_with_ncurses= true\n" 417 "leverage_prebuilt= false\n" 418 "\n"); 419 if (fclose(fp)) 420 dfatal_errno("Unable to write to %s\n", ConfigBase1); 421 free(path); 422 423 asprintf(&path, "%s/LiveSystem-make.conf", ConfigBase1); 424 fp = fopen(path, "w"); 425 dassert_errno(fp, "Unable to create %s", path); 426 fprintf(fp, "%s", 427 "#\n" 428 "# Various dports options that might be of interest\n" 429 "#\n" 430 "#LICENSES_ACCEPTED= NONE\n" 431 "#DISABLE_LICENSES= yes\n" 432 "#DEFAULT_VERSIONS= ssl=openssl\n" 433 "#FORCE_PACKAGE= yes\n" 434 "#DPORTS_BUILDER= yes\n" 435 "#\n" 436 "# Turn these on to generate debug binaries. However, these\n" 437 "# options will seriously bloat memory use and storage use,\n" 438 "# do not use lightly\n" 439 "#\n" 440 "#STRIP=\n" 441 "#WITH_DEBUG=yes\n" 442 ); 443 if (fclose(fp)) 444 dfatal_errno("Unable to write to %s\n", ConfigBase1); 445 free(path); 446 } 447 448 __dead2 static void 449 usage(int ecode) 450 { 451 if (ecode == 2) { 452 fprintf(stderr, "Run 'dsynth help' for usage\n"); 453 exit(1); 454 } 455 456 fprintf(stderr, 457 "dsynth [options] directive\n" 458 " -d - Debug verbosity (-dd disables ncurses)\n" 459 " -h - Display this screen and exit\n" 460 " -m gb - Load management based on pkgdep memory\n" 461 " -p profile - Override profile selected in dsynth.ini\n" 462 " -s n - Set initial DynamicMaxWorkers\n" 463 " -v - Print version info and exit\n" 464 " -y - Automatically answer yes to dsynth questions\n" 465 " -D - Enable DEVELOPER mode\n" 466 " -P - Include the check-plist stage\n" 467 " -S - Disable ncurses\n" 468 " -N - Do not nice-up sub-processes (else nice +10)\n" 469 "\n" 470 " init - Initialize /etc/dsynth\n" 471 " status - Dry-run of 'upgrade-system'\n" 472 " cleanup - Clean-up mounts\n" 473 " configure - Bring up configuration menu\n" 474 " upgrade-system - Incremental build and upgrade using pkg list\n" 475 " from local system, then upgrade the local\n" 476 " system.\n" 477 " prepare-system - 'upgrade-system' but stops after building\n" 478 " rebuild-repository - Rebuild database files for current repository\n" 479 " purge-distfiles - Delete obsolete source distribution files\n" 480 " status-everything - Dry-run of 'everything'\n" 481 " everything - Build entire dports tree and repo database\n" 482 " (-D everything infers -P)\n" 483 " version - Print version info and exit\n" 484 " help - Display this screen and exit\n" 485 " status [ports] - Dry-run of 'build' with given list\n" 486 " build [ports] - Incrementally build dports based on the given\n" 487 " list, but asks before updating the repo\n" 488 " database and system\n" 489 " just-build [ports] - 'build' but skips post-build steps\n" 490 " install [ports] - 'build' but upgrades system without asking\n" 491 " force [ports] - 'build' but deletes existing packages first\n" 492 " test [ports] - 'build' w/DEVELOPER=yes and pre-deletes pkgs\n" 493 " (also infers -P)\n" 494 " debug [ports] - like 'test' but leaves mounts intact\n" 495 " monitor [datfile] - Monitor a running dsynth\n" 496 "\n" 497 " [ports] is a space-delimited list of origins, e.g. editors/joe. It\n" 498 " may also be a path to a file containing one origin per line.\n" 499 ); 500 501 exit(ecode); 502 } 503