1 /* 2 * Copyright (c) 2019 The DragonFly Project. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Matthew Dillon <dillon@backplane.com> 6 * 7 * This code uses concepts and configuration based on 'synth', by 8 * John R. Marino <draco@marino.st>, which was written in ada. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in 18 * the documentation and/or other materials provided with the 19 * distribution. 20 * 3. Neither the name of The DragonFly Project nor the names of its 21 * contributors may be used to endorse or promote products derived 22 * from this software without specific, prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 25 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 26 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 27 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 28 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 29 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 30 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 31 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 32 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 33 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 34 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 35 * SUCH DAMAGE. 36 */ 37 38 #include "dsynth.h" 39 40 static void DoInit(void); 41 static void usage(int ecode) __dead2; 42 43 int OverridePkgDeleteOpt; 44 int YesOpt; 45 int DebugOpt; 46 int MaskProbeAbort; 47 int ColorOpt = 1; 48 int NullStdinOpt = 1; 49 int SlowStartOpt = -1; 50 long PkgDepMemoryTarget; 51 long PkgDepScaleTarget = 100; /* 1.00 */ 52 char *DSynthExecPath; 53 char *ProfileOverrideOpt; 54 int NiceOpt = 10; 55 56 int 57 main(int ac, char **av) 58 { 59 pkg_t *pkgs; 60 int isworker; 61 int c; 62 int sopt; 63 64 /* 65 * Get our exec path so we can self-exec clean WORKER 66 * processes. 67 */ 68 { 69 size_t len; 70 const int name[] = { 71 CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1, 72 }; 73 if (sysctl(name, 4, NULL, &len, NULL, 0) < 0) 74 dfatal_errno("Cannot get binary path"); 75 DSynthExecPath = malloc(len + 1); 76 if (sysctl(name, 4, DSynthExecPath, &len, NULL, 0) < 0) 77 dfatal_errno("Cannot get binary path"); 78 DSynthExecPath[len] = 0; 79 } 80 81 /* 82 * Override profile in dsynth.ini (can be further overridden 83 * with the -p profile option). 84 */ 85 ProfileOverrideOpt = getenv("DSYNTH_PROFILE"); 86 87 /* 88 * Process options and make sure the directive is present 89 */ 90 sopt = 0; 91 while ((c = getopt(ac, av, "dhm:p:vxys:DPM:NS")) != -1) { 92 switch(c) { 93 case 'x': 94 ++OverridePkgDeleteOpt; 95 break; 96 case 'y': 97 ++YesOpt; 98 break; 99 case 'D': 100 WorkerProcFlags |= WORKER_PROC_DEVELOPER; 101 break; 102 case 'P': 103 WorkerProcFlags |= WORKER_PROC_CHECK_PLIST; 104 break; 105 case 'S': 106 UseNCurses = 0; 107 if (++sopt == 2) 108 ColorOpt = 0; 109 break; 110 case 'N': 111 NiceOpt = 0; 112 break; 113 case 'd': 114 ++DebugOpt; 115 if (DebugOpt >= 2) 116 UseNCurses = 0; 117 break; 118 case 'h': 119 usage(0); 120 /* NOT REACHED */ 121 exit(0); 122 case 'v': 123 printf("dsynth %s\n", DSYNTH_VERSION); 124 exit(0); 125 case 's': 126 /* 127 * Start with N jobs, increasing to the configured 128 * maximum slowly. 0 to disable (starts with the 129 * full count). 130 */ 131 SlowStartOpt = strtol(optarg, NULL, 0); 132 break; 133 case 'm': 134 PkgDepMemoryTarget = strtoul(optarg, NULL, 0); 135 PkgDepMemoryTarget *= ONEGB; 136 break; 137 case 'M': 138 PkgDepScaleTarget = strtod(optarg, NULL) * 100; 139 if (PkgDepScaleTarget < 1) 140 PkgDepScaleTarget = 1; 141 if (PkgDepScaleTarget > 9900) 142 PkgDepScaleTarget = 9900; 143 break; 144 case 'p': 145 ProfileOverrideOpt = optarg; 146 break; 147 default: 148 fprintf(stderr, "Unknown option: %c\n", c); 149 usage(2); 150 /* NOT REACHED */ 151 break; 152 } 153 } 154 ac -= optind; 155 av += optind; 156 pkgs = NULL; 157 if (ac < 1) { 158 fprintf(stderr, "Missing directive\n"); 159 usage(2); 160 /* NOT REACHED */ 161 } 162 163 /* 164 * Directives which do not require a working configuration 165 */ 166 if (strcmp(av[0], "init") == 0) { 167 DoInit(); 168 exit(0); 169 /* NOT REACHED */ 170 } 171 if (strcmp(av[0], "help") == 0) { 172 usage(0); 173 exit(0); 174 /* NOT REACHED */ 175 } 176 if (strcmp(av[0], "version") == 0) { 177 printf("dsynth %s\n", DSYNTH_VERSION); 178 exit(0); 179 /* NOT REACHED */ 180 } 181 182 /* 183 * Preconfiguration. 184 */ 185 if (strcmp(av[0], "WORKER") == 0) { 186 isworker = 1; 187 } else { 188 isworker = 0; 189 } 190 191 signal(SIGPIPE, SIG_IGN); 192 ParseConfiguration(isworker); 193 194 /* 195 * Setup some environment for bulk operations (pkglist scan). 196 * These are not used by the builder (the builder will replicate 197 * all of these). 198 * 199 * NOTE: PKG_SUFX - pkg versions older than 1.17 200 * PKG_COMPRESSION_FORMAT - pkg versions >= 1.17 201 */ 202 addbuildenv("PORTSDIR", DPortsPath, 203 BENV_ENVIRONMENT | BENV_PKGLIST); 204 addbuildenv("BATCH", "yes", 205 BENV_ENVIRONMENT | BENV_PKGLIST); 206 addbuildenv("PKG_COMPRESSION_FORMAT", UsePkgSufx, 207 BENV_ENVIRONMENT | BENV_PKGLIST); 208 addbuildenv("PKG_SUFX", UsePkgSufx, 209 BENV_ENVIRONMENT | BENV_PKGLIST); 210 addbuildenv("PACKAGE_BUILDING", "yes", 211 BENV_ENVIRONMENT | BENV_PKGLIST); 212 addbuildenv("ARCH", ArchitectureName, 213 BENV_ENVIRONMENT | BENV_PKGLIST); 214 215 #if 0 216 /* 217 * 218 */ 219 addbuildenv("OSTYPE", OperatingSystemName, 220 BENV_ENVIRONMENT | BENV_PKGLIST); 221 addbuildenv("MACHTYPE", MachineName, 222 BENV_ENVIRONMENT | BENV_PKGLIST); 223 #endif 224 /* 225 * SlowStart auto adjust. We nominally start with 1 job and increase 226 * it to the maximum every 5 seconds to give various dynamic management 227 * parameters time to stabilize. 228 * 229 * This can take a while on a many-core box with a high jobs setting, 230 * so increase the initial jobs in such cases. 231 */ 232 if (SlowStartOpt > MaxWorkers) 233 SlowStartOpt = MaxWorkers; 234 if (SlowStartOpt < 0) { 235 if (MaxWorkers < 16) 236 SlowStartOpt = 1; 237 else 238 SlowStartOpt = MaxWorkers / 4; 239 } 240 241 /* 242 * Special directive for when dsynth execs itself to manage 243 * a worker chroot. 244 */ 245 if (isworker) { 246 WorkerProcess(ac, av); 247 exit(0); 248 } 249 250 /* 251 * Build initialization and directive handling 252 */ 253 DoInitBuild(-1); 254 255 /* 256 * Directives that use the configuration but are not interlocked 257 * against a running dsynth. 258 */ 259 if (strcmp(av[0], "monitor") == 0) { 260 char *spath; 261 char *lpath; 262 263 if (ac == 1) { 264 asprintf(&spath, "%s/%s", StatsBase, STATS_FILE); 265 asprintf(&lpath, "%s/%s", StatsBase, STATS_LOCKFILE); 266 MonitorDirective(spath, lpath); 267 free(spath); 268 free(lpath); 269 } else { 270 MonitorDirective(av[1], NULL); 271 } 272 exit(0); 273 /* NOT REACHED */ 274 } 275 276 /* 277 * Front-end exec (not a WORKER exec), normal startup. We have 278 * the configuration so the first thing we need to do is check 279 * the lock file. 280 */ 281 { 282 char *lkpath; 283 int fd; 284 285 asprintf(&lkpath, "%s/.lock", BuildBase); 286 fd = open(lkpath, O_RDWR | O_CREAT | O_CLOEXEC, 0644); 287 if (fd < 0) 288 dfatal_errno("Unable to create %s", lkpath); 289 if (flock(fd, LOCK_EX | LOCK_NB) < 0) { 290 dfatal("Another dsynth is using %s, exiting", 291 BuildBase); 292 } 293 /* leave descriptor open */ 294 } 295 296 if (strcmp(av[0], "debug") == 0) { 297 DoCleanBuild(1); 298 OptimizeEnv(); 299 pkgs = ParsePackageList(ac - 1, av + 1, 1); 300 RemovePackages(pkgs); 301 DoBuild(pkgs); 302 } else if (strcmp(av[0], "status") == 0) { 303 OptimizeEnv(); 304 if (ac - 1) 305 pkgs = ParsePackageList(ac - 1, av + 1, 0); 306 else 307 pkgs = GetLocalPackageList(); 308 DoStatus(pkgs); 309 } else if (strcmp(av[0], "cleanup") == 0) { 310 DoCleanBuild(0); 311 } else if (strcmp(av[0], "configure") == 0) { 312 DoCleanBuild(0); 313 DoConfigure(); 314 } else if (strcmp(av[0], "upgrade-system") == 0) { 315 DoCleanBuild(1); 316 OptimizeEnv(); 317 pkgs = GetLocalPackageList(); 318 DoBuild(pkgs); 319 DoRebuildRepo(0); 320 DoUpgradePkgs(pkgs, 0); 321 } else if (strcmp(av[0], "prepare-system") == 0) { 322 DeleteObsoletePkgs = 1; 323 DoCleanBuild(1); 324 OptimizeEnv(); 325 pkgs = GetLocalPackageList(); 326 DoBuild(pkgs); 327 DoRebuildRepo(0); 328 } else if (strcmp(av[0], "rebuild-repository") == 0) { 329 OptimizeEnv(); 330 DoRebuildRepo(0); 331 } else if (strcmp(av[0], "purge-distfiles") == 0) { 332 OptimizeEnv(); 333 pkgs = GetFullPackageList(); 334 PurgeDistfiles(pkgs); 335 } else if (strcmp(av[0], "reset-db") == 0) { 336 char *dbmpath; 337 338 asprintf(&dbmpath, "%s/ports_crc.db", BuildBase); 339 remove(dbmpath); 340 printf("%s reset, will be regenerated on next build\n", 341 dbmpath); 342 free(dbmpath); 343 } else if (strcmp(av[0], "status-everything") == 0) { 344 OptimizeEnv(); 345 pkgs = GetFullPackageList(); 346 DoStatus(pkgs); 347 } else if (strcmp(av[0], "everything") == 0) { 348 if (WorkerProcFlags & WORKER_PROC_DEVELOPER) 349 WorkerProcFlags |= WORKER_PROC_CHECK_PLIST; 350 MaskProbeAbort = 1; 351 DeleteObsoletePkgs = 1; 352 DoCleanBuild(1); 353 OptimizeEnv(); 354 pkgs = GetFullPackageList(); 355 DoBuild(pkgs); 356 DoRebuildRepo(1); 357 } else if (strcmp(av[0], "build") == 0) { 358 DoCleanBuild(1); 359 OptimizeEnv(); 360 pkgs = ParsePackageList(ac - 1, av + 1, 0); 361 DoBuild(pkgs); 362 DoRebuildRepo(1); 363 DoUpgradePkgs(pkgs, 1); 364 } else if (strcmp(av[0], "just-build") == 0) { 365 DoCleanBuild(1); 366 OptimizeEnv(); 367 pkgs = ParsePackageList(ac - 1, av + 1, 0); 368 DoBuild(pkgs); 369 } else if (strcmp(av[0], "install") == 0) { 370 DoCleanBuild(1); 371 OptimizeEnv(); 372 pkgs = ParsePackageList(ac - 1, av + 1, 0); 373 DoBuild(pkgs); 374 DoRebuildRepo(0); 375 DoUpgradePkgs(pkgs, 0); 376 } else if (strcmp(av[0], "force") == 0) { 377 DoCleanBuild(1); 378 OptimizeEnv(); 379 pkgs = ParsePackageList(ac - 1, av + 1, 0); 380 RemovePackages(pkgs); 381 DoBuild(pkgs); 382 DoRebuildRepo(1); 383 DoUpgradePkgs(pkgs, 1); 384 } else if (strcmp(av[0], "test") == 0) { 385 WorkerProcFlags |= WORKER_PROC_CHECK_PLIST | WORKER_PROC_INSTALL | WORKER_PROC_DEINSTALL; 386 DoCleanBuild(1); 387 OptimizeEnv(); 388 pkgs = ParsePackageList(ac - 1, av + 1, 0); 389 RemovePackages(pkgs); 390 WorkerProcFlags |= WORKER_PROC_DEVELOPER; 391 DoBuild(pkgs); 392 } else { 393 fprintf(stderr, "Unknown directive '%s'\n", av[0]); 394 usage(2); 395 } 396 return 0; 397 } 398 399 static void 400 DoInit(void) 401 { 402 struct stat st; 403 char *path; 404 FILE *fp; 405 406 if (stat(ConfigBase1, &st) == 0) { 407 dfatal("init will not overwrite %s", ConfigBase1); 408 } 409 if (stat(ConfigBase2, &st) == 0) { 410 dfatal("init will not create %s if %s exists", 411 ConfigBase2, ConfigBase1); 412 } 413 if (mkdir(ConfigBase1, 0755) < 0) 414 dfatal_errno("Unable to mkdir %s", ConfigBase1); 415 416 asprintf(&path, "%s/dsynth.ini", ConfigBase1); 417 fp = fopen(path, "w"); 418 dassert_errno(fp, "Unable to create %s", path); 419 fprintf(fp, "%s", 420 "; This Synth configuration file is automatically generated\n" 421 "; Take care when hand editing!\n" 422 "\n" 423 "[Global Configuration]\n" 424 "profile_selected= LiveSystem\n" 425 "\n" 426 "[LiveSystem]\n" 427 "Operating_system= DragonFly\n" 428 "Directory_packages= /build/synth/live_packages\n" 429 "Directory_repository= /build/synth/live_packages/All\n" 430 "Directory_portsdir= /build/synth/dports\n" 431 "Directory_options= /build/synth/options\n" 432 "Directory_distfiles= /build/synth/distfiles\n" 433 "Directory_buildbase= /build/synth/build\n" 434 "Directory_logs= /build/synth/logs\n" 435 "Directory_ccache= disabled\n" 436 "Directory_system= /\n" 437 "Package_suffix= .txz\n" 438 "Number_of_builders= 0\n" 439 "Max_jobs_per_builder= 0\n" 440 "Tmpfs_workdir= true\n" 441 "Tmpfs_localbase= true\n" 442 "Display_with_ncurses= true\n" 443 "leverage_prebuilt= false\n" 444 "\n"); 445 if (fclose(fp)) 446 dfatal_errno("Unable to write to %s\n", ConfigBase1); 447 free(path); 448 449 asprintf(&path, "%s/LiveSystem-make.conf", ConfigBase1); 450 fp = fopen(path, "w"); 451 dassert_errno(fp, "Unable to create %s", path); 452 fprintf(fp, "%s", 453 "#\n" 454 "# Various dports options that might be of interest\n" 455 "#\n" 456 "#LICENSES_ACCEPTED= NONE\n" 457 "#DISABLE_LICENSES= yes\n" 458 "#DEFAULT_VERSIONS= ssl=openssl\n" 459 "#FORCE_PACKAGE= yes\n" 460 "#DPORTS_BUILDER= yes\n" 461 "#\n" 462 "# Turn these on to generate debug binaries. However, these\n" 463 "# options will seriously bloat memory use and storage use,\n" 464 "# do not use lightly\n" 465 "#\n" 466 "#STRIP=\n" 467 "#WITH_DEBUG=yes\n" 468 ); 469 if (fclose(fp)) 470 dfatal_errno("Unable to write to %s\n", ConfigBase1); 471 free(path); 472 } 473 474 __dead2 static void 475 usage(int ecode) 476 { 477 if (ecode == 2) { 478 fprintf(stderr, "Run 'dsynth help' for usage\n"); 479 exit(1); 480 } 481 482 fprintf(stderr, 483 "dsynth [options] directive\n" 484 " -d - Debug verbosity (-dd disables ncurses)\n" 485 " -h - Display this screen and exit\n" 486 " -m gb - Load management based on pkgdep memory\n" 487 " -p profile - Override profile selected in dsynth.ini\n" 488 " -s n - Set initial DynamicMaxWorkers\n" 489 " -v - Print version info and exit\n" 490 " -x - Do not rebuild packages with dependencies\n" 491 " which require rebuilding\n" 492 " -xx - Do not rebuild packages whos dports trees\n" 493 " change\n" 494 " -y - Automatically answer yes to dsynth questions\n" 495 " -D - Enable DEVELOPER mode\n" 496 " -P - Include the check-plist stage\n" 497 " -S - Disable ncurses\n" 498 " -N - Do not nice-up sub-processes (else nice +10)\n" 499 "\n" 500 " init - Initialize /etc/dsynth\n" 501 " status - Dry-run of 'upgrade-system'\n" 502 " cleanup - Clean-up mounts\n" 503 " configure - Bring up configuration menu\n" 504 " upgrade-system - Incremental build and upgrade using pkg list\n" 505 " from local system, then upgrade the local\n" 506 " system.\n" 507 " prepare-system - 'upgrade-system' but stops after building\n" 508 " rebuild-repository - Rebuild database files for current repository\n" 509 " purge-distfiles - Delete obsolete source distribution files\n" 510 " reset-db - Delete ports_crc.db, regenerate next build\n" 511 " status-everything - Dry-run of 'everything'\n" 512 " everything - Build entire dports tree and repo database\n" 513 " (-D everything infers -P)\n" 514 " version - Print version info and exit\n" 515 " help - Display this screen and exit\n" 516 " status [ports] - Dry-run of 'build' with given list\n" 517 " build [ports] - Incrementally build dports based on the given\n" 518 " list, but asks before updating the repo\n" 519 " database and system\n" 520 " just-build [ports] - 'build' but skips post-build steps\n" 521 " install [ports] - 'build' but upgrades system without asking\n" 522 " force [ports] - 'build' but deletes existing packages first\n" 523 " test [ports] - 'build' w/DEVELOPER=yes and pre-deletes pkgs\n" 524 " (also infers -P)\n" 525 " debug [ports] - like 'test' but leaves mounts intact\n" 526 " monitor [datfile] - Monitor a running dsynth\n" 527 "\n" 528 " [ports] is a space-delimited list of origins, e.g. editors/joe. It\n" 529 " may also be a path to a file containing one origin per line.\n" 530 ); 531 532 exit(ecode); 533 } 534