1 /* 2 * Copyright (c) 2019 The DragonFly Project. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Matthew Dillon <dillon@backplane.com> 6 * 7 * This code uses concepts and configuration based on 'synth', by 8 * John R. Marino <draco@marino.st>, which was written in ada. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in 18 * the documentation and/or other materials provided with the 19 * distribution. 20 * 3. Neither the name of The DragonFly Project nor the names of its 21 * contributors may be used to endorse or promote products derived 22 * from this software without specific, prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 25 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 26 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 27 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 28 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 29 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 30 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 31 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 32 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 33 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 34 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 35 * SUCH DAMAGE. 36 */ 37 38 #include "dsynth.h" 39 40 static void DoInit(void); 41 static void usage(int ecode) __dead2; 42 43 int OverridePkgDeleteOpt; 44 int FetchOnlyOpt; 45 int YesOpt; 46 int DebugOpt; 47 int MaskProbeAbort; 48 int ColorOpt = 1; 49 int NullStdinOpt = 1; 50 int SlowStartOpt = -1; 51 long PkgDepMemoryTarget; 52 long PkgDepScaleTarget = 100; /* 1.00 */ 53 char *DSynthExecPath; 54 char *ProfileOverrideOpt; 55 int NiceOpt = 10; 56 57 int 58 main(int ac, char **av) 59 { 60 pkg_t *pkgs; 61 int isworker; 62 int c; 63 int sopt; 64 65 /* 66 * Get our exec path so we can self-exec clean WORKER 67 * processes. 68 */ 69 { 70 size_t len; 71 const int name[] = { 72 CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1, 73 }; 74 if (sysctl(name, 4, NULL, &len, NULL, 0) < 0) 75 dfatal_errno("Cannot get binary path"); 76 DSynthExecPath = malloc(len + 1); 77 if (sysctl(name, 4, DSynthExecPath, &len, NULL, 0) < 0) 78 dfatal_errno("Cannot get binary path"); 79 DSynthExecPath[len] = 0; 80 } 81 82 /* 83 * Override profile in dsynth.ini (can be further overridden 84 * with the -p profile option). 85 */ 86 ProfileOverrideOpt = getenv("DSYNTH_PROFILE"); 87 88 /* 89 * Process options and make sure the directive is present 90 */ 91 sopt = 0; 92 while ((c = getopt(ac, av, "dhm:p:vxys:DPM:NS")) != -1) { 93 switch(c) { 94 case 'x': 95 ++OverridePkgDeleteOpt; 96 break; 97 case 'y': 98 ++YesOpt; 99 break; 100 case 'D': 101 WorkerProcFlags |= WORKER_PROC_DEVELOPER; 102 break; 103 case 'P': 104 WorkerProcFlags |= WORKER_PROC_CHECK_PLIST; 105 break; 106 case 'S': 107 UseNCurses = 0; 108 if (++sopt == 2) 109 ColorOpt = 0; 110 break; 111 case 'N': 112 NiceOpt = 0; 113 break; 114 case 'd': 115 ++DebugOpt; 116 if (DebugOpt >= 2) 117 UseNCurses = 0; 118 break; 119 case 'h': 120 usage(0); 121 /* NOT REACHED */ 122 exit(0); 123 case 'v': 124 printf("dsynth %s\n", DSYNTH_VERSION); 125 exit(0); 126 case 's': 127 /* 128 * Start with N jobs, increasing to the configured 129 * maximum slowly. 0 to disable (starts with the 130 * full count). 131 */ 132 SlowStartOpt = strtol(optarg, NULL, 0); 133 break; 134 case 'm': 135 PkgDepMemoryTarget = strtoul(optarg, NULL, 0); 136 PkgDepMemoryTarget *= ONEGB; 137 break; 138 case 'M': 139 PkgDepScaleTarget = strtod(optarg, NULL) * 100; 140 if (PkgDepScaleTarget < 1) 141 PkgDepScaleTarget = 1; 142 if (PkgDepScaleTarget > 9900) 143 PkgDepScaleTarget = 9900; 144 break; 145 case 'p': 146 ProfileOverrideOpt = optarg; 147 break; 148 default: 149 fprintf(stderr, "Unknown option: %c\n", c); 150 usage(2); 151 /* NOT REACHED */ 152 break; 153 } 154 } 155 ac -= optind; 156 av += optind; 157 pkgs = NULL; 158 if (ac < 1) { 159 fprintf(stderr, "Missing directive\n"); 160 usage(2); 161 /* NOT REACHED */ 162 } 163 164 /* 165 * Directives which do not require a working configuration 166 */ 167 if (strcmp(av[0], "init") == 0) { 168 DoInit(); 169 exit(0); 170 /* NOT REACHED */ 171 } 172 if (strcmp(av[0], "help") == 0) { 173 usage(0); 174 exit(0); 175 /* NOT REACHED */ 176 } 177 if (strcmp(av[0], "version") == 0) { 178 printf("dsynth %s\n", DSYNTH_VERSION); 179 exit(0); 180 /* NOT REACHED */ 181 } 182 183 /* 184 * Preconfiguration. 185 */ 186 if (strcmp(av[0], "WORKER") == 0) { 187 isworker = 1; 188 } else { 189 isworker = 0; 190 } 191 192 signal(SIGPIPE, SIG_IGN); 193 ParseConfiguration(isworker); 194 195 /* 196 * Setup some environment for bulk operations (pkglist scan). 197 * These are not used by the builder (the builder will replicate 198 * all of these). 199 * 200 * NOTE: PKG_SUFX - pkg versions older than 1.17 201 * PKG_COMPRESSION_FORMAT - pkg versions >= 1.17 202 */ 203 addbuildenv("PORTSDIR", DPortsPath, 204 BENV_ENVIRONMENT | BENV_PKGLIST); 205 addbuildenv("BATCH", "yes", 206 BENV_ENVIRONMENT | BENV_PKGLIST); 207 addbuildenv("PKG_COMPRESSION_FORMAT", UsePkgSufx, 208 BENV_ENVIRONMENT | BENV_PKGLIST); 209 addbuildenv("PKG_SUFX", UsePkgSufx, 210 BENV_ENVIRONMENT | BENV_PKGLIST); 211 addbuildenv("PACKAGE_BUILDING", "yes", 212 BENV_ENVIRONMENT | BENV_PKGLIST); 213 addbuildenv("ARCH", ArchitectureName, 214 BENV_ENVIRONMENT | BENV_PKGLIST); 215 216 #if 0 217 /* 218 * 219 */ 220 addbuildenv("OSTYPE", OperatingSystemName, 221 BENV_ENVIRONMENT | BENV_PKGLIST); 222 addbuildenv("MACHTYPE", MachineName, 223 BENV_ENVIRONMENT | BENV_PKGLIST); 224 #endif 225 /* 226 * SlowStart auto adjust. We nominally start with 1 job and increase 227 * it to the maximum every 5 seconds to give various dynamic management 228 * parameters time to stabilize. 229 * 230 * This can take a while on a many-core box with a high jobs setting, 231 * so increase the initial jobs in such cases. 232 */ 233 if (SlowStartOpt > MaxWorkers) 234 SlowStartOpt = MaxWorkers; 235 if (SlowStartOpt < 0) { 236 if (MaxWorkers < 16) 237 SlowStartOpt = 1; 238 else 239 SlowStartOpt = MaxWorkers / 4; 240 } 241 242 /* 243 * Special directive for when dsynth execs itself to manage 244 * a worker chroot. 245 */ 246 if (isworker) { 247 WorkerProcess(ac, av); 248 exit(0); 249 } 250 251 /* 252 * Build initialization and directive handling 253 */ 254 DoInitBuild(-1); 255 256 /* 257 * Directives that use the configuration but are not interlocked 258 * against a running dsynth. 259 */ 260 if (strcmp(av[0], "monitor") == 0) { 261 char *spath; 262 char *lpath; 263 264 if (ac == 1) { 265 asprintf(&spath, "%s/%s", StatsBase, STATS_FILE); 266 asprintf(&lpath, "%s/%s", StatsBase, STATS_LOCKFILE); 267 MonitorDirective(spath, lpath); 268 free(spath); 269 free(lpath); 270 } else { 271 MonitorDirective(av[1], NULL); 272 } 273 exit(0); 274 /* NOT REACHED */ 275 } 276 277 /* 278 * Front-end exec (not a WORKER exec), normal startup. We have 279 * the configuration so the first thing we need to do is check 280 * the lock file. 281 */ 282 { 283 char *lkpath; 284 int fd; 285 286 asprintf(&lkpath, "%s/.lock", BuildBase); 287 fd = open(lkpath, O_RDWR | O_CREAT | O_CLOEXEC, 0644); 288 if (fd < 0) 289 dfatal_errno("Unable to create %s", lkpath); 290 if (flock(fd, LOCK_EX | LOCK_NB) < 0) { 291 dfatal("Another dsynth is using %s, exiting", 292 BuildBase); 293 } 294 /* leave descriptor open */ 295 } 296 297 if (strcmp(av[0], "debug") == 0) { 298 DoCleanBuild(1); 299 OptimizeEnv(); 300 pkgs = ParsePackageList(ac - 1, av + 1, 1); 301 RemovePackages(pkgs); 302 DoBuild(pkgs); 303 } else if (strcmp(av[0], "status") == 0) { 304 OptimizeEnv(); 305 if (ac - 1) 306 pkgs = ParsePackageList(ac - 1, av + 1, 0); 307 else 308 pkgs = GetLocalPackageList(); 309 DoStatus(pkgs); 310 } else if (strcmp(av[0], "cleanup") == 0) { 311 DoCleanBuild(0); 312 } else if (strcmp(av[0], "configure") == 0) { 313 DoCleanBuild(0); 314 DoConfigure(); 315 } else if (strcmp(av[0], "fetch-only") == 0) { 316 if (SlowStartOpt == -1) 317 SlowStartOpt = 999; 318 if (PkgDepScaleTarget == 100) 319 PkgDepScaleTarget = 999; 320 ++FetchOnlyOpt; 321 ++YesOpt; 322 WorkerProcFlags |= WORKER_PROC_FETCHONLY; 323 DoCleanBuild(1); 324 OptimizeEnv(); 325 if (ac == 2 && strcmp(av[1], "everything") == 0) { 326 MaskProbeAbort = 1; 327 pkgs = GetFullPackageList(); 328 } else { 329 pkgs = ParsePackageList(ac - 1, av + 1, 0); 330 } 331 DoBuild(pkgs); 332 } else if (strcmp(av[0], "upgrade-system") == 0) { 333 DoCleanBuild(1); 334 OptimizeEnv(); 335 pkgs = GetLocalPackageList(); 336 DoBuild(pkgs); 337 DoRebuildRepo(0); 338 DoUpgradePkgs(pkgs, 0); 339 } else if (strcmp(av[0], "prepare-system") == 0) { 340 DeleteObsoletePkgs = 1; 341 DoCleanBuild(1); 342 OptimizeEnv(); 343 pkgs = GetLocalPackageList(); 344 DoBuild(pkgs); 345 DoRebuildRepo(0); 346 } else if (strcmp(av[0], "rebuild-repository") == 0) { 347 OptimizeEnv(); 348 DoRebuildRepo(0); 349 } else if (strcmp(av[0], "purge-distfiles") == 0) { 350 OptimizeEnv(); 351 pkgs = GetFullPackageList(); 352 PurgeDistfiles(pkgs); 353 } else if (strcmp(av[0], "reset-db") == 0) { 354 char *dbmpath; 355 356 asprintf(&dbmpath, "%s/ports_crc.db", BuildBase); 357 remove(dbmpath); 358 printf("%s reset, will be regenerated on next build\n", 359 dbmpath); 360 free(dbmpath); 361 } else if (strcmp(av[0], "status-everything") == 0) { 362 OptimizeEnv(); 363 pkgs = GetFullPackageList(); 364 DoStatus(pkgs); 365 } else if (strcmp(av[0], "everything") == 0) { 366 if (WorkerProcFlags & WORKER_PROC_DEVELOPER) 367 WorkerProcFlags |= WORKER_PROC_CHECK_PLIST; 368 MaskProbeAbort = 1; 369 DeleteObsoletePkgs = 1; 370 DoCleanBuild(1); 371 OptimizeEnv(); 372 pkgs = GetFullPackageList(); 373 DoBuild(pkgs); 374 DoRebuildRepo(1); 375 } else if (strcmp(av[0], "build") == 0) { 376 DoCleanBuild(1); 377 OptimizeEnv(); 378 pkgs = ParsePackageList(ac - 1, av + 1, 0); 379 DoBuild(pkgs); 380 DoRebuildRepo(1); 381 DoUpgradePkgs(pkgs, 1); 382 } else if (strcmp(av[0], "just-build") == 0) { 383 DoCleanBuild(1); 384 OptimizeEnv(); 385 pkgs = ParsePackageList(ac - 1, av + 1, 0); 386 DoBuild(pkgs); 387 } else if (strcmp(av[0], "install") == 0) { 388 DoCleanBuild(1); 389 OptimizeEnv(); 390 pkgs = ParsePackageList(ac - 1, av + 1, 0); 391 DoBuild(pkgs); 392 DoRebuildRepo(0); 393 DoUpgradePkgs(pkgs, 0); 394 } else if (strcmp(av[0], "force") == 0) { 395 DoCleanBuild(1); 396 OptimizeEnv(); 397 pkgs = ParsePackageList(ac - 1, av + 1, 0); 398 RemovePackages(pkgs); 399 DoBuild(pkgs); 400 DoRebuildRepo(1); 401 DoUpgradePkgs(pkgs, 1); 402 } else if (strcmp(av[0], "test") == 0) { 403 WorkerProcFlags |= WORKER_PROC_CHECK_PLIST | WORKER_PROC_INSTALL | WORKER_PROC_DEINSTALL; 404 DoCleanBuild(1); 405 OptimizeEnv(); 406 pkgs = ParsePackageList(ac - 1, av + 1, 0); 407 RemovePackages(pkgs); 408 WorkerProcFlags |= WORKER_PROC_DEVELOPER; 409 DoBuild(pkgs); 410 } else { 411 fprintf(stderr, "Unknown directive '%s'\n", av[0]); 412 usage(2); 413 } 414 return 0; 415 } 416 417 static void 418 DoInit(void) 419 { 420 struct stat st; 421 char *path; 422 FILE *fp; 423 424 if (stat(ConfigBase1, &st) == 0) { 425 dfatal("init will not overwrite %s", ConfigBase1); 426 } 427 if (stat(ConfigBase2, &st) == 0) { 428 dfatal("init will not create %s if %s exists", 429 ConfigBase2, ConfigBase1); 430 } 431 if (mkdir(ConfigBase1, 0755) < 0) 432 dfatal_errno("Unable to mkdir %s", ConfigBase1); 433 434 asprintf(&path, "%s/dsynth.ini", ConfigBase1); 435 fp = fopen(path, "w"); 436 dassert_errno(fp, "Unable to create %s", path); 437 fprintf(fp, "%s", 438 "; This Synth configuration file is automatically generated\n" 439 "; Take care when hand editing!\n" 440 "\n" 441 "[Global Configuration]\n" 442 "profile_selected= LiveSystem\n" 443 "\n" 444 "[LiveSystem]\n" 445 "Operating_system= DragonFly\n" 446 "Directory_packages= /build/synth/live_packages\n" 447 "Directory_repository= /build/synth/live_packages/All\n" 448 "Directory_portsdir= /build/synth/dports\n" 449 "Directory_options= /build/synth/options\n" 450 "Directory_distfiles= /build/synth/distfiles\n" 451 "Directory_buildbase= /build/synth/build\n" 452 "Directory_logs= /build/synth/logs\n" 453 "Directory_ccache= disabled\n" 454 "Directory_system= /\n" 455 "Package_suffix= .txz\n" 456 "Number_of_builders= 0\n" 457 "Max_jobs_per_builder= 0\n" 458 "Tmpfs_workdir= true\n" 459 "Tmpfs_localbase= true\n" 460 "Display_with_ncurses= true\n" 461 "leverage_prebuilt= false\n" 462 "\n"); 463 if (fclose(fp)) 464 dfatal_errno("Unable to write to %s\n", ConfigBase1); 465 free(path); 466 467 asprintf(&path, "%s/LiveSystem-make.conf", ConfigBase1); 468 fp = fopen(path, "w"); 469 dassert_errno(fp, "Unable to create %s", path); 470 fprintf(fp, "%s", 471 "#\n" 472 "# Various dports options that might be of interest\n" 473 "#\n" 474 "#LICENSES_ACCEPTED= NONE\n" 475 "#DISABLE_LICENSES= yes\n" 476 "#DEFAULT_VERSIONS= ssl=openssl\n" 477 "#FORCE_PACKAGE= yes\n" 478 "#DPORTS_BUILDER= yes\n" 479 "#\n" 480 "# Turn these on to generate debug binaries. However, these\n" 481 "# options will seriously bloat memory use and storage use,\n" 482 "# do not use lightly\n" 483 "#\n" 484 "#STRIP=\n" 485 "#WITH_DEBUG=yes\n" 486 ); 487 if (fclose(fp)) 488 dfatal_errno("Unable to write to %s\n", ConfigBase1); 489 free(path); 490 } 491 492 __dead2 static void 493 usage(int ecode) 494 { 495 if (ecode == 2) { 496 fprintf(stderr, "Run 'dsynth help' for usage\n"); 497 exit(1); 498 } 499 500 fprintf(stderr, 501 "dsynth [options] directive\n" 502 " -d - Debug verbosity (-dd disables ncurses)\n" 503 " -h - Display this screen and exit\n" 504 " -m gb - Load management based on pkgdep memory\n" 505 " -p profile - Override profile selected in dsynth.ini\n" 506 " -s n - Set initial DynamicMaxWorkers\n" 507 " -v - Print version info and exit\n" 508 " -x - Do not rebuild packages with dependencies\n" 509 " which require rebuilding\n" 510 " -xx - Do not rebuild packages whos dports trees\n" 511 " change\n" 512 " -y - Automatically answer yes to dsynth questions\n" 513 " -D - Enable DEVELOPER mode\n" 514 " -P - Include the check-plist stage\n" 515 " -S - Disable ncurses\n" 516 " -N - Do not nice-up sub-processes (else nice +10)\n" 517 "\n" 518 " init - Initialize /etc/dsynth\n" 519 " status - Dry-run of 'upgrade-system'\n" 520 " cleanup - Clean-up mounts\n" 521 " configure - Bring up configuration menu\n" 522 " upgrade-system - Incremental build and upgrade using pkg list\n" 523 " from local system, then upgrade the local\n" 524 " system.\n" 525 " prepare-system - 'upgrade-system' but stops after building\n" 526 " rebuild-repository - Rebuild database files for current repository\n" 527 " purge-distfiles - Delete obsolete source distribution files\n" 528 " reset-db - Delete ports_crc.db, regenerate next build\n" 529 " status-everything - Dry-run of 'everything'\n" 530 " everything - Build entire dports tree and repo database\n" 531 " (-D everything infers -P)\n" 532 " version - Print version info and exit\n" 533 " help - Display this screen and exit\n" 534 " status [ports] - Dry-run of 'build' with given list\n" 535 " build [ports] - Incrementally build dports based on the given\n" 536 " list, but asks before updating the repo\n" 537 " database and system\n" 538 " just-build [ports] - 'build' but skips post-build steps\n" 539 " install [ports] - 'build' but upgrades system without asking\n" 540 " force [ports] - 'build' but deletes existing packages first\n" 541 " test [ports] - 'build' w/DEVELOPER=yes and pre-deletes pkgs\n" 542 " (also infers -P)\n" 543 " debug [ports] - like 'test' but leaves mounts intact\n" 544 " fetch-only [ports] - Fetch src dists only ('everything' ok)\n" 545 " monitor [datfile] - Monitor a running dsynth\n" 546 "\n" 547 " [ports] is a space-delimited list of origins, e.g. editors/joe. It\n" 548 " may also be a path to a file containing one origin per line.\n" 549 ); 550 551 exit(ecode); 552 } 553