1 /* 2 * Copyright (c) 2019 The DragonFly Project. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Matthew Dillon <dillon@backplane.com> 6 * 7 * This code uses concepts and configuration based on 'synth', by 8 * John R. Marino <draco@marino.st>, which was written in ada. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in 18 * the documentation and/or other materials provided with the 19 * distribution. 20 * 3. Neither the name of The DragonFly Project nor the names of its 21 * contributors may be used to endorse or promote products derived 22 * from this software without specific, prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 25 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 26 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 27 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 28 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 29 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 30 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 31 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 32 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 33 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 34 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 35 * SUCH DAMAGE. 36 */ 37 38 #include "dsynth.h" 39 40 int UseCCache; 41 int UseUsrSrc; 42 int UseTmpfs; 43 int NumCores = 1; 44 int MaxBulk = 8; 45 int MaxWorkers = 8; 46 int MaxJobs = 8; 47 int UseTmpfsWork = 1; 48 int UseTmpfsBase = 1; 49 int UseNCurses = -1; /* indicates default operation (enabled) */ 50 int LeveragePrebuilt = 0; 51 int WorkerProcFlags = 0; 52 long PhysMem; 53 const char *OperatingSystemName = "Unknown"; /* e.g. "DragonFly" */ 54 const char *ArchitectureName = "unknown"; /* e.g. "x86_64" */ 55 const char *MachineName = "unknown"; /* e.g. "x86_64" */ 56 const char *VersionName = "unknown"; /* e.g. "DragonFly 5.7-SYNTH" */ 57 const char *VersionOnlyName = "unknown"; /* e.g. "5.7-SYNTH" */ 58 const char *VersionFromParamHeader = "unknown"; /* e.g. "500704" */ 59 const char *ReleaseName = "unknown"; /* e.g. "5.7" */ 60 const char *DPortsPath = "/usr/dports"; 61 const char *CCachePath = DISABLED_STR; 62 const char *PackagesPath = "/build/synth/live_packages"; 63 const char *RepositoryPath = "/build/synth/live_packages/All"; 64 const char *OptionsPath = "/build/synth/options"; 65 const char *DistFilesPath = "/build/synth/distfiles"; 66 const char *BuildBase = "/build/synth/build"; 67 const char *LogsPath = "/build/synth/logs"; 68 const char *SystemPath = "/"; 69 const char *ProfileLabel = "[LiveSystem]"; /* with the brackets */ 70 const char *Profile = "LiveSystem"; /* without the brackets */ 71 72 /* 73 * Hooks are scripts in ConfigBase 74 */ 75 int UsingHooks; 76 const char *HookRunStart; 77 const char *HookRunEnd; 78 const char *HookPkgSuccess; 79 const char *HookPkgFailure; 80 const char *HookPkgIgnored; 81 const char *HookPkgSkipped; 82 83 const char *ConfigBase; /* The config base we found */ 84 const char *ConfigBase1 = "/etc/dsynth"; 85 const char *ConfigBase2 = "/usr/local/etc/dsynth"; 86 87 static void parseConfigFile(const char *path); 88 static void parseProfile(const char *cpath, const char *path); 89 static char *stripwhite(char *str); 90 static int truefalse(const char *str); 91 static char *dokernsysctl(int m1, int m2); 92 static void getElfInfo(const char *path); 93 static char *checkhook(const char *scriptname); 94 95 void 96 ParseConfiguration(int isworker) 97 { 98 struct stat st; 99 size_t len; 100 int reln; 101 char *synth_config; 102 char *buf; 103 104 /* 105 * Get the default OperatingSystemName, ArchitectureName, and 106 * ReleaseName. 107 */ 108 OperatingSystemName = dokernsysctl(CTL_KERN, KERN_OSTYPE); 109 ArchitectureName = dokernsysctl(CTL_HW, HW_MACHINE_ARCH); 110 MachineName = dokernsysctl(CTL_HW, HW_MACHINE); 111 ReleaseName = dokernsysctl(CTL_KERN, KERN_OSRELEASE); 112 113 /* 114 * Retrieve resource information from the system. Note that 115 * NumCores and PhysMem will also be used for dynamic load 116 * management. 117 */ 118 NumCores = 1; 119 len = sizeof(NumCores); 120 if (sysctlbyname("hw.ncpu", &NumCores, &len, NULL, 0) < 0) 121 dfatal_errno("Cannot get hw.ncpu"); 122 123 len = sizeof(PhysMem); 124 if (sysctlbyname("hw.physmem", &PhysMem, &len, NULL, 0) < 0) 125 dfatal_errno("Cannot get hw.physmem"); 126 if (PkgDepMemoryTarget == 0) 127 PkgDepMemoryTarget = PhysMem / 2; 128 129 /* 130 * Calculate nominal defaults. 131 */ 132 MaxBulk = NumCores; 133 MaxWorkers = MaxBulk / 2; 134 if (MaxWorkers > (int)((PhysMem + (ONEGB/2)) / ONEGB)) 135 MaxWorkers = (PhysMem + (ONEGB/2)) / ONEGB; 136 137 if (MaxBulk < 1) 138 MaxBulk = 1; 139 if (MaxWorkers < 1) 140 MaxWorkers = 1; 141 if (MaxJobs < 1) 142 MaxJobs = 1; 143 144 /* 145 * Configuration file must exist. Look for it in 146 * "/etc/dsynth" and "/usr/local/etc/dsynth". 147 */ 148 ConfigBase = ConfigBase1; 149 asprintf(&synth_config, "%s/dsynth.ini", ConfigBase1); 150 if (stat(synth_config, &st) < 0) { 151 ConfigBase = ConfigBase2; 152 asprintf(&synth_config, "%s/dsynth.ini", ConfigBase2); 153 } 154 155 if (stat(synth_config, &st) < 0) { 156 dfatal("Configuration file missing, " 157 "could not find %s/dsynth.ini or %s/dsynth.ini\n", 158 ConfigBase1, 159 ConfigBase2); 160 } 161 162 /* 163 * Check to see what hooks we have 164 */ 165 HookRunStart = checkhook("hook_run_start"); 166 HookRunEnd = checkhook("hook_run_end"); 167 HookPkgSuccess = checkhook("hook_pkg_success"); 168 HookPkgFailure = checkhook("hook_pkg_failure"); 169 HookPkgIgnored = checkhook("hook_pkg_ignored"); 170 HookPkgSkipped = checkhook("hook_pkg_skipped"); 171 172 /* 173 * Parse the configuration file(s). This may override some of 174 * the above defaults. 175 */ 176 parseConfigFile(synth_config); 177 parseProfile(synth_config, ProfileLabel); 178 179 /* 180 * Figure out whether CCache is configured. Also set UseUsrSrc 181 * if it exists under the system path. 182 * 183 * Not supported for the moment 184 */ 185 if (strcmp(CCachePath, "disabled") != 0) { 186 dfatal("Directory_ccache is not supported, please\n" 187 " set to 'disabled'\n"); 188 /* NOT REACHED */ 189 UseCCache = 1; 190 } 191 asprintf(&buf, "%s/usr/src/sys/Makefile", SystemPath); 192 if (stat(buf, &st) == 0) 193 UseUsrSrc = 1; 194 free(buf); 195 196 /* 197 * If this is a dsynth WORKER exec it handles a single slot, 198 * just set MaxWorkers to 1. 199 */ 200 if (isworker) 201 MaxWorkers = 1; 202 203 /* 204 * Final check 205 */ 206 if (stat(DPortsPath, &st) < 0) 207 dfatal("Directory missing: %s", DPortsPath); 208 if (stat(PackagesPath, &st) < 0) 209 dfatal("Directory missing: %s", PackagesPath); 210 if (stat(OptionsPath, &st) < 0) 211 dfatal("Directory missing: %s", OptionsPath); 212 if (stat(DistFilesPath, &st) < 0) 213 dfatal("Directory missing: %s", DistFilesPath); 214 if (stat(BuildBase, &st) < 0) 215 dfatal("Directory missing: %s", BuildBase); 216 if (stat(LogsPath, &st) < 0) 217 dfatal("Directory missing: %s", LogsPath); 218 if (stat(SystemPath, &st) < 0) 219 dfatal("Directory missing: %s", SystemPath); 220 if (UseCCache && stat(CCachePath, &st) < 0) 221 dfatal("Directory missing: %s", CCachePath); 222 223 /* 224 * Now use the SystemPath to retrieve file information from /bin/sh, 225 * and use this to set OperatingSystemName, ArchitectureName, 226 * MachineName, and ReleaseName. 227 * 228 * Since this method is used to build for specific releases, require 229 * that it succeed. 230 */ 231 asprintf(&buf, "%s/bin/sh", SystemPath); 232 getElfInfo(buf); 233 free(buf); 234 235 /* 236 * Calculate VersionName from OperatingSystemName and ReleaseName. 237 */ 238 if (strchr(ReleaseName, '-')) { 239 reln = strchr(ReleaseName, '-') - ReleaseName; 240 asprintf(&buf, "%s %*.*s-SYNTH", 241 OperatingSystemName, 242 reln, reln, ReleaseName); 243 VersionName = buf; 244 asprintf(&buf, "%*.*s-SYNTH", reln, reln, ReleaseName); 245 VersionOnlyName = buf; 246 } else { 247 asprintf(&buf, "%s %s-SYNTH", 248 OperatingSystemName, 249 ReleaseName); 250 asprintf(&buf, "%s-SYNTH", ReleaseName); 251 VersionOnlyName = buf; 252 } 253 254 /* 255 * Get __DragonFly_version from the system header via SystemPath 256 */ 257 { 258 char *ptr; 259 FILE *fp; 260 261 asprintf(&buf, "%s/usr/include/sys/param.h", SystemPath); 262 fp = fopen(buf, "r"); 263 if (fp == NULL) 264 dpanic_errno("Cannot open %s", buf); 265 while ((ptr = fgetln(fp, &len)) != NULL) { 266 if (len == 0 || ptr[len-1] != '\n') 267 continue; 268 ptr[len-1] = 0; 269 if (strncmp(ptr, "#define __DragonFly_version", 27)) 270 continue; 271 ptr += 27; 272 ptr = strtok(ptr, " \t\r\n"); 273 VersionFromParamHeader = strdup(ptr); 274 break; 275 } 276 fclose(fp); 277 } 278 279 /* 280 * If RepositoryPath is under PackagesPath, make sure it 281 * is created. 282 */ 283 if (strncmp(RepositoryPath, PackagesPath, strlen(PackagesPath)) == 0) { 284 if (stat(RepositoryPath, &st) < 0) { 285 if (mkdir(RepositoryPath, 0755) < 0) 286 dfatal_errno("Cannot mkdir '%s'", 287 RepositoryPath); 288 } 289 } 290 291 if (stat(RepositoryPath, &st) < 0) 292 dfatal("Directory missing: %s", RepositoryPath); 293 } 294 295 void 296 DoConfigure(void) 297 { 298 dfatal("Not Implemented"); 299 } 300 301 static void 302 parseConfigFile(const char *path) 303 { 304 char buf[1024]; 305 char copy[1024]; 306 FILE *fp; 307 char *l1; 308 char *l2; 309 size_t len; 310 int mode = -1; 311 int lineno = 0; 312 313 fp = fopen(path, "r"); 314 if (fp == NULL) { 315 ddprintf(0, "Warning: Config file %s does not exist\n", path); 316 return; 317 } 318 if (DebugOpt >= 2) 319 ddprintf(0, "ParseConfig %s\n", path); 320 321 while (fgets(buf, sizeof(buf), fp) != NULL) { 322 ++lineno; 323 len = strlen(buf); 324 if (len == 0 || buf[len-1] != '\n') 325 continue; 326 buf[--len] = 0; 327 328 /* 329 * Remove any trailing whitespace, ignore empty lines. 330 */ 331 while (len > 0 && isspace(buf[len-1])) 332 --len; 333 if (len == 0) 334 continue; 335 buf[len] = 0; 336 337 /* 338 * ignore comments 339 */ 340 if (buf[0] == ';' || buf[0] == '#') 341 continue; 342 if (buf[0] == '[') { 343 if (strcmp(buf, "[Global Configuration]") == 0) 344 mode = 0; /* parse global config */ 345 else if (strcmp(buf, ProfileLabel) == 0) 346 mode = 1; /* use profile */ 347 else 348 mode = -1; /* ignore profile */ 349 continue; 350 } 351 352 bcopy(buf, copy, len + 1); 353 354 l1 = strtok(copy, "="); 355 if (l1 == NULL) { 356 dfatal("Syntax error in config line %d: %s\n", 357 lineno, buf); 358 } 359 l2 = strtok(NULL, " \t\n"); 360 if (l2 == NULL) { 361 dfatal("Syntax error in config line %d: %s\n", 362 lineno, buf); 363 } 364 l1 = stripwhite(l1); 365 l2 = stripwhite(l2); 366 367 switch(mode) { 368 case 0: 369 /* 370 * Global Configuration 371 */ 372 if (strcmp(l1, "profile_selected") == 0) { 373 Profile = strdup(l2); 374 asprintf(&l2, "[%s]", l2); 375 ProfileLabel = l2; 376 } else { 377 dfatal("Unknown directive in config " 378 "line %d: %s\n", lineno, buf); 379 } 380 break; 381 case 1: 382 /* 383 * Selected Profile 384 */ 385 l2 = strdup(l2); 386 if (strcmp(l1, "Operating_system") == 0) { 387 OperatingSystemName = l2; 388 } else if (strcmp(l1, "Directory_packages") == 0) { 389 PackagesPath = l2; 390 } else if (strcmp(l1, "Directory_repository") == 0) { 391 RepositoryPath = l2; 392 } else if (strcmp(l1, "Directory_portsdir") == 0) { 393 DPortsPath = l2; 394 } else if (strcmp(l1, "Directory_options") == 0) { 395 OptionsPath = l2; 396 } else if (strcmp(l1, "Directory_distfiles") == 0) { 397 DistFilesPath = l2; 398 } else if (strcmp(l1, "Directory_buildbase") == 0) { 399 BuildBase = l2; 400 } else if (strcmp(l1, "Directory_logs") == 0) { 401 LogsPath = l2; 402 } else if (strcmp(l1, "Directory_ccache") == 0) { 403 CCachePath = l2; 404 } else if (strcmp(l1, "Directory_system") == 0) { 405 SystemPath = l2; 406 } else if (strcmp(l1, "Number_of_builders") == 0) { 407 MaxWorkers = strtol(l2, NULL, 0); 408 if (MaxWorkers == 0) 409 MaxWorkers = NumCores / 2 + 1; 410 else 411 if (MaxWorkers < 0 || MaxWorkers > MAXWORKERS) { 412 dfatal("Config: Number_of_builders " 413 "must range %d..%d", 414 1, MAXWORKERS); 415 } 416 free(l2); 417 } else if (strcmp(l1, "Max_jobs_per_builder") == 0) { 418 MaxJobs = strtol(l2, NULL, 0); 419 if (MaxJobs == 0) { 420 MaxJobs = NumCores; 421 } else 422 if (MaxJobs < 0 || MaxJobs > MAXJOBS) { 423 dfatal("Config: Max_jobs_per_builder " 424 "must range %d..%d", 425 1, MAXJOBS); 426 } 427 free(l2); 428 } else if (strcmp(l1, "Tmpfs_workdir") == 0) { 429 UseTmpfsWork = truefalse(l2); 430 dassert(UseTmpfsWork == 1, 431 "Config: Tmpfs_workdir must be " 432 "set to true, 'false' not supported"); 433 } else if (strcmp(l1, "Tmpfs_localbase") == 0) { 434 UseTmpfsBase = truefalse(l2); 435 dassert(UseTmpfsBase == 1, 436 "Config: Tmpfs_localbase must be " 437 "set to true, 'false' not supported"); 438 } else if (strcmp(l1, "Display_with_ncurses") == 0) { 439 if (UseNCurses == -1) 440 UseNCurses = truefalse(l2); 441 } else if (strcmp(l1, "leverage_prebuilt") == 0) { 442 LeveragePrebuilt = truefalse(l2); 443 dassert(LeveragePrebuilt == 0, 444 "Config: leverage_prebuilt not " 445 "supported and must be set to false"); 446 } else { 447 dfatal("Unknown directive in profile section " 448 "line %d: %s\n", lineno, buf); 449 } 450 break; 451 default: 452 /* 453 * Ignore unselected profile 454 */ 455 break; 456 } 457 } 458 fclose(fp); 459 } 460 461 /* 462 * NOTE: profile has brackets, e.g. "[LiveSystem]". 463 */ 464 static void 465 parseProfile(const char *cpath, const char *profile) 466 { 467 char buf[1024]; 468 char copy[1024]; 469 char *ppath; 470 FILE *fp; 471 char *l1; 472 char *l2; 473 int len; 474 int plen; 475 int lineno = 0; 476 477 len = strlen(cpath); 478 while (len && cpath[len-1] != '/') 479 --len; 480 if (len == 0) 481 ++len; 482 plen = strlen(profile); 483 ddassert(plen > 2 && profile[0] == '[' && profile[plen-1] == ']'); 484 485 asprintf(&ppath, "%*.*s%*.*s-make.conf", 486 len, len, cpath, plen - 2, plen - 2, profile + 1); 487 fp = fopen(ppath, "r"); 488 if (fp == NULL) { 489 ddprintf(0, "Warning: Profile %s does not exist\n", ppath); 490 return; 491 } 492 if (DebugOpt >= 2) 493 ddprintf(0, "ParseProfile %s\n", ppath); 494 free(ppath); 495 496 while (fgets(buf, sizeof(buf), fp) != NULL) { 497 ++lineno; 498 len = strlen(buf); 499 if (len == 0 || buf[len-1] != '\n') 500 continue; 501 buf[--len] = 0; 502 503 /* 504 * Remove any trailing whitespace, ignore empty lines. 505 */ 506 while (len > 0 && isspace(buf[len-1])) 507 --len; 508 buf[len] = 0; 509 stripwhite(buf); 510 len = strlen(buf); 511 if (len == 0) 512 continue; 513 514 /* 515 * Ignore comments. 516 */ 517 if (buf[0] == ';' || buf[0] == '#') 518 continue; 519 520 bcopy(buf, copy, len + 1); 521 l1 = strtok(copy, "="); 522 if (l1 == NULL) { 523 dfatal("Syntax error in profile line %d: %s\n", 524 lineno, buf); 525 } 526 l2 = strtok(NULL, " \t\n"); 527 if (l2 == NULL) { 528 dfatal("Syntax error in profile line %d: %s\n", 529 lineno, buf); 530 } 531 l1 = stripwhite(l1); 532 l2 = stripwhite(l2); 533 534 /* 535 * Add to builder environment 536 */ 537 addbuildenv(l1, l2, BENV_MAKECONF); 538 if (DebugOpt >= 2) 539 ddprintf(4, "%s=%s\n", l1, l2); 540 } 541 fclose(fp); 542 if (DebugOpt >= 2) 543 ddprintf(0, "ParseProfile finished\n"); 544 } 545 546 static char * 547 stripwhite(char *str) 548 { 549 size_t len; 550 551 len = strlen(str); 552 while (len > 0 && isspace(str[len-1])) 553 --len; 554 str[len] =0; 555 556 while (*str && isspace(*str)) 557 ++str; 558 return str; 559 } 560 561 static int 562 truefalse(const char *str) 563 { 564 if (strcmp(str, "0") == 0) 565 return 0; 566 if (strcmp(str, "1") == 0) 567 return 1; 568 if (strcasecmp(str, "false") == 0) 569 return 0; 570 if (strcasecmp(str, "true") == 0) 571 return 1; 572 dfatal("syntax error for boolean '%s': " 573 "must be '0', '1', 'false', or 'true'", str); 574 return 0; 575 } 576 577 static char * 578 dokernsysctl(int m1, int m2) 579 { 580 int mib[] = { m1, m2 }; 581 char buf[1024]; 582 size_t len; 583 584 len = sizeof(buf) - 1; 585 if (sysctl(mib, 2, buf, &len, NULL, 0) < 0) 586 dfatal_errno("sysctl for system/architecture"); 587 buf[len] = 0; 588 return(strdup(buf)); 589 } 590 591 struct NoteTag { 592 Elf_Note note; 593 char osname1[12]; 594 int version; /* e.g. 500702 -> 5.7 */ 595 int x1; 596 int x2; 597 int x3; 598 char osname2[12]; 599 int zero; 600 }; 601 602 static void 603 getElfInfo(const char *path) 604 { 605 struct NoteTag note; 606 char *cmd; 607 char *base; 608 FILE *fp; 609 size_t size; 610 size_t n; 611 int r; 612 uint32_t addr; 613 uint32_t v[4]; 614 615 asprintf(&cmd, "readelf -x .note.tag %s", path); 616 fp = popen(cmd, "r"); 617 dassert_errno(fp, "Cannot run: %s", cmd); 618 n = 0; 619 620 while (n != sizeof(note) && 621 (base = fgetln(fp, &size)) != NULL && size) { 622 base[--size] = 0; 623 if (strncmp(base, " 0x", 3) != 0) 624 continue; 625 r = sscanf(base, "%x %x %x %x %x", 626 &addr, &v[0], &v[1], &v[2], &v[3]); 627 v[0] = ntohl(v[0]); 628 v[1] = ntohl(v[1]); 629 v[2] = ntohl(v[2]); 630 v[3] = ntohl(v[3]); 631 if (r < 2) 632 continue; 633 r = (r - 1) * sizeof(v[0]); 634 if (n + r > sizeof(note)) 635 r = sizeof(note) - n; 636 bcopy((char *)v, (char *)¬e + n, r); 637 n += r; 638 } 639 pclose(fp); 640 641 if (n != sizeof(note)) 642 dfatal("Unable to parse output from: %s", cmd); 643 if (strncmp(OperatingSystemName, note.osname1, sizeof(note.osname1))) { 644 dfatal("%s ELF, mismatch OS name %.*s vs %s", 645 path, (int)sizeof(note.osname1), 646 note.osname1, OperatingSystemName); 647 } 648 free(cmd); 649 asprintf(&cmd, "%d.%d", 650 note.version / 100000, 651 (note.version % 100000) / 100); 652 ReleaseName = cmd; 653 } 654 655 static char * 656 checkhook(const char *scriptname) 657 { 658 struct stat st; 659 char *path; 660 661 asprintf(&path, "%s/%s", ConfigBase, scriptname); 662 if (stat(path, &st) < 0 || (st.st_mode & 0111) == 0) { 663 free(path); 664 return NULL; 665 } 666 UsingHooks = 1; 667 668 return path; 669 } 670