1 /* 2 * Copyright (c) 2019 The DragonFly Project. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Matthew Dillon <dillon@backplane.com> 6 * 7 * This code uses concepts and configuration based on 'synth', by 8 * John R. Marino <draco@marino.st>, which was written in ada. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in 18 * the documentation and/or other materials provided with the 19 * distribution. 20 * 3. Neither the name of The DragonFly Project nor the names of its 21 * contributors may be used to endorse or promote products derived 22 * from this software without specific, prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 25 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 26 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 27 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 28 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 29 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 30 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 31 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 32 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 33 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 34 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 35 * SUCH DAMAGE. 36 */ 37 38 #include "dsynth.h" 39 40 int UseCCache; 41 int UseUsrSrc; 42 int UseTmpfs; 43 int NumCores = 1; 44 int MaxBulk = 8; 45 int MaxWorkers = 8; 46 int MaxJobs = 8; 47 int UseTmpfsWork = 1; 48 int UseTmpfsBase = 1; 49 int UseNCurses = -1; /* indicates default operation (enabled) */ 50 int LeveragePrebuilt = 0; 51 int WorkerProcFlags = 0; 52 long PhysMem; 53 const char *OperatingSystemName = "Unknown"; 54 const char *ArchitectureName = "unknown"; 55 const char *MachineName = "unknown"; 56 const char *VersionName = "unknown"; 57 const char *ReleaseName = "unknown"; 58 const char *DPortsPath = "/usr/dports"; 59 const char *CCachePath = DISABLED_STR; 60 const char *PackagesPath = "/build/synth/live_packages"; 61 const char *RepositoryPath = "/build/synth/live_packages/All"; 62 const char *OptionsPath = "/build/synth/options"; 63 const char *DistFilesPath = "/build/synth/distfiles"; 64 const char *BuildBase = "/build/synth/build"; 65 const char *LogsPath = "/build/synth/logs"; 66 const char *SystemPath = "/"; 67 const char *ProfileLabel = "[LiveSystem]"; /* with the brackets */ 68 const char *Profile = "LiveSystem"; /* without the brackets */ 69 70 /* 71 * Hooks are scripts in ConfigBase 72 */ 73 int UsingHooks; 74 const char *HookRunStart; 75 const char *HookRunEnd; 76 const char *HookPkgSuccess; 77 const char *HookPkgFailure; 78 const char *HookPkgIgnored; 79 const char *HookPkgSkipped; 80 81 const char *ConfigBase; /* The config base we found */ 82 const char *ConfigBase1 = "/etc/dsynth"; 83 const char *ConfigBase2 = "/usr/local/etc/dsynth"; 84 85 static void parseConfigFile(const char *path); 86 static void parseProfile(const char *cpath, const char *path); 87 static char *stripwhite(char *str); 88 static int truefalse(const char *str); 89 static char *dokernsysctl(int m1, int m2); 90 static void getElfInfo(const char *path); 91 static char *checkhook(const char *scriptname); 92 93 void 94 ParseConfiguration(int isworker) 95 { 96 struct stat st; 97 size_t len; 98 int reln; 99 char *synth_config; 100 char *buf; 101 102 /* 103 * Get the default OperatingSystemName, ArchitectureName, and 104 * ReleaseName. 105 */ 106 OperatingSystemName = dokernsysctl(CTL_KERN, KERN_OSTYPE); 107 ArchitectureName = dokernsysctl(CTL_HW, HW_MACHINE_ARCH); 108 MachineName = dokernsysctl(CTL_HW, HW_MACHINE); 109 ReleaseName = dokernsysctl(CTL_KERN, KERN_OSRELEASE); 110 111 /* 112 * Retrieve resource information from the system. Note that 113 * NumCores and PhysMem will also be used for dynamic load 114 * management. 115 */ 116 NumCores = 1; 117 len = sizeof(NumCores); 118 if (sysctlbyname("hw.ncpu", &NumCores, &len, NULL, 0) < 0) 119 dfatal_errno("Cannot get hw.ncpu"); 120 121 len = sizeof(PhysMem); 122 if (sysctlbyname("hw.physmem", &PhysMem, &len, NULL, 0) < 0) 123 dfatal_errno("Cannot get hw.physmem"); 124 if (PkgDepMemoryTarget == 0) 125 PkgDepMemoryTarget = PhysMem / 2; 126 127 /* 128 * Calculate nominal defaults. 129 */ 130 MaxBulk = NumCores; 131 MaxWorkers = MaxBulk / 2; 132 if (MaxWorkers > (int)((PhysMem + (ONEGB/2)) / ONEGB)) 133 MaxWorkers = (PhysMem + (ONEGB/2)) / ONEGB; 134 135 if (MaxBulk < 1) 136 MaxBulk = 1; 137 if (MaxWorkers < 1) 138 MaxWorkers = 1; 139 if (MaxJobs < 1) 140 MaxJobs = 1; 141 142 /* 143 * Configuration file must exist. Look for it in 144 * "/etc/dsynth" and "/usr/local/etc/dsynth". 145 */ 146 ConfigBase = ConfigBase1; 147 asprintf(&synth_config, "%s/dsynth.ini", ConfigBase1); 148 if (stat(synth_config, &st) < 0) { 149 ConfigBase = ConfigBase2; 150 asprintf(&synth_config, "%s/dsynth.ini", ConfigBase2); 151 } 152 153 if (stat(synth_config, &st) < 0) { 154 dfatal("Configuration file missing, " 155 "could not find %s/dsynth.ini or %s/dsynth.ini\n", 156 ConfigBase1, 157 ConfigBase2); 158 } 159 160 /* 161 * Check to see what hooks we have 162 */ 163 HookRunStart = checkhook("hook_run_start"); 164 HookRunEnd = checkhook("hook_run_end"); 165 HookPkgSuccess = checkhook("hook_pkg_success"); 166 HookPkgFailure = checkhook("hook_pkg_failure"); 167 HookPkgIgnored = checkhook("hook_pkg_ignored"); 168 HookPkgSkipped = checkhook("hook_pkg_skipped"); 169 170 /* 171 * Parse the configuration file(s). This may override some of 172 * the above defaults. 173 */ 174 parseConfigFile(synth_config); 175 parseProfile(synth_config, ProfileLabel); 176 177 /* 178 * Figure out whether CCache is configured. Also set UseUsrSrc 179 * if it exists under the system path. 180 * 181 * Not supported for the moment 182 */ 183 if (strcmp(CCachePath, "disabled") != 0) { 184 dfatal("Directory_ccache is not supported, please\n" 185 " set to 'disabled'\n"); 186 /* NOT REACHED */ 187 UseCCache = 1; 188 } 189 asprintf(&buf, "%s/usr/src/sys/Makefile", SystemPath); 190 if (stat(buf, &st) == 0) 191 UseUsrSrc = 1; 192 free(buf); 193 194 /* 195 * If this is a dsynth WORKER exec it handles a single slot, 196 * just set MaxWorkers to 1. 197 */ 198 if (isworker) 199 MaxWorkers = 1; 200 201 /* 202 * Final check 203 */ 204 if (stat(DPortsPath, &st) < 0) 205 dfatal("Directory missing: %s", DPortsPath); 206 if (stat(PackagesPath, &st) < 0) 207 dfatal("Directory missing: %s", PackagesPath); 208 if (stat(OptionsPath, &st) < 0) 209 dfatal("Directory missing: %s", OptionsPath); 210 if (stat(DistFilesPath, &st) < 0) 211 dfatal("Directory missing: %s", DistFilesPath); 212 if (stat(BuildBase, &st) < 0) 213 dfatal("Directory missing: %s", BuildBase); 214 if (stat(LogsPath, &st) < 0) 215 dfatal("Directory missing: %s", LogsPath); 216 if (stat(SystemPath, &st) < 0) 217 dfatal("Directory missing: %s", SystemPath); 218 if (UseCCache && stat(CCachePath, &st) < 0) 219 dfatal("Directory missing: %s", CCachePath); 220 221 /* 222 * Now use the SystemPath to retrieve file information from /bin/sh, 223 * and use this to set OperatingSystemName, ArchitectureName, 224 * MachineName, and ReleaseName. 225 * 226 * Since this method is used to build for specific releases, require 227 * that it succeed. 228 */ 229 asprintf(&buf, "%s/bin/sh", SystemPath); 230 getElfInfo(buf); 231 free(buf); 232 233 /* 234 * Calculate VersionName from OperatingSystemName and ReleaseName. 235 */ 236 if (strchr(ReleaseName, '-')) { 237 reln = strchr(ReleaseName, '-') - ReleaseName; 238 asprintf(&buf, "%s %*.*s-SYNTH", 239 OperatingSystemName, 240 reln, reln, ReleaseName); 241 } else { 242 asprintf(&buf, "%s %s-SYNTH", 243 OperatingSystemName, 244 ReleaseName); 245 } 246 VersionName = buf; 247 248 /* 249 * If RepositoryPath is under PackagesPath, make sure it 250 * is created. 251 */ 252 if (strncmp(RepositoryPath, PackagesPath, strlen(PackagesPath)) == 0) { 253 if (stat(RepositoryPath, &st) < 0) { 254 if (mkdir(RepositoryPath, 0755) < 0) 255 dfatal_errno("Cannot mkdir '%s'", 256 RepositoryPath); 257 } 258 } 259 260 if (stat(RepositoryPath, &st) < 0) 261 dfatal("Directory missing: %s", RepositoryPath); 262 } 263 264 void 265 DoConfigure(void) 266 { 267 dfatal("Not Implemented"); 268 } 269 270 static void 271 parseConfigFile(const char *path) 272 { 273 char buf[1024]; 274 char copy[1024]; 275 FILE *fp; 276 char *l1; 277 char *l2; 278 size_t len; 279 int mode = -1; 280 int lineno = 0; 281 282 fp = fopen(path, "r"); 283 if (fp == NULL) { 284 ddprintf(0, "Warning: Config file %s does not exist\n", path); 285 return; 286 } 287 if (DebugOpt >= 2) 288 ddprintf(0, "ParseConfig %s\n", path); 289 290 while (fgets(buf, sizeof(buf), fp) != NULL) { 291 ++lineno; 292 len = strlen(buf); 293 if (len == 0 || buf[len-1] != '\n') 294 continue; 295 buf[--len] = 0; 296 297 /* 298 * Remove any trailing whitespace, ignore empty lines. 299 */ 300 while (len > 0 && isspace(buf[len-1])) 301 --len; 302 if (len == 0) 303 continue; 304 buf[len] = 0; 305 306 /* 307 * ignore comments 308 */ 309 if (buf[0] == ';' || buf[0] == '#') 310 continue; 311 if (buf[0] == '[') { 312 if (strcmp(buf, "[Global Configuration]") == 0) 313 mode = 0; /* parse global config */ 314 else if (strcmp(buf, ProfileLabel) == 0) 315 mode = 1; /* use profile */ 316 else 317 mode = -1; /* ignore profile */ 318 continue; 319 } 320 321 bcopy(buf, copy, len + 1); 322 323 l1 = strtok(copy, "="); 324 if (l1 == NULL) { 325 dfatal("Syntax error in config line %d: %s\n", 326 lineno, buf); 327 } 328 l2 = strtok(NULL, " \t\n"); 329 if (l2 == NULL) { 330 dfatal("Syntax error in config line %d: %s\n", 331 lineno, buf); 332 } 333 l1 = stripwhite(l1); 334 l2 = stripwhite(l2); 335 336 switch(mode) { 337 case 0: 338 /* 339 * Global Configuration 340 */ 341 if (strcmp(l1, "profile_selected") == 0) { 342 Profile = strdup(l2); 343 asprintf(&l2, "[%s]", l2); 344 ProfileLabel = l2; 345 } else { 346 dfatal("Unknown directive in config " 347 "line %d: %s\n", lineno, buf); 348 } 349 break; 350 case 1: 351 /* 352 * Selected Profile 353 */ 354 l2 = strdup(l2); 355 if (strcmp(l1, "Operating_system") == 0) { 356 OperatingSystemName = l2; 357 } else if (strcmp(l1, "Directory_packages") == 0) { 358 PackagesPath = l2; 359 } else if (strcmp(l1, "Directory_repository") == 0) { 360 RepositoryPath = l2; 361 } else if (strcmp(l1, "Directory_portsdir") == 0) { 362 DPortsPath = l2; 363 } else if (strcmp(l1, "Directory_options") == 0) { 364 OptionsPath = l2; 365 } else if (strcmp(l1, "Directory_distfiles") == 0) { 366 DistFilesPath = l2; 367 } else if (strcmp(l1, "Directory_buildbase") == 0) { 368 BuildBase = l2; 369 } else if (strcmp(l1, "Directory_logs") == 0) { 370 LogsPath = l2; 371 } else if (strcmp(l1, "Directory_ccache") == 0) { 372 CCachePath = l2; 373 } else if (strcmp(l1, "Directory_system") == 0) { 374 SystemPath = l2; 375 } else if (strcmp(l1, "Number_of_builders") == 0) { 376 MaxWorkers = strtol(l2, NULL, 0); 377 if (MaxWorkers == 0) 378 MaxWorkers = NumCores / 2 + 1; 379 else 380 if (MaxWorkers < 0 || MaxWorkers > MAXWORKERS) { 381 dfatal("Config: Number_of_builders " 382 "must range %d..%d", 383 1, MAXWORKERS); 384 } 385 free(l2); 386 } else if (strcmp(l1, "Max_jobs_per_builder") == 0) { 387 MaxJobs = strtol(l2, NULL, 0); 388 if (MaxJobs == 0) { 389 MaxJobs = NumCores; 390 } else 391 if (MaxJobs < 0 || MaxJobs > MAXJOBS) { 392 dfatal("Config: Max_jobs_per_builder " 393 "must range %d..%d", 394 1, MAXJOBS); 395 } 396 free(l2); 397 } else if (strcmp(l1, "Tmpfs_workdir") == 0) { 398 UseTmpfsWork = truefalse(l2); 399 dassert(UseTmpfsWork == 1, 400 "Config: Tmpfs_workdir must be " 401 "set to true, 'false' not supported"); 402 } else if (strcmp(l1, "Tmpfs_localbase") == 0) { 403 UseTmpfsBase = truefalse(l2); 404 dassert(UseTmpfsBase == 1, 405 "Config: Tmpfs_localbase must be " 406 "set to true, 'false' not supported"); 407 } else if (strcmp(l1, "Display_with_ncurses") == 0) { 408 if (UseNCurses == -1) 409 UseNCurses = truefalse(l2); 410 } else if (strcmp(l1, "leverage_prebuilt") == 0) { 411 LeveragePrebuilt = truefalse(l2); 412 dassert(LeveragePrebuilt == 0, 413 "Config: leverage_prebuilt not " 414 "supported and must be set to false"); 415 } else { 416 dfatal("Unknown directive in profile section " 417 "line %d: %s\n", lineno, buf); 418 } 419 break; 420 default: 421 /* 422 * Ignore unselected profile 423 */ 424 break; 425 } 426 } 427 fclose(fp); 428 } 429 430 /* 431 * NOTE: profile has brackets, e.g. "[LiveSystem]". 432 */ 433 static void 434 parseProfile(const char *cpath, const char *profile) 435 { 436 char buf[1024]; 437 char copy[1024]; 438 char *ppath; 439 FILE *fp; 440 char *l1; 441 char *l2; 442 int len; 443 int plen; 444 int lineno = 0; 445 446 len = strlen(cpath); 447 while (len && cpath[len-1] != '/') 448 --len; 449 if (len == 0) 450 ++len; 451 plen = strlen(profile); 452 ddassert(plen > 2 && profile[0] == '[' && profile[plen-1] == ']'); 453 454 asprintf(&ppath, "%*.*s%*.*s-make.conf", 455 len, len, cpath, plen - 2, plen - 2, profile + 1); 456 fp = fopen(ppath, "r"); 457 if (fp == NULL) { 458 ddprintf(0, "Warning: Profile %s does not exist\n", ppath); 459 return; 460 } 461 if (DebugOpt >= 2) 462 ddprintf(0, "ParseProfile %s\n", ppath); 463 free(ppath); 464 465 while (fgets(buf, sizeof(buf), fp) != NULL) { 466 ++lineno; 467 len = strlen(buf); 468 if (len == 0 || buf[len-1] != '\n') 469 continue; 470 buf[--len] = 0; 471 472 /* 473 * Remove any trailing whitespace, ignore empty lines. 474 */ 475 while (len > 0 && isspace(buf[len-1])) 476 --len; 477 buf[len] = 0; 478 stripwhite(buf); 479 len = strlen(buf); 480 if (len == 0) 481 continue; 482 483 /* 484 * Ignore comments. 485 */ 486 if (buf[0] == ';' || buf[0] == '#') 487 continue; 488 489 bcopy(buf, copy, len + 1); 490 l1 = strtok(copy, "="); 491 if (l1 == NULL) { 492 dfatal("Syntax error in profile line %d: %s\n", 493 lineno, buf); 494 } 495 l2 = strtok(NULL, " \t\n"); 496 if (l2 == NULL) { 497 dfatal("Syntax error in profile line %d: %s\n", 498 lineno, buf); 499 } 500 l1 = stripwhite(l1); 501 l2 = stripwhite(l2); 502 503 /* 504 * Add to builder environment 505 */ 506 addbuildenv(l1, l2, BENV_MAKECONF); 507 if (DebugOpt >= 2) 508 ddprintf(4, "%s=%s\n", l1, l2); 509 } 510 fclose(fp); 511 if (DebugOpt >= 2) 512 ddprintf(0, "ParseProfile finished\n"); 513 } 514 515 static char * 516 stripwhite(char *str) 517 { 518 size_t len; 519 520 len = strlen(str); 521 while (len > 0 && isspace(str[len-1])) 522 --len; 523 str[len] =0; 524 525 while (*str && isspace(*str)) 526 ++str; 527 return str; 528 } 529 530 static int 531 truefalse(const char *str) 532 { 533 if (strcmp(str, "0") == 0) 534 return 0; 535 if (strcmp(str, "1") == 0) 536 return 1; 537 if (strcasecmp(str, "false") == 0) 538 return 0; 539 if (strcasecmp(str, "true") == 0) 540 return 1; 541 dfatal("syntax error for boolean '%s': " 542 "must be '0', '1', 'false', or 'true'", str); 543 return 0; 544 } 545 546 static char * 547 dokernsysctl(int m1, int m2) 548 { 549 int mib[] = { m1, m2 }; 550 char buf[1024]; 551 size_t len; 552 553 len = sizeof(buf) - 1; 554 if (sysctl(mib, 2, buf, &len, NULL, 0) < 0) 555 dfatal_errno("sysctl for system/architecture"); 556 buf[len] = 0; 557 return(strdup(buf)); 558 } 559 560 struct NoteTag { 561 Elf_Note note; 562 char osname1[12]; 563 int version; /* e.g. 500702 -> 5.7 */ 564 int x1; 565 int x2; 566 int x3; 567 char osname2[12]; 568 int zero; 569 }; 570 571 static void 572 getElfInfo(const char *path) 573 { 574 struct NoteTag note; 575 char *cmd; 576 char *base; 577 FILE *fp; 578 size_t size; 579 size_t n; 580 int r; 581 uint32_t addr; 582 uint32_t v[4]; 583 584 asprintf(&cmd, "readelf -x .note.tag %s", path); 585 fp = popen(cmd, "r"); 586 dassert_errno(fp, "Cannot run: %s", cmd); 587 n = 0; 588 589 while (n != sizeof(note) && 590 (base = fgetln(fp, &size)) != NULL && size) { 591 base[--size] = 0; 592 if (strncmp(base, " 0x", 3) != 0) 593 continue; 594 r = sscanf(base, "%x %x %x %x %x", 595 &addr, &v[0], &v[1], &v[2], &v[3]); 596 v[0] = ntohl(v[0]); 597 v[1] = ntohl(v[1]); 598 v[2] = ntohl(v[2]); 599 v[3] = ntohl(v[3]); 600 if (r < 2) 601 continue; 602 r = (r - 1) * sizeof(v[0]); 603 if (n + r > sizeof(note)) 604 r = sizeof(note) - n; 605 bcopy((char *)v, (char *)¬e + n, r); 606 n += r; 607 } 608 pclose(fp); 609 610 if (n != sizeof(note)) 611 dfatal("Unable to parse output from: %s", cmd); 612 if (strncmp(OperatingSystemName, note.osname1, sizeof(note.osname1))) { 613 dfatal("%s ELF, mismatch OS name %.*s vs %s", 614 path, (int)sizeof(note.osname1), 615 note.osname1, OperatingSystemName); 616 } 617 free(cmd); 618 asprintf(&cmd, "%d.%d", 619 note.version / 100000, 620 (note.version % 100000) / 100); 621 ReleaseName = cmd; 622 } 623 624 static char * 625 checkhook(const char *scriptname) 626 { 627 struct stat st; 628 char *path; 629 630 asprintf(&path, "%s/%s", ConfigBase, scriptname); 631 if (stat(path, &st) < 0 || (st.st_mode & 0111) == 0) { 632 free(path); 633 return NULL; 634 } 635 UsingHooks = 1; 636 637 return path; 638 } 639