1# Copyright (c) Facebook, Inc. and its affiliates. 2# 3# This source code is licensed under the MIT license found in the 4# LICENSE file in the root directory of this source tree. 5 6from __future__ import absolute_import, division, print_function, unicode_literals 7 8import io 9import os 10 11from .builder import ( 12 AutoconfBuilder, 13 Boost, 14 CargoBuilder, 15 CMakeBuilder, 16 BistroBuilder, 17 Iproute2Builder, 18 MakeBuilder, 19 NinjaBootstrap, 20 NopBuilder, 21 OpenNSABuilder, 22 OpenSSLBuilder, 23 SqliteBuilder, 24 CMakeBootStrapBuilder, 25) 26from .expr import parse_expr 27from .fetcher import ( 28 ArchiveFetcher, 29 GitFetcher, 30 PreinstalledNopFetcher, 31 ShipitTransformerFetcher, 32 SimpleShipitTransformerFetcher, 33 SystemPackageFetcher, 34) 35from .py_wheel_builder import PythonWheelBuilder 36 37 38try: 39 import configparser 40except ImportError: 41 import ConfigParser as configparser 42 43REQUIRED = "REQUIRED" 44OPTIONAL = "OPTIONAL" 45 46SCHEMA = { 47 "manifest": { 48 "optional_section": False, 49 "fields": { 50 "name": REQUIRED, 51 "fbsource_path": OPTIONAL, 52 "shipit_project": OPTIONAL, 53 "shipit_fbcode_builder": OPTIONAL, 54 }, 55 }, 56 "dependencies": {"optional_section": True, "allow_values": False}, 57 "depends.environment": {"optional_section": True}, 58 "git": { 59 "optional_section": True, 60 "fields": {"repo_url": REQUIRED, "rev": OPTIONAL, "depth": OPTIONAL}, 61 }, 62 "download": { 63 "optional_section": True, 64 "fields": {"url": REQUIRED, "sha256": REQUIRED}, 65 }, 66 "build": { 67 "optional_section": True, 68 "fields": { 69 "builder": REQUIRED, 70 "subdir": OPTIONAL, 71 "build_in_src_dir": OPTIONAL, 72 "disable_env_override_pkgconfig": OPTIONAL, 73 "disable_env_override_path": OPTIONAL, 74 }, 75 }, 76 "msbuild": {"optional_section": True, "fields": {"project": REQUIRED}}, 77 "cargo": { 78 "optional_section": True, 79 "fields": { 80 "build_doc": OPTIONAL, 81 "workspace_dir": OPTIONAL, 82 "manifests_to_build": OPTIONAL, 83 }, 84 }, 85 "cmake.defines": {"optional_section": True}, 86 "autoconf.args": {"optional_section": True}, 87 "rpms": {"optional_section": True}, 88 "debs": {"optional_section": True}, 89 "preinstalled.env": {"optional_section": True}, 90 "b2.args": {"optional_section": True}, 91 "make.build_args": {"optional_section": True}, 92 "make.install_args": {"optional_section": True}, 93 "make.test_args": {"optional_section": True}, 94 "header-only": {"optional_section": True, "fields": {"includedir": REQUIRED}}, 95 "shipit.pathmap": {"optional_section": True}, 96 "shipit.strip": {"optional_section": True}, 97 "install.files": {"optional_section": True}, 98} 99 100# These sections are allowed to vary for different platforms 101# using the expression syntax to enable/disable sections 102ALLOWED_EXPR_SECTIONS = [ 103 "autoconf.args", 104 "build", 105 "cmake.defines", 106 "dependencies", 107 "make.build_args", 108 "make.install_args", 109 "b2.args", 110 "download", 111 "git", 112 "install.files", 113] 114 115 116def parse_conditional_section_name(name, section_def): 117 expr = name[len(section_def) + 1 :] 118 return parse_expr(expr, ManifestContext.ALLOWED_VARIABLES) 119 120 121def validate_allowed_fields(file_name, section, config, allowed_fields): 122 for field in config.options(section): 123 if not allowed_fields.get(field): 124 raise Exception( 125 ("manifest file %s section '%s' contains " "unknown field '%s'") 126 % (file_name, section, field) 127 ) 128 129 for field in allowed_fields: 130 if allowed_fields[field] == REQUIRED and not config.has_option(section, field): 131 raise Exception( 132 ("manifest file %s section '%s' is missing " "required field '%s'") 133 % (file_name, section, field) 134 ) 135 136 137def validate_allow_values(file_name, section, config): 138 for field in config.options(section): 139 value = config.get(section, field) 140 if value is not None: 141 raise Exception( 142 ( 143 "manifest file %s section '%s' has '%s = %s' but " 144 "this section doesn't allow specifying values " 145 "for its entries" 146 ) 147 % (file_name, section, field, value) 148 ) 149 150 151def validate_section(file_name, section, config): 152 section_def = SCHEMA.get(section) 153 if not section_def: 154 for name in ALLOWED_EXPR_SECTIONS: 155 if section.startswith(name + "."): 156 # Verify that the conditional parses, but discard it 157 try: 158 parse_conditional_section_name(section, name) 159 except Exception as exc: 160 raise Exception( 161 ("manifest file %s section '%s' has invalid " "conditional: %s") 162 % (file_name, section, str(exc)) 163 ) 164 section_def = SCHEMA.get(name) 165 canonical_section_name = name 166 break 167 if not section_def: 168 raise Exception( 169 "manifest file %s contains unknown section '%s'" % (file_name, section) 170 ) 171 else: 172 canonical_section_name = section 173 174 allowed_fields = section_def.get("fields") 175 if allowed_fields: 176 validate_allowed_fields(file_name, section, config, allowed_fields) 177 elif not section_def.get("allow_values", True): 178 validate_allow_values(file_name, section, config) 179 return canonical_section_name 180 181 182class ManifestParser(object): 183 def __init__(self, file_name, fp=None): 184 # allow_no_value enables listing parameters in the 185 # autoconf.args section one per line 186 config = configparser.RawConfigParser(allow_no_value=True) 187 config.optionxform = str # make it case sensitive 188 189 if fp is None: 190 with open(file_name, "r") as fp: 191 config.read_file(fp) 192 elif isinstance(fp, type("")): 193 # For testing purposes, parse from a string (str 194 # or unicode) 195 config.read_file(io.StringIO(fp)) 196 else: 197 config.read_file(fp) 198 199 # validate against the schema 200 seen_sections = set() 201 202 for section in config.sections(): 203 seen_sections.add(validate_section(file_name, section, config)) 204 205 for section in SCHEMA.keys(): 206 section_def = SCHEMA[section] 207 if ( 208 not section_def.get("optional_section", False) 209 and section not in seen_sections 210 ): 211 raise Exception( 212 "manifest file %s is missing required section %s" 213 % (file_name, section) 214 ) 215 216 self._config = config 217 self.name = config.get("manifest", "name") 218 self.fbsource_path = self.get("manifest", "fbsource_path") 219 self.shipit_project = self.get("manifest", "shipit_project") 220 self.shipit_fbcode_builder = self.get("manifest", "shipit_fbcode_builder") 221 222 if self.name != os.path.basename(file_name): 223 raise Exception( 224 "filename of the manifest '%s' does not match the manifest name '%s'" 225 % (file_name, self.name) 226 ) 227 228 def get(self, section, key, defval=None, ctx=None): 229 ctx = ctx or {} 230 231 for s in self._config.sections(): 232 if s == section: 233 if self._config.has_option(s, key): 234 return self._config.get(s, key) 235 return defval 236 237 if s.startswith(section + "."): 238 expr = parse_conditional_section_name(s, section) 239 if not expr.eval(ctx): 240 continue 241 242 if self._config.has_option(s, key): 243 return self._config.get(s, key) 244 245 return defval 246 247 def get_section_as_args(self, section, ctx=None): 248 """Intended for use with the make.[build_args/install_args] and 249 autoconf.args sections, this method collects the entries and returns an 250 array of strings. 251 If the manifest contains conditional sections, ctx is used to 252 evaluate the condition and merge in the values. 253 """ 254 args = [] 255 ctx = ctx or {} 256 257 for s in self._config.sections(): 258 if s != section: 259 if not s.startswith(section + "."): 260 continue 261 expr = parse_conditional_section_name(s, section) 262 if not expr.eval(ctx): 263 continue 264 for field in self._config.options(s): 265 value = self._config.get(s, field) 266 if value is None: 267 args.append(field) 268 else: 269 args.append("%s=%s" % (field, value)) 270 return args 271 272 def get_section_as_ordered_pairs(self, section, ctx=None): 273 """Used for eg: shipit.pathmap which has strong 274 ordering requirements""" 275 res = [] 276 ctx = ctx or {} 277 278 for s in self._config.sections(): 279 if s != section: 280 if not s.startswith(section + "."): 281 continue 282 expr = parse_conditional_section_name(s, section) 283 if not expr.eval(ctx): 284 continue 285 286 for key in self._config.options(s): 287 value = self._config.get(s, key) 288 res.append((key, value)) 289 return res 290 291 def get_section_as_dict(self, section, ctx=None): 292 d = {} 293 ctx = ctx or {} 294 295 for s in self._config.sections(): 296 if s != section: 297 if not s.startswith(section + "."): 298 continue 299 expr = parse_conditional_section_name(s, section) 300 if not expr.eval(ctx): 301 continue 302 for field in self._config.options(s): 303 value = self._config.get(s, field) 304 d[field] = value 305 return d 306 307 def update_hash(self, hasher, ctx): 308 """Compute a hash over the configuration for the given 309 context. The goal is for the hash to change if the config 310 for that context changes, but not if a change is made to 311 the config only for a different platform than that expressed 312 by ctx. The hash is intended to be used to help invalidate 313 a future cache for the third party build products. 314 The hasher argument is a hash object returned from hashlib.""" 315 for section in sorted(SCHEMA.keys()): 316 hasher.update(section.encode("utf-8")) 317 318 # Note: at the time of writing, nothing in the implementation 319 # relies on keys in any config section being ordered. 320 # In theory we could have conflicting flags in different 321 # config sections and later flags override earlier flags. 322 # For the purposes of computing a hash we're not super 323 # concerned about this: manifest changes should be rare 324 # enough and we'd rather that this trigger an invalidation 325 # than strive for a cache hit at this time. 326 pairs = self.get_section_as_ordered_pairs(section, ctx) 327 pairs.sort(key=lambda pair: pair[0]) 328 for key, value in pairs: 329 hasher.update(key.encode("utf-8")) 330 if value is not None: 331 hasher.update(value.encode("utf-8")) 332 333 def is_first_party_project(self): 334 """returns true if this is an FB first-party project""" 335 return self.shipit_project is not None 336 337 def get_required_system_packages(self, ctx): 338 """Returns dictionary of packager system -> list of packages""" 339 return { 340 "rpm": self.get_section_as_args("rpms", ctx), 341 "deb": self.get_section_as_args("debs", ctx), 342 } 343 344 def _is_satisfied_by_preinstalled_environment(self, ctx): 345 envs = self.get_section_as_args("preinstalled.env", ctx) 346 if not envs: 347 return False 348 for key in envs: 349 val = os.environ.get(key, None) 350 print(f"Testing ENV[{key}]: {repr(val)}") 351 if val is None: 352 return False 353 if len(val) == 0: 354 return False 355 356 return True 357 358 def create_fetcher(self, build_options, ctx): 359 use_real_shipit = ( 360 ShipitTransformerFetcher.available() and build_options.use_shipit 361 ) 362 if ( 363 not use_real_shipit 364 and self.fbsource_path 365 and build_options.fbsource_dir 366 and self.shipit_project 367 ): 368 return SimpleShipitTransformerFetcher(build_options, self) 369 370 if ( 371 self.fbsource_path 372 and build_options.fbsource_dir 373 and self.shipit_project 374 and ShipitTransformerFetcher.available() 375 ): 376 # We can use the code from fbsource 377 return ShipitTransformerFetcher(build_options, self.shipit_project) 378 379 # Can we satisfy this dep with system packages? 380 if build_options.allow_system_packages: 381 if self._is_satisfied_by_preinstalled_environment(ctx): 382 return PreinstalledNopFetcher() 383 384 packages = self.get_required_system_packages(ctx) 385 package_fetcher = SystemPackageFetcher(build_options, packages) 386 if package_fetcher.packages_are_installed(): 387 return package_fetcher 388 389 repo_url = self.get("git", "repo_url", ctx=ctx) 390 if repo_url: 391 rev = self.get("git", "rev") 392 depth = self.get("git", "depth") 393 return GitFetcher(build_options, self, repo_url, rev, depth) 394 395 url = self.get("download", "url", ctx=ctx) 396 if url: 397 # We need to defer this import until now to avoid triggering 398 # a cycle when the facebook/__init__.py is loaded. 399 try: 400 from getdeps.facebook.lfs import LFSCachingArchiveFetcher 401 402 return LFSCachingArchiveFetcher( 403 build_options, self, url, self.get("download", "sha256", ctx=ctx) 404 ) 405 except ImportError: 406 # This FB internal module isn't shippped to github, 407 # so just use its base class 408 return ArchiveFetcher( 409 build_options, self, url, self.get("download", "sha256", ctx=ctx) 410 ) 411 412 raise KeyError( 413 "project %s has no fetcher configuration matching %s" % (self.name, ctx) 414 ) 415 416 def create_builder( # noqa:C901 417 self, 418 build_options, 419 src_dir, 420 build_dir, 421 inst_dir, 422 ctx, 423 loader, 424 final_install_prefix=None, 425 extra_cmake_defines=None, 426 ): 427 builder = self.get("build", "builder", ctx=ctx) 428 if not builder: 429 raise Exception("project %s has no builder for %r" % (self.name, ctx)) 430 build_in_src_dir = self.get("build", "build_in_src_dir", "false", ctx=ctx) 431 if build_in_src_dir == "true": 432 # Some scripts don't work when they are configured and build in 433 # a different directory than source (or when the build directory 434 # is not a subdir of source). 435 build_dir = src_dir 436 subdir = self.get("build", "subdir", None, ctx=ctx) 437 if subdir is not None: 438 build_dir = os.path.join(build_dir, subdir) 439 print("build_dir is %s" % build_dir) # just to quiet lint 440 441 if builder == "make" or builder == "cmakebootstrap": 442 build_args = self.get_section_as_args("make.build_args", ctx) 443 install_args = self.get_section_as_args("make.install_args", ctx) 444 test_args = self.get_section_as_args("make.test_args", ctx) 445 if builder == "cmakebootstrap": 446 return CMakeBootStrapBuilder( 447 build_options, 448 ctx, 449 self, 450 src_dir, 451 None, 452 inst_dir, 453 build_args, 454 install_args, 455 test_args, 456 ) 457 else: 458 return MakeBuilder( 459 build_options, 460 ctx, 461 self, 462 src_dir, 463 None, 464 inst_dir, 465 build_args, 466 install_args, 467 test_args, 468 ) 469 470 if builder == "autoconf": 471 args = self.get_section_as_args("autoconf.args", ctx) 472 return AutoconfBuilder( 473 build_options, ctx, self, src_dir, build_dir, inst_dir, args 474 ) 475 476 if builder == "boost": 477 args = self.get_section_as_args("b2.args", ctx) 478 return Boost(build_options, ctx, self, src_dir, build_dir, inst_dir, args) 479 480 if builder == "bistro": 481 return BistroBuilder( 482 build_options, 483 ctx, 484 self, 485 src_dir, 486 build_dir, 487 inst_dir, 488 ) 489 490 if builder == "cmake": 491 defines = self.get_section_as_dict("cmake.defines", ctx) 492 return CMakeBuilder( 493 build_options, 494 ctx, 495 self, 496 src_dir, 497 build_dir, 498 inst_dir, 499 defines, 500 final_install_prefix, 501 extra_cmake_defines, 502 ) 503 504 if builder == "python-wheel": 505 return PythonWheelBuilder( 506 build_options, ctx, self, src_dir, build_dir, inst_dir 507 ) 508 509 if builder == "sqlite": 510 return SqliteBuilder(build_options, ctx, self, src_dir, build_dir, inst_dir) 511 512 if builder == "ninja_bootstrap": 513 return NinjaBootstrap( 514 build_options, ctx, self, build_dir, src_dir, inst_dir 515 ) 516 517 if builder == "nop": 518 return NopBuilder(build_options, ctx, self, src_dir, inst_dir) 519 520 if builder == "openssl": 521 return OpenSSLBuilder( 522 build_options, ctx, self, build_dir, src_dir, inst_dir 523 ) 524 525 if builder == "iproute2": 526 return Iproute2Builder( 527 build_options, ctx, self, src_dir, build_dir, inst_dir 528 ) 529 530 if builder == "cargo": 531 build_doc = self.get("cargo", "build_doc", False, ctx) 532 workspace_dir = self.get("cargo", "workspace_dir", None, ctx) 533 manifests_to_build = self.get("cargo", "manifests_to_build", None, ctx) 534 return CargoBuilder( 535 build_options, 536 ctx, 537 self, 538 src_dir, 539 build_dir, 540 inst_dir, 541 build_doc, 542 workspace_dir, 543 manifests_to_build, 544 loader, 545 ) 546 547 if builder == "OpenNSA": 548 return OpenNSABuilder(build_options, ctx, self, src_dir, inst_dir) 549 550 raise KeyError("project %s has no known builder" % (self.name)) 551 552 553class ManifestContext(object): 554 """ProjectContext contains a dictionary of values to use when evaluating boolean 555 expressions in a project manifest. 556 557 This object should be passed as the `ctx` parameter in ManifestParser.get() calls. 558 """ 559 560 ALLOWED_VARIABLES = {"os", "distro", "distro_vers", "fb", "test"} 561 562 def __init__(self, ctx_dict): 563 assert set(ctx_dict.keys()) == self.ALLOWED_VARIABLES 564 self.ctx_dict = ctx_dict 565 566 def get(self, key): 567 return self.ctx_dict[key] 568 569 def set(self, key, value): 570 assert key in self.ALLOWED_VARIABLES 571 self.ctx_dict[key] = value 572 573 def copy(self): 574 return ManifestContext(dict(self.ctx_dict)) 575 576 def __str__(self): 577 s = ", ".join( 578 "%s=%s" % (key, value) for key, value in sorted(self.ctx_dict.items()) 579 ) 580 return "{" + s + "}" 581 582 583class ContextGenerator(object): 584 """ContextGenerator allows creating ManifestContext objects on a per-project basis. 585 This allows us to evaluate different projects with slightly different contexts. 586 587 For instance, this can be used to only enable tests for some projects.""" 588 589 def __init__(self, default_ctx): 590 self.default_ctx = ManifestContext(default_ctx) 591 self.ctx_by_project = {} 592 593 def set_value_for_project(self, project_name, key, value): 594 project_ctx = self.ctx_by_project.get(project_name) 595 if project_ctx is None: 596 project_ctx = self.default_ctx.copy() 597 self.ctx_by_project[project_name] = project_ctx 598 project_ctx.set(key, value) 599 600 def set_value_for_all_projects(self, key, value): 601 self.default_ctx.set(key, value) 602 for ctx in self.ctx_by_project.values(): 603 ctx.set(key, value) 604 605 def get_context(self, project_name): 606 return self.ctx_by_project.get(project_name, self.default_ctx) 607