1# This Source Code Form is subject to the terms of the Mozilla Public 2# License, v. 2.0. If a copy of the MPL was not distributed with this 3# file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5from __future__ import absolute_import, print_function, unicode_literals 6 7from collections import defaultdict 8from copy import deepcopy 9import glob 10import json 11import os 12import six 13import subprocess 14import sys 15 16from mozbuild.backend.base import BuildBackend 17import mozpack.path as mozpath 18from mozbuild.frontend.sandbox import alphabetical_sorted 19from mozbuild.frontend.data import GnProjectData 20from mozbuild.util import expand_variables, mkdir 21 22 23license_header = """# This Source Code Form is subject to the terms of the Mozilla Public 24# License, v. 2.0. If a copy of the MPL was not distributed with this 25# file, You can obtain one at http://mozilla.org/MPL/2.0/. 26""" 27 28generated_header = """ 29 ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### 30 ### DO NOT edit it by hand. ### 31""" 32 33 34class MozbuildWriter(object): 35 def __init__(self, fh): 36 self._fh = fh 37 self.indent = "" 38 self._indent_increment = 4 39 40 # We need to correlate a small amount of state here to figure out 41 # which library template to use ("Library()" or "SharedLibrary()") 42 self._library_name = None 43 self._shared_library = None 44 45 def mb_serialize(self, v): 46 if isinstance(v, (bool, list)): 47 return repr(v) 48 return '"%s"' % v 49 50 def finalize(self): 51 if self._library_name: 52 self.write("\n") 53 if self._shared_library: 54 self.write_ln( 55 "SharedLibrary(%s)" % self.mb_serialize(self._library_name) 56 ) 57 else: 58 self.write_ln("Library(%s)" % self.mb_serialize(self._library_name)) 59 60 def write(self, content): 61 self._fh.write(content) 62 63 def write_ln(self, line): 64 self.write(self.indent) 65 self.write(line) 66 self.write("\n") 67 68 def write_attrs(self, context_attrs): 69 for k in sorted(context_attrs.keys()): 70 v = context_attrs[k] 71 if isinstance(v, (list, set)): 72 self.write_mozbuild_list(k, v) 73 elif isinstance(v, dict): 74 self.write_mozbuild_dict(k, v) 75 else: 76 self.write_mozbuild_value(k, v) 77 78 def write_mozbuild_list(self, key, value): 79 if value: 80 self.write("\n") 81 self.write(self.indent + key) 82 self.write(" += [\n " + self.indent) 83 self.write( 84 (",\n " + self.indent).join( 85 alphabetical_sorted(self.mb_serialize(v) for v in value) 86 ) 87 ) 88 self.write("\n") 89 self.write_ln("]") 90 91 def write_mozbuild_value(self, key, value): 92 if value: 93 if key == "LIBRARY_NAME": 94 self._library_name = value 95 elif key == "FORCE_SHARED_LIB": 96 self._shared_library = True 97 else: 98 self.write("\n") 99 self.write_ln("%s = %s" % (key, self.mb_serialize(value))) 100 self.write("\n") 101 102 def write_mozbuild_dict(self, key, value): 103 # Templates we need to use instead of certain values. 104 replacements = ( 105 ( 106 ("COMPILE_FLAGS", '"WARNINGS_AS_ERRORS"', "[]"), 107 "AllowCompilerWarnings()", 108 ), 109 ) 110 if value: 111 self.write("\n") 112 for k in sorted(value.keys()): 113 v = value[k] 114 subst_vals = key, self.mb_serialize(k), self.mb_serialize(v) 115 wrote_ln = False 116 for flags, tmpl in replacements: 117 if subst_vals == flags: 118 self.write_ln(tmpl) 119 wrote_ln = True 120 121 if not wrote_ln: 122 self.write_ln("%s[%s] = %s" % subst_vals) 123 124 def write_condition(self, values): 125 def mk_condition(k, v): 126 if not v: 127 return 'not CONFIG["%s"]' % k 128 return 'CONFIG["%s"] == %s' % (k, self.mb_serialize(v)) 129 130 self.write("\n") 131 self.write("if ") 132 self.write( 133 " and ".join(mk_condition(k, values[k]) for k in sorted(values.keys())) 134 ) 135 self.write(":\n") 136 self.indent += " " * self._indent_increment 137 138 def terminate_condition(self): 139 assert len(self.indent) >= self._indent_increment 140 self.indent = self.indent[self._indent_increment :] 141 142 143def find_deps(all_targets, target): 144 all_deps = set([target]) 145 for dep in all_targets[target]["deps"]: 146 if dep not in all_deps: 147 all_deps |= find_deps(all_targets, dep) 148 return all_deps 149 150 151def filter_gn_config(gn_result, config, sandbox_vars, input_vars, gn_target): 152 # Translates the raw output of gn into just what we'll need to generate a 153 # mozbuild configuration. 154 gn_out = {"targets": {}, "sandbox_vars": sandbox_vars, "gn_gen_args": input_vars} 155 156 gn_mozbuild_vars = ("MOZ_DEBUG", "OS_TARGET", "HOST_CPU_ARCH", "CPU_ARCH") 157 158 mozbuild_args = {k: config.substs.get(k) for k in gn_mozbuild_vars} 159 gn_out["mozbuild_args"] = mozbuild_args 160 all_deps = find_deps(gn_result["targets"], gn_target) 161 162 for target_fullname in all_deps: 163 raw_spec = gn_result["targets"][target_fullname] 164 165 # TODO: 'executable' will need to be handled here at some point as well. 166 if raw_spec["type"] not in ("static_library", "shared_library", "source_set"): 167 continue 168 169 spec = {} 170 for spec_attr in ( 171 "type", 172 "sources", 173 "defines", 174 "include_dirs", 175 "cflags", 176 "deps", 177 "libs", 178 ): 179 spec[spec_attr] = raw_spec.get(spec_attr, []) 180 gn_out["targets"][target_fullname] = spec 181 182 return gn_out 183 184 185def process_gn_config( 186 gn_config, srcdir, config, output, non_unified_sources, sandbox_vars, mozilla_flags 187): 188 # Translates a json gn config into attributes that can be used to write out 189 # moz.build files for this configuration. 190 191 # Much of this code is based on similar functionality in `gyp_reader.py`. 192 193 mozbuild_attrs = {"mozbuild_args": gn_config.get("mozbuild_args", None), "dirs": {}} 194 195 targets = gn_config["targets"] 196 197 project_relsrcdir = mozpath.relpath(srcdir, config.topsrcdir) 198 199 non_unified_sources = set([mozpath.normpath(s) for s in non_unified_sources]) 200 201 def target_info(fullname): 202 path, name = target_fullname.split(":") 203 # Stripping '//' gives us a path relative to the project root, 204 # adding a suffix avoids name collisions with libraries already 205 # in the tree (like "webrtc"). 206 return path.lstrip("//"), name + "_gn" 207 208 # Process all targets from the given gn project and its dependencies. 209 for target_fullname, spec in six.iteritems(targets): 210 211 target_path, target_name = target_info(target_fullname) 212 context_attrs = {} 213 214 # Remove leading 'lib' from the target_name if any, and use as 215 # library name. 216 name = target_name 217 if spec["type"] in ("static_library", "shared_library", "source_set"): 218 if name.startswith("lib"): 219 name = name[3:] 220 context_attrs["LIBRARY_NAME"] = six.ensure_text(name) 221 else: 222 raise Exception( 223 "The following GN target type is not currently " 224 'consumed by moz.build: "%s". It may need to be ' 225 "added, or you may need to re-run the " 226 "`GnConfigGen` step." % spec["type"] 227 ) 228 229 if spec["type"] == "shared_library": 230 context_attrs["FORCE_SHARED_LIB"] = True 231 232 sources = [] 233 unified_sources = [] 234 extensions = set() 235 use_defines_in_asflags = False 236 237 for f in spec.get("sources", []): 238 f = f.lstrip("//") 239 ext = mozpath.splitext(f)[-1] 240 extensions.add(ext) 241 src = "%s/%s" % (project_relsrcdir, f) 242 if ext == ".h": 243 continue 244 elif ext == ".def": 245 context_attrs["SYMBOLS_FILE"] = src 246 elif ext != ".S" and src not in non_unified_sources: 247 unified_sources.append("/%s" % src) 248 else: 249 sources.append("/%s" % src) 250 # The Mozilla build system doesn't use DEFINES for building 251 # ASFILES. 252 if ext == ".s": 253 use_defines_in_asflags = True 254 255 context_attrs["SOURCES"] = sources 256 context_attrs["UNIFIED_SOURCES"] = unified_sources 257 258 context_attrs["DEFINES"] = {} 259 for define in spec.get("defines", []): 260 if "=" in define: 261 name, value = define.split("=", 1) 262 context_attrs["DEFINES"][name] = value 263 else: 264 context_attrs["DEFINES"][define] = True 265 266 context_attrs["LOCAL_INCLUDES"] = [] 267 for include in spec.get("include_dirs", []): 268 # GN will have resolved all these paths relative to the root of 269 # the project indicated by "//". 270 if include.startswith("//"): 271 include = include[2:] 272 # moz.build expects all LOCAL_INCLUDES to exist, so ensure they do. 273 if include.startswith("/"): 274 resolved = mozpath.abspath(mozpath.join(config.topsrcdir, include[1:])) 275 else: 276 resolved = mozpath.abspath(mozpath.join(srcdir, include)) 277 if not os.path.exists(resolved): 278 # GN files may refer to include dirs that are outside of the 279 # tree or we simply didn't vendor. Print a warning in this case. 280 if not resolved.endswith("gn-output/gen"): 281 print( 282 "Included path: '%s' does not exist, dropping include from GN " 283 "configuration." % resolved, 284 file=sys.stderr, 285 ) 286 continue 287 if not include.startswith("/"): 288 include = "/%s/%s" % (project_relsrcdir, include) 289 context_attrs["LOCAL_INCLUDES"] += [include] 290 291 context_attrs["ASFLAGS"] = spec.get("asflags_mozilla", []) 292 if use_defines_in_asflags and context_attrs["DEFINES"]: 293 context_attrs["ASFLAGS"] += ["-D" + d for d in context_attrs["DEFINES"]] 294 flags = [_f for _f in spec.get("cflags", []) if _f in mozilla_flags] 295 if flags: 296 suffix_map = { 297 ".c": "CFLAGS", 298 ".cpp": "CXXFLAGS", 299 ".cc": "CXXFLAGS", 300 ".m": "CMFLAGS", 301 ".mm": "CMMFLAGS", 302 } 303 variables = (suffix_map[e] for e in extensions if e in suffix_map) 304 for var in variables: 305 for f in flags: 306 # We may be getting make variable references out of the 307 # gn data, and we don't want those in emitted data, so 308 # substitute them with their actual value. 309 f = expand_variables(f, config.substs).split() 310 if not f: 311 continue 312 # the result may be a string or a list. 313 if isinstance(f, six.string_types): 314 context_attrs.setdefault(var, []).append(f) 315 else: 316 context_attrs.setdefault(var, []).extend(f) 317 318 context_attrs["OS_LIBS"] = [] 319 for lib in spec.get("libs", []): 320 lib_name = os.path.splitext(lib)[0] 321 if lib.endswith(".framework"): 322 context_attrs["OS_LIBS"] += ["-framework " + lib_name] 323 else: 324 context_attrs["OS_LIBS"] += [lib_name] 325 326 # Add some features to all contexts. Put here in case LOCAL_INCLUDES 327 # order matters. 328 context_attrs["LOCAL_INCLUDES"] += [ 329 "!/ipc/ipdl/_ipdlheaders", 330 "/ipc/chromium/src", 331 "/ipc/glue", 332 "/tools/profiler/public", 333 ] 334 # These get set via VC project file settings for normal GYP builds. 335 # TODO: Determine if these defines are needed for GN builds. 336 if gn_config["mozbuild_args"]["OS_TARGET"] == "WINNT": 337 context_attrs["DEFINES"]["UNICODE"] = True 338 context_attrs["DEFINES"]["_UNICODE"] = True 339 340 context_attrs["COMPILE_FLAGS"] = {"OS_INCLUDES": []} 341 342 for key, value in sandbox_vars.items(): 343 if context_attrs.get(key) and isinstance(context_attrs[key], list): 344 # If we have a key from sandbox_vars that's also been 345 # populated here we use the value from sandbox_vars as our 346 # basis rather than overriding outright. 347 context_attrs[key] = value + context_attrs[key] 348 elif context_attrs.get(key) and isinstance(context_attrs[key], dict): 349 context_attrs[key].update(value) 350 else: 351 context_attrs[key] = value 352 353 target_relsrcdir = mozpath.join(project_relsrcdir, target_path, target_name) 354 mozbuild_attrs["dirs"][target_relsrcdir] = context_attrs 355 356 return mozbuild_attrs 357 358 359def find_common_attrs(config_attributes): 360 # Returns the intersection of the given configs and prunes the inputs 361 # to no longer contain these common attributes. 362 363 common_attrs = deepcopy(config_attributes[0]) 364 365 def make_intersection(reference, input_attrs): 366 # Modifies `reference` so that after calling this function it only 367 # contains parts it had in common with in `input_attrs`. 368 369 for k, input_value in input_attrs.items(): 370 # Anything in `input_attrs` must match what's already in 371 # `reference`. 372 common_value = reference.get(k) 373 if common_value: 374 if isinstance(input_value, list): 375 reference[k] = [ 376 i 377 for i in common_value 378 if input_value.count(i) == common_value.count(i) 379 ] 380 elif isinstance(input_value, dict): 381 reference[k] = { 382 key: value 383 for key, value in common_value.items() 384 if key in input_value and value == input_value[key] 385 } 386 elif input_value != common_value: 387 del reference[k] 388 elif k in reference: 389 del reference[k] 390 391 # Additionally, any keys in `reference` that aren't in `input_attrs` 392 # must be deleted. 393 for k in set(reference.keys()) - set(input_attrs.keys()): 394 del reference[k] 395 396 def make_difference(reference, input_attrs): 397 # Modifies `input_attrs` so that after calling this function it contains 398 # no parts it has in common with in `reference`. 399 for k, input_value in list(six.iteritems(input_attrs)): 400 common_value = reference.get(k) 401 if common_value: 402 if isinstance(input_value, list): 403 input_attrs[k] = [ 404 i 405 for i in input_value 406 if common_value.count(i) != input_value.count(i) 407 ] 408 elif isinstance(input_value, dict): 409 input_attrs[k] = { 410 key: value 411 for key, value in input_value.items() 412 if key not in common_value 413 } 414 else: 415 del input_attrs[k] 416 417 for config_attr_set in config_attributes[1:]: 418 make_intersection(common_attrs, config_attr_set) 419 420 for config_attr_set in config_attributes: 421 make_difference(common_attrs, config_attr_set) 422 423 return common_attrs 424 425 426def write_mozbuild( 427 config, 428 srcdir, 429 output, 430 non_unified_sources, 431 gn_config_files, 432 mozilla_flags, 433 write_mozbuild_variables, 434): 435 436 all_mozbuild_results = [] 437 438 for path in sorted(gn_config_files): 439 with open(path, "r") as fh: 440 gn_config = json.load(fh) 441 mozbuild_attrs = process_gn_config( 442 gn_config, 443 srcdir, 444 config, 445 output, 446 non_unified_sources, 447 gn_config["sandbox_vars"], 448 mozilla_flags, 449 ) 450 all_mozbuild_results.append(mozbuild_attrs) 451 452 # Translate {config -> {dirs -> build info}} into 453 # {dirs -> [(config, build_info)]} 454 configs_by_dir = defaultdict(list) 455 for config_attrs in all_mozbuild_results: 456 mozbuild_args = config_attrs["mozbuild_args"] 457 dirs = config_attrs["dirs"] 458 for d, build_data in dirs.items(): 459 configs_by_dir[d].append((mozbuild_args, build_data)) 460 461 for relsrcdir, configs in sorted(configs_by_dir.items()): 462 target_srcdir = mozpath.join(config.topsrcdir, relsrcdir) 463 mkdir(target_srcdir) 464 465 target_mozbuild = mozpath.join(target_srcdir, "moz.build") 466 with open(target_mozbuild, "w") as fh: 467 mb = MozbuildWriter(fh) 468 mb.write(license_header) 469 mb.write("\n") 470 mb.write(generated_header) 471 472 try: 473 if relsrcdir in write_mozbuild_variables["INCLUDE_TK_CFLAGS_DIRS"]: 474 mb.write('if CONFIG["MOZ_WIDGET_TOOLKIT"] == "gtk":\n') 475 mb.write(' CXXFLAGS += CONFIG["TK_CFLAGS"]\n') 476 except KeyError: 477 pass 478 479 all_args = [args for args, _ in configs] 480 481 # Start with attributes that will be a part of the mozconfig 482 # for every configuration, then factor by other potentially useful 483 # combinations. 484 for attrs in ( 485 (), 486 ("MOZ_DEBUG",), 487 ("OS_TARGET",), 488 ("CPU_ARCH",), 489 ("MOZ_DEBUG", "OS_TARGET"), 490 ("OS_TARGET", "CPU_ARCH"), 491 ("OS_TARGET", "CPU_ARCH", "MOZ_DEBUG"), 492 ("MOZ_DEBUG", "OS_TARGET", "CPU_ARCH", "HOST_CPU_ARCH"), 493 ): 494 conditions = set() 495 for args in all_args: 496 cond = tuple(((k, args.get(k) or "") for k in attrs)) 497 conditions.add(cond) 498 499 for cond in sorted(conditions): 500 common_attrs = find_common_attrs( 501 [ 502 attrs 503 for args, attrs in configs 504 if all((args.get(k) or "") == v for k, v in cond) 505 ] 506 ) 507 if any(common_attrs.values()): 508 if cond: 509 mb.write_condition(dict(cond)) 510 mb.write_attrs(common_attrs) 511 if cond: 512 mb.terminate_condition() 513 514 mb.finalize() 515 516 dirs_mozbuild = mozpath.join(srcdir, "moz.build") 517 with open(dirs_mozbuild, "w") as fh: 518 mb = MozbuildWriter(fh) 519 mb.write(license_header) 520 mb.write("\n") 521 mb.write(generated_header) 522 523 # Not every srcdir is present for every config, which needs to be 524 # reflected in the generated root moz.build. 525 dirs_by_config = { 526 tuple(v["mozbuild_args"].items()): set(v["dirs"].keys()) 527 for v in all_mozbuild_results 528 } 529 530 for attrs in ((), ("OS_TARGET",), ("OS_TARGET", "CPU_ARCH")): 531 532 conditions = set() 533 for args in dirs_by_config.keys(): 534 cond = tuple(((k, dict(args).get(k) or "") for k in attrs)) 535 conditions.add(cond) 536 537 for cond in sorted(conditions): 538 common_dirs = None 539 for args, dir_set in dirs_by_config.items(): 540 if all((dict(args).get(k) or "") == v for k, v in cond): 541 if common_dirs is None: 542 common_dirs = deepcopy(dir_set) 543 else: 544 common_dirs &= dir_set 545 546 for args, dir_set in dirs_by_config.items(): 547 if all(dict(args).get(k) == v for k, v in cond): 548 dir_set -= common_dirs 549 550 if common_dirs: 551 if cond: 552 mb.write_condition(dict(cond)) 553 mb.write_mozbuild_list("DIRS", ["/%s" % d for d in common_dirs]) 554 if cond: 555 mb.terminate_condition() 556 557 558def generate_gn_config( 559 config, 560 srcdir, 561 output, 562 non_unified_sources, 563 gn_binary, 564 input_variables, 565 sandbox_variables, 566 gn_target, 567): 568 def str_for_arg(v): 569 if v in (True, False): 570 return str(v).lower() 571 return '"%s"' % v 572 573 gn_args = "--args=%s" % " ".join( 574 ["%s=%s" % (k, str_for_arg(v)) for k, v in six.iteritems(input_variables)] 575 ) 576 gn_arg_string = "_".join( 577 [str(input_variables[k]) for k in sorted(input_variables.keys())] 578 ) 579 out_dir = mozpath.join(output, "gn-output") 580 gen_args = [config.substs["GN"], "gen", out_dir, gn_args, "--ide=json"] 581 print('Running "%s"' % " ".join(gen_args), file=sys.stderr) 582 subprocess.check_call(gen_args, cwd=srcdir, stderr=subprocess.STDOUT) 583 584 gn_config_file = mozpath.join(out_dir, "project.json") 585 586 with open(gn_config_file, "r") as fh: 587 gn_out = json.load(fh) 588 gn_out = filter_gn_config( 589 gn_out, config, sandbox_variables, input_variables, gn_target 590 ) 591 592 os.remove(gn_config_file) 593 594 gn_out_file = mozpath.join(out_dir, gn_arg_string + ".json") 595 with open(gn_out_file, "w") as fh: 596 json.dump(gn_out, fh, indent=4, sort_keys=True, separators=(",", ": ")) 597 print("Wrote gn config to %s" % gn_out_file) 598 599 600class GnConfigGenBackend(BuildBackend): 601 def consume_object(self, obj): 602 if isinstance(obj, GnProjectData): 603 gn_binary = obj.config.substs.get("GN") 604 if not gn_binary: 605 raise Exception( 606 "The GN program must be present to generate GN configs." 607 ) 608 609 generate_gn_config( 610 obj.config, 611 mozpath.join(obj.srcdir, obj.target_dir), 612 mozpath.join(obj.objdir, obj.target_dir), 613 obj.non_unified_sources, 614 gn_binary, 615 obj.gn_input_variables, 616 obj.gn_sandbox_variables, 617 obj.gn_target, 618 ) 619 return True 620 621 def consume_finished(self): 622 pass 623 624 625class GnMozbuildWriterBackend(BuildBackend): 626 def consume_object(self, obj): 627 if isinstance(obj, GnProjectData): 628 gn_config_files = glob.glob( 629 mozpath.join(obj.srcdir, "gn-configs", "*.json") 630 ) 631 if not gn_config_files: 632 # Check the objdir for a gn-config in to aide debugging in cases 633 # someone is running both steps on the same machine and want to 634 # sanity check moz.build generation for a particular config. 635 gn_config_files = glob.glob( 636 mozpath.join(obj.objdir, obj.target_dir, "gn-output", "*.json") 637 ) 638 if gn_config_files: 639 print( 640 "Writing moz.build files based on the following gn configs: %s" 641 % gn_config_files 642 ) 643 write_mozbuild( 644 obj.config, 645 mozpath.join(obj.srcdir, obj.target_dir), 646 mozpath.join(obj.objdir, obj.target_dir), 647 obj.non_unified_sources, 648 gn_config_files, 649 obj.mozilla_flags, 650 obj.write_mozbuild_variables, 651 ) 652 else: 653 print( 654 "Ignoring gn project '%s', no config files found in '%s'" 655 % (obj.srcdir, mozpath.join(obj.srcdir, "gn-configs")) 656 ) 657 return True 658 659 def consume_finished(self): 660 pass 661