1# This Source Code Form is subject to the terms of the Mozilla Public 2# License, v. 2.0. If a copy of the MPL was not distributed with this 3# file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5from __future__ import print_function 6 7from collections import defaultdict 8from copy import deepcopy 9import glob 10import json 11import os 12import subprocess 13import sys 14import types 15 16from mozbuild.backend.base import BuildBackend 17import mozpack.path as mozpath 18from mozbuild.frontend.sandbox import alphabetical_sorted 19from mozbuild.frontend.data import GnProjectData 20from mozbuild.util import expand_variables, mkdir 21 22 23license_header = """# This Source Code Form is subject to the terms of the Mozilla Public 24# License, v. 2.0. If a copy of the MPL was not distributed with this 25# file, You can obtain one at http://mozilla.org/MPL/2.0/. 26""" 27 28generated_header = """ 29 ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### 30 ### DO NOT edit it by hand. ### 31""" 32 33 34class MozbuildWriter(object): 35 def __init__(self, fh): 36 self._fh = fh 37 self.indent = '' 38 self._indent_increment = 4 39 40 # We need to correlate a small amount of state here to figure out 41 # which library template to use ("Library()" or "SharedLibrary()") 42 self._library_name = None 43 self._shared_library = None 44 45 def mb_serialize(self, v): 46 if isinstance(v, (bool, list)): 47 return repr(v) 48 return '"%s"' % v 49 50 def finalize(self): 51 if self._library_name: 52 self.write('\n') 53 if self._shared_library: 54 self.write_ln("SharedLibrary(%s)" % self.mb_serialize(self._library_name)) 55 else: 56 self.write_ln("Library(%s)" % self.mb_serialize(self._library_name)) 57 58 def write(self, content): 59 self._fh.write(content) 60 61 def write_ln(self, line): 62 self.write(self.indent) 63 self.write(line) 64 self.write('\n') 65 66 def write_attrs(self, context_attrs): 67 for k in sorted(context_attrs.keys()): 68 v = context_attrs[k] 69 if isinstance(v, (list, set)): 70 self.write_mozbuild_list(k, v) 71 elif isinstance(v, dict): 72 self.write_mozbuild_dict(k, v) 73 else: 74 self.write_mozbuild_value(k, v) 75 76 def write_mozbuild_list(self, key, value): 77 if value: 78 self.write('\n') 79 self.write(self.indent + key) 80 self.write(' += [\n ' + self.indent) 81 self.write((',\n ' + self.indent).join(alphabetical_sorted(self.mb_serialize(v) for v in value))) 82 self.write('\n') 83 self.write_ln(']') 84 85 def write_mozbuild_value(self, key, value): 86 if value: 87 if key == 'LIBRARY_NAME': 88 self._library_name = value 89 elif key == 'FORCE_SHARED_LIB': 90 self._shared_library = True 91 else: 92 self.write('\n') 93 self.write_ln('%s = %s' % (key, self.mb_serialize(value))) 94 self.write('\n') 95 96 def write_mozbuild_dict(self, key, value): 97 # Templates we need to use instead of certain values. 98 replacements = ( 99 (('COMPILE_FLAGS', '"WARNINGS_AS_ERRORS"', '[]'), 'AllowCompilerWarnings()'), 100 ) 101 if value: 102 self.write('\n') 103 for k in sorted(value.keys()): 104 v = value[k] 105 subst_vals = key, self.mb_serialize(k), self.mb_serialize(v) 106 wrote_ln = False 107 for flags, tmpl in replacements: 108 if subst_vals == flags: 109 self.write_ln(tmpl) 110 wrote_ln = True 111 112 if not wrote_ln: 113 self.write_ln("%s[%s] = %s" % subst_vals) 114 115 116 def write_condition(self, values): 117 def mk_condition(k, v): 118 if not v: 119 return 'not CONFIG["%s"]' % k 120 return 'CONFIG["%s"] == %s' % (k, self.mb_serialize(v)) 121 122 self.write('\n') 123 self.write('if ') 124 self.write(' and '.join(mk_condition(k, values[k]) for k in sorted(values.keys()))) 125 self.write(':\n') 126 self.indent += ' ' * self._indent_increment 127 128 def terminate_condition(self): 129 assert len(self.indent) >= self._indent_increment 130 self.indent = self.indent[self._indent_increment:] 131 132 133def find_deps(all_targets, target): 134 all_deps = set([target]) 135 for dep in all_targets[target]['deps']: 136 if dep not in all_deps: 137 all_deps |= find_deps(all_targets, dep) 138 return all_deps 139 140 141def filter_gn_config(gn_result, config, sandbox_vars, input_vars): 142 # Translates the raw output of gn into just what we'll need to generate a 143 # mozbuild configuration. 144 gn_out = { 145 'targets': {}, 146 'sandbox_vars': sandbox_vars, 147 'gn_gen_args': input_vars, 148 } 149 150 gn_mozbuild_vars = ( 151 'MOZ_DEBUG', 152 'OS_TARGET', 153 'HOST_CPU_ARCH', 154 'CPU_ARCH', 155 ) 156 157 mozbuild_args = {k: config.substs.get(k) for k in gn_mozbuild_vars} 158 gn_out['mozbuild_args'] = mozbuild_args 159 all_deps = find_deps(gn_result['targets'], "//:default") 160 161 for target_fullname in all_deps: 162 raw_spec = gn_result['targets'][target_fullname] 163 164 # TODO: 'executable' will need to be handled here at some point as well. 165 if raw_spec['type'] not in ('static_library', 'shared_library', 166 'source_set'): 167 continue 168 169 spec = {} 170 for spec_attr in ('type', 'sources', 'defines', 'include_dirs', 171 'cflags', 'deps', 'libs'): 172 spec[spec_attr] = raw_spec.get(spec_attr, []) 173 gn_out['targets'][target_fullname] = spec 174 175 return gn_out 176 177 178def process_gn_config(gn_config, srcdir, config, output, non_unified_sources, 179 sandbox_vars, mozilla_flags): 180 # Translates a json gn config into attributes that can be used to write out 181 # moz.build files for this configuration. 182 183 # Much of this code is based on similar functionality in `gyp_reader.py`. 184 185 mozbuild_attrs = {'mozbuild_args': gn_config.get('mozbuild_args', None), 186 'dirs': {}} 187 188 targets = gn_config["targets"] 189 190 project_relsrcdir = mozpath.relpath(srcdir, config.topsrcdir) 191 192 def target_info(fullname): 193 path, name = target_fullname.split(':') 194 # Stripping '//' gives us a path relative to the project root, 195 # adding a suffix avoids name collisions with libraries already 196 # in the tree (like "webrtc"). 197 return path.lstrip('//'), name + '_gn' 198 199 # Process all targets from the given gn project and its dependencies. 200 for target_fullname, spec in targets.iteritems(): 201 202 target_path, target_name = target_info(target_fullname) 203 context_attrs = {} 204 205 # Remove leading 'lib' from the target_name if any, and use as 206 # library name. 207 name = target_name 208 if spec['type'] in ('static_library', 'shared_library', 'source_set'): 209 if name.startswith('lib'): 210 name = name[3:] 211 context_attrs['LIBRARY_NAME'] = name.decode('utf-8') 212 else: 213 raise Exception('The following GN target type is not currently ' 214 'consumed by moz.build: "%s". It may need to be ' 215 'added, or you may need to re-run the ' 216 '`GnConfigGen` step.' % spec['type']) 217 218 if spec['type'] == 'shared_library': 219 context_attrs['FORCE_SHARED_LIB'] = True 220 221 sources = [] 222 unified_sources = [] 223 extensions = set() 224 use_defines_in_asflags = False 225 226 for f in spec.get('sources', []): 227 f = f.lstrip("//") 228 ext = mozpath.splitext(f)[-1] 229 extensions.add(ext) 230 src = '%s/%s' % (project_relsrcdir, f) 231 if ext == '.h': 232 continue 233 elif ext == '.def': 234 context_attrs['SYMBOLS_FILE'] = src 235 elif ext != '.S' and src not in non_unified_sources: 236 unified_sources.append('/%s' % src) 237 else: 238 sources.append('/%s' % src) 239 # The Mozilla build system doesn't use DEFINES for building 240 # ASFILES. 241 if ext == '.s': 242 use_defines_in_asflags = True 243 244 context_attrs['SOURCES'] = sources 245 context_attrs['UNIFIED_SOURCES'] = unified_sources 246 247 context_attrs['DEFINES'] = {} 248 for define in spec.get('defines', []): 249 if '=' in define: 250 name, value = define.split('=', 1) 251 context_attrs['DEFINES'][name] = value 252 else: 253 context_attrs['DEFINES'][define] = True 254 255 context_attrs['LOCAL_INCLUDES'] = [] 256 for include in spec.get('include_dirs', []): 257 # GN will have resolved all these paths relative to the root of 258 # the project indicated by "//". 259 if include.startswith('//'): 260 include = include[2:] 261 # moz.build expects all LOCAL_INCLUDES to exist, so ensure they do. 262 if include.startswith('/'): 263 resolved = mozpath.abspath(mozpath.join(config.topsrcdir, include[1:])) 264 else: 265 resolved = mozpath.abspath(mozpath.join(srcdir, include)) 266 if not os.path.exists(resolved): 267 # GN files may refer to include dirs that are outside of the 268 # tree or we simply didn't vendor. Print a warning in this case. 269 if not resolved.endswith('gn-output/gen'): 270 print("Included path: '%s' does not exist, dropping include from GN " 271 "configuration." % resolved, file=sys.stderr) 272 continue 273 if not include.startswith('/'): 274 include = '/%s/%s' % (project_relsrcdir, include) 275 context_attrs['LOCAL_INCLUDES'] += [include] 276 277 context_attrs['ASFLAGS'] = spec.get('asflags_mozilla', []) 278 if use_defines_in_asflags and defines: 279 context_attrs['ASFLAGS'] += ['-D' + d for d in defines] 280 flags = [f for f in spec.get('cflags', []) if f in mozilla_flags] 281 if flags: 282 suffix_map = { 283 '.c': 'CFLAGS', 284 '.cpp': 'CXXFLAGS', 285 '.cc': 'CXXFLAGS', 286 '.m': 'CMFLAGS', 287 '.mm': 'CMMFLAGS', 288 } 289 variables = (suffix_map[e] for e in extensions if e in suffix_map) 290 for var in variables: 291 for f in flags: 292 # We may be getting make variable references out of the 293 # gn data, and we don't want those in emitted data, so 294 # substitute them with their actual value. 295 f = expand_variables(f, config.substs).split() 296 if not f: 297 continue 298 # the result may be a string or a list. 299 if isinstance(f, types.StringTypes): 300 context_attrs.setdefault(var, []).append(f) 301 else: 302 context_attrs.setdefault(var, []).extend(f) 303 304 context_attrs['OS_LIBS'] = [] 305 for lib in spec.get('libs', []): 306 lib_name = os.path.splitext(lib)[0] 307 if lib.endswith('.framework'): 308 context_attrs['OS_LIBS'] += ['-framework ' + lib_name] 309 else: 310 context_attrs['OS_LIBS'] += [lib_name] 311 312 # Add some features to all contexts. Put here in case LOCAL_INCLUDES 313 # order matters. 314 context_attrs['LOCAL_INCLUDES'] += [ 315 '!/ipc/ipdl/_ipdlheaders', 316 '/ipc/chromium/src', 317 '/ipc/glue', 318 ] 319 # These get set via VC project file settings for normal GYP builds. 320 # TODO: Determine if these defines are needed for GN builds. 321 if gn_config['mozbuild_args']['OS_TARGET'] == 'WINNT': 322 context_attrs['DEFINES']['UNICODE'] = True 323 context_attrs['DEFINES']['_UNICODE'] = True 324 325 context_attrs['COMPILE_FLAGS'] = { 326 'STL': [], 327 'OS_INCLUDES': [], 328 } 329 330 for key, value in sandbox_vars.items(): 331 if context_attrs.get(key) and isinstance(context_attrs[key], list): 332 # If we have a key from sandbox_vars that's also been 333 # populated here we use the value from sandbox_vars as our 334 # basis rather than overriding outright. 335 context_attrs[key] = value + context_attrs[key] 336 elif context_attrs.get(key) and isinstance(context_attrs[key], dict): 337 context_attrs[key].update(value) 338 else: 339 context_attrs[key] = value 340 341 target_relsrcdir = mozpath.join(project_relsrcdir, target_path, target_name) 342 mozbuild_attrs['dirs'][target_relsrcdir] = context_attrs 343 344 return mozbuild_attrs 345 346 347def find_common_attrs(config_attributes): 348 # Returns the intersection of the given configs and prunes the inputs 349 # to no longer contain these common attributes. 350 351 common_attrs = deepcopy(config_attributes[0]) 352 353 def make_intersection(reference, input_attrs): 354 # Modifies `reference` so that after calling this function it only 355 # contains parts it had in common with in `input_attrs`. 356 357 for k, input_value in input_attrs.items(): 358 # Anything in `input_attrs` must match what's already in 359 # `reference`. 360 common_value = reference.get(k) 361 if common_value: 362 if isinstance(input_value, list): 363 input_value = set(input_value) 364 reference[k] = [i for i in common_value if i in input_value] 365 elif isinstance(input_value, dict): 366 reference[k] = {key: value for key, value in common_value.items() 367 if key in input_value and value == input_value[key]} 368 elif input_value != common_value: 369 del reference[k] 370 elif k in reference: 371 del reference[k] 372 373 # Additionally, any keys in `reference` that aren't in `input_attrs` 374 # must be deleted. 375 for k in set(reference.keys()) - set(input_attrs.keys()): 376 del reference[k] 377 378 def make_difference(reference, input_attrs): 379 # Modifies `input_attrs` so that after calling this function it contains 380 # no parts it has in common with in `reference`. 381 for k, input_value in input_attrs.items(): 382 common_value = reference.get(k) 383 if common_value: 384 if isinstance(input_value, list): 385 common_value = set(common_value) 386 input_attrs[k] = [i for i in input_value if i not in common_value] 387 elif isinstance(input_value, dict): 388 input_attrs[k] = {key: value for key, value in input_value.items() 389 if key not in common_value} 390 else: 391 del input_attrs[k] 392 393 for config_attr_set in config_attributes[1:]: 394 make_intersection(common_attrs, config_attr_set) 395 396 for config_attr_set in config_attributes: 397 make_difference(common_attrs, config_attr_set) 398 399 return common_attrs 400 401 402def write_mozbuild(config, srcdir, output, non_unified_sources, gn_config_files, 403 mozilla_flags): 404 405 all_mozbuild_results = [] 406 407 for path in gn_config_files: 408 with open(path, 'r') as fh: 409 gn_config = json.load(fh) 410 mozbuild_attrs = process_gn_config(gn_config, srcdir, config, 411 output, non_unified_sources, 412 gn_config['sandbox_vars'], 413 mozilla_flags) 414 all_mozbuild_results.append(mozbuild_attrs) 415 416 # Translate {config -> {dirs -> build info}} into 417 # {dirs -> [(config, build_info)]} 418 configs_by_dir = defaultdict(list) 419 for config_attrs in all_mozbuild_results: 420 mozbuild_args = config_attrs['mozbuild_args'] 421 dirs = config_attrs['dirs'] 422 for d, build_data in dirs.items(): 423 configs_by_dir[d].append((mozbuild_args, build_data)) 424 425 for relsrcdir, configs in configs_by_dir.items(): 426 target_srcdir = mozpath.join(config.topsrcdir, relsrcdir) 427 mkdir(target_srcdir) 428 429 target_mozbuild = mozpath.join(target_srcdir, 'moz.build') 430 with open(target_mozbuild, 'w') as fh: 431 mb = MozbuildWriter(fh) 432 mb.write(license_header) 433 mb.write('\n') 434 mb.write(generated_header) 435 436 all_attr_sets = [attrs for _, attrs in configs] 437 all_args = [args for args, _ in configs] 438 439 # Start with attributes that will be a part of the mozconfig 440 # for every configuration, then factor by other potentially useful 441 # combinations. 442 for attrs in ((), 443 ('MOZ_DEBUG',), ('OS_TARGET',), ('MOZ_DEBUG', 'OS_TARGET',), 444 ('MOZ_DEBUG', 'OS_TARGET', 'CPU_ARCH', 'HOST_CPU_ARCH')): 445 conditions = set() 446 for args in all_args: 447 cond = tuple(((k, args.get(k)) for k in attrs)) 448 conditions.add(cond) 449 for cond in sorted(conditions): 450 common_attrs = find_common_attrs([attrs for args, attrs in configs if 451 all(args.get(k) == v for k, v in cond)]) 452 if any(common_attrs.values()): 453 if cond: 454 mb.write_condition(dict(cond)) 455 mb.write_attrs(common_attrs) 456 if cond: 457 mb.terminate_condition() 458 459 mb.finalize() 460 461 dirs_mozbuild = mozpath.join(srcdir, 'moz.build') 462 with open(dirs_mozbuild, 'w') as fh: 463 mb = MozbuildWriter(fh) 464 mb.write(license_header) 465 mb.write('\n') 466 mb.write(generated_header) 467 468 # Not every srcdir is present for every config, which needs to be 469 # reflected in the generated root moz.build. 470 dirs_by_config = {tuple(v['mozbuild_args'].items()): set(v['dirs'].keys()) 471 for v in all_mozbuild_results} 472 473 for attrs in ((), ('OS_TARGET',), ('OS_TARGET', 'CPU_ARCH')): 474 475 conditions = set() 476 for args in dirs_by_config.keys(): 477 cond = tuple(((k, dict(args).get(k)) for k in attrs)) 478 conditions.add(cond) 479 480 for cond in conditions: 481 common_dirs = None 482 for args, dir_set in dirs_by_config.items(): 483 if all(dict(args).get(k) == v for k, v in cond): 484 if common_dirs is None: 485 common_dirs = deepcopy(dir_set) 486 else: 487 common_dirs &= dir_set 488 489 for args, dir_set in dirs_by_config.items(): 490 if all(dict(args).get(k) == v for k, v in cond): 491 dir_set -= common_dirs 492 493 if common_dirs: 494 if cond: 495 mb.write_condition(dict(cond)) 496 mb.write_mozbuild_list('DIRS', 497 ['/%s' % d for d in common_dirs]) 498 if cond: 499 mb.terminate_condition() 500 501 502def generate_gn_config(config, srcdir, output, non_unified_sources, gn_binary, 503 input_variables, sandbox_variables): 504 505 def str_for_arg(v): 506 if v in (True, False): 507 return str(v).lower() 508 return '"%s"' % v 509 510 gn_args = '--args=%s' % ' '.join(['%s=%s' % (k, str_for_arg(v)) for k, v 511 in input_variables.iteritems()]) 512 gn_arg_string = '_'.join([str(input_variables[k]) for k in sorted(input_variables.keys())]) 513 out_dir = mozpath.join(output, 'gn-output') 514 gen_args = [ 515 config.substs['GN'], 'gen', out_dir, gn_args, '--ide=json', 516 ] 517 print("Running \"%s\"" % ' '.join(gen_args), file=sys.stderr) 518 subprocess.check_call(gen_args, cwd=srcdir, stderr=subprocess.STDOUT) 519 520 521 gn_config_file = mozpath.join(out_dir, 'project.json') 522 523 with open(gn_config_file, 'r') as fh: 524 gn_out = json.load(fh) 525 gn_out = filter_gn_config(gn_out, config, sandbox_variables, 526 input_variables) 527 528 os.remove(gn_config_file) 529 530 gn_out_file = mozpath.join(out_dir, gn_arg_string + '.json') 531 with open(gn_out_file, 'w') as fh: 532 json.dump(gn_out, fh, indent=4, sort_keys=True, separators=(',', ': ')) 533 print("Wrote gn config to %s" % gn_out_file) 534 535 536class GnConfigGenBackend(BuildBackend): 537 538 def consume_object(self, obj): 539 if isinstance(obj, GnProjectData): 540 gn_binary = obj.config.substs.get('GN') 541 if not gn_binary: 542 raise Exception("The GN program must be present to generate GN configs.") 543 544 generate_gn_config(obj.config, mozpath.join(obj.srcdir, obj.target_dir), 545 mozpath.join(obj.objdir, obj.target_dir), 546 obj.non_unified_sources, gn_binary, 547 obj.gn_input_variables, obj.gn_sandbox_variables) 548 return True 549 550 def consume_finished(self): 551 pass 552 553 554class GnMozbuildWriterBackend(BuildBackend): 555 556 def consume_object(self, obj): 557 if isinstance(obj, GnProjectData): 558 gn_config_files = glob.glob(mozpath.join(obj.srcdir, 'gn-configs', '*.json')) 559 if not gn_config_files: 560 # Check the objdir for a gn-config in to aide debugging in cases 561 # someone is running both steps on the same machine and want to 562 # sanity check moz.build generation for a particular config. 563 gn_config_files = glob.glob(mozpath.join(obj.objdir, obj.target_dir, 564 'gn-output', '*.json')) 565 if gn_config_files: 566 print("Writing moz.build files based on the following gn configs: %s" % 567 gn_config_files) 568 write_mozbuild(obj.config, mozpath.join(obj.srcdir, obj.target_dir), 569 mozpath.join(obj.objdir, obj.target_dir), 570 obj.non_unified_sources, gn_config_files, 571 obj.mozilla_flags) 572 else: 573 print("Ignoring gn project '%s', no config files found in '%s'" % 574 (obj.srcdir, mozpath.join(obj.srcdir, 'gn-configs'))) 575 return True 576 577 def consume_finished(self): 578 pass 579