1#!/usr/bin/python 2# Copyright 2020 The Chromium Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""Tests for mb.py.""" 7 8from __future__ import print_function 9from __future__ import absolute_import 10 11import json 12import os 13import re 14import StringIO 15import sys 16import unittest 17 18sys.path.insert( 19 0, 20 os.path.abspath( 21 os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))) 22 23from mb import mb 24 25 26class FakeMBW(mb.MetaBuildWrapper): 27 def __init__(self, win32=False): 28 super(FakeMBW, self).__init__() 29 30 # Override vars for test portability. 31 if win32: 32 self.chromium_src_dir = 'c:\\fake_src' 33 self.default_config_master = 'c:\\fake_src\\tools\\mb\\mb_config.pyl' 34 35 self.default_config_bucket = 'c:\\fake_src\\tools\\mb\\mb_config_bucket.pyl' # pylint: disable=line-too-long 36 self.default_isolate_map = ('c:\\fake_src\\testing\\buildbot\\' 37 'gn_isolate_map.pyl') 38 self.platform = 'win32' 39 self.executable = 'c:\\python\\python.exe' 40 self.sep = '\\' 41 self.cwd = 'c:\\fake_src\\out\\Default' 42 else: 43 self.chromium_src_dir = '/fake_src' 44 self.default_config_master = '/fake_src/tools/mb/mb_config.pyl' 45 self.default_config_bucket = '/fake_src/tools/mb/mb_config_bucket.pyl' 46 self.default_isolate_map = '/fake_src/testing/buildbot/gn_isolate_map.pyl' 47 self.executable = '/usr/bin/python' 48 self.platform = 'linux2' 49 self.sep = '/' 50 self.cwd = '/fake_src/out/Default' 51 52 self.files = {} 53 self.calls = [] 54 self.cmds = [] 55 self.cross_compile = None 56 self.out = '' 57 self.err = '' 58 self.rmdirs = [] 59 60 def ExpandUser(self, path): 61 return '$HOME/%s' % path 62 63 def Exists(self, path): 64 return self.files.get(self._AbsPath(path)) is not None 65 66 def MaybeMakeDirectory(self, path): 67 abpath = self._AbsPath(path) 68 self.files[abpath] = True 69 70 def PathJoin(self, *comps): 71 return self.sep.join(comps) 72 73 def ReadFile(self, path): 74 return self.files[self._AbsPath(path)] 75 76 def WriteFile(self, path, contents, force_verbose=False): 77 if self.args.dryrun or self.args.verbose or force_verbose: 78 self.Print('\nWriting """\\\n%s""" to %s.\n' % (contents, path)) 79 abpath = self._AbsPath(path) 80 self.files[abpath] = contents 81 82 def Call(self, cmd, env=None, buffer_output=True, stdin=None): 83 self.calls.append(cmd) 84 if self.cmds: 85 return self.cmds.pop(0) 86 return 0, '', '' 87 88 def Print(self, *args, **kwargs): 89 sep = kwargs.get('sep', ' ') 90 end = kwargs.get('end', '\n') 91 f = kwargs.get('file', sys.stdout) 92 if f == sys.stderr: 93 self.err += sep.join(args) + end 94 else: 95 self.out += sep.join(args) + end 96 97 def TempFile(self, mode='w'): 98 return FakeFile(self.files) 99 100 def RemoveFile(self, path): 101 abpath = self._AbsPath(path) 102 self.files[abpath] = None 103 104 def RemoveDirectory(self, path): 105 abpath = self._AbsPath(path) 106 self.rmdirs.append(abpath) 107 files_to_delete = [f for f in self.files if f.startswith(abpath)] 108 for f in files_to_delete: 109 self.files[f] = None 110 111 def _AbsPath(self, path): 112 if not ((self.platform == 'win32' and path.startswith('c:')) or 113 (self.platform != 'win32' and path.startswith('/'))): 114 path = self.PathJoin(self.cwd, path) 115 if self.sep == '\\': 116 return re.sub(r'\\+', r'\\', path) 117 else: 118 return re.sub('/+', '/', path) 119 120 121class FakeFile(object): 122 def __init__(self, files): 123 self.name = '/tmp/file' 124 self.buf = '' 125 self.files = files 126 127 def write(self, contents): 128 self.buf += contents 129 130 def close(self): 131 self.files[self.name] = self.buf 132 133 134TEST_CONFIG = """\ 135{ 136 'masters': { 137 'chromium': {}, 138 'fake_master': { 139 'fake_builder': 'rel_bot', 140 'fake_debug_builder': 'debug_goma', 141 'fake_simplechrome_builder': 'cros_chrome_sdk', 142 'fake_args_bot': '//build/args/bots/fake_master/fake_args_bot.gn', 143 'fake_multi_phase': { 'phase_1': 'phase_1', 'phase_2': 'phase_2'}, 144 'fake_args_file': 'args_file_goma', 145 'fake_ios_error': 'ios_error', 146 }, 147 }, 148 'configs': { 149 'args_file_goma': ['args_file', 'goma'], 150 'cros_chrome_sdk': ['cros_chrome_sdk'], 151 'rel_bot': ['rel', 'goma', 'fake_feature1'], 152 'debug_goma': ['debug', 'goma'], 153 'phase_1': ['phase_1'], 154 'phase_2': ['phase_2'], 155 'ios_error': ['error'], 156 }, 157 'mixins': { 158 'cros_chrome_sdk': { 159 'cros_passthrough': True, 160 }, 161 'error': { 162 'gn_args': 'error', 163 }, 164 'fake_feature1': { 165 'gn_args': 'enable_doom_melon=true', 166 }, 167 'goma': { 168 'gn_args': 'use_goma=true', 169 }, 170 'args_file': { 171 'args_file': '//build/args/fake.gn', 172 }, 173 'phase_1': { 174 'gn_args': 'phase=1', 175 }, 176 'phase_2': { 177 'gn_args': 'phase=2', 178 }, 179 'rel': { 180 'gn_args': 'is_debug=false', 181 }, 182 'debug': { 183 'gn_args': 'is_debug=true', 184 }, 185 }, 186} 187""" 188 189TEST_CONFIG_BUCKET = """\ 190{ 191 'public_artifact_builders': {}, 192 'buckets': { 193 'ci': { 194 'fake_builder': 'rel_bot', 195 'fake_debug_builder': 'debug_goma', 196 'fake_simplechrome_builder': 'cros_chrome_sdk', 197 'fake_args_bot': '//build/args/bots/fake_master/fake_args_bot.gn', 198 'fake_multi_phase': { 'phase_1': 'phase_1', 'phase_2': 'phase_2'}, 199 'fake_args_file': 'args_file_goma', 200 } 201 }, 202 'configs': { 203 'args_file_goma': ['args_file', 'goma'], 204 'cros_chrome_sdk': ['cros_chrome_sdk'], 205 'rel_bot': ['rel', 'goma', 'fake_feature1'], 206 'debug_goma': ['debug', 'goma'], 207 'phase_1': ['phase_1'], 208 'phase_2': ['phase_2'], 209 }, 210 'mixins': { 211 'cros_chrome_sdk': { 212 'cros_passthrough': True, 213 }, 214 'fake_feature1': { 215 'gn_args': 'enable_doom_melon=true', 216 }, 217 'goma': { 218 'gn_args': 'use_goma=true', 219 }, 220 'args_file': { 221 'args_file': '//build/args/fake.gn', 222 }, 223 'phase_1': { 224 'gn_args': 'phase=1', 225 }, 226 'phase_2': { 227 'gn_args': 'phase=2', 228 }, 229 'rel': { 230 'gn_args': 'is_debug=false', 231 }, 232 'debug': { 233 'gn_args': 'is_debug=true', 234 }, 235 }, 236} 237""" 238 239# Same as previous config but with fake_debug_builder 240# and fake_simplechrome_builder configs swapped 241TEST_CONFIG_BUCKET_2 = """\ 242{ 243 'public_artifact_builders': {}, 244 'buckets': { 245 'ci': { 246 'fake_builder': 'rel_bot', 247 'fake_debug_builder': 'cros_chrome_sdk', 248 'fake_simplechrome_builder': 'debug_goma', 249 'fake_args_bot': '//build/args/bots/fake_master/fake_args_bot.gn', 250 'fake_multi_phase': { 'phase_1': 'phase_1', 'phase_2': 'phase_2'}, 251 'fake_args_file': 'args_file_goma', 252 } 253 }, 254 'configs': { 255 'args_file_goma': ['args_file', 'goma'], 256 'cros_chrome_sdk': ['cros_chrome_sdk'], 257 'rel_bot': ['rel', 'goma', 'fake_feature1'], 258 'debug_goma': ['debug', 'goma'], 259 'phase_1': ['phase_1'], 260 'phase_2': ['phase_2'], 261 }, 262 'mixins': { 263 'cros_chrome_sdk': { 264 'cros_passthrough': True, 265 }, 266 'fake_feature1': { 267 'gn_args': 'enable_doom_melon=true', 268 }, 269 'goma': { 270 'gn_args': 'use_goma=true', 271 }, 272 'args_file': { 273 'args_file': '//build/args/fake.gn', 274 }, 275 'phase_1': { 276 'gn_args': 'phase=1', 277 }, 278 'phase_2': { 279 'gn_args': 'phase=2', 280 }, 281 'rel': { 282 'gn_args': 'is_debug=false', 283 }, 284 'debug': { 285 'gn_args': 'is_debug=true', 286 }, 287 }, 288} 289""" 290 291TEST_BAD_CONFIG = """\ 292{ 293 'configs': { 294 'rel_bot_1': ['rel', 'chrome_with_codecs'], 295 'rel_bot_2': ['rel', 'bad_nested_config'], 296 }, 297 'masters': { 298 'chromium': { 299 'a': 'rel_bot_1', 300 'b': 'rel_bot_2', 301 }, 302 }, 303 'mixins': { 304 'chrome_with_codecs': { 305 'gn_args': 'proprietary_codecs=true', 306 }, 307 'bad_nested_config': { 308 'mixins': ['chrome_with_codecs'], 309 }, 310 'rel': { 311 'gn_args': 'is_debug=false', 312 }, 313 }, 314} 315""" 316 317TEST_BAD_CONFIG_BUCKET = """\ 318{ 319 'public_artifact_builders': { 320 'fake_bucket_a': ['fake_builder_a', 'fake_builder_b'], 321 }, 322 'configs': { 323 'rel_bot_1': ['rel', 'chrome_with_codecs'], 324 'rel_bot_2': ['rel', 'bad_nested_config'], 325 }, 326 'buckets': { 327 'fake_bucket_a': { 328 'fake_builder_a': 'rel_bot_1', 329 'fake_builder_b': 'rel_bot_2', 330 }, 331 }, 332 'mixins': { 333 'chrome_with_codecs': { 334 'gn_args': 'proprietary_codecs=true', 335 }, 336 'bad_nested_config': { 337 'mixins': ['chrome_with_codecs'], 338 }, 339 'rel': { 340 'gn_args': 'is_debug=false', 341 }, 342 }, 343} 344""" 345 346 347TEST_ARGS_FILE_TWICE_CONFIG = """\ 348{ 349 'masters': { 350 'chromium': {}, 351 'fake_master': { 352 'fake_args_file_twice': 'args_file_twice', 353 }, 354 }, 355 'configs': { 356 'args_file_twice': ['args_file', 'args_file'], 357 }, 358 'mixins': { 359 'args_file': { 360 'args_file': '//build/args/fake.gn', 361 }, 362 }, 363} 364""" 365 366 367TEST_ARGS_FILE_TWICE_CONFIG_BUCKET = """\ 368{ 369 'public_artifact_builders': {}, 370 'buckets': { 371 'chromium': {}, 372 'fake_bucket': { 373 'fake_args_file_twice': 'args_file_twice', 374 }, 375 }, 376 'configs': { 377 'args_file_twice': ['args_file', 'args_file'], 378 }, 379 'mixins': { 380 'args_file': { 381 'args_file': '//build/args/fake.gn', 382 }, 383 }, 384} 385""" 386 387TEST_DUP_CONFIG = """\ 388{ 389 'masters': { 390 'chromium': {}, 391 'fake_master': { 392 'fake_builder': 'some_config', 393 'other_builder': 'some_other_config', 394 }, 395 }, 396 'configs': { 397 'some_config': ['args_file'], 398 'some_other_config': ['args_file'], 399 }, 400 'mixins': { 401 'args_file': { 402 'args_file': '//build/args/fake.gn', 403 }, 404 }, 405} 406""" 407 408TEST_DUP_CONFIG_BUCKET = """\ 409{ 410 'public_artifact_builders': {}, 411 'buckets': { 412 'ci': {}, 413 'fake_bucket': { 414 'fake_builder': 'some_config', 415 'other_builder': 'some_other_config', 416 }, 417 }, 418 'configs': { 419 'some_config': ['args_file'], 420 'some_other_config': ['args_file'], 421 }, 422 'mixins': { 423 'args_file': { 424 'args_file': '//build/args/fake.gn', 425 }, 426 }, 427} 428""" 429 430TRYSERVER_CONFIG = """\ 431{ 432 'masters': { 433 'not_a_tryserver': { 434 'fake_builder': 'fake_config', 435 }, 436 'tryserver.chromium.linux': { 437 'try_builder': 'fake_config', 438 }, 439 'tryserver.chromium.mac': { 440 'try_builder2': 'fake_config', 441 }, 442 }, 443 'configs': {}, 444 'mixins': {}, 445} 446""" 447 448 449class UnitTest(unittest.TestCase): 450 def fake_mbw(self, files=None, win32=False): 451 mbw = FakeMBW(win32=win32) 452 mbw.files.setdefault(mbw.default_config_master, TEST_CONFIG) 453 mbw.files.setdefault(mbw.default_config_bucket, TEST_CONFIG_BUCKET) 454 mbw.files.setdefault( 455 mbw.ToAbsPath('//testing/buildbot/gn_isolate_map.pyl'), 456 '''{ 457 "foo_unittests": { 458 "label": "//foo:foo_unittests", 459 "type": "console_test_launcher", 460 "args": [], 461 }, 462 }''') 463 mbw.files.setdefault( 464 mbw.ToAbsPath('//build/args/bots/fake_master/fake_args_bot.gn'), 465 'is_debug = false\n') 466 if files: 467 for path, contents in files.items(): 468 mbw.files[path] = contents 469 return mbw 470 471 def check(self, args, mbw=None, files=None, out=None, err=None, ret=None, 472 env=None): 473 if not mbw: 474 mbw = self.fake_mbw(files) 475 476 try: 477 prev_env = os.environ.copy() 478 os.environ = env if env else prev_env 479 actual_ret = mbw.Main(args) 480 finally: 481 os.environ = prev_env 482 self.assertEqual(actual_ret, ret) 483 if out is not None: 484 self.assertEqual(mbw.out, out) 485 if err is not None: 486 self.assertEqual(mbw.err, err) 487 return mbw 488 489 def test_analyze(self): 490 files = {'/tmp/in.json': '''{\ 491 "files": ["foo/foo_unittest.cc"], 492 "test_targets": ["foo_unittests"], 493 "additional_compile_targets": ["all"] 494 }''', 495 '/tmp/out.json.gn': '''{\ 496 "status": "Found dependency", 497 "compile_targets": ["//foo:foo_unittests"], 498 "test_targets": ["//foo:foo_unittests"] 499 }'''} 500 501 mbw = self.fake_mbw(files) 502 mbw.Call = lambda cmd, env=None, buffer_output=True, stdin=None: (0, '', '') 503 504 self.check(['analyze', '-c', 'debug_goma', '//out/Default', 505 '/tmp/in.json', '/tmp/out.json'], mbw=mbw, ret=0) 506 out = json.loads(mbw.files['/tmp/out.json']) 507 self.assertEqual(out, { 508 'status': 'Found dependency', 509 'compile_targets': ['foo:foo_unittests'], 510 'test_targets': ['foo_unittests'] 511 }) 512 513 def test_analyze_optimizes_compile_for_all(self): 514 files = {'/tmp/in.json': '''{\ 515 "files": ["foo/foo_unittest.cc"], 516 "test_targets": ["foo_unittests"], 517 "additional_compile_targets": ["all"] 518 }''', 519 '/tmp/out.json.gn': '''{\ 520 "status": "Found dependency", 521 "compile_targets": ["//foo:foo_unittests", "all"], 522 "test_targets": ["//foo:foo_unittests"] 523 }'''} 524 525 mbw = self.fake_mbw(files) 526 mbw.Call = lambda cmd, env=None, buffer_output=True, stdin=None: (0, '', '') 527 528 self.check(['analyze', '-c', 'debug_goma', '//out/Default', 529 '/tmp/in.json', '/tmp/out.json'], mbw=mbw, ret=0) 530 out = json.loads(mbw.files['/tmp/out.json']) 531 532 # check that 'foo_unittests' is not in the compile_targets 533 self.assertEqual(['all'], out['compile_targets']) 534 535 def test_analyze_handles_other_toolchains(self): 536 files = {'/tmp/in.json': '''{\ 537 "files": ["foo/foo_unittest.cc"], 538 "test_targets": ["foo_unittests"], 539 "additional_compile_targets": ["all"] 540 }''', 541 '/tmp/out.json.gn': '''{\ 542 "status": "Found dependency", 543 "compile_targets": ["//foo:foo_unittests", 544 "//foo:foo_unittests(bar)"], 545 "test_targets": ["//foo:foo_unittests"] 546 }'''} 547 548 mbw = self.fake_mbw(files) 549 mbw.Call = lambda cmd, env=None, buffer_output=True, stdin=None: (0, '', '') 550 551 self.check(['analyze', '-c', 'debug_goma', '//out/Default', 552 '/tmp/in.json', '/tmp/out.json'], mbw=mbw, ret=0) 553 out = json.loads(mbw.files['/tmp/out.json']) 554 555 # crbug.com/736215: If GN returns a label containing a toolchain, 556 # MB (and Ninja) don't know how to handle it; to work around this, 557 # we give up and just build everything we were asked to build. The 558 # output compile_targets should include all of the input test_targets and 559 # additional_compile_targets. 560 self.assertEqual(['all', 'foo_unittests'], out['compile_targets']) 561 562 def test_analyze_handles_way_too_many_results(self): 563 too_many_files = ', '.join(['"//foo:foo%d"' % i for i in xrange(40 * 1024)]) 564 files = {'/tmp/in.json': '''{\ 565 "files": ["foo/foo_unittest.cc"], 566 "test_targets": ["foo_unittests"], 567 "additional_compile_targets": ["all"] 568 }''', 569 '/tmp/out.json.gn': '''{\ 570 "status": "Found dependency", 571 "compile_targets": [''' + too_many_files + '''], 572 "test_targets": ["//foo:foo_unittests"] 573 }'''} 574 575 mbw = self.fake_mbw(files) 576 mbw.Call = lambda cmd, env=None, buffer_output=True, stdin=None: (0, '', '') 577 578 self.check(['analyze', '-c', 'debug_goma', '//out/Default', 579 '/tmp/in.json', '/tmp/out.json'], mbw=mbw, ret=0) 580 out = json.loads(mbw.files['/tmp/out.json']) 581 582 # If GN returns so many compile targets that we might have command-line 583 # issues, we should give up and just build everything we were asked to 584 # build. The output compile_targets should include all of the input 585 # test_targets and additional_compile_targets. 586 self.assertEqual(['all', 'foo_unittests'], out['compile_targets']) 587 588 def test_gen(self): 589 mbw = self.fake_mbw() 590 self.check(['gen', '-c', 'debug_goma', '//out/Default', '-g', '/goma'], 591 mbw=mbw, ret=0) 592 self.assertMultiLineEqual(mbw.files['/fake_src/out/Default/args.gn'], 593 ('goma_dir = "/goma"\n' 594 'is_debug = true\n' 595 'use_goma = true\n')) 596 597 # Make sure we log both what is written to args.gn and the command line. 598 self.assertIn('Writing """', mbw.out) 599 self.assertIn('/fake_src/buildtools/linux64/gn gen //out/Default --check', 600 mbw.out) 601 602 mbw = self.fake_mbw(win32=True) 603 self.check(['gen', '-c', 'debug_goma', '-g', 'c:\\goma', '//out/Debug'], 604 mbw=mbw, ret=0) 605 self.assertMultiLineEqual(mbw.files['c:\\fake_src\\out\\Debug\\args.gn'], 606 ('goma_dir = "c:\\\\goma"\n' 607 'is_debug = true\n' 608 'use_goma = true\n')) 609 self.assertIn('c:\\fake_src\\buildtools\\win\\gn.exe gen //out/Debug ' 610 '--check\n', mbw.out) 611 612 mbw = self.fake_mbw() 613 self.check(['gen', '-m', 'fake_master', '-b', 'fake_args_bot', 614 '//out/Debug'], 615 mbw=mbw, ret=0) 616 self.assertEqual( 617 mbw.files['/fake_src/out/Debug/args.gn'], 618 'import("//build/args/bots/fake_master/fake_args_bot.gn")\n') 619 620 def test_gen_args_file_mixins(self): 621 mbw = self.fake_mbw() 622 self.check(['gen', '-m', 'fake_master', '-b', 'fake_args_file', 623 '//out/Debug'], mbw=mbw, ret=0) 624 625 self.assertEqual( 626 mbw.files['/fake_src/out/Debug/args.gn'], 627 ('import("//build/args/fake.gn")\n' 628 'use_goma = true\n')) 629 630 def test_gen_args_file_twice_bucket(self): 631 mbw = self.fake_mbw() 632 mbw.files[mbw.default_config_bucket] = TEST_ARGS_FILE_TWICE_CONFIG_BUCKET 633 self.check([ 634 'gen', '-u', 'fake_bucket', '-b', 'fake_args_file_twice', '//out/Debug' 635 ], 636 mbw=mbw, 637 ret=1) 638 639 def test_gen_args_file_twice(self): 640 mbw = self.fake_mbw() 641 mbw.files[mbw.default_config_master] = TEST_ARGS_FILE_TWICE_CONFIG 642 self.check(['gen', '-m', 'fake_master', '-b', 'fake_args_file_twice', 643 '//out/Debug'], mbw=mbw, ret=1) 644 645 def test_gen_fails(self): 646 mbw = self.fake_mbw() 647 mbw.Call = lambda cmd, env=None, buffer_output=True, stdin=None: (1, '', '') 648 self.check(['gen', '-c', 'debug_goma', '//out/Default'], mbw=mbw, ret=1) 649 650 def test_gen_swarming(self): 651 files = { 652 '/tmp/swarming_targets': 'base_unittests\n', 653 '/fake_src/testing/buildbot/gn_isolate_map.pyl': ( 654 "{'base_unittests': {" 655 " 'label': '//base:base_unittests'," 656 " 'type': 'raw'," 657 " 'args': []," 658 "}}\n" 659 ), 660 } 661 662 mbw = self.fake_mbw(files) 663 664 def fake_call(cmd, env=None, buffer_output=True, stdin=None): 665 del cmd 666 del env 667 del buffer_output 668 del stdin 669 mbw.files['/fake_src/out/Default/base_unittests.runtime_deps'] = ( 670 'base_unittests\n') 671 return 0, '', '' 672 673 mbw.Call = fake_call 674 675 self.check(['gen', 676 '-c', 'debug_goma', 677 '--swarming-targets-file', '/tmp/swarming_targets', 678 '//out/Default'], mbw=mbw, ret=0) 679 self.assertIn('/fake_src/out/Default/base_unittests.isolate', 680 mbw.files) 681 self.assertIn('/fake_src/out/Default/base_unittests.isolated.gen.json', 682 mbw.files) 683 684 def test_gen_swarming_script(self): 685 files = { 686 '/tmp/swarming_targets': 'cc_perftests\n', 687 '/fake_src/testing/buildbot/gn_isolate_map.pyl': ( 688 "{'cc_perftests': {" 689 " 'label': '//cc:cc_perftests'," 690 " 'type': 'script'," 691 " 'script': '/fake_src/out/Default/test_script.py'," 692 " 'args': []," 693 "}}\n" 694 ), 695 } 696 mbw = self.fake_mbw(files=files) 697 698 def fake_call(cmd, env=None, buffer_output=True, stdin=None): 699 del cmd 700 del env 701 del buffer_output 702 del stdin 703 mbw.files['/fake_src/out/Default/cc_perftests.runtime_deps'] = ( 704 'cc_perftests\n') 705 return 0, '', '' 706 707 mbw.Call = fake_call 708 709 self.check(['gen', 710 '-c', 'debug_goma', 711 '--swarming-targets-file', '/tmp/swarming_targets', 712 '--isolate-map-file', 713 '/fake_src/testing/buildbot/gn_isolate_map.pyl', 714 '//out/Default'], mbw=mbw, ret=0) 715 self.assertIn('/fake_src/out/Default/cc_perftests.isolate', 716 mbw.files) 717 self.assertIn('/fake_src/out/Default/cc_perftests.isolated.gen.json', 718 mbw.files) 719 720 def test_multiple_isolate_maps(self): 721 files = { 722 '/tmp/swarming_targets': 'cc_perftests\n', 723 '/fake_src/testing/buildbot/gn_isolate_map.pyl': ( 724 "{'cc_perftests': {" 725 " 'label': '//cc:cc_perftests'," 726 " 'type': 'raw'," 727 " 'args': []," 728 "}}\n" 729 ), 730 '/fake_src/testing/buildbot/gn_isolate_map2.pyl': ( 731 "{'cc_perftests2': {" 732 " 'label': '//cc:cc_perftests'," 733 " 'type': 'raw'," 734 " 'args': []," 735 "}}\n" 736 ), 737 } 738 mbw = self.fake_mbw(files=files) 739 740 def fake_call(cmd, env=None, buffer_output=True, stdin=None): 741 del cmd 742 del env 743 del buffer_output 744 del stdin 745 mbw.files['/fake_src/out/Default/cc_perftests.runtime_deps'] = ( 746 'cc_perftests_fuzzer\n') 747 return 0, '', '' 748 749 mbw.Call = fake_call 750 751 self.check(['gen', 752 '-c', 'debug_goma', 753 '--swarming-targets-file', '/tmp/swarming_targets', 754 '--isolate-map-file', 755 '/fake_src/testing/buildbot/gn_isolate_map.pyl', 756 '--isolate-map-file', 757 '/fake_src/testing/buildbot/gn_isolate_map2.pyl', 758 '//out/Default'], mbw=mbw, ret=0) 759 self.assertIn('/fake_src/out/Default/cc_perftests.isolate', 760 mbw.files) 761 self.assertIn('/fake_src/out/Default/cc_perftests.isolated.gen.json', 762 mbw.files) 763 764 765 def test_duplicate_isolate_maps(self): 766 files = { 767 '/tmp/swarming_targets': 'cc_perftests\n', 768 '/fake_src/testing/buildbot/gn_isolate_map.pyl': ( 769 "{'cc_perftests': {" 770 " 'label': '//cc:cc_perftests'," 771 " 'type': 'raw'," 772 " 'args': []," 773 "}}\n" 774 ), 775 '/fake_src/testing/buildbot/gn_isolate_map2.pyl': ( 776 "{'cc_perftests': {" 777 " 'label': '//cc:cc_perftests'," 778 " 'type': 'raw'," 779 " 'args': []," 780 "}}\n" 781 ), 782 'c:\\fake_src\out\Default\cc_perftests.exe.runtime_deps': ( 783 "cc_perftests\n" 784 ), 785 } 786 mbw = self.fake_mbw(files=files, win32=True) 787 # Check that passing duplicate targets into mb fails. 788 self.check(['gen', 789 '-c', 'debug_goma', 790 '--swarming-targets-file', '/tmp/swarming_targets', 791 '--isolate-map-file', 792 '/fake_src/testing/buildbot/gn_isolate_map.pyl', 793 '--isolate-map-file', 794 '/fake_src/testing/buildbot/gn_isolate_map2.pyl', 795 '//out/Default'], mbw=mbw, ret=1) 796 797 798 def test_isolate(self): 799 files = { 800 '/fake_src/out/Default/toolchain.ninja': "", 801 '/fake_src/testing/buildbot/gn_isolate_map.pyl': ( 802 "{'base_unittests': {" 803 " 'label': '//base:base_unittests'," 804 " 'type': 'raw'," 805 " 'args': []," 806 "}}\n" 807 ), 808 '/fake_src/out/Default/base_unittests.runtime_deps': ( 809 "base_unittests\n" 810 ), 811 } 812 self.check(['isolate', '-c', 'debug_goma', '//out/Default', 813 'base_unittests'], files=files, ret=0) 814 815 # test running isolate on an existing build_dir 816 files['/fake_src/out/Default/args.gn'] = 'is_debug = True\n' 817 self.check(['isolate', '//out/Default', 'base_unittests'], 818 files=files, ret=0) 819 820 self.check(['isolate', '//out/Default', 'base_unittests'], 821 files=files, ret=0) 822 823 def test_isolate_dir(self): 824 files = { 825 '/fake_src/out/Default/toolchain.ninja': "", 826 '/fake_src/testing/buildbot/gn_isolate_map.pyl': ( 827 "{'base_unittests': {" 828 " 'label': '//base:base_unittests'," 829 " 'type': 'raw'," 830 " 'args': []," 831 "}}\n" 832 ), 833 } 834 mbw = self.fake_mbw(files=files) 835 mbw.cmds.append((0, '', '')) # Result of `gn gen` 836 mbw.cmds.append((0, '', '')) # Result of `autoninja` 837 838 # Result of `gn desc runtime_deps` 839 mbw.cmds.append((0, 'base_unitests\n../../test_data/\n', '')) 840 self.check(['isolate', '-c', 'debug_goma', '//out/Default', 841 'base_unittests'], mbw=mbw, ret=0, err='') 842 843 def test_isolate_generated_dir(self): 844 files = { 845 '/fake_src/out/Default/toolchain.ninja': "", 846 '/fake_src/testing/buildbot/gn_isolate_map.pyl': ( 847 "{'base_unittests': {" 848 " 'label': '//base:base_unittests'," 849 " 'type': 'raw'," 850 " 'args': []," 851 "}}\n" 852 ), 853 } 854 mbw = self.fake_mbw(files=files) 855 mbw.cmds.append((0, '', '')) # Result of `gn gen` 856 mbw.cmds.append((0, '', '')) # Result of `autoninja` 857 858 # Result of `gn desc runtime_deps` 859 mbw.cmds.append((0, 'base_unitests\ntest_data/\n', '')) 860 expected_err = ('error: gn `data` items may not list generated directories;' 861 ' list files in directory instead for:\n' 862 '//out/Default/test_data/\n') 863 self.check(['isolate', '-c', 'debug_goma', '//out/Default', 864 'base_unittests'], mbw=mbw, ret=1) 865 self.assertEqual(mbw.out[-len(expected_err):], expected_err) 866 867 868 def test_run(self): 869 files = { 870 '/fake_src/testing/buildbot/gn_isolate_map.pyl': ( 871 "{'base_unittests': {" 872 " 'label': '//base:base_unittests'," 873 " 'type': 'raw'," 874 " 'args': []," 875 "}}\n" 876 ), 877 '/fake_src/out/Default/base_unittests.runtime_deps': ( 878 "base_unittests\n" 879 ), 880 } 881 self.check(['run', '-c', 'debug_goma', '//out/Default', 882 'base_unittests'], files=files, ret=0) 883 884 def test_run_swarmed(self): 885 files = { 886 '/fake_src/testing/buildbot/gn_isolate_map.pyl': 887 ("{'base_unittests': {" 888 " 'label': '//base:base_unittests'," 889 " 'type': 'raw'," 890 " 'args': []," 891 "}}\n"), 892 '/fake_src/out/Default/base_unittests.runtime_deps': 893 ("base_unittests\n"), 894 '/fake_src/out/Default/base_unittests.archive.json': 895 ("{\"base_unittests\":\"fake_hash\"}"), 896 '/fake_src/third_party/depot_tools/cipd_manifest.txt': 897 ("# vpython\n" 898 "/some/vpython/pkg git_revision:deadbeef\n"), 899 } 900 901 mbw = self.fake_mbw(files=files) 902 original_impl = mbw.ToSrcRelPath 903 904 def to_src_rel_path_stub(path): 905 if path.endswith('base_unittests.archive.json'): 906 return 'base_unittests.archive.json' 907 return original_impl(path) 908 909 mbw.ToSrcRelPath = to_src_rel_path_stub 910 911 self.check(['run', '-s', '-c', 'debug_goma', '//out/Default', 912 'base_unittests'], mbw=mbw, ret=0) 913 self.check(['run', '-s', '-c', 'debug_goma', '-d', 'os', 'Win7', 914 '//out/Default', 'base_unittests'], mbw=mbw, ret=0) 915 916 def test_lookup(self): 917 self.check(['lookup', '-c', 'debug_goma'], ret=0, 918 out=('\n' 919 'Writing """\\\n' 920 'is_debug = true\n' 921 'use_goma = true\n' 922 '""" to _path_/args.gn.\n\n' 923 '/fake_src/buildtools/linux64/gn gen _path_\n')) 924 925 def test_quiet_lookup(self): 926 self.check(['lookup', '-c', 'debug_goma', '--quiet'], ret=0, 927 out=('is_debug = true\n' 928 'use_goma = true\n')) 929 930 def test_lookup_goma_dir_expansion(self): 931 self.check(['lookup', '-c', 'rel_bot', '-g', '/foo'], ret=0, 932 out=('\n' 933 'Writing """\\\n' 934 'enable_doom_melon = true\n' 935 'goma_dir = "/foo"\n' 936 'is_debug = false\n' 937 'use_goma = true\n' 938 '""" to _path_/args.gn.\n\n' 939 '/fake_src/buildtools/linux64/gn gen _path_\n')) 940 941 def test_lookup_simplechrome(self): 942 simplechrome_env = { 943 'GN_ARGS': 'is_chromeos=1 target_os="chromeos"', 944 } 945 self.check(['lookup', '-c', 'cros_chrome_sdk'], ret=0, env=simplechrome_env) 946 947 def test_help(self): 948 orig_stdout = sys.stdout 949 try: 950 sys.stdout = StringIO.StringIO() 951 self.assertRaises(SystemExit, self.check, ['-h']) 952 self.assertRaises(SystemExit, self.check, ['help']) 953 self.assertRaises(SystemExit, self.check, ['help', 'gen']) 954 finally: 955 sys.stdout = orig_stdout 956 957 def test_multiple_phases(self): 958 # Check that not passing a --phase to a multi-phase builder fails. 959 mbw = self.check(['lookup', '-m', 'fake_master', '-b', 'fake_multi_phase'], 960 ret=1) 961 self.assertIn('Must specify a build --phase', mbw.out) 962 963 # Check that passing a --phase to a single-phase builder fails. 964 mbw = self.check(['lookup', '-m', 'fake_master', '-b', 'fake_builder', 965 '--phase', 'phase_1'], ret=1) 966 self.assertIn('Must not specify a build --phase', mbw.out) 967 968 # Check that passing a wrong phase key to a multi-phase builder fails. 969 mbw = self.check(['lookup', '-m', 'fake_master', '-b', 'fake_multi_phase', 970 '--phase', 'wrong_phase'], ret=1) 971 self.assertIn('Phase wrong_phase doesn\'t exist', mbw.out) 972 973 # Check that passing a correct phase key to a multi-phase builder passes. 974 mbw = self.check(['lookup', '-m', 'fake_master', '-b', 'fake_multi_phase', 975 '--phase', 'phase_1'], ret=0) 976 self.assertIn('phase = 1', mbw.out) 977 978 mbw = self.check(['lookup', '-m', 'fake_master', '-b', 'fake_multi_phase', 979 '--phase', 'phase_2'], ret=0) 980 self.assertIn('phase = 2', mbw.out) 981 982 def test_recursive_lookup(self): 983 files = { 984 '/fake_src/build/args/fake.gn': ( 985 'enable_doom_melon = true\n' 986 'enable_antidoom_banana = true\n' 987 ) 988 } 989 self.check(['lookup', '-m', 'fake_master', '-b', 'fake_args_file', 990 '--recursive'], files=files, ret=0, 991 out=('enable_antidoom_banana = true\n' 992 'enable_doom_melon = true\n' 993 'use_goma = true\n')) 994 995 def test_recursive_lookup_bucket(self): 996 files = { 997 '/fake_src/build/args/fake.gn': ('enable_doom_melon = true\n' 998 'enable_antidoom_banana = true\n') 999 } 1000 self.check(['lookup', '-u', 'ci', '-b', 'fake_args_file', '--recursive'], 1001 files=files, 1002 ret=0, 1003 out=('enable_antidoom_banana = true\n' 1004 'enable_doom_melon = true\n' 1005 'use_goma = true\n')) 1006 1007 def test_validate(self): 1008 mbw = self.fake_mbw() 1009 self.check(['validate'], mbw=mbw, ret=0) 1010 1011 def test_validate_inconsistent(self): 1012 mbw = self.fake_mbw() 1013 mbw.files[mbw.default_config_bucket] = TEST_CONFIG_BUCKET_2 1014 self.check(['validate'], mbw=mbw, ret=1) 1015 self.assertIn('mb_config_buckets.pyl doesn\'t match', mbw.out) 1016 1017 def test_bad_validate(self): 1018 mbw = self.fake_mbw() 1019 mbw.files[mbw.default_config_master] = TEST_BAD_CONFIG 1020 self.check(['validate', '-f', mbw.default_config_master], mbw=mbw, ret=1) 1021 1022 def test_bad_validate_bucket(self): 1023 mbw = self.fake_mbw() 1024 mbw.files[mbw.default_config_bucket] = TEST_BAD_CONFIG_BUCKET 1025 self.check(['validate', '-f', mbw.default_config_bucket], mbw=mbw, ret=1) 1026 1027 def test_duplicate_validate(self): 1028 mbw = self.fake_mbw() 1029 mbw.files[mbw.default_config_master] = TEST_DUP_CONFIG 1030 self.check(['validate'], mbw=mbw, ret=1) 1031 self.assertIn( 1032 'Duplicate configs detected. When evaluated fully, the ' 1033 'following configs are all equivalent: \'some_config\', ' 1034 '\'some_other_config\'.', mbw.out) 1035 1036 def test_duplicate_validate_bucket(self): 1037 mbw = self.fake_mbw() 1038 mbw.files[mbw.default_config_bucket] = TEST_DUP_CONFIG_BUCKET 1039 self.check(['validate'], mbw=mbw, ret=1) 1040 self.assertIn('Duplicate configs detected. When evaluated fully, the ' 1041 'following configs are all equivalent: \'some_config\', ' 1042 '\'some_other_config\'.', mbw.out) 1043 1044 def test_build_command_unix(self): 1045 files = { 1046 '/fake_src/out/Default/toolchain.ninja': '', 1047 '/fake_src/testing/buildbot/gn_isolate_map.pyl': ( 1048 '{"base_unittests": {' 1049 ' "label": "//base:base_unittests",' 1050 ' "type": "raw",' 1051 ' "args": [],' 1052 '}}\n') 1053 } 1054 1055 mbw = self.fake_mbw(files) 1056 self.check(['run', '//out/Default', 'base_unittests'], mbw=mbw, ret=0) 1057 self.assertIn(['autoninja', '-C', 'out/Default', 'base_unittests'], 1058 mbw.calls) 1059 1060 def test_build_command_windows(self): 1061 files = { 1062 'c:\\fake_src\\out\\Default\\toolchain.ninja': '', 1063 'c:\\fake_src\\testing\\buildbot\\gn_isolate_map.pyl': ( 1064 '{"base_unittests": {' 1065 ' "label": "//base:base_unittests",' 1066 ' "type": "raw",' 1067 ' "args": [],' 1068 '}}\n') 1069 } 1070 1071 mbw = self.fake_mbw(files, True) 1072 self.check(['run', '//out/Default', 'base_unittests'], mbw=mbw, ret=0) 1073 self.assertIn(['autoninja.bat', '-C', 'out\\Default', 'base_unittests'], 1074 mbw.calls) 1075 1076 def test_ios_error_config_with_ios_json(self): 1077 """Ensures that ios_error config finds the correct iOS JSON file for args""" 1078 files = { 1079 '/fake_src/ios/build/bots/fake_master/fake_ios_error.json': 1080 ('{"gn_args": ["is_debug=true"]}\n') 1081 } 1082 mbw = self.fake_mbw(files) 1083 self.check(['lookup', '-m', 'fake_master', '-b', 'fake_ios_error'], 1084 mbw=mbw, 1085 ret=0, 1086 out=('\n' 1087 'Writing """\\\n' 1088 'is_debug = true\n' 1089 '""" to _path_/args.gn.\n\n' 1090 '/fake_src/buildtools/linux64/gn gen _path_\n')) 1091 1092 def test_bot_definition_in_ios_json_only(self): 1093 """Ensures that logic checks iOS JSON file for args 1094 1095 When builder definition is not present, ensure that ios/build/bots/ is 1096 checked. 1097 """ 1098 files = { 1099 '/fake_src/ios/build/bots/fake_master/fake_ios_bot.json': 1100 ('{"gn_args": ["is_debug=true"]}\n') 1101 } 1102 mbw = self.fake_mbw(files) 1103 self.check(['lookup', '-m', 'fake_master', '-b', 'fake_ios_bot'], 1104 mbw=mbw, 1105 ret=0, 1106 out=('\n' 1107 'Writing """\\\n' 1108 'is_debug = true\n' 1109 '""" to _path_/args.gn.\n\n' 1110 '/fake_src/buildtools/linux64/gn gen _path_\n')) 1111 1112 def test_ios_error_config_missing_json_definition(self): 1113 """Ensures MBErr is thrown 1114 1115 Expect MBErr with 'No iOS definition ...' for iOS bots when the bot config 1116 is ios_error, but there is no iOS JSON definition for it. 1117 """ 1118 mbw = self.fake_mbw() 1119 self.check(['lookup', '-m', 'fake_master', '-b', 'fake_ios_error'], 1120 mbw=mbw, 1121 ret=1) 1122 self.assertIn('MBErr: No iOS definition was found.', mbw.out) 1123 1124 def test_bot_missing_definition(self): 1125 """Ensures builder missing MBErr is thrown 1126 1127 Expect the original MBErr to be thrown for iOS bots when the bot definition 1128 doesn't exist at all. 1129 """ 1130 mbw = self.fake_mbw() 1131 self.check(['lookup', '-m', 'fake_master', '-b', 'random_bot'], 1132 mbw=mbw, 1133 ret=1) 1134 self.assertIn('MBErr: Builder name "random_bot" not found under masters', 1135 mbw.out) 1136 1137 1138if __name__ == '__main__': 1139 unittest.main() 1140