1# SPDX-License-Identifier: GPL-2.0+ 2# Copyright (c) 2014 Google, Inc 3# 4 5import os 6import shutil 7import sys 8import tempfile 9import unittest 10 11from buildman import board 12from buildman import bsettings 13from buildman import cmdline 14from buildman import control 15from buildman import toolchain 16from patman import command 17from patman import gitutil 18from patman import terminal 19from patman import tools 20 21settings_data = ''' 22# Buildman settings file 23 24[toolchain] 25 26[toolchain-alias] 27 28[make-flags] 29src=/home/sjg/c/src 30chroot=/home/sjg/c/chroot 31vboot=VBOOT_DEBUG=1 MAKEFLAGS_VBOOT=DEBUG=1 CFLAGS_EXTRA_VBOOT=-DUNROLL_LOOPS VBOOT_SOURCE=${src}/platform/vboot_reference 32chromeos_coreboot=VBOOT=${chroot}/build/link/usr ${vboot} 33chromeos_daisy=VBOOT=${chroot}/build/daisy/usr ${vboot} 34chromeos_peach=VBOOT=${chroot}/build/peach_pit/usr ${vboot} 35''' 36 37boards = [ 38 ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 1', 'board0', ''], 39 ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 2', 'board1', ''], 40 ['Active', 'powerpc', 'powerpc', '', 'Tester', 'PowerPC board 1', 'board2', ''], 41 ['Active', 'sandbox', 'sandbox', '', 'Tester', 'Sandbox board', 'board4', ''], 42] 43 44commit_shortlog = """4aca821 patman: Avoid changing the order of tags 4539403bb patman: Use --no-pager' to stop git from forking a pager 46db6e6f2 patman: Remove the -a option 47f2ccf03 patman: Correct unit tests to run correctly 481d097f9 patman: Fix indentation in terminal.py 49d073747 patman: Support the 'reverse' option for 'git log 50""" 51 52commit_log = ["""commit 7f6b8315d18f683c5181d0c3694818c1b2a20dcd 53Author: Masahiro Yamada <yamada.m@jp.panasonic.com> 54Date: Fri Aug 22 19:12:41 2014 +0900 55 56 buildman: refactor help message 57 58 "buildman [options]" is displayed by default. 59 60 Append the rest of help messages to parser.usage 61 instead of replacing it. 62 63 Besides, "-b <branch>" is not mandatory since commit fea5858e. 64 Drop it from the usage. 65 66 Signed-off-by: Masahiro Yamada <yamada.m@jp.panasonic.com> 67""", 68"""commit d0737479be6baf4db5e2cdbee123e96bc5ed0ba8 69Author: Simon Glass <sjg@chromium.org> 70Date: Thu Aug 14 16:48:25 2014 -0600 71 72 patman: Support the 'reverse' option for 'git log' 73 74 This option is currently not supported, but needs to be, for buildman to 75 operate as expected. 76 77 Series-changes: 7 78 - Add new patch to fix the 'reverse' bug 79 80 Series-version: 8 81 82 Change-Id: I79078f792e8b390b8a1272a8023537821d45feda 83 Reported-by: York Sun <yorksun@freescale.com> 84 Signed-off-by: Simon Glass <sjg@chromium.org> 85 86""", 87"""commit 1d097f9ab487c5019152fd47bda126839f3bf9fc 88Author: Simon Glass <sjg@chromium.org> 89Date: Sat Aug 9 11:44:32 2014 -0600 90 91 patman: Fix indentation in terminal.py 92 93 This code came from a different project with 2-character indentation. Fix 94 it for U-Boot. 95 96 Series-changes: 6 97 - Add new patch to fix indentation in teminal.py 98 99 Change-Id: I5a74d2ebbb3cc12a665f5c725064009ac96e8a34 100 Signed-off-by: Simon Glass <sjg@chromium.org> 101 102""", 103"""commit f2ccf03869d1e152c836515a3ceb83cdfe04a105 104Author: Simon Glass <sjg@chromium.org> 105Date: Sat Aug 9 11:08:24 2014 -0600 106 107 patman: Correct unit tests to run correctly 108 109 It seems that doctest behaves differently now, and some of the unit tests 110 do not run. Adjust the tests to work correctly. 111 112 ./tools/patman/patman --test 113 <unittest.result.TestResult run=10 errors=0 failures=0> 114 115 Series-changes: 6 116 - Add new patch to fix patman unit tests 117 118 Change-Id: I3d2ca588f4933e1f9d6b1665a00e4ae58269ff3b 119 120""", 121"""commit db6e6f2f9331c5a37647d6668768d4a40b8b0d1c 122Author: Simon Glass <sjg@chromium.org> 123Date: Sat Aug 9 12:06:02 2014 -0600 124 125 patman: Remove the -a option 126 127 It seems that this is no longer needed, since checkpatch.pl will catch 128 whitespace problems in patches. Also the option is not widely used, so 129 it seems safe to just remove it. 130 131 Series-changes: 6 132 - Add new patch to remove patman's -a option 133 134 Suggested-by: Masahiro Yamada <yamada.m@jp.panasonic.com> 135 Change-Id: I5821a1c75154e532c46513486ca40b808de7e2cc 136 137""", 138"""commit 39403bb4f838153028a6f21ca30bf100f3791133 139Author: Simon Glass <sjg@chromium.org> 140Date: Thu Aug 14 21:50:52 2014 -0600 141 142 patman: Use --no-pager' to stop git from forking a pager 143 144""", 145"""commit 4aca821e27e97925c039e69fd37375b09c6f129c 146Author: Simon Glass <sjg@chromium.org> 147Date: Fri Aug 22 15:57:39 2014 -0600 148 149 patman: Avoid changing the order of tags 150 151 patman collects tags that it sees in the commit and places them nicely 152 sorted at the end of the patch. However, this is not really necessary and 153 in fact is apparently not desirable. 154 155 Series-changes: 9 156 - Add new patch to avoid changing the order of tags 157 158 Series-version: 9 159 160 Suggested-by: Masahiro Yamada <yamada.m@jp.panasonic.com> 161 Change-Id: Ib1518588c1a189ad5c3198aae76f8654aed8d0db 162"""] 163 164TEST_BRANCH = '__testbranch' 165 166class TestFunctional(unittest.TestCase): 167 """Functional test for buildman. 168 169 This aims to test from just below the invocation of buildman (parsing 170 of arguments) to 'make' and 'git' invocation. It is not a true 171 emd-to-end test, as it mocks git, make and the tool chain. But this 172 makes it easier to detect when the builder is doing the wrong thing, 173 since in many cases this test code will fail. For example, only a 174 very limited subset of 'git' arguments is supported - anything 175 unexpected will fail. 176 """ 177 def setUp(self): 178 self._base_dir = tempfile.mkdtemp() 179 self._output_dir = tempfile.mkdtemp() 180 self._git_dir = os.path.join(self._base_dir, 'src') 181 self._buildman_pathname = sys.argv[0] 182 self._buildman_dir = os.path.dirname(os.path.realpath(sys.argv[0])) 183 command.test_result = self._HandleCommand 184 self.setupToolchains() 185 self._toolchains.Add('arm-gcc', test=False) 186 self._toolchains.Add('powerpc-gcc', test=False) 187 bsettings.Setup(None) 188 bsettings.AddFile(settings_data) 189 self._boards = board.Boards() 190 for brd in boards: 191 self._boards.AddBoard(board.Board(*brd)) 192 193 # Directories where the source been cloned 194 self._clone_dirs = [] 195 self._commits = len(commit_shortlog.splitlines()) + 1 196 self._total_builds = self._commits * len(boards) 197 198 # Number of calls to make 199 self._make_calls = 0 200 201 # Map of [board, commit] to error messages 202 self._error = {} 203 204 self._test_branch = TEST_BRANCH 205 206 # Avoid sending any output and clear all terminal output 207 terminal.SetPrintTestMode() 208 terminal.GetPrintTestLines() 209 210 def tearDown(self): 211 shutil.rmtree(self._base_dir) 212 #shutil.rmtree(self._output_dir) 213 214 def setupToolchains(self): 215 self._toolchains = toolchain.Toolchains() 216 self._toolchains.Add('gcc', test=False) 217 218 def _RunBuildman(self, *args): 219 return command.RunPipe([[self._buildman_pathname] + list(args)], 220 capture=True, capture_stderr=True) 221 222 def _RunControl(self, *args, clean_dir=False, boards=None): 223 sys.argv = [sys.argv[0]] + list(args) 224 options, args = cmdline.ParseArgs() 225 result = control.DoBuildman(options, args, toolchains=self._toolchains, 226 make_func=self._HandleMake, boards=boards or self._boards, 227 clean_dir=clean_dir) 228 self._builder = control.builder 229 return result 230 231 def testFullHelp(self): 232 command.test_result = None 233 result = self._RunBuildman('-H') 234 help_file = os.path.join(self._buildman_dir, 'README') 235 # Remove possible extraneous strings 236 extra = '::::::::::::::\n' + help_file + '\n::::::::::::::\n' 237 gothelp = result.stdout.replace(extra, '') 238 self.assertEqual(len(gothelp), os.path.getsize(help_file)) 239 self.assertEqual(0, len(result.stderr)) 240 self.assertEqual(0, result.return_code) 241 242 def testHelp(self): 243 command.test_result = None 244 result = self._RunBuildman('-h') 245 help_file = os.path.join(self._buildman_dir, 'README') 246 self.assertTrue(len(result.stdout) > 1000) 247 self.assertEqual(0, len(result.stderr)) 248 self.assertEqual(0, result.return_code) 249 250 def testGitSetup(self): 251 """Test gitutils.Setup(), from outside the module itself""" 252 command.test_result = command.CommandResult(return_code=1) 253 gitutil.Setup() 254 self.assertEqual(gitutil.use_no_decorate, False) 255 256 command.test_result = command.CommandResult(return_code=0) 257 gitutil.Setup() 258 self.assertEqual(gitutil.use_no_decorate, True) 259 260 def _HandleCommandGitLog(self, args): 261 if args[-1] == '--': 262 args = args[:-1] 263 if '-n0' in args: 264 return command.CommandResult(return_code=0) 265 elif args[-1] == 'upstream/master..%s' % self._test_branch: 266 return command.CommandResult(return_code=0, stdout=commit_shortlog) 267 elif args[:3] == ['--no-color', '--no-decorate', '--reverse']: 268 if args[-1] == self._test_branch: 269 count = int(args[3][2:]) 270 return command.CommandResult(return_code=0, 271 stdout=''.join(commit_log[:count])) 272 273 # Not handled, so abort 274 print('git log', args) 275 sys.exit(1) 276 277 def _HandleCommandGitConfig(self, args): 278 config = args[0] 279 if config == 'sendemail.aliasesfile': 280 return command.CommandResult(return_code=0) 281 elif config.startswith('branch.badbranch'): 282 return command.CommandResult(return_code=1) 283 elif config == 'branch.%s.remote' % self._test_branch: 284 return command.CommandResult(return_code=0, stdout='upstream\n') 285 elif config == 'branch.%s.merge' % self._test_branch: 286 return command.CommandResult(return_code=0, 287 stdout='refs/heads/master\n') 288 289 # Not handled, so abort 290 print('git config', args) 291 sys.exit(1) 292 293 def _HandleCommandGit(self, in_args): 294 """Handle execution of a git command 295 296 This uses a hacked-up parser. 297 298 Args: 299 in_args: Arguments after 'git' from the command line 300 """ 301 git_args = [] # Top-level arguments to git itself 302 sub_cmd = None # Git sub-command selected 303 args = [] # Arguments to the git sub-command 304 for arg in in_args: 305 if sub_cmd: 306 args.append(arg) 307 elif arg[0] == '-': 308 git_args.append(arg) 309 else: 310 if git_args and git_args[-1] in ['--git-dir', '--work-tree']: 311 git_args.append(arg) 312 else: 313 sub_cmd = arg 314 if sub_cmd == 'config': 315 return self._HandleCommandGitConfig(args) 316 elif sub_cmd == 'log': 317 return self._HandleCommandGitLog(args) 318 elif sub_cmd == 'clone': 319 return command.CommandResult(return_code=0) 320 elif sub_cmd == 'checkout': 321 return command.CommandResult(return_code=0) 322 323 # Not handled, so abort 324 print('git', git_args, sub_cmd, args) 325 sys.exit(1) 326 327 def _HandleCommandNm(self, args): 328 return command.CommandResult(return_code=0) 329 330 def _HandleCommandObjdump(self, args): 331 return command.CommandResult(return_code=0) 332 333 def _HandleCommandObjcopy(self, args): 334 return command.CommandResult(return_code=0) 335 336 def _HandleCommandSize(self, args): 337 return command.CommandResult(return_code=0) 338 339 def _HandleCommand(self, **kwargs): 340 """Handle a command execution. 341 342 The command is in kwargs['pipe-list'], as a list of pipes, each a 343 list of commands. The command should be emulated as required for 344 testing purposes. 345 346 Returns: 347 A CommandResult object 348 """ 349 pipe_list = kwargs['pipe_list'] 350 wc = False 351 if len(pipe_list) != 1: 352 if pipe_list[1] == ['wc', '-l']: 353 wc = True 354 else: 355 print('invalid pipe', kwargs) 356 sys.exit(1) 357 cmd = pipe_list[0][0] 358 args = pipe_list[0][1:] 359 result = None 360 if cmd == 'git': 361 result = self._HandleCommandGit(args) 362 elif cmd == './scripts/show-gnu-make': 363 return command.CommandResult(return_code=0, stdout='make') 364 elif cmd.endswith('nm'): 365 return self._HandleCommandNm(args) 366 elif cmd.endswith('objdump'): 367 return self._HandleCommandObjdump(args) 368 elif cmd.endswith('objcopy'): 369 return self._HandleCommandObjcopy(args) 370 elif cmd.endswith( 'size'): 371 return self._HandleCommandSize(args) 372 373 if not result: 374 # Not handled, so abort 375 print('unknown command', kwargs) 376 sys.exit(1) 377 378 if wc: 379 result.stdout = len(result.stdout.splitlines()) 380 return result 381 382 def _HandleMake(self, commit, brd, stage, cwd, *args, **kwargs): 383 """Handle execution of 'make' 384 385 Args: 386 commit: Commit object that is being built 387 brd: Board object that is being built 388 stage: Stage that we are at (mrproper, config, build) 389 cwd: Directory where make should be run 390 args: Arguments to pass to make 391 kwargs: Arguments to pass to command.RunPipe() 392 """ 393 self._make_calls += 1 394 if stage == 'mrproper': 395 return command.CommandResult(return_code=0) 396 elif stage == 'config': 397 return command.CommandResult(return_code=0, 398 combined='Test configuration complete') 399 elif stage == 'build': 400 stderr = '' 401 out_dir = '' 402 for arg in args: 403 if arg.startswith('O='): 404 out_dir = arg[2:] 405 fname = os.path.join(cwd or '', out_dir, 'u-boot') 406 tools.WriteFile(fname, b'U-Boot') 407 if type(commit) is not str: 408 stderr = self._error.get((brd.target, commit.sequence)) 409 if stderr: 410 return command.CommandResult(return_code=1, stderr=stderr) 411 return command.CommandResult(return_code=0) 412 413 # Not handled, so abort 414 print('make', stage) 415 sys.exit(1) 416 417 # Example function to print output lines 418 def print_lines(self, lines): 419 print(len(lines)) 420 for line in lines: 421 print(line) 422 #self.print_lines(terminal.GetPrintTestLines()) 423 424 def testNoBoards(self): 425 """Test that buildman aborts when there are no boards""" 426 self._boards = board.Boards() 427 with self.assertRaises(SystemExit): 428 self._RunControl() 429 430 def testCurrentSource(self): 431 """Very simple test to invoke buildman on the current source""" 432 self.setupToolchains(); 433 self._RunControl('-o', self._output_dir) 434 lines = terminal.GetPrintTestLines() 435 self.assertIn('Building current source for %d boards' % len(boards), 436 lines[0].text) 437 438 def testBadBranch(self): 439 """Test that we can detect an invalid branch""" 440 with self.assertRaises(ValueError): 441 self._RunControl('-b', 'badbranch') 442 443 def testBadToolchain(self): 444 """Test that missing toolchains are detected""" 445 self.setupToolchains(); 446 ret_code = self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir) 447 lines = terminal.GetPrintTestLines() 448 449 # Buildman always builds the upstream commit as well 450 self.assertIn('Building %d commits for %d boards' % 451 (self._commits, len(boards)), lines[0].text) 452 self.assertEqual(self._builder.count, self._total_builds) 453 454 # Only sandbox should succeed, the others don't have toolchains 455 self.assertEqual(self._builder.fail, 456 self._total_builds - self._commits) 457 self.assertEqual(ret_code, 100) 458 459 for commit in range(self._commits): 460 for board in self._boards.GetList(): 461 if board.arch != 'sandbox': 462 errfile = self._builder.GetErrFile(commit, board.target) 463 fd = open(errfile) 464 self.assertEqual(fd.readlines(), 465 ['No tool chain for %s\n' % board.arch]) 466 fd.close() 467 468 def testBranch(self): 469 """Test building a branch with all toolchains present""" 470 self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir) 471 self.assertEqual(self._builder.count, self._total_builds) 472 self.assertEqual(self._builder.fail, 0) 473 474 def testCount(self): 475 """Test building a specific number of commitst""" 476 self._RunControl('-b', TEST_BRANCH, '-c2', '-o', self._output_dir) 477 self.assertEqual(self._builder.count, 2 * len(boards)) 478 self.assertEqual(self._builder.fail, 0) 479 # Each board has a config, and then one make per commit 480 self.assertEqual(self._make_calls, len(boards) * (1 + 2)) 481 482 def testIncremental(self): 483 """Test building a branch twice - the second time should do nothing""" 484 self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir) 485 486 # Each board has a mrproper, config, and then one make per commit 487 self.assertEqual(self._make_calls, len(boards) * (self._commits + 1)) 488 self._make_calls = 0 489 self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir, clean_dir=False) 490 self.assertEqual(self._make_calls, 0) 491 self.assertEqual(self._builder.count, self._total_builds) 492 self.assertEqual(self._builder.fail, 0) 493 494 def testForceBuild(self): 495 """The -f flag should force a rebuild""" 496 self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir) 497 self._make_calls = 0 498 self._RunControl('-b', TEST_BRANCH, '-f', '-o', self._output_dir, clean_dir=False) 499 # Each board has a config and one make per commit 500 self.assertEqual(self._make_calls, len(boards) * (self._commits + 1)) 501 502 def testForceReconfigure(self): 503 """The -f flag should force a rebuild""" 504 self._RunControl('-b', TEST_BRANCH, '-C', '-o', self._output_dir) 505 # Each commit has a config and make 506 self.assertEqual(self._make_calls, len(boards) * self._commits * 2) 507 508 def testForceReconfigure(self): 509 """The -f flag should force a rebuild""" 510 self._RunControl('-b', TEST_BRANCH, '-C', '-o', self._output_dir) 511 # Each commit has a config and make 512 self.assertEqual(self._make_calls, len(boards) * self._commits * 2) 513 514 def testMrproper(self): 515 """The -f flag should force a rebuild""" 516 self._RunControl('-b', TEST_BRANCH, '-m', '-o', self._output_dir) 517 # Each board has a mkproper, config and then one make per commit 518 self.assertEqual(self._make_calls, len(boards) * (self._commits + 2)) 519 520 def testErrors(self): 521 """Test handling of build errors""" 522 self._error['board2', 1] = 'fred\n' 523 self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir) 524 self.assertEqual(self._builder.count, self._total_builds) 525 self.assertEqual(self._builder.fail, 1) 526 527 # Remove the error. This should have no effect since the commit will 528 # not be rebuilt 529 del self._error['board2', 1] 530 self._make_calls = 0 531 self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir, clean_dir=False) 532 self.assertEqual(self._builder.count, self._total_builds) 533 self.assertEqual(self._make_calls, 0) 534 self.assertEqual(self._builder.fail, 1) 535 536 # Now use the -F flag to force rebuild of the bad commit 537 self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir, '-F', clean_dir=False) 538 self.assertEqual(self._builder.count, self._total_builds) 539 self.assertEqual(self._builder.fail, 0) 540 self.assertEqual(self._make_calls, 2) 541 542 def testBranchWithSlash(self): 543 """Test building a branch with a '/' in the name""" 544 self._test_branch = '/__dev/__testbranch' 545 self._RunControl('-b', self._test_branch, clean_dir=False) 546 self.assertEqual(self._builder.count, self._total_builds) 547 self.assertEqual(self._builder.fail, 0) 548 549 def testEnvironment(self): 550 """Test that the done and environment files are written to out-env""" 551 self._RunControl('-o', self._output_dir) 552 board0_dir = os.path.join(self._output_dir, 'current', 'board0') 553 self.assertTrue(os.path.exists(os.path.join(board0_dir, 'done'))) 554 self.assertTrue(os.path.exists(os.path.join(board0_dir, 'out-env'))) 555 556 def testWorkInOutput(self): 557 """Test the -w option which should write directly to the output dir""" 558 board_list = board.Boards() 559 board_list.AddBoard(board.Board(*boards[0])) 560 self._RunControl('-o', self._output_dir, '-w', clean_dir=False, 561 boards=board_list) 562 self.assertTrue( 563 os.path.exists(os.path.join(self._output_dir, 'u-boot'))) 564 self.assertTrue( 565 os.path.exists(os.path.join(self._output_dir, 'done'))) 566 self.assertTrue( 567 os.path.exists(os.path.join(self._output_dir, 'out-env'))) 568 569 def testWorkInOutputFail(self): 570 """Test the -w option failures""" 571 with self.assertRaises(SystemExit) as e: 572 self._RunControl('-o', self._output_dir, '-w', clean_dir=False) 573 self.assertIn("single board", str(e.exception)) 574 self.assertFalse( 575 os.path.exists(os.path.join(self._output_dir, 'u-boot'))) 576 577 board_list = board.Boards() 578 board_list.AddBoard(board.Board(*boards[0])) 579 with self.assertRaises(SystemExit) as e: 580 self._RunControl('-b', self._test_branch, '-o', self._output_dir, 581 '-w', clean_dir=False, boards=board_list) 582 self.assertIn("single commit", str(e.exception)) 583 584 board_list = board.Boards() 585 board_list.AddBoard(board.Board(*boards[0])) 586 with self.assertRaises(SystemExit) as e: 587 self._RunControl('-w', clean_dir=False) 588 self.assertIn("specify -o", str(e.exception)) 589