1# SPDX-License-Identifier: GPL-2.0+
2# Copyright (c) 2014 Google, Inc
3#
4
5import os
6import shutil
7import sys
8import tempfile
9import unittest
10
11from buildman import board
12from buildman import bsettings
13from buildman import cmdline
14from buildman import control
15from buildman import toolchain
16from patman import command
17from patman import gitutil
18from patman import terminal
19from patman import tools
20
21settings_data = '''
22# Buildman settings file
23
24[toolchain]
25
26[toolchain-alias]
27
28[make-flags]
29src=/home/sjg/c/src
30chroot=/home/sjg/c/chroot
31vboot=VBOOT_DEBUG=1 MAKEFLAGS_VBOOT=DEBUG=1 CFLAGS_EXTRA_VBOOT=-DUNROLL_LOOPS VBOOT_SOURCE=${src}/platform/vboot_reference
32chromeos_coreboot=VBOOT=${chroot}/build/link/usr ${vboot}
33chromeos_daisy=VBOOT=${chroot}/build/daisy/usr ${vboot}
34chromeos_peach=VBOOT=${chroot}/build/peach_pit/usr ${vboot}
35'''
36
37boards = [
38    ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 1', 'board0',  ''],
39    ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 2', 'board1', ''],
40    ['Active', 'powerpc', 'powerpc', '', 'Tester', 'PowerPC board 1', 'board2', ''],
41    ['Active', 'sandbox', 'sandbox', '', 'Tester', 'Sandbox board', 'board4', ''],
42]
43
44commit_shortlog = """4aca821 patman: Avoid changing the order of tags
4539403bb patman: Use --no-pager' to stop git from forking a pager
46db6e6f2 patman: Remove the -a option
47f2ccf03 patman: Correct unit tests to run correctly
481d097f9 patman: Fix indentation in terminal.py
49d073747 patman: Support the 'reverse' option for 'git log
50"""
51
52commit_log = ["""commit 7f6b8315d18f683c5181d0c3694818c1b2a20dcd
53Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
54Date:   Fri Aug 22 19:12:41 2014 +0900
55
56    buildman: refactor help message
57
58    "buildman [options]" is displayed by default.
59
60    Append the rest of help messages to parser.usage
61    instead of replacing it.
62
63    Besides, "-b <branch>" is not mandatory since commit fea5858e.
64    Drop it from the usage.
65
66    Signed-off-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
67""",
68"""commit d0737479be6baf4db5e2cdbee123e96bc5ed0ba8
69Author: Simon Glass <sjg@chromium.org>
70Date:   Thu Aug 14 16:48:25 2014 -0600
71
72    patman: Support the 'reverse' option for 'git log'
73
74    This option is currently not supported, but needs to be, for buildman to
75    operate as expected.
76
77    Series-changes: 7
78    - Add new patch to fix the 'reverse' bug
79
80    Series-version: 8
81
82    Change-Id: I79078f792e8b390b8a1272a8023537821d45feda
83    Reported-by: York Sun <yorksun@freescale.com>
84    Signed-off-by: Simon Glass <sjg@chromium.org>
85
86""",
87"""commit 1d097f9ab487c5019152fd47bda126839f3bf9fc
88Author: Simon Glass <sjg@chromium.org>
89Date:   Sat Aug 9 11:44:32 2014 -0600
90
91    patman: Fix indentation in terminal.py
92
93    This code came from a different project with 2-character indentation. Fix
94    it for U-Boot.
95
96    Series-changes: 6
97    - Add new patch to fix indentation in teminal.py
98
99    Change-Id: I5a74d2ebbb3cc12a665f5c725064009ac96e8a34
100    Signed-off-by: Simon Glass <sjg@chromium.org>
101
102""",
103"""commit f2ccf03869d1e152c836515a3ceb83cdfe04a105
104Author: Simon Glass <sjg@chromium.org>
105Date:   Sat Aug 9 11:08:24 2014 -0600
106
107    patman: Correct unit tests to run correctly
108
109    It seems that doctest behaves differently now, and some of the unit tests
110    do not run. Adjust the tests to work correctly.
111
112     ./tools/patman/patman --test
113    <unittest.result.TestResult run=10 errors=0 failures=0>
114
115    Series-changes: 6
116    - Add new patch to fix patman unit tests
117
118    Change-Id: I3d2ca588f4933e1f9d6b1665a00e4ae58269ff3b
119
120""",
121"""commit db6e6f2f9331c5a37647d6668768d4a40b8b0d1c
122Author: Simon Glass <sjg@chromium.org>
123Date:   Sat Aug 9 12:06:02 2014 -0600
124
125    patman: Remove the -a option
126
127    It seems that this is no longer needed, since checkpatch.pl will catch
128    whitespace problems in patches. Also the option is not widely used, so
129    it seems safe to just remove it.
130
131    Series-changes: 6
132    - Add new patch to remove patman's -a option
133
134    Suggested-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
135    Change-Id: I5821a1c75154e532c46513486ca40b808de7e2cc
136
137""",
138"""commit 39403bb4f838153028a6f21ca30bf100f3791133
139Author: Simon Glass <sjg@chromium.org>
140Date:   Thu Aug 14 21:50:52 2014 -0600
141
142    patman: Use --no-pager' to stop git from forking a pager
143
144""",
145"""commit 4aca821e27e97925c039e69fd37375b09c6f129c
146Author: Simon Glass <sjg@chromium.org>
147Date:   Fri Aug 22 15:57:39 2014 -0600
148
149    patman: Avoid changing the order of tags
150
151    patman collects tags that it sees in the commit and places them nicely
152    sorted at the end of the patch. However, this is not really necessary and
153    in fact is apparently not desirable.
154
155    Series-changes: 9
156    - Add new patch to avoid changing the order of tags
157
158    Series-version: 9
159
160    Suggested-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
161    Change-Id: Ib1518588c1a189ad5c3198aae76f8654aed8d0db
162"""]
163
164TEST_BRANCH = '__testbranch'
165
166class TestFunctional(unittest.TestCase):
167    """Functional test for buildman.
168
169    This aims to test from just below the invocation of buildman (parsing
170    of arguments) to 'make' and 'git' invocation. It is not a true
171    emd-to-end test, as it mocks git, make and the tool chain. But this
172    makes it easier to detect when the builder is doing the wrong thing,
173    since in many cases this test code will fail. For example, only a
174    very limited subset of 'git' arguments is supported - anything
175    unexpected will fail.
176    """
177    def setUp(self):
178        self._base_dir = tempfile.mkdtemp()
179        self._output_dir = tempfile.mkdtemp()
180        self._git_dir = os.path.join(self._base_dir, 'src')
181        self._buildman_pathname = sys.argv[0]
182        self._buildman_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
183        command.test_result = self._HandleCommand
184        self.setupToolchains()
185        self._toolchains.Add('arm-gcc', test=False)
186        self._toolchains.Add('powerpc-gcc', test=False)
187        bsettings.Setup(None)
188        bsettings.AddFile(settings_data)
189        self._boards = board.Boards()
190        for brd in boards:
191            self._boards.AddBoard(board.Board(*brd))
192
193        # Directories where the source been cloned
194        self._clone_dirs = []
195        self._commits = len(commit_shortlog.splitlines()) + 1
196        self._total_builds = self._commits * len(boards)
197
198        # Number of calls to make
199        self._make_calls = 0
200
201        # Map of [board, commit] to error messages
202        self._error = {}
203
204        self._test_branch = TEST_BRANCH
205
206        # Avoid sending any output and clear all terminal output
207        terminal.SetPrintTestMode()
208        terminal.GetPrintTestLines()
209
210    def tearDown(self):
211        shutil.rmtree(self._base_dir)
212        #shutil.rmtree(self._output_dir)
213
214    def setupToolchains(self):
215        self._toolchains = toolchain.Toolchains()
216        self._toolchains.Add('gcc', test=False)
217
218    def _RunBuildman(self, *args):
219        return command.RunPipe([[self._buildman_pathname] + list(args)],
220                capture=True, capture_stderr=True)
221
222    def _RunControl(self, *args, clean_dir=False, boards=None):
223        sys.argv = [sys.argv[0]] + list(args)
224        options, args = cmdline.ParseArgs()
225        result = control.DoBuildman(options, args, toolchains=self._toolchains,
226                make_func=self._HandleMake, boards=boards or self._boards,
227                clean_dir=clean_dir)
228        self._builder = control.builder
229        return result
230
231    def testFullHelp(self):
232        command.test_result = None
233        result = self._RunBuildman('-H')
234        help_file = os.path.join(self._buildman_dir, 'README')
235        # Remove possible extraneous strings
236        extra = '::::::::::::::\n' + help_file + '\n::::::::::::::\n'
237        gothelp = result.stdout.replace(extra, '')
238        self.assertEqual(len(gothelp), os.path.getsize(help_file))
239        self.assertEqual(0, len(result.stderr))
240        self.assertEqual(0, result.return_code)
241
242    def testHelp(self):
243        command.test_result = None
244        result = self._RunBuildman('-h')
245        help_file = os.path.join(self._buildman_dir, 'README')
246        self.assertTrue(len(result.stdout) > 1000)
247        self.assertEqual(0, len(result.stderr))
248        self.assertEqual(0, result.return_code)
249
250    def testGitSetup(self):
251        """Test gitutils.Setup(), from outside the module itself"""
252        command.test_result = command.CommandResult(return_code=1)
253        gitutil.Setup()
254        self.assertEqual(gitutil.use_no_decorate, False)
255
256        command.test_result = command.CommandResult(return_code=0)
257        gitutil.Setup()
258        self.assertEqual(gitutil.use_no_decorate, True)
259
260    def _HandleCommandGitLog(self, args):
261        if args[-1] == '--':
262            args = args[:-1]
263        if '-n0' in args:
264            return command.CommandResult(return_code=0)
265        elif args[-1] == 'upstream/master..%s' % self._test_branch:
266            return command.CommandResult(return_code=0, stdout=commit_shortlog)
267        elif args[:3] == ['--no-color', '--no-decorate', '--reverse']:
268            if args[-1] == self._test_branch:
269                count = int(args[3][2:])
270                return command.CommandResult(return_code=0,
271                                            stdout=''.join(commit_log[:count]))
272
273        # Not handled, so abort
274        print('git log', args)
275        sys.exit(1)
276
277    def _HandleCommandGitConfig(self, args):
278        config = args[0]
279        if config == 'sendemail.aliasesfile':
280            return command.CommandResult(return_code=0)
281        elif config.startswith('branch.badbranch'):
282            return command.CommandResult(return_code=1)
283        elif config == 'branch.%s.remote' % self._test_branch:
284            return command.CommandResult(return_code=0, stdout='upstream\n')
285        elif config == 'branch.%s.merge' % self._test_branch:
286            return command.CommandResult(return_code=0,
287                                         stdout='refs/heads/master\n')
288
289        # Not handled, so abort
290        print('git config', args)
291        sys.exit(1)
292
293    def _HandleCommandGit(self, in_args):
294        """Handle execution of a git command
295
296        This uses a hacked-up parser.
297
298        Args:
299            in_args: Arguments after 'git' from the command line
300        """
301        git_args = []           # Top-level arguments to git itself
302        sub_cmd = None          # Git sub-command selected
303        args = []               # Arguments to the git sub-command
304        for arg in in_args:
305            if sub_cmd:
306                args.append(arg)
307            elif arg[0] == '-':
308                git_args.append(arg)
309            else:
310                if git_args and git_args[-1] in ['--git-dir', '--work-tree']:
311                    git_args.append(arg)
312                else:
313                    sub_cmd = arg
314        if sub_cmd == 'config':
315            return self._HandleCommandGitConfig(args)
316        elif sub_cmd == 'log':
317            return self._HandleCommandGitLog(args)
318        elif sub_cmd == 'clone':
319            return command.CommandResult(return_code=0)
320        elif sub_cmd == 'checkout':
321            return command.CommandResult(return_code=0)
322
323        # Not handled, so abort
324        print('git', git_args, sub_cmd, args)
325        sys.exit(1)
326
327    def _HandleCommandNm(self, args):
328        return command.CommandResult(return_code=0)
329
330    def _HandleCommandObjdump(self, args):
331        return command.CommandResult(return_code=0)
332
333    def _HandleCommandObjcopy(self, args):
334        return command.CommandResult(return_code=0)
335
336    def _HandleCommandSize(self, args):
337        return command.CommandResult(return_code=0)
338
339    def _HandleCommand(self, **kwargs):
340        """Handle a command execution.
341
342        The command is in kwargs['pipe-list'], as a list of pipes, each a
343        list of commands. The command should be emulated as required for
344        testing purposes.
345
346        Returns:
347            A CommandResult object
348        """
349        pipe_list = kwargs['pipe_list']
350        wc = False
351        if len(pipe_list) != 1:
352            if pipe_list[1] == ['wc', '-l']:
353                wc = True
354            else:
355                print('invalid pipe', kwargs)
356                sys.exit(1)
357        cmd = pipe_list[0][0]
358        args = pipe_list[0][1:]
359        result = None
360        if cmd == 'git':
361            result = self._HandleCommandGit(args)
362        elif cmd == './scripts/show-gnu-make':
363            return command.CommandResult(return_code=0, stdout='make')
364        elif cmd.endswith('nm'):
365            return self._HandleCommandNm(args)
366        elif cmd.endswith('objdump'):
367            return self._HandleCommandObjdump(args)
368        elif cmd.endswith('objcopy'):
369            return self._HandleCommandObjcopy(args)
370        elif cmd.endswith( 'size'):
371            return self._HandleCommandSize(args)
372
373        if not result:
374            # Not handled, so abort
375            print('unknown command', kwargs)
376            sys.exit(1)
377
378        if wc:
379            result.stdout = len(result.stdout.splitlines())
380        return result
381
382    def _HandleMake(self, commit, brd, stage, cwd, *args, **kwargs):
383        """Handle execution of 'make'
384
385        Args:
386            commit: Commit object that is being built
387            brd: Board object that is being built
388            stage: Stage that we are at (mrproper, config, build)
389            cwd: Directory where make should be run
390            args: Arguments to pass to make
391            kwargs: Arguments to pass to command.RunPipe()
392        """
393        self._make_calls += 1
394        if stage == 'mrproper':
395            return command.CommandResult(return_code=0)
396        elif stage == 'config':
397            return command.CommandResult(return_code=0,
398                    combined='Test configuration complete')
399        elif stage == 'build':
400            stderr = ''
401            out_dir = ''
402            for arg in args:
403                if arg.startswith('O='):
404                    out_dir = arg[2:]
405            fname = os.path.join(cwd or '', out_dir, 'u-boot')
406            tools.WriteFile(fname, b'U-Boot')
407            if type(commit) is not str:
408                stderr = self._error.get((brd.target, commit.sequence))
409            if stderr:
410                return command.CommandResult(return_code=1, stderr=stderr)
411            return command.CommandResult(return_code=0)
412
413        # Not handled, so abort
414        print('make', stage)
415        sys.exit(1)
416
417    # Example function to print output lines
418    def print_lines(self, lines):
419        print(len(lines))
420        for line in lines:
421            print(line)
422        #self.print_lines(terminal.GetPrintTestLines())
423
424    def testNoBoards(self):
425        """Test that buildman aborts when there are no boards"""
426        self._boards = board.Boards()
427        with self.assertRaises(SystemExit):
428            self._RunControl()
429
430    def testCurrentSource(self):
431        """Very simple test to invoke buildman on the current source"""
432        self.setupToolchains();
433        self._RunControl('-o', self._output_dir)
434        lines = terminal.GetPrintTestLines()
435        self.assertIn('Building current source for %d boards' % len(boards),
436                      lines[0].text)
437
438    def testBadBranch(self):
439        """Test that we can detect an invalid branch"""
440        with self.assertRaises(ValueError):
441            self._RunControl('-b', 'badbranch')
442
443    def testBadToolchain(self):
444        """Test that missing toolchains are detected"""
445        self.setupToolchains();
446        ret_code = self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir)
447        lines = terminal.GetPrintTestLines()
448
449        # Buildman always builds the upstream commit as well
450        self.assertIn('Building %d commits for %d boards' %
451                (self._commits, len(boards)), lines[0].text)
452        self.assertEqual(self._builder.count, self._total_builds)
453
454        # Only sandbox should succeed, the others don't have toolchains
455        self.assertEqual(self._builder.fail,
456                         self._total_builds - self._commits)
457        self.assertEqual(ret_code, 100)
458
459        for commit in range(self._commits):
460            for board in self._boards.GetList():
461                if board.arch != 'sandbox':
462                  errfile = self._builder.GetErrFile(commit, board.target)
463                  fd = open(errfile)
464                  self.assertEqual(fd.readlines(),
465                          ['No tool chain for %s\n' % board.arch])
466                  fd.close()
467
468    def testBranch(self):
469        """Test building a branch with all toolchains present"""
470        self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir)
471        self.assertEqual(self._builder.count, self._total_builds)
472        self.assertEqual(self._builder.fail, 0)
473
474    def testCount(self):
475        """Test building a specific number of commitst"""
476        self._RunControl('-b', TEST_BRANCH, '-c2', '-o', self._output_dir)
477        self.assertEqual(self._builder.count, 2 * len(boards))
478        self.assertEqual(self._builder.fail, 0)
479        # Each board has a config, and then one make per commit
480        self.assertEqual(self._make_calls, len(boards) * (1 + 2))
481
482    def testIncremental(self):
483        """Test building a branch twice - the second time should do nothing"""
484        self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir)
485
486        # Each board has a mrproper, config, and then one make per commit
487        self.assertEqual(self._make_calls, len(boards) * (self._commits + 1))
488        self._make_calls = 0
489        self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir, clean_dir=False)
490        self.assertEqual(self._make_calls, 0)
491        self.assertEqual(self._builder.count, self._total_builds)
492        self.assertEqual(self._builder.fail, 0)
493
494    def testForceBuild(self):
495        """The -f flag should force a rebuild"""
496        self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir)
497        self._make_calls = 0
498        self._RunControl('-b', TEST_BRANCH, '-f', '-o', self._output_dir, clean_dir=False)
499        # Each board has a config and one make per commit
500        self.assertEqual(self._make_calls, len(boards) * (self._commits + 1))
501
502    def testForceReconfigure(self):
503        """The -f flag should force a rebuild"""
504        self._RunControl('-b', TEST_BRANCH, '-C', '-o', self._output_dir)
505        # Each commit has a config and make
506        self.assertEqual(self._make_calls, len(boards) * self._commits * 2)
507
508    def testForceReconfigure(self):
509        """The -f flag should force a rebuild"""
510        self._RunControl('-b', TEST_BRANCH, '-C', '-o', self._output_dir)
511        # Each commit has a config and make
512        self.assertEqual(self._make_calls, len(boards) * self._commits * 2)
513
514    def testMrproper(self):
515        """The -f flag should force a rebuild"""
516        self._RunControl('-b', TEST_BRANCH, '-m', '-o', self._output_dir)
517        # Each board has a mkproper, config and then one make per commit
518        self.assertEqual(self._make_calls, len(boards) * (self._commits + 2))
519
520    def testErrors(self):
521        """Test handling of build errors"""
522        self._error['board2', 1] = 'fred\n'
523        self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir)
524        self.assertEqual(self._builder.count, self._total_builds)
525        self.assertEqual(self._builder.fail, 1)
526
527        # Remove the error. This should have no effect since the commit will
528        # not be rebuilt
529        del self._error['board2', 1]
530        self._make_calls = 0
531        self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir, clean_dir=False)
532        self.assertEqual(self._builder.count, self._total_builds)
533        self.assertEqual(self._make_calls, 0)
534        self.assertEqual(self._builder.fail, 1)
535
536        # Now use the -F flag to force rebuild of the bad commit
537        self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir, '-F', clean_dir=False)
538        self.assertEqual(self._builder.count, self._total_builds)
539        self.assertEqual(self._builder.fail, 0)
540        self.assertEqual(self._make_calls, 2)
541
542    def testBranchWithSlash(self):
543        """Test building a branch with a '/' in the name"""
544        self._test_branch = '/__dev/__testbranch'
545        self._RunControl('-b', self._test_branch, clean_dir=False)
546        self.assertEqual(self._builder.count, self._total_builds)
547        self.assertEqual(self._builder.fail, 0)
548
549    def testEnvironment(self):
550        """Test that the done and environment files are written to out-env"""
551        self._RunControl('-o', self._output_dir)
552        board0_dir = os.path.join(self._output_dir, 'current', 'board0')
553        self.assertTrue(os.path.exists(os.path.join(board0_dir, 'done')))
554        self.assertTrue(os.path.exists(os.path.join(board0_dir, 'out-env')))
555
556    def testWorkInOutput(self):
557        """Test the -w option which should write directly to the output dir"""
558        board_list = board.Boards()
559        board_list.AddBoard(board.Board(*boards[0]))
560        self._RunControl('-o', self._output_dir, '-w', clean_dir=False,
561                         boards=board_list)
562        self.assertTrue(
563            os.path.exists(os.path.join(self._output_dir, 'u-boot')))
564        self.assertTrue(
565            os.path.exists(os.path.join(self._output_dir, 'done')))
566        self.assertTrue(
567            os.path.exists(os.path.join(self._output_dir, 'out-env')))
568
569    def testWorkInOutputFail(self):
570        """Test the -w option failures"""
571        with self.assertRaises(SystemExit) as e:
572            self._RunControl('-o', self._output_dir, '-w', clean_dir=False)
573        self.assertIn("single board", str(e.exception))
574        self.assertFalse(
575            os.path.exists(os.path.join(self._output_dir, 'u-boot')))
576
577        board_list = board.Boards()
578        board_list.AddBoard(board.Board(*boards[0]))
579        with self.assertRaises(SystemExit) as e:
580            self._RunControl('-b', self._test_branch, '-o', self._output_dir,
581                             '-w', clean_dir=False, boards=board_list)
582        self.assertIn("single commit", str(e.exception))
583
584        board_list = board.Boards()
585        board_list.AddBoard(board.Board(*boards[0]))
586        with self.assertRaises(SystemExit) as e:
587            self._RunControl('-w', clean_dir=False)
588        self.assertIn("specify -o", str(e.exception))
589