1# SPDX-License-Identifier: GPL-2.0+
2# Copyright (c) 2014 Google, Inc
3#
4
5import os
6import shutil
7import sys
8import tempfile
9import unittest
10
11from buildman import board
12from buildman import bsettings
13from buildman import cmdline
14from buildman import control
15from buildman import toolchain
16from patman import command
17from patman import gitutil
18from patman import terminal
19from patman import test_util
20from patman import tools
21
22settings_data = '''
23# Buildman settings file
24
25[toolchain]
26
27[toolchain-alias]
28
29[make-flags]
30src=/home/sjg/c/src
31chroot=/home/sjg/c/chroot
32vboot=VBOOT_DEBUG=1 MAKEFLAGS_VBOOT=DEBUG=1 CFLAGS_EXTRA_VBOOT=-DUNROLL_LOOPS VBOOT_SOURCE=${src}/platform/vboot_reference
33chromeos_coreboot=VBOOT=${chroot}/build/link/usr ${vboot}
34chromeos_daisy=VBOOT=${chroot}/build/daisy/usr ${vboot}
35chromeos_peach=VBOOT=${chroot}/build/peach_pit/usr ${vboot}
36'''
37
38boards = [
39    ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 1', 'board0',  ''],
40    ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 2', 'board1', ''],
41    ['Active', 'powerpc', 'powerpc', '', 'Tester', 'PowerPC board 1', 'board2', ''],
42    ['Active', 'sandbox', 'sandbox', '', 'Tester', 'Sandbox board', 'board4', ''],
43]
44
45commit_shortlog = """4aca821 patman: Avoid changing the order of tags
4639403bb patman: Use --no-pager' to stop git from forking a pager
47db6e6f2 patman: Remove the -a option
48f2ccf03 patman: Correct unit tests to run correctly
491d097f9 patman: Fix indentation in terminal.py
50d073747 patman: Support the 'reverse' option for 'git log
51"""
52
53commit_log = ["""commit 7f6b8315d18f683c5181d0c3694818c1b2a20dcd
54Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
55Date:   Fri Aug 22 19:12:41 2014 +0900
56
57    buildman: refactor help message
58
59    "buildman [options]" is displayed by default.
60
61    Append the rest of help messages to parser.usage
62    instead of replacing it.
63
64    Besides, "-b <branch>" is not mandatory since commit fea5858e.
65    Drop it from the usage.
66
67    Signed-off-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
68""",
69"""commit d0737479be6baf4db5e2cdbee123e96bc5ed0ba8
70Author: Simon Glass <sjg@chromium.org>
71Date:   Thu Aug 14 16:48:25 2014 -0600
72
73    patman: Support the 'reverse' option for 'git log'
74
75    This option is currently not supported, but needs to be, for buildman to
76    operate as expected.
77
78    Series-changes: 7
79    - Add new patch to fix the 'reverse' bug
80
81    Series-version: 8
82
83    Change-Id: I79078f792e8b390b8a1272a8023537821d45feda
84    Reported-by: York Sun <yorksun@freescale.com>
85    Signed-off-by: Simon Glass <sjg@chromium.org>
86
87""",
88"""commit 1d097f9ab487c5019152fd47bda126839f3bf9fc
89Author: Simon Glass <sjg@chromium.org>
90Date:   Sat Aug 9 11:44:32 2014 -0600
91
92    patman: Fix indentation in terminal.py
93
94    This code came from a different project with 2-character indentation. Fix
95    it for U-Boot.
96
97    Series-changes: 6
98    - Add new patch to fix indentation in teminal.py
99
100    Change-Id: I5a74d2ebbb3cc12a665f5c725064009ac96e8a34
101    Signed-off-by: Simon Glass <sjg@chromium.org>
102
103""",
104"""commit f2ccf03869d1e152c836515a3ceb83cdfe04a105
105Author: Simon Glass <sjg@chromium.org>
106Date:   Sat Aug 9 11:08:24 2014 -0600
107
108    patman: Correct unit tests to run correctly
109
110    It seems that doctest behaves differently now, and some of the unit tests
111    do not run. Adjust the tests to work correctly.
112
113     ./tools/patman/patman --test
114    <unittest.result.TestResult run=10 errors=0 failures=0>
115
116    Series-changes: 6
117    - Add new patch to fix patman unit tests
118
119    Change-Id: I3d2ca588f4933e1f9d6b1665a00e4ae58269ff3b
120
121""",
122"""commit db6e6f2f9331c5a37647d6668768d4a40b8b0d1c
123Author: Simon Glass <sjg@chromium.org>
124Date:   Sat Aug 9 12:06:02 2014 -0600
125
126    patman: Remove the -a option
127
128    It seems that this is no longer needed, since checkpatch.pl will catch
129    whitespace problems in patches. Also the option is not widely used, so
130    it seems safe to just remove it.
131
132    Series-changes: 6
133    - Add new patch to remove patman's -a option
134
135    Suggested-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
136    Change-Id: I5821a1c75154e532c46513486ca40b808de7e2cc
137
138""",
139"""commit 39403bb4f838153028a6f21ca30bf100f3791133
140Author: Simon Glass <sjg@chromium.org>
141Date:   Thu Aug 14 21:50:52 2014 -0600
142
143    patman: Use --no-pager' to stop git from forking a pager
144
145""",
146"""commit 4aca821e27e97925c039e69fd37375b09c6f129c
147Author: Simon Glass <sjg@chromium.org>
148Date:   Fri Aug 22 15:57:39 2014 -0600
149
150    patman: Avoid changing the order of tags
151
152    patman collects tags that it sees in the commit and places them nicely
153    sorted at the end of the patch. However, this is not really necessary and
154    in fact is apparently not desirable.
155
156    Series-changes: 9
157    - Add new patch to avoid changing the order of tags
158
159    Series-version: 9
160
161    Suggested-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
162    Change-Id: Ib1518588c1a189ad5c3198aae76f8654aed8d0db
163"""]
164
165TEST_BRANCH = '__testbranch'
166
167class TestFunctional(unittest.TestCase):
168    """Functional test for buildman.
169
170    This aims to test from just below the invocation of buildman (parsing
171    of arguments) to 'make' and 'git' invocation. It is not a true
172    emd-to-end test, as it mocks git, make and the tool chain. But this
173    makes it easier to detect when the builder is doing the wrong thing,
174    since in many cases this test code will fail. For example, only a
175    very limited subset of 'git' arguments is supported - anything
176    unexpected will fail.
177    """
178    def setUp(self):
179        self._base_dir = tempfile.mkdtemp()
180        self._output_dir = tempfile.mkdtemp()
181        self._git_dir = os.path.join(self._base_dir, 'src')
182        self._buildman_pathname = sys.argv[0]
183        self._buildman_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
184        command.test_result = self._HandleCommand
185        self.setupToolchains()
186        self._toolchains.Add('arm-gcc', test=False)
187        self._toolchains.Add('powerpc-gcc', test=False)
188        bsettings.Setup(None)
189        bsettings.AddFile(settings_data)
190        self._boards = board.Boards()
191        for brd in boards:
192            self._boards.AddBoard(board.Board(*brd))
193
194        # Directories where the source been cloned
195        self._clone_dirs = []
196        self._commits = len(commit_shortlog.splitlines()) + 1
197        self._total_builds = self._commits * len(boards)
198
199        # Number of calls to make
200        self._make_calls = 0
201
202        # Map of [board, commit] to error messages
203        self._error = {}
204
205        self._test_branch = TEST_BRANCH
206
207        # Avoid sending any output and clear all terminal output
208        terminal.SetPrintTestMode()
209        terminal.GetPrintTestLines()
210
211    def tearDown(self):
212        shutil.rmtree(self._base_dir)
213        #shutil.rmtree(self._output_dir)
214
215    def setupToolchains(self):
216        self._toolchains = toolchain.Toolchains()
217        self._toolchains.Add('gcc', test=False)
218
219    def _RunBuildman(self, *args):
220        return command.RunPipe([[self._buildman_pathname] + list(args)],
221                capture=True, capture_stderr=True)
222
223    def _RunControl(self, *args, boards=None, clean_dir=False,
224                    test_thread_exceptions=False):
225        """Run buildman
226
227        Args:
228            args: List of arguments to pass
229            boards:
230            clean_dir: Used for tests only, indicates that the existing output_dir
231                should be removed before starting the build
232            test_thread_exceptions: Uses for tests only, True to make the threads
233                raise an exception instead of reporting their result. This simulates
234                a failure in the code somewhere
235
236        Returns:
237            result code from buildman
238        """
239        sys.argv = [sys.argv[0]] + list(args)
240        options, args = cmdline.ParseArgs()
241        result = control.DoBuildman(options, args, toolchains=self._toolchains,
242                make_func=self._HandleMake, boards=boards or self._boards,
243                clean_dir=clean_dir,
244                test_thread_exceptions=test_thread_exceptions)
245        self._builder = control.builder
246        return result
247
248    def testFullHelp(self):
249        command.test_result = None
250        result = self._RunBuildman('-H')
251        help_file = os.path.join(self._buildman_dir, 'README')
252        # Remove possible extraneous strings
253        extra = '::::::::::::::\n' + help_file + '\n::::::::::::::\n'
254        gothelp = result.stdout.replace(extra, '')
255        self.assertEqual(len(gothelp), os.path.getsize(help_file))
256        self.assertEqual(0, len(result.stderr))
257        self.assertEqual(0, result.return_code)
258
259    def testHelp(self):
260        command.test_result = None
261        result = self._RunBuildman('-h')
262        help_file = os.path.join(self._buildman_dir, 'README')
263        self.assertTrue(len(result.stdout) > 1000)
264        self.assertEqual(0, len(result.stderr))
265        self.assertEqual(0, result.return_code)
266
267    def testGitSetup(self):
268        """Test gitutils.Setup(), from outside the module itself"""
269        command.test_result = command.CommandResult(return_code=1)
270        gitutil.Setup()
271        self.assertEqual(gitutil.use_no_decorate, False)
272
273        command.test_result = command.CommandResult(return_code=0)
274        gitutil.Setup()
275        self.assertEqual(gitutil.use_no_decorate, True)
276
277    def _HandleCommandGitLog(self, args):
278        if args[-1] == '--':
279            args = args[:-1]
280        if '-n0' in args:
281            return command.CommandResult(return_code=0)
282        elif args[-1] == 'upstream/master..%s' % self._test_branch:
283            return command.CommandResult(return_code=0, stdout=commit_shortlog)
284        elif args[:3] == ['--no-color', '--no-decorate', '--reverse']:
285            if args[-1] == self._test_branch:
286                count = int(args[3][2:])
287                return command.CommandResult(return_code=0,
288                                            stdout=''.join(commit_log[:count]))
289
290        # Not handled, so abort
291        print('git log', args)
292        sys.exit(1)
293
294    def _HandleCommandGitConfig(self, args):
295        config = args[0]
296        if config == 'sendemail.aliasesfile':
297            return command.CommandResult(return_code=0)
298        elif config.startswith('branch.badbranch'):
299            return command.CommandResult(return_code=1)
300        elif config == 'branch.%s.remote' % self._test_branch:
301            return command.CommandResult(return_code=0, stdout='upstream\n')
302        elif config == 'branch.%s.merge' % self._test_branch:
303            return command.CommandResult(return_code=0,
304                                         stdout='refs/heads/master\n')
305
306        # Not handled, so abort
307        print('git config', args)
308        sys.exit(1)
309
310    def _HandleCommandGit(self, in_args):
311        """Handle execution of a git command
312
313        This uses a hacked-up parser.
314
315        Args:
316            in_args: Arguments after 'git' from the command line
317        """
318        git_args = []           # Top-level arguments to git itself
319        sub_cmd = None          # Git sub-command selected
320        args = []               # Arguments to the git sub-command
321        for arg in in_args:
322            if sub_cmd:
323                args.append(arg)
324            elif arg[0] == '-':
325                git_args.append(arg)
326            else:
327                if git_args and git_args[-1] in ['--git-dir', '--work-tree']:
328                    git_args.append(arg)
329                else:
330                    sub_cmd = arg
331        if sub_cmd == 'config':
332            return self._HandleCommandGitConfig(args)
333        elif sub_cmd == 'log':
334            return self._HandleCommandGitLog(args)
335        elif sub_cmd == 'clone':
336            return command.CommandResult(return_code=0)
337        elif sub_cmd == 'checkout':
338            return command.CommandResult(return_code=0)
339        elif sub_cmd == 'worktree':
340            return command.CommandResult(return_code=0)
341
342        # Not handled, so abort
343        print('git', git_args, sub_cmd, args)
344        sys.exit(1)
345
346    def _HandleCommandNm(self, args):
347        return command.CommandResult(return_code=0)
348
349    def _HandleCommandObjdump(self, args):
350        return command.CommandResult(return_code=0)
351
352    def _HandleCommandObjcopy(self, args):
353        return command.CommandResult(return_code=0)
354
355    def _HandleCommandSize(self, args):
356        return command.CommandResult(return_code=0)
357
358    def _HandleCommand(self, **kwargs):
359        """Handle a command execution.
360
361        The command is in kwargs['pipe-list'], as a list of pipes, each a
362        list of commands. The command should be emulated as required for
363        testing purposes.
364
365        Returns:
366            A CommandResult object
367        """
368        pipe_list = kwargs['pipe_list']
369        wc = False
370        if len(pipe_list) != 1:
371            if pipe_list[1] == ['wc', '-l']:
372                wc = True
373            else:
374                print('invalid pipe', kwargs)
375                sys.exit(1)
376        cmd = pipe_list[0][0]
377        args = pipe_list[0][1:]
378        result = None
379        if cmd == 'git':
380            result = self._HandleCommandGit(args)
381        elif cmd == './scripts/show-gnu-make':
382            return command.CommandResult(return_code=0, stdout='make')
383        elif cmd.endswith('nm'):
384            return self._HandleCommandNm(args)
385        elif cmd.endswith('objdump'):
386            return self._HandleCommandObjdump(args)
387        elif cmd.endswith('objcopy'):
388            return self._HandleCommandObjcopy(args)
389        elif cmd.endswith( 'size'):
390            return self._HandleCommandSize(args)
391
392        if not result:
393            # Not handled, so abort
394            print('unknown command', kwargs)
395            sys.exit(1)
396
397        if wc:
398            result.stdout = len(result.stdout.splitlines())
399        return result
400
401    def _HandleMake(self, commit, brd, stage, cwd, *args, **kwargs):
402        """Handle execution of 'make'
403
404        Args:
405            commit: Commit object that is being built
406            brd: Board object that is being built
407            stage: Stage that we are at (mrproper, config, build)
408            cwd: Directory where make should be run
409            args: Arguments to pass to make
410            kwargs: Arguments to pass to command.RunPipe()
411        """
412        self._make_calls += 1
413        if stage == 'mrproper':
414            return command.CommandResult(return_code=0)
415        elif stage == 'config':
416            return command.CommandResult(return_code=0,
417                    combined='Test configuration complete')
418        elif stage == 'build':
419            stderr = ''
420            out_dir = ''
421            for arg in args:
422                if arg.startswith('O='):
423                    out_dir = arg[2:]
424            fname = os.path.join(cwd or '', out_dir, 'u-boot')
425            tools.WriteFile(fname, b'U-Boot')
426            if type(commit) is not str:
427                stderr = self._error.get((brd.target, commit.sequence))
428            if stderr:
429                return command.CommandResult(return_code=1, stderr=stderr)
430            return command.CommandResult(return_code=0)
431
432        # Not handled, so abort
433        print('make', stage)
434        sys.exit(1)
435
436    # Example function to print output lines
437    def print_lines(self, lines):
438        print(len(lines))
439        for line in lines:
440            print(line)
441        #self.print_lines(terminal.GetPrintTestLines())
442
443    def testNoBoards(self):
444        """Test that buildman aborts when there are no boards"""
445        self._boards = board.Boards()
446        with self.assertRaises(SystemExit):
447            self._RunControl()
448
449    def testCurrentSource(self):
450        """Very simple test to invoke buildman on the current source"""
451        self.setupToolchains();
452        self._RunControl('-o', self._output_dir)
453        lines = terminal.GetPrintTestLines()
454        self.assertIn('Building current source for %d boards' % len(boards),
455                      lines[0].text)
456
457    def testBadBranch(self):
458        """Test that we can detect an invalid branch"""
459        with self.assertRaises(ValueError):
460            self._RunControl('-b', 'badbranch')
461
462    def testBadToolchain(self):
463        """Test that missing toolchains are detected"""
464        self.setupToolchains();
465        ret_code = self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir)
466        lines = terminal.GetPrintTestLines()
467
468        # Buildman always builds the upstream commit as well
469        self.assertIn('Building %d commits for %d boards' %
470                (self._commits, len(boards)), lines[0].text)
471        self.assertEqual(self._builder.count, self._total_builds)
472
473        # Only sandbox should succeed, the others don't have toolchains
474        self.assertEqual(self._builder.fail,
475                         self._total_builds - self._commits)
476        self.assertEqual(ret_code, 100)
477
478        for commit in range(self._commits):
479            for board in self._boards.GetList():
480                if board.arch != 'sandbox':
481                  errfile = self._builder.GetErrFile(commit, board.target)
482                  fd = open(errfile)
483                  self.assertEqual(fd.readlines(),
484                          ['No tool chain for %s\n' % board.arch])
485                  fd.close()
486
487    def testBranch(self):
488        """Test building a branch with all toolchains present"""
489        self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir)
490        self.assertEqual(self._builder.count, self._total_builds)
491        self.assertEqual(self._builder.fail, 0)
492
493    def testCount(self):
494        """Test building a specific number of commitst"""
495        self._RunControl('-b', TEST_BRANCH, '-c2', '-o', self._output_dir)
496        self.assertEqual(self._builder.count, 2 * len(boards))
497        self.assertEqual(self._builder.fail, 0)
498        # Each board has a config, and then one make per commit
499        self.assertEqual(self._make_calls, len(boards) * (1 + 2))
500
501    def testIncremental(self):
502        """Test building a branch twice - the second time should do nothing"""
503        self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir)
504
505        # Each board has a mrproper, config, and then one make per commit
506        self.assertEqual(self._make_calls, len(boards) * (self._commits + 1))
507        self._make_calls = 0
508        self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir, clean_dir=False)
509        self.assertEqual(self._make_calls, 0)
510        self.assertEqual(self._builder.count, self._total_builds)
511        self.assertEqual(self._builder.fail, 0)
512
513    def testForceBuild(self):
514        """The -f flag should force a rebuild"""
515        self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir)
516        self._make_calls = 0
517        self._RunControl('-b', TEST_BRANCH, '-f', '-o', self._output_dir, clean_dir=False)
518        # Each board has a config and one make per commit
519        self.assertEqual(self._make_calls, len(boards) * (self._commits + 1))
520
521    def testForceReconfigure(self):
522        """The -f flag should force a rebuild"""
523        self._RunControl('-b', TEST_BRANCH, '-C', '-o', self._output_dir)
524        # Each commit has a config and make
525        self.assertEqual(self._make_calls, len(boards) * self._commits * 2)
526
527    def testForceReconfigure(self):
528        """The -f flag should force a rebuild"""
529        self._RunControl('-b', TEST_BRANCH, '-C', '-o', self._output_dir)
530        # Each commit has a config and make
531        self.assertEqual(self._make_calls, len(boards) * self._commits * 2)
532
533    def testMrproper(self):
534        """The -f flag should force a rebuild"""
535        self._RunControl('-b', TEST_BRANCH, '-m', '-o', self._output_dir)
536        # Each board has a mkproper, config and then one make per commit
537        self.assertEqual(self._make_calls, len(boards) * (self._commits + 2))
538
539    def testErrors(self):
540        """Test handling of build errors"""
541        self._error['board2', 1] = 'fred\n'
542        self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir)
543        self.assertEqual(self._builder.count, self._total_builds)
544        self.assertEqual(self._builder.fail, 1)
545
546        # Remove the error. This should have no effect since the commit will
547        # not be rebuilt
548        del self._error['board2', 1]
549        self._make_calls = 0
550        self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir, clean_dir=False)
551        self.assertEqual(self._builder.count, self._total_builds)
552        self.assertEqual(self._make_calls, 0)
553        self.assertEqual(self._builder.fail, 1)
554
555        # Now use the -F flag to force rebuild of the bad commit
556        self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir, '-F', clean_dir=False)
557        self.assertEqual(self._builder.count, self._total_builds)
558        self.assertEqual(self._builder.fail, 0)
559        self.assertEqual(self._make_calls, 2)
560
561    def testBranchWithSlash(self):
562        """Test building a branch with a '/' in the name"""
563        self._test_branch = '/__dev/__testbranch'
564        self._RunControl('-b', self._test_branch, clean_dir=False)
565        self.assertEqual(self._builder.count, self._total_builds)
566        self.assertEqual(self._builder.fail, 0)
567
568    def testEnvironment(self):
569        """Test that the done and environment files are written to out-env"""
570        self._RunControl('-o', self._output_dir)
571        board0_dir = os.path.join(self._output_dir, 'current', 'board0')
572        self.assertTrue(os.path.exists(os.path.join(board0_dir, 'done')))
573        self.assertTrue(os.path.exists(os.path.join(board0_dir, 'out-env')))
574
575    def testEnvironmentUnicode(self):
576        """Test there are no unicode errors when the env has non-ASCII chars"""
577        try:
578            varname = b'buildman_test_var'
579            os.environb[varname] = b'strange\x80chars'
580            self.assertEqual(0, self._RunControl('-o', self._output_dir))
581            board0_dir = os.path.join(self._output_dir, 'current', 'board0')
582            self.assertTrue(os.path.exists(os.path.join(board0_dir, 'done')))
583            self.assertTrue(os.path.exists(os.path.join(board0_dir, 'out-env')))
584        finally:
585            del os.environb[varname]
586
587    def testWorkInOutput(self):
588        """Test the -w option which should write directly to the output dir"""
589        board_list = board.Boards()
590        board_list.AddBoard(board.Board(*boards[0]))
591        self._RunControl('-o', self._output_dir, '-w', clean_dir=False,
592                         boards=board_list)
593        self.assertTrue(
594            os.path.exists(os.path.join(self._output_dir, 'u-boot')))
595        self.assertTrue(
596            os.path.exists(os.path.join(self._output_dir, 'done')))
597        self.assertTrue(
598            os.path.exists(os.path.join(self._output_dir, 'out-env')))
599
600    def testWorkInOutputFail(self):
601        """Test the -w option failures"""
602        with self.assertRaises(SystemExit) as e:
603            self._RunControl('-o', self._output_dir, '-w', clean_dir=False)
604        self.assertIn("single board", str(e.exception))
605        self.assertFalse(
606            os.path.exists(os.path.join(self._output_dir, 'u-boot')))
607
608        board_list = board.Boards()
609        board_list.AddBoard(board.Board(*boards[0]))
610        with self.assertRaises(SystemExit) as e:
611            self._RunControl('-b', self._test_branch, '-o', self._output_dir,
612                             '-w', clean_dir=False, boards=board_list)
613        self.assertIn("single commit", str(e.exception))
614
615        board_list = board.Boards()
616        board_list.AddBoard(board.Board(*boards[0]))
617        with self.assertRaises(SystemExit) as e:
618            self._RunControl('-w', clean_dir=False)
619        self.assertIn("specify -o", str(e.exception))
620
621    def testThreadExceptions(self):
622        """Test that exceptions in threads are reported"""
623        with test_util.capture_sys_output() as (stdout, stderr):
624            self.assertEqual(102, self._RunControl('-o', self._output_dir,
625                                                   test_thread_exceptions=True))
626        self.assertIn('Thread exception: test exception', stdout.getvalue())
627