1# SPDX-License-Identifier: GPL-2.0+
2# Copyright (c) 2013 The Chromium OS Authors.
3#
4
5import multiprocessing
6import os
7import shutil
8import subprocess
9import sys
10
11from buildman import board
12from buildman import bsettings
13from buildman import toolchain
14from buildman.builder import Builder
15from patman import command
16from patman import gitutil
17from patman import patchstream
18from patman import terminal
19from patman.terminal import Print
20
21def GetPlural(count):
22    """Returns a plural 's' if count is not 1"""
23    return 's' if count != 1 else ''
24
25def GetActionSummary(is_summary, commits, selected, options):
26    """Return a string summarising the intended action.
27
28    Returns:
29        Summary string.
30    """
31    if commits:
32        count = len(commits)
33        count = (count + options.step - 1) // options.step
34        commit_str = '%d commit%s' % (count, GetPlural(count))
35    else:
36        commit_str = 'current source'
37    str = '%s %s for %d boards' % (
38        'Summary of' if is_summary else 'Building', commit_str,
39        len(selected))
40    str += ' (%d thread%s, %d job%s per thread)' % (options.threads,
41            GetPlural(options.threads), options.jobs, GetPlural(options.jobs))
42    return str
43
44def ShowActions(series, why_selected, boards_selected, builder, options,
45                board_warnings):
46    """Display a list of actions that we would take, if not a dry run.
47
48    Args:
49        series: Series object
50        why_selected: Dictionary where each key is a buildman argument
51                provided by the user, and the value is the list of boards
52                brought in by that argument. For example, 'arm' might bring
53                in 400 boards, so in this case the key would be 'arm' and
54                the value would be a list of board names.
55        boards_selected: Dict of selected boards, key is target name,
56                value is Board object
57        builder: The builder that will be used to build the commits
58        options: Command line options object
59        board_warnings: List of warnings obtained from board selected
60    """
61    col = terminal.Color()
62    print('Dry run, so not doing much. But I would do this:')
63    print()
64    if series:
65        commits = series.commits
66    else:
67        commits = None
68    print(GetActionSummary(False, commits, boards_selected,
69            options))
70    print('Build directory: %s' % builder.base_dir)
71    if commits:
72        for upto in range(0, len(series.commits), options.step):
73            commit = series.commits[upto]
74            print('   ', col.Color(col.YELLOW, commit.hash[:8], bright=False), end=' ')
75            print(commit.subject)
76    print()
77    for arg in why_selected:
78        if arg != 'all':
79            print(arg, ': %d boards' % len(why_selected[arg]))
80            if options.verbose:
81                print('   %s' % ' '.join(why_selected[arg]))
82    print(('Total boards to build for each commit: %d\n' %
83            len(why_selected['all'])))
84    if board_warnings:
85        for warning in board_warnings:
86            print(col.Color(col.YELLOW, warning))
87
88def ShowToolchainPrefix(boards, toolchains):
89    """Show information about a the tool chain used by one or more boards
90
91    The function checks that all boards use the same toolchain, then prints
92    the correct value for CROSS_COMPILE.
93
94    Args:
95        boards: Boards object containing selected boards
96        toolchains: Toolchains object containing available toolchains
97
98    Return:
99        None on success, string error message otherwise
100    """
101    boards = boards.GetSelectedDict()
102    tc_set = set()
103    for brd in boards.values():
104        tc_set.add(toolchains.Select(brd.arch))
105    if len(tc_set) != 1:
106        return 'Supplied boards must share one toolchain'
107        return False
108    tc = tc_set.pop()
109    print(tc.GetEnvArgs(toolchain.VAR_CROSS_COMPILE))
110    return None
111
112def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
113               clean_dir=False, test_thread_exceptions=False):
114    """The main control code for buildman
115
116    Args:
117        options: Command line options object
118        args: Command line arguments (list of strings)
119        toolchains: Toolchains to use - this should be a Toolchains()
120                object. If None, then it will be created and scanned
121        make_func: Make function to use for the builder. This is called
122                to execute 'make'. If this is None, the normal function
123                will be used, which calls the 'make' tool with suitable
124                arguments. This setting is useful for tests.
125        board: Boards() object to use, containing a list of available
126                boards. If this is None it will be created and scanned.
127        clean_dir: Used for tests only, indicates that the existing output_dir
128            should be removed before starting the build
129        test_thread_exceptions: Uses for tests only, True to make the threads
130            raise an exception instead of reporting their result. This simulates
131            a failure in the code somewhere
132    """
133    global builder
134
135    if options.full_help:
136        pager = os.getenv('PAGER')
137        if not pager:
138            pager = 'more'
139        fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])),
140                             'README')
141        command.Run(pager, fname)
142        return 0
143
144    gitutil.Setup()
145    col = terminal.Color()
146
147    options.git_dir = os.path.join(options.git, '.git')
148
149    no_toolchains = toolchains is None
150    if no_toolchains:
151        toolchains = toolchain.Toolchains(options.override_toolchain)
152
153    if options.fetch_arch:
154        if options.fetch_arch == 'list':
155            sorted_list = toolchains.ListArchs()
156            print(col.Color(col.BLUE, 'Available architectures: %s\n' %
157                            ' '.join(sorted_list)))
158            return 0
159        else:
160            fetch_arch = options.fetch_arch
161            if fetch_arch == 'all':
162                fetch_arch = ','.join(toolchains.ListArchs())
163                print(col.Color(col.CYAN, '\nDownloading toolchains: %s' %
164                                fetch_arch))
165            for arch in fetch_arch.split(','):
166                print()
167                ret = toolchains.FetchAndInstall(arch)
168                if ret:
169                    return ret
170            return 0
171
172    if no_toolchains:
173        toolchains.GetSettings()
174        toolchains.Scan(options.list_tool_chains and options.verbose)
175    if options.list_tool_chains:
176        toolchains.List()
177        print()
178        return 0
179
180    if options.incremental:
181        print(col.Color(col.RED,
182                        'Warning: -I has been removed. See documentation'))
183    if not options.output_dir:
184        if options.work_in_output:
185            sys.exit(col.Color(col.RED, '-w requires that you specify -o'))
186        options.output_dir = '..'
187
188    # Work out what subset of the boards we are building
189    if not boards:
190        if not os.path.exists(options.output_dir):
191            os.makedirs(options.output_dir)
192        board_file = os.path.join(options.output_dir, 'boards.cfg')
193        our_path = os.path.dirname(os.path.realpath(__file__))
194        genboardscfg = os.path.join(our_path, '../genboardscfg.py')
195        if not os.path.exists(genboardscfg):
196            genboardscfg = os.path.join(options.git, 'tools/genboardscfg.py')
197        status = subprocess.call([genboardscfg, '-q', '-o', board_file])
198        if status != 0:
199            # Older versions don't support -q
200            status = subprocess.call([genboardscfg, '-o', board_file])
201            if status != 0:
202                sys.exit("Failed to generate boards.cfg")
203
204        boards = board.Boards()
205        boards.ReadBoards(board_file)
206
207    exclude = []
208    if options.exclude:
209        for arg in options.exclude:
210            exclude += arg.split(',')
211
212    if options.boards:
213        requested_boards = []
214        for b in options.boards:
215            requested_boards += b.split(',')
216    else:
217        requested_boards = None
218    why_selected, board_warnings = boards.SelectBoards(args, exclude,
219                                                       requested_boards)
220    selected = boards.GetSelected()
221    if not len(selected):
222        sys.exit(col.Color(col.RED, 'No matching boards found'))
223
224    if options.print_prefix:
225        err = ShowToolchainPrefix(boards, toolchains)
226        if err:
227            sys.exit(col.Color(col.RED, err))
228        return 0
229
230    # Work out how many commits to build. We want to build everything on the
231    # branch. We also build the upstream commit as a control so we can see
232    # problems introduced by the first commit on the branch.
233    count = options.count
234    has_range = options.branch and '..' in options.branch
235    if count == -1:
236        if not options.branch:
237            count = 1
238        else:
239            if has_range:
240                count, msg = gitutil.CountCommitsInRange(options.git_dir,
241                                                         options.branch)
242            else:
243                count, msg = gitutil.CountCommitsInBranch(options.git_dir,
244                                                          options.branch)
245            if count is None:
246                sys.exit(col.Color(col.RED, msg))
247            elif count == 0:
248                sys.exit(col.Color(col.RED, "Range '%s' has no commits" %
249                                   options.branch))
250            if msg:
251                print(col.Color(col.YELLOW, msg))
252            count += 1   # Build upstream commit also
253
254    if not count:
255        str = ("No commits found to process in branch '%s': "
256               "set branch's upstream or use -c flag" % options.branch)
257        sys.exit(col.Color(col.RED, str))
258    if options.work_in_output:
259        if len(selected) != 1:
260            sys.exit(col.Color(col.RED,
261                               '-w can only be used with a single board'))
262        if count != 1:
263            sys.exit(col.Color(col.RED,
264                               '-w can only be used with a single commit'))
265
266    # Read the metadata from the commits. First look at the upstream commit,
267    # then the ones in the branch. We would like to do something like
268    # upstream/master~..branch but that isn't possible if upstream/master is
269    # a merge commit (it will list all the commits that form part of the
270    # merge)
271    # Conflicting tags are not a problem for buildman, since it does not use
272    # them. For example, Series-version is not useful for buildman. On the
273    # other hand conflicting tags will cause an error. So allow later tags
274    # to overwrite earlier ones by setting allow_overwrite=True
275    if options.branch:
276        if count == -1:
277            if has_range:
278                range_expr = options.branch
279            else:
280                range_expr = gitutil.GetRangeInBranch(options.git_dir,
281                                                      options.branch)
282            upstream_commit = gitutil.GetUpstream(options.git_dir,
283                                                  options.branch)
284            series = patchstream.get_metadata_for_list(upstream_commit,
285                options.git_dir, 1, series=None, allow_overwrite=True)
286
287            series = patchstream.get_metadata_for_list(range_expr,
288                    options.git_dir, None, series, allow_overwrite=True)
289        else:
290            # Honour the count
291            series = patchstream.get_metadata_for_list(options.branch,
292                    options.git_dir, count, series=None, allow_overwrite=True)
293    else:
294        series = None
295        if not options.dry_run:
296            options.verbose = True
297            if not options.summary:
298                options.show_errors = True
299
300    # By default we have one thread per CPU. But if there are not enough jobs
301    # we can have fewer threads and use a high '-j' value for make.
302    if options.threads is None:
303        options.threads = min(multiprocessing.cpu_count(), len(selected))
304    if not options.jobs:
305        options.jobs = max(1, (multiprocessing.cpu_count() +
306                len(selected) - 1) // len(selected))
307
308    if not options.step:
309        options.step = len(series.commits) - 1
310
311    gnu_make = command.Output(os.path.join(options.git,
312            'scripts/show-gnu-make'), raise_on_error=False).rstrip()
313    if not gnu_make:
314        sys.exit('GNU Make not found')
315
316    # Create a new builder with the selected options.
317    output_dir = options.output_dir
318    if options.branch:
319        dirname = options.branch.replace('/', '_')
320        # As a special case allow the board directory to be placed in the
321        # output directory itself rather than any subdirectory.
322        if not options.no_subdirs:
323            output_dir = os.path.join(options.output_dir, dirname)
324        if clean_dir and os.path.exists(output_dir):
325            shutil.rmtree(output_dir)
326    builder = Builder(toolchains, output_dir, options.git_dir,
327            options.threads, options.jobs, gnu_make=gnu_make, checkout=True,
328            show_unknown=options.show_unknown, step=options.step,
329            no_subdirs=options.no_subdirs, full_path=options.full_path,
330            verbose_build=options.verbose_build,
331            mrproper=options.mrproper,
332            per_board_out_dir=options.per_board_out_dir,
333            config_only=options.config_only,
334            squash_config_y=not options.preserve_config_y,
335            warnings_as_errors=options.warnings_as_errors,
336            work_in_output=options.work_in_output,
337            test_thread_exceptions=test_thread_exceptions)
338    builder.force_config_on_failure = not options.quick
339    if make_func:
340        builder.do_make = make_func
341
342    # For a dry run, just show our actions as a sanity check
343    if options.dry_run:
344        ShowActions(series, why_selected, selected, builder, options,
345                    board_warnings)
346    else:
347        builder.force_build = options.force_build
348        builder.force_build_failures = options.force_build_failures
349        builder.force_reconfig = options.force_reconfig
350        builder.in_tree = options.in_tree
351
352        # Work out which boards to build
353        board_selected = boards.GetSelectedDict()
354
355        if series:
356            commits = series.commits
357            # Number the commits for test purposes
358            for commit in range(len(commits)):
359                commits[commit].sequence = commit
360        else:
361            commits = None
362
363        Print(GetActionSummary(options.summary, commits, board_selected,
364                               options))
365
366        # We can't show function sizes without board details at present
367        if options.show_bloat:
368            options.show_detail = True
369        builder.SetDisplayOptions(
370            options.show_errors, options.show_sizes, options.show_detail,
371            options.show_bloat, options.list_error_boards, options.show_config,
372            options.show_environment, options.filter_dtb_warnings,
373            options.filter_migration_warnings)
374        if options.summary:
375            builder.ShowSummary(commits, board_selected)
376        else:
377            fail, warned, excs = builder.BuildBoards(
378                commits, board_selected, options.keep_outputs, options.verbose)
379            if excs:
380                return 102
381            elif fail:
382                return 100
383            elif warned and not options.ignore_warnings:
384                return 101
385    return 0
386