1# Copyright (c) 2013 The Chromium OS Authors.
2#
3# SPDX-License-Identifier:	GPL-2.0+
4#
5
6import multiprocessing
7import os
8import shutil
9import sys
10
11import board
12import bsettings
13from builder import Builder
14import gitutil
15import patchstream
16import terminal
17from terminal import Print
18import toolchain
19import command
20import subprocess
21
22def GetPlural(count):
23    """Returns a plural 's' if count is not 1"""
24    return 's' if count != 1 else ''
25
26def GetActionSummary(is_summary, commits, selected, options):
27    """Return a string summarising the intended action.
28
29    Returns:
30        Summary string.
31    """
32    if commits:
33        count = len(commits)
34        count = (count + options.step - 1) / options.step
35        commit_str = '%d commit%s' % (count, GetPlural(count))
36    else:
37        commit_str = 'current source'
38    str = '%s %s for %d boards' % (
39        'Summary of' if is_summary else 'Building', commit_str,
40        len(selected))
41    str += ' (%d thread%s, %d job%s per thread)' % (options.threads,
42            GetPlural(options.threads), options.jobs, GetPlural(options.jobs))
43    return str
44
45def ShowActions(series, why_selected, boards_selected, builder, options):
46    """Display a list of actions that we would take, if not a dry run.
47
48    Args:
49        series: Series object
50        why_selected: Dictionary where each key is a buildman argument
51                provided by the user, and the value is the boards brought
52                in by that argument. For example, 'arm' might bring in
53                400 boards, so in this case the key would be 'arm' and
54                the value would be a list of board names.
55        boards_selected: Dict of selected boards, key is target name,
56                value is Board object
57        builder: The builder that will be used to build the commits
58        options: Command line options object
59    """
60    col = terminal.Color()
61    print 'Dry run, so not doing much. But I would do this:'
62    print
63    if series:
64        commits = series.commits
65    else:
66        commits = None
67    print GetActionSummary(False, commits, boards_selected,
68            options)
69    print 'Build directory: %s' % builder.base_dir
70    if commits:
71        for upto in range(0, len(series.commits), options.step):
72            commit = series.commits[upto]
73            print '   ', col.Color(col.YELLOW, commit.hash[:8], bright=False),
74            print commit.subject
75    print
76    for arg in why_selected:
77        if arg != 'all':
78            print arg, ': %d boards' % why_selected[arg]
79    print ('Total boards to build for each commit: %d\n' %
80            why_selected['all'])
81
82def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
83               clean_dir=False):
84    """The main control code for buildman
85
86    Args:
87        options: Command line options object
88        args: Command line arguments (list of strings)
89        toolchains: Toolchains to use - this should be a Toolchains()
90                object. If None, then it will be created and scanned
91        make_func: Make function to use for the builder. This is called
92                to execute 'make'. If this is None, the normal function
93                will be used, which calls the 'make' tool with suitable
94                arguments. This setting is useful for tests.
95        board: Boards() object to use, containing a list of available
96                boards. If this is None it will be created and scanned.
97    """
98    global builder
99
100    if options.full_help:
101        pager = os.getenv('PAGER')
102        if not pager:
103            pager = 'more'
104        fname = os.path.join(os.path.dirname(sys.argv[0]), 'README')
105        command.Run(pager, fname)
106        return 0
107
108    gitutil.Setup()
109
110    options.git_dir = os.path.join(options.git, '.git')
111
112    if not toolchains:
113        toolchains = toolchain.Toolchains()
114        toolchains.GetSettings()
115        toolchains.Scan(options.list_tool_chains)
116    if options.list_tool_chains:
117        toolchains.List()
118        print
119        return 0
120
121    if options.fetch_arch:
122        if options.fetch_arch == 'list':
123            sorted_list = toolchains.ListArchs()
124            print 'Available architectures: %s\n' % ' '.join(sorted_list)
125            return 0
126        else:
127            fetch_arch = options.fetch_arch
128            if fetch_arch == 'all':
129                fetch_arch = ','.join(toolchains.ListArchs())
130                print 'Downloading toolchains: %s\n' % fetch_arch
131            for arch in fetch_arch.split(','):
132                ret = toolchains.FetchAndInstall(arch)
133                if ret:
134                    return ret
135            return 0
136
137    # Work out how many commits to build. We want to build everything on the
138    # branch. We also build the upstream commit as a control so we can see
139    # problems introduced by the first commit on the branch.
140    col = terminal.Color()
141    count = options.count
142    has_range = options.branch and '..' in options.branch
143    if count == -1:
144        if not options.branch:
145            count = 1
146        else:
147            if has_range:
148                count, msg = gitutil.CountCommitsInRange(options.git_dir,
149                                                         options.branch)
150            else:
151                count, msg = gitutil.CountCommitsInBranch(options.git_dir,
152                                                          options.branch)
153            if count is None:
154                sys.exit(col.Color(col.RED, msg))
155            elif count == 0:
156                sys.exit(col.Color(col.RED, "Range '%s' has no commits" %
157                                   options.branch))
158            if msg:
159                print col.Color(col.YELLOW, msg)
160            count += 1   # Build upstream commit also
161
162    if not count:
163        str = ("No commits found to process in branch '%s': "
164               "set branch's upstream or use -c flag" % options.branch)
165        sys.exit(col.Color(col.RED, str))
166
167    # Work out what subset of the boards we are building
168    if not boards:
169        board_file = os.path.join(options.git, 'boards.cfg')
170        status = subprocess.call([os.path.join(options.git,
171                                                'tools/genboardscfg.py')])
172        if status != 0:
173                sys.exit("Failed to generate boards.cfg")
174
175        boards = board.Boards()
176        boards.ReadBoards(os.path.join(options.git, 'boards.cfg'))
177
178    exclude = []
179    if options.exclude:
180        for arg in options.exclude:
181            exclude += arg.split(',')
182
183    why_selected = boards.SelectBoards(args, exclude)
184    selected = boards.GetSelected()
185    if not len(selected):
186        sys.exit(col.Color(col.RED, 'No matching boards found'))
187
188    # Read the metadata from the commits. First look at the upstream commit,
189    # then the ones in the branch. We would like to do something like
190    # upstream/master~..branch but that isn't possible if upstream/master is
191    # a merge commit (it will list all the commits that form part of the
192    # merge)
193    # Conflicting tags are not a problem for buildman, since it does not use
194    # them. For example, Series-version is not useful for buildman. On the
195    # other hand conflicting tags will cause an error. So allow later tags
196    # to overwrite earlier ones by setting allow_overwrite=True
197    if options.branch:
198        if count == -1:
199            if has_range:
200                range_expr = options.branch
201            else:
202                range_expr = gitutil.GetRangeInBranch(options.git_dir,
203                                                      options.branch)
204            upstream_commit = gitutil.GetUpstream(options.git_dir,
205                                                  options.branch)
206            series = patchstream.GetMetaDataForList(upstream_commit,
207                options.git_dir, 1, series=None, allow_overwrite=True)
208
209            series = patchstream.GetMetaDataForList(range_expr,
210                    options.git_dir, None, series, allow_overwrite=True)
211        else:
212            # Honour the count
213            series = patchstream.GetMetaDataForList(options.branch,
214                    options.git_dir, count, series=None, allow_overwrite=True)
215    else:
216        series = None
217        options.verbose = True
218        if not options.summary:
219            options.show_errors = True
220
221    # By default we have one thread per CPU. But if there are not enough jobs
222    # we can have fewer threads and use a high '-j' value for make.
223    if not options.threads:
224        options.threads = min(multiprocessing.cpu_count(), len(selected))
225    if not options.jobs:
226        options.jobs = max(1, (multiprocessing.cpu_count() +
227                len(selected) - 1) / len(selected))
228
229    if not options.step:
230        options.step = len(series.commits) - 1
231
232    gnu_make = command.Output(os.path.join(options.git,
233                                           'scripts/show-gnu-make')).rstrip()
234    if not gnu_make:
235        sys.exit('GNU Make not found')
236
237    # Create a new builder with the selected options.
238    output_dir = options.output_dir
239    if options.branch:
240        dirname = options.branch.replace('/', '_')
241        # As a special case allow the board directory to be placed in the
242        # output directory itself rather than any subdirectory.
243        if not options.no_subdirs:
244            output_dir = os.path.join(options.output_dir, dirname)
245    if (clean_dir and output_dir != options.output_dir and
246            os.path.exists(output_dir)):
247        shutil.rmtree(output_dir)
248    builder = Builder(toolchains, output_dir, options.git_dir,
249            options.threads, options.jobs, gnu_make=gnu_make, checkout=True,
250            show_unknown=options.show_unknown, step=options.step,
251            no_subdirs=options.no_subdirs, full_path=options.full_path,
252            verbose_build=options.verbose_build)
253    builder.force_config_on_failure = not options.quick
254    if make_func:
255        builder.do_make = make_func
256
257    # For a dry run, just show our actions as a sanity check
258    if options.dry_run:
259        ShowActions(series, why_selected, selected, builder, options)
260    else:
261        builder.force_build = options.force_build
262        builder.force_build_failures = options.force_build_failures
263        builder.force_reconfig = options.force_reconfig
264        builder.in_tree = options.in_tree
265
266        # Work out which boards to build
267        board_selected = boards.GetSelectedDict()
268
269        if series:
270            commits = series.commits
271            # Number the commits for test purposes
272            for commit in range(len(commits)):
273                commits[commit].sequence = commit
274        else:
275            commits = None
276
277        Print(GetActionSummary(options.summary, commits, board_selected,
278                                options))
279
280        # We can't show function sizes without board details at present
281        if options.show_bloat:
282            options.show_detail = True
283        builder.SetDisplayOptions(options.show_errors, options.show_sizes,
284                                  options.show_detail, options.show_bloat,
285                                  options.list_error_boards,
286                                  options.show_config)
287        if options.summary:
288            builder.ShowSummary(commits, board_selected)
289        else:
290            fail, warned = builder.BuildBoards(commits, board_selected,
291                                options.keep_outputs, options.verbose)
292            if fail:
293                return 128
294            elif warned:
295                return 129
296    return 0
297