1# SPDX-License-Identifier: GPL-2.0+ 2# Copyright (c) 2013 The Chromium OS Authors. 3# 4 5import multiprocessing 6import os 7import shutil 8import subprocess 9import sys 10 11from buildman import board 12from buildman import bsettings 13from buildman import toolchain 14from buildman.builder import Builder 15from patman import command 16from patman import gitutil 17from patman import patchstream 18from patman import terminal 19from patman.terminal import Print 20 21def GetPlural(count): 22 """Returns a plural 's' if count is not 1""" 23 return 's' if count != 1 else '' 24 25def GetActionSummary(is_summary, commits, selected, options): 26 """Return a string summarising the intended action. 27 28 Returns: 29 Summary string. 30 """ 31 if commits: 32 count = len(commits) 33 count = (count + options.step - 1) // options.step 34 commit_str = '%d commit%s' % (count, GetPlural(count)) 35 else: 36 commit_str = 'current source' 37 str = '%s %s for %d boards' % ( 38 'Summary of' if is_summary else 'Building', commit_str, 39 len(selected)) 40 str += ' (%d thread%s, %d job%s per thread)' % (options.threads, 41 GetPlural(options.threads), options.jobs, GetPlural(options.jobs)) 42 return str 43 44def ShowActions(series, why_selected, boards_selected, builder, options, 45 board_warnings): 46 """Display a list of actions that we would take, if not a dry run. 47 48 Args: 49 series: Series object 50 why_selected: Dictionary where each key is a buildman argument 51 provided by the user, and the value is the list of boards 52 brought in by that argument. For example, 'arm' might bring 53 in 400 boards, so in this case the key would be 'arm' and 54 the value would be a list of board names. 55 boards_selected: Dict of selected boards, key is target name, 56 value is Board object 57 builder: The builder that will be used to build the commits 58 options: Command line options object 59 board_warnings: List of warnings obtained from board selected 60 """ 61 col = terminal.Color() 62 print('Dry run, so not doing much. But I would do this:') 63 print() 64 if series: 65 commits = series.commits 66 else: 67 commits = None 68 print(GetActionSummary(False, commits, boards_selected, 69 options)) 70 print('Build directory: %s' % builder.base_dir) 71 if commits: 72 for upto in range(0, len(series.commits), options.step): 73 commit = series.commits[upto] 74 print(' ', col.Color(col.YELLOW, commit.hash[:8], bright=False), end=' ') 75 print(commit.subject) 76 print() 77 for arg in why_selected: 78 if arg != 'all': 79 print(arg, ': %d boards' % len(why_selected[arg])) 80 if options.verbose: 81 print(' %s' % ' '.join(why_selected[arg])) 82 print(('Total boards to build for each commit: %d\n' % 83 len(why_selected['all']))) 84 if board_warnings: 85 for warning in board_warnings: 86 print(col.Color(col.YELLOW, warning)) 87 88def ShowToolchainPrefix(boards, toolchains): 89 """Show information about a the tool chain used by one or more boards 90 91 The function checks that all boards use the same toolchain, then prints 92 the correct value for CROSS_COMPILE. 93 94 Args: 95 boards: Boards object containing selected boards 96 toolchains: Toolchains object containing available toolchains 97 98 Return: 99 None on success, string error message otherwise 100 """ 101 boards = boards.GetSelectedDict() 102 tc_set = set() 103 for brd in boards.values(): 104 tc_set.add(toolchains.Select(brd.arch)) 105 if len(tc_set) != 1: 106 return 'Supplied boards must share one toolchain' 107 return False 108 tc = tc_set.pop() 109 print(tc.GetEnvArgs(toolchain.VAR_CROSS_COMPILE)) 110 return None 111 112def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, 113 clean_dir=False, test_thread_exceptions=False): 114 """The main control code for buildman 115 116 Args: 117 options: Command line options object 118 args: Command line arguments (list of strings) 119 toolchains: Toolchains to use - this should be a Toolchains() 120 object. If None, then it will be created and scanned 121 make_func: Make function to use for the builder. This is called 122 to execute 'make'. If this is None, the normal function 123 will be used, which calls the 'make' tool with suitable 124 arguments. This setting is useful for tests. 125 board: Boards() object to use, containing a list of available 126 boards. If this is None it will be created and scanned. 127 clean_dir: Used for tests only, indicates that the existing output_dir 128 should be removed before starting the build 129 test_thread_exceptions: Uses for tests only, True to make the threads 130 raise an exception instead of reporting their result. This simulates 131 a failure in the code somewhere 132 """ 133 global builder 134 135 if options.full_help: 136 pager = os.getenv('PAGER') 137 if not pager: 138 pager = 'more' 139 fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), 140 'README') 141 command.Run(pager, fname) 142 return 0 143 144 gitutil.Setup() 145 col = terminal.Color() 146 147 options.git_dir = os.path.join(options.git, '.git') 148 149 no_toolchains = toolchains is None 150 if no_toolchains: 151 toolchains = toolchain.Toolchains(options.override_toolchain) 152 153 if options.fetch_arch: 154 if options.fetch_arch == 'list': 155 sorted_list = toolchains.ListArchs() 156 print(col.Color(col.BLUE, 'Available architectures: %s\n' % 157 ' '.join(sorted_list))) 158 return 0 159 else: 160 fetch_arch = options.fetch_arch 161 if fetch_arch == 'all': 162 fetch_arch = ','.join(toolchains.ListArchs()) 163 print(col.Color(col.CYAN, '\nDownloading toolchains: %s' % 164 fetch_arch)) 165 for arch in fetch_arch.split(','): 166 print() 167 ret = toolchains.FetchAndInstall(arch) 168 if ret: 169 return ret 170 return 0 171 172 if no_toolchains: 173 toolchains.GetSettings() 174 toolchains.Scan(options.list_tool_chains and options.verbose) 175 if options.list_tool_chains: 176 toolchains.List() 177 print() 178 return 0 179 180 if options.incremental: 181 print(col.Color(col.RED, 182 'Warning: -I has been removed. See documentation')) 183 if not options.output_dir: 184 if options.work_in_output: 185 sys.exit(col.Color(col.RED, '-w requires that you specify -o')) 186 options.output_dir = '..' 187 188 # Work out what subset of the boards we are building 189 if not boards: 190 if not os.path.exists(options.output_dir): 191 os.makedirs(options.output_dir) 192 board_file = os.path.join(options.output_dir, 'boards.cfg') 193 our_path = os.path.dirname(os.path.realpath(__file__)) 194 genboardscfg = os.path.join(our_path, '../genboardscfg.py') 195 if not os.path.exists(genboardscfg): 196 genboardscfg = os.path.join(options.git, 'tools/genboardscfg.py') 197 status = subprocess.call([genboardscfg, '-q', '-o', board_file]) 198 if status != 0: 199 # Older versions don't support -q 200 status = subprocess.call([genboardscfg, '-o', board_file]) 201 if status != 0: 202 sys.exit("Failed to generate boards.cfg") 203 204 boards = board.Boards() 205 boards.ReadBoards(board_file) 206 207 exclude = [] 208 if options.exclude: 209 for arg in options.exclude: 210 exclude += arg.split(',') 211 212 if options.boards: 213 requested_boards = [] 214 for b in options.boards: 215 requested_boards += b.split(',') 216 else: 217 requested_boards = None 218 why_selected, board_warnings = boards.SelectBoards(args, exclude, 219 requested_boards) 220 selected = boards.GetSelected() 221 if not len(selected): 222 sys.exit(col.Color(col.RED, 'No matching boards found')) 223 224 if options.print_prefix: 225 err = ShowToolchainPrefix(boards, toolchains) 226 if err: 227 sys.exit(col.Color(col.RED, err)) 228 return 0 229 230 # Work out how many commits to build. We want to build everything on the 231 # branch. We also build the upstream commit as a control so we can see 232 # problems introduced by the first commit on the branch. 233 count = options.count 234 has_range = options.branch and '..' in options.branch 235 if count == -1: 236 if not options.branch: 237 count = 1 238 else: 239 if has_range: 240 count, msg = gitutil.CountCommitsInRange(options.git_dir, 241 options.branch) 242 else: 243 count, msg = gitutil.CountCommitsInBranch(options.git_dir, 244 options.branch) 245 if count is None: 246 sys.exit(col.Color(col.RED, msg)) 247 elif count == 0: 248 sys.exit(col.Color(col.RED, "Range '%s' has no commits" % 249 options.branch)) 250 if msg: 251 print(col.Color(col.YELLOW, msg)) 252 count += 1 # Build upstream commit also 253 254 if not count: 255 str = ("No commits found to process in branch '%s': " 256 "set branch's upstream or use -c flag" % options.branch) 257 sys.exit(col.Color(col.RED, str)) 258 if options.work_in_output: 259 if len(selected) != 1: 260 sys.exit(col.Color(col.RED, 261 '-w can only be used with a single board')) 262 if count != 1: 263 sys.exit(col.Color(col.RED, 264 '-w can only be used with a single commit')) 265 266 # Read the metadata from the commits. First look at the upstream commit, 267 # then the ones in the branch. We would like to do something like 268 # upstream/master~..branch but that isn't possible if upstream/master is 269 # a merge commit (it will list all the commits that form part of the 270 # merge) 271 # Conflicting tags are not a problem for buildman, since it does not use 272 # them. For example, Series-version is not useful for buildman. On the 273 # other hand conflicting tags will cause an error. So allow later tags 274 # to overwrite earlier ones by setting allow_overwrite=True 275 if options.branch: 276 if count == -1: 277 if has_range: 278 range_expr = options.branch 279 else: 280 range_expr = gitutil.GetRangeInBranch(options.git_dir, 281 options.branch) 282 upstream_commit = gitutil.GetUpstream(options.git_dir, 283 options.branch) 284 series = patchstream.get_metadata_for_list(upstream_commit, 285 options.git_dir, 1, series=None, allow_overwrite=True) 286 287 series = patchstream.get_metadata_for_list(range_expr, 288 options.git_dir, None, series, allow_overwrite=True) 289 else: 290 # Honour the count 291 series = patchstream.get_metadata_for_list(options.branch, 292 options.git_dir, count, series=None, allow_overwrite=True) 293 else: 294 series = None 295 if not options.dry_run: 296 options.verbose = True 297 if not options.summary: 298 options.show_errors = True 299 300 # By default we have one thread per CPU. But if there are not enough jobs 301 # we can have fewer threads and use a high '-j' value for make. 302 if options.threads is None: 303 options.threads = min(multiprocessing.cpu_count(), len(selected)) 304 if not options.jobs: 305 options.jobs = max(1, (multiprocessing.cpu_count() + 306 len(selected) - 1) // len(selected)) 307 308 if not options.step: 309 options.step = len(series.commits) - 1 310 311 gnu_make = command.Output(os.path.join(options.git, 312 'scripts/show-gnu-make'), raise_on_error=False).rstrip() 313 if not gnu_make: 314 sys.exit('GNU Make not found') 315 316 # Create a new builder with the selected options. 317 output_dir = options.output_dir 318 if options.branch: 319 dirname = options.branch.replace('/', '_') 320 # As a special case allow the board directory to be placed in the 321 # output directory itself rather than any subdirectory. 322 if not options.no_subdirs: 323 output_dir = os.path.join(options.output_dir, dirname) 324 if clean_dir and os.path.exists(output_dir): 325 shutil.rmtree(output_dir) 326 builder = Builder(toolchains, output_dir, options.git_dir, 327 options.threads, options.jobs, gnu_make=gnu_make, checkout=True, 328 show_unknown=options.show_unknown, step=options.step, 329 no_subdirs=options.no_subdirs, full_path=options.full_path, 330 verbose_build=options.verbose_build, 331 mrproper=options.mrproper, 332 per_board_out_dir=options.per_board_out_dir, 333 config_only=options.config_only, 334 squash_config_y=not options.preserve_config_y, 335 warnings_as_errors=options.warnings_as_errors, 336 work_in_output=options.work_in_output, 337 test_thread_exceptions=test_thread_exceptions) 338 builder.force_config_on_failure = not options.quick 339 if make_func: 340 builder.do_make = make_func 341 342 # For a dry run, just show our actions as a sanity check 343 if options.dry_run: 344 ShowActions(series, why_selected, selected, builder, options, 345 board_warnings) 346 else: 347 builder.force_build = options.force_build 348 builder.force_build_failures = options.force_build_failures 349 builder.force_reconfig = options.force_reconfig 350 builder.in_tree = options.in_tree 351 352 # Work out which boards to build 353 board_selected = boards.GetSelectedDict() 354 355 if series: 356 commits = series.commits 357 # Number the commits for test purposes 358 for commit in range(len(commits)): 359 commits[commit].sequence = commit 360 else: 361 commits = None 362 363 Print(GetActionSummary(options.summary, commits, board_selected, 364 options)) 365 366 # We can't show function sizes without board details at present 367 if options.show_bloat: 368 options.show_detail = True 369 builder.SetDisplayOptions( 370 options.show_errors, options.show_sizes, options.show_detail, 371 options.show_bloat, options.list_error_boards, options.show_config, 372 options.show_environment, options.filter_dtb_warnings, 373 options.filter_migration_warnings) 374 if options.summary: 375 builder.ShowSummary(commits, board_selected) 376 else: 377 fail, warned, excs = builder.BuildBoards( 378 commits, board_selected, options.keep_outputs, options.verbose) 379 if excs: 380 return 102 381 elif fail: 382 return 100 383 elif warned and not options.ignore_warnings: 384 return 101 385 return 0 386