1# This Source Code Form is subject to the terms of the Mozilla Public 2# License, v. 2.0. If a copy of the MPL was not distributed with this 3# file, # You can obtain one at http://mozilla.org/MPL/2.0/. 4 5from __future__ import absolute_import, print_function, unicode_literals 6 7import argparse 8import hashlib 9import itertools 10import json 11import logging 12import operator 13import os 14import re 15import subprocess 16import sys 17 18from collections import OrderedDict 19 20import mozpack.path as mozpath 21 22from mach.decorators import ( 23 CommandArgument, 24 CommandArgumentGroup, 25 CommandProvider, 26 Command, 27 SettingsProvider, 28 SubCommand, 29) 30 31from mach.main import Mach 32 33from mozbuild.base import ( 34 BuildEnvironmentNotFoundException, 35 MachCommandBase, 36 MachCommandConditions as conditions, 37 MozbuildObject, 38) 39from mozbuild.util import ensureParentDir 40 41from mozbuild.backend import ( 42 backends, 43) 44 45 46BUILD_WHAT_HELP = ''' 47What to build. Can be a top-level make target or a relative directory. If 48multiple options are provided, they will be built serially. Takes dependency 49information from `topsrcdir/build/dumbmake-dependencies` to build additional 50targets as needed. BUILDING ONLY PARTS OF THE TREE CAN RESULT IN BAD TREE 51STATE. USE AT YOUR OWN RISK. 52'''.strip() 53 54 55EXCESSIVE_SWAP_MESSAGE = ''' 56=================== 57PERFORMANCE WARNING 58 59Your machine experienced a lot of swap activity during the build. This is 60possibly a sign that your machine doesn't have enough physical memory or 61not enough available memory to perform the build. It's also possible some 62other system activity during the build is to blame. 63 64If you feel this message is not appropriate for your machine configuration, 65please file a Core :: Build Config bug at 66https://bugzilla.mozilla.org/enter_bug.cgi?product=Core&component=Build%20Config 67and tell us about your machine and build configuration so we can adjust the 68warning heuristic. 69=================== 70''' 71 72 73class StoreDebugParamsAndWarnAction(argparse.Action): 74 def __call__(self, parser, namespace, values, option_string=None): 75 sys.stderr.write('The --debugparams argument is deprecated. Please ' + 76 'use --debugger-args instead.\n\n') 77 setattr(namespace, self.dest, values) 78 79 80@CommandProvider 81class Watch(MachCommandBase): 82 """Interface to watch and re-build the tree.""" 83 84 @Command('watch', category='post-build', description='Watch and re-build the tree.', 85 conditions=[conditions.is_firefox]) 86 @CommandArgument('-v', '--verbose', action='store_true', 87 help='Verbose output for what commands the watcher is running.') 88 def watch(self, verbose=False): 89 """Watch and re-build the source tree.""" 90 91 if not conditions.is_artifact_build(self): 92 print('mach watch requires an artifact build. See ' 93 'https://developer.mozilla.org/docs/Mozilla/Developer_guide/Build_Instructions/Simple_Firefox_build') 94 return 1 95 96 if not self.substs.get('WATCHMAN', None): 97 print('mach watch requires watchman to be installed. See ' 98 'https://developer.mozilla.org/docs/Mozilla/Developer_guide/Build_Instructions/Incremental_builds_with_filesystem_watching') 99 return 1 100 101 self._activate_virtualenv() 102 try: 103 self.virtualenv_manager.install_pip_package('pywatchman==1.3.0') 104 except Exception: 105 print('Could not install pywatchman from pip. See ' 106 'https://developer.mozilla.org/docs/Mozilla/Developer_guide/Build_Instructions/Incremental_builds_with_filesystem_watching') 107 return 1 108 109 from mozbuild.faster_daemon import Daemon 110 daemon = Daemon(self.config_environment) 111 112 try: 113 return daemon.watch() 114 except KeyboardInterrupt: 115 # Suppress ugly stack trace when user hits Ctrl-C. 116 sys.exit(3) 117 118 119@CommandProvider 120class Build(MachCommandBase): 121 """Interface to build the tree.""" 122 123 @Command('build', category='build', description='Build the tree.') 124 @CommandArgument('--jobs', '-j', default='0', metavar='jobs', type=int, 125 help='Number of concurrent jobs to run. Default is the number of CPUs.') 126 @CommandArgument('-C', '--directory', default=None, 127 help='Change to a subdirectory of the build directory first.') 128 @CommandArgument('what', default=None, nargs='*', help=BUILD_WHAT_HELP) 129 @CommandArgument('-X', '--disable-extra-make-dependencies', 130 default=False, action='store_true', 131 help='Do not add extra make dependencies.') 132 @CommandArgument('-v', '--verbose', action='store_true', 133 help='Verbose output for what commands the build is running.') 134 @CommandArgument('--keep-going', action='store_true', 135 help='Keep building after an error has occurred') 136 def build(self, what=None, disable_extra_make_dependencies=None, jobs=0, 137 directory=None, verbose=False, keep_going=False): 138 """Build the source tree. 139 140 With no arguments, this will perform a full build. 141 142 Positional arguments define targets to build. These can be make targets 143 or patterns like "<dir>/<target>" to indicate a make target within a 144 directory. 145 146 There are a few special targets that can be used to perform a partial 147 build faster than what `mach build` would perform: 148 149 * binaries - compiles and links all C/C++ sources and produces shared 150 libraries and executables (binaries). 151 152 * faster - builds JavaScript, XUL, CSS, etc files. 153 154 "binaries" and "faster" almost fully complement each other. However, 155 there are build actions not captured by either. If things don't appear to 156 be rebuilding, perform a vanilla `mach build` to rebuild the world. 157 """ 158 from mozbuild.controller.building import ( 159 BuildDriver, 160 ) 161 162 self.log_manager.enable_all_structured_loggers() 163 164 driver = self._spawn(BuildDriver) 165 return driver.build( 166 what=what, 167 disable_extra_make_dependencies=disable_extra_make_dependencies, 168 jobs=jobs, 169 directory=directory, 170 verbose=verbose, 171 keep_going=keep_going, 172 mach_context=self._mach_context) 173 174 @Command('configure', category='build', 175 description='Configure the tree (run configure and config.status).') 176 @CommandArgument('options', default=None, nargs=argparse.REMAINDER, 177 help='Configure options') 178 def configure(self, options=None, buildstatus_messages=False, line_handler=None): 179 from mozbuild.controller.building import ( 180 BuildDriver, 181 ) 182 183 self.log_manager.enable_all_structured_loggers() 184 driver = self._spawn(BuildDriver) 185 186 return driver.configure( 187 options=options, 188 buildstatus_messages=buildstatus_messages, 189 line_handler=line_handler) 190 191 @Command('resource-usage', category='post-build', 192 description='Show information about system resource usage for a build.') 193 @CommandArgument('--address', default='localhost', 194 help='Address the HTTP server should listen on.') 195 @CommandArgument('--port', type=int, default=0, 196 help='Port number the HTTP server should listen on.') 197 @CommandArgument('--browser', default='firefox', 198 help='Web browser to automatically open. See webbrowser Python module.') 199 @CommandArgument('--url', 200 help='URL of JSON document to display') 201 def resource_usage(self, address=None, port=None, browser=None, url=None): 202 import webbrowser 203 from mozbuild.html_build_viewer import BuildViewerServer 204 205 server = BuildViewerServer(address, port) 206 207 if url: 208 server.add_resource_json_url('url', url) 209 else: 210 last = self._get_state_filename('build_resources.json') 211 if not os.path.exists(last): 212 print('Build resources not available. If you have performed a ' 213 'build and receive this message, the psutil Python package ' 214 'likely failed to initialize properly.') 215 return 1 216 217 server.add_resource_json_file('last', last) 218 try: 219 webbrowser.get(browser).open_new_tab(server.url) 220 except Exception: 221 print('Cannot get browser specified, trying the default instead.') 222 try: 223 browser = webbrowser.get().open_new_tab(server.url) 224 except Exception: 225 print('Please open %s in a browser.' % server.url) 226 227 print('Hit CTRL+c to stop server.') 228 server.run() 229 230 @Command('build-backend', category='build', 231 description='Generate a backend used to build the tree.') 232 @CommandArgument('-d', '--diff', action='store_true', 233 help='Show a diff of changes.') 234 # It would be nice to filter the choices below based on 235 # conditions, but that is for another day. 236 @CommandArgument('-b', '--backend', nargs='+', choices=sorted(backends), 237 help='Which backend to build.') 238 @CommandArgument('-v', '--verbose', action='store_true', 239 help='Verbose output.') 240 @CommandArgument('-n', '--dry-run', action='store_true', 241 help='Do everything except writing files out.') 242 def build_backend(self, backend, diff=False, verbose=False, dry_run=False): 243 python = self.virtualenv_manager.python_path 244 config_status = os.path.join(self.topobjdir, 'config.status') 245 246 if not os.path.exists(config_status): 247 print('config.status not found. Please run |mach configure| ' 248 'or |mach build| prior to building the %s build backend.' 249 % backend) 250 return 1 251 252 args = [python, config_status] 253 if backend: 254 args.append('--backend') 255 args.extend(backend) 256 if diff: 257 args.append('--diff') 258 if verbose: 259 args.append('--verbose') 260 if dry_run: 261 args.append('--dry-run') 262 263 return self._run_command_in_objdir(args=args, pass_thru=True, 264 ensure_exit_code=False) 265 266 267@CommandProvider 268class CargoProvider(MachCommandBase): 269 """Invoke cargo in useful ways.""" 270 271 @Command('cargo', category='build', 272 description='Invoke cargo in useful ways.') 273 def cargo(self): 274 self.parser.print_usage() 275 return 1 276 277 @SubCommand('cargo', 'check', 278 description='Run `cargo check` on a given crate. Defaults to gkrust.') 279 @CommandArgument('--all-crates', default=None, action='store_true', 280 help='Check all of the crates in the tree.') 281 @CommandArgument('crates', default=None, nargs='*', help='The crate name(s) to check.') 282 def check(self, all_crates=None, crates=None): 283 # XXX duplication with `mach vendor rust` 284 crates_and_roots = { 285 'gkrust': 'toolkit/library/rust', 286 'gkrust-gtest': 'toolkit/library/gtest/rust', 287 'js': 'js/rust', 288 'mozjs_sys': 'js/src', 289 'geckodriver': 'testing/geckodriver', 290 } 291 292 if all_crates: 293 crates = crates_and_roots.keys() 294 elif crates == None or crates == []: 295 crates = ['gkrust'] 296 297 for crate in crates: 298 root = crates_and_roots.get(crate, None) 299 if not root: 300 print('Cannot locate crate %s. Please check your spelling or ' 301 'add the crate information to the list.' % crate) 302 return 1 303 304 check_targets = [ 305 'force-cargo-library-check', 306 'force-cargo-host-library-check', 307 'force-cargo-program-check', 308 'force-cargo-host-program-check', 309 ] 310 311 ret = self._run_make(srcdir=False, directory=root, 312 ensure_exit_code=0, silent=True, 313 print_directory=False, target=check_targets) 314 if ret != 0: 315 return ret 316 317 return 0 318 319@CommandProvider 320class Doctor(MachCommandBase): 321 """Provide commands for diagnosing common build environment problems""" 322 @Command('doctor', category='devenv', 323 description='') 324 @CommandArgument('--fix', default=None, action='store_true', 325 help='Attempt to fix found problems.') 326 def doctor(self, fix=None): 327 self._activate_virtualenv() 328 from mozbuild.doctor import Doctor 329 doctor = Doctor(self.topsrcdir, self.topobjdir, fix) 330 return doctor.check_all() 331 332@CommandProvider 333class Clobber(MachCommandBase): 334 NO_AUTO_LOG = True 335 CLOBBER_CHOICES = ['objdir', 'python'] 336 @Command('clobber', category='build', 337 description='Clobber the tree (delete the object directory).') 338 @CommandArgument('what', default=['objdir'], nargs='*', 339 help='Target to clobber, must be one of {{{}}} (default objdir).'.format( 340 ', '.join(CLOBBER_CHOICES))) 341 @CommandArgument('--full', action='store_true', 342 help='Perform a full clobber') 343 def clobber(self, what, full=False): 344 """Clean up the source and object directories. 345 346 Performing builds and running various commands generate various files. 347 348 Sometimes it is necessary to clean up these files in order to make 349 things work again. This command can be used to perform that cleanup. 350 351 By default, this command removes most files in the current object 352 directory (where build output is stored). Some files (like Visual 353 Studio project files) are not removed by default. If you would like 354 to remove the object directory in its entirety, run with `--full`. 355 356 The `python` target will clean up various generated Python files from 357 the source directory and will remove untracked files from well-known 358 directories containing Python packages. Run this to remove .pyc files, 359 compiled C extensions, etc. Note: all files not tracked or ignored by 360 version control in well-known Python package directories will be 361 deleted. Run the `status` command of your VCS to see if any untracked 362 files you haven't committed yet will be deleted. 363 """ 364 invalid = set(what) - set(self.CLOBBER_CHOICES) 365 if invalid: 366 print('Unknown clobber target(s): {}'.format(', '.join(invalid))) 367 return 1 368 369 ret = 0 370 if 'objdir' in what: 371 from mozbuild.controller.clobber import Clobberer 372 try: 373 Clobberer(self.topsrcdir, self.topobjdir).remove_objdir(full) 374 except OSError as e: 375 if sys.platform.startswith('win'): 376 if isinstance(e, WindowsError) and e.winerror in (5,32): 377 self.log(logging.ERROR, 'file_access_error', {'error': e}, 378 "Could not clobber because a file was in use. If the " 379 "application is running, try closing it. {error}") 380 return 1 381 raise 382 383 if 'python' in what: 384 if conditions.is_hg(self): 385 cmd = ['hg', 'purge', '--all', '-I', 'glob:**.py[cdo]', 386 '-I', 'path:python/', '-I', 'path:third_party/python/'] 387 elif conditions.is_git(self): 388 cmd = ['git', 'clean', '-f', '-x', '*.py[cdo]', 'python/', 389 'third_party/python/'] 390 else: 391 # We don't know what is tracked/untracked if we don't have VCS. 392 # So we can't clean python/ and third_party/python/. 393 cmd = ['find', '.', '-type', 'f', '-name', '*.py[cdo]', 394 '-delete'] 395 ret = subprocess.call(cmd, cwd=self.topsrcdir) 396 return ret 397 398 @property 399 def substs(self): 400 try: 401 return super(Clobber, self).substs 402 except BuildEnvironmentNotFoundException: 403 return {} 404 405@CommandProvider 406class Logs(MachCommandBase): 407 """Provide commands to read mach logs.""" 408 NO_AUTO_LOG = True 409 410 @Command('show-log', category='post-build', 411 description='Display mach logs') 412 @CommandArgument('log_file', nargs='?', type=argparse.FileType('rb'), 413 help='Filename to read log data from. Defaults to the log of the last ' 414 'mach command.') 415 def show_log(self, log_file=None): 416 if not log_file: 417 path = self._get_state_filename('last_log.json') 418 log_file = open(path, 'rb') 419 420 if os.isatty(sys.stdout.fileno()): 421 env = dict(os.environ) 422 if 'LESS' not in env: 423 # Sensible default flags if none have been set in the user 424 # environment. 425 env[b'LESS'] = b'FRX' 426 less = subprocess.Popen(['less'], stdin=subprocess.PIPE, env=env) 427 # Various objects already have a reference to sys.stdout, so we 428 # can't just change it, we need to change the file descriptor under 429 # it to redirect to less's input. 430 # First keep a copy of the sys.stdout file descriptor. 431 output_fd = os.dup(sys.stdout.fileno()) 432 os.dup2(less.stdin.fileno(), sys.stdout.fileno()) 433 434 startTime = 0 435 for line in log_file: 436 created, action, params = json.loads(line) 437 if not startTime: 438 startTime = created 439 self.log_manager.terminal_handler.formatter.start_time = \ 440 created 441 if 'line' in params: 442 record = logging.makeLogRecord({ 443 'created': created, 444 'name': self._logger.name, 445 'levelno': logging.INFO, 446 'msg': '{line}', 447 'params': params, 448 'action': action, 449 }) 450 self._logger.handle(record) 451 452 if self.log_manager.terminal: 453 # Close less's input so that it knows that we're done sending data. 454 less.stdin.close() 455 # Since the less's input file descriptor is now also the stdout 456 # file descriptor, we still actually have a non-closed system file 457 # descriptor for less's input. Replacing sys.stdout's file 458 # descriptor with what it was before we replaced it will properly 459 # close less's input. 460 os.dup2(output_fd, sys.stdout.fileno()) 461 less.wait() 462 463 464@CommandProvider 465class Warnings(MachCommandBase): 466 """Provide commands for inspecting warnings.""" 467 468 @property 469 def database_path(self): 470 return self._get_state_filename('warnings.json') 471 472 @property 473 def database(self): 474 from mozbuild.compilation.warnings import WarningsDatabase 475 476 path = self.database_path 477 478 database = WarningsDatabase() 479 480 if os.path.exists(path): 481 database.load_from_file(path) 482 483 return database 484 485 @Command('warnings-summary', category='post-build', 486 description='Show a summary of compiler warnings.') 487 @CommandArgument('-C', '--directory', default=None, 488 help='Change to a subdirectory of the build directory first.') 489 @CommandArgument('report', default=None, nargs='?', 490 help='Warnings report to display. If not defined, show the most ' 491 'recent report.') 492 def summary(self, directory=None, report=None): 493 database = self.database 494 495 if directory: 496 dirpath = self.join_ensure_dir(self.topsrcdir, directory) 497 if not dirpath: 498 return 1 499 else: 500 dirpath = None 501 502 type_counts = database.type_counts(dirpath) 503 sorted_counts = sorted(type_counts.iteritems(), 504 key=operator.itemgetter(1)) 505 506 total = 0 507 for k, v in sorted_counts: 508 print('%d\t%s' % (v, k)) 509 total += v 510 511 print('%d\tTotal' % total) 512 513 @Command('warnings-list', category='post-build', 514 description='Show a list of compiler warnings.') 515 @CommandArgument('-C', '--directory', default=None, 516 help='Change to a subdirectory of the build directory first.') 517 @CommandArgument('--flags', default=None, nargs='+', 518 help='Which warnings flags to match.') 519 @CommandArgument('report', default=None, nargs='?', 520 help='Warnings report to display. If not defined, show the most ' 521 'recent report.') 522 def list(self, directory=None, flags=None, report=None): 523 database = self.database 524 525 by_name = sorted(database.warnings) 526 527 topsrcdir = mozpath.normpath(self.topsrcdir) 528 529 if directory: 530 directory = mozpath.normsep(directory) 531 dirpath = self.join_ensure_dir(topsrcdir, directory) 532 if not dirpath: 533 return 1 534 535 if flags: 536 # Flatten lists of flags. 537 flags = set(itertools.chain(*[flaglist.split(',') for flaglist in flags])) 538 539 for warning in by_name: 540 filename = mozpath.normsep(warning['filename']) 541 542 if filename.startswith(topsrcdir): 543 filename = filename[len(topsrcdir) + 1:] 544 545 if directory and not filename.startswith(directory): 546 continue 547 548 if flags and warning['flag'] not in flags: 549 continue 550 551 if warning['column'] is not None: 552 print('%s:%d:%d [%s] %s' % (filename, warning['line'], 553 warning['column'], warning['flag'], warning['message'])) 554 else: 555 print('%s:%d [%s] %s' % (filename, warning['line'], 556 warning['flag'], warning['message'])) 557 558 def join_ensure_dir(self, dir1, dir2): 559 dir1 = mozpath.normpath(dir1) 560 dir2 = mozpath.normsep(dir2) 561 joined_path = mozpath.join(dir1, dir2) 562 if os.path.isdir(joined_path): 563 return joined_path 564 else: 565 print('Specified directory not found.') 566 return None 567 568@CommandProvider 569class GTestCommands(MachCommandBase): 570 @Command('gtest', category='testing', 571 description='Run GTest unit tests (C++ tests).') 572 @CommandArgument('gtest_filter', default=b"*", nargs='?', metavar='gtest_filter', 573 help="test_filter is a ':'-separated list of wildcard patterns (called the positive patterns)," 574 "optionally followed by a '-' and another ':'-separated pattern list (called the negative patterns).") 575 @CommandArgument('--jobs', '-j', default='1', nargs='?', metavar='jobs', type=int, 576 help='Run the tests in parallel using multiple processes.') 577 @CommandArgument('--tbpl-parser', '-t', action='store_true', 578 help='Output test results in a format that can be parsed by TBPL.') 579 @CommandArgument('--shuffle', '-s', action='store_true', 580 help='Randomize the execution order of tests.') 581 582 @CommandArgumentGroup('debugging') 583 @CommandArgument('--debug', action='store_true', group='debugging', 584 help='Enable the debugger. Not specifying a --debugger option will result in the default debugger being used.') 585 @CommandArgument('--debugger', default=None, type=str, group='debugging', 586 help='Name of debugger to use.') 587 @CommandArgument('--debugger-args', default=None, metavar='params', type=str, 588 group='debugging', 589 help='Command-line arguments to pass to the debugger itself; split as the Bourne shell would.') 590 591 def gtest(self, shuffle, jobs, gtest_filter, tbpl_parser, debug, debugger, 592 debugger_args): 593 594 # We lazy build gtest because it's slow to link 595 self._run_make(directory="testing/gtest", target='gtest', 596 print_directory=False, ensure_exit_code=True) 597 598 app_path = self.get_binary_path('app') 599 args = [app_path, '-unittest', '--gtest_death_test_style=threadsafe']; 600 601 if debug or debugger or debugger_args: 602 args = self.prepend_debugger_args(args, debugger, debugger_args) 603 604 cwd = os.path.join(self.topobjdir, '_tests', 'gtest') 605 606 if not os.path.isdir(cwd): 607 os.makedirs(cwd) 608 609 # Use GTest environment variable to control test execution 610 # For details see: 611 # https://code.google.com/p/googletest/wiki/AdvancedGuide#Running_Test_Programs:_Advanced_Options 612 gtest_env = {b'GTEST_FILTER': gtest_filter} 613 614 # Note: we must normalize the path here so that gtest on Windows sees 615 # a MOZ_GMP_PATH which has only Windows dir seperators, because 616 # nsIFile cannot open the paths with non-Windows dir seperators. 617 xre_path = os.path.join(os.path.normpath(self.topobjdir), "dist", "bin") 618 gtest_env["MOZ_XRE_DIR"] = xre_path 619 gtest_env["MOZ_GMP_PATH"] = os.pathsep.join( 620 os.path.join(xre_path, p, "1.0") 621 for p in ('gmp-fake', 'gmp-fakeopenh264') 622 ) 623 624 gtest_env[b"MOZ_RUN_GTEST"] = b"True" 625 626 if shuffle: 627 gtest_env[b"GTEST_SHUFFLE"] = b"True" 628 629 if tbpl_parser: 630 gtest_env[b"MOZ_TBPL_PARSER"] = b"True" 631 632 if jobs == 1: 633 return self.run_process(args=args, 634 append_env=gtest_env, 635 cwd=cwd, 636 ensure_exit_code=False, 637 pass_thru=True) 638 639 from mozprocess import ProcessHandlerMixin 640 import functools 641 def handle_line(job_id, line): 642 # Prepend the jobId 643 line = '[%d] %s' % (job_id + 1, line.strip()) 644 self.log(logging.INFO, "GTest", {'line': line}, '{line}') 645 646 gtest_env["GTEST_TOTAL_SHARDS"] = str(jobs) 647 processes = {} 648 for i in range(0, jobs): 649 gtest_env["GTEST_SHARD_INDEX"] = str(i) 650 processes[i] = ProcessHandlerMixin([app_path, "-unittest"], 651 cwd=cwd, 652 env=gtest_env, 653 processOutputLine=[functools.partial(handle_line, i)], 654 universal_newlines=True) 655 processes[i].run() 656 657 exit_code = 0 658 for process in processes.values(): 659 status = process.wait() 660 if status: 661 exit_code = status 662 663 # Clamp error code to 255 to prevent overflowing multiple of 664 # 256 into 0 665 if exit_code > 255: 666 exit_code = 255 667 668 return exit_code 669 670 def prepend_debugger_args(self, args, debugger, debugger_args): 671 ''' 672 Given an array with program arguments, prepend arguments to run it under a 673 debugger. 674 675 :param args: The executable and arguments used to run the process normally. 676 :param debugger: The debugger to use, or empty to use the default debugger. 677 :param debugger_args: Any additional parameters to pass to the debugger. 678 ''' 679 680 import mozdebug 681 682 if not debugger: 683 # No debugger name was provided. Look for the default ones on 684 # current OS. 685 debugger = mozdebug.get_default_debugger_name(mozdebug.DebuggerSearch.KeepLooking) 686 687 if debugger: 688 debuggerInfo = mozdebug.get_debugger_info(debugger, debugger_args) 689 if not debuggerInfo: 690 print("Could not find a suitable debugger in your PATH.") 691 return 1 692 693 # Parameters come from the CLI. We need to convert them before 694 # their use. 695 if debugger_args: 696 from mozbuild import shellutil 697 try: 698 debugger_args = shellutil.split(debugger_args) 699 except shellutil.MetaCharacterException as e: 700 print("The --debugger_args you passed require a real shell to parse them.") 701 print("(We can't handle the %r character.)" % e.char) 702 return 1 703 704 # Prepend the debugger args. 705 args = [debuggerInfo.path] + debuggerInfo.args + args 706 return args 707 708@CommandProvider 709class ClangCommands(MachCommandBase): 710 @Command('clang-complete', category='devenv', 711 description='Generate a .clang_complete file.') 712 def clang_complete(self): 713 import shlex 714 715 build_vars = {} 716 717 def on_line(line): 718 elements = [s.strip() for s in line.split('=', 1)] 719 720 if len(elements) != 2: 721 return 722 723 build_vars[elements[0]] = elements[1] 724 725 try: 726 old_logger = self.log_manager.replace_terminal_handler(None) 727 self._run_make(target='showbuild', log=False, line_handler=on_line) 728 finally: 729 self.log_manager.replace_terminal_handler(old_logger) 730 731 def print_from_variable(name): 732 if name not in build_vars: 733 return 734 735 value = build_vars[name] 736 737 value = value.replace('-I.', '-I%s' % self.topobjdir) 738 value = value.replace(' .', ' %s' % self.topobjdir) 739 value = value.replace('-I..', '-I%s/..' % self.topobjdir) 740 value = value.replace(' ..', ' %s/..' % self.topobjdir) 741 742 args = shlex.split(value) 743 for i in range(0, len(args) - 1): 744 arg = args[i] 745 746 if arg.startswith(('-I', '-D')): 747 print(arg) 748 continue 749 750 if arg.startswith('-include'): 751 print(arg + ' ' + args[i + 1]) 752 continue 753 754 print_from_variable('COMPILE_CXXFLAGS') 755 756 print('-I%s/ipc/chromium/src' % self.topsrcdir) 757 print('-I%s/ipc/glue' % self.topsrcdir) 758 print('-I%s/ipc/ipdl/_ipdlheaders' % self.topobjdir) 759 760 761@CommandProvider 762class Package(MachCommandBase): 763 """Package the built product for distribution.""" 764 765 @Command('package', category='post-build', 766 description='Package the built product for distribution as an APK, DMG, etc.') 767 @CommandArgument('-v', '--verbose', action='store_true', 768 help='Verbose output for what commands the packaging process is running.') 769 def package(self, verbose=False): 770 ret = self._run_make(directory=".", target='package', 771 silent=not verbose, ensure_exit_code=False) 772 if ret == 0: 773 self.notify('Packaging complete') 774 return ret 775 776@CommandProvider 777class Install(MachCommandBase): 778 """Install a package.""" 779 780 @Command('install', category='post-build', 781 description='Install the package on the machine, or on a device.') 782 @CommandArgument('--verbose', '-v', action='store_true', 783 help='Print verbose output when installing to an Android emulator.') 784 def install(self, verbose=False): 785 if conditions.is_android(self): 786 from mozrunner.devices.android_device import verify_android_device 787 verify_android_device(self, verbose=verbose) 788 ret = self._run_make(directory=".", target='install', ensure_exit_code=False) 789 if ret == 0: 790 self.notify('Install complete') 791 return ret 792 793@SettingsProvider 794class RunSettings(): 795 config_settings = [ 796 ('runprefs.*', 'string', """ 797Pass a pref into Firefox when using `mach run`, of the form `foo.bar=value`. 798Prefs will automatically be cast into the appropriate type. Integers can be 799single quoted to force them to be strings. 800""".strip()), 801 ] 802 803@CommandProvider 804class RunProgram(MachCommandBase): 805 """Run the compiled program.""" 806 807 prog_group = 'the compiled program' 808 809 @Command('run', category='post-build', 810 description='Run the compiled program, possibly under a debugger or DMD.') 811 @CommandArgument('params', nargs='...', group=prog_group, 812 help='Command-line arguments to be passed through to the program. Not specifying a --profile or -P option will result in a temporary profile being used.') 813 @CommandArgumentGroup(prog_group) 814 @CommandArgument('--remote', '-r', action='store_true', group=prog_group, 815 help='Do not pass the --no-remote argument by default.') 816 @CommandArgument('--background', '-b', action='store_true', group=prog_group, 817 help='Do not pass the --foreground argument by default on Mac.') 818 @CommandArgument('--noprofile', '-n', action='store_true', group=prog_group, 819 help='Do not pass the --profile argument by default.') 820 @CommandArgument('--disable-e10s', action='store_true', group=prog_group, 821 help='Run the program with electrolysis disabled.') 822 @CommandArgument('--enable-crash-reporter', action='store_true', group=prog_group, 823 help='Run the program with the crash reporter enabled.') 824 @CommandArgument('--setpref', action='append', default=[], group=prog_group, 825 help='Set the specified pref before starting the program. Can be set multiple times. Prefs can also be set in ~/.mozbuild/machrc in the [runprefs] section - see `./mach settings` for more information.') 826 827 @CommandArgumentGroup('debugging') 828 @CommandArgument('--debug', action='store_true', group='debugging', 829 help='Enable the debugger. Not specifying a --debugger option will result in the default debugger being used.') 830 @CommandArgument('--debugger', default=None, type=str, group='debugging', 831 help='Name of debugger to use.') 832 @CommandArgument('--debugger-args', default=None, metavar='params', type=str, 833 group='debugging', 834 help='Command-line arguments to pass to the debugger itself; split as the Bourne shell would.') 835 @CommandArgument('--debugparams', action=StoreDebugParamsAndWarnAction, 836 default=None, type=str, dest='debugger_args', group='debugging', 837 help=argparse.SUPPRESS) 838 839 @CommandArgumentGroup('DMD') 840 @CommandArgument('--dmd', action='store_true', group='DMD', 841 help='Enable DMD. The following arguments have no effect without this.') 842 @CommandArgument('--mode', choices=['live', 'dark-matter', 'cumulative', 'scan'], group='DMD', 843 help='Profiling mode. The default is \'dark-matter\'.') 844 @CommandArgument('--stacks', choices=['partial', 'full'], group='DMD', 845 help='Allocation stack trace coverage. The default is \'partial\'.') 846 @CommandArgument('--show-dump-stats', action='store_true', group='DMD', 847 help='Show stats when doing dumps.') 848 def run(self, params, remote, background, noprofile, disable_e10s, 849 enable_crash_reporter, setpref, debug, debugger, 850 debugger_args, dmd, mode, stacks, show_dump_stats): 851 852 if conditions.is_android(self): 853 # Running Firefox for Android is completely different 854 if dmd: 855 print("DMD is not supported for Firefox for Android") 856 return 1 857 from mozrunner.devices.android_device import verify_android_device, run_firefox_for_android 858 if not (debug or debugger or debugger_args): 859 verify_android_device(self, install=True) 860 return run_firefox_for_android(self, params) 861 verify_android_device(self, install=True, debugger=True) 862 args = [''] 863 864 else: 865 from mozprofile import Profile, Preferences 866 867 try: 868 binpath = self.get_binary_path('app') 869 except Exception as e: 870 print("It looks like your program isn't built.", 871 "You can run |mach build| to build it.") 872 print(e) 873 return 1 874 875 args = [binpath] 876 877 if params: 878 args.extend(params) 879 880 if not remote: 881 args.append('-no-remote') 882 883 if not background and sys.platform == 'darwin': 884 args.append('-foreground') 885 886 no_profile_option_given = \ 887 all(p not in params for p in ['-profile', '--profile', '-P']) 888 if no_profile_option_given and not noprofile: 889 prefs = { 890 'browser.shell.checkDefaultBrowser': False, 891 'general.warnOnAboutConfig': False, 892 } 893 prefs.update(self._mach_context.settings.runprefs) 894 prefs.update([p.split('=', 1) for p in setpref]) 895 for pref in prefs: 896 prefs[pref] = Preferences.cast(prefs[pref]) 897 898 path = os.path.join(self.topobjdir, 'tmp', 'scratch_user') 899 profile = Profile(path, preferences=prefs) 900 args.append('-profile') 901 args.append(profile.profile) 902 903 if not no_profile_option_given and setpref: 904 print("setpref is only supported if a profile is not specified") 905 return 1 906 907 extra_env = { 908 'MOZ_DEVELOPER_REPO_DIR': self.topsrcdir, 909 'MOZ_DEVELOPER_OBJ_DIR': self.topobjdir, 910 'RUST_BACKTRACE': 'full', 911 } 912 913 if not enable_crash_reporter: 914 extra_env['MOZ_CRASHREPORTER_DISABLE'] = '1' 915 else: 916 extra_env['MOZ_CRASHREPORTER'] = '1' 917 918 if disable_e10s: 919 extra_env['MOZ_FORCE_DISABLE_E10S'] = '1' 920 921 if debug or debugger or debugger_args: 922 if 'INSIDE_EMACS' in os.environ: 923 self.log_manager.terminal_handler.setLevel(logging.WARNING) 924 925 import mozdebug 926 if not debugger: 927 # No debugger name was provided. Look for the default ones on 928 # current OS. 929 debugger = mozdebug.get_default_debugger_name(mozdebug.DebuggerSearch.KeepLooking) 930 931 if debugger: 932 self.debuggerInfo = mozdebug.get_debugger_info(debugger, debugger_args) 933 if not self.debuggerInfo: 934 print("Could not find a suitable debugger in your PATH.") 935 return 1 936 937 # Parameters come from the CLI. We need to convert them before 938 # their use. 939 if debugger_args: 940 from mozbuild import shellutil 941 try: 942 debugger_args = shellutil.split(debugger_args) 943 except shellutil.MetaCharacterException as e: 944 print("The --debugger-args you passed require a real shell to parse them.") 945 print("(We can't handle the %r character.)" % e.char) 946 return 1 947 948 # Prepend the debugger args. 949 args = [self.debuggerInfo.path] + self.debuggerInfo.args + args 950 951 if dmd: 952 dmd_params = [] 953 954 if mode: 955 dmd_params.append('--mode=' + mode) 956 if stacks: 957 dmd_params.append('--stacks=' + stacks) 958 if show_dump_stats: 959 dmd_params.append('--show-dump-stats=yes') 960 961 if dmd_params: 962 extra_env['DMD'] = ' '.join(dmd_params) 963 else: 964 extra_env['DMD'] = '1' 965 966 return self.run_process(args=args, ensure_exit_code=False, 967 pass_thru=True, append_env=extra_env) 968 969@CommandProvider 970class Buildsymbols(MachCommandBase): 971 """Produce a package of debug symbols suitable for use with Breakpad.""" 972 973 @Command('buildsymbols', category='post-build', 974 description='Produce a package of Breakpad-format symbols.') 975 def buildsymbols(self): 976 return self._run_make(directory=".", target='buildsymbols', ensure_exit_code=False) 977 978@CommandProvider 979class Makefiles(MachCommandBase): 980 @Command('empty-makefiles', category='build-dev', 981 description='Find empty Makefile.in in the tree.') 982 def empty(self): 983 import pymake.parser 984 import pymake.parserdata 985 986 IGNORE_VARIABLES = { 987 'DEPTH': ('@DEPTH@',), 988 'topsrcdir': ('@top_srcdir@',), 989 'srcdir': ('@srcdir@',), 990 'relativesrcdir': ('@relativesrcdir@',), 991 'VPATH': ('@srcdir@',), 992 } 993 994 IGNORE_INCLUDES = [ 995 'include $(DEPTH)/config/autoconf.mk', 996 'include $(topsrcdir)/config/config.mk', 997 'include $(topsrcdir)/config/rules.mk', 998 ] 999 1000 def is_statement_relevant(s): 1001 if isinstance(s, pymake.parserdata.SetVariable): 1002 exp = s.vnameexp 1003 if not exp.is_static_string: 1004 return True 1005 1006 if exp.s not in IGNORE_VARIABLES: 1007 return True 1008 1009 return s.value not in IGNORE_VARIABLES[exp.s] 1010 1011 if isinstance(s, pymake.parserdata.Include): 1012 if s.to_source() in IGNORE_INCLUDES: 1013 return False 1014 1015 return True 1016 1017 for path in self._makefile_ins(): 1018 relpath = os.path.relpath(path, self.topsrcdir) 1019 try: 1020 statements = [s for s in pymake.parser.parsefile(path) 1021 if is_statement_relevant(s)] 1022 1023 if not statements: 1024 print(relpath) 1025 except pymake.parser.SyntaxError: 1026 print('Warning: Could not parse %s' % relpath, file=sys.stderr) 1027 1028 def _makefile_ins(self): 1029 for root, dirs, files in os.walk(self.topsrcdir): 1030 for f in files: 1031 if f == 'Makefile.in': 1032 yield os.path.join(root, f) 1033 1034@CommandProvider 1035class MachDebug(MachCommandBase): 1036 @Command('environment', category='build-dev', 1037 description='Show info about the mach and build environment.') 1038 @CommandArgument('--format', default='pretty', 1039 choices=['pretty', 'configure', 'json'], 1040 help='Print data in the given format.') 1041 @CommandArgument('--output', '-o', type=str, 1042 help='Output to the given file.') 1043 @CommandArgument('--verbose', '-v', action='store_true', 1044 help='Print verbose output.') 1045 def environment(self, format, output=None, verbose=False): 1046 func = getattr(self, '_environment_%s' % format.replace('.', '_')) 1047 1048 if output: 1049 # We want to preserve mtimes if the output file already exists 1050 # and the content hasn't changed. 1051 from mozbuild.util import FileAvoidWrite 1052 with FileAvoidWrite(output) as out: 1053 return func(out, verbose) 1054 return func(sys.stdout, verbose) 1055 1056 def _environment_pretty(self, out, verbose): 1057 state_dir = self._mach_context.state_dir 1058 import platform 1059 print('platform:\n\t%s' % platform.platform(), file=out) 1060 print('python version:\n\t%s' % sys.version, file=out) 1061 print('python prefix:\n\t%s' % sys.prefix, file=out) 1062 print('mach cwd:\n\t%s' % self._mach_context.cwd, file=out) 1063 print('os cwd:\n\t%s' % os.getcwd(), file=out) 1064 print('mach directory:\n\t%s' % self._mach_context.topdir, file=out) 1065 print('state directory:\n\t%s' % state_dir, file=out) 1066 1067 print('object directory:\n\t%s' % self.topobjdir, file=out) 1068 1069 if self.mozconfig['path']: 1070 print('mozconfig path:\n\t%s' % self.mozconfig['path'], file=out) 1071 if self.mozconfig['configure_args']: 1072 print('mozconfig configure args:', file=out) 1073 for arg in self.mozconfig['configure_args']: 1074 print('\t%s' % arg, file=out) 1075 1076 if self.mozconfig['make_extra']: 1077 print('mozconfig extra make args:', file=out) 1078 for arg in self.mozconfig['make_extra']: 1079 print('\t%s' % arg, file=out) 1080 1081 if self.mozconfig['make_flags']: 1082 print('mozconfig make flags:', file=out) 1083 for arg in self.mozconfig['make_flags']: 1084 print('\t%s' % arg, file=out) 1085 1086 config = None 1087 1088 try: 1089 config = self.config_environment 1090 1091 except Exception: 1092 pass 1093 1094 if config: 1095 print('config topsrcdir:\n\t%s' % config.topsrcdir, file=out) 1096 print('config topobjdir:\n\t%s' % config.topobjdir, file=out) 1097 1098 if verbose: 1099 print('config substitutions:', file=out) 1100 for k in sorted(config.substs): 1101 print('\t%s: %s' % (k, config.substs[k]), file=out) 1102 1103 print('config defines:', file=out) 1104 for k in sorted(config.defines): 1105 print('\t%s' % k, file=out) 1106 1107 def _environment_json(self, out, verbose): 1108 import json 1109 class EnvironmentEncoder(json.JSONEncoder): 1110 def default(self, obj): 1111 if isinstance(obj, MozbuildObject): 1112 result = { 1113 'topsrcdir': obj.topsrcdir, 1114 'topobjdir': obj.topobjdir, 1115 'mozconfig': obj.mozconfig, 1116 } 1117 if verbose: 1118 result['substs'] = obj.substs 1119 result['defines'] = obj.defines 1120 return result 1121 elif isinstance(obj, set): 1122 return list(obj) 1123 return json.JSONEncoder.default(self, obj) 1124 json.dump(self, cls=EnvironmentEncoder, sort_keys=True, fp=out) 1125 1126class ArtifactSubCommand(SubCommand): 1127 def __call__(self, func): 1128 after = SubCommand.__call__(self, func) 1129 jobchoices = { 1130 'android-api-16', 1131 'android-x86', 1132 'linux', 1133 'linux64', 1134 'macosx64', 1135 'win32', 1136 'win64' 1137 } 1138 args = [ 1139 CommandArgument('--tree', metavar='TREE', type=str, 1140 help='Firefox tree.'), 1141 CommandArgument('--job', metavar='JOB', choices=jobchoices, 1142 help='Build job.'), 1143 CommandArgument('--verbose', '-v', action='store_true', 1144 help='Print verbose output.'), 1145 ] 1146 for arg in args: 1147 after = arg(after) 1148 return after 1149 1150 1151@CommandProvider 1152class PackageFrontend(MachCommandBase): 1153 """Fetch and install binary artifacts from Mozilla automation.""" 1154 1155 @Command('artifact', category='post-build', 1156 description='Use pre-built artifacts to build Firefox.') 1157 def artifact(self): 1158 '''Download, cache, and install pre-built binary artifacts to build Firefox. 1159 1160 Use |mach build| as normal to freshen your installed binary libraries: 1161 artifact builds automatically download, cache, and install binary 1162 artifacts from Mozilla automation, replacing whatever may be in your 1163 object directory. Use |mach artifact last| to see what binary artifacts 1164 were last used. 1165 1166 Never build libxul again! 1167 1168 ''' 1169 pass 1170 1171 def _make_artifacts(self, tree=None, job=None, skip_cache=False): 1172 state_dir = self._mach_context.state_dir 1173 cache_dir = os.path.join(state_dir, 'package-frontend') 1174 1175 here = os.path.abspath(os.path.dirname(__file__)) 1176 build_obj = MozbuildObject.from_environment(cwd=here) 1177 1178 hg = None 1179 if conditions.is_hg(build_obj): 1180 hg = build_obj.substs['HG'] 1181 1182 git = None 1183 if conditions.is_git(build_obj): 1184 git = build_obj.substs['GIT'] 1185 1186 from mozbuild.artifacts import Artifacts 1187 artifacts = Artifacts(tree, self.substs, self.defines, job, 1188 log=self.log, cache_dir=cache_dir, 1189 skip_cache=skip_cache, hg=hg, git=git, 1190 topsrcdir=self.topsrcdir) 1191 return artifacts 1192 1193 @ArtifactSubCommand('artifact', 'install', 1194 'Install a good pre-built artifact.') 1195 @CommandArgument('source', metavar='SRC', nargs='?', type=str, 1196 help='Where to fetch and install artifacts from. Can be omitted, in ' 1197 'which case the current hg repository is inspected; an hg revision; ' 1198 'a remote URL; or a local file.', 1199 default=None) 1200 @CommandArgument('--skip-cache', action='store_true', 1201 help='Skip all local caches to force re-fetching remote artifacts.', 1202 default=False) 1203 def artifact_install(self, source=None, skip_cache=False, tree=None, job=None, verbose=False): 1204 self._set_log_level(verbose) 1205 artifacts = self._make_artifacts(tree=tree, job=job, skip_cache=skip_cache) 1206 1207 return artifacts.install_from(source, self.distdir) 1208 1209 @ArtifactSubCommand('artifact', 'clear-cache', 1210 'Delete local artifacts and reset local artifact cache.') 1211 def artifact_clear_cache(self, tree=None, job=None, verbose=False): 1212 self._set_log_level(verbose) 1213 artifacts = self._make_artifacts(tree=tree, job=job) 1214 artifacts.clear_cache() 1215 return 0 1216 1217 @SubCommand('artifact', 'toolchain') 1218 @CommandArgument('--verbose', '-v', action='store_true', 1219 help='Print verbose output.') 1220 @CommandArgument('--cache-dir', metavar='DIR', 1221 help='Directory where to store the artifacts cache') 1222 @CommandArgument('--skip-cache', action='store_true', 1223 help='Skip all local caches to force re-fetching remote artifacts.', 1224 default=False) 1225 @CommandArgument('--from-build', metavar='BUILD', nargs='+', 1226 help='Get toolchains resulting from the given build(s)') 1227 @CommandArgument('--tooltool-manifest', metavar='MANIFEST', 1228 help='Explicit tooltool manifest to process') 1229 @CommandArgument('--authentication-file', metavar='FILE', 1230 help='Use the RelengAPI token found in the given file to authenticate') 1231 @CommandArgument('--tooltool-url', metavar='URL', 1232 help='Use the given url as tooltool server') 1233 @CommandArgument('--no-unpack', action='store_true', 1234 help='Do not unpack any downloaded file') 1235 @CommandArgument('--retry', type=int, default=0, 1236 help='Number of times to retry failed downloads') 1237 @CommandArgument('--artifact-manifest', metavar='FILE', 1238 help='Store a manifest about the downloaded taskcluster artifacts') 1239 @CommandArgument('files', nargs='*', 1240 help='A list of files to download, in the form path@task-id, in ' 1241 'addition to the files listed in the tooltool manifest.') 1242 def artifact_toolchain(self, verbose=False, cache_dir=None, 1243 skip_cache=False, from_build=(), 1244 tooltool_manifest=None, authentication_file=None, 1245 tooltool_url=None, no_unpack=False, retry=None, 1246 artifact_manifest=None, files=()): 1247 '''Download, cache and install pre-built toolchains. 1248 ''' 1249 from mozbuild.artifacts import ArtifactCache 1250 from mozbuild.action.tooltool import ( 1251 FileRecord, 1252 open_manifest, 1253 unpack_file, 1254 ) 1255 from requests.adapters import HTTPAdapter 1256 import redo 1257 import requests 1258 import shutil 1259 1260 from taskgraph.util.taskcluster import ( 1261 get_artifact_url, 1262 ) 1263 1264 self._set_log_level(verbose) 1265 # Normally, we'd use self.log_manager.enable_unstructured(), 1266 # but that enables all logging, while we only really want tooltool's 1267 # and it also makes structured log output twice. 1268 # So we manually do what it does, and limit that to the tooltool 1269 # logger. 1270 if self.log_manager.terminal_handler: 1271 logging.getLogger('mozbuild.action.tooltool').addHandler( 1272 self.log_manager.terminal_handler) 1273 logging.getLogger('redo').addHandler( 1274 self.log_manager.terminal_handler) 1275 self.log_manager.terminal_handler.addFilter( 1276 self.log_manager.structured_filter) 1277 if not cache_dir: 1278 cache_dir = os.path.join(self._mach_context.state_dir, 'toolchains') 1279 1280 tooltool_url = (tooltool_url or 1281 'https://tooltool.mozilla-releng.net').rstrip('/') 1282 1283 cache = ArtifactCache(cache_dir=cache_dir, log=self.log, 1284 skip_cache=skip_cache) 1285 1286 if authentication_file: 1287 with open(authentication_file, 'rb') as f: 1288 token = f.read().strip() 1289 1290 class TooltoolAuthenticator(HTTPAdapter): 1291 def send(self, request, *args, **kwargs): 1292 request.headers['Authorization'] = \ 1293 'Bearer {}'.format(token) 1294 return super(TooltoolAuthenticator, self).send( 1295 request, *args, **kwargs) 1296 1297 cache._download_manager.session.mount( 1298 tooltool_url, TooltoolAuthenticator()) 1299 1300 class DownloadRecord(FileRecord): 1301 def __init__(self, url, *args, **kwargs): 1302 super(DownloadRecord, self).__init__(*args, **kwargs) 1303 self.url = url 1304 self.basename = self.filename 1305 1306 def fetch_with(self, cache): 1307 self.filename = cache.fetch(self.url) 1308 return self.filename 1309 1310 def validate(self): 1311 if self.size is None and self.digest is None: 1312 return True 1313 return super(DownloadRecord, self).validate() 1314 1315 class ArtifactRecord(DownloadRecord): 1316 def __init__(self, task_id, artifact_name): 1317 cot = cache._download_manager.session.get( 1318 get_artifact_url(task_id, 'public/chain-of-trust.json')) 1319 cot.raise_for_status() 1320 digest = algorithm = None 1321 data = json.loads(cot.content) 1322 for algorithm, digest in (data.get('artifacts', {}) 1323 .get(artifact_name, {}).items()): 1324 pass 1325 1326 name = os.path.basename(artifact_name) 1327 artifact_url = get_artifact_url(task_id, artifact_name, 1328 use_proxy=not artifact_name.startswith('public/')) 1329 super(ArtifactRecord, self).__init__( 1330 artifact_url, name, 1331 None, digest, algorithm, unpack=True) 1332 1333 records = OrderedDict() 1334 downloaded = [] 1335 1336 if tooltool_manifest: 1337 manifest = open_manifest(tooltool_manifest) 1338 for record in manifest.file_records: 1339 url = '{}/{}/{}'.format(tooltool_url, record.algorithm, 1340 record.digest) 1341 records[record.filename] = DownloadRecord( 1342 url, record.filename, record.size, record.digest, 1343 record.algorithm, unpack=record.unpack, 1344 version=record.version, visibility=record.visibility, 1345 setup=record.setup) 1346 1347 if from_build: 1348 if 'TASK_ID' in os.environ: 1349 self.log(logging.ERROR, 'artifact', {}, 1350 'Do not use --from-build in automation; all dependencies ' 1351 'should be determined in the decision task.') 1352 return 1 1353 from taskgraph.optimize import IndexSearch 1354 from taskgraph.parameters import Parameters 1355 from taskgraph.generator import load_tasks_for_kind 1356 params = Parameters( 1357 level=os.environ.get('MOZ_SCM_LEVEL', '3'), 1358 strict=False, 1359 ) 1360 1361 root_dir = mozpath.join(self.topsrcdir, 'taskcluster/ci') 1362 toolchains = load_tasks_for_kind(params, 'toolchain', root_dir=root_dir) 1363 1364 aliases = {} 1365 for t in toolchains.values(): 1366 alias = t.attributes.get('toolchain-alias') 1367 if alias: 1368 aliases['toolchain-{}'.format(alias)] = \ 1369 t.task['metadata']['name'] 1370 1371 for b in from_build: 1372 user_value = b 1373 1374 if not b.startswith('toolchain-'): 1375 b = 'toolchain-{}'.format(b) 1376 1377 task = toolchains.get(aliases.get(b, b)) 1378 if not task: 1379 self.log(logging.ERROR, 'artifact', {'build': user_value}, 1380 'Could not find a toolchain build named `{build}`') 1381 return 1 1382 1383 task_id = IndexSearch().should_replace_task( 1384 task, {}, task.optimization.get('index-search', [])) 1385 artifact_name = task.attributes.get('toolchain-artifact') 1386 if task_id in (True, False) or not artifact_name: 1387 self.log(logging.ERROR, 'artifact', {'build': user_value}, 1388 'Could not find artifacts for a toolchain build ' 1389 'named `{build}`. Local commits and other changes ' 1390 'in your checkout may cause this error. Try ' 1391 'updating to a fresh checkout of mozilla-central ' 1392 'to use artifact builds.') 1393 return 1 1394 1395 record = ArtifactRecord(task_id, artifact_name) 1396 records[record.filename] = record 1397 1398 # Handle the list of files of the form path@task-id on the command 1399 # line. Each of those give a path to an artifact to download. 1400 for f in files: 1401 if '@' not in f: 1402 self.log(logging.ERROR, 'artifact', {}, 1403 'Expected a list of files of the form path@task-id') 1404 return 1 1405 name, task_id = f.rsplit('@', 1) 1406 record = ArtifactRecord(task_id, name) 1407 records[record.filename] = record 1408 1409 for record in records.itervalues(): 1410 self.log(logging.INFO, 'artifact', {'name': record.basename}, 1411 'Downloading {name}') 1412 valid = False 1413 # sleeptime is 60 per retry.py, used by tooltool_wrapper.sh 1414 for attempt, _ in enumerate(redo.retrier(attempts=retry+1, 1415 sleeptime=60)): 1416 try: 1417 record.fetch_with(cache) 1418 except (requests.exceptions.HTTPError, 1419 requests.exceptions.ChunkedEncodingError, 1420 requests.exceptions.ConnectionError) as e: 1421 1422 if isinstance(e, requests.exceptions.HTTPError): 1423 # The relengapi proxy likes to return error 400 bad request 1424 # which seems improbably to be due to our (simple) GET 1425 # being borked. 1426 status = e.response.status_code 1427 should_retry = status >= 500 or status == 400 1428 else: 1429 should_retry = True 1430 1431 if should_retry or attempt < retry: 1432 level = logging.WARN 1433 else: 1434 level = logging.ERROR 1435 # e.message is not always a string, so convert it first. 1436 self.log(level, 'artifact', {}, str(e.message)) 1437 if not should_retry: 1438 break 1439 if attempt < retry: 1440 self.log(logging.INFO, 'artifact', {}, 1441 'Will retry in a moment...') 1442 continue 1443 try: 1444 valid = record.validate() 1445 except Exception: 1446 pass 1447 if not valid: 1448 os.unlink(record.filename) 1449 if attempt < retry: 1450 self.log(logging.INFO, 'artifact', {}, 1451 'Will retry in a moment...') 1452 continue 1453 1454 downloaded.append(record) 1455 break 1456 1457 if not valid: 1458 self.log(logging.ERROR, 'artifact', {'name': record.basename}, 1459 'Failed to download {name}') 1460 return 1 1461 1462 artifacts = {} if artifact_manifest else None 1463 1464 for record in downloaded: 1465 local = os.path.join(os.getcwd(), record.basename) 1466 if os.path.exists(local): 1467 os.unlink(local) 1468 # unpack_file needs the file with its final name to work 1469 # (https://github.com/mozilla/build-tooltool/issues/38), so we 1470 # need to copy it, even though we remove it later. Use hard links 1471 # when possible. 1472 try: 1473 os.link(record.filename, local) 1474 except Exception: 1475 shutil.copy(record.filename, local) 1476 # Keep a sha256 of each downloaded file, for the chain-of-trust 1477 # validation. 1478 if artifact_manifest is not None: 1479 with open(local) as fh: 1480 h = hashlib.sha256() 1481 while True: 1482 data = fh.read(1024 * 1024) 1483 if not data: 1484 break 1485 h.update(data) 1486 artifacts[record.url] = { 1487 'sha256': h.hexdigest(), 1488 } 1489 if record.unpack and not no_unpack: 1490 unpack_file(local, record.setup) 1491 os.unlink(local) 1492 1493 if not downloaded: 1494 self.log(logging.ERROR, 'artifact', {}, 'Nothing to download') 1495 if files: 1496 return 1 1497 1498 if artifacts: 1499 ensureParentDir(artifact_manifest) 1500 with open(artifact_manifest, 'w') as fh: 1501 json.dump(artifacts, fh, indent=4, sort_keys=True) 1502 1503 return 0 1504 1505class StaticAnalysisSubCommand(SubCommand): 1506 def __call__(self, func): 1507 after = SubCommand.__call__(self, func) 1508 args = [ 1509 CommandArgument('--verbose', '-v', action='store_true', 1510 help='Print verbose output.'), 1511 ] 1512 for arg in args: 1513 after = arg(after) 1514 return after 1515 1516 1517class StaticAnalysisMonitor(object): 1518 def __init__(self, srcdir, objdir, total): 1519 self._total = total 1520 self._processed = 0 1521 self._current = None 1522 self._srcdir = srcdir 1523 1524 from mozbuild.compilation.warnings import ( 1525 WarningsCollector, 1526 WarningsDatabase, 1527 ) 1528 1529 self._warnings_database = WarningsDatabase() 1530 1531 def on_warning(warning): 1532 filename = warning['filename'] 1533 self._warnings_database.insert(warning) 1534 1535 self._warnings_collector = WarningsCollector(on_warning, objdir=objdir) 1536 1537 @property 1538 def num_files(self): 1539 return self._total 1540 1541 @property 1542 def num_files_processed(self): 1543 return self._processed 1544 1545 @property 1546 def current_file(self): 1547 return self._current 1548 1549 @property 1550 def warnings_db(self): 1551 return self._warnings_database 1552 1553 def on_line(self, line): 1554 warning = None 1555 1556 try: 1557 warning = self._warnings_collector.process_line(line) 1558 except: 1559 pass 1560 1561 if line.find('clang-tidy') != -1: 1562 filename = line.split(' ')[-1] 1563 if os.path.isfile(filename): 1564 self._current = os.path.relpath(filename, self._srcdir) 1565 else: 1566 self._current = None 1567 self._processed = self._processed + 1 1568 return (warning, False) 1569 return (warning, True) 1570 1571 1572@CommandProvider 1573class StaticAnalysis(MachCommandBase): 1574 """Utilities for running C++ static analysis checks and format.""" 1575 1576 # List of file extension to consider (should start with dot) 1577 _format_include_extensions = ('.cpp', '.c', '.cc', '.h') 1578 # File contaning all paths to exclude from formatting 1579 _format_ignore_file = '.clang-format-ignore' 1580 1581 @Command('static-analysis', category='testing', 1582 description='Run C++ static analysis checks') 1583 def static_analysis(self): 1584 # If not arguments are provided, just print a help message. 1585 mach = Mach(os.getcwd()) 1586 mach.run(['static-analysis', '--help']) 1587 1588 @StaticAnalysisSubCommand('static-analysis', 'check', 1589 'Run the checks using the helper tool') 1590 @CommandArgument('source', nargs='*', default=['.*'], 1591 help='Source files to be analyzed (regex on path). ' 1592 'Can be omitted, in which case the entire code base ' 1593 'is analyzed. The source argument is ignored if ' 1594 'there is anything fed through stdin, in which case ' 1595 'the analysis is only performed on the files changed ' 1596 'in the patch streamed through stdin. This is called ' 1597 'the diff mode.') 1598 @CommandArgument('--checks', '-c', default='-*', metavar='checks', 1599 help='Static analysis checks to enable. By default, this enables only ' 1600 'custom Mozilla checks, but can be any clang-tidy checks syntax.') 1601 @CommandArgument('--jobs', '-j', default='0', metavar='jobs', type=int, 1602 help='Number of concurrent jobs to run. Default is the number of CPUs.') 1603 @CommandArgument('--strip', '-p', default='1', metavar='NUM', 1604 help='Strip NUM leading components from file names in diff mode.') 1605 @CommandArgument('--fix', '-f', default=False, action='store_true', 1606 help='Try to autofix errors detected by clang-tidy checkers.') 1607 @CommandArgument('--header-filter', '-h-f', default='', metavar='header_filter', 1608 help='Regular expression matching the names of the headers to ' 1609 'output diagnostics from. Diagnostics from the main file ' 1610 'of each translation unit are always displayed') 1611 def check(self, source=None, jobs=2, strip=1, verbose=False, 1612 checks='-*', fix=False, header_filter=''): 1613 from mozbuild.controller.building import ( 1614 StaticAnalysisFooter, 1615 StaticAnalysisOutputManager, 1616 ) 1617 1618 self._set_log_level(verbose) 1619 self.log_manager.enable_all_structured_loggers() 1620 1621 rc = self._build_compile_db(verbose=verbose) 1622 if rc != 0: 1623 return rc 1624 1625 rc = self._build_export(jobs=jobs, verbose=verbose) 1626 if rc != 0: 1627 return rc 1628 1629 rc = self._get_clang_tools(verbose=verbose) 1630 if rc != 0: 1631 return rc 1632 1633 python = self.virtualenv_manager.python_path 1634 1635 if checks == '-*': 1636 checks = self._get_checks() 1637 1638 common_args = ['-clang-tidy-binary', self._clang_tidy_path, 1639 '-clang-apply-replacements-binary', self._clang_apply_replacements, 1640 '-checks=%s' % checks, 1641 '-extra-arg=-DMOZ_CLANG_PLUGIN'] 1642 1643 # Flag header-filter is passed to 'run-clang-tidy' in order to limit 1644 # the diagnostic messages only to the specified header files. 1645 # When no value is specified the default value is considered to be the source 1646 # in order to limit the dianostic message to the source files or folders. 1647 common_args.append('-header-filter=%s' % 1648 (header_filter if len(header_filter) else ''.join(source))) 1649 1650 if fix: 1651 common_args.append('-fix') 1652 1653 compile_db = json.loads(open(self._compile_db, 'r').read()) 1654 total = 0 1655 import re 1656 name_re = re.compile('(' + ')|('.join(source) + ')') 1657 for f in compile_db: 1658 if name_re.search(f['file']): 1659 total = total + 1 1660 1661 if not total: 1662 return 0 1663 1664 args = [python, self._run_clang_tidy_path, '-p', self.topobjdir] 1665 args += ['-j', str(jobs)] + source + common_args 1666 cwd = self.topobjdir 1667 1668 monitor = StaticAnalysisMonitor(self.topsrcdir, self.topobjdir, total) 1669 1670 footer = StaticAnalysisFooter(self.log_manager.terminal, monitor) 1671 with StaticAnalysisOutputManager(self.log_manager, monitor, footer) as output: 1672 rc = self.run_process(args=args, line_handler=output.on_line, cwd=cwd) 1673 1674 self.log(logging.WARNING, 'warning_summary', 1675 {'count': len(monitor.warnings_db)}, 1676 '{count} warnings present.') 1677 return rc 1678 1679 @StaticAnalysisSubCommand('static-analysis', 'install', 1680 'Install the static analysis helper tool') 1681 @CommandArgument('source', nargs='?', type=str, 1682 help='Where to fetch a local archive containing the static-analysis and format helper tool.' 1683 'It will be installed in ~/.mozbuild/clang-tools/.' 1684 'Can be omitted, in which case the latest clang-tools ' 1685 ' helper for the platform would be automatically ' 1686 'detected and installed.') 1687 @CommandArgument('--skip-cache', action='store_true', 1688 help='Skip all local caches to force re-fetching the helper tool.', 1689 default=False) 1690 def install(self, source=None, skip_cache=False, verbose=False): 1691 self._set_log_level(verbose) 1692 rc = self._get_clang_tools(force=True, skip_cache=skip_cache, 1693 source=source, verbose=verbose) 1694 return rc 1695 1696 @StaticAnalysisSubCommand('static-analysis', 'clear-cache', 1697 'Delete local helpers and reset static analysis helper tool cache') 1698 def clear_cache(self, verbose=False): 1699 self._set_log_level(verbose) 1700 rc = self._get_clang_tools(force=True, download_if_needed=False, 1701 verbose=verbose) 1702 if rc != 0: 1703 return rc 1704 1705 self._artifact_manager.artifact_clear_cache() 1706 1707 @StaticAnalysisSubCommand('static-analysis', 'print-checks', 1708 'Print a list of the static analysis checks performed by default') 1709 def print_checks(self, verbose=False): 1710 self._set_log_level(verbose) 1711 rc = self._get_clang_tools(verbose=verbose) 1712 if rc != 0: 1713 return rc 1714 args = [self._clang_tidy_path, '-list-checks', '-checks=-*,mozilla-*'] 1715 return self._run_command_in_objdir(args=args, pass_thru=True) 1716 1717 @Command('clang-format', category='misc', description='Run clang-format on current changes') 1718 @CommandArgument('--show', '-s', action='store_true', default=False, 1719 help='Show diff output on instead of applying changes') 1720 @CommandArgument('--path', '-p', nargs='+', default=None, 1721 help='Specify the path(s) to reformat') 1722 def clang_format(self, show, path, verbose=False): 1723 # Run clang-format or clang-format-diff on the local changes 1724 # or files/directories 1725 if path is not None: 1726 path = self._conv_to_abspath(path) 1727 1728 os.chdir(self.topsrcdir) 1729 1730 rc = self._get_clang_tools(verbose=verbose) 1731 if rc != 0: 1732 return rc 1733 1734 if path is None: 1735 return self._run_clang_format_diff(self._clang_format_diff, 1736 self._clang_format_path, show) 1737 else: 1738 return self._run_clang_format_path(self._clang_format_path, show, path) 1739 1740 def _get_checks(self): 1741 checks = '-*' 1742 import yaml 1743 with open(mozpath.join(self.topsrcdir, "tools", "clang-tidy", "config.yaml")) as f: 1744 try: 1745 config = yaml.load(f) 1746 for item in config['clang_checkers']: 1747 if item['publish']: 1748 checks += ',' + item['name'] 1749 except Exception: 1750 print('Looks like config.yaml is not valid, so we are unable to ' 1751 'determine default checkers, using \'-checks=-*,mozilla-*\'') 1752 checks += ',mozilla-*' 1753 return checks 1754 1755 def _get_config_environment(self): 1756 ran_configure = False 1757 config = None 1758 builder = Build(self._mach_context) 1759 1760 try: 1761 config = self.config_environment 1762 except Exception: 1763 print('Looks like configure has not run yet, running it now...') 1764 rc = builder.configure() 1765 if rc != 0: 1766 return (rc, config, ran_configure) 1767 ran_configure = True 1768 try: 1769 config = self.config_environment 1770 except Exception as e: 1771 pass 1772 1773 return (0, config, ran_configure) 1774 1775 def _build_compile_db(self, verbose=False): 1776 self._compile_db = mozpath.join(self.topobjdir, 'compile_commands.json') 1777 if os.path.exists(self._compile_db): 1778 return 0 1779 else: 1780 rc, config, ran_configure = self._get_config_environment() 1781 if rc != 0: 1782 return rc 1783 1784 if ran_configure: 1785 # Configure may have created the compilation database if the 1786 # mozconfig enables building the CompileDB backend by default, 1787 # So we recurse to see if the file exists once again. 1788 return self._build_compile_db(verbose=verbose) 1789 1790 if config: 1791 print('Looks like a clang compilation database has not been ' 1792 'created yet, creating it now...') 1793 builder = Build(self._mach_context) 1794 rc = builder.build_backend(['CompileDB'], verbose=verbose) 1795 if rc != 0: 1796 return rc 1797 assert os.path.exists(self._compile_db) 1798 return 0 1799 1800 def _build_export(self, jobs, verbose=False): 1801 def on_line(line): 1802 self.log(logging.INFO, 'build_output', {'line': line}, '{line}') 1803 1804 builder = Build(self._mach_context) 1805 # First install what we can through install manifests. 1806 rc = builder._run_make(directory=self.topobjdir, target='pre-export', 1807 line_handler=None, silent=not verbose) 1808 if rc != 0: 1809 return rc 1810 1811 # Then build the rest of the build dependencies by running the full 1812 # export target, because we can't do anything better. 1813 return builder._run_make(directory=self.topobjdir, target='export', 1814 line_handler=None, silent=not verbose, 1815 num_jobs=jobs) 1816 1817 def _conv_to_abspath(self, paths): 1818 # Converts all the paths to absolute pathnames 1819 tmp_path = [] 1820 for f in paths: 1821 tmp_path.append(os.path.abspath(f)) 1822 return tmp_path 1823 1824 def _get_clang_tools(self, force=False, skip_cache=False, 1825 source=None, download_if_needed=True, 1826 verbose=False): 1827 rc, config, _ = self._get_config_environment() 1828 1829 if rc != 0: 1830 return rc 1831 1832 clang_tools_path = mozpath.join(self._mach_context.state_dir, 1833 "clang-tools") 1834 self._clang_tidy_path = mozpath.join(clang_tools_path, "clang", "bin", 1835 "clang-tidy" + config.substs.get('BIN_SUFFIX', '')) 1836 self._clang_format_path = mozpath.join(clang_tools_path, "clang", "bin", 1837 "clang-format" + config.substs.get('BIN_SUFFIX', '')) 1838 self._clang_apply_replacements = mozpath.join(clang_tools_path, "clang", "bin", 1839 "clang-apply-replacements" + config.substs.get('BIN_SUFFIX', '')) 1840 self._run_clang_tidy_path = mozpath.join(clang_tools_path, "clang", "share", 1841 "clang", "run-clang-tidy.py") 1842 self._clang_format_diff = mozpath.join(clang_tools_path, "clang", "share", 1843 "clang", "clang-format-diff.py") 1844 1845 if os.path.exists(self._clang_tidy_path) and \ 1846 os.path.exists(self._clang_format_path) and \ 1847 os.path.exists(self._clang_apply_replacements) and \ 1848 os.path.exists(self._run_clang_tidy_path) and \ 1849 not force: 1850 return 0 1851 else: 1852 if os.path.isdir(clang_tools_path) and download_if_needed: 1853 # The directory exists, perhaps it's corrupted? Delete it 1854 # and start from scratch. 1855 import shutil 1856 shutil.rmtree(clang_tools_path) 1857 return self._get_clang_tools(force=force, skip_cache=skip_cache, 1858 source=source, verbose=verbose, 1859 download_if_needed=download_if_needed) 1860 1861 # Create base directory where we store clang binary 1862 os.mkdir(clang_tools_path) 1863 1864 if source: 1865 return self._get_clang_tools_from_source(source) 1866 1867 self._artifact_manager = PackageFrontend(self._mach_context) 1868 1869 if not download_if_needed: 1870 return 0 1871 1872 job, _ = self.platform 1873 1874 if job is None: 1875 raise Exception('The current platform isn\'t supported. ' 1876 'Currently only the following platforms are ' 1877 'supported: win32/win64, linux64 and macosx64.') 1878 else: 1879 job += '-clang-tidy' 1880 1881 # We want to unpack data in the clang-tidy mozbuild folder 1882 currentWorkingDir = os.getcwd() 1883 os.chdir(clang_tools_path) 1884 rc = self._artifact_manager.artifact_toolchain(verbose=verbose, 1885 skip_cache=skip_cache, 1886 from_build=[job], 1887 no_unpack=False, 1888 retry=0) 1889 # Change back the cwd 1890 os.chdir(currentWorkingDir) 1891 1892 return rc 1893 1894 def _get_clang_tools_from_source(self, filename): 1895 from mozbuild.action.tooltool import unpack_file 1896 clang_tidy_path = mozpath.join(self._mach_context.state_dir, 1897 "clang-tools") 1898 1899 currentWorkingDir = os.getcwd() 1900 os.chdir(clang_tidy_path) 1901 1902 unpack_file(filename) 1903 1904 # Change back the cwd 1905 os.chdir(currentWorkingDir) 1906 1907 clang_path = mozpath.join(clang_tidy_path, 'clang') 1908 1909 if not os.path.isdir(clang_path): 1910 raise Exception('Extracted the archive but didn\'t find ' 1911 'the expected output') 1912 1913 assert os.path.exists(self._clang_tidy_path) 1914 assert os.path.exists(self._clang_format_path) 1915 assert os.path.exists(self._clang_apply_replacements) 1916 assert os.path.exists(self._run_clang_tidy_path) 1917 return 0 1918 1919 def _get_clang_format_diff_command(self): 1920 if self.repository.name == 'hg': 1921 args = ["hg", "diff", "-U0", "-r" ".^"] 1922 for dot_extension in self._format_include_extensions: 1923 args += ['--include', 'glob:**{0}'.format(dot_extension)] 1924 args += ['--exclude', 'listfile:{0}'.format(self._format_ignore_file)] 1925 else: 1926 args = ["git", "diff", "--no-color", "-U0", "HEAD", "--"] 1927 for dot_extension in self._format_include_extensions: 1928 args += ['*{0}'.format(dot_extension)] 1929 # git-diff doesn't support an 'exclude-from-files' param, but 1930 # allow to add individual exclude pattern since v1.9, see 1931 # https://git-scm.com/docs/gitglossary#gitglossary-aiddefpathspecapathspec 1932 with open(self._format_ignore_file, 'rb') as exclude_pattern_file: 1933 for pattern in exclude_pattern_file.readlines(): 1934 pattern = pattern.rstrip() 1935 pattern = pattern.replace('.*', '**') 1936 if not pattern or pattern.startswith('#'): 1937 continue # empty or comment 1938 magics = ['exclude'] 1939 if pattern.startswith('^'): 1940 magics += ['top'] 1941 pattern = pattern[1:] 1942 args += [':({0}){1}'.format(','.join(magics), pattern)] 1943 return args 1944 1945 def _run_clang_format_diff(self, clang_format_diff, clang_format, show): 1946 # Run clang-format on the diff 1947 # Note that this will potentially miss a lot things 1948 from subprocess import Popen, PIPE, check_output, CalledProcessError 1949 1950 diff_process = Popen(self._get_clang_format_diff_command(), stdout=PIPE) 1951 args = [sys.executable, clang_format_diff, "-p1", "-binary=%s" % clang_format] 1952 1953 if not show: 1954 args.append("-i") 1955 try: 1956 output = check_output(args, stdin=diff_process.stdout) 1957 if show: 1958 # We want to print the diffs 1959 print(output) 1960 return 0 1961 except CalledProcessError as e: 1962 # Something wrong happend 1963 print("clang-format: An error occured while running clang-format-diff.") 1964 return e.returncode 1965 1966 def _is_ignored_path(self, ignored_dir_re, f): 1967 # Remove upto topsrcdir in pathname and match 1968 if f.startswith(self.topsrcdir + '/'): 1969 match_f = f[len(self.topsrcdir + '/'):] 1970 else: 1971 match_f = f 1972 return re.match(ignored_dir_re, match_f) 1973 1974 def _generate_path_list(self, paths): 1975 path_to_third_party = os.path.join(self.topsrcdir, self._format_ignore_file) 1976 ignored_dir = [] 1977 with open(path_to_third_party, 'r') as fh: 1978 for line in fh: 1979 # Remove comments and empty lines 1980 if line.startswith('#') or len(line.strip()) == 0: 1981 continue 1982 # The regexp is to make sure we are managing relative paths 1983 ignored_dir.append(r"^[\./]*" + line.rstrip()) 1984 1985 # Generates the list of regexp 1986 ignored_dir_re = '(%s)' % '|'.join(ignored_dir) 1987 extensions = self._format_include_extensions 1988 1989 path_list = [] 1990 for f in paths: 1991 if self._is_ignored_path(ignored_dir_re, f): 1992 # Early exit if we have provided an ignored directory 1993 print("clang-format: Ignored third party code '{0}'".format(f)) 1994 continue 1995 1996 if os.path.isdir(f): 1997 # Processing a directory, generate the file list 1998 for folder, subs, files in os.walk(f): 1999 subs.sort() 2000 for filename in sorted(files): 2001 f_in_dir = os.path.join(folder, filename) 2002 if (f_in_dir.endswith(extensions) 2003 and not self._is_ignored_path(ignored_dir_re, f_in_dir)): 2004 # Supported extension and accepted path 2005 path_list.append(f_in_dir) 2006 else: 2007 if f.endswith(extensions): 2008 path_list.append(f) 2009 2010 return path_list 2011 2012 def _run_clang_format_path(self, clang_format, show, paths): 2013 # Run clang-format on files or directories directly 2014 from subprocess import check_output, CalledProcessError 2015 2016 args = [clang_format, "-i"] 2017 2018 path_list = self._generate_path_list(paths) 2019 2020 if path_list == []: 2021 return 2022 2023 print("Processing %d file(s)..." % len(path_list)) 2024 2025 batchsize = 200 2026 2027 for i in range(0, len(path_list), batchsize): 2028 l = path_list[i: (i + batchsize)] 2029 # Run clang-format on the list 2030 try: 2031 check_output(args + l) 2032 except CalledProcessError as e: 2033 # Something wrong happend 2034 print("clang-format: An error occured while running clang-format.") 2035 return e.returncode 2036 2037 if show: 2038 # show the diff 2039 if self.repository.name == 'hg': 2040 diff_command = ["hg", "diff"] + paths 2041 else: 2042 assert self.repository.name == 'git' 2043 diff_command = ["git", "diff"] + paths 2044 try: 2045 output = check_output(diff_command) 2046 print(output) 2047 except CalledProcessError as e: 2048 # Something wrong happend 2049 print("clang-format: Unable to run the diff command.") 2050 return e.returncode 2051 return 0 2052 2053@CommandProvider 2054class Vendor(MachCommandBase): 2055 """Vendor third-party dependencies into the source repository.""" 2056 2057 @Command('vendor', category='misc', 2058 description='Vendor third-party dependencies into the source repository.') 2059 def vendor(self): 2060 self.parser.print_usage() 2061 sys.exit(1) 2062 2063 @SubCommand('vendor', 'rust', 2064 description='Vendor rust crates from crates.io into third_party/rust') 2065 @CommandArgument('--ignore-modified', action='store_true', 2066 help='Ignore modified files in current checkout', 2067 default=False) 2068 @CommandArgument('--build-peers-said-large-imports-were-ok', action='store_true', 2069 help='Permit overly-large files to be added to the repository', 2070 default=False) 2071 def vendor_rust(self, **kwargs): 2072 from mozbuild.vendor_rust import VendorRust 2073 vendor_command = self._spawn(VendorRust) 2074 vendor_command.vendor(**kwargs) 2075 2076 @SubCommand('vendor', 'aom', 2077 description='Vendor av1 video codec reference implementation into the source repository.') 2078 @CommandArgument('-r', '--revision', 2079 help='Repository tag or commit to update to.') 2080 @CommandArgument('--repo', 2081 help='Repository url to pull a snapshot from. Supports github and googlesource.') 2082 @CommandArgument('--ignore-modified', action='store_true', 2083 help='Ignore modified files in current checkout', 2084 default=False) 2085 def vendor_aom(self, **kwargs): 2086 from mozbuild.vendor_aom import VendorAOM 2087 vendor_command = self._spawn(VendorAOM) 2088 vendor_command.vendor(**kwargs) 2089 2090 2091@CommandProvider 2092class WebRTCGTestCommands(GTestCommands): 2093 @Command('webrtc-gtest', category='testing', 2094 description='Run WebRTC.org GTest unit tests.') 2095 @CommandArgument('gtest_filter', default=b"*", nargs='?', metavar='gtest_filter', 2096 help="test_filter is a ':'-separated list of wildcard patterns (called the positive patterns)," 2097 "optionally followed by a '-' and another ':'-separated pattern list (called the negative patterns).") 2098 @CommandArgumentGroup('debugging') 2099 @CommandArgument('--debug', action='store_true', group='debugging', 2100 help='Enable the debugger. Not specifying a --debugger option will result in the default debugger being used.') 2101 @CommandArgument('--debugger', default=None, type=str, group='debugging', 2102 help='Name of debugger to use.') 2103 @CommandArgument('--debugger-args', default=None, metavar='params', type=str, 2104 group='debugging', 2105 help='Command-line arguments to pass to the debugger itself; split as the Bourne shell would.') 2106 def gtest(self, gtest_filter, debug, debugger, 2107 debugger_args): 2108 app_path = self.get_binary_path('webrtc-gtest') 2109 args = [app_path] 2110 2111 if debug or debugger or debugger_args: 2112 args = self.prepend_debugger_args(args, debugger, debugger_args) 2113 2114 # Used to locate resources used by tests 2115 cwd = os.path.join(self.topsrcdir, 'media', 'webrtc', 'trunk') 2116 2117 if not os.path.isdir(cwd): 2118 print('Unable to find working directory for tests: %s' % cwd) 2119 return 1 2120 2121 gtest_env = { 2122 # These tests are not run under ASAN upstream, so we need to 2123 # disable some checks. 2124 b'ASAN_OPTIONS': 'alloc_dealloc_mismatch=0', 2125 # Use GTest environment variable to control test execution 2126 # For details see: 2127 # https://code.google.com/p/googletest/wiki/AdvancedGuide#Running_Test_Programs:_Advanced_Options 2128 b'GTEST_FILTER': gtest_filter 2129 } 2130 2131 return self.run_process(args=args, 2132 append_env=gtest_env, 2133 cwd=cwd, 2134 ensure_exit_code=False, 2135 pass_thru=True) 2136 2137@CommandProvider 2138class Repackage(MachCommandBase): 2139 '''Repackages artifacts into different formats. 2140 2141 This is generally used after packages are signed by the signing 2142 scriptworkers in order to bundle things up into shippable formats, such as a 2143 .dmg on OSX or an installer exe on Windows. 2144 ''' 2145 @Command('repackage', category='misc', 2146 description='Repackage artifacts into different formats.') 2147 def repackage(self): 2148 print("Usage: ./mach repackage [dmg|installer|mar] [args...]") 2149 2150 @SubCommand('repackage', 'dmg', 2151 description='Repackage a tar file into a .dmg for OSX') 2152 @CommandArgument('--input', '-i', type=str, required=True, 2153 help='Input filename') 2154 @CommandArgument('--output', '-o', type=str, required=True, 2155 help='Output filename') 2156 def repackage_dmg(self, input, output): 2157 if not os.path.exists(input): 2158 print('Input file does not exist: %s' % input) 2159 return 1 2160 2161 if not os.path.exists(os.path.join(self.topobjdir, 'config.status')): 2162 print('config.status not found. Please run |mach configure| ' 2163 'prior to |mach repackage|.') 2164 return 1 2165 2166 from mozbuild.repackaging.dmg import repackage_dmg 2167 repackage_dmg(input, output) 2168 2169 @SubCommand('repackage', 'installer', 2170 description='Repackage into a Windows installer exe') 2171 @CommandArgument('--tag', type=str, required=True, 2172 help='The .tag file used to build the installer') 2173 @CommandArgument('--setupexe', type=str, required=True, 2174 help='setup.exe file inside the installer') 2175 @CommandArgument('--package', type=str, required=False, 2176 help='Optional package .zip for building a full installer') 2177 @CommandArgument('--output', '-o', type=str, required=True, 2178 help='Output filename') 2179 @CommandArgument('--package-name', type=str, required=False, 2180 help='Name of the package being rebuilt') 2181 @CommandArgument('--sfx-stub', type=str, required=True, 2182 help='Path to the self-extraction stub.') 2183 def repackage_installer(self, tag, setupexe, package, output, package_name, sfx_stub): 2184 from mozbuild.repackaging.installer import repackage_installer 2185 repackage_installer( 2186 topsrcdir=self.topsrcdir, 2187 tag=tag, 2188 setupexe=setupexe, 2189 package=package, 2190 output=output, 2191 package_name=package_name, 2192 sfx_stub=sfx_stub, 2193 ) 2194 2195 @SubCommand('repackage', 'mar', 2196 description='Repackage into complete MAR file') 2197 @CommandArgument('--input', '-i', type=str, required=True, 2198 help='Input filename') 2199 @CommandArgument('--mar', type=str, required=True, 2200 help='Mar binary path') 2201 @CommandArgument('--output', '-o', type=str, required=True, 2202 help='Output filename') 2203 @CommandArgument('--format', type=str, default='lzma', 2204 choices=('lzma', 'bz2'), 2205 help='Mar format') 2206 def repackage_mar(self, input, mar, output, format): 2207 from mozbuild.repackaging.mar import repackage_mar 2208 repackage_mar(self.topsrcdir, input, mar, output, format) 2209