1#!/usr/bin/python 2# Copyright 2017 The Chromium Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5"""This script helps to generate code coverage report. 6 7 It uses Clang Source-based Code Coverage - 8 https://clang.llvm.org/docs/SourceBasedCodeCoverage.html 9 10 In order to generate code coverage report, you need to first add 11 "use_clang_coverage=true" and "is_component_build=false" GN flags to args.gn 12 file in your build output directory (e.g. out/coverage). 13 14 * Example usage: 15 16 gn gen out/coverage \\ 17 --args="use_clang_coverage=true is_component_build=false\\ 18 is_debug=false dcheck_always_on=true" 19 gclient runhooks 20 python tools/code_coverage/coverage.py crypto_unittests url_unittests \\ 21 -b out/coverage -o out/report -c 'out/coverage/crypto_unittests' \\ 22 -c 'out/coverage/url_unittests --gtest_filter=URLParser.PathURL' \\ 23 -f url/ -f crypto/ 24 25 The command above builds crypto_unittests and url_unittests targets and then 26 runs them with specified command line arguments. For url_unittests, it only 27 runs the test URLParser.PathURL. The coverage report is filtered to include 28 only files and sub-directories under url/ and crypto/ directories. 29 30 If you want to run tests that try to draw to the screen but don't have a 31 display connected, you can run tests in headless mode with xvfb. 32 33 * Sample flow for running a test target with xvfb (e.g. unit_tests): 34 35 python tools/code_coverage/coverage.py unit_tests -b out/coverage \\ 36 -o out/report -c 'python testing/xvfb.py out/coverage/unit_tests' 37 38 If you are building a fuzz target, you need to add "use_libfuzzer=true" GN 39 flag as well. 40 41 * Sample workflow for a fuzz target (e.g. pdfium_fuzzer): 42 43 python tools/code_coverage/coverage.py pdfium_fuzzer \\ 44 -b out/coverage -o out/report \\ 45 -c 'out/coverage/pdfium_fuzzer -runs=0 <corpus_dir>' \\ 46 -f third_party/pdfium 47 48 where: 49 <corpus_dir> - directory containing samples files for this format. 50 51 To learn more about generating code coverage reports for fuzz targets, see 52 https://chromium.googlesource.com/chromium/src/+/master/testing/libfuzzer/efficient_fuzzer.md#Code-Coverage 53 54 * Sample workflow for running Blink web tests: 55 56 python tools/code_coverage/coverage.py blink_tests \\ 57 -wt -b out/coverage -o out/report -f third_party/blink 58 59 If you need to pass arguments to run_web_tests.py, use 60 -wt='arguments to run_web_tests.py e.g. test directories' 61 62 For more options, please refer to tools/code_coverage/coverage.py -h. 63 64 For an overview of how code coverage works in Chromium, please refer to 65 https://chromium.googlesource.com/chromium/src/+/master/docs/testing/code_coverage.md 66""" 67 68from __future__ import print_function 69 70import sys 71 72import argparse 73import json 74import logging 75import multiprocessing 76import os 77import platform 78import re 79import shlex 80import shutil 81import subprocess 82import urllib2 83 84sys.path.append( 85 os.path.join( 86 os.path.dirname(__file__), os.path.pardir, os.path.pardir, 87 'third_party')) 88from collections import defaultdict 89 90import coverage_utils 91 92# Absolute path to the code coverage tools binary. These paths can be 93# overwritten by user specified coverage tool paths. 94# Absolute path to the root of the checkout. 95SRC_ROOT_PATH = os.path.join(os.path.abspath(os.path.dirname(__file__)), 96 os.path.pardir, os.path.pardir) 97LLVM_BIN_DIR = os.path.join( 98 os.path.join(SRC_ROOT_PATH, 'third_party', 'llvm-build', 'Release+Asserts'), 99 'bin') 100LLVM_COV_PATH = os.path.join(LLVM_BIN_DIR, 'llvm-cov') 101LLVM_PROFDATA_PATH = os.path.join(LLVM_BIN_DIR, 'llvm-profdata') 102 103 104# Build directory, the value is parsed from command line arguments. 105BUILD_DIR = None 106 107# Output directory for generated artifacts, the value is parsed from command 108# line arguemnts. 109OUTPUT_DIR = None 110 111# Name of the file extension for profraw data files. 112PROFRAW_FILE_EXTENSION = 'profraw' 113 114# Name of the final profdata file, and this file needs to be passed to 115# "llvm-cov" command in order to call "llvm-cov show" to inspect the 116# line-by-line coverage of specific files. 117PROFDATA_FILE_NAME = os.extsep.join(['coverage', 'profdata']) 118 119# Name of the file with summary information generated by llvm-cov export. 120SUMMARY_FILE_NAME = os.extsep.join(['summary', 'json']) 121 122# Build arg required for generating code coverage data. 123CLANG_COVERAGE_BUILD_ARG = 'use_clang_coverage' 124 125LOGS_DIR_NAME = 'logs' 126 127# Used to extract a mapping between directories and components. 128COMPONENT_MAPPING_URL = ( 129 'https://storage.googleapis.com/chromium-owners/component_map.json') 130 131# Caches the results returned by _GetBuildArgs, don't use this variable 132# directly, call _GetBuildArgs instead. 133_BUILD_ARGS = None 134 135# Retry failed merges. 136MERGE_RETRIES = 3 137 138# Message to guide user to file a bug when everything else fails. 139FILE_BUG_MESSAGE = ( 140 'If it persists, please file a bug with the command you used, git revision ' 141 'and args.gn config here: ' 142 'https://bugs.chromium.org/p/chromium/issues/entry?' 143 'components=Infra%3ETest%3ECodeCoverage') 144 145# String to replace with actual llvm profile path. 146LLVM_PROFILE_FILE_PATH_SUBSTITUTION = '<llvm_profile_file_path>' 147 148def _ConfigureLLVMCoverageTools(args): 149 """Configures llvm coverage tools.""" 150 if args.coverage_tools_dir: 151 llvm_bin_dir = coverage_utils.GetFullPath(args.coverage_tools_dir) 152 global LLVM_COV_PATH 153 global LLVM_PROFDATA_PATH 154 LLVM_COV_PATH = os.path.join(llvm_bin_dir, 'llvm-cov') 155 LLVM_PROFDATA_PATH = os.path.join(llvm_bin_dir, 'llvm-profdata') 156 else: 157 subprocess.check_call( 158 ['tools/clang/scripts/update.py', '--package', 'coverage_tools']) 159 160 if coverage_utils.GetHostPlatform() == 'win': 161 LLVM_COV_PATH += '.exe' 162 LLVM_PROFDATA_PATH += '.exe' 163 164 coverage_tools_exist = ( 165 os.path.exists(LLVM_COV_PATH) and os.path.exists(LLVM_PROFDATA_PATH)) 166 assert coverage_tools_exist, ('Cannot find coverage tools, please make sure ' 167 'both \'%s\' and \'%s\' exist.') % ( 168 LLVM_COV_PATH, LLVM_PROFDATA_PATH) 169 170 171def _GetPathWithLLVMSymbolizerDir(): 172 """Add llvm-symbolizer directory to path for symbolized stacks.""" 173 path = os.getenv('PATH') 174 dirs = path.split(os.pathsep) 175 if LLVM_BIN_DIR in dirs: 176 return path 177 178 return path + os.pathsep + LLVM_BIN_DIR 179 180 181def _GetTargetOS(): 182 """Returns the target os specified in args.gn file. 183 184 Returns an empty string is target_os is not specified. 185 """ 186 build_args = _GetBuildArgs() 187 return build_args['target_os'] if 'target_os' in build_args else '' 188 189 190def _IsIOS(): 191 """Returns true if the target_os specified in args.gn file is ios""" 192 return _GetTargetOS() == 'ios' 193 194 195def _GeneratePerFileLineByLineCoverageInFormat(binary_paths, profdata_file_path, 196 filters, ignore_filename_regex, 197 output_format): 198 """Generates per file line-by-line coverage in html or text using 199 'llvm-cov show'. 200 201 For a file with absolute path /a/b/x.cc, a html/txt report is generated as: 202 OUTPUT_DIR/coverage/a/b/x.cc.[html|txt]. For html format, an index html file 203 is also generated as: OUTPUT_DIR/index.html. 204 205 Args: 206 binary_paths: A list of paths to the instrumented binaries. 207 profdata_file_path: A path to the profdata file. 208 filters: A list of directories and files to get coverage for. 209 ignore_filename_regex: A regular expression for skipping source code files 210 with certain file paths. 211 output_format: The output format of generated report files. 212 """ 213 # llvm-cov show [options] -instr-profile PROFILE BIN [-object BIN,...] 214 # [[-object BIN]] [SOURCES] 215 # NOTE: For object files, the first one is specified as a positional argument, 216 # and the rest are specified as keyword argument. 217 logging.debug('Generating per file line by line coverage reports using ' 218 '"llvm-cov show" command.') 219 220 subprocess_cmd = [ 221 LLVM_COV_PATH, 'show', '-format={}'.format(output_format), 222 '-output-dir={}'.format(OUTPUT_DIR), 223 '-instr-profile={}'.format(profdata_file_path), binary_paths[0] 224 ] 225 subprocess_cmd.extend( 226 ['-object=' + binary_path for binary_path in binary_paths[1:]]) 227 _AddArchArgumentForIOSIfNeeded(subprocess_cmd, len(binary_paths)) 228 if coverage_utils.GetHostPlatform() in ['linux', 'mac']: 229 subprocess_cmd.extend(['-Xdemangler', 'c++filt', '-Xdemangler', '-n']) 230 subprocess_cmd.extend(filters) 231 if ignore_filename_regex: 232 subprocess_cmd.append('-ignore-filename-regex=%s' % ignore_filename_regex) 233 234 subprocess.check_call(subprocess_cmd) 235 236 logging.debug('Finished running "llvm-cov show" command.') 237 238 239def _GetLogsDirectoryPath(): 240 """Path to the logs directory.""" 241 return os.path.join( 242 coverage_utils.GetCoverageReportRootDirPath(OUTPUT_DIR), LOGS_DIR_NAME) 243 244 245def _GetProfdataFilePath(): 246 """Path to the resulting .profdata file.""" 247 return os.path.join( 248 coverage_utils.GetCoverageReportRootDirPath(OUTPUT_DIR), 249 PROFDATA_FILE_NAME) 250 251 252def _GetSummaryFilePath(): 253 """The JSON file that contains coverage summary written by llvm-cov export.""" 254 return os.path.join( 255 coverage_utils.GetCoverageReportRootDirPath(OUTPUT_DIR), 256 SUMMARY_FILE_NAME) 257 258 259def _CreateCoverageProfileDataForTargets(targets, commands, jobs_count=None): 260 """Builds and runs target to generate the coverage profile data. 261 262 Args: 263 targets: A list of targets to build with coverage instrumentation. 264 commands: A list of commands used to run the targets. 265 jobs_count: Number of jobs to run in parallel for building. If None, a 266 default value is derived based on CPUs availability. 267 268 Returns: 269 A relative path to the generated profdata file. 270 """ 271 _BuildTargets(targets, jobs_count) 272 target_profdata_file_paths = _GetTargetProfDataPathsByExecutingCommands( 273 targets, commands) 274 coverage_profdata_file_path = ( 275 _CreateCoverageProfileDataFromTargetProfDataFiles( 276 target_profdata_file_paths)) 277 278 for target_profdata_file_path in target_profdata_file_paths: 279 os.remove(target_profdata_file_path) 280 281 return coverage_profdata_file_path 282 283 284def _BuildTargets(targets, jobs_count): 285 """Builds target with Clang coverage instrumentation. 286 287 This function requires current working directory to be the root of checkout. 288 289 Args: 290 targets: A list of targets to build with coverage instrumentation. 291 jobs_count: Number of jobs to run in parallel for compilation. If None, a 292 default value is derived based on CPUs availability. 293 """ 294 logging.info('Building %s.', str(targets)) 295 autoninja = 'autoninja' 296 if coverage_utils.GetHostPlatform() == 'win': 297 autoninja += '.bat' 298 299 subprocess_cmd = [autoninja, '-C', BUILD_DIR] 300 if jobs_count is not None: 301 subprocess_cmd.append('-j' + str(jobs_count)) 302 303 subprocess_cmd.extend(targets) 304 subprocess.check_call(subprocess_cmd) 305 logging.debug('Finished building %s.', str(targets)) 306 307 308def _GetTargetProfDataPathsByExecutingCommands(targets, commands): 309 """Runs commands and returns the relative paths to the profraw data files. 310 311 Args: 312 targets: A list of targets built with coverage instrumentation. 313 commands: A list of commands used to run the targets. 314 315 Returns: 316 A list of relative paths to the generated profraw data files. 317 """ 318 logging.debug('Executing the test commands.') 319 320 # Remove existing profraw data files. 321 report_root_dir = coverage_utils.GetCoverageReportRootDirPath(OUTPUT_DIR) 322 for file_or_dir in os.listdir(report_root_dir): 323 if file_or_dir.endswith(PROFRAW_FILE_EXTENSION): 324 os.remove(os.path.join(report_root_dir, file_or_dir)) 325 326 # Ensure that logs directory exists. 327 if not os.path.exists(_GetLogsDirectoryPath()): 328 os.makedirs(_GetLogsDirectoryPath()) 329 330 profdata_file_paths = [] 331 332 # Run all test targets to generate profraw data files. 333 for target, command in zip(targets, commands): 334 output_file_name = os.extsep.join([target + '_output', 'log']) 335 output_file_path = os.path.join(_GetLogsDirectoryPath(), output_file_name) 336 337 profdata_file_path = None 338 for _ in xrange(MERGE_RETRIES): 339 logging.info('Running command: "%s", the output is redirected to "%s".', 340 command, output_file_path) 341 342 if _IsIOSCommand(command): 343 # On iOS platform, due to lack of write permissions, profraw files are 344 # generated outside of the OUTPUT_DIR, and the exact paths are contained 345 # in the output of the command execution. 346 output = _ExecuteIOSCommand(command, output_file_path) 347 else: 348 # On other platforms, profraw files are generated inside the OUTPUT_DIR. 349 output = _ExecuteCommand(target, command, output_file_path) 350 351 profraw_file_paths = [] 352 if _IsIOS(): 353 profraw_file_paths = [_GetProfrawDataFileByParsingOutput(output)] 354 else: 355 for file_or_dir in os.listdir(report_root_dir): 356 if file_or_dir.endswith(PROFRAW_FILE_EXTENSION): 357 profraw_file_paths.append( 358 os.path.join(report_root_dir, file_or_dir)) 359 360 assert profraw_file_paths, ( 361 'Running target "%s" failed to generate any profraw data file, ' 362 'please make sure the binary exists, is properly instrumented and ' 363 'does not crash. %s' % (target, FILE_BUG_MESSAGE)) 364 365 assert isinstance(profraw_file_paths, list), ( 366 'Variable \'profraw_file_paths\' is expected to be of type \'list\', ' 367 'but it is a %s. %s' % (type(profraw_file_paths), FILE_BUG_MESSAGE)) 368 369 try: 370 profdata_file_path = _CreateTargetProfDataFileFromProfRawFiles( 371 target, profraw_file_paths) 372 break 373 except Exception: 374 logging.info('Retrying...') 375 finally: 376 # Remove profraw files now so that they are not used in next iteration. 377 for profraw_file_path in profraw_file_paths: 378 os.remove(profraw_file_path) 379 380 assert profdata_file_path, ( 381 'Failed to merge target "%s" profraw files after %d retries. %s' % 382 (target, MERGE_RETRIES, FILE_BUG_MESSAGE)) 383 profdata_file_paths.append(profdata_file_path) 384 385 logging.debug('Finished executing the test commands.') 386 387 return profdata_file_paths 388 389 390def _GetEnvironmentVars(profraw_file_path): 391 """Return environment vars for subprocess, given a profraw file path.""" 392 env = os.environ.copy() 393 env.update({ 394 'LLVM_PROFILE_FILE': profraw_file_path, 395 'PATH': _GetPathWithLLVMSymbolizerDir() 396 }) 397 return env 398 399 400def _SplitCommand(command): 401 """Split a command string into parts in a platform-specific way.""" 402 if coverage_utils.GetHostPlatform() == 'win': 403 return command.split() 404 return shlex.split(command) 405 406 407def _ExecuteCommand(target, command, output_file_path): 408 """Runs a single command and generates a profraw data file.""" 409 # Per Clang "Source-based Code Coverage" doc: 410 # 411 # "%p" expands out to the process ID. It's not used by this scripts due to: 412 # 1) If a target program spawns too many processess, it may exhaust all disk 413 # space available. For example, unit_tests writes thousands of .profraw 414 # files each of size 1GB+. 415 # 2) If a target binary uses shared libraries, coverage profile data for them 416 # will be missing, resulting in incomplete coverage reports. 417 # 418 # "%Nm" expands out to the instrumented binary's signature. When this pattern 419 # is specified, the runtime creates a pool of N raw profiles which are used 420 # for on-line profile merging. The runtime takes care of selecting a raw 421 # profile from the pool, locking it, and updating it before the program exits. 422 # N must be between 1 and 9. The merge pool specifier can only occur once per 423 # filename pattern. 424 # 425 # "%1m" is used when tests run in single process, such as fuzz targets. 426 # 427 # For other cases, "%4m" is chosen as it creates some level of parallelism, 428 # but it's not too big to consume too much computing resource or disk space. 429 profile_pattern_string = '%1m' if _IsFuzzerTarget(target) else '%4m' 430 expected_profraw_file_name = os.extsep.join( 431 [target, profile_pattern_string, PROFRAW_FILE_EXTENSION]) 432 expected_profraw_file_path = os.path.join( 433 coverage_utils.GetCoverageReportRootDirPath(OUTPUT_DIR), 434 expected_profraw_file_name) 435 command = command.replace(LLVM_PROFILE_FILE_PATH_SUBSTITUTION, 436 expected_profraw_file_path) 437 438 try: 439 # Some fuzz targets or tests may write into stderr, redirect it as well. 440 with open(output_file_path, 'wb') as output_file_handle: 441 subprocess.check_call(_SplitCommand(command), 442 stdout=output_file_handle, 443 stderr=subprocess.STDOUT, 444 env=_GetEnvironmentVars(expected_profraw_file_path)) 445 except subprocess.CalledProcessError as e: 446 logging.warning('Command: "%s" exited with non-zero return code.', command) 447 448 return open(output_file_path, 'rb').read() 449 450 451def _IsFuzzerTarget(target): 452 """Returns true if the target is a fuzzer target.""" 453 build_args = _GetBuildArgs() 454 use_libfuzzer = ('use_libfuzzer' in build_args and 455 build_args['use_libfuzzer'] == 'true') 456 return use_libfuzzer and target.endswith('_fuzzer') 457 458 459def _ExecuteIOSCommand(command, output_file_path): 460 """Runs a single iOS command and generates a profraw data file. 461 462 iOS application doesn't have write access to folders outside of the app, so 463 it's impossible to instruct the app to flush the profraw data file to the 464 desired location. The profraw data file will be generated somewhere within the 465 application's Documents folder, and the full path can be obtained by parsing 466 the output. 467 """ 468 assert _IsIOSCommand(command) 469 470 # After running tests, iossim generates a profraw data file, it won't be 471 # needed anyway, so dump it into the OUTPUT_DIR to avoid polluting the 472 # checkout. 473 iossim_profraw_file_path = os.path.join( 474 OUTPUT_DIR, os.extsep.join(['iossim', PROFRAW_FILE_EXTENSION])) 475 command = command.replace(LLVM_PROFILE_FILE_PATH_SUBSTITUTION, 476 iossim_profraw_file_path) 477 478 try: 479 with open(output_file_path, 'wb') as output_file_handle: 480 subprocess.check_call(_SplitCommand(command), 481 stdout=output_file_handle, 482 stderr=subprocess.STDOUT, 483 env=_GetEnvironmentVars(iossim_profraw_file_path)) 484 except subprocess.CalledProcessError as e: 485 # iossim emits non-zero return code even if tests run successfully, so 486 # ignore the return code. 487 pass 488 489 return open(output_file_path, 'rb').read() 490 491 492def _GetProfrawDataFileByParsingOutput(output): 493 """Returns the path to the profraw data file obtained by parsing the output. 494 495 The output of running the test target has no format, but it is guaranteed to 496 have a single line containing the path to the generated profraw data file. 497 NOTE: This should only be called when target os is iOS. 498 """ 499 assert _IsIOS() 500 501 output_by_lines = ''.join(output).splitlines() 502 profraw_file_pattern = re.compile('.*Coverage data at (.*coverage\.profraw).') 503 504 for line in output_by_lines: 505 result = profraw_file_pattern.match(line) 506 if result: 507 return result.group(1) 508 509 assert False, ('No profraw data file was generated, did you call ' 510 'coverage_util::ConfigureCoverageReportPath() in test setup? ' 511 'Please refer to base/test/test_support_ios.mm for example.') 512 513 514def _CreateCoverageProfileDataFromTargetProfDataFiles(profdata_file_paths): 515 """Returns a relative path to coverage profdata file by merging target 516 profdata files. 517 518 Args: 519 profdata_file_paths: A list of relative paths to the profdata data files 520 that are to be merged. 521 522 Returns: 523 A relative path to the merged coverage profdata file. 524 525 Raises: 526 CalledProcessError: An error occurred merging profdata files. 527 """ 528 logging.info('Creating the coverage profile data file.') 529 logging.debug('Merging target profraw files to create target profdata file.') 530 profdata_file_path = _GetProfdataFilePath() 531 try: 532 subprocess_cmd = [ 533 LLVM_PROFDATA_PATH, 'merge', '-o', profdata_file_path, '-sparse=true' 534 ] 535 subprocess_cmd.extend(profdata_file_paths) 536 537 output = subprocess.check_output(subprocess_cmd) 538 logging.debug('Merge output: %s', output) 539 except subprocess.CalledProcessError as error: 540 logging.error( 541 'Failed to merge target profdata files to create coverage profdata. %s', 542 FILE_BUG_MESSAGE) 543 raise error 544 545 logging.debug('Finished merging target profdata files.') 546 logging.info('Code coverage profile data is created as: "%s".', 547 profdata_file_path) 548 return profdata_file_path 549 550 551def _CreateTargetProfDataFileFromProfRawFiles(target, profraw_file_paths): 552 """Returns a relative path to target profdata file by merging target 553 profraw files. 554 555 Args: 556 profraw_file_paths: A list of relative paths to the profdata data files 557 that are to be merged. 558 559 Returns: 560 A relative path to the merged coverage profdata file. 561 562 Raises: 563 CalledProcessError: An error occurred merging profdata files. 564 """ 565 logging.info('Creating target profile data file.') 566 logging.debug('Merging target profraw files to create target profdata file.') 567 profdata_file_path = os.path.join(OUTPUT_DIR, '%s.profdata' % target) 568 569 try: 570 subprocess_cmd = [ 571 LLVM_PROFDATA_PATH, 'merge', '-o', profdata_file_path, '-sparse=true' 572 ] 573 subprocess_cmd.extend(profraw_file_paths) 574 575 output = subprocess.check_output(subprocess_cmd) 576 logging.debug('Merge output: %s', output) 577 except subprocess.CalledProcessError as error: 578 logging.error( 579 'Failed to merge target profraw files to create target profdata.') 580 raise error 581 582 logging.debug('Finished merging target profraw files.') 583 logging.info('Target "%s" profile data is created as: "%s".', target, 584 profdata_file_path) 585 return profdata_file_path 586 587 588def _GeneratePerFileCoverageSummary(binary_paths, profdata_file_path, filters, 589 ignore_filename_regex): 590 """Generates per file coverage summary using "llvm-cov export" command.""" 591 # llvm-cov export [options] -instr-profile PROFILE BIN [-object BIN,...] 592 # [[-object BIN]] [SOURCES]. 593 # NOTE: For object files, the first one is specified as a positional argument, 594 # and the rest are specified as keyword argument. 595 logging.debug('Generating per-file code coverage summary using "llvm-cov ' 596 'export -summary-only" command.') 597 for path in binary_paths: 598 if not os.path.exists(path): 599 logging.error("Binary %s does not exist", path) 600 subprocess_cmd = [ 601 LLVM_COV_PATH, 'export', '-summary-only', 602 '-instr-profile=' + profdata_file_path, binary_paths[0] 603 ] 604 subprocess_cmd.extend( 605 ['-object=' + binary_path for binary_path in binary_paths[1:]]) 606 _AddArchArgumentForIOSIfNeeded(subprocess_cmd, len(binary_paths)) 607 subprocess_cmd.extend(filters) 608 if ignore_filename_regex: 609 subprocess_cmd.append('-ignore-filename-regex=%s' % ignore_filename_regex) 610 611 export_output = subprocess.check_output(subprocess_cmd) 612 613 # Write output on the disk to be used by code coverage bot. 614 with open(_GetSummaryFilePath(), 'w') as f: 615 f.write(export_output) 616 617 return export_output 618 619 620def _AddArchArgumentForIOSIfNeeded(cmd_list, num_archs): 621 """Appends -arch arguments to the command list if it's ios platform. 622 623 iOS binaries are universal binaries, and require specifying the architecture 624 to use, and one architecture needs to be specified for each binary. 625 """ 626 if _IsIOS(): 627 cmd_list.extend(['-arch=x86_64'] * num_archs) 628 629 630def _GetBinaryPath(command): 631 """Returns a relative path to the binary to be run by the command. 632 633 Currently, following types of commands are supported (e.g. url_unittests): 634 1. Run test binary direcly: "out/coverage/url_unittests <arguments>" 635 2. Use xvfb. 636 2.1. "python testing/xvfb.py out/coverage/url_unittests <arguments>" 637 2.2. "testing/xvfb.py out/coverage/url_unittests <arguments>" 638 3. Use iossim to run tests on iOS platform, please refer to testing/iossim.mm 639 for its usage. 640 3.1. "out/Coverage-iphonesimulator/iossim 641 <iossim_arguments> -c <app_arguments> 642 out/Coverage-iphonesimulator/url_unittests.app" 643 644 Args: 645 command: A command used to run a target. 646 647 Returns: 648 A relative path to the binary. 649 """ 650 xvfb_script_name = os.extsep.join(['xvfb', 'py']) 651 652 command_parts = _SplitCommand(command) 653 if os.path.basename(command_parts[0]) == 'python': 654 assert os.path.basename(command_parts[1]) == xvfb_script_name, ( 655 'This tool doesn\'t understand the command: "%s".' % command) 656 return command_parts[2] 657 658 if os.path.basename(command_parts[0]) == xvfb_script_name: 659 return command_parts[1] 660 661 if _IsIOSCommand(command): 662 # For a given application bundle, the binary resides in the bundle and has 663 # the same name with the application without the .app extension. 664 app_path = command_parts[1].rstrip(os.path.sep) 665 app_name = os.path.splitext(os.path.basename(app_path))[0] 666 return os.path.join(app_path, app_name) 667 668 if coverage_utils.GetHostPlatform() == 'win' \ 669 and not command_parts[0].endswith('.exe'): 670 return command_parts[0] + '.exe' 671 672 return command_parts[0] 673 674 675def _IsIOSCommand(command): 676 """Returns true if command is used to run tests on iOS platform.""" 677 return os.path.basename(_SplitCommand(command)[0]) == 'iossim' 678 679 680def _VerifyTargetExecutablesAreInBuildDirectory(commands): 681 """Verifies that the target executables specified in the commands are inside 682 the given build directory.""" 683 for command in commands: 684 binary_path = _GetBinaryPath(command) 685 binary_absolute_path = coverage_utils.GetFullPath(binary_path) 686 assert binary_absolute_path.startswith(BUILD_DIR + os.sep), ( 687 'Target executable "%s" in command: "%s" is outside of ' 688 'the given build directory: "%s".' % (binary_path, command, BUILD_DIR)) 689 690 691def _ValidateBuildingWithClangCoverage(): 692 """Asserts that targets are built with Clang coverage enabled.""" 693 build_args = _GetBuildArgs() 694 695 if (CLANG_COVERAGE_BUILD_ARG not in build_args or 696 build_args[CLANG_COVERAGE_BUILD_ARG] != 'true'): 697 assert False, ('\'{} = true\' is required in args.gn.' 698 ).format(CLANG_COVERAGE_BUILD_ARG) 699 700 701def _ValidateCurrentPlatformIsSupported(): 702 """Asserts that this script suports running on the current platform""" 703 target_os = _GetTargetOS() 704 if target_os: 705 current_platform = target_os 706 else: 707 current_platform = coverage_utils.GetHostPlatform() 708 709 assert current_platform in [ 710 'linux', 'mac', 'chromeos', 'ios', 'win' 711 ], ('Coverage is only supported on linux, mac, chromeos, ios and win.') 712 713 714def _GetBuildArgs(): 715 """Parses args.gn file and returns results as a dictionary. 716 717 Returns: 718 A dictionary representing the build args. 719 """ 720 global _BUILD_ARGS 721 if _BUILD_ARGS is not None: 722 return _BUILD_ARGS 723 724 _BUILD_ARGS = {} 725 build_args_path = os.path.join(BUILD_DIR, 'args.gn') 726 assert os.path.exists(build_args_path), ('"%s" is not a build directory, ' 727 'missing args.gn file.' % BUILD_DIR) 728 with open(build_args_path) as build_args_file: 729 build_args_lines = build_args_file.readlines() 730 731 for build_arg_line in build_args_lines: 732 build_arg_without_comments = build_arg_line.split('#')[0] 733 key_value_pair = build_arg_without_comments.split('=') 734 if len(key_value_pair) != 2: 735 continue 736 737 key = key_value_pair[0].strip() 738 739 # Values are wrapped within a pair of double-quotes, so remove the leading 740 # and trailing double-quotes. 741 value = key_value_pair[1].strip().strip('"') 742 _BUILD_ARGS[key] = value 743 744 return _BUILD_ARGS 745 746 747def _VerifyPathsAndReturnAbsolutes(paths): 748 """Verifies that the paths specified in |paths| exist and returns absolute 749 versions. 750 751 Args: 752 paths: A list of files or directories. 753 """ 754 absolute_paths = [] 755 for path in paths: 756 absolute_path = os.path.join(SRC_ROOT_PATH, path) 757 assert os.path.exists(absolute_path), ('Path: "%s" doesn\'t exist.' % path) 758 759 absolute_paths.append(absolute_path) 760 761 return absolute_paths 762 763 764def _GetBinaryPathsFromTargets(targets, build_dir): 765 """Return binary paths from target names.""" 766 # FIXME: Derive output binary from target build definitions rather than 767 # assuming that it is always the same name. 768 binary_paths = [] 769 for target in targets: 770 binary_path = os.path.join(build_dir, target) 771 if coverage_utils.GetHostPlatform() == 'win': 772 binary_path += '.exe' 773 774 if os.path.exists(binary_path): 775 binary_paths.append(binary_path) 776 else: 777 logging.warning( 778 'Target binary "%s" not found in build directory, skipping.', 779 os.path.basename(binary_path)) 780 781 return binary_paths 782 783 784def _GetCommandForWebTests(arguments): 785 """Return command to run for blink web tests.""" 786 command_list = [ 787 'python', 'testing/xvfb.py', 'python', 788 'third_party/blink/tools/run_web_tests.py', 789 '--additional-driver-flag=--no-sandbox', 790 '--additional-env-var=LLVM_PROFILE_FILE=%s' % 791 LLVM_PROFILE_FILE_PATH_SUBSTITUTION, 792 '--child-processes=%d' % max(1, int(multiprocessing.cpu_count() / 2)), 793 '--disable-breakpad', '--no-show-results', '--skip-failing-tests', 794 '--target=%s' % os.path.basename(BUILD_DIR), '--time-out-ms=30000' 795 ] 796 if arguments.strip(): 797 command_list.append(arguments) 798 return ' '.join(command_list) 799 800 801def _GetBinaryPathForWebTests(): 802 """Return binary path used to run blink web tests.""" 803 host_platform = coverage_utils.GetHostPlatform() 804 if host_platform == 'win': 805 return os.path.join(BUILD_DIR, 'content_shell.exe') 806 elif host_platform == 'linux': 807 return os.path.join(BUILD_DIR, 'content_shell') 808 elif host_platform == 'mac': 809 return os.path.join(BUILD_DIR, 'Content Shell.app', 'Contents', 'MacOS', 810 'Content Shell') 811 else: 812 assert False, 'This platform is not supported for web tests.' 813 814 815def _SetupOutputDir(): 816 """Setup output directory.""" 817 if os.path.exists(OUTPUT_DIR): 818 shutil.rmtree(OUTPUT_DIR) 819 820 # Creates |OUTPUT_DIR| and its platform sub-directory. 821 os.makedirs(coverage_utils.GetCoverageReportRootDirPath(OUTPUT_DIR)) 822 823 824def _SetMacXcodePath(): 825 """Set DEVELOPER_DIR to the path to hermetic Xcode.app on Mac OS X.""" 826 if sys.platform != 'darwin': 827 return 828 829 xcode_path = os.path.join(SRC_ROOT_PATH, 'build', 'mac_files', 'Xcode.app') 830 if os.path.exists(xcode_path): 831 os.environ['DEVELOPER_DIR'] = xcode_path 832 833 834def _ParseCommandArguments(): 835 """Adds and parses relevant arguments for tool comands. 836 837 Returns: 838 A dictionary representing the arguments. 839 """ 840 arg_parser = argparse.ArgumentParser() 841 arg_parser.usage = __doc__ 842 843 arg_parser.add_argument( 844 '-b', 845 '--build-dir', 846 type=str, 847 required=True, 848 help='The build directory, the path needs to be relative to the root of ' 849 'the checkout.') 850 851 arg_parser.add_argument( 852 '-o', 853 '--output-dir', 854 type=str, 855 required=True, 856 help='Output directory for generated artifacts.') 857 858 arg_parser.add_argument( 859 '-c', 860 '--command', 861 action='append', 862 required=False, 863 help='Commands used to run test targets, one test target needs one and ' 864 'only one command, when specifying commands, one should assume the ' 865 'current working directory is the root of the checkout. This option is ' 866 'incompatible with -p/--profdata-file option.') 867 868 arg_parser.add_argument( 869 '-wt', 870 '--web-tests', 871 nargs='?', 872 type=str, 873 const=' ', 874 required=False, 875 help='Run blink web tests. Support passing arguments to run_web_tests.py') 876 877 arg_parser.add_argument( 878 '-p', 879 '--profdata-file', 880 type=str, 881 required=False, 882 help='Path to profdata file to use for generating code coverage reports. ' 883 'This can be useful if you generated the profdata file seperately in ' 884 'your own test harness. This option is ignored if run command(s) are ' 885 'already provided above using -c/--command option.') 886 887 arg_parser.add_argument( 888 '-f', 889 '--filters', 890 action='append', 891 required=False, 892 help='Directories or files to get code coverage for, and all files under ' 893 'the directories are included recursively.') 894 895 arg_parser.add_argument( 896 '-i', 897 '--ignore-filename-regex', 898 type=str, 899 help='Skip source code files with file paths that match the given ' 900 'regular expression. For example, use -i=\'.*/out/.*|.*/third_party/.*\' ' 901 'to exclude files in third_party/ and out/ folders from the report.') 902 903 arg_parser.add_argument( 904 '--no-file-view', 905 action='store_true', 906 help='Don\'t generate the file view in the coverage report. When there ' 907 'are large number of html files, the file view becomes heavy and may ' 908 'cause the browser to freeze, and this argument comes handy.') 909 910 arg_parser.add_argument( 911 '--no-component-view', 912 action='store_true', 913 help='Don\'t generate the component view in the coverage report.') 914 915 arg_parser.add_argument( 916 '--coverage-tools-dir', 917 type=str, 918 help='Path of the directory where LLVM coverage tools (llvm-cov, ' 919 'llvm-profdata) exist. This should be only needed if you are testing ' 920 'against a custom built clang revision. Otherwise, we pick coverage ' 921 'tools automatically from your current source checkout.') 922 923 arg_parser.add_argument( 924 '-j', 925 '--jobs', 926 type=int, 927 default=None, 928 help='Run N jobs to build in parallel. If not specified, a default value ' 929 'will be derived based on CPUs and goma availability. Please refer to ' 930 '\'autoninja -h\' for more details.') 931 932 arg_parser.add_argument( 933 '--format', 934 type=str, 935 default='html', 936 help='Output format of the "llvm-cov show" command. The supported ' 937 'formats are "text" and "html".') 938 939 arg_parser.add_argument( 940 '-v', 941 '--verbose', 942 action='store_true', 943 help='Prints additional output for diagnostics.') 944 945 arg_parser.add_argument( 946 '-l', '--log_file', type=str, help='Redirects logs to a file.') 947 948 arg_parser.add_argument( 949 'targets', 950 nargs='+', 951 help='The names of the test targets to run. If multiple run commands are ' 952 'specified using the -c/--command option, then the order of targets and ' 953 'commands must match, otherwise coverage generation will fail.') 954 955 args = arg_parser.parse_args() 956 return args 957 958 959def Main(): 960 """Execute tool commands.""" 961 962 # Change directory to source root to aid in relative paths calculations. 963 os.chdir(SRC_ROOT_PATH) 964 965 # Setup coverage binaries even when script is called with empty params. This 966 # is used by coverage bot for initial setup. 967 if len(sys.argv) == 1: 968 subprocess.check_call( 969 ['tools/clang/scripts/update.py', '--package', 'coverage_tools']) 970 print(__doc__) 971 return 972 973 args = _ParseCommandArguments() 974 coverage_utils.ConfigureLogging(verbose=args.verbose, log_file=args.log_file) 975 _ConfigureLLVMCoverageTools(args) 976 977 global BUILD_DIR 978 BUILD_DIR = coverage_utils.GetFullPath(args.build_dir) 979 980 global OUTPUT_DIR 981 OUTPUT_DIR = coverage_utils.GetFullPath(args.output_dir) 982 983 assert args.web_tests or args.command or args.profdata_file, ( 984 'Need to either provide commands to run using -c/--command option OR ' 985 'provide prof-data file as input using -p/--profdata-file option OR ' 986 'run web tests using -wt/--run-web-tests.') 987 988 assert not args.command or (len(args.targets) == len(args.command)), ( 989 'Number of targets must be equal to the number of test commands.') 990 991 assert os.path.exists(BUILD_DIR), ( 992 'Build directory: "%s" doesn\'t exist. ' 993 'Please run "gn gen" to generate.' % BUILD_DIR) 994 995 _ValidateCurrentPlatformIsSupported() 996 _ValidateBuildingWithClangCoverage() 997 998 absolute_filter_paths = [] 999 if args.filters: 1000 absolute_filter_paths = _VerifyPathsAndReturnAbsolutes(args.filters) 1001 1002 _SetupOutputDir() 1003 1004 # Get .profdata file and list of binary paths. 1005 if args.web_tests: 1006 commands = [_GetCommandForWebTests(args.web_tests)] 1007 profdata_file_path = _CreateCoverageProfileDataForTargets( 1008 args.targets, commands, args.jobs) 1009 binary_paths = [_GetBinaryPathForWebTests()] 1010 elif args.command: 1011 for i in range(len(args.command)): 1012 assert not 'run_web_tests.py' in args.command[i], ( 1013 'run_web_tests.py is not supported via --command argument. ' 1014 'Please use --run-web-tests argument instead.') 1015 1016 # A list of commands are provided. Run them to generate profdata file, and 1017 # create a list of binary paths from parsing commands. 1018 _VerifyTargetExecutablesAreInBuildDirectory(args.command) 1019 profdata_file_path = _CreateCoverageProfileDataForTargets( 1020 args.targets, args.command, args.jobs) 1021 binary_paths = [_GetBinaryPath(command) for command in args.command] 1022 else: 1023 # An input prof-data file is already provided. Just calculate binary paths. 1024 profdata_file_path = args.profdata_file 1025 binary_paths = _GetBinaryPathsFromTargets(args.targets, args.build_dir) 1026 1027 # If the checkout uses the hermetic xcode binaries, then otool must be 1028 # directly invoked. The indirection via /usr/bin/otool won't work unless 1029 # there's an actual system install of Xcode. 1030 otool_path = None 1031 if sys.platform == 'darwin': 1032 hermetic_otool_path = os.path.join( 1033 SRC_ROOT_PATH, 'build', 'mac_files', 'xcode_binaries', 'Contents', 1034 'Developer', 'Toolchains', 'XcodeDefault.xctoolchain', 'usr', 'bin', 1035 'otool') 1036 if os.path.exists(hermetic_otool_path): 1037 otool_path = hermetic_otool_path 1038 if sys.platform.startswith('linux') or sys.platform.startswith('darwin'): 1039 binary_paths.extend( 1040 coverage_utils.GetSharedLibraries(binary_paths, BUILD_DIR, otool_path)) 1041 1042 assert args.format == 'html' or args.format == 'text', ( 1043 '%s is not a valid output format for "llvm-cov show". Only "text" and ' 1044 '"html" formats are supported.' % (args.format)) 1045 logging.info('Generating code coverage report in %s (this can take a while ' 1046 'depending on size of target!).' % (args.format)) 1047 per_file_summary_data = _GeneratePerFileCoverageSummary( 1048 binary_paths, profdata_file_path, absolute_filter_paths, 1049 args.ignore_filename_regex) 1050 _GeneratePerFileLineByLineCoverageInFormat( 1051 binary_paths, profdata_file_path, absolute_filter_paths, 1052 args.ignore_filename_regex, args.format) 1053 component_mappings = None 1054 if not args.no_component_view: 1055 component_mappings = json.load(urllib2.urlopen(COMPONENT_MAPPING_URL)) 1056 1057 # Call prepare here. 1058 processor = coverage_utils.CoverageReportPostProcessor( 1059 OUTPUT_DIR, 1060 SRC_ROOT_PATH, 1061 per_file_summary_data, 1062 no_component_view=args.no_component_view, 1063 no_file_view=args.no_file_view, 1064 component_mappings=component_mappings) 1065 1066 if args.format == 'html': 1067 processor.PrepareHtmlReport() 1068 1069 1070if __name__ == '__main__': 1071 sys.exit(Main()) 1072