1#!/usr/bin/env python 2# Copyright 2017 The PDFium Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5"""Generates a coverage report for given tests. 6 7Requires that 'use_clang_coverage = true' is set in args.gn. 8Prefers that 'is_component_build = false' is set in args.gn. 9""" 10 11import argparse 12from collections import namedtuple 13import fnmatch 14import os 15import pprint 16import subprocess 17import sys 18 19# Add src dir to path to avoid having to set PYTHONPATH. 20sys.path.append( 21 os.path.abspath( 22 os.path.join( 23 os.path.dirname(__file__), os.path.pardir, os.path.pardir, 24 os.path.pardir))) 25 26from testing.tools.common import GetBooleanGnArg 27 28# 'binary' is the file that is to be run for the test. 29# 'use_test_runner' indicates if 'binary' depends on test_runner.py and thus 30# requires special handling. 31# 'opt_args' are optional arguments to pass to the test 'binary'. 32TestSpec = namedtuple('TestSpec', 'binary, use_test_runner, opt_args') 33 34# All of the coverage tests that the script knows how to run. 35COVERAGE_TESTS = { 36 'pdfium_unittests': 37 TestSpec('pdfium_unittests', False, []), 38 'pdfium_embeddertests': 39 TestSpec('pdfium_embeddertests', False, []), 40 'corpus_tests': 41 TestSpec('run_corpus_tests.py', True, []), 42 'corpus_tests_javascript_disabled': 43 TestSpec('run_corpus_tests.py', True, ['--disable-javascript']), 44 'corpus_tests_xfa_disabled': 45 TestSpec('run_corpus_tests.py', True, ['--disable-xfa']), 46 'javascript_tests': 47 TestSpec('run_javascript_tests.py', True, []), 48 'javascript_tests_javascript_disabled': 49 TestSpec('run_javascript_tests.py', True, ['--disable-javascript']), 50 'javascript_tests_xfa_disabled': 51 TestSpec('run_javascript_tests.py', True, ['--disable-xfa']), 52 'pixel_tests': 53 TestSpec('run_pixel_tests.py', True, []), 54 'pixel_tests_javascript_disabled': 55 TestSpec('run_pixel_tests.py', True, ['--disable-javascript']), 56 'pixel_tests_xfa_disabled': 57 TestSpec('run_pixel_tests.py', True, ['--disable-xfa']), 58} 59 60 61class CoverageExecutor(object): 62 63 def __init__(self, parser, args): 64 """Initialize executor based on the current script environment 65 66 Args: 67 parser: argparse.ArgumentParser for handling improper inputs. 68 args: Dictionary of arguments passed into the calling script. 69 """ 70 self.dry_run = args['dry_run'] 71 self.verbose = args['verbose'] 72 73 self.source_directory = args['source_directory'] 74 if not os.path.isdir(self.source_directory): 75 parser.error("'%s' needs to be a directory" % self.source_directory) 76 77 self.llvm_directory = os.path.join(self.source_directory, 'third_party', 78 'llvm-build', 'Release+Asserts', 'bin') 79 if not os.path.isdir(self.llvm_directory): 80 parser.error("Cannot find LLVM bin directory , expected it to be in '%s'" 81 % self.llvm_directory) 82 83 self.build_directory = args['build_directory'] 84 if not os.path.isdir(self.build_directory): 85 parser.error("'%s' needs to be a directory" % self.build_directory) 86 87 (self.coverage_tests, 88 self.build_targets) = self.calculate_coverage_tests(args) 89 if not self.coverage_tests: 90 parser.error( 91 'No valid tests in set to be run. This is likely due to bad command ' 92 'line arguments') 93 94 if not GetBooleanGnArg('use_clang_coverage', self.build_directory, 95 self.verbose): 96 parser.error( 97 'use_clang_coverage does not appear to be set to true for build, but ' 98 'is needed') 99 100 self.use_goma = GetBooleanGnArg('use_goma', self.build_directory, 101 self.verbose) 102 103 self.output_directory = args['output_directory'] 104 if not os.path.exists(self.output_directory): 105 if not self.dry_run: 106 os.makedirs(self.output_directory) 107 elif not os.path.isdir(self.output_directory): 108 parser.error('%s exists, but is not a directory' % self.output_directory) 109 elif len(os.listdir(self.output_directory)) > 0: 110 parser.error('%s is not empty, cowardly refusing to continue' % 111 self.output_directory) 112 113 self.prof_data = os.path.join(self.output_directory, 'pdfium.profdata') 114 115 def check_output(self, args, dry_run=False, env=None): 116 """Dry run aware wrapper of subprocess.check_output()""" 117 if dry_run: 118 print "Would have run '%s'" % ' '.join(args) 119 return '' 120 121 output = subprocess.check_output(args, env=env) 122 123 if self.verbose: 124 print "check_output(%s) returned '%s'" % (args, output) 125 return output 126 127 def call(self, args, dry_run=False, env=None): 128 """Dry run aware wrapper of subprocess.call()""" 129 if dry_run: 130 print "Would have run '%s'" % ' '.join(args) 131 return 0 132 133 output = subprocess.call(args, env=env) 134 135 if self.verbose: 136 print 'call(%s) returned %s' % (args, output) 137 return output 138 139 def call_silent(self, args, dry_run=False, env=None): 140 """Dry run aware wrapper of subprocess.call() that eats output from call""" 141 if dry_run: 142 print "Would have run '%s'" % ' '.join(args) 143 return 0 144 145 with open(os.devnull, 'w') as f: 146 output = subprocess.call(args, env=env, stdout=f) 147 148 if self.verbose: 149 print 'call_silent(%s) returned %s' % (args, output) 150 return output 151 152 def calculate_coverage_tests(self, args): 153 """Determine which tests should be run.""" 154 testing_tools_directory = os.path.join(self.source_directory, 'testing', 155 'tools') 156 tests = args['tests'] if args['tests'] else COVERAGE_TESTS.keys() 157 coverage_tests = {} 158 build_targets = set() 159 for name in tests: 160 test_spec = COVERAGE_TESTS[name] 161 if test_spec.use_test_runner: 162 binary_path = os.path.join(testing_tools_directory, test_spec.binary) 163 build_targets.add('pdfium_diff') 164 build_targets.add('pdfium_test') 165 else: 166 binary_path = os.path.join(self.build_directory, test_spec.binary) 167 build_targets.add(name) 168 coverage_tests[name] = TestSpec(binary_path, test_spec.use_test_runner, 169 test_spec.opt_args) 170 171 build_targets = list(build_targets) 172 173 return coverage_tests, build_targets 174 175 def build_binaries(self): 176 """Build all the binaries that are going to be needed for coverage 177 generation.""" 178 call_args = ['ninja'] 179 if self.use_goma: 180 call_args += ['-j', '250'] 181 call_args += ['-C', self.build_directory] 182 call_args += self.build_targets 183 return self.call(call_args, dry_run=self.dry_run) == 0 184 185 def generate_coverage(self, name, spec): 186 """Generate the coverage data for a test 187 188 Args: 189 name: Name associated with the test to be run. This is used as a label 190 in the coverage data, so should be unique across all of the tests 191 being run. 192 spec: Tuple containing the TestSpec. 193 """ 194 if self.verbose: 195 print "Generating coverage for test '%s', using data '%s'" % (name, spec) 196 if not os.path.exists(spec.binary): 197 print('Unable to generate coverage for %s, since it appears to not exist' 198 ' @ %s') % (name, spec.binary) 199 return False 200 201 binary_args = [spec.binary] 202 if spec.opt_args: 203 binary_args.extend(spec.opt_args) 204 profile_pattern_string = '%8m' 205 expected_profraw_file = '%s.%s.profraw' % (name, profile_pattern_string) 206 expected_profraw_path = os.path.join(self.output_directory, 207 expected_profraw_file) 208 209 env = { 210 'LLVM_PROFILE_FILE': expected_profraw_path, 211 'PATH': os.getenv('PATH') + os.pathsep + self.llvm_directory 212 } 213 214 if spec.use_test_runner: 215 # Test runner performs multi-threading in the wrapper script, not the test 216 # binary, so need to limit the number of instances of the binary being run 217 # to the max value in LLVM_PROFILE_FILE, which is 8. 218 binary_args.extend(['-j', '8', '--build-dir', self.build_directory]) 219 if self.call(binary_args, dry_run=self.dry_run, env=env) and self.verbose: 220 print('Running %s appears to have failed, which might affect ' 221 'results') % spec.binary 222 223 return True 224 225 def merge_raw_coverage_results(self): 226 """Merge raw coverage data sets into one one file for report generation.""" 227 llvm_profdata_bin = os.path.join(self.llvm_directory, 'llvm-profdata') 228 229 raw_data = [] 230 raw_data_pattern = '*.profraw' 231 for file_name in os.listdir(self.output_directory): 232 if fnmatch.fnmatch(file_name, raw_data_pattern): 233 raw_data.append(os.path.join(self.output_directory, file_name)) 234 235 return self.call( 236 [llvm_profdata_bin, 'merge', '-o', self.prof_data, '-sparse=true'] + 237 raw_data) == 0 238 239 def generate_html_report(self): 240 """Generate HTML report by calling upstream coverage.py""" 241 coverage_bin = os.path.join(self.source_directory, 'tools', 'code_coverage', 242 'coverage.py') 243 report_directory = os.path.join(self.output_directory, 'HTML') 244 245 coverage_args = ['-p', self.prof_data] 246 coverage_args += ['-b', self.build_directory] 247 coverage_args += ['-o', report_directory] 248 coverage_args += self.build_targets 249 250 # Whitelist the directories of interest 251 coverage_args += ['-f', 'core'] 252 coverage_args += ['-f', 'fpdfsdk'] 253 coverage_args += ['-f', 'fxbarcode'] 254 coverage_args += ['-f', 'fxjs'] 255 coverage_args += ['-f', 'public'] 256 coverage_args += ['-f', 'samples'] 257 coverage_args += ['-f', 'xfa'] 258 259 # Blacklist test files 260 coverage_args += ['-i', '.*test.*'] 261 262 # Component view is only useful for Chromium 263 coverage_args += ['--no-component-view'] 264 265 return self.call([coverage_bin] + coverage_args) == 0 266 267 def run(self): 268 """Setup environment, execute the tests and generate coverage report""" 269 if not self.fetch_profiling_tools(): 270 print 'Unable to fetch profiling tools' 271 return False 272 273 if not self.build_binaries(): 274 print 'Failed to successfully build binaries' 275 return False 276 277 for name in self.coverage_tests.keys(): 278 if not self.generate_coverage(name, self.coverage_tests[name]): 279 print 'Failed to successfully generate coverage data' 280 return False 281 282 if not self.merge_raw_coverage_results(): 283 print 'Failed to successfully merge raw coverage results' 284 return False 285 286 if not self.generate_html_report(): 287 print 'Failed to successfully generate HTML report' 288 return False 289 290 return True 291 292 def fetch_profiling_tools(self): 293 """Call coverage.py with no args to ensure profiling tools are present.""" 294 return self.call_silent( 295 os.path.join(self.source_directory, 'tools', 'code_coverage', 296 'coverage.py')) == 0 297 298 299def main(): 300 parser = argparse.ArgumentParser() 301 parser.formatter_class = argparse.RawDescriptionHelpFormatter 302 parser.description = 'Generates a coverage report for given tests.' 303 304 parser.add_argument( 305 '-s', 306 '--source_directory', 307 help='Location of PDFium source directory, defaults to CWD', 308 default=os.getcwd()) 309 build_default = os.path.join('out', 'Coverage') 310 parser.add_argument( 311 '-b', 312 '--build_directory', 313 help= 314 'Location of PDFium build directory with coverage enabled, defaults to ' 315 '%s under CWD' % build_default, 316 default=os.path.join(os.getcwd(), build_default)) 317 output_default = 'coverage_report' 318 parser.add_argument( 319 '-o', 320 '--output_directory', 321 help='Location to write out coverage report to, defaults to %s under CWD ' 322 % output_default, 323 default=os.path.join(os.getcwd(), output_default)) 324 parser.add_argument( 325 '-n', 326 '--dry-run', 327 help='Output commands instead of executing them', 328 action='store_true') 329 parser.add_argument( 330 '-v', 331 '--verbose', 332 help='Output additional diagnostic information', 333 action='store_true') 334 parser.add_argument( 335 'tests', 336 help='Tests to be run, defaults to all. Valid entries are %s' % 337 COVERAGE_TESTS.keys(), 338 nargs='*') 339 340 args = vars(parser.parse_args()) 341 if args['verbose']: 342 pprint.pprint(args) 343 344 executor = CoverageExecutor(parser, args) 345 if executor.run(): 346 return 0 347 return 1 348 349 350if __name__ == '__main__': 351 sys.exit(main()) 352