1#!/usr/bin/env python
2# Copyright 2017 The PDFium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5"""Generates a coverage report for given tests.
6
7Requires that 'use_clang_coverage = true' is set in args.gn.
8Prefers that 'is_component_build = false' is set in args.gn.
9"""
10
11import argparse
12from collections import namedtuple
13import fnmatch
14import os
15import pprint
16import subprocess
17import sys
18
19# Add src dir to path to avoid having to set PYTHONPATH.
20sys.path.append(
21    os.path.abspath(
22        os.path.join(
23            os.path.dirname(__file__), os.path.pardir, os.path.pardir,
24            os.path.pardir)))
25
26from testing.tools.common import GetBooleanGnArg
27
28# 'binary' is the file that is to be run for the test.
29# 'use_test_runner' indicates if 'binary' depends on test_runner.py and thus
30# requires special handling.
31# 'opt_args' are optional arguments to pass to the test 'binary'.
32TestSpec = namedtuple('TestSpec', 'binary, use_test_runner, opt_args')
33
34# All of the coverage tests that the script knows how to run.
35COVERAGE_TESTS = {
36    'pdfium_unittests':
37        TestSpec('pdfium_unittests', False, []),
38    'pdfium_embeddertests':
39        TestSpec('pdfium_embeddertests', False, []),
40    'corpus_tests':
41        TestSpec('run_corpus_tests.py', True, []),
42    'corpus_tests_javascript_disabled':
43        TestSpec('run_corpus_tests.py', True, ['--disable-javascript']),
44    'corpus_tests_xfa_disabled':
45        TestSpec('run_corpus_tests.py', True, ['--disable-xfa']),
46    'javascript_tests':
47        TestSpec('run_javascript_tests.py', True, []),
48    'javascript_tests_javascript_disabled':
49        TestSpec('run_javascript_tests.py', True, ['--disable-javascript']),
50    'javascript_tests_xfa_disabled':
51        TestSpec('run_javascript_tests.py', True, ['--disable-xfa']),
52    'pixel_tests':
53        TestSpec('run_pixel_tests.py', True, []),
54    'pixel_tests_javascript_disabled':
55        TestSpec('run_pixel_tests.py', True, ['--disable-javascript']),
56    'pixel_tests_xfa_disabled':
57        TestSpec('run_pixel_tests.py', True, ['--disable-xfa']),
58}
59
60
61class CoverageExecutor(object):
62
63  def __init__(self, parser, args):
64    """Initialize executor based on the current script environment
65
66    Args:
67        parser: argparse.ArgumentParser for handling improper inputs.
68        args: Dictionary of arguments passed into the calling script.
69    """
70    self.dry_run = args['dry_run']
71    self.verbose = args['verbose']
72
73    self.source_directory = args['source_directory']
74    if not os.path.isdir(self.source_directory):
75      parser.error("'%s' needs to be a directory" % self.source_directory)
76
77    self.llvm_directory = os.path.join(self.source_directory, 'third_party',
78                                       'llvm-build', 'Release+Asserts', 'bin')
79    if not os.path.isdir(self.llvm_directory):
80      parser.error("Cannot find LLVM bin directory , expected it to be in '%s'"
81                   % self.llvm_directory)
82
83    self.build_directory = args['build_directory']
84    if not os.path.isdir(self.build_directory):
85      parser.error("'%s' needs to be a directory" % self.build_directory)
86
87    (self.coverage_tests,
88     self.build_targets) = self.calculate_coverage_tests(args)
89    if not self.coverage_tests:
90      parser.error(
91          'No valid tests in set to be run. This is likely due to bad command '
92          'line arguments')
93
94    if not GetBooleanGnArg('use_clang_coverage', self.build_directory,
95                           self.verbose):
96      parser.error(
97          'use_clang_coverage does not appear to be set to true for build, but '
98          'is needed')
99
100    self.use_goma = GetBooleanGnArg('use_goma', self.build_directory,
101                                    self.verbose)
102
103    self.output_directory = args['output_directory']
104    if not os.path.exists(self.output_directory):
105      if not self.dry_run:
106        os.makedirs(self.output_directory)
107    elif not os.path.isdir(self.output_directory):
108      parser.error('%s exists, but is not a directory' % self.output_directory)
109    elif len(os.listdir(self.output_directory)) > 0:
110      parser.error('%s is not empty, cowardly refusing to continue' %
111                   self.output_directory)
112
113    self.prof_data = os.path.join(self.output_directory, 'pdfium.profdata')
114
115  def check_output(self, args, dry_run=False, env=None):
116    """Dry run aware wrapper of subprocess.check_output()"""
117    if dry_run:
118      print "Would have run '%s'" % ' '.join(args)
119      return ''
120
121    output = subprocess.check_output(args, env=env)
122
123    if self.verbose:
124      print "check_output(%s) returned '%s'" % (args, output)
125    return output
126
127  def call(self, args, dry_run=False, env=None):
128    """Dry run aware wrapper of subprocess.call()"""
129    if dry_run:
130      print "Would have run '%s'" % ' '.join(args)
131      return 0
132
133    output = subprocess.call(args, env=env)
134
135    if self.verbose:
136      print 'call(%s) returned %s' % (args, output)
137    return output
138
139  def call_silent(self, args, dry_run=False, env=None):
140    """Dry run aware wrapper of subprocess.call() that eats output from call"""
141    if dry_run:
142      print "Would have run '%s'" % ' '.join(args)
143      return 0
144
145    with open(os.devnull, 'w') as f:
146      output = subprocess.call(args, env=env, stdout=f)
147
148    if self.verbose:
149      print 'call_silent(%s) returned %s' % (args, output)
150    return output
151
152  def calculate_coverage_tests(self, args):
153    """Determine which tests should be run."""
154    testing_tools_directory = os.path.join(self.source_directory, 'testing',
155                                           'tools')
156    tests = args['tests'] if args['tests'] else COVERAGE_TESTS.keys()
157    coverage_tests = {}
158    build_targets = set()
159    for name in tests:
160      test_spec = COVERAGE_TESTS[name]
161      if test_spec.use_test_runner:
162        binary_path = os.path.join(testing_tools_directory, test_spec.binary)
163        build_targets.add('pdfium_diff')
164        build_targets.add('pdfium_test')
165      else:
166        binary_path = os.path.join(self.build_directory, test_spec.binary)
167        build_targets.add(name)
168      coverage_tests[name] = TestSpec(binary_path, test_spec.use_test_runner,
169                                      test_spec.opt_args)
170
171    build_targets = list(build_targets)
172
173    return coverage_tests, build_targets
174
175  def build_binaries(self):
176    """Build all the binaries that are going to be needed for coverage
177    generation."""
178    call_args = ['ninja']
179    if self.use_goma:
180      call_args += ['-j', '250']
181    call_args += ['-C', self.build_directory]
182    call_args += self.build_targets
183    return self.call(call_args, dry_run=self.dry_run) == 0
184
185  def generate_coverage(self, name, spec):
186    """Generate the coverage data for a test
187
188    Args:
189        name: Name associated with the test to be run. This is used as a label
190              in the coverage data, so should be unique across all of the tests
191              being run.
192        spec: Tuple containing the TestSpec.
193    """
194    if self.verbose:
195      print "Generating coverage for test '%s', using data '%s'" % (name, spec)
196    if not os.path.exists(spec.binary):
197      print('Unable to generate coverage for %s, since it appears to not exist'
198            ' @ %s') % (name, spec.binary)
199      return False
200
201    binary_args = [spec.binary]
202    if spec.opt_args:
203      binary_args.extend(spec.opt_args)
204    profile_pattern_string = '%8m'
205    expected_profraw_file = '%s.%s.profraw' % (name, profile_pattern_string)
206    expected_profraw_path = os.path.join(self.output_directory,
207                                         expected_profraw_file)
208
209    env = {
210        'LLVM_PROFILE_FILE': expected_profraw_path,
211        'PATH': os.getenv('PATH') + os.pathsep + self.llvm_directory
212    }
213
214    if spec.use_test_runner:
215      # Test runner performs multi-threading in the wrapper script, not the test
216      # binary, so need to limit the number of instances of the binary being run
217      # to the max value in LLVM_PROFILE_FILE, which is 8.
218      binary_args.extend(['-j', '8', '--build-dir', self.build_directory])
219    if self.call(binary_args, dry_run=self.dry_run, env=env) and self.verbose:
220      print('Running %s appears to have failed, which might affect '
221            'results') % spec.binary
222
223    return True
224
225  def merge_raw_coverage_results(self):
226    """Merge raw coverage data sets into one one file for report generation."""
227    llvm_profdata_bin = os.path.join(self.llvm_directory, 'llvm-profdata')
228
229    raw_data = []
230    raw_data_pattern = '*.profraw'
231    for file_name in os.listdir(self.output_directory):
232      if fnmatch.fnmatch(file_name, raw_data_pattern):
233        raw_data.append(os.path.join(self.output_directory, file_name))
234
235    return self.call(
236        [llvm_profdata_bin, 'merge', '-o', self.prof_data, '-sparse=true'] +
237        raw_data) == 0
238
239  def generate_html_report(self):
240    """Generate HTML report by calling upstream coverage.py"""
241    coverage_bin = os.path.join(self.source_directory, 'tools', 'code_coverage',
242                                'coverage.py')
243    report_directory = os.path.join(self.output_directory, 'HTML')
244
245    coverage_args = ['-p', self.prof_data]
246    coverage_args += ['-b', self.build_directory]
247    coverage_args += ['-o', report_directory]
248    coverage_args += self.build_targets
249
250    # Whitelist the directories of interest
251    coverage_args += ['-f', 'core']
252    coverage_args += ['-f', 'fpdfsdk']
253    coverage_args += ['-f', 'fxbarcode']
254    coverage_args += ['-f', 'fxjs']
255    coverage_args += ['-f', 'public']
256    coverage_args += ['-f', 'samples']
257    coverage_args += ['-f', 'xfa']
258
259    # Blacklist test files
260    coverage_args += ['-i', '.*test.*']
261
262    # Component view is only useful for Chromium
263    coverage_args += ['--no-component-view']
264
265    return self.call([coverage_bin] + coverage_args) == 0
266
267  def run(self):
268    """Setup environment, execute the tests and generate coverage report"""
269    if not self.fetch_profiling_tools():
270      print 'Unable to fetch profiling tools'
271      return False
272
273    if not self.build_binaries():
274      print 'Failed to successfully build binaries'
275      return False
276
277    for name in self.coverage_tests.keys():
278      if not self.generate_coverage(name, self.coverage_tests[name]):
279        print 'Failed to successfully generate coverage data'
280        return False
281
282    if not self.merge_raw_coverage_results():
283      print 'Failed to successfully merge raw coverage results'
284      return False
285
286    if not self.generate_html_report():
287      print 'Failed to successfully generate HTML report'
288      return False
289
290    return True
291
292  def fetch_profiling_tools(self):
293    """Call coverage.py with no args to ensure profiling tools are present."""
294    return self.call_silent(
295        os.path.join(self.source_directory, 'tools', 'code_coverage',
296                     'coverage.py')) == 0
297
298
299def main():
300  parser = argparse.ArgumentParser()
301  parser.formatter_class = argparse.RawDescriptionHelpFormatter
302  parser.description = 'Generates a coverage report for given tests.'
303
304  parser.add_argument(
305      '-s',
306      '--source_directory',
307      help='Location of PDFium source directory, defaults to CWD',
308      default=os.getcwd())
309  build_default = os.path.join('out', 'Coverage')
310  parser.add_argument(
311      '-b',
312      '--build_directory',
313      help=
314      'Location of PDFium build directory with coverage enabled, defaults to '
315      '%s under CWD' % build_default,
316      default=os.path.join(os.getcwd(), build_default))
317  output_default = 'coverage_report'
318  parser.add_argument(
319      '-o',
320      '--output_directory',
321      help='Location to write out coverage report to, defaults to %s under CWD '
322      % output_default,
323      default=os.path.join(os.getcwd(), output_default))
324  parser.add_argument(
325      '-n',
326      '--dry-run',
327      help='Output commands instead of executing them',
328      action='store_true')
329  parser.add_argument(
330      '-v',
331      '--verbose',
332      help='Output additional diagnostic information',
333      action='store_true')
334  parser.add_argument(
335      'tests',
336      help='Tests to be run, defaults to all. Valid entries are %s' %
337      COVERAGE_TESTS.keys(),
338      nargs='*')
339
340  args = vars(parser.parse_args())
341  if args['verbose']:
342    pprint.pprint(args)
343
344  executor = CoverageExecutor(parser, args)
345  if executor.run():
346    return 0
347  return 1
348
349
350if __name__ == '__main__':
351  sys.exit(main())
352