1#!/usr/bin/env python
2# Copyright 2015 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6from __future__ import print_function
7
8import glob
9import os
10import subprocess
11import sys
12
13
14class ClangPluginTest(object):
15  """Test harness for clang plugins."""
16
17  def __init__(self, test_base, clang_path, plugin_name, reset_results):
18    """Constructor.
19
20    Args:
21      test_base: Path to the directory containing the tests.
22      clang_path: Path to the clang binary.
23      plugin_name: Name of the plugin.
24      reset_results: If true, resets expected results to the actual test output.
25    """
26    self._test_base = test_base
27    self._clang_path = clang_path
28    self._plugin_name = plugin_name
29    self._reset_results = reset_results
30
31  def AddPluginArg(self, clang_cmd, plugin_arg):
32    """Helper to add an argument for the tested plugin."""
33    clang_cmd.extend(['-Xclang', '-plugin-arg-%s' % self._plugin_name,
34                      '-Xclang', plugin_arg])
35
36  def AdjustClangArguments(self, clang_cmd):
37    """Tests can override this to customize the command line for clang."""
38    pass
39
40  def Run(self):
41    """Runs the tests.
42
43    The working directory is temporarily changed to self._test_base while
44    running the tests.
45
46    Returns: the number of failing tests.
47    """
48    print('Using clang %s...' % self._clang_path)
49
50    os.chdir(self._test_base)
51
52    clang_cmd = [self._clang_path, '-c', '-std=c++14']
53    clang_cmd.extend(['-Xclang', '-add-plugin', '-Xclang', self._plugin_name])
54    self.AdjustClangArguments(clang_cmd)
55
56    passing = []
57    failing = []
58    tests = glob.glob('*.cpp')
59    for test in tests:
60      sys.stdout.write('Testing %s... ' % test)
61      test_name, _ = os.path.splitext(test)
62
63      cmd = clang_cmd[:]
64      try:
65        # Some tests need to run with extra flags.
66        cmd.extend(open('%s.flags' % test_name).read().split())
67      except IOError:
68        pass
69      cmd.append(test)
70
71      print("cmd", cmd)
72      failure_message = self.RunOneTest(test_name, cmd)
73      if failure_message:
74        print('failed: %s' % failure_message)
75        failing.append(test_name)
76      else:
77        print('passed!')
78        passing.append(test_name)
79
80    print('Ran %d tests: %d succeeded, %d failed' % (
81        len(passing) + len(failing), len(passing), len(failing)))
82    for test in failing:
83      print('    %s' % test)
84    return len(failing)
85
86  def RunOneTest(self, test_name, cmd):
87    try:
88      actual = subprocess.check_output(cmd,
89                                       stderr=subprocess.STDOUT,
90                                       universal_newlines=True)
91    except subprocess.CalledProcessError as e:
92      # Some plugin tests intentionally trigger compile errors, so just ignore
93      # an exit code that indicates failure.
94      actual = e.output
95    except Exception as e:
96      return 'could not execute %s (%s)' % (cmd, e)
97
98    return self.ProcessOneResult(test_name, actual)
99
100  def ProcessOneResult(self, test_name, actual):
101    """Tests can override this for custom result processing."""
102    # On Windows, clang emits CRLF as the end of line marker. Normalize it to LF
103    # to match posix systems.
104    actual = actual.replace('\r\n', '\n')
105
106    result_file = '%s.txt%s' % (test_name, '' if self._reset_results else
107                                '.actual')
108    try:
109      expected = open('%s.txt' % test_name).read()
110    except IOError:
111      open(result_file, 'w').write(actual)
112      return 'no expected file found'
113
114    if expected != actual:
115      open(result_file, 'w').write(actual)
116      error = 'expected and actual differed\n'
117      error += 'Actual:\n' + actual
118      error += 'Expected:\n' + expected
119      return error
120