1# Copyright (c) 2012 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5
6import json
7import logging
8import os
9import time
10import traceback
11
12import buildbot_report
13import constants
14
15
16class BaseTestResult(object):
17  """A single result from a unit test."""
18
19  def __init__(self, name, log):
20    self.name = name
21    self.log = log.replace('\r', '')
22
23
24class SingleTestResult(BaseTestResult):
25  """Result information for a single test.
26
27  Args:
28    full_name: Full name of the test.
29    start_date: Date in milliseconds when the test began running.
30    dur: Duration of the test run in milliseconds.
31    log: An optional string listing any errors.
32  """
33
34  def __init__(self, full_name, start_date, dur, log=''):
35    BaseTestResult.__init__(self, full_name, log)
36    name_pieces = full_name.rsplit('#')
37    if len(name_pieces) > 1:
38      self.test_name = name_pieces[1]
39      self.class_name = name_pieces[0]
40    else:
41      self.class_name = full_name
42      self.test_name = full_name
43    self.start_date = start_date
44    self.dur = dur
45
46
47class TestResults(object):
48  """Results of a test run."""
49
50  def __init__(self):
51    self.ok = []
52    self.failed = []
53    self.crashed = []
54    self.unknown = []
55    self.timed_out = False
56    self.overall_fail = False
57
58  @staticmethod
59  def FromRun(ok=None, failed=None, crashed=None, timed_out=False,
60              overall_fail=False):
61    ret = TestResults()
62    ret.ok = ok or []
63    ret.failed = failed or []
64    ret.crashed = crashed or []
65    ret.timed_out = timed_out
66    ret.overall_fail = overall_fail
67    return ret
68
69  @staticmethod
70  def FromTestResults(results):
71    """Combines a list of results in a single TestResults object."""
72    ret = TestResults()
73    for t in results:
74      ret.ok += t.ok
75      ret.failed += t.failed
76      ret.crashed += t.crashed
77      ret.unknown += t.unknown
78      if t.timed_out:
79        ret.timed_out = True
80      if t.overall_fail:
81        ret.overall_fail = True
82    return ret
83
84  @staticmethod
85  def FromPythonException(test_name, start_date_ms, exc_info):
86    """Constructs a TestResults with exception information for the given test.
87
88    Args:
89      test_name: name of the test which raised an exception.
90      start_date_ms: the starting time for the test.
91      exc_info: exception info, ostensibly from sys.exc_info().
92
93    Returns:
94      A TestResults object with a SingleTestResult in the failed list.
95    """
96    exc_type, exc_value, exc_traceback = exc_info
97    trace_info = ''.join(traceback.format_exception(exc_type, exc_value,
98                                                    exc_traceback))
99    log_msg = 'Exception:\n' + trace_info
100    duration_ms = (int(time.time()) * 1000) - start_date_ms
101
102    exc_result = SingleTestResult(
103                     full_name='PythonWrapper#' + test_name,
104                     start_date=start_date_ms,
105                     dur=duration_ms,
106                     log=(str(exc_type) + ' ' + log_msg))
107
108    results = TestResults()
109    results.failed.append(exc_result)
110    return results
111
112  def _Log(self, sorted_list):
113    for t in sorted_list:
114      logging.critical(t.name)
115      if t.log:
116        logging.critical(t.log)
117
118  def GetAllBroken(self):
119    """Returns the all broken tests including failed, crashed, unknown."""
120    return self.failed + self.crashed + self.unknown
121
122  def LogFull(self, test_group, test_suite, build_type):
123    """Output broken test logs, summarize in a log file and the test output."""
124    # Output all broken tests or 'passed' if none broken.
125    logging.critical('*' * 80)
126    logging.critical('Final result')
127    if self.failed:
128      logging.critical('Failed:')
129      self._Log(sorted(self.failed))
130    if self.crashed:
131      logging.critical('Crashed:')
132      self._Log(sorted(self.crashed))
133    if self.unknown:
134      logging.critical('Unknown:')
135      self._Log(sorted(self.unknown))
136    if not self.GetAllBroken():
137      logging.critical('Passed')
138    logging.critical('*' * 80)
139
140    # Summarize in a log file, if tests are running on bots.
141    if test_group and test_suite and os.environ.get('BUILDBOT_BUILDERNAME'):
142      log_file_path = os.path.join(constants.CHROME_DIR, 'out',
143                                   build_type, 'test_logs')
144      if not os.path.exists(log_file_path):
145        os.mkdir(log_file_path)
146      full_file_name = os.path.join(log_file_path, test_group)
147      if not os.path.exists(full_file_name):
148        with open(full_file_name, 'w') as log_file:
149          print >> log_file, '\n%s results for %s build %s:' % (
150              test_group, os.environ.get('BUILDBOT_BUILDERNAME'),
151              os.environ.get('BUILDBOT_BUILDNUMBER'))
152      log_contents = ['  %s result : %d tests ran' % (test_suite,
153                                                      len(self.ok) +
154                                                      len(self.failed) +
155                                                      len(self.crashed) +
156                                                      len(self.unknown))]
157      content_pairs = [('passed', len(self.ok)), ('failed', len(self.failed)),
158                       ('crashed', len(self.crashed))]
159      for (result, count) in content_pairs:
160        if count:
161          log_contents.append(', %d tests %s' % (count, result))
162      with open(full_file_name, 'a') as log_file:
163        print >> log_file, ''.join(log_contents)
164      content = {'test_group': test_group,
165                 'ok': [t.name for t in self.ok],
166                 'failed': [t.name for t in self.failed],
167                 'crashed': [t.name for t in self.failed],
168                 'unknown': [t.name for t in self.unknown],}
169      with open(os.path.join(log_file_path, 'results.json'), 'a') as json_file:
170        print >> json_file, json.dumps(content)
171
172    # Summarize in the test output.
173    summary_string = 'Summary:\n'
174    summary_string += 'RAN=%d\n' % (len(self.ok) + len(self.failed) +
175                                    len(self.crashed) + len(self.unknown))
176    summary_string += 'PASSED=%d\n' % (len(self.ok))
177    summary_string += 'FAILED=%d %s\n' % (len(self.failed),
178                                          [t.name for t in self.failed])
179    summary_string += 'CRASHED=%d %s\n' % (len(self.crashed),
180                                           [t.name for t in self.crashed])
181    summary_string += 'UNKNOWN=%d %s\n' % (len(self.unknown),
182                                           [t.name for t in self.unknown])
183    logging.critical(summary_string)
184    return summary_string
185
186  def PrintAnnotation(self):
187    """Print buildbot annotations for test results."""
188    if self.timed_out:
189      buildbot_report.PrintWarning()
190    elif self.failed or self.crashed or self.overall_fail:
191      buildbot_report.PrintError()
192    else:
193      print 'Step success!'  # No annotation needed
194