1# Copyright (c) 2012 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5 6import json 7import logging 8import os 9import time 10import traceback 11 12import buildbot_report 13import constants 14 15 16class BaseTestResult(object): 17 """A single result from a unit test.""" 18 19 def __init__(self, name, log): 20 self.name = name 21 self.log = log.replace('\r', '') 22 23 24class SingleTestResult(BaseTestResult): 25 """Result information for a single test. 26 27 Args: 28 full_name: Full name of the test. 29 start_date: Date in milliseconds when the test began running. 30 dur: Duration of the test run in milliseconds. 31 log: An optional string listing any errors. 32 """ 33 34 def __init__(self, full_name, start_date, dur, log=''): 35 BaseTestResult.__init__(self, full_name, log) 36 name_pieces = full_name.rsplit('#') 37 if len(name_pieces) > 1: 38 self.test_name = name_pieces[1] 39 self.class_name = name_pieces[0] 40 else: 41 self.class_name = full_name 42 self.test_name = full_name 43 self.start_date = start_date 44 self.dur = dur 45 46 47class TestResults(object): 48 """Results of a test run.""" 49 50 def __init__(self): 51 self.ok = [] 52 self.failed = [] 53 self.crashed = [] 54 self.unknown = [] 55 self.timed_out = False 56 self.overall_fail = False 57 58 @staticmethod 59 def FromRun(ok=None, failed=None, crashed=None, timed_out=False, 60 overall_fail=False): 61 ret = TestResults() 62 ret.ok = ok or [] 63 ret.failed = failed or [] 64 ret.crashed = crashed or [] 65 ret.timed_out = timed_out 66 ret.overall_fail = overall_fail 67 return ret 68 69 @staticmethod 70 def FromTestResults(results): 71 """Combines a list of results in a single TestResults object.""" 72 ret = TestResults() 73 for t in results: 74 ret.ok += t.ok 75 ret.failed += t.failed 76 ret.crashed += t.crashed 77 ret.unknown += t.unknown 78 if t.timed_out: 79 ret.timed_out = True 80 if t.overall_fail: 81 ret.overall_fail = True 82 return ret 83 84 @staticmethod 85 def FromPythonException(test_name, start_date_ms, exc_info): 86 """Constructs a TestResults with exception information for the given test. 87 88 Args: 89 test_name: name of the test which raised an exception. 90 start_date_ms: the starting time for the test. 91 exc_info: exception info, ostensibly from sys.exc_info(). 92 93 Returns: 94 A TestResults object with a SingleTestResult in the failed list. 95 """ 96 exc_type, exc_value, exc_traceback = exc_info 97 trace_info = ''.join(traceback.format_exception(exc_type, exc_value, 98 exc_traceback)) 99 log_msg = 'Exception:\n' + trace_info 100 duration_ms = (int(time.time()) * 1000) - start_date_ms 101 102 exc_result = SingleTestResult( 103 full_name='PythonWrapper#' + test_name, 104 start_date=start_date_ms, 105 dur=duration_ms, 106 log=(str(exc_type) + ' ' + log_msg)) 107 108 results = TestResults() 109 results.failed.append(exc_result) 110 return results 111 112 def _Log(self, sorted_list): 113 for t in sorted_list: 114 logging.critical(t.name) 115 if t.log: 116 logging.critical(t.log) 117 118 def GetAllBroken(self): 119 """Returns the all broken tests including failed, crashed, unknown.""" 120 return self.failed + self.crashed + self.unknown 121 122 def LogFull(self, test_group, test_suite, build_type): 123 """Output broken test logs, summarize in a log file and the test output.""" 124 # Output all broken tests or 'passed' if none broken. 125 logging.critical('*' * 80) 126 logging.critical('Final result') 127 if self.failed: 128 logging.critical('Failed:') 129 self._Log(sorted(self.failed)) 130 if self.crashed: 131 logging.critical('Crashed:') 132 self._Log(sorted(self.crashed)) 133 if self.unknown: 134 logging.critical('Unknown:') 135 self._Log(sorted(self.unknown)) 136 if not self.GetAllBroken(): 137 logging.critical('Passed') 138 logging.critical('*' * 80) 139 140 # Summarize in a log file, if tests are running on bots. 141 if test_group and test_suite and os.environ.get('BUILDBOT_BUILDERNAME'): 142 log_file_path = os.path.join(constants.CHROME_DIR, 'out', 143 build_type, 'test_logs') 144 if not os.path.exists(log_file_path): 145 os.mkdir(log_file_path) 146 full_file_name = os.path.join(log_file_path, test_group) 147 if not os.path.exists(full_file_name): 148 with open(full_file_name, 'w') as log_file: 149 print >> log_file, '\n%s results for %s build %s:' % ( 150 test_group, os.environ.get('BUILDBOT_BUILDERNAME'), 151 os.environ.get('BUILDBOT_BUILDNUMBER')) 152 log_contents = [' %s result : %d tests ran' % (test_suite, 153 len(self.ok) + 154 len(self.failed) + 155 len(self.crashed) + 156 len(self.unknown))] 157 content_pairs = [('passed', len(self.ok)), ('failed', len(self.failed)), 158 ('crashed', len(self.crashed))] 159 for (result, count) in content_pairs: 160 if count: 161 log_contents.append(', %d tests %s' % (count, result)) 162 with open(full_file_name, 'a') as log_file: 163 print >> log_file, ''.join(log_contents) 164 content = {'test_group': test_group, 165 'ok': [t.name for t in self.ok], 166 'failed': [t.name for t in self.failed], 167 'crashed': [t.name for t in self.failed], 168 'unknown': [t.name for t in self.unknown],} 169 with open(os.path.join(log_file_path, 'results.json'), 'a') as json_file: 170 print >> json_file, json.dumps(content) 171 172 # Summarize in the test output. 173 summary_string = 'Summary:\n' 174 summary_string += 'RAN=%d\n' % (len(self.ok) + len(self.failed) + 175 len(self.crashed) + len(self.unknown)) 176 summary_string += 'PASSED=%d\n' % (len(self.ok)) 177 summary_string += 'FAILED=%d %s\n' % (len(self.failed), 178 [t.name for t in self.failed]) 179 summary_string += 'CRASHED=%d %s\n' % (len(self.crashed), 180 [t.name for t in self.crashed]) 181 summary_string += 'UNKNOWN=%d %s\n' % (len(self.unknown), 182 [t.name for t in self.unknown]) 183 logging.critical(summary_string) 184 return summary_string 185 186 def PrintAnnotation(self): 187 """Print buildbot annotations for test results.""" 188 if self.timed_out: 189 buildbot_report.PrintWarning() 190 elif self.failed or self.crashed or self.overall_fail: 191 buildbot_report.PrintError() 192 else: 193 print 'Step success!' # No annotation needed 194