1# This Source Code Form is subject to the terms of the Mozilla Public 2# License, v. 2.0. If a copy of the MPL was not distributed with this file, 3# You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 6import time 7import os 8import mozinfo 9 10 11class TestContext(object): 12 """ Stores context data about the test """ 13 14 attrs = ['hostname', 'arch', 'env', 'os', 'os_version', 'tree', 'revision', 15 'product', 'logfile', 'testgroup', 'harness', 'buildtype'] 16 17 def __init__(self, hostname='localhost', tree='', revision='', product='', 18 logfile=None, arch='', operating_system='', testgroup='', 19 harness='moztest', buildtype=''): 20 self.hostname = hostname 21 self.arch = arch or mozinfo.processor 22 self.env = os.environ.copy() 23 self.os = operating_system or mozinfo.os 24 self.os_version = mozinfo.version 25 self.tree = tree 26 self.revision = revision 27 self.product = product 28 self.logfile = logfile 29 self.testgroup = testgroup 30 self.harness = harness 31 self.buildtype = buildtype 32 33 def __str__(self): 34 return '%s (%s, %s)' % (self.hostname, self.os, self.arch) 35 36 def __repr__(self): 37 return '<%s>' % self.__str__() 38 39 def __eq__(self, other): 40 if not isinstance(other, TestContext): 41 return False 42 diffs = [a for a in self.attrs if getattr(self, a) != getattr(other, a)] 43 return len(diffs) == 0 44 45 def __hash__(self): 46 def get(attr): 47 value = getattr(self, attr) 48 if isinstance(value, dict): 49 value = frozenset(value.items()) 50 return value 51 return hash(frozenset([get(a) for a in self.attrs])) 52 53 54class TestResult(object): 55 """ Stores test result data """ 56 57 FAIL_RESULTS = [ 58 'UNEXPECTED-PASS', 59 'UNEXPECTED-FAIL', 60 'ERROR', 61 ] 62 COMPUTED_RESULTS = FAIL_RESULTS + [ 63 'PASS', 64 'KNOWN-FAIL', 65 'SKIPPED', 66 ] 67 POSSIBLE_RESULTS = [ 68 'PASS', 69 'FAIL', 70 'SKIP', 71 'ERROR', 72 ] 73 74 def __init__(self, name, test_class='', time_start=None, context=None, 75 result_expected='PASS'): 76 """ Create a TestResult instance. 77 name = name of the test that is running 78 test_class = the class that the test belongs to 79 time_start = timestamp (seconds since UNIX epoch) of when the test started 80 running; if not provided, defaults to the current time 81 ! Provide 0 if you only have the duration 82 context = TestContext instance; can be None 83 result_expected = string representing the expected outcome of the test""" 84 85 msg = "Result '%s' not in possible results: %s" %\ 86 (result_expected, ', '.join(self.POSSIBLE_RESULTS)) 87 assert isinstance(name, basestring), "name has to be a string" 88 assert result_expected in self.POSSIBLE_RESULTS, msg 89 90 self.name = name 91 self.test_class = test_class 92 self.context = context 93 self.time_start = time_start if time_start is not None else time.time() 94 self.time_end = None 95 self._result_expected = result_expected 96 self._result_actual = None 97 self.result = None 98 self.filename = None 99 self.description = None 100 self.output = [] 101 self.reason = None 102 103 @property 104 def test_name(self): 105 return '%s.py %s.%s' % (self.test_class.split('.')[0], 106 self.test_class, 107 self.name) 108 109 def __str__(self): 110 return '%s | %s (%s) | %s' % (self.result or 'PENDING', 111 self.name, self.test_class, self.reason) 112 113 def __repr__(self): 114 return '<%s>' % self.__str__() 115 116 def calculate_result(self, expected, actual): 117 if actual == 'ERROR': 118 return 'ERROR' 119 if actual == 'SKIP': 120 return 'SKIPPED' 121 122 if expected == 'PASS': 123 if actual == 'PASS': 124 return 'PASS' 125 if actual == 'FAIL': 126 return 'UNEXPECTED-FAIL' 127 128 if expected == 'FAIL': 129 if actual == 'PASS': 130 return 'UNEXPECTED-PASS' 131 if actual == 'FAIL': 132 return 'KNOWN-FAIL' 133 134 # if actual is skip or error, we return at the beginning, so if we get 135 # here it is definitely some kind of error 136 return 'ERROR' 137 138 def infer_results(self, computed_result): 139 assert computed_result in self.COMPUTED_RESULTS 140 if computed_result == 'UNEXPECTED-PASS': 141 expected = 'FAIL' 142 actual = 'PASS' 143 elif computed_result == 'UNEXPECTED-FAIL': 144 expected = 'PASS' 145 actual = 'FAIL' 146 elif computed_result == 'KNOWN-FAIL': 147 expected = actual = 'FAIL' 148 elif computed_result == 'SKIPPED': 149 expected = actual = 'SKIP' 150 else: 151 return 152 self._result_expected = expected 153 self._result_actual = actual 154 155 def finish(self, result, time_end=None, output=None, reason=None): 156 """ Marks the test as finished, storing its end time and status 157 ! Provide the duration as time_end if you only have that. """ 158 159 if result in self.POSSIBLE_RESULTS: 160 self._result_actual = result 161 self.result = self.calculate_result(self._result_expected, 162 self._result_actual) 163 elif result in self.COMPUTED_RESULTS: 164 self.infer_results(result) 165 self.result = result 166 else: 167 valid = self.POSSIBLE_RESULTS + self.COMPUTED_RESULTS 168 msg = "Result '%s' not valid. Need one of: %s" %\ 169 (result, ', '.join(valid)) 170 raise ValueError(msg) 171 172 # use lists instead of multiline strings 173 if isinstance(output, basestring): 174 output = output.splitlines() 175 176 self.time_end = time_end if time_end is not None else time.time() 177 self.output = output or self.output 178 self.reason = reason 179 180 @property 181 def finished(self): 182 """ Boolean saying if the test is finished or not """ 183 return self.result is not None 184 185 @property 186 def duration(self): 187 """ Returns the time it took for the test to finish. If the test is 188 not finished, returns the elapsed time so far """ 189 if self.result is not None: 190 return self.time_end - self.time_start 191 else: 192 # returns the elapsed time 193 return time.time() - self.time_start 194 195 196class TestResultCollection(list): 197 """ Container class that stores test results """ 198 199 resultClass = TestResult 200 201 def __init__(self, suite_name, time_taken=0, resultClass=None): 202 list.__init__(self) 203 self.suite_name = suite_name 204 self.time_taken = time_taken 205 if resultClass is not None: 206 self.resultClass = resultClass 207 208 def __str__(self): 209 return "%s (%.2fs)\n%s" % (self.suite_name, self.time_taken, 210 list.__str__(self)) 211 212 def subset(self, predicate): 213 tests = self.filter(predicate) 214 duration = 0 215 sub = TestResultCollection(self.suite_name) 216 for t in tests: 217 sub.append(t) 218 duration += t.duration 219 sub.time_taken = duration 220 return sub 221 222 @property 223 def contexts(self): 224 """ List of unique contexts for the test results contained """ 225 cs = [tr.context for tr in self] 226 return list(set(cs)) 227 228 def filter(self, predicate): 229 """ Returns a generator of TestResults that satisfy a given predicate """ 230 return (tr for tr in self if predicate(tr)) 231 232 def tests_with_result(self, result): 233 """ Returns a generator of TestResults with the given result """ 234 msg = "Result '%s' not in possible results: %s" %\ 235 (result, ', '.join(self.resultClass.COMPUTED_RESULTS)) 236 assert result in self.resultClass.COMPUTED_RESULTS, msg 237 return self.filter(lambda t: t.result == result) 238 239 @property 240 def tests(self): 241 """ Generator of all tests in the collection """ 242 return (t for t in self) 243 244 def add_result(self, test, result_expected='PASS', 245 result_actual='PASS', output='', context=None): 246 def get_class(test): 247 return test.__class__.__module__ + '.' + test.__class__.__name__ 248 249 t = self.resultClass(name=str(test).split()[0], test_class=get_class(test), 250 time_start=0, result_expected=result_expected, 251 context=context) 252 t.finish(result_actual, time_end=0, reason=relevant_line(output), 253 output=output) 254 self.append(t) 255 256 @property 257 def num_failures(self): 258 fails = 0 259 for t in self: 260 if t.result in self.resultClass.FAIL_RESULTS: 261 fails += 1 262 return fails 263 264 def add_unittest_result(self, result, context=None): 265 """ Adds the python unittest result provided to the collection""" 266 if hasattr(result, 'time_taken'): 267 self.time_taken += result.time_taken 268 269 for test, output in result.errors: 270 self.add_result(test, result_actual='ERROR', output=output) 271 272 for test, output in result.failures: 273 self.add_result(test, result_actual='FAIL', 274 output=output) 275 276 if hasattr(result, 'unexpectedSuccesses'): 277 for test in result.unexpectedSuccesses: 278 self.add_result(test, result_expected='FAIL', 279 result_actual='PASS') 280 281 if hasattr(result, 'skipped'): 282 for test, output in result.skipped: 283 self.add_result(test, result_expected='SKIP', 284 result_actual='SKIP', output=output) 285 286 if hasattr(result, 'expectedFailures'): 287 for test, output in result.expectedFailures: 288 self.add_result(test, result_expected='FAIL', 289 result_actual='FAIL', output=output) 290 291 # unittest does not store these by default 292 if hasattr(result, 'tests_passed'): 293 for test in result.tests_passed: 294 self.add_result(test) 295 296 @classmethod 297 def from_unittest_results(cls, context, *results): 298 """ Creates a TestResultCollection containing the given python 299 unittest results """ 300 301 if not results: 302 return cls('from unittest') 303 304 # all the TestResult instances share the same context 305 context = context or TestContext() 306 307 collection = cls('from %s' % results[0].__class__.__name__) 308 309 for result in results: 310 collection.add_unittest_result(result, context) 311 312 return collection 313 314 315# used to get exceptions/errors from tracebacks 316def relevant_line(s): 317 KEYWORDS = ('Error:', 'Exception:', 'error:', 'exception:') 318 lines = s.splitlines() 319 for line in lines: 320 for keyword in KEYWORDS: 321 if keyword in line: 322 return line 323 return 'N/A' 324