1# This Source Code Form is subject to the terms of the Mozilla Public
2# License, v. 2.0. If a copy of the MPL was not distributed with this file,
3# You can obtain one at http://mozilla.org/MPL/2.0/.
4
5
6import time
7import os
8import mozinfo
9
10
11class TestContext(object):
12    """ Stores context data about the test """
13
14    attrs = ['hostname', 'arch', 'env', 'os', 'os_version', 'tree', 'revision',
15             'product', 'logfile', 'testgroup', 'harness', 'buildtype']
16
17    def __init__(self, hostname='localhost', tree='', revision='', product='',
18                 logfile=None, arch='', operating_system='', testgroup='',
19                 harness='moztest', buildtype=''):
20        self.hostname = hostname
21        self.arch = arch or mozinfo.processor
22        self.env = os.environ.copy()
23        self.os = operating_system or mozinfo.os
24        self.os_version = mozinfo.version
25        self.tree = tree
26        self.revision = revision
27        self.product = product
28        self.logfile = logfile
29        self.testgroup = testgroup
30        self.harness = harness
31        self.buildtype = buildtype
32
33    def __str__(self):
34        return '%s (%s, %s)' % (self.hostname, self.os, self.arch)
35
36    def __repr__(self):
37        return '<%s>' % self.__str__()
38
39    def __eq__(self, other):
40        if not isinstance(other, TestContext):
41            return False
42        diffs = [a for a in self.attrs if getattr(self, a) != getattr(other, a)]
43        return len(diffs) == 0
44
45    def __hash__(self):
46        def get(attr):
47            value = getattr(self, attr)
48            if isinstance(value, dict):
49                value = frozenset(value.items())
50            return value
51        return hash(frozenset([get(a) for a in self.attrs]))
52
53
54class TestResult(object):
55    """ Stores test result data """
56
57    FAIL_RESULTS = [
58        'UNEXPECTED-PASS',
59        'UNEXPECTED-FAIL',
60        'ERROR',
61    ]
62    COMPUTED_RESULTS = FAIL_RESULTS + [
63        'PASS',
64        'KNOWN-FAIL',
65        'SKIPPED',
66    ]
67    POSSIBLE_RESULTS = [
68        'PASS',
69        'FAIL',
70        'SKIP',
71        'ERROR',
72    ]
73
74    def __init__(self, name, test_class='', time_start=None, context=None,
75                 result_expected='PASS'):
76        """ Create a TestResult instance.
77        name = name of the test that is running
78        test_class = the class that the test belongs to
79        time_start = timestamp (seconds since UNIX epoch) of when the test started
80                     running; if not provided, defaults to the current time
81                     ! Provide 0 if you only have the duration
82        context = TestContext instance; can be None
83        result_expected = string representing the expected outcome of the test"""
84
85        msg = "Result '%s' not in possible results: %s" %\
86              (result_expected, ', '.join(self.POSSIBLE_RESULTS))
87        assert isinstance(name, basestring), "name has to be a string"
88        assert result_expected in self.POSSIBLE_RESULTS, msg
89
90        self.name = name
91        self.test_class = test_class
92        self.context = context
93        self.time_start = time_start if time_start is not None else time.time()
94        self.time_end = None
95        self._result_expected = result_expected
96        self._result_actual = None
97        self.result = None
98        self.filename = None
99        self.description = None
100        self.output = []
101        self.reason = None
102
103    @property
104    def test_name(self):
105        return '%s.py %s.%s' % (self.test_class.split('.')[0],
106                                self.test_class,
107                                self.name)
108
109    def __str__(self):
110        return '%s | %s (%s) | %s' % (self.result or 'PENDING',
111                                      self.name, self.test_class, self.reason)
112
113    def __repr__(self):
114        return '<%s>' % self.__str__()
115
116    def calculate_result(self, expected, actual):
117        if actual == 'ERROR':
118            return 'ERROR'
119        if actual == 'SKIP':
120            return 'SKIPPED'
121
122        if expected == 'PASS':
123            if actual == 'PASS':
124                return 'PASS'
125            if actual == 'FAIL':
126                return 'UNEXPECTED-FAIL'
127
128        if expected == 'FAIL':
129            if actual == 'PASS':
130                return 'UNEXPECTED-PASS'
131            if actual == 'FAIL':
132                return 'KNOWN-FAIL'
133
134        # if actual is skip or error, we return at the beginning, so if we get
135        # here it is definitely some kind of error
136        return 'ERROR'
137
138    def infer_results(self, computed_result):
139        assert computed_result in self.COMPUTED_RESULTS
140        if computed_result == 'UNEXPECTED-PASS':
141            expected = 'FAIL'
142            actual = 'PASS'
143        elif computed_result == 'UNEXPECTED-FAIL':
144            expected = 'PASS'
145            actual = 'FAIL'
146        elif computed_result == 'KNOWN-FAIL':
147            expected = actual = 'FAIL'
148        elif computed_result == 'SKIPPED':
149            expected = actual = 'SKIP'
150        else:
151            return
152        self._result_expected = expected
153        self._result_actual = actual
154
155    def finish(self, result, time_end=None, output=None, reason=None):
156        """ Marks the test as finished, storing its end time and status
157        ! Provide the duration as time_end if you only have that. """
158
159        if result in self.POSSIBLE_RESULTS:
160            self._result_actual = result
161            self.result = self.calculate_result(self._result_expected,
162                                                self._result_actual)
163        elif result in self.COMPUTED_RESULTS:
164            self.infer_results(result)
165            self.result = result
166        else:
167            valid = self.POSSIBLE_RESULTS + self.COMPUTED_RESULTS
168            msg = "Result '%s' not valid. Need one of: %s" %\
169                  (result, ', '.join(valid))
170            raise ValueError(msg)
171
172        # use lists instead of multiline strings
173        if isinstance(output, basestring):
174            output = output.splitlines()
175
176        self.time_end = time_end if time_end is not None else time.time()
177        self.output = output or self.output
178        self.reason = reason
179
180    @property
181    def finished(self):
182        """ Boolean saying if the test is finished or not """
183        return self.result is not None
184
185    @property
186    def duration(self):
187        """ Returns the time it took for the test to finish. If the test is
188        not finished, returns the elapsed time so far """
189        if self.result is not None:
190            return self.time_end - self.time_start
191        else:
192            # returns the elapsed time
193            return time.time() - self.time_start
194
195
196class TestResultCollection(list):
197    """ Container class that stores test results """
198
199    resultClass = TestResult
200
201    def __init__(self, suite_name, time_taken=0, resultClass=None):
202        list.__init__(self)
203        self.suite_name = suite_name
204        self.time_taken = time_taken
205        if resultClass is not None:
206            self.resultClass = resultClass
207
208    def __str__(self):
209        return "%s (%.2fs)\n%s" % (self.suite_name, self.time_taken,
210                                   list.__str__(self))
211
212    def subset(self, predicate):
213        tests = self.filter(predicate)
214        duration = 0
215        sub = TestResultCollection(self.suite_name)
216        for t in tests:
217            sub.append(t)
218            duration += t.duration
219        sub.time_taken = duration
220        return sub
221
222    @property
223    def contexts(self):
224        """ List of unique contexts for the test results contained """
225        cs = [tr.context for tr in self]
226        return list(set(cs))
227
228    def filter(self, predicate):
229        """ Returns a generator of TestResults that satisfy a given predicate """
230        return (tr for tr in self if predicate(tr))
231
232    def tests_with_result(self, result):
233        """ Returns a generator of TestResults with the given result """
234        msg = "Result '%s' not in possible results: %s" %\
235              (result, ', '.join(self.resultClass.COMPUTED_RESULTS))
236        assert result in self.resultClass.COMPUTED_RESULTS, msg
237        return self.filter(lambda t: t.result == result)
238
239    @property
240    def tests(self):
241        """ Generator of all tests in the collection """
242        return (t for t in self)
243
244    def add_result(self, test, result_expected='PASS',
245                   result_actual='PASS', output='', context=None):
246        def get_class(test):
247            return test.__class__.__module__ + '.' + test.__class__.__name__
248
249        t = self.resultClass(name=str(test).split()[0], test_class=get_class(test),
250                             time_start=0, result_expected=result_expected,
251                             context=context)
252        t.finish(result_actual, time_end=0, reason=relevant_line(output),
253                 output=output)
254        self.append(t)
255
256    @property
257    def num_failures(self):
258        fails = 0
259        for t in self:
260            if t.result in self.resultClass.FAIL_RESULTS:
261                fails += 1
262        return fails
263
264    def add_unittest_result(self, result, context=None):
265        """ Adds the python unittest result provided to the collection"""
266        if hasattr(result, 'time_taken'):
267            self.time_taken += result.time_taken
268
269        for test, output in result.errors:
270            self.add_result(test, result_actual='ERROR', output=output)
271
272        for test, output in result.failures:
273            self.add_result(test, result_actual='FAIL',
274                            output=output)
275
276        if hasattr(result, 'unexpectedSuccesses'):
277            for test in result.unexpectedSuccesses:
278                self.add_result(test, result_expected='FAIL',
279                                result_actual='PASS')
280
281        if hasattr(result, 'skipped'):
282            for test, output in result.skipped:
283                self.add_result(test, result_expected='SKIP',
284                                result_actual='SKIP', output=output)
285
286        if hasattr(result, 'expectedFailures'):
287            for test, output in result.expectedFailures:
288                self.add_result(test, result_expected='FAIL',
289                                result_actual='FAIL', output=output)
290
291        # unittest does not store these by default
292        if hasattr(result, 'tests_passed'):
293            for test in result.tests_passed:
294                self.add_result(test)
295
296    @classmethod
297    def from_unittest_results(cls, context, *results):
298        """ Creates a TestResultCollection containing the given python
299        unittest results """
300
301        if not results:
302            return cls('from unittest')
303
304        # all the TestResult instances share the same context
305        context = context or TestContext()
306
307        collection = cls('from %s' % results[0].__class__.__name__)
308
309        for result in results:
310            collection.add_unittest_result(result, context)
311
312        return collection
313
314
315# used to get exceptions/errors from tracebacks
316def relevant_line(s):
317    KEYWORDS = ('Error:', 'Exception:', 'error:', 'exception:')
318    lines = s.splitlines()
319    for line in lines:
320        for keyword in KEYWORDS:
321            if keyword in line:
322                return line
323    return 'N/A'
324