1from __future__ import absolute_import, division, print_function
2
3import math
4import mozinfo
5
6
7class Bisect(object):
8
9    "Class for creating, bisecting and summarizing for --bisect-chunk option."
10
11    def __init__(self, harness):
12        super(Bisect, self).__init__()
13        self.summary = []
14        self.contents = {}
15        self.repeat = 10
16        self.failcount = 0
17        self.max_failures = 3
18
19    def setup(self, tests):
20        """This method is used to initialize various variables that are required
21        for test bisection"""
22        status = 0
23        self.contents.clear()
24        # We need totalTests key in contents for sanity check
25        self.contents["totalTests"] = tests
26        self.contents["tests"] = tests
27        self.contents["loop"] = 0
28        return status
29
30    def reset(self, expectedError, result):
31        """This method is used to initialize self.expectedError and self.result
32        for each loop in runtests."""
33        self.expectedError = expectedError
34        self.result = result
35
36    def get_tests_for_bisection(self, options, tests):
37        """Make a list of tests for bisection from a given list of tests"""
38        bisectlist = []
39        for test in tests:
40            bisectlist.append(test)
41            if test.endswith(options.bisectChunk):
42                break
43
44        return bisectlist
45
46    def pre_test(self, options, tests, status):
47        """This method is used to call other methods for setting up variables and
48        getting the list of tests for bisection."""
49        if options.bisectChunk == "default":
50            return tests
51        # The second condition in 'if' is required to verify that the failing
52        # test is the last one.
53        elif "loop" not in self.contents or not self.contents["tests"][-1].endswith(
54            options.bisectChunk
55        ):
56            tests = self.get_tests_for_bisection(options, tests)
57            status = self.setup(tests)
58
59        return self.next_chunk_binary(options, status)
60
61    def post_test(self, options, expectedError, result):
62        """This method is used to call other methods to summarize results and check whether a
63        sanity check is done or not."""
64        self.reset(expectedError, result)
65        status = self.summarize_chunk(options)
66        # Check whether sanity check has to be done. Also it is necessary to check whether
67        # options.bisectChunk is present in self.expectedError as we do not want to run
68        # if it is "default".
69        if status == -1 and options.bisectChunk in self.expectedError:
70            # In case we have a debug build, we don't want to run a sanity
71            # check, will take too much time.
72            if mozinfo.info["debug"]:
73                return status
74
75            testBleedThrough = self.contents["testsToRun"][0]
76            tests = self.contents["totalTests"]
77            tests.remove(testBleedThrough)
78            # To make sure that the failing test is dependent on some other
79            # test.
80            if options.bisectChunk in testBleedThrough:
81                return status
82
83            status = self.setup(tests)
84            self.summary.append("Sanity Check:")
85
86        return status
87
88    def next_chunk_reverse(self, options, status):
89        "This method is used to bisect the tests in a reverse search fashion."
90
91        # Base Cases.
92        if self.contents["loop"] <= 1:
93            self.contents["testsToRun"] = self.contents["tests"]
94            if self.contents["loop"] == 1:
95                self.contents["testsToRun"] = [self.contents["tests"][-1]]
96            self.contents["loop"] += 1
97            return self.contents["testsToRun"]
98
99        if "result" in self.contents:
100            if self.contents["result"] == "PASS":
101                chunkSize = self.contents["end"] - self.contents["start"]
102                self.contents["end"] = self.contents["start"] - 1
103                self.contents["start"] = self.contents["end"] - chunkSize
104
105            # self.contents['result'] will be expected error only if it fails.
106            elif self.contents["result"] == "FAIL":
107                self.contents["tests"] = self.contents["testsToRun"]
108                status = 1  # for initializing
109
110        # initialize
111        if status:
112            totalTests = len(self.contents["tests"])
113            chunkSize = int(math.ceil(totalTests / 10.0))
114            self.contents["start"] = totalTests - chunkSize - 1
115            self.contents["end"] = totalTests - 2
116
117        start = self.contents["start"]
118        end = self.contents["end"] + 1
119        self.contents["testsToRun"] = self.contents["tests"][start:end]
120        self.contents["testsToRun"].append(self.contents["tests"][-1])
121        self.contents["loop"] += 1
122
123        return self.contents["testsToRun"]
124
125    def next_chunk_binary(self, options, status):
126        "This method is used to bisect the tests in a binary search fashion."
127
128        # Base cases.
129        if self.contents["loop"] <= 1:
130            self.contents["testsToRun"] = self.contents["tests"]
131            if self.contents["loop"] == 1:
132                self.contents["testsToRun"] = [self.contents["tests"][-1]]
133            self.contents["loop"] += 1
134            return self.contents["testsToRun"]
135
136        # Initialize the contents dict.
137        if status:
138            totalTests = len(self.contents["tests"])
139            self.contents["start"] = 0
140            self.contents["end"] = totalTests - 2
141
142        # pylint --py3k W1619
143        mid = (self.contents["start"] + self.contents["end"]) / 2
144        if "result" in self.contents:
145            if self.contents["result"] == "PASS":
146                self.contents["end"] = mid
147
148            elif self.contents["result"] == "FAIL":
149                self.contents["start"] = mid + 1
150
151        mid = (self.contents["start"] + self.contents["end"]) / 2
152        start = mid + 1
153        end = self.contents["end"] + 1
154        self.contents["testsToRun"] = self.contents["tests"][start:end]
155        if not self.contents["testsToRun"]:
156            self.contents["testsToRun"].append(self.contents["tests"][mid])
157        self.contents["testsToRun"].append(self.contents["tests"][-1])
158        self.contents["loop"] += 1
159
160        return self.contents["testsToRun"]
161
162    def summarize_chunk(self, options):
163        "This method is used summarize the results after the list of tests is run."
164        if options.bisectChunk == "default":
165            # if no expectedError that means all the tests have successfully
166            # passed.
167            if len(self.expectedError) == 0:
168                return -1
169            options.bisectChunk = self.expectedError.keys()[0]
170            self.summary.append("\tFound Error in test: %s" % options.bisectChunk)
171            return 0
172
173        # If options.bisectChunk is not in self.result then we need to move to
174        # the next run.
175        if options.bisectChunk not in self.result:
176            return -1
177
178        self.summary.append("\tPass %d:" % self.contents["loop"])
179        if len(self.contents["testsToRun"]) > 1:
180            self.summary.append(
181                "\t\t%d test files(start,end,failing). [%s, %s, %s]"
182                % (
183                    len(self.contents["testsToRun"]),
184                    self.contents["testsToRun"][0],
185                    self.contents["testsToRun"][-2],
186                    self.contents["testsToRun"][-1],
187                )
188            )
189        else:
190            self.summary.append("\t\t1 test file [%s]" % self.contents["testsToRun"][0])
191            return self.check_for_intermittent(options)
192
193        if self.result[options.bisectChunk] == "PASS":
194            self.summary.append("\t\tno failures found.")
195            if self.contents["loop"] == 1:
196                status = -1
197            else:
198                self.contents["result"] = "PASS"
199                status = 0
200
201        elif self.result[options.bisectChunk] == "FAIL":
202            if "expectedError" not in self.contents:
203                self.summary.append("\t\t%s failed." % self.contents["testsToRun"][-1])
204                self.contents["expectedError"] = self.expectedError[options.bisectChunk]
205                status = 0
206
207            elif (
208                self.expectedError[options.bisectChunk]
209                == self.contents["expectedError"]
210            ):
211                self.summary.append(
212                    "\t\t%s failed with expected error."
213                    % self.contents["testsToRun"][-1]
214                )
215                self.contents["result"] = "FAIL"
216                status = 0
217
218                # This code checks for test-bleedthrough. Should work for any
219                # algorithm.
220                numberOfTests = len(self.contents["testsToRun"])
221                if numberOfTests < 3:
222                    # This means that only 2 tests are run. Since the last test
223                    # is the failing test itself therefore the bleedthrough
224                    # test is the first test
225                    self.summary.append(
226                        "TEST-UNEXPECTED-FAIL | %s | Bleedthrough detected, this test is the "
227                        "root cause for many of the above failures"
228                        % self.contents["testsToRun"][0]
229                    )
230                    status = -1
231            else:
232                self.summary.append(
233                    "\t\t%s failed with different error."
234                    % self.contents["testsToRun"][-1]
235                )
236                status = -1
237
238        return status
239
240    def check_for_intermittent(self, options):
241        "This method is used to check whether a test is an intermittent."
242        if self.result[options.bisectChunk] == "PASS":
243            self.summary.append(
244                "\t\tThe test %s passed." % self.contents["testsToRun"][0]
245            )
246            if self.repeat > 0:
247                # loop is set to 1 to again run the single test.
248                self.contents["loop"] = 1
249                self.repeat -= 1
250                return 0
251            else:
252                if self.failcount > 0:
253                    # -1 is being returned as the test is intermittent, so no need to bisect
254                    # further.
255                    return -1
256                # If the test does not fail even once, then proceed to next chunk for bisection.
257                # loop is set to 2 to proceed on bisection.
258                self.contents["loop"] = 2
259                return 1
260        elif self.result[options.bisectChunk] == "FAIL":
261            self.summary.append(
262                "\t\tThe test %s failed." % self.contents["testsToRun"][0]
263            )
264            self.failcount += 1
265            self.contents["loop"] = 1
266            self.repeat -= 1
267            # self.max_failures is the maximum number of times a test is allowed
268            # to fail to be called an intermittent. If a test fails more than
269            # limit set, it is a perma-fail.
270            if self.failcount < self.max_failures:
271                if self.repeat == 0:
272                    # -1 is being returned as the test is intermittent, so no need to bisect
273                    # further.
274                    return -1
275                return 0
276            else:
277                self.summary.append(
278                    "TEST-UNEXPECTED-FAIL | %s | Bleedthrough detected, this test is the "
279                    "root cause for many of the above failures"
280                    % self.contents["testsToRun"][0]
281                )
282                return -1
283
284    def print_summary(self):
285        "This method is used to print the recorded summary."
286        print("Bisection summary:")
287        for line in self.summary:
288            print(line)
289