1from __future__ import absolute_import, division, print_function 2 3import math 4import mozinfo 5 6 7class Bisect(object): 8 9 "Class for creating, bisecting and summarizing for --bisect-chunk option." 10 11 def __init__(self, harness): 12 super(Bisect, self).__init__() 13 self.summary = [] 14 self.contents = {} 15 self.repeat = 10 16 self.failcount = 0 17 self.max_failures = 3 18 19 def setup(self, tests): 20 """This method is used to initialize various variables that are required 21 for test bisection""" 22 status = 0 23 self.contents.clear() 24 # We need totalTests key in contents for sanity check 25 self.contents["totalTests"] = tests 26 self.contents["tests"] = tests 27 self.contents["loop"] = 0 28 return status 29 30 def reset(self, expectedError, result): 31 """This method is used to initialize self.expectedError and self.result 32 for each loop in runtests.""" 33 self.expectedError = expectedError 34 self.result = result 35 36 def get_tests_for_bisection(self, options, tests): 37 """Make a list of tests for bisection from a given list of tests""" 38 bisectlist = [] 39 for test in tests: 40 bisectlist.append(test) 41 if test.endswith(options.bisectChunk): 42 break 43 44 return bisectlist 45 46 def pre_test(self, options, tests, status): 47 """This method is used to call other methods for setting up variables and 48 getting the list of tests for bisection.""" 49 if options.bisectChunk == "default": 50 return tests 51 # The second condition in 'if' is required to verify that the failing 52 # test is the last one. 53 elif "loop" not in self.contents or not self.contents["tests"][-1].endswith( 54 options.bisectChunk 55 ): 56 tests = self.get_tests_for_bisection(options, tests) 57 status = self.setup(tests) 58 59 return self.next_chunk_binary(options, status) 60 61 def post_test(self, options, expectedError, result): 62 """This method is used to call other methods to summarize results and check whether a 63 sanity check is done or not.""" 64 self.reset(expectedError, result) 65 status = self.summarize_chunk(options) 66 # Check whether sanity check has to be done. Also it is necessary to check whether 67 # options.bisectChunk is present in self.expectedError as we do not want to run 68 # if it is "default". 69 if status == -1 and options.bisectChunk in self.expectedError: 70 # In case we have a debug build, we don't want to run a sanity 71 # check, will take too much time. 72 if mozinfo.info["debug"]: 73 return status 74 75 testBleedThrough = self.contents["testsToRun"][0] 76 tests = self.contents["totalTests"] 77 tests.remove(testBleedThrough) 78 # To make sure that the failing test is dependent on some other 79 # test. 80 if options.bisectChunk in testBleedThrough: 81 return status 82 83 status = self.setup(tests) 84 self.summary.append("Sanity Check:") 85 86 return status 87 88 def next_chunk_reverse(self, options, status): 89 "This method is used to bisect the tests in a reverse search fashion." 90 91 # Base Cases. 92 if self.contents["loop"] <= 1: 93 self.contents["testsToRun"] = self.contents["tests"] 94 if self.contents["loop"] == 1: 95 self.contents["testsToRun"] = [self.contents["tests"][-1]] 96 self.contents["loop"] += 1 97 return self.contents["testsToRun"] 98 99 if "result" in self.contents: 100 if self.contents["result"] == "PASS": 101 chunkSize = self.contents["end"] - self.contents["start"] 102 self.contents["end"] = self.contents["start"] - 1 103 self.contents["start"] = self.contents["end"] - chunkSize 104 105 # self.contents['result'] will be expected error only if it fails. 106 elif self.contents["result"] == "FAIL": 107 self.contents["tests"] = self.contents["testsToRun"] 108 status = 1 # for initializing 109 110 # initialize 111 if status: 112 totalTests = len(self.contents["tests"]) 113 chunkSize = int(math.ceil(totalTests / 10.0)) 114 self.contents["start"] = totalTests - chunkSize - 1 115 self.contents["end"] = totalTests - 2 116 117 start = self.contents["start"] 118 end = self.contents["end"] + 1 119 self.contents["testsToRun"] = self.contents["tests"][start:end] 120 self.contents["testsToRun"].append(self.contents["tests"][-1]) 121 self.contents["loop"] += 1 122 123 return self.contents["testsToRun"] 124 125 def next_chunk_binary(self, options, status): 126 "This method is used to bisect the tests in a binary search fashion." 127 128 # Base cases. 129 if self.contents["loop"] <= 1: 130 self.contents["testsToRun"] = self.contents["tests"] 131 if self.contents["loop"] == 1: 132 self.contents["testsToRun"] = [self.contents["tests"][-1]] 133 self.contents["loop"] += 1 134 return self.contents["testsToRun"] 135 136 # Initialize the contents dict. 137 if status: 138 totalTests = len(self.contents["tests"]) 139 self.contents["start"] = 0 140 self.contents["end"] = totalTests - 2 141 142 # pylint --py3k W1619 143 mid = (self.contents["start"] + self.contents["end"]) / 2 144 if "result" in self.contents: 145 if self.contents["result"] == "PASS": 146 self.contents["end"] = mid 147 148 elif self.contents["result"] == "FAIL": 149 self.contents["start"] = mid + 1 150 151 mid = (self.contents["start"] + self.contents["end"]) / 2 152 start = mid + 1 153 end = self.contents["end"] + 1 154 self.contents["testsToRun"] = self.contents["tests"][start:end] 155 if not self.contents["testsToRun"]: 156 self.contents["testsToRun"].append(self.contents["tests"][mid]) 157 self.contents["testsToRun"].append(self.contents["tests"][-1]) 158 self.contents["loop"] += 1 159 160 return self.contents["testsToRun"] 161 162 def summarize_chunk(self, options): 163 "This method is used summarize the results after the list of tests is run." 164 if options.bisectChunk == "default": 165 # if no expectedError that means all the tests have successfully 166 # passed. 167 if len(self.expectedError) == 0: 168 return -1 169 options.bisectChunk = self.expectedError.keys()[0] 170 self.summary.append("\tFound Error in test: %s" % options.bisectChunk) 171 return 0 172 173 # If options.bisectChunk is not in self.result then we need to move to 174 # the next run. 175 if options.bisectChunk not in self.result: 176 return -1 177 178 self.summary.append("\tPass %d:" % self.contents["loop"]) 179 if len(self.contents["testsToRun"]) > 1: 180 self.summary.append( 181 "\t\t%d test files(start,end,failing). [%s, %s, %s]" 182 % ( 183 len(self.contents["testsToRun"]), 184 self.contents["testsToRun"][0], 185 self.contents["testsToRun"][-2], 186 self.contents["testsToRun"][-1], 187 ) 188 ) 189 else: 190 self.summary.append("\t\t1 test file [%s]" % self.contents["testsToRun"][0]) 191 return self.check_for_intermittent(options) 192 193 if self.result[options.bisectChunk] == "PASS": 194 self.summary.append("\t\tno failures found.") 195 if self.contents["loop"] == 1: 196 status = -1 197 else: 198 self.contents["result"] = "PASS" 199 status = 0 200 201 elif self.result[options.bisectChunk] == "FAIL": 202 if "expectedError" not in self.contents: 203 self.summary.append("\t\t%s failed." % self.contents["testsToRun"][-1]) 204 self.contents["expectedError"] = self.expectedError[options.bisectChunk] 205 status = 0 206 207 elif ( 208 self.expectedError[options.bisectChunk] 209 == self.contents["expectedError"] 210 ): 211 self.summary.append( 212 "\t\t%s failed with expected error." 213 % self.contents["testsToRun"][-1] 214 ) 215 self.contents["result"] = "FAIL" 216 status = 0 217 218 # This code checks for test-bleedthrough. Should work for any 219 # algorithm. 220 numberOfTests = len(self.contents["testsToRun"]) 221 if numberOfTests < 3: 222 # This means that only 2 tests are run. Since the last test 223 # is the failing test itself therefore the bleedthrough 224 # test is the first test 225 self.summary.append( 226 "TEST-UNEXPECTED-FAIL | %s | Bleedthrough detected, this test is the " 227 "root cause for many of the above failures" 228 % self.contents["testsToRun"][0] 229 ) 230 status = -1 231 else: 232 self.summary.append( 233 "\t\t%s failed with different error." 234 % self.contents["testsToRun"][-1] 235 ) 236 status = -1 237 238 return status 239 240 def check_for_intermittent(self, options): 241 "This method is used to check whether a test is an intermittent." 242 if self.result[options.bisectChunk] == "PASS": 243 self.summary.append( 244 "\t\tThe test %s passed." % self.contents["testsToRun"][0] 245 ) 246 if self.repeat > 0: 247 # loop is set to 1 to again run the single test. 248 self.contents["loop"] = 1 249 self.repeat -= 1 250 return 0 251 else: 252 if self.failcount > 0: 253 # -1 is being returned as the test is intermittent, so no need to bisect 254 # further. 255 return -1 256 # If the test does not fail even once, then proceed to next chunk for bisection. 257 # loop is set to 2 to proceed on bisection. 258 self.contents["loop"] = 2 259 return 1 260 elif self.result[options.bisectChunk] == "FAIL": 261 self.summary.append( 262 "\t\tThe test %s failed." % self.contents["testsToRun"][0] 263 ) 264 self.failcount += 1 265 self.contents["loop"] = 1 266 self.repeat -= 1 267 # self.max_failures is the maximum number of times a test is allowed 268 # to fail to be called an intermittent. If a test fails more than 269 # limit set, it is a perma-fail. 270 if self.failcount < self.max_failures: 271 if self.repeat == 0: 272 # -1 is being returned as the test is intermittent, so no need to bisect 273 # further. 274 return -1 275 return 0 276 else: 277 self.summary.append( 278 "TEST-UNEXPECTED-FAIL | %s | Bleedthrough detected, this test is the " 279 "root cause for many of the above failures" 280 % self.contents["testsToRun"][0] 281 ) 282 return -1 283 284 def print_summary(self): 285 "This method is used to print the recorded summary." 286 print("Bisection summary:") 287 for line in self.summary: 288 print(line) 289