1# Copyright 2018 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5from __future__ import print_function 6from __future__ import division 7from __future__ import absolute_import 8 9import logging 10import math 11import unittest 12import sys 13 14from dashboard.pinpoint import test 15from dashboard.pinpoint.models import job_state 16from dashboard.pinpoint.models.change import change_test 17from dashboard.pinpoint.models.quest import quest_test 18 19 20class ExploreTest(test.TestCase): 21 22 def setUp(self): 23 # Intercept the logging messages, so that we can see them when we have test 24 # output in failures. 25 self.logger = logging.getLogger() 26 self.logger.level = logging.DEBUG 27 self.stream_handler = logging.StreamHandler(sys.stdout) 28 self.logger.addHandler(self.stream_handler) 29 self.addCleanup(self.logger.removeHandler, self.stream_handler) 30 super(ExploreTest, self).setUp() 31 32 def testDifferentWithMidpoint(self): 33 quests = [ 34 quest_test.QuestByChange({ 35 change_test.Change(1): quest_test.QuestPass(), 36 change_test.Change(9): quest_test.QuestFail(), 37 }) 38 ] 39 state = job_state.JobState(quests, comparison_mode=job_state.PERFORMANCE) 40 state.AddChange(change_test.Change(1)) 41 state.AddChange(change_test.Change(9)) 42 43 state.ScheduleWork() 44 state.Explore() 45 46 # The Changes are different. Add the speculated modpoints in a two-level 47 # deep bisection tree, which include 3, 5 (the midpoint between 0 and 9), 48 # and 7. 49 expected = [ 50 change_test.Change(1), 51 change_test.Change(3), 52 change_test.Change(5), 53 change_test.Change(7), 54 change_test.Change(9), 55 ] 56 57 # View the whole diff in case of failure. 58 self.maxDiff = None 59 self.assertEqual(state._changes, expected) 60 attempt_count_1 = len(state._attempts[change_test.Change(1)]) 61 attempt_count_2 = len(state._attempts[change_test.Change(3)]) 62 attempt_count_3 = len(state._attempts[change_test.Change(5)]) 63 attempt_count_4 = len(state._attempts[change_test.Change(7)]) 64 attempt_count_5 = len(state._attempts[change_test.Change(9)]) 65 self.assertEqual(attempt_count_1, attempt_count_2) 66 self.assertEqual(attempt_count_2, attempt_count_3) 67 self.assertEqual(attempt_count_3, attempt_count_4) 68 self.assertEqual(attempt_count_4, attempt_count_5) 69 self.assertEqual([], state.Differences()) 70 71 def testDifferentNoMidpoint(self): 72 quests = [ 73 quest_test.QuestByChange({ 74 change_test.Change(1): quest_test.QuestPass(), 75 change_test.Change(2): quest_test.QuestFail(), 76 }) 77 ] 78 state = job_state.JobState(quests, comparison_mode=job_state.PERFORMANCE) 79 state.AddChange(change_test.Change(1)) 80 state.AddChange(change_test.Change(2)) 81 82 state.ScheduleWork() 83 state.Explore() 84 85 # The Changes are different, but there's no midpoint. We're done. 86 self.assertEqual(len(state._changes), 2) 87 attempt_count_1 = len(state._attempts[change_test.Change(1)]) 88 attempt_count_2 = len(state._attempts[change_test.Change(2)]) 89 self.assertEqual(attempt_count_1, attempt_count_2) 90 self.assertEqual([(change_test.Change(1), change_test.Change(2))], 91 state.Differences()) 92 93 def testPending(self): 94 quests = [quest_test.QuestSpin()] 95 state = job_state.JobState(quests, comparison_mode=job_state.PERFORMANCE) 96 state.AddChange(change_test.Change(1)) 97 state.AddChange(change_test.Change(9)) 98 99 state.Explore() 100 101 # The results are pending. Do not add any Attempts or Changes. 102 self.assertEqual(len(state._changes), 2) 103 attempt_count_1 = len(state._attempts[change_test.Change(1)]) 104 attempt_count_2 = len(state._attempts[change_test.Change(9)]) 105 self.assertEqual(attempt_count_1, attempt_count_2) 106 self.assertEqual([], state.Differences()) 107 108 def testSame(self): 109 quests = [quest_test.QuestPass()] 110 state = job_state.JobState(quests, comparison_mode=job_state.FUNCTIONAL) 111 state.AddChange(change_test.Change(1)) 112 state.AddChange(change_test.Change(9)) 113 for _ in range(5): 114 # More Attempts give more confidence that they are, indeed, the same. 115 state.AddAttempts(change_test.Change(1)) 116 state.AddAttempts(change_test.Change(9)) 117 118 state.ScheduleWork() 119 state.Explore() 120 121 # The Changes are the same. Do not add any Attempts or Changes. 122 self.assertEqual(len(state._changes), 2) 123 attempt_count_1 = len(state._attempts[change_test.Change(1)]) 124 attempt_count_2 = len(state._attempts[change_test.Change(9)]) 125 self.assertEqual(attempt_count_1, attempt_count_2) 126 self.assertEqual([], state.Differences()) 127 128 def testUnknown(self): 129 quests = [quest_test.QuestPass()] 130 state = job_state.JobState( 131 quests, comparison_mode=job_state.FUNCTIONAL, comparison_magnitude=0.2) 132 state.AddChange(change_test.Change(1)) 133 state.AddChange(change_test.Change(9)) 134 135 state.ScheduleWork() 136 state.Explore() 137 138 # Need more information. Add more attempts to one Change. 139 self.assertEqual(len(state._changes), 2) 140 attempt_count_1 = len(state._attempts[change_test.Change(1)]) 141 attempt_count_2 = len(state._attempts[change_test.Change(9)]) 142 self.assertGreaterEqual(attempt_count_1, attempt_count_2) 143 144 state.ScheduleWork() 145 state.Explore() 146 147 # We still need more information. Add more attempts to the other Change. 148 self.assertEqual(len(state._changes), 2) 149 attempt_count_1 = len(state._attempts[change_test.Change(1)]) 150 attempt_count_2 = len(state._attempts[change_test.Change(9)]) 151 self.assertEqual(attempt_count_1, attempt_count_2) 152 153 def testDifferences(self): 154 quests = [quest_test.QuestPass()] 155 state = job_state.JobState( 156 quests, comparison_mode=job_state.FUNCTIONAL, comparison_magnitude=0.2) 157 change_a = change_test.Change(1) 158 change_b = change_test.Change(9) 159 state.AddChange(change_a) 160 state.AddChange(change_b) 161 self.assertEqual([], state.Differences()) 162 163 164class ScheduleWorkTest(unittest.TestCase): 165 166 def testNoAttempts(self): 167 state = job_state.JobState(()) 168 self.assertFalse(state.ScheduleWork()) 169 170 def testWorkLeft(self): 171 quests = [ 172 quest_test.QuestCycle(quest_test.QuestPass(), quest_test.QuestSpin()) 173 ] 174 state = job_state.JobState(quests) 175 state.AddChange(change_test.Change(123)) 176 self.assertTrue(state.ScheduleWork()) 177 178 def testNoWorkLeft(self): 179 quests = [quest_test.QuestPass()] 180 state = job_state.JobState(quests) 181 state.AddChange(change_test.Change(123)) 182 self.assertTrue(state.ScheduleWork()) 183 self.assertFalse(state.ScheduleWork()) 184 185 def testAllAttemptsFail(self): 186 quests = [ 187 quest_test.QuestCycle(quest_test.QuestFail(), quest_test.QuestFail(), 188 quest_test.QuestFail2()) 189 ] 190 state = job_state.JobState(quests) 191 state.AddChange(change_test.Change(123)) 192 expected_regexp = ('.*7/10.*\nInformationalError: Expected error for ' 193 'testing.$') 194 self.assertTrue(state.ScheduleWork()) 195 with self.assertRaisesRegexp(Exception, expected_regexp): 196 self.assertFalse(state.ScheduleWork()) 197 198 199class MeanTest(unittest.TestCase): 200 201 def testValidValues(self): 202 self.assertEqual(2, job_state.Mean([1, 2, 3])) 203 204 def testInvalidValues(self): 205 self.assertEqual(2, job_state.Mean([1, 2, 3, None])) 206 207 def testNoValues(self): 208 self.assertTrue(math.isnan(job_state.Mean([None]))) 209 210 211class FirstOrLastChangeFailedTest(unittest.TestCase): 212 213 def testNoChanges(self): 214 state = job_state.JobState(()) 215 self.assertFalse(state.FirstOrLastChangeFailed()) 216 217 def testNoFailedAttempts(self): 218 quests = [quest_test.QuestPass()] 219 state = job_state.JobState(quests) 220 state.AddChange(change_test.Change(1)) 221 state.AddChange(change_test.Change(2)) 222 state.ScheduleWork() 223 self.assertFalse(state.FirstOrLastChangeFailed()) 224 225 def testFailedAttempt(self): 226 quests = [quest_test.QuestPass()] 227 state = job_state.JobState(quests) 228 state.AddChange(change_test.Change(1)) 229 state.AddChange(change_test.Change(2)) 230 state.ScheduleWork() 231 for attempt in state._attempts[change_test.Change(1)]: 232 attempt._last_execution._exception = "Failed" 233 self.assertTrue(state.FirstOrLastChangeFailed()) 234 235 def testNoAttempts(self): 236 quests = [quest_test.QuestPass()] 237 state = job_state.JobState(quests) 238 state.AddChange(change_test.Change(1)) 239 state.AddChange(change_test.Change(2)) 240 state.ScheduleWork() 241 # It shouldn't happen that a change has no attempts, but we should cope 242 # gracefully if that somehow happens. 243 del state._attempts[change_test.Change(1)][:] 244 del state._attempts[change_test.Change(2)][:] 245 self.assertFalse(state.FirstOrLastChangeFailed()) 246