1# Copyright 2018 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5from __future__ import print_function
6from __future__ import division
7from __future__ import absolute_import
8
9import logging
10import math
11import unittest
12import sys
13
14from dashboard.pinpoint import test
15from dashboard.pinpoint.models import job_state
16from dashboard.pinpoint.models.change import change_test
17from dashboard.pinpoint.models.quest import quest_test
18
19
20class ExploreTest(test.TestCase):
21
22  def setUp(self):
23    # Intercept the logging messages, so that we can see them when we have test
24    # output in failures.
25    self.logger = logging.getLogger()
26    self.logger.level = logging.DEBUG
27    self.stream_handler = logging.StreamHandler(sys.stdout)
28    self.logger.addHandler(self.stream_handler)
29    self.addCleanup(self.logger.removeHandler, self.stream_handler)
30    super(ExploreTest, self).setUp()
31
32  def testDifferentWithMidpoint(self):
33    quests = [
34        quest_test.QuestByChange({
35            change_test.Change(1): quest_test.QuestPass(),
36            change_test.Change(9): quest_test.QuestFail(),
37        })
38    ]
39    state = job_state.JobState(quests, comparison_mode=job_state.PERFORMANCE)
40    state.AddChange(change_test.Change(1))
41    state.AddChange(change_test.Change(9))
42
43    state.ScheduleWork()
44    state.Explore()
45
46    # The Changes are different. Add the speculated modpoints in a two-level
47    # deep bisection tree, which include 3, 5 (the midpoint between 0 and 9),
48    # and 7.
49    expected = [
50        change_test.Change(1),
51        change_test.Change(3),
52        change_test.Change(5),
53        change_test.Change(7),
54        change_test.Change(9),
55    ]
56
57    # View the whole diff in case of failure.
58    self.maxDiff = None
59    self.assertEqual(state._changes, expected)
60    attempt_count_1 = len(state._attempts[change_test.Change(1)])
61    attempt_count_2 = len(state._attempts[change_test.Change(3)])
62    attempt_count_3 = len(state._attempts[change_test.Change(5)])
63    attempt_count_4 = len(state._attempts[change_test.Change(7)])
64    attempt_count_5 = len(state._attempts[change_test.Change(9)])
65    self.assertEqual(attempt_count_1, attempt_count_2)
66    self.assertEqual(attempt_count_2, attempt_count_3)
67    self.assertEqual(attempt_count_3, attempt_count_4)
68    self.assertEqual(attempt_count_4, attempt_count_5)
69    self.assertEqual([], state.Differences())
70
71  def testDifferentNoMidpoint(self):
72    quests = [
73        quest_test.QuestByChange({
74            change_test.Change(1): quest_test.QuestPass(),
75            change_test.Change(2): quest_test.QuestFail(),
76        })
77    ]
78    state = job_state.JobState(quests, comparison_mode=job_state.PERFORMANCE)
79    state.AddChange(change_test.Change(1))
80    state.AddChange(change_test.Change(2))
81
82    state.ScheduleWork()
83    state.Explore()
84
85    # The Changes are different, but there's no midpoint. We're done.
86    self.assertEqual(len(state._changes), 2)
87    attempt_count_1 = len(state._attempts[change_test.Change(1)])
88    attempt_count_2 = len(state._attempts[change_test.Change(2)])
89    self.assertEqual(attempt_count_1, attempt_count_2)
90    self.assertEqual([(change_test.Change(1), change_test.Change(2))],
91                     state.Differences())
92
93  def testPending(self):
94    quests = [quest_test.QuestSpin()]
95    state = job_state.JobState(quests, comparison_mode=job_state.PERFORMANCE)
96    state.AddChange(change_test.Change(1))
97    state.AddChange(change_test.Change(9))
98
99    state.Explore()
100
101    # The results are pending. Do not add any Attempts or Changes.
102    self.assertEqual(len(state._changes), 2)
103    attempt_count_1 = len(state._attempts[change_test.Change(1)])
104    attempt_count_2 = len(state._attempts[change_test.Change(9)])
105    self.assertEqual(attempt_count_1, attempt_count_2)
106    self.assertEqual([], state.Differences())
107
108  def testSame(self):
109    quests = [quest_test.QuestPass()]
110    state = job_state.JobState(quests, comparison_mode=job_state.FUNCTIONAL)
111    state.AddChange(change_test.Change(1))
112    state.AddChange(change_test.Change(9))
113    for _ in range(5):
114      # More Attempts give more confidence that they are, indeed, the same.
115      state.AddAttempts(change_test.Change(1))
116      state.AddAttempts(change_test.Change(9))
117
118    state.ScheduleWork()
119    state.Explore()
120
121    # The Changes are the same. Do not add any Attempts or Changes.
122    self.assertEqual(len(state._changes), 2)
123    attempt_count_1 = len(state._attempts[change_test.Change(1)])
124    attempt_count_2 = len(state._attempts[change_test.Change(9)])
125    self.assertEqual(attempt_count_1, attempt_count_2)
126    self.assertEqual([], state.Differences())
127
128  def testUnknown(self):
129    quests = [quest_test.QuestPass()]
130    state = job_state.JobState(
131        quests, comparison_mode=job_state.FUNCTIONAL, comparison_magnitude=0.2)
132    state.AddChange(change_test.Change(1))
133    state.AddChange(change_test.Change(9))
134
135    state.ScheduleWork()
136    state.Explore()
137
138    # Need more information. Add more attempts to one Change.
139    self.assertEqual(len(state._changes), 2)
140    attempt_count_1 = len(state._attempts[change_test.Change(1)])
141    attempt_count_2 = len(state._attempts[change_test.Change(9)])
142    self.assertGreaterEqual(attempt_count_1, attempt_count_2)
143
144    state.ScheduleWork()
145    state.Explore()
146
147    # We still need more information. Add more attempts to the other Change.
148    self.assertEqual(len(state._changes), 2)
149    attempt_count_1 = len(state._attempts[change_test.Change(1)])
150    attempt_count_2 = len(state._attempts[change_test.Change(9)])
151    self.assertEqual(attempt_count_1, attempt_count_2)
152
153  def testDifferences(self):
154    quests = [quest_test.QuestPass()]
155    state = job_state.JobState(
156        quests, comparison_mode=job_state.FUNCTIONAL, comparison_magnitude=0.2)
157    change_a = change_test.Change(1)
158    change_b = change_test.Change(9)
159    state.AddChange(change_a)
160    state.AddChange(change_b)
161    self.assertEqual([], state.Differences())
162
163
164class ScheduleWorkTest(unittest.TestCase):
165
166  def testNoAttempts(self):
167    state = job_state.JobState(())
168    self.assertFalse(state.ScheduleWork())
169
170  def testWorkLeft(self):
171    quests = [
172        quest_test.QuestCycle(quest_test.QuestPass(), quest_test.QuestSpin())
173    ]
174    state = job_state.JobState(quests)
175    state.AddChange(change_test.Change(123))
176    self.assertTrue(state.ScheduleWork())
177
178  def testNoWorkLeft(self):
179    quests = [quest_test.QuestPass()]
180    state = job_state.JobState(quests)
181    state.AddChange(change_test.Change(123))
182    self.assertTrue(state.ScheduleWork())
183    self.assertFalse(state.ScheduleWork())
184
185  def testAllAttemptsFail(self):
186    quests = [
187        quest_test.QuestCycle(quest_test.QuestFail(), quest_test.QuestFail(),
188                              quest_test.QuestFail2())
189    ]
190    state = job_state.JobState(quests)
191    state.AddChange(change_test.Change(123))
192    expected_regexp = ('.*7/10.*\nInformationalError: Expected error for '
193                       'testing.$')
194    self.assertTrue(state.ScheduleWork())
195    with self.assertRaisesRegexp(Exception, expected_regexp):
196      self.assertFalse(state.ScheduleWork())
197
198
199class MeanTest(unittest.TestCase):
200
201  def testValidValues(self):
202    self.assertEqual(2, job_state.Mean([1, 2, 3]))
203
204  def testInvalidValues(self):
205    self.assertEqual(2, job_state.Mean([1, 2, 3, None]))
206
207  def testNoValues(self):
208    self.assertTrue(math.isnan(job_state.Mean([None])))
209
210
211class FirstOrLastChangeFailedTest(unittest.TestCase):
212
213  def testNoChanges(self):
214    state = job_state.JobState(())
215    self.assertFalse(state.FirstOrLastChangeFailed())
216
217  def testNoFailedAttempts(self):
218    quests = [quest_test.QuestPass()]
219    state = job_state.JobState(quests)
220    state.AddChange(change_test.Change(1))
221    state.AddChange(change_test.Change(2))
222    state.ScheduleWork()
223    self.assertFalse(state.FirstOrLastChangeFailed())
224
225  def testFailedAttempt(self):
226    quests = [quest_test.QuestPass()]
227    state = job_state.JobState(quests)
228    state.AddChange(change_test.Change(1))
229    state.AddChange(change_test.Change(2))
230    state.ScheduleWork()
231    for attempt in state._attempts[change_test.Change(1)]:
232      attempt._last_execution._exception = "Failed"
233    self.assertTrue(state.FirstOrLastChangeFailed())
234
235  def testNoAttempts(self):
236    quests = [quest_test.QuestPass()]
237    state = job_state.JobState(quests)
238    state.AddChange(change_test.Change(1))
239    state.AddChange(change_test.Change(2))
240    state.ScheduleWork()
241    # It shouldn't happen that a change has no attempts, but we should cope
242    # gracefully if that somehow happens.
243    del state._attempts[change_test.Change(1)][:]
244    del state._attempts[change_test.Change(2)][:]
245    self.assertFalse(state.FirstOrLastChangeFailed())
246