1# Copyright 2019 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import os 6import shutil 7import tempfile 8import unittest 9 10import mock 11 12from cli_tools.pinboard import pinboard 13from core.external_modules import pandas as pd 14 15 16def StateItem(revision, **kwargs): 17 item = {'revision': revision, 18 'timestamp': kwargs.pop('timestamp', '2019-03-15'), 19 'jobs': []} 20 with_bots = False 21 if kwargs.get('with_bots'): 22 with_bots = kwargs.pop('with_bots') 23 for job_id, status in sorted(kwargs.items()): 24 job = {'id': job_id, 'status': status} 25 if with_bots: 26 job['bot'] = job_id 27 item['jobs'].append(job) 28 29 return item 30 31 32@unittest.skipIf(pd is None, 'pandas not available') 33class PinboardToolTests(unittest.TestCase): 34 def setUp(self): 35 self.cache_dir = tempfile.mkdtemp() 36 os.mkdir(os.path.join(self.cache_dir, 'job_results')) 37 mock.patch( 38 'cli_tools.pinboard.pinboard.CACHED_DATA_DIR', 39 new=self.cache_dir).start() 40 self.subprocess = mock.patch( 41 'cli_tools.pinboard.pinboard.subprocess').start() 42 self.upload_to_cloud = mock.patch( 43 'cli_tools.pinboard.pinboard.UploadToCloudStorage').start() 44 self.download_from_cloud = mock.patch( 45 'cli_tools.pinboard.pinboard.DownloadFromCloudStorage').start() 46 self.download_from_cloud.return_value = False 47 48 def tearDown(self): 49 mock.patch.stopall() 50 shutil.rmtree(self.cache_dir) 51 52 @mock.patch('cli_tools.pinboard.pinboard.GetLastCommitOfDate') 53 @mock.patch('cli_tools.pinboard.pinboard.LoadJsonFile') 54 def testStartPinpointJobs(self, load_configs, get_last_commit): 55 load_configs.return_value = [{ 56 'name': 'config1', 57 'configuration': 'AndroidGo' 58 }, { 59 'name': 'config2', 60 'configuration': 'Pixel2' 61 }] 62 get_last_commit.return_value = ('2a66bac4', '2019-03-17T23:50:16-07:00') 63 self.subprocess.check_output.side_effect = [ 64 'Started: https://pinpoint.example.com/job/14b4c451f40000\n', 65 'Started: https://pinpoint.example.com/job/11fae481f40000\n'] 66 state = [] 67 68 pinboard.StartPinpointJobs(state, '2019-03-17') 69 70 self.assertEqual(state, [{ 71 'revision': 72 '2a66bac4', 73 'timestamp': 74 '2019-03-17T23:50:16-07:00', 75 'jobs': [{ 76 'id': '14b4c451f40000', 77 'status': 'queued', 78 'bot': 'AndroidGo' 79 }, { 80 'id': '11fae481f40000', 81 'status': 'queued', 82 'bot': 'Pixel2' 83 }] 84 }]) 85 86 def testCollectPinpointResults(self): 87 state = [ 88 StateItem('a100', job1='completed', job2='completed'), 89 StateItem('a200', job3='completed', job4='running'), 90 StateItem('a300', job5='running', job6='running')] 91 92 # Write some fake "previous" results for first revision. 93 df = pd.DataFrame({'revision': ['a100']}) 94 df.to_csv(pinboard.RevisionResultsFile(state[0]), index=False) 95 96 self.subprocess.check_output.side_effect = [ 97 'job4: completed\n', 98 'job5: running\njob6: failed\n', 99 'getting csv data ...\n' 100 ] 101 expected_state = [ 102 StateItem('a100', job1='completed', job2='completed'), 103 StateItem('a200', job3='completed', job4='completed'), 104 StateItem('a300', job5='running', job6='failed')] 105 106 pinboard.CollectPinpointResults(state) 107 108 self.assertEqual(state, expected_state) 109 self.subprocess.check_output.assert_has_calls([ 110 mock.call(['vpython', pinboard.PINPOINT_CLI, 'status', 'job4'], 111 universal_newlines=True), 112 mock.call(['vpython', pinboard.PINPOINT_CLI, 'status', 'job5', 'job6'], 113 universal_newlines=True), 114 mock.call([ 115 'vpython', pinboard.PINPOINT_CLI, 'get-csv', '--output', 116 pinboard.RevisionResultsFile(state[1]), '--', 'job3', 'job4']) 117 ]) 118 119 def testUpdateJobsState(self): 120 state = pinboard.LoadJobsState() 121 self.assertEqual(state, []) 122 123 # Update state with new data a couple of times. 124 state.append(StateItem('a100')) 125 pinboard.UpdateJobsState(state) 126 state.append(StateItem('a200')) 127 pinboard.UpdateJobsState(state) 128 129 # No new data. Should be a no-op. 130 pinboard.UpdateJobsState(state) 131 132 stored_state = pinboard.LoadJobsState() 133 self.assertEqual(stored_state, state) 134 self.assertEqual([i['revision'] for i in stored_state], ['a100', 'a200']) 135 self.assertEqual(self.upload_to_cloud.call_count, 2) 136 137 @mock.patch('cli_tools.pinboard.pinboard.GetRevisionResults') 138 @mock.patch('cli_tools.pinboard.pinboard.TimeAgo') 139 def testAggregateAndUploadResults(self, time_ago, get_revision_results): 140 state = [ 141 StateItem('a100', timestamp='2019-03-15', job1='completed'), 142 StateItem('a200', timestamp='2019-03-16', job2='completed'), 143 StateItem('a300', timestamp='2019-03-17', job3='failed'), 144 StateItem('a400', timestamp='2019-03-18', job4='completed'), 145 StateItem('a500', timestamp='2019-03-19', job5='completed'), 146 ] 147 148 def GetFakeResults(item): 149 df = pd.DataFrame(index=[0]) 150 df['revision'] = item['revision'] 151 df['label'] = 'with_patch' 152 df['benchmark'] = 'loading' 153 df['name'] = 'Total:duration' 154 df['timestamp'] = pd.Timestamp(item['timestamp']) 155 df['count'] = 1 if item['revision'] != 'a400' else 0 156 return df 157 158 get_revision_results.side_effect = GetFakeResults 159 time_ago.return_value = pd.Timestamp('2018-10-20') 160 161 # Only process first few revisions. 162 new_items, cached_df = pinboard.GetItemsToUpdate(state[:3]) 163 pinboard.AggregateAndUploadResults(new_items, cached_df) 164 dataset_file = pinboard.CachedFilePath(pinboard.DATASET_CSV_FILE) 165 df = pd.read_csv(dataset_file) 166 self.assertEqual(set(df['revision']), set(['a100', 'a200'])) 167 self.assertTrue((df[df['reference']]['revision'] == 'a200').all()) 168 169 # Incrementally process the rest. 170 new_items, cached_df = pinboard.GetItemsToUpdate(state) 171 pinboard.AggregateAndUploadResults(new_items, cached_df) 172 dataset_file = pinboard.CachedFilePath(pinboard.DATASET_CSV_FILE) 173 df = pd.read_csv(dataset_file) 174 self.assertEqual(set(df['revision']), set(['a100', 'a200', 'a500'])) 175 self.assertTrue((df[df['reference']]['revision'] == 'a500').all()) 176 177 # No new revisions. This should be a no-op. 178 new_items, cached_df = pinboard.GetItemsToUpdate(state) 179 pinboard.AggregateAndUploadResults(new_items, cached_df) 180 181 self.assertEqual(get_revision_results.call_count, 4) 182 # Uploads twice (the pkl and csv) on each call to aggregate results. 183 self.assertEqual(self.upload_to_cloud.call_count, 2 * 2) 184 185 def testGetRevisionResults_different_bots(self): 186 item = StateItem( 187 '2a66ba', 188 timestamp='2019-03-17T23:50:16-07:00', 189 with_bots=True, 190 job1='completed', 191 job2='completed') 192 csv = [ 193 'change,benchmark,story,name,unit,mean,job_id\n', 194 '2a66ba,loading,story1,Total:duration,ms_smallerIsBetter,300.0,job1\n', 195 '2a66ba,loading,story2,Total:duration,ms_smallerIsBetter,400.0,job2\n', 196 '2a66ba+patch,loading,story1,Total:duration,ms_smallerIsBetter,100.0,' + 197 'job1\n', 198 '2a66ba+patch,loading,story2,Total:duration,ms_smallerIsBetter,200.0,' + 199 'job2\n', 200 '2a66ba,loading,story1,Other:metric,count_smallerIsBetter,1.0,job1\n' 201 ] 202 expected_results = [ 203 ('without_patch', 'job1', 0.3, '2018-03-17T12:00:00'), 204 ('with_patch', 'job1', 0.1, '2019-03-17T12:00:00'), 205 ('without_patch', 'job2', 0.4, '2018-03-17T12:00:00'), 206 ('with_patch', 'job2', 0.2, '2019-03-17T12:00:00'), 207 ] 208 209 filename = pinboard.RevisionResultsFile(item) 210 with open(filename, 'w') as f: 211 f.writelines(csv) 212 213 with mock.patch( 214 'cli_tools.pinboard.pinboard.ACTIVE_STORIES', new=['story1', 'story2']): 215 df = pinboard.GetRevisionResults(item) 216 217 self.assertEqual(len(df.index), 4) # Only two rows of output. 218 self.assertTrue((df['revision'] == '2a66ba').all()) 219 self.assertTrue((df['benchmark'] == 'loading').all()) 220 self.assertTrue((df['name'] == 'Total:duration').all()) 221 self.assertTrue((df['count'] == 1).all()) 222 df = df.set_index(['label', 'bot'], verify_integrity=True) 223 for label, bot, value, timestamp in expected_results: 224 self.assertEqual(df.loc[label, bot]['mean'], value) 225 self.assertEqual(df.loc[label, bot]['timestamp'], pd.Timestamp(timestamp)) 226 227 def testGetRevisionResults_simple(self): 228 item = StateItem('2a66ba', timestamp='2019-03-17T23:50:16-07:00') 229 csv = [ 230 'change,benchmark,story,name,unit,mean,job_id\n', 231 '2a66ba,loading,story1,Total:duration,ms_smallerIsBetter,300.0,job1\n', 232 '2a66ba,loading,story2,Total:duration,ms_smallerIsBetter,400.0,job1\n', 233 '2a66ba+patch,loading,story1,Total:duration,ms_smallerIsBetter,100.0,' + 234 'job1\n', 235 '2a66ba+patch,loading,story2,Total:duration,ms_smallerIsBetter,200.0,' + 236 'job1\n', 237 '2a66ba,loading,story1,Other:metric,count_smallerIsBetter,1.0,job1\n' 238 ] 239 expected_results = [ 240 ('without_patch', 0.35, '2018-03-17T12:00:00'), 241 ('with_patch', 0.15, '2019-03-17T12:00:00'), 242 ] 243 244 filename = pinboard.RevisionResultsFile(item) 245 with open(filename, 'w') as f: 246 f.writelines(csv) 247 248 with mock.patch('cli_tools.pinboard.pinboard.ACTIVE_STORIES', 249 new=['story1', 'story2']): 250 df = pinboard.GetRevisionResults(item) 251 252 self.assertEqual(len(df.index), 2) # Only two rows of output. 253 self.assertTrue((df['revision'] == '2a66ba').all()) 254 self.assertTrue((df['benchmark'] == 'loading').all()) 255 self.assertTrue((df['name'] == 'Total:duration').all()) 256 self.assertTrue((df['count'] == 2).all()) 257 df = df.set_index('label', verify_integrity=True) 258 for label, value, timestamp in expected_results: 259 self.assertEqual(df.loc[label, 'mean'], value) 260 self.assertEqual(df.loc[label, 'timestamp'], pd.Timestamp(timestamp)) 261 262 def testGetRevisionResults_empty(self): 263 item = StateItem('2a66ba', timestamp='2019-03-17T23:50:16-07:00') 264 csv = [ 265 'change,benchmark,story,name,unit,mean,job_id\n', 266 '2a66ba,loading,story1,Other:metric,count_smallerIsBetter,1.0,job1\n' 267 ] 268 269 filename = pinboard.RevisionResultsFile(item) 270 with open(filename, 'w') as f: 271 f.writelines(csv) 272 273 df = pinboard.GetRevisionResults(item) 274 self.assertEqual(len(df.index), 1) # Only one row of output. 275 row = df.iloc[0] 276 self.assertEqual(row['revision'], '2a66ba') 277 self.assertEqual(row['count'], 0) 278 279 @mock.patch('cli_tools.pinboard.pinboard.FindCommit') 280 def testGetLastCommitOfDate_simple(self, find_commit): 281 commit_before = ('2a66bac4', '2019-03-17T23:50:16-07:00') 282 commit_after = ('5aefdb31', '2019-03-18T02:41:58-07:00') 283 find_commit.side_effect = [commit_after, commit_before] 284 285 date = pd.Timestamp('2019-03-17 04:01:01', tz=pinboard.TZ) 286 return_value = pinboard.GetLastCommitOfDate(date) 287 288 cutoff_date = pd.Timestamp('2019-03-18 00:00:00', tz=pinboard.TZ) 289 find_commit.assert_has_calls([ 290 mock.call(after_date=cutoff_date), 291 mock.call(before_date=cutoff_date)]) 292 self.assertEqual(return_value, commit_before) 293 294 @mock.patch('cli_tools.pinboard.pinboard.FindCommit') 295 def testGetLastCommitOfDate_failed(self, find_commit): 296 commit_before = ('2a66bac4', '2019-03-17T23:50:16-07:00') 297 find_commit.side_effect = [None, commit_before] 298 299 date = pd.Timestamp('2019-03-17 04:01:01', tz=pinboard.TZ) 300 with self.assertRaises(ValueError): 301 pinboard.GetLastCommitOfDate(date) 302 303 cutoff_date = pd.Timestamp('2019-03-18 00:00:00', tz=pinboard.TZ) 304 find_commit.assert_has_calls([ 305 mock.call(after_date=cutoff_date)]) 306 307 def testFindCommit_simple(self): 308 self.subprocess.check_output.return_value = '2a66bac4:1552891816\n' 309 date = pd.Timestamp('2019-03-18T00:00:00', tz=pinboard.TZ) 310 revision, timestamp = pinboard.FindCommit(before_date=date) 311 self.subprocess.check_output.assert_called_once_with( 312 ['git', 'log', '--max-count', '1', '--format=format:%H:%ct', 313 '--before', '2019-03-18T00:00:00-07:00', 'origin/master'], 314 cwd=pinboard.TOOLS_PERF_DIR) 315 self.assertEqual(revision, '2a66bac4') 316 self.assertEqual(timestamp, '2019-03-17T23:50:16-07:00') 317 318 def testFindCommit_notFound(self): 319 self.subprocess.check_output.return_value = '' 320 date = pd.Timestamp('2019-03-18T00:00:00', tz=pinboard.TZ) 321 return_value = pinboard.FindCommit(after_date=date) 322 self.subprocess.check_output.assert_called_once_with( 323 ['git', 'log', '--max-count', '1', '--format=format:%H:%ct', 324 '--after', '2019-03-18T00:00:00-07:00', 'origin/master'], 325 cwd=pinboard.TOOLS_PERF_DIR) 326 self.assertIsNone(return_value) 327