1# Copyright 2019 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import os
6import shutil
7import tempfile
8import unittest
9
10import mock
11
12from cli_tools.pinboard import pinboard
13from core.external_modules import pandas as pd
14
15
16def StateItem(revision, **kwargs):
17  item = {'revision': revision,
18          'timestamp': kwargs.pop('timestamp', '2019-03-15'),
19          'jobs': []}
20  with_bots = False
21  if kwargs.get('with_bots'):
22    with_bots = kwargs.pop('with_bots')
23  for job_id, status in sorted(kwargs.items()):
24    job = {'id': job_id, 'status': status}
25    if with_bots:
26      job['bot'] = job_id
27    item['jobs'].append(job)
28
29  return item
30
31
32@unittest.skipIf(pd is None, 'pandas not available')
33class PinboardToolTests(unittest.TestCase):
34  def setUp(self):
35    self.cache_dir = tempfile.mkdtemp()
36    os.mkdir(os.path.join(self.cache_dir, 'job_results'))
37    mock.patch(
38        'cli_tools.pinboard.pinboard.CACHED_DATA_DIR',
39        new=self.cache_dir).start()
40    self.subprocess = mock.patch(
41        'cli_tools.pinboard.pinboard.subprocess').start()
42    self.upload_to_cloud = mock.patch(
43        'cli_tools.pinboard.pinboard.UploadToCloudStorage').start()
44    self.download_from_cloud = mock.patch(
45        'cli_tools.pinboard.pinboard.DownloadFromCloudStorage').start()
46    self.download_from_cloud.return_value = False
47
48  def tearDown(self):
49    mock.patch.stopall()
50    shutil.rmtree(self.cache_dir)
51
52  @mock.patch('cli_tools.pinboard.pinboard.GetLastCommitOfDate')
53  @mock.patch('cli_tools.pinboard.pinboard.LoadJsonFile')
54  def testStartPinpointJobs(self, load_configs, get_last_commit):
55    load_configs.return_value = [{
56        'name': 'config1',
57        'configuration': 'AndroidGo'
58    }, {
59        'name': 'config2',
60        'configuration': 'Pixel2'
61    }]
62    get_last_commit.return_value = ('2a66bac4', '2019-03-17T23:50:16-07:00')
63    self.subprocess.check_output.side_effect = [
64        'Started: https://pinpoint.example.com/job/14b4c451f40000\n',
65        'Started: https://pinpoint.example.com/job/11fae481f40000\n']
66    state = []
67
68    pinboard.StartPinpointJobs(state, '2019-03-17')
69
70    self.assertEqual(state, [{
71        'revision':
72        '2a66bac4',
73        'timestamp':
74        '2019-03-17T23:50:16-07:00',
75        'jobs': [{
76            'id': '14b4c451f40000',
77            'status': 'queued',
78            'bot': 'AndroidGo'
79        }, {
80            'id': '11fae481f40000',
81            'status': 'queued',
82            'bot': 'Pixel2'
83        }]
84    }])
85
86  def testCollectPinpointResults(self):
87    state = [
88        StateItem('a100', job1='completed', job2='completed'),
89        StateItem('a200', job3='completed', job4='running'),
90        StateItem('a300', job5='running', job6='running')]
91
92    # Write some fake "previous" results for first revision.
93    df = pd.DataFrame({'revision': ['a100']})
94    df.to_csv(pinboard.RevisionResultsFile(state[0]), index=False)
95
96    self.subprocess.check_output.side_effect = [
97        'job4: completed\n',
98        'job5: running\njob6: failed\n',
99        'getting csv data ...\n'
100    ]
101    expected_state = [
102        StateItem('a100', job1='completed', job2='completed'),
103        StateItem('a200', job3='completed', job4='completed'),
104        StateItem('a300', job5='running', job6='failed')]
105
106    pinboard.CollectPinpointResults(state)
107
108    self.assertEqual(state, expected_state)
109    self.subprocess.check_output.assert_has_calls([
110        mock.call(['vpython', pinboard.PINPOINT_CLI, 'status', 'job4'],
111                  universal_newlines=True),
112        mock.call(['vpython', pinboard.PINPOINT_CLI, 'status', 'job5', 'job6'],
113                  universal_newlines=True),
114        mock.call([
115            'vpython', pinboard.PINPOINT_CLI, 'get-csv', '--output',
116            pinboard.RevisionResultsFile(state[1]), '--', 'job3', 'job4'])
117    ])
118
119  def testUpdateJobsState(self):
120    state = pinboard.LoadJobsState()
121    self.assertEqual(state, [])
122
123    # Update state with new data a couple of times.
124    state.append(StateItem('a100'))
125    pinboard.UpdateJobsState(state)
126    state.append(StateItem('a200'))
127    pinboard.UpdateJobsState(state)
128
129    # No new data. Should be a no-op.
130    pinboard.UpdateJobsState(state)
131
132    stored_state = pinboard.LoadJobsState()
133    self.assertEqual(stored_state, state)
134    self.assertEqual([i['revision'] for i in stored_state], ['a100', 'a200'])
135    self.assertEqual(self.upload_to_cloud.call_count, 2)
136
137  @mock.patch('cli_tools.pinboard.pinboard.GetRevisionResults')
138  @mock.patch('cli_tools.pinboard.pinboard.TimeAgo')
139  def testAggregateAndUploadResults(self, time_ago, get_revision_results):
140    state = [
141        StateItem('a100', timestamp='2019-03-15', job1='completed'),
142        StateItem('a200', timestamp='2019-03-16', job2='completed'),
143        StateItem('a300', timestamp='2019-03-17', job3='failed'),
144        StateItem('a400', timestamp='2019-03-18', job4='completed'),
145        StateItem('a500', timestamp='2019-03-19', job5='completed'),
146    ]
147
148    def GetFakeResults(item):
149      df = pd.DataFrame(index=[0])
150      df['revision'] = item['revision']
151      df['label'] = 'with_patch'
152      df['benchmark'] = 'loading'
153      df['name'] = 'Total:duration'
154      df['timestamp'] = pd.Timestamp(item['timestamp'])
155      df['count'] = 1 if item['revision'] != 'a400' else 0
156      return df
157
158    get_revision_results.side_effect = GetFakeResults
159    time_ago.return_value = pd.Timestamp('2018-10-20')
160
161    # Only process first few revisions.
162    new_items, cached_df = pinboard.GetItemsToUpdate(state[:3])
163    pinboard.AggregateAndUploadResults(new_items, cached_df)
164    dataset_file = pinboard.CachedFilePath(pinboard.DATASET_CSV_FILE)
165    df = pd.read_csv(dataset_file)
166    self.assertEqual(set(df['revision']), set(['a100', 'a200']))
167    self.assertTrue((df[df['reference']]['revision'] == 'a200').all())
168
169    # Incrementally process the rest.
170    new_items, cached_df = pinboard.GetItemsToUpdate(state)
171    pinboard.AggregateAndUploadResults(new_items, cached_df)
172    dataset_file = pinboard.CachedFilePath(pinboard.DATASET_CSV_FILE)
173    df = pd.read_csv(dataset_file)
174    self.assertEqual(set(df['revision']), set(['a100', 'a200', 'a500']))
175    self.assertTrue((df[df['reference']]['revision'] == 'a500').all())
176
177    # No new revisions. This should be a no-op.
178    new_items, cached_df = pinboard.GetItemsToUpdate(state)
179    pinboard.AggregateAndUploadResults(new_items, cached_df)
180
181    self.assertEqual(get_revision_results.call_count, 4)
182    # Uploads twice (the pkl and csv) on each call to aggregate results.
183    self.assertEqual(self.upload_to_cloud.call_count, 2 * 2)
184
185  def testGetRevisionResults_different_bots(self):
186    item = StateItem(
187        '2a66ba',
188        timestamp='2019-03-17T23:50:16-07:00',
189        with_bots=True,
190        job1='completed',
191        job2='completed')
192    csv = [
193        'change,benchmark,story,name,unit,mean,job_id\n',
194        '2a66ba,loading,story1,Total:duration,ms_smallerIsBetter,300.0,job1\n',
195        '2a66ba,loading,story2,Total:duration,ms_smallerIsBetter,400.0,job2\n',
196        '2a66ba+patch,loading,story1,Total:duration,ms_smallerIsBetter,100.0,' +
197        'job1\n',
198        '2a66ba+patch,loading,story2,Total:duration,ms_smallerIsBetter,200.0,' +
199        'job2\n',
200        '2a66ba,loading,story1,Other:metric,count_smallerIsBetter,1.0,job1\n'
201    ]
202    expected_results = [
203        ('without_patch', 'job1', 0.3, '2018-03-17T12:00:00'),
204        ('with_patch', 'job1', 0.1, '2019-03-17T12:00:00'),
205        ('without_patch', 'job2', 0.4, '2018-03-17T12:00:00'),
206        ('with_patch', 'job2', 0.2, '2019-03-17T12:00:00'),
207    ]
208
209    filename = pinboard.RevisionResultsFile(item)
210    with open(filename, 'w') as f:
211      f.writelines(csv)
212
213    with mock.patch(
214        'cli_tools.pinboard.pinboard.ACTIVE_STORIES', new=['story1', 'story2']):
215      df = pinboard.GetRevisionResults(item)
216
217    self.assertEqual(len(df.index), 4)  # Only two rows of output.
218    self.assertTrue((df['revision'] == '2a66ba').all())
219    self.assertTrue((df['benchmark'] == 'loading').all())
220    self.assertTrue((df['name'] == 'Total:duration').all())
221    self.assertTrue((df['count'] == 1).all())
222    df = df.set_index(['label', 'bot'], verify_integrity=True)
223    for label, bot, value, timestamp in expected_results:
224      self.assertEqual(df.loc[label, bot]['mean'], value)
225      self.assertEqual(df.loc[label, bot]['timestamp'], pd.Timestamp(timestamp))
226
227  def testGetRevisionResults_simple(self):
228    item = StateItem('2a66ba', timestamp='2019-03-17T23:50:16-07:00')
229    csv = [
230        'change,benchmark,story,name,unit,mean,job_id\n',
231        '2a66ba,loading,story1,Total:duration,ms_smallerIsBetter,300.0,job1\n',
232        '2a66ba,loading,story2,Total:duration,ms_smallerIsBetter,400.0,job1\n',
233        '2a66ba+patch,loading,story1,Total:duration,ms_smallerIsBetter,100.0,' +
234        'job1\n',
235        '2a66ba+patch,loading,story2,Total:duration,ms_smallerIsBetter,200.0,' +
236        'job1\n',
237        '2a66ba,loading,story1,Other:metric,count_smallerIsBetter,1.0,job1\n'
238    ]
239    expected_results = [
240        ('without_patch', 0.35, '2018-03-17T12:00:00'),
241        ('with_patch', 0.15, '2019-03-17T12:00:00'),
242    ]
243
244    filename = pinboard.RevisionResultsFile(item)
245    with open(filename, 'w') as f:
246      f.writelines(csv)
247
248    with mock.patch('cli_tools.pinboard.pinboard.ACTIVE_STORIES',
249                    new=['story1', 'story2']):
250      df = pinboard.GetRevisionResults(item)
251
252    self.assertEqual(len(df.index), 2)  # Only two rows of output.
253    self.assertTrue((df['revision'] == '2a66ba').all())
254    self.assertTrue((df['benchmark'] == 'loading').all())
255    self.assertTrue((df['name'] == 'Total:duration').all())
256    self.assertTrue((df['count'] == 2).all())
257    df = df.set_index('label', verify_integrity=True)
258    for label, value, timestamp in expected_results:
259      self.assertEqual(df.loc[label, 'mean'], value)
260      self.assertEqual(df.loc[label, 'timestamp'], pd.Timestamp(timestamp))
261
262  def testGetRevisionResults_empty(self):
263    item = StateItem('2a66ba', timestamp='2019-03-17T23:50:16-07:00')
264    csv = [
265        'change,benchmark,story,name,unit,mean,job_id\n',
266        '2a66ba,loading,story1,Other:metric,count_smallerIsBetter,1.0,job1\n'
267    ]
268
269    filename = pinboard.RevisionResultsFile(item)
270    with open(filename, 'w') as f:
271      f.writelines(csv)
272
273    df = pinboard.GetRevisionResults(item)
274    self.assertEqual(len(df.index), 1)  # Only one row of output.
275    row = df.iloc[0]
276    self.assertEqual(row['revision'], '2a66ba')
277    self.assertEqual(row['count'], 0)
278
279  @mock.patch('cli_tools.pinboard.pinboard.FindCommit')
280  def testGetLastCommitOfDate_simple(self, find_commit):
281    commit_before = ('2a66bac4', '2019-03-17T23:50:16-07:00')
282    commit_after = ('5aefdb31', '2019-03-18T02:41:58-07:00')
283    find_commit.side_effect = [commit_after, commit_before]
284
285    date = pd.Timestamp('2019-03-17 04:01:01', tz=pinboard.TZ)
286    return_value = pinboard.GetLastCommitOfDate(date)
287
288    cutoff_date = pd.Timestamp('2019-03-18 00:00:00', tz=pinboard.TZ)
289    find_commit.assert_has_calls([
290        mock.call(after_date=cutoff_date),
291        mock.call(before_date=cutoff_date)])
292    self.assertEqual(return_value, commit_before)
293
294  @mock.patch('cli_tools.pinboard.pinboard.FindCommit')
295  def testGetLastCommitOfDate_failed(self, find_commit):
296    commit_before = ('2a66bac4', '2019-03-17T23:50:16-07:00')
297    find_commit.side_effect = [None, commit_before]
298
299    date = pd.Timestamp('2019-03-17 04:01:01', tz=pinboard.TZ)
300    with self.assertRaises(ValueError):
301      pinboard.GetLastCommitOfDate(date)
302
303    cutoff_date = pd.Timestamp('2019-03-18 00:00:00', tz=pinboard.TZ)
304    find_commit.assert_has_calls([
305        mock.call(after_date=cutoff_date)])
306
307  def testFindCommit_simple(self):
308    self.subprocess.check_output.return_value = '2a66bac4:1552891816\n'
309    date = pd.Timestamp('2019-03-18T00:00:00', tz=pinboard.TZ)
310    revision, timestamp = pinboard.FindCommit(before_date=date)
311    self.subprocess.check_output.assert_called_once_with(
312        ['git', 'log', '--max-count', '1', '--format=format:%H:%ct',
313         '--before', '2019-03-18T00:00:00-07:00', 'origin/master'],
314        cwd=pinboard.TOOLS_PERF_DIR)
315    self.assertEqual(revision, '2a66bac4')
316    self.assertEqual(timestamp, '2019-03-17T23:50:16-07:00')
317
318  def testFindCommit_notFound(self):
319    self.subprocess.check_output.return_value = ''
320    date = pd.Timestamp('2019-03-18T00:00:00', tz=pinboard.TZ)
321    return_value = pinboard.FindCommit(after_date=date)
322    self.subprocess.check_output.assert_called_once_with(
323        ['git', 'log', '--max-count', '1', '--format=format:%H:%ct',
324         '--after', '2019-03-18T00:00:00-07:00', 'origin/master'],
325        cwd=pinboard.TOOLS_PERF_DIR)
326    self.assertIsNone(return_value)
327