1#!/usr/bin/env python
2#
3# Copyright 2013 The Chromium Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""Updates the Chrome reference builds.
8
9Usage:
10  $ /path/to/update_reference_build.py
11  $ git commit -a
12  $ git cl upload
13"""
14
15import argparse
16import collections
17import logging
18import os
19import shutil
20import subprocess
21import sys
22import tempfile
23import urllib2
24import zipfile
25
26sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'py_utils'))
27
28from py_utils import cloud_storage
29from dependency_manager import base_config
30
31
32_CHROME_BINARIES_CONFIG = os.path.join(
33    os.path.dirname(os.path.abspath(__file__)), '..', '..', 'common',
34    'py_utils', 'py_utils', 'chrome_binaries.json')
35
36_CHROME_GS_BUCKET = 'chrome-unsigned'
37_CHROMIUM_GS_BUCKET = 'chromium-browser-snapshots'
38
39# How many commit positions to search below and above omaha branch position to
40# find closest chromium build snapshot. The value 10 is chosen because it looks
41# more than sufficient from manual inspection of the bucket.
42_CHROMIUM_SNAPSHOT_SEARCH_WINDOW = 10
43
44# Remove a platform name from this list to disable updating it.
45# Add one to enable updating it. (Must also update _PLATFORM_MAP.)
46_PLATFORMS_TO_UPDATE = ['mac_x86_64', 'win_x86', 'win_AMD64', 'linux_x86_64',
47                        'android_k_armeabi-v7a', 'android_l_arm64-v8a',
48                        'android_l_armeabi-v7a', 'android_n_armeabi-v7a',
49                        'android_n_arm64-v8a', 'android_n_bundle_armeabi-v7a',
50                        'android_n_bundle_arm64-v8a']
51
52# Add platforms here if you also want to update chromium binary for it.
53# Must add chromium_info for it in _PLATFORM_MAP.
54_CHROMIUM_PLATFORMS = ['mac_x86_64', 'win_x86', 'win_AMD64', 'linux_x86_64']
55
56# Remove a channel name from this list to disable updating it.
57# Add one to enable updating it.
58_CHANNELS_TO_UPDATE = ['stable', 'canary', 'dev']
59
60
61# Omaha is Chrome's autoupdate server. It reports the current versions used
62# by each platform on each channel.
63_OMAHA_PLATFORMS = { 'stable':  ['mac', 'linux', 'win', 'android'],
64                    'dev':  ['linux'], 'canary': ['mac', 'win']}
65
66
67# All of the information we need to update each platform.
68#   omaha: name omaha uses for the platforms.
69#   zip_name: name of the zip file to be retrieved from cloud storage.
70#   gs_build: name of the Chrome build platform used in cloud storage.
71#   chromium_info: information needed to update chromium (optional).
72#   destination: Name of the folder to download the reference build to.
73UpdateInfo = collections.namedtuple('UpdateInfo',
74    'omaha, gs_folder, gs_build, chromium_info, zip_name')
75# build_dir: name of the build directory in _CHROMIUM_GS_BUCKET.
76# zip_name: name of the zip file to be retrieved from cloud storage.
77ChromiumInfo = collections.namedtuple('ChromiumInfo', 'build_dir, zip_name')
78_PLATFORM_MAP = {'mac_x86_64': UpdateInfo(
79                     omaha='mac',
80                     gs_folder='desktop-*',
81                     gs_build='mac64',
82                     chromium_info=ChromiumInfo(
83                         build_dir='Mac',
84                         zip_name='chrome-mac.zip'),
85                     zip_name='chrome-mac.zip'),
86                 'win_x86': UpdateInfo(
87                     omaha='win',
88                     gs_folder='desktop-*',
89                     gs_build='win-clang',
90                     chromium_info=ChromiumInfo(
91                         build_dir='Win',
92                         zip_name='chrome-win.zip'),
93                     zip_name='chrome-win-clang.zip'),
94                 'win_AMD64': UpdateInfo(
95                     omaha='win',
96                     gs_folder='desktop-*',
97                     gs_build='win64-clang',
98                     chromium_info=ChromiumInfo(
99                        build_dir='Win_x64',
100                        zip_name='chrome-win.zip'),
101                     zip_name='chrome-win64-clang.zip'),
102                 'linux_x86_64': UpdateInfo(
103                     omaha='linux',
104                     gs_folder='desktop-*',
105                     gs_build='linux64',
106                     chromium_info=ChromiumInfo(
107                         build_dir='Linux_x64',
108                         zip_name='chrome-linux.zip'),
109                     zip_name='chrome-linux64.zip'),
110                 'android_k_armeabi-v7a': UpdateInfo(
111                     omaha='android',
112                     gs_folder='android-*',
113                     gs_build='arm',
114                     chromium_info=None,
115                     zip_name='Chrome.apk'),
116                 'android_l_arm64-v8a': UpdateInfo(
117                     omaha='android',
118                     gs_folder='android-*',
119                     gs_build='arm_64',
120                     chromium_info=None,
121                     zip_name='ChromeModern.apk'),
122                 'android_l_armeabi-v7a': UpdateInfo(
123                     omaha='android',
124                     gs_folder='android-*',
125                     gs_build='arm',
126                     chromium_info=None,
127                     zip_name='Chrome.apk'),
128                 'android_n_armeabi-v7a': UpdateInfo(
129                     omaha='android',
130                     gs_folder='android-*',
131                     gs_build='arm',
132                     chromium_info=None,
133                     zip_name='Monochrome.apk'),
134                 'android_n_arm64-v8a': UpdateInfo(
135                     omaha='android',
136                     gs_folder='android-*',
137                     gs_build='arm_64',
138                     chromium_info=None,
139                     zip_name='Monochrome.apk'),
140                 'android_n_bundle_armeabi-v7a': UpdateInfo(
141                     omaha='android',
142                     gs_folder='android-*',
143                     gs_build='arm',
144                     chromium_info=None,
145                     zip_name='Monochrome.apks'),
146                 'android_n_bundle_arm64-v8a': UpdateInfo(
147                     omaha='android',
148                     gs_folder='android-*',
149                     gs_build='arm_64',
150                     chromium_info=None,
151                     zip_name='Monochrome.apks')
152
153}
154
155
156VersionInfo = collections.namedtuple('VersionInfo',
157                                     'version, branch_base_position')
158
159
160def _ChannelVersionsMap(channel):
161  rows = _OmahaReportVersionInfo(channel)
162  omaha_versions_map = _OmahaVersionsMap(rows, channel)
163  channel_versions_map = {}
164  for platform in _PLATFORMS_TO_UPDATE:
165    omaha_platform = _PLATFORM_MAP[platform].omaha
166    if omaha_platform in omaha_versions_map:
167      channel_versions_map[platform] = omaha_versions_map[omaha_platform]
168  return channel_versions_map
169
170
171def _OmahaReportVersionInfo(channel):
172  url ='https://omahaproxy.appspot.com/all?channel=%s' % channel
173  lines = urllib2.urlopen(url).readlines()
174  return [l.split(',') for l in lines]
175
176
177def _OmahaVersionsMap(rows, channel):
178  platforms = _OMAHA_PLATFORMS.get(channel, [])
179  if (len(rows) < 1 or
180      rows[0][0:3] != ['os', 'channel', 'current_version'] or
181      rows[0][7] != 'branch_base_position'):
182    raise ValueError(
183        'Omaha report is not in the expected form: %s.' % rows)
184  versions_map = {}
185  for row in rows[1:]:
186    if row[1] != channel:
187      raise ValueError(
188          'Omaha report contains a line with the channel %s' % row[1])
189    if row[0] in platforms:
190      versions_map[row[0]] = VersionInfo(version=row[2],
191                                         branch_base_position=int(row[7]))
192  logging.warn('versions map: %s' % versions_map)
193  if not all(platform in versions_map for platform in platforms):
194    raise ValueError(
195        'Omaha report did not contain all desired platforms '
196        'for channel %s' % channel)
197  return versions_map
198
199
200RemotePath = collections.namedtuple('RemotePath', 'bucket, path')
201
202
203def _ResolveChromeRemotePath(platform_info, version_info):
204  # Path example: desktop-*/30.0.1595.0/precise32/chrome-precise32.zip
205  return RemotePath(bucket=_CHROME_GS_BUCKET,
206                    path=('%s/%s/%s/%s' % (platform_info.gs_folder,
207                                           version_info.version,
208                                           platform_info.gs_build,
209                                           platform_info.zip_name)))
210
211
212def _FindClosestChromiumSnapshot(base_position, build_dir):
213  """Returns the closest chromium snapshot available in cloud storage.
214
215  Chromium snapshots are pulled from _CHROMIUM_BUILD_DIR in CHROMIUM_GS_BUCKET.
216
217  Continuous chromium snapshots do not always contain the exact release build.
218  This function queries the storage bucket and find the closest snapshot within
219  +/-_CHROMIUM_SNAPSHOT_SEARCH_WINDOW to find the closest build.
220  """
221  min_position = base_position - _CHROMIUM_SNAPSHOT_SEARCH_WINDOW
222  max_position = base_position + _CHROMIUM_SNAPSHOT_SEARCH_WINDOW
223
224  # Getting the full list of objects in cloud storage bucket is prohibitively
225  # slow. It's faster to list objects with a prefix. Assuming we're looking at
226  # +/- 10 commit positions, for commit position 123456, we want to look at
227  # positions between 123446 an 123466. We do this by getting all snapshots
228  # with prefix 12344*, 12345*, and 12346*. This may get a few more snapshots
229  # that we intended, but that's fine since we take the min distance anyways.
230  min_position_prefix = min_position / 10;
231  max_position_prefix = max_position / 10;
232
233  available_positions = []
234  for position_prefix in range(min_position_prefix, max_position_prefix + 1):
235    query = '%s/%d*' % (build_dir, position_prefix)
236    try:
237      ls_results = cloud_storage.ListDirs(_CHROMIUM_GS_BUCKET, query)
238    except cloud_storage.NotFoundError:
239      # It's fine if there is no chromium snapshot available for one prefix.
240      # We will look at the rest of the prefixes.
241      continue
242
243    for entry in ls_results:
244      # entry looks like '/Linux_x64/${commit_position}/'.
245      position = int(entry.split('/')[2])
246      available_positions.append(position)
247
248  if len(available_positions) == 0:
249    raise ValueError('No chromium build found +/-%d commit positions of %d' %
250                     (_CHROMIUM_SNAPSHOT_SEARCH_WINDOW, base_position))
251
252  distance_function = lambda position: abs(position - base_position)
253  min_distance_snapshot = min(available_positions, key=distance_function)
254  return min_distance_snapshot
255
256
257def _ResolveChromiumRemotePath(channel, platform, version_info):
258  platform_info = _PLATFORM_MAP[platform]
259  branch_base_position = version_info.branch_base_position
260  omaha_version = version_info.version
261  build_dir = platform_info.chromium_info.build_dir
262  # Look through chromium-browser-snapshots for closest match.
263  closest_snapshot = _FindClosestChromiumSnapshot(
264      branch_base_position, build_dir)
265  if closest_snapshot != branch_base_position:
266    print ('Channel %s corresponds to commit position ' % channel +
267            '%d on %s, ' % (branch_base_position, platform) +
268            'but closest chromium snapshot available on ' +
269            '%s is %d' % (_CHROMIUM_GS_BUCKET, closest_snapshot))
270  return RemotePath(bucket=_CHROMIUM_GS_BUCKET,
271                    path = ('%s/%s/%s' % (build_dir, closest_snapshot,
272                                        platform_info.chromium_info.zip_name)))
273
274
275def _QueuePlatformUpdate(binary, platform, version_info, config, channel):
276  """ platform: the name of the platform for the browser to
277      be downloaded & updated from cloud storage. """
278  platform_info = _PLATFORM_MAP[platform]
279
280  if binary == 'chrome':
281    remote_path = _ResolveChromeRemotePath(platform_info, version_info)
282  elif binary == 'chromium':
283    remote_path = _ResolveChromiumRemotePath(channel, platform, version_info)
284  else:
285    raise ValueError('binary must be \'chrome\' or \'chromium\'')
286
287  if not cloud_storage.Exists(remote_path.bucket, remote_path.path):
288    cloud_storage_path = 'gs://%s/%s' % (remote_path.bucket, remote_path.path)
289    logging.warn('Failed to find %s build for version %s at path %s.' % (
290        platform, version_info.version, cloud_storage_path))
291    logging.warn('Skipping this update for this platform/channel.')
292    return
293
294  reference_builds_folder = os.path.join(
295      os.path.dirname(os.path.abspath(__file__)), 'chrome_telemetry_build',
296      'reference_builds', binary, channel)
297  if not os.path.exists(reference_builds_folder):
298    os.makedirs(reference_builds_folder)
299  local_dest_path = os.path.join(reference_builds_folder,
300                                 platform,
301                                 platform_info.zip_name)
302  cloud_storage.Get(remote_path.bucket, remote_path.path, local_dest_path)
303  _ModifyBuildIfNeeded(binary, local_dest_path, platform)
304  config.AddCloudStorageDependencyUpdateJob('%s_%s' % (binary, channel),
305      platform, local_dest_path, version=version_info.version,
306      execute_job=False)
307
308
309def _ModifyBuildIfNeeded(binary, location, platform):
310  """Hook to modify the build before saving it for Telemetry to use.
311
312  This can be used to remove various utilities that cause noise in a
313  test environment. Right now, it is just used to remove Keystone,
314  which is a tool used to autoupdate Chrome.
315  """
316  if binary != 'chrome':
317    return
318
319  if platform == 'mac_x86_64':
320    _RemoveKeystoneFromBuild(location)
321    return
322
323  if 'mac' in platform:
324    raise NotImplementedError(
325        'Platform <%s> sounds like it is an OSX version. If so, we may need to '
326        'remove Keystone from it per crbug.com/932615. Please edit this script'
327        ' and teach it what needs to be done :).')
328
329
330def _RemoveKeystoneFromBuild(location):
331  """Removes the Keystone autoupdate binary from the chrome mac zipfile."""
332  logging.info('Removing keystone from mac build at %s' % location)
333  temp_folder = tempfile.mkdtemp(prefix='RemoveKeystoneFromBuild')
334  try:
335    subprocess.check_call(['unzip', '-q', location, '-d', temp_folder])
336    keystone_folder = os.path.join(
337        temp_folder, 'chrome-mac', 'Google Chrome.app', 'Contents',
338        'Frameworks', 'Google Chrome Framework.framework', 'Frameworks',
339        'KeystoneRegistration.framework')
340    shutil.rmtree(keystone_folder)
341    os.remove(location)
342    subprocess.check_call(['zip', '--quiet', '--recurse-paths', '--symlinks',
343                           location, 'chrome-mac'],
344                           cwd=temp_folder)
345  finally:
346    shutil.rmtree(temp_folder)
347
348
349def _NeedsUpdate(config, binary, channel, platform, version_info):
350  channel_version = version_info.version
351  print 'Checking %s (%s channel) on %s' % (binary, channel, platform)
352  current_version = config.GetVersion('%s_%s' % (binary, channel), platform)
353  print 'current: %s, channel: %s' % (current_version, channel_version)
354  if current_version and current_version == channel_version:
355    print 'Already up to date.'
356    return False
357  return True
358
359
360def UpdateBuilds(args):
361  config = base_config.BaseConfig(_CHROME_BINARIES_CONFIG, writable=True)
362  for channel in _CHANNELS_TO_UPDATE:
363    channel_versions_map = _ChannelVersionsMap(channel)
364    for platform in channel_versions_map:
365      version_info = channel_versions_map.get(platform)
366      if args.update_chrome:
367        if _NeedsUpdate(config, 'chrome', channel, platform, version_info):
368          _QueuePlatformUpdate('chrome', platform, version_info, config,
369                               channel)
370      if args.update_chromium and platform in _CHROMIUM_PLATFORMS:
371        if _NeedsUpdate(config, 'chromium', channel, platform, version_info):
372          _QueuePlatformUpdate('chromium', platform, version_info,
373                               config, channel)
374
375  print 'Updating builds with downloaded binaries'
376  config.ExecuteUpdateJobs(force=True)
377
378
379def main():
380  logging.getLogger().setLevel(logging.DEBUG)
381  parser = argparse.ArgumentParser(
382      description='Update reference binaries used by perf bots.')
383  parser.add_argument('--no-update-chrome', action='store_false',
384                      dest='update_chrome', default=True,
385                      help='do not update chrome binaries')
386  parser.add_argument('--no-update-chromium', action='store_false',
387                      dest='update_chromium', default=True,
388                      help='do not update chromium binaries')
389  args = parser.parse_args()
390  UpdateBuilds(args)
391
392if __name__ == '__main__':
393  main()
394