1#!/usr/bin/env python 2# 3# Copyright 2013 The Chromium Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7"""Updates the Chrome reference builds. 8 9Usage: 10 $ /path/to/update_reference_build.py 11 $ git commit -a 12 $ git cl upload 13""" 14 15import argparse 16import collections 17import logging 18import os 19import shutil 20import subprocess 21import sys 22import tempfile 23import urllib2 24import zipfile 25 26sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'py_utils')) 27 28from py_utils import cloud_storage 29from dependency_manager import base_config 30 31 32_CHROME_BINARIES_CONFIG = os.path.join( 33 os.path.dirname(os.path.abspath(__file__)), '..', '..', 'common', 34 'py_utils', 'py_utils', 'chrome_binaries.json') 35 36_CHROME_GS_BUCKET = 'chrome-unsigned' 37_CHROMIUM_GS_BUCKET = 'chromium-browser-snapshots' 38 39# How many commit positions to search below and above omaha branch position to 40# find closest chromium build snapshot. The value 10 is chosen because it looks 41# more than sufficient from manual inspection of the bucket. 42_CHROMIUM_SNAPSHOT_SEARCH_WINDOW = 10 43 44# Remove a platform name from this list to disable updating it. 45# Add one to enable updating it. (Must also update _PLATFORM_MAP.) 46_PLATFORMS_TO_UPDATE = ['mac_x86_64', 'win_x86', 'win_AMD64', 'linux_x86_64', 47 'android_k_armeabi-v7a', 'android_l_arm64-v8a', 48 'android_l_armeabi-v7a', 'android_n_armeabi-v7a', 49 'android_n_arm64-v8a', 'android_n_bundle_armeabi-v7a', 50 'android_n_bundle_arm64-v8a'] 51 52# Add platforms here if you also want to update chromium binary for it. 53# Must add chromium_info for it in _PLATFORM_MAP. 54_CHROMIUM_PLATFORMS = ['mac_x86_64', 'win_x86', 'win_AMD64', 'linux_x86_64'] 55 56# Remove a channel name from this list to disable updating it. 57# Add one to enable updating it. 58_CHANNELS_TO_UPDATE = ['stable', 'canary', 'dev'] 59 60 61# Omaha is Chrome's autoupdate server. It reports the current versions used 62# by each platform on each channel. 63_OMAHA_PLATFORMS = { 'stable': ['mac', 'linux', 'win', 'android'], 64 'dev': ['linux'], 'canary': ['mac', 'win']} 65 66 67# All of the information we need to update each platform. 68# omaha: name omaha uses for the platforms. 69# zip_name: name of the zip file to be retrieved from cloud storage. 70# gs_build: name of the Chrome build platform used in cloud storage. 71# chromium_info: information needed to update chromium (optional). 72# destination: Name of the folder to download the reference build to. 73UpdateInfo = collections.namedtuple('UpdateInfo', 74 'omaha, gs_folder, gs_build, chromium_info, zip_name') 75# build_dir: name of the build directory in _CHROMIUM_GS_BUCKET. 76# zip_name: name of the zip file to be retrieved from cloud storage. 77ChromiumInfo = collections.namedtuple('ChromiumInfo', 'build_dir, zip_name') 78_PLATFORM_MAP = {'mac_x86_64': UpdateInfo( 79 omaha='mac', 80 gs_folder='desktop-*', 81 gs_build='mac64', 82 chromium_info=ChromiumInfo( 83 build_dir='Mac', 84 zip_name='chrome-mac.zip'), 85 zip_name='chrome-mac.zip'), 86 'win_x86': UpdateInfo( 87 omaha='win', 88 gs_folder='desktop-*', 89 gs_build='win-clang', 90 chromium_info=ChromiumInfo( 91 build_dir='Win', 92 zip_name='chrome-win.zip'), 93 zip_name='chrome-win-clang.zip'), 94 'win_AMD64': UpdateInfo( 95 omaha='win', 96 gs_folder='desktop-*', 97 gs_build='win64-clang', 98 chromium_info=ChromiumInfo( 99 build_dir='Win_x64', 100 zip_name='chrome-win.zip'), 101 zip_name='chrome-win64-clang.zip'), 102 'linux_x86_64': UpdateInfo( 103 omaha='linux', 104 gs_folder='desktop-*', 105 gs_build='linux64', 106 chromium_info=ChromiumInfo( 107 build_dir='Linux_x64', 108 zip_name='chrome-linux.zip'), 109 zip_name='chrome-linux64.zip'), 110 'android_k_armeabi-v7a': UpdateInfo( 111 omaha='android', 112 gs_folder='android-*', 113 gs_build='arm', 114 chromium_info=None, 115 zip_name='Chrome.apk'), 116 'android_l_arm64-v8a': UpdateInfo( 117 omaha='android', 118 gs_folder='android-*', 119 gs_build='arm_64', 120 chromium_info=None, 121 zip_name='ChromeModern.apk'), 122 'android_l_armeabi-v7a': UpdateInfo( 123 omaha='android', 124 gs_folder='android-*', 125 gs_build='arm', 126 chromium_info=None, 127 zip_name='Chrome.apk'), 128 'android_n_armeabi-v7a': UpdateInfo( 129 omaha='android', 130 gs_folder='android-*', 131 gs_build='arm', 132 chromium_info=None, 133 zip_name='Monochrome.apk'), 134 'android_n_arm64-v8a': UpdateInfo( 135 omaha='android', 136 gs_folder='android-*', 137 gs_build='arm_64', 138 chromium_info=None, 139 zip_name='Monochrome.apk'), 140 'android_n_bundle_armeabi-v7a': UpdateInfo( 141 omaha='android', 142 gs_folder='android-*', 143 gs_build='arm', 144 chromium_info=None, 145 zip_name='Monochrome.apks'), 146 'android_n_bundle_arm64-v8a': UpdateInfo( 147 omaha='android', 148 gs_folder='android-*', 149 gs_build='arm_64', 150 chromium_info=None, 151 zip_name='Monochrome.apks') 152 153} 154 155 156VersionInfo = collections.namedtuple('VersionInfo', 157 'version, branch_base_position') 158 159 160def _ChannelVersionsMap(channel): 161 rows = _OmahaReportVersionInfo(channel) 162 omaha_versions_map = _OmahaVersionsMap(rows, channel) 163 channel_versions_map = {} 164 for platform in _PLATFORMS_TO_UPDATE: 165 omaha_platform = _PLATFORM_MAP[platform].omaha 166 if omaha_platform in omaha_versions_map: 167 channel_versions_map[platform] = omaha_versions_map[omaha_platform] 168 return channel_versions_map 169 170 171def _OmahaReportVersionInfo(channel): 172 url ='https://omahaproxy.appspot.com/all?channel=%s' % channel 173 lines = urllib2.urlopen(url).readlines() 174 return [l.split(',') for l in lines] 175 176 177def _OmahaVersionsMap(rows, channel): 178 platforms = _OMAHA_PLATFORMS.get(channel, []) 179 if (len(rows) < 1 or 180 rows[0][0:3] != ['os', 'channel', 'current_version'] or 181 rows[0][7] != 'branch_base_position'): 182 raise ValueError( 183 'Omaha report is not in the expected form: %s.' % rows) 184 versions_map = {} 185 for row in rows[1:]: 186 if row[1] != channel: 187 raise ValueError( 188 'Omaha report contains a line with the channel %s' % row[1]) 189 if row[0] in platforms: 190 versions_map[row[0]] = VersionInfo(version=row[2], 191 branch_base_position=int(row[7])) 192 logging.warn('versions map: %s' % versions_map) 193 if not all(platform in versions_map for platform in platforms): 194 raise ValueError( 195 'Omaha report did not contain all desired platforms ' 196 'for channel %s' % channel) 197 return versions_map 198 199 200RemotePath = collections.namedtuple('RemotePath', 'bucket, path') 201 202 203def _ResolveChromeRemotePath(platform_info, version_info): 204 # Path example: desktop-*/30.0.1595.0/precise32/chrome-precise32.zip 205 return RemotePath(bucket=_CHROME_GS_BUCKET, 206 path=('%s/%s/%s/%s' % (platform_info.gs_folder, 207 version_info.version, 208 platform_info.gs_build, 209 platform_info.zip_name))) 210 211 212def _FindClosestChromiumSnapshot(base_position, build_dir): 213 """Returns the closest chromium snapshot available in cloud storage. 214 215 Chromium snapshots are pulled from _CHROMIUM_BUILD_DIR in CHROMIUM_GS_BUCKET. 216 217 Continuous chromium snapshots do not always contain the exact release build. 218 This function queries the storage bucket and find the closest snapshot within 219 +/-_CHROMIUM_SNAPSHOT_SEARCH_WINDOW to find the closest build. 220 """ 221 min_position = base_position - _CHROMIUM_SNAPSHOT_SEARCH_WINDOW 222 max_position = base_position + _CHROMIUM_SNAPSHOT_SEARCH_WINDOW 223 224 # Getting the full list of objects in cloud storage bucket is prohibitively 225 # slow. It's faster to list objects with a prefix. Assuming we're looking at 226 # +/- 10 commit positions, for commit position 123456, we want to look at 227 # positions between 123446 an 123466. We do this by getting all snapshots 228 # with prefix 12344*, 12345*, and 12346*. This may get a few more snapshots 229 # that we intended, but that's fine since we take the min distance anyways. 230 min_position_prefix = min_position / 10; 231 max_position_prefix = max_position / 10; 232 233 available_positions = [] 234 for position_prefix in range(min_position_prefix, max_position_prefix + 1): 235 query = '%s/%d*' % (build_dir, position_prefix) 236 try: 237 ls_results = cloud_storage.ListDirs(_CHROMIUM_GS_BUCKET, query) 238 except cloud_storage.NotFoundError: 239 # It's fine if there is no chromium snapshot available for one prefix. 240 # We will look at the rest of the prefixes. 241 continue 242 243 for entry in ls_results: 244 # entry looks like '/Linux_x64/${commit_position}/'. 245 position = int(entry.split('/')[2]) 246 available_positions.append(position) 247 248 if len(available_positions) == 0: 249 raise ValueError('No chromium build found +/-%d commit positions of %d' % 250 (_CHROMIUM_SNAPSHOT_SEARCH_WINDOW, base_position)) 251 252 distance_function = lambda position: abs(position - base_position) 253 min_distance_snapshot = min(available_positions, key=distance_function) 254 return min_distance_snapshot 255 256 257def _ResolveChromiumRemotePath(channel, platform, version_info): 258 platform_info = _PLATFORM_MAP[platform] 259 branch_base_position = version_info.branch_base_position 260 omaha_version = version_info.version 261 build_dir = platform_info.chromium_info.build_dir 262 # Look through chromium-browser-snapshots for closest match. 263 closest_snapshot = _FindClosestChromiumSnapshot( 264 branch_base_position, build_dir) 265 if closest_snapshot != branch_base_position: 266 print ('Channel %s corresponds to commit position ' % channel + 267 '%d on %s, ' % (branch_base_position, platform) + 268 'but closest chromium snapshot available on ' + 269 '%s is %d' % (_CHROMIUM_GS_BUCKET, closest_snapshot)) 270 return RemotePath(bucket=_CHROMIUM_GS_BUCKET, 271 path = ('%s/%s/%s' % (build_dir, closest_snapshot, 272 platform_info.chromium_info.zip_name))) 273 274 275def _QueuePlatformUpdate(binary, platform, version_info, config, channel): 276 """ platform: the name of the platform for the browser to 277 be downloaded & updated from cloud storage. """ 278 platform_info = _PLATFORM_MAP[platform] 279 280 if binary == 'chrome': 281 remote_path = _ResolveChromeRemotePath(platform_info, version_info) 282 elif binary == 'chromium': 283 remote_path = _ResolveChromiumRemotePath(channel, platform, version_info) 284 else: 285 raise ValueError('binary must be \'chrome\' or \'chromium\'') 286 287 if not cloud_storage.Exists(remote_path.bucket, remote_path.path): 288 cloud_storage_path = 'gs://%s/%s' % (remote_path.bucket, remote_path.path) 289 logging.warn('Failed to find %s build for version %s at path %s.' % ( 290 platform, version_info.version, cloud_storage_path)) 291 logging.warn('Skipping this update for this platform/channel.') 292 return 293 294 reference_builds_folder = os.path.join( 295 os.path.dirname(os.path.abspath(__file__)), 'chrome_telemetry_build', 296 'reference_builds', binary, channel) 297 if not os.path.exists(reference_builds_folder): 298 os.makedirs(reference_builds_folder) 299 local_dest_path = os.path.join(reference_builds_folder, 300 platform, 301 platform_info.zip_name) 302 cloud_storage.Get(remote_path.bucket, remote_path.path, local_dest_path) 303 _ModifyBuildIfNeeded(binary, local_dest_path, platform) 304 config.AddCloudStorageDependencyUpdateJob('%s_%s' % (binary, channel), 305 platform, local_dest_path, version=version_info.version, 306 execute_job=False) 307 308 309def _ModifyBuildIfNeeded(binary, location, platform): 310 """Hook to modify the build before saving it for Telemetry to use. 311 312 This can be used to remove various utilities that cause noise in a 313 test environment. Right now, it is just used to remove Keystone, 314 which is a tool used to autoupdate Chrome. 315 """ 316 if binary != 'chrome': 317 return 318 319 if platform == 'mac_x86_64': 320 _RemoveKeystoneFromBuild(location) 321 return 322 323 if 'mac' in platform: 324 raise NotImplementedError( 325 'Platform <%s> sounds like it is an OSX version. If so, we may need to ' 326 'remove Keystone from it per crbug.com/932615. Please edit this script' 327 ' and teach it what needs to be done :).') 328 329 330def _RemoveKeystoneFromBuild(location): 331 """Removes the Keystone autoupdate binary from the chrome mac zipfile.""" 332 logging.info('Removing keystone from mac build at %s' % location) 333 temp_folder = tempfile.mkdtemp(prefix='RemoveKeystoneFromBuild') 334 try: 335 subprocess.check_call(['unzip', '-q', location, '-d', temp_folder]) 336 keystone_folder = os.path.join( 337 temp_folder, 'chrome-mac', 'Google Chrome.app', 'Contents', 338 'Frameworks', 'Google Chrome Framework.framework', 'Frameworks', 339 'KeystoneRegistration.framework') 340 shutil.rmtree(keystone_folder) 341 os.remove(location) 342 subprocess.check_call(['zip', '--quiet', '--recurse-paths', '--symlinks', 343 location, 'chrome-mac'], 344 cwd=temp_folder) 345 finally: 346 shutil.rmtree(temp_folder) 347 348 349def _NeedsUpdate(config, binary, channel, platform, version_info): 350 channel_version = version_info.version 351 print 'Checking %s (%s channel) on %s' % (binary, channel, platform) 352 current_version = config.GetVersion('%s_%s' % (binary, channel), platform) 353 print 'current: %s, channel: %s' % (current_version, channel_version) 354 if current_version and current_version == channel_version: 355 print 'Already up to date.' 356 return False 357 return True 358 359 360def UpdateBuilds(args): 361 config = base_config.BaseConfig(_CHROME_BINARIES_CONFIG, writable=True) 362 for channel in _CHANNELS_TO_UPDATE: 363 channel_versions_map = _ChannelVersionsMap(channel) 364 for platform in channel_versions_map: 365 version_info = channel_versions_map.get(platform) 366 if args.update_chrome: 367 if _NeedsUpdate(config, 'chrome', channel, platform, version_info): 368 _QueuePlatformUpdate('chrome', platform, version_info, config, 369 channel) 370 if args.update_chromium and platform in _CHROMIUM_PLATFORMS: 371 if _NeedsUpdate(config, 'chromium', channel, platform, version_info): 372 _QueuePlatformUpdate('chromium', platform, version_info, 373 config, channel) 374 375 print 'Updating builds with downloaded binaries' 376 config.ExecuteUpdateJobs(force=True) 377 378 379def main(): 380 logging.getLogger().setLevel(logging.DEBUG) 381 parser = argparse.ArgumentParser( 382 description='Update reference binaries used by perf bots.') 383 parser.add_argument('--no-update-chrome', action='store_false', 384 dest='update_chrome', default=True, 385 help='do not update chrome binaries') 386 parser.add_argument('--no-update-chromium', action='store_false', 387 dest='update_chromium', default=True, 388 help='do not update chromium binaries') 389 args = parser.parse_args() 390 UpdateBuilds(args) 391 392if __name__ == '__main__': 393 main() 394