1#!/usr/bin/env python 2# 3# Copyright 2017 The Chromium Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6r"""This script downloads / packages & uploads Android SDK packages. 7 8 It could be run when we need to update sdk packages to latest version. 9 It has 2 usages: 10 1) download: downloading a new version of the SDK via sdkmanager 11 2) package: wrapping SDK directory into CIPD-compatible packages and 12 uploading the new packages via CIPD to server. 13 Providing '--dry-run' option to show what packages to be 14 created and uploaded without actually doing either. 15 16 Both downloading and uploading allows to either specify a package, or 17 deal with default packages (build-tools, platform-tools, platforms and 18 tools). 19 20 Example usage: 21 1) updating default packages: 22 $ update_sdk.py download 23 (optional) $ update_sdk.py package --dry-run 24 $ update_sdk.py package 25 2) updating a specified package: 26 $ update_sdk.py download -p "build-tools;27.0.3" 27 (optional) $ update_sdk.py package --dry-run -p build-tools \ 28 --version 27.0.3 29 $ update_sdk.py package -p build-tools --version 27.0.3 30 31 Note that `package` could update the package argument to the checkout 32 version in .gn file //build/config/android/config.gni. If having git 33 changes, please prepare to upload a CL that updates the SDK version. 34""" 35 36from __future__ import print_function 37 38import argparse 39import os 40import re 41import shutil 42import subprocess 43import sys 44import tempfile 45 46_SRC_ROOT = os.path.realpath( 47 os.path.join(os.path.dirname(__file__), '..', '..', '..')) 48 49_SRC_DEPS_PATH = os.path.join(_SRC_ROOT, 'DEPS') 50 51_SDK_PUBLIC_ROOT = os.path.join(_SRC_ROOT, 'third_party', 'android_sdk', 52 'public') 53 54_SDKMANAGER_PATH = os.path.join(_SRC_ROOT, 'third_party', 'android_sdk', 55 'public', 'tools', 'bin', 'sdkmanager') 56 57_ANDROID_CONFIG_GNI_PATH = os.path.join(_SRC_ROOT, 'build', 'config', 'android', 58 'config.gni') 59 60_TOOLS_LIB_PATH = os.path.join(_SDK_PUBLIC_ROOT, 'tools', 'lib') 61 62_DEFAULT_DOWNLOAD_PACKAGES = [ 63 'build-tools', 'platform-tools', 'platforms', 'tools' 64] 65 66# TODO(shenghuazhang): Search package versions from available packages through 67# the sdkmanager, instead of hardcoding the package names w/ version. 68# TODO(yliuyliu): we might not need the latest version if unstable, 69# will double-check this later. 70_DEFAULT_PACKAGES_DICT = { 71 'build-tools': 'build-tools;27.0.3', 72 'platforms': 'platforms;android-28', 73 'sources': 'sources;android-28', 74} 75 76_GN_ARGUMENTS_TO_UPDATE = { 77 'build-tools': 'default_android_sdk_build_tools_version', 78 'tools': 'android_sdk_tools_version_suffix', 79 'platforms': 'default_android_sdk_version', 80} 81 82_COMMON_JAR_SUFFIX_PATTERN = re.compile( 83 r'^common' # file name begins with 'common' 84 r'(-[\d\.]+(-dev)?)' # group of suffix e.g.'-26.0.0-dev', '-25.3.2' 85 r'\.jar$' # ends with .jar 86) 87 88 89def _DownloadSdk(arguments): 90 """Download sdk package from sdkmanager. 91 92 If package isn't provided, update build-tools, platform-tools, platforms, 93 and tools. 94 95 Args: 96 arguments: The arguments parsed from argparser. 97 """ 98 for pkg in arguments.package: 99 # If package is not a sdk-style path, try to match a default path to it. 100 if pkg in _DEFAULT_PACKAGES_DICT: 101 print('Coercing %s to %s' % (pkg, _DEFAULT_PACKAGES_DICT[pkg])) 102 pkg = _DEFAULT_PACKAGES_DICT[pkg] 103 104 download_sdk_cmd = [ 105 _SDKMANAGER_PATH, '--install', 106 '--sdk_root=%s' % arguments.sdk_root, pkg 107 ] 108 if arguments.verbose: 109 download_sdk_cmd.append('--verbose') 110 111 subprocess.check_call(download_sdk_cmd) 112 113 114def _FindPackageVersion(package, sdk_root): 115 """Find sdk package version. 116 117 Two options for package version: 118 1) Use the version in name if package name contains ';version' 119 2) For simple name package, search its version from 'Installed packages' 120 via `sdkmanager --list` 121 122 Args: 123 package: The Android SDK package. 124 sdk_root: The Android SDK root path. 125 126 Returns: 127 The version of package. 128 129 Raises: 130 Exception: cannot find the version of package. 131 """ 132 sdkmanager_list_cmd = [ 133 _SDKMANAGER_PATH, 134 '--list', 135 '--sdk_root=%s' % sdk_root, 136 ] 137 138 if package in _DEFAULT_PACKAGES_DICT: 139 # Get the version after ';' from package name 140 package = _DEFAULT_PACKAGES_DICT[package] 141 return package.split(';')[1] 142 else: 143 # Get the package version via `sdkmanager --list`. The logic is: 144 # Check through 'Installed packages' which is at the first section of 145 # `sdkmanager --list` output, example: 146 # Installed packages:=====================] 100% Computing updates... 147 # Path | Version | Description 148 # ------- | ------- | ------- 149 # build-tools;27.0.3 | 27.0.3 | Android SDK Build-Tools 27.0.3 150 # emulator | 26.0.3 | Android Emulator 151 # platforms;android-27 | 1 | Android SDK Platform 27 152 # tools | 26.1.1 | Android SDK Tools 153 # 154 # Available Packages: 155 # .... 156 # When found a line containing the package path, grap its version between 157 # the first and second '|'. Since the 'Installed packages' list section ends 158 # by the first new line, the check loop should be ended when reaches a '\n'. 159 output = subprocess.check_output(sdkmanager_list_cmd) 160 for line in output.splitlines(): 161 if ' ' + package + ' ' in line: 162 # if found package path, catch its version which in the first '|...|' 163 return line.split('|')[1].strip() 164 if line == '\n': # Reaches the end of 'Installed packages' list 165 break 166 raise Exception('Cannot find the version of package %s' % package) 167 168 169def _ReplaceVersionInFile(file_path, pattern, version, dry_run=False): 170 """Replace the version of sdk package argument in file. 171 172 Check whether the version in file is the same as the new version first. 173 Replace the version if not dry run. 174 175 Args: 176 file_path: Path to the file to update the version of sdk package argument. 177 pattern: Pattern for the sdk package argument. Must capture at least one 178 group that the first group is the argument line excluding version. 179 version: The new version of the package. 180 dry_run: Bool. To show what packages would be created and packages, without 181 actually doing either. 182 """ 183 with tempfile.NamedTemporaryFile() as temp_file: 184 with open(file_path) as f: 185 for line in f: 186 new_line = re.sub(pattern, r'\g<1>\g<2>%s\g<3>\n' % version, line) 187 if new_line != line: 188 print(' Note: file "%s" argument ' % file_path + 189 '"%s" would be updated to "%s".' % (line.strip(), version)) 190 temp_file.write(new_line) 191 if not dry_run: 192 temp_file.flush() 193 shutil.move(temp_file.name, file_path) 194 temp_file.delete = False 195 196 197def GetCipdPackagePath(pkg_yaml_file): 198 """Find CIPD package path in .yaml file. 199 200 There should one line in .yaml file, e.g.: 201 "package: chrome_internal/third_party/android_sdk/internal/q/add-ons" or 202 "package: chromium/third_party/android_sdk/public/platforms" 203 204 Args: 205 pkg_yaml_file: The yaml file to find CIPD package path. 206 207 Returns: 208 The CIPD package path in yaml file. 209 """ 210 cipd_package_path = '' 211 with open(pkg_yaml_file) as f: 212 pattern = re.compile( 213 # Match the argument with "package: " 214 r'(^\s*package:\s*)' 215 # The CIPD package path we want 216 r'([\w\/-]+)' 217 # End of string 218 r'(\s*?$)') 219 for line in f: 220 found = re.match(pattern, line) 221 if found: 222 cipd_package_path = found.group(2) 223 break 224 return cipd_package_path 225 226 227def UploadSdkPackage(sdk_root, dry_run, service_url, package, yaml_file, 228 verbose): 229 """Build and upload a package instance file to CIPD. 230 231 This would also update gn and ensure files to the package version as 232 uploading to CIPD. 233 234 Args: 235 sdk_root: Root of the sdk packages. 236 dry_run: Bool. To show what packages would be created and packages, without 237 actually doing either. 238 service_url: The url of the CIPD service. 239 package: The package to be uploaded to CIPD. 240 yaml_file: Path to the yaml file that defines what to put into the package. 241 Default as //third_party/android_sdk/public/cipd_*.yaml 242 verbose: Enable more logging. 243 244 Returns: 245 New instance ID when CIPD package created. 246 247 Raises: 248 IOError: cannot find .yaml file, CIPD package path or instance ID for 249 package. 250 CalledProcessError: cipd command failed to create package. 251 """ 252 pkg_yaml_file = yaml_file or os.path.join(sdk_root, 'cipd_%s.yaml' % package) 253 if not os.path.exists(pkg_yaml_file): 254 raise IOError('Cannot find .yaml file for package %s' % package) 255 256 cipd_package_path = GetCipdPackagePath(pkg_yaml_file) 257 if not cipd_package_path: 258 raise IOError('Cannot find CIPD package path in %s' % pkg_yaml_file) 259 260 if dry_run: 261 print('This `package` command (without -n/--dry-run) would create and ' + 262 'upload the package %s to CIPD.' % package) 263 else: 264 upload_sdk_cmd = [ 265 'cipd', 'create', '-pkg-def', pkg_yaml_file, '-service-url', service_url 266 ] 267 if verbose: 268 upload_sdk_cmd.extend(['-log-level', 'debug']) 269 270 output = subprocess.check_output(upload_sdk_cmd) 271 272 # Need to match pattern to find new instance ID. 273 # e.g.: chromium/third_party/android_sdk/public/platforms:\ 274 # Kg2t9p0YnQk8bldUv4VA3o156uPXLUfIFAmVZ-Gm5ewC 275 pattern = re.compile( 276 # Match the argument with "Instance: %s:" for cipd_package_path 277 (r'(^\s*Instance: %s:)' % cipd_package_path) + 278 # instance ID e.g. DLK621q5_Bga5EsOr7cp6bHWWxFKx6UHLu_Ix_m3AckC. 279 r'([-\w.]+)' 280 # End of string 281 r'(\s*?$)') 282 for line in output.splitlines(): 283 found = re.match(pattern, line) 284 if found: 285 # Return new instance ID. 286 return found.group(2) 287 # Raises error if instance ID not found. 288 raise IOError('Cannot find instance ID by creating package %s' % package) 289 290 291def UpdateInstanceId(package, 292 deps_path, 293 dry_run, 294 new_instance_id, 295 release_version=None): 296 """Find the sdk pkg version in DEPS and modify it as cipd uploading version. 297 298 TODO(shenghuazhang): use DEPS edition operations after issue crbug.com/760633 299 fixed. 300 301 DEPS file hooks sdk package with version with suffix -crX, e.g. '26.0.2-cr1'. 302 If pkg_version is the base number of the existing version in DEPS, e.g. 303 '26.0.2', return '26.0.2-cr2' as the version uploading to CIPD. If not the 304 base number, return ${pkg_version}-cr0. 305 306 Args: 307 package: The name of the package. 308 deps_path: Path to deps file which gclient hooks sdk pkg w/ versions. 309 dry_run: Bool. To show what packages would be created and packages, without 310 actually doing either. 311 new_instance_id: New instance ID after CIPD package created. 312 release_version: Android sdk release version e.g. 'o_mr1', 'p'. 313 """ 314 var_package = package 315 if release_version: 316 var_package = release_version + '_' + var_package 317 package_var_pattern = re.compile( 318 # Match the argument with "'android_sdk_*_version': '" with whitespaces. 319 r'(^\s*\'android_sdk_%s_version\'\s*:\s*\')' % var_package + 320 # instance ID e.g. DLK621q5_Bga5EsOr7cp6bHWWxFKx6UHLu_Ix_m3AckC. 321 r'([-\w.]+)' 322 # End of string 323 r'(\',?$)') 324 325 with tempfile.NamedTemporaryFile() as temp_file: 326 with open(deps_path) as f: 327 for line in f: 328 new_line = line 329 found = re.match(package_var_pattern, line) 330 if found: 331 instance_id = found.group(2) 332 new_line = re.sub(package_var_pattern, 333 r'\g<1>%s\g<3>' % new_instance_id, line) 334 print( 335 ' Note: deps file "%s" argument ' % deps_path + 336 '"%s" would be updated to "%s".' % (instance_id, new_instance_id)) 337 temp_file.write(new_line) 338 339 if not dry_run: 340 temp_file.flush() 341 shutil.move(temp_file.name, deps_path) 342 temp_file.delete = False 343 344 345def ChangeVersionInGNI(package, arg_version, gn_args_dict, gni_file_path, 346 dry_run): 347 """Change the sdk package version in config.gni file.""" 348 if package in gn_args_dict: 349 version_config_name = gn_args_dict.get(package) 350 # Regex to parse the line of sdk package version gn argument, e.g. 351 # ' default_android_sdk_version = "27"'. Capture a group for the line 352 # excluding the version. 353 gn_arg_pattern = re.compile( 354 # Match the argument with '=' and whitespaces. Capture a group for it. 355 r'(^\s*%s\s*=\s*)' % version_config_name + 356 # Optional quote. 357 r'("?)' + 358 # Version number. E.g. 27, 27.0.3, -26.0.0-dev 359 r'(?:[-\w\s.]+)' + 360 # Optional quote. 361 r'("?)' + 362 # End of string 363 r'$') 364 365 _ReplaceVersionInFile(gni_file_path, gn_arg_pattern, arg_version, dry_run) 366 367 368def GetToolsSuffix(tools_lib_path): 369 """Get the gn config of package 'tools' suffix. 370 371 Check jar file name of 'common*.jar' in tools/lib, which could be 372 'common.jar', common-<version>-dev.jar' or 'common-<version>.jar'. 373 If suffix exists, return the suffix. 374 375 Args: 376 tools_lib_path: The path of tools/lib. 377 378 Returns: 379 The suffix of tools package. 380 """ 381 tools_lib_jars_list = os.listdir(tools_lib_path) 382 for file_name in tools_lib_jars_list: 383 found = re.match(_COMMON_JAR_SUFFIX_PATTERN, file_name) 384 if found: 385 return found.group(1) 386 387 388def _GetArgVersion(pkg_version, package): 389 """Get the argument version. 390 391 Args: 392 pkg_version: The package version. 393 package: The package name. 394 395 Returns: 396 The argument version. 397 """ 398 # Remove all chars except for digits and dots in version 399 arg_version = re.sub(r'[^\d\.]', '', pkg_version) 400 401 if package == 'tools': 402 suffix = GetToolsSuffix(_TOOLS_LIB_PATH) 403 if suffix: 404 arg_version = suffix 405 else: 406 arg_version = '-%s' % arg_version 407 return arg_version 408 409 410def _UploadSdkPackage(arguments): 411 """Upload SDK packages to CIPD. 412 413 Args: 414 arguments: The arguments parsed by argparser. 415 416 Raises: 417 IOError: Don't use --version/--yaml-file for default packages. 418 """ 419 packages = arguments.package 420 if not packages: 421 packages = _DEFAULT_DOWNLOAD_PACKAGES 422 if arguments.version or arguments.yaml_file: 423 raise IOError("Don't use --version/--yaml-file for default packages.") 424 425 for package in packages: 426 pkg_version = arguments.version 427 if not pkg_version: 428 pkg_version = _FindPackageVersion(package, arguments.sdk_root) 429 430 # Upload SDK package to CIPD, and update the package instance ID hooking 431 # in DEPS file. 432 new_instance_id = UploadSdkPackage( 433 os.path.join(arguments.sdk_root, '..'), arguments.dry_run, 434 arguments.service_url, package, arguments.yaml_file, arguments.verbose) 435 UpdateInstanceId(package, _SRC_DEPS_PATH, arguments.dry_run, 436 new_instance_id) 437 438 if package in _GN_ARGUMENTS_TO_UPDATE: 439 # Update the package version config in gn file 440 arg_version = _GetArgVersion(pkg_version, package) 441 ChangeVersionInGNI(package, arg_version, _GN_ARGUMENTS_TO_UPDATE, 442 _ANDROID_CONFIG_GNI_PATH, arguments.dry_run) 443 444 445def main(): 446 parser = argparse.ArgumentParser( 447 description='A script to download Android SDK packages ' 448 'via sdkmanager and upload to CIPD.') 449 450 subparsers = parser.add_subparsers(title='commands') 451 452 download_parser = subparsers.add_parser( 453 'download', 454 help='Download sdk package to the latest version from sdkmanager.') 455 download_parser.set_defaults(func=_DownloadSdk) 456 download_parser.add_argument( 457 '-p', 458 '--package', 459 nargs=1, 460 default=_DEFAULT_DOWNLOAD_PACKAGES, 461 help='The package of the SDK needs to be installed/updated. ' 462 'Note that package name should be a sdk-style path e.g. ' 463 '"platforms;android-27" or "platform-tools". If package ' 464 'is not specified, update "build-tools;27.0.3", "tools" ' 465 '"platform-tools" and "platforms;android-27" by default.') 466 download_parser.add_argument( 467 '--sdk-root', help='base path to the Android SDK root') 468 download_parser.add_argument( 469 '-v', '--verbose', action='store_true', help='print debug information') 470 471 package_parser = subparsers.add_parser( 472 'package', help='Create and upload package instance file to CIPD.') 473 package_parser.set_defaults(func=_UploadSdkPackage) 474 package_parser.add_argument( 475 '-n', 476 '--dry-run', 477 action='store_true', 478 help='Dry run won\'t trigger creating instances or uploading packages. ' 479 'It shows what packages would be created and uploaded to CIPD. ' 480 'It also shows the possible updates of sdk version on files.') 481 package_parser.add_argument( 482 '-p', 483 '--package', 484 nargs=1, 485 help='The package to be uploaded to CIPD. Note that package ' 486 'name is a simple path e.g. "platforms" or "build-tools" ' 487 'which matches package name on CIPD service. Default by ' 488 'build-tools, platform-tools, platforms and tools') 489 package_parser.add_argument( 490 '--version', 491 help='Version of the uploading package instance through CIPD.') 492 package_parser.add_argument( 493 '--yaml-file', 494 help='Path to *.yaml file that defines what to put into the package.' 495 'Default as //third_party/android_sdk/public/cipd_<package>.yaml') 496 package_parser.add_argument( 497 '--service-url', 498 help='The url of the CIPD service.', 499 default='https://chrome-infra-packages.appspot.com') 500 package_parser.add_argument( 501 '--sdk-root', help='base path to the Android SDK root') 502 package_parser.add_argument( 503 '-v', '--verbose', action='store_true', help='print debug information') 504 505 args = parser.parse_args() 506 507 if not args.sdk_root: 508 args.sdk_root = _SDK_PUBLIC_ROOT 509 510 args.func(args) 511 512 513if __name__ == '__main__': 514 sys.exit(main()) 515