1#!/usr/bin/env python
2#
3# Copyright 2017 The Chromium Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6r"""This script downloads / packages & uploads Android SDK packages.
7
8   It could be run when we need to update sdk packages to latest version.
9   It has 2 usages:
10     1) download: downloading a new version of the SDK via sdkmanager
11     2) package: wrapping SDK directory into CIPD-compatible packages and
12                 uploading the new packages via CIPD to server.
13                 Providing '--dry-run' option to show what packages to be
14                 created and uploaded without actually doing either.
15
16   Both downloading and uploading allows to either specify a package, or
17   deal with default packages (build-tools, platform-tools, platforms and
18   tools).
19
20   Example usage:
21     1) updating default packages:
22        $ update_sdk.py download
23        (optional) $ update_sdk.py package --dry-run
24        $ update_sdk.py package
25     2) updating a specified package:
26        $ update_sdk.py download -p "build-tools;27.0.3"
27        (optional) $ update_sdk.py package --dry-run -p build-tools \
28                     --version 27.0.3
29        $ update_sdk.py package -p build-tools --version 27.0.3
30
31   Note that `package` could update the package argument to the checkout
32   version in .gn file //build/config/android/config.gni. If having git
33   changes, please prepare to upload a CL that updates the SDK version.
34"""
35
36from __future__ import print_function
37
38import argparse
39import os
40import re
41import shutil
42import subprocess
43import sys
44import tempfile
45
46_SRC_ROOT = os.path.realpath(
47    os.path.join(os.path.dirname(__file__), '..', '..', '..'))
48
49_SRC_DEPS_PATH = os.path.join(_SRC_ROOT, 'DEPS')
50
51_SDK_PUBLIC_ROOT = os.path.join(_SRC_ROOT, 'third_party', 'android_sdk',
52                                'public')
53
54_SDKMANAGER_PATH = os.path.join(_SRC_ROOT, 'third_party', 'android_sdk',
55                                'public', 'tools', 'bin', 'sdkmanager')
56
57_ANDROID_CONFIG_GNI_PATH = os.path.join(_SRC_ROOT, 'build', 'config', 'android',
58                                        'config.gni')
59
60_TOOLS_LIB_PATH = os.path.join(_SDK_PUBLIC_ROOT, 'tools', 'lib')
61
62_DEFAULT_DOWNLOAD_PACKAGES = [
63    'build-tools', 'platform-tools', 'platforms', 'tools'
64]
65
66# TODO(shenghuazhang): Search package versions from available packages through
67# the sdkmanager, instead of hardcoding the package names w/ version.
68# TODO(yliuyliu): we might not need the latest version if unstable,
69# will double-check this later.
70_DEFAULT_PACKAGES_DICT = {
71    'build-tools': 'build-tools;27.0.3',
72    'platforms': 'platforms;android-28',
73    'sources': 'sources;android-28',
74}
75
76_GN_ARGUMENTS_TO_UPDATE = {
77    'build-tools': 'default_android_sdk_build_tools_version',
78    'tools': 'android_sdk_tools_version_suffix',
79    'platforms': 'default_android_sdk_version',
80}
81
82_COMMON_JAR_SUFFIX_PATTERN = re.compile(
83    r'^common'  # file name begins with 'common'
84    r'(-[\d\.]+(-dev)?)'  # group of suffix e.g.'-26.0.0-dev', '-25.3.2'
85    r'\.jar$'  # ends with .jar
86)
87
88
89def _DownloadSdk(arguments):
90  """Download sdk package from sdkmanager.
91
92  If package isn't provided, update build-tools, platform-tools, platforms,
93  and tools.
94
95  Args:
96    arguments: The arguments parsed from argparser.
97  """
98  for pkg in arguments.package:
99    # If package is not a sdk-style path, try to match a default path to it.
100    if pkg in _DEFAULT_PACKAGES_DICT:
101      print('Coercing %s to %s' % (pkg, _DEFAULT_PACKAGES_DICT[pkg]))
102      pkg = _DEFAULT_PACKAGES_DICT[pkg]
103
104    download_sdk_cmd = [
105        _SDKMANAGER_PATH, '--install',
106        '--sdk_root=%s' % arguments.sdk_root, pkg
107    ]
108    if arguments.verbose:
109      download_sdk_cmd.append('--verbose')
110
111    subprocess.check_call(download_sdk_cmd)
112
113
114def _FindPackageVersion(package, sdk_root):
115  """Find sdk package version.
116
117  Two options for package version:
118    1) Use the version in name if package name contains ';version'
119    2) For simple name package, search its version from 'Installed packages'
120       via `sdkmanager --list`
121
122  Args:
123    package: The Android SDK package.
124    sdk_root: The Android SDK root path.
125
126  Returns:
127    The version of package.
128
129  Raises:
130    Exception: cannot find the version of package.
131  """
132  sdkmanager_list_cmd = [
133      _SDKMANAGER_PATH,
134      '--list',
135      '--sdk_root=%s' % sdk_root,
136  ]
137
138  if package in _DEFAULT_PACKAGES_DICT:
139    # Get the version after ';' from package name
140    package = _DEFAULT_PACKAGES_DICT[package]
141    return package.split(';')[1]
142  else:
143    # Get the package version via `sdkmanager --list`. The logic is:
144    # Check through 'Installed packages' which is at the first section of
145    # `sdkmanager --list` output, example:
146    #   Installed packages:=====================] 100% Computing updates...
147    #     Path                        | Version | Description
148    #     -------                     | ------- | -------
149    #     build-tools;27.0.3          | 27.0.3  | Android SDK Build-Tools 27.0.3
150    #     emulator                    | 26.0.3  | Android Emulator
151    #     platforms;android-27        | 1       | Android SDK Platform 27
152    #     tools                       | 26.1.1  | Android SDK Tools
153    #
154    #   Available Packages:
155    #   ....
156    # When found a line containing the package path, grap its version between
157    # the first and second '|'. Since the 'Installed packages' list section ends
158    # by the first new line, the check loop should be ended when reaches a '\n'.
159    output = subprocess.check_output(sdkmanager_list_cmd)
160    for line in output.splitlines():
161      if ' ' + package + ' ' in line:
162        # if found package path, catch its version which in the first '|...|'
163        return line.split('|')[1].strip()
164      if line == '\n':  # Reaches the end of 'Installed packages' list
165        break
166    raise Exception('Cannot find the version of package %s' % package)
167
168
169def _ReplaceVersionInFile(file_path, pattern, version, dry_run=False):
170  """Replace the version of sdk package argument in file.
171
172  Check whether the version in file is the same as the new version first.
173  Replace the version if not dry run.
174
175  Args:
176    file_path: Path to the file to update the version of sdk package argument.
177    pattern: Pattern for the sdk package argument. Must capture at least one
178      group that the first group is the argument line excluding version.
179    version: The new version of the package.
180    dry_run: Bool. To show what packages would be created and packages, without
181      actually doing either.
182  """
183  with tempfile.NamedTemporaryFile() as temp_file:
184    with open(file_path) as f:
185      for line in f:
186        new_line = re.sub(pattern, r'\g<1>\g<2>%s\g<3>\n' % version, line)
187        if new_line != line:
188          print('    Note: file "%s" argument ' % file_path +
189                '"%s" would be updated to "%s".' % (line.strip(), version))
190        temp_file.write(new_line)
191    if not dry_run:
192      temp_file.flush()
193      shutil.move(temp_file.name, file_path)
194      temp_file.delete = False
195
196
197def GetCipdPackagePath(pkg_yaml_file):
198  """Find CIPD package path in .yaml file.
199
200  There should one line in .yaml file, e.g.:
201  "package: chrome_internal/third_party/android_sdk/internal/q/add-ons" or
202  "package: chromium/third_party/android_sdk/public/platforms"
203
204  Args:
205    pkg_yaml_file: The yaml file to find CIPD package path.
206
207  Returns:
208    The CIPD package path in yaml file.
209  """
210  cipd_package_path = ''
211  with open(pkg_yaml_file) as f:
212    pattern = re.compile(
213        # Match the argument with "package: "
214        r'(^\s*package:\s*)'
215        # The CIPD package path we want
216        r'([\w\/-]+)'
217        # End of string
218        r'(\s*?$)')
219    for line in f:
220      found = re.match(pattern, line)
221      if found:
222        cipd_package_path = found.group(2)
223        break
224  return cipd_package_path
225
226
227def UploadSdkPackage(sdk_root, dry_run, service_url, package, yaml_file,
228                     verbose):
229  """Build and upload a package instance file to CIPD.
230
231  This would also update gn and ensure files to the package version as
232  uploading to CIPD.
233
234  Args:
235    sdk_root: Root of the sdk packages.
236    dry_run: Bool. To show what packages would be created and packages, without
237      actually doing either.
238    service_url: The url of the CIPD service.
239    package: The package to be uploaded to CIPD.
240    yaml_file: Path to the yaml file that defines what to put into the package.
241      Default as //third_party/android_sdk/public/cipd_*.yaml
242    verbose: Enable more logging.
243
244  Returns:
245    New instance ID when CIPD package created.
246
247  Raises:
248    IOError: cannot find .yaml file, CIPD package path or instance ID for
249    package.
250    CalledProcessError: cipd command failed to create package.
251  """
252  pkg_yaml_file = yaml_file or os.path.join(sdk_root, 'cipd_%s.yaml' % package)
253  if not os.path.exists(pkg_yaml_file):
254    raise IOError('Cannot find .yaml file for package %s' % package)
255
256  cipd_package_path = GetCipdPackagePath(pkg_yaml_file)
257  if not cipd_package_path:
258    raise IOError('Cannot find CIPD package path in %s' % pkg_yaml_file)
259
260  if dry_run:
261    print('This `package` command (without -n/--dry-run) would create and ' +
262          'upload the package %s to CIPD.' % package)
263  else:
264    upload_sdk_cmd = [
265        'cipd', 'create', '-pkg-def', pkg_yaml_file, '-service-url', service_url
266    ]
267    if verbose:
268      upload_sdk_cmd.extend(['-log-level', 'debug'])
269
270    output = subprocess.check_output(upload_sdk_cmd)
271
272    # Need to match pattern to find new instance ID.
273    # e.g.: chromium/third_party/android_sdk/public/platforms:\
274    # Kg2t9p0YnQk8bldUv4VA3o156uPXLUfIFAmVZ-Gm5ewC
275    pattern = re.compile(
276        # Match the argument with "Instance: %s:" for cipd_package_path
277        (r'(^\s*Instance: %s:)' % cipd_package_path) +
278        # instance ID e.g. DLK621q5_Bga5EsOr7cp6bHWWxFKx6UHLu_Ix_m3AckC.
279        r'([-\w.]+)'
280        # End of string
281        r'(\s*?$)')
282    for line in output.splitlines():
283      found = re.match(pattern, line)
284      if found:
285        # Return new instance ID.
286        return found.group(2)
287    # Raises error if instance ID not found.
288    raise IOError('Cannot find instance ID by creating package %s' % package)
289
290
291def UpdateInstanceId(package,
292                     deps_path,
293                     dry_run,
294                     new_instance_id,
295                     release_version=None):
296  """Find the sdk pkg version in DEPS and modify it as cipd uploading version.
297
298  TODO(shenghuazhang): use DEPS edition operations after issue crbug.com/760633
299  fixed.
300
301  DEPS file hooks sdk package with version with suffix -crX, e.g. '26.0.2-cr1'.
302  If pkg_version is the base number of the existing version in DEPS, e.g.
303  '26.0.2', return '26.0.2-cr2' as the version uploading to CIPD. If not the
304  base number, return ${pkg_version}-cr0.
305
306  Args:
307    package: The name of the package.
308    deps_path: Path to deps file which gclient hooks sdk pkg w/ versions.
309    dry_run: Bool. To show what packages would be created and packages, without
310      actually doing either.
311    new_instance_id: New instance ID after CIPD package created.
312    release_version: Android sdk release version e.g. 'o_mr1', 'p'.
313  """
314  var_package = package
315  if release_version:
316    var_package = release_version + '_' + var_package
317  package_var_pattern = re.compile(
318      # Match the argument with "'android_sdk_*_version': '" with whitespaces.
319      r'(^\s*\'android_sdk_%s_version\'\s*:\s*\')' % var_package +
320      # instance ID e.g. DLK621q5_Bga5EsOr7cp6bHWWxFKx6UHLu_Ix_m3AckC.
321      r'([-\w.]+)'
322      # End of string
323      r'(\',?$)')
324
325  with tempfile.NamedTemporaryFile() as temp_file:
326    with open(deps_path) as f:
327      for line in f:
328        new_line = line
329        found = re.match(package_var_pattern, line)
330        if found:
331          instance_id = found.group(2)
332          new_line = re.sub(package_var_pattern,
333                            r'\g<1>%s\g<3>' % new_instance_id, line)
334          print(
335              '    Note: deps file "%s" argument ' % deps_path +
336              '"%s" would be updated to "%s".' % (instance_id, new_instance_id))
337        temp_file.write(new_line)
338
339    if not dry_run:
340      temp_file.flush()
341      shutil.move(temp_file.name, deps_path)
342      temp_file.delete = False
343
344
345def ChangeVersionInGNI(package, arg_version, gn_args_dict, gni_file_path,
346                       dry_run):
347  """Change the sdk package version in config.gni file."""
348  if package in gn_args_dict:
349    version_config_name = gn_args_dict.get(package)
350    # Regex to parse the line of sdk package version gn argument, e.g.
351    # '  default_android_sdk_version = "27"'. Capture a group for the line
352    # excluding the version.
353    gn_arg_pattern = re.compile(
354        # Match the argument with '=' and whitespaces. Capture a group for it.
355        r'(^\s*%s\s*=\s*)' % version_config_name +
356        # Optional quote.
357        r'("?)' +
358        # Version number. E.g. 27, 27.0.3, -26.0.0-dev
359        r'(?:[-\w\s.]+)' +
360        # Optional quote.
361        r'("?)' +
362        # End of string
363        r'$')
364
365    _ReplaceVersionInFile(gni_file_path, gn_arg_pattern, arg_version, dry_run)
366
367
368def GetToolsSuffix(tools_lib_path):
369  """Get the gn config of package 'tools' suffix.
370
371  Check jar file name of 'common*.jar' in tools/lib, which could be
372  'common.jar', common-<version>-dev.jar' or 'common-<version>.jar'.
373  If suffix exists, return the suffix.
374
375  Args:
376    tools_lib_path: The path of tools/lib.
377
378  Returns:
379    The suffix of tools package.
380  """
381  tools_lib_jars_list = os.listdir(tools_lib_path)
382  for file_name in tools_lib_jars_list:
383    found = re.match(_COMMON_JAR_SUFFIX_PATTERN, file_name)
384    if found:
385      return found.group(1)
386
387
388def _GetArgVersion(pkg_version, package):
389  """Get the argument version.
390
391  Args:
392    pkg_version: The package version.
393    package: The package name.
394
395  Returns:
396    The argument version.
397  """
398  # Remove all chars except for digits and dots in version
399  arg_version = re.sub(r'[^\d\.]', '', pkg_version)
400
401  if package == 'tools':
402    suffix = GetToolsSuffix(_TOOLS_LIB_PATH)
403    if suffix:
404      arg_version = suffix
405    else:
406      arg_version = '-%s' % arg_version
407  return arg_version
408
409
410def _UploadSdkPackage(arguments):
411  """Upload SDK packages to CIPD.
412
413  Args:
414    arguments: The arguments parsed by argparser.
415
416  Raises:
417    IOError: Don't use --version/--yaml-file for default packages.
418  """
419  packages = arguments.package
420  if not packages:
421    packages = _DEFAULT_DOWNLOAD_PACKAGES
422    if arguments.version or arguments.yaml_file:
423      raise IOError("Don't use --version/--yaml-file for default packages.")
424
425  for package in packages:
426    pkg_version = arguments.version
427    if not pkg_version:
428      pkg_version = _FindPackageVersion(package, arguments.sdk_root)
429
430    # Upload SDK package to CIPD, and update the package instance ID hooking
431    # in DEPS file.
432    new_instance_id = UploadSdkPackage(
433        os.path.join(arguments.sdk_root, '..'), arguments.dry_run,
434        arguments.service_url, package, arguments.yaml_file, arguments.verbose)
435    UpdateInstanceId(package, _SRC_DEPS_PATH, arguments.dry_run,
436                     new_instance_id)
437
438    if package in _GN_ARGUMENTS_TO_UPDATE:
439      # Update the package version config in gn file
440      arg_version = _GetArgVersion(pkg_version, package)
441      ChangeVersionInGNI(package, arg_version, _GN_ARGUMENTS_TO_UPDATE,
442                         _ANDROID_CONFIG_GNI_PATH, arguments.dry_run)
443
444
445def main():
446  parser = argparse.ArgumentParser(
447      description='A script to download Android SDK packages '
448      'via sdkmanager and upload to CIPD.')
449
450  subparsers = parser.add_subparsers(title='commands')
451
452  download_parser = subparsers.add_parser(
453      'download',
454      help='Download sdk package to the latest version from sdkmanager.')
455  download_parser.set_defaults(func=_DownloadSdk)
456  download_parser.add_argument(
457      '-p',
458      '--package',
459      nargs=1,
460      default=_DEFAULT_DOWNLOAD_PACKAGES,
461      help='The package of the SDK needs to be installed/updated. '
462      'Note that package name should be a sdk-style path e.g. '
463      '"platforms;android-27" or "platform-tools". If package '
464      'is not specified, update "build-tools;27.0.3", "tools" '
465      '"platform-tools" and "platforms;android-27" by default.')
466  download_parser.add_argument(
467      '--sdk-root', help='base path to the Android SDK root')
468  download_parser.add_argument(
469      '-v', '--verbose', action='store_true', help='print debug information')
470
471  package_parser = subparsers.add_parser(
472      'package', help='Create and upload package instance file to CIPD.')
473  package_parser.set_defaults(func=_UploadSdkPackage)
474  package_parser.add_argument(
475      '-n',
476      '--dry-run',
477      action='store_true',
478      help='Dry run won\'t trigger creating instances or uploading packages. '
479      'It shows what packages would be created and uploaded to CIPD. '
480      'It also shows the possible updates of sdk version on files.')
481  package_parser.add_argument(
482      '-p',
483      '--package',
484      nargs=1,
485      help='The package to be uploaded to CIPD. Note that package '
486      'name is a simple path e.g. "platforms" or "build-tools" '
487      'which matches package name on CIPD service. Default by '
488      'build-tools, platform-tools, platforms and tools')
489  package_parser.add_argument(
490      '--version',
491      help='Version of the uploading package instance through CIPD.')
492  package_parser.add_argument(
493      '--yaml-file',
494      help='Path to *.yaml file that defines what to put into the package.'
495      'Default as //third_party/android_sdk/public/cipd_<package>.yaml')
496  package_parser.add_argument(
497      '--service-url',
498      help='The url of the CIPD service.',
499      default='https://chrome-infra-packages.appspot.com')
500  package_parser.add_argument(
501      '--sdk-root', help='base path to the Android SDK root')
502  package_parser.add_argument(
503      '-v', '--verbose', action='store_true', help='print debug information')
504
505  args = parser.parse_args()
506
507  if not args.sdk_root:
508    args.sdk_root = _SDK_PUBLIC_ROOT
509
510  args.func(args)
511
512
513if __name__ == '__main__':
514  sys.exit(main())
515