1#!/usr/bin/env python
2# Copyright (c) 2012 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""This script is used to download prebuilt clang binaries. It runs as a
7"gclient hook" in Chromium checkouts.
8
9It can also be run stand-alone as a convenient way of installing a well-tested
10near-tip-of-tree clang version:
11
12  $ curl -s https://raw.githubusercontent.com/chromium/chromium/main/tools/clang/scripts/update.py | python - --output-dir=/tmp/clang
13
14(Note that the output dir may be deleted and re-created if it exists.)
15"""
16
17from __future__ import division
18from __future__ import print_function
19import argparse
20import os
21import platform
22import shutil
23import stat
24import sys
25import tarfile
26import tempfile
27import time
28
29try:
30  from urllib2 import HTTPError, URLError, urlopen
31except ImportError: # For Py3 compatibility
32  from urllib.error import HTTPError, URLError
33  from urllib.request import urlopen
34
35import zipfile
36
37
38# Do NOT CHANGE this if you don't know what you're doing -- see
39# https://chromium.googlesource.com/chromium/src/+/main/docs/updating_clang.md
40# Reverting problematic clang rolls is safe, though.
41# This is the output of `git describe` and is usable as a commit-ish.
42CLANG_REVISION = 'llvmorg-14-init-4918-ge787678c'
43CLANG_SUB_REVISION = 1
44
45PACKAGE_VERSION = '%s-%s' % (CLANG_REVISION, CLANG_SUB_REVISION)
46RELEASE_VERSION = '14.0.0'
47
48CDS_URL = os.environ.get('CDS_CLANG_BUCKET_OVERRIDE',
49    'https://commondatastorage.googleapis.com/chromium-browser-clang')
50
51# Path constants. (All of these should be absolute paths.)
52THIS_DIR = os.path.abspath(os.path.dirname(__file__))
53CHROMIUM_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..', '..'))
54LLVM_BUILD_DIR = os.path.join(CHROMIUM_DIR, 'third_party', 'llvm-build',
55                              'Release+Asserts')
56
57STAMP_FILE = os.path.normpath(
58    os.path.join(LLVM_BUILD_DIR, 'cr_build_revision'))
59OLD_STAMP_FILE = os.path.normpath(
60    os.path.join(LLVM_BUILD_DIR, '..', 'cr_build_revision'))
61FORCE_HEAD_REVISION_FILE = os.path.normpath(os.path.join(LLVM_BUILD_DIR, '..',
62                                                   'force_head_revision'))
63
64
65def RmTree(dir):
66  """Delete dir."""
67  def ChmodAndRetry(func, path, _):
68    # Subversion can leave read-only files around.
69    if not os.access(path, os.W_OK):
70      os.chmod(path, stat.S_IWUSR)
71      return func(path)
72    raise
73  shutil.rmtree(dir, onerror=ChmodAndRetry)
74
75
76def ReadStampFile(path):
77  """Return the contents of the stamp file, or '' if it doesn't exist."""
78  try:
79    with open(path, 'r') as f:
80      return f.read().rstrip()
81  except IOError:
82    return ''
83
84
85def WriteStampFile(s, path):
86  """Write s to the stamp file."""
87  EnsureDirExists(os.path.dirname(path))
88  with open(path, 'w') as f:
89    f.write(s)
90    f.write('\n')
91
92
93def DownloadUrl(url, output_file):
94  """Download url into output_file."""
95  CHUNK_SIZE = 4096
96  TOTAL_DOTS = 10
97  num_retries = 3
98  retry_wait_s = 5  # Doubled at each retry.
99
100  while True:
101    try:
102      sys.stdout.write('Downloading %s ' % url)
103      sys.stdout.flush()
104      response = urlopen(url)
105      total_size = int(response.info().get('Content-Length').strip())
106      bytes_done = 0
107      dots_printed = 0
108      while True:
109        chunk = response.read(CHUNK_SIZE)
110        if not chunk:
111          break
112        output_file.write(chunk)
113        bytes_done += len(chunk)
114        num_dots = TOTAL_DOTS * bytes_done // total_size
115        sys.stdout.write('.' * (num_dots - dots_printed))
116        sys.stdout.flush()
117        dots_printed = num_dots
118      if bytes_done != total_size:
119        raise URLError("only got %d of %d bytes" %
120                       (bytes_done, total_size))
121      print(' Done.')
122      return
123    except URLError as e:
124      sys.stdout.write('\n')
125      print(e)
126      if num_retries == 0 or isinstance(e, HTTPError) and e.code == 404:
127        raise e
128      num_retries -= 1
129      print('Retrying in %d s ...' % retry_wait_s)
130      sys.stdout.flush()
131      time.sleep(retry_wait_s)
132      retry_wait_s *= 2
133
134
135def EnsureDirExists(path):
136  if not os.path.exists(path):
137    os.makedirs(path)
138
139
140def DownloadAndUnpack(url, output_dir, path_prefixes=None):
141  """Download an archive from url and extract into output_dir. If path_prefixes
142     is not None, only extract files whose paths within the archive start with
143     any prefix in path_prefixes."""
144  with tempfile.TemporaryFile() as f:
145    DownloadUrl(url, f)
146    f.seek(0)
147    EnsureDirExists(output_dir)
148    if url.endswith('.zip'):
149      assert path_prefixes is None
150      zipfile.ZipFile(f).extractall(path=output_dir)
151    else:
152      t = tarfile.open(mode='r:gz', fileobj=f)
153      members = None
154      if path_prefixes is not None:
155        members = [m for m in t.getmembers()
156                   if any(m.name.startswith(p) for p in path_prefixes)]
157      t.extractall(path=output_dir, members=members)
158
159
160def GetPlatformUrlPrefix(host_os):
161  _HOST_OS_URL_MAP = {
162      'linux': 'Linux_x64',
163      'mac': 'Mac',
164      'mac-arm64': 'Mac_arm64',
165      'win': 'Win',
166  }
167  return CDS_URL + '/' + _HOST_OS_URL_MAP[host_os] + '/'
168
169
170def DownloadAndUnpackPackage(package_file, output_dir, host_os):
171  cds_file = "%s-%s.tgz" % (package_file, PACKAGE_VERSION)
172  cds_full_url = GetPlatformUrlPrefix(host_os) + cds_file
173  try:
174    DownloadAndUnpack(cds_full_url, output_dir)
175  except URLError:
176    print('Failed to download prebuilt clang package %s' % cds_file)
177    print('Use build.py if you want to build locally.')
178    print('Exiting.')
179    sys.exit(1)
180
181
182def DownloadAndUnpackClangMacRuntime(output_dir):
183  cds_file = "clang-%s.tgz" % PACKAGE_VERSION
184  # We run this only for the runtime libraries, and 'mac' and 'mac-arm64' both
185  # have the same (universal) runtime libraries. It doesn't matter which one
186  # we download here.
187  cds_full_url = GetPlatformUrlPrefix('mac') + cds_file
188  path_prefixes = [
189      'lib/clang/' + RELEASE_VERSION + '/lib/darwin', 'include/c++/v1'
190  ]
191  try:
192    DownloadAndUnpack(cds_full_url, output_dir, path_prefixes)
193  except URLError:
194    print('Failed to download prebuilt clang %s' % cds_file)
195    print('Use build.py if you want to build locally.')
196    print('Exiting.')
197    sys.exit(1)
198
199
200# TODO(hans): Create a clang-win-runtime package instead.
201def DownloadAndUnpackClangWinRuntime(output_dir):
202  cds_file = "clang-%s.tgz" % PACKAGE_VERSION
203  cds_full_url = GetPlatformUrlPrefix('win') + cds_file
204  path_prefixes = [
205      'lib/clang/' + RELEASE_VERSION + '/lib/windows', 'bin/llvm-symbolizer.exe'
206  ]
207  try:
208    DownloadAndUnpack(cds_full_url, output_dir, path_prefixes)
209  except URLError:
210    print('Failed to download prebuilt clang %s' % cds_file)
211    print('Use build.py if you want to build locally.')
212    print('Exiting.')
213    sys.exit(1)
214
215
216def UpdatePackage(package_name, host_os):
217  stamp_file = None
218  package_file = None
219
220  stamp_file = os.path.join(LLVM_BUILD_DIR, package_name + '_revision')
221  if package_name == 'clang':
222    stamp_file = STAMP_FILE
223    package_file = 'clang'
224  elif package_name == 'clang-tidy':
225    package_file = 'clang-tidy'
226  elif package_name == 'objdump':
227    package_file = 'llvmobjdump'
228  elif package_name == 'translation_unit':
229    package_file = 'translation_unit'
230  elif package_name == 'coverage_tools':
231    stamp_file = os.path.join(LLVM_BUILD_DIR, 'cr_coverage_revision')
232    package_file = 'llvm-code-coverage'
233  elif package_name == 'libclang':
234    package_file = 'libclang'
235  else:
236    print('Unknown package: "%s".' % package_name)
237    return 1
238
239  assert stamp_file is not None
240  assert package_file is not None
241
242  # TODO(hans): Create a clang-win-runtime package and use separate DEPS hook.
243  target_os = []
244  if package_name == 'clang':
245    try:
246      GCLIENT_CONFIG = os.path.join(os.path.dirname(CHROMIUM_DIR), '.gclient')
247      env = {}
248      exec (open(GCLIENT_CONFIG).read(), env, env)
249      target_os = env.get('target_os', target_os)
250    except:
251      pass
252
253  if os.path.exists(OLD_STAMP_FILE):
254    # Delete the old stamp file so it doesn't look like an old version of clang
255    # is available in case the user rolls back to an old version of this script
256    # during a bisect for example (crbug.com/988933).
257    os.remove(OLD_STAMP_FILE)
258
259  expected_stamp = ','.join([PACKAGE_VERSION] + target_os)
260  if ReadStampFile(stamp_file) == expected_stamp:
261    return 0
262
263  # Updating the main clang package nukes the output dir. Any other packages
264  # need to be updated *after* the clang package.
265  if package_name == 'clang' and os.path.exists(LLVM_BUILD_DIR):
266    RmTree(LLVM_BUILD_DIR)
267
268  DownloadAndUnpackPackage(package_file, LLVM_BUILD_DIR, host_os)
269
270  if package_name == 'clang' and 'mac' in target_os:
271    DownloadAndUnpackClangMacRuntime(LLVM_BUILD_DIR)
272  if package_name == 'clang' and 'win' in target_os:
273    # When doing win/cross builds on other hosts, get the Windows runtime
274    # libraries, and llvm-symbolizer.exe (needed in asan builds).
275    DownloadAndUnpackClangWinRuntime(LLVM_BUILD_DIR)
276
277  WriteStampFile(expected_stamp, stamp_file)
278  return 0
279
280
281def main():
282  _PLATFORM_HOST_OS_MAP = {
283      'darwin': 'mac',
284      'cygwin': 'win',
285      'linux2': 'linux',
286      'win32': 'win',
287  }
288  default_host_os = _PLATFORM_HOST_OS_MAP.get(sys.platform, sys.platform)
289  if default_host_os == 'mac' and platform.machine() == 'arm64':
290    default_host_os = 'mac-arm64'
291
292  parser = argparse.ArgumentParser(description='Update clang.')
293  parser.add_argument('--output-dir',
294                      help='Where to extract the package.')
295  parser.add_argument('--package',
296                      help='What package to update (default: clang)',
297                      default='clang')
298  parser.add_argument('--host-os',
299                      help='Which host OS to download for (default: %s)' %
300                      default_host_os,
301                      default=default_host_os,
302                      choices=('linux', 'mac', 'mac-arm64', 'win'))
303  parser.add_argument('--print-revision', action='store_true',
304                      help='Print current clang revision and exit.')
305  parser.add_argument('--llvm-force-head-revision', action='store_true',
306                      help='Print locally built revision with --print-revision')
307  parser.add_argument('--print-clang-version', action='store_true',
308                      help=('Print current clang release version (e.g. 9.0.0) '
309                            'and exit.'))
310  parser.add_argument('--verify-version',
311                      help='Verify that clang has the passed-in version.')
312  args = parser.parse_args()
313
314  if args.verify_version and args.verify_version != RELEASE_VERSION:
315    print('RELEASE_VERSION is %s but --verify-version argument was %s.' % (
316        RELEASE_VERSION, args.verify_version))
317    print('clang_version in build/toolchain/toolchain.gni is likely outdated.')
318    return 1
319
320  if args.print_clang_version:
321    print(RELEASE_VERSION)
322    return 0
323
324  if args.print_revision:
325    if args.llvm_force_head_revision:
326      force_head_revision = ReadStampFile(FORCE_HEAD_REVISION_FILE)
327      if force_head_revision == '':
328        print('No locally built version found!')
329        return 1
330      print(force_head_revision)
331      return 0
332
333    print(PACKAGE_VERSION)
334    return 0
335
336  if args.llvm_force_head_revision:
337    print('--llvm-force-head-revision can only be used for --print-revision')
338    return 1
339
340  if args.output_dir:
341    global LLVM_BUILD_DIR, STAMP_FILE
342    LLVM_BUILD_DIR = os.path.abspath(args.output_dir)
343    STAMP_FILE = os.path.join(LLVM_BUILD_DIR, 'cr_build_revision')
344
345  return UpdatePackage(args.package, args.host_os)
346
347
348if __name__ == '__main__':
349  sys.exit(main())
350