1#!/usr/bin/env python 2# Copyright (c) 2012 The Chromium Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""This script is used to download prebuilt clang binaries. It runs as a 7"gclient hook" in Chromium checkouts. 8 9It can also be run stand-alone as a convenient way of installing a well-tested 10near-tip-of-tree clang version: 11 12 $ curl -s https://raw.githubusercontent.com/chromium/chromium/master/tools/clang/scripts/update.py | python - --output-dir=/tmp/clang 13 14(Note that the output dir may be deleted and re-created if it exists.) 15""" 16 17from __future__ import division 18from __future__ import print_function 19import argparse 20import os 21import shutil 22import stat 23import sys 24import tarfile 25import tempfile 26import time 27 28try: 29 from urllib2 import HTTPError, URLError, urlopen 30except ImportError: # For Py3 compatibility 31 from urllib.error import HTTPError, URLError 32 from urllib.request import urlopen 33 34import zipfile 35 36 37# Do NOT CHANGE this if you don't know what you're doing -- see 38# https://chromium.googlesource.com/chromium/src/+/master/docs/updating_clang.md 39# Reverting problematic clang rolls is safe, though. 40# This is the output of `git describe` and is usable as a commit-ish. 41CLANG_REVISION = 'llvmorg-12-init-11462-g418f18c6' 42CLANG_SUB_REVISION = 1 43 44PACKAGE_VERSION = '%s-%s' % (CLANG_REVISION, CLANG_SUB_REVISION) 45RELEASE_VERSION = '12.0.0' 46 47 48CDS_URL = os.environ.get('CDS_CLANG_BUCKET_OVERRIDE', 49 'https://commondatastorage.googleapis.com/chromium-browser-clang') 50 51# Path constants. (All of these should be absolute paths.) 52THIS_DIR = os.path.abspath(os.path.dirname(__file__)) 53CHROMIUM_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..', '..')) 54LLVM_BUILD_DIR = os.path.join(CHROMIUM_DIR, 'third_party', 'llvm-build', 55 'Release+Asserts') 56 57STAMP_FILE = os.path.normpath( 58 os.path.join(LLVM_BUILD_DIR, 'cr_build_revision')) 59OLD_STAMP_FILE = os.path.normpath( 60 os.path.join(LLVM_BUILD_DIR, '..', 'cr_build_revision')) 61FORCE_HEAD_REVISION_FILE = os.path.normpath(os.path.join(LLVM_BUILD_DIR, '..', 62 'force_head_revision')) 63 64 65def RmTree(dir): 66 """Delete dir.""" 67 def ChmodAndRetry(func, path, _): 68 # Subversion can leave read-only files around. 69 if not os.access(path, os.W_OK): 70 os.chmod(path, stat.S_IWUSR) 71 return func(path) 72 raise 73 shutil.rmtree(dir, onerror=ChmodAndRetry) 74 75 76def ReadStampFile(path): 77 """Return the contents of the stamp file, or '' if it doesn't exist.""" 78 try: 79 with open(path, 'r') as f: 80 return f.read().rstrip() 81 except IOError: 82 return '' 83 84 85def WriteStampFile(s, path): 86 """Write s to the stamp file.""" 87 EnsureDirExists(os.path.dirname(path)) 88 with open(path, 'w') as f: 89 f.write(s) 90 f.write('\n') 91 92 93def DownloadUrl(url, output_file): 94 """Download url into output_file.""" 95 CHUNK_SIZE = 4096 96 TOTAL_DOTS = 10 97 num_retries = 3 98 retry_wait_s = 5 # Doubled at each retry. 99 100 while True: 101 try: 102 sys.stdout.write('Downloading %s ' % url) 103 sys.stdout.flush() 104 response = urlopen(url) 105 total_size = int(response.info().get('Content-Length').strip()) 106 bytes_done = 0 107 dots_printed = 0 108 while True: 109 chunk = response.read(CHUNK_SIZE) 110 if not chunk: 111 break 112 output_file.write(chunk) 113 bytes_done += len(chunk) 114 num_dots = TOTAL_DOTS * bytes_done // total_size 115 sys.stdout.write('.' * (num_dots - dots_printed)) 116 sys.stdout.flush() 117 dots_printed = num_dots 118 if bytes_done != total_size: 119 raise URLError("only got %d of %d bytes" % 120 (bytes_done, total_size)) 121 print(' Done.') 122 return 123 except URLError as e: 124 sys.stdout.write('\n') 125 print(e) 126 if num_retries == 0 or isinstance(e, HTTPError) and e.code == 404: 127 raise e 128 num_retries -= 1 129 print('Retrying in %d s ...' % retry_wait_s) 130 sys.stdout.flush() 131 time.sleep(retry_wait_s) 132 retry_wait_s *= 2 133 134 135def EnsureDirExists(path): 136 if not os.path.exists(path): 137 os.makedirs(path) 138 139 140def DownloadAndUnpack(url, output_dir, path_prefixes=None): 141 """Download an archive from url and extract into output_dir. If path_prefixes 142 is not None, only extract files whose paths within the archive start with 143 any prefix in path_prefixes.""" 144 with tempfile.TemporaryFile() as f: 145 DownloadUrl(url, f) 146 f.seek(0) 147 EnsureDirExists(output_dir) 148 if url.endswith('.zip'): 149 assert path_prefixes is None 150 zipfile.ZipFile(f).extractall(path=output_dir) 151 else: 152 t = tarfile.open(mode='r:gz', fileobj=f) 153 members = None 154 if path_prefixes is not None: 155 members = [m for m in t.getmembers() 156 if any(m.name.startswith(p) for p in path_prefixes)] 157 t.extractall(path=output_dir, members=members) 158 159 160def GetPlatformUrlPrefix(host_os): 161 _HOST_OS_URL_MAP = { 162 'linux': 'Linux_x64', 163 'mac': 'Mac', 164 'win': 'Win', 165 } 166 return CDS_URL + '/' + _HOST_OS_URL_MAP[host_os] + '/' 167 168 169def DownloadAndUnpackPackage(package_file, output_dir, host_os): 170 cds_file = "%s-%s.tgz" % (package_file, PACKAGE_VERSION) 171 cds_full_url = GetPlatformUrlPrefix(host_os) + cds_file 172 try: 173 DownloadAndUnpack(cds_full_url, output_dir) 174 except URLError: 175 print('Failed to download prebuilt clang package %s' % cds_file) 176 print('Use build.py if you want to build locally.') 177 print('Exiting.') 178 sys.exit(1) 179 180 181# TODO(hans): Create a clang-win-runtime package instead. 182def DownloadAndUnpackClangWinRuntime(output_dir): 183 cds_file = "clang-%s.tgz" % PACKAGE_VERSION 184 cds_full_url = GetPlatformUrlPrefix('win') + cds_file 185 path_prefixes = [ 'lib/clang/' + RELEASE_VERSION + '/lib/', 186 'bin/llvm-symbolizer.exe' ] 187 try: 188 DownloadAndUnpack(cds_full_url, output_dir, path_prefixes) 189 except URLError: 190 print('Failed to download prebuilt clang %s' % cds_file) 191 print('Use build.py if you want to build locally.') 192 print('Exiting.') 193 sys.exit(1) 194 195 196def UpdatePackage(package_name, host_os): 197 stamp_file = None 198 package_file = None 199 200 stamp_file = os.path.join(LLVM_BUILD_DIR, package_name + '_revision') 201 if package_name == 'clang': 202 stamp_file = STAMP_FILE 203 package_file = 'clang' 204 elif package_name == 'clang-tidy': 205 package_file = 'clang-tidy' 206 elif package_name == 'lld_mac': 207 package_file = 'lld' 208 if host_os != 'mac': 209 print( 210 'The lld_mac package cannot be downloaded for non-macs.', 211 file=sys.stderr) 212 print( 213 'On non-mac, lld is included in the clang package.', file=sys.stderr) 214 return 1 215 elif package_name == 'objdump': 216 package_file = 'llvmobjdump' 217 elif package_name == 'translation_unit': 218 package_file = 'translation_unit' 219 elif package_name == 'coverage_tools': 220 stamp_file = os.path.join(LLVM_BUILD_DIR, 'cr_coverage_revision') 221 package_file = 'llvm-code-coverage' 222 elif package_name == 'libclang': 223 package_file = 'libclang' 224 else: 225 print('Unknown package: "%s".' % package_name) 226 return 1 227 228 assert stamp_file is not None 229 assert package_file is not None 230 231 # TODO(hans): Create a clang-win-runtime package and use separate DEPS hook. 232 target_os = [] 233 if package_name == 'clang': 234 try: 235 GCLIENT_CONFIG = os.path.join(os.path.dirname(CHROMIUM_DIR), '.gclient') 236 env = {} 237 exec (open(GCLIENT_CONFIG).read(), env, env) 238 target_os = env.get('target_os', target_os) 239 except: 240 pass 241 242 if os.path.exists(OLD_STAMP_FILE): 243 # Delete the old stamp file so it doesn't look like an old version of clang 244 # is available in case the user rolls back to an old version of this script 245 # during a bisect for example (crbug.com/988933). 246 os.remove(OLD_STAMP_FILE) 247 248 expected_stamp = ','.join([PACKAGE_VERSION] + target_os) 249 if ReadStampFile(stamp_file) == expected_stamp: 250 return 0 251 252 # Updating the main clang package nukes the output dir. Any other packages 253 # need to be updated *after* the clang package. 254 if package_name == 'clang' and os.path.exists(LLVM_BUILD_DIR): 255 RmTree(LLVM_BUILD_DIR) 256 257 DownloadAndUnpackPackage(package_file, LLVM_BUILD_DIR, host_os) 258 259 if package_name == 'clang' and 'win' in target_os: 260 # When doing win/cross builds on other hosts, get the Windows runtime 261 # libraries, and llvm-symbolizer.exe (needed in asan builds). 262 DownloadAndUnpackClangWinRuntime(LLVM_BUILD_DIR) 263 264 WriteStampFile(expected_stamp, stamp_file) 265 return 0 266 267 268def main(): 269 _PLATFORM_HOST_OS_MAP = { 270 'darwin': 'mac', 271 'cygwin': 'win', 272 'linux2': 'linux', 273 'win32': 'win', 274 } 275 default_host_os = _PLATFORM_HOST_OS_MAP.get(sys.platform, sys.platform) 276 277 parser = argparse.ArgumentParser(description='Update clang.') 278 parser.add_argument('--output-dir', 279 help='Where to extract the package.') 280 parser.add_argument('--package', 281 help='What package to update (default: clang)', 282 default='clang') 283 parser.add_argument( 284 '--host-os', 285 help='Which host OS to download for (default: %s)' % default_host_os, 286 default=default_host_os, 287 choices=('linux', 'mac', 'win')) 288 parser.add_argument('--force-local-build', action='store_true', 289 help='(no longer used)') 290 parser.add_argument('--print-revision', action='store_true', 291 help='Print current clang revision and exit.') 292 parser.add_argument('--llvm-force-head-revision', action='store_true', 293 help='Print locally built revision with --print-revision') 294 parser.add_argument('--print-clang-version', action='store_true', 295 help=('Print current clang release version (e.g. 9.0.0) ' 296 'and exit.')) 297 parser.add_argument('--verify-version', 298 help='Verify that clang has the passed-in version.') 299 args = parser.parse_args() 300 301 if args.force_local_build: 302 print(('update.py --force-local-build is no longer used to build clang; ' 303 'use build.py instead.')) 304 return 1 305 306 if args.verify_version and args.verify_version != RELEASE_VERSION: 307 print('RELEASE_VERSION is %s but --verify-version argument was %s.' % ( 308 RELEASE_VERSION, args.verify_version)) 309 print('clang_version in build/toolchain/toolchain.gni is likely outdated.') 310 return 1 311 312 if args.print_clang_version: 313 print(RELEASE_VERSION) 314 return 0 315 316 if args.print_revision: 317 if args.llvm_force_head_revision: 318 force_head_revision = ReadStampFile(FORCE_HEAD_REVISION_FILE) 319 if force_head_revision == '': 320 print('No locally built version found!') 321 return 1 322 print(force_head_revision) 323 return 0 324 325 print(PACKAGE_VERSION) 326 return 0 327 328 if args.llvm_force_head_revision: 329 print('--llvm-force-head-revision can only be used for --print-revision') 330 return 1 331 332 if args.output_dir: 333 global LLVM_BUILD_DIR, STAMP_FILE 334 LLVM_BUILD_DIR = os.path.abspath(args.output_dir) 335 STAMP_FILE = os.path.join(LLVM_BUILD_DIR, 'cr_build_revision') 336 337 return UpdatePackage(args.package, args.host_os) 338 339 340if __name__ == '__main__': 341 sys.exit(main()) 342