1#!/usr/bin/env python 2# Copyright (c) 2012 The Chromium Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""This script is used to download prebuilt clang binaries. It runs as a 7"gclient hook" in Chromium checkouts. 8 9It can also be run stand-alone as a convenient way of installing a well-tested 10near-tip-of-tree clang version: 11 12 $ curl -s https://raw.githubusercontent.com/chromium/chromium/main/tools/clang/scripts/update.py | python - --output-dir=/tmp/clang 13 14(Note that the output dir may be deleted and re-created if it exists.) 15""" 16 17from __future__ import division 18from __future__ import print_function 19import argparse 20import os 21import platform 22import shutil 23import stat 24import sys 25import tarfile 26import tempfile 27import time 28 29try: 30 from urllib2 import HTTPError, URLError, urlopen 31except ImportError: # For Py3 compatibility 32 from urllib.error import HTTPError, URLError 33 from urllib.request import urlopen 34 35import zipfile 36 37 38# Do NOT CHANGE this if you don't know what you're doing -- see 39# https://chromium.googlesource.com/chromium/src/+/main/docs/updating_clang.md 40# Reverting problematic clang rolls is safe, though. 41# This is the output of `git describe` and is usable as a commit-ish. 42CLANG_REVISION = 'llvmorg-14-init-4918-ge787678c' 43CLANG_SUB_REVISION = 1 44 45PACKAGE_VERSION = '%s-%s' % (CLANG_REVISION, CLANG_SUB_REVISION) 46RELEASE_VERSION = '14.0.0' 47 48CDS_URL = os.environ.get('CDS_CLANG_BUCKET_OVERRIDE', 49 'https://commondatastorage.googleapis.com/chromium-browser-clang') 50 51# Path constants. (All of these should be absolute paths.) 52THIS_DIR = os.path.abspath(os.path.dirname(__file__)) 53CHROMIUM_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..', '..')) 54LLVM_BUILD_DIR = os.path.join(CHROMIUM_DIR, 'third_party', 'llvm-build', 55 'Release+Asserts') 56 57STAMP_FILE = os.path.normpath( 58 os.path.join(LLVM_BUILD_DIR, 'cr_build_revision')) 59OLD_STAMP_FILE = os.path.normpath( 60 os.path.join(LLVM_BUILD_DIR, '..', 'cr_build_revision')) 61FORCE_HEAD_REVISION_FILE = os.path.normpath(os.path.join(LLVM_BUILD_DIR, '..', 62 'force_head_revision')) 63 64 65def RmTree(dir): 66 """Delete dir.""" 67 def ChmodAndRetry(func, path, _): 68 # Subversion can leave read-only files around. 69 if not os.access(path, os.W_OK): 70 os.chmod(path, stat.S_IWUSR) 71 return func(path) 72 raise 73 shutil.rmtree(dir, onerror=ChmodAndRetry) 74 75 76def ReadStampFile(path): 77 """Return the contents of the stamp file, or '' if it doesn't exist.""" 78 try: 79 with open(path, 'r') as f: 80 return f.read().rstrip() 81 except IOError: 82 return '' 83 84 85def WriteStampFile(s, path): 86 """Write s to the stamp file.""" 87 EnsureDirExists(os.path.dirname(path)) 88 with open(path, 'w') as f: 89 f.write(s) 90 f.write('\n') 91 92 93def DownloadUrl(url, output_file): 94 """Download url into output_file.""" 95 CHUNK_SIZE = 4096 96 TOTAL_DOTS = 10 97 num_retries = 3 98 retry_wait_s = 5 # Doubled at each retry. 99 100 while True: 101 try: 102 sys.stdout.write('Downloading %s ' % url) 103 sys.stdout.flush() 104 response = urlopen(url) 105 total_size = int(response.info().get('Content-Length').strip()) 106 bytes_done = 0 107 dots_printed = 0 108 while True: 109 chunk = response.read(CHUNK_SIZE) 110 if not chunk: 111 break 112 output_file.write(chunk) 113 bytes_done += len(chunk) 114 num_dots = TOTAL_DOTS * bytes_done // total_size 115 sys.stdout.write('.' * (num_dots - dots_printed)) 116 sys.stdout.flush() 117 dots_printed = num_dots 118 if bytes_done != total_size: 119 raise URLError("only got %d of %d bytes" % 120 (bytes_done, total_size)) 121 print(' Done.') 122 return 123 except URLError as e: 124 sys.stdout.write('\n') 125 print(e) 126 if num_retries == 0 or isinstance(e, HTTPError) and e.code == 404: 127 raise e 128 num_retries -= 1 129 print('Retrying in %d s ...' % retry_wait_s) 130 sys.stdout.flush() 131 time.sleep(retry_wait_s) 132 retry_wait_s *= 2 133 134 135def EnsureDirExists(path): 136 if not os.path.exists(path): 137 os.makedirs(path) 138 139 140def DownloadAndUnpack(url, output_dir, path_prefixes=None): 141 """Download an archive from url and extract into output_dir. If path_prefixes 142 is not None, only extract files whose paths within the archive start with 143 any prefix in path_prefixes.""" 144 with tempfile.TemporaryFile() as f: 145 DownloadUrl(url, f) 146 f.seek(0) 147 EnsureDirExists(output_dir) 148 if url.endswith('.zip'): 149 assert path_prefixes is None 150 zipfile.ZipFile(f).extractall(path=output_dir) 151 else: 152 t = tarfile.open(mode='r:gz', fileobj=f) 153 members = None 154 if path_prefixes is not None: 155 members = [m for m in t.getmembers() 156 if any(m.name.startswith(p) for p in path_prefixes)] 157 t.extractall(path=output_dir, members=members) 158 159 160def GetPlatformUrlPrefix(host_os): 161 _HOST_OS_URL_MAP = { 162 'linux': 'Linux_x64', 163 'mac': 'Mac', 164 'mac-arm64': 'Mac_arm64', 165 'win': 'Win', 166 } 167 return CDS_URL + '/' + _HOST_OS_URL_MAP[host_os] + '/' 168 169 170def DownloadAndUnpackPackage(package_file, output_dir, host_os): 171 cds_file = "%s-%s.tgz" % (package_file, PACKAGE_VERSION) 172 cds_full_url = GetPlatformUrlPrefix(host_os) + cds_file 173 try: 174 DownloadAndUnpack(cds_full_url, output_dir) 175 except URLError: 176 print('Failed to download prebuilt clang package %s' % cds_file) 177 print('Use build.py if you want to build locally.') 178 print('Exiting.') 179 sys.exit(1) 180 181 182def DownloadAndUnpackClangMacRuntime(output_dir): 183 cds_file = "clang-%s.tgz" % PACKAGE_VERSION 184 # We run this only for the runtime libraries, and 'mac' and 'mac-arm64' both 185 # have the same (universal) runtime libraries. It doesn't matter which one 186 # we download here. 187 cds_full_url = GetPlatformUrlPrefix('mac') + cds_file 188 path_prefixes = [ 189 'lib/clang/' + RELEASE_VERSION + '/lib/darwin', 'include/c++/v1' 190 ] 191 try: 192 DownloadAndUnpack(cds_full_url, output_dir, path_prefixes) 193 except URLError: 194 print('Failed to download prebuilt clang %s' % cds_file) 195 print('Use build.py if you want to build locally.') 196 print('Exiting.') 197 sys.exit(1) 198 199 200# TODO(hans): Create a clang-win-runtime package instead. 201def DownloadAndUnpackClangWinRuntime(output_dir): 202 cds_file = "clang-%s.tgz" % PACKAGE_VERSION 203 cds_full_url = GetPlatformUrlPrefix('win') + cds_file 204 path_prefixes = [ 205 'lib/clang/' + RELEASE_VERSION + '/lib/windows', 'bin/llvm-symbolizer.exe' 206 ] 207 try: 208 DownloadAndUnpack(cds_full_url, output_dir, path_prefixes) 209 except URLError: 210 print('Failed to download prebuilt clang %s' % cds_file) 211 print('Use build.py if you want to build locally.') 212 print('Exiting.') 213 sys.exit(1) 214 215 216def UpdatePackage(package_name, host_os): 217 stamp_file = None 218 package_file = None 219 220 stamp_file = os.path.join(LLVM_BUILD_DIR, package_name + '_revision') 221 if package_name == 'clang': 222 stamp_file = STAMP_FILE 223 package_file = 'clang' 224 elif package_name == 'clang-tidy': 225 package_file = 'clang-tidy' 226 elif package_name == 'objdump': 227 package_file = 'llvmobjdump' 228 elif package_name == 'translation_unit': 229 package_file = 'translation_unit' 230 elif package_name == 'coverage_tools': 231 stamp_file = os.path.join(LLVM_BUILD_DIR, 'cr_coverage_revision') 232 package_file = 'llvm-code-coverage' 233 elif package_name == 'libclang': 234 package_file = 'libclang' 235 else: 236 print('Unknown package: "%s".' % package_name) 237 return 1 238 239 assert stamp_file is not None 240 assert package_file is not None 241 242 # TODO(hans): Create a clang-win-runtime package and use separate DEPS hook. 243 target_os = [] 244 if package_name == 'clang': 245 try: 246 GCLIENT_CONFIG = os.path.join(os.path.dirname(CHROMIUM_DIR), '.gclient') 247 env = {} 248 exec (open(GCLIENT_CONFIG).read(), env, env) 249 target_os = env.get('target_os', target_os) 250 except: 251 pass 252 253 if os.path.exists(OLD_STAMP_FILE): 254 # Delete the old stamp file so it doesn't look like an old version of clang 255 # is available in case the user rolls back to an old version of this script 256 # during a bisect for example (crbug.com/988933). 257 os.remove(OLD_STAMP_FILE) 258 259 expected_stamp = ','.join([PACKAGE_VERSION] + target_os) 260 if ReadStampFile(stamp_file) == expected_stamp: 261 return 0 262 263 # Updating the main clang package nukes the output dir. Any other packages 264 # need to be updated *after* the clang package. 265 if package_name == 'clang' and os.path.exists(LLVM_BUILD_DIR): 266 RmTree(LLVM_BUILD_DIR) 267 268 DownloadAndUnpackPackage(package_file, LLVM_BUILD_DIR, host_os) 269 270 if package_name == 'clang' and 'mac' in target_os: 271 DownloadAndUnpackClangMacRuntime(LLVM_BUILD_DIR) 272 if package_name == 'clang' and 'win' in target_os: 273 # When doing win/cross builds on other hosts, get the Windows runtime 274 # libraries, and llvm-symbolizer.exe (needed in asan builds). 275 DownloadAndUnpackClangWinRuntime(LLVM_BUILD_DIR) 276 277 WriteStampFile(expected_stamp, stamp_file) 278 return 0 279 280 281def main(): 282 _PLATFORM_HOST_OS_MAP = { 283 'darwin': 'mac', 284 'cygwin': 'win', 285 'linux2': 'linux', 286 'win32': 'win', 287 } 288 default_host_os = _PLATFORM_HOST_OS_MAP.get(sys.platform, sys.platform) 289 if default_host_os == 'mac' and platform.machine() == 'arm64': 290 default_host_os = 'mac-arm64' 291 292 parser = argparse.ArgumentParser(description='Update clang.') 293 parser.add_argument('--output-dir', 294 help='Where to extract the package.') 295 parser.add_argument('--package', 296 help='What package to update (default: clang)', 297 default='clang') 298 parser.add_argument('--host-os', 299 help='Which host OS to download for (default: %s)' % 300 default_host_os, 301 default=default_host_os, 302 choices=('linux', 'mac', 'mac-arm64', 'win')) 303 parser.add_argument('--print-revision', action='store_true', 304 help='Print current clang revision and exit.') 305 parser.add_argument('--llvm-force-head-revision', action='store_true', 306 help='Print locally built revision with --print-revision') 307 parser.add_argument('--print-clang-version', action='store_true', 308 help=('Print current clang release version (e.g. 9.0.0) ' 309 'and exit.')) 310 parser.add_argument('--verify-version', 311 help='Verify that clang has the passed-in version.') 312 args = parser.parse_args() 313 314 if args.verify_version and args.verify_version != RELEASE_VERSION: 315 print('RELEASE_VERSION is %s but --verify-version argument was %s.' % ( 316 RELEASE_VERSION, args.verify_version)) 317 print('clang_version in build/toolchain/toolchain.gni is likely outdated.') 318 return 1 319 320 if args.print_clang_version: 321 print(RELEASE_VERSION) 322 return 0 323 324 if args.print_revision: 325 if args.llvm_force_head_revision: 326 force_head_revision = ReadStampFile(FORCE_HEAD_REVISION_FILE) 327 if force_head_revision == '': 328 print('No locally built version found!') 329 return 1 330 print(force_head_revision) 331 return 0 332 333 print(PACKAGE_VERSION) 334 return 0 335 336 if args.llvm_force_head_revision: 337 print('--llvm-force-head-revision can only be used for --print-revision') 338 return 1 339 340 if args.output_dir: 341 global LLVM_BUILD_DIR, STAMP_FILE 342 LLVM_BUILD_DIR = os.path.abspath(args.output_dir) 343 STAMP_FILE = os.path.join(LLVM_BUILD_DIR, 'cr_build_revision') 344 345 return UpdatePackage(args.package, args.host_os) 346 347 348if __name__ == '__main__': 349 sys.exit(main()) 350