1#!/usr/bin/env python
2# Copyright (c) 2012 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""This script is used to download prebuilt clang binaries."""
7
8import os
9import shutil
10import subprocess
11import stat
12import sys
13import tarfile
14import tempfile
15import time
16import urllib2
17
18
19# CLANG_REVISION and CLANG_SUB_REVISION determine the build of clang
20# to use. These should be synced with tools/clang/scripts/update.py in
21# Chromium.
22CLANG_REVISION = '4e0d9925d6a3561449bdd8def27fd3f3f1b3fb9f'
23CLANG_SVN_REVISION = 'n346557'
24CLANG_SUB_REVISION = 1
25
26PACKAGE_VERSION = '%s-%s-%s' % (CLANG_SVN_REVISION, CLANG_REVISION[:8],
27                                CLANG_SUB_REVISION)
28
29# Path constants. (All of these should be absolute paths.)
30THIS_DIR = os.path.abspath(os.path.dirname(__file__))
31LLVM_BUILD_DIR = os.path.join(THIS_DIR, 'llvm-build')
32STAMP_FILE = os.path.join(LLVM_BUILD_DIR, 'cr_build_revision')
33
34# URL for pre-built binaries.
35CDS_URL = os.environ.get('CDS_CLANG_BUCKET_OVERRIDE',
36    'https://commondatastorage.googleapis.com/chromium-browser-clang')
37
38# Bump after VC updates.
39DIA_DLL = {
40  '2013': 'msdia120.dll',
41  '2015': 'msdia140.dll',
42  '2017': 'msdia140.dll',
43}
44
45
46def DownloadUrl(url, output_file):
47  """Download url into output_file."""
48  CHUNK_SIZE = 4096
49  TOTAL_DOTS = 10
50  num_retries = 3
51  retry_wait_s = 5  # Doubled at each retry.
52
53  while True:
54    try:
55      sys.stdout.write('Downloading %s ' % url)
56      sys.stdout.flush()
57      response = urllib2.urlopen(url)
58      total_size = int(response.info().getheader('Content-Length').strip())
59      bytes_done = 0
60      dots_printed = 0
61      while True:
62        chunk = response.read(CHUNK_SIZE)
63        if not chunk:
64          break
65        output_file.write(chunk)
66        bytes_done += len(chunk)
67        num_dots = TOTAL_DOTS * bytes_done / total_size
68        sys.stdout.write('.' * (num_dots - dots_printed))
69        sys.stdout.flush()
70        dots_printed = num_dots
71      if bytes_done != total_size:
72        raise urllib2.URLError("only got %d of %d bytes" %
73                               (bytes_done, total_size))
74      print ' Done.'
75      return
76    except urllib2.URLError as e:
77      sys.stdout.write('\n')
78      print e
79      if num_retries == 0 or isinstance(e, urllib2.HTTPError) and e.code == 404:
80        raise e
81      num_retries -= 1
82      print 'Retrying in %d s ...' % retry_wait_s
83      time.sleep(retry_wait_s)
84      retry_wait_s *= 2
85
86
87def EnsureDirExists(path):
88  if not os.path.exists(path):
89    print "Creating directory %s" % path
90    os.makedirs(path)
91
92
93def DownloadAndUnpack(url, output_dir):
94  with tempfile.TemporaryFile() as f:
95    DownloadUrl(url, f)
96    f.seek(0)
97    EnsureDirExists(output_dir)
98    tarfile.open(mode='r:gz', fileobj=f).extractall(path=output_dir)
99
100
101def ReadStampFile(path=STAMP_FILE):
102  """Return the contents of the stamp file, or '' if it doesn't exist."""
103  try:
104    with open(path, 'r') as f:
105      return f.read().rstrip()
106  except IOError:
107    return ''
108
109
110def WriteStampFile(s, path=STAMP_FILE):
111  """Write s to the stamp file."""
112  EnsureDirExists(os.path.dirname(path))
113  with open(path, 'w') as f:
114    f.write(s)
115    f.write('\n')
116
117
118def RmTree(dir):
119  """Delete dir."""
120  def ChmodAndRetry(func, path, _):
121    # Subversion can leave read-only files around.
122    if not os.access(path, os.W_OK):
123      os.chmod(path, stat.S_IWUSR)
124      return func(path)
125    raise
126
127  shutil.rmtree(dir, onerror=ChmodAndRetry)
128
129
130def CopyFile(src, dst):
131  """Copy a file from src to dst."""
132  print "Copying %s to %s" % (src, dst)
133  shutil.copy(src, dst)
134
135
136vs_version = None
137def GetVSVersion():
138  global vs_version
139  if vs_version:
140    return vs_version
141
142  # Try using the toolchain in depot_tools.
143  # This sets environment variables used by SelectVisualStudioVersion below.
144  sys.path.append(THIS_DIR)
145  import vs_toolchain
146  vs_toolchain.SetEnvironmentAndGetRuntimeDllDirs()
147
148  # Use gyp to find the MSVS installation, either in depot_tools as per above,
149  # or a system-wide installation otherwise.
150  sys.path.append(os.path.join(THIS_DIR, 'gyp', 'pylib'))
151  import gyp.MSVSVersion
152  vs_version = gyp.MSVSVersion.SelectVisualStudioVersion(
153      vs_toolchain.GetVisualStudioVersion())
154  return vs_version
155
156
157def CopyDiaDllTo(target_dir):
158  # This script always wants to use the 64-bit msdia*.dll.
159  dia_path = os.path.join(GetVSVersion().Path(), 'DIA SDK', 'bin', 'amd64')
160  dia_dll = os.path.join(dia_path, DIA_DLL[GetVSVersion().ShortName()])
161  CopyFile(dia_dll, target_dir)
162
163
164def UpdateClang():
165  cds_file = "clang-%s.tgz" %  PACKAGE_VERSION
166  if sys.platform == 'win32' or sys.platform == 'cygwin':
167    cds_full_url = CDS_URL + '/Win/' + cds_file
168  elif sys.platform.startswith('linux'):
169    cds_full_url = CDS_URL + '/Linux_x64/' + cds_file
170  else:
171    return 0
172
173  print 'Updating Clang to %s...' % PACKAGE_VERSION
174
175  if ReadStampFile() == PACKAGE_VERSION:
176    print 'Clang is already up to date.'
177    return 0
178
179  # Reset the stamp file in case the build is unsuccessful.
180  WriteStampFile('')
181
182  print 'Downloading prebuilt clang'
183  if os.path.exists(LLVM_BUILD_DIR):
184    RmTree(LLVM_BUILD_DIR)
185  try:
186    DownloadAndUnpack(cds_full_url, LLVM_BUILD_DIR)
187    print 'clang %s unpacked' % PACKAGE_VERSION
188    if sys.platform == 'win32':
189      CopyDiaDllTo(os.path.join(LLVM_BUILD_DIR, 'bin'))
190    WriteStampFile(PACKAGE_VERSION)
191    return 0
192  except urllib2.URLError:
193    print 'Failed to download prebuilt clang %s' % cds_file
194    print 'Exiting.'
195    return 1
196
197
198def main():
199  # Don't buffer stdout, so that print statements are immediately flushed.
200  sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
201  return UpdateClang()
202
203
204if __name__ == '__main__':
205  sys.exit(main())
206