1# This Source Code Form is subject to the terms of the Mozilla Public 2# License, v. 2.0. If a copy of the MPL was not distributed with this 3# file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5from __future__ import absolute_import, print_function 6 7from distutils.spawn import find_executable 8from distutils.version import LooseVersion 9import json 10import os 11import platform 12import shutil 13import subprocess 14from subprocess import PIPE 15 16import servo.packages as packages 17from servo.util import extract, download_file, host_triple 18 19 20def run_as_root(command): 21 if os.geteuid() != 0: 22 command.insert(0, 'sudo') 23 return subprocess.call(command) 24 25 26def install_salt_dependencies(context, force): 27 install = False 28 if context.distro == 'Ubuntu': 29 pkgs = ['build-essential', 'libssl-dev', 'libffi-dev', 'python-dev'] 30 command = ['apt-get', 'install'] 31 if subprocess.call(['dpkg', '-s'] + pkgs, stdout=PIPE, stderr=PIPE) != 0: 32 install = True 33 elif context.distro in ['CentOS', 'CentOS Linux', 'Fedora']: 34 installed_pkgs = str(subprocess.check_output(['rpm', '-qa'])).replace('\n', '|') 35 pkgs = ['gcc', 'libffi-devel', 'python-devel', 'openssl-devel'] 36 for p in pkgs: 37 command = ['dnf', 'install'] 38 if "|{}".format(p) not in installed_pkgs: 39 install = True 40 break 41 42 if install: 43 if force: 44 command.append('-y') 45 print("Installing missing Salt dependencies...") 46 run_as_root(command + pkgs) 47 48 49def salt(context, force=False): 50 # Ensure Salt dependencies are installed 51 install_salt_dependencies(context, force) 52 # Ensure Salt is installed in the virtualenv 53 # It's not instaled globally because it's a large, non-required dependency, 54 # and the installation fails on Windows 55 print("Checking Salt installation...", end='') 56 reqs_path = os.path.join(context.topdir, 'python', 'requirements-salt.txt') 57 process = subprocess.Popen( 58 ["pip", "install", "-q", "-I", "-r", reqs_path], 59 stdout=PIPE, 60 stderr=PIPE 61 ) 62 process.wait() 63 if process.returncode: 64 out, err = process.communicate() 65 print('failed to install Salt via pip:') 66 print('Output: {}\nError: {}'.format(out, err)) 67 return 1 68 print("done") 69 70 salt_root = os.path.join(context.sharedir, 'salt') 71 config_dir = os.path.join(salt_root, 'etc', 'salt') 72 pillar_dir = os.path.join(config_dir, 'pillars') 73 74 # In order to allow `mach bootstrap` to work from any CWD, 75 # the `root_dir` must be an absolute path. 76 # We place it under `context.sharedir` because 77 # Salt caches data (e.g. gitfs files) in its `var` subdirectory. 78 # Hence, dynamically generate the config with an appropriate `root_dir` 79 # and serialize it as JSON (which is valid YAML). 80 config = { 81 'hash_type': 'sha384', 82 'master': 'localhost', 83 'root_dir': salt_root, 84 'state_output': 'changes', 85 'state_tabular': True, 86 } 87 if 'SERVO_SALTFS_ROOT' in os.environ: 88 config.update({ 89 'fileserver_backend': ['roots'], 90 'file_roots': { 91 'base': [os.path.abspath(os.environ['SERVO_SALTFS_ROOT'])], 92 }, 93 }) 94 else: 95 config.update({ 96 'fileserver_backend': ['git'], 97 'gitfs_env_whitelist': 'base', 98 'gitfs_provider': 'gitpython', 99 'gitfs_remotes': [ 100 'https://github.com/servo/saltfs.git', 101 ], 102 }) 103 104 if not os.path.exists(config_dir): 105 os.makedirs(config_dir, mode=0o700) 106 with open(os.path.join(config_dir, 'minion'), 'w') as config_file: 107 config_file.write(json.dumps(config) + '\n') 108 109 # Similarly, the pillar data is created dynamically 110 # and temporarily serialized to disk. 111 # This dynamism is not yet used, but will be in the future 112 # to enable Android bootstrapping by using 113 # context.sharedir as a location for Android packages. 114 pillar = { 115 'top.sls': { 116 'base': { 117 '*': ['bootstrap'], 118 }, 119 }, 120 'bootstrap.sls': { 121 'fully_managed': False, 122 }, 123 } 124 if os.path.exists(pillar_dir): 125 shutil.rmtree(pillar_dir) 126 os.makedirs(pillar_dir, mode=0o700) 127 for filename in pillar: 128 with open(os.path.join(pillar_dir, filename), 'w') as pillar_file: 129 pillar_file.write(json.dumps(pillar[filename]) + '\n') 130 131 cmd = [ 132 # sudo escapes from the venv, need to use full path 133 find_executable('salt-call'), 134 '--local', 135 '--config-dir={}'.format(config_dir), 136 '--pillar-root={}'.format(pillar_dir), 137 'state.apply', 138 'servo-build-dependencies', 139 ] 140 141 if not force: 142 print('Running bootstrap in dry-run mode to show changes') 143 # Because `test=True` mode runs each state individually without 144 # considering how required/previous states affect the system, 145 # it will often report states with requisites as failing due 146 # to the requisites not actually being run, 147 # even though these are spurious and will succeed during 148 # the actual highstate. 149 # Hence `--retcode-passthrough` is not helpful in dry-run mode, 150 # so only detect failures of the actual salt-call binary itself. 151 retcode = run_as_root(cmd + ['test=True']) 152 if retcode != 0: 153 print('Something went wrong while bootstrapping') 154 return retcode 155 156 proceed = raw_input( 157 'Proposed changes are above, proceed with bootstrap? [y/N]: ' 158 ) 159 if proceed.lower() not in ['y', 'yes']: 160 return 0 161 162 print('') 163 164 print('Running Salt bootstrap') 165 retcode = run_as_root(cmd + ['--retcode-passthrough']) 166 if retcode == 0: 167 print('Salt bootstrapping complete') 168 else: 169 print('Salt bootstrapping encountered errors') 170 return retcode 171 172 173def windows_msvc(context, force=False): 174 '''Bootstrapper for MSVC building on Windows.''' 175 176 deps_dir = os.path.join(context.sharedir, "msvc-dependencies") 177 deps_url = "https://servo-deps.s3.amazonaws.com/msvc-deps/" 178 179 def version(package): 180 return packages.WINDOWS_MSVC[package] 181 182 def package_dir(package): 183 return os.path.join(deps_dir, package, version(package)) 184 185 def check_cmake(version): 186 cmake_path = find_executable("cmake") 187 if cmake_path: 188 cmake = subprocess.Popen([cmake_path, "--version"], stdout=PIPE) 189 cmake_version = cmake.stdout.read().splitlines()[0].replace("cmake version ", "") 190 if LooseVersion(cmake_version) >= LooseVersion(version): 191 return True 192 return False 193 194 to_install = {} 195 for package in packages.WINDOWS_MSVC: 196 # Don't install CMake if it already exists in PATH 197 if package == "cmake" and check_cmake(version("cmake")): 198 continue 199 200 if not os.path.isdir(package_dir(package)): 201 to_install[package] = version(package) 202 203 if not to_install: 204 return 0 205 206 print("Installing missing MSVC dependencies...") 207 for package in to_install: 208 full_spec = '{}-{}'.format(package, version(package)) 209 210 parent_dir = os.path.dirname(package_dir(package)) 211 if not os.path.isdir(parent_dir): 212 os.makedirs(parent_dir) 213 214 zip_path = package_dir(package) + ".zip" 215 if not os.path.isfile(zip_path): 216 zip_url = "{}{}.zip".format(deps_url, full_spec) 217 download_file(full_spec, zip_url, zip_path) 218 219 print("Extracting {}...".format(full_spec), end='') 220 extract(zip_path, deps_dir) 221 print("done") 222 223 extracted_path = os.path.join(deps_dir, full_spec) 224 os.rename(extracted_path, package_dir(package)) 225 226 return 0 227 228 229def bootstrap(context, force=False): 230 '''Dispatches to the right bootstrapping function for the OS.''' 231 232 bootstrapper = None 233 234 if "windows-msvc" in host_triple(): 235 bootstrapper = windows_msvc 236 elif "linux-gnu" in host_triple(): 237 distro, version, _ = platform.linux_distribution() 238 if distro.lower() in [ 239 'centos', 240 'centos linux', 241 'debian', 242 'fedora', 243 'ubuntu', 244 ]: 245 context.distro = distro 246 bootstrapper = salt 247 248 if bootstrapper is None: 249 print('Bootstrap support is not yet available for your OS.') 250 return 1 251 252 return bootstrapper(context, force=force) 253