1# This Source Code Form is subject to the terms of the Mozilla Public
2# License, v. 2.0. If a copy of the MPL was not distributed with this
3# file, You can obtain one at http://mozilla.org/MPL/2.0/.
4
5from __future__ import absolute_import, print_function
6
7from distutils.spawn import find_executable
8from distutils.version import LooseVersion
9import json
10import os
11import platform
12import shutil
13import subprocess
14from subprocess import PIPE
15
16import servo.packages as packages
17from servo.util import extract, download_file, host_triple
18
19
20def run_as_root(command):
21    if os.geteuid() != 0:
22        command.insert(0, 'sudo')
23    return subprocess.call(command)
24
25
26def install_salt_dependencies(context, force):
27    install = False
28    if context.distro == 'Ubuntu':
29        pkgs = ['build-essential', 'libssl-dev', 'libffi-dev', 'python-dev']
30        command = ['apt-get', 'install']
31        if subprocess.call(['dpkg', '-s'] + pkgs, stdout=PIPE, stderr=PIPE) != 0:
32            install = True
33    elif context.distro in ['CentOS', 'CentOS Linux', 'Fedora']:
34        installed_pkgs = str(subprocess.check_output(['rpm', '-qa'])).replace('\n', '|')
35        pkgs = ['gcc', 'libffi-devel', 'python-devel', 'openssl-devel']
36        for p in pkgs:
37            command = ['dnf', 'install']
38            if "|{}".format(p) not in installed_pkgs:
39                install = True
40                break
41
42    if install:
43        if force:
44            command.append('-y')
45        print("Installing missing Salt dependencies...")
46        run_as_root(command + pkgs)
47
48
49def salt(context, force=False):
50    # Ensure Salt dependencies are installed
51    install_salt_dependencies(context, force)
52    # Ensure Salt is installed in the virtualenv
53    # It's not instaled globally because it's a large, non-required dependency,
54    # and the installation fails on Windows
55    print("Checking Salt installation...", end='')
56    reqs_path = os.path.join(context.topdir, 'python', 'requirements-salt.txt')
57    process = subprocess.Popen(
58        ["pip", "install", "-q", "-I", "-r", reqs_path],
59        stdout=PIPE,
60        stderr=PIPE
61    )
62    process.wait()
63    if process.returncode:
64        out, err = process.communicate()
65        print('failed to install Salt via pip:')
66        print('Output: {}\nError: {}'.format(out, err))
67        return 1
68    print("done")
69
70    salt_root = os.path.join(context.sharedir, 'salt')
71    config_dir = os.path.join(salt_root, 'etc', 'salt')
72    pillar_dir = os.path.join(config_dir, 'pillars')
73
74    # In order to allow `mach bootstrap` to work from any CWD,
75    # the `root_dir` must be an absolute path.
76    # We place it under `context.sharedir` because
77    # Salt caches data (e.g. gitfs files) in its `var` subdirectory.
78    # Hence, dynamically generate the config with an appropriate `root_dir`
79    # and serialize it as JSON (which is valid YAML).
80    config = {
81        'hash_type': 'sha384',
82        'master': 'localhost',
83        'root_dir': salt_root,
84        'state_output': 'changes',
85        'state_tabular': True,
86    }
87    if 'SERVO_SALTFS_ROOT' in os.environ:
88        config.update({
89            'fileserver_backend': ['roots'],
90            'file_roots': {
91                'base': [os.path.abspath(os.environ['SERVO_SALTFS_ROOT'])],
92            },
93        })
94    else:
95        config.update({
96            'fileserver_backend': ['git'],
97            'gitfs_env_whitelist': 'base',
98            'gitfs_provider': 'gitpython',
99            'gitfs_remotes': [
100                'https://github.com/servo/saltfs.git',
101            ],
102        })
103
104    if not os.path.exists(config_dir):
105        os.makedirs(config_dir, mode=0o700)
106    with open(os.path.join(config_dir, 'minion'), 'w') as config_file:
107        config_file.write(json.dumps(config) + '\n')
108
109    # Similarly, the pillar data is created dynamically
110    # and temporarily serialized to disk.
111    # This dynamism is not yet used, but will be in the future
112    # to enable Android bootstrapping by using
113    # context.sharedir as a location for Android packages.
114    pillar = {
115        'top.sls': {
116            'base': {
117                '*': ['bootstrap'],
118            },
119        },
120        'bootstrap.sls': {
121            'fully_managed': False,
122        },
123    }
124    if os.path.exists(pillar_dir):
125        shutil.rmtree(pillar_dir)
126    os.makedirs(pillar_dir, mode=0o700)
127    for filename in pillar:
128        with open(os.path.join(pillar_dir, filename), 'w') as pillar_file:
129            pillar_file.write(json.dumps(pillar[filename]) + '\n')
130
131    cmd = [
132        # sudo escapes from the venv, need to use full path
133        find_executable('salt-call'),
134        '--local',
135        '--config-dir={}'.format(config_dir),
136        '--pillar-root={}'.format(pillar_dir),
137        'state.apply',
138        'servo-build-dependencies',
139    ]
140
141    if not force:
142        print('Running bootstrap in dry-run mode to show changes')
143        # Because `test=True` mode runs each state individually without
144        # considering how required/previous states affect the system,
145        # it will often report states with requisites as failing due
146        # to the requisites not actually being run,
147        # even though these are spurious and will succeed during
148        # the actual highstate.
149        # Hence `--retcode-passthrough` is not helpful in dry-run mode,
150        # so only detect failures of the actual salt-call binary itself.
151        retcode = run_as_root(cmd + ['test=True'])
152        if retcode != 0:
153            print('Something went wrong while bootstrapping')
154            return retcode
155
156        proceed = raw_input(
157            'Proposed changes are above, proceed with bootstrap? [y/N]: '
158        )
159        if proceed.lower() not in ['y', 'yes']:
160            return 0
161
162        print('')
163
164    print('Running Salt bootstrap')
165    retcode = run_as_root(cmd + ['--retcode-passthrough'])
166    if retcode == 0:
167        print('Salt bootstrapping complete')
168    else:
169        print('Salt bootstrapping encountered errors')
170    return retcode
171
172
173def windows_msvc(context, force=False):
174    '''Bootstrapper for MSVC building on Windows.'''
175
176    deps_dir = os.path.join(context.sharedir, "msvc-dependencies")
177    deps_url = "https://servo-deps.s3.amazonaws.com/msvc-deps/"
178
179    def version(package):
180        return packages.WINDOWS_MSVC[package]
181
182    def package_dir(package):
183        return os.path.join(deps_dir, package, version(package))
184
185    def check_cmake(version):
186        cmake_path = find_executable("cmake")
187        if cmake_path:
188            cmake = subprocess.Popen([cmake_path, "--version"], stdout=PIPE)
189            cmake_version = cmake.stdout.read().splitlines()[0].replace("cmake version ", "")
190            if LooseVersion(cmake_version) >= LooseVersion(version):
191                return True
192        return False
193
194    to_install = {}
195    for package in packages.WINDOWS_MSVC:
196        # Don't install CMake if it already exists in PATH
197        if package == "cmake" and check_cmake(version("cmake")):
198            continue
199
200        if not os.path.isdir(package_dir(package)):
201            to_install[package] = version(package)
202
203    if not to_install:
204        return 0
205
206    print("Installing missing MSVC dependencies...")
207    for package in to_install:
208        full_spec = '{}-{}'.format(package, version(package))
209
210        parent_dir = os.path.dirname(package_dir(package))
211        if not os.path.isdir(parent_dir):
212            os.makedirs(parent_dir)
213
214        zip_path = package_dir(package) + ".zip"
215        if not os.path.isfile(zip_path):
216            zip_url = "{}{}.zip".format(deps_url, full_spec)
217            download_file(full_spec, zip_url, zip_path)
218
219        print("Extracting {}...".format(full_spec), end='')
220        extract(zip_path, deps_dir)
221        print("done")
222
223        extracted_path = os.path.join(deps_dir, full_spec)
224        os.rename(extracted_path, package_dir(package))
225
226    return 0
227
228
229def bootstrap(context, force=False):
230    '''Dispatches to the right bootstrapping function for the OS.'''
231
232    bootstrapper = None
233
234    if "windows-msvc" in host_triple():
235        bootstrapper = windows_msvc
236    elif "linux-gnu" in host_triple():
237        distro, version, _ = platform.linux_distribution()
238        if distro.lower() in [
239            'centos',
240            'centos linux',
241            'debian',
242            'fedora',
243            'ubuntu',
244        ]:
245            context.distro = distro
246            bootstrapper = salt
247
248    if bootstrapper is None:
249        print('Bootstrap support is not yet available for your OS.')
250        return 1
251
252    return bootstrapper(context, force=force)
253