1#!/usr/bin/env python
2# This Source Code Form is subject to the terms of the Mozilla Public
3# License, v. 2.0. If a copy of the MPL was not distributed with this file,
4# You can obtain one at http://mozilla.org/MPL/2.0/.
5
6# This script provides one-line bootstrap support to configure systems to build
7# the tree. It does so by cloning the repo before calling directly into `mach
8# bootstrap`.
9
10# Note that this script can't assume anything in particular about the host
11# Python environment (except that it's run with a sufficiently recent version of
12# Python 3), so we are restricted to stdlib modules.
13
14from __future__ import absolute_import, print_function, unicode_literals
15
16import sys
17
18major, minor = sys.version_info[:2]
19if (major < 3) or (major == 3 and minor < 5):
20    print(
21        "Bootstrap currently only runs on Python 3.5+."
22        "Please try re-running with python3.5+."
23    )
24    sys.exit(1)
25
26import os
27import shutil
28import stat
29import subprocess
30import tempfile
31import zipfile
32
33from optparse import OptionParser
34from urllib.request import urlopen
35
36CLONE_MERCURIAL_PULL_FAIL = """
37Failed to pull from hg.mozilla.org.
38
39This is most likely because of unstable network connection.
40Try running `cd %s && hg pull https://hg.mozilla.org/mozilla-unified` manually,
41or download a mercurial bundle and use it:
42https://developer.mozilla.org/en-US/docs/Mozilla/Developer_guide/Source_Code/Mercurial/Bundles"""
43
44WINDOWS = sys.platform.startswith("win32") or sys.platform.startswith("msys")
45VCS_HUMAN_READABLE = {
46    "hg": "Mercurial",
47    "git": "Git",
48}
49
50
51def which(name):
52    """Python implementation of which.
53
54    It returns the path of an executable or None if it couldn't be found.
55    """
56    # git-cinnabar.exe doesn't exist, but .exe versions of the other executables
57    # do.
58    if WINDOWS and name != "git-cinnabar":
59        name += ".exe"
60    search_dirs = os.environ["PATH"].split(os.pathsep)
61
62    for path in search_dirs:
63        test = os.path.join(path, name)
64        if os.path.isfile(test) and os.access(test, os.X_OK):
65            return test
66
67    return None
68
69
70def validate_clone_dest(dest):
71    dest = os.path.abspath(dest)
72
73    if not os.path.exists(dest):
74        return dest
75
76    if not os.path.isdir(dest):
77        print("ERROR! Destination %s exists but is not a directory." % dest)
78        return None
79
80    if not os.listdir(dest):
81        return dest
82    else:
83        print("ERROR! Destination directory %s exists but is nonempty." % dest)
84        return None
85
86
87def input_clone_dest(vcs, no_interactive):
88    repo_name = "mozilla-unified"
89    print("Cloning into %s using %s..." % (repo_name, VCS_HUMAN_READABLE[vcs]))
90    while True:
91        dest = None
92        if not no_interactive:
93            dest = input(
94                "Destination directory for clone (leave empty to use "
95                "default destination of %s): " % repo_name
96            ).strip()
97        if not dest:
98            dest = repo_name
99        dest = validate_clone_dest(os.path.expanduser(dest))
100        if dest:
101            return dest
102        if no_interactive:
103            return None
104
105
106def hg_clone_firefox(hg, dest):
107    # We create an empty repo then modify the config before adding data.
108    # This is necessary to ensure storage settings are optimally
109    # configured.
110    args = [
111        hg,
112        # The unified repo is generaldelta, so ensure the client is as
113        # well.
114        "--config",
115        "format.generaldelta=true",
116        "init",
117        dest,
118    ]
119    res = subprocess.call(args)
120    if res:
121        print("unable to create destination repo; please try cloning manually")
122        return None
123
124    # Strictly speaking, this could overwrite a config based on a template
125    # the user has installed. Let's pretend this problem doesn't exist
126    # unless someone complains about it.
127    with open(os.path.join(dest, ".hg", "hgrc"), "a") as fh:
128        fh.write("[paths]\n")
129        fh.write("default = https://hg.mozilla.org/mozilla-unified\n")
130        fh.write("\n")
131
132        # The server uses aggressivemergedeltas which can blow up delta chain
133        # length. This can cause performance to tank due to delta chains being
134        # too long. Limit the delta chain length to something reasonable
135        # to bound revlog read time.
136        fh.write("[format]\n")
137        fh.write("# This is necessary to keep performance in check\n")
138        fh.write("maxchainlen = 10000\n")
139
140    res = subprocess.call(
141        [hg, "pull", "https://hg.mozilla.org/mozilla-unified"], cwd=dest
142    )
143    print("")
144    if res:
145        print(CLONE_MERCURIAL_PULL_FAIL % dest)
146        return None
147
148    print('updating to "central" - the development head of Gecko and Firefox')
149    res = subprocess.call([hg, "update", "-r", "central"], cwd=dest)
150    if res:
151        print(
152            "error updating; you will need to `cd %s && hg update -r central` "
153            "manually" % dest
154        )
155    return dest
156
157
158def git_clone_firefox(git, dest, watchman):
159    tempdir = None
160    cinnabar = None
161    env = dict(os.environ)
162    try:
163        cinnabar = which("git-cinnabar")
164        if not cinnabar:
165            cinnabar_url = (
166                "https://github.com/glandium/git-cinnabar/archive/" "master.zip"
167            )
168            # If git-cinnabar isn't installed already, that's fine; we can
169            # download a temporary copy. `mach bootstrap` will clone a full copy
170            # of the repo in the state dir; we don't want to copy all that logic
171            # to this tiny bootstrapping script.
172            tempdir = tempfile.mkdtemp()
173            with open(os.path.join(tempdir, "git-cinnabar.zip"), mode="w+b") as archive:
174                with urlopen(cinnabar_url) as repo:
175                    shutil.copyfileobj(repo, archive)
176                archive.seek(0)
177                with zipfile.ZipFile(archive) as zipf:
178                    zipf.extractall(path=tempdir)
179            cinnabar_dir = os.path.join(tempdir, "git-cinnabar-master")
180            cinnabar = os.path.join(cinnabar_dir, "git-cinnabar")
181            # Make git-cinnabar and git-remote-hg executable.
182            st = os.stat(cinnabar)
183            os.chmod(cinnabar, st.st_mode | stat.S_IEXEC)
184            st = os.stat(os.path.join(cinnabar_dir, "git-remote-hg"))
185            os.chmod(
186                os.path.join(cinnabar_dir, "git-remote-hg"), st.st_mode | stat.S_IEXEC
187            )
188            env["PATH"] = cinnabar_dir + os.pathsep + env["PATH"]
189            subprocess.check_call(
190                ["git", "cinnabar", "download"], cwd=cinnabar_dir, env=env
191            )
192            print(
193                "WARNING! git-cinnabar is required for Firefox development  "
194                "with git. After the clone is complete, the bootstrapper "
195                "will ask if you would like to configure git; answer yes, "
196                "and be sure to add git-cinnabar to your PATH according to "
197                "the bootstrapper output."
198            )
199
200        # We're guaranteed to have `git-cinnabar` installed now.
201        # Configure git per the git-cinnabar requirements.
202        subprocess.check_call(
203            [
204                git,
205                "clone",
206                "-b",
207                "bookmarks/central",
208                "hg::https://hg.mozilla.org/mozilla-unified",
209                dest,
210            ],
211            env=env,
212        )
213        subprocess.check_call([git, "config", "fetch.prune", "true"], cwd=dest, env=env)
214        subprocess.check_call([git, "config", "pull.ff", "only"], cwd=dest, env=env)
215
216        watchman_sample = os.path.join(dest, ".git/hooks/fsmonitor-watchman.sample")
217        # Older versions of git didn't include fsmonitor-watchman.sample.
218        if watchman and os.path.exists(watchman_sample):
219            print("Configuring watchman")
220            watchman_config = os.path.join(dest, ".git/hooks/query-watchman")
221            if not os.path.exists(watchman_config):
222                print("Copying %s to %s" % (watchman_sample, watchman_config))
223                copy_args = [
224                    "cp",
225                    ".git/hooks/fsmonitor-watchman.sample",
226                    ".git/hooks/query-watchman",
227                ]
228                subprocess.check_call(copy_args, cwd=dest)
229
230            config_args = [git, "config", "core.fsmonitor", ".git/hooks/query-watchman"]
231            subprocess.check_call(config_args, cwd=dest, env=env)
232        return dest
233    finally:
234        if not cinnabar:
235            print(
236                "Failed to install git-cinnabar. Try performing a manual "
237                "installation: https://github.com/glandium/git-cinnabar/wiki/"
238                "Mozilla:-A-git-workflow-for-Gecko-development"
239            )
240        if tempdir:
241            shutil.rmtree(tempdir)
242
243
244def clone(vcs, no_interactive):
245    hg = which("hg")
246    if not hg:
247        print(
248            "Mercurial is not installed. Mercurial is required to clone "
249            "Firefox%s." % (", even when cloning with Git" if vcs == "git" else "")
250        )
251        try:
252            # We're going to recommend people install the Mercurial package with
253            # pip3. That will work if `pip3` installs binaries to a location
254            # that's in the PATH, but it might not be. To help out, if we CAN
255            # import "mercurial" (in which case it's already been installed),
256            # offer that as a solution.
257            import mercurial  # noqa: F401
258
259            print(
260                "Hint: have you made sure that Mercurial is installed to a "
261                "location in your PATH?"
262            )
263        except ImportError:
264            print("Try installing hg with `pip3 install Mercurial`.")
265        return None
266    if vcs == "hg":
267        binary = hg
268    else:
269        binary = which(vcs)
270        if not binary:
271            print("Git is not installed.")
272            print("Try installing git using your system package manager.")
273            return None
274
275    dest = input_clone_dest(vcs, no_interactive)
276    if not dest:
277        return None
278
279    print("Cloning Firefox %s repository to %s" % (VCS_HUMAN_READABLE[vcs], dest))
280    if vcs == "hg":
281        return hg_clone_firefox(binary, dest)
282    else:
283        watchman = which("watchman")
284        return git_clone_firefox(binary, dest, watchman)
285
286
287def bootstrap(srcdir, application_choice, no_interactive, no_system_changes):
288    args = [sys.executable, os.path.join(srcdir, "mach")]
289
290    if no_interactive:
291        # --no-interactive is a global argument, not a command argument,
292        # so it needs to be specified before "bootstrap" is appended.
293        args += ["--no-interactive"]
294
295    args += ["bootstrap"]
296
297    if application_choice:
298        args += ["--application-choice", application_choice]
299    if no_system_changes:
300        args += ["--no-system-changes"]
301
302    print("Running `%s`" % " ".join(args))
303    return subprocess.call(args, cwd=srcdir)
304
305
306def main(args):
307    parser = OptionParser()
308    parser.add_option(
309        "--application-choice",
310        dest="application_choice",
311        help='Pass in an application choice (see "APPLICATIONS" in '
312        "python/mozboot/mozboot/bootstrap.py) instead of using the "
313        "default interactive prompt.",
314    )
315    parser.add_option(
316        "--vcs",
317        dest="vcs",
318        default="hg",
319        choices=["git", "hg"],
320        help="VCS (hg or git) to use for downloading the source code, "
321        "instead of using the default interactive prompt.",
322    )
323    parser.add_option(
324        "--no-interactive",
325        dest="no_interactive",
326        action="store_true",
327        help="Answer yes to any (Y/n) interactive prompts.",
328    )
329    parser.add_option(
330        "--no-system-changes",
331        dest="no_system_changes",
332        action="store_true",
333        help="Only executes actions that leave the system " "configuration alone.",
334    )
335
336    options, leftover = parser.parse_args(args)
337
338    try:
339        srcdir = clone(options.vcs, options.no_interactive)
340        if not srcdir:
341            return 1
342        print("Clone complete.")
343        return bootstrap(
344            srcdir,
345            options.application_choice,
346            options.no_interactive,
347            options.no_system_changes,
348        )
349    except Exception:
350        print("Could not bootstrap Firefox! Consider filing a bug.")
351        raise
352
353
354if __name__ == "__main__":
355    sys.exit(main(sys.argv))
356