1#!/usr/bin/env python 2# This Source Code Form is subject to the terms of the Mozilla Public 3# License, v. 2.0. If a copy of the MPL was not distributed with this file, 4# You can obtain one at http://mozilla.org/MPL/2.0/. 5 6# This script provides one-line bootstrap support to configure systems to build 7# the tree. It does so by cloning the repo before calling directly into `mach 8# bootstrap`. 9 10# Note that this script can't assume anything in particular about the host 11# Python environment (except that it's run with a sufficiently recent version of 12# Python 3), so we are restricted to stdlib modules. 13 14from __future__ import absolute_import, print_function, unicode_literals 15 16import sys 17 18major, minor = sys.version_info[:2] 19if (major < 3) or (major == 3 and minor < 5): 20 print( 21 "Bootstrap currently only runs on Python 3.5+." 22 "Please try re-running with python3.5+." 23 ) 24 sys.exit(1) 25 26import os 27import shutil 28import stat 29import subprocess 30import tempfile 31import zipfile 32 33from optparse import OptionParser 34from urllib.request import urlopen 35 36CLONE_MERCURIAL_PULL_FAIL = """ 37Failed to pull from hg.mozilla.org. 38 39This is most likely because of unstable network connection. 40Try running `cd %s && hg pull https://hg.mozilla.org/mozilla-unified` manually, 41or download a mercurial bundle and use it: 42https://developer.mozilla.org/en-US/docs/Mozilla/Developer_guide/Source_Code/Mercurial/Bundles""" 43 44WINDOWS = sys.platform.startswith("win32") or sys.platform.startswith("msys") 45VCS_HUMAN_READABLE = { 46 "hg": "Mercurial", 47 "git": "Git", 48} 49 50 51def which(name): 52 """Python implementation of which. 53 54 It returns the path of an executable or None if it couldn't be found. 55 """ 56 # git-cinnabar.exe doesn't exist, but .exe versions of the other executables 57 # do. 58 if WINDOWS and name != "git-cinnabar": 59 name += ".exe" 60 search_dirs = os.environ["PATH"].split(os.pathsep) 61 62 for path in search_dirs: 63 test = os.path.join(path, name) 64 if os.path.isfile(test) and os.access(test, os.X_OK): 65 return test 66 67 return None 68 69 70def validate_clone_dest(dest): 71 dest = os.path.abspath(dest) 72 73 if not os.path.exists(dest): 74 return dest 75 76 if not os.path.isdir(dest): 77 print("ERROR! Destination %s exists but is not a directory." % dest) 78 return None 79 80 if not os.listdir(dest): 81 return dest 82 else: 83 print("ERROR! Destination directory %s exists but is nonempty." % dest) 84 return None 85 86 87def input_clone_dest(vcs, no_interactive): 88 repo_name = "mozilla-unified" 89 print("Cloning into %s using %s..." % (repo_name, VCS_HUMAN_READABLE[vcs])) 90 while True: 91 dest = None 92 if not no_interactive: 93 dest = input( 94 "Destination directory for clone (leave empty to use " 95 "default destination of %s): " % repo_name 96 ).strip() 97 if not dest: 98 dest = repo_name 99 dest = validate_clone_dest(os.path.expanduser(dest)) 100 if dest: 101 return dest 102 if no_interactive: 103 return None 104 105 106def hg_clone_firefox(hg, dest): 107 # We create an empty repo then modify the config before adding data. 108 # This is necessary to ensure storage settings are optimally 109 # configured. 110 args = [ 111 hg, 112 # The unified repo is generaldelta, so ensure the client is as 113 # well. 114 "--config", 115 "format.generaldelta=true", 116 "init", 117 dest, 118 ] 119 res = subprocess.call(args) 120 if res: 121 print("unable to create destination repo; please try cloning manually") 122 return None 123 124 # Strictly speaking, this could overwrite a config based on a template 125 # the user has installed. Let's pretend this problem doesn't exist 126 # unless someone complains about it. 127 with open(os.path.join(dest, ".hg", "hgrc"), "a") as fh: 128 fh.write("[paths]\n") 129 fh.write("default = https://hg.mozilla.org/mozilla-unified\n") 130 fh.write("\n") 131 132 # The server uses aggressivemergedeltas which can blow up delta chain 133 # length. This can cause performance to tank due to delta chains being 134 # too long. Limit the delta chain length to something reasonable 135 # to bound revlog read time. 136 fh.write("[format]\n") 137 fh.write("# This is necessary to keep performance in check\n") 138 fh.write("maxchainlen = 10000\n") 139 140 res = subprocess.call( 141 [hg, "pull", "https://hg.mozilla.org/mozilla-unified"], cwd=dest 142 ) 143 print("") 144 if res: 145 print(CLONE_MERCURIAL_PULL_FAIL % dest) 146 return None 147 148 print('updating to "central" - the development head of Gecko and Firefox') 149 res = subprocess.call([hg, "update", "-r", "central"], cwd=dest) 150 if res: 151 print( 152 "error updating; you will need to `cd %s && hg update -r central` " 153 "manually" % dest 154 ) 155 return dest 156 157 158def git_clone_firefox(git, dest, watchman): 159 tempdir = None 160 cinnabar = None 161 env = dict(os.environ) 162 try: 163 cinnabar = which("git-cinnabar") 164 if not cinnabar: 165 cinnabar_url = ( 166 "https://github.com/glandium/git-cinnabar/archive/" "master.zip" 167 ) 168 # If git-cinnabar isn't installed already, that's fine; we can 169 # download a temporary copy. `mach bootstrap` will clone a full copy 170 # of the repo in the state dir; we don't want to copy all that logic 171 # to this tiny bootstrapping script. 172 tempdir = tempfile.mkdtemp() 173 with open(os.path.join(tempdir, "git-cinnabar.zip"), mode="w+b") as archive: 174 with urlopen(cinnabar_url) as repo: 175 shutil.copyfileobj(repo, archive) 176 archive.seek(0) 177 with zipfile.ZipFile(archive) as zipf: 178 zipf.extractall(path=tempdir) 179 cinnabar_dir = os.path.join(tempdir, "git-cinnabar-master") 180 cinnabar = os.path.join(cinnabar_dir, "git-cinnabar") 181 # Make git-cinnabar and git-remote-hg executable. 182 st = os.stat(cinnabar) 183 os.chmod(cinnabar, st.st_mode | stat.S_IEXEC) 184 st = os.stat(os.path.join(cinnabar_dir, "git-remote-hg")) 185 os.chmod( 186 os.path.join(cinnabar_dir, "git-remote-hg"), st.st_mode | stat.S_IEXEC 187 ) 188 env["PATH"] = cinnabar_dir + os.pathsep + env["PATH"] 189 subprocess.check_call( 190 ["git", "cinnabar", "download"], cwd=cinnabar_dir, env=env 191 ) 192 print( 193 "WARNING! git-cinnabar is required for Firefox development " 194 "with git. After the clone is complete, the bootstrapper " 195 "will ask if you would like to configure git; answer yes, " 196 "and be sure to add git-cinnabar to your PATH according to " 197 "the bootstrapper output." 198 ) 199 200 # We're guaranteed to have `git-cinnabar` installed now. 201 # Configure git per the git-cinnabar requirements. 202 subprocess.check_call( 203 [ 204 git, 205 "clone", 206 "-b", 207 "bookmarks/central", 208 "hg::https://hg.mozilla.org/mozilla-unified", 209 dest, 210 ], 211 env=env, 212 ) 213 subprocess.check_call([git, "config", "fetch.prune", "true"], cwd=dest, env=env) 214 subprocess.check_call([git, "config", "pull.ff", "only"], cwd=dest, env=env) 215 216 watchman_sample = os.path.join(dest, ".git/hooks/fsmonitor-watchman.sample") 217 # Older versions of git didn't include fsmonitor-watchman.sample. 218 if watchman and os.path.exists(watchman_sample): 219 print("Configuring watchman") 220 watchman_config = os.path.join(dest, ".git/hooks/query-watchman") 221 if not os.path.exists(watchman_config): 222 print("Copying %s to %s" % (watchman_sample, watchman_config)) 223 copy_args = [ 224 "cp", 225 ".git/hooks/fsmonitor-watchman.sample", 226 ".git/hooks/query-watchman", 227 ] 228 subprocess.check_call(copy_args, cwd=dest) 229 230 config_args = [git, "config", "core.fsmonitor", ".git/hooks/query-watchman"] 231 subprocess.check_call(config_args, cwd=dest, env=env) 232 return dest 233 finally: 234 if not cinnabar: 235 print( 236 "Failed to install git-cinnabar. Try performing a manual " 237 "installation: https://github.com/glandium/git-cinnabar/wiki/" 238 "Mozilla:-A-git-workflow-for-Gecko-development" 239 ) 240 if tempdir: 241 shutil.rmtree(tempdir) 242 243 244def clone(vcs, no_interactive): 245 hg = which("hg") 246 if not hg: 247 print( 248 "Mercurial is not installed. Mercurial is required to clone " 249 "Firefox%s." % (", even when cloning with Git" if vcs == "git" else "") 250 ) 251 try: 252 # We're going to recommend people install the Mercurial package with 253 # pip3. That will work if `pip3` installs binaries to a location 254 # that's in the PATH, but it might not be. To help out, if we CAN 255 # import "mercurial" (in which case it's already been installed), 256 # offer that as a solution. 257 import mercurial # noqa: F401 258 259 print( 260 "Hint: have you made sure that Mercurial is installed to a " 261 "location in your PATH?" 262 ) 263 except ImportError: 264 print("Try installing hg with `pip3 install Mercurial`.") 265 return None 266 if vcs == "hg": 267 binary = hg 268 else: 269 binary = which(vcs) 270 if not binary: 271 print("Git is not installed.") 272 print("Try installing git using your system package manager.") 273 return None 274 275 dest = input_clone_dest(vcs, no_interactive) 276 if not dest: 277 return None 278 279 print("Cloning Firefox %s repository to %s" % (VCS_HUMAN_READABLE[vcs], dest)) 280 if vcs == "hg": 281 return hg_clone_firefox(binary, dest) 282 else: 283 watchman = which("watchman") 284 return git_clone_firefox(binary, dest, watchman) 285 286 287def bootstrap(srcdir, application_choice, no_interactive, no_system_changes): 288 args = [sys.executable, os.path.join(srcdir, "mach")] 289 290 if no_interactive: 291 # --no-interactive is a global argument, not a command argument, 292 # so it needs to be specified before "bootstrap" is appended. 293 args += ["--no-interactive"] 294 295 args += ["bootstrap"] 296 297 if application_choice: 298 args += ["--application-choice", application_choice] 299 if no_system_changes: 300 args += ["--no-system-changes"] 301 302 print("Running `%s`" % " ".join(args)) 303 return subprocess.call(args, cwd=srcdir) 304 305 306def main(args): 307 parser = OptionParser() 308 parser.add_option( 309 "--application-choice", 310 dest="application_choice", 311 help='Pass in an application choice (see "APPLICATIONS" in ' 312 "python/mozboot/mozboot/bootstrap.py) instead of using the " 313 "default interactive prompt.", 314 ) 315 parser.add_option( 316 "--vcs", 317 dest="vcs", 318 default="hg", 319 choices=["git", "hg"], 320 help="VCS (hg or git) to use for downloading the source code, " 321 "instead of using the default interactive prompt.", 322 ) 323 parser.add_option( 324 "--no-interactive", 325 dest="no_interactive", 326 action="store_true", 327 help="Answer yes to any (Y/n) interactive prompts.", 328 ) 329 parser.add_option( 330 "--no-system-changes", 331 dest="no_system_changes", 332 action="store_true", 333 help="Only executes actions that leave the system " "configuration alone.", 334 ) 335 336 options, leftover = parser.parse_args(args) 337 338 try: 339 srcdir = clone(options.vcs, options.no_interactive) 340 if not srcdir: 341 return 1 342 print("Clone complete.") 343 return bootstrap( 344 srcdir, 345 options.application_choice, 346 options.no_interactive, 347 options.no_system_changes, 348 ) 349 except Exception: 350 print("Could not bootstrap Firefox! Consider filing a bug.") 351 raise 352 353 354if __name__ == "__main__": 355 sys.exit(main(sys.argv)) 356