1#!/usr/bin/env python 2# Copyright (c) Facebook, Inc. and its affiliates. 3from __future__ import absolute_import 4from __future__ import division 5from __future__ import print_function 6from __future__ import unicode_literals 7 8""" 9 10This is a small DSL to describe builds of Facebook's open-source projects 11that are published to Github from a single internal repo, including projects 12that depend on folly, wangle, proxygen, fbthrift, etc. 13 14This file defines the interface of the DSL, and common utilieis, but you 15will have to instantiate a specific builder, with specific options, in 16order to get work done -- see e.g. make_docker_context.py. 17 18== Design notes == 19 20Goals: 21 22 - A simple declarative language for what needs to be checked out & built, 23 how, in what order. 24 25 - The same specification should work for external continuous integration 26 builds (e.g. Travis + Docker) and for internal VM-based continuous 27 integration builds. 28 29 - One should be able to build without root, and to install to a prefix. 30 31Non-goals: 32 33 - General usefulness. The only point of this is to make it easier to build 34 and test Facebook's open-source services. 35 36Ideas for the future -- these may not be very good :) 37 38 - Especially on Ubuntu 14.04 the current initial setup is inefficient: 39 we add PPAs after having installed a bunch of packages -- this prompts 40 reinstalls of large amounts of code. We also `apt-get update` a few 41 times. 42 43 - A "shell script" builder. Like DockerFBCodeBuilder, but outputs a 44 shell script that runs outside of a container. Or maybe even 45 synchronously executes the shell commands, `make`-style. 46 47 - A "Makefile" generator. That might make iterating on builds even quicker 48 than what you can currently get with Docker build caching. 49 50 - Generate a rebuild script that can be run e.g. inside the built Docker 51 container by tagging certain steps with list-inheriting Python objects: 52 * do change directories 53 * do NOT `git clone` -- if we want to update code this should be a 54 separate script that e.g. runs rebase on top of specific targets 55 across all the repos. 56 * do NOT install software (most / all setup can be skipped) 57 * do NOT `autoreconf` or `configure` 58 * do `make` and `cmake` 59 60 - If we get non-Debian OSes, part of ccache setup should be factored out. 61""" 62 63import os 64import re 65 66from shell_quoting import path_join, shell_join, ShellQuoted 67 68 69def _read_project_github_hashes(): 70 base_dir = "deps/github_hashes/" # trailing slash used in regex below 71 for dirname, _, files in os.walk(base_dir): 72 for filename in files: 73 path = os.path.join(dirname, filename) 74 with open(path) as f: 75 m_proj = re.match("^" + base_dir + "(.*)-rev\.txt$", path) 76 if m_proj is None: 77 raise RuntimeError("Not a hash file? {0}".format(path)) 78 m_hash = re.match("^Subproject commit ([0-9a-f]+)\n$", f.read()) 79 if m_hash is None: 80 raise RuntimeError("No hash in {0}".format(path)) 81 yield m_proj.group(1), m_hash.group(1) 82 83 84class FBCodeBuilder(object): 85 def __init__(self, **kwargs): 86 self._options_do_not_access = kwargs # Use .option() instead. 87 # This raises upon detecting options that are specified but unused, 88 # because otherwise it is very easy to make a typo in option names. 89 self.options_used = set() 90 # Mark 'projects_dir' used even if the build installs no github 91 # projects. This is needed because driver programs like 92 # `shell_builder.py` unconditionally set this for all builds. 93 self._github_dir = self.option("projects_dir") 94 self._github_hashes = dict(_read_project_github_hashes()) 95 96 def __repr__(self): 97 return "{0}({1})".format( 98 self.__class__.__name__, 99 ", ".join( 100 "{0}={1}".format(k, repr(v)) 101 for k, v in self._options_do_not_access.items() 102 ), 103 ) 104 105 def option(self, name, default=None): 106 value = self._options_do_not_access.get(name, default) 107 if value is None: 108 raise RuntimeError("Option {0} is required".format(name)) 109 self.options_used.add(name) 110 return value 111 112 def has_option(self, name): 113 return name in self._options_do_not_access 114 115 def add_option(self, name, value): 116 if name in self._options_do_not_access: 117 raise RuntimeError("Option {0} already set".format(name)) 118 self._options_do_not_access[name] = value 119 120 # 121 # Abstract parts common to every installation flow 122 # 123 124 def render(self, steps): 125 """ 126 127 Converts nested actions to your builder's expected output format. 128 Typically takes the output of build(). 129 130 """ 131 res = self._render_impl(steps) # Implementation-dependent 132 # Now that the output is rendered, we expect all options to have 133 # been used. 134 unused_options = set(self._options_do_not_access) 135 unused_options -= self.options_used 136 if unused_options: 137 raise RuntimeError( 138 "Unused options: {0} -- please check if you made a typo " 139 "in any of them. Those that are truly not useful should " 140 "be not be set so that this typo detection can be useful.".format( 141 unused_options 142 ) 143 ) 144 return res 145 146 def build(self, steps): 147 if not steps: 148 raise RuntimeError( 149 "Please ensure that the config you are passing " "contains steps" 150 ) 151 return [self.setup(), self.diagnostics()] + steps 152 153 def setup(self): 154 "Your builder may want to install packages here." 155 raise NotImplementedError 156 157 def diagnostics(self): 158 "Log some system diagnostics before/after setup for ease of debugging" 159 # The builder's repr is not used in a command to avoid pointlessly 160 # invalidating Docker's build cache. 161 return self.step( 162 "Diagnostics", 163 [ 164 self.comment("Builder {0}".format(repr(self))), 165 self.run(ShellQuoted("hostname")), 166 self.run(ShellQuoted("cat /etc/issue || echo no /etc/issue")), 167 self.run(ShellQuoted("g++ --version || echo g++ not installed")), 168 self.run(ShellQuoted("cmake --version || echo cmake not installed")), 169 ], 170 ) 171 172 def step(self, name, actions): 173 "A labeled collection of actions or other steps" 174 raise NotImplementedError 175 176 def run(self, shell_cmd): 177 "Run this bash command" 178 raise NotImplementedError 179 180 def set_env(self, key, value): 181 'Set the environment "key" to value "value"' 182 raise NotImplementedError 183 184 def workdir(self, dir): 185 "Create this directory if it does not exist, and change into it" 186 raise NotImplementedError 187 188 def copy_local_repo(self, dir, dest_name): 189 """ 190 Copy the local repo at `dir` into this step's `workdir()`, analog of: 191 cp -r /path/to/folly folly 192 """ 193 raise NotImplementedError 194 195 def python_deps(self): 196 return [ 197 "wheel", 198 "cython==0.28.6", 199 ] 200 201 def debian_deps(self): 202 return [ 203 "autoconf-archive", 204 "bison", 205 "build-essential", 206 "cmake", 207 "curl", 208 "flex", 209 "git", 210 "gperf", 211 "joe", 212 "libboost-all-dev", 213 "libcap-dev", 214 "libdouble-conversion-dev", 215 "libevent-dev", 216 "libgflags-dev", 217 "libgoogle-glog-dev", 218 "libkrb5-dev", 219 "libpcre3-dev", 220 "libpthread-stubs0-dev", 221 "libnuma-dev", 222 "libsasl2-dev", 223 "libsnappy-dev", 224 "libsqlite3-dev", 225 "libssl-dev", 226 "libtool", 227 "netcat-openbsd", 228 "pkg-config", 229 "sudo", 230 "unzip", 231 "wget", 232 "python3-venv", 233 ] 234 235 # 236 # Specific build helpers 237 # 238 239 def install_debian_deps(self): 240 actions = [ 241 self.run( 242 ShellQuoted("apt-get update && apt-get install -yq {deps}").format( 243 deps=shell_join( 244 " ", (ShellQuoted(dep) for dep in self.debian_deps()) 245 ) 246 ) 247 ), 248 ] 249 gcc_version = self.option("gcc_version") 250 251 # Make the selected GCC the default before building anything 252 actions.extend( 253 [ 254 self.run( 255 ShellQuoted("apt-get install -yq {c} {cpp}").format( 256 c=ShellQuoted("gcc-{v}").format(v=gcc_version), 257 cpp=ShellQuoted("g++-{v}").format(v=gcc_version), 258 ) 259 ), 260 self.run( 261 ShellQuoted( 262 "update-alternatives --install /usr/bin/gcc gcc {c} 40 " 263 "--slave /usr/bin/g++ g++ {cpp}" 264 ).format( 265 c=ShellQuoted("/usr/bin/gcc-{v}").format(v=gcc_version), 266 cpp=ShellQuoted("/usr/bin/g++-{v}").format(v=gcc_version), 267 ) 268 ), 269 self.run(ShellQuoted("update-alternatives --config gcc")), 270 ] 271 ) 272 273 actions.extend(self.debian_ccache_setup_steps()) 274 275 return self.step("Install packages for Debian-based OS", actions) 276 277 def create_python_venv(self): 278 actions = [] 279 if self.option("PYTHON_VENV", "OFF") == "ON": 280 actions.append( 281 self.run( 282 ShellQuoted("python3 -m venv {p}").format( 283 p=path_join(self.option("prefix"), "venv") 284 ) 285 ) 286 ) 287 return actions 288 289 def python_venv(self): 290 actions = [] 291 if self.option("PYTHON_VENV", "OFF") == "ON": 292 actions.append( 293 ShellQuoted("source {p}").format( 294 p=path_join(self.option("prefix"), "venv", "bin", "activate") 295 ) 296 ) 297 298 actions.append( 299 self.run( 300 ShellQuoted("python3 -m pip install {deps}").format( 301 deps=shell_join( 302 " ", (ShellQuoted(dep) for dep in self.python_deps()) 303 ) 304 ) 305 ) 306 ) 307 return actions 308 309 def enable_rust_toolchain(self, toolchain="stable", is_bootstrap=True): 310 choices = set(["stable", "beta", "nightly"]) 311 312 assert toolchain in choices, ( 313 "while enabling rust toolchain: {} is not in {}" 314 ).format(toolchain, choices) 315 316 rust_toolchain_opt = (toolchain, is_bootstrap) 317 prev_opt = self.option("rust_toolchain", rust_toolchain_opt) 318 assert prev_opt == rust_toolchain_opt, ( 319 "while enabling rust toolchain: previous toolchain already set to" 320 " {}, but trying to set it to {} now" 321 ).format(prev_opt, rust_toolchain_opt) 322 323 self.add_option("rust_toolchain", rust_toolchain_opt) 324 325 def rust_toolchain(self): 326 actions = [] 327 if self.option("rust_toolchain", False): 328 (toolchain, is_bootstrap) = self.option("rust_toolchain") 329 rust_dir = path_join(self.option("prefix"), "rust") 330 actions = [ 331 self.set_env("CARGO_HOME", rust_dir), 332 self.set_env("RUSTUP_HOME", rust_dir), 333 self.set_env("RUSTC_BOOTSTRAP", "1" if is_bootstrap else "0"), 334 self.run( 335 ShellQuoted( 336 "curl -sSf https://build.travis-ci.com/files/rustup-init.sh" 337 " | sh -s --" 338 " --default-toolchain={r} " 339 " --profile=minimal" 340 " --no-modify-path" 341 " -y" 342 ).format(p=rust_dir, r=toolchain) 343 ), 344 self.set_env( 345 "PATH", 346 ShellQuoted("{p}:$PATH").format(p=path_join(rust_dir, "bin")), 347 ), 348 self.run(ShellQuoted("rustup update")), 349 self.run(ShellQuoted("rustc --version")), 350 self.run(ShellQuoted("rustup --version")), 351 self.run(ShellQuoted("cargo --version")), 352 ] 353 return actions 354 355 def debian_ccache_setup_steps(self): 356 return [] # It's ok to ship a renderer without ccache support. 357 358 def github_project_workdir(self, project, path): 359 # Only check out a non-default branch if requested. This especially 360 # makes sense when building from a local repo. 361 git_hash = self.option( 362 "{0}:git_hash".format(project), 363 # Any repo that has a hash in deps/github_hashes defaults to 364 # that, with the goal of making builds maximally consistent. 365 self._github_hashes.get(project, ""), 366 ) 367 maybe_change_branch = ( 368 [ 369 self.run(ShellQuoted("git checkout {hash}").format(hash=git_hash)), 370 ] 371 if git_hash 372 else [] 373 ) 374 375 local_repo_dir = self.option("{0}:local_repo_dir".format(project), "") 376 return self.step( 377 "Check out {0}, workdir {1}".format(project, path), 378 [ 379 self.workdir(self._github_dir), 380 self.run( 381 ShellQuoted("git clone {opts} https://github.com/{p}").format( 382 p=project, 383 opts=ShellQuoted( 384 self.option("{}:git_clone_opts".format(project), "") 385 ), 386 ) 387 ) 388 if not local_repo_dir 389 else self.copy_local_repo(local_repo_dir, os.path.basename(project)), 390 self.workdir( 391 path_join(self._github_dir, os.path.basename(project), path), 392 ), 393 ] 394 + maybe_change_branch, 395 ) 396 397 def fb_github_project_workdir(self, project_and_path, github_org="facebook"): 398 "This helper lets Facebook-internal CI special-cases FB projects" 399 project, path = project_and_path.split("/", 1) 400 return self.github_project_workdir(github_org + "/" + project, path) 401 402 def _make_vars(self, make_vars): 403 return shell_join( 404 " ", 405 ( 406 ShellQuoted("{k}={v}").format(k=k, v=v) 407 for k, v in ({} if make_vars is None else make_vars).items() 408 ), 409 ) 410 411 def parallel_make(self, make_vars=None): 412 return self.run( 413 ShellQuoted("make -j {n} VERBOSE=1 {vars}").format( 414 n=self.option("make_parallelism"), 415 vars=self._make_vars(make_vars), 416 ) 417 ) 418 419 def make_and_install(self, make_vars=None): 420 return [ 421 self.parallel_make(make_vars), 422 self.run( 423 ShellQuoted("make install VERBOSE=1 {vars}").format( 424 vars=self._make_vars(make_vars), 425 ) 426 ), 427 ] 428 429 def configure(self, name=None): 430 autoconf_options = {} 431 if name is not None: 432 autoconf_options.update( 433 self.option("{0}:autoconf_options".format(name), {}) 434 ) 435 return [ 436 self.run( 437 ShellQuoted( 438 'LDFLAGS="$LDFLAGS -L"{p}"/lib -Wl,-rpath="{p}"/lib" ' 439 'CFLAGS="$CFLAGS -I"{p}"/include" ' 440 'CPPFLAGS="$CPPFLAGS -I"{p}"/include" ' 441 "PY_PREFIX={p} " 442 "./configure --prefix={p} {args}" 443 ).format( 444 p=self.option("prefix"), 445 args=shell_join( 446 " ", 447 ( 448 ShellQuoted("{k}={v}").format(k=k, v=v) 449 for k, v in autoconf_options.items() 450 ), 451 ), 452 ) 453 ), 454 ] 455 456 def autoconf_install(self, name): 457 return self.step( 458 "Build and install {0}".format(name), 459 [ 460 self.run(ShellQuoted("autoreconf -ivf")), 461 ] 462 + self.configure() 463 + self.make_and_install(), 464 ) 465 466 def cmake_configure(self, name, cmake_path=".."): 467 cmake_defines = { 468 "BUILD_SHARED_LIBS": "ON", 469 "CMAKE_INSTALL_PREFIX": self.option("prefix"), 470 } 471 472 # Hacks to add thriftpy3 support 473 if "BUILD_THRIFT_PY3" in os.environ and "folly" in name: 474 cmake_defines["PYTHON_EXTENSIONS"] = "True" 475 476 if "BUILD_THRIFT_PY3" in os.environ and "fbthrift" in name: 477 cmake_defines["thriftpy3"] = "ON" 478 479 cmake_defines.update(self.option("{0}:cmake_defines".format(name), {})) 480 return [ 481 self.run( 482 ShellQuoted( 483 'CXXFLAGS="$CXXFLAGS -fPIC -isystem "{p}"/include" ' 484 'CFLAGS="$CFLAGS -fPIC -isystem "{p}"/include" ' 485 "cmake {args} {cmake_path}" 486 ).format( 487 p=self.option("prefix"), 488 args=shell_join( 489 " ", 490 ( 491 ShellQuoted("-D{k}={v}").format(k=k, v=v) 492 for k, v in cmake_defines.items() 493 ), 494 ), 495 cmake_path=cmake_path, 496 ) 497 ), 498 ] 499 500 def cmake_install(self, name, cmake_path=".."): 501 return self.step( 502 "Build and install {0}".format(name), 503 self.cmake_configure(name, cmake_path) + self.make_and_install(), 504 ) 505 506 def cargo_build(self, name): 507 return self.step( 508 "Build {0}".format(name), 509 [ 510 self.run( 511 ShellQuoted("cargo build -j {n}").format( 512 n=self.option("make_parallelism") 513 ) 514 ) 515 ], 516 ) 517 518 def fb_github_autoconf_install(self, project_and_path, github_org="facebook"): 519 return [ 520 self.fb_github_project_workdir(project_and_path, github_org), 521 self.autoconf_install(project_and_path), 522 ] 523 524 def fb_github_cmake_install( 525 self, project_and_path, cmake_path="..", github_org="facebook" 526 ): 527 return [ 528 self.fb_github_project_workdir(project_and_path, github_org), 529 self.cmake_install(project_and_path, cmake_path), 530 ] 531 532 def fb_github_cargo_build(self, project_and_path, github_org="facebook"): 533 return [ 534 self.fb_github_project_workdir(project_and_path, github_org), 535 self.cargo_build(project_and_path), 536 ] 537