1#!/usr/bin/env python 2# Copyright (c) Facebook, Inc. and its affiliates. 3from __future__ import absolute_import 4from __future__ import division 5from __future__ import print_function 6from __future__ import unicode_literals 7 8""" 9 10Extends FBCodeBuilder to produce Docker context directories. 11 12In order to get the largest iteration-time savings from Docker's build 13caching, you will want to: 14 - Use fine-grained steps as appropriate (e.g. separate make & make install), 15 - Start your action sequence with the lowest-risk steps, and with the steps 16 that change the least often, and 17 - Put the steps that you are debugging towards the very end. 18 19""" 20import logging 21import os 22import shutil 23import tempfile 24 25from fbcode_builder import FBCodeBuilder 26from shell_quoting import raw_shell, shell_comment, shell_join, ShellQuoted, path_join 27from utils import recursively_flatten_list, run_command 28 29 30class DockerFBCodeBuilder(FBCodeBuilder): 31 def _user(self): 32 return self.option("user", "root") 33 34 def _change_user(self): 35 return ShellQuoted("USER {u}").format(u=self._user()) 36 37 def setup(self): 38 # Please add RPM-based OSes here as appropriate. 39 # 40 # To allow exercising non-root installs -- we change users after the 41 # system packages are installed. TODO: For users not defined in the 42 # image, we should probably `useradd`. 43 return self.step( 44 "Setup", 45 [ 46 # Docker's FROM does not understand shell quoting. 47 ShellQuoted("FROM {}".format(self.option("os_image"))), 48 # /bin/sh syntax is a pain 49 ShellQuoted('SHELL ["/bin/bash", "-c"]'), 50 ] 51 + self.install_debian_deps() 52 + [self._change_user()] 53 + [self.workdir(self.option("prefix"))] 54 + self.create_python_venv() 55 + self.python_venv() 56 + self.rust_toolchain(), 57 ) 58 59 def python_venv(self): 60 # To both avoid calling venv activate on each RUN command AND to ensure 61 # it is present when the resulting container is run add to PATH 62 actions = [] 63 if self.option("PYTHON_VENV", "OFF") == "ON": 64 actions = ShellQuoted("ENV PATH={p}:$PATH").format( 65 p=path_join(self.option("prefix"), "venv", "bin") 66 ) 67 return actions 68 69 def step(self, name, actions): 70 assert "\n" not in name, "Name {0} would span > 1 line".format(name) 71 b = ShellQuoted("") 72 return [ShellQuoted("### {0} ###".format(name)), b] + actions + [b] 73 74 def run(self, shell_cmd): 75 return ShellQuoted("RUN {cmd}").format(cmd=shell_cmd) 76 77 def set_env(self, key, value): 78 return ShellQuoted("ENV {key}={val}").format(key=key, val=value) 79 80 def workdir(self, dir): 81 return [ 82 # As late as Docker 1.12.5, this results in `build` being owned 83 # by root:root -- the explicit `mkdir` works around the bug: 84 # USER nobody 85 # WORKDIR build 86 ShellQuoted("USER root"), 87 ShellQuoted("RUN mkdir -p {d} && chown {u} {d}").format( 88 d=dir, u=self._user() 89 ), 90 self._change_user(), 91 ShellQuoted("WORKDIR {dir}").format(dir=dir), 92 ] 93 94 def comment(self, comment): 95 # This should not be a command since we don't want comment changes 96 # to invalidate the Docker build cache. 97 return shell_comment(comment) 98 99 def copy_local_repo(self, repo_dir, dest_name): 100 fd, archive_path = tempfile.mkstemp( 101 prefix="local_repo_{0}_".format(dest_name), 102 suffix=".tgz", 103 dir=os.path.abspath(self.option("docker_context_dir")), 104 ) 105 os.close(fd) 106 run_command("tar", "czf", archive_path, ".", cwd=repo_dir) 107 return [ 108 ShellQuoted("ADD {archive} {dest_name}").format( 109 archive=os.path.basename(archive_path), dest_name=dest_name 110 ), 111 # Docker permissions make very little sense... see also workdir() 112 ShellQuoted("USER root"), 113 ShellQuoted("RUN chown -R {u} {d}").format(d=dest_name, u=self._user()), 114 self._change_user(), 115 ] 116 117 def _render_impl(self, steps): 118 return raw_shell(shell_join("\n", recursively_flatten_list(steps))) 119 120 def debian_ccache_setup_steps(self): 121 source_ccache_tgz = self.option("ccache_tgz", "") 122 if not source_ccache_tgz: 123 logging.info("Docker ccache not enabled") 124 return [] 125 126 dest_ccache_tgz = os.path.join(self.option("docker_context_dir"), "ccache.tgz") 127 128 try: 129 try: 130 os.link(source_ccache_tgz, dest_ccache_tgz) 131 except OSError: 132 logging.exception( 133 "Hard-linking {s} to {d} failed, falling back to copy".format( 134 s=source_ccache_tgz, d=dest_ccache_tgz 135 ) 136 ) 137 shutil.copyfile(source_ccache_tgz, dest_ccache_tgz) 138 except Exception: 139 logging.exception( 140 "Failed to copy or link {s} to {d}, aborting".format( 141 s=source_ccache_tgz, d=dest_ccache_tgz 142 ) 143 ) 144 raise 145 146 return [ 147 # Separate layer so that in development we avoid re-downloads. 148 self.run(ShellQuoted("apt-get install -yq ccache")), 149 ShellQuoted("ADD ccache.tgz /"), 150 ShellQuoted( 151 # Set CCACHE_DIR before the `ccache` invocations below. 152 "ENV CCACHE_DIR=/ccache " 153 # No clang support for now, so it's easiest to hardcode gcc. 154 'CC="ccache gcc" CXX="ccache g++" ' 155 # Always log for ease of debugging. For real FB projects, 156 # this log is several megabytes, so dumping it to stdout 157 # would likely exceed the Travis log limit of 4MB. 158 # 159 # On a local machine, `docker cp` will get you the data. To 160 # get the data out from Travis, I would compress and dump 161 # uuencoded bytes to the log -- for Bistro this was about 162 # 600kb or 8000 lines: 163 # 164 # apt-get install sharutils 165 # bzip2 -9 < /tmp/ccache.log | uuencode -m ccache.log.bz2 166 "CCACHE_LOGFILE=/tmp/ccache.log" 167 ), 168 self.run( 169 ShellQuoted( 170 # Future: Skipping this part made this Docker step instant, 171 # saving ~1min of build time. It's unclear if it is the 172 # chown or the du, but probably the chown -- since a large 173 # part of the cost is incurred at image save time. 174 # 175 # ccache.tgz may be empty, or may have the wrong 176 # permissions. 177 "mkdir -p /ccache && time chown -R nobody /ccache && " 178 "time du -sh /ccache && " 179 # Reset stats so `docker_build_with_ccache.sh` can print 180 # useful values at the end of the run. 181 "echo === Prev run stats === && ccache -s && ccache -z && " 182 # Record the current time to let travis_build.sh figure out 183 # the number of bytes in the cache that are actually used -- 184 # this is crucial for tuning the maximum cache size. 185 "date +%s > /FBCODE_BUILDER_CCACHE_START_TIME && " 186 # The build running as `nobody` should be able to write here 187 "chown nobody /tmp/ccache.log" 188 ) 189 ), 190 ] 191