1#!/usr/bin/env python3 2 3# Build the Debian packages using Docker images. 4# 5# This script builds the Docker images and then executes them sequentially, each 6# one building a Debian package for the targeted operating system. It is 7# designed to be a "single command" to produce all the images. 8# 9# By default, builds for all known distributions, but a list of distributions 10# can be passed on the commandline for debugging. 11 12import argparse 13import json 14import os 15import signal 16import subprocess 17import sys 18import threading 19from concurrent.futures import ThreadPoolExecutor 20from typing import Optional, Sequence 21 22DISTS = ( 23 "debian:buster", # oldstable: EOL 2022-08 24 "debian:bullseye", 25 "debian:bookworm", 26 "debian:sid", 27 "ubuntu:focal", # 20.04 LTS (our EOL forced by Py38 on 2024-10-14) 28 "ubuntu:hirsute", # 21.04 (EOL 2022-01-05) 29 "ubuntu:impish", # 21.10 (EOL 2022-07) 30) 31 32DESC = """\ 33Builds .debs for synapse, using a Docker image for the build environment. 34 35By default, builds for all known distributions, but a list of distributions 36can be passed on the commandline for debugging. 37""" 38 39projdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) 40 41 42class Builder(object): 43 def __init__( 44 self, redirect_stdout=False, docker_build_args: Optional[Sequence[str]] = None 45 ): 46 self.redirect_stdout = redirect_stdout 47 self._docker_build_args = tuple(docker_build_args or ()) 48 self.active_containers = set() 49 self._lock = threading.Lock() 50 self._failed = False 51 52 def run_build(self, dist, skip_tests=False): 53 """Build deb for a single distribution""" 54 55 if self._failed: 56 print("not building %s due to earlier failure" % (dist,)) 57 raise Exception("failed") 58 59 try: 60 self._inner_build(dist, skip_tests) 61 except Exception as e: 62 print("build of %s failed: %s" % (dist, e), file=sys.stderr) 63 self._failed = True 64 raise 65 66 def _inner_build(self, dist, skip_tests=False): 67 tag = dist.split(":", 1)[1] 68 69 # Make the dir where the debs will live. 70 # 71 # Note that we deliberately put this outside the source tree, otherwise 72 # we tend to get source packages which are full of debs. (We could hack 73 # around that with more magic in the build_debian.sh script, but that 74 # doesn't solve the problem for natively-run dpkg-buildpakage). 75 debsdir = os.path.join(projdir, "../debs") 76 os.makedirs(debsdir, exist_ok=True) 77 78 if self.redirect_stdout: 79 logfile = os.path.join(debsdir, "%s.buildlog" % (tag,)) 80 print("building %s: directing output to %s" % (dist, logfile)) 81 stdout = open(logfile, "w") 82 else: 83 stdout = None 84 85 # first build a docker image for the build environment 86 build_args = ( 87 ( 88 "docker", 89 "build", 90 "--tag", 91 "dh-venv-builder:" + tag, 92 "--build-arg", 93 "distro=" + dist, 94 "-f", 95 "docker/Dockerfile-dhvirtualenv", 96 ) 97 + self._docker_build_args 98 + ("docker",) 99 ) 100 101 subprocess.check_call( 102 build_args, 103 stdout=stdout, 104 stderr=subprocess.STDOUT, 105 cwd=projdir, 106 ) 107 108 container_name = "synapse_build_" + tag 109 with self._lock: 110 self.active_containers.add(container_name) 111 112 # then run the build itself 113 subprocess.check_call( 114 [ 115 "docker", 116 "run", 117 "--rm", 118 "--name", 119 container_name, 120 "--volume=" + projdir + ":/synapse/source:ro", 121 "--volume=" + debsdir + ":/debs", 122 "-e", 123 "TARGET_USERID=%i" % (os.getuid(),), 124 "-e", 125 "TARGET_GROUPID=%i" % (os.getgid(),), 126 "-e", 127 "DEB_BUILD_OPTIONS=%s" % ("nocheck" if skip_tests else ""), 128 "dh-venv-builder:" + tag, 129 ], 130 stdout=stdout, 131 stderr=subprocess.STDOUT, 132 ) 133 134 with self._lock: 135 self.active_containers.remove(container_name) 136 137 if stdout is not None: 138 stdout.close() 139 print("Completed build of %s" % (dist,)) 140 141 def kill_containers(self): 142 with self._lock: 143 active = list(self.active_containers) 144 145 for c in active: 146 print("killing container %s" % (c,)) 147 subprocess.run( 148 [ 149 "docker", 150 "kill", 151 c, 152 ], 153 stdout=subprocess.DEVNULL, 154 ) 155 with self._lock: 156 self.active_containers.remove(c) 157 158 159def run_builds(builder, dists, jobs=1, skip_tests=False): 160 def sig(signum, _frame): 161 print("Caught SIGINT") 162 builder.kill_containers() 163 164 signal.signal(signal.SIGINT, sig) 165 166 with ThreadPoolExecutor(max_workers=jobs) as e: 167 res = e.map(lambda dist: builder.run_build(dist, skip_tests), dists) 168 169 # make sure we consume the iterable so that exceptions are raised. 170 for _ in res: 171 pass 172 173 174if __name__ == "__main__": 175 parser = argparse.ArgumentParser( 176 description=DESC, 177 ) 178 parser.add_argument( 179 "-j", 180 "--jobs", 181 type=int, 182 default=1, 183 help="specify the number of builds to run in parallel", 184 ) 185 parser.add_argument( 186 "--no-check", 187 action="store_true", 188 help="skip running tests after building", 189 ) 190 parser.add_argument( 191 "--docker-build-arg", 192 action="append", 193 help="specify an argument to pass to docker build", 194 ) 195 parser.add_argument( 196 "--show-dists-json", 197 action="store_true", 198 help="instead of building the packages, just list the dists to build for, as a json array", 199 ) 200 parser.add_argument( 201 "dist", 202 nargs="*", 203 default=DISTS, 204 help="a list of distributions to build for. Default: %(default)s", 205 ) 206 args = parser.parse_args() 207 if args.show_dists_json: 208 print(json.dumps(DISTS)) 209 else: 210 builder = Builder( 211 redirect_stdout=(args.jobs > 1), docker_build_args=args.docker_build_arg 212 ) 213 run_builds( 214 builder, 215 dists=args.dist, 216 jobs=args.jobs, 217 skip_tests=args.no_check, 218 ) 219