1#!/usr/bin/env python3
2
3# Build the Debian packages using Docker images.
4#
5# This script builds the Docker images and then executes them sequentially, each
6# one building a Debian package for the targeted operating system. It is
7# designed to be a "single command" to produce all the images.
8#
9# By default, builds for all known distributions, but a list of distributions
10# can be passed on the commandline for debugging.
11
12import argparse
13import json
14import os
15import signal
16import subprocess
17import sys
18import threading
19from concurrent.futures import ThreadPoolExecutor
20from typing import Optional, Sequence
21
22DISTS = (
23    "debian:buster",  # oldstable: EOL 2022-08
24    "debian:bullseye",
25    "debian:bookworm",
26    "debian:sid",
27    "ubuntu:focal",  # 20.04 LTS (our EOL forced by Py38 on 2024-10-14)
28    "ubuntu:hirsute",  # 21.04 (EOL 2022-01-05)
29    "ubuntu:impish",  # 21.10  (EOL 2022-07)
30)
31
32DESC = """\
33Builds .debs for synapse, using a Docker image for the build environment.
34
35By default, builds for all known distributions, but a list of distributions
36can be passed on the commandline for debugging.
37"""
38
39projdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
40
41
42class Builder(object):
43    def __init__(
44        self, redirect_stdout=False, docker_build_args: Optional[Sequence[str]] = None
45    ):
46        self.redirect_stdout = redirect_stdout
47        self._docker_build_args = tuple(docker_build_args or ())
48        self.active_containers = set()
49        self._lock = threading.Lock()
50        self._failed = False
51
52    def run_build(self, dist, skip_tests=False):
53        """Build deb for a single distribution"""
54
55        if self._failed:
56            print("not building %s due to earlier failure" % (dist,))
57            raise Exception("failed")
58
59        try:
60            self._inner_build(dist, skip_tests)
61        except Exception as e:
62            print("build of %s failed: %s" % (dist, e), file=sys.stderr)
63            self._failed = True
64            raise
65
66    def _inner_build(self, dist, skip_tests=False):
67        tag = dist.split(":", 1)[1]
68
69        # Make the dir where the debs will live.
70        #
71        # Note that we deliberately put this outside the source tree, otherwise
72        # we tend to get source packages which are full of debs. (We could hack
73        # around that with more magic in the build_debian.sh script, but that
74        # doesn't solve the problem for natively-run dpkg-buildpakage).
75        debsdir = os.path.join(projdir, "../debs")
76        os.makedirs(debsdir, exist_ok=True)
77
78        if self.redirect_stdout:
79            logfile = os.path.join(debsdir, "%s.buildlog" % (tag,))
80            print("building %s: directing output to %s" % (dist, logfile))
81            stdout = open(logfile, "w")
82        else:
83            stdout = None
84
85        # first build a docker image for the build environment
86        build_args = (
87            (
88                "docker",
89                "build",
90                "--tag",
91                "dh-venv-builder:" + tag,
92                "--build-arg",
93                "distro=" + dist,
94                "-f",
95                "docker/Dockerfile-dhvirtualenv",
96            )
97            + self._docker_build_args
98            + ("docker",)
99        )
100
101        subprocess.check_call(
102            build_args,
103            stdout=stdout,
104            stderr=subprocess.STDOUT,
105            cwd=projdir,
106        )
107
108        container_name = "synapse_build_" + tag
109        with self._lock:
110            self.active_containers.add(container_name)
111
112        # then run the build itself
113        subprocess.check_call(
114            [
115                "docker",
116                "run",
117                "--rm",
118                "--name",
119                container_name,
120                "--volume=" + projdir + ":/synapse/source:ro",
121                "--volume=" + debsdir + ":/debs",
122                "-e",
123                "TARGET_USERID=%i" % (os.getuid(),),
124                "-e",
125                "TARGET_GROUPID=%i" % (os.getgid(),),
126                "-e",
127                "DEB_BUILD_OPTIONS=%s" % ("nocheck" if skip_tests else ""),
128                "dh-venv-builder:" + tag,
129            ],
130            stdout=stdout,
131            stderr=subprocess.STDOUT,
132        )
133
134        with self._lock:
135            self.active_containers.remove(container_name)
136
137        if stdout is not None:
138            stdout.close()
139            print("Completed build of %s" % (dist,))
140
141    def kill_containers(self):
142        with self._lock:
143            active = list(self.active_containers)
144
145        for c in active:
146            print("killing container %s" % (c,))
147            subprocess.run(
148                [
149                    "docker",
150                    "kill",
151                    c,
152                ],
153                stdout=subprocess.DEVNULL,
154            )
155            with self._lock:
156                self.active_containers.remove(c)
157
158
159def run_builds(builder, dists, jobs=1, skip_tests=False):
160    def sig(signum, _frame):
161        print("Caught SIGINT")
162        builder.kill_containers()
163
164    signal.signal(signal.SIGINT, sig)
165
166    with ThreadPoolExecutor(max_workers=jobs) as e:
167        res = e.map(lambda dist: builder.run_build(dist, skip_tests), dists)
168
169    # make sure we consume the iterable so that exceptions are raised.
170    for _ in res:
171        pass
172
173
174if __name__ == "__main__":
175    parser = argparse.ArgumentParser(
176        description=DESC,
177    )
178    parser.add_argument(
179        "-j",
180        "--jobs",
181        type=int,
182        default=1,
183        help="specify the number of builds to run in parallel",
184    )
185    parser.add_argument(
186        "--no-check",
187        action="store_true",
188        help="skip running tests after building",
189    )
190    parser.add_argument(
191        "--docker-build-arg",
192        action="append",
193        help="specify an argument to pass to docker build",
194    )
195    parser.add_argument(
196        "--show-dists-json",
197        action="store_true",
198        help="instead of building the packages, just list the dists to build for, as a json array",
199    )
200    parser.add_argument(
201        "dist",
202        nargs="*",
203        default=DISTS,
204        help="a list of distributions to build for. Default: %(default)s",
205    )
206    args = parser.parse_args()
207    if args.show_dists_json:
208        print(json.dumps(DISTS))
209    else:
210        builder = Builder(
211            redirect_stdout=(args.jobs > 1), docker_build_args=args.docker_build_arg
212        )
213        run_builds(
214            builder,
215            dists=args.dist,
216            jobs=args.jobs,
217            skip_tests=args.no_check,
218        )
219