xref: /qemu/tests/docker/docker.py (revision f03868bd) 
1#!/usr/bin/env python2
2#
3# Docker controlling module
4#
5# Copyright (c) 2016 Red Hat Inc.
6#
7# Authors:
8#  Fam Zheng <famz@redhat.com>
9#
10# This work is licensed under the terms of the GNU GPL, version 2
11# or (at your option) any later version. See the COPYING file in
12# the top-level directory.
13
14from __future__ import print_function
15import os
16import sys
17sys.path.append(os.path.join(os.path.dirname(__file__),
18                             '..', '..', 'scripts'))
19import argparse
20import subprocess
21import json
22import hashlib
23import atexit
24import uuid
25import tempfile
26import re
27import signal
28from tarfile import TarFile, TarInfo
29from StringIO import StringIO
30from shutil import copy, rmtree
31from pwd import getpwuid
32
33
34FILTERED_ENV_NAMES = ['ftp_proxy', 'http_proxy', 'https_proxy']
35
36
37DEVNULL = open(os.devnull, 'wb')
38
39
40def _text_checksum(text):
41    """Calculate a digest string unique to the text content"""
42    return hashlib.sha1(text).hexdigest()
43
44def _file_checksum(filename):
45    return _text_checksum(open(filename, 'rb').read())
46
47def _guess_docker_command():
48    """ Guess a working docker command or raise exception if not found"""
49    commands = [["docker"], ["sudo", "-n", "docker"]]
50    for cmd in commands:
51        try:
52            if subprocess.call(cmd + ["images"],
53                               stdout=DEVNULL, stderr=DEVNULL) == 0:
54                return cmd
55        except OSError:
56            pass
57    commands_txt = "\n".join(["  " + " ".join(x) for x in commands])
58    raise Exception("Cannot find working docker command. Tried:\n%s" % \
59                    commands_txt)
60
61def _copy_with_mkdir(src, root_dir, sub_path='.'):
62    """Copy src into root_dir, creating sub_path as needed."""
63    dest_dir = os.path.normpath("%s/%s" % (root_dir, sub_path))
64    try:
65        os.makedirs(dest_dir)
66    except OSError:
67        # we can safely ignore already created directories
68        pass
69
70    dest_file = "%s/%s" % (dest_dir, os.path.basename(src))
71    copy(src, dest_file)
72
73
74def _get_so_libs(executable):
75    """Return a list of libraries associated with an executable.
76
77    The paths may be symbolic links which would need to be resolved to
78    ensure theright data is copied."""
79
80    libs = []
81    ldd_re = re.compile(r"(/.*/)(\S*)")
82    try:
83        ldd_output = subprocess.check_output(["ldd", executable])
84        for line in ldd_output.split("\n"):
85            search = ldd_re.search(line)
86            if search and len(search.groups()) == 2:
87                so_path = search.groups()[0]
88                so_lib = search.groups()[1]
89                libs.append("%s/%s" % (so_path, so_lib))
90    except subprocess.CalledProcessError:
91        print("%s had no associated libraries (static build?)" % (executable))
92
93    return libs
94
95def _copy_binary_with_libs(src, dest_dir):
96    """Copy a binary executable and all its dependant libraries.
97
98    This does rely on the host file-system being fairly multi-arch
99    aware so the file don't clash with the guests layout."""
100
101    _copy_with_mkdir(src, dest_dir, "/usr/bin")
102
103    libs = _get_so_libs(src)
104    if libs:
105        for l in libs:
106            so_path = os.path.dirname(l)
107            _copy_with_mkdir(l , dest_dir, so_path)
108
109def _read_qemu_dockerfile(img_name):
110    df = os.path.join(os.path.dirname(__file__), "dockerfiles",
111                      img_name + ".docker")
112    return open(df, "r").read()
113
114def _dockerfile_preprocess(df):
115    out = ""
116    for l in df.splitlines():
117        if len(l.strip()) == 0 or l.startswith("#"):
118            continue
119        from_pref = "FROM qemu:"
120        if l.startswith(from_pref):
121            # TODO: Alternatively we could replace this line with "FROM $ID"
122            # where $ID is the image's hex id obtained with
123            #    $ docker images $IMAGE --format="{{.Id}}"
124            # but unfortunately that's not supported by RHEL 7.
125            inlining = _read_qemu_dockerfile(l[len(from_pref):])
126            out += _dockerfile_preprocess(inlining)
127            continue
128        out += l + "\n"
129    return out
130
131class Docker(object):
132    """ Running Docker commands """
133    def __init__(self):
134        self._command = _guess_docker_command()
135        self._instances = []
136        atexit.register(self._kill_instances)
137        signal.signal(signal.SIGTERM, self._kill_instances)
138        signal.signal(signal.SIGHUP, self._kill_instances)
139
140    def _do(self, cmd, quiet=True, **kwargs):
141        if quiet:
142            kwargs["stdout"] = DEVNULL
143        return subprocess.call(self._command + cmd, **kwargs)
144
145    def _do_check(self, cmd, quiet=True, **kwargs):
146        if quiet:
147            kwargs["stdout"] = DEVNULL
148        return subprocess.check_call(self._command + cmd, **kwargs)
149
150    def _do_kill_instances(self, only_known, only_active=True):
151        cmd = ["ps", "-q"]
152        if not only_active:
153            cmd.append("-a")
154        for i in self._output(cmd).split():
155            resp = self._output(["inspect", i])
156            labels = json.loads(resp)[0]["Config"]["Labels"]
157            active = json.loads(resp)[0]["State"]["Running"]
158            if not labels:
159                continue
160            instance_uuid = labels.get("com.qemu.instance.uuid", None)
161            if not instance_uuid:
162                continue
163            if only_known and instance_uuid not in self._instances:
164                continue
165            print("Terminating", i)
166            if active:
167                self._do(["kill", i])
168            self._do(["rm", i])
169
170    def clean(self):
171        self._do_kill_instances(False, False)
172        return 0
173
174    def _kill_instances(self, *args, **kwargs):
175        return self._do_kill_instances(True)
176
177    def _output(self, cmd, **kwargs):
178        return subprocess.check_output(self._command + cmd,
179                                       stderr=subprocess.STDOUT,
180                                       **kwargs)
181
182    def get_image_dockerfile_checksum(self, tag):
183        resp = self._output(["inspect", tag])
184        labels = json.loads(resp)[0]["Config"].get("Labels", {})
185        return labels.get("com.qemu.dockerfile-checksum", "")
186
187    def build_image(self, tag, docker_dir, dockerfile,
188                    quiet=True, user=False, argv=None, extra_files_cksum=[]):
189        if argv == None:
190            argv = []
191
192        tmp_df = tempfile.NamedTemporaryFile(dir=docker_dir, suffix=".docker")
193        tmp_df.write(dockerfile)
194
195        if user:
196            uid = os.getuid()
197            uname = getpwuid(uid).pw_name
198            tmp_df.write("\n")
199            tmp_df.write("RUN id %s 2>/dev/null || useradd -u %d -U %s" %
200                         (uname, uid, uname))
201
202        tmp_df.write("\n")
203        tmp_df.write("LABEL com.qemu.dockerfile-checksum=%s" %
204                     _text_checksum("\n".join([dockerfile] +
205                                    extra_files_cksum)))
206        tmp_df.flush()
207
208        self._do_check(["build", "-t", tag, "-f", tmp_df.name] + argv + \
209                       [docker_dir],
210                       quiet=quiet)
211
212    def update_image(self, tag, tarball, quiet=True):
213        "Update a tagged image using "
214
215        self._do_check(["build", "-t", tag, "-"], quiet=quiet, stdin=tarball)
216
217    def image_matches_dockerfile(self, tag, dockerfile):
218        try:
219            checksum = self.get_image_dockerfile_checksum(tag)
220        except Exception:
221            return False
222        return checksum == _text_checksum(_dockerfile_preprocess(dockerfile))
223
224    def run(self, cmd, keep, quiet):
225        label = uuid.uuid1().hex
226        if not keep:
227            self._instances.append(label)
228        ret = self._do_check(["run", "--label",
229                             "com.qemu.instance.uuid=" + label] + cmd,
230                             quiet=quiet)
231        if not keep:
232            self._instances.remove(label)
233        return ret
234
235    def command(self, cmd, argv, quiet):
236        return self._do([cmd] + argv, quiet=quiet)
237
238class SubCommand(object):
239    """A SubCommand template base class"""
240    name = None # Subcommand name
241    def shared_args(self, parser):
242        parser.add_argument("--quiet", action="store_true",
243                            help="Run quietly unless an error occured")
244
245    def args(self, parser):
246        """Setup argument parser"""
247        pass
248    def run(self, args, argv):
249        """Run command.
250        args: parsed argument by argument parser.
251        argv: remaining arguments from sys.argv.
252        """
253        pass
254
255class RunCommand(SubCommand):
256    """Invoke docker run and take care of cleaning up"""
257    name = "run"
258    def args(self, parser):
259        parser.add_argument("--keep", action="store_true",
260                            help="Don't remove image when command completes")
261    def run(self, args, argv):
262        return Docker().run(argv, args.keep, quiet=args.quiet)
263
264class BuildCommand(SubCommand):
265    """ Build docker image out of a dockerfile. Arguments: <tag> <dockerfile>"""
266    name = "build"
267    def args(self, parser):
268        parser.add_argument("--include-executable", "-e",
269                            help="""Specify a binary that will be copied to the
270                            container together with all its dependent
271                            libraries""")
272        parser.add_argument("--extra-files", "-f", nargs='*',
273                            help="""Specify files that will be copied in the
274                            Docker image, fulfilling the ADD directive from the
275                            Dockerfile""")
276        parser.add_argument("--add-current-user", "-u", dest="user",
277                            action="store_true",
278                            help="Add the current user to image's passwd")
279        parser.add_argument("tag",
280                            help="Image Tag")
281        parser.add_argument("dockerfile",
282                            help="Dockerfile name")
283
284    def run(self, args, argv):
285        dockerfile = open(args.dockerfile, "rb").read()
286        tag = args.tag
287
288        dkr = Docker()
289        if "--no-cache" not in argv and \
290           dkr.image_matches_dockerfile(tag, dockerfile):
291            if not args.quiet:
292                print("Image is up to date.")
293        else:
294            # Create a docker context directory for the build
295            docker_dir = tempfile.mkdtemp(prefix="docker_build")
296
297            # Is there a .pre file to run in the build context?
298            docker_pre = os.path.splitext(args.dockerfile)[0]+".pre"
299            if os.path.exists(docker_pre):
300                stdout = DEVNULL if args.quiet else None
301                rc = subprocess.call(os.path.realpath(docker_pre),
302                                     cwd=docker_dir, stdout=stdout)
303                if rc == 3:
304                    print("Skip")
305                    return 0
306                elif rc != 0:
307                    print("%s exited with code %d" % (docker_pre, rc))
308                    return 1
309
310            # Copy any extra files into the Docker context. These can be
311            # included by the use of the ADD directive in the Dockerfile.
312            cksum = []
313            if args.include_executable:
314                # FIXME: there is no checksum of this executable and the linked
315                # libraries, once the image built any change of this executable
316                # or any library won't trigger another build.
317                _copy_binary_with_libs(args.include_executable, docker_dir)
318            for filename in args.extra_files or []:
319                _copy_with_mkdir(filename, docker_dir)
320                cksum += [_file_checksum(filename)]
321
322            argv += ["--build-arg=" + k.lower() + "=" + v
323                        for k, v in os.environ.iteritems()
324                        if k.lower() in FILTERED_ENV_NAMES]
325            dkr.build_image(tag, docker_dir, dockerfile,
326                            quiet=args.quiet, user=args.user, argv=argv,
327                            extra_files_cksum=cksum)
328
329            rmtree(docker_dir)
330
331        return 0
332
333class UpdateCommand(SubCommand):
334    """ Update a docker image with new executables. Arguments: <tag> <executable>"""
335    name = "update"
336    def args(self, parser):
337        parser.add_argument("tag",
338                            help="Image Tag")
339        parser.add_argument("executable",
340                            help="Executable to copy")
341
342    def run(self, args, argv):
343        # Create a temporary tarball with our whole build context and
344        # dockerfile for the update
345        tmp = tempfile.NamedTemporaryFile(suffix="dckr.tar.gz")
346        tmp_tar = TarFile(fileobj=tmp, mode='w')
347
348        # Add the executable to the tarball
349        bn = os.path.basename(args.executable)
350        ff = "/usr/bin/%s" % bn
351        tmp_tar.add(args.executable, arcname=ff)
352
353        # Add any associated libraries
354        libs = _get_so_libs(args.executable)
355        if libs:
356            for l in libs:
357                tmp_tar.add(os.path.realpath(l), arcname=l)
358
359        # Create a Docker buildfile
360        df = StringIO()
361        df.write("FROM %s\n" % args.tag)
362        df.write("ADD . /\n")
363        df.seek(0)
364
365        df_tar = TarInfo(name="Dockerfile")
366        df_tar.size = len(df.buf)
367        tmp_tar.addfile(df_tar, fileobj=df)
368
369        tmp_tar.close()
370
371        # reset the file pointers
372        tmp.flush()
373        tmp.seek(0)
374
375        # Run the build with our tarball context
376        dkr = Docker()
377        dkr.update_image(args.tag, tmp, quiet=args.quiet)
378
379        return 0
380
381class CleanCommand(SubCommand):
382    """Clean up docker instances"""
383    name = "clean"
384    def run(self, args, argv):
385        Docker().clean()
386        return 0
387
388class ImagesCommand(SubCommand):
389    """Run "docker images" command"""
390    name = "images"
391    def run(self, args, argv):
392        return Docker().command("images", argv, args.quiet)
393
394
395class ProbeCommand(SubCommand):
396    """Probe if we can run docker automatically"""
397    name = "probe"
398
399    def run(self, args, argv):
400        try:
401            docker = Docker()
402            if docker._command[0] == "docker":
403                print("yes")
404            elif docker._command[0] == "sudo":
405                print("sudo")
406        except Exception:
407            print("no")
408
409        return
410
411
412def main():
413    parser = argparse.ArgumentParser(description="A Docker helper",
414            usage="%s <subcommand> ..." % os.path.basename(sys.argv[0]))
415    subparsers = parser.add_subparsers(title="subcommands", help=None)
416    for cls in SubCommand.__subclasses__():
417        cmd = cls()
418        subp = subparsers.add_parser(cmd.name, help=cmd.__doc__)
419        cmd.shared_args(subp)
420        cmd.args(subp)
421        subp.set_defaults(cmdobj=cmd)
422    args, argv = parser.parse_known_args()
423    return args.cmdobj.run(args, argv)
424
425if __name__ == "__main__":
426    sys.exit(main())
427