1# This Source Code Form is subject to the terms of the Mozilla Public
2# License, v. 2.0. If a copy of the MPL was not distributed with this
3# file, You can obtain one at http://mozilla.org/MPL/2.0/.
4
5from __future__ import absolute_import, print_function, unicode_literals
6
7import logging
8import os
9import re
10import json
11
12import six
13from six import text_type
14import mozpack.path as mozpath
15import taskgraph
16from taskgraph.transforms.base import TransformSequence
17from .. import GECKO
18from taskgraph.util.docker import (
19    create_context_tar,
20    generate_context_hash,
21    image_path,
22)
23from taskgraph.util.schema import Schema
24from voluptuous import (
25    Optional,
26    Required,
27)
28from .task import task_description_schema
29
30logger = logging.getLogger(__name__)
31
32CONTEXTS_DIR = "docker-contexts"
33
34DIGEST_RE = re.compile("^[0-9a-f]{64}$")
35
36IMAGE_BUILDER_IMAGE = (
37    "mozillareleases/image_builder:5.0.0"
38    "@sha256:"
39    "e510a9a9b80385f71c112d61b2f2053da625aff2b6d430411ac42e424c58953f"
40)
41
42transforms = TransformSequence()
43
44docker_image_schema = Schema(
45    {
46        # Name of the docker image.
47        Required("name"): text_type,
48        # Name of the parent docker image.
49        Optional("parent"): text_type,
50        # Treeherder symbol.
51        Required("symbol"): text_type,
52        # relative path (from config.path) to the file the docker image was defined
53        # in.
54        Optional("job-from"): text_type,
55        # Arguments to use for the Dockerfile.
56        Optional("args"): {text_type: text_type},
57        # Name of the docker image definition under taskcluster/docker, when
58        # different from the docker image name.
59        Optional("definition"): text_type,
60        # List of package tasks this docker image depends on.
61        Optional("packages"): [text_type],
62        Optional(
63            "index",
64            description="information for indexing this build so its artifacts can be discovered",
65        ): task_description_schema["index"],
66        Optional(
67            "cache",
68            description="Whether this image should be cached based on inputs.",
69        ): bool,
70    }
71)
72
73
74transforms.add_validate(docker_image_schema)
75
76
77@transforms.add
78def fill_template(config, tasks):
79    if not taskgraph.fast and config.write_artifacts:
80        if not os.path.isdir(CONTEXTS_DIR):
81            os.makedirs(CONTEXTS_DIR)
82
83    for task in tasks:
84        image_name = task.pop("name")
85        job_symbol = task.pop("symbol")
86        args = task.pop("args", {})
87        packages = task.pop("packages", [])
88        parent = task.pop("parent", None)
89
90        for p in packages:
91            if "packages-{}".format(p) not in config.kind_dependencies_tasks:
92                raise Exception(
93                    "Missing package job for {}-{}: {}".format(
94                        config.kind, image_name, p
95                    )
96                )
97
98        if not taskgraph.fast:
99            context_path = mozpath.relpath(image_path(image_name), GECKO)
100            if config.write_artifacts:
101                context_file = os.path.join(
102                    CONTEXTS_DIR, "{}.tar.gz".format(image_name)
103                )
104                logger.info(
105                    "Writing {} for docker image {}".format(context_file, image_name)
106                )
107                context_hash = create_context_tar(
108                    GECKO, context_path, context_file, image_name, args
109                )
110            else:
111                context_hash = generate_context_hash(
112                    GECKO, context_path, image_name, args
113                )
114        else:
115            if config.write_artifacts:
116                raise Exception("Can't write artifacts if `taskgraph.fast` is set.")
117            context_hash = "0" * 40
118        digest_data = [context_hash]
119        digest_data += [json.dumps(args, sort_keys=True)]
120
121        description = "Build the docker image {} for use by dependent tasks".format(
122            image_name
123        )
124
125        args["DOCKER_IMAGE_PACKAGES"] = " ".join("<{}>".format(p) for p in packages)
126
127        # Adjust the zstandard compression level based on the execution level.
128        # We use faster compression for level 1 because we care more about
129        # end-to-end times. We use slower/better compression for other levels
130        # because images are read more often and it is worth the trade-off to
131        # burn more CPU once to reduce image size.
132        zstd_level = "3" if int(config.params["level"]) == 1 else "10"
133
134        # include some information that is useful in reconstructing this task
135        # from JSON
136        taskdesc = {
137            "label": "{}-{}".format(config.kind, image_name),
138            "description": description,
139            "attributes": {
140                "image_name": image_name,
141                "artifact_prefix": "public",
142            },
143            "expires-after": "1 year",
144            "scopes": [],
145            "treeherder": {
146                "symbol": job_symbol,
147                "platform": "taskcluster-images/opt",
148                "kind": "other",
149                "tier": 1,
150            },
151            "run-on-projects": [],
152            "worker-type": "images",
153            "worker": {
154                "implementation": "docker-worker",
155                "os": "linux",
156                "artifacts": [
157                    {
158                        "type": "file",
159                        "path": "/workspace/image.tar.zst",
160                        "name": "public/image.tar.zst",
161                    }
162                ],
163                "env": {
164                    "CONTEXT_TASK_ID": {"task-reference": "<decision>"},
165                    "CONTEXT_PATH": "public/docker-contexts/{}.tar.gz".format(
166                        image_name
167                    ),
168                    "HASH": context_hash,
169                    "PROJECT": config.params["project"],
170                    "IMAGE_NAME": image_name,
171                    "DOCKER_IMAGE_ZSTD_LEVEL": zstd_level,
172                    "DOCKER_BUILD_ARGS": {
173                        "task-reference": six.ensure_text(json.dumps(args))
174                    },
175                    "GECKO_BASE_REPOSITORY": config.params["base_repository"],
176                    "GECKO_HEAD_REPOSITORY": config.params["head_repository"],
177                    "GECKO_HEAD_REV": config.params["head_rev"],
178                },
179                "chain-of-trust": True,
180                "max-run-time": 7200,
181                # FIXME: We aren't currently propagating the exit code
182            },
183        }
184        # Retry for 'funsize-update-generator' if exit status code is -1
185        if image_name in ["funsize-update-generator"]:
186            taskdesc["worker"]["retry-exit-status"] = [-1]
187
188        worker = taskdesc["worker"]
189
190        if image_name == "image_builder":
191            worker["docker-image"] = IMAGE_BUILDER_IMAGE
192            digest_data.append("image-builder-image:{}".format(IMAGE_BUILDER_IMAGE))
193        else:
194            worker["docker-image"] = {"in-tree": "image_builder"}
195            deps = taskdesc.setdefault("dependencies", {})
196            deps["docker-image"] = "{}-image_builder".format(config.kind)
197
198        if packages:
199            deps = taskdesc.setdefault("dependencies", {})
200            for p in sorted(packages):
201                deps[p] = "packages-{}".format(p)
202
203        if parent:
204            deps = taskdesc.setdefault("dependencies", {})
205            deps["parent"] = "{}-{}".format(config.kind, parent)
206            worker["env"]["PARENT_TASK_ID"] = {
207                "task-reference": "<parent>",
208            }
209        if "index" in task:
210            taskdesc["index"] = task["index"]
211
212        if task.get("cache", True) and not taskgraph.fast:
213            taskdesc["cache"] = {
214                "type": "docker-images.v2",
215                "name": image_name,
216                "digest-data": digest_data,
217            }
218
219        yield taskdesc
220