1# This Source Code Form is subject to the terms of the Mozilla Public 2# License, v. 2.0. If a copy of the MPL was not distributed with this 3# file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5from __future__ import absolute_import, print_function, unicode_literals 6 7import logging 8import os 9import re 10import json 11 12import six 13from six import text_type 14import mozpack.path as mozpath 15import taskgraph 16from taskgraph.transforms.base import TransformSequence 17from .. import GECKO 18from taskgraph.util.docker import ( 19 create_context_tar, 20 generate_context_hash, 21 image_path, 22) 23from taskgraph.util.schema import Schema 24from voluptuous import ( 25 Optional, 26 Required, 27) 28from .task import task_description_schema 29 30logger = logging.getLogger(__name__) 31 32CONTEXTS_DIR = "docker-contexts" 33 34DIGEST_RE = re.compile("^[0-9a-f]{64}$") 35 36IMAGE_BUILDER_IMAGE = ( 37 "mozillareleases/image_builder:5.0.0" 38 "@sha256:" 39 "e510a9a9b80385f71c112d61b2f2053da625aff2b6d430411ac42e424c58953f" 40) 41 42transforms = TransformSequence() 43 44docker_image_schema = Schema( 45 { 46 # Name of the docker image. 47 Required("name"): text_type, 48 # Name of the parent docker image. 49 Optional("parent"): text_type, 50 # Treeherder symbol. 51 Required("symbol"): text_type, 52 # relative path (from config.path) to the file the docker image was defined 53 # in. 54 Optional("job-from"): text_type, 55 # Arguments to use for the Dockerfile. 56 Optional("args"): {text_type: text_type}, 57 # Name of the docker image definition under taskcluster/docker, when 58 # different from the docker image name. 59 Optional("definition"): text_type, 60 # List of package tasks this docker image depends on. 61 Optional("packages"): [text_type], 62 Optional( 63 "index", 64 description="information for indexing this build so its artifacts can be discovered", 65 ): task_description_schema["index"], 66 Optional( 67 "cache", 68 description="Whether this image should be cached based on inputs.", 69 ): bool, 70 } 71) 72 73 74transforms.add_validate(docker_image_schema) 75 76 77@transforms.add 78def fill_template(config, tasks): 79 if not taskgraph.fast and config.write_artifacts: 80 if not os.path.isdir(CONTEXTS_DIR): 81 os.makedirs(CONTEXTS_DIR) 82 83 for task in tasks: 84 image_name = task.pop("name") 85 job_symbol = task.pop("symbol") 86 args = task.pop("args", {}) 87 packages = task.pop("packages", []) 88 parent = task.pop("parent", None) 89 90 for p in packages: 91 if "packages-{}".format(p) not in config.kind_dependencies_tasks: 92 raise Exception( 93 "Missing package job for {}-{}: {}".format( 94 config.kind, image_name, p 95 ) 96 ) 97 98 if not taskgraph.fast: 99 context_path = mozpath.relpath(image_path(image_name), GECKO) 100 if config.write_artifacts: 101 context_file = os.path.join( 102 CONTEXTS_DIR, "{}.tar.gz".format(image_name) 103 ) 104 logger.info( 105 "Writing {} for docker image {}".format(context_file, image_name) 106 ) 107 context_hash = create_context_tar( 108 GECKO, context_path, context_file, image_name, args 109 ) 110 else: 111 context_hash = generate_context_hash( 112 GECKO, context_path, image_name, args 113 ) 114 else: 115 if config.write_artifacts: 116 raise Exception("Can't write artifacts if `taskgraph.fast` is set.") 117 context_hash = "0" * 40 118 digest_data = [context_hash] 119 digest_data += [json.dumps(args, sort_keys=True)] 120 121 description = "Build the docker image {} for use by dependent tasks".format( 122 image_name 123 ) 124 125 args["DOCKER_IMAGE_PACKAGES"] = " ".join("<{}>".format(p) for p in packages) 126 127 # Adjust the zstandard compression level based on the execution level. 128 # We use faster compression for level 1 because we care more about 129 # end-to-end times. We use slower/better compression for other levels 130 # because images are read more often and it is worth the trade-off to 131 # burn more CPU once to reduce image size. 132 zstd_level = "3" if int(config.params["level"]) == 1 else "10" 133 134 # include some information that is useful in reconstructing this task 135 # from JSON 136 taskdesc = { 137 "label": "{}-{}".format(config.kind, image_name), 138 "description": description, 139 "attributes": { 140 "image_name": image_name, 141 "artifact_prefix": "public", 142 }, 143 "expires-after": "1 year", 144 "scopes": [], 145 "treeherder": { 146 "symbol": job_symbol, 147 "platform": "taskcluster-images/opt", 148 "kind": "other", 149 "tier": 1, 150 }, 151 "run-on-projects": [], 152 "worker-type": "images", 153 "worker": { 154 "implementation": "docker-worker", 155 "os": "linux", 156 "artifacts": [ 157 { 158 "type": "file", 159 "path": "/workspace/image.tar.zst", 160 "name": "public/image.tar.zst", 161 } 162 ], 163 "env": { 164 "CONTEXT_TASK_ID": {"task-reference": "<decision>"}, 165 "CONTEXT_PATH": "public/docker-contexts/{}.tar.gz".format( 166 image_name 167 ), 168 "HASH": context_hash, 169 "PROJECT": config.params["project"], 170 "IMAGE_NAME": image_name, 171 "DOCKER_IMAGE_ZSTD_LEVEL": zstd_level, 172 "DOCKER_BUILD_ARGS": { 173 "task-reference": six.ensure_text(json.dumps(args)) 174 }, 175 "GECKO_BASE_REPOSITORY": config.params["base_repository"], 176 "GECKO_HEAD_REPOSITORY": config.params["head_repository"], 177 "GECKO_HEAD_REV": config.params["head_rev"], 178 }, 179 "chain-of-trust": True, 180 "max-run-time": 7200, 181 # FIXME: We aren't currently propagating the exit code 182 }, 183 } 184 # Retry for 'funsize-update-generator' if exit status code is -1 185 if image_name in ["funsize-update-generator"]: 186 taskdesc["worker"]["retry-exit-status"] = [-1] 187 188 worker = taskdesc["worker"] 189 190 if image_name == "image_builder": 191 worker["docker-image"] = IMAGE_BUILDER_IMAGE 192 digest_data.append("image-builder-image:{}".format(IMAGE_BUILDER_IMAGE)) 193 else: 194 worker["docker-image"] = {"in-tree": "image_builder"} 195 deps = taskdesc.setdefault("dependencies", {}) 196 deps["docker-image"] = "{}-image_builder".format(config.kind) 197 198 if packages: 199 deps = taskdesc.setdefault("dependencies", {}) 200 for p in sorted(packages): 201 deps[p] = "packages-{}".format(p) 202 203 if parent: 204 deps = taskdesc.setdefault("dependencies", {}) 205 deps["parent"] = "{}-{}".format(config.kind, parent) 206 worker["env"]["PARENT_TASK_ID"] = { 207 "task-reference": "<parent>", 208 } 209 if "index" in task: 210 taskdesc["index"] = task["index"] 211 212 if task.get("cache", True) and not taskgraph.fast: 213 taskdesc["cache"] = { 214 "type": "docker-images.v2", 215 "name": image_name, 216 "digest-data": digest_data, 217 } 218 219 yield taskdesc 220